diff --git a/straceanalyse.py b/straceanalyse.py index a11f4f6..cf15c10 100644 --- a/straceanalyse.py +++ b/straceanalyse.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """ -JTL-Shop Performance Analyzer - Analysiert ALLE PHP-FPM Prozesse +JTL-Shop Performance Analyzer - Mit Script-Detection (Complete Fixed Version) """ import subprocess @@ -15,6 +15,7 @@ class ShopPerformanceAnalyzer: self.domain = domain self.results = { 'missing_files': Counter(), + 'missing_files_context': {}, 'syscalls': Counter(), 'mysql_queries': [], 'redis_operations': Counter(), @@ -24,6 +25,35 @@ class ShopPerformanceAnalyzer: } self.debug = False self.output_dir = f"/root/shop_analysis_{domain}_{datetime.now().strftime('%Y%m%d_%H%M%S')}" + + def get_process_info(self, pid): + """Hole Informationen ueber den Prozess""" + info = { + 'cwd': None, + 'request_uri': None, + 'script_filename': None + } + + try: + cwd_link = f'/proc/{pid}/cwd' + if os.path.exists(cwd_link): + info['cwd'] = os.readlink(cwd_link) + except: + pass + + try: + with open(f'/proc/{pid}/environ', 'rb') as f: + environ = f.read().decode('utf-8', errors='ignore') + env_vars = environ.split('\x00') + for var in env_vars: + if var.startswith('REQUEST_URI='): + info['request_uri'] = var.split('=', 1)[1] + elif var.startswith('SCRIPT_FILENAME='): + info['script_filename'] = var.split('=', 1)[1] + except: + pass + + return info def get_php_fpm_pids(self): """Finde alle PHP-FPM PIDs fuer den Shop""" @@ -43,7 +73,7 @@ class ShopPerformanceAnalyzer: 'strace', '-p', str(pid), '-f', - '-s', '300', + '-s', '500', '-e', 'trace=all', '-T' ] @@ -67,17 +97,28 @@ class ShopPerformanceAnalyzer: except Exception as e: return "" - def analyze_strace_output(self, output): - """Analysiere strace Output""" + def analyze_strace_output(self, output, pid): + """Analysiere strace Output mit Context""" if not output or len(output) < 10: return lines = output.split('\n') + proc_info = self.get_process_info(pid) + + last_php_file = proc_info.get('script_filename') or 'unknown' + current_request = proc_info.get('request_uri') or 'unknown' + for line in lines: if not line.strip(): continue - + + # Track PHP File opens + if 'openat' in line or 'open(' in line: + php_match = re.search(r'"([^"]+\.php)"', line) + if php_match: + last_php_file = php_match.group(1) + syscall_match = re.match(r'^(\w+)\(', line) if syscall_match: self.results['syscalls'][syscall_match.group(1)] += 1 @@ -94,6 +135,23 @@ class ShopPerformanceAnalyzer: filepath = file_match.group(1) self.results['missing_files'][filepath] += 1 self.results['errors']['ENOENT'] += 1 + + if filepath not in self.results['missing_files_context']: + self.results['missing_files_context'][filepath] = { + 'count': 0, + 'php_scripts': set(), + 'requests': set(), + 'pids': set() + } + + context = self.results['missing_files_context'][filepath] + context['count'] += 1 + if last_php_file: + context['php_scripts'].add(last_php_file) + if current_request: + context['requests'].add(current_request) + context['pids'].add(pid) + break if 'EAGAIN' in line: @@ -131,25 +189,82 @@ class ShopPerformanceAnalyzer: self.results['slow_operations'].append(f'Slow I/O Wait ({time_match.group(1)}s)') def export_missing_files(self): - """Exportiere fehlende Dateien in verschiedene Formate""" + """Exportiere fehlende Dateien mit Script-Context""" if not self.results['missing_files']: return None os.makedirs(self.output_dir, exist_ok=True) - # 1. Komplette Liste (sortiert nach Haeufigkeit) + # 1. Komplette Liste mit Context list_file = os.path.join(self.output_dir, 'missing_files_all.txt') with open(list_file, 'w') as f: - f.write(f"# Fehlende Dateien fuer {self.domain}\n") + f.write(f"# Fehlende Dateien fuer {self.domain} - MIT SCRIPT CONTEXT\n") f.write(f"# Erstellt: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n") f.write(f"# Total: {len(self.results['missing_files'])} Dateien\n") f.write(f"# Zugriffe: {sum(self.results['missing_files'].values())}\n") f.write("#" + "="*70 + "\n\n") for filepath, count in self.results['missing_files'].most_common(): + f.write(f"\n{'='*70}\n") f.write(f"[{count:4d}x] {filepath}\n") + f.write(f"{'='*70}\n") + + if filepath in self.results['missing_files_context']: + ctx = self.results['missing_files_context'][filepath] + + php_scripts = [s for s in ctx['php_scripts'] if s is not None] + real_scripts = [s for s in php_scripts if s != 'unknown'] + + if real_scripts: + f.write(f"\n Aufgerufen von:\n") + for script in sorted(real_scripts): + short_script = script.replace(f'/var/www/vhosts/{self.domain}/httpdocs/', '') + f.write(f" * {short_script}\n") + else: + f.write(f"\n Aufgerufen von: unknown (konnte nicht ermittelt werden)\n") + + requests = [r for r in ctx['requests'] if r is not None and r != 'unknown'] + if requests: + f.write(f"\n Bei Requests:\n") + for req in sorted(requests): + f.write(f" * {req}\n") + + f.write(f"\n PIDs: {', '.join(map(str, sorted(ctx['pids'])))}\n") + + f.write("\n") - # 2. Nach Kategorie sortiert + # 2. Script-zu-Dateien Mapping + script_mapping_file = os.path.join(self.output_dir, 'missing_files_by_script.txt') + with open(script_mapping_file, 'w') as f: + f.write(f"# Fehlende Dateien gruppiert nach aufrufendem Script\n") + f.write(f"# Domain: {self.domain}\n") + f.write(f"# Erstellt: {datetime.now().strftime('%Y-%m-%d %H:%M:%S')}\n\n") + + script_to_files = defaultdict(list) + for filepath, ctx in self.results['missing_files_context'].items(): + php_scripts = [s for s in ctx['php_scripts'] if s is not None] + if not php_scripts: + php_scripts = ['unknown'] + + for script in php_scripts: + script_to_files[script].append((filepath, ctx['count'])) + + for script, files in sorted(script_to_files.items()): + if script and script != 'unknown': + short_script = script.replace(f'/var/www/vhosts/{self.domain}/httpdocs/', '') + else: + short_script = 'unknown (konnte nicht ermittelt werden)' + + f.write(f"\n{'='*70}\n") + f.write(f"SCRIPT: {short_script}\n") + f.write(f"{'='*70}\n") + f.write(f"Anzahl fehlender Dateien: {len(files)}\n\n") + + for filepath, count in sorted(files, key=lambda x: x[1], reverse=True): + short_path = filepath.replace(f'/var/www/vhosts/{self.domain}/httpdocs/', '') + f.write(f" [{count:4d}x] {short_path}\n") + + # 3. Nach Kategorie sortiert category_file = os.path.join(self.output_dir, 'missing_files_by_category.txt') with open(category_file, 'w') as f: f.write(f"# Fehlende Dateien nach Kategorie - {self.domain}\n") @@ -181,7 +296,7 @@ class ShopPerformanceAnalyzer: for filepath, count in sorted(items, key=lambda x: x[1], reverse=True): f.write(f"[{count:4d}x] {filepath}\n") - # 3. Bash Script zum Erstellen von Platzhaltern + # 4. Bash Script script_file = os.path.join(self.output_dir, 'create_placeholders.sh') with open(script_file, 'w') as f: f.write("#!/bin/bash\n") @@ -199,7 +314,6 @@ class ShopPerformanceAnalyzer: f.write('CREATED=0\n') f.write('SKIPPED=0\n\n') - # Nur Bilder image_files = [fp for fp in self.results['missing_files'].keys() if any(ext in fp.lower() for ext in ['.jpg', '.jpeg', '.png', '.gif', '.webp'])] @@ -224,10 +338,10 @@ class ShopPerformanceAnalyzer: os.chmod(script_file, 0o755) - # 4. CSV Export + # 5. CSV Export mit Script-Info csv_file = os.path.join(self.output_dir, 'missing_files.csv') with open(csv_file, 'w') as f: - f.write("Zugriffe,Kategorie,Dateipfad\n") + f.write("Zugriffe,Kategorie,Dateipfad,Aufgerufen_von_Script\n") for filepath, count in self.results['missing_files'].most_common(): if 'manufacturer' in filepath: @@ -242,9 +356,19 @@ class ShopPerformanceAnalyzer: category = 'Sonstig' filepath_safe = filepath.replace('"', '""') - f.write(f'{count},{category},"{filepath_safe}"\n') + + scripts = 'unknown' + if filepath in self.results['missing_files_context']: + ctx = self.results['missing_files_context'][filepath] + php_scripts = [s for s in ctx['php_scripts'] if s is not None] + real_scripts = [s.replace(f'/var/www/vhosts/{self.domain}/httpdocs/', '') + for s in php_scripts if s != 'unknown'] + if real_scripts: + scripts = '; '.join(real_scripts) + + f.write(f'{count},{category},"{filepath_safe}","{scripts}"\n') - # 5. Hersteller IDs extrahieren + # 6. Hersteller IDs manufacturer_file = os.path.join(self.output_dir, 'missing_manufacturer_ids.txt') manufacturer_ids = set() for filepath in self.results['missing_files'].keys(): @@ -261,7 +385,7 @@ class ShopPerformanceAnalyzer: for mid in sorted(manufacturer_ids, key=int): f.write(f"{mid}\n") - # 6. Nur Dateipfade (fuer weitere Verarbeitung) + # 7. Nur Pfade paths_only_file = os.path.join(self.output_dir, 'missing_files_paths_only.txt') with open(paths_only_file, 'w') as f: for filepath in self.results['missing_files'].keys(): @@ -280,11 +404,6 @@ class ShopPerformanceAnalyzer: if total_syscalls == 0: print("WARNUNG: Keine Syscalls aufgezeichnet!") - print(" Moegliche Gruende:") - print(" - Prozesse sind gerade idle (wenig Traffic)") - print(" - Strace hat keine Berechtigung") - print(" - Prozesse wurden zwischen Analyse beendet\n") - print(" Versuche: python3 script.py spiel-und-modellbau.com 10\n") return print("SYSCALL STATISTIK") @@ -295,44 +414,39 @@ class ShopPerformanceAnalyzer: print(f" {syscall:20s}: {count:6d} ({percentage:5.1f}%) {bar}") print() - # Fehlende Dateien missing_count = len(self.results['missing_files']) if missing_count > 0: - print("FEHLENDE DATEIEN (ENOENT)") + print("FEHLENDE DATEIEN (ENOENT) - MIT SCRIPT CONTEXT") print("-" * 80) print(f" WARNUNG: {missing_count} verschiedene Dateien nicht gefunden!") - print(f" WARNUNG: {sum(self.results['missing_files'].values())} Zugriffe auf nicht-existierende Dateien!\n") + print(f" WARNUNG: {sum(self.results['missing_files'].values())} Zugriffe\n") - # Kategorien - categories = defaultdict(int) - for filepath in self.results['missing_files'].keys(): - if 'manufacturer' in filepath: - categories['Hersteller-Bilder'] += 1 - elif 'product' in filepath or 'artikel' in filepath: - categories['Produkt-Bilder'] += 1 - elif 'variation' in filepath: - categories['Variationen-Bilder'] += 1 - elif 'category' in filepath or 'kategorie' in filepath: - categories['Kategorie-Bilder'] += 1 - else: - categories['Sonstige'] += 1 - - print(" Kategorien:") - for category, count in sorted(categories.items(), key=lambda x: x[1], reverse=True): - print(f" * {category:25s}: {count:4d} Dateien") - print() - - print(" Top 15 fehlende Dateien:") - for path, count in self.results['missing_files'].most_common(15): + print(" Top 10 fehlende Dateien (mit aufrufendem Script):") + for path, count in list(self.results['missing_files'].most_common(10)): short_path = path.replace(f'/var/www/vhosts/{self.domain}/httpdocs/', '') - print(f" [{count:3d}x] {short_path}") + print(f"\n [{count:3d}x] {short_path}") + + if path in self.results['missing_files_context']: + ctx = self.results['missing_files_context'][path] + + php_scripts = [s for s in ctx['php_scripts'] if s is not None] + real_scripts = [s for s in php_scripts if s != 'unknown'] + + if real_scripts: + print(f" Aufgerufen von:") + for script in list(real_scripts)[:3]: + short_script = script.replace(f'/var/www/vhosts/{self.domain}/httpdocs/', '') + print(f" -> {short_script}") + if len(real_scripts) > 3: + print(f" -> ... und {len(real_scripts)-3} weitere") + else: + print(f" Aufgerufen von: unknown (konnte nicht ermittelt werden)") - if len(self.results['missing_files']) > 15: - print(f"\n INFO: ... und {len(self.results['missing_files'])-15} weitere") - print(f" INFO: Vollstaendige Liste siehe Export-Dateien!") + if len(self.results['missing_files']) > 10: + print(f"\n INFO: ... und {len(self.results['missing_files'])-10} weitere") + print(f"\n INFO: Vollstaendige Liste siehe Export-Dateien!") print() - # Errors if self.results['errors']: print("FEHLER") print("-" * 80) @@ -340,46 +454,6 @@ class ShopPerformanceAnalyzer: print(f" {error:20s}: {count:6d}x") print() - # MySQL Queries - if self.results['mysql_queries']: - print("MYSQL QUERIES") - print("-" * 80) - query_counter = Counter(self.results['mysql_queries']) - print(f" Total Queries: {len(self.results['mysql_queries'])}") - print(f" Unique Queries: {len(query_counter)}") - print("\n Haeufigste Queries:") - for query, count in query_counter.most_common(10): - print(f" [{count:3d}x] {query[:70]}...") - print() - - # File Paths - if self.results['file_paths']: - print("HAEUFIGSTE DATEIZUGRIFFE") - print("-" * 80) - for path, count in self.results['file_paths'].most_common(15): - print(f" [{count:3d}x] {path}") - print() - - # Redis - if self.results['redis_operations']: - print("REDIS OPERATIONEN") - print("-" * 80) - total_redis = sum(self.results['redis_operations'].values()) - for op, count in self.results['redis_operations'].most_common(): - percentage = (count / total_redis * 100) if total_redis > 0 else 0 - print(f" {op:15s}: {count:6d}x ({percentage:5.1f}%)") - print() - - # Slow Operations - if self.results['slow_operations']: - print("LANGSAME OPERATIONEN") - print("-" * 80) - slow_counter = Counter(self.results['slow_operations']) - for op, count in slow_counter.items(): - print(f" WARNUNG: {op}: {count}x") - print() - - # Export fehlende Dateien if missing_count > 0: print("="*80) print("EXPORTIERE FEHLENDE DATEIEN") @@ -389,125 +463,23 @@ class ShopPerformanceAnalyzer: if export_dir: print(f"Dateien exportiert nach: {export_dir}\n") print(" Erstellt:") - print(f" missing_files_all.txt - Komplette Liste (sortiert nach Haeufigkeit)") + print(f" missing_files_all.txt - Komplette Liste MIT Script-Context") + print(f" missing_files_by_script.txt - Gruppiert nach PHP-Script") print(f" missing_files_by_category.txt - Nach Kategorie gruppiert") - print(f" missing_files.csv - CSV fuer Excel") - print(f" create_placeholders.sh - Bash Script (ausfuehrbar)") + print(f" missing_files.csv - CSV mit Script-Info") + print(f" create_placeholders.sh - Bash Script") print(f" missing_manufacturer_ids.txt - Hersteller IDs") - print(f" missing_files_paths_only.txt - Nur Pfade (fuer Scripts)\n") + print(f" missing_files_paths_only.txt - Nur Pfade\n") print(" Quick-Fix:") print(f" bash {export_dir}/create_placeholders.sh\n") - # Handlungsempfehlungen - self.generate_recommendations(total_syscalls, missing_count) - - def generate_recommendations(self, total_syscalls, missing_count): - """Generiere Handlungsempfehlungen""" - print("="*80) - print("HANDLUNGSEMPFEHLUNGEN FUER DEN KUNDEN") - print("="*80 + "\n") - - recommendations = [] - priority = 1 - - if missing_count > 5: - manufacturer_missing = sum(1 for p in self.results['missing_files'] if 'manufacturer' in p) - product_missing = sum(1 for p in self.results['missing_files'] if 'product' in p or 'artikel' in p) - - if manufacturer_missing > 0: - recommendations.append({ - 'priority': priority, - 'severity': 'KRITISCH', - 'problem': f'{manufacturer_missing} Hersteller-Bilder fehlen', - 'impact': f'Jedes fehlende Bild = 6-8 stat() Calls. Bei {sum(v for k,v in self.results["missing_files"].items() if "manufacturer" in k)} Zugriffen!', - 'solution': '1. JTL-Shop Admin einloggen\n2. Bilder -> Hersteller-Bilder -> "Fehlende generieren"\n3. ODER: Bash Script ausfuehren (siehe Export)', - 'files': [f"{self.output_dir}/create_placeholders.sh", - f"{self.output_dir}/missing_manufacturer_ids.txt"] - }) - priority += 1 - - if product_missing > 0: - recommendations.append({ - 'priority': priority, - 'severity': 'WICHTIG', - 'problem': f'{product_missing} Produkt-Bilder fehlen', - 'impact': 'Erhoehte I/O Last', - 'solution': 'JTL-Shop Admin -> Bilder -> "Bildcache regenerieren"' - }) - priority += 1 - - stat_calls = sum(count for syscall, count in self.results['syscalls'].items() - if 'stat' in syscall.lower()) - if stat_calls > 500: - recommendations.append({ - 'priority': priority, - 'severity': 'WICHTIG', - 'problem': f'{stat_calls} Filesystem stat() Calls', - 'impact': 'Filesystem-Thrashing, langsame Response-Times', - 'solution': 'PHP Realpath Cache erhoehen in PHP-Einstellungen', - 'technical': 'Plesk -> Domain -> PHP Settings:\nrealpath_cache_size = 4096K\nrealpath_cache_ttl = 600' - }) - priority += 1 - - imagemagick_count = sum(1 for op in self.results['slow_operations'] if 'ImageMagick' in op) - if imagemagick_count > 3: - recommendations.append({ - 'priority': priority, - 'severity': 'KRITISCH', - 'problem': f'ImageMagick wird {imagemagick_count}x aufgerufen', - 'impact': 'CPU-intensive Bildverarbeitung bei jedem Request!', - 'solution': '1. Bild-Cache in JTL-Shop aktivieren\n2. Alle Bildgroessen vorher generieren\n3. Pruefen ob Bilder wirklich vorhanden sind' - }) - priority += 1 - - if len(self.results['mysql_queries']) > 50: - recommendations.append({ - 'priority': priority, - 'severity': 'WICHTIG', - 'problem': f'{len(self.results["mysql_queries"])} MySQL Queries', - 'impact': 'N+1 Query Problem, Database Overhead', - 'solution': 'JTL-Shop: System -> Cache -> Object Cache aktivieren (Redis)' - }) - priority += 1 - - eagain_count = self.results['errors'].get('EAGAIN', 0) - if eagain_count > 100: - recommendations.append({ - 'priority': priority, - 'severity': 'WICHTIG', - 'problem': f'{eagain_count}x EAGAIN', - 'impact': 'Redis/MySQL Verbindungen ueberlastet', - 'solution': 'Redis Connection Pool erhoehen oder PHP-FPM Worker erhoehen' - }) - priority += 1 - - if recommendations: - for rec in recommendations: - print(f"[{rec['severity']}] PRIORITAET {rec['priority']}: {rec['problem']}") - print(f" Impact: {rec['impact']}") - print(f" Loesung: {rec['solution']}") - if 'files' in rec: - print(f" Dateien:") - for file in rec['files']: - print(f" * {file}") - if 'technical' in rec: - lines = rec['technical'].split('\n') - print(f" Technisch:") - for line in lines: - print(f" {line}") - print() - else: - print("Keine kritischen Probleme gefunden!\n") - print("="*80) print("ZUSAMMENFASSUNG") print("="*80) print(f" * Total Syscalls: {total_syscalls}") print(f" * Fehlende Dateien: {missing_count}") print(f" * MySQL Queries: {len(self.results['mysql_queries'])}") - print(f" * Redis Operations: {sum(self.results['redis_operations'].values())}") - print(f" * Handlungsempfehlungen: {len(recommendations)}") if missing_count > 0: print(f"\n Export-Verzeichnis: {self.output_dir}") print() @@ -515,17 +487,12 @@ class ShopPerformanceAnalyzer: def main(): if len(sys.argv) < 2: print("\n" + "="*80) - print("JTL-Shop Performance Analyzer") + print("JTL-Shop Performance Analyzer - Mit Script Detection") print("="*80) print("\nUsage: python3 shop_analyzer.py [duration] [max_processes]") print("\nExamples:") - print(" python3 shop_analyzer.py spiel-und-modellbau.com") print(" python3 shop_analyzer.py spiel-und-modellbau.com 10") - print(" python3 shop_analyzer.py spiel-und-modellbau.com 10 20 # Max 20 Prozesse") - print("\nParameter:") - print(" domain - Shop Domain") - print(" duration - Sekunden pro Prozess (default: 5)") - print(" max_processes - Max Anzahl Prozesse (default: alle)") + print(" python3 shop_analyzer.py spiel-und-modellbau.com 10 20") print() sys.exit(1) @@ -533,13 +500,8 @@ def main(): duration = int(sys.argv[2]) if len(sys.argv) > 2 else 5 max_processes = int(sys.argv[3]) if len(sys.argv) > 3 else None - print(f"\nStarte Performance-Analyse fuer: {domain}") - print(f"Analyse-Dauer: {duration} Sekunden pro Prozess") - if max_processes: - print(f"Max Prozesse: {max_processes}") - else: - print(f"Prozesse: ALLE gefundenen PHP-FPM Worker") - print() + print(f"\nStarte Performance-Analyse mit Script-Detection fuer: {domain}") + print(f"Analyse-Dauer: {duration} Sekunden pro Prozess\n") analyzer = ShopPerformanceAnalyzer(domain) @@ -548,25 +510,17 @@ def main(): print("Keine PHP-FPM Prozesse gefunden!") sys.exit(1) - # Limit anwenden falls gesetzt if max_processes and len(pids) > max_processes: - print(f"{len(pids)} PHP-FPM Prozesse gefunden (analysiere {max_processes})") pids = pids[:max_processes] - else: - print(f"{len(pids)} PHP-FPM Prozesse gefunden (analysiere alle)") - - print(f" PIDs: {pids}\n") - - # Progress bar setup - total = len(pids) - analyzed = 0 - failed = 0 + print(f"{len(pids)} PHP-FPM Prozesse gefunden\n") print("Analyse laeuft...") print("-" * 80) + total = len(pids) + analyzed = 0 + for i, pid in enumerate(pids, 1): - # Progress indicator percent = int((i / total) * 100) bar_length = 40 filled = int((percent / 100) * bar_length) @@ -576,22 +530,15 @@ def main(): output = analyzer.run_strace(pid, duration) if output and len(output) > 100: - analyzer.analyze_strace_output(output) + analyzer.analyze_strace_output(output, pid) analyzed += 1 - else: - failed += 1 print(f"\r[{'#' * bar_length}] 100% | Fertig!{' ' * 30}") print("-" * 80) - print(f"\nAnalyse abgeschlossen!") - print(f" * Erfolgreich analysiert: {analyzed}/{total}") - if failed > 0: - print(f" * Idle/Keine Daten: {failed}") - print() + print(f"\nAnalyse abgeschlossen! ({analyzed} Prozesse erfolgreich)\n") if analyzed == 0: - print("Konnte keine Daten sammeln!") - print(" Shop hat gerade wenig Traffic. Versuche spaeter nochmal oder erhoehe duration.\n") + print("Konnte keine Daten sammeln!\n") sys.exit(1) analyzer.generate_report()