diff --git a/mariadb-optimization.py b/mariadb-optimization.py index f81c21c..2fc1250 100644 --- a/mariadb-optimization.py +++ b/mariadb-optimization.py @@ -1,13 +1,14 @@ #!/usr/bin/env python3 """ -JTL Shop MariaDB Index-Optimierung -================================== -Erstellt fehlende Indizes und aktiviert FULLTEXT-Suche auf allen JTL-Shops. +JTL Shop MariaDB Optimierung +============================ +Erstellt fehlende Indizes, aktiviert FULLTEXT-Suche und optimiert MariaDB-Konfiguration. Features: - Backup aller Sucheinstellungen vor Änderungen - Erstellt 5 Index-Typen auf allen betroffenen Shops - Aktiviert FULLTEXT-Suche +- Generiert optimierte MariaDB-Konfiguration - Generiert Rollback-Skript für Notfälle Autor: Claude @@ -17,6 +18,8 @@ Datum: 2025-12-12 import subprocess import json import sys +import math +import os from datetime import datetime from pathlib import Path @@ -26,6 +29,8 @@ TIMESTAMP = datetime.now().strftime("%Y%m%d_%H%M%S") BACKUP_FILE = f"/root/jtl_search_backup_{TIMESTAMP}.json" ROLLBACK_FILE = f"/root/jtl_search_rollback_{TIMESTAMP}.py" LOG_FILE = f"/root/jtl_index_log_{TIMESTAMP}.txt" +MARIADB_CONFIG_FILE = "/etc/mysql/conf.d/ZZ-ztl-final-override.cnf" +MARIADB_CONFIG_BACKUP = f"/root/mariadb_config_backup_{TIMESTAMP}.cnf" # Index-Definitionen INDEXES = [ @@ -81,7 +86,7 @@ class Logger: def run_mysql(query, database=None): """Führt MySQL-Query aus und gibt Ergebnis zurück.""" - cmd = ["mysql", "-N", "-e", query] + cmd = ["mysql", "-N", "-B", "-e", query] if database: cmd.extend(["-D", database]) @@ -167,6 +172,113 @@ def create_index(shop, index_name, table, columns, is_fulltext=False): return err +def get_total_database_size_gb(): + """Ermittelt die Gesamtgröße aller Datenbanken in GB.""" + query = """ + SELECT ROUND(SUM(data_length + index_length) / 1024 / 1024 / 1024, 2) + FROM information_schema.tables + WHERE table_schema NOT IN ('information_schema', 'mysql', 'performance_schema', 'sys'); + """ + result, err = run_mysql(query) + if err or not result: + return None + try: + return float(result) + except ValueError: + return None + + +def calculate_buffer_pool_size(db_size_gb): + """Berechnet Buffer Pool Size: DB-Größe × 1.2, aufgerundet auf ganze GB.""" + recommended = db_size_gb * 1.2 + return math.ceil(recommended) + + +def generate_mariadb_config(buffer_pool_gb): + """Generiert die MariaDB-Konfiguration mit dynamischem Buffer Pool.""" + config = f"""# JTL-Shop MariaDB Optimierung +# Generiert am: {datetime.now().strftime("%Y-%m-%d %H:%M:%S")} +# Buffer Pool berechnet: DB-Größe × 1.2 = {buffer_pool_gb}G + +[mysqld] +# InnoDB Buffer Pool (dynamisch berechnet) +innodb_buffer_pool_size = {buffer_pool_gb}G +innodb_buffer_pool_instances = 16 +innodb_buffer_pool_dump_at_shutdown = 1 +innodb_buffer_pool_load_at_startup = 1 +innodb_log_file_size = 512M +innodb_log_buffer_size = 64M +innodb_flush_log_at_trx_commit = 2 + +# InnoDB Performance +innodb_flush_method = O_DIRECT +innodb_file_per_table = 1 +innodb_io_capacity = 2000 +innodb_io_capacity_max = 4000 +innodb_read_io_threads = 16 +innodb_write_io_threads = 16 + +# Query Cache deaktiviert (bei MariaDB 10.1.7+ empfohlen) +query_cache_type = 0 +query_cache_size = 0 + +# Table Cache für viele Shops +table_definition_cache = 20000 +table_open_cache = 15000 +open_files_limit = 30000 + +# Performance +skip_name_resolve = 1 +optimizer_search_depth = 0 +key_buffer_size = 64M + +# Slow Query Log +slow_query_log = 1 +slow_query_log_file = /var/log/mysql/slow.log +long_query_time = 1.0 +log_queries_not_using_indexes = 0 + +# Connections +max_connections = 400 +thread_cache_size = 100 +wait_timeout = 120 + +# Buffer für JTL-Queries +join_buffer_size = 4M +sort_buffer_size = 4M +read_buffer_size = 2M +read_rnd_buffer_size = 2M +""" + return config + + +def backup_existing_config(): + """Erstellt ein Backup der existierenden Config, falls vorhanden.""" + if os.path.exists(MARIADB_CONFIG_FILE): + with open(MARIADB_CONFIG_FILE, 'r') as f: + content = f.read() + with open(MARIADB_CONFIG_BACKUP, 'w') as f: + f.write(content) + return True + return False + + +def write_mariadb_config(config_content): + """Schreibt die MariaDB-Konfiguration.""" + with open(MARIADB_CONFIG_FILE, 'w') as f: + f.write(config_content) + + +def restart_mariadb(): + """Startet MariaDB neu.""" + result = subprocess.run( + ["systemctl", "restart", "mariadb"], + capture_output=True, + text=True + ) + return result.returncode == 0, result.stderr + + def generate_rollback_script(backup_data): """Generiert ein Rollback-Skript.""" script = f'''#!/usr/bin/env python3 @@ -225,7 +337,7 @@ def main(): logger = Logger(LOG_FILE) print("=" * 60) - print("JTL Shop MariaDB Index-Optimierung") + print("JTL Shop MariaDB Optimierung") print("=" * 60) print() @@ -355,35 +467,85 @@ def main(): print() + # Phase 5: MariaDB-Konfiguration optimieren + print("=" * 60) + print("PHASE 5: MariaDB-Konfiguration optimieren") + print("=" * 60) + + config_stats = {"success": False, "restart": False} + + # Datenbankgröße ermitteln + logger.log("Ermittle Gesamtgröße aller Datenbanken...") + db_size_gb = get_total_database_size_gb() + + if db_size_gb is None: + logger.log("Konnte Datenbankgröße nicht ermitteln. Überspringe Config-Optimierung.", "ERROR") + else: + logger.log(f" Gesamtgröße: {db_size_gb} GB") + + # Buffer Pool berechnen + buffer_pool_gb = calculate_buffer_pool_size(db_size_gb) + logger.log(f" Berechneter Buffer Pool: {db_size_gb} GB × 1.2 = {buffer_pool_gb} GB") + + # Backup der existierenden Config + if backup_existing_config(): + logger.log(f" Bestehendes Config-Backup erstellt: {MARIADB_CONFIG_BACKUP}") + else: + logger.log(f" Keine bestehende Config gefunden, erstelle neu") + + # Config generieren und schreiben + config_content = generate_mariadb_config(buffer_pool_gb) + write_mariadb_config(config_content) + logger.log(f" Config geschrieben: {MARIADB_CONFIG_FILE}") + config_stats["success"] = True + + # MariaDB neustarten + logger.log(" Starte MariaDB neu...") + success, err = restart_mariadb() + if success: + logger.log(" [OK] MariaDB erfolgreich neugestartet") + config_stats["restart"] = True + else: + logger.log(f" [FEHLER] MariaDB Neustart fehlgeschlagen: {err}", "ERROR") + + print() + # Zusammenfassung print("=" * 60) print("ZUSAMMENFASSUNG") print("=" * 60) print() print("Standard-Indizes:") - print(f" Erstellt: {index_stats['created']}") - print(f" Übersprungen: {index_stats['skipped']}") + print(f" Erstellt: {index_stats['created']}") + print(f" Übersprungen: {index_stats['skipped']}") print(f" Fehlgeschlagen: {index_stats['failed']}") print() print("FULLTEXT-Index:") - print(f" Erstellt: {fulltext_stats['created']}") - print(f" Übersprungen: {fulltext_stats['skipped']}") + print(f" Erstellt: {fulltext_stats['created']}") + print(f" Übersprungen: {fulltext_stats['skipped']}") print(f" Fehlgeschlagen: {fulltext_stats['failed']}") print() print("Sucheinstellung (suche_fulltext = 'Y'):") - print(f" Aktualisiert: {search_stats['updated']}") - print(f" Übersprungen: {search_stats['skipped']}") + print(f" Aktualisiert: {search_stats['updated']}") + print(f" Übersprungen: {search_stats['skipped']}") print(f" Fehlgeschlagen: {search_stats['failed']}") print() + print("MariaDB-Konfiguration:") + print(f" Config erstellt: {'Ja' if config_stats['success'] else 'Nein'}") + print(f" MariaDB Restart: {'Erfolgreich' if config_stats['restart'] else 'Fehlgeschlagen/Übersprungen'}") + print() print("=" * 60) print("WICHTIGE DATEIEN") print("=" * 60) - print(f" Backup: {BACKUP_FILE}") - print(f" Rollback: {ROLLBACK_FILE}") - print(f" Log: {LOG_FILE}") + print(f" Search Backup: {BACKUP_FILE}") + print(f" Rollback-Skript: {ROLLBACK_FILE}") + print(f" Config Backup: {MARIADB_CONFIG_BACKUP}") + print(f" MariaDB Config: {MARIADB_CONFIG_FILE}") + print(f" Log: {LOG_FILE}") print() - print("Bei Problemen Rollback ausführen mit:") - print(f" python3 {ROLLBACK_FILE}") + print("Bei Problemen:") + print(f" Rollback Search: python3 {ROLLBACK_FILE}") + print(f" Rollback Config: cp {MARIADB_CONFIG_BACKUP} {MARIADB_CONFIG_FILE} && systemctl restart mariadb") print() # Log speichern