From db8b0d0f6703280cc3de355ef4ff7d56b2262721 Mon Sep 17 00:00:00 2001 From: Thomas Ciesla Date: Fri, 9 Jan 2026 15:55:35 +0100 Subject: [PATCH] fix request handling --- jtl-wafi-agent.py | 110 +++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 98 insertions(+), 12 deletions(-) diff --git a/jtl-wafi-agent.py b/jtl-wafi-agent.py index 5618fc8..3ca4365 100644 --- a/jtl-wafi-agent.py +++ b/jtl-wafi-agent.py @@ -1331,9 +1331,9 @@ class LiveStatsTracker: self.blocked_paths: Dict[str, List[float]] = {} # Path -> [blocked timestamps] self.ip_404s: Dict[str, List[float]] = {} # IP -> [404 timestamps] - # IP Request Details (für IP-Detail Popup) + # IP Request Details (für IP-Detail Popup) - werden NICHT nach window_seconds gelöscht! self.ip_request_details: Dict[str, List[Dict]] = {} # IP -> [{ts, path, ua}, ...] - self.max_request_details = 100 # Max Anzahl Details pro IP + self.max_request_details = 500 # Max Anzahl Details pro IP (nicht zeitbasiert) # Human/Bot Counters self.human_requests: List[float] = [] @@ -1357,15 +1357,12 @@ class LiveStatsTracker: self.human_requests = [t for t in self.human_requests if t > cutoff] self.bot_requests = [t for t in self.bot_requests if t > cutoff] - # Request Details cleanup - for ip in list(self.ip_request_details.keys()): - self.ip_request_details[ip] = [r for r in self.ip_request_details[ip] if r['ts'] > cutoff] - if not self.ip_request_details[ip]: - del self.ip_request_details[ip] + # Request Details werden NICHT zeitbasiert gelöscht - nur durch max_request_details limitiert + # Damit bleiben alle Requests im Speicher für das IP-Detail-Popup - def record_request(self, ip: str, path: str, is_bot: bool, is_blocked: bool = False, is_404: bool = False, user_agent: str = ''): - """Zeichnet einen Request auf.""" - now = time.time() + def record_request(self, ip: str, path: str, is_bot: bool, is_blocked: bool = False, is_404: bool = False, user_agent: str = '', timestamp: float = None): + """Zeichnet einen Request auf. timestamp=None nutzt aktuelle Zeit.""" + now = timestamp if timestamp is not None else time.time() # IP-Request if ip not in self.ip_requests: @@ -1526,13 +1523,102 @@ _shop_stats_trackers: Dict[str, LiveStatsTracker] = {} def get_shop_stats_tracker(shop: str, window_seconds: int = 300) -> LiveStatsTracker: """Gibt den Stats-Tracker für einen Shop zurück (erstellt falls nötig).""" if shop not in _shop_stats_trackers: - _shop_stats_trackers[shop] = LiveStatsTracker(shop, window_seconds) + tracker = LiveStatsTracker(shop, window_seconds) + # Initialen Log-Scan durchführen um Request-Details zu laden + _load_initial_request_details(tracker, shop) + _shop_stats_trackers[shop] = tracker elif _shop_stats_trackers[shop].window_seconds != window_seconds: # Window geändert - neuen Tracker erstellen - _shop_stats_trackers[shop] = LiveStatsTracker(shop, window_seconds) + tracker = LiveStatsTracker(shop, window_seconds) + _load_initial_request_details(tracker, shop) + _shop_stats_trackers[shop] = tracker return _shop_stats_trackers[shop] +def _load_initial_request_details(tracker: LiveStatsTracker, shop: str, max_lines: int = 5000): + """ + Lädt initiale Request-Details aus dem Log-File. + Wird beim Erstellen eines neuen LiveStatsTracker aufgerufen. + """ + log_file = os.path.join(VHOSTS_DIR, shop, 'httpdocs', SHOP_LOG_FILE) + + if not os.path.isfile(log_file): + return + + try: + # Letzte N Zeilen des Logs lesen (effizient von hinten) + with open(log_file, 'r') as f: + # Gehe ans Ende der Datei + f.seek(0, 2) + file_size = f.tell() + + if file_size == 0: + return + + # Lies maximal 500KB von hinten + read_size = min(512000, file_size) + f.seek(file_size - read_size) + + # Überspringe erste (möglicherweise unvollständige) Zeile + if read_size < file_size: + f.readline() + + lines = f.readlines() + + # Nur die letzten max_lines Zeilen verarbeiten + lines = lines[-max_lines:] + + for line in lines: + try: + # IP extrahieren + ip = None + if 'IP: ' in line: + ip = line.split('IP: ')[1].split(' |')[0].strip() + + if not ip: + continue + + # Zeitstempel extrahieren (Format: [2024-01-09 12:34:56]) + timestamp = None + if line.startswith('[') and ']' in line: + try: + ts_str = line[1:line.index(']')] + ts_dt = datetime.strptime(ts_str, '%Y-%m-%d %H:%M:%S') + timestamp = ts_dt.timestamp() + except: + pass + + # Path extrahieren + path = '/' + if 'Path: ' in line: + path = line.split('Path: ')[1].split(' |')[0].strip() + elif 'URI: ' in line: + path = line.split('URI: ')[1].split(' |')[0].strip() + + # User-Agent extrahieren + user_agent = '' + if 'UA: ' in line: + user_agent = line.split('UA: ')[1].split(' |')[0].strip() + elif 'User-Agent: ' in line: + user_agent = line.split('User-Agent: ')[1].split(' |')[0].strip() + + # Bot/Blocked erkennen + is_bot = any(x in line for x in ['BOT: ', 'BOT:', 'BLOCKED_BOT:', 'MONITOR_BOT:', 'BANNED_BOT:']) + is_blocked = any(x in line for x in ['BANNED', 'BLOCKED']) + is_404 = '404' in line + + # Request aufzeichnen (mit originalem Zeitstempel falls vorhanden) + tracker.record_request(ip, path, is_bot, is_blocked, is_404, user_agent, timestamp) + + except Exception: + continue + + logger.info(f"Initiale Request-Details für {shop} geladen: {len(tracker.ip_request_details)} IPs") + + except Exception as e: + logger.warning(f"Fehler beim Laden der initialen Request-Details für {shop}: {e}") + + # ============================================================================= # BAN/WHITELIST MANAGEMENT # =============================================================================