From 7151eeb91bade311c04bcf99eb7230472a36b66e Mon Sep 17 00:00:00 2001 From: thomasciesla Date: Tue, 30 Dec 2025 16:00:30 +0100 Subject: [PATCH] jtl-wafi-agent.py aktualisiert --- jtl-wafi-agent.py | 400 +++++++++++++++++++++++++++++++++++----------- 1 file changed, 305 insertions(+), 95 deletions(-) diff --git a/jtl-wafi-agent.py b/jtl-wafi-agent.py index bf4724a..19f636a 100644 --- a/jtl-wafi-agent.py +++ b/jtl-wafi-agent.py @@ -19,6 +19,7 @@ import sys import json import time import socket +import struct import hashlib import logging import asyncio @@ -86,6 +87,11 @@ IP_API_URL = "http://ip-api.com/json/{ip}?fields=status,country,countryCode,isp, IP_API_RATE_LIMIT = 45 # Requests pro Minute (Free Tier) IP_INFO_CACHE_TTL = 86400 # 24 Stunden Cache +# ============================================================================= +# GLOBAL COUNTRY CACHE (ipdeny.com - shared between all shops) +# ============================================================================= +GLOBAL_COUNTRY_CACHE_DIR = "/var/lib/jtl-wafi/country_cache" + # ============================================================================= # BAN/WHITELIST FILES # ============================================================================= @@ -1031,17 +1037,107 @@ def restart_php_fpm(domain: str) -> dict: # ============================================================================= -# IP-INFO CACHE (ip-api.com) +# IP-INFO CACHE (ipdeny.com for country, ip-api.com for ISP/ASN) # ============================================================================= # Global IP-Info Cache _ip_info_cache: Dict[str, Dict[str, Any]] = {} _ip_api_last_request = 0.0 _ip_api_request_count = 0 +# Global Country Ranges Cache (loaded once from ipdeny.com files) +_country_ranges_cache: Dict[str, List[tuple]] = {} # country -> [(network_int, mask_int), ...] +_country_cache_loaded = False + + +def _load_global_country_cache(): + """Lädt alle Country-Ranges aus dem globalen Cache.""" + global _country_ranges_cache, _country_cache_loaded + + if _country_cache_loaded: + return + + if not os.path.isdir(GLOBAL_COUNTRY_CACHE_DIR): + logger.debug(f"Global Country Cache nicht vorhanden: {GLOBAL_COUNTRY_CACHE_DIR}") + _country_cache_loaded = True + return + + for country in COUNTRY_CODES: + cache_file = os.path.join(GLOBAL_COUNTRY_CACHE_DIR, f"{country}.ranges") + if not os.path.isfile(cache_file): + continue + + try: + with open(cache_file, 'r') as f: + content = f.read() + + # Parse PHP serialized format: a:N:{i:0;s:LEN:"CIDR";...} + ranges = [] + import re + for match in re.finditer(r's:\d+:"([^"]+)"', content): + cidr = match.group(1) + if '/' in cidr: + try: + subnet, mask = cidr.split('/') + subnet_int = struct.unpack('!I', socket.inet_aton(subnet))[0] + mask_int = (0xFFFFFFFF << (32 - int(mask))) & 0xFFFFFFFF + ranges.append((subnet_int, mask_int)) + except: + pass + + if ranges: + _country_ranges_cache[country.upper()] = ranges + logger.debug(f"Country ranges geladen: {country.upper()} ({len(ranges)} ranges)") + except Exception as e: + logger.debug(f"Fehler beim Laden von {country}: {e}") + + _country_cache_loaded = True + logger.info(f"Global Country Cache geladen: {len(_country_ranges_cache)} Länder") + + +def get_country_for_ip_cached(ip: str) -> str: + """ + Ermittelt das Land für eine IP aus dem globalen ipdeny.com Cache. + Konsistent mit PHP-Templates! + + Returns: + 2-Letter Country Code (z.B. 'DE') oder 'XX' wenn unbekannt + """ + global _country_ranges_cache + + # Cache laden falls noch nicht geschehen + if not _country_cache_loaded: + _load_global_country_cache() + + try: + ip_int = struct.unpack('!I', socket.inet_aton(ip))[0] + except: + return 'XX' + + # Suche in allen Ländern (häufigste zuerst) + priority_countries = ['DE', 'AT', 'CH', 'US', 'GB', 'FR', 'NL', 'IT', 'ES', 'PL'] + + for country in priority_countries: + if country in _country_ranges_cache: + for subnet_int, mask_int in _country_ranges_cache[country]: + if (ip_int & mask_int) == (subnet_int & mask_int): + return country + + # Restliche Länder + for country, ranges in _country_ranges_cache.items(): + if country in priority_countries: + continue + for subnet_int, mask_int in ranges: + if (ip_int & mask_int) == (subnet_int & mask_int): + return country + + return 'XX' + def get_ip_info(ip: str) -> Dict[str, Any]: """ - Holt IP-Informationen von ip-api.com mit Caching. + Holt IP-Informationen mit konsistenter Country-Detection. + - Country: aus ipdeny.com Cache (konsistent mit PHP!) + - ISP/ASN: aus ip-api.com (optional, mit Rate-Limiting) Returns: Dict mit country, countryCode, isp, org, as (ASN) @@ -1054,45 +1150,50 @@ def get_ip_info(ip: str) -> Dict[str, Any]: if time.time() - cached.get('_cached_at', 0) < IP_INFO_CACHE_TTL: return cached - # Rate-Limiting + # Country aus ipdeny.com Cache (konsistent mit PHP!) + country_code = get_country_for_ip_cached(ip) + + # Basis-Ergebnis mit Country aus lokalem Cache + result = { + 'country': country_code if country_code != 'XX' else 'Unknown', + 'countryCode': country_code, + 'isp': '', + 'org': '', + 'as': '', + '_cached_at': time.time() + } + + # Optional: ISP/ASN von ip-api.com holen (mit Rate-Limiting) now = time.time() - if now - _ip_api_last_request < 60: - if _ip_api_request_count >= IP_API_RATE_LIMIT: - logger.debug(f"IP-API Rate-Limit erreicht, verwende Cache für {ip}") - return _ip_info_cache.get(ip, {'country': 'Unknown', 'countryCode': 'XX', 'isp': '', 'org': '', 'as': ''}) - else: + if now - _ip_api_last_request >= 60: _ip_api_last_request = now _ip_api_request_count = 0 - try: - url = IP_API_URL.format(ip=ip) - request = urllib.request.Request( - url, - headers={'User-Agent': f'JTL-WAFi-Agent/{VERSION}'} - ) - with urllib.request.urlopen(request, timeout=5) as response: - data = json.loads(response.read().decode('utf-8')) + if _ip_api_request_count < IP_API_RATE_LIMIT: + try: + url = IP_API_URL.format(ip=ip) + request = urllib.request.Request( + url, + headers={'User-Agent': f'JTL-WAFi-Agent/{VERSION}'} + ) + with urllib.request.urlopen(request, timeout=3) as response: + data = json.loads(response.read().decode('utf-8')) - _ip_api_request_count += 1 + _ip_api_request_count += 1 - if data.get('status') == 'success': - result = { - 'country': data.get('country', 'Unknown'), - 'countryCode': data.get('countryCode', 'XX'), - 'isp': data.get('isp', ''), - 'org': data.get('org', ''), - 'as': data.get('as', ''), - '_cached_at': time.time() - } - _ip_info_cache[ip] = result - return result - else: - logger.debug(f"IP-API Fehler für {ip}: {data.get('message', 'Unknown error')}") - return {'country': 'Unknown', 'countryCode': 'XX', 'isp': '', 'org': '', 'as': ''} + if data.get('status') == 'success': + result['isp'] = data.get('isp', '') + result['org'] = data.get('org', '') + result['as'] = data.get('as', '') + # Country NUR überschreiben wenn lokaler Cache 'XX' war + if country_code == 'XX' and data.get('countryCode'): + result['country'] = data.get('country', 'Unknown') + result['countryCode'] = data.get('countryCode', 'XX') + except Exception as e: + logger.debug(f"IP-API Request fehlgeschlagen für {ip}: {e}") - except Exception as e: - logger.debug(f"IP-API Request fehlgeschlagen für {ip}: {e}") - return {'country': 'Unknown', 'countryCode': 'XX', 'isp': '', 'org': '', 'as': ''} + _ip_info_cache[ip] = result + return result def get_cached_ip_info(ip: str) -> Optional[Dict[str, Any]]: @@ -1975,54 +2076,40 @@ if (!$is_whitelisted && file_exists($ip_ban_file)) {{ }} }} -// === Country Detection via IP === +// === Country Detection (local cache only - no HTTP during page load!) === function get_country_for_ip($ip, $country_cache_dir) {{ - // Most common countries first for performance + // Common countries to check (ordered by traffic likelihood) $countries = ['de', 'at', 'ch', 'us', 'gb', 'fr', 'nl', 'it', 'es', 'pl', 'be', 'se', 'no', 'dk', 'fi', 'ru', 'cn', 'jp', 'kr', 'in', 'br', 'au', 'ca', 'ua', 'cz', 'pt', 'ie', 'gr', 'hu', 'ro', 'bg', 'hr', 'sk', 'si', 'lt', 'lv', 'ee', 'lu', 'mt', 'cy', 'tr', 'il', 'za', 'mx', 'ar', 'cl', 'co', 'pe', 've', 'eg', 'ng', 'ke', 'ma', 'tn', 'pk', 'bd', 'vn', 'th', 'my', 'sg', 'id', 'ph', 'tw', 'hk', 'nz', 'ae', 'sa', 'qa', 'kw', 'bh', 'om', 'ir', 'iq']; + $ip_long = ip2long($ip); + if ($ip_long === false) return 'XX'; + foreach ($countries as $country) {{ $cache_file = "$country_cache_dir/$country.ranges"; - $ranges = []; - // Load from cache or download - if (file_exists($cache_file) && (time() - filemtime($cache_file)) < 86400) {{ - $ranges = @unserialize(@file_get_contents($cache_file)); - }} + // Only read from cache - NO HTTP requests during page load! + if (!file_exists($cache_file)) continue; - if (empty($ranges) || !is_array($ranges)) {{ - // Download fresh from ipdeny.com - $url = "https://www.ipdeny.com/ipblocks/data/aggregated/$country-aggregated.zone"; - $ctx = stream_context_create(['http' => ['timeout' => 10]]); - $content = @file_get_contents($url, false, $ctx); - if ($content !== false) {{ - $ranges = []; - foreach (explode("\\n", trim($content)) as $line) {{ - $line = trim($line); - if (!empty($line) && strpos($line, '/') !== false) {{ - $ranges[] = $line; - }} - }} - if (count($ranges) > 100) {{ - @file_put_contents($cache_file, serialize($ranges)); - }} - }} - }} + $ranges = @unserialize(@file_get_contents($cache_file)); + if (empty($ranges) || !is_array($ranges)) continue; - // Check if IP is in this country - if (!empty($ranges)) {{ - foreach ($ranges as $range) {{ - if (ip_in_cidr($ip, $range)) {{ - return strtoupper($country); - }} + foreach ($ranges as $cidr) {{ + if (strpos($cidr, '/') === false) continue; + list($subnet, $mask) = explode('/', $cidr); + $subnet_long = ip2long($subnet); + if ($subnet_long === false) continue; + $mask_long = -1 << (32 - (int)$mask); + if (($ip_long & $mask_long) === ($subnet_long & $mask_long)) {{ + return strtoupper($country); }} }} }} - return 'XX'; // Unknown country + return 'XX'; // Unknown (cache not loaded or IP not found) }} // === Bot IP Ranges (für getarnte Bots) === @@ -2140,6 +2227,8 @@ $user_agent = $_SERVER['HTTP_USER_AGENT'] ?? ''; $uri = $_SERVER['REQUEST_URI'] ?? '/'; // Ensure directories exist +if (!is_dir($ratelimit_dir)) @mkdir($ratelimit_dir, 0777, true); +if (!is_dir($country_cache_dir)) @mkdir($country_cache_dir, 0777, true); // Always needed for country detection if ($bot_mode) {{ if (!is_dir($bot_bans_dir)) @mkdir($bot_bans_dir, 0777, true); if (!is_dir($bot_counts_dir)) @mkdir($bot_counts_dir, 0777, true); @@ -2147,7 +2236,6 @@ if ($bot_mode) {{ if ($country_mode) {{ if (!is_dir($country_bans_dir)) @mkdir($country_bans_dir, 0777, true); if (!is_dir($country_counts_dir)) @mkdir($country_counts_dir, 0777, true); - if (!is_dir($country_cache_dir)) @mkdir($country_cache_dir, 0777, true); }} // === IP Ban/Whitelist Files === @@ -2212,50 +2300,40 @@ if ($is_whitelisted) {{ return; }} -// === Country Detection === +// === Country Detection (local cache only - no HTTP during page load!) === function get_country_for_ip($ip, $country_cache_dir) {{ + // Common countries to check (ordered by traffic likelihood) $countries = ['de', 'at', 'ch', 'us', 'gb', 'fr', 'nl', 'it', 'es', 'pl', 'be', 'se', 'no', 'dk', 'fi', 'ru', 'cn', 'jp', 'kr', 'in', 'br', 'au', 'ca', 'ua', 'cz', 'pt', 'ie', 'gr', 'hu', 'ro', 'bg', 'hr', 'sk', 'si', 'lt', 'lv', 'ee', 'lu', 'mt', 'cy', 'tr', 'il', 'za', 'mx', 'ar', 'cl', 'co', 'pe', 've', 'eg', 'ng', 'ke', 'ma', 'tn', 'pk', 'bd', 'vn', 'th', 'my', 'sg', 'id', 'ph', 'tw', 'hk', 'nz', 'ae', 'sa', 'qa', 'kw', 'bh', 'om', 'ir', 'iq']; + $ip_long = ip2long($ip); + if ($ip_long === false) return 'XX'; + foreach ($countries as $country) {{ $cache_file = "$country_cache_dir/$country.ranges"; - $ranges = []; - if (file_exists($cache_file) && (time() - filemtime($cache_file)) < 86400) {{ - $ranges = @unserialize(@file_get_contents($cache_file)); - }} + // Only read from cache - NO HTTP requests during page load! + if (!file_exists($cache_file)) continue; - if (empty($ranges) || !is_array($ranges)) {{ - $url = "https://www.ipdeny.com/ipblocks/data/aggregated/$country-aggregated.zone"; - $ctx = stream_context_create(['http' => ['timeout' => 10]]); - $content = @file_get_contents($url, false, $ctx); - if ($content !== false) {{ - $ranges = []; - foreach (explode("\\n", trim($content)) as $line) {{ - $line = trim($line); - if (!empty($line) && strpos($line, '/') !== false) {{ - $ranges[] = $line; - }} - }} - if (count($ranges) > 100) {{ - @file_put_contents($cache_file, serialize($ranges)); - }} - }} - }} + $ranges = @unserialize(@file_get_contents($cache_file)); + if (empty($ranges) || !is_array($ranges)) continue; - if (!empty($ranges)) {{ - foreach ($ranges as $range) {{ - if (ip_in_cidr($ip, $range)) {{ - return strtoupper($country); - }} + foreach ($ranges as $cidr) {{ + if (strpos($cidr, '/') === false) continue; + list($subnet, $mask) = explode('/', $cidr); + $subnet_long = ip2long($subnet); + if ($subnet_long === false) continue; + $mask_long = -1 << (32 - (int)$mask); + if (($ip_long & $mask_long) === ($subnet_long & $mask_long)) {{ + return strtoupper($country); }} }} }} - return 'XX'; + return 'XX'; // Unknown (cache not loaded or IP not found) }} // === Bot IP Ranges === @@ -2644,6 +2722,131 @@ def get_active_shops() -> List[str]: return active +# ============================================================================= +# COUNTRY IP RANGES DOWNLOAD (ipdeny.com) +# ============================================================================= +COUNTRY_CODES = [ + 'de', 'at', 'ch', 'us', 'gb', 'fr', 'nl', 'it', 'es', 'pl', 'be', 'se', 'no', 'dk', 'fi', + 'ru', 'cn', 'jp', 'kr', 'in', 'br', 'au', 'ca', 'ua', 'cz', 'pt', 'ie', 'gr', 'hu', 'ro', + 'bg', 'hr', 'sk', 'si', 'lt', 'lv', 'ee', 'lu', 'mt', 'cy', 'tr', 'il', 'za', 'mx', 'ar', + 'cl', 'co', 'pe', 've', 'eg', 'ng', 'ke', 'ma', 'tn', 'pk', 'bd', 'vn', 'th', 'my', 'sg', + 'id', 'ph', 'tw', 'hk', 'nz', 'ae', 'sa', 'qa', 'kw', 'bh', 'om', 'ir', 'iq' +] + + +def download_country_ranges(force: bool = False) -> int: + """ + Lädt IP-Ranges für alle Länder von ipdeny.com herunter. + Speichert NUR in den GLOBALEN Cache (world-readable für PHP via Symlink). + + Args: + force: Cache ignorieren und neu laden + + Returns: + Anzahl der erfolgreich geladenen Länder + """ + global _country_ranges_cache, _country_cache_loaded + + # Globales Cache-Verzeichnis erstellen (world-readable) + os.makedirs(GLOBAL_COUNTRY_CACHE_DIR, exist_ok=True) + os.chmod(GLOBAL_COUNTRY_CACHE_DIR, 0o755) + + downloaded = 0 + + for country in COUNTRY_CODES: + global_cache_file = os.path.join(GLOBAL_COUNTRY_CACHE_DIR, f"{country}.ranges") + + # Prüfe ob globaler Cache existiert und aktuell ist (24h) + if not force and os.path.isfile(global_cache_file): + try: + if time.time() - os.path.getmtime(global_cache_file) < 86400: + downloaded += 1 + continue + except: + pass + + # Download von ipdeny.com + url = f"https://www.ipdeny.com/ipblocks/data/aggregated/{country}-aggregated.zone" + try: + req = urllib.request.Request(url, headers={'User-Agent': 'JTL-WAFi/3.1'}) + with urllib.request.urlopen(req, timeout=10) as response: + content = response.read().decode('utf-8') + + ranges = [] + for line in content.strip().split('\n'): + line = line.strip() + if line and '/' in line: + ranges.append(line) + + if len(ranges) > 100: # Sanity check + # PHP serialize format (for PHP compatibility) + php_serialized = f'a:{len(ranges)}:{{' + ''.join( + f'i:{i};s:{len(r)}:"{r}";' for i, r in enumerate(ranges) + ) + '}' + + # In globalen Cache speichern (world-readable) + with open(global_cache_file, 'w') as f: + f.write(php_serialized) + os.chmod(global_cache_file, 0o644) + + downloaded += 1 + logger.debug(f"Country ranges geladen: {country.upper()} ({len(ranges)} ranges)") + except Exception as e: + logger.debug(f"Fehler beim Laden von {country}: {e}") + + # Globalen Cache neu laden + _country_cache_loaded = False + _load_global_country_cache() + + return downloaded + + +def download_country_ranges_async(): + """Lädt Country-Ranges im Hintergrund in den globalen Cache.""" + def _download(): + count = download_country_ranges(force=False) + logger.info(f"Country IP-Ranges geladen: {count}/{len(COUNTRY_CODES)} Länder") + + thread = threading.Thread(target=_download, daemon=True) + thread.start() + + +def ensure_country_cache_symlink(ratelimit_path: str) -> bool: + """ + Erstellt einen Symlink vom Shop-spezifischen Pfad zum globalen Country-Cache. + + Args: + ratelimit_path: Pfad zum jtl-wafi_ratelimit Verzeichnis des Shops + + Returns: + True wenn erfolgreich, False sonst + """ + symlink_path = os.path.join(ratelimit_path, 'country_cache') + + try: + # Falls bereits existiert + if os.path.islink(symlink_path): + # Prüfe ob Symlink korrekt ist + if os.readlink(symlink_path) == GLOBAL_COUNTRY_CACHE_DIR: + return True + # Falsches Ziel - entfernen und neu erstellen + os.remove(symlink_path) + elif os.path.isdir(symlink_path): + # Altes Verzeichnis existiert - entfernen (war vorher Kopie) + shutil.rmtree(symlink_path) + elif os.path.exists(symlink_path): + os.remove(symlink_path) + + # Symlink erstellen + os.symlink(GLOBAL_COUNTRY_CACHE_DIR, symlink_path) + logger.debug(f"Country-Cache Symlink erstellt: {symlink_path} -> {GLOBAL_COUNTRY_CACHE_DIR}") + return True + + except Exception as e: + logger.error(f"Fehler beim Erstellen des Country-Cache Symlinks: {e}") + return False + + # ============================================================================= # ACTIVATE / DEACTIVATE BLOCKING # ============================================================================= @@ -2806,6 +3009,9 @@ def activate_blocking(shop: str, silent: bool = True, monitor_only=monitor_only ) + # Step 5: Country-Cache Symlink erstellen (zeigt auf globalen Cache) + ensure_country_cache_symlink(ratelimit_path) + if not silent: logger.info(f"Blocking aktiviert für {shop}") @@ -4265,6 +4471,10 @@ def main(): print("❌ Root-Rechte erforderlich!") sys.exit(1) + # Country IP-Ranges laden/initialisieren (im Hintergrund) + logger.info("Initialisiere Country IP-Ranges Cache...") + download_country_ranges_async() + # Agent starten agent = JTLWAFiAgent(dashboard_url=args.url) agent.run()