jtl-wafi-agent.py aktualisiert

jtl-wafi-agent.py

@@ -19,6 +19,7 @@ import sys
 import json
 import time
 import socket
+import struct
 import hashlib
 import logging
 import asyncio
@@ -86,6 +87,11 @@ IP_API_URL = "http://ip-api.com/json/{ip}?fields=status,country,countryCode,isp,
 IP_API_RATE_LIMIT = 45  # Requests pro Minute (Free Tier)
 IP_INFO_CACHE_TTL = 86400  # 24 Stunden Cache
 
+# =============================================================================
+# GLOBAL COUNTRY CACHE (ipdeny.com - shared between all shops)
+# =============================================================================
+GLOBAL_COUNTRY_CACHE_DIR = "/var/lib/jtl-wafi/country_cache"
+
 # =============================================================================
 # BAN/WHITELIST FILES
 # =============================================================================
@@ -1031,17 +1037,107 @@ def restart_php_fpm(domain: str) -> dict:
 
 
 # =============================================================================
-# IP-INFO CACHE (ip-api.com)
+# IP-INFO CACHE (ipdeny.com for country, ip-api.com for ISP/ASN)
 # =============================================================================
 # Global IP-Info Cache
 _ip_info_cache: Dict[str, Dict[str, Any]] = {}
 _ip_api_last_request = 0.0
 _ip_api_request_count = 0
 
+# Global Country Ranges Cache (loaded once from ipdeny.com files)
+_country_ranges_cache: Dict[str, List[tuple]] = {}  # country -> [(network_int, mask_int), ...]
+_country_cache_loaded = False
+
+
+def _load_global_country_cache():
+    """Lädt alle Country-Ranges aus dem globalen Cache."""
+    global _country_ranges_cache, _country_cache_loaded
+
+    if _country_cache_loaded:
+        return
+
+    if not os.path.isdir(GLOBAL_COUNTRY_CACHE_DIR):
+        logger.debug(f"Global Country Cache nicht vorhanden: {GLOBAL_COUNTRY_CACHE_DIR}")
+        _country_cache_loaded = True
+        return
+
+    for country in COUNTRY_CODES:
+        cache_file = os.path.join(GLOBAL_COUNTRY_CACHE_DIR, f"{country}.ranges")
+        if not os.path.isfile(cache_file):
+            continue
+
+        try:
+            with open(cache_file, 'r') as f:
+                content = f.read()
+
+            # Parse PHP serialized format: a:N:{i:0;s:LEN:"CIDR";...}
+            ranges = []
+            import re
+            for match in re.finditer(r's:\d+:"([^"]+)"', content):
+                cidr = match.group(1)
+                if '/' in cidr:
+                    try:
+                        subnet, mask = cidr.split('/')
+                        subnet_int = struct.unpack('!I', socket.inet_aton(subnet))[0]
+                        mask_int = (0xFFFFFFFF << (32 - int(mask))) & 0xFFFFFFFF
+                        ranges.append((subnet_int, mask_int))
+                    except:
+                        pass
+
+            if ranges:
+                _country_ranges_cache[country.upper()] = ranges
+                logger.debug(f"Country ranges geladen: {country.upper()} ({len(ranges)} ranges)")
+        except Exception as e:
+            logger.debug(f"Fehler beim Laden von {country}: {e}")
+
+    _country_cache_loaded = True
+    logger.info(f"Global Country Cache geladen: {len(_country_ranges_cache)} Länder")
+
+
+def get_country_for_ip_cached(ip: str) -> str:
+    """
+    Ermittelt das Land für eine IP aus dem globalen ipdeny.com Cache.
+    Konsistent mit PHP-Templates!
+
+    Returns:
+        2-Letter Country Code (z.B. 'DE') oder 'XX' wenn unbekannt
+    """
+    global _country_ranges_cache
+
+    # Cache laden falls noch nicht geschehen
+    if not _country_cache_loaded:
+        _load_global_country_cache()
+
+    try:
+        ip_int = struct.unpack('!I', socket.inet_aton(ip))[0]
+    except:
+        return 'XX'
+
+    # Suche in allen Ländern (häufigste zuerst)
+    priority_countries = ['DE', 'AT', 'CH', 'US', 'GB', 'FR', 'NL', 'IT', 'ES', 'PL']
+
+    for country in priority_countries:
+        if country in _country_ranges_cache:
+            for subnet_int, mask_int in _country_ranges_cache[country]:
+                if (ip_int & mask_int) == (subnet_int & mask_int):
+                    return country
+
+    # Restliche Länder
+    for country, ranges in _country_ranges_cache.items():
+        if country in priority_countries:
+            continue
+        for subnet_int, mask_int in ranges:
+            if (ip_int & mask_int) == (subnet_int & mask_int):
+                return country
+
+    return 'XX'
+
 
 def get_ip_info(ip: str) -> Dict[str, Any]:
     """
-    Holt IP-Informationen von ip-api.com mit Caching.
+    Holt IP-Informationen mit konsistenter Country-Detection.
+    - Country: aus ipdeny.com Cache (konsistent mit PHP!)
+    - ISP/ASN: aus ip-api.com (optional, mit Rate-Limiting)
 
     Returns:
         Dict mit country, countryCode, isp, org, as (ASN)
@@ -1054,45 +1150,50 @@ def get_ip_info(ip: str) -> Dict[str, Any]:
         if time.time() - cached.get('_cached_at', 0) < IP_INFO_CACHE_TTL:
             return cached
 
-    # Rate-Limiting
+    # Country aus ipdeny.com Cache (konsistent mit PHP!)
+    country_code = get_country_for_ip_cached(ip)
+
+    # Basis-Ergebnis mit Country aus lokalem Cache
+    result = {
+        'country': country_code if country_code != 'XX' else 'Unknown',
+        'countryCode': country_code,
+        'isp': '',
+        'org': '',
+        'as': '',
+        '_cached_at': time.time()
+    }
+
+    # Optional: ISP/ASN von ip-api.com holen (mit Rate-Limiting)
     now = time.time()
-    if now - _ip_api_last_request < 60:
+    if now - _ip_api_last_request >= 60:
-        if _ip_api_request_count >= IP_API_RATE_LIMIT:
-            logger.debug(f"IP-API Rate-Limit erreicht, verwende Cache für {ip}")
-            return _ip_info_cache.get(ip, {'country': 'Unknown', 'countryCode': 'XX', 'isp': '', 'org': '', 'as': ''})
-    else:
         _ip_api_last_request = now
         _ip_api_request_count = 0
 
-    try:
+    if _ip_api_request_count < IP_API_RATE_LIMIT:
-        url = IP_API_URL.format(ip=ip)
+        try:
-        request = urllib.request.Request(
+            url = IP_API_URL.format(ip=ip)
-            url,
+            request = urllib.request.Request(
-            headers={'User-Agent': f'JTL-WAFi-Agent/{VERSION}'}
+                url,
-        )
+                headers={'User-Agent': f'JTL-WAFi-Agent/{VERSION}'}
-        with urllib.request.urlopen(request, timeout=5) as response:
+            )
-            data = json.loads(response.read().decode('utf-8'))
+            with urllib.request.urlopen(request, timeout=3) as response:
+                data = json.loads(response.read().decode('utf-8'))
 
-        _ip_api_request_count += 1
+            _ip_api_request_count += 1
 
-        if data.get('status') == 'success':
+            if data.get('status') == 'success':
-            result = {
+                result['isp'] = data.get('isp', '')
-                'country': data.get('country', 'Unknown'),
+                result['org'] = data.get('org', '')
-                'countryCode': data.get('countryCode', 'XX'),
+                result['as'] = data.get('as', '')
-                'isp': data.get('isp', ''),
+                # Country NUR überschreiben wenn lokaler Cache 'XX' war
-                'org': data.get('org', ''),
+                if country_code == 'XX' and data.get('countryCode'):
-                'as': data.get('as', ''),
+                    result['country'] = data.get('country', 'Unknown')
-                '_cached_at': time.time()
+                    result['countryCode'] = data.get('countryCode', 'XX')
-            }
+        except Exception as e:
-            _ip_info_cache[ip] = result
+            logger.debug(f"IP-API Request fehlgeschlagen für {ip}: {e}")
-            return result
-        else:
-            logger.debug(f"IP-API Fehler für {ip}: {data.get('message', 'Unknown error')}")
-            return {'country': 'Unknown', 'countryCode': 'XX', 'isp': '', 'org': '', 'as': ''}
 
-    except Exception as e:
+    _ip_info_cache[ip] = result
-        logger.debug(f"IP-API Request fehlgeschlagen für {ip}: {e}")
+    return result
-        return {'country': 'Unknown', 'countryCode': 'XX', 'isp': '', 'org': '', 'as': ''}
 
 
 def get_cached_ip_info(ip: str) -> Optional[Dict[str, Any]]:
@@ -1975,54 +2076,40 @@ if (!$is_whitelisted && file_exists($ip_ban_file)) {{
     }}
 }}
 
-// === Country Detection via IP ===
+// === Country Detection (local cache only - no HTTP during page load!) ===
 function get_country_for_ip($ip, $country_cache_dir) {{
-    // Most common countries first for performance
+    // Common countries to check (ordered by traffic likelihood)
     $countries = ['de', 'at', 'ch', 'us', 'gb', 'fr', 'nl', 'it', 'es', 'pl', 'be', 'se', 'no', 'dk', 'fi',
                   'ru', 'cn', 'jp', 'kr', 'in', 'br', 'au', 'ca', 'ua', 'cz', 'pt', 'ie', 'gr', 'hu', 'ro',
                   'bg', 'hr', 'sk', 'si', 'lt', 'lv', 'ee', 'lu', 'mt', 'cy', 'tr', 'il', 'za', 'mx', 'ar',
                   'cl', 'co', 'pe', 've', 'eg', 'ng', 'ke', 'ma', 'tn', 'pk', 'bd', 'vn', 'th', 'my', 'sg',
                   'id', 'ph', 'tw', 'hk', 'nz', 'ae', 'sa', 'qa', 'kw', 'bh', 'om', 'ir', 'iq'];
 
+    $ip_long = ip2long($ip);
+    if ($ip_long === false) return 'XX';
+
     foreach ($countries as $country) {{
         $cache_file = "$country_cache_dir/$country.ranges";
-        $ranges = [];
 
-        // Load from cache or download
+        // Only read from cache - NO HTTP requests during page load!
-        if (file_exists($cache_file) && (time() - filemtime($cache_file)) < 86400) {{
+        if (!file_exists($cache_file)) continue;
-            $ranges = @unserialize(@file_get_contents($cache_file));
-        }}
 
-        if (empty($ranges) || !is_array($ranges)) {{
+        $ranges = @unserialize(@file_get_contents($cache_file));
-            // Download fresh from ipdeny.com
+        if (empty($ranges) || !is_array($ranges)) continue;
-            $url = "https://www.ipdeny.com/ipblocks/data/aggregated/$country-aggregated.zone";
-            $ctx = stream_context_create(['http' => ['timeout' => 10]]);
-            $content = @file_get_contents($url, false, $ctx);
-            if ($content !== false) {{
-                $ranges = [];
-                foreach (explode("\\n", trim($content)) as $line) {{
-                    $line = trim($line);
-                    if (!empty($line) && strpos($line, '/') !== false) {{
-                        $ranges[] = $line;
-                    }}
-                }}
-                if (count($ranges) > 100) {{
-                    @file_put_contents($cache_file, serialize($ranges));
-                }}
-            }}
-        }}
 
-        // Check if IP is in this country
+        foreach ($ranges as $cidr) {{
-        if (!empty($ranges)) {{
+            if (strpos($cidr, '/') === false) continue;
-            foreach ($ranges as $range) {{
+            list($subnet, $mask) = explode('/', $cidr);
-                if (ip_in_cidr($ip, $range)) {{
+            $subnet_long = ip2long($subnet);
-                    return strtoupper($country);
+            if ($subnet_long === false) continue;
-                }}
+            $mask_long = -1 << (32 - (int)$mask);
+            if (($ip_long & $mask_long) === ($subnet_long & $mask_long)) {{
+                return strtoupper($country);
             }}
         }}
     }}
 
-    return 'XX';  // Unknown country
+    return 'XX';  // Unknown (cache not loaded or IP not found)
 }}
 
 // === Bot IP Ranges (für getarnte Bots) ===
@@ -2140,6 +2227,8 @@ $user_agent = $_SERVER['HTTP_USER_AGENT'] ?? '';
 $uri = $_SERVER['REQUEST_URI'] ?? '/';
 
 // Ensure directories exist
+if (!is_dir($ratelimit_dir)) @mkdir($ratelimit_dir, 0777, true);
+if (!is_dir($country_cache_dir)) @mkdir($country_cache_dir, 0777, true);  // Always needed for country detection
 if ($bot_mode) {{
     if (!is_dir($bot_bans_dir)) @mkdir($bot_bans_dir, 0777, true);
     if (!is_dir($bot_counts_dir)) @mkdir($bot_counts_dir, 0777, true);
@@ -2147,7 +2236,6 @@ if ($bot_mode) {{
 if ($country_mode) {{
     if (!is_dir($country_bans_dir)) @mkdir($country_bans_dir, 0777, true);
     if (!is_dir($country_counts_dir)) @mkdir($country_counts_dir, 0777, true);
-    if (!is_dir($country_cache_dir)) @mkdir($country_cache_dir, 0777, true);
 }}
 
 // === IP Ban/Whitelist Files ===
@@ -2212,50 +2300,40 @@ if ($is_whitelisted) {{
     return;
 }}
 
-// === Country Detection ===
+// === Country Detection (local cache only - no HTTP during page load!) ===
 function get_country_for_ip($ip, $country_cache_dir) {{
+    // Common countries to check (ordered by traffic likelihood)
     $countries = ['de', 'at', 'ch', 'us', 'gb', 'fr', 'nl', 'it', 'es', 'pl', 'be', 'se', 'no', 'dk', 'fi',
                   'ru', 'cn', 'jp', 'kr', 'in', 'br', 'au', 'ca', 'ua', 'cz', 'pt', 'ie', 'gr', 'hu', 'ro',
                   'bg', 'hr', 'sk', 'si', 'lt', 'lv', 'ee', 'lu', 'mt', 'cy', 'tr', 'il', 'za', 'mx', 'ar',
                   'cl', 'co', 'pe', 've', 'eg', 'ng', 'ke', 'ma', 'tn', 'pk', 'bd', 'vn', 'th', 'my', 'sg',
                   'id', 'ph', 'tw', 'hk', 'nz', 'ae', 'sa', 'qa', 'kw', 'bh', 'om', 'ir', 'iq'];
 
+    $ip_long = ip2long($ip);
+    if ($ip_long === false) return 'XX';
+
     foreach ($countries as $country) {{
         $cache_file = "$country_cache_dir/$country.ranges";
-        $ranges = [];
 
-        if (file_exists($cache_file) && (time() - filemtime($cache_file)) < 86400) {{
+        // Only read from cache - NO HTTP requests during page load!
-            $ranges = @unserialize(@file_get_contents($cache_file));
+        if (!file_exists($cache_file)) continue;
-        }}
 
-        if (empty($ranges) || !is_array($ranges)) {{
+        $ranges = @unserialize(@file_get_contents($cache_file));
-            $url = "https://www.ipdeny.com/ipblocks/data/aggregated/$country-aggregated.zone";
+        if (empty($ranges) || !is_array($ranges)) continue;
-            $ctx = stream_context_create(['http' => ['timeout' => 10]]);
-            $content = @file_get_contents($url, false, $ctx);
-            if ($content !== false) {{
-                $ranges = [];
-                foreach (explode("\\n", trim($content)) as $line) {{
-                    $line = trim($line);
-                    if (!empty($line) && strpos($line, '/') !== false) {{
-                        $ranges[] = $line;
-                    }}
-                }}
-                if (count($ranges) > 100) {{
-                    @file_put_contents($cache_file, serialize($ranges));
-                }}
-            }}
-        }}
 
-        if (!empty($ranges)) {{
+        foreach ($ranges as $cidr) {{
-            foreach ($ranges as $range) {{
+            if (strpos($cidr, '/') === false) continue;
-                if (ip_in_cidr($ip, $range)) {{
+            list($subnet, $mask) = explode('/', $cidr);
-                    return strtoupper($country);
+            $subnet_long = ip2long($subnet);
-                }}
+            if ($subnet_long === false) continue;
+            $mask_long = -1 << (32 - (int)$mask);
+            if (($ip_long & $mask_long) === ($subnet_long & $mask_long)) {{
+                return strtoupper($country);
             }}
         }}
     }}
 
-    return 'XX';
+    return 'XX';  // Unknown (cache not loaded or IP not found)
 }}
 
 // === Bot IP Ranges ===
@@ -2644,6 +2722,131 @@ def get_active_shops() -> List[str]:
     return active
 
 
+# =============================================================================
+# COUNTRY IP RANGES DOWNLOAD (ipdeny.com)
+# =============================================================================
+COUNTRY_CODES = [
+    'de', 'at', 'ch', 'us', 'gb', 'fr', 'nl', 'it', 'es', 'pl', 'be', 'se', 'no', 'dk', 'fi',
+    'ru', 'cn', 'jp', 'kr', 'in', 'br', 'au', 'ca', 'ua', 'cz', 'pt', 'ie', 'gr', 'hu', 'ro',
+    'bg', 'hr', 'sk', 'si', 'lt', 'lv', 'ee', 'lu', 'mt', 'cy', 'tr', 'il', 'za', 'mx', 'ar',
+    'cl', 'co', 'pe', 've', 'eg', 'ng', 'ke', 'ma', 'tn', 'pk', 'bd', 'vn', 'th', 'my', 'sg',
+    'id', 'ph', 'tw', 'hk', 'nz', 'ae', 'sa', 'qa', 'kw', 'bh', 'om', 'ir', 'iq'
+]
+
+
+def download_country_ranges(force: bool = False) -> int:
+    """
+    Lädt IP-Ranges für alle Länder von ipdeny.com herunter.
+    Speichert NUR in den GLOBALEN Cache (world-readable für PHP via Symlink).
+
+    Args:
+        force: Cache ignorieren und neu laden
+
+    Returns:
+        Anzahl der erfolgreich geladenen Länder
+    """
+    global _country_ranges_cache, _country_cache_loaded
+
+    # Globales Cache-Verzeichnis erstellen (world-readable)
+    os.makedirs(GLOBAL_COUNTRY_CACHE_DIR, exist_ok=True)
+    os.chmod(GLOBAL_COUNTRY_CACHE_DIR, 0o755)
+
+    downloaded = 0
+
+    for country in COUNTRY_CODES:
+        global_cache_file = os.path.join(GLOBAL_COUNTRY_CACHE_DIR, f"{country}.ranges")
+
+        # Prüfe ob globaler Cache existiert und aktuell ist (24h)
+        if not force and os.path.isfile(global_cache_file):
+            try:
+                if time.time() - os.path.getmtime(global_cache_file) < 86400:
+                    downloaded += 1
+                    continue
+            except:
+                pass
+
+        # Download von ipdeny.com
+        url = f"https://www.ipdeny.com/ipblocks/data/aggregated/{country}-aggregated.zone"
+        try:
+            req = urllib.request.Request(url, headers={'User-Agent': 'JTL-WAFi/3.1'})
+            with urllib.request.urlopen(req, timeout=10) as response:
+                content = response.read().decode('utf-8')
+
+            ranges = []
+            for line in content.strip().split('\n'):
+                line = line.strip()
+                if line and '/' in line:
+                    ranges.append(line)
+
+            if len(ranges) > 100:  # Sanity check
+                # PHP serialize format (for PHP compatibility)
+                php_serialized = f'a:{len(ranges)}:{{' + ''.join(
+                    f'i:{i};s:{len(r)}:"{r}";' for i, r in enumerate(ranges)
+                ) + '}'
+
+                # In globalen Cache speichern (world-readable)
+                with open(global_cache_file, 'w') as f:
+                    f.write(php_serialized)
+                os.chmod(global_cache_file, 0o644)
+
+                downloaded += 1
+                logger.debug(f"Country ranges geladen: {country.upper()} ({len(ranges)} ranges)")
+        except Exception as e:
+            logger.debug(f"Fehler beim Laden von {country}: {e}")
+
+    # Globalen Cache neu laden
+    _country_cache_loaded = False
+    _load_global_country_cache()
+
+    return downloaded
+
+
+def download_country_ranges_async():
+    """Lädt Country-Ranges im Hintergrund in den globalen Cache."""
+    def _download():
+        count = download_country_ranges(force=False)
+        logger.info(f"Country IP-Ranges geladen: {count}/{len(COUNTRY_CODES)} Länder")
+
+    thread = threading.Thread(target=_download, daemon=True)
+    thread.start()
+
+
+def ensure_country_cache_symlink(ratelimit_path: str) -> bool:
+    """
+    Erstellt einen Symlink vom Shop-spezifischen Pfad zum globalen Country-Cache.
+
+    Args:
+        ratelimit_path: Pfad zum jtl-wafi_ratelimit Verzeichnis des Shops
+
+    Returns:
+        True wenn erfolgreich, False sonst
+    """
+    symlink_path = os.path.join(ratelimit_path, 'country_cache')
+
+    try:
+        # Falls bereits existiert
+        if os.path.islink(symlink_path):
+            # Prüfe ob Symlink korrekt ist
+            if os.readlink(symlink_path) == GLOBAL_COUNTRY_CACHE_DIR:
+                return True
+            # Falsches Ziel - entfernen und neu erstellen
+            os.remove(symlink_path)
+        elif os.path.isdir(symlink_path):
+            # Altes Verzeichnis existiert - entfernen (war vorher Kopie)
+            shutil.rmtree(symlink_path)
+        elif os.path.exists(symlink_path):
+            os.remove(symlink_path)
+
+        # Symlink erstellen
+        os.symlink(GLOBAL_COUNTRY_CACHE_DIR, symlink_path)
+        logger.debug(f"Country-Cache Symlink erstellt: {symlink_path} -> {GLOBAL_COUNTRY_CACHE_DIR}")
+        return True
+
+    except Exception as e:
+        logger.error(f"Fehler beim Erstellen des Country-Cache Symlinks: {e}")
+        return False
+
+
 # =============================================================================
 # ACTIVATE / DEACTIVATE BLOCKING
 # =============================================================================
@@ -2806,6 +3009,9 @@ def activate_blocking(shop: str, silent: bool = True,
             monitor_only=monitor_only
         )
 
+        # Step 5: Country-Cache Symlink erstellen (zeigt auf globalen Cache)
+        ensure_country_cache_symlink(ratelimit_path)
+
         if not silent:
             logger.info(f"Blocking aktiviert für {shop}")
 
@@ -4265,6 +4471,10 @@ def main():
         print("❌ Root-Rechte erforderlich!")
         sys.exit(1)
 
+    # Country IP-Ranges laden/initialisieren (im Hintergrund)
+    logger.info("Initialisiere Country IP-Ranges Cache...")
+    download_country_ranges_async()
+
     # Agent starten
     agent = JTLWAFiAgent(dashboard_url=args.url)
     agent.run()