thomasciesla/JTL-WAFI
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

jtl-wafi-agent.py aktualisiert

Browse Source
This commit is contained in:
thomasciesla
2025-12-30 16:00:30 +01:00
parent 2ac35b119a
commit 7151eeb91b
1 changed files with 305 additions and 95 deletions
Download Patch File Download Diff File Expand all files Collapse all files

376
jtl-wafi-agent.py
View File

@@ -19,6 +19,7 @@ import sys
import json
import time
import socket
import struct
import hashlib
import logging
import asyncio
@@ -86,6 +87,11 @@ IP_API_URL = "http://ip-api.com/json/{ip}?fields=status,country,countryCode,isp,
IP_API_RATE_LIMIT = 45 # Requests pro Minute (Free Tier)
IP_INFO_CACHE_TTL = 86400 # 24 Stunden Cache
# =============================================================================
# GLOBAL COUNTRY CACHE (ipdeny.com - shared between all shops)
# =============================================================================
GLOBAL_COUNTRY_CACHE_DIR = "/var/lib/jtl-wafi/country_cache"
# =============================================================================
# BAN/WHITELIST FILES
# =============================================================================
@@ -1031,17 +1037,107 @@ def restart_php_fpm(domain: str) -> dict:
# =============================================================================
# IP-INFO CACHE (ip-api.com)
# IP-INFO CACHE (ipdeny.com for country, ip-api.com for ISP/ASN)
# =============================================================================
# Global IP-Info Cache
_ip_info_cache: Dict[str, Dict[str, Any]] = {}
_ip_api_last_request = 0.0
_ip_api_request_count = 0
# Global Country Ranges Cache (loaded once from ipdeny.com files)
_country_ranges_cache: Dict[str, List[tuple]] = {} # country -> [(network_int, mask_int), ...]
_country_cache_loaded = False
def _load_global_country_cache():
"""Lädt alle Country-Ranges aus dem globalen Cache."""
global _country_ranges_cache, _country_cache_loaded
if _country_cache_loaded:
return
if not os.path.isdir(GLOBAL_COUNTRY_CACHE_DIR):
logger.debug(f"Global Country Cache nicht vorhanden: {GLOBAL_COUNTRY_CACHE_DIR}")
_country_cache_loaded = True
return
for country in COUNTRY_CODES:
cache_file = os.path.join(GLOBAL_COUNTRY_CACHE_DIR, f"{country}.ranges")
if not os.path.isfile(cache_file):
continue
try:
with open(cache_file, 'r') as f:
content = f.read()
# Parse PHP serialized format: a:N:{i:0;s:LEN:"CIDR";...}
ranges = []
import re
for match in re.finditer(r's:\d+:"([^"]+)"', content):
cidr = match.group(1)
if '/' in cidr:
try:
subnet, mask = cidr.split('/')
subnet_int = struct.unpack('!I', socket.inet_aton(subnet))[0]
mask_int = (0xFFFFFFFF << (32 - int(mask))) & 0xFFFFFFFF
ranges.append((subnet_int, mask_int))
except:
pass
if ranges:
_country_ranges_cache[country.upper()] = ranges
logger.debug(f"Country ranges geladen: {country.upper()} ({len(ranges)} ranges)")
except Exception as e:
logger.debug(f"Fehler beim Laden von {country}: {e}")
_country_cache_loaded = True
logger.info(f"Global Country Cache geladen: {len(_country_ranges_cache)} Länder")
def get_country_for_ip_cached(ip: str) -> str:
"""
Ermittelt das Land für eine IP aus dem globalen ipdeny.com Cache.
Konsistent mit PHP-Templates!
Returns:
2-Letter Country Code (z.B. 'DE') oder 'XX' wenn unbekannt
"""
global _country_ranges_cache
# Cache laden falls noch nicht geschehen
if not _country_cache_loaded:
_load_global_country_cache()
try:
ip_int = struct.unpack('!I', socket.inet_aton(ip))[0]
except:
return 'XX'
# Suche in allen Ländern (häufigste zuerst)
priority_countries = ['DE', 'AT', 'CH', 'US', 'GB', 'FR', 'NL', 'IT', 'ES', 'PL']
for country in priority_countries:
if country in _country_ranges_cache:
for subnet_int, mask_int in _country_ranges_cache[country]:
if (ip_int & mask_int) == (subnet_int & mask_int):
return country
# Restliche Länder
for country, ranges in _country_ranges_cache.items():
if country in priority_countries:
continue
for subnet_int, mask_int in ranges:
if (ip_int & mask_int) == (subnet_int & mask_int):
return country
return 'XX'
def get_ip_info(ip: str) -> Dict[str, Any]:
"""
Holt IP-Informationen von ip-api.com mit Caching.
Holt IP-Informationen mit konsistenter Country-Detection.
- Country: aus ipdeny.com Cache (konsistent mit PHP!)
- ISP/ASN: aus ip-api.com (optional, mit Rate-Limiting)
Returns:
Dict mit country, countryCode, isp, org, as (ASN)
@@ -1054,45 +1150,50 @@ def get_ip_info(ip: str) -> Dict[str, Any]:
if time.time() - cached.get('_cached_at', 0) < IP_INFO_CACHE_TTL:
return cached
# Rate-Limiting
# Country aus ipdeny.com Cache (konsistent mit PHP!)
country_code = get_country_for_ip_cached(ip)
# Basis-Ergebnis mit Country aus lokalem Cache
result = {
'country': country_code if country_code != 'XX' else 'Unknown',
'countryCode': country_code,
'isp': '',
'org': '',
'as': '',
'_cached_at': time.time()
}
# Optional: ISP/ASN von ip-api.com holen (mit Rate-Limiting)
now = time.time()
if now - _ip_api_last_request < 60:
if _ip_api_request_count >= IP_API_RATE_LIMIT:
logger.debug(f"IP-API Rate-Limit erreicht, verwende Cache für {ip}")
return _ip_info_cache.get(ip, {'country': 'Unknown', 'countryCode': 'XX', 'isp': '', 'org': '', 'as': ''})
else:
if now - _ip_api_last_request >= 60:
_ip_api_last_request = now
_ip_api_request_count = 0
if _ip_api_request_count < IP_API_RATE_LIMIT:
try:
url = IP_API_URL.format(ip=ip)
request = urllib.request.Request(
url,
headers={'User-Agent': f'JTL-WAFi-Agent/{VERSION}'}
)
with urllib.request.urlopen(request, timeout=5) as response:
with urllib.request.urlopen(request, timeout=3) as response:
data = json.loads(response.read().decode('utf-8'))
_ip_api_request_count += 1
if data.get('status') == 'success':
result = {
'country': data.get('country', 'Unknown'),
'countryCode': data.get('countryCode', 'XX'),
'isp': data.get('isp', ''),
'org': data.get('org', ''),
'as': data.get('as', ''),
'_cached_at': time.time()
}
_ip_info_cache[ip] = result
return result
else:
logger.debug(f"IP-API Fehler für {ip}: {data.get('message', 'Unknown error')}")
return {'country': 'Unknown', 'countryCode': 'XX', 'isp': '', 'org': '', 'as': ''}
result['isp'] = data.get('isp', '')
result['org'] = data.get('org', '')
result['as'] = data.get('as', '')
# Country NUR überschreiben wenn lokaler Cache 'XX' war
if country_code == 'XX' and data.get('countryCode'):
result['country'] = data.get('country', 'Unknown')
result['countryCode'] = data.get('countryCode', 'XX')
except Exception as e:
logger.debug(f"IP-API Request fehlgeschlagen für {ip}: {e}")
return {'country': 'Unknown', 'countryCode': 'XX', 'isp': '', 'org': '', 'as': ''}
_ip_info_cache[ip] = result
return result
def get_cached_ip_info(ip: str) -> Optional[Dict[str, Any]]:
@@ -1975,54 +2076,40 @@ if (!$is_whitelisted && file_exists($ip_ban_file)) {{
}}
}}
// === Country Detection via IP ===
// === Country Detection (local cache only - no HTTP during page load!) ===
function get_country_for_ip($ip, $country_cache_dir) {{
// Most common countries first for performance
// Common countries to check (ordered by traffic likelihood)
$countries = ['de', 'at', 'ch', 'us', 'gb', 'fr', 'nl', 'it', 'es', 'pl', 'be', 'se', 'no', 'dk', 'fi',
'ru', 'cn', 'jp', 'kr', 'in', 'br', 'au', 'ca', 'ua', 'cz', 'pt', 'ie', 'gr', 'hu', 'ro',
'bg', 'hr', 'sk', 'si', 'lt', 'lv', 'ee', 'lu', 'mt', 'cy', 'tr', 'il', 'za', 'mx', 'ar',
'cl', 'co', 'pe', 've', 'eg', 'ng', 'ke', 'ma', 'tn', 'pk', 'bd', 'vn', 'th', 'my', 'sg',
'id', 'ph', 'tw', 'hk', 'nz', 'ae', 'sa', 'qa', 'kw', 'bh', 'om', 'ir', 'iq'];
$ip_long = ip2long($ip);
if ($ip_long === false) return 'XX';
foreach ($countries as $country) {{
$cache_file = "$country_cache_dir/$country.ranges";
$ranges = [];
// Load from cache or download
if (file_exists($cache_file) && (time() - filemtime($cache_file)) < 86400) {{
// Only read from cache - NO HTTP requests during page load!
if (!file_exists($cache_file)) continue;
$ranges = @unserialize(@file_get_contents($cache_file));
}}
if (empty($ranges) || !is_array($ranges)) continue;
if (empty($ranges) || !is_array($ranges)) {{
// Download fresh from ipdeny.com
$url = "https://www.ipdeny.com/ipblocks/data/aggregated/$country-aggregated.zone";
$ctx = stream_context_create(['http' => ['timeout' => 10]]);
$content = @file_get_contents($url, false, $ctx);
if ($content !== false) {{
$ranges = [];
foreach (explode("\\n", trim($content)) as $line) {{
$line = trim($line);
if (!empty($line) && strpos($line, '/') !== false) {{
$ranges[] = $line;
}}
}}
if (count($ranges) > 100) {{
@file_put_contents($cache_file, serialize($ranges));
}}
}}
}}
// Check if IP is in this country
if (!empty($ranges)) {{
foreach ($ranges as $range) {{
if (ip_in_cidr($ip, $range)) {{
foreach ($ranges as $cidr) {{
if (strpos($cidr, '/') === false) continue;
list($subnet, $mask) = explode('/', $cidr);
$subnet_long = ip2long($subnet);
if ($subnet_long === false) continue;
$mask_long = -1 << (32 - (int)$mask);
if (($ip_long & $mask_long) === ($subnet_long & $mask_long)) {{
return strtoupper($country);
}}
}}
}}
}}
return 'XX'; // Unknown country
return 'XX'; // Unknown (cache not loaded or IP not found)
}}
// === Bot IP Ranges (für getarnte Bots) ===
@@ -2140,6 +2227,8 @@ $user_agent = $_SERVER['HTTP_USER_AGENT'] ?? '';
$uri = $_SERVER['REQUEST_URI'] ?? '/';
// Ensure directories exist
if (!is_dir($ratelimit_dir)) @mkdir($ratelimit_dir, 0777, true);
if (!is_dir($country_cache_dir)) @mkdir($country_cache_dir, 0777, true); // Always needed for country detection
if ($bot_mode) {{
if (!is_dir($bot_bans_dir)) @mkdir($bot_bans_dir, 0777, true);
if (!is_dir($bot_counts_dir)) @mkdir($bot_counts_dir, 0777, true);
@@ -2147,7 +2236,6 @@ if ($bot_mode) {{
if ($country_mode) {{
if (!is_dir($country_bans_dir)) @mkdir($country_bans_dir, 0777, true);
if (!is_dir($country_counts_dir)) @mkdir($country_counts_dir, 0777, true);
if (!is_dir($country_cache_dir)) @mkdir($country_cache_dir, 0777, true);
}}
// === IP Ban/Whitelist Files ===
@@ -2212,50 +2300,40 @@ if ($is_whitelisted) {{
return;
}}
// === Country Detection ===
// === Country Detection (local cache only - no HTTP during page load!) ===
function get_country_for_ip($ip, $country_cache_dir) {{
// Common countries to check (ordered by traffic likelihood)
$countries = ['de', 'at', 'ch', 'us', 'gb', 'fr', 'nl', 'it', 'es', 'pl', 'be', 'se', 'no', 'dk', 'fi',
'ru', 'cn', 'jp', 'kr', 'in', 'br', 'au', 'ca', 'ua', 'cz', 'pt', 'ie', 'gr', 'hu', 'ro',
'bg', 'hr', 'sk', 'si', 'lt', 'lv', 'ee', 'lu', 'mt', 'cy', 'tr', 'il', 'za', 'mx', 'ar',
'cl', 'co', 'pe', 've', 'eg', 'ng', 'ke', 'ma', 'tn', 'pk', 'bd', 'vn', 'th', 'my', 'sg',
'id', 'ph', 'tw', 'hk', 'nz', 'ae', 'sa', 'qa', 'kw', 'bh', 'om', 'ir', 'iq'];
$ip_long = ip2long($ip);
if ($ip_long === false) return 'XX';
foreach ($countries as $country) {{
$cache_file = "$country_cache_dir/$country.ranges";
$ranges = [];
if (file_exists($cache_file) && (time() - filemtime($cache_file)) < 86400) {{
// Only read from cache - NO HTTP requests during page load!
if (!file_exists($cache_file)) continue;
$ranges = @unserialize(@file_get_contents($cache_file));
}}
if (empty($ranges) || !is_array($ranges)) continue;
if (empty($ranges) || !is_array($ranges)) {{
$url = "https://www.ipdeny.com/ipblocks/data/aggregated/$country-aggregated.zone";
$ctx = stream_context_create(['http' => ['timeout' => 10]]);
$content = @file_get_contents($url, false, $ctx);
if ($content !== false) {{
$ranges = [];
foreach (explode("\\n", trim($content)) as $line) {{
$line = trim($line);
if (!empty($line) && strpos($line, '/') !== false) {{
$ranges[] = $line;
}}
}}
if (count($ranges) > 100) {{
@file_put_contents($cache_file, serialize($ranges));
}}
}}
}}
if (!empty($ranges)) {{
foreach ($ranges as $range) {{
if (ip_in_cidr($ip, $range)) {{
foreach ($ranges as $cidr) {{
if (strpos($cidr, '/') === false) continue;
list($subnet, $mask) = explode('/', $cidr);
$subnet_long = ip2long($subnet);
if ($subnet_long === false) continue;
$mask_long = -1 << (32 - (int)$mask);
if (($ip_long & $mask_long) === ($subnet_long & $mask_long)) {{
return strtoupper($country);
}}
}}
}}
}}
return 'XX';
return 'XX'; // Unknown (cache not loaded or IP not found)
}}
// === Bot IP Ranges ===
@@ -2644,6 +2722,131 @@ def get_active_shops() -> List[str]:
return active
# =============================================================================
# COUNTRY IP RANGES DOWNLOAD (ipdeny.com)
# =============================================================================
COUNTRY_CODES = [
'de', 'at', 'ch', 'us', 'gb', 'fr', 'nl', 'it', 'es', 'pl', 'be', 'se', 'no', 'dk', 'fi',
'ru', 'cn', 'jp', 'kr', 'in', 'br', 'au', 'ca', 'ua', 'cz', 'pt', 'ie', 'gr', 'hu', 'ro',
'bg', 'hr', 'sk', 'si', 'lt', 'lv', 'ee', 'lu', 'mt', 'cy', 'tr', 'il', 'za', 'mx', 'ar',
'cl', 'co', 'pe', 've', 'eg', 'ng', 'ke', 'ma', 'tn', 'pk', 'bd', 'vn', 'th', 'my', 'sg',
'id', 'ph', 'tw', 'hk', 'nz', 'ae', 'sa', 'qa', 'kw', 'bh', 'om', 'ir', 'iq'
]
def download_country_ranges(force: bool = False) -> int:
"""
Lädt IP-Ranges für alle Länder von ipdeny.com herunter.
Speichert NUR in den GLOBALEN Cache (world-readable für PHP via Symlink).
Args:
force: Cache ignorieren und neu laden
Returns:
Anzahl der erfolgreich geladenen Länder
"""
global _country_ranges_cache, _country_cache_loaded
# Globales Cache-Verzeichnis erstellen (world-readable)
os.makedirs(GLOBAL_COUNTRY_CACHE_DIR, exist_ok=True)
os.chmod(GLOBAL_COUNTRY_CACHE_DIR, 0o755)
downloaded = 0
for country in COUNTRY_CODES:
global_cache_file = os.path.join(GLOBAL_COUNTRY_CACHE_DIR, f"{country}.ranges")
# Prüfe ob globaler Cache existiert und aktuell ist (24h)
if not force and os.path.isfile(global_cache_file):
try:
if time.time() - os.path.getmtime(global_cache_file) < 86400:
downloaded += 1
continue
except:
pass
# Download von ipdeny.com
url = f"https://www.ipdeny.com/ipblocks/data/aggregated/{country}-aggregated.zone"
try:
req = urllib.request.Request(url, headers={'User-Agent': 'JTL-WAFi/3.1'})
with urllib.request.urlopen(req, timeout=10) as response:
content = response.read().decode('utf-8')
ranges = []
for line in content.strip().split('\n'):
line = line.strip()
if line and '/' in line:
ranges.append(line)
if len(ranges) > 100: # Sanity check
# PHP serialize format (for PHP compatibility)
php_serialized = f'a:{len(ranges)}:{{' + ''.join(
f'i:{i};s:{len(r)}:"{r}";' for i, r in enumerate(ranges)
) + '}'
# In globalen Cache speichern (world-readable)
with open(global_cache_file, 'w') as f:
f.write(php_serialized)
os.chmod(global_cache_file, 0o644)
downloaded += 1
logger.debug(f"Country ranges geladen: {country.upper()} ({len(ranges)} ranges)")
except Exception as e:
logger.debug(f"Fehler beim Laden von {country}: {e}")
# Globalen Cache neu laden
_country_cache_loaded = False
_load_global_country_cache()
return downloaded
def download_country_ranges_async():
"""Lädt Country-Ranges im Hintergrund in den globalen Cache."""
def _download():
count = download_country_ranges(force=False)
logger.info(f"Country IP-Ranges geladen: {count}/{len(COUNTRY_CODES)} Länder")
thread = threading.Thread(target=_download, daemon=True)
thread.start()
def ensure_country_cache_symlink(ratelimit_path: str) -> bool:
"""
Erstellt einen Symlink vom Shop-spezifischen Pfad zum globalen Country-Cache.
Args:
ratelimit_path: Pfad zum jtl-wafi_ratelimit Verzeichnis des Shops
Returns:
True wenn erfolgreich, False sonst
"""
symlink_path = os.path.join(ratelimit_path, 'country_cache')
try:
# Falls bereits existiert
if os.path.islink(symlink_path):
# Prüfe ob Symlink korrekt ist
if os.readlink(symlink_path) == GLOBAL_COUNTRY_CACHE_DIR:
return True
# Falsches Ziel - entfernen und neu erstellen
os.remove(symlink_path)
elif os.path.isdir(symlink_path):
# Altes Verzeichnis existiert - entfernen (war vorher Kopie)
shutil.rmtree(symlink_path)
elif os.path.exists(symlink_path):
os.remove(symlink_path)
# Symlink erstellen
os.symlink(GLOBAL_COUNTRY_CACHE_DIR, symlink_path)
logger.debug(f"Country-Cache Symlink erstellt: {symlink_path} -> {GLOBAL_COUNTRY_CACHE_DIR}")
return True
except Exception as e:
logger.error(f"Fehler beim Erstellen des Country-Cache Symlinks: {e}")
return False
# =============================================================================
# ACTIVATE / DEACTIVATE BLOCKING
# =============================================================================
@@ -2806,6 +3009,9 @@ def activate_blocking(shop: str, silent: bool = True,
monitor_only=monitor_only
)
# Step 5: Country-Cache Symlink erstellen (zeigt auf globalen Cache)
ensure_country_cache_symlink(ratelimit_path)
if not silent:
logger.info(f"Blocking aktiviert für {shop}")
@@ -4265,6 +4471,10 @@ def main():
print("❌ Root-Rechte erforderlich!")
sys.exit(1)
# Country IP-Ranges laden/initialisieren (im Hintergrund)
logger.info("Initialisiere Country IP-Ranges Cache...")
download_country_ranges_async()
# Agent starten
agent = JTLWAFiAgent(dashboard_url=args.url)
agent.run()