#!/usr/bin/env python3 """ JTL-WAFi Agent v3.0.0 - WebSocket Real-Time Agent Features: - WebSocket-basierte Echtzeit-Kommunikation - Integrierter Shop-Manager (kein separates Script mehr) - Token-basierte Authentifizierung - On-Demand Live-Log-Streaming - Auto-Reconnect mit Exponential Backoff - Auto-Update: Agent über Git aktualisieren - PHP-FPM Restart: OPcache automatisch leeren v3.0.0: Auto-Update, FPM-Restart, Country-Detection Bugfix """ import os import sys import json import time import socket import struct import hashlib import logging import asyncio import ssl import shutil import subprocess import re import ipaddress import signal import platform import threading import fcntl import urllib.request import urllib.error from datetime import datetime, timedelta, timezone from pathlib import Path from collections import Counter from typing import Optional, Dict, Any, List, Set, Callable from logging.handlers import RotatingFileHandler # ============================================================================= # VERSION # ============================================================================= VERSION = "3.1.0" # ============================================================================= # PFADE - AGENT # ============================================================================= CONFIG_DIR = "/etc/jtl-wafi" TOKEN_FILE = "/etc/jtl-wafi/token" LOG_FILE = "/var/log/jtl-wafi.log" # ============================================================================= # PFADE - SHOPS # ============================================================================= VHOSTS_DIR = "/var/www/vhosts" ACTIVE_SHOPS_FILE = "/var/lib/jtl-wafi/active_shops.json" # ============================================================================= # SHOP-DATEIEN # ============================================================================= BACKUP_SUFFIX = ".jtl-wafi_backup" BLOCKING_FILE = "jtl-wafi.php" CACHE_FILE = "jtl-wafi_ip_ranges.cache" SHOP_LOG_FILE = "jtl-wafi_blocked.log" RATELIMIT_DIR = "jtl-wafi_ratelimit" # ============================================================================= # WEBSOCKET # ============================================================================= DEFAULT_DASHBOARD_URL = "wss://shop000.jtl-hosting.de:8000/ws/agent" HEARTBEAT_INTERVAL = 60 # Sekunden RECONNECT_BASE_DELAY = 1 # Sekunden RECONNECT_MAX_DELAY = 60 # Sekunden STATS_UPDATE_INTERVAL = 10 # Sekunden # ============================================================================= # AUTO-UPDATE # ============================================================================= AGENT_UPDATE_URL = "https://git.jtl-hosting.de/thomasciesla/JTL-WAFI/raw/branch/main/jtl-wafi-agent.py" # ============================================================================= # IP-API CONFIGURATION # ============================================================================= IP_API_URL = "http://ip-api.com/json/{ip}?fields=status,country,countryCode,isp,org,as" IP_API_RATE_LIMIT = 45 # Requests pro Minute (Free Tier) IP_INFO_CACHE_TTL = 86400 # 24 Stunden Cache # ============================================================================= # GLOBAL COUNTRY CACHE (ipdeny.com - shared between all shops) # ============================================================================= GLOBAL_COUNTRY_CACHE_DIR = "/var/lib/jtl-wafi/country_cache" # ============================================================================= # BAN/WHITELIST FILES # ============================================================================= BANNED_IPS_FILE = "jtl-wafi_banned_ips.json" WHITELISTED_IPS_FILE = "jtl-wafi_whitelisted_ips.json" AUTO_BAN_CONFIG_FILE = "jtl-wafi_autoban_config.json" # ============================================================================= # STATISTICS WINDOWS (Sekunden) # ============================================================================= STATS_WINDOWS = { '5m': 300, '10m': 600, '15m': 900, '30m': 1800, '60m': 3600 } DEFAULT_STATS_WINDOW = '5m' # ============================================================================= # LOG ROTATION # ============================================================================= LOG_MAX_SIZE = 10 * 1024 * 1024 # 10 MB LOG_BACKUP_COUNT = 3 # ============================================================================= # RATE-LIMIT DEFAULTS # ============================================================================= DEFAULT_RATE_LIMIT = 30 # Requests pro Minute DEFAULT_BAN_DURATION = 5 # Minuten # ============================================================================= # COUNTRY PRESETS (ersetzt GEO_REGIONS) # ============================================================================= COUNTRY_PRESET_EU_PLUS = [ "de", "at", "ch", "be", "cy", "ee", "es", "fi", "fr", "gb", "gr", "hr", "ie", "it", "lt", "lu", "lv", "mt", "nl", "pt", "si", "sk" ] COUNTRY_PRESET_DACH = ["de", "at", "ch"] # ============================================================================= # LINK11 # ============================================================================= LINK11_IP = "128.65.223.106" # ============================================================================= # DNS CACHE # ============================================================================= DNS_CACHE = {} # ============================================================================= # COUNTRIES - Vollständige Länderliste (ISO 3166-1 Alpha-2) # ============================================================================= COUNTRIES = { "ad": {"name": "Andorra", "flag": "🇦🇩"}, "ae": {"name": "VAE", "flag": "🇦🇪"}, "af": {"name": "Afghanistan", "flag": "🇦🇫"}, "ag": {"name": "Antigua und Barbuda", "flag": "🇦🇬"}, "al": {"name": "Albanien", "flag": "🇦🇱"}, "am": {"name": "Armenien", "flag": "🇦🇲"}, "ao": {"name": "Angola", "flag": "🇦🇴"}, "ar": {"name": "Argentinien", "flag": "🇦🇷"}, "at": {"name": "Österreich", "flag": "🇦🇹"}, "au": {"name": "Australien", "flag": "🇦🇺"}, "az": {"name": "Aserbaidschan", "flag": "🇦🇿"}, "ba": {"name": "Bosnien-Herzegowina", "flag": "🇧🇦"}, "bb": {"name": "Barbados", "flag": "🇧🇧"}, "bd": {"name": "Bangladesch", "flag": "🇧🇩"}, "be": {"name": "Belgien", "flag": "🇧🇪"}, "bf": {"name": "Burkina Faso", "flag": "🇧🇫"}, "bg": {"name": "Bulgarien", "flag": "🇧🇬"}, "bh": {"name": "Bahrain", "flag": "🇧🇭"}, "bi": {"name": "Burundi", "flag": "🇧🇮"}, "bj": {"name": "Benin", "flag": "🇧🇯"}, "bn": {"name": "Brunei", "flag": "🇧🇳"}, "bo": {"name": "Bolivien", "flag": "🇧🇴"}, "br": {"name": "Brasilien", "flag": "🇧🇷"}, "bs": {"name": "Bahamas", "flag": "🇧🇸"}, "bt": {"name": "Bhutan", "flag": "🇧🇹"}, "bw": {"name": "Botswana", "flag": "🇧🇼"}, "by": {"name": "Belarus", "flag": "🇧🇾"}, "bz": {"name": "Belize", "flag": "🇧🇿"}, "ca": {"name": "Kanada", "flag": "🇨🇦"}, "cd": {"name": "DR Kongo", "flag": "🇨🇩"}, "cf": {"name": "Zentralafrikanische Republik", "flag": "🇨🇫"}, "cg": {"name": "Kongo", "flag": "🇨🇬"}, "ch": {"name": "Schweiz", "flag": "🇨🇭"}, "ci": {"name": "Elfenbeinküste", "flag": "🇨🇮"}, "cl": {"name": "Chile", "flag": "🇨🇱"}, "cm": {"name": "Kamerun", "flag": "🇨🇲"}, "cn": {"name": "China", "flag": "🇨🇳"}, "co": {"name": "Kolumbien", "flag": "🇨🇴"}, "cr": {"name": "Costa Rica", "flag": "🇨🇷"}, "cu": {"name": "Kuba", "flag": "🇨🇺"}, "cv": {"name": "Kap Verde", "flag": "🇨🇻"}, "cy": {"name": "Zypern", "flag": "🇨🇾"}, "cz": {"name": "Tschechien", "flag": "🇨🇿"}, "de": {"name": "Deutschland", "flag": "🇩🇪"}, "dj": {"name": "Dschibuti", "flag": "🇩🇯"}, "dk": {"name": "Dänemark", "flag": "🇩🇰"}, "dm": {"name": "Dominica", "flag": "🇩🇲"}, "do": {"name": "Dominikanische Republik", "flag": "🇩🇴"}, "dz": {"name": "Algerien", "flag": "🇩🇿"}, "ec": {"name": "Ecuador", "flag": "🇪🇨"}, "ee": {"name": "Estland", "flag": "🇪🇪"}, "eg": {"name": "Ägypten", "flag": "🇪🇬"}, "er": {"name": "Eritrea", "flag": "🇪🇷"}, "es": {"name": "Spanien", "flag": "🇪🇸"}, "et": {"name": "Äthiopien", "flag": "🇪🇹"}, "fi": {"name": "Finnland", "flag": "🇫🇮"}, "fj": {"name": "Fidschi", "flag": "🇫🇯"}, "fr": {"name": "Frankreich", "flag": "🇫🇷"}, "ga": {"name": "Gabun", "flag": "🇬🇦"}, "gb": {"name": "Großbritannien", "flag": "🇬🇧"}, "gd": {"name": "Grenada", "flag": "🇬🇩"}, "ge": {"name": "Georgien", "flag": "🇬🇪"}, "gh": {"name": "Ghana", "flag": "🇬🇭"}, "gm": {"name": "Gambia", "flag": "🇬🇲"}, "gn": {"name": "Guinea", "flag": "🇬🇳"}, "gq": {"name": "Äquatorialguinea", "flag": "🇬🇶"}, "gr": {"name": "Griechenland", "flag": "🇬🇷"}, "gt": {"name": "Guatemala", "flag": "🇬🇹"}, "gw": {"name": "Guinea-Bissau", "flag": "🇬🇼"}, "gy": {"name": "Guyana", "flag": "🇬🇾"}, "hk": {"name": "Hongkong", "flag": "🇭🇰"}, "hn": {"name": "Honduras", "flag": "🇭🇳"}, "hr": {"name": "Kroatien", "flag": "🇭🇷"}, "ht": {"name": "Haiti", "flag": "🇭🇹"}, "hu": {"name": "Ungarn", "flag": "🇭🇺"}, "id": {"name": "Indonesien", "flag": "🇮🇩"}, "ie": {"name": "Irland", "flag": "🇮🇪"}, "il": {"name": "Israel", "flag": "🇮🇱"}, "in": {"name": "Indien", "flag": "🇮🇳"}, "iq": {"name": "Irak", "flag": "🇮🇶"}, "ir": {"name": "Iran", "flag": "🇮🇷"}, "is": {"name": "Island", "flag": "🇮🇸"}, "it": {"name": "Italien", "flag": "🇮🇹"}, "jm": {"name": "Jamaika", "flag": "🇯🇲"}, "jo": {"name": "Jordanien", "flag": "🇯🇴"}, "jp": {"name": "Japan", "flag": "🇯🇵"}, "ke": {"name": "Kenia", "flag": "🇰🇪"}, "kg": {"name": "Kirgisistan", "flag": "🇰🇬"}, "kh": {"name": "Kambodscha", "flag": "🇰🇭"}, "km": {"name": "Komoren", "flag": "🇰🇲"}, "kn": {"name": "St. Kitts und Nevis", "flag": "🇰🇳"}, "kp": {"name": "Nordkorea", "flag": "🇰🇵"}, "kr": {"name": "Südkorea", "flag": "🇰🇷"}, "kw": {"name": "Kuwait", "flag": "🇰🇼"}, "kz": {"name": "Kasachstan", "flag": "🇰🇿"}, "la": {"name": "Laos", "flag": "🇱🇦"}, "lb": {"name": "Libanon", "flag": "🇱🇧"}, "lc": {"name": "St. Lucia", "flag": "🇱🇨"}, "li": {"name": "Liechtenstein", "flag": "🇱🇮"}, "lk": {"name": "Sri Lanka", "flag": "🇱🇰"}, "lr": {"name": "Liberia", "flag": "🇱🇷"}, "ls": {"name": "Lesotho", "flag": "🇱🇸"}, "lt": {"name": "Litauen", "flag": "🇱🇹"}, "lu": {"name": "Luxemburg", "flag": "🇱🇺"}, "lv": {"name": "Lettland", "flag": "🇱🇻"}, "ly": {"name": "Libyen", "flag": "🇱🇾"}, "ma": {"name": "Marokko", "flag": "🇲🇦"}, "mc": {"name": "Monaco", "flag": "🇲🇨"}, "md": {"name": "Moldau", "flag": "🇲🇩"}, "me": {"name": "Montenegro", "flag": "🇲🇪"}, "mg": {"name": "Madagaskar", "flag": "🇲🇬"}, "mk": {"name": "Nordmazedonien", "flag": "🇲🇰"}, "ml": {"name": "Mali", "flag": "🇲🇱"}, "mm": {"name": "Myanmar", "flag": "🇲🇲"}, "mn": {"name": "Mongolei", "flag": "🇲🇳"}, "mo": {"name": "Macau", "flag": "🇲🇴"}, "mr": {"name": "Mauretanien", "flag": "🇲🇷"}, "mt": {"name": "Malta", "flag": "🇲🇹"}, "mu": {"name": "Mauritius", "flag": "🇲🇺"}, "mv": {"name": "Malediven", "flag": "🇲🇻"}, "mw": {"name": "Malawi", "flag": "🇲🇼"}, "mx": {"name": "Mexiko", "flag": "🇲🇽"}, "my": {"name": "Malaysia", "flag": "🇲🇾"}, "mz": {"name": "Mosambik", "flag": "🇲🇿"}, "na": {"name": "Namibia", "flag": "🇳🇦"}, "ne": {"name": "Niger", "flag": "🇳🇪"}, "ng": {"name": "Nigeria", "flag": "🇳🇬"}, "ni": {"name": "Nicaragua", "flag": "🇳🇮"}, "nl": {"name": "Niederlande", "flag": "🇳🇱"}, "no": {"name": "Norwegen", "flag": "🇳🇴"}, "np": {"name": "Nepal", "flag": "🇳🇵"}, "nz": {"name": "Neuseeland", "flag": "🇳🇿"}, "om": {"name": "Oman", "flag": "🇴🇲"}, "pa": {"name": "Panama", "flag": "🇵🇦"}, "pe": {"name": "Peru", "flag": "🇵🇪"}, "pg": {"name": "Papua-Neuguinea", "flag": "🇵🇬"}, "ph": {"name": "Philippinen", "flag": "🇵🇭"}, "pk": {"name": "Pakistan", "flag": "🇵🇰"}, "pl": {"name": "Polen", "flag": "🇵🇱"}, "pt": {"name": "Portugal", "flag": "🇵🇹"}, "py": {"name": "Paraguay", "flag": "🇵🇾"}, "qa": {"name": "Katar", "flag": "🇶🇦"}, "ro": {"name": "Rumänien", "flag": "🇷🇴"}, "rs": {"name": "Serbien", "flag": "🇷🇸"}, "ru": {"name": "Russland", "flag": "🇷🇺"}, "rw": {"name": "Ruanda", "flag": "🇷🇼"}, "sa": {"name": "Saudi-Arabien", "flag": "🇸🇦"}, "sb": {"name": "Salomonen", "flag": "🇸🇧"}, "sc": {"name": "Seychellen", "flag": "🇸🇨"}, "sd": {"name": "Sudan", "flag": "🇸🇩"}, "se": {"name": "Schweden", "flag": "🇸🇪"}, "sg": {"name": "Singapur", "flag": "🇸🇬"}, "si": {"name": "Slowenien", "flag": "🇸🇮"}, "sk": {"name": "Slowakei", "flag": "🇸🇰"}, "sl": {"name": "Sierra Leone", "flag": "🇸🇱"}, "sm": {"name": "San Marino", "flag": "🇸🇲"}, "sn": {"name": "Senegal", "flag": "🇸🇳"}, "so": {"name": "Somalia", "flag": "🇸🇴"}, "sr": {"name": "Suriname", "flag": "🇸🇷"}, "ss": {"name": "Südsudan", "flag": "🇸🇸"}, "sv": {"name": "El Salvador", "flag": "🇸🇻"}, "sy": {"name": "Syrien", "flag": "🇸🇾"}, "sz": {"name": "Eswatini", "flag": "🇸🇿"}, "td": {"name": "Tschad", "flag": "🇹🇩"}, "tg": {"name": "Togo", "flag": "🇹🇬"}, "th": {"name": "Thailand", "flag": "🇹🇭"}, "tj": {"name": "Tadschikistan", "flag": "🇹🇯"}, "tl": {"name": "Osttimor", "flag": "🇹🇱"}, "tm": {"name": "Turkmenistan", "flag": "🇹🇲"}, "tn": {"name": "Tunesien", "flag": "🇹🇳"}, "to": {"name": "Tonga", "flag": "🇹🇴"}, "tr": {"name": "Türkei", "flag": "🇹🇷"}, "tt": {"name": "Trinidad und Tobago", "flag": "🇹🇹"}, "tw": {"name": "Taiwan", "flag": "🇹🇼"}, "tz": {"name": "Tansania", "flag": "🇹🇿"}, "ua": {"name": "Ukraine", "flag": "🇺🇦"}, "ug": {"name": "Uganda", "flag": "🇺🇬"}, "us": {"name": "USA", "flag": "🇺🇸"}, "uy": {"name": "Uruguay", "flag": "🇺🇾"}, "uz": {"name": "Usbekistan", "flag": "🇺🇿"}, "va": {"name": "Vatikanstadt", "flag": "🇻🇦"}, "vc": {"name": "St. Vincent und die Grenadinen", "flag": "🇻🇨"}, "ve": {"name": "Venezuela", "flag": "🇻🇪"}, "vn": {"name": "Vietnam", "flag": "🇻🇳"}, "vu": {"name": "Vanuatu", "flag": "🇻🇺"}, "ws": {"name": "Samoa", "flag": "🇼🇸"}, "xk": {"name": "Kosovo", "flag": "🇽🇰"}, "ye": {"name": "Jemen", "flag": "🇾🇪"}, "za": {"name": "Südafrika", "flag": "🇿🇦"}, "zm": {"name": "Sambia", "flag": "🇿🇲"}, "zw": {"name": "Simbabwe", "flag": "🇿🇼"}, } # Default Country Rate-Limit Settings DEFAULT_COUNTRY_RATE_LIMIT = 100 # Requests pro Minute pro Land DEFAULT_COUNTRY_BAN_DURATION = 600 # Sekunden (10 Minuten) # ============================================================================= # BOT IP RANGES - Für Bots die sich mit normalem User-Agent tarnen # ============================================================================= BOT_IP_RANGES = { # Alibaba Cloud / Alibaba Spider - tarnt sich oft mit normalem UA # Große zusammengefasste Blöcke basierend auf APNIC/WHOIS-Daten 'Alibaba-Bot': [ # === Alibaba Cloud Singapore (ASEPL-SG) - Hauptblöcke === '43.0.0.0/9', # 43.0.0.0 - 43.127.255.255 (8.3 Mio IPs) '8.128.0.0/10', # 8.128.0.0 - 8.191.255.255 (4.2 Mio IPs) '8.208.0.0/12', # 8.208.0.0 - 8.223.255.255 (1 Mio IPs) # === Alibaba Cloud Singapore - Weitere Blöcke === '47.74.0.0/15', # 47.74.0.0 - 47.75.255.255 '47.76.0.0/16', # 47.76.0.0 - 47.76.255.255 '47.88.0.0/15', # 47.88.0.0 - 47.89.255.255 '47.90.0.0/15', # 47.90.0.0 - 47.91.255.255 '47.241.0.0/16', # 47.241.0.0 - 47.241.255.255 '47.52.0.0/15', # 47.52.0.0 - 47.53.255.255 (HK) '47.56.0.0/15', # 47.56.0.0 - 47.57.255.255 (HK) '149.129.0.0/16', # 149.129.0.0 - 149.129.255.255 '161.117.0.0/16', # 161.117.0.0 - 161.117.255.255 '170.33.0.0/16', # 170.33.0.0 - 170.33.255.255 # === Alibaba Cloud China (Aliyun) === '39.96.0.0/13', # 39.96.0.0 - 39.103.255.255 '39.104.0.0/14', # 39.104.0.0 - 39.107.255.255 '39.108.0.0/16', # 39.108.0.0 - 39.108.255.255 '101.132.0.0/15', # 101.132.0.0 - 101.133.255.255 '106.14.0.0/15', # 106.14.0.0 - 106.15.255.255 '112.124.0.0/16', # 112.124.0.0 - 112.124.255.255 '114.55.0.0/16', # 114.55.0.0 - 114.55.255.255 '115.28.0.0/15', # 115.28.0.0 - 115.29.255.255 '116.62.0.0/16', # 116.62.0.0 - 116.62.255.255 '118.31.0.0/16', # 118.31.0.0 - 118.31.255.255 '119.23.0.0/16', # 119.23.0.0 - 119.23.255.255 '120.24.0.0/14', # 120.24.0.0 - 120.27.255.255 '120.55.0.0/16', # 120.55.0.0 - 120.55.255.255 '120.76.0.0/14', # 120.76.0.0 - 120.79.255.255 '121.40.0.0/14', # 121.40.0.0 - 121.43.255.255 '121.196.0.0/14', # 121.196.0.0 - 121.199.255.255 '139.196.0.0/16', # 139.196.0.0 - 139.196.255.255 '139.224.0.0/16', # 139.224.0.0 - 139.224.255.255 '140.205.0.0/16', # 140.205.0.0 - 140.205.255.255 '182.92.0.0/16', # 182.92.0.0 - 182.92.255.255 # === Alibaba Sonstige === '203.107.0.0/16', # Alibaba DNS '103.206.40.0/22', # Alibaba Cloud SG '185.218.176.0/22', # Alibaba Cloud ], } # ============================================================================= # BOT DETECTION PATTERNS # ============================================================================= BOT_PATTERNS = { # ========================================================================= # AI/LLM SERVICES # ========================================================================= 'ChatGPT-User': r'chatgpt-user', 'ChatGPT-Operator': r'chatgpt-operator', 'ChatGPT-Agent': r'chatgpt agent', 'ChatGPT': r'chatgpt', 'GPTBot (OpenAI)': r'gptbot', 'OAI-SearchBot (OpenAI)': r'oai-searchbot', 'OpenAI': r'openai', 'ClaudeBot (Anthropic)': r'claudebot', 'Claude-User': r'claude-user', 'Claude-Web': r'claude-web', 'Claude-SearchBot': r'claude-searchbot', 'Anthropic-AI': r'anthropic-ai', 'Anthropic': r'anthropic', 'Gemini-Deep-Research': r'gemini-deep-research', 'Google-NotebookLM': r'google-notebooklm', 'NotebookLM': r'notebooklm', 'GoogleAgent-Mariner': r'googleagent-mariner', 'PerplexityBot': r'perplexitybot', 'Perplexity-User': r'perplexity-user', 'Perplexity': r'perplexity', 'Cohere-AI': r'cohere-ai', 'Cohere-Training-Crawler': r'cohere-training-data-crawler', 'Cohere': r'cohere', 'MistralAI-User': r'mistralai-user', 'MistralAI': r'mistralai', 'Mistral': r'mistral', 'DeepSeekBot': r'deepseekbot', 'DeepSeek': r'deepseek', 'Bytespider (TikTok/ByteDance)': r'bytespider', 'TikTokSpider': r'tiktokspider', 'ByteDance': r'bytedance', 'AI2Bot-DeepResearchEval': r'ai2bot-deepresearcheval', 'AI2Bot-Dolma': r'ai2bot-dolma', 'AI2Bot (Allen Institute)': r'ai2bot', 'CCBot (Common Crawl)': r'ccbot', 'Diffbot': r'diffbot', 'img2dataset': r'img2dataset', 'LAIONDownloader': r'laiondownloader', 'LAION-HuggingFace': r'laion-huggingface', 'LAION': r'laion', 'HuggingFace': r'huggingface', 'BedrockBot (AWS)': r'bedrockbot', 'DuckAssistBot': r'duckassistbot', 'PhindBot': r'phindbot', 'YouBot': r'youbot', 'iAskSpider': r'iaskspider', 'iAskBot': r'iaskbot', 'ChatGLM-Spider': r'chatglm-spider', 'Panscient': r'panscient', 'Devin (Cognition)': r'devin', 'Manus-User': r'manus-user', 'TwinAgent': r'twinagent', 'NovaAct': r'novaact', 'FirecrawlAgent': r'firecrawlagent', 'Firecrawl': r'firecrawl', 'Crawl4AI': r'crawl4ai', 'Crawlspace': r'crawlspace', 'Cloudflare-AutoRAG': r'cloudflare-autorag', 'TerraCotta': r'terracotta', 'Thinkbot': r'thinkbot', # ========================================================================= # SUCHMASCHINEN # ========================================================================= 'Googlebot-Image': r'googlebot-image', 'Googlebot-Video': r'googlebot-video', 'Googlebot-News': r'googlebot-news', 'Googlebot-Discovery': r'googlebot-discovery', 'Googlebot': r'googlebot', 'Google-Extended': r'google-extended', 'Google-CloudVertexBot': r'google-cloudvertexbot', 'Google-Firebase': r'google-firebase', 'Google-InspectionTool': r'google-inspectiontool', 'GoogleOther-Image': r'googleother-image', 'GoogleOther-Video': r'googleother-video', 'GoogleOther': r'googleother', 'Storebot-Google': r'storebot-google', 'AdsBot-Google': r'adsbot-google', 'Bingbot (Microsoft)': r'bingbot', 'BingPreview': r'bingpreview', 'MSNBot': r'msnbot', 'Baiduspider': r'baiduspider', 'Baidu': r'baidu', 'YandexBot': r'yandexbot', 'YandexAdditionalBot': r'yandexadditionalbot', 'YandexAdditional': r'yandexadditional', 'Yandex': r'yandex', 'DuckDuckBot': r'duckduckbot', 'DuckDuckGo': r'duckduckgo', 'Applebot-Extended': r'applebot-extended', 'Applebot': r'applebot', 'Yahoo Slurp': r'slurp', 'Sogou': r'sogou', 'Sosospider': r'sosospider', 'NaverBot': r'naverbot', 'Naver': r'naver', 'SeznamBot': r'seznambot', 'MojeekBot': r'mojeekbot', 'QwantBot': r'qwantbot', 'PetalBot (Huawei)': r'petalbot', 'CocCocBot': r'coccocbot', 'Exabot': r'exabot', 'BraveBot': r'bravebot', 'Bravest': r'bravest', 'SeekportBot': r'seekportbot', # ========================================================================= # SEO & MARKETING TOOLS # ========================================================================= 'AhrefsBot': r'ahrefsbot', 'Ahrefs': r'ahrefs', 'SemrushBot-OCOB': r'semrushbot-ocob', 'SemrushBot-SWA': r'semrushbot-swa', 'SemrushBot': r'semrushbot', 'Semrush': r'semrush', 'MJ12Bot (Majestic)': r'mj12bot', 'Majestic': r'majestic', 'DotBot (Moz)': r'dotbot', 'RogerBot (Moz)': r'rogerbot', 'Screaming Frog': r'screaming frog', 'BLEXBot': r'blexbot', 'DataForSEOBot': r'dataforseobot', 'Linkdex': r'linkdex', 'SearchmetricsBot': r'searchmetricsbot', # ========================================================================= # SOCIAL MEDIA # ========================================================================= 'Facebook External Hit': r'facebookexternalhit', 'FacebookBot': r'facebookbot', 'Facebot': r'facebot', 'Meta-ExternalAgent': r'meta-externalagent', 'Meta-ExternalFetcher': r'meta-externalfetcher', 'Meta-WebIndexer': r'meta-webindexer', 'Facebook': r'facebook', 'Twitterbot': r'twitterbot', 'Twitter': r'twitter', 'Instagram': r'instagram', 'LinkedInBot': r'linkedinbot', 'LinkedIn': r'linkedin', 'Pinterestbot': r'pinterestbot', 'Pinterest': r'pinterest', 'WhatsApp': r'whatsapp', 'TelegramBot': r'telegrambot', 'Telegram': r'telegram', 'DiscordBot': r'discordbot', 'Discord': r'discord', 'Slackbot': r'slackbot', 'Slack': r'slack', 'Quora-Bot': r'quora-bot', 'Snapchat': r'snapchat', 'RedditBot': r'redditbot', # ========================================================================= # E-COMMERCE & PREISVERGLEICH # ========================================================================= 'Amazonbot': r'amazonbot', 'Amazon-Kendra': r'amazon-kendra', 'AmazonBuyForMe': r'amazonbuyforme', 'AMZNKAssocBot': r'amznkassocbot', 'Alibaba-Bot': r'alibaba|alibabagroup|aliyun|alicdn|alimama|taobao|tmall|1688\.com', 'AlibabaSpider': r'alibabaspider', 'Aliyun': r'aliyun', 'GeedoShopProductFinder': r'geedoshopproductfinder', 'Geedo': r'geedo', 'ShopWiki': r'shopwiki', 'PriceGrabber': r'pricegrabber', 'Shopify': r'shopify', 'Idealo': r'idealo', 'Guenstiger.de': r'guenstiger', 'Billiger.de': r'billiger', 'Ladenzeile': r'ladenzeile', 'Kelkoo': r'kelkoo', 'PriceRunner': r'pricerunner', # ========================================================================= # ARCHIV & RESEARCH # ========================================================================= 'Archive.org Bot': r'archive\.org_bot|archive-org-bot', 'Internet Archive': r'ia_archiver|ia-archiver', 'Wayback Machine': r'wayback', 'Heritrix': r'heritrix', 'Apache Nutch': r'nutch', 'Common Crawl': r'commoncrawl', # ========================================================================= # MONITORING & UPTIME # ========================================================================= 'UptimeRobot': r'uptimerobot', 'Pingdom': r'pingdom', 'StatusCake': r'statuscake', 'Site24x7': r'site24x7', 'NewRelic': r'newrelic', 'Datadog': r'datadog', 'GTmetrix': r'gtmetrix', 'PageSpeed Insights': r'pagespeed', 'Chrome Lighthouse': r'chrome-lighthouse', # ========================================================================= # DOWNLOAD & SCRAPER TOOLS # ========================================================================= 'HTTrack': r'httrack', 'Teleport Pro': r'teleportpro|teleport pro', 'Teleport': r'teleport', 'GetRight': r'getright', 'FlashGet': r'flashget', 'LeechFTP': r'leechftp', 'LeechGet': r'leechget', 'Leech': r'leech', 'Offline Explorer': r'offline explorer', 'Offline Navigator': r'offline navigator', 'Offline Tool': r'offline', 'WebCopier': r'webcopier', 'WebCopy': r'webcopy', 'WebRipper': r'webripper', 'WebReaper': r'webreaper', 'WebStripper': r'webstripper', 'WebSauger': r'websauger', 'WebZIP': r'webzip', 'WebWhacker': r'webwhacker', 'WebBandit': r'webbandit', 'SiteSucker': r'sitesucker', 'SiteSnagger': r'sitesnagger', 'BlackWidow': r'blackwidow', 'Mass Downloader': r'mass downloader', 'Download Demon': r'download demon', 'Download Ninja': r'download ninja', 'Download Master': r'download master', 'FreshDownload': r'freshdownload', 'SmartDownload': r'smartdownload', 'RealDownload': r'realdownload', 'StarDownloader': r'stardownloader', 'Net Vampire': r'net vampire', 'NetAnts': r'netants', 'NetZIP': r'netzip', 'Go!Zilla': r'go!zilla|gozilla', 'Grabber': r'grabber', 'PageGrabber': r'pagegrabber', 'EirGrabber': r'eirgrabber', 'EmailSiphon': r'emailsiphon', 'EmailCollector': r'emailcollector', 'EmailWolf': r'emailwolf', 'Email Extractor': r'email extractor', 'ExtractorPro': r'extractorpro', 'HarvestMan': r'harvestman', 'Harvest': r'harvest', 'Collector': r'collector', 'Vacuum': r'vacuum', 'WebVac': r'webvac', 'Zeus': r'zeus', 'ScrapeBox': r'scrapebox', 'Xenu Link Sleuth': r'xenu', 'Larbin': r'larbin', 'Grub': r'grub', # ========================================================================= # HTTP LIBRARIES & FRAMEWORKS # ========================================================================= 'Python-Requests': r'python-requests', 'Python-urllib': r'python-urllib', 'Python-HTTPX': r'python-httpx', 'Python HTTP': r'python/', 'aiohttp': r'aiohttp', 'HTTPX': r'httpx/', 'cURL': r'curl/|^curl', 'Wget': r'wget/|^wget', 'Go-HTTP-Client': r'go-http-client', 'Go HTTP': r'go http|go-http', 'Java HTTP Client': r'java/|java ', 'Apache-HttpClient': r'apache-httpclient', 'Jakarta Commons': r'jakarta', 'Axios': r'axios/|axios', 'Node-Fetch': r'node-fetch', 'Got (Node.js)': r'got/', 'libwww-perl': r'libwww-perl', 'LWP (Perl)': r'lwp::|lwp/', 'WWW-Mechanize': r'www-mechanize', 'Mechanize': r'mechanize', 'Scrapy': r'scrapy/|scrapy', 'HTTP.rb': r'http\.rb', 'Typhoeus': r'typhoeus', 'OkHttp': r'okhttp/|okhttp', 'CFNetwork': r'cfnetwork', 'WinHTTP': r'winhttp', 'Indy Library': r'indy library', 'Chilkat': r'chilkat', 'httplib': r'httplib', 'ApacheBench': r'apachebench', 'Guzzle (PHP)': r'guzzle', 'Requests': r'requests/', # ========================================================================= # SECURITY SCANNER # ========================================================================= 'Nessus': r'nessus', 'SQLMap': r'sqlmap', 'Netsparker': r'netsparker', 'Nikto': r'nikto', 'Acunetix': r'acunetix', 'Burp Suite': r'burpsuite|burp', 'OWASP ZAP': r'owasp zap', 'OpenVAS': r'openvas', 'Nmap': r'nmap', 'Masscan': r'masscan', 'WPScan': r'wpscan', # ========================================================================= # HEADLESS BROWSERS & AUTOMATION # ========================================================================= 'PhantomJS': r'phantomjs', 'Headless Chrome': r'headlesschrome', 'Headless Browser': r'headless', 'Selenium': r'selenium', 'Puppeteer': r'puppeteer', 'Playwright': r'playwright', 'Cypress': r'cypress', # ========================================================================= # FEED READER & RSS # ========================================================================= 'FeedFetcher': r'feedfetcher', 'FeedParser': r'feedparser', 'Feedly': r'feedly', 'Inoreader': r'inoreader', 'NewsBlur': r'newsblur', # ========================================================================= # WEITERE BEKANNTE BOTS # ========================================================================= 'OmgiliBot': r'omgilibot', 'Omgili': r'omgili', 'Webzio-Extended': r'webzio-extended', 'Webzio': r'webzio', 'Timpibot': r'timpibot', 'PanguBot': r'pangubot', 'ImagesiftBot': r'imagesiftbot', 'Kangaroo Bot': r'kangaroo bot', 'QualifiedBot': r'qualifiedbot', 'VelenPublicWebCrawler': r'velenpublicwebcrawler', 'Linguee Bot': r'linguee bot', 'Linguee': r'linguee', 'QuillBot': r'quillbot', 'TurnitinBot': r'turnitinbot', 'Turnitin': r'turnitin', 'ZanistaBot': r'zanistabot', 'WRTNBot': r'wrtnbot', 'WARDBot': r'wardbot', 'ShapBot': r'shapbot', 'LinerBot': r'linerbot', 'LinkupBot': r'linkupbot', 'KlaviyoAIBot': r'klaviyoaibot', 'KunatoCrawler': r'kunatocrawler', 'IbouBot': r'iboubot', 'BuddyBot': r'buddybot', 'BrightBot': r'brightbot', 'Channel3Bot': r'channel3bot', 'Andibot': r'andibot', 'Anomura': r'anomura', 'Awario': r'awario', 'BigSur.ai': r'bigsur', 'Cotoyogi': r'cotoyogi', 'AddSearchBot': r'addsearchbot', 'aiHitBot': r'aihitbot', 'Atlassian-Bot': r'atlassian-bot', 'RainBot': r'rainbot', 'TinyTestBot': r'tinytestbot', 'Brandwatch': r'brandwatch', 'Meltwater': r'meltwater', 'Netvibes': r'netvibes', 'BitlyBot': r'bitlybot', 'Mail.ru Bot': r'mail\.ru', 'YaK': r'yak', } # Generische Patterns (Fallback für unbekannte Bots) GENERIC_BOT_PATTERNS = [ 'bot', 'crawler', 'spider', 'scraper', 'fetch', 'scan', 'check', 'monitor', 'probe', 'index', 'archive', 'capture', 'reader', 'download', 'mirror', 'ripper', 'collector', 'extractor', 'siphon', 'copier', 'sucker', 'bandit', 'stripper', 'whacker', 'reaper', 'robot', 'agent', 'seeker', 'finder', 'walker', 'roam', 'snagger', ] # ============================================================================= # LOGGING SETUP # ============================================================================= def setup_logging(debug: bool = False): """Konfiguriert Logging mit Rotation.""" log_level = logging.DEBUG if debug else logging.INFO # Formatter formatter = logging.Formatter( '%(asctime)s [%(levelname)s] %(message)s', datefmt='%Y-%m-%d %H:%M:%S' ) # Console Handler console_handler = logging.StreamHandler() console_handler.setFormatter(formatter) console_handler.setLevel(log_level) # File Handler mit Rotation handlers = [console_handler] log_dir = os.path.dirname(LOG_FILE) if log_dir and os.path.exists(log_dir): try: file_handler = RotatingFileHandler( LOG_FILE, maxBytes=LOG_MAX_SIZE, backupCount=LOG_BACKUP_COUNT ) file_handler.setFormatter(formatter) file_handler.setLevel(log_level) handlers.append(file_handler) except PermissionError: pass # Logger konfigurieren logger = logging.getLogger('jtl_wafi_agent') logger.setLevel(log_level) logger.handlers = handlers return logger # Global Logger (wird in main() initialisiert) logger = logging.getLogger('jtl_wafi_agent') # ============================================================================= # UTILITY FUNCTIONS # ============================================================================= def utc_now() -> datetime: """Gibt aktuelle UTC Zeit zurück.""" return datetime.now(timezone.utc) def utc_now_iso() -> str: """Gibt aktuelle UTC Zeit als ISO-String zurück.""" return utc_now().strftime('%Y-%m-%dT%H:%M:%SZ') def format_duration(minutes: float) -> str: """Formatiert Minuten als lesbare Dauer.""" if minutes < 60: return f"{int(minutes)}m" hours = minutes / 60 if hours < 24: return f"{int(hours)}h {int(minutes % 60)}m" return f"{int(hours / 24)}d {int(hours % 24)}h" # ============================================================================= # OWNERSHIP HELPER FUNCTIONS # ============================================================================= def get_most_common_owner(httpdocs_path: str): """ Ermittelt die häufigste uid:gid-Kombination im httpdocs-Verzeichnis. Gibt (uid, gid) zurück oder (None, None) wenn nicht ermittelbar. """ if not os.path.isdir(httpdocs_path): return None, None owner_counts = Counter() try: for entry in os.listdir(httpdocs_path): entry_path = os.path.join(httpdocs_path, entry) try: stat_info = os.stat(entry_path) owner_counts[(stat_info.st_uid, stat_info.st_gid)] += 1 except (OSError, IOError): continue except (OSError, IOError): return None, None if not owner_counts: return None, None most_common = owner_counts.most_common(1)[0][0] return most_common def set_owner(path: str, uid: int, gid: int, recursive: bool = False): """ Setzt Owner und Gruppe für eine Datei oder einen Ordner. Optional rekursiv für Verzeichnisse. """ if uid is None or gid is None: return try: os.chown(path, uid, gid) if recursive and os.path.isdir(path): for root, dirs, files in os.walk(path): for d in dirs: try: os.chown(os.path.join(root, d), uid, gid) except (OSError, IOError): pass for f in files: try: os.chown(os.path.join(root, f), uid, gid) except (OSError, IOError): pass except (OSError, IOError): pass # ============================================================================= # PHP-FPM RESTART FUNCTIONS # ============================================================================= def extract_domain_from_path(shop_path: str) -> Optional[str]: """ Extrahiert Domain aus Pfad wie /var/www/vhosts/example.com/httpdocs. Args: shop_path: Shop-Pfad oder Domain Returns: Domain-String oder None """ # Wenn es ein Pfad ist if '/' in shop_path: match = re.search(r'/var/www/vhosts/([^/]+)', shop_path) if match: return match.group(1) # Ansonsten ist es vermutlich schon eine Domain return shop_path def find_php_fpm_service(domain: str) -> Optional[str]: """ Findet den PHP-FPM Service für eine Domain. Sucht nach Services im Format: plesk-php{version}-fpm_{domain}_{id}.service Args: domain: Shop-Domain (z.B. "example.de") Returns: Service-Name oder None wenn nicht gefunden """ try: result = subprocess.run( ['systemctl', 'list-units', '--type=service', '--all', '--no-legend'], capture_output=True, text=True, timeout=10 ) if result.returncode != 0: logger.warning(f"systemctl list-units fehlgeschlagen: {result.stderr}") return None for line in result.stdout.split('\n'): # Suche nach plesk-php*-fpm_{domain}_ if 'plesk-php' in line and f'fpm_{domain}_' in line: # Extrahiere Service-Namen (erstes Feld) parts = line.split() if parts: service_name = parts[0] logger.debug(f"PHP-FPM Service gefunden für {domain}: {service_name}") return service_name logger.debug(f"Kein PHP-FPM Service für {domain} gefunden") return None except subprocess.TimeoutExpired: logger.warning("systemctl list-units Timeout") return None except Exception as e: logger.warning(f"Fehler beim Suchen des PHP-FPM Service: {e}") return None def restart_php_fpm(domain: str) -> dict: """ Startet den PHP-FPM Service für eine Domain neu. Args: domain: Shop-Domain (z.B. "example.de") Returns: Dict mit 'success', 'service' und 'message' """ # Domain aus Pfad extrahieren falls nötig clean_domain = extract_domain_from_path(domain) if not clean_domain: return { 'success': False, 'service': None, 'message': f'Konnte Domain nicht extrahieren aus: {domain}' } service = find_php_fpm_service(clean_domain) if not service: return { 'success': False, 'service': None, 'message': f'Kein PHP-FPM Service gefunden für {clean_domain}' } try: result = subprocess.run( ['systemctl', 'restart', service], capture_output=True, text=True, timeout=30 ) if result.returncode == 0: logger.info(f"PHP-FPM Service {service} neugestartet für {clean_domain}") return { 'success': True, 'service': service, 'message': f'OPcache geleert ({service})' } else: error_msg = result.stderr.strip() if result.stderr else 'Unknown error' logger.error(f"Fehler beim Neustart von {service}: {error_msg}") return { 'success': False, 'service': service, 'message': f'Restart fehlgeschlagen: {error_msg}' } except subprocess.TimeoutExpired: logger.error(f"Timeout beim Neustart von {service}") return { 'success': False, 'service': service, 'message': 'Restart Timeout (>30s)' } except Exception as e: logger.error(f"Exception beim PHP-FPM Restart: {e}") return { 'success': False, 'service': service, 'message': str(e) } # ============================================================================= # IP-INFO CACHE (ipdeny.com for country, ip-api.com for ISP/ASN) # ============================================================================= # Global IP-Info Cache _ip_info_cache: Dict[str, Dict[str, Any]] = {} _ip_api_last_request = 0.0 _ip_api_request_count = 0 # WHOIS Cache (1 Stunde TTL) WHOIS_CACHE_TTL = 3600 # 1 Stunde _whois_cache: Dict[str, Dict[str, Any]] = {} # Global Country Ranges Cache (loaded once from ipdeny.com files) _country_ranges_cache: Dict[str, List[tuple]] = {} # country -> [(network_int, mask_int), ...] _country_cache_loaded = False def _load_global_country_cache(): """Lädt alle Country-Ranges aus dem globalen Cache.""" global _country_ranges_cache, _country_cache_loaded if _country_cache_loaded: return if not os.path.isdir(GLOBAL_COUNTRY_CACHE_DIR): logger.debug(f"Global Country Cache nicht vorhanden: {GLOBAL_COUNTRY_CACHE_DIR}") _country_cache_loaded = True return for country in COUNTRY_CODES: cache_file = os.path.join(GLOBAL_COUNTRY_CACHE_DIR, f"{country}.ranges") if not os.path.isfile(cache_file): continue try: with open(cache_file, 'r') as f: content = f.read() # Parse PHP serialized format: a:N:{i:0;s:LEN:"CIDR";...} ranges = [] import re for match in re.finditer(r's:\d+:"([^"]+)"', content): cidr = match.group(1) if '/' in cidr: try: subnet, mask = cidr.split('/') subnet_int = struct.unpack('!I', socket.inet_aton(subnet))[0] mask_int = (0xFFFFFFFF << (32 - int(mask))) & 0xFFFFFFFF ranges.append((subnet_int, mask_int)) except: pass if ranges: _country_ranges_cache[country.upper()] = ranges logger.debug(f"Country ranges geladen: {country.upper()} ({len(ranges)} ranges)") except Exception as e: logger.debug(f"Fehler beim Laden von {country}: {e}") _country_cache_loaded = True logger.info(f"Global Country Cache geladen: {len(_country_ranges_cache)} Länder") def get_country_for_ip_cached(ip: str) -> str: """ Ermittelt das Land für eine IP aus dem globalen ipdeny.com Cache. Konsistent mit PHP-Templates! Returns: 2-Letter Country Code (z.B. 'DE') oder 'XX' wenn unbekannt """ global _country_ranges_cache # Cache laden falls noch nicht geschehen if not _country_cache_loaded: _load_global_country_cache() try: ip_int = struct.unpack('!I', socket.inet_aton(ip))[0] except: return 'XX' # Suche in allen Ländern (häufigste zuerst) priority_countries = ['DE', 'AT', 'CH', 'US', 'GB', 'FR', 'NL', 'IT', 'ES', 'PL'] for country in priority_countries: if country in _country_ranges_cache: for subnet_int, mask_int in _country_ranges_cache[country]: if (ip_int & mask_int) == (subnet_int & mask_int): return country # Restliche Länder for country, ranges in _country_ranges_cache.items(): if country in priority_countries: continue for subnet_int, mask_int in ranges: if (ip_int & mask_int) == (subnet_int & mask_int): return country return 'XX' def get_ip_info(ip: str) -> Dict[str, Any]: """ Holt IP-Informationen mit konsistenter Country-Detection. - Country: aus ipdeny.com Cache (konsistent mit PHP!) - ISP/ASN: aus ip-api.com (optional, mit Rate-Limiting) Returns: Dict mit country, countryCode, isp, org, as (ASN) """ global _ip_info_cache, _ip_api_last_request, _ip_api_request_count # Prüfe Cache if ip in _ip_info_cache: cached = _ip_info_cache[ip] if time.time() - cached.get('_cached_at', 0) < IP_INFO_CACHE_TTL: return cached # Country aus ipdeny.com Cache (konsistent mit PHP!) country_code = get_country_for_ip_cached(ip) # Basis-Ergebnis mit Country aus lokalem Cache result = { 'country': country_code if country_code != 'XX' else 'Unknown', 'countryCode': country_code, 'isp': '', 'org': '', 'as': '', '_cached_at': time.time() } # Optional: ISP/ASN von ip-api.com holen (mit Rate-Limiting) now = time.time() if now - _ip_api_last_request >= 60: _ip_api_last_request = now _ip_api_request_count = 0 if _ip_api_request_count < IP_API_RATE_LIMIT: try: url = IP_API_URL.format(ip=ip) request = urllib.request.Request( url, headers={'User-Agent': f'JTL-WAFi-Agent/{VERSION}'} ) with urllib.request.urlopen(request, timeout=3) as response: data = json.loads(response.read().decode('utf-8')) _ip_api_request_count += 1 if data.get('status') == 'success': result['isp'] = data.get('isp', '') result['org'] = data.get('org', '') result['as'] = data.get('as', '') # Country NUR überschreiben wenn lokaler Cache 'XX' war if country_code == 'XX' and data.get('countryCode'): result['country'] = data.get('country', 'Unknown') result['countryCode'] = data.get('countryCode', 'XX') except Exception as e: logger.debug(f"IP-API Request fehlgeschlagen für {ip}: {e}") _ip_info_cache[ip] = result return result def get_cached_ip_info(ip: str) -> Optional[Dict[str, Any]]: """Gibt gecachte IP-Info zurück ohne API-Call.""" return _ip_info_cache.get(ip) def whois_lookup(ip: str) -> Dict[str, Any]: """ Führt WHOIS-Lookup für eine IP durch (lokales whois-Tool). Cached Ergebnisse für 1 Stunde. Returns: Dict mit: netname, org, asn, country, abuse, range, raw """ global _whois_cache # Cache prüfen if ip in _whois_cache: cached = _whois_cache[ip] if time.time() - cached.get('_cached_at', 0) < WHOIS_CACHE_TTL: return cached result = { 'ip': ip, 'netname': '', 'org': '', 'asn': '', 'country': '', 'abuse': '', 'range': '', 'descr': '', 'raw': '', '_cached_at': time.time() } try: # WHOIS-Kommando ausführen proc = subprocess.run( ['whois', ip], capture_output=True, text=True, timeout=10 ) if proc.returncode != 0: logger.debug(f"WHOIS fehlgeschlagen für {ip}: {proc.stderr}") _whois_cache[ip] = result return result raw_output = proc.stdout result['raw'] = raw_output # Parsing - verschiedene WHOIS-Formate unterstützen (RIPE, ARIN, APNIC, etc.) lines = raw_output.split('\n') for line in lines: line_lower = line.lower().strip() # Netname if line_lower.startswith('netname:'): result['netname'] = line.split(':', 1)[1].strip() # Organisation elif line_lower.startswith('org-name:') or line_lower.startswith('orgname:'): result['org'] = line.split(':', 1)[1].strip() elif line_lower.startswith('organization:') and not result['org']: result['org'] = line.split(':', 1)[1].strip() elif line_lower.startswith('descr:') and not result['org']: # Fallback zu descr wenn keine org gefunden val = line.split(':', 1)[1].strip() if val and not result['descr']: result['descr'] = val # ASN elif line_lower.startswith('origin:') or line_lower.startswith('originas:'): result['asn'] = line.split(':', 1)[1].strip() # Country elif line_lower.startswith('country:') and not result['country']: result['country'] = line.split(':', 1)[1].strip().upper() # Abuse Contact elif 'abuse' in line_lower and '@' in line: # E-Mail aus der Zeile extrahieren import re email_match = re.search(r'[\w\.-]+@[\w\.-]+\.\w+', line) if email_match and not result['abuse']: result['abuse'] = email_match.group(0) # IP Range (inetnum/NetRange) elif line_lower.startswith('inetnum:') or line_lower.startswith('netrange:'): result['range'] = line.split(':', 1)[1].strip() elif line_lower.startswith('cidr:') and not result['range']: result['range'] = line.split(':', 1)[1].strip() # Fallback: descr als org wenn org leer if not result['org'] and result['descr']: result['org'] = result['descr'] logger.debug(f"WHOIS für {ip}: netname={result['netname']}, org={result['org']}, asn={result['asn']}") except subprocess.TimeoutExpired: logger.warning(f"WHOIS Timeout für {ip}") except FileNotFoundError: logger.error("WHOIS-Tool nicht installiert! Bitte 'whois' installieren.") except Exception as e: logger.error(f"WHOIS Fehler für {ip}: {e}") _whois_cache[ip] = result return result # ============================================================================= # LIVE STATISTICS TRACKER # ============================================================================= class LiveStatsTracker: """ Trackt Live-Statistiken für einen Shop mit Rolling Window. Speichert IP-Zugriffe, Requests und erkennt verdächtige IPs. """ def __init__(self, shop: str, window_seconds: int = 300): self.shop = shop self.window_seconds = window_seconds # Rolling Window Data (deque mit Timestamps) self.ip_requests: Dict[str, List[float]] = {} # IP -> [timestamps] self.path_requests: Dict[str, List[float]] = {} # Path -> [timestamps] self.blocked_paths: Dict[str, List[float]] = {} # Path -> [blocked timestamps] self.ip_404s: Dict[str, List[float]] = {} # IP -> [404 timestamps] # IP Request Details (für IP-Detail Popup) - werden NICHT nach window_seconds gelöscht! self.ip_request_details: Dict[str, List[Dict]] = {} # IP -> [{ts, path, ua}, ...] self.max_request_details = 500 # Max Anzahl Details pro IP (nicht zeitbasiert) # Human/Bot Counters self.human_requests: List[float] = [] self.bot_requests: List[float] = [] # IP-Info Cache (pro Shop) self.ip_info: Dict[str, Dict] = {} def cleanup_old_data(self): """Entfernt Daten außerhalb des Zeitfensters.""" cutoff = time.time() - self.window_seconds def cleanup_dict(d: Dict[str, List[float]]) -> Dict[str, List[float]]: return {k: [t for t in v if t > cutoff] for k, v in d.items() if any(t > cutoff for t in v)} self.ip_requests = cleanup_dict(self.ip_requests) self.path_requests = cleanup_dict(self.path_requests) self.blocked_paths = cleanup_dict(self.blocked_paths) self.ip_404s = cleanup_dict(self.ip_404s) self.human_requests = [t for t in self.human_requests if t > cutoff] self.bot_requests = [t for t in self.bot_requests if t > cutoff] # Request Details werden NICHT zeitbasiert gelöscht - nur durch max_request_details limitiert # Damit bleiben alle Requests im Speicher für das IP-Detail-Popup def record_request(self, ip: str, path: str, is_bot: bool, is_blocked: bool = False, is_404: bool = False, user_agent: str = '', timestamp: float = None): """Zeichnet einen Request auf. timestamp=None nutzt aktuelle Zeit.""" now = timestamp if timestamp is not None else time.time() # IP-Request if ip not in self.ip_requests: self.ip_requests[ip] = [] self.ip_requests[ip].append(now) # Path-Request if path not in self.path_requests: self.path_requests[path] = [] self.path_requests[path].append(now) # Blocked Path if is_blocked: if path not in self.blocked_paths: self.blocked_paths[path] = [] self.blocked_paths[path].append(now) # 404 Error if is_404: if ip not in self.ip_404s: self.ip_404s[ip] = [] self.ip_404s[ip].append(now) # Human/Bot Counter if is_bot: self.bot_requests.append(now) else: self.human_requests.append(now) # IP-Info abrufen (async im Hintergrund) if ip not in self.ip_info: self.ip_info[ip] = get_ip_info(ip) # Request Details speichern (für IP-Detail Popup) if ip not in self.ip_request_details: self.ip_request_details[ip] = [] self.ip_request_details[ip].append({ 'ts': now, 'path': path, 'ua': user_agent, 'blocked': is_blocked }) # Limitieren auf max_request_details if len(self.ip_request_details[ip]) > self.max_request_details: self.ip_request_details[ip] = self.ip_request_details[ip][-self.max_request_details:] def get_ip_requests(self, ip: str, limit: int = 0) -> List[Dict]: """Gibt die Requests einer IP zurück. limit=0 für alle.""" self.cleanup_old_data() if ip not in self.ip_request_details: return [] # Neueste zuerst all_requests = list(reversed(self.ip_request_details[ip])) if limit > 0: return all_requests[:limit] return all_requests def get_top_ips(self, limit: int = 10) -> List[Dict[str, Any]]: """Gibt Top IPs mit Zusatzinfos zurück.""" self.cleanup_old_data() ip_counts = [(ip, len(times)) for ip, times in self.ip_requests.items()] ip_counts.sort(key=lambda x: x[1], reverse=True) result = [] for ip, count in ip_counts[:limit]: info = self.ip_info.get(ip) or get_ip_info(ip) result.append({ 'ip': ip, 'count': count, 'country': info.get('countryCode', 'XX'), 'country_name': info.get('country', 'Unknown'), 'org': info.get('org') or info.get('isp', ''), 'asn': info.get('as', '') }) return result def get_top_requests(self, limit: int = 10) -> List[Dict[str, Any]]: """Gibt Top Requests zurück.""" self.cleanup_old_data() path_counts = [(path, len(times)) for path, times in self.path_requests.items()] path_counts.sort(key=lambda x: x[1], reverse=True) return [{'path': path, 'count': count} for path, count in path_counts[:limit]] def get_top_blocked(self, limit: int = 10) -> List[Dict[str, Any]]: """Gibt Top geblockte Requests zurück.""" self.cleanup_old_data() blocked_counts = [(path, len(times)) for path, times in self.blocked_paths.items()] blocked_counts.sort(key=lambda x: x[1], reverse=True) return [{'path': path, 'count': count} for path, count in blocked_counts[:limit]] def get_suspicious_ips(self, min_requests: int = 50, min_404s: int = 5) -> List[Dict[str, Any]]: """ Erkennt verdächtige IPs basierend auf: - Hohe Request-Rate - Viele 404-Errors """ self.cleanup_old_data() suspicious = [] for ip, times in self.ip_requests.items(): count = len(times) error_count = len(self.ip_404s.get(ip, [])) # Verdächtig wenn: viele Requests ODER viele 404s is_suspicious = count >= min_requests or error_count >= min_404s if is_suspicious: info = self.ip_info.get(ip) or get_ip_info(ip) suspicious.append({ 'ip': ip, 'count': count, 'errors': error_count, 'country': info.get('countryCode', 'XX'), 'country_name': info.get('country', 'Unknown'), 'org': info.get('org') or info.get('isp', ''), 'asn': info.get('as', ''), 'reason': 'High rate' if count >= min_requests else 'Many 404s' }) suspicious.sort(key=lambda x: x['count'], reverse=True) return suspicious[:10] def get_human_rpm(self) -> float: """Gibt Human Requests pro Minute zurück.""" self.cleanup_old_data() minutes = self.window_seconds / 60 return round(len(self.human_requests) / minutes, 2) if minutes > 0 else 0.0 def get_bot_rpm(self) -> float: """Gibt Bot Requests pro Minute zurück.""" self.cleanup_old_data() minutes = self.window_seconds / 60 return round(len(self.bot_requests) / minutes, 2) if minutes > 0 else 0.0 def get_stats(self) -> Dict[str, Any]: """Gibt alle Statistiken zurück.""" self.cleanup_old_data() return { 'top_ips': self.get_top_ips(), 'top_requests': self.get_top_requests(), 'top_blocked': self.get_top_blocked(), 'suspicious_ips': self.get_suspicious_ips(), 'human_rpm': self.get_human_rpm(), 'bot_rpm': self.get_bot_rpm(), 'window_seconds': self.window_seconds } # Global Stats Trackers (pro Shop) _shop_stats_trackers: Dict[str, LiveStatsTracker] = {} def get_shop_stats_tracker(shop: str, window_seconds: int = 300) -> LiveStatsTracker: """Gibt den Stats-Tracker für einen Shop zurück (erstellt falls nötig).""" if shop not in _shop_stats_trackers: tracker = LiveStatsTracker(shop, window_seconds) # Initialen Log-Scan durchführen um Request-Details zu laden _load_initial_request_details(tracker, shop) _shop_stats_trackers[shop] = tracker elif _shop_stats_trackers[shop].window_seconds != window_seconds: # Window geändert - neuen Tracker erstellen tracker = LiveStatsTracker(shop, window_seconds) _load_initial_request_details(tracker, shop) _shop_stats_trackers[shop] = tracker # Falls ip_request_details leer sind, nochmal laden (z.B. nach Reset oder bei erstem WHOIS) tracker = _shop_stats_trackers[shop] if not tracker.ip_request_details: logger.info(f"Tracker für {shop} hat keine ip_request_details - lade aus Log") _load_initial_request_details(tracker, shop) else: logger.info(f"Tracker für {shop} bereits geladen: {len(tracker.ip_request_details)} IPs in ip_request_details") return tracker def _load_initial_request_details(tracker: LiveStatsTracker, shop: str, max_lines: int = 5000): """ Lädt initiale Request-Details aus dem Log-File. Wird beim Erstellen eines neuen LiveStatsTracker aufgerufen. """ log_file = os.path.join(VHOSTS_DIR, shop, 'httpdocs', SHOP_LOG_FILE) if not os.path.isfile(log_file): return try: # Letzte N Zeilen des Logs lesen (effizient von hinten) with open(log_file, 'r') as f: # Gehe ans Ende der Datei f.seek(0, 2) file_size = f.tell() if file_size == 0: return # Lies maximal 500KB von hinten read_size = min(512000, file_size) f.seek(file_size - read_size) # Überspringe erste (möglicherweise unvollständige) Zeile if read_size < file_size: f.readline() lines = f.readlines() # Nur die letzten max_lines Zeilen verarbeiten lines = lines[-max_lines:] logger.info(f"_load_initial_request_details: {len(lines)} Zeilen gelesen aus {log_file}") parsed_count = 0 for line in lines: try: # IP extrahieren ip = None if 'IP: ' in line: ip = line.split('IP: ')[1].split(' |')[0].strip() if not ip: continue # Zeitstempel extrahieren (Format: [2024-01-09 12:34:56]) timestamp = None if line.startswith('[') and ']' in line: try: ts_str = line[1:line.index(']')] ts_dt = datetime.strptime(ts_str, '%Y-%m-%d %H:%M:%S') timestamp = ts_dt.timestamp() except: pass # User-Agent extrahieren (muss VOR URI gemacht werden wegen Format) user_agent = '' if ' | UA: ' in line: user_agent = line.split(' | UA: ')[1].strip() elif 'User-Agent: ' in line: user_agent = line.split('User-Agent: ')[1].split(' |')[0].strip() # Path/URI extrahieren path = '/' if 'URI: ' in line: # Format: ... | URI: /path | UA: ... uri_part = line.split('URI: ')[1] # Falls UA danach kommt, abschneiden if ' | UA: ' in uri_part: path = uri_part.split(' | UA: ')[0].strip() else: path = uri_part.strip() elif 'Path: ' in line: path = line.split('Path: ')[1].split(' |')[0].strip() # Bot/Blocked erkennen is_bot = any(x in line for x in ['BOT: ', 'BOT:', 'BLOCKED_BOT:', 'MONITOR_BOT:', 'BANNED_BOT:']) is_blocked = any(x in line for x in ['BANNED', 'BLOCKED']) is_404 = '404' in line # NUR ip_request_details füllen (nicht die Rolling-Window Stats!) # Die Rolling-Window-Stats werden nur durch Live-Logs gefüllt if ip not in tracker.ip_request_details: tracker.ip_request_details[ip] = [] # Timestamp verwenden (oder aktuelle Zeit wenn nicht vorhanden) ts = timestamp if timestamp else time.time() tracker.ip_request_details[ip].append({ 'ts': ts, 'path': path, 'ua': user_agent, 'blocked': is_blocked }) # Limitieren auf max_request_details if len(tracker.ip_request_details[ip]) > tracker.max_request_details: tracker.ip_request_details[ip] = tracker.ip_request_details[ip][-tracker.max_request_details:] parsed_count += 1 except Exception as e: logger.debug(f"Fehler beim Parsen von Zeile: {e}") continue total_requests = sum(len(reqs) for reqs in tracker.ip_request_details.values()) logger.info(f"Initiale Request-Details für {shop} geladen: {len(tracker.ip_request_details)} IPs, {total_requests} Requests total (parsed {parsed_count} von {len(lines)} Zeilen)") # Debug: Zeige die ersten 5 IPs mit Request-Anzahl sample_ips = list(tracker.ip_request_details.items())[:5] for sample_ip, sample_reqs in sample_ips: logger.info(f" - IP {sample_ip}: {len(sample_reqs)} Requests") except Exception as e: logger.warning(f"Fehler beim Laden der initialen Request-Details für {shop}: {e}") import traceback logger.debug(traceback.format_exc()) # ============================================================================= # BAN/WHITELIST MANAGEMENT # ============================================================================= def get_banned_ips(shop: str) -> Dict[str, Dict[str, Any]]: """Lädt gebannte IPs für einen Shop.""" httpdocs = os.path.join(VHOSTS_DIR, shop, 'httpdocs') ban_file = os.path.join(httpdocs, BANNED_IPS_FILE) if not os.path.isfile(ban_file): return {} try: with open(ban_file, 'r') as f: data = json.load(f) # Abgelaufene Bans entfernen now = time.time() active_bans = {} for ip, ban_data in data.items(): expires = ban_data.get('expires', 0) if expires == -1 or expires > now: # -1 = permanent active_bans[ip] = ban_data return active_bans except: return {} def save_banned_ips(shop: str, banned_ips: Dict[str, Dict[str, Any]]): """Speichert gebannte IPs für einen Shop.""" httpdocs = os.path.join(VHOSTS_DIR, shop, 'httpdocs') ban_file = os.path.join(httpdocs, BANNED_IPS_FILE) try: with open(ban_file, 'w') as f: json.dump(banned_ips, f, indent=2) # Owner setzen uid, gid = get_most_common_owner(httpdocs) set_owner(ban_file, uid, gid) except Exception as e: logger.error(f"Fehler beim Speichern der Bans für {shop}: {e}") def ban_ip(shop: str, ip: str, duration: int, reason: str = "manual") -> Dict[str, Any]: """ Bannt eine IP für einen Shop. Args: shop: Shop-Domain ip: IP-Adresse duration: Dauer in Sekunden (-1 für permanent) reason: Grund für den Ban Returns: Dict mit success, message """ banned_ips = get_banned_ips(shop) now = time.time() expires = -1 if duration == -1 else int(now + duration) banned_ips[ip] = { 'banned_at': int(now), 'expires': expires, 'reason': reason, 'banned_by': 'manual' } save_banned_ips(shop, banned_ips) logger.info(f"IP {ip} gebannt für {shop}: {reason} (Dauer: {'permanent' if duration == -1 else f'{duration}s'})") return { 'success': True, 'message': f'IP {ip} gebannt', 'expires': expires } def unban_ip(shop: str, ip: str) -> Dict[str, Any]: """Entfernt Ban für eine IP.""" banned_ips = get_banned_ips(shop) if ip in banned_ips: del banned_ips[ip] save_banned_ips(shop, banned_ips) logger.info(f"IP {ip} entbannt für {shop}") return {'success': True, 'message': f'IP {ip} entbannt'} return {'success': False, 'message': 'IP nicht in Ban-Liste'} def get_whitelisted_ips(shop: str) -> Dict[str, Dict[str, Any]]: """Lädt Whitelist für einen Shop.""" httpdocs = os.path.join(VHOSTS_DIR, shop, 'httpdocs') whitelist_file = os.path.join(httpdocs, WHITELISTED_IPS_FILE) if not os.path.isfile(whitelist_file): return {} try: with open(whitelist_file, 'r') as f: return json.load(f) except: return {} def save_whitelisted_ips(shop: str, whitelisted_ips: Dict[str, Dict[str, Any]]): """Speichert Whitelist für einen Shop.""" httpdocs = os.path.join(VHOSTS_DIR, shop, 'httpdocs') whitelist_file = os.path.join(httpdocs, WHITELISTED_IPS_FILE) try: with open(whitelist_file, 'w') as f: json.dump(whitelisted_ips, f, indent=2) uid, gid = get_most_common_owner(httpdocs) set_owner(whitelist_file, uid, gid) except Exception as e: logger.error(f"Fehler beim Speichern der Whitelist für {shop}: {e}") def whitelist_ip(shop: str, ip: str, description: str = "") -> Dict[str, Any]: """ Fügt IP zur Whitelist hinzu. Args: shop: Shop-Domain ip: IP-Adresse oder CIDR (z.B. 192.168.1.0/24) description: Beschreibung """ whitelisted = get_whitelisted_ips(shop) whitelisted[ip] = { 'added_at': int(time.time()), 'description': description, 'added_by': 'manual' } save_whitelisted_ips(shop, whitelisted) # Wenn IP gebannt ist, entbannen banned = get_banned_ips(shop) if ip in banned: del banned[ip] save_banned_ips(shop, banned) logger.info(f"IP {ip} zur Whitelist hinzugefügt für {shop}") return {'success': True, 'message': f'IP {ip} zur Whitelist hinzugefügt'} def remove_from_whitelist(shop: str, ip: str) -> Dict[str, Any]: """Entfernt IP von der Whitelist.""" whitelisted = get_whitelisted_ips(shop) if ip in whitelisted: del whitelisted[ip] save_whitelisted_ips(shop, whitelisted) logger.info(f"IP {ip} von Whitelist entfernt für {shop}") return {'success': True, 'message': f'IP {ip} von Whitelist entfernt'} return {'success': False, 'message': 'IP nicht in Whitelist'} # ============================================================================= # AUTO-BAN CONFIGURATION # ============================================================================= def get_autoban_config(shop: str) -> Dict[str, Any]: """Lädt Auto-Ban Konfiguration für einen Shop.""" httpdocs = os.path.join(VHOSTS_DIR, shop, 'httpdocs') config_file = os.path.join(httpdocs, AUTO_BAN_CONFIG_FILE) default_config = { 'enabled': False, 'max_requests_per_minute': 100, 'max_404s_per_minute': 10, 'max_login_attempts': 5, 'ban_duration': 3600 # 1 Stunde } if not os.path.isfile(config_file): return default_config try: with open(config_file, 'r') as f: config = json.load(f) return {**default_config, **config} except: return default_config def save_autoban_config(shop: str, config: Dict[str, Any]): """Speichert Auto-Ban Konfiguration.""" httpdocs = os.path.join(VHOSTS_DIR, shop, 'httpdocs') config_file = os.path.join(httpdocs, AUTO_BAN_CONFIG_FILE) try: with open(config_file, 'w') as f: json.dump(config, f, indent=2) uid, gid = get_most_common_owner(httpdocs) set_owner(config_file, uid, gid) except Exception as e: logger.error(f"Fehler beim Speichern der Auto-Ban Config für {shop}: {e}") def check_autoban(shop: str, ip: str, stats_tracker: LiveStatsTracker) -> Optional[str]: """ Prüft ob eine IP automatisch gebannt werden soll. Returns: Grund für Ban oder None wenn kein Ban nötig """ config = get_autoban_config(shop) if not config.get('enabled', False): return None # Whitelist Check whitelisted = get_whitelisted_ips(shop) for wl_ip in whitelisted.keys(): if '/' in wl_ip: if ip_in_cidr(ip, wl_ip): return None elif ip == wl_ip: return None # Bereits gebannt? banned = get_banned_ips(shop) if ip in banned: return None # Request-Rate prüfen ip_requests = stats_tracker.ip_requests.get(ip, []) cutoff = time.time() - 60 recent_requests = len([t for t in ip_requests if t > cutoff]) if recent_requests >= config.get('max_requests_per_minute', 100): ban_ip(shop, ip, config.get('ban_duration', 3600), f"Auto-Ban: {recent_requests} req/min") return f"Rate limit exceeded ({recent_requests} req/min)" # 404-Rate prüfen ip_404s = stats_tracker.ip_404s.get(ip, []) recent_404s = len([t for t in ip_404s if t > cutoff]) if recent_404s >= config.get('max_404s_per_minute', 10): ban_ip(shop, ip, config.get('ban_duration', 3600), f"Auto-Ban: {recent_404s} 404s/min") return f"Too many 404 errors ({recent_404s}/min)" return None # ============================================================================= # BOT DETECTION FUNCTIONS # ============================================================================= def ip_in_cidr(ip_str: str, cidr_str: str) -> bool: """Prüft ob eine IP in einem CIDR-Netz liegt.""" try: ip = ipaddress.ip_address(ip_str) network = ipaddress.ip_network(cidr_str, strict=False) return ip in network except ValueError: return False def detect_bot(user_agent: str, ip: str = None) -> str: """ Erkennt Bots anhand des User-Agents und/oder der IP. IP-basierte Erkennung hat Priorität (für getarnte Bots). Gibt den Anzeigenamen zurück. """ # SCHRITT 1: IP-basierte Erkennung (höchste Priorität) if ip: for bot_name, ip_ranges in BOT_IP_RANGES.items(): for cidr in ip_ranges: if ip_in_cidr(ip, cidr): return bot_name if not user_agent or user_agent == 'Unknown': return 'Unbekannt' # SCHRITT 2: Spezifische User-Agent Patterns for bot_name, pattern in BOT_PATTERNS.items(): if re.search(pattern, user_agent, re.IGNORECASE): return bot_name # SCHRITT 3: Generische Patterns als Fallback ua_lower = user_agent.lower() for pattern in GENERIC_BOT_PATTERNS: if pattern in ua_lower: return f'Bot ({pattern})' return 'Unbekannt' # ============================================================================= # LINK11 CHECK # ============================================================================= def check_link11(domain: str) -> Dict[str, Any]: """Prüft ob eine Domain hinter Link11 ist.""" global DNS_CACHE if domain in DNS_CACHE: return DNS_CACHE[domain] try: ip = socket.gethostbyname(domain) is_link11 = (ip == LINK11_IP) DNS_CACHE[domain] = {'is_link11': is_link11, 'ip': ip} return DNS_CACHE[domain] except socket.gaierror: DNS_CACHE[domain] = {'is_link11': False, 'ip': 'N/A'} return DNS_CACHE[domain] def get_country_preset(preset_name: str) -> List[str]: """Gibt die Länder eines Presets zurück.""" if preset_name == "eu_plus": return COUNTRY_PRESET_EU_PLUS elif preset_name == "dach": return COUNTRY_PRESET_DACH return [] # ============================================================================= # PHP TEMPLATE GENERATORS # ============================================================================= def generate_php_unlimited_countries(countries: List[str]) -> str: """Generiert PHP-Array der unlimitierten Länder.""" return ", ".join([f"'{c.lower()}'" for c in countries]) def generate_php_bot_patterns() -> str: """Generiert PHP-Array der Bot-Patterns.""" patterns = [] for bot_name, pattern in BOT_PATTERNS.items(): escaped_pattern = pattern.replace("'", "\\'").replace("/", "\\/") safe_bot_name = bot_name.replace("'", "\\'") patterns.append(f"'{safe_bot_name}' => '/{escaped_pattern}/i'") return ",\n ".join(patterns) def generate_php_generic_patterns() -> str: """Generiert PHP-Array der generischen Patterns.""" patterns = [f"'{pattern}'" for pattern in GENERIC_BOT_PATTERNS] return ", ".join(patterns) def generate_php_bot_ip_ranges() -> str: """Generiert PHP-Array für IP-basierte Bot-Erkennung.""" lines = [] for bot_name, ip_ranges in BOT_IP_RANGES.items(): safe_bot_name = bot_name.replace("'", "\\'") ranges_str = ", ".join([f"'{r}'" for r in ip_ranges]) lines.append(f"'{safe_bot_name}' => [{ranges_str}]") return ",\n ".join(lines) # ============================================================================= # PHP TEMPLATES - BOT RATE-LIMITING # ============================================================================= BOT_SCRIPT_TEMPLATE = ''' $expiry_date) return; $log_file = __DIR__ . '/{log_file}'; $ratelimit_dir = __DIR__ . '/{ratelimit_dir}'; $bans_dir = $ratelimit_dir . '/bans'; $counts_dir = $ratelimit_dir . '/counts'; // Rate-Limit Configuration $rate_limit = {rate_limit}; // Requests per minute for this bot-type $ban_duration = {ban_duration}; // Ban duration in seconds $window_size = 60; // Window size in seconds (1 minute) $cleanup_probability = 100; // 1 in X chance to run cleanup $visitor_ip = $_SERVER['REMOTE_ADDR'] ?? ''; $user_agent = $_SERVER['HTTP_USER_AGENT'] ?? ''; $uri = $_SERVER['REQUEST_URI'] ?? '/'; // Ensure directories exist if (!is_dir($bans_dir)) @mkdir($bans_dir, 0777, true); if (!is_dir($counts_dir)) @mkdir($counts_dir, 0777, true); // === IP Ban/Whitelist Files === $ip_ban_file = __DIR__ . '/jtl-wafi_banned_ips.json'; $ip_whitelist_file = __DIR__ . '/jtl-wafi_whitelisted_ips.json'; // === IP-in-CIDR Check Function === function ip_in_cidr($ip, $cidr) {{ if (strpos($cidr, '/') === false) return false; list($subnet, $mask) = explode('/', $cidr); $ip_long = ip2long($ip); $subnet_long = ip2long($subnet); if ($ip_long === false || $subnet_long === false) return false; $mask_long = -1 << (32 - (int)$mask); return ($ip_long & $mask_long) === ($subnet_long & $mask_long); }} // === Check IP Ban/Whitelist Function === function check_ip_in_list($ip, $list) {{ if (empty($list) || !is_array($list)) return false; foreach (array_keys($list) as $entry) {{ if (strpos($entry, '/') !== false) {{ if (ip_in_cidr($ip, $entry)) return $entry; }} elseif ($entry === $ip) {{ return $entry; }} }} return false; }} // === STEP 0: Check IP Whitelist (skip all rate limiting if whitelisted) === $is_whitelisted = false; if (file_exists($ip_whitelist_file)) {{ $whitelist = @json_decode(@file_get_contents($ip_whitelist_file), true); if (check_ip_in_list($visitor_ip, $whitelist)) {{ $is_whitelisted = true; }} }} // === STEP 0.5: Check IP Ban (block immediately if banned) === if (!$is_whitelisted && file_exists($ip_ban_file)) {{ $bans = @json_decode(@file_get_contents($ip_ban_file), true); if (is_array($bans) && isset($bans[$visitor_ip])) {{ $ban_data = $bans[$visitor_ip]; $expires = $ban_data['expires'] ?? 0; if ($expires === -1 || $expires > time()) {{ $timestamp = date('Y-m-d H:i:s'); $reason = $ban_data['reason'] ?? 'IP banned'; $remaining = $expires === -1 ? 'permanent' : ($expires - time()) . 's'; @file_put_contents($log_file, "[$timestamp] BLOCKED_IP: $visitor_ip | Reason: $reason | Remaining: $remaining\\n", FILE_APPEND | LOCK_EX); header('HTTP/1.1 403 Forbidden'); if ($expires !== -1) header('Retry-After: ' . ($expires - time())); exit; }} }} }} // Whitelisted IPs skip all rate limiting if ($is_whitelisted) {{ $timestamp = date('Y-m-d H:i:s'); @file_put_contents($log_file, "[$timestamp] WHITELISTED | IP: $visitor_ip | URI: $uri | UA: $user_agent\\n", FILE_APPEND | LOCK_EX); return; }} // === Bot IP Ranges (für getarnte Bots) === $bot_ip_ranges = [ {bot_ip_ranges} ]; // === Bot Detection Patterns (User-Agent) === $bot_patterns = [ {bot_patterns} ]; $generic_patterns = [{generic_patterns}]; // === STEP 0: IP-basierte Bot-Erkennung (höchste Priorität) === $detected_bot = null; if (!empty($visitor_ip)) {{ foreach ($bot_ip_ranges as $bot_name => $ip_ranges) {{ foreach ($ip_ranges as $cidr) {{ if (ip_in_cidr($visitor_ip, $cidr)) {{ $detected_bot = $bot_name; break 2; // Aus beiden Schleifen ausbrechen }} }} }} }} // === STEP 1: User-Agent-basierte Erkennung (falls IP nicht erkannt) === if ($detected_bot === null && !empty($user_agent)) {{ // Check specific patterns first foreach ($bot_patterns as $bot_name => $pattern) {{ if (preg_match($pattern, $user_agent)) {{ $detected_bot = $bot_name; break; }} }} // Check generic patterns as fallback if ($detected_bot === null) {{ $ua_lower = strtolower($user_agent); foreach ($generic_patterns as $pattern) {{ if (strpos($ua_lower, $pattern) !== false) {{ $detected_bot = "Bot ($pattern)"; break; }} }} }} }} // Not a bot - allow through without any rate limiting if ($detected_bot === null) return; // === Create hash based on BOT-TYPE only (not IP!) === $bot_hash = md5($detected_bot); // === STEP 2: Check if this bot-type is banned === $ban_file = "$bans_dir/$bot_hash.ban"; if (file_exists($ban_file)) {{ $ban_content = @file_get_contents($ban_file); $ban_parts = explode('|', $ban_content, 2); $ban_until = (int)$ban_parts[0]; if (time() < $ban_until) {{ // Bot-type is banned - log and block $timestamp = date('Y-m-d H:i:s'); $remaining = $ban_until - time(); @file_put_contents($log_file, "[$timestamp] BLOCKED (banned): $detected_bot | IP: $visitor_ip | Remaining: {{$remaining}}s\\n", FILE_APPEND | LOCK_EX); header('HTTP/1.1 403 Forbidden'); header('Retry-After: ' . $remaining); exit; }} // Ban expired - remove file @unlink($ban_file); }} // === STEP 3: Rate-Limit Check for this bot-type === $count_file = "$counts_dir/$bot_hash.count"; $current_time = time(); $count = 1; $window_start = $current_time; if (file_exists($count_file)) {{ $fp = @fopen($count_file, 'c+'); if ($fp && flock($fp, LOCK_EX)) {{ $content = fread($fp, 100); if (!empty($content)) {{ $parts = explode('|', $content); if (count($parts) === 2) {{ $window_start = (int)$parts[0]; $count = (int)$parts[1]; if ($current_time - $window_start > $window_size) {{ // New window $window_start = $current_time; $count = 1; }} else {{ $count++; }} }} }} ftruncate($fp, 0); rewind($fp); fwrite($fp, "$window_start|$count"); flock($fp, LOCK_UN); fclose($fp); }} }} else {{ @file_put_contents($count_file, "$window_start|$count", LOCK_EX); }} // === STEP 4: Check if limit exceeded === if ($count > $rate_limit) {{ // Create ban for this bot-type (store timestamp|botname) $ban_until = $current_time + $ban_duration; @file_put_contents($ban_file, "$ban_until|$detected_bot", LOCK_EX); // Log the ban $timestamp = date('Y-m-d H:i:s'); $ban_minutes = $ban_duration / 60; @file_put_contents($log_file, "[$timestamp] BANNED: $detected_bot | IP: $visitor_ip | Exceeded $rate_limit req/min | Ban: {{$ban_minutes}}m | Total requests: $count\\n", FILE_APPEND | LOCK_EX); // Block this request header('HTTP/1.1 403 Forbidden'); header('Retry-After: ' . $ban_duration); exit; }} // === STEP 5: Under limit - log and ALLOW through === $timestamp = date('Y-m-d H:i:s'); $uri = $_SERVER['REQUEST_URI'] ?? '/'; @file_put_contents($log_file, "[$timestamp] BOT: $detected_bot | IP: $visitor_ip | Count: $count/$rate_limit | URI: $uri | UA: $user_agent\\n", FILE_APPEND | LOCK_EX); // === STEP 6: Probabilistic cleanup === if (rand(1, $cleanup_probability) === 1) {{ $now = time(); foreach (glob("$bans_dir/*.ban") as $f) {{ $ban_content = @file_get_contents($f); $ban_parts = explode('|', $ban_content, 2); $ban_time = (int)$ban_parts[0]; if ($now > $ban_time) @unlink($f); }} foreach (glob("$counts_dir/*.count") as $f) {{ if ($now - filemtime($f) > $window_size * 2) @unlink($f); }} }} // Bot is under rate limit - ALLOW through (no exit, no 403) return; ''' # ============================================================================= # PHP TEMPLATES - MONITOR (Full Monitoring without Blocking) # ============================================================================= MONITOR_TEMPLATE = ''' $expiry_date) return; $log_file = __DIR__ . '/{log_file}'; $ratelimit_dir = __DIR__ . '/{ratelimit_dir}'; $country_cache_dir = $ratelimit_dir . '/country_cache'; $visitor_ip = $_SERVER['REMOTE_ADDR'] ?? ''; $user_agent = $_SERVER['HTTP_USER_AGENT'] ?? ''; $uri = $_SERVER['REQUEST_URI'] ?? '/'; // Ensure directories exist if (!is_dir($ratelimit_dir)) @mkdir($ratelimit_dir, 0777, true); if (!is_dir($country_cache_dir)) @mkdir($country_cache_dir, 0777, true); // === IP Ban/Whitelist Files === $ip_ban_file = __DIR__ . '/jtl-wafi_banned_ips.json'; $ip_whitelist_file = __DIR__ . '/jtl-wafi_whitelisted_ips.json'; // === IP-in-CIDR Check Function === function ip_in_cidr($ip, $cidr) {{ if (strpos($cidr, '/') === false) return false; list($subnet, $mask) = explode('/', $cidr); $ip_long = ip2long($ip); $subnet_long = ip2long($subnet); if ($ip_long === false || $subnet_long === false) return false; $mask_long = -1 << (32 - (int)$mask); return ($ip_long & $mask_long) === ($subnet_long & $mask_long); }} // === Check IP Ban/Whitelist Function === function check_ip_in_list($ip, $list) {{ if (empty($list) || !is_array($list)) return false; foreach (array_keys($list) as $entry) {{ if (strpos($entry, '/') !== false) {{ if (ip_in_cidr($ip, $entry)) return $entry; }} elseif ($entry === $ip) {{ return $entry; }} }} return false; }} // === Check IP Whitelist === $is_whitelisted = false; if (file_exists($ip_whitelist_file)) {{ $whitelist = @json_decode(@file_get_contents($ip_whitelist_file), true); if (check_ip_in_list($visitor_ip, $whitelist)) {{ $is_whitelisted = true; }} }} // === Check IP Ban (block even in monitor mode) === if (!$is_whitelisted && file_exists($ip_ban_file)) {{ $bans = @json_decode(@file_get_contents($ip_ban_file), true); if (is_array($bans) && isset($bans[$visitor_ip])) {{ $ban_data = $bans[$visitor_ip]; $expires = $ban_data['expires'] ?? 0; if ($expires === -1 || $expires > time()) {{ $timestamp = date('Y-m-d H:i:s'); $reason = $ban_data['reason'] ?? 'IP banned'; $remaining = $expires === -1 ? 'permanent' : ($expires - time()) . 's'; @file_put_contents($log_file, "[$timestamp] BLOCKED_IP: $visitor_ip | Reason: $reason | Remaining: $remaining\\n", FILE_APPEND | LOCK_EX); header('HTTP/1.1 403 Forbidden'); if ($expires !== -1) header('Retry-After: ' . ($expires - time())); exit; }} }} }} // === Country Detection (local cache only - no HTTP during page load!) === function get_country_for_ip($ip, $country_cache_dir) {{ // Common countries to check (ordered by traffic likelihood) $countries = ['de', 'at', 'ch', 'us', 'gb', 'fr', 'nl', 'it', 'es', 'pl', 'be', 'se', 'no', 'dk', 'fi', 'ru', 'cn', 'jp', 'kr', 'in', 'br', 'au', 'ca', 'ua', 'cz', 'pt', 'ie', 'gr', 'hu', 'ro', 'bg', 'hr', 'sk', 'si', 'lt', 'lv', 'ee', 'lu', 'mt', 'cy', 'tr', 'il', 'za', 'mx', 'ar', 'cl', 'co', 'pe', 've', 'eg', 'ng', 'ke', 'ma', 'tn', 'pk', 'bd', 'vn', 'th', 'my', 'sg', 'id', 'ph', 'tw', 'hk', 'nz', 'ae', 'sa', 'qa', 'kw', 'bh', 'om', 'ir', 'iq']; $ip_long = ip2long($ip); if ($ip_long === false) return 'XX'; foreach ($countries as $country) {{ $cache_file = "$country_cache_dir/$country.ranges"; // Only read from cache - NO HTTP requests during page load! if (!file_exists($cache_file)) continue; $ranges = @unserialize(@file_get_contents($cache_file)); if (empty($ranges) || !is_array($ranges)) continue; foreach ($ranges as $cidr) {{ if (strpos($cidr, '/') === false) continue; list($subnet, $mask) = explode('/', $cidr); $subnet_long = ip2long($subnet); if ($subnet_long === false) continue; $mask_long = -1 << (32 - (int)$mask); if (($ip_long & $mask_long) === ($subnet_long & $mask_long)) {{ return strtoupper($country); }} }} }} return 'XX'; // Unknown (cache not loaded or IP not found) }} // === Bot IP Ranges (für getarnte Bots) === $bot_ip_ranges = [ {bot_ip_ranges} ]; // === Bot Detection Patterns (User-Agent) === $bot_patterns = [ {bot_patterns} ]; $generic_patterns = [{generic_patterns}]; // === Detect Bot === $detected_bot = null; if (!empty($visitor_ip)) {{ foreach ($bot_ip_ranges as $bot_name => $ip_ranges) {{ foreach ($ip_ranges as $cidr) {{ if (ip_in_cidr($visitor_ip, $cidr)) {{ $detected_bot = $bot_name; break 2; }} }} }} }} if ($detected_bot === null && !empty($user_agent)) {{ foreach ($bot_patterns as $bot_name => $pattern) {{ if (preg_match($pattern, $user_agent)) {{ $detected_bot = $bot_name; break; }} }} if ($detected_bot === null) {{ $ua_lower = strtolower($user_agent); foreach ($generic_patterns as $pattern) {{ if (strpos($ua_lower, $pattern) !== false) {{ $detected_bot = "Bot ($pattern)"; break; }} }} }} }} // === Detect Country === $country = 'XX'; if (!empty($visitor_ip) && filter_var($visitor_ip, FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE) !== false) {{ $country = get_country_for_ip($visitor_ip, $country_cache_dir); }} // === Log Entry === $timestamp = date('Y-m-d H:i:s'); $uri = $_SERVER['REQUEST_URI'] ?? '/'; if ($detected_bot !== null) {{ @file_put_contents($log_file, "[$timestamp] MONITOR_BOT: $detected_bot | IP: $visitor_ip | Country: $country | URI: $uri | UA: $user_agent\\n", FILE_APPEND | LOCK_EX); }} else {{ @file_put_contents($log_file, "[$timestamp] MONITOR_HUMAN | IP: $visitor_ip | Country: $country | URI: $uri | UA: $user_agent\\n", FILE_APPEND | LOCK_EX); }} // === ALLOW REQUEST THROUGH - NO BLOCKING === return; ''' # ============================================================================= # PHP TEMPLATES - COMBINED (Bot + Country Rate-Limiting) # ============================================================================= COMBINED_SCRIPT_TEMPLATE = ''' check bot rate-limit * 4. If human AND country_mode AND country NOT unlimited -> check country rate-limit * 5. Always log (for stats) */ $expiry_date = strtotime('{expiry_timestamp}'); if (time() > $expiry_date) return; // === Configuration === $bot_mode = {bot_mode}; // true oder false $country_mode = {country_mode}; // true oder false $bot_rate_limit = {bot_rate_limit}; $bot_ban_duration = {bot_ban_duration}; $country_rate_limit = {country_rate_limit}; $country_ban_duration = {country_ban_duration}; $unlimited_countries = [{unlimited_countries}]; // z.B. 'de', 'at', 'ch', ... $window_size = 60; $cleanup_probability = 100; // === Paths === $log_file = __DIR__ . '/{log_file}'; $ratelimit_dir = __DIR__ . '/{ratelimit_dir}'; $bot_bans_dir = $ratelimit_dir . '/bans'; $bot_counts_dir = $ratelimit_dir . '/counts'; $country_bans_dir = $ratelimit_dir . '/country_bans'; $country_counts_dir = $ratelimit_dir . '/country_counts'; $country_cache_dir = $ratelimit_dir . '/country_cache'; $visitor_ip = $_SERVER['REMOTE_ADDR'] ?? ''; $user_agent = $_SERVER['HTTP_USER_AGENT'] ?? ''; $uri = $_SERVER['REQUEST_URI'] ?? '/'; // Ensure directories exist if (!is_dir($ratelimit_dir)) @mkdir($ratelimit_dir, 0777, true); if (!is_dir($country_cache_dir)) @mkdir($country_cache_dir, 0777, true); // Always needed for country detection if ($bot_mode) {{ if (!is_dir($bot_bans_dir)) @mkdir($bot_bans_dir, 0777, true); if (!is_dir($bot_counts_dir)) @mkdir($bot_counts_dir, 0777, true); }} if ($country_mode) {{ if (!is_dir($country_bans_dir)) @mkdir($country_bans_dir, 0777, true); if (!is_dir($country_counts_dir)) @mkdir($country_counts_dir, 0777, true); }} // === IP Ban/Whitelist Files === $ip_ban_file = __DIR__ . '/jtl-wafi_banned_ips.json'; $ip_whitelist_file = __DIR__ . '/jtl-wafi_whitelisted_ips.json'; // === IP-in-CIDR Check Function === function ip_in_cidr($ip, $cidr) {{ if (strpos($cidr, '/') === false) return false; list($subnet, $mask) = explode('/', $cidr); $ip_long = ip2long($ip); $subnet_long = ip2long($subnet); if ($ip_long === false || $subnet_long === false) return false; $mask_long = -1 << (32 - (int)$mask); return ($ip_long & $mask_long) === ($subnet_long & $mask_long); }} // === Check IP Ban/Whitelist Function === function check_ip_in_list($ip, $list) {{ if (empty($list) || !is_array($list)) return false; foreach (array_keys($list) as $entry) {{ if (strpos($entry, '/') !== false) {{ if (ip_in_cidr($ip, $entry)) return $entry; }} elseif ($entry === $ip) {{ return $entry; }} }} return false; }} // === STEP 0: Check IP Whitelist === $is_whitelisted = false; if (file_exists($ip_whitelist_file)) {{ $whitelist = @json_decode(@file_get_contents($ip_whitelist_file), true); if (check_ip_in_list($visitor_ip, $whitelist)) {{ $is_whitelisted = true; }} }} // === STEP 0.5: Check IP Ban === if (!$is_whitelisted && file_exists($ip_ban_file)) {{ $bans = @json_decode(@file_get_contents($ip_ban_file), true); if (is_array($bans) && isset($bans[$visitor_ip])) {{ $ban_data = $bans[$visitor_ip]; $expires = $ban_data['expires'] ?? 0; if ($expires === -1 || $expires > time()) {{ $timestamp = date('Y-m-d H:i:s'); $reason = $ban_data['reason'] ?? 'IP banned'; $remaining = $expires === -1 ? 'permanent' : ($expires - time()) . 's'; @file_put_contents($log_file, "[$timestamp] BLOCKED_IP: $visitor_ip | Reason: $reason | Remaining: $remaining\\n", FILE_APPEND | LOCK_EX); header('HTTP/1.1 403 Forbidden'); if ($expires !== -1) header('Retry-After: ' . ($expires - time())); exit; }} }} }} // Whitelisted IPs skip all rate limiting if ($is_whitelisted) {{ $timestamp = date('Y-m-d H:i:s'); @file_put_contents($log_file, "[$timestamp] WHITELISTED | IP: $visitor_ip | URI: $uri | UA: $user_agent\\n", FILE_APPEND | LOCK_EX); return; }} // === Country Detection (local cache only - no HTTP during page load!) === function get_country_for_ip($ip, $country_cache_dir) {{ // Common countries to check (ordered by traffic likelihood) $countries = ['de', 'at', 'ch', 'us', 'gb', 'fr', 'nl', 'it', 'es', 'pl', 'be', 'se', 'no', 'dk', 'fi', 'ru', 'cn', 'jp', 'kr', 'in', 'br', 'au', 'ca', 'ua', 'cz', 'pt', 'ie', 'gr', 'hu', 'ro', 'bg', 'hr', 'sk', 'si', 'lt', 'lv', 'ee', 'lu', 'mt', 'cy', 'tr', 'il', 'za', 'mx', 'ar', 'cl', 'co', 'pe', 've', 'eg', 'ng', 'ke', 'ma', 'tn', 'pk', 'bd', 'vn', 'th', 'my', 'sg', 'id', 'ph', 'tw', 'hk', 'nz', 'ae', 'sa', 'qa', 'kw', 'bh', 'om', 'ir', 'iq']; $ip_long = ip2long($ip); if ($ip_long === false) return 'XX'; foreach ($countries as $country) {{ $cache_file = "$country_cache_dir/$country.ranges"; // Only read from cache - NO HTTP requests during page load! if (!file_exists($cache_file)) continue; $ranges = @unserialize(@file_get_contents($cache_file)); if (empty($ranges) || !is_array($ranges)) continue; foreach ($ranges as $cidr) {{ if (strpos($cidr, '/') === false) continue; list($subnet, $mask) = explode('/', $cidr); $subnet_long = ip2long($subnet); if ($subnet_long === false) continue; $mask_long = -1 << (32 - (int)$mask); if (($ip_long & $mask_long) === ($subnet_long & $mask_long)) {{ return strtoupper($country); }} }} }} return 'XX'; // Unknown (cache not loaded or IP not found) }} // === Bot IP Ranges === $bot_ip_ranges = [ {bot_ip_ranges} ]; // === Bot Detection Patterns === $bot_patterns = [ {bot_patterns} ]; $generic_patterns = [{generic_patterns}]; // === STEP 1: Detect Bot === $detected_bot = null; if (!empty($visitor_ip)) {{ foreach ($bot_ip_ranges as $bot_name => $ip_ranges) {{ foreach ($ip_ranges as $cidr) {{ if (ip_in_cidr($visitor_ip, $cidr)) {{ $detected_bot = $bot_name; break 2; }} }} }} }} if ($detected_bot === null && !empty($user_agent)) {{ foreach ($bot_patterns as $bot_name => $pattern) {{ if (preg_match($pattern, $user_agent)) {{ $detected_bot = $bot_name; break; }} }} if ($detected_bot === null) {{ $ua_lower = strtolower($user_agent); foreach ($generic_patterns as $pattern) {{ if (strpos($ua_lower, $pattern) !== false) {{ $detected_bot = "Bot ($pattern)"; break; }} }} }} }} // === STEP 2: Detect Country (always - for logging and stats) === $country = 'XX'; if (!empty($visitor_ip) && filter_var($visitor_ip, FILTER_VALIDATE_IP, FILTER_FLAG_NO_PRIV_RANGE | FILTER_FLAG_NO_RES_RANGE) !== false) {{ $country = get_country_for_ip($visitor_ip, $country_cache_dir); }} $timestamp = date('Y-m-d H:i:s'); $uri = $_SERVER['REQUEST_URI'] ?? '/'; $current_time = time(); // === STEP 3: Bot Rate-Limiting (if bot detected AND bot_mode active) === if ($detected_bot !== null && $bot_mode) {{ $bot_hash = md5($detected_bot); $ban_file = "$bot_bans_dir/$bot_hash.ban"; // Check if bot is banned if (file_exists($ban_file)) {{ $ban_content = @file_get_contents($ban_file); $ban_parts = explode('|', $ban_content, 2); $ban_until = (int)$ban_parts[0]; if ($current_time < $ban_until) {{ $remaining = $ban_until - $current_time; @file_put_contents($log_file, "[$timestamp] BLOCKED_BOT: $detected_bot | IP: $visitor_ip | Country: $country | Remaining: {{$remaining}}s\\n", FILE_APPEND | LOCK_EX); header('HTTP/1.1 403 Forbidden'); header('Retry-After: ' . $remaining); exit; }} @unlink($ban_file); }} // Rate-limit check $count_file = "$bot_counts_dir/$bot_hash.count"; $count = 1; $window_start = $current_time; if (file_exists($count_file)) {{ $fp = @fopen($count_file, 'c+'); if ($fp && flock($fp, LOCK_EX)) {{ $content = fread($fp, 100); if (!empty($content)) {{ $parts = explode('|', $content); if (count($parts) === 2) {{ $window_start = (int)$parts[0]; $count = (int)$parts[1]; if ($current_time - $window_start > $window_size) {{ $window_start = $current_time; $count = 1; }} else {{ $count++; }} }} }} ftruncate($fp, 0); rewind($fp); fwrite($fp, "$window_start|$count"); flock($fp, LOCK_UN); fclose($fp); }} }} else {{ @file_put_contents($count_file, "$window_start|$count", LOCK_EX); }} // Check if limit exceeded if ($count > $bot_rate_limit) {{ $ban_until = $current_time + $bot_ban_duration; @file_put_contents($ban_file, "$ban_until|$detected_bot", LOCK_EX); $ban_minutes = $bot_ban_duration / 60; @file_put_contents($log_file, "[$timestamp] BANNED_BOT: $detected_bot | IP: $visitor_ip | Country: $country | Exceeded $bot_rate_limit req/min | Ban: {{$ban_minutes}}m\\n", FILE_APPEND | LOCK_EX); header('HTTP/1.1 403 Forbidden'); header('Retry-After: ' . $bot_ban_duration); exit; }} // Bot under limit - log and allow @file_put_contents($log_file, "[$timestamp] BOT: $detected_bot | IP: $visitor_ip | Country: $country | Count: $count/$bot_rate_limit | URI: $uri | UA: $user_agent\\n", FILE_APPEND | LOCK_EX); // Cleanup (probabilistic) if (rand(1, $cleanup_probability) === 1) {{ foreach (glob("$bot_bans_dir/*.ban") as $f) {{ $ban_content = @file_get_contents($f); $ban_time = (int)explode('|', $ban_content, 2)[0]; if ($current_time > $ban_time) @unlink($f); }} foreach (glob("$bot_counts_dir/*.count") as $f) {{ if ($current_time - filemtime($f) > $window_size * 2) @unlink($f); }} }} return; // Bot handled, don't check country }} // === STEP 4: Country Rate-Limiting (if human AND country_mode active) === if ($detected_bot === null && $country_mode) {{ $country_lower = strtolower($country); // Check if country is unlimited (still log but don't rate-limit) if (in_array($country_lower, $unlimited_countries)) {{ @file_put_contents($log_file, "[$timestamp] HUMAN | IP: $visitor_ip | Country: $country | URI: $uri | UA: $user_agent\\n", FILE_APPEND | LOCK_EX); return; // Unlimited country - allow through }} // Country needs rate-limiting $ban_file = "$country_bans_dir/$country_lower.ban"; // Check if country is banned if (file_exists($ban_file)) {{ $ban_until = (int)@file_get_contents($ban_file); if ($current_time < $ban_until) {{ $remaining = $ban_until - $current_time; @file_put_contents($log_file, "[$timestamp] BLOCKED_COUNTRY: $country | IP: $visitor_ip | Remaining: {{$remaining}}s\\n", FILE_APPEND | LOCK_EX); header('HTTP/1.1 403 Forbidden'); header('Retry-After: ' . $remaining); exit; }} @unlink($ban_file); }} // Rate-limit check $count_file = "$country_counts_dir/$country_lower.count"; $count = 1; $window_start = $current_time; if (file_exists($count_file)) {{ $fp = @fopen($count_file, 'c+'); if ($fp && flock($fp, LOCK_EX)) {{ $content = fread($fp, 100); if (!empty($content)) {{ $parts = explode('|', $content); if (count($parts) === 2) {{ $window_start = (int)$parts[0]; $count = (int)$parts[1]; if ($current_time - $window_start > $window_size) {{ $window_start = $current_time; $count = 1; }} else {{ $count++; }} }} }} ftruncate($fp, 0); rewind($fp); fwrite($fp, "$window_start|$count"); flock($fp, LOCK_UN); fclose($fp); }} }} else {{ @file_put_contents($count_file, "$window_start|$count", LOCK_EX); }} // Check if limit exceeded if ($count > $country_rate_limit) {{ $ban_until = $current_time + $country_ban_duration; @file_put_contents($ban_file, "$ban_until", LOCK_EX); $ban_minutes = $country_ban_duration / 60; @file_put_contents($log_file, "[$timestamp] BANNED_COUNTRY: $country | IP: $visitor_ip | Exceeded $country_rate_limit req/min | Ban: {{$ban_minutes}}m\\n", FILE_APPEND | LOCK_EX); header('HTTP/1.1 403 Forbidden'); header('Retry-After: ' . $country_ban_duration); exit; }} // Human under limit - log and allow @file_put_contents($log_file, "[$timestamp] HUMAN | IP: $visitor_ip | Country: $country | Count: $count/$country_rate_limit | URI: $uri | UA: $user_agent\\n", FILE_APPEND | LOCK_EX); // Cleanup (probabilistic) if (rand(1, $cleanup_probability) === 1) {{ foreach (glob("$country_bans_dir/*.ban") as $f) {{ $ban_time = (int)@file_get_contents($f); if ($current_time > $ban_time) @unlink($f); }} foreach (glob("$country_counts_dir/*.count") as $f) {{ if ($current_time - filemtime($f) > $window_size * 2) @unlink($f); }} }} return; }} // === STEP 5: No rate-limiting active, just log === if ($detected_bot !== null) {{ @file_put_contents($log_file, "[$timestamp] BOT: $detected_bot | IP: $visitor_ip | Country: $country | URI: $uri | UA: $user_agent\\n", FILE_APPEND | LOCK_EX); }} else {{ @file_put_contents($log_file, "[$timestamp] HUMAN | IP: $visitor_ip | Country: $country | URI: $uri | UA: $user_agent\\n", FILE_APPEND | LOCK_EX); }} return; ''' # ============================================================================= # SHOP REGISTRY FUNCTIONS # ============================================================================= def add_shop_to_active(shop: str, bot_mode: bool = True, bot_rate_limit: int = None, bot_ban_duration: int = None, country_mode: bool = False, country_rate_limit: int = None, country_ban_duration: int = None, unlimited_countries: List[str] = None, monitor_only: bool = False): """ Registriert einen Shop als aktiv mit v2.5 Konfiguration. Args: shop: Domain des Shops bot_mode: Bot-Rate-Limiting aktivieren bot_rate_limit: Requests/min pro Bot-Type bot_ban_duration: Ban-Dauer in Sekunden für Bots country_mode: Country-Rate-Limiting aktivieren country_rate_limit: Requests/min pro Land country_ban_duration: Ban-Dauer in Sekunden für Länder unlimited_countries: Liste der Länder ohne Rate-Limit monitor_only: Nur Monitoring, kein Blocking """ os.makedirs(os.path.dirname(ACTIVE_SHOPS_FILE), exist_ok=True) shop_data = { "activated": datetime.now().isoformat(), "expiry": (datetime.now() + timedelta(hours=72)).isoformat(), "bot_mode": bot_mode, "country_mode": country_mode, "monitor_only": monitor_only } if bot_rate_limit is not None: shop_data["bot_rate_limit"] = bot_rate_limit if bot_ban_duration is not None: shop_data["bot_ban_duration"] = bot_ban_duration if country_rate_limit is not None: shop_data["country_rate_limit"] = country_rate_limit if country_ban_duration is not None: shop_data["country_ban_duration"] = country_ban_duration if unlimited_countries: shop_data["unlimited_countries"] = unlimited_countries # File Locking für sichere parallele Zugriffe lock_file = ACTIVE_SHOPS_FILE + '.lock' with open(lock_file, 'w') as lf: fcntl.flock(lf, fcntl.LOCK_EX) # Exklusiver Lock - wartet wenn gesperrt try: shops = {} if os.path.isfile(ACTIVE_SHOPS_FILE): try: with open(ACTIVE_SHOPS_FILE, 'r') as f: shops = json.load(f) except: shops = {} shops[shop] = shop_data with open(ACTIVE_SHOPS_FILE, 'w') as f: json.dump(shops, f, indent=2) finally: fcntl.flock(lf, fcntl.LOCK_UN) # Lock freigeben def remove_shop_from_active(shop: str): """Entfernt einen Shop aus der aktiven Liste.""" if not os.path.isfile(ACTIVE_SHOPS_FILE): return # File Locking für sichere parallele Zugriffe lock_file = ACTIVE_SHOPS_FILE + '.lock' with open(lock_file, 'w') as lf: fcntl.flock(lf, fcntl.LOCK_EX) # Exklusiver Lock - wartet wenn gesperrt try: if not os.path.isfile(ACTIVE_SHOPS_FILE): return with open(ACTIVE_SHOPS_FILE, 'r') as f: shops = json.load(f) if shop in shops: del shops[shop] with open(ACTIVE_SHOPS_FILE, 'w') as f: json.dump(shops, f, indent=2) except: pass finally: fcntl.flock(lf, fcntl.LOCK_UN) # Lock freigeben def get_shop_config(shop: str) -> Dict[str, Any]: """Gibt die komplette Konfiguration eines Shops zurück.""" if not os.path.isfile(ACTIVE_SHOPS_FILE): return {} try: with open(ACTIVE_SHOPS_FILE, 'r') as f: return json.load(f).get(shop, {}) except: return {} def get_shop_bot_config(shop: str) -> tuple: """Gibt Bot-Rate-Limit Config zurück (enabled, rate_limit, ban_duration).""" config = get_shop_config(shop) return ( config.get("bot_mode", False), config.get("bot_rate_limit"), config.get("bot_ban_duration") ) def get_shop_country_config(shop: str) -> tuple: """Gibt Country-Rate-Limit Config zurück (enabled, rate_limit, ban_duration, unlimited).""" config = get_shop_config(shop) return ( config.get("country_mode", False), config.get("country_rate_limit"), config.get("country_ban_duration"), config.get("unlimited_countries", []) ) def get_shop_monitor_mode(shop: str) -> bool: """Gibt zurück ob ein Shop im Monitor-Only Modus ist.""" return get_shop_config(shop).get("monitor_only", False) def get_shop_activation_time(shop: str) -> Optional[datetime]: """Gibt die Aktivierungszeit eines Shops zurück.""" if not os.path.isfile(ACTIVE_SHOPS_FILE): return None try: with open(ACTIVE_SHOPS_FILE, 'r') as f: activated_str = json.load(f).get(shop, {}).get("activated") return datetime.fromisoformat(activated_str) if activated_str else None except: return None def get_available_shops() -> List[str]: """Gibt Liste aller verfügbaren Shops zurück.""" shops = [] if not os.path.exists(VHOSTS_DIR): return shops try: for entry in os.listdir(VHOSTS_DIR): shop_path = os.path.join(VHOSTS_DIR, entry) if os.path.isdir(shop_path) and entry not in ['chroot', 'system', 'default']: httpdocs = os.path.join(shop_path, 'httpdocs') if os.path.isdir(httpdocs) and os.path.isfile(os.path.join(httpdocs, 'index.php')): shops.append(entry) except: pass return sorted(shops) def get_active_shops() -> List[str]: """Gibt Liste aller aktiven Shops zurück.""" active = [] for shop in get_available_shops(): httpdocs = os.path.join(VHOSTS_DIR, shop, 'httpdocs') if os.path.isfile(os.path.join(httpdocs, BLOCKING_FILE)) or \ os.path.isfile(os.path.join(httpdocs, f'index.php{BACKUP_SUFFIX}')): active.append(shop) return active # ============================================================================= # COUNTRY IP RANGES DOWNLOAD (ipdeny.com) # ============================================================================= COUNTRY_CODES = [ 'de', 'at', 'ch', 'us', 'gb', 'fr', 'nl', 'it', 'es', 'pl', 'be', 'se', 'no', 'dk', 'fi', 'ru', 'cn', 'jp', 'kr', 'in', 'br', 'au', 'ca', 'ua', 'cz', 'pt', 'ie', 'gr', 'hu', 'ro', 'bg', 'hr', 'sk', 'si', 'lt', 'lv', 'ee', 'lu', 'mt', 'cy', 'tr', 'il', 'za', 'mx', 'ar', 'cl', 'co', 'pe', 've', 'eg', 'ng', 'ke', 'ma', 'tn', 'pk', 'bd', 'vn', 'th', 'my', 'sg', 'id', 'ph', 'tw', 'hk', 'nz', 'ae', 'sa', 'qa', 'kw', 'bh', 'om', 'ir', 'iq' ] def download_country_ranges(force: bool = False) -> int: """ Lädt IP-Ranges für alle Länder von ipdeny.com herunter. Speichert NUR in den GLOBALEN Cache (world-readable für PHP via Symlink). Args: force: Cache ignorieren und neu laden Returns: Anzahl der erfolgreich geladenen Länder """ global _country_ranges_cache, _country_cache_loaded # Globales Cache-Verzeichnis erstellen (world-readable) os.makedirs(GLOBAL_COUNTRY_CACHE_DIR, exist_ok=True) os.chmod(GLOBAL_COUNTRY_CACHE_DIR, 0o755) downloaded = 0 for country in COUNTRY_CODES: global_cache_file = os.path.join(GLOBAL_COUNTRY_CACHE_DIR, f"{country}.ranges") # Prüfe ob globaler Cache existiert und aktuell ist (24h) if not force and os.path.isfile(global_cache_file): try: if time.time() - os.path.getmtime(global_cache_file) < 86400: downloaded += 1 continue except: pass # Download von ipdeny.com url = f"https://www.ipdeny.com/ipblocks/data/aggregated/{country}-aggregated.zone" try: req = urllib.request.Request(url, headers={'User-Agent': 'JTL-WAFi/3.1'}) with urllib.request.urlopen(req, timeout=10) as response: content = response.read().decode('utf-8') ranges = [] for line in content.strip().split('\n'): line = line.strip() if line and '/' in line: ranges.append(line) if len(ranges) > 100: # Sanity check # PHP serialize format (for PHP compatibility) php_serialized = f'a:{len(ranges)}:{{' + ''.join( f'i:{i};s:{len(r)}:"{r}";' for i, r in enumerate(ranges) ) + '}' # In globalen Cache speichern (world-readable) with open(global_cache_file, 'w') as f: f.write(php_serialized) os.chmod(global_cache_file, 0o644) downloaded += 1 logger.debug(f"Country ranges geladen: {country.upper()} ({len(ranges)} ranges)") except Exception as e: logger.debug(f"Fehler beim Laden von {country}: {e}") # Globalen Cache neu laden _country_cache_loaded = False _load_global_country_cache() return downloaded def download_country_ranges_async(): """Lädt Country-Ranges im Hintergrund in den globalen Cache.""" def _download(): count = download_country_ranges(force=False) logger.info(f"Country IP-Ranges geladen: {count}/{len(COUNTRY_CODES)} Länder") thread = threading.Thread(target=_download, daemon=True) thread.start() def copy_country_cache_to_shop(ratelimit_path: str) -> int: """ Kopiert Country-Cache vom globalen Verzeichnis in den Shop. (Symlinks funktionieren nicht wegen PHP open_basedir Restriction) Args: ratelimit_path: Pfad zum jtl-wafi_ratelimit Verzeichnis des Shops Returns: Anzahl der kopierten Dateien """ shop_cache_dir = os.path.join(ratelimit_path, 'country_cache') try: # Shop-Cache Verzeichnis erstellen os.makedirs(shop_cache_dir, exist_ok=True) # Falls Symlink existiert (von alter Version), entfernen if os.path.islink(shop_cache_dir): os.remove(shop_cache_dir) os.makedirs(shop_cache_dir, exist_ok=True) # Prüfe ob globaler Cache existiert if not os.path.isdir(GLOBAL_COUNTRY_CACHE_DIR): logger.debug(f"Globaler Country-Cache nicht vorhanden: {GLOBAL_COUNTRY_CACHE_DIR}") return 0 copied = 0 for filename in os.listdir(GLOBAL_COUNTRY_CACHE_DIR): if filename.endswith('.ranges'): src = os.path.join(GLOBAL_COUNTRY_CACHE_DIR, filename) dst = os.path.join(shop_cache_dir, filename) # Nur kopieren wenn Quelle neuer ist oder Ziel nicht existiert if not os.path.isfile(dst) or os.path.getmtime(src) > os.path.getmtime(dst): shutil.copy2(src, dst) copied += 1 if copied > 0: logger.debug(f"Country-Cache kopiert: {copied} Dateien nach {shop_cache_dir}") return copied except Exception as e: logger.error(f"Fehler beim Kopieren des Country-Cache: {e}") return 0 def copy_country_cache_to_shop_async(ratelimit_path: str): """Kopiert Country-Cache im Hintergrund.""" def _copy(): # Erst sicherstellen, dass globaler Cache existiert if not os.path.isdir(GLOBAL_COUNTRY_CACHE_DIR) or not os.listdir(GLOBAL_COUNTRY_CACHE_DIR): download_country_ranges(force=False) copy_country_cache_to_shop(ratelimit_path) thread = threading.Thread(target=_copy, daemon=True) thread.start() # ============================================================================= # ACTIVATE / DEACTIVATE BLOCKING # ============================================================================= def activate_blocking(shop: str, silent: bool = True, bot_mode: bool = True, bot_rate_limit: int = None, bot_ban_duration: int = None, country_mode: bool = False, country_rate_limit: int = None, country_ban_duration: int = None, unlimited_countries: List[str] = None, monitor_only: bool = False) -> bool: """ Aktiviert Blocking für einen Shop (v2.5). Args: shop: Domain des Shops silent: Keine Konsolenausgabe bot_mode: Bot-Rate-Limiting aktivieren bot_rate_limit: Requests/min pro Bot-Type bot_ban_duration: Ban-Dauer in Sekunden für Bots country_mode: Country-Rate-Limiting aktivieren country_rate_limit: Requests/min pro Land country_ban_duration: Ban-Dauer in Sekunden für Länder unlimited_countries: Liste der Länder ohne Rate-Limit monitor_only: Nur Monitoring, kein Blocking Returns: True wenn erfolgreich, False sonst """ httpdocs = os.path.join(VHOSTS_DIR, shop, 'httpdocs') index_php = os.path.join(httpdocs, 'index.php') backup_php = os.path.join(httpdocs, f'index.php{BACKUP_SUFFIX}') blocking_file = os.path.join(httpdocs, BLOCKING_FILE) ratelimit_path = os.path.join(httpdocs, RATELIMIT_DIR) # Defaults setzen if bot_rate_limit is None: bot_rate_limit = DEFAULT_RATE_LIMIT if bot_ban_duration is None: bot_ban_duration = DEFAULT_BAN_DURATION * 60 if country_rate_limit is None: country_rate_limit = DEFAULT_COUNTRY_RATE_LIMIT if country_ban_duration is None: country_ban_duration = DEFAULT_COUNTRY_BAN_DURATION if unlimited_countries is None: unlimited_countries = [] # Prüfe ob bereits aktiv if os.path.isfile(backup_php): if not silent: logger.warning(f"Blocking bereits aktiv für {shop}") return False if not os.path.isfile(index_php): if not silent: logger.error(f"index.php nicht gefunden für {shop}") return False # Ermittle Owner uid, gid = get_most_common_owner(httpdocs) # Log-Meldung erstellen mode_parts = [] if monitor_only: mode_parts.append("MONITOR") else: if bot_mode: mode_parts.append(f"Bot({bot_rate_limit}/min)") if country_mode: mode_parts.append(f"Country({country_rate_limit}/min, {len(unlimited_countries)} unlimited)") mode_str = " + ".join(mode_parts) if mode_parts else "None" if not silent: logger.info(f"Aktiviere {shop}: {mode_str}") try: # Step 1: Backup und PHP-Blocking aktivieren shutil.copy2(index_php, backup_php) set_owner(backup_php, uid, gid) with open(index_php, 'r', encoding='utf-8') as f: content = f.read() lines = content.split('\n') insert_line = 0 for i, line in enumerate(lines): if 'declare(strict_types' in line: insert_line = i + 1 break elif ' bool: """ Deaktiviert Blocking für einen Shop. Args: shop: Domain des Shops silent: Keine Konsolenausgabe Returns: True wenn erfolgreich, False sonst """ httpdocs = os.path.join(VHOSTS_DIR, shop, 'httpdocs') index_php = os.path.join(httpdocs, 'index.php') backup_php = os.path.join(httpdocs, f'index.php{BACKUP_SUFFIX}') ratelimit_path = os.path.join(httpdocs, RATELIMIT_DIR) if not silent: logger.info(f"Deaktiviere Blocking für: {shop}") try: # Step 1: Original wiederherstellen if os.path.isfile(backup_php): shutil.move(backup_php, index_php) else: # Manuell require entfernen if os.path.isfile(index_php): with open(index_php, 'r') as f: content = f.read() lines = [l for l in content.split('\n') if BLOCKING_FILE not in l] with open(index_php, 'w') as f: f.write('\n'.join(lines)) # Step 2: Dateien löschen for f in [os.path.join(httpdocs, x) for x in [BLOCKING_FILE, CACHE_FILE, SHOP_LOG_FILE]]: if os.path.isfile(f): os.remove(f) # Step 3: Rate-Limit Verzeichnis löschen if os.path.isdir(ratelimit_path): shutil.rmtree(ratelimit_path) # Step 4: Deregistrieren remove_shop_from_active(shop) if not silent: logger.info(f"✅ Blocking deaktiviert für {shop}") return True except Exception as e: logger.error(f"Fehler beim Deaktivieren von {shop}: {e}") return False # ============================================================================= # SHOP LOG STATS # ============================================================================= def levenshtein_similarity(s1: str, s2: str) -> float: """ Berechnet die Ähnlichkeit zweier Strings basierend auf Levenshtein-Distanz. Returns: Float zwischen 0.0 (komplett unterschiedlich) und 1.0 (identisch) """ if s1 == s2: return 1.0 len1, len2 = len(s1), len(s2) if len1 == 0 or len2 == 0: return 0.0 # Optimierung: Bei sehr unterschiedlicher Länge ist Ähnlichkeit gering if abs(len1 - len2) > max(len1, len2) * 0.5: return 0.0 # Levenshtein-Distanz berechnen (dynamische Programmierung) if len1 > len2: s1, s2, len1, len2 = s2, s1, len2, len1 current_row = range(len1 + 1) for i in range(1, len2 + 1): previous_row, current_row = current_row, [i] + [0] * len1 for j in range(1, len1 + 1): add, delete, change = previous_row[j] + 1, current_row[j-1] + 1, previous_row[j-1] if s1[j-1] != s2[i-1]: change += 1 current_row[j] = min(add, delete, change) distance = current_row[len1] max_len = max(len1, len2) return 1.0 - (distance / max_len) def group_similar_urls(urls: Dict[str, int], similarity_threshold: float = 0.80) -> Dict[str, int]: """ Gruppiert ähnliche URLs basierend auf schnellem Präfix-Ansatz. Optimiert für Performance - keine Levenshtein mehr wegen CPU-Last. Args: urls: Dict mit {url: count} similarity_threshold: Nicht mehr verwendet (Kompatibilität) Returns: Dict mit {repräsentative_url: summierter_count} """ if not urls: return {} # Nur Top 500 URLs verarbeiten (Performance) sorted_urls = sorted(urls.items(), key=lambda x: x[1], reverse=True)[:500] # Nach normalisiertem Pfad gruppieren (Query-Params entfernen, auf 50 Zeichen kürzen) prefix_groups = {} for url, count in sorted_urls: # URL normalisieren: Query-Parameter entfernen path = url.split('?')[0] if '?' in url else url # Auf erste 50 Zeichen kürzen für Gruppierung normalized = path[:50] if normalized not in prefix_groups: prefix_groups[normalized] = {'urls': [], 'total': 0} prefix_groups[normalized]['urls'].append(url) prefix_groups[normalized]['total'] += count # Ergebnis erstellen result = {} for normalized, data in prefix_groups.items(): url_count = len(data['urls']) total = data['total'] if url_count > 1: # Mehrere ähnliche URLs display_url = normalized if len(display_url) > 47: display_url = display_url[:44] + '...' display_url += f' ({url_count}x)' else: # Einzelne URL display_url = data['urls'][0] if len(display_url) > 60: display_url = display_url[:57] + '...' result[display_url] = total return result def get_shop_log_stats(shop: str) -> Dict[str, Any]: """ Sammelt Statistiken aus dem Shop-Log (v2.5). Returns: Dict mit log_entries, bot_bans, country_bans, active_bot_bans, active_country_bans, banned_bots, banned_countries, req_per_min, unique_ips, unique_bots, unique_countries, top_bots, top_ips, top_countries """ httpdocs = os.path.join(VHOSTS_DIR, shop, 'httpdocs') log_file = os.path.join(httpdocs, SHOP_LOG_FILE) ratelimit_path = os.path.join(httpdocs, RATELIMIT_DIR) stats = { 'log_entries': 0, 'bot_bans': 0, 'country_bans': 0, 'active_bot_bans': 0, 'active_country_bans': 0, 'banned_bots': [], 'banned_countries': [], 'req_per_min': 0.0, 'unique_ips': 0, 'unique_bots': 0, 'unique_countries': 0, 'top_bots': {}, 'top_ips': [], 'top_countries': {}, 'top_requests': {}, 'human_requests': 0, 'bot_requests': 0 } ips = {} bots = {} countries = {} uris = {} # Log-Datei auswerten if os.path.isfile(log_file): try: with open(log_file, 'r') as f: for line in f: stats['log_entries'] += 1 ip = None detected_bot = None country = None # Bot-Bans erkennen if 'BANNED_BOT: ' in line or 'BANNED: ' in line: stats['bot_bans'] += 1 try: if 'BANNED_BOT: ' in line: detected_bot = line.split('BANNED_BOT: ')[1].split(' |')[0].strip() else: detected_bot = line.split('BANNED: ')[1].split(' |')[0].strip() except: pass # Country-Bans erkennen elif 'BANNED_COUNTRY: ' in line: stats['country_bans'] += 1 try: country = line.split('BANNED_COUNTRY: ')[1].split(' |')[0].strip() except: pass # Bot-Requests erkennen elif 'BOT: ' in line or 'BLOCKED_BOT: ' in line or 'MONITOR_BOT: ' in line: stats['bot_requests'] += 1 try: if 'BOT: ' in line: detected_bot = line.split('BOT: ')[1].split(' |')[0].strip() elif 'BLOCKED_BOT: ' in line: detected_bot = line.split('BLOCKED_BOT: ')[1].split(' |')[0].strip() elif 'MONITOR_BOT: ' in line: detected_bot = line.split('MONITOR_BOT: ')[1].split(' |')[0].strip() except: pass # Human-Requests erkennen elif 'HUMAN' in line or 'BLOCKED_COUNTRY: ' in line or 'MONITOR_HUMAN' in line: stats['human_requests'] += 1 # IP extrahieren if 'IP: ' in line: try: ip = line.split('IP: ')[1].split(' |')[0].strip() except: pass # Country extrahieren if 'Country: ' in line: try: country = line.split('Country: ')[1].split(' |')[0].strip() except: pass # URI extrahieren uri = None if 'URI: ' in line: try: uri = line.split('URI: ')[1].split(' |')[0].strip() # Leere URIs ignorieren if uri and uri != '/': uris[uri] = uris.get(uri, 0) + 1 except: pass # Statistiken sammeln if ip: ips[ip] = ips.get(ip, 0) + 1 if detected_bot: bots[detected_bot] = bots.get(detected_bot, 0) + 1 if country and country != 'XX': countries[country] = countries.get(country, 0) + 1 except: pass # Aktive Bot-Bans zählen bans_dir = os.path.join(ratelimit_path, 'bans') if os.path.isdir(bans_dir): now = time.time() try: for ban_file in os.listdir(bans_dir): if ban_file.endswith('.ban'): try: with open(os.path.join(bans_dir, ban_file), 'r') as f: content = f.read().strip() parts = content.split('|', 1) ban_until = int(parts[0]) bot_name = parts[1] if len(parts) > 1 else "Unbekannt" if now < ban_until: stats['active_bot_bans'] += 1 stats['banned_bots'].append(bot_name) except: pass except: pass # Aktive Country-Bans zählen country_bans_dir = os.path.join(ratelimit_path, 'country_bans') if os.path.isdir(country_bans_dir): now = time.time() try: for ban_file in os.listdir(country_bans_dir): if ban_file.endswith('.ban'): try: with open(os.path.join(country_bans_dir, ban_file), 'r') as f: ban_until = int(f.read().strip()) if now < ban_until: stats['active_country_bans'] += 1 country_code = ban_file.replace('.ban', '').upper() stats['banned_countries'].append(country_code) except: pass except: pass # Statistiken berechnen stats['unique_ips'] = len(ips) stats['unique_bots'] = len(bots) stats['unique_countries'] = len(countries) # Top Bots (max 20) sorted_bots = sorted(bots.items(), key=lambda x: x[1], reverse=True)[:20] stats['top_bots'] = dict(sorted_bots) # Top IPs (max 20) - mit Country-Info (nur aus Cache, keine API-Aufrufe!) sorted_ips = sorted(ips.items(), key=lambda x: x[1], reverse=True)[:20] top_ips_list = [] for ip, count in sorted_ips: # Nur schnellen Country-Lookup aus lokalem Cache, keine externen API-Aufrufe country_code = get_country_for_ip_cached(ip) top_ips_list.append({ 'ip': ip, 'count': count, 'country': country_code if country_code else 'XX', 'org': '', 'asn': '' }) stats['top_ips'] = top_ips_list # Top Countries (max 20) sorted_countries = sorted(countries.items(), key=lambda x: x[1], reverse=True)[:20] stats['top_countries'] = dict(sorted_countries) # Top Requests (max 20) - mit Ähnlichkeits-Gruppierung if uris: grouped_uris = group_similar_urls(uris, similarity_threshold=0.85) sorted_uris = sorted(grouped_uris.items(), key=lambda x: x[1], reverse=True)[:20] stats['top_requests'] = dict(sorted_uris) # Req/min berechnen activation_time = get_shop_activation_time(shop) if activation_time and stats['log_entries'] > 0: runtime_minutes = (datetime.now() - activation_time).total_seconds() / 60 if runtime_minutes > 0: stats['req_per_min'] = round(stats['log_entries'] / runtime_minutes, 2) stats['human_rpm'] = round(stats['human_requests'] / runtime_minutes, 2) stats['bot_rpm'] = round(stats['bot_requests'] / runtime_minutes, 2) else: stats['human_rpm'] = 0.0 stats['bot_rpm'] = 0.0 else: stats['human_rpm'] = 0.0 stats['bot_rpm'] = 0.0 return stats # ============================================================================= # LOG WATCHER - On-Demand Live-Streaming # ============================================================================= class LogWatcher: """ Überwacht Shop-Log-Dateien für Live-Streaming. Nur aktiv wenn explizit angefordert. """ def __init__(self, callback: Callable[[str, str], None]): """ Args: callback: Wird mit (shop, line) aufgerufen für neue Einträge """ self.callback = callback self.watching: Set[str] = set() self.file_positions: Dict[str, int] = {} self.running = False self._thread: Optional[threading.Thread] = None self._lock = threading.Lock() def start(self): """Startet den Watcher-Thread.""" if self.running: return self.running = True self._thread = threading.Thread(target=self._watch_loop, daemon=True) self._thread.start() logger.debug("LogWatcher gestartet") def stop(self): """Stoppt den Watcher-Thread.""" self.running = False if self._thread: self._thread.join(timeout=2) logger.debug("LogWatcher gestoppt") def start_watching(self, shop: str): """Startet Überwachung für einen Shop.""" with self._lock: self.watching.add(shop) log_file = os.path.join(VHOSTS_DIR, shop, 'httpdocs', SHOP_LOG_FILE) if os.path.isfile(log_file): # Start at end of file self.file_positions[shop] = os.path.getsize(log_file) else: self.file_positions[shop] = 0 logger.debug(f"LogWatcher: Überwache {shop}") def stop_watching(self, shop: str): """Stoppt Überwachung für einen Shop.""" with self._lock: self.watching.discard(shop) self.file_positions.pop(shop, None) logger.debug(f"LogWatcher: Beende Überwachung von {shop}") def _watch_loop(self): """Hauptschleife - prüft alle 500ms auf neue Einträge.""" while self.running: with self._lock: shops_to_watch = list(self.watching) for shop in shops_to_watch: try: log_file = os.path.join(VHOSTS_DIR, shop, 'httpdocs', SHOP_LOG_FILE) if not os.path.isfile(log_file): continue current_size = os.path.getsize(log_file) last_pos = self.file_positions.get(shop, 0) if current_size > last_pos: # Neue Daten vorhanden with open(log_file, 'r') as f: f.seek(last_pos) new_content = f.read() self.file_positions[shop] = f.tell() for line in new_content.strip().split('\n'): if line: try: self.callback(shop, line) except Exception as e: logger.error(f"LogWatcher Callback Fehler: {e}") elif current_size < last_pos: # Log wurde rotiert self.file_positions[shop] = 0 except Exception as e: logger.error(f"LogWatcher Fehler für {shop}: {e}") time.sleep(0.5) # ============================================================================= # LOG ROTATION # ============================================================================= def rotate_shop_logs(): """Rotiert alle Shop-Logs die > 10MB sind.""" for shop in get_active_shops(): try: httpdocs = os.path.join(VHOSTS_DIR, shop, 'httpdocs') log_file = os.path.join(httpdocs, SHOP_LOG_FILE) if not os.path.isfile(log_file): continue size = os.path.getsize(log_file) if size <= LOG_MAX_SIZE: continue logger.info(f"Rotiere Log für {shop} ({size / 1024 / 1024:.1f} MB)") # Rotiere: log -> log.1 -> log.2 -> log.3 (delete) for i in range(LOG_BACKUP_COUNT - 1, 0, -1): src = f"{log_file}.{i}" dst = f"{log_file}.{i + 1}" if os.path.exists(src): if i + 1 > LOG_BACKUP_COUNT: os.remove(src) else: shutil.move(src, dst) shutil.move(log_file, f"{log_file}.1") # Neues Log mit korrekter Ownership uid, gid = get_most_common_owner(httpdocs) with open(log_file, 'w') as f: f.write(f"# Log rotated at {utc_now_iso()}\n") set_owner(log_file, uid, gid) except Exception as e: logger.error(f"Log-Rotation Fehler für {shop}: {e}") # ============================================================================= # GEOIP AGENT - WebSocket Client # ============================================================================= class JTLWAFiAgent: """ WebSocket-basierter Agent für Echtzeit-Kommunikation mit dem Dashboard. """ def __init__(self, dashboard_url: str = DEFAULT_DASHBOARD_URL): self.dashboard_url = dashboard_url self.hostname = socket.getfqdn() self.agent_id = hashlib.md5(self.hostname.encode()).hexdigest()[:16] self.token: Optional[str] = None self.approved = False self.running = False self.ws = None self.reconnect_delay = RECONNECT_BASE_DELAY # Log Watcher für Live-Streaming self.log_watcher = LogWatcher(callback=self._on_log_entry) # State Tracking self.last_stats: Dict[str, Dict] = {} self.last_stats_time = 0 self.last_heartbeat_time = 0 self.last_log_rotation_time = 0 # Token laden falls vorhanden self._load_token() def _load_token(self): """Lädt gespeicherten Token aus Datei.""" if os.path.isfile(TOKEN_FILE): try: with open(TOKEN_FILE, 'r') as f: self.token = f.read().strip() if self.token: logger.info(f"Token geladen aus {TOKEN_FILE}") except Exception as e: logger.warning(f"Konnte Token nicht laden: {e}") def _save_token(self, token: str): """Speichert Token in Datei.""" try: os.makedirs(os.path.dirname(TOKEN_FILE), exist_ok=True) with open(TOKEN_FILE, 'w') as f: f.write(token) os.chmod(TOKEN_FILE, 0o600) self.token = token logger.info(f"Token gespeichert in {TOKEN_FILE}") except Exception as e: logger.error(f"Konnte Token nicht speichern: {e}") def _get_os_info(self) -> Dict[str, str]: """Sammelt OS-Informationen.""" return { "system": platform.system(), "release": platform.release(), "machine": platform.machine() } def _get_system_stats(self) -> Dict[str, Any]: """Sammelt System-Statistiken.""" stats = {} try: load = os.getloadavg() stats["load_1m"] = round(load[0], 2) stats["load_5m"] = round(load[1], 2) stats["load_15m"] = round(load[2], 2) except: pass try: with open('/proc/meminfo', 'r') as f: meminfo = {} for line in f: parts = line.split(':') if len(parts) == 2: meminfo[parts[0].strip()] = int(parts[1].strip().split()[0]) total = meminfo.get('MemTotal', 1) available = meminfo.get('MemAvailable', meminfo.get('MemFree', 0)) stats["memory_percent"] = round((1 - available / total) * 100, 1) except: pass try: with open('/proc/uptime', 'r') as f: stats["uptime_seconds"] = int(float(f.read().split()[0])) except: pass return stats def _get_shops_summary(self) -> Dict[str, int]: """Gibt Shop-Zusammenfassung zurück.""" available = get_available_shops() active = get_active_shops() return {"total": len(available), "active": len(active)} def _get_all_shops_data(self) -> List[Dict[str, Any]]: """Sammelt Daten aller Shops (v2.5).""" shops_data = [] available = get_available_shops() active = get_active_shops() for shop in available: is_active = shop in active link11_info = check_link11(shop) shop_data = { "domain": shop, "status": "active" if is_active else "inactive", "link11": link11_info['is_link11'], "link11_ip": link11_info['ip'] } if is_active: config = get_shop_config(shop) activation_time = get_shop_activation_time(shop) # v2.5 Konfiguration shop_data["bot_mode"] = config.get("bot_mode", False) shop_data["bot_rate_limit"] = config.get("bot_rate_limit") shop_data["bot_ban_duration"] = config.get("bot_ban_duration") shop_data["country_mode"] = config.get("country_mode", False) shop_data["country_rate_limit"] = config.get("country_rate_limit") shop_data["country_ban_duration"] = config.get("country_ban_duration") shop_data["unlimited_countries"] = config.get("unlimited_countries", []) shop_data["monitor_only"] = config.get("monitor_only", False) if activation_time: shop_data["activated"] = activation_time.isoformat() runtime = (datetime.now() - activation_time).total_seconds() / 60 shop_data["runtime_minutes"] = round(runtime, 1) # Stats sammeln shop_data["stats"] = get_shop_log_stats(shop) shops_data.append(shop_data) return shops_data def _on_log_entry(self, shop: str, line: str): """Callback für neue Log-Einträge.""" if not self.ws or not self.approved: return # Event senden asyncio.run_coroutine_threadsafe( self._send_event('log.entry', {'shop': shop, 'line': line}), self._loop ) # Live Stats Tracker updaten try: tracker = get_shop_stats_tracker(shop) # IP extrahieren ip = None if 'IP: ' in line: ip = line.split('IP: ')[1].split(' |')[0].strip() # User-Agent extrahieren (muss VOR URI gemacht werden wegen Format) user_agent = '' if ' | UA: ' in line: user_agent = line.split(' | UA: ')[1].strip() elif 'User-Agent: ' in line: user_agent = line.split('User-Agent: ')[1].split(' |')[0].strip() # Path/URI extrahieren path = '/' if 'URI: ' in line: # Format: ... | URI: /path | UA: ... uri_part = line.split('URI: ')[1] # Falls UA danach kommt, abschneiden if ' | UA: ' in uri_part: path = uri_part.split(' | UA: ')[0].strip() else: path = uri_part.strip() elif 'Path: ' in line: path = line.split('Path: ')[1].split(' |')[0].strip() # Bot/Human erkennen is_bot = any(x in line for x in ['BOT: ', 'BOT:', 'BLOCKED_BOT:', 'MONITOR_BOT:', 'BANNED_BOT:']) # Blocked erkennen is_blocked = any(x in line for x in ['BANNED', 'BLOCKED']) # 404 erkennen is_404 = '404' in line if ip: tracker.record_request(ip, path, is_bot, is_blocked, is_404, user_agent) except Exception as e: logger.debug(f"LiveStats record error: {e}") # Prüfe auf Ban-Events if 'BANNED: ' in line: try: bot_name = line.split('BANNED: ')[1].split(' |')[0].strip() asyncio.run_coroutine_threadsafe( self._send_event('bot.banned', { 'shop': shop, 'bot_name': bot_name, 'line': line }), self._loop ) except: pass async def _send_event(self, event_type: str, data: Dict[str, Any]): """Sendet ein Event an das Dashboard.""" if not self.ws: return try: message = json.dumps({ 'type': event_type, 'data': data }) await self.ws.send(message) logger.debug(f"Gesendet: {event_type}") except Exception as e: logger.error(f"Fehler beim Senden von {event_type}: {e}") async def _send_connect(self): """Sendet agent.connect Event.""" await self._send_event('agent.connect', { 'hostname': self.hostname, 'agent_id': self.agent_id, 'token': self.token, 'version': VERSION, 'os_info': self._get_os_info(), 'shops_summary': self._get_shops_summary() }) async def _send_heartbeat(self): """Sendet agent.heartbeat Event.""" await self._send_event('agent.heartbeat', { 'agent_id': self.agent_id, 'system': self._get_system_stats(), 'shops_summary': self._get_shops_summary() }) self.last_heartbeat_time = time.time() async def _send_full_update(self): """Sendet shop.full_update Event.""" await self._send_event('shop.full_update', { 'agent_id': self.agent_id, 'hostname': self.hostname, 'shops': self._get_all_shops_data(), 'system': self._get_system_stats() }) async def _send_stats_update(self): """Sendet Stats-Updates für aktive Shops.""" for shop in get_active_shops(): stats = get_shop_log_stats(shop) # Nur senden wenn sich etwas geändert hat last = self.last_stats.get(shop, {}) if stats != last: await self._send_event('shop.stats', { 'domain': shop, 'stats': stats }) self.last_stats[shop] = stats self.last_stats_time = time.time() async def _handle_message(self, message: str): """Verarbeitet eingehende Nachrichten vom Dashboard.""" try: data = json.loads(message) event_type = data.get('type') event_data = data.get('data', {}) logger.debug(f"Empfangen: {event_type}") if event_type == 'auth.approved': # Token speichern token = event_data.get('token') if token: self._save_token(token) self.approved = True logger.info("✅ Agent wurde im Dashboard freigegeben!") # Full Update senden await self._send_full_update() elif event_type == 'command.activate': await self._handle_activate_command(event_data) elif event_type == 'command.deactivate': await self._handle_deactivate_command(event_data) elif event_type == 'command.update': await self._handle_update_command(event_data) elif event_type == 'command.livestats': await self._handle_livestats_command(event_data) elif event_type == 'command.ban': await self._handle_ban_command(event_data) elif event_type == 'command.unban': await self._handle_unban_command(event_data) elif event_type == 'command.whitelist': await self._handle_whitelist_command(event_data) elif event_type == 'command.unwhitelist': await self._handle_unwhitelist_command(event_data) elif event_type == 'command.get_bans': await self._handle_get_bans_command(event_data) elif event_type == 'command.get_whitelist': await self._handle_get_whitelist_command(event_data) elif event_type == 'command.autoban_config': await self._handle_autoban_config_command(event_data) elif event_type == 'command.whois': await self._handle_whois_command(event_data) elif event_type == 'command.reset_stats': await self._handle_reset_stats_command(event_data) elif event_type == 'log.subscribe': shop = event_data.get('shop') if shop: self.log_watcher.start_watching(shop) elif event_type == 'log.unsubscribe': shop = event_data.get('shop') if shop: self.log_watcher.stop_watching(shop) elif event_type == 'ping': await self._send_event('pong', {'agent_id': self.agent_id}) except json.JSONDecodeError: logger.error(f"Ungültiges JSON empfangen: {message[:100]}") except Exception as e: logger.error(f"Fehler bei Nachrichtenverarbeitung: {e}") async def _handle_activate_command(self, data: Dict[str, Any]): """Verarbeitet activate-Command (v2.5).""" command_id = data.get('command_id', 'unknown') shop = data.get('shop') # v2.5 Parameter bot_mode = data.get('bot_mode', True) bot_rate_limit = data.get('bot_rate_limit') bot_ban_duration = data.get('bot_ban_duration') country_mode = data.get('country_mode', False) country_rate_limit = data.get('country_rate_limit') country_ban_duration = data.get('country_ban_duration') unlimited_countries = data.get('unlimited_countries', []) monitor_only = data.get('monitor_only', False) restart_fpm = data.get('restart_fpm', False) # Log-Meldung erstellen mode_parts = [] if monitor_only: mode_parts.append("MONITOR") else: if bot_mode: mode_parts.append(f"Bot({bot_rate_limit or 'default'}/min)") if country_mode: mode_parts.append(f"Country({country_rate_limit or 'default'}/min)") mode_str = " + ".join(mode_parts) if mode_parts else "None" logger.info(f"Aktiviere {shop}: {mode_str}") try: success = activate_blocking( shop=shop, silent=True, bot_mode=bot_mode, bot_rate_limit=bot_rate_limit, bot_ban_duration=bot_ban_duration, country_mode=country_mode, country_rate_limit=country_rate_limit, country_ban_duration=country_ban_duration, unlimited_countries=unlimited_countries, monitor_only=monitor_only ) if success: # PHP-FPM Restart wenn gewünscht fpm_result = None if restart_fpm: fpm_result = restart_php_fpm(shop) if fpm_result['success']: logger.info(f"PHP-FPM Restart erfolgreich: {fpm_result['message']}") else: logger.warning(f"PHP-FPM Restart fehlgeschlagen: {fpm_result['message']}") message = f'Shop {shop} aktiviert ({mode_str})' if fpm_result: message += f' | FPM: {fpm_result["message"]}' await self._send_event('command.result', { 'command_id': command_id, 'status': 'success', 'message': message, 'shop': shop, 'fpm_restart': fpm_result }) # Full Update senden await self._send_full_update() else: await self._send_event('command.result', { 'command_id': command_id, 'status': 'error', 'message': f'Aktivierung fehlgeschlagen', 'shop': shop }) except Exception as e: await self._send_event('command.result', { 'command_id': command_id, 'status': 'error', 'message': str(e), 'shop': shop }) async def _handle_deactivate_command(self, data: Dict[str, Any]): """Verarbeitet deactivate-Command.""" global _shop_stats_trackers command_id = data.get('command_id', 'unknown') shop = data.get('shop') restart_fpm = data.get('restart_fpm', False) logger.info(f"Deaktiviere {shop}") try: success = deactivate_blocking(shop, silent=True) if success: # Log-File löschen log_file = os.path.join(VHOSTS_DIR, shop, 'httpdocs', SHOP_LOG_FILE) if os.path.isfile(log_file): try: os.remove(log_file) logger.info(f"Log-File gelöscht: {log_file}") except Exception as e: logger.warning(f"Konnte Log-File nicht löschen: {e}") # Stats-Tracker für diesen Shop löschen if shop in _shop_stats_trackers: del _shop_stats_trackers[shop] logger.info(f"Stats-Tracker für {shop} gelöscht") # PHP-FPM Restart wenn gewünscht fpm_result = None if restart_fpm: fpm_result = restart_php_fpm(shop) if fpm_result['success']: logger.info(f"PHP-FPM Restart erfolgreich: {fpm_result['message']}") else: logger.warning(f"PHP-FPM Restart fehlgeschlagen: {fpm_result['message']}") message = f'Shop {shop} deaktiviert (Logs & Stats gelöscht)' if fpm_result: message += f' | FPM: {fpm_result["message"]}' await self._send_event('command.result', { 'command_id': command_id, 'status': 'success', 'message': message, 'shop': shop, 'fpm_restart': fpm_result }) # Full Update senden await self._send_full_update() else: await self._send_event('command.result', { 'command_id': command_id, 'status': 'error', 'message': f'Deaktivierung fehlgeschlagen', 'shop': shop }) except Exception as e: await self._send_event('command.result', { 'command_id': command_id, 'status': 'error', 'message': str(e), 'shop': shop }) async def _handle_update_command(self, data: Dict[str, Any]): """Verarbeitet update-Command - Agent selbst updaten.""" command_id = data.get('command_id', 'unknown') logger.info("Update-Command empfangen - starte Self-Update...") try: import urllib.request # 1. Download neue Version logger.info(f"Lade neue Version von {AGENT_UPDATE_URL}...") request = urllib.request.Request( AGENT_UPDATE_URL, headers={'User-Agent': f'JTL-WAFi-Agent/{VERSION}'} ) with urllib.request.urlopen(request, timeout=30) as response: new_content = response.read().decode('utf-8') # 2. Syntax-Check logger.info("Prüfe Syntax der neuen Version...") compile(new_content, '', 'exec') # 3. Version extrahieren (optional, für Log) new_version = "unknown" for line in new_content.split('\n')[:50]: if 'VERSION = ' in line: new_version = line.split('=')[1].strip().strip('"\'') break logger.info(f"Neue Version: {new_version} (aktuell: {VERSION})") # 4. Script-Pfad ermitteln script_path = os.path.abspath(__file__) backup_path = script_path + '.backup' # 5. Backup erstellen logger.info(f"Erstelle Backup: {backup_path}") shutil.copy(script_path, backup_path) # 6. Neue Version schreiben logger.info(f"Schreibe neue Version nach {script_path}...") with open(script_path, 'w', encoding='utf-8') as f: f.write(new_content) # 7. Erfolg melden await self._send_event('command.result', { 'command_id': command_id, 'status': 'success', 'message': f'Update erfolgreich ({VERSION} -> {new_version}). Agent wird neugestartet...' }) # Kurz warten damit Nachricht gesendet wird await asyncio.sleep(1) # 8. Neustart via os.execv (ersetzt den aktuellen Prozess) logger.info("Starte Agent neu via os.execv...") os.execv(sys.executable, [sys.executable, script_path]) except urllib.error.URLError as e: error_msg = f'Download fehlgeschlagen: {str(e)}' logger.error(error_msg) await self._send_event('command.result', { 'command_id': command_id, 'status': 'error', 'message': error_msg }) except SyntaxError as e: error_msg = f'Syntax-Fehler in neuer Version: {str(e)}' logger.error(error_msg) await self._send_event('command.result', { 'command_id': command_id, 'status': 'error', 'message': error_msg }) except Exception as e: error_msg = f'Update fehlgeschlagen: {str(e)}' logger.error(error_msg) await self._send_event('command.result', { 'command_id': command_id, 'status': 'error', 'message': error_msg }) async def _handle_livestats_command(self, data: Dict[str, Any]): """Gibt Live-Statistiken für einen Shop zurück.""" command_id = data.get('command_id', 'unknown') shop = data.get('shop') window = data.get('window', DEFAULT_STATS_WINDOW) if not shop: await self._send_event('livestats.result', { 'command_id': command_id, 'success': False, 'error': 'Shop nicht angegeben' }) return try: window_seconds = STATS_WINDOWS.get(window, STATS_WINDOWS[DEFAULT_STATS_WINDOW]) tracker = get_shop_stats_tracker(shop, window_seconds) stats = tracker.get_stats() # Banned/Whitelisted IPs hinzufügen stats['banned_ips'] = get_banned_ips(shop) stats['whitelisted_ips'] = get_whitelisted_ips(shop) stats['autoban_config'] = get_autoban_config(shop) await self._send_event('livestats.result', { 'command_id': command_id, 'success': True, 'shop': shop, 'stats': stats }) except Exception as e: logger.error(f"Fehler bei livestats für {shop}: {e}") await self._send_event('livestats.result', { 'command_id': command_id, 'success': False, 'error': str(e) }) async def _handle_ban_command(self, data: Dict[str, Any]): """Bannt eine IP für einen Shop.""" command_id = data.get('command_id', 'unknown') shop = data.get('shop') ip = data.get('ip') duration = data.get('duration', 3600) # Default: 1 Stunde reason = data.get('reason', 'Manual ban') if not shop or not ip: await self._send_event('command.result', { 'command_id': command_id, 'status': 'error', 'message': 'Shop oder IP nicht angegeben' }) return try: result = ban_ip(shop, ip, duration, reason) await self._send_event('command.result', { 'command_id': command_id, 'status': 'success' if result['success'] else 'error', 'message': result['message'], 'shop': shop, 'ip': ip }) except Exception as e: await self._send_event('command.result', { 'command_id': command_id, 'status': 'error', 'message': str(e) }) async def _handle_unban_command(self, data: Dict[str, Any]): """Entfernt Ban für eine IP.""" command_id = data.get('command_id', 'unknown') shop = data.get('shop') ip = data.get('ip') if not shop or not ip: await self._send_event('command.result', { 'command_id': command_id, 'status': 'error', 'message': 'Shop oder IP nicht angegeben' }) return try: result = unban_ip(shop, ip) await self._send_event('command.result', { 'command_id': command_id, 'status': 'success' if result['success'] else 'error', 'message': result['message'], 'shop': shop, 'ip': ip }) except Exception as e: await self._send_event('command.result', { 'command_id': command_id, 'status': 'error', 'message': str(e) }) async def _handle_whitelist_command(self, data: Dict[str, Any]): """Fügt IP zur Whitelist hinzu.""" command_id = data.get('command_id', 'unknown') shop = data.get('shop') ip = data.get('ip') description = data.get('description', '') if not shop or not ip: await self._send_event('command.result', { 'command_id': command_id, 'status': 'error', 'message': 'Shop oder IP nicht angegeben' }) return try: result = whitelist_ip(shop, ip, description) await self._send_event('command.result', { 'command_id': command_id, 'status': 'success' if result['success'] else 'error', 'message': result['message'], 'shop': shop, 'ip': ip }) except Exception as e: await self._send_event('command.result', { 'command_id': command_id, 'status': 'error', 'message': str(e) }) async def _handle_unwhitelist_command(self, data: Dict[str, Any]): """Entfernt IP von Whitelist.""" command_id = data.get('command_id', 'unknown') shop = data.get('shop') ip = data.get('ip') if not shop or not ip: await self._send_event('command.result', { 'command_id': command_id, 'status': 'error', 'message': 'Shop oder IP nicht angegeben' }) return try: result = remove_from_whitelist(shop, ip) await self._send_event('command.result', { 'command_id': command_id, 'status': 'success' if result['success'] else 'error', 'message': result['message'], 'shop': shop, 'ip': ip }) except Exception as e: await self._send_event('command.result', { 'command_id': command_id, 'status': 'error', 'message': str(e) }) async def _handle_get_bans_command(self, data: Dict[str, Any]): """Gibt gebannte IPs für einen Shop zurück.""" command_id = data.get('command_id', 'unknown') shop = data.get('shop') if not shop: await self._send_event('bans.result', { 'command_id': command_id, 'success': False, 'error': 'Shop nicht angegeben' }) return try: banned = get_banned_ips(shop) await self._send_event('bans.result', { 'command_id': command_id, 'success': True, 'shop': shop, 'banned_ips': banned }) except Exception as e: await self._send_event('bans.result', { 'command_id': command_id, 'success': False, 'error': str(e) }) async def _handle_get_whitelist_command(self, data: Dict[str, Any]): """Gibt Whitelist für einen Shop zurück.""" command_id = data.get('command_id', 'unknown') shop = data.get('shop') if not shop: await self._send_event('whitelist.result', { 'command_id': command_id, 'success': False, 'error': 'Shop nicht angegeben' }) return try: whitelisted = get_whitelisted_ips(shop) await self._send_event('whitelist.result', { 'command_id': command_id, 'success': True, 'shop': shop, 'whitelisted_ips': whitelisted }) except Exception as e: await self._send_event('whitelist.result', { 'command_id': command_id, 'success': False, 'error': str(e) }) async def _handle_autoban_config_command(self, data: Dict[str, Any]): """Konfiguriert Auto-Ban für einen Shop.""" command_id = data.get('command_id', 'unknown') shop = data.get('shop') config = data.get('config') if not shop: await self._send_event('command.result', { 'command_id': command_id, 'status': 'error', 'message': 'Shop nicht angegeben' }) return try: if config: # Konfiguration speichern save_autoban_config(shop, config) await self._send_event('command.result', { 'command_id': command_id, 'status': 'success', 'message': f'Auto-Ban Konfiguration für {shop} gespeichert', 'shop': shop }) else: # Konfiguration abrufen current_config = get_autoban_config(shop) await self._send_event('autoban_config.result', { 'command_id': command_id, 'success': True, 'shop': shop, 'config': current_config }) except Exception as e: await self._send_event('command.result', { 'command_id': command_id, 'status': 'error', 'message': str(e) }) async def _handle_whois_command(self, data: Dict[str, Any]): """Führt WHOIS-Lookup für eine IP durch und sendet Ergebnis mit Request-Historie.""" command_id = data.get('command_id', 'unknown') ip = data.get('ip') shop = data.get('shop') # Optional: für Request-Historie if not ip: await self._send_event('whois_result', { 'command_id': command_id, 'success': False, 'error': 'Keine IP angegeben' }) return try: # WHOIS-Lookup durchführen (wird gecacht) result = whois_lookup(ip) # Request-Historie für diese IP sammeln - IMMER aus allen aktiven Shops # (auch wenn ein spezifischer Shop angegeben ist, könnte die IP in mehreren Shops sein) requests = [] shops_to_check = list(get_active_shops()) logger.info(f"WHOIS für {ip}: Prüfe {len(shops_to_check)} Shops: {shops_to_check}") for s in shops_to_check: tracker = get_shop_stats_tracker(s) logger.info(f"Shop {s}: Tracker hat {len(tracker.ip_request_details)} IPs in ip_request_details") # Debug: Zeige alle IPs im Tracker all_ips = list(tracker.ip_request_details.keys())[:10] # Erste 10 logger.info(f"Shop {s}: Erste 10 IPs im Tracker: {all_ips}") shop_requests = tracker.get_ip_requests(ip) # Falls keine Requests für diese IP gefunden, versuche nochmal aus Log zu laden # (könnte sein, dass neue Log-Einträge existieren die noch nicht geladen wurden) if not shop_requests: # Prüfe ob Log-File existiert und neuer ist als Tracker-Erstellung log_file = os.path.join(VHOSTS_DIR, s, 'httpdocs', SHOP_LOG_FILE) if os.path.isfile(log_file): logger.info(f"Shop {s}: IP {ip} nicht gefunden - Reload aus Log") _load_initial_request_details(tracker, s) shop_requests = tracker.get_ip_requests(ip) if shop_requests: logger.info(f"Shop {s}: {len(shop_requests)} Requests für IP {ip}") else: logger.info(f"Shop {s}: Keine Requests für IP {ip}") # Kopie erstellen um Original nicht zu modifizieren for r in shop_requests: req_copy = r.copy() req_copy['shop'] = s requests.append(req_copy) # Nach Zeit sortieren (neueste zuerst) if requests: requests.sort(key=lambda x: x['ts'], reverse=True) # Ergebnis senden await self._send_event('whois_result', { 'command_id': command_id, 'success': True, 'ip': ip, 'netname': result.get('netname', ''), 'org': result.get('org', ''), 'asn': result.get('asn', ''), 'country': result.get('country', ''), 'abuse': result.get('abuse', ''), 'range': result.get('range', ''), 'requests': requests }) logger.info(f"WHOIS für {ip}: {result.get('org', 'Unknown')} ({result.get('asn', 'N/A')}) - {len(requests)} Requests") except Exception as e: logger.error(f"WHOIS Fehler für {ip}: {e}") await self._send_event('whois_result', { 'command_id': command_id, 'success': False, 'ip': ip, 'error': str(e) }) async def _handle_reset_stats_command(self, data: Dict[str, Any]): """Setzt alle In-Memory Caches, Statistiken und Log-Files zurück.""" global _ip_info_cache, _whois_cache, _shop_stats_trackers command_id = data.get('command_id', 'unknown') try: # IP-Info Cache leeren _ip_info_cache.clear() logger.info("IP-Info Cache geleert") # WHOIS Cache leeren _whois_cache.clear() logger.info("WHOIS Cache geleert") # LiveStats Tracker für alle Shops leeren _shop_stats_trackers.clear() logger.info("LiveStats Tracker geleert") # Log-Files aller aktiven Shops löschen deleted_logs = 0 for shop in get_active_shops(): log_file = os.path.join(VHOSTS_DIR, shop, 'httpdocs', SHOP_LOG_FILE) if os.path.isfile(log_file): try: os.remove(log_file) deleted_logs += 1 logger.info(f"Log-File gelöscht: {log_file}") except Exception as e: logger.warning(f"Konnte Log-File nicht löschen {log_file}: {e}") await self._send_event('command.result', { 'command_id': command_id, 'status': 'success', 'message': f'Agent {self.hostname}: Alle Caches zurückgesetzt, {deleted_logs} Log-Files gelöscht' }) logger.info(f"Reset Stats: Alle In-Memory Daten zurückgesetzt, {deleted_logs} Log-Files gelöscht") except Exception as e: logger.error(f"Reset Stats Fehler: {e}") await self._send_event('command.result', { 'command_id': command_id, 'status': 'error', 'message': str(e) }) async def _periodic_tasks(self): """Führt periodische Tasks aus.""" while self.running: try: now = time.time() # Heartbeat (alle 60 Sekunden) if now - self.last_heartbeat_time >= HEARTBEAT_INTERVAL: if self.approved: await self._send_heartbeat() # Stats Update (alle 10 Sekunden) if now - self.last_stats_time >= STATS_UPDATE_INTERVAL: if self.approved: await self._send_stats_update() # Log Rotation (alle 5 Minuten prüfen) if now - self.last_log_rotation_time >= 300: rotate_shop_logs() self.last_log_rotation_time = now await asyncio.sleep(1) except asyncio.CancelledError: break except Exception as e: logger.error(f"Fehler in periodic_tasks: {e}") await asyncio.sleep(5) async def connect(self): """Stellt WebSocket-Verbindung her.""" # SSL Context der Self-Signed Certificates akzeptiert ssl_context = ssl.create_default_context() ssl_context.check_hostname = False ssl_context.verify_mode = ssl.CERT_NONE try: # websockets importieren import websockets logger.info(f"Verbinde zu {self.dashboard_url}...") async with websockets.connect( self.dashboard_url, ssl=ssl_context, ping_interval=30, ping_timeout=10, close_timeout=5 ) as websocket: self.ws = websocket self.reconnect_delay = RECONNECT_BASE_DELAY logger.info("✅ WebSocket verbunden") # Connect-Event senden await self._send_connect() # Periodic Tasks starten periodic_task = asyncio.create_task(self._periodic_tasks()) try: # Nachrichten empfangen async for message in websocket: await self._handle_message(message) finally: periodic_task.cancel() try: await periodic_task except asyncio.CancelledError: pass except ImportError: logger.error("websockets-Modul nicht installiert! Installiere mit: pip install websockets") raise except Exception as e: logger.error(f"WebSocket Fehler: {e}") raise finally: self.ws = None self.approved = False async def run_async(self): """Hauptschleife mit Auto-Reconnect.""" self._loop = asyncio.get_running_loop() self.running = True self._shutdown_event = asyncio.Event() # Signal Handler für asyncio def signal_handler(): logger.info("Beende Agent (Signal empfangen)...") self.running = False self._shutdown_event.set() # WebSocket schließen falls verbunden if self.ws: asyncio.create_task(self._close_websocket()) # Signale registrieren for sig in (signal.SIGINT, signal.SIGTERM): self._loop.add_signal_handler(sig, signal_handler) # LogWatcher starten self.log_watcher.start() try: while self.running: try: await self.connect() except asyncio.CancelledError: break except Exception as e: if not self.running: break logger.warning(f"Verbindung getrennt: {e}") if self.running: logger.info(f"Reconnect in {self.reconnect_delay}s...") try: # Warte mit Timeout, damit Signal schnell reagiert await asyncio.wait_for( self._shutdown_event.wait(), timeout=self.reconnect_delay ) break # Shutdown signal received except asyncio.TimeoutError: pass # Normal timeout, reconnect # Exponential Backoff self.reconnect_delay = min( self.reconnect_delay * 2, RECONNECT_MAX_DELAY ) finally: # LogWatcher stoppen self.log_watcher.stop() # Signal Handler entfernen for sig in (signal.SIGINT, signal.SIGTERM): self._loop.remove_signal_handler(sig) async def _close_websocket(self): """Schließt WebSocket-Verbindung.""" if self.ws: try: await self.ws.close() except: pass def run(self): """Startet den Agent.""" logger.info("=" * 60) logger.info(f"JTL-WAFi Agent v{VERSION} (WebSocket Real-Time)") logger.info(f"Hostname: {self.hostname}") logger.info(f"Agent-ID: {self.agent_id}") logger.info(f"Dashboard: {self.dashboard_url}") logger.info(f"Token: {'vorhanden' if self.token else 'nicht vorhanden'}") logger.info("=" * 60) # Asyncio Loop starten try: asyncio.run(self.run_async()) except KeyboardInterrupt: pass logger.info("Agent beendet.") # ============================================================================= # CLI INTERFACE # ============================================================================= def create_systemd_service(): """Erstellt systemd Service-Datei.""" service = """[Unit] Description=JTL-WAFi Agent v3.0 (WebSocket) After=network.target [Service] Type=simple ExecStart=/usr/bin/python3 /opt/jtl-wafi/jtl_wafi_agent.py Restart=always RestartSec=10 User=root Environment=PYTHONUNBUFFERED=1 [Install] WantedBy=multi-user.target """ service_path = "/etc/systemd/system/jtl-wafi-agent.service" try: with open(service_path, 'w') as f: f.write(service) print(f"✅ Service erstellt: {service_path}") print(" Aktivieren mit: systemctl daemon-reload && systemctl enable --now jtl-wafi-agent") except PermissionError: print("❌ Root-Rechte erforderlich!") sys.exit(1) def check_dependencies(): """Prüft ob alle Abhängigkeiten installiert sind.""" missing = [] try: import websockets except ImportError: missing.append("websockets") if missing: print("❌ Fehlende Abhängigkeiten:") for dep in missing: print(f" - {dep}") print(f"\nInstallieren mit: pip install {' '.join(missing)}") return False return True def main(): """Hauptfunktion mit CLI-Argumenten.""" import argparse parser = argparse.ArgumentParser( description=f"JTL-WAFi Agent v{VERSION} - WebSocket Real-Time Agent" ) parser.add_argument( "--url", default=DEFAULT_DASHBOARD_URL, help=f"Dashboard WebSocket URL (default: {DEFAULT_DASHBOARD_URL})" ) parser.add_argument( "--debug", action="store_true", help="Debug-Logging aktivieren" ) parser.add_argument( "--install-service", action="store_true", help="Systemd Service installieren" ) parser.add_argument( "--check-deps", action="store_true", help="Abhängigkeiten prüfen" ) args = parser.parse_args() # Logging initialisieren global logger logger = setup_logging(debug=args.debug) if args.install_service: create_systemd_service() return if args.check_deps: if check_dependencies(): print("✅ Alle Abhängigkeiten sind installiert") return # Abhängigkeiten prüfen if not check_dependencies(): sys.exit(1) # Root-Check if os.geteuid() != 0: print("❌ Root-Rechte erforderlich!") sys.exit(1) # Country IP-Ranges laden/initialisieren (im Hintergrund) logger.info("Initialisiere Country IP-Ranges Cache...") download_country_ranges_async() # Agent starten agent = JTLWAFiAgent(dashboard_url=args.url) agent.run() if __name__ == "__main__": main()