Serverjonas/search
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add stuff

Browse Source
This commit is contained in:
jhkaktus
2026-10-09 16:42:15 +02:00
parent 02bd184043
commit 19730e1711
13 changed files with 3448 additions and 995 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crawler.py
+506 -42
View File
@@ -14,7 +14,17 @@ Features:
bounded retries with rollback + backoff)
- Smart priority queue: prefers never-crawled URLs, new domains and
new hosts, with a spread bonus for little-explored territory
- Per-host crawl delay
- Host-aware scheduling: a worker never waits on a busy host - when
the current domain is rate-limited or blocked, the worker
automatically picks another domain and returns once the host is
crawlable again
- Per-host crawl delay (robots.txt) and per-host block backoff:
HTTP 429/5xx/403 and network errors pause the host with an
exponential backoff, Retry-After is honored
- Transient failures are requeued (bounded by JONASSEARCH_MAX_ATTEMPTS)
instead of being lost
- Non-crawlable targets (localhost, raw IPs, non-real domains) are
rejected before they reach the queue
- robots.txt support with in-memory cache
- Graceful shutdown
- Basic crawler statistics
@@ -52,10 +62,12 @@ import signal
import time
import logging
import argparse
import ipaddress
import subprocess
import threading
import multiprocessing as mp
from collections import defaultdict
from email.utils import parsedate_to_datetime
from html import unescape
from pathlib import Path
from urllib.parse import parse_qsl, urljoin, urlparse, urldefrag
@@ -138,7 +150,7 @@ USER_AGENT = os.getenv(
# change; completely independent from the deployment generation (the
# generation is a global deployment counter in PostgreSQL, the version
# identifies the crawler code).
CRAWLER_VERSION = int(os.getenv("JONASSEARCH_CRAWLER_VERSION", "1"))
CRAWLER_VERSION = int(os.getenv("JONASSEARCH_CRAWLER_VERSION", "3"))
# Exit codes. 42 is reserved exclusively for "I must be updated" so
# the launcher can distinguish update, crash and normal exit.
@@ -173,6 +185,16 @@ LAUNCHER_RESTART_DELAY = float(
# Only crawl normal HTTP(S) pages.
ALLOWED_SCHEMES = {"http", "https"}
# Host suffixes reserved for loopback/private networks (never delegated
# in public DNS). Together with the localhost/IP checks in
# is_non_crawlable_host() they keep non-real domains out of the queue.
NON_CRAWLABLE_HOST_SUFFIXES = (
".internal",
".local",
".localdomain",
".localhost",
)
# Ignore obvious non-HTML resources.
BLOCKED_EXTENSIONS = {
".7z", ".avi", ".bin", ".bmp", ".bz2", ".class", ".csv", ".doc",
@@ -244,6 +266,69 @@ QUEUE_INSERT_PAGE_SIZE = 500
QUEUE_INSERT_MAX_ATTEMPTS = 3
QUEUE_INSERT_RETRY_BASE_DELAY = 0.5
# ============================================================
# HOST SCHEDULING / BLOCK HANDLING CONFIG
# ============================================================
# The queue claim examines candidates in priority order and picks the
# first URL whose host is currently crawlable (not rate-limited, not
# blocked). Hosts that would have to wait are skipped, so a worker
# automatically crawls another domain while the blocked one cools
# down - no worker sits idle in front of a 429.
# Pending URLs examined per claim round (FOR UPDATE SKIP LOCKED, so
# several crawler processes stay safe on one shared queue).
QUEUE_CLAIM_BATCH = int(os.getenv("JONASSEARCH_CLAIM_BATCH", "25"))
# Maximum number of candidates examined per claim (batch * rounds).
# Keeps the common case (a ready host in the first batch) cheap while
# still finding crawlable hosts behind a wall of blocked ones.
QUEUE_CLAIM_SCAN_LIMIT = int(
os.getenv("JONASSEARCH_CLAIM_SCAN_LIMIT", "200")
)
# Cap for the "nothing is crawlable right now" sleep, so a host blocked
# for a long time cannot park the whole worker; after the cap the queue
# is simply re-scanned.
QUEUE_MAX_IDLE_SLEEP = float(os.getenv("JONASSEARCH_MAX_IDLE_SLEEP", "30"))
# A URL may be attempted this many times (transient errors are
# requeued) before it is marked failed for good, so permanently broken
# hosts cannot spin forever.
QUEUE_MAX_ATTEMPTS = int(os.getenv("JONASSEARCH_MAX_ATTEMPTS", "5"))
# Rate waits longer than this are NOT slept inline: the item is put
# back and the worker immediately looks for another domain instead of
# blocking on a slow host (e.g. a large robots.txt crawl-delay).
MAX_INLINE_RATE_WAIT = float(os.getenv("JONASSEARCH_MAX_INLINE_WAIT", "10"))
# Backoff for blocked hosts, per HTTP status: (base seconds, cap
# seconds). The pause doubles with every consecutive block of the same
# host, so persistent blocks fade out of the schedule while a single
# hiccup costs only the base delay.
HOST_BLOCK_RULES = {
401: (600.0, 7200.0), # auth required: probably permanent, long pause
403: (600.0, 7200.0), # forbidden
408: (30.0, 600.0), # request timeout
425: (30.0, 600.0), # too early
429: (60.0, 1800.0), # rate limited (Retry-After is honored on top)
500: (30.0, 600.0),
502: (30.0, 600.0),
503: (30.0, 900.0),
504: (30.0, 600.0),
}
# Network-level failures (DNS, timeout, connection reset) get a short
# backoff as well - a dead host must not be retried in a tight loop.
HOST_BLOCK_NETWORK_BASE = 30.0
HOST_BLOCK_NETWORK_MAX = 300.0
# Absolute upper bound for one host pause (also caps Retry-After).
HOST_BLOCK_HARD_CAP = 7200.0
# HTTP statuses whose queue items are requeued (retried later) instead
# of being failed terminally.
TRANSIENT_HTTP_STATUSES = {408, 425, 429, 500, 502, 503, 504}
# ============================================================
# URL PRIORITY CONFIG
# ============================================================
@@ -646,35 +731,135 @@ def write_stats(conn, crawled, queued, failed):
""", (crawled, queued, failed))
def get_next_queue_item(conn):
def claim_next_queue_item(conn, scheduler, robots):
"""
Atomically claim one pending URL.
Atomically claim the best currently-crawlable pending URL.
SKIP LOCKED is the important part: several crawler processes can
consume the same PostgreSQL queue without waiting for each other.
Candidates are read in priority order (same ORDER BY as before) in
small batches with FOR UPDATE SKIP LOCKED, so several crawler
processes can still share one queue without waiting for each
other. For every candidate the HostScheduler decides whether its
host may be contacted right now (per-host crawl delay, active
block backoff). The first ready candidate is claimed; hosts that
would have to wait are skipped, so the worker automatically works
on another domain while a rate-limited or blocked host cools down.
Returns (item, wait_hint):
item (id, url, source_url) of the claimed URL, or None
wait_hint seconds until the soonest examined host becomes
crawlable again (None when the queue is empty)
Writes only - the commit happens in safe_commit().
"""
now = time.monotonic()
seen_ids = []
scanned = 0
wait_hint = None
item = None
while item is None and scanned < QUEUE_CLAIM_SCAN_LIMIT:
with conn.cursor() as cur:
if seen_ids:
cur.execute("""
SELECT id, url, source_url
FROM crawl_queue
WHERE status = 'pending'
AND NOT (id = ANY(%s))
ORDER BY priority DESC, discovered_at ASC, id ASC
FOR UPDATE SKIP LOCKED
LIMIT %s
""", (seen_ids, QUEUE_CLAIM_BATCH))
else:
cur.execute("""
SELECT id, url, source_url
FROM crawl_queue
WHERE status = 'pending'
ORDER BY priority DESC, discovered_at ASC, id ASC
FOR UPDATE SKIP LOCKED
LIMIT %s
""", (QUEUE_CLAIM_BATCH,))
candidates = cur.fetchall()
if not candidates:
break
for queue_id, url, source_url in candidates:
seen_ids.append(queue_id)
scanned += 1
wait_for = scheduler.wait_time(url, robots.delay(url), now)
if wait_for > 0.0:
if wait_hint is None or wait_for < wait_hint:
wait_hint = wait_for
continue
# Host is crawlable: claim it. The row is already locked by
# this transaction (FOR UPDATE SKIP LOCKED), so the
# status='pending' guard is just belt and braces against
# races with other claim paths.
with conn.cursor() as cur:
cur.execute("""
UPDATE crawl_queue
SET status = 'processing',
attempts = attempts + 1,
last_attempt_at = now()
WHERE id = %s AND status = 'pending'
RETURNING id, url, source_url
""", (queue_id,))
row = cur.fetchone()
if row:
item = row
break
if len(candidates) < QUEUE_CLAIM_BATCH:
break
return item, wait_hint
def defer_queue_item(conn, queue_id):
"""
Put a claimed item back WITHOUT burning an attempt: the host turned
out not to be crawlable at fetch time (e.g. a robots.txt crawl-delay
larger than MAX_INLINE_RATE_WAIT was discovered only after
claiming). The item stays pending and is retried when the host's
schedule allows it - the worker crawls other domains until then.
Writes only - the commit happens in safe_commit().
"""
with conn.cursor() as cur:
cur.execute("""
WITH next_item AS (
SELECT id
FROM crawl_queue
WHERE status = 'pending'
ORDER BY priority DESC, discovered_at ASC, id ASC
FOR UPDATE SKIP LOCKED
LIMIT 1
)
UPDATE crawl_queue q
SET status = 'processing',
attempts = attempts + 1,
last_attempt_at = now()
FROM next_item
WHERE q.id = next_item.id
RETURNING q.id, q.url, q.source_url
""")
UPDATE crawl_queue
SET status = 'pending',
attempts = GREATEST(attempts - 1, 0)
WHERE id = %s
""", (queue_id,))
def requeue_queue_item(conn, queue_id):
"""
Requeue an item after a transient failure (HTTP 429/5xx, network
errors): the host got a backoff, the URL is retried afterwards.
After QUEUE_MAX_ATTEMPTS attempts the item is marked failed for
good, so a permanently broken host cannot spin forever.
Returns the new status ('pending' or 'failed'), or None.
Writes only - the commit happens in safe_commit().
"""
with conn.cursor() as cur:
cur.execute("""
UPDATE crawl_queue
SET status = CASE
WHEN attempts >= %s THEN 'failed'
ELSE 'pending'
END
WHERE id = %s
RETURNING status
""", (QUEUE_MAX_ATTEMPTS, queue_id))
row = cur.fetchone()
return row
return row[0] if row else None
def finish_queue_item(conn, queue_id, success=True):
@@ -1290,6 +1475,54 @@ def clean_text_list(values):
# URL NORMALIZATION
# ============================================================
def is_non_crawlable_host(hostname):
"""
True for hosts the crawler must never touch: loopback names,
private-use suffixes, raw IP literals (IPv4/IPv6) and other
non-resolvable (non-real) host names. A public web crawler can only
follow real, publicly resolving domains - everything else is a LAN
target, a loopback service or a literal address.
Applied centrally in normalize_url(), so neither the seed URL nor
any discovered link ever reaches the queue.
"""
if not hostname:
return True
host = hostname.strip(".").lower()
if not host:
return True
if host == "localhost" or host == "localhost.localdomain":
return True
if host.endswith(NON_CRAWLABLE_HOST_SUFFIXES):
return True
# Raw IP literals (IPv4 and IPv6) are not real domains. Note that
# urlparse already strips the brackets of IPv6 URLs, so "::1" is
# compared directly.
try:
ipaddress.ip_address(host)
return True
except ValueError:
pass
# Single-label names (no dot) can never be public domains.
if "." not in host:
return True
return False
def is_non_crawlable_url(url):
"""
Convenience wrapper for queue entries: True when the URL points at
a non-crawlable host (localhost, raw IP, non-real domain) or cannot
be parsed at all. Unparseable URLs are treated as non-crawlable so
one malformed entry can never crash the worker.
"""
try:
hostname = (urlparse(url).hostname or "").lower()
except ValueError:
return True
return is_non_crawlable_host(hostname)
def normalize_url(url, base_url=None):
if not url:
return None
@@ -1319,6 +1552,11 @@ def normalize_url(url, base_url=None):
if not hostname:
return None
# Never queue loopback, LAN or raw-IP targets (localhost, IPs,
# non-real domains).
if is_non_crawlable_host(hostname):
return None
# Drop default ports.
port = parsed.port
if (scheme == "http" and port == 80) or (scheme == "https" and port == 443):
@@ -1423,20 +1661,159 @@ class RobotsCache:
# HTTP
# ============================================================
class RateLimiter:
class HostScheduler:
"""
Per-process host availability tracker - the heart of the automatic
domain switching.
The scheduler keeps, per host:
last_request: monotonic time of the last request start, so
per-host delays (DEFAULT_DELAY / robots.txt
crawl-delay) are honored.
blocked_until: monotonic time until which the host is considered
blocked (HTTP 429/5xx/403, network errors,
Retry-After).
block_count: consecutive blocks of the host; the pause doubles
with every consecutive block and resets on a
successful fetch.
claim_next_queue_item() asks wait_time() for every queue candidate:
a host that would have to wait is skipped, so the worker crawls
another domain instead and returns to the blocked host when its
backoff has expired. No worker ever sleeps in front of a
rate-limited host while other domains are crawlable.
"""
def __init__(self):
self.last_request = defaultdict(float)
self.last_request = {}
self.blocked_until = {}
self.block_count = {}
@staticmethod
def _host(url):
try:
return (urlparse(url).netloc or "").lower()
except ValueError:
return ""
def wait_time(self, url, delay, now=None):
"""Seconds until url's host may be contacted again (0.0 = now)."""
if now is None:
now = time.monotonic()
host = self._host(url)
if not host:
return 0.0
next_allowed = max(
self.last_request.get(host, 0.0) + delay,
self.blocked_until.get(host, 0.0),
)
return max(0.0, next_allowed - now)
def is_ready(self, url, delay, now=None):
return self.wait_time(url, delay, now) <= 0.0
def wait(self, url, delay):
host = urlparse(url).netloc.lower()
now = time.monotonic()
wait_for = delay - (now - self.last_request[host])
"""
Enforce the per-host delay directly before a request.
Returns True when the request may start now (the wait was slept
inline). Returns False when the wait would exceed
MAX_INLINE_RATE_WAIT (e.g. a large robots.txt crawl-delay that
claim_next_queue_item could not know yet, because robots.txt is
only loaded during the first fetch): the caller then defers the
item instead of parking the worker, and the host's schedule
stays untouched, so the item is retried exactly when the delay
has passed.
"""
wait_for = self.wait_time(url, delay)
if wait_for > MAX_INLINE_RATE_WAIT:
return False
if wait_for > 0:
logging.info("[RATE] %s: %.1fs", host, wait_for)
if wait_for >= 1.0:
logging.info("[RATE ] %s: %.1fs", self._host(url), wait_for)
time.sleep(wait_for)
self.last_request[self._host(url)] = time.monotonic()
return True
self.last_request[host] = time.monotonic()
def _register_block(self, host, seconds):
count = self.block_count.get(host, 0) + 1
self.block_count[host] = count
blocked = min(seconds, HOST_BLOCK_HARD_CAP)
self.blocked_until[host] = time.monotonic() + blocked
return count, blocked
def note_success(self, url):
"""Successful fetch: the host's consecutive-block counter resets."""
self.block_count.pop(self._host(url), None)
def note_http_error(self, url, status, retry_after=None):
"""
Pause the host after a block-ish HTTP status (429, 5xx, 403,
...). An exponential backoff applies; a Retry-After header is
honored on top of it.
"""
rule = HOST_BLOCK_RULES.get(status)
if rule is None:
return
host = self._host(url)
base, cap = rule
seconds = min(cap, base * (2 ** self.block_count.get(host, 0)))
if retry_after is not None:
seconds = max(seconds, retry_after)
count, blocked = self._register_block(host, seconds)
logging.info(
"[BLOCK] %s: HTTP %d -> host paused for %.0fs (block #%d)",
host, status, blocked, count,
)
def note_network_error(self, url):
"""Short backoff after DNS/timeout/connection failures."""
host = self._host(url)
seconds = min(
HOST_BLOCK_NETWORK_MAX,
HOST_BLOCK_NETWORK_BASE * (2 ** self.block_count.get(host, 0)),
)
count, blocked = self._register_block(host, seconds)
logging.info(
"[BLOCK] %s: network error -> host paused for %.0fs (block #%d)",
host, blocked, count,
)
def _retry_after_seconds(headers):
"""
Parse a Retry-After header (delta-seconds or HTTP date). Returns
the number of seconds, or None when absent/unparseable.
"""
try:
value = headers.get("Retry-After") if headers is not None else None
except Exception:
return None
if not value:
return None
value = value.strip()
if value.isdigit():
return float(value)
try:
retry_at = parsedate_to_datetime(value)
return max(0.0, retry_at.timestamp() - time.time())
except Exception:
return None
def is_transient_error(error):
"""
True for fetch errors worth retrying later (the item is requeued
and the host gets a backoff): network-level failures and the HTTP
statuses in TRANSIENT_HTTP_STATUSES. Everything else (404, robots,
too large, not HTML, ...) stays a terminal failure.
"""
if not error:
return False
if error.startswith("network:"):
return True
match = re.match(r"^http_(\d{3})$", error)
return bool(match) and int(match.group(1)) in TRANSIENT_HTTP_STATUSES
def is_skipped_url(url):
@@ -1448,14 +1825,20 @@ def is_skipped_url(url):
return any(pattern.match(url) for pattern in SKIP_URL_PATTERNS)
def fetch(url, limiter, robots):
def fetch(url, scheduler, robots):
parsed = urlparse(url)
if not robots.allowed(url):
return None, None, "robots"
delay = robots.delay(url)
limiter.wait(url, delay)
if not scheduler.wait(url, delay):
# The host needs a longer break than we are willing to sleep
# for inline (e.g. a large robots.txt crawl-delay discovered
# only now, because robots.txt is loaded on the first fetch):
# report it so the worker defers the item and crawls another
# domain in the meantime.
return None, None, "host_busy"
req = Request(
url,
@@ -1501,8 +1884,18 @@ def fetch(url, limiter, robots):
return status, text, None
except HTTPError as e:
# Blocked or rate-limited? Put the host on backoff so the next
# claims automatically prefer other domains.
scheduler.note_http_error(
url,
e.code,
retry_after=_retry_after_seconds(
getattr(e, "headers", None)
),
)
return e.code, None, f"http_{e.code}"
except (URLError, TimeoutError, OSError) as e:
scheduler.note_network_error(url)
return None, None, f"network:{e}"
except Exception as e:
return None, None, f"error:{e}"
@@ -1702,10 +2095,11 @@ def worker(worker_id):
conn = None
robots = RobotsCache()
limiter = RateLimiter()
scheduler = HostScheduler()
crawled = 0
failed = 0
retried = 0
last_stats = time.monotonic()
current_queue_id = None
update_requested = False
@@ -1742,11 +2136,26 @@ def worker(worker_id):
update_requested = True
break
item = safe_commit(conn, lambda: get_next_queue_item(conn))
item, wait_hint = safe_commit(
conn,
lambda: claim_next_queue_item(conn, scheduler, robots),
)
if not item:
logging.info("Queue empty. Worker waiting...")
time.sleep(5)
if wait_hint:
# Every candidate host is rate-limited or blocked
# right now; sleep until the soonest one becomes
# crawlable (capped, then the queue is re-scanned).
sleep_for = min(wait_hint, QUEUE_MAX_IDLE_SLEEP)
logging.info(
"[WAIT ] All candidate hosts busy or blocked; "
"worker sleeps %.1fs",
sleep_for,
)
else:
sleep_for = 5
logging.info("Queue empty. Worker waiting...")
time.sleep(sleep_for)
continue
queue_id, url, source_url = item
@@ -1770,12 +2179,53 @@ def worker(worker_id):
)
continue
# Hosts that may never be crawled (localhost, raw IPs,
# non-real domains) can still sit in the queue from before
# this rule existed: drop them here so they are not
# fetched and not retried endlessly.
if is_non_crawlable_url(url):
logging.info("[SKIP ] %s (non-crawlable host)", url)
safe_commit(
conn,
lambda: finish_queue_item(
conn, queue_id, success=True
),
)
continue
started = time.monotonic()
status, html, error = fetch(url, limiter, robots)
status, html, error = fetch(url, scheduler, robots)
if error:
logging.info("[HTTP ] %s -> %s", url, error)
if error == "host_busy":
# Host needs a longer break than we sleep for
# inline: put the item back (without burning an
# attempt) and crawl another domain instead.
safe_commit(
conn,
lambda: defer_queue_item(conn, queue_id),
)
continue
if is_transient_error(error):
# The host is already on backoff (set in fetch);
# requeue the URL for a later attempt.
new_status = safe_commit(
conn,
lambda: requeue_queue_item(conn, queue_id),
)
retried += 1
logging.info(
"[RETRY] %s -> %s (max %d attempts)",
url,
new_status or "requeued",
QUEUE_MAX_ATTEMPTS,
)
continue
safe_commit(
conn,
lambda: finish_queue_item(
@@ -1786,6 +2236,7 @@ def worker(worker_id):
continue
logging.info("[HTTP ] %s -> HTTP %s", url, status)
scheduler.note_success(url)
# A redirect normally arrives as the final URL in urllib.
# The requested URL remains the canonical queue identity.
@@ -1941,8 +2392,8 @@ def worker(worker_id):
conn.close()
logging.info(
"Stopped. crawled=%d failed=%d",
crawled, failed
"Stopped. crawled=%d failed=%d retried=%d",
crawled, failed, retried
)
if update_requested:
@@ -1961,6 +2412,7 @@ def main():
print(f"Start: {START_URL}")
print(f"Workers: {WORKERS}")
print(f"Delay: {DEFAULT_DELAY:.1f} Sekunden")
print("Auto-Switch: aktiv (blockierte Domains werden übersprungen)")
print(f"Robots-Cache: {ROBOTS_CACHE_SECONDS // 3600} Stunden")
print(f"DB: {DB_HOST}:{DB_PORT}/{DB_NAME}")
print(f"Version: {CRAWLER_VERSION}")
@@ -1975,7 +2427,19 @@ def main():
safe_commit(conn, lambda: requeue_stale_items(conn))
# Seed URL. ON CONFLICT makes restarting harmless.
queue_urls(conn, [normalize_url(START_URL)], None)
seed_url = normalize_url(START_URL)
if seed_url is None:
# Either an invalid URL or a non-crawlable host (localhost,
# raw IP, non-real domain): crawling can never work, so fail
# clearly instead of spinning on an empty queue.
conn.close()
logging.error(
"Start URL %s is not crawlable (invalid or non-public "
"host such as localhost or an IP address).",
START_URL,
)
return EXIT_ERROR
queue_urls(conn, [seed_url], None)
conn.close()
@@ -2043,6 +2507,7 @@ def main():
sys.exit(EXIT_ERROR)
print("Crawler stopped.")
return EXIT_OK
# ============================================================
@@ -2257,8 +2722,7 @@ def main_cli(argv=None):
if args.init_deployment:
return cmd_init_deployment()
main()
return EXIT_OK
return main()
if __name__ == "__main__":