Serverjonas/search
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add launcher

Browse Source
This commit is contained in:
jhkaktus
2026-09-13 01:57:46 +02:00
parent 2e1701c442
commit 682c0f4971
6 changed files with 2165 additions and 209 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crawler.py
+706 -37
View File
@@ -18,6 +18,13 @@ Features:
- robots.txt support with in-memory cache
- Graceful shutdown
- Basic crawler statistics
- Generation-based rolling updates: the crawler remembers the
deployment generation it started with, checks it before every
relevant commit (safe point) and exits with exit code 42 when a new
generation requires an update - the open transaction is rolled
back, never half-committed
- Launcher support: --version, --self-update N (verify a staged
release) and --update-deployment N (bump generation + required_version)
Expected PostgreSQL schema:
pages
@@ -30,6 +37,8 @@ Connection defaults can be overridden with environment variables:
JONASSEARCH_DB_NAME
JONASSEARCH_DB_USER
JONASSEARCH_DB_PASSWORD
JONASSEARCH_UPDATE_CHECK_SECONDS
JONASSEARCH_RELEASES_DIR
Example:
python3 crawler.py
@@ -37,10 +46,14 @@ Example:
import os
import re
import sys
import math
import signal
import time
import logging
import argparse
import subprocess
import threading
import multiprocessing as mp
from collections import defaultdict
from html import unescape
@@ -59,11 +72,29 @@ try:
except ImportError:
BeautifulSoup = None
# .env zuerst aus dem Arbeitsverzeichnis laden (Standard), zusätzlich aus
# dem Projektordner – so findet der Prozess die Konfiguration auch dann,
# wenn er von woanders gestartet wird (z. B. via systemd).
load_dotenv()
load_dotenv(Path(__file__).resolve().parent / ".env")
def _load_env_files():
"""
.env zuerst aus dem Arbeitsverzeichnis laden (Standard), zusätzlich
aus dem Projektordner – so findet der Prozess die Konfiguration
auch dann, wenn er von woanders gestartet wird (z. B. via systemd).
Eine nicht lesbare .env darf den Crawler nicht abstürzen lassen
(z. B. falscher Benutzer unter systemd): dann laufen die bereits
gesetzten Umgebungsvariablen unverändert weiter.
"""
for dotenv_path in (None, Path(__file__).resolve().parent / ".env"):
try:
load_dotenv(dotenv_path)
except OSError as exc:
logging.warning(
"[CONFIG] %s not readable (%s); continuing with the "
"current environment.",
dotenv_path or ".env (working directory)",
exc,
)
_load_env_files()
# ============================================================
@@ -99,6 +130,46 @@ USER_AGENT = os.getenv(
"JonasSearchBot/0.1 (+https://jonassearch.local/bot)"
)
# ============================================================
# DEPLOYMENT / UPDATE CONFIG
# ============================================================
# Version of THIS crawler code. Increased with every released crawler
# change; completely independent from the deployment generation (the
# generation is a global deployment counter in PostgreSQL, the version
# identifies the crawler code).
CRAWLER_VERSION = int(os.getenv("JONASSEARCH_CRAWLER_VERSION", "1"))
# Exit codes. 42 is reserved exclusively for "I must be updated" so
# the launcher can distinguish update, crash and normal exit.
EXIT_OK = 0
EXIT_ERROR = 1
EXIT_UPDATE_REQUIRED = 42
# How often the background checker re-reads the generation. It only
# sets an internal flag; the actual exit happens at the next safe
# commit point, never in the middle of a transaction.
UPDATE_CHECK_INTERVAL_SECONDS = float(
os.getenv("JONASSEARCH_UPDATE_CHECK_SECONDS", "30")
)
# Staging directory for updates: the launcher stages a new version in
# releases/<VERSION>/crawler.py, verifies it with --self-update and
# only then activates it. Relative paths resolve against the crawler
# file's directory, so they work from any working directory.
_RELEASES_RAW = os.getenv("JONASSEARCH_RELEASES_DIR", "releases")
CRAWLER_RELEASES_DIR = Path(_RELEASES_RAW)
if not CRAWLER_RELEASES_DIR.is_absolute():
CRAWLER_RELEASES_DIR = (
Path(__file__).resolve().parent / CRAWLER_RELEASES_DIR
)
# How long the launcher waits before restarting the crawler after it
# exited with EXIT_UPDATE_REQUIRED.
LAUNCHER_RESTART_DELAY = float(
os.getenv("JONASSEARCH_LAUNCHER_RESTART_DELAY", "2")
)
# Only crawl normal HTTP(S) pages.
ALLOWED_SCHEMES = {"http", "https"}
@@ -223,6 +294,252 @@ HOST_STATS_CACHE_SECONDS = 30
HOST_STATS_SAMPLE_LIMIT = 20000
# ============================================================
# DEPLOYMENT / UPDATE (generation-based rolling updates)
# ============================================================
#
# The deployment_state table is the global switch for rolling
# updates (exactly one row, id = 1):
#
# CREATE TABLE IF NOT EXISTS deployment_state (
# id INTEGER PRIMARY KEY,
# generation BIGINT NOT NULL,
# required_version INTEGER NOT NULL
# )
#
# - generation: global deployment counter. The launcher bumps
# it when a new crawler version goes live.
# - required_version: the crawler version this generation requires.
#
# Every crawler instance reads the generation ONCE at startup ("my
# generation is N"). Before every relevant PostgreSQL commit
# (safe_commit) and additionally in a periodic background check it
# compares the current generation with its own:
#
# - identical -> everything fine, continue.
# - different -> the open transaction is rolled back cleanly and the
# process exits with EXIT_UPDATE_REQUIRED (42). The
# launcher recognizes that exit code, provisions the
# new version and starts it; the new process reads the
# new generation at startup and runs on.
#
# No process is ever killed mid-transaction: exits happen only at the
# crawler's own safe commit points.
#
# CRAWLER_VERSION (CONFIG section) is the version of THIS crawler
# code. It is NOT the generation and must be increased with every
# released crawler change.
class UpdateRequired(Exception):
"""
Raised at a safe point (before a commit) when a new deployment
generation is active. safe_commit() rolls the open transaction
back BEFORE raising, so no work is ever committed half-done.
"""
class UpdateSignal:
"""
Per-process update state (every crawler worker process owns one).
generation: the deployment generation this process started with.
pending: True once a check noticed a new generation; the
process stops at the next safe commit point.
stop_checker: tells the background checker thread to finish.
"""
def __init__(self):
self.generation = None
self.pending = False
self.stop_checker = False
def check(self):
"""Raise UpdateRequired if an update is pending."""
if self.pending:
raise UpdateRequired(
"new deployment generation active; update required"
)
update_signal = UpdateSignal()
def ensure_deployment_state_table(conn):
"""
Idempotent bootstrap for deployment_state (fresh databases):
creates the table if needed and makes sure exactly one row with
id = 1 exists. Safe to run from several crawler instances at once
(CREATE TABLE IF NOT EXISTS + ON CONFLICT DO NOTHING).
"""
with conn.cursor() as cur:
cur.execute("""
CREATE TABLE IF NOT EXISTS deployment_state (
id INTEGER PRIMARY KEY,
generation BIGINT NOT NULL,
required_version INTEGER NOT NULL
)
""")
cur.execute("""
INSERT INTO deployment_state (id, generation, required_version)
VALUES (1, 1, %s)
ON CONFLICT (id) DO NOTHING
""", (CRAWLER_VERSION,))
conn.commit()
def _fetch_generation_row(conn):
"""
Read the deployment_state row (id = 1). If the table does not exist
yet, it is bootstrapped (see ensure_deployment_state_table) and the
row is read again. Returns the row or None (table without row).
"""
try:
with conn.cursor() as cur:
cur.execute(
"SELECT generation FROM deployment_state WHERE id = 1"
)
row = cur.fetchone()
conn.rollback() # read-only; leave no open transaction behind
return row
except psycopg2.errors.UndefinedTable:
conn.rollback()
logging.warning(
"deployment_state table missing; creating it with "
"generation 1 and required_version %d.",
CRAWLER_VERSION,
)
ensure_deployment_state_table(conn)
with conn.cursor() as cur:
cur.execute(
"SELECT generation FROM deployment_state WHERE id = 1"
)
row = cur.fetchone()
conn.rollback()
return row
def get_current_generation(conn=None):
"""
Read the current deployment generation. Called once per process at
startup: this becomes "my generation" (update_signal.generation).
"""
own_conn = conn is None
if own_conn:
conn = db_connect()
try:
row = _fetch_generation_row(conn)
if row is None:
logging.warning(
"deployment_state has no row with id = 1; inserting "
"generation 1 with required_version %d.",
CRAWLER_VERSION,
)
ensure_deployment_state_table(conn)
row = _fetch_generation_row(conn)
return int(row[0])
finally:
if own_conn:
conn.close()
def periodic_update_check():
"""
Regular update check OUTSIDE any transaction, so instances that
currently do not commit still notice a new generation.
Runs as a daemon thread per worker and uses its own short-lived
connection, so it can never interfere with the worker's open
transaction. A new generation only sets update_signal.pending;
the actual exit happens at the next safe commit point. A failed
check (e.g. short DB hiccup) is simply retried next interval.
"""
while not (
STOP.is_set()
or update_signal.pending
or update_signal.stop_checker
):
time.sleep(UPDATE_CHECK_INTERVAL_SECONDS)
if STOP.is_set() or update_signal.pending or update_signal.stop_checker:
break
try:
conn = db_connect()
except Exception:
continue
try:
with conn.cursor() as cur:
cur.execute(
"SELECT generation FROM deployment_state WHERE id = 1"
)
row = cur.fetchone()
conn.rollback()
except Exception:
try:
conn.rollback()
except Exception:
pass
continue
finally:
try:
conn.close()
except Exception:
pass
if row is None:
continue
try:
current = int(row[0])
except (TypeError, ValueError):
continue
if (
not update_signal.pending
and update_signal.generation is not None
and current != update_signal.generation
):
update_signal.pending = True
logging.warning(
"[UPDATE] Deployment generation changed: %d -> %d. "
"This worker (version %d) stops at the next safe "
"commit point.",
update_signal.generation,
current,
CRAWLER_VERSION,
)
def safe_commit(conn, operation=None):
"""
The ONLY commit path for crawling work.
1. If an update is pending, roll the open transaction back and
raise UpdateRequired: the caller's writes are discarded
cleanly, nothing is committed half-done.
2. Otherwise run the writes (operation) and commit them.
Every crawler commit that does real work goes through here
(page saves, queue inserts, queue claim/finish, stats), so a
deployment switch can never cut a transaction in half. The
one-off startup commits (seeding, stats table) are routed
through this function as well.
"""
if update_signal.pending:
try:
conn.rollback()
except Exception:
logging.exception("Rollback before update exit failed.")
raise UpdateRequired(
"new deployment generation active; update required"
)
result = operation() if operation is not None else None
conn.commit()
return result
# ============================================================
# LOGGING / SHUTDOWN
# ============================================================
@@ -262,6 +579,8 @@ def db_connect():
def ensure_stats_table(conn):
"""Create crawler_stats. Writes only - commit happens in
safe_commit()."""
with conn.cursor() as cur:
cur.execute("""
CREATE TABLE IF NOT EXISTS crawler_stats (
@@ -272,16 +591,16 @@ def ensure_stats_table(conn):
failed INTEGER NOT NULL DEFAULT 0
)
""")
conn.commit()
def write_stats(conn, crawled, queued, failed):
"""Insert one stats row. Writes only - commit happens in
safe_commit()."""
with conn.cursor() as cur:
cur.execute("""
INSERT INTO crawler_stats (crawled, queued, failed)
VALUES (%s, %s, %s)
""", (crawled, queued, failed))
conn.commit()
def get_next_queue_item(conn):
@@ -290,6 +609,8 @@ def get_next_queue_item(conn):
SKIP LOCKED is the important part: several crawler processes can
consume the same PostgreSQL queue without waiting for each other.
Writes only - the commit happens in safe_commit().
"""
with conn.cursor() as cur:
cur.execute("""
@@ -310,23 +631,25 @@ def get_next_queue_item(conn):
RETURNING q.id, q.url, q.source_url
""")
row = cur.fetchone()
conn.commit()
return row
def finish_queue_item(conn, queue_id, success=True):
"""Mark a queue item done/failed. Writes only - commit happens in
safe_commit()."""
with conn.cursor() as cur:
cur.execute("""
UPDATE crawl_queue
SET status = %s
WHERE id = %s
""", ("done" if success else "failed", queue_id))
conn.commit()
def requeue_stale_items(conn, older_than_seconds=3600):
"""
Recover URLs left in 'processing' after a crash.
Writes only - commit happens in safe_commit().
"""
with conn.cursor() as cur:
cur.execute("""
@@ -336,7 +659,6 @@ def requeue_stale_items(conn, older_than_seconds=3600):
AND last_attempt_at < now() - (%s * interval '1 second')
""", (older_than_seconds,))
changed = cur.rowcount
conn.commit()
if changed:
logging.info("Requeued %d stale processing URLs.", changed)
@@ -346,9 +668,10 @@ def _insert_queue_rows(conn, rows):
Insert pre-ordered queue rows and return the number of newly
inserted rows.
All pages run inside ONE transaction (commit only at the end), so
the batch stays atomic as before. rowcount is accumulated per page
because execute_values() does not aggregate rowcount across pages.
All pages run inside ONE transaction (committed by safe_commit()
after this function returns), so the batch stays atomic as before.
rowcount is accumulated per page because execute_values() does not
aggregate rowcount across pages.
"""
inserted = 0
with conn.cursor() as cur:
@@ -367,7 +690,6 @@ def _insert_queue_rows(conn, rows):
)
inserted += max(cur.rowcount, 0)
conn.commit()
return inserted
@@ -441,7 +763,9 @@ def queue_urls(conn, links, source_url):
for attempt in range(1, QUEUE_INSERT_MAX_ATTEMPTS + 1):
try:
inserted = _insert_queue_rows(conn, rows)
inserted = safe_commit(
conn, lambda: _insert_queue_rows(conn, rows)
)
if attempt > 1:
logging.info(
"[QUEUE] Insert succeeded on retry %d/%d.",
@@ -1222,6 +1546,10 @@ def parse_html(html, base_url):
# ============================================================
def save_page(conn, url, source_url, status, content_type, parsed, crawl_ms):
"""
Store/refresh one page. Writes only - commit happens in
safe_commit().
"""
# Sanitize everything that came from the web before it reaches
# PostgreSQL. NUL bytes are legal in HTML but illegal in text columns.
url = clean_text(url)
@@ -1292,8 +1620,6 @@ def save_page(conn, url, source_url, status, content_type, parsed, crawl_ms):
crawl_ms,
))
conn.commit()
# ============================================================
# WORKER
@@ -1310,17 +1636,42 @@ def worker(worker_id):
crawled = 0
failed = 0
last_stats = time.monotonic()
current_queue_id = None
update_requested = False
try:
conn = db_connect()
# Remember the generation this instance started with. Read ONCE
# per process: five crawler instances all remember their own
# generation (here: the same one, until a deployment bump).
update_signal.generation = get_current_generation(conn)
logging.info(
"Started. DB=%s:%s/%s",
DB_HOST, DB_PORT, DB_NAME
"Started. DB=%s:%s/%s version=%d generation=%d",
DB_HOST, DB_PORT, DB_NAME,
CRAWLER_VERSION, update_signal.generation,
)
# Regular update check OUTSIDE the transactions (own short-lived
# connection, daemon thread). It only sets update_signal.pending;
# the exit itself happens at the next safe commit point, never
# in the middle of a transaction.
checker_thread = threading.Thread(
target=periodic_update_check,
name=f"{process_name}-update-check",
daemon=True,
)
checker_thread.start()
while not STOP.is_set():
item = get_next_queue_item(conn)
# Safe point between page cycles: if the periodic check
# noticed a new generation, stop before starting new work.
if update_signal.pending:
update_requested = True
break
item = safe_commit(conn, lambda: get_next_queue_item(conn))
if not item:
logging.info("Queue empty. Worker waiting...")
@@ -1328,6 +1679,7 @@ def worker(worker_id):
continue
queue_id, url, source_url = item
current_queue_id = queue_id
logging.info("=" * 70)
logging.info("[CRAWL] %s", url)
@@ -1339,7 +1691,12 @@ def worker(worker_id):
if error:
logging.info("[HTTP ] %s -> %s", url, error)
finish_queue_item(conn, queue_id, success=False)
safe_commit(
conn,
lambda: finish_queue_item(
conn, queue_id, success=False
),
)
failed += 1
continue
@@ -1348,7 +1705,12 @@ def worker(worker_id):
# A redirect normally arrives as the final URL in urllib.
# The requested URL remains the canonical queue identity.
if html is None:
finish_queue_item(conn, queue_id, success=False)
safe_commit(
conn,
lambda: finish_queue_item(
conn, queue_id, success=False
),
)
failed += 1
continue
@@ -1356,7 +1718,12 @@ def worker(worker_id):
parsed = parse_html(html, url)
except Exception as exc:
logging.exception("[PARSE] %s: %s", url, exc)
finish_queue_item(conn, queue_id, success=False)
safe_commit(
conn,
lambda: finish_queue_item(
conn, queue_id, success=False
),
)
failed += 1
continue
@@ -1367,14 +1734,17 @@ def worker(worker_id):
logging.info("[HEADS ] %d", len(parsed["headings"]))
logging.info("[LINKS ] %d gefunden", len(parsed["links"]))
save_page(
conn=conn,
url=url,
source_url=source_url,
status=status,
content_type="text/html",
parsed=parsed,
crawl_ms=crawl_ms,
safe_commit(
conn,
lambda: save_page(
conn=conn,
url=url,
source_url=source_url,
status=status,
content_type="text/html",
parsed=parsed,
crawl_ms=crawl_ms,
),
)
inserted = queue_urls(
@@ -1390,9 +1760,13 @@ def worker(worker_id):
inserted,
)
finish_queue_item(conn, queue_id, success=True)
safe_commit(
conn,
lambda: finish_queue_item(conn, queue_id, success=True),
)
crawled += 1
current_queue_id = None
# Periodic stats; not on every page to avoid unnecessary DB load.
if time.monotonic() - last_stats >= 30:
@@ -1403,15 +1777,58 @@ def worker(worker_id):
)
queued = cur.fetchone()[0]
write_stats(conn, crawled, queued, failed)
safe_commit(
conn,
lambda: write_stats(conn, crawled, queued, failed),
)
last_stats = time.monotonic()
except UpdateRequired:
raise
except Exception:
conn.rollback()
except UpdateRequired:
# A safe point (safe_commit) noticed the new generation and
# rolled back the open transaction BEFORE raising. Nothing was
# committed half-done.
update_requested = True
# Best effort: give the claimed item back to the queue so the
# next version can crawl it immediately. (Should this fail,
# requeue_stale_items() recovers it later anyway.) This commit
# is wanted shutdown bookkeeping, not crawled content.
try:
if current_queue_id is not None:
with conn.cursor() as cur:
cur.execute(
"UPDATE crawl_queue "
"SET status = 'pending' "
"WHERE id = %s AND status = 'processing'",
(current_queue_id,),
)
conn.commit()
logging.info(
"[UPDATE] Queue item %d requeued for the next version.",
current_queue_id,
)
except Exception:
try:
conn.rollback()
except Exception:
pass
logging.warning(
"[UPDATE] %s stops for update (started with generation %s, "
"version %d).",
process_name,
update_signal.generation,
CRAWLER_VERSION,
)
except Exception:
logging.exception("Worker crashed.")
raise
finally:
update_signal.stop_checker = True
if conn is not None:
conn.close()
@@ -1420,6 +1837,10 @@ def worker(worker_id):
crawled, failed
)
if update_requested:
# Reserved exit code, exclusively for "update required".
sys.exit(EXIT_UPDATE_REQUIRED)
# ============================================================
# STARTUP
@@ -1434,12 +1855,16 @@ def main():
print(f"Delay: {DEFAULT_DELAY:.1f} Sekunden")
print(f"Robots-Cache: {ROBOTS_CACHE_SECONDS // 3600} Stunden")
print(f"DB: {DB_HOST}:{DB_PORT}/{DB_NAME}")
print(f"Version: {CRAWLER_VERSION}")
print()
conn = db_connect()
ensure_deployment_state_table(conn)
ensure_stats_table(conn)
requeue_stale_items(conn)
safe_commit(conn)
safe_commit(conn, lambda: requeue_stale_items(conn))
# Seed URL. ON CONFLICT makes restarting harmless.
queue_urls(conn, [normalize_url(START_URL)], None)
@@ -1453,18 +1878,34 @@ def main():
p.start()
workers.append(p)
update_requested = False
worker_crashed = False
try:
while any(p.is_alive() for p in workers):
time.sleep(1)
# If a worker dies unexpectedly, stop the remaining workers.
for p in workers:
if p.exitcode not in (None, 0):
if p.exitcode == EXIT_UPDATE_REQUIRED:
# This worker reached its safe point and wants the
# update. Stop the remaining workers as well (they
# also exit at their own safe points, never
# mid-transaction) and propagate exit code 42.
if not update_requested:
logging.warning(
"[UPDATE] Worker %s requests update; the "
"other workers stop at their safe points.",
p.name,
)
update_requested = True
STOP.set()
elif p.exitcode not in (None, 0):
logging.error(
"Worker %s exited with code %s.",
p.name,
p.exitcode,
)
worker_crashed = True
STOP.set()
except KeyboardInterrupt:
@@ -1481,9 +1922,237 @@ def main():
p.terminate()
p.join()
if update_requested:
# Reserved exit code: "update required". Only the launcher may
# interpret this code; normal errors use EXIT_ERROR (1).
print("Crawler stopped: update required.")
sys.exit(EXIT_UPDATE_REQUIRED)
if worker_crashed:
# A normal error must NOT look like a normal end (exit 0) or an
# update (42): the launcher restarts on this code unchanged.
print("Crawler stopped: worker error.")
sys.exit(EXIT_ERROR)
print("Crawler stopped.")
# ============================================================
# CLI (version / deployment / update checks)
# ============================================================
def build_arg_parser():
parser = argparse.ArgumentParser(
prog="crawler.py",
description="JonasSearch PostgreSQL crawler with "
"generation-based rolling updates.",
)
parser.add_argument(
"--version",
action="store_true",
help="print the crawler version (just the number) and exit",
)
parser.add_argument(
"--self-update",
type=int,
metavar="VERSION",
help="verify a staged crawler release in "
"%s/<VERSION>/crawler.py without activating it "
"(exit 0 = startable, 1 = broken/missing)" % CRAWLER_RELEASES_DIR,
)
parser.add_argument(
"--update-deployment",
type=int,
metavar="VERSION",
help="bump the deployment generation and set required_version "
"to VERSION. Run ONLY after the new version has been "
"provisioned and verified - from this moment all running "
"crawlers stop at their next safe commit point.",
)
parser.add_argument(
"--init-deployment",
action="store_true",
help="create deployment_state if needed and seed the initial "
"row (id=1, generation 1, required_version = this "
"crawler's version)",
)
return parser
def cmd_version():
# Deliberately machine-readable: the launcher parses this output.
print(CRAWLER_VERSION)
return EXIT_OK
def _basic_self_check():
"""
Minimal smoke test used by --self-update when no releases directory
exists yet: this file must at least compile.
"""
try:
with open(__file__, "r", encoding="utf-8") as fh:
compile(fh.read(), __file__, "exec")
except Exception as exc:
logging.error("[UPDATE] Self-check failed: %s", exc)
return EXIT_ERROR
print("[UPDATE] Basic self-check passed.")
return EXIT_OK
def cmd_self_update(version):
"""
Verify that a staged release for `version` exists and is startable
- WITHOUT activating it. The launcher runs this with the CURRENT
(old) interpreter before swapping anything, so a broken release can
never become active (fallback guarantee).
Checks:
1. releases/<VERSION>/crawler.py exists,
2. `python3 releases/<VERSION>/crawler.py --version` prints exactly
`version` and exits with EXIT_OK (imports, syntax and the
version constant of the new file are all checked).
"""
release_file = CRAWLER_RELEASES_DIR / str(version) / "crawler.py"
if not CRAWLER_RELEASES_DIR.is_dir():
logging.warning(
"[UPDATE] No releases directory (%s); running the basic "
"self-check only.",
CRAWLER_RELEASES_DIR,
)
return _basic_self_check()
if not release_file.is_file():
logging.error(
"[UPDATE] Staged release missing: %s", release_file
)
return EXIT_ERROR
print(f"[UPDATE] Checking staged release: {release_file}")
try:
result = subprocess.run(
[sys.executable, str(release_file), "--version"],
capture_output=True,
text=True,
timeout=60,
)
except Exception as exc:
logging.error("[UPDATE] Release check could not run: %s", exc)
return EXIT_ERROR
printed = (result.stdout or "").strip()
if result.returncode != EXIT_OK or printed != str(version):
logging.error(
"[UPDATE] Staged release check FAILED (exit code %d, "
"version output %r, stderr: %s). The release is NOT "
"activated; the old version stays in place.",
result.returncode,
printed,
(result.stderr or "").strip()[-500:],
)
return EXIT_ERROR
print(f"[UPDATE] Release {version} verified and startable.")
return EXIT_OK
def cmd_update_deployment(required_version):
"""
The global switch point: atomically raise the deployment generation
and set the required crawler version in ONE statement.
Run only AFTER the new version has been provisioned and verified on
the machines (see --self-update). From this moment every running
old crawler sees the new generation at its next safe commit point,
rolls its open transaction back and exits with EXIT_UPDATE_REQUIRED
- no process is ever killed mid-transaction.
"""
conn = None
try:
conn = db_connect()
ensure_deployment_state_table(conn)
with conn.cursor() as cur:
cur.execute(
"""
UPDATE deployment_state
SET generation = generation + 1,
required_version = %s
WHERE id = 1
RETURNING generation, required_version
""",
(required_version,),
)
row = cur.fetchone()
if row is None:
logging.error(
"deployment_state row (id = 1) missing; nothing updated."
)
conn.rollback()
return EXIT_ERROR
conn.commit()
generation, activated_version = row
print(
f"[UPDATE] Deployment generation -> {generation}, "
f"required_version -> {activated_version}. Running crawlers "
"stop at their next safe commit point with exit code "
f"{EXIT_UPDATE_REQUIRED}."
)
return EXIT_OK
except Exception:
logging.exception("Deployment update failed.")
if conn is not None:
try:
conn.rollback()
except Exception:
pass
return EXIT_ERROR
finally:
if conn is not None:
conn.close()
def cmd_init_deployment():
"""
One-time initialization of deployment_state: create the table and
the single current-state row (id = 1) with generation 1 and the
currently running crawler version. Restarting is harmless.
"""
conn = None
try:
conn = db_connect()
ensure_deployment_state_table(conn)
row = _fetch_generation_row(conn)
generation = int(row[0]) if row else 1
print(
f"[UPDATE] deployment_state ready: generation {generation}, "
f"required_version {CRAWLER_VERSION}."
)
return EXIT_OK
except Exception:
logging.exception("Initializing deployment_state failed.")
return EXIT_ERROR
finally:
if conn is not None:
conn.close()
def main_cli(argv=None):
args = build_arg_parser().parse_args(argv)
if args.version:
return cmd_version()
if args.self_update is not None:
return cmd_self_update(args.self_update)
if args.update_deployment is not None:
return cmd_update_deployment(args.update_deployment)
if args.init_deployment:
return cmd_init_deployment()
main()
return EXIT_OK
if __name__ == "__main__":
mp.set_start_method("spawn", force=True)
main()
sys.exit(main_cli())