Serverjonas/search
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
aa0bc4e960b8643ff110967b5e3bb0a2d45457dd
search/launcher.py
T
jhkaktus 682c0f4971 add launcher
2026-09-13 01:57:46 +02:00

621 lines
20 KiB
Python
Raw Blame History

#!/usr/bin/env python3
"""
JonasSearch Crawler Launcher
Starts and monitors the crawler and handles the generation-based
rolling update mechanism (see crawler.py, section
"DEPLOYMENT / UPDATE").
Division of labor:
crawler.py (old version)
- remembers the deployment generation it started with
- checks it before every relevant commit (safe point) and
periodically in the background
- when a new generation appears it rolls the open transaction
back and exits with exit code 42 (EXIT_UPDATE_REQUIRED)
launcher.py (this file)
- starts the crawler and waits for it to end
- interprets the exit code:
0 -> normal end (e.g. Ctrl+C): nothing to do
42 -> update required: provision the new version,
bump the generation, start the new version
else-> error/crash: restart the crawler unchanged
- update flow (in this order, so a broken update can never
take the crawler down):
1. stage the new version completely on this machine
(releases/<VERSION>/crawler.py)
2. verify it startable: the OLD crawler verifies the
staged release via `--self-update VERSION`
3. only then raise generation + required_version in
PostgreSQL (--update-deployment VERSION)
4. restart the crawler (now the new version), which
reads the new generation at startup and runs on
Exit codes of the launcher itself mirror the crawler's: 0 normal end,
42 update cycle completed, 1 configuration/provisioning error.
Usage:
python3 launcher.py # start crawler (loop until Ctrl+C)
python3 launcher.py --once # run one crawler cycle, then exit
python3 launcher.py --check-update # only trigger the update flow
python3 launcher.py --status # show crawler + deployment state
"""
import os
import sys
import time
import shutil
import signal
import logging
import argparse
import subprocess
from pathlib import Path
from urllib.request import Request, urlopen
from urllib.error import URLError
from dotenv import load_dotenv
# .env loading mirrors crawler.py: working directory first, then the
# launcher's own directory, so relative paths work from anywhere. An
# unreadable .env must not crash the launcher.
for _dotenv_path in (None, Path(__file__).resolve().parent / ".env"):
try:
load_dotenv(_dotenv_path)
except OSError as exc:
logging.warning(
"[CONFIG] %s not readable (%s); continuing with the "
"current environment.",
_dotenv_path or ".env (working directory)",
exc,
)
# Configure logging BEFORE importing crawler.py - importing the module
# would otherwise install its own logging.basicConfig first.
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [launcher] %(levelname)s %(message)s",
datefmt="%H:%M:%S",
)
log = logging.getLogger("launcher")
# Make the crawler's CONFIG available (DB settings, CRAWLER_VERSION,
# exit codes, CRAWLER_RELEASES_DIR). crawler.py is import-safe: all
# side effects live under `if __name__ == "__main__":`.
sys.path.insert(0, str(Path(__file__).resolve().parent))
import crawler as crawler_mod # noqa: E402
# crawler.py registers SIGINT/SIGTERM handlers at import time (for its
# spawned workers). The launcher needs the default behavior back so
# Ctrl+C raises KeyboardInterrupt here; the crawler child processes
# spawn fresh and get their own handlers.
signal.signal(signal.SIGINT, signal.default_int_handler)
signal.signal(signal.SIGTERM, signal.default_int_handler)
# ============================================================
# CONFIG
# ============================================================
# The crawler file this launcher starts (active/old version).
CRAWLER_ENTRY = Path(
os.getenv(
"JONASSEARCH_LAUNCHER_ENTRY",
str(Path(__file__).resolve().parent / "crawler.py"),
)
)
# Where new versions are staged before activation.
# crawler.py resolves relative JONASSEARCH_RELEASES_DIR paths against
# the crawler file's directory - mirror that here so both agree.
RELEASES_DIR = crawler_mod.CRAWLER_RELEASES_DIR
# Index of the currently active version, kept next to the crawler.
ACTIVE_MARKER = CRAWLER_ENTRY.parent / ".active-crawler-version"
# How new versions arrive on this machine (step 16 of the concept -
# deliberately kept simple and pluggable). Supported sources:
# - JONASSEARCH_UPDATE_SOURCE_URL: HTTP(S) download of crawler.py
# (the file is downloaded fresh for every update, so no local
# cache is needed)
# - JONASSEARCH_UPDATE_SOURCE_DIR: local directory containing
# crawler.py (e.g. a shared folder or a git checkout)
# Without a source the launcher can only verify and activate a release
# that was placed into releases/<VERSION>/crawler.py externally.
UPDATE_SOURCE_URL = os.getenv("JONASSEARCH_UPDATE_SOURCE_URL", "").strip()
UPDATE_SOURCE_DIR = os.getenv("JONASSEARCH_UPDATE_SOURCE_DIR", "").strip()
# Restart behavior after a crash (not an update!).
RESTART_DELAY_ON_ERROR = float(
os.getenv("JONASSEARCH_LAUNCHER_ERROR_RESTART_DELAY", "10")
)
# ============================================================
# HELPERS
# ============================================================
def run_crawler_command(entry, *args, timeout=None):
"""Run a crawler.py subcommand and return the CompletedProcess."""
return subprocess.run(
[sys.executable, str(entry), *args],
capture_output=True,
text=True,
timeout=timeout,
)
def read_active_version():
"""Version number of the currently active crawler (marker file)."""
try:
return int(ACTIVE_MARKER.read_text(encoding="utf-8").strip())
except Exception:
return None
def write_active_version(version):
ACTIVE_MARKER.write_text(f"{version}\n", encoding="utf-8")
def read_installed_version(entry):
"""Ask the crawler itself which version it is (source of truth)."""
try:
result = run_crawler_command(entry, "--version", timeout=30)
except Exception as exc:
log.error("Could not query crawler version (%s): %s", entry, exc)
return None
if result.returncode != crawler_mod.EXIT_OK:
log.error(
"crawler.py --version failed (exit %d): %s",
result.returncode,
(result.stderr or "").strip()[-300:],
)
return None
try:
return int((result.stdout or "").strip())
except ValueError:
log.error(
"crawler.py --version printed no version number: %r",
(result.stdout or "")[:100],
)
return None
def fetch_remote_version():
"""Ask the update source which version it offers (may be None)."""
if UPDATE_SOURCE_URL:
try:
req = Request(UPDATE_SOURCE_URL, headers={"User-Agent": "launcher"})
with urlopen(req, timeout=crawler_mod.REQUEST_TIMEOUT) as resp:
text = resp.read(1024 * 1024).decode("utf-8", errors="replace")
return int(text.strip())
except (URLError, ValueError) as exc:
log.error(
"Could not query update source %s: %s", UPDATE_SOURCE_URL, exc
)
return None
if UPDATE_SOURCE_DIR:
version_file = Path(UPDATE_SOURCE_DIR) / "VERSION"
try:
return int(version_file.read_text(encoding="utf-8").strip())
except Exception as exc:
log.error(
"Could not read VERSION in update source %s: %s",
version_file,
exc,
)
return None
return None
# ============================================================
# UPDATE FLOW (steps 8, 9 and 15 of the concept)
# ============================================================
def stage_release(version):
"""
Step 1: make the new version fully available on this machine,
WITHOUT activating it: releases/<VERSION>/crawler.py.
Already staged releases are kept (idempotent). Without a configured
update source the release must already exist here.
"""
target_dir = RELEASES_DIR / str(version)
target_file = target_dir / "crawler.py"
if target_file.is_file():
log.info("Release %d already staged at %s.", version, target_file)
return target_file
if UPDATE_SOURCE_URL:
base = UPDATE_SOURCE_URL.rsplit("/", 1)[0]
file_url = f"{base}/{version}/crawler.py"
log.info("Downloading release %d from %s ...", version, file_url)
req = Request(file_url, headers={"User-Agent": "launcher"})
with urlopen(req, timeout=crawler_mod.REQUEST_TIMEOUT) as resp:
data = resp.read(64 * 1024 * 1024)
target_dir.mkdir(parents=True, exist_ok=True)
tmp_file = target_dir / "crawler.py.part"
tmp_file.write_bytes(data)
# Only complete files get their final name.
tmp_file.replace(target_file)
log.info("Release %d staged at %s.", version, target_file)
return target_file
if UPDATE_SOURCE_DIR:
source_file = Path(UPDATE_SOURCE_DIR) / "crawler.py"
if not source_file.is_file():
log.error("crawler.py missing in update source %s.", source_file)
return None
target_dir.mkdir(parents=True, exist_ok=True)
shutil.copy2(source_file, target_file)
log.info("Release %d staged at %s.", version, target_file)
return target_file
log.error(
"Release %d is not staged (%s) and no update source is "
"configured (JONASSEARCH_UPDATE_SOURCE_URL or "
"JONASSEARCH_UPDATE_SOURCE_DIR). The generation was NOT "
"changed.",
version,
target_file,
)
return None
def verify_release(version):
"""
Step 2: verify the staged release is complete and startable -
executed by the CURRENT (old) crawler, before anything is switched.
If this fails, the old version stays active (fallback guarantee).
"""
result = run_crawler_command(
CRAWLER_ENTRY, "--self-update", str(version), timeout=120
)
if result.returncode != crawler_mod.EXIT_OK:
log.error(
"Release %d FAILED the verification; nothing was switched.\n%s",
version,
(result.stdout or "") + (result.stderr or ""),
)
return False
return True
def activate_release(version, staged_file):
"""
Step 3: switch the active crawler to the verified release.
The old file is kept as crawler.py.bak-<VERSION> so there is always
a working fallback. The marker file records the active version;
the database switch itself happens in bump_deployment().
"""
if staged_file.resolve() == CRAWLER_ENTRY.resolve():
write_active_version(version)
return True
backup = CRAWLER_ENTRY.with_name(
"crawler.py.bak-{}".format(
read_active_version()
or read_installed_version(CRAWLER_ENTRY)
or "unknown"
)
)
try:
shutil.copy2(CRAWLER_ENTRY, backup)
shutil.copy2(staged_file, CRAWLER_ENTRY)
except OSError as exc:
log.error(
"Switching to release %d failed (%s); the old version "
"stays active.",
version,
exc,
)
return False
# Sanity check: the switched file must identify itself correctly.
installed = read_installed_version(CRAWLER_ENTRY)
if installed != version:
log.error(
"Switched crawler reports version %s instead of %d; "
"restoring the backup.",
installed,
version,
)
shutil.copy2(backup, CRAWLER_ENTRY)
return False
write_active_version(version)
log.info(
"Release %d activated (backup of the old version: %s).",
version,
backup,
)
return True
def bump_deployment(version):
"""
Step 4 - the global switch point: generation + 1 and
required_version = version in PostgreSQL. Only AFTER this moment do
running old crawlers notice the update (at their next safe commit
point) and exit with code 42.
"""
result = run_crawler_command(
CRAWLER_ENTRY, "--update-deployment", str(version), timeout=60
)
if result.returncode != crawler_mod.EXIT_OK:
log.error(
"Deployment update to version %d failed:\n%s",
version,
(result.stderr or "")[-500:],
)
return False
log.info((result.stdout or "").strip())
return True
def perform_update(required_version):
"""
Full update flow in the safe order (concept steps 8, 9 and 15):
stage -> verify -> activate -> bump generation
Returns True only if every step succeeded.
"""
log.info("Update to crawler version %d starting ...", required_version)
staged = stage_release(required_version)
if staged is None:
return False
if not verify_release(required_version):
return False
if not activate_release(required_version, staged):
return False
if not bump_deployment(required_version):
return False
log.info("Update to crawler version %d completed.", required_version)
return True
# ============================================================
# LAUNCH LOOP
# ============================================================
def start_crawler_process():
log.info(
"Starting crawler: %s (version %s)",
CRAWLER_ENTRY,
read_installed_version(CRAWLER_ENTRY),
)
# The crawler inherits stdout/stderr, so its logs stay visible.
return subprocess.Popen([sys.executable, str(CRAWLER_ENTRY)])
def wait_for_process(process):
"""Block until the process ends; return its exit code."""
try:
return process.wait()
except KeyboardInterrupt:
log.info("Ctrl+C received; stopping crawler ...")
process.terminate()
try:
return process.wait(timeout=10)
except subprocess.TimeoutExpired:
process.kill()
return process.wait()
def read_required_version():
"""
Read required_version from deployment_state (id = 1). Returns None
on database errors or a missing row.
"""
try:
conn = crawler_mod.db_connect()
try:
with conn.cursor() as cur:
cur.execute(
"SELECT required_version FROM deployment_state "
"WHERE id = 1"
)
row = cur.fetchone()
conn.rollback()
finally:
conn.close()
except Exception as exc:
log.error("Deployment state could not be read: %s", exc)
return None
return int(row[0]) if row else None
def check_and_update():
"""
Compare installed version with the required version from
PostgreSQL and run the update flow when necessary.
"""
try:
generation = crawler_mod.get_current_generation()
except Exception as exc:
log.error("Deployment state could not be read: %s", exc)
return False
row_required = read_required_version()
if row_required is None:
log.error("deployment_state has no row with id = 1.")
return False
required_version = row_required
installed = read_installed_version(CRAWLER_ENTRY)
remote = fetch_remote_version()
log.info(
"Deployment: generation %d | required_version %d | "
"installed %s | update source offers %s",
generation,
required_version,
installed,
remote,
)
if installed is None:
return False
if installed != required_version:
# The crawler would exit with 42 immediately anyway - provide
# the new version first so the restart is seamless.
return perform_update(required_version)
if remote is not None and remote > installed:
# New version announced but not yet deployed: stage it now and
# only then flip the generation (concept step 16).
return perform_update(remote)
log.info("Crawler is up to date.")
return True
def main():
parser = argparse.ArgumentParser(
prog="launcher.py",
description="Starts and monitors the JonasSearch crawler and "
"handles generation-based rolling updates.",
)
parser.add_argument(
"--once",
action="store_true",
help="run one crawler cycle and exit (no restart loop)",
)
parser.add_argument(
"--check-update",
action="store_true",
help="check for a new version, run the update flow if needed "
"and exit",
)
parser.add_argument(
"--status",
action="store_true",
help="show deployment state, installed version and exit",
)
args = parser.parse_args()
if args.status:
try:
generation = crawler_mod.get_current_generation()
except Exception as exc:
log.error("Deployment state could not be read: %s", exc)
return crawler_mod.EXIT_ERROR
installed = read_installed_version(CRAWLER_ENTRY)
required_version = read_required_version()
print(
f"generation: {generation}\n"
f"required_version: {required_version}\n"
f"installed: {installed}\n"
f"crawler file: {CRAWLER_ENTRY}\n"
f"releases dir: {RELEASES_DIR}"
)
return (
crawler_mod.EXIT_OK
if required_version is not None
else crawler_mod.EXIT_ERROR
)
if args.check_update:
return crawler_mod.EXIT_OK if check_and_update() else crawler_mod.EXIT_ERROR
log.info(
"Launcher started. Crawler: %s | releases: %s",
CRAWLER_ENTRY,
RELEASES_DIR,
)
update_cycle_ran = False
while True:
process = start_crawler_process()
exit_code = wait_for_process(process)
if exit_code == crawler_mod.EXIT_UPDATE_REQUIRED:
# The crawler reached a safe point itself and wants the
# update. New version first, then the generation bump, then
# the restart (see perform_update()).
log.warning(
"Crawler exited with code %d: update required.",
crawler_mod.EXIT_UPDATE_REQUIRED,
)
required_version = read_required_version()
if required_version is None:
log.error(
"Cannot determine the required version; starting the "
"existing crawler again."
)
time.sleep(RESTART_DELAY_ON_ERROR)
continue
installed = read_installed_version(CRAWLER_ENTRY)
if installed == required_version:
# Rare case: the version was already switched externally
# (e.g. by --check-update) - nothing to provision.
log.info("Version %d already active.", required_version)
elif not perform_update(required_version):
log.error(
"Update failed. Starting the existing version again "
"after %ds (it will exit with 42 at the next safe "
"point and the update will be retried).",
RESTART_DELAY_ON_ERROR,
)
time.sleep(RESTART_DELAY_ON_ERROR)
continue
update_cycle_ran = True
if args.once:
# One cycle only: report the update to the caller
# (exit code 42) instead of restarting ourselves.
break
try:
current_generation = crawler_mod.get_current_generation()
except Exception:
current_generation = "?"
log.info(
"Restarting the crawler (it reads generation %s at "
"startup) ...",
current_generation,
)
time.sleep(crawler_mod.LAUNCHER_RESTART_DELAY)
continue
if exit_code == crawler_mod.EXIT_OK:
# Normal end (e.g. Ctrl+C on the crawler): do not restart.
log.info("Crawler ended normally (exit 0).")
break
# Anything else is an error/crash - NOT an update.
log.error(
"Crawler crashed (exit code %s). Restart in %ds ...",
exit_code,
RESTART_DELAY_ON_ERROR,
)
time.sleep(RESTART_DELAY_ON_ERROR)
if args.once:
break
log.info("Launcher stopped.")
return (
crawler_mod.EXIT_UPDATE_REQUIRED
if update_cycle_ran
else crawler_mod.EXIT_OK
)
if __name__ == "__main__":
sys.exit(main())
Reference in New Issue View Git Blame Copy Permalink