add vorschläge

priorityweb/app.py

@@ -0,0 +1,920 @@
+#!/usr/bin/env python3
+"""
+JonasSearch Vorschlagsverfeinerer Gamma 0.1
+
+Lets users submit domains that are put at a very high priority in the
+crawler's PostgreSQL queue (crawl_queue), so they get crawled next.
+
+Runs on PORT = statserver port + 1 (default 8082).
+
+How the queue jump works
+------------------------
+The crawler claims work with:
+
+    ORDER BY priority DESC, discovered_at ASC, id ASC
+
+Normal crawler-discovered URLs sit at priority 0..500. A suggestion is
+upserted at SUGGEST_PRIORITY (1,000,000), which sorts it above the whole
+existing queue WITHOUT touching the other ~13M pending rows. The table
+is far too large for mass UPDATEs: such a query blocks the crawler for
+minutes. Multiple suggestions all share SUGGEST_PRIORITY and are
+crawled in submission order (discovered_at, id), so no demotion is
+needed there either.
+"""
+
+import os
+import re
+import threading
+import time
+from datetime import datetime, timezone
+
+import psycopg2
+from flask import Flask, jsonify, render_template_string, request
+from psycopg2.extras import RealDictCursor
+
+DB_HOST = os.getenv("JONASSEARCH_DB_HOST", "127.0.0.1")
+DB_PORT = int(os.getenv("JONASSEARCH_DB_PORT", "25432"))
+DB_NAME = os.getenv("JONASSEARCH_DB_NAME", "jonassearch")
+DB_USER = os.getenv("JONASSEARCH_DB_USER", "jonassearch")
+DB_PASSWORD = os.getenv("JONASSEARCH_DB_PASSWORD", "")
+
+# Port of the statserver (jonassearch-stats.service, web/app.py).
+STATSERVER_PORT = int(os.getenv("JONASSEARCH_STATSERVER_PORT", "8081"))
+PORT = int(os.getenv("PORT", str(STATSERVER_PORT + 1)))
+
+# Very high priority for user suggestions. The current pending maximum
+# in the queue is 500 (crawler-assigned), so this always sorts on top.
+SUGGEST_PRIORITY = 1_000_000
+
+# Hard safety net: this UI must never run away on the huge queue table.
+STATEMENT_TIMEOUT_MS = int(os.getenv("JONASSEARCH_STATEMENT_TIMEOUT_MS", "15000"))
+
+# Max suggestions shown in the recent list.
+MAX_RECENT = 50
+
+# Very defensive domain validation (ASCII only).
+DOMAIN_RE = re.compile(
+    r"^(?=.{1,253}\.)([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)+"
+    r"[a-z]{2,63}$"
+)
+
+app = Flask(__name__)
+
+
+def db():
+    return psycopg2.connect(
+        host=DB_HOST,
+        port=DB_PORT,
+        dbname=DB_NAME,
+        user=DB_USER,
+        password=DB_PASSWORD,
+        connect_timeout=5,
+        application_name="jonassearch-priority",
+        options=f"-c statement_timeout={STATEMENT_TIMEOUT_MS}",
+    )
+
+
+def query(sql, params=None):
+    conn = db()
+    try:
+        with conn.cursor(cursor_factory=RealDictCursor) as cur:
+            cur.execute(sql, params)
+            if cur.description:
+                return cur.fetchall()
+            return []
+    finally:
+        conn.close()
+
+
+def clean_domain(raw):
+    """
+    Turn arbitrary user input into a registrable domain.
+
+    Accepts "example.com", "https://www.example.com/pfad",
+    "www.example.com", strips scheme, path, port, "www." prefix.
+    Returns lowercase domain or None if invalid.
+    """
+    if not raw:
+        return None
+
+    value = raw.strip().lower()
+
+    # Accept full URLs too.
+    if "://" in value:
+        value = value.split("://", 1)[1]
+
+    # Cut off path, query, port and userinfo.
+    value = value.split("/", 1)[0]
+    value = value.split("?", 1)[0]
+    value = value.split("#", 1)[0]
+    value = value.split("@", 1)[-1]
+    value = value.rsplit(":", 1)[0]
+
+    # Strip a leading www. so we prioritize the whole domain.
+    if value.startswith("www."):
+        value = value[4:]
+
+    value = value.rstrip(".")
+
+    if not DOMAIN_RE.match(value):
+        return None
+
+    return value
+
+
+def ensure_suggestions_table(conn):
+    """Bookkeeping table for the UI; the queue itself stays untouched."""
+    with conn.cursor() as cur:
+        cur.execute("""
+            CREATE TABLE IF NOT EXISTS domain_suggestions (
+                id BIGSERIAL PRIMARY KEY,
+                domain TEXT NOT NULL UNIQUE,
+                requested_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+                request_count INTEGER NOT NULL DEFAULT 1,
+                last_status TEXT
+            )
+        """)
+    conn.commit()
+
+
+def ensure_suggestions_table_standalone():
+    """Create the bookkeeping table with its own short-lived connection."""
+    conn = db()
+    try:
+        ensure_suggestions_table(conn)
+    finally:
+        conn.close()
+
+
+def suggest_domain(conn, domain):
+    """
+    Put the domain at the very front of the crawl queue.
+
+    Uses the passed connection (so tests can run against temp tables)
+    and only touches rows by their unique url index — never the whole
+    table.
+    """
+    url = f"https://{domain}/"
+
+    with conn.cursor() as cur:
+        # The domain itself: insert at top priority. If it already
+        # exists, pull it back to pending at top priority — unless a
+        # worker is processing it right now, in which case we leave it
+        # alone and let the worker finish.
+        cur.execute("""
+            INSERT INTO crawl_queue (url, source_url, priority, status)
+            VALUES (%s, %s, %s, 'pending')
+            ON CONFLICT (url) DO UPDATE SET
+                priority = EXCLUDED.priority,
+                status = 'pending',
+                last_attempt_at = NULL
+            WHERE crawl_queue.status IS DISTINCT FROM 'processing'
+        """, (url, "vorschlagsverfeinerer", SUGGEST_PRIORITY))
+
+        # UI bookkeeping.
+        cur.execute("""
+            INSERT INTO domain_suggestions (domain, last_status)
+            VALUES (%s, 'queued')
+            ON CONFLICT (domain) DO UPDATE SET
+                requested_at = now(),
+                request_count = domain_suggestions.request_count + 1,
+                last_status = 'queued'
+        """, (domain,))
+
+    conn.commit()
+
+
+def urls_ahead_of(domain):
+    """
+    How many pending URLs are claimed before this suggestion.
+
+    The crawler picks: priority DESC, then discovered_at ASC, then id
+    ASC. All suggestions share SUGGEST_PRIORITY, so only other
+    suggestions submitted earlier can be ahead.
+    """
+    row = query("""
+        SELECT COUNT(*) AS ahead
+        FROM crawl_queue
+        WHERE status = 'pending'
+          AND (
+              priority > %(P)s
+              OR (
+                  priority = %(P)s
+                  AND (discovered_at, id) < (
+                      SELECT discovered_at, id
+                      FROM crawl_queue
+                      WHERE url = %(url)s
+                  )
+              )
+          )
+    """, {"P": SUGGEST_PRIORITY, "url": f"https://{domain}/"})
+
+    return int(row[0]["ahead"] or 0)
+
+
+# ============================================================
+# ROUTES
+# ============================================================
+
+@app.get("/")
+def index():
+    return render_template_string(HTML, port=PORT)
+
+
+@app.get("/api/suggest")
+def api_suggest():
+    """Convenience: /api/suggest?domain=example.com also works."""
+    return _handle_suggest(request.args.get("domain", ""))
+
+
+@app.post("/api/suggest")
+def api_suggest_post():
+    body = request.get_json(silent=True) or {}
+    domain_input = body.get("domain") or request.form.get("domain", "")
+    return _handle_suggest(domain_input)
+
+
+def _handle_suggest(raw_domain):
+    domain = clean_domain(raw_domain)
+
+    if not domain:
+        return jsonify({
+            "ok": False,
+            "error": (
+                "Bitte eine gültige Domain eingeben, z. B. "
+                "example.com oder https://www.example.com/seite"
+            ),
+        }), 400
+
+    conn = None
+    try:
+        conn = db()
+        suggest_domain(conn, domain)
+    except Exception as exc:
+        if conn is not None:
+            conn.rollback()
+            conn.close()
+            conn = None
+
+        # First run without the bookkeeping table: create it and retry
+        # once (42P01 = undefined_table).
+        if getattr(exc, "pgcode", None) == "42P01":
+            try:
+                ensure_suggestions_table_standalone()
+                conn = db()
+                suggest_domain(conn, domain)
+            except Exception as retry_exc:
+                if conn is not None:
+                    conn.rollback()
+                return jsonify({
+                    "ok": False,
+                    "error": f"Datenbankfehler: {retry_exc}",
+                }), 503
+        else:
+            return jsonify({
+                "ok": False,
+                "error": f"Datenbankfehler: {exc}",
+            }), 503
+    finally:
+        if conn is not None:
+            conn.close()
+
+    try:
+        ahead = urls_ahead_of(domain)
+    except Exception:
+        ahead = None
+
+    return jsonify({
+        "ok": True,
+        "domain": domain,
+        "priority": SUGGEST_PRIORITY,
+        "urls_ahead": ahead,
+    })
+
+
+@app.get("/api/suggestions")
+def api_suggestions():
+    """Recent suggestions for the UI list."""
+    try:
+        rows = query("""
+            SELECT domain,
+                   requested_at,
+                   request_count,
+                   last_status,
+                   EXISTS (
+                       SELECT 1 FROM crawl_queue q
+                       WHERE q.url = 'https://' || domain_suggestions.domain || '/'
+                         AND q.status = 'pending'
+                   ) AS still_pending
+            FROM domain_suggestions
+            ORDER BY requested_at DESC
+            LIMIT %s
+        """, (MAX_RECENT,))
+    except Exception:
+        rows = []
+
+    return jsonify([
+        {
+            "domain": r["domain"],
+            "requested_at": r["requested_at"].isoformat(),
+            "request_count": int(r["request_count"]),
+            "last_status": r["last_status"],
+            "still_pending": bool(r["still_pending"]),
+        }
+        for r in rows
+    ])
+
+
+# ============================================================
+# QUEUE SNAPSHOT (background refresh)
+# ============================================================
+
+# The pending/processing counts scan a 13M-row queue (several seconds
+# of index-only scan). They must never run inside a request: a
+# background thread refreshes them every 60 s and /api/queue serves
+# the cached snapshot. The "boosted" count only reads the tiny
+# priority >= SUGGEST_PRIORITY index range and is always fast.
+
+QUEUE_REFRESH_SECONDS = int(os.getenv("JONASSEARCH_QUEUE_REFRESH_SECONDS", "60"))
+
+_queue_lock = threading.Lock()
+_queue_snapshot = None
+
+
+def compute_queue_snapshot():
+    """Full queue stats; may take several seconds on the big table."""
+    row = query("""
+        SELECT
+            (SELECT COUNT(*) FROM crawl_queue
+             WHERE status = 'pending') AS pending,
+            (SELECT COUNT(*) FROM crawl_queue
+             WHERE status = 'processing') AS processing,
+            (SELECT COUNT(*) FROM crawl_queue
+             WHERE status = 'pending'
+               AND priority >= %s) AS boosted
+    """, (SUGGEST_PRIORITY,))[0]
+
+    return {
+        "pending": int(row["pending"] or 0),
+        "processing": int(row["processing"] or 0),
+        "boosted": int(row["boosted"] or 0),
+    }
+
+
+def _queue_worker():
+    global _queue_snapshot
+
+    while True:
+        try:
+            snapshot = compute_queue_snapshot()
+            with _queue_lock:
+                _queue_snapshot = snapshot
+            time.sleep(QUEUE_REFRESH_SECONDS)
+        except Exception:
+            # DB hiccup: retry shortly; the UI keeps the last snapshot.
+            time.sleep(15)
+
+
+def start_queue_worker():
+    thread = threading.Thread(
+        target=_queue_worker,
+        name="queue-stats",
+        daemon=True,
+    )
+    thread.start()
+
+
+@app.get("/api/queue")
+def api_queue():
+    """Small live status for the header widgets (cached, see above)."""
+    with _queue_lock:
+        snapshot = _queue_snapshot
+
+    if snapshot is not None:
+        return jsonify({
+            **snapshot,
+            "online": True,
+            "timestamp": datetime.now(timezone.utc).isoformat(),
+        })
+
+    # No snapshot yet (service just started): fall back to the fast
+    # boosted-only count instead of blocking the request for seconds.
+    try:
+        row = query("""
+            SELECT COUNT(*) AS boosted
+            FROM crawl_queue
+            WHERE status = 'pending'
+              AND priority >= %s
+        """, (SUGGEST_PRIORITY,))[0]
+    except Exception:
+        return jsonify({"online": False}), 503
+
+    return jsonify({
+        "online": True,
+        "pending": None,
+        "processing": None,
+        "boosted": int(row["boosted"] or 0),
+        "timestamp": datetime.now(timezone.utc).isoformat(),
+    })
+
+
+@app.get("/api/health")
+def api_health():
+    try:
+        query("SELECT 1")
+        return jsonify({"status": "ok"})
+    except Exception as exc:
+        return jsonify({"status": "error", "error": str(exc)}), 503
+
+
+# ============================================================
+# HTML (inline, no extra static files needed)
+# ============================================================
+
+HTML = """
+<!DOCTYPE html>
+<html lang="de">
+<head>
+    <meta charset="UTF-8">
+    <meta name="viewport" content="width=device-width, initial-scale=1.0">
+    <title>JonasSearch · Vorschlagsverfeinerer</title>
+    <style>
+        :root {
+            font-family: Inter, system-ui, -apple-system, BlinkMacSystemFont,
+                "Segoe UI", sans-serif;
+            color-scheme: dark;
+
+            --bg: #080a0f;
+            --panel: #10131a;
+            --panel-2: #151922;
+            --border: #252b36;
+            --text: #f2f4f8;
+            --muted: #8b94a5;
+            --accent: #7dd3fc;
+            --green: #4ade80;
+            --red: #fb7185;
+            --gold: #facc15;
+        }
+
+        * { box-sizing: border-box; }
+
+        body {
+            margin: 0;
+            min-height: 100vh;
+            background: radial-gradient(circle at top, #151b27 0, var(--bg) 45%);
+            color: var(--text);
+        }
+
+        header {
+            max-width: 1100px;
+            margin: auto;
+            padding: 48px 32px 32px;
+            display: flex;
+            justify-content: space-between;
+            align-items: flex-start;
+        }
+
+        .eyebrow {
+            color: var(--accent);
+            font-size: 12px;
+            font-weight: 800;
+            letter-spacing: 0.18em;
+        }
+
+        h1 { margin: 5px 0; font-size: 36px; line-height: 1; }
+        header p { margin: 10px 0 0; color: var(--muted); }
+
+        .status {
+            display: flex; align-items: center; gap: 9px;
+            color: var(--muted); font-size: 14px;
+        }
+
+        #statusDot {
+            width: 9px; height: 9px; border-radius: 50%;
+            display: inline-block; background: var(--muted);
+        }
+        #statusDot.online { background: var(--green); }
+        #statusDot.offline { background: var(--red); }
+
+        main {
+            max-width: 1100px;
+            margin: auto;
+            padding: 0 32px 50px;
+        }
+
+        .hero {
+            background: rgba(16, 19, 26, 0.92);
+            border: 1px solid var(--border);
+            border-radius: 14px;
+            box-shadow: 0 15px 50px rgba(0, 0, 0, 0.22);
+            padding: 28px;
+            margin-bottom: 16px;
+        }
+
+        .hero h2 { margin: 0 0 6px; font-size: 20px; }
+        .hero .hint { margin: 0 0 20px; color: var(--muted); font-size: 13px; }
+
+        .hint code {
+            background: var(--panel-2);
+            border: 1px solid var(--border);
+            border-radius: 6px;
+            padding: 2px 6px;
+            font-size: 12px;
+            color: var(--accent);
+        }
+
+        .input-row { display: flex; gap: 12px; flex-wrap: wrap; }
+
+        input[type="text"] {
+            flex: 1;
+            min-width: 260px;
+            background: var(--panel-2);
+            border: 1px solid var(--border);
+            border-radius: 10px;
+            padding: 14px 16px;
+            color: var(--text);
+            font-size: 15px;
+            outline: none;
+            transition: border-color 0.2s ease;
+        }
+
+        input[type="text"]:focus { border-color: var(--accent); }
+
+        input[type="text"]::placeholder { color: #5b6473; }
+
+        button {
+            background: linear-gradient(180deg, #38bdf8, #0ea5e9);
+            border: none;
+            color: #06222f;
+            font-weight: 800;
+            font-size: 14px;
+            padding: 14px 24px;
+            border-radius: 10px;
+            cursor: pointer;
+            transition: transform 0.1s ease, filter 0.2s ease;
+            white-space: nowrap;
+        }
+
+        button:hover { filter: brightness(1.1); }
+        button:active { transform: scale(0.98); }
+        button:disabled { opacity: 0.55; cursor: wait; }
+
+        .message {
+            margin-top: 16px;
+            padding: 12px 14px;
+            border-radius: 10px;
+            font-size: 14px;
+            display: none;
+        }
+
+        .message.ok {
+            display: block;
+            background: rgba(74, 222, 128, 0.09);
+            border: 1px solid rgba(74, 222, 128, 0.35);
+            color: var(--green);
+        }
+
+        .message.err {
+            display: block;
+            background: rgba(251, 113, 133, 0.09);
+            border: 1px solid rgba(251, 113, 133, 0.35);
+            color: var(--red);
+        }
+
+        .message b { color: inherit; }
+
+        .grid {
+            display: grid;
+            grid-template-columns: repeat(3, 1fr);
+            gap: 16px;
+            margin-bottom: 16px;
+        }
+
+        .card {
+            background: rgba(16, 19, 26, 0.92);
+            border: 1px solid var(--border);
+            border-radius: 14px;
+            padding: 20px 24px;
+            box-shadow: 0 15px 50px rgba(0, 0, 0, 0.22);
+        }
+
+        .card span {
+            display: block;
+            color: var(--muted);
+            font-size: 11px;
+            font-weight: 700;
+            letter-spacing: 0.08em;
+        }
+
+        .card strong {
+            display: block;
+            margin-top: 8px;
+            font-size: 28px;
+            letter-spacing: -0.04em;
+        }
+
+        .card.gold strong { color: var(--gold); }
+
+        .list-card {
+            background: rgba(16, 19, 26, 0.92);
+            border: 1px solid var(--border);
+            border-radius: 14px;
+            padding: 24px;
+            box-shadow: 0 15px 50px rgba(0, 0, 0, 0.22);
+        }
+
+        .list-card h2 { margin: 0 0 4px; font-size: 18px; }
+        .list-card .sub { margin: 0 0 18px; color: var(--muted); font-size: 13px; }
+
+        .suggestion {
+            display: flex;
+            justify-content: space-between;
+            align-items: center;
+            gap: 14px;
+            padding: 12px 0;
+            border-bottom: 1px solid var(--border);
+        }
+
+        .suggestion:last-child { border-bottom: none; }
+
+        .suggestion .domain {
+            font-weight: 600;
+            font-size: 14px;
+            word-break: break-all;
+        }
+
+        .suggestion .meta {
+            color: var(--muted);
+            font-size: 12px;
+            white-space: nowrap;
+        }
+
+        .badge {
+            display: inline-block;
+            margin-left: 8px;
+            padding: 2px 9px;
+            border-radius: 100px;
+            font-size: 11px;
+            font-weight: 700;
+        }
+
+        .badge.queued {
+            background: rgba(250, 204, 21, 0.12);
+            color: var(--gold);
+            border: 1px solid rgba(250, 204, 21, 0.3);
+        }
+
+        .badge.crawled {
+            background: rgba(74, 222, 128, 0.12);
+            color: var(--green);
+            border: 1px solid rgba(74, 222, 128, 0.3);
+        }
+
+        .empty { color: var(--muted); font-size: 14px; padding: 14px 0; }
+
+        footer {
+            max-width: 1100px;
+            margin: auto;
+            padding: 0 32px 30px;
+            color: var(--muted);
+            font-size: 12px;
+        }
+
+        @media (max-width: 760px) {
+            header { padding: 30px 18px 24px; flex-direction: column; gap: 20px; }
+            main { padding: 0 18px 30px; }
+            .grid { grid-template-columns: 1fr; }
+            footer { padding: 0 18px 30px; }
+        }
+    </style>
+</head>
+
+<body>
+
+<header>
+    <div>
+        <div class="eyebrow">SERVERJONASSEARCH</div>
+        <h1>Vorschlagsverfeinerer</h1>
+        <p>Gamma 0.1 · Domain vorschlagen und die Warteschlange überholen</p>
+    </div>
+
+    <div class="status">
+        <span id="statusDot"></span>
+        <span id="statusText">Verbinde…</span>
+    </div>
+</header>
+
+<main>
+
+    <section class="hero">
+        <h2>Domain priorisieren</h2>
+        <p class="hint">
+            Eingabe als Domain oder komplette URL — z.&nbsp;B.
+            <code>example.com</code> oder
+            <code>https://www.example.com/seite</code>.
+            Der Vorschlag landet mit sehr hoher Priorität direkt vorne in der
+            Crawl-Queue.
+        </p>
+
+        <form id="suggestForm">
+            <div class="input-row">
+                <input type="text" id="domainInput"
+                       placeholder="example.com" autocomplete="off"
+                       spellcheck="false">
+                <button type="submit" id="submitBtn">🚀 Priorisieren</button>
+            </div>
+        </form>
+
+        <div id="message" class="message"></div>
+    </section>
+
+    <section class="grid">
+
+        <article class="card" title="Letzte Messung des Crawlers">
+            <span>QUEUE PENDING</span>
+            <strong id="queuePending">—</strong>
+        </article>
+
+        <article class="card">
+            <span>IN BEARBEITUNG</span>
+            <strong id="queueProcessing">—</strong>
+        </article>
+
+        <article class="card gold">
+            <span>PRIORISIERTE DOMAINS</span>
+            <strong id="queueBoosted">—</strong>
+        </article>
+
+    </section>
+
+    <section class="list-card">
+        <h2>Letzte Vorschläge</h2>
+        <p class="sub">Die zuletzt priorisierten Domains und ihr Queue-Status</p>
+        <div id="suggestions">
+            <div class="empty">Lade…</div>
+        </div>
+    </section>
+
+</main>
+
+<footer>
+    JonasSearch Gamma 0.1 · Vorschlagsverfeinerer · Port {{ port }}
+</footer>
+
+<script>
+    const $ = id => document.getElementById(id);
+
+    function fmt(n) {
+        return (n === null || n === undefined)
+            ? "—"
+            : new Intl.NumberFormat("de-DE").format(n);
+    }
+
+    function setStatus(online) {
+        $("statusDot").className = online ? "online" : "offline";
+        $("statusText").textContent = online ? "Online" : "Offline";
+    }
+
+    function showMessage(kind, html) {
+        const box = $("message");
+        box.className = "message " + kind;
+        box.innerHTML = html;
+    }
+
+    function escapeHTML(s) {
+        return String(s)
+            .replaceAll("&", "&amp;")
+            .replaceAll("<", "&lt;")
+            .replaceAll(">", "&gt;")
+            .replaceAll('"', "&quot;")
+            .replaceAll("'", "&#039;");
+    }
+
+    async function refreshQueue() {
+        try {
+            const r = await fetch("/api/queue", { cache: "no-store" });
+            const d = await r.json();
+            setStatus(d.online !== false);
+            $("queuePending").textContent = fmt(d.pending);
+            $("queueProcessing").textContent = fmt(d.processing);
+            $("queueBoosted").textContent = fmt(d.boosted);
+        } catch {
+            setStatus(false);
+        }
+    }
+
+    async function refreshSuggestions() {
+        try {
+            const r = await fetch("/api/suggestions", { cache: "no-store" });
+            const items = await r.json();
+            const box = $("suggestions");
+
+            if (!items.length) {
+                box.innerHTML =
+                    '<div class="empty">Noch keine Vorschläge — ' +
+                    'sei der Erste! 🚀</div>';
+                return;
+            }
+
+            box.innerHTML = items.map(x => {
+                const when = new Date(x.requested_at)
+                    .toLocaleString("de-DE");
+                const times = x.request_count > 1
+                    ? ` · ${x.request_count}× vorgeschlagen` : "";
+                const badge = x.still_pending
+                    ? '<span class="badge queued">in Queue</span>'
+                    : '<span class="badge crawled">verarbeitet</span>';
+                return `
+                    <div class="suggestion">
+                        <div class="domain">
+                            ${escapeHTML(x.domain)}${badge}
+                        </div>
+                        <div class="meta">
+                            ${escapeHTML(when)}${escapeHTML(times)}
+                        </div>
+                    </div>
+                `;
+            }).join("");
+        } catch {
+            $("suggestions").innerHTML =
+                '<div class="empty">Vorschläge konnten nicht geladen werden.</div>';
+        }
+    }
+
+    $("suggestForm").addEventListener("submit", async (event) => {
+        event.preventDefault();
+
+        const btn = $("submitBtn");
+        const domain = $("domainInput").value.trim();
+
+        btn.disabled = true;
+
+        try {
+            const r = await fetch("/api/suggest", {
+                method: "POST",
+                headers: { "Content-Type": "application/json" },
+                body: JSON.stringify({ domain })
+            });
+
+            const d = await r.json();
+
+            if (d.ok) {
+                const ahead = d.urls_ahead === null || d.urls_ahead === undefined
+                    ? ""
+                    : ` — noch <b>${fmt(d.urls_ahead)}</b> Vorschläge vor dir`;
+                showMessage("ok",
+                    `✅ <b>${escapeHTML(d.domain)}</b> steht jetzt ` +
+                    `mit Priorität ${d.priority} vorne in der Queue` +
+                    ahead + ".");
+                $("domainInput").value = "";
+                refreshSuggestions();
+            } else {
+                showMessage("err",
+                    "⚠️ " + escapeHTML(d.error || "Unbekannter Fehler"));
+            }
+        } catch {
+            showMessage("err",
+                "⚠️ Server nicht erreichbar — ist der Dienst gestartet?");
+        } finally {
+            btn.disabled = false;
+            refreshQueue();
+        }
+    });
+
+    refreshQueue();
+    refreshSuggestions();
+    setInterval(refreshQueue, 15000);
+    setInterval(refreshSuggestions, 30000);
+</script>
+
+</body>
+</html>
+"""
+
+
+if __name__ == "__main__":
+    host = os.getenv("HOST", "0.0.0.0")
+
+    # Bookkeeping table for first run.
+    try:
+        ensure_suggestions_table_standalone()
+    except Exception as exc:
+        print(f"WARN: could not ensure domain_suggestions table: {exc}")
+
+    start_queue_worker()
+
+    print("=" * 70)
+    print("     JonasSearch Vorschlagsverfeinerer Gamma 0.1")
+    print("=" * 70)
+    print(f"Statserver : Port {STATSERVER_PORT}")
+    print(f"HTTP       : {host}:{PORT} (Statserver + 1)")
+    print(f"DB         : {DB_HOST}:{DB_PORT}/{DB_NAME}")
+    print(f"Priorität  : {SUGGEST_PRIORITY}")
+    print("=" * 70)
+
+    app.run(
+        host=host,
+        port=PORT,
+        debug=False,
+        threaded=True,
+    )

priorityweb/jonassearch-priority.service

@@ -0,0 +1,36 @@
+# JonasSearch Vorschlagsverfeinerer (priority suggestion web UI)
+#
+# Installieren (als root):
+#   cp /srv/serverjonas/jonassearch/priorityweb/jonassearch-priority.service \
+#      /etc/systemd/system/
+#   systemctl daemon-reload
+#   systemctl enable --now jonassearch-priority.service
+#
+# Port: Statserver (8081) + 1 = 8082
+
+[Unit]
+Description=JonasSearch Vorschlagsverfeinerer (priority domain suggestions)
+After=network-online.target postgresql@17-jonassearch.service
+Wants=network-online.target
+
+[Service]
+Type=simple
+
+User=jonassearch
+Group=jonassearch
+
+WorkingDirectory=/srv/serverjonas/jonassearch
+
+EnvironmentFile=/srv/serverjonas/search/.env
+
+ExecStart=/usr/bin/python3 /srv/serverjonas/jonassearch/priorityweb/app.py
+
+Restart=on-failure
+RestartSec=10
+
+# Sauberer Shutdown
+KillSignal=SIGTERM
+TimeoutStopSec=30
+
+[Install]
+WantedBy=multi-user.target

priorityweb/test_app.py

@@ -0,0 +1,474 @@
+#!/usr/bin/env python3
+"""
+Simple local test suite for the Vorschlagsverfeinerer.
+
+DB-backed tests use temp tables shadowing the real ones, so the
+production crawl_queue is never touched.
+"""
+
+import os
+import sys
+import unittest
+from unittest import mock
+
+sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
+
+
+def _load_env_file():
+    """
+    Load ../.env like the systemd EnvironmentFile does, so the tests
+    also run outside systemd (local manual runs). Must happen before
+    app.py is imported, which reads the DB variables at import time.
+    """
+    env_path = os.path.join(
+        os.path.dirname(os.path.abspath(__file__)),
+        os.pardir,
+        ".env",
+    )
+
+    try:
+        with open(env_path) as handle:
+            for line in handle:
+                line = line.strip()
+                if not line or line.startswith("#") or "=" not in line:
+                    continue
+                key, _, value = line.partition("=")
+                os.environ.setdefault(key.strip(), value.strip())
+    except OSError:
+        pass
+
+
+_load_env_file()
+
+import app as priority_app  # noqa: E402
+
+
+class CleanDomainTest(unittest.TestCase):
+
+    def test_plain_domain(self):
+        self.assertEqual(
+            priority_app.clean_domain("example.com"), "example.com"
+        )
+
+    def test_uppercase_and_spaces(self):
+        self.assertEqual(
+            priority_app.clean_domain("  EXAMPLE.COM  "), "example.com"
+        )
+
+    def test_www_prefix(self):
+        self.assertEqual(
+            priority_app.clean_domain("www.example.com"), "example.com"
+        )
+
+    def test_full_url(self):
+        self.assertEqual(
+            priority_app.clean_domain("https://www.example.com/pfad?x=1"),
+            "example.com",
+        )
+
+    def test_http_url_with_port(self):
+        self.assertEqual(
+            priority_app.clean_domain("http://example.com:8080/index.html"),
+            "example.com",
+        )
+
+    def test_trailing_dot(self):
+        self.assertEqual(
+            priority_app.clean_domain("example.com."), "example.com"
+        )
+
+    def test_subdomain(self):
+        self.assertEqual(
+            priority_app.clean_domain("docs.example.co.uk"),
+            "docs.example.co.uk",
+        )
+
+    def test_invalid_empty(self):
+        self.assertIsNone(priority_app.clean_domain(""))
+
+    def test_invalid_no_tld(self):
+        self.assertIsNone(priority_app.clean_domain("localhost"))
+
+    def test_invalid_injection(self):
+        self.assertIsNone(
+            priority_app.clean_domain("example.com'; DROP TABLE pages; --")
+        )
+
+    def test_invalid_javascript_scheme(self):
+        self.assertIsNone(
+            priority_app.clean_domain("javascript:alert(1)")
+        )
+
+
+class QueuePriorityTest(unittest.TestCase):
+    """
+    Test the queue-jump SQL against a real database connection whose
+    temp tables shadow the production tables.
+    """
+
+    def setUp(self):
+        self.conn = priority_app.db()
+
+        with self.conn.cursor() as cur:
+            cur.execute("""
+                CREATE TEMP TABLE crawl_queue (
+                    id BIGSERIAL PRIMARY KEY,
+                    url TEXT NOT NULL UNIQUE,
+                    source_url TEXT,
+                    discovered_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+                    priority INTEGER NOT NULL DEFAULT 0,
+                    status TEXT NOT NULL DEFAULT 'pending'
+                        CHECK (status IN
+                            ('pending','processing','done','failed')),
+                    attempts INTEGER NOT NULL DEFAULT 0,
+                    last_attempt_at TIMESTAMPTZ
+                )
+            """)
+            cur.execute("""
+                CREATE TEMP TABLE domain_suggestions (
+                    id BIGSERIAL PRIMARY KEY,
+                    domain TEXT NOT NULL UNIQUE,
+                    requested_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+                    request_count INTEGER NOT NULL DEFAULT 1,
+                    last_status TEXT
+                )
+            """)
+
+        self.conn.commit()
+
+    def tearDown(self):
+        self.conn.rollback()
+        self.conn.close()
+
+    def _rows(self, sql, params=None):
+        with self.conn.cursor() as cur:
+            cur.execute(sql, params)
+            return cur.fetchall()
+
+    def test_suggestion_jumps_queue(self):
+        with self.conn.cursor() as cur:
+            # Simulate queue state: normal pending URLs, current max
+            # priority 500 like in production.
+            cur.execute("""
+                INSERT INTO crawl_queue (url, priority, status) VALUES
+                ('https://a.example/', 0, 'pending'),
+                ('https://b.example/', 0, 'pending'),
+                ('https://canva.example/', 500, 'pending')
+            """)
+        self.conn.commit()
+
+        priority_app.suggest_domain(self.conn, "cool.example")
+
+        # The crawler's exact claim order must put the suggestion first.
+        order = self._rows("""
+            SELECT url FROM crawl_queue
+            WHERE status = 'pending'
+            ORDER BY priority DESC, discovered_at ASC, id ASC
+        """)
+        self.assertEqual(order[0][0], "https://cool.example/")
+
+        row = self._rows(
+            "SELECT priority, status FROM crawl_queue "
+            "WHERE url = 'https://cool.example/'"
+        )[0]
+        self.assertEqual(row, (priority_app.SUGGEST_PRIORITY, "pending"))
+
+        # Other rows were not touched.
+        self.assertEqual(
+            self._rows(
+                "SELECT priority FROM crawl_queue "
+                "WHERE url = 'https://canva.example/'"
+            )[0][0],
+            500,
+        )
+
+        # Suggestion bookkeeping was written.
+        self.assertEqual(
+            self._rows("SELECT domain FROM domain_suggestions"),
+            [("cool.example",)],
+        )
+
+    def test_existing_failed_url_is_requeued(self):
+        with self.conn.cursor() as cur:
+            cur.execute("""
+                INSERT INTO crawl_queue (url, priority, status) VALUES
+                ('https://old.example/', 0, 'failed'),
+                ('https://x.example/', 0, 'pending')
+            """)
+        self.conn.commit()
+
+        priority_app.suggest_domain(self.conn, "old.example")
+
+        row = self._rows(
+            "SELECT priority, status FROM crawl_queue "
+            "WHERE url = 'https://old.example/'"
+        )[0]
+        self.assertEqual(row, (priority_app.SUGGEST_PRIORITY, "pending"))
+
+    def test_processing_row_is_left_alone(self):
+        with self.conn.cursor() as cur:
+            cur.execute("""
+                INSERT INTO crawl_queue (url, priority, status) VALUES
+                ('https://busy.example/', 123, 'processing')
+            """)
+        self.conn.commit()
+
+        priority_app.suggest_domain(self.conn, "busy.example")
+
+        # A worker is crawling it right now: do not interfere.
+        row = self._rows(
+            "SELECT priority, status FROM crawl_queue "
+            "WHERE url = 'https://busy.example/'"
+        )[0]
+        self.assertEqual(row, (123, "processing"))
+
+        # But it is still recorded as a suggestion.
+        self.assertEqual(
+            self._rows(
+                "SELECT request_count FROM domain_suggestions "
+                "WHERE domain = 'busy.example'"
+            )[0][0],
+            1,
+        )
+
+    def test_multiple_suggestions_crawl_in_order(self):
+        priority_app.suggest_domain(self.conn, "first.example")
+        priority_app.suggest_domain(self.conn, "second.example")
+
+        order = self._rows("""
+            SELECT url FROM crawl_queue
+            WHERE status = 'pending'
+            ORDER BY priority DESC, discovered_at ASC, id ASC
+        """)
+
+        self.assertEqual(
+            [row[0] for row in order[:2]],
+            ["https://first.example/", "https://second.example/"],
+        )
+
+
+class UrlsAheadTest(unittest.TestCase):
+    """urls_ahead_of must count only earlier suggestions."""
+
+    def setUp(self):
+        self.conn = priority_app.db()
+
+        with self.conn.cursor() as cur:
+            cur.execute("""
+                CREATE TEMP TABLE crawl_queue (
+                    id BIGSERIAL PRIMARY KEY,
+                    url TEXT NOT NULL UNIQUE,
+                    source_url TEXT,
+                    discovered_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+                    priority INTEGER NOT NULL DEFAULT 0,
+                    status TEXT NOT NULL DEFAULT 'pending',
+                    attempts INTEGER NOT NULL DEFAULT 0,
+                    last_attempt_at TIMESTAMPTZ
+                )
+            """)
+            cur.execute("""
+                INSERT INTO crawl_queue (url, priority, status) VALUES
+                ('https://earlier.example/', 1000000, 'pending'),
+                ('https://me.example/',      1000000, 'pending'),
+                ('https://later.example/',   1000000, 'pending'),
+                ('https://normal.example/',  0,       'pending'),
+                ('https://gone.example/',    1000000, 'done')
+            """)
+            # Force a strict discovered_at ordering.
+            cur.execute("""
+                UPDATE crawl_queue
+                SET discovered_at = now() - (
+                    CASE url
+                        WHEN 'https://earlier.example/' THEN interval '30 s'
+                        WHEN 'https://me.example/'      THEN interval '20 s'
+                        ELSE interval '10 s'
+                    END
+                )
+                WHERE url LIKE 'https://%.example/'
+            """)
+
+        self.conn.commit()
+
+    def tearDown(self):
+        self.conn.rollback()
+        self.conn.close()
+
+    def test_only_earlier_suggestions_count(self):
+        def fake_query(sql, params=None):
+            with self.conn.cursor() as cur:
+                cur.execute(sql, params)
+                rows = cur.fetchall()
+                cols = [d.name for d in cur.description]
+            return [dict(zip(cols, row)) for row in rows]
+
+        with mock.patch.object(priority_app, "query", fake_query):
+            ahead = priority_app.urls_ahead_of("me.example")
+
+        # Only 'earlier.example' is pending, same priority, older.
+        self.assertEqual(ahead, 1)
+
+
+class ApiTest(unittest.TestCase):
+    """HTTP-layer tests with mocked database access."""
+
+    def setUp(self):
+        priority_app.app.config["TESTING"] = True
+        self.client = priority_app.app.test_client()
+
+    def test_index_serves_html(self):
+        response = self.client.get("/")
+        self.assertEqual(response.status_code, 200)
+        self.assertIn(b"Vorschlagsverfeinerer", response.data)
+        self.assertIn(b"domainInput", response.data)
+
+    def test_suggest_valid_domain(self):
+        with mock.patch.object(priority_app, "db"), \
+             mock.patch.object(priority_app, "suggest_domain"), \
+             mock.patch.object(priority_app, "urls_ahead_of",
+                               return_value=2):
+            response = self.client.post(
+                "/api/suggest", json={"domain": "example.com"}
+            )
+
+        self.assertEqual(response.status_code, 200)
+        data = response.get_json()
+        self.assertTrue(data["ok"])
+        self.assertEqual(data["domain"], "example.com")
+        self.assertEqual(data["urls_ahead"], 2)
+        self.assertEqual(data["priority"], priority_app.SUGGEST_PRIORITY)
+
+    def test_suggest_invalid_domain(self):
+        response = self.client.post(
+            "/api/suggest", json={"domain": "not a domain!"}
+        )
+
+        self.assertEqual(response.status_code, 400)
+        self.assertFalse(response.get_json()["ok"])
+
+    def test_suggest_get_variant(self):
+        with mock.patch.object(priority_app, "db"), \
+             mock.patch.object(priority_app, "suggest_domain"), \
+             mock.patch.object(priority_app, "urls_ahead_of",
+                               side_effect=RuntimeError):
+            response = self.client.get("/api/suggest?domain=example.org")
+
+        self.assertEqual(response.status_code, 200)
+        data = response.get_json()
+        self.assertTrue(data["ok"])
+        self.assertEqual(data["urls_ahead"], None)
+
+    def test_suggest_creates_missing_table_and_retries(self):
+        """
+        First run without domain_suggestions: the insert fails with
+        42P01, the table gets created, and the suggestion succeeds.
+        """
+        calls = {"suggest": 0}
+
+        class FakeUndefinedTable(Exception):
+            pgcode = "42P01"
+
+        def fake_db():
+            return mock.MagicMock()
+
+        def fake_suggest(conn, domain):
+            calls["suggest"] += 1
+            if calls["suggest"] == 1:
+                raise FakeUndefinedTable(
+                    "relation \"domain_suggestions\" does not exist"
+                )
+
+        with mock.patch.object(priority_app, "db", side_effect=fake_db), \
+             mock.patch.object(priority_app, "suggest_domain",
+                               side_effect=fake_suggest), \
+             mock.patch.object(priority_app,
+                               "ensure_suggestions_table_standalone") as \
+                    ensure_mock, \
+             mock.patch.object(priority_app, "urls_ahead_of", return_value=0):
+            response = self.client.post(
+                "/api/suggest", json={"domain": "example.com"}
+            )
+
+        self.assertEqual(response.status_code, 200)
+        self.assertTrue(response.get_json()["ok"])
+        self.assertEqual(calls["suggest"], 2)
+        ensure_mock.assert_called_once()
+
+    def test_suggest_db_error_returns_503(self):
+        with mock.patch.object(priority_app, "db",
+                               side_effect=RuntimeError("db down")):
+            response = self.client.post(
+                "/api/suggest", json={"domain": "example.com"}
+            )
+
+        self.assertEqual(response.status_code, 503)
+        self.assertFalse(response.get_json()["ok"])
+
+    def test_queue_endpoint_serves_snapshot(self):
+        snapshot = {"pending": 3, "processing": 1, "boosted": 2}
+        with mock.patch.object(priority_app, "_queue_lock"):
+            priority_app._queue_snapshot = dict(snapshot)
+            response = self.client.get("/api/queue")
+
+        self.assertEqual(response.status_code, 200)
+        data = response.get_json()
+        self.assertTrue(data["online"])
+        self.assertEqual(data["pending"], 3)
+        self.assertEqual(data["processing"], 1)
+        self.assertEqual(data["boosted"], 2)
+
+    def test_queue_endpoint_fallback_counts_boosted_only(self):
+        old = priority_app._queue_snapshot
+        priority_app._queue_snapshot = None
+        try:
+            with mock.patch.object(
+                priority_app, "query",
+                return_value=[{"boosted": 7}],
+            ):
+                response = self.client.get("/api/queue")
+        finally:
+            priority_app._queue_snapshot = old
+
+        self.assertEqual(response.status_code, 200)
+        data = response.get_json()
+        self.assertTrue(data["online"])
+        self.assertIsNone(data["pending"])
+        self.assertEqual(data["boosted"], 7)
+
+    def test_queue_endpoint_db_down(self):
+        old = priority_app._queue_snapshot
+        priority_app._queue_snapshot = None
+        try:
+            with mock.patch.object(priority_app, "query",
+                                   side_effect=RuntimeError("no db")):
+                response = self.client.get("/api/queue")
+        finally:
+            priority_app._queue_snapshot = old
+
+        self.assertEqual(response.status_code, 503)
+        self.assertFalse(response.get_json()["online"])
+
+    def test_suggestions_endpoint_empty(self):
+        with mock.patch.object(priority_app, "query",
+                               side_effect=RuntimeError("no table")):
+            response = self.client.get("/api/suggestions")
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.get_json(), [])
+
+    def test_health_ok(self):
+        with mock.patch.object(priority_app, "query", return_value=[]):
+            response = self.client.get("/api/health")
+
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.get_json()["status"], "ok")
+
+    def test_health_db_down(self):
+        with mock.patch.object(priority_app, "query",
+                               side_effect=RuntimeError("down")):
+            response = self.client.get("/api/health")
+
+        self.assertEqual(response.status_code, 503)
+
+
+if __name__ == "__main__":
+    unittest.main(verbosity=2)

web/app.py

@@ -10,7 +10,7 @@ from pathlib import Path
 from flask import Flask, jsonify, render_template_string, Response
 
 DB_DATA = Path(__file__).parent / "page_counts.json"
-INTERVAL = 15 * 60  # 15 Minuten
+INTERVAL = 30 * 60  # 30 Minuten
 
 app = Flask(__name__)