921 lines
26 KiB
Python
921 lines
26 KiB
Python
#!/usr/bin/env python3
|
||
"""
|
||
JonasSearch Vorschlagsverfeinerer Gamma 0.1
|
||
|
||
Lets users submit domains that are put at a very high priority in the
|
||
crawler's PostgreSQL queue (crawl_queue), so they get crawled next.
|
||
|
||
Runs on PORT = statserver port + 1 (default 8082).
|
||
|
||
How the queue jump works
|
||
------------------------
|
||
The crawler claims work with:
|
||
|
||
ORDER BY priority DESC, discovered_at ASC, id ASC
|
||
|
||
Normal crawler-discovered URLs sit at priority 0..500. A suggestion is
|
||
upserted at SUGGEST_PRIORITY (1,000,000), which sorts it above the whole
|
||
existing queue WITHOUT touching the other ~13M pending rows. The table
|
||
is far too large for mass UPDATEs: such a query blocks the crawler for
|
||
minutes. Multiple suggestions all share SUGGEST_PRIORITY and are
|
||
crawled in submission order (discovered_at, id), so no demotion is
|
||
needed there either.
|
||
"""
|
||
|
||
import os
|
||
import re
|
||
import threading
|
||
import time
|
||
from datetime import datetime, timezone
|
||
|
||
import psycopg2
|
||
from flask import Flask, jsonify, render_template_string, request
|
||
from psycopg2.extras import RealDictCursor
|
||
|
||
DB_HOST = os.getenv("JONASSEARCH_DB_HOST", "127.0.0.1")
|
||
DB_PORT = int(os.getenv("JONASSEARCH_DB_PORT", "25432"))
|
||
DB_NAME = os.getenv("JONASSEARCH_DB_NAME", "jonassearch")
|
||
DB_USER = os.getenv("JONASSEARCH_DB_USER", "jonassearch")
|
||
DB_PASSWORD = os.getenv("JONASSEARCH_DB_PASSWORD", "")
|
||
|
||
# Port of the statserver (jonassearch-stats.service, web/app.py).
|
||
STATSERVER_PORT = int(os.getenv("JONASSEARCH_STATSERVER_PORT", "8081"))
|
||
PORT = int(os.getenv("PORT", str(STATSERVER_PORT + 1)))
|
||
|
||
# Very high priority for user suggestions. The current pending maximum
|
||
# in the queue is 500 (crawler-assigned), so this always sorts on top.
|
||
SUGGEST_PRIORITY = 1_000_000
|
||
|
||
# Hard safety net: this UI must never run away on the huge queue table.
|
||
STATEMENT_TIMEOUT_MS = int(os.getenv("JONASSEARCH_STATEMENT_TIMEOUT_MS", "15000"))
|
||
|
||
# Max suggestions shown in the recent list.
|
||
MAX_RECENT = 50
|
||
|
||
# Very defensive domain validation (ASCII only).
|
||
DOMAIN_RE = re.compile(
|
||
r"^(?=.{1,253}\.)([a-z0-9]([a-z0-9-]{0,61}[a-z0-9])?\.)+"
|
||
r"[a-z]{2,63}$"
|
||
)
|
||
|
||
app = Flask(__name__)
|
||
|
||
|
||
def db():
|
||
return psycopg2.connect(
|
||
host=DB_HOST,
|
||
port=DB_PORT,
|
||
dbname=DB_NAME,
|
||
user=DB_USER,
|
||
password=DB_PASSWORD,
|
||
connect_timeout=5,
|
||
application_name="jonassearch-priority",
|
||
options=f"-c statement_timeout={STATEMENT_TIMEOUT_MS}",
|
||
)
|
||
|
||
|
||
def query(sql, params=None):
|
||
conn = db()
|
||
try:
|
||
with conn.cursor(cursor_factory=RealDictCursor) as cur:
|
||
cur.execute(sql, params)
|
||
if cur.description:
|
||
return cur.fetchall()
|
||
return []
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def clean_domain(raw):
|
||
"""
|
||
Turn arbitrary user input into a registrable domain.
|
||
|
||
Accepts "example.com", "https://www.example.com/pfad",
|
||
"www.example.com", strips scheme, path, port, "www." prefix.
|
||
Returns lowercase domain or None if invalid.
|
||
"""
|
||
if not raw:
|
||
return None
|
||
|
||
value = raw.strip().lower()
|
||
|
||
# Accept full URLs too.
|
||
if "://" in value:
|
||
value = value.split("://", 1)[1]
|
||
|
||
# Cut off path, query, port and userinfo.
|
||
value = value.split("/", 1)[0]
|
||
value = value.split("?", 1)[0]
|
||
value = value.split("#", 1)[0]
|
||
value = value.split("@", 1)[-1]
|
||
value = value.rsplit(":", 1)[0]
|
||
|
||
# Strip a leading www. so we prioritize the whole domain.
|
||
if value.startswith("www."):
|
||
value = value[4:]
|
||
|
||
value = value.rstrip(".")
|
||
|
||
if not DOMAIN_RE.match(value):
|
||
return None
|
||
|
||
return value
|
||
|
||
|
||
def ensure_suggestions_table(conn):
|
||
"""Bookkeeping table for the UI; the queue itself stays untouched."""
|
||
with conn.cursor() as cur:
|
||
cur.execute("""
|
||
CREATE TABLE IF NOT EXISTS domain_suggestions (
|
||
id BIGSERIAL PRIMARY KEY,
|
||
domain TEXT NOT NULL UNIQUE,
|
||
requested_at TIMESTAMPTZ NOT NULL DEFAULT now(),
|
||
request_count INTEGER NOT NULL DEFAULT 1,
|
||
last_status TEXT
|
||
)
|
||
""")
|
||
conn.commit()
|
||
|
||
|
||
def ensure_suggestions_table_standalone():
|
||
"""Create the bookkeeping table with its own short-lived connection."""
|
||
conn = db()
|
||
try:
|
||
ensure_suggestions_table(conn)
|
||
finally:
|
||
conn.close()
|
||
|
||
|
||
def suggest_domain(conn, domain):
|
||
"""
|
||
Put the domain at the very front of the crawl queue.
|
||
|
||
Uses the passed connection (so tests can run against temp tables)
|
||
and only touches rows by their unique url index — never the whole
|
||
table.
|
||
"""
|
||
url = f"https://{domain}/"
|
||
|
||
with conn.cursor() as cur:
|
||
# The domain itself: insert at top priority. If it already
|
||
# exists, pull it back to pending at top priority — unless a
|
||
# worker is processing it right now, in which case we leave it
|
||
# alone and let the worker finish.
|
||
cur.execute("""
|
||
INSERT INTO crawl_queue (url, source_url, priority, status)
|
||
VALUES (%s, %s, %s, 'pending')
|
||
ON CONFLICT (url) DO UPDATE SET
|
||
priority = EXCLUDED.priority,
|
||
status = 'pending',
|
||
last_attempt_at = NULL
|
||
WHERE crawl_queue.status IS DISTINCT FROM 'processing'
|
||
""", (url, "vorschlagsverfeinerer", SUGGEST_PRIORITY))
|
||
|
||
# UI bookkeeping.
|
||
cur.execute("""
|
||
INSERT INTO domain_suggestions (domain, last_status)
|
||
VALUES (%s, 'queued')
|
||
ON CONFLICT (domain) DO UPDATE SET
|
||
requested_at = now(),
|
||
request_count = domain_suggestions.request_count + 1,
|
||
last_status = 'queued'
|
||
""", (domain,))
|
||
|
||
conn.commit()
|
||
|
||
|
||
def urls_ahead_of(domain):
|
||
"""
|
||
How many pending URLs are claimed before this suggestion.
|
||
|
||
The crawler picks: priority DESC, then discovered_at ASC, then id
|
||
ASC. All suggestions share SUGGEST_PRIORITY, so only other
|
||
suggestions submitted earlier can be ahead.
|
||
"""
|
||
row = query("""
|
||
SELECT COUNT(*) AS ahead
|
||
FROM crawl_queue
|
||
WHERE status = 'pending'
|
||
AND (
|
||
priority > %(P)s
|
||
OR (
|
||
priority = %(P)s
|
||
AND (discovered_at, id) < (
|
||
SELECT discovered_at, id
|
||
FROM crawl_queue
|
||
WHERE url = %(url)s
|
||
)
|
||
)
|
||
)
|
||
""", {"P": SUGGEST_PRIORITY, "url": f"https://{domain}/"})
|
||
|
||
return int(row[0]["ahead"] or 0)
|
||
|
||
|
||
# ============================================================
|
||
# ROUTES
|
||
# ============================================================
|
||
|
||
@app.get("/")
|
||
def index():
|
||
return render_template_string(HTML, port=PORT)
|
||
|
||
|
||
@app.get("/api/suggest")
|
||
def api_suggest():
|
||
"""Convenience: /api/suggest?domain=example.com also works."""
|
||
return _handle_suggest(request.args.get("domain", ""))
|
||
|
||
|
||
@app.post("/api/suggest")
|
||
def api_suggest_post():
|
||
body = request.get_json(silent=True) or {}
|
||
domain_input = body.get("domain") or request.form.get("domain", "")
|
||
return _handle_suggest(domain_input)
|
||
|
||
|
||
def _handle_suggest(raw_domain):
|
||
domain = clean_domain(raw_domain)
|
||
|
||
if not domain:
|
||
return jsonify({
|
||
"ok": False,
|
||
"error": (
|
||
"Bitte eine gültige Domain eingeben, z. B. "
|
||
"example.com oder https://www.example.com/seite"
|
||
),
|
||
}), 400
|
||
|
||
conn = None
|
||
try:
|
||
conn = db()
|
||
suggest_domain(conn, domain)
|
||
except Exception as exc:
|
||
if conn is not None:
|
||
conn.rollback()
|
||
conn.close()
|
||
conn = None
|
||
|
||
# First run without the bookkeeping table: create it and retry
|
||
# once (42P01 = undefined_table).
|
||
if getattr(exc, "pgcode", None) == "42P01":
|
||
try:
|
||
ensure_suggestions_table_standalone()
|
||
conn = db()
|
||
suggest_domain(conn, domain)
|
||
except Exception as retry_exc:
|
||
if conn is not None:
|
||
conn.rollback()
|
||
return jsonify({
|
||
"ok": False,
|
||
"error": f"Datenbankfehler: {retry_exc}",
|
||
}), 503
|
||
else:
|
||
return jsonify({
|
||
"ok": False,
|
||
"error": f"Datenbankfehler: {exc}",
|
||
}), 503
|
||
finally:
|
||
if conn is not None:
|
||
conn.close()
|
||
|
||
try:
|
||
ahead = urls_ahead_of(domain)
|
||
except Exception:
|
||
ahead = None
|
||
|
||
return jsonify({
|
||
"ok": True,
|
||
"domain": domain,
|
||
"priority": SUGGEST_PRIORITY,
|
||
"urls_ahead": ahead,
|
||
})
|
||
|
||
|
||
@app.get("/api/suggestions")
|
||
def api_suggestions():
|
||
"""Recent suggestions for the UI list."""
|
||
try:
|
||
rows = query("""
|
||
SELECT domain,
|
||
requested_at,
|
||
request_count,
|
||
last_status,
|
||
EXISTS (
|
||
SELECT 1 FROM crawl_queue q
|
||
WHERE q.url = 'https://' || domain_suggestions.domain || '/'
|
||
AND q.status = 'pending'
|
||
) AS still_pending
|
||
FROM domain_suggestions
|
||
ORDER BY requested_at DESC
|
||
LIMIT %s
|
||
""", (MAX_RECENT,))
|
||
except Exception:
|
||
rows = []
|
||
|
||
return jsonify([
|
||
{
|
||
"domain": r["domain"],
|
||
"requested_at": r["requested_at"].isoformat(),
|
||
"request_count": int(r["request_count"]),
|
||
"last_status": r["last_status"],
|
||
"still_pending": bool(r["still_pending"]),
|
||
}
|
||
for r in rows
|
||
])
|
||
|
||
|
||
# ============================================================
|
||
# QUEUE SNAPSHOT (background refresh)
|
||
# ============================================================
|
||
|
||
# The pending/processing counts scan a 13M-row queue (several seconds
|
||
# of index-only scan). They must never run inside a request: a
|
||
# background thread refreshes them every 60 s and /api/queue serves
|
||
# the cached snapshot. The "boosted" count only reads the tiny
|
||
# priority >= SUGGEST_PRIORITY index range and is always fast.
|
||
|
||
QUEUE_REFRESH_SECONDS = int(os.getenv("JONASSEARCH_QUEUE_REFRESH_SECONDS", "60"))
|
||
|
||
_queue_lock = threading.Lock()
|
||
_queue_snapshot = None
|
||
|
||
|
||
def compute_queue_snapshot():
|
||
"""Full queue stats; may take several seconds on the big table."""
|
||
row = query("""
|
||
SELECT
|
||
(SELECT COUNT(*) FROM crawl_queue
|
||
WHERE status = 'pending') AS pending,
|
||
(SELECT COUNT(*) FROM crawl_queue
|
||
WHERE status = 'processing') AS processing,
|
||
(SELECT COUNT(*) FROM crawl_queue
|
||
WHERE status = 'pending'
|
||
AND priority >= %s) AS boosted
|
||
""", (SUGGEST_PRIORITY,))[0]
|
||
|
||
return {
|
||
"pending": int(row["pending"] or 0),
|
||
"processing": int(row["processing"] or 0),
|
||
"boosted": int(row["boosted"] or 0),
|
||
}
|
||
|
||
|
||
def _queue_worker():
|
||
global _queue_snapshot
|
||
|
||
while True:
|
||
try:
|
||
snapshot = compute_queue_snapshot()
|
||
with _queue_lock:
|
||
_queue_snapshot = snapshot
|
||
time.sleep(QUEUE_REFRESH_SECONDS)
|
||
except Exception:
|
||
# DB hiccup: retry shortly; the UI keeps the last snapshot.
|
||
time.sleep(15)
|
||
|
||
|
||
def start_queue_worker():
|
||
thread = threading.Thread(
|
||
target=_queue_worker,
|
||
name="queue-stats",
|
||
daemon=True,
|
||
)
|
||
thread.start()
|
||
|
||
|
||
@app.get("/api/queue")
|
||
def api_queue():
|
||
"""Small live status for the header widgets (cached, see above)."""
|
||
with _queue_lock:
|
||
snapshot = _queue_snapshot
|
||
|
||
if snapshot is not None:
|
||
return jsonify({
|
||
**snapshot,
|
||
"online": True,
|
||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||
})
|
||
|
||
# No snapshot yet (service just started): fall back to the fast
|
||
# boosted-only count instead of blocking the request for seconds.
|
||
try:
|
||
row = query("""
|
||
SELECT COUNT(*) AS boosted
|
||
FROM crawl_queue
|
||
WHERE status = 'pending'
|
||
AND priority >= %s
|
||
""", (SUGGEST_PRIORITY,))[0]
|
||
except Exception:
|
||
return jsonify({"online": False}), 503
|
||
|
||
return jsonify({
|
||
"online": True,
|
||
"pending": None,
|
||
"processing": None,
|
||
"boosted": int(row["boosted"] or 0),
|
||
"timestamp": datetime.now(timezone.utc).isoformat(),
|
||
})
|
||
|
||
|
||
@app.get("/api/health")
|
||
def api_health():
|
||
try:
|
||
query("SELECT 1")
|
||
return jsonify({"status": "ok"})
|
||
except Exception as exc:
|
||
return jsonify({"status": "error", "error": str(exc)}), 503
|
||
|
||
|
||
# ============================================================
|
||
# HTML (inline, no extra static files needed)
|
||
# ============================================================
|
||
|
||
HTML = """
|
||
<!DOCTYPE html>
|
||
<html lang="de">
|
||
<head>
|
||
<meta charset="UTF-8">
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0">
|
||
<title>JonasSearch · Vorschlagsverfeinerer</title>
|
||
<style>
|
||
:root {
|
||
font-family: Inter, system-ui, -apple-system, BlinkMacSystemFont,
|
||
"Segoe UI", sans-serif;
|
||
color-scheme: dark;
|
||
|
||
--bg: #080a0f;
|
||
--panel: #10131a;
|
||
--panel-2: #151922;
|
||
--border: #252b36;
|
||
--text: #f2f4f8;
|
||
--muted: #8b94a5;
|
||
--accent: #7dd3fc;
|
||
--green: #4ade80;
|
||
--red: #fb7185;
|
||
--gold: #facc15;
|
||
}
|
||
|
||
* { box-sizing: border-box; }
|
||
|
||
body {
|
||
margin: 0;
|
||
min-height: 100vh;
|
||
background: radial-gradient(circle at top, #151b27 0, var(--bg) 45%);
|
||
color: var(--text);
|
||
}
|
||
|
||
header {
|
||
max-width: 1100px;
|
||
margin: auto;
|
||
padding: 48px 32px 32px;
|
||
display: flex;
|
||
justify-content: space-between;
|
||
align-items: flex-start;
|
||
}
|
||
|
||
.eyebrow {
|
||
color: var(--accent);
|
||
font-size: 12px;
|
||
font-weight: 800;
|
||
letter-spacing: 0.18em;
|
||
}
|
||
|
||
h1 { margin: 5px 0; font-size: 36px; line-height: 1; }
|
||
header p { margin: 10px 0 0; color: var(--muted); }
|
||
|
||
.status {
|
||
display: flex; align-items: center; gap: 9px;
|
||
color: var(--muted); font-size: 14px;
|
||
}
|
||
|
||
#statusDot {
|
||
width: 9px; height: 9px; border-radius: 50%;
|
||
display: inline-block; background: var(--muted);
|
||
}
|
||
#statusDot.online { background: var(--green); }
|
||
#statusDot.offline { background: var(--red); }
|
||
|
||
main {
|
||
max-width: 1100px;
|
||
margin: auto;
|
||
padding: 0 32px 50px;
|
||
}
|
||
|
||
.hero {
|
||
background: rgba(16, 19, 26, 0.92);
|
||
border: 1px solid var(--border);
|
||
border-radius: 14px;
|
||
box-shadow: 0 15px 50px rgba(0, 0, 0, 0.22);
|
||
padding: 28px;
|
||
margin-bottom: 16px;
|
||
}
|
||
|
||
.hero h2 { margin: 0 0 6px; font-size: 20px; }
|
||
.hero .hint { margin: 0 0 20px; color: var(--muted); font-size: 13px; }
|
||
|
||
.hint code {
|
||
background: var(--panel-2);
|
||
border: 1px solid var(--border);
|
||
border-radius: 6px;
|
||
padding: 2px 6px;
|
||
font-size: 12px;
|
||
color: var(--accent);
|
||
}
|
||
|
||
.input-row { display: flex; gap: 12px; flex-wrap: wrap; }
|
||
|
||
input[type="text"] {
|
||
flex: 1;
|
||
min-width: 260px;
|
||
background: var(--panel-2);
|
||
border: 1px solid var(--border);
|
||
border-radius: 10px;
|
||
padding: 14px 16px;
|
||
color: var(--text);
|
||
font-size: 15px;
|
||
outline: none;
|
||
transition: border-color 0.2s ease;
|
||
}
|
||
|
||
input[type="text"]:focus { border-color: var(--accent); }
|
||
|
||
input[type="text"]::placeholder { color: #5b6473; }
|
||
|
||
button {
|
||
background: linear-gradient(180deg, #38bdf8, #0ea5e9);
|
||
border: none;
|
||
color: #06222f;
|
||
font-weight: 800;
|
||
font-size: 14px;
|
||
padding: 14px 24px;
|
||
border-radius: 10px;
|
||
cursor: pointer;
|
||
transition: transform 0.1s ease, filter 0.2s ease;
|
||
white-space: nowrap;
|
||
}
|
||
|
||
button:hover { filter: brightness(1.1); }
|
||
button:active { transform: scale(0.98); }
|
||
button:disabled { opacity: 0.55; cursor: wait; }
|
||
|
||
.message {
|
||
margin-top: 16px;
|
||
padding: 12px 14px;
|
||
border-radius: 10px;
|
||
font-size: 14px;
|
||
display: none;
|
||
}
|
||
|
||
.message.ok {
|
||
display: block;
|
||
background: rgba(74, 222, 128, 0.09);
|
||
border: 1px solid rgba(74, 222, 128, 0.35);
|
||
color: var(--green);
|
||
}
|
||
|
||
.message.err {
|
||
display: block;
|
||
background: rgba(251, 113, 133, 0.09);
|
||
border: 1px solid rgba(251, 113, 133, 0.35);
|
||
color: var(--red);
|
||
}
|
||
|
||
.message b { color: inherit; }
|
||
|
||
.grid {
|
||
display: grid;
|
||
grid-template-columns: repeat(3, 1fr);
|
||
gap: 16px;
|
||
margin-bottom: 16px;
|
||
}
|
||
|
||
.card {
|
||
background: rgba(16, 19, 26, 0.92);
|
||
border: 1px solid var(--border);
|
||
border-radius: 14px;
|
||
padding: 20px 24px;
|
||
box-shadow: 0 15px 50px rgba(0, 0, 0, 0.22);
|
||
}
|
||
|
||
.card span {
|
||
display: block;
|
||
color: var(--muted);
|
||
font-size: 11px;
|
||
font-weight: 700;
|
||
letter-spacing: 0.08em;
|
||
}
|
||
|
||
.card strong {
|
||
display: block;
|
||
margin-top: 8px;
|
||
font-size: 28px;
|
||
letter-spacing: -0.04em;
|
||
}
|
||
|
||
.card.gold strong { color: var(--gold); }
|
||
|
||
.list-card {
|
||
background: rgba(16, 19, 26, 0.92);
|
||
border: 1px solid var(--border);
|
||
border-radius: 14px;
|
||
padding: 24px;
|
||
box-shadow: 0 15px 50px rgba(0, 0, 0, 0.22);
|
||
}
|
||
|
||
.list-card h2 { margin: 0 0 4px; font-size: 18px; }
|
||
.list-card .sub { margin: 0 0 18px; color: var(--muted); font-size: 13px; }
|
||
|
||
.suggestion {
|
||
display: flex;
|
||
justify-content: space-between;
|
||
align-items: center;
|
||
gap: 14px;
|
||
padding: 12px 0;
|
||
border-bottom: 1px solid var(--border);
|
||
}
|
||
|
||
.suggestion:last-child { border-bottom: none; }
|
||
|
||
.suggestion .domain {
|
||
font-weight: 600;
|
||
font-size: 14px;
|
||
word-break: break-all;
|
||
}
|
||
|
||
.suggestion .meta {
|
||
color: var(--muted);
|
||
font-size: 12px;
|
||
white-space: nowrap;
|
||
}
|
||
|
||
.badge {
|
||
display: inline-block;
|
||
margin-left: 8px;
|
||
padding: 2px 9px;
|
||
border-radius: 100px;
|
||
font-size: 11px;
|
||
font-weight: 700;
|
||
}
|
||
|
||
.badge.queued {
|
||
background: rgba(250, 204, 21, 0.12);
|
||
color: var(--gold);
|
||
border: 1px solid rgba(250, 204, 21, 0.3);
|
||
}
|
||
|
||
.badge.crawled {
|
||
background: rgba(74, 222, 128, 0.12);
|
||
color: var(--green);
|
||
border: 1px solid rgba(74, 222, 128, 0.3);
|
||
}
|
||
|
||
.empty { color: var(--muted); font-size: 14px; padding: 14px 0; }
|
||
|
||
footer {
|
||
max-width: 1100px;
|
||
margin: auto;
|
||
padding: 0 32px 30px;
|
||
color: var(--muted);
|
||
font-size: 12px;
|
||
}
|
||
|
||
@media (max-width: 760px) {
|
||
header { padding: 30px 18px 24px; flex-direction: column; gap: 20px; }
|
||
main { padding: 0 18px 30px; }
|
||
.grid { grid-template-columns: 1fr; }
|
||
footer { padding: 0 18px 30px; }
|
||
}
|
||
</style>
|
||
</head>
|
||
|
||
<body>
|
||
|
||
<header>
|
||
<div>
|
||
<div class="eyebrow">SERVERJONASSEARCH</div>
|
||
<h1>Vorschlagsverfeinerer</h1>
|
||
<p>Gamma 0.1 · Domain vorschlagen und die Warteschlange überholen</p>
|
||
</div>
|
||
|
||
<div class="status">
|
||
<span id="statusDot"></span>
|
||
<span id="statusText">Verbinde…</span>
|
||
</div>
|
||
</header>
|
||
|
||
<main>
|
||
|
||
<section class="hero">
|
||
<h2>Domain priorisieren</h2>
|
||
<p class="hint">
|
||
Eingabe als Domain oder komplette URL — z. B.
|
||
<code>example.com</code> oder
|
||
<code>https://www.example.com/seite</code>.
|
||
Der Vorschlag landet mit sehr hoher Priorität direkt vorne in der
|
||
Crawl-Queue.
|
||
</p>
|
||
|
||
<form id="suggestForm">
|
||
<div class="input-row">
|
||
<input type="text" id="domainInput"
|
||
placeholder="example.com" autocomplete="off"
|
||
spellcheck="false">
|
||
<button type="submit" id="submitBtn">🚀 Priorisieren</button>
|
||
</div>
|
||
</form>
|
||
|
||
<div id="message" class="message"></div>
|
||
</section>
|
||
|
||
<section class="grid">
|
||
|
||
<article class="card" title="Letzte Messung des Crawlers">
|
||
<span>QUEUE PENDING</span>
|
||
<strong id="queuePending">—</strong>
|
||
</article>
|
||
|
||
<article class="card">
|
||
<span>IN BEARBEITUNG</span>
|
||
<strong id="queueProcessing">—</strong>
|
||
</article>
|
||
|
||
<article class="card gold">
|
||
<span>PRIORISIERTE DOMAINS</span>
|
||
<strong id="queueBoosted">—</strong>
|
||
</article>
|
||
|
||
</section>
|
||
|
||
<section class="list-card">
|
||
<h2>Letzte Vorschläge</h2>
|
||
<p class="sub">Die zuletzt priorisierten Domains und ihr Queue-Status</p>
|
||
<div id="suggestions">
|
||
<div class="empty">Lade…</div>
|
||
</div>
|
||
</section>
|
||
|
||
</main>
|
||
|
||
<footer>
|
||
JonasSearch Gamma 0.1 · Vorschlagsverfeinerer · Port {{ port }}
|
||
</footer>
|
||
|
||
<script>
|
||
const $ = id => document.getElementById(id);
|
||
|
||
function fmt(n) {
|
||
return (n === null || n === undefined)
|
||
? "—"
|
||
: new Intl.NumberFormat("de-DE").format(n);
|
||
}
|
||
|
||
function setStatus(online) {
|
||
$("statusDot").className = online ? "online" : "offline";
|
||
$("statusText").textContent = online ? "Online" : "Offline";
|
||
}
|
||
|
||
function showMessage(kind, html) {
|
||
const box = $("message");
|
||
box.className = "message " + kind;
|
||
box.innerHTML = html;
|
||
}
|
||
|
||
function escapeHTML(s) {
|
||
return String(s)
|
||
.replaceAll("&", "&")
|
||
.replaceAll("<", "<")
|
||
.replaceAll(">", ">")
|
||
.replaceAll('"', """)
|
||
.replaceAll("'", "'");
|
||
}
|
||
|
||
async function refreshQueue() {
|
||
try {
|
||
const r = await fetch("/api/queue", { cache: "no-store" });
|
||
const d = await r.json();
|
||
setStatus(d.online !== false);
|
||
$("queuePending").textContent = fmt(d.pending);
|
||
$("queueProcessing").textContent = fmt(d.processing);
|
||
$("queueBoosted").textContent = fmt(d.boosted);
|
||
} catch {
|
||
setStatus(false);
|
||
}
|
||
}
|
||
|
||
async function refreshSuggestions() {
|
||
try {
|
||
const r = await fetch("/api/suggestions", { cache: "no-store" });
|
||
const items = await r.json();
|
||
const box = $("suggestions");
|
||
|
||
if (!items.length) {
|
||
box.innerHTML =
|
||
'<div class="empty">Noch keine Vorschläge — ' +
|
||
'sei der Erste! 🚀</div>';
|
||
return;
|
||
}
|
||
|
||
box.innerHTML = items.map(x => {
|
||
const when = new Date(x.requested_at)
|
||
.toLocaleString("de-DE");
|
||
const times = x.request_count > 1
|
||
? ` · ${x.request_count}× vorgeschlagen` : "";
|
||
const badge = x.still_pending
|
||
? '<span class="badge queued">in Queue</span>'
|
||
: '<span class="badge crawled">verarbeitet</span>';
|
||
return `
|
||
<div class="suggestion">
|
||
<div class="domain">
|
||
${escapeHTML(x.domain)}${badge}
|
||
</div>
|
||
<div class="meta">
|
||
${escapeHTML(when)}${escapeHTML(times)}
|
||
</div>
|
||
</div>
|
||
`;
|
||
}).join("");
|
||
} catch {
|
||
$("suggestions").innerHTML =
|
||
'<div class="empty">Vorschläge konnten nicht geladen werden.</div>';
|
||
}
|
||
}
|
||
|
||
$("suggestForm").addEventListener("submit", async (event) => {
|
||
event.preventDefault();
|
||
|
||
const btn = $("submitBtn");
|
||
const domain = $("domainInput").value.trim();
|
||
|
||
btn.disabled = true;
|
||
|
||
try {
|
||
const r = await fetch("/api/suggest", {
|
||
method: "POST",
|
||
headers: { "Content-Type": "application/json" },
|
||
body: JSON.stringify({ domain })
|
||
});
|
||
|
||
const d = await r.json();
|
||
|
||
if (d.ok) {
|
||
const ahead = d.urls_ahead === null || d.urls_ahead === undefined
|
||
? ""
|
||
: ` — noch <b>${fmt(d.urls_ahead)}</b> Vorschläge vor dir`;
|
||
showMessage("ok",
|
||
`✅ <b>${escapeHTML(d.domain)}</b> steht jetzt ` +
|
||
`mit Priorität ${d.priority} vorne in der Queue` +
|
||
ahead + ".");
|
||
$("domainInput").value = "";
|
||
refreshSuggestions();
|
||
} else {
|
||
showMessage("err",
|
||
"⚠️ " + escapeHTML(d.error || "Unbekannter Fehler"));
|
||
}
|
||
} catch {
|
||
showMessage("err",
|
||
"⚠️ Server nicht erreichbar — ist der Dienst gestartet?");
|
||
} finally {
|
||
btn.disabled = false;
|
||
refreshQueue();
|
||
}
|
||
});
|
||
|
||
refreshQueue();
|
||
refreshSuggestions();
|
||
setInterval(refreshQueue, 15000);
|
||
setInterval(refreshSuggestions, 30000);
|
||
</script>
|
||
|
||
</body>
|
||
</html>
|
||
"""
|
||
|
||
|
||
if __name__ == "__main__":
|
||
host = os.getenv("HOST", "0.0.0.0")
|
||
|
||
# Bookkeeping table for first run.
|
||
try:
|
||
ensure_suggestions_table_standalone()
|
||
except Exception as exc:
|
||
print(f"WARN: could not ensure domain_suggestions table: {exc}")
|
||
|
||
start_queue_worker()
|
||
|
||
print("=" * 70)
|
||
print(" JonasSearch Vorschlagsverfeinerer Gamma 0.1")
|
||
print("=" * 70)
|
||
print(f"Statserver : Port {STATSERVER_PORT}")
|
||
print(f"HTTP : {host}:{PORT} (Statserver + 1)")
|
||
print(f"DB : {DB_HOST}:{DB_PORT}/{DB_NAME}")
|
||
print(f"Priorität : {SUGGEST_PRIORITY}")
|
||
print("=" * 70)
|
||
|
||
app.run(
|
||
host=host,
|
||
port=PORT,
|
||
debug=False,
|
||
threaded=True,
|
||
)
|