Serverjonas/search
2
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

add pictures

Browse Source
This commit is contained in:
jhkaktus
2026-10-17 18:32:59 +02:00
parent c5f963f08f
commit aa0bc4e960
10 changed files with 1273 additions and 64 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crawler.py
+281 -33
View File
@@ -7,7 +7,11 @@ PostgreSQL-based parallel crawler.
Features:
- PostgreSQL queue with row-level locking (FOR UPDATE SKIP LOCKED)
- Multiple crawler workers can safely share one queue
- Stores title, headings, cleaned text and links
- Stores title, headings, cleaned text, links and image URLs
- Image URLs from <img src/srcset>, <picture>/<source>, og:image,
twitter:image and link rel="image_src" are resolved to absolute
http(s) URLs, deduplicated and stored in pages.pictures
(migration: add_pages_pictures.sql)
- Search vector is maintained by the PostgreSQL trigger
- URL deduplication via UNIQUE(url)
- Deadlock-safe parallel queue inserts (deterministic ordering,
@@ -37,7 +41,7 @@ Features:
release) and --update-deployment N (bump generation + required_version)
Expected PostgreSQL schema:
pages
pages (incl. pictures text[], see add_pages_pictures.sql)
crawl_queue
crawler_stats
@@ -150,7 +154,7 @@ USER_AGENT = os.getenv(
# change; completely independent from the deployment generation (the
# generation is a global deployment counter in PostgreSQL, the version
# identifies the crawler code).
CRAWLER_VERSION = int(os.getenv("JONASSEARCH_CRAWLER_VERSION", "3"))
CRAWLER_VERSION = int(os.getenv("JONASSEARCH_CRAWLER_VERSION", "5"))
# Exit codes. 42 is reserved exclusively for "I must be updated" so
# the launcher can distinguish update, crash and normal exit.
@@ -235,6 +239,15 @@ MAX_HEADING_LENGTH = 1000
MAX_TEXT_LENGTH = 2_000_000
MAX_LINKS = 10_000
# Image URLs per page. A page with thousands of <img>/srcset entries
# would otherwise bloat pages.pictures for no search value; the first
# images in document order are kept.
MAX_PICTURES = 1_000
# Hard cap for one image URL (text[] has no B-tree limit, but absurd
# data: URLs / tracking parameters are not worth storing).
MAX_IMAGE_URL_LENGTH = 2_000
# Cap for the text that feeds the full-text index. PostgreSQL
# hard-limits a single tsvector value to 1,048,575 bytes (not
# configurable), and the trigger pages_search_vector_update() builds
@@ -281,11 +294,23 @@ QUEUE_CLAIM_BATCH = int(os.getenv("JONASSEARCH_CLAIM_BATCH", "25"))
# Maximum number of candidates examined per claim (batch * rounds).
# Keeps the common case (a ready host in the first batch) cheap while
# still finding crawlable hosts behind a wall of blocked ones.
# still finding crawlable hosts behind a small wall of blocked ones.
# Larger walls are handled in SQL: hosts that are known to be busy are
# excluded from the candidate query itself (see busy_hosts()), so a
# blocked host with thousands of queued URLs cannot hide the rest of
# the queue behind it.
QUEUE_CLAIM_SCAN_LIMIT = int(
os.getenv("JONASSEARCH_CLAIM_SCAN_LIMIT", "200")
)
# Upper bound for the host list that is passed into the claim query.
# Longest-running blocks are kept first, because those form the wall at
# the head of the priority order; hosts beyond the cap are still
# skipped by the per-candidate check, they are just not pre-filtered.
QUEUE_CLAIM_BUSY_HOST_LIMIT = int(
os.getenv("JONASSEARCH_CLAIM_BUSY_HOST_LIMIT", "1000")
)
# Cap for the "nothing is crawlable right now" sleep, so a host blocked
# for a long time cannot park the whole worker; after the cap the queue
# is simply re-scanned.
@@ -738,11 +763,18 @@ def claim_next_queue_item(conn, scheduler, robots):
Candidates are read in priority order (same ORDER BY as before) in
small batches with FOR UPDATE SKIP LOCKED, so several crawler
processes can still share one queue without waiting for each
other. For every candidate the HostScheduler decides whether its
host may be contacted right now (per-host crawl delay, active
block backoff). The first ready candidate is claimed; hosts that
would have to wait are skipped, so the worker automatically works
on another domain while a rate-limited or blocked host cools down.
other. Hosts that the HostScheduler already knows to be busy
(per-host crawl delay, active block backoff) are excluded by the
query itself. For the remaining candidates the scheduler decides
whether a host may be contacted right now. The first ready
candidate is claimed, so the worker automatically works on another
domain while a rate-limited or blocked host cools down.
The SQL-level host filter is what keeps a blocked host from
starving the whole queue: blocked rows stay 'pending' at the top
of the priority order, and without the filter a wall of them could
fill every scan window forever while tens of thousands of other
hosts wait behind it.
Returns (item, wait_hint):
item (id, url, source_url) of the claimed URL, or None
@@ -757,27 +789,28 @@ def claim_next_queue_item(conn, scheduler, robots):
wait_hint = None
item = None
busy_hosts = scheduler.busy_hosts(now)
# Longest waits first: those hosts form the wall at the head of the
# priority order and are the most valuable ones to exclude.
excluded_hosts = [
host for host, _wait in sorted(
busy_hosts.items(), key=lambda pair: pair[1], reverse=True
)
][:QUEUE_CLAIM_BUSY_HOST_LIMIT]
while item is None and scanned < QUEUE_CLAIM_SCAN_LIMIT:
with conn.cursor() as cur:
if seen_ids:
cur.execute("""
SELECT id, url, source_url
FROM crawl_queue
WHERE status = 'pending'
AND NOT (id = ANY(%s))
ORDER BY priority DESC, discovered_at ASC, id ASC
FOR UPDATE SKIP LOCKED
LIMIT %s
""", (seen_ids, QUEUE_CLAIM_BATCH))
else:
cur.execute("""
SELECT id, url, source_url
FROM crawl_queue
WHERE status = 'pending'
ORDER BY priority DESC, discovered_at ASC, id ASC
FOR UPDATE SKIP LOCKED
LIMIT %s
""", (QUEUE_CLAIM_BATCH,))
cur.execute("""
SELECT id, url, source_url
FROM crawl_queue
WHERE status = 'pending'
AND NOT (id = ANY(%s::bigint[]))
AND lower(split_part(split_part(url, '://', 2), '/', 1))
<> ALL(%s::text[])
ORDER BY priority DESC, discovered_at ASC, id ASC
FOR UPDATE SKIP LOCKED
LIMIT %s
""", (seen_ids, excluded_hosts, QUEUE_CLAIM_BATCH))
candidates = cur.fetchall()
if not candidates:
@@ -815,6 +848,18 @@ def claim_next_queue_item(conn, scheduler, robots):
if len(candidates) < QUEUE_CLAIM_BATCH:
break
if item is None and scanned == 0 and excluded_hosts:
# The busy-host filter removed every pending URL from the query
# before it could return a single candidate. Report when the
# soonest host is crawlable again - but only if there really are
# pending URLs; otherwise the queue is simply empty.
with conn.cursor() as cur:
cur.execute(
"SELECT 1 FROM crawl_queue WHERE status = 'pending' LIMIT 1"
)
if cur.fetchone() is not None:
wait_hint = min(busy_hosts.values())
return item, wait_hint
@@ -1584,6 +1629,48 @@ def normalize_url(url, base_url=None):
return normalized
def normalize_image_url(raw, base_url):
"""
Turn an image reference from HTML into an absolute http(s) URL.
Unlike normalize_url() this does NOT reject image file extensions
(that list exists to keep binary resources out of the page queue).
Relative paths and protocol-relative URLs are resolved against the
page URL; fragments are dropped because they do not identify a
different image resource. data:, javascript:, ftp: and other
non-http(s) sources are rejected, as are empty/whitespace values,
non-public hosts and URLs that are unreasonably long.
"""
if not isinstance(raw, str):
return None
raw = unescape(raw).strip()
if not raw:
return None
try:
absolute = urljoin(base_url, raw)
parsed = urlparse(absolute)
except ValueError:
return None
if parsed.scheme.lower() not in ALLOWED_SCHEMES:
return None
if not parsed.hostname:
return None
if is_non_crawlable_host(parsed.hostname):
return None
# Fragments never select a different image file.
clean, _fragment = urldefrag(absolute)
if any(char.isspace() for char in clean):
return None
if len(clean) > MAX_IMAGE_URL_LENGTH:
return None
return clean
# ============================================================
# ROBOTS
# ============================================================
@@ -1688,6 +1775,10 @@ class HostScheduler:
self.last_request = {}
self.blocked_until = {}
self.block_count = {}
# Delay last used for a host (robots.txt crawl-delay or
# DEFAULT_DELAY), so busy_hosts() can report the host's schedule
# without looking at any URL again.
self.host_delay = {}
@staticmethod
def _host(url):
@@ -1703,12 +1794,54 @@ class HostScheduler:
host = self._host(url)
if not host:
return 0.0
self.host_delay[host] = delay
last = self.last_request.get(host)
# A host that was never contacted has no pending crawl delay:
# monotonic() has an arbitrary origin, so treating "never" as 0
# and adding a large robots delay would wrongly block the host
# for a long time on a freshly booted machine.
rate_until = last + delay if last is not None else 0.0
next_allowed = max(
self.last_request.get(host, 0.0) + delay,
rate_until,
self.blocked_until.get(host, 0.0),
)
return max(0.0, next_allowed - now)
def busy_hosts(self, now=None):
"""
Map of hosts that cannot be contacted right now to the number
of seconds they still have to wait.
The queue claim passes these hosts into SQL: skipping busy hosts
only within the small candidate window would let a wall of
blocked URLs at the head of the priority order hide every other
crawlable host in the queue (the wall stays pending at the top,
so each rescan looks at the very same rows forever).
Expired entries are pruned here because this runs once per
claim - the per-process state then cannot grow without bound on
a queue with millions of distinct hosts.
"""
if now is None:
now = time.monotonic()
busy = {}
for host, last in list(self.last_request.items()):
next_allowed = last + self.host_delay.get(host, 0.0)
if next_allowed > now:
busy[host] = next_allowed - now
else:
del self.last_request[host]
for host, until in list(self.blocked_until.items()):
if until > now:
busy[host] = max(busy.get(host, 0.0), until - now)
else:
del self.blocked_until[host]
return busy
def is_ready(self, url, delay, now=None):
return self.wait_time(url, delay, now) <= 0.0
@@ -1725,14 +1858,16 @@ class HostScheduler:
stays untouched, so the item is retried exactly when the delay
has passed.
"""
host = self._host(url)
self.host_delay[host] = delay
wait_for = self.wait_time(url, delay)
if wait_for > MAX_INLINE_RATE_WAIT:
return False
if wait_for > 0:
if wait_for >= 1.0:
logging.info("[RATE ] %s: %.1fs", self._host(url), wait_for)
logging.info("[RATE ] %s: %.1fs", host, wait_for)
time.sleep(wait_for)
self.last_request[self._host(url)] = time.monotonic()
self.last_request[host] = time.monotonic()
return True
def _register_block(self, host, seconds):
@@ -1926,6 +2061,104 @@ def clean_text_bs4(soup):
return text[:MAX_TEXT_LENGTH]
# Common attributes used for lazy-loaded images. They are not
# standardized, but these three are reliable enough in practice.
IMAGE_LAZY_SRC_ATTRS = ("data-src", "data-lazy-src", "data-original")
# srcset appears on <img> and on <source> inside <picture>.
IMAGE_SRCSET_ATTRS = ("srcset", "data-srcset")
# Social/social-preview image metadata.
IMAGE_META_KEYS = frozenset({
"og:image",
"og:image:url",
"og:image:secure_url",
"twitter:image",
"twitter:image:src",
})
def _srcset_candidates(value):
"""
Yield the URL part of every srcset candidate.
srcset format: "img.jpg 1x, img@2x.jpg 2x" (or a width descriptor).
Parsed like the HTML spec: whitespace separates URL and descriptor,
a trailing comma ends a candidate. This keeps commas inside URLs
(including data: URLs, which are rejected later) from being split
into bogus relative candidates.
"""
if not value:
return
tokens = str(value).split()
index = 0
while index < len(tokens):
candidate = tokens[index]
index += 1
if not candidate:
continue
ends_candidate = candidate.endswith(",")
candidate = candidate.rstrip(",").strip()
if not candidate:
continue
if not ends_candidate:
# The next token is the descriptor ("2x", "640w"), if any.
index += 1
yield candidate
def extract_pictures(soup, base_url):
"""
All absolute, deduplicated image URLs of a parsed document.
Sources: <img src>, lazy-loading data-* attributes, <img srcset>,
<picture> via <source srcset>, Open-Graph/Twitter preview images and
<link rel="image_src">. Order of first appearance is preserved.
"""
pictures = []
seen = set()
def add(raw):
if len(pictures) >= MAX_PICTURES:
return
normalized = normalize_image_url(raw, base_url)
if not normalized or normalized in seen:
return
seen.add(normalized)
pictures.append(normalized)
for tag in soup.find_all("img"):
add(tag.get("src"))
for attr in IMAGE_LAZY_SRC_ATTRS:
add(tag.get(attr))
for attr in IMAGE_SRCSET_ATTRS:
for candidate in _srcset_candidates(tag.get(attr)):
add(candidate)
for tag in soup.find_all("source"):
for attr in IMAGE_SRCSET_ATTRS:
for candidate in _srcset_candidates(tag.get(attr)):
add(candidate)
for tag in soup.find_all("meta"):
key = (tag.get("property") or tag.get("name") or "").strip().lower()
if key in IMAGE_META_KEYS:
add(tag.get("content"))
for tag in soup.find_all("link"):
rel = tag.get("rel") or []
if isinstance(rel, str):
rel = rel.lower().split()
if any(str(value).lower() == "image_src" for value in rel):
add(tag.get("href"))
return pictures
def parse_html(html, base_url):
if BeautifulSoup is None:
raise RuntimeError(
@@ -1965,6 +2198,10 @@ def parse_html(html, base_url):
if len(links) >= MAX_LINKS:
break
# Extract images BEFORE clean_text_bs4() mutates the tree (it
# removes navigation, noscript, svg, ... elements).
pictures = extract_pictures(soup, base_url)
content = clean_text_bs4(soup)
domain = urlparse(base_url).hostname or ""
@@ -1981,6 +2218,7 @@ def parse_html(html, base_url):
"headings": headings,
"content": content,
"links": links,
"pictures": pictures,
"domain": domain,
"tld": tld,
}
@@ -2006,6 +2244,11 @@ def save_page(conn, url, source_url, status, content_type, parsed, crawl_ms):
"headings": clean_text_list(parsed["headings"]),
"content": clean_text(parsed["content"]),
"links": clean_text_list(parsed["links"]),
# parse_html() already deduplicates; repeat it here so a caller
# with a hand-built parsed dict can never write duplicates.
"pictures": list(dict.fromkeys(
clean_text_list(parsed.get("pictures") or [])
))[:MAX_PICTURES],
"domain": clean_text(parsed["domain"]),
"tld": clean_text(parsed["tld"]),
}
@@ -2037,6 +2280,7 @@ def save_page(conn, url, source_url, status, content_type, parsed, crawl_ms):
headings,
content,
links,
pictures,
domain,
tld,
source_url,
@@ -2047,7 +2291,7 @@ def save_page(conn, url, source_url, status, content_type, parsed, crawl_ms):
crawl_time_ms
)
VALUES (
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s,
COALESCE(
(SELECT discovered_at
FROM crawl_queue
@@ -2062,6 +2306,7 @@ def save_page(conn, url, source_url, status, content_type, parsed, crawl_ms):
headings = EXCLUDED.headings,
content = EXCLUDED.content,
links = EXCLUDED.links,
pictures = EXCLUDED.pictures,
domain = EXCLUDED.domain,
tld = EXCLUDED.tld,
source_url = EXCLUDED.source_url,
@@ -2075,6 +2320,7 @@ def save_page(conn, url, source_url, status, content_type, parsed, crawl_ms):
safe_parsed["headings"],
index_content,
safe_parsed["links"],
safe_parsed["pictures"],
safe_parsed["domain"],
safe_parsed["tld"],
source_url,
@@ -2149,7 +2395,8 @@ def worker(worker_id):
sleep_for = min(wait_hint, QUEUE_MAX_IDLE_SLEEP)
logging.info(
"[WAIT ] All candidate hosts busy or blocked; "
"worker sleeps %.1fs",
"soonest host free in %.1fs, worker sleeps %.1fs",
wait_hint,
sleep_for,
)
else:
@@ -2269,6 +2516,7 @@ def worker(worker_id):
logging.info("[DOMAIN] %s", parsed["domain"])
logging.info("[HEADS ] %d", len(parsed["headings"]))
logging.info("[LINKS ] %d gefunden", len(parsed["links"]))
logging.info("[IMGS ] %d gefunden", len(parsed["pictures"]))
try:
safe_commit(