From f0ff7c894b3b6b73e41cd297bd8f1a1d24e83587 Mon Sep 17 00:00:00 2001
From: Victor Giers <mail@victorgiers.com>
Date: Tue, 16 Jun 2026 19:05:57 +0200
Subject: [PATCH] Add fallback rerank logic for web search

---
 backend/agent/tools/web.py | 26 ++++++++++++++++++--------
 backend/websearch.py       | 20 ++++++++++++++++----
 2 files changed, 34 insertions(+), 12 deletions(-)

diff --git a/backend/agent/tools/web.py b/backend/agent/tools/web.py
index 8141850..02f0799 100644
--- a/backend/agent/tools/web.py
+++ b/backend/agent/tools/web.py
@@ -14,6 +14,7 @@ from ...websearch import (
     DEFAULT_HEADERS,
     HTTP_LIMITS,
     HTTP_TIMEOUT,
+    fallback_rerank,
     fetch_website_snapshot,
     render_recent_context,
     rerank,
@@ -236,13 +237,22 @@ async def web_rerank_handler(arguments: Dict[str, Any], _context: ToolExecutionC
         if _is_noise_result(url, title):
             continue
         docs.append((url, text))
-    ranked = await rerank(
-        arguments["prompt"],
-        docs,
-        model=arguments.get("model") or "",
-        context_excerpt=arguments.get("context_excerpt") or "",
-        embed_model=arguments.get("rerank_model"),
-    )
+    context_excerpt = arguments.get("context_excerpt") or ""
+    try:
+        ranked = await rerank(
+            arguments["prompt"],
+            docs,
+            model=arguments.get("model") or "",
+            context_excerpt=context_excerpt,
+            embed_model=arguments.get("rerank_model"),
+        )
+    except Exception as exc:
+        ranked = fallback_rerank(
+            arguments["prompt"],
+            docs,
+            context_excerpt,
+            reason=f"rerank_exception:{type(exc).__name__}",
+        )
     maximum = max(1, min(int(arguments.get("maximum_results") or 6), 12))
     minimum_score = float(arguments.get("minimum_score") or 0)
     selected = []
@@ -341,7 +351,7 @@ def register_web_tools(registry: NativeToolProvider) -> None:
     ))
     registry.register(ToolDefinition(
         name="heimgeist.web_rerank",
-        description="Rank extracted web pages using Heimgeist's embedding-based reranker.",
+        description="Rank extracted web pages using Heimgeist's embedding-based reranker with lexical fallback.",
         input_schema={
             "type": "object",
             "properties": {
diff --git a/backend/websearch.py b/backend/websearch.py
index 7a79188..fd47ac8 100644
--- a/backend/websearch.py
+++ b/backend/websearch.py
@@ -587,13 +587,18 @@ async def rerank(
                 embed_model = alt
             else:
                 print(f"[web] embed() FAILED (models tried={tried + [alt]}, meta={meta or meta2})")
-                return [(u, t, 0.0) for (u, t) in docs]
+                return fallback_rerank(
+                    prompt,
+                    docs,
+                    context_excerpt,
+                    reason=f"model_failed:{embed_model}",
+                )
 
     # split q vs passages and update cache
     q_emb = embeddings[0] if embeddings else []
     if not q_emb:
         print("[web] embed() empty query vector — aborting rerank")
-        return [(u, t, 0.0) for (u, t) in docs]
+        return fallback_rerank(prompt, docs, context_excerpt, reason="empty_query_vector")
 
     # positions >=1 correspond to passages (only those that weren’t cached)
     for pos, emb_vec in enumerate(embeddings[1:], start=1):
@@ -605,6 +610,9 @@ async def rerank(
 
     # build aligned passage vectors
     p_emb_list: List[List[float]] = [emb_cache.get(k, []) for k in keys]
+    if docs and not any(len(vec) for vec in p_emb_list):
+        print("[web] embed() empty passage vectors — aborting rerank")
+        return fallback_rerank(prompt, docs, context_excerpt, reason="empty_passage_vectors")
 
     # logging
     q_dim = len(q_emb)
@@ -796,8 +804,12 @@ async def enrich_prompt(
             pass
     except Exception:
         print("[web] ERROR in rerank:\n" + traceback.format_exc())
-        print(f"[web] enrich_prompt total: {time.perf_counter() - start_all:.3f}s")
-        return _no_results_enriched("rerank_failed", queries)
+        ranked = fallback_rerank(
+            user_prompt,
+            docs,
+            context_excerpt,
+            reason="rerank_exception",
+        )
 
     # 5) build prompt
     try: