Improve agent reliability by implementing evidence-first chat grounding, expanding search vocabulary, refining research question formulation, and adding comprehensive query generation tests.

2026-06-15 20:55:47 +02:00
parent ba51bb9a1b
commit 9c368b97c5
3 changed files with 72 additions and 7 deletions
--- a/backend/agent/tools/web.py
+++ b/backend/agent/tools/web.py
@@ -66,6 +66,7 @@ GENERIC_QUERY_TERMS = {
    "events",
    "for",
    "from",
+    "got",
    "heute",
    "headline",
    "headlines",
@@ -77,6 +78,7 @@ GENERIC_QUERY_TERMS = {
    "mean",
    "meaning",
    "meant",
+    "more",
    "nachrichten",
    "nah",
    "news",
@@ -87,6 +89,8 @@ GENERIC_QUERY_TERMS = {
    "outlet",
    "outlets",
    "recent",
+    "quote",
+    "quoted",
    "said",
    "say",
    "search",
--- a/backend/agent/workflows/builtins.py
+++ b/backend/agent/workflows/builtins.py
@@ -35,11 +35,18 @@ def edge(source: str, target: str, source_handle: str = "output", suffix: str =
    }


-def chat_arguments(context_blocks: List[Dict[str, str]] | None = None) -> Dict[str, Any]:
+EVIDENCE_FIRST_SYSTEM_PROMPT = (
+    "Answer the current user request using the supplied retrieved context as the primary evidence. "
+    "If retrieved context conflicts with older chat history, prefer the retrieved context and explain the correction briefly. "
+    "Do not recommend that the user consult external sources when relevant retrieved sources are already present."
+)
+
+
+def chat_arguments(context_blocks: List[Dict[str, str]] | None = None, system_prompt: str = "") -> Dict[str, Any]:
    return {
        "model": {"$ref": "run.chat_model"},
        "messages": {"$ref": "run.messages"},
-        "system_prompt": "",
+        "system_prompt": system_prompt,
        "context_blocks": context_blocks or [],
        "attachments": {"$ref": "run.attachments"},
        "vision_model": {"$ref": "run.vision_model"},
@@ -102,7 +109,7 @@ def web_graph() -> Dict[str, Any]:
                "prompt": {"$ref": "input.prompt"}, "pages": {"$ref": "nodes.fetch.output.pages"}, "model": {"$ref": "run.chat_model"},
                "rerank_model": None, "context_excerpt": "", "maximum_results": 6, "minimum_score": 55,
            }}),
-            node("chat", "tool", 1260, 100, {"tool": "heimgeist.chat", "arguments": chat_arguments([{"$ref": "nodes.rerank.output"}])}),
+            node("chat", "tool", 1260, 100, {"tool": "heimgeist.chat", "arguments": chat_arguments([{"$ref": "nodes.rerank.output"}], EVIDENCE_FIRST_SYSTEM_PROMPT)}),
            node("output", "output", 1540, 100, {"value": {"$ref": "nodes.chat.output"}}),
        ],
        [edge("input", "queries"), edge("queries", "search"), edge("search", "fetch"), edge("fetch", "rerank"), edge("rerank", "chat"), edge("chat", "output")],
@@ -144,7 +151,7 @@ KNOWLEDGE_WEB = graph(
        }}),
        node("merge", "merge", 1300, 130, {"values": [{"$ref": "nodes.knowledge.output"}, {"$ref": "nodes.rerank.output"}], "deduplicate_by": "url"}),
        node("limit", "limit", 1520, 130, {"value": {"$ref": "nodes.merge.output"}, "maximum_chars": 22000}),
-        node("chat", "tool", 1760, 130, {"tool": "heimgeist.chat", "arguments": chat_arguments([{"$ref": "nodes.limit.output"}])}),
+        node("chat", "tool", 1760, 130, {"tool": "heimgeist.chat", "arguments": chat_arguments([{"$ref": "nodes.limit.output"}], EVIDENCE_FIRST_SYSTEM_PROMPT)}),
        node("output", "output", 2040, 130, {"value": {"$ref": "nodes.chat.output"}}),
    ],
    [
@@ -195,8 +202,16 @@ RESEARCH = graph(
    [
        node("input", "input", 0, 180),
        node("analyze", "prompt", 220, 180, {
-            "model_source": "chat_model", "system_template": "Extract the core research question.",
-            "user_template": "{{input.prompt}}", "output_mode": "text", "temperature": 0.1,
+            "model_source": "chat_model",
+            "system_template": (
+                "Rewrite the user's current request as one concrete web research question. "
+                "Resolve follow-ups, pronouns, and phrases like 'that agreement' using the recent conversation. "
+                "If the request follows a current-news answer, preserve the current-news entities and claims from that answer. "
+                "Do not introduce historical topics or named agreements unless they appear in the current request or recent conversation. "
+                "Return one concise search-ready question only."
+            ),
+            "user_template": "Current request:\n{{input.prompt}}\n\nRecent conversation:\n{{run.messages}}",
+            "output_mode": "text", "temperature": 0.1,
        }),
        node("queries1", "tool", 470, 80, {"tool": "heimgeist.web_generate_queries", "arguments": {
            "prompt": {"$ref": "nodes.analyze.output.content"}, "model": {"$ref": "run.chat_model"}, "messages": {"$ref": "run.messages"},
@@ -230,7 +245,7 @@ RESEARCH = graph(
        node("empty2", "template", 1910, -80, {"template": "", "values": {}}),
        node("merge", "merge", 2820, 80, {"values": [{"$ref": "nodes.rank1.output"}, {"$ref": "nodes.rank2.output"}], "deduplicate_by": "url", "allow_missing": True}),
        node("limit", "limit", 3050, 80, {"value": {"$ref": "nodes.merge.output"}, "maximum_chars": 28000}),
-        node("chat", "tool", 3280, 80, {"tool": "heimgeist.chat", "arguments": chat_arguments([{"$ref": "nodes.limit.output"}])}),
+        node("chat", "tool", 3280, 80, {"tool": "heimgeist.chat", "arguments": chat_arguments([{"$ref": "nodes.limit.output"}], EVIDENCE_FIRST_SYSTEM_PROMPT)}),
        node("output", "output", 3520, 80, {"value": {"$ref": "nodes.chat.output"}}),
    ],
    [
--- a/backend/tests/test_agent_web_tools.py
+++ b/backend/tests/test_agent_web_tools.py
@@ -45,6 +45,52 @@ class WebToolTests(unittest.IsolatedAsyncioTestCase):
        self.assertTrue(queries)
        self.assertTrue(all("iran" in query.lower() for query in queries))

+    async def test_generate_queries_filters_followup_filler_and_keeps_quoted_topic(self):
+        registry = NativeToolProvider()
+        register_web_tools(registry)
+
+        prompt = (
+            "nah i mean what you got from the news, you said\n"
+            "```7. The Iranian government describes ending the war in Lebanon as "
+            "\"inseparable\" from this broader Iran-US agreement, while Hezbollah also issued a statement.```"
+        )
+        result_payload = {
+            "queries": [
+                "nah mean you got latest news",
+                "Iranian government Lebanon Iran-US agreement Hezbollah statement",
+            ]
+        }
+        with patch(
+            "backend.agent.tools.web.chat_typed",
+            new=AsyncMock(return_value=OllamaChatResult(content=json.dumps(result_payload))),
+        ):
+            result = await registry.call_tool(
+                "heimgeist.web_generate_queries",
+                {"prompt": prompt, "model": "model", "messages": []},
+                context(registry),
+            )
+
+        queries = result["queries"]
+        self.assertNotIn("nah mean you got latest news", queries)
+        self.assertTrue(any("iran" in query.lower() and "lebanon" in query.lower() for query in queries))
+
+    async def test_generate_queries_accepts_followup_synonyms_and_libanon_spelling(self):
+        registry = NativeToolProvider()
+        register_web_tools(registry)
+
+        result_payload = {"queries": ["US Iran deal Lebanon ceasefire"]}
+        with patch(
+            "backend.agent.tools.web.chat_typed",
+            new=AsyncMock(return_value=OllamaChatResult(content=json.dumps(result_payload))),
+        ):
+            result = await registry.call_tool(
+                "heimgeist.web_generate_queries",
+                {"prompt": "tell me more about the agreement to end war in libanon", "model": "model", "messages": []},
+                context(registry),
+            )
+
+        self.assertIn("US Iran deal Lebanon ceasefire", result["queries"])
+
    async def test_search_filters_dictionary_noise_for_current_lookup(self):
        registry = NativeToolProvider()
        register_web_tools(registry)