giers10/Heimgeist
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Enhance RAG corpus enrichment: Update prompt version, improve response validation, increase token limits, and refactor data normalization logic.

Browse Source
This commit is contained in:
Victor Giers
2026-06-15 02:11:07 +02:00
parent 90a994f72e
commit 7b42e25135
1 changed files with 44 additions and 28 deletions
Download Patch File Download Diff File Expand all files Collapse all files

72
backend/rag/corpus_enricher.py
View File

@@ -91,7 +91,7 @@ except Exception:
# Constants & helpers
# -------------------------
PROMPT_VERSION = "v4.0-standard-deep"
PROMPT_VERSION = "v4.1-standard-deep"
ENTITY_CANON = {
"PERSON": "PERSON",
@@ -301,8 +301,7 @@ def ollama_generate_json(
return json_loads(m.group(0))
except Exception:
pass
# last resort minimal structure
return {"headline": "", "summary": raw, "keywords": [], "entities": [], "qa": []}
raise ValueError("Ollama returned incomplete or invalid JSON.")
def ollama_generate_text(
host: str,
@@ -379,6 +378,25 @@ def build_user_main(text: str, summary_lang: str, doc_hint: str, want_qa: int, *
"TEXT:\n" + text
)
def unwrap_enrichment_response(value: Any) -> Dict[str, Any]:
if not isinstance(value, dict):
raise ValueError("Enrichment response is not a JSON object.")
nested = value.get("summary")
if not value.get("headline") and isinstance(nested, str) and nested.lstrip().startswith("{"):
try:
decoded = json_loads(nested)
except Exception as exc:
raise ValueError("Enrichment response contains truncated nested JSON.") from exc
if isinstance(decoded, dict):
value = decoded
if not isinstance(value.get("summary"), str) or not str(value.get("summary") or "").strip():
raise ValueError("Enrichment response is missing a summary.")
for field in ("keywords", "entities", "qa"):
if field in value and not isinstance(value[field], list):
raise ValueError(f"Enrichment response field '{field}' must be a list.")
return value
def build_user_qa_topup(text: str, summary_lang: str, need: int) -> str:
need = max(1, min(3, int(need)))
return (
@@ -786,7 +804,7 @@ def enrich_one(
"temperature": 0.2,
"repeat_penalty": 1.1,
"top_p": 0.9,
"num_predict": 360 if include_qa else 240,
"num_predict": 1000 if include_qa else 700,
}
with sem:
@@ -795,33 +813,31 @@ def enrich_one(
try:
out = ollama_generate_json(args.ollama, args.model, system, user,
keep_alive=args.keep_alive, timeout=args.timeout, options=options)
out = unwrap_enrichment_response(out)
# sanitize + normalize structure
if not isinstance(out, dict):
out = {"headline": "", "summary": sanitize_text(str(out)), "keywords": [], "entities": [], "qa": []}
else:
for k in ("headline", "summary"):
if k in out and isinstance(out[k], str):
out[k] = sanitize_text(out[k])
for k in ("headline", "summary"):
if k in out and isinstance(out[k], str):
out[k] = sanitize_text(out[k])
# normalize arrays to expected types
out["keywords"] = [sanitize_text(str(x)) for x in out.get("keywords", []) if str(x).strip()]
ents = []
for e in out.get("entities", []) or []:
if isinstance(e, dict):
name = sanitize_text(str(e.get("name", "")))
typ = sanitize_text(str(e.get("type", "OTHER")))
if name:
ents.append({"name": name, "type": typ})
out["entities"] = ents
# normalize arrays to expected types
out["keywords"] = [sanitize_text(str(x)) for x in out.get("keywords", []) if str(x).strip()]
ents = []
for e in out.get("entities", []) or []:
if isinstance(e, dict):
name = sanitize_text(str(e.get("name", "")))
typ = sanitize_text(str(e.get("type", "OTHER")))
if name:
ents.append({"name": name, "type": typ})
out["entities"] = ents
qas = []
for qa in out.get("qa", []) or []:
if isinstance(qa, dict):
q = sanitize_text(str(qa.get("q", "")))
a = sanitize_text(str(qa.get("a", "")))
if q and a:
qas.append({"q": q, "a": a})
out["qa"] = qas
qas = []
for qa in out.get("qa", []) or []:
if isinstance(qa, dict):
q = sanitize_text(str(qa.get("q", "")))
a = sanitize_text(str(qa.get("a", "")))
if q and a:
qas.append({"q": q, "a": a})
out["qa"] = qas
# post-enforce schema + language
fixed = enforce_schema_and_language(