Feat: Add and integrate enrichment model support across backend and frontend

This commit is contained in:
2026-06-15 02:41:49 +02:00
parent 26ef19f4c6
commit bfb7f870c6
3 changed files with 42 additions and 5 deletions

View File

@@ -12,6 +12,7 @@ DEFAULT_BACKEND_API_URL = "http://127.0.0.1:8000"
DEFAULT_OLLAMA_API_URL = "http://127.0.0.1:11434"
DEFAULT_EMBED_MODEL = "nomic-embed-text:latest"
DEFAULT_RERANK_MODEL = DEFAULT_EMBED_MODEL
DEFAULT_ENRICHMENT_MODEL = "qwen3:4b"
DEFAULT_TRANSCRIPTION_MODEL = "base"
BGE_EMBED_MODEL = "bge-m3:latest"
DEFAULT_SETTINGS: Dict[str, Any] = {
@@ -21,6 +22,7 @@ DEFAULT_SETTINGS: Dict[str, Any] = {
"visionModel": "",
"embedModel": DEFAULT_EMBED_MODEL,
"rerankModel": DEFAULT_RERANK_MODEL,
"enrichmentModel": DEFAULT_ENRICHMENT_MODEL,
"transcriptionModel": DEFAULT_TRANSCRIPTION_MODEL,
}
@@ -125,6 +127,10 @@ def load_app_settings() -> Dict[str, Any]:
settings["visionModel"] = settings.get("chatModel", "")
settings["embedModel"] = normalize_embed_model(settings.get("embedModel"))
settings["rerankModel"] = normalize_rerank_model(settings.get("rerankModel"))
settings["enrichmentModel"] = normalize_model_name(
settings.get("enrichmentModel"),
DEFAULT_ENRICHMENT_MODEL,
)
settings["chatModel"] = normalize_model_name(settings.get("chatModel"))
settings["visionModel"] = normalize_model_name(settings.get("visionModel"))
settings["transcriptionModel"] = normalize_transcription_model(settings.get("transcriptionModel"))
@@ -147,6 +153,11 @@ def get_rerank_model_preference() -> str:
return normalize_rerank_model(settings.get("rerankModel"))
def get_enrichment_model_preference() -> str:
settings = load_app_settings()
return normalize_model_name(settings.get("enrichmentModel"), DEFAULT_ENRICHMENT_MODEL)
def get_transcription_model_preference() -> str:
settings = load_app_settings()
return normalize_transcription_model(settings.get("transcriptionModel"))

View File

@@ -22,7 +22,9 @@ from pydantic import BaseModel
from .app_settings import (
DEFAULT_EMBED_MODEL as DEFAULT_EMBED_MODEL_SETTING,
DEFAULT_ENRICHMENT_MODEL,
get_embed_model_preference,
get_enrichment_model_preference,
get_ollama_api_url,
)
from .paths import library_root
@@ -35,7 +37,7 @@ LIB_ROOT = library_root()
RAW_CORPUS_PROFILE = "per-file-default-v1"
PREPARE_PROFILE = "selective-enrich-v2"
DEFAULT_EMBED_MODEL = DEFAULT_EMBED_MODEL_SETTING
DEFAULT_ENRICH_MODEL = "qwen3:4b"
DEFAULT_ENRICH_MODEL = DEFAULT_ENRICHMENT_MODEL
DEFAULT_ENRICH_MIN_CHARS = 240
DEFAULT_ENRICH_MAX_TEXT = 6000
DEFAULT_ENRICH_CONCURRENCY = max(1, min(4, (os.cpu_count() or 4) // 2))
@@ -106,6 +108,10 @@ def _default_embed_model() -> str:
return get_embed_model_preference()
def _default_enrichment_model() -> str:
return get_enrichment_model_preference()
def _resolve_ollama_url(value: Optional[str] = None) -> str:
if isinstance(value, str) and value.strip():
return value.strip().rstrip("/")
@@ -631,13 +637,15 @@ def _persist_item_metadata(slug: str, enhanced_path: Path) -> None:
"status": "ready" if ok else "fallback",
"headline": str(record.get("headline") or ""),
"summary": str(record.get("summary") or ""),
"keywords": list(record.get("keywords") or [])[:12],
"entities": list(record.get("entities") or [])[:12],
"keywords": list(record.get("keywords") or []),
"entities": list(record.get("entities") or []),
"qa": list(record.get("qa") or []),
"language": record.get("lang"),
"level": str(enrichment_meta.get("level") or "standard"),
"model": enrichment_meta.get("model"),
"strategy": enrichment_meta.get("strategy"),
"qa_count": len(record.get("qa") or []),
"quality_flags": list(enrichment_meta.get("quality_flags") or []),
"updated_at": now_iso(),
"error": enrichment_meta.get("error") if not ok else None,
}
@@ -997,7 +1005,7 @@ def _run_selected_enrichment(slug: str, on_progress=None, **opts) -> Dict[str, A
shadow_out=paths["shadow_partial"],
on_progress=on_progress,
ollama=_resolve_ollama_url(opts.get("ollama")),
model=opts.get("enrich_model", DEFAULT_ENRICH_MODEL),
model=opts.get("enrich_model") or _default_enrichment_model() or DEFAULT_ENRICH_MODEL,
summary_lang=opts.get("summary_lang", "auto"),
concurrency=opts.get("enrich_concurrency", DEFAULT_ENRICH_CONCURRENCY),
min_chars=opts.get("min_chars", DEFAULT_ENRICH_MIN_CHARS),
@@ -1060,7 +1068,7 @@ def _run_prepare_pipeline(slug: str, on_progress=None, **opts):
slug,
on_progress=enrich_progress,
ollama=_resolve_ollama_url(opts.get("ollama")),
enrich_model=opts.get("enrich_model", DEFAULT_ENRICH_MODEL),
enrich_model=opts.get("enrich_model") or _default_enrichment_model() or DEFAULT_ENRICH_MODEL,
summary_lang=opts.get("summary_lang", "auto"),
enrich_concurrency=opts.get("enrich_concurrency", DEFAULT_ENRICH_CONCURRENCY),
min_chars=opts.get("min_chars", DEFAULT_ENRICH_MIN_CHARS),

View File

@@ -28,6 +28,7 @@ const BACKEND_SIDECAR_NAME: &str = "heimgeist-backend";
const BACKEND_STARTUP_TIMEOUT: Duration = Duration::from_secs(45);
const DEFAULT_OLLAMA_API_URL: &str = "http://127.0.0.1:11434";
const DEFAULT_EMBED_MODEL: &str = "nomic-embed-text:latest";
const DEFAULT_ENRICHMENT_MODEL: &str = "qwen3:4b";
const DEFAULT_TRANSCRIPTION_MODEL: &str = "base";
const BGE_EMBED_MODEL: &str = "bge-m3:latest";
const DEFAULT_UI_SCALE: f64 = 1.0;
@@ -121,6 +122,10 @@ fn default_settings() -> SettingsMap {
settings.insert("visionModel".into(), json!(""));
settings.insert("embedModel".into(), json!(DEFAULT_EMBED_MODEL));
settings.insert("rerankModel".into(), json!(DEFAULT_EMBED_MODEL));
settings.insert(
"enrichmentModel".into(),
json!(DEFAULT_ENRICHMENT_MODEL),
);
settings.insert(
"transcriptionModel".into(),
json!(DEFAULT_TRANSCRIPTION_MODEL),
@@ -244,6 +249,14 @@ fn migrate_settings(source: Option<SettingsMap>) -> (SettingsMap, bool) {
migrated = true;
}
if !source.contains_key("enrichmentModel") {
next.insert(
"enrichmentModel".into(),
json!(DEFAULT_ENRICHMENT_MODEL),
);
migrated = true;
}
if !source.contains_key("transcriptionModel") {
next.insert(
"transcriptionModel".into(),
@@ -263,6 +276,10 @@ fn normalize_settings(settings: &mut SettingsMap) {
let vision_model = normalize_model_name(settings.get("visionModel"), "");
let embed_model = normalize_embed_model(settings.get("embedModel"));
let rerank_model = normalize_embed_model(settings.get("rerankModel"));
let enrichment_model = normalize_model_name(
settings.get("enrichmentModel"),
DEFAULT_ENRICHMENT_MODEL,
);
let transcription_model = normalize_model_name(
settings.get("transcriptionModel"),
DEFAULT_TRANSCRIPTION_MODEL,
@@ -280,6 +297,7 @@ fn normalize_settings(settings: &mut SettingsMap) {
settings.insert("visionModel".into(), json!(vision_model));
settings.insert("embedModel".into(), json!(embed_model));
settings.insert("rerankModel".into(), json!(rerank_model));
settings.insert("enrichmentModel".into(), json!(enrichment_model));
settings.insert("transcriptionModel".into(), json!(transcription_model));
settings.insert("uiScale".into(), json!(ui_scale));
settings.insert(