Feat: Add and integrate enrichment model support across backend and frontend
This commit is contained in:
@@ -12,6 +12,7 @@ DEFAULT_BACKEND_API_URL = "http://127.0.0.1:8000"
|
||||
DEFAULT_OLLAMA_API_URL = "http://127.0.0.1:11434"
|
||||
DEFAULT_EMBED_MODEL = "nomic-embed-text:latest"
|
||||
DEFAULT_RERANK_MODEL = DEFAULT_EMBED_MODEL
|
||||
DEFAULT_ENRICHMENT_MODEL = "qwen3:4b"
|
||||
DEFAULT_TRANSCRIPTION_MODEL = "base"
|
||||
BGE_EMBED_MODEL = "bge-m3:latest"
|
||||
DEFAULT_SETTINGS: Dict[str, Any] = {
|
||||
@@ -21,6 +22,7 @@ DEFAULT_SETTINGS: Dict[str, Any] = {
|
||||
"visionModel": "",
|
||||
"embedModel": DEFAULT_EMBED_MODEL,
|
||||
"rerankModel": DEFAULT_RERANK_MODEL,
|
||||
"enrichmentModel": DEFAULT_ENRICHMENT_MODEL,
|
||||
"transcriptionModel": DEFAULT_TRANSCRIPTION_MODEL,
|
||||
}
|
||||
|
||||
@@ -125,6 +127,10 @@ def load_app_settings() -> Dict[str, Any]:
|
||||
settings["visionModel"] = settings.get("chatModel", "")
|
||||
settings["embedModel"] = normalize_embed_model(settings.get("embedModel"))
|
||||
settings["rerankModel"] = normalize_rerank_model(settings.get("rerankModel"))
|
||||
settings["enrichmentModel"] = normalize_model_name(
|
||||
settings.get("enrichmentModel"),
|
||||
DEFAULT_ENRICHMENT_MODEL,
|
||||
)
|
||||
settings["chatModel"] = normalize_model_name(settings.get("chatModel"))
|
||||
settings["visionModel"] = normalize_model_name(settings.get("visionModel"))
|
||||
settings["transcriptionModel"] = normalize_transcription_model(settings.get("transcriptionModel"))
|
||||
@@ -147,6 +153,11 @@ def get_rerank_model_preference() -> str:
|
||||
return normalize_rerank_model(settings.get("rerankModel"))
|
||||
|
||||
|
||||
def get_enrichment_model_preference() -> str:
|
||||
settings = load_app_settings()
|
||||
return normalize_model_name(settings.get("enrichmentModel"), DEFAULT_ENRICHMENT_MODEL)
|
||||
|
||||
|
||||
def get_transcription_model_preference() -> str:
|
||||
settings = load_app_settings()
|
||||
return normalize_transcription_model(settings.get("transcriptionModel"))
|
||||
|
||||
@@ -22,7 +22,9 @@ from pydantic import BaseModel
|
||||
|
||||
from .app_settings import (
|
||||
DEFAULT_EMBED_MODEL as DEFAULT_EMBED_MODEL_SETTING,
|
||||
DEFAULT_ENRICHMENT_MODEL,
|
||||
get_embed_model_preference,
|
||||
get_enrichment_model_preference,
|
||||
get_ollama_api_url,
|
||||
)
|
||||
from .paths import library_root
|
||||
@@ -35,7 +37,7 @@ LIB_ROOT = library_root()
|
||||
RAW_CORPUS_PROFILE = "per-file-default-v1"
|
||||
PREPARE_PROFILE = "selective-enrich-v2"
|
||||
DEFAULT_EMBED_MODEL = DEFAULT_EMBED_MODEL_SETTING
|
||||
DEFAULT_ENRICH_MODEL = "qwen3:4b"
|
||||
DEFAULT_ENRICH_MODEL = DEFAULT_ENRICHMENT_MODEL
|
||||
DEFAULT_ENRICH_MIN_CHARS = 240
|
||||
DEFAULT_ENRICH_MAX_TEXT = 6000
|
||||
DEFAULT_ENRICH_CONCURRENCY = max(1, min(4, (os.cpu_count() or 4) // 2))
|
||||
@@ -106,6 +108,10 @@ def _default_embed_model() -> str:
|
||||
return get_embed_model_preference()
|
||||
|
||||
|
||||
def _default_enrichment_model() -> str:
|
||||
return get_enrichment_model_preference()
|
||||
|
||||
|
||||
def _resolve_ollama_url(value: Optional[str] = None) -> str:
|
||||
if isinstance(value, str) and value.strip():
|
||||
return value.strip().rstrip("/")
|
||||
@@ -631,13 +637,15 @@ def _persist_item_metadata(slug: str, enhanced_path: Path) -> None:
|
||||
"status": "ready" if ok else "fallback",
|
||||
"headline": str(record.get("headline") or ""),
|
||||
"summary": str(record.get("summary") or ""),
|
||||
"keywords": list(record.get("keywords") or [])[:12],
|
||||
"entities": list(record.get("entities") or [])[:12],
|
||||
"keywords": list(record.get("keywords") or []),
|
||||
"entities": list(record.get("entities") or []),
|
||||
"qa": list(record.get("qa") or []),
|
||||
"language": record.get("lang"),
|
||||
"level": str(enrichment_meta.get("level") or "standard"),
|
||||
"model": enrichment_meta.get("model"),
|
||||
"strategy": enrichment_meta.get("strategy"),
|
||||
"qa_count": len(record.get("qa") or []),
|
||||
"quality_flags": list(enrichment_meta.get("quality_flags") or []),
|
||||
"updated_at": now_iso(),
|
||||
"error": enrichment_meta.get("error") if not ok else None,
|
||||
}
|
||||
@@ -997,7 +1005,7 @@ def _run_selected_enrichment(slug: str, on_progress=None, **opts) -> Dict[str, A
|
||||
shadow_out=paths["shadow_partial"],
|
||||
on_progress=on_progress,
|
||||
ollama=_resolve_ollama_url(opts.get("ollama")),
|
||||
model=opts.get("enrich_model", DEFAULT_ENRICH_MODEL),
|
||||
model=opts.get("enrich_model") or _default_enrichment_model() or DEFAULT_ENRICH_MODEL,
|
||||
summary_lang=opts.get("summary_lang", "auto"),
|
||||
concurrency=opts.get("enrich_concurrency", DEFAULT_ENRICH_CONCURRENCY),
|
||||
min_chars=opts.get("min_chars", DEFAULT_ENRICH_MIN_CHARS),
|
||||
@@ -1060,7 +1068,7 @@ def _run_prepare_pipeline(slug: str, on_progress=None, **opts):
|
||||
slug,
|
||||
on_progress=enrich_progress,
|
||||
ollama=_resolve_ollama_url(opts.get("ollama")),
|
||||
enrich_model=opts.get("enrich_model", DEFAULT_ENRICH_MODEL),
|
||||
enrich_model=opts.get("enrich_model") or _default_enrichment_model() or DEFAULT_ENRICH_MODEL,
|
||||
summary_lang=opts.get("summary_lang", "auto"),
|
||||
enrich_concurrency=opts.get("enrich_concurrency", DEFAULT_ENRICH_CONCURRENCY),
|
||||
min_chars=opts.get("min_chars", DEFAULT_ENRICH_MIN_CHARS),
|
||||
|
||||
@@ -28,6 +28,7 @@ const BACKEND_SIDECAR_NAME: &str = "heimgeist-backend";
|
||||
const BACKEND_STARTUP_TIMEOUT: Duration = Duration::from_secs(45);
|
||||
const DEFAULT_OLLAMA_API_URL: &str = "http://127.0.0.1:11434";
|
||||
const DEFAULT_EMBED_MODEL: &str = "nomic-embed-text:latest";
|
||||
const DEFAULT_ENRICHMENT_MODEL: &str = "qwen3:4b";
|
||||
const DEFAULT_TRANSCRIPTION_MODEL: &str = "base";
|
||||
const BGE_EMBED_MODEL: &str = "bge-m3:latest";
|
||||
const DEFAULT_UI_SCALE: f64 = 1.0;
|
||||
@@ -121,6 +122,10 @@ fn default_settings() -> SettingsMap {
|
||||
settings.insert("visionModel".into(), json!(""));
|
||||
settings.insert("embedModel".into(), json!(DEFAULT_EMBED_MODEL));
|
||||
settings.insert("rerankModel".into(), json!(DEFAULT_EMBED_MODEL));
|
||||
settings.insert(
|
||||
"enrichmentModel".into(),
|
||||
json!(DEFAULT_ENRICHMENT_MODEL),
|
||||
);
|
||||
settings.insert(
|
||||
"transcriptionModel".into(),
|
||||
json!(DEFAULT_TRANSCRIPTION_MODEL),
|
||||
@@ -244,6 +249,14 @@ fn migrate_settings(source: Option<SettingsMap>) -> (SettingsMap, bool) {
|
||||
migrated = true;
|
||||
}
|
||||
|
||||
if !source.contains_key("enrichmentModel") {
|
||||
next.insert(
|
||||
"enrichmentModel".into(),
|
||||
json!(DEFAULT_ENRICHMENT_MODEL),
|
||||
);
|
||||
migrated = true;
|
||||
}
|
||||
|
||||
if !source.contains_key("transcriptionModel") {
|
||||
next.insert(
|
||||
"transcriptionModel".into(),
|
||||
@@ -263,6 +276,10 @@ fn normalize_settings(settings: &mut SettingsMap) {
|
||||
let vision_model = normalize_model_name(settings.get("visionModel"), "");
|
||||
let embed_model = normalize_embed_model(settings.get("embedModel"));
|
||||
let rerank_model = normalize_embed_model(settings.get("rerankModel"));
|
||||
let enrichment_model = normalize_model_name(
|
||||
settings.get("enrichmentModel"),
|
||||
DEFAULT_ENRICHMENT_MODEL,
|
||||
);
|
||||
let transcription_model = normalize_model_name(
|
||||
settings.get("transcriptionModel"),
|
||||
DEFAULT_TRANSCRIPTION_MODEL,
|
||||
@@ -280,6 +297,7 @@ fn normalize_settings(settings: &mut SettingsMap) {
|
||||
settings.insert("visionModel".into(), json!(vision_model));
|
||||
settings.insert("embedModel".into(), json!(embed_model));
|
||||
settings.insert("rerankModel".into(), json!(rerank_model));
|
||||
settings.insert("enrichmentModel".into(), json!(enrichment_model));
|
||||
settings.insert("transcriptionModel".into(), json!(transcription_model));
|
||||
settings.insert("uiScale".into(), json!(ui_scale));
|
||||
settings.insert(
|
||||
|
||||
Reference in New Issue
Block a user