Feat: Add automatic deep enrichment feature

Implements logic and UI for automatically queuing deep enrichment (Q&A generation) after content indexing, updating settings across backend, frontend, and desktop layers.
This commit is contained in:
2026-06-15 04:40:41 +02:00
parent 36b6b84129
commit 88c4a5b0d4
5 changed files with 100 additions and 0 deletions

View File

@@ -14,6 +14,7 @@ DEFAULT_EMBED_MODEL = "nomic-embed-text:latest"
DEFAULT_RERANK_MODEL = DEFAULT_EMBED_MODEL
DEFAULT_ENRICHMENT_MODEL = "qwen3:4b"
DEFAULT_TRANSCRIPTION_MODEL = "base"
DEFAULT_AUTO_DEEP_ENRICHMENT = True
BGE_EMBED_MODEL = "bge-m3:latest"
DEFAULT_SETTINGS: Dict[str, Any] = {
"backendApiUrl": DEFAULT_BACKEND_API_URL,
@@ -24,6 +25,7 @@ DEFAULT_SETTINGS: Dict[str, Any] = {
"rerankModel": DEFAULT_RERANK_MODEL,
"enrichmentModel": DEFAULT_ENRICHMENT_MODEL,
"transcriptionModel": DEFAULT_TRANSCRIPTION_MODEL,
"autoDeepEnrichment": DEFAULT_AUTO_DEEP_ENRICHMENT,
}
@@ -98,6 +100,18 @@ def normalize_transcription_model(value: Any) -> str:
return normalize_model_name(value, DEFAULT_TRANSCRIPTION_MODEL)
def normalize_boolean(value: Any, default: bool = False) -> bool:
if isinstance(value, bool):
return value
if isinstance(value, str):
normalized = value.strip().lower()
if normalized in {"true", "1", "yes", "on"}:
return True
if normalized in {"false", "0", "no", "off", ""}:
return False
return default
def load_app_settings() -> Dict[str, Any]:
path = settings_path()
try:
@@ -134,6 +148,10 @@ def load_app_settings() -> Dict[str, Any]:
settings["chatModel"] = normalize_model_name(settings.get("chatModel"))
settings["visionModel"] = normalize_model_name(settings.get("visionModel"))
settings["transcriptionModel"] = normalize_transcription_model(settings.get("transcriptionModel"))
settings["autoDeepEnrichment"] = normalize_boolean(
settings.get("autoDeepEnrichment"),
DEFAULT_AUTO_DEEP_ENRICHMENT,
)
return settings
@@ -161,3 +179,11 @@ def get_enrichment_model_preference() -> str:
def get_transcription_model_preference() -> str:
settings = load_app_settings()
return normalize_transcription_model(settings.get("transcriptionModel"))
def get_auto_deep_enrichment_preference() -> bool:
settings = load_app_settings()
return normalize_boolean(
settings.get("autoDeepEnrichment"),
DEFAULT_AUTO_DEEP_ENRICHMENT,
)

View File

@@ -23,6 +23,7 @@ from pydantic import BaseModel
from .app_settings import (
DEFAULT_EMBED_MODEL as DEFAULT_EMBED_MODEL_SETTING,
DEFAULT_ENRICHMENT_MODEL,
get_auto_deep_enrichment_preference,
get_embed_model_preference,
get_enrichment_model_preference,
get_ollama_api_url,
@@ -42,6 +43,7 @@ DEFAULT_ENRICH_MIN_CHARS = 240
DEFAULT_ENRICH_MAX_TEXT = 6000
DEFAULT_ENRICH_CONCURRENCY = max(1, min(4, (os.cpu_count() or 4) // 2))
ITEM_METADATA_VERSION = 2
AUTO_DEEP_ENRICHMENT_DISABLED_KEY = "auto_deep_enrichment_disabled"
JOB_EXECUTOR = ThreadPoolExecutor(max_workers=2)
JOBS: Dict[str, Dict[str, Any]] = {}
@@ -553,6 +555,30 @@ def _mark_entry_pending(entry: Dict[str, Any]) -> None:
entry["metadata"].pop("error", None)
def _enable_automatic_deep_enrichment(slug: str, data: Optional[Dict[str, Any]] = None) -> bool:
if not get_auto_deep_enrichment_preference():
return False
library = data if data is not None else read_library(slug)
changed = False
for entry in library.get("files", []):
metadata = entry.get("metadata") if isinstance(entry.get("metadata"), dict) else {}
if metadata.get("status") not in {"ready", "fallback"}:
continue
if _file_enrich_enabled(entry) or bool(entry.get(AUTO_DEEP_ENRICHMENT_DISABLED_KEY)):
continue
entry["enrich_enabled"] = True
_mark_entry_pending(entry)
changed = True
if not changed:
return False
_set_pending_prepare_signature(library, _prepare_signature(library.get("files", [])))
write_library(slug, library)
return True
async def _save_library_change(slug: str, data: Dict[str, Any]) -> Optional[str]:
signature = _prepare_signature(data.get("files", []))
_set_pending_prepare_signature(data, signature)
@@ -873,6 +899,9 @@ async def _handle_post_job_state(slug: str, job_type: str, status: str) -> None:
return
if payload["states"].get("is_indexed"):
if job_type == "prepare" and status == "succeeded" and _enable_automatic_deep_enrichment(slug, data):
await _ensure_prepare_job(slug)
return
_mark_all_files_ready(slug)
_clear_pending_prepare(slug)
return
@@ -1238,6 +1267,14 @@ async def list_libraries():
try:
data = _read_json(meta)
payload = library_payload(data)
if (
payload["states"].get("is_indexed")
and not _has_active_job(data["slug"])
and _enable_automatic_deep_enrichment(data["slug"], data)
):
await _ensure_prepare_job(data["slug"])
data = read_library(data["slug"])
payload = library_payload(data)
has_failed_item = any(
str(entry.get("sync_status") or "") == "failed"
for entry in data.get("files", [])
@@ -1698,6 +1735,10 @@ async def update_file_enrichment(slug: str, req: UpdateFileEnrichmentRequest):
return {"job_id": None, "library": library_payload(data)}
target["enrich_enabled"] = desired
if desired:
target.pop(AUTO_DEEP_ENRICHMENT_DISABLED_KEY, None)
else:
target[AUTO_DEEP_ENRICHMENT_DISABLED_KEY] = True
target["sync_status"] = "pending"
target.pop("sync_error", None)
target.pop("synced_at", None)