Feat: Add automatic deep enrichment feature
Implements logic and UI for automatically queuing deep enrichment (Q&A generation) after content indexing, updating settings across backend, frontend, and desktop layers.
This commit is contained in:
@@ -14,6 +14,7 @@ DEFAULT_EMBED_MODEL = "nomic-embed-text:latest"
|
||||
DEFAULT_RERANK_MODEL = DEFAULT_EMBED_MODEL
|
||||
DEFAULT_ENRICHMENT_MODEL = "qwen3:4b"
|
||||
DEFAULT_TRANSCRIPTION_MODEL = "base"
|
||||
DEFAULT_AUTO_DEEP_ENRICHMENT = True
|
||||
BGE_EMBED_MODEL = "bge-m3:latest"
|
||||
DEFAULT_SETTINGS: Dict[str, Any] = {
|
||||
"backendApiUrl": DEFAULT_BACKEND_API_URL,
|
||||
@@ -24,6 +25,7 @@ DEFAULT_SETTINGS: Dict[str, Any] = {
|
||||
"rerankModel": DEFAULT_RERANK_MODEL,
|
||||
"enrichmentModel": DEFAULT_ENRICHMENT_MODEL,
|
||||
"transcriptionModel": DEFAULT_TRANSCRIPTION_MODEL,
|
||||
"autoDeepEnrichment": DEFAULT_AUTO_DEEP_ENRICHMENT,
|
||||
}
|
||||
|
||||
|
||||
@@ -98,6 +100,18 @@ def normalize_transcription_model(value: Any) -> str:
|
||||
return normalize_model_name(value, DEFAULT_TRANSCRIPTION_MODEL)
|
||||
|
||||
|
||||
def normalize_boolean(value: Any, default: bool = False) -> bool:
|
||||
if isinstance(value, bool):
|
||||
return value
|
||||
if isinstance(value, str):
|
||||
normalized = value.strip().lower()
|
||||
if normalized in {"true", "1", "yes", "on"}:
|
||||
return True
|
||||
if normalized in {"false", "0", "no", "off", ""}:
|
||||
return False
|
||||
return default
|
||||
|
||||
|
||||
def load_app_settings() -> Dict[str, Any]:
|
||||
path = settings_path()
|
||||
try:
|
||||
@@ -134,6 +148,10 @@ def load_app_settings() -> Dict[str, Any]:
|
||||
settings["chatModel"] = normalize_model_name(settings.get("chatModel"))
|
||||
settings["visionModel"] = normalize_model_name(settings.get("visionModel"))
|
||||
settings["transcriptionModel"] = normalize_transcription_model(settings.get("transcriptionModel"))
|
||||
settings["autoDeepEnrichment"] = normalize_boolean(
|
||||
settings.get("autoDeepEnrichment"),
|
||||
DEFAULT_AUTO_DEEP_ENRICHMENT,
|
||||
)
|
||||
|
||||
return settings
|
||||
|
||||
@@ -161,3 +179,11 @@ def get_enrichment_model_preference() -> str:
|
||||
def get_transcription_model_preference() -> str:
|
||||
settings = load_app_settings()
|
||||
return normalize_transcription_model(settings.get("transcriptionModel"))
|
||||
|
||||
|
||||
def get_auto_deep_enrichment_preference() -> bool:
|
||||
settings = load_app_settings()
|
||||
return normalize_boolean(
|
||||
settings.get("autoDeepEnrichment"),
|
||||
DEFAULT_AUTO_DEEP_ENRICHMENT,
|
||||
)
|
||||
|
||||
@@ -23,6 +23,7 @@ from pydantic import BaseModel
|
||||
from .app_settings import (
|
||||
DEFAULT_EMBED_MODEL as DEFAULT_EMBED_MODEL_SETTING,
|
||||
DEFAULT_ENRICHMENT_MODEL,
|
||||
get_auto_deep_enrichment_preference,
|
||||
get_embed_model_preference,
|
||||
get_enrichment_model_preference,
|
||||
get_ollama_api_url,
|
||||
@@ -42,6 +43,7 @@ DEFAULT_ENRICH_MIN_CHARS = 240
|
||||
DEFAULT_ENRICH_MAX_TEXT = 6000
|
||||
DEFAULT_ENRICH_CONCURRENCY = max(1, min(4, (os.cpu_count() or 4) // 2))
|
||||
ITEM_METADATA_VERSION = 2
|
||||
AUTO_DEEP_ENRICHMENT_DISABLED_KEY = "auto_deep_enrichment_disabled"
|
||||
|
||||
JOB_EXECUTOR = ThreadPoolExecutor(max_workers=2)
|
||||
JOBS: Dict[str, Dict[str, Any]] = {}
|
||||
@@ -553,6 +555,30 @@ def _mark_entry_pending(entry: Dict[str, Any]) -> None:
|
||||
entry["metadata"].pop("error", None)
|
||||
|
||||
|
||||
def _enable_automatic_deep_enrichment(slug: str, data: Optional[Dict[str, Any]] = None) -> bool:
|
||||
if not get_auto_deep_enrichment_preference():
|
||||
return False
|
||||
|
||||
library = data if data is not None else read_library(slug)
|
||||
changed = False
|
||||
for entry in library.get("files", []):
|
||||
metadata = entry.get("metadata") if isinstance(entry.get("metadata"), dict) else {}
|
||||
if metadata.get("status") not in {"ready", "fallback"}:
|
||||
continue
|
||||
if _file_enrich_enabled(entry) or bool(entry.get(AUTO_DEEP_ENRICHMENT_DISABLED_KEY)):
|
||||
continue
|
||||
entry["enrich_enabled"] = True
|
||||
_mark_entry_pending(entry)
|
||||
changed = True
|
||||
|
||||
if not changed:
|
||||
return False
|
||||
|
||||
_set_pending_prepare_signature(library, _prepare_signature(library.get("files", [])))
|
||||
write_library(slug, library)
|
||||
return True
|
||||
|
||||
|
||||
async def _save_library_change(slug: str, data: Dict[str, Any]) -> Optional[str]:
|
||||
signature = _prepare_signature(data.get("files", []))
|
||||
_set_pending_prepare_signature(data, signature)
|
||||
@@ -873,6 +899,9 @@ async def _handle_post_job_state(slug: str, job_type: str, status: str) -> None:
|
||||
return
|
||||
|
||||
if payload["states"].get("is_indexed"):
|
||||
if job_type == "prepare" and status == "succeeded" and _enable_automatic_deep_enrichment(slug, data):
|
||||
await _ensure_prepare_job(slug)
|
||||
return
|
||||
_mark_all_files_ready(slug)
|
||||
_clear_pending_prepare(slug)
|
||||
return
|
||||
@@ -1238,6 +1267,14 @@ async def list_libraries():
|
||||
try:
|
||||
data = _read_json(meta)
|
||||
payload = library_payload(data)
|
||||
if (
|
||||
payload["states"].get("is_indexed")
|
||||
and not _has_active_job(data["slug"])
|
||||
and _enable_automatic_deep_enrichment(data["slug"], data)
|
||||
):
|
||||
await _ensure_prepare_job(data["slug"])
|
||||
data = read_library(data["slug"])
|
||||
payload = library_payload(data)
|
||||
has_failed_item = any(
|
||||
str(entry.get("sync_status") or "") == "failed"
|
||||
for entry in data.get("files", [])
|
||||
@@ -1698,6 +1735,10 @@ async def update_file_enrichment(slug: str, req: UpdateFileEnrichmentRequest):
|
||||
return {"job_id": None, "library": library_payload(data)}
|
||||
|
||||
target["enrich_enabled"] = desired
|
||||
if desired:
|
||||
target.pop(AUTO_DEEP_ENRICHMENT_DISABLED_KEY, None)
|
||||
else:
|
||||
target[AUTO_DEEP_ENRICHMENT_DISABLED_KEY] = True
|
||||
target["sync_status"] = "pending"
|
||||
target.pop("sync_error", None)
|
||||
target.pop("synced_at", None)
|
||||
|
||||
Reference in New Issue
Block a user