diff --git a/backend/app_settings.py b/backend/app_settings.py index ba3e6b7..424e9e3 100644 --- a/backend/app_settings.py +++ b/backend/app_settings.py @@ -14,6 +14,7 @@ DEFAULT_EMBED_MODEL = "nomic-embed-text:latest" DEFAULT_RERANK_MODEL = DEFAULT_EMBED_MODEL DEFAULT_ENRICHMENT_MODEL = "qwen3:4b" DEFAULT_TRANSCRIPTION_MODEL = "base" +DEFAULT_AUTO_DEEP_ENRICHMENT = True BGE_EMBED_MODEL = "bge-m3:latest" DEFAULT_SETTINGS: Dict[str, Any] = { "backendApiUrl": DEFAULT_BACKEND_API_URL, @@ -24,6 +25,7 @@ DEFAULT_SETTINGS: Dict[str, Any] = { "rerankModel": DEFAULT_RERANK_MODEL, "enrichmentModel": DEFAULT_ENRICHMENT_MODEL, "transcriptionModel": DEFAULT_TRANSCRIPTION_MODEL, + "autoDeepEnrichment": DEFAULT_AUTO_DEEP_ENRICHMENT, } @@ -98,6 +100,18 @@ def normalize_transcription_model(value: Any) -> str: return normalize_model_name(value, DEFAULT_TRANSCRIPTION_MODEL) +def normalize_boolean(value: Any, default: bool = False) -> bool: + if isinstance(value, bool): + return value + if isinstance(value, str): + normalized = value.strip().lower() + if normalized in {"true", "1", "yes", "on"}: + return True + if normalized in {"false", "0", "no", "off", ""}: + return False + return default + + def load_app_settings() -> Dict[str, Any]: path = settings_path() try: @@ -134,6 +148,10 @@ def load_app_settings() -> Dict[str, Any]: settings["chatModel"] = normalize_model_name(settings.get("chatModel")) settings["visionModel"] = normalize_model_name(settings.get("visionModel")) settings["transcriptionModel"] = normalize_transcription_model(settings.get("transcriptionModel")) + settings["autoDeepEnrichment"] = normalize_boolean( + settings.get("autoDeepEnrichment"), + DEFAULT_AUTO_DEEP_ENRICHMENT, + ) return settings @@ -161,3 +179,11 @@ def get_enrichment_model_preference() -> str: def get_transcription_model_preference() -> str: settings = load_app_settings() return normalize_transcription_model(settings.get("transcriptionModel")) + + +def get_auto_deep_enrichment_preference() -> bool: + settings = load_app_settings() + return normalize_boolean( + settings.get("autoDeepEnrichment"), + DEFAULT_AUTO_DEEP_ENRICHMENT, + ) diff --git a/backend/local_rag.py b/backend/local_rag.py index 195d35d..13fbf2c 100644 --- a/backend/local_rag.py +++ b/backend/local_rag.py @@ -23,6 +23,7 @@ from pydantic import BaseModel from .app_settings import ( DEFAULT_EMBED_MODEL as DEFAULT_EMBED_MODEL_SETTING, DEFAULT_ENRICHMENT_MODEL, + get_auto_deep_enrichment_preference, get_embed_model_preference, get_enrichment_model_preference, get_ollama_api_url, @@ -42,6 +43,7 @@ DEFAULT_ENRICH_MIN_CHARS = 240 DEFAULT_ENRICH_MAX_TEXT = 6000 DEFAULT_ENRICH_CONCURRENCY = max(1, min(4, (os.cpu_count() or 4) // 2)) ITEM_METADATA_VERSION = 2 +AUTO_DEEP_ENRICHMENT_DISABLED_KEY = "auto_deep_enrichment_disabled" JOB_EXECUTOR = ThreadPoolExecutor(max_workers=2) JOBS: Dict[str, Dict[str, Any]] = {} @@ -553,6 +555,30 @@ def _mark_entry_pending(entry: Dict[str, Any]) -> None: entry["metadata"].pop("error", None) +def _enable_automatic_deep_enrichment(slug: str, data: Optional[Dict[str, Any]] = None) -> bool: + if not get_auto_deep_enrichment_preference(): + return False + + library = data if data is not None else read_library(slug) + changed = False + for entry in library.get("files", []): + metadata = entry.get("metadata") if isinstance(entry.get("metadata"), dict) else {} + if metadata.get("status") not in {"ready", "fallback"}: + continue + if _file_enrich_enabled(entry) or bool(entry.get(AUTO_DEEP_ENRICHMENT_DISABLED_KEY)): + continue + entry["enrich_enabled"] = True + _mark_entry_pending(entry) + changed = True + + if not changed: + return False + + _set_pending_prepare_signature(library, _prepare_signature(library.get("files", []))) + write_library(slug, library) + return True + + async def _save_library_change(slug: str, data: Dict[str, Any]) -> Optional[str]: signature = _prepare_signature(data.get("files", [])) _set_pending_prepare_signature(data, signature) @@ -873,6 +899,9 @@ async def _handle_post_job_state(slug: str, job_type: str, status: str) -> None: return if payload["states"].get("is_indexed"): + if job_type == "prepare" and status == "succeeded" and _enable_automatic_deep_enrichment(slug, data): + await _ensure_prepare_job(slug) + return _mark_all_files_ready(slug) _clear_pending_prepare(slug) return @@ -1238,6 +1267,14 @@ async def list_libraries(): try: data = _read_json(meta) payload = library_payload(data) + if ( + payload["states"].get("is_indexed") + and not _has_active_job(data["slug"]) + and _enable_automatic_deep_enrichment(data["slug"], data) + ): + await _ensure_prepare_job(data["slug"]) + data = read_library(data["slug"]) + payload = library_payload(data) has_failed_item = any( str(entry.get("sync_status") or "") == "failed" for entry in data.get("files", []) @@ -1698,6 +1735,10 @@ async def update_file_enrichment(slug: str, req: UpdateFileEnrichmentRequest): return {"job_id": None, "library": library_payload(data)} target["enrich_enabled"] = desired + if desired: + target.pop(AUTO_DEEP_ENRICHMENT_DISABLED_KEY, None) + else: + target[AUTO_DEEP_ENRICHMENT_DISABLED_KEY] = True target["sync_status"] = "pending" target.pop("sync_error", None) target.pop("synced_at", None) diff --git a/src-tauri/src/main.rs b/src-tauri/src/main.rs index d906b08..5bfa237 100644 --- a/src-tauri/src/main.rs +++ b/src-tauri/src/main.rs @@ -127,6 +127,7 @@ fn default_settings() -> SettingsMap { "transcriptionModel".into(), json!(DEFAULT_TRANSCRIPTION_MODEL), ); + settings.insert("autoDeepEnrichment".into(), json!(true)); settings.insert("colorScheme".into(), json!("Default")); settings.insert("uiScale".into(), json!(DEFAULT_UI_SCALE)); settings.insert("openDevToolsOnStartup".into(), json!(false)); @@ -259,6 +260,11 @@ fn migrate_settings(source: Option) -> (SettingsMap, bool) { migrated = true; } + if !source.contains_key("autoDeepEnrichment") { + next.insert("autoDeepEnrichment".into(), json!(true)); + migrated = true; + } + normalize_settings(&mut next); (next, migrated) } @@ -276,6 +282,8 @@ fn normalize_settings(settings: &mut SettingsMap) { settings.get("transcriptionModel"), DEFAULT_TRANSCRIPTION_MODEL, ); + let auto_deep_enrichment = + normalize_boolean_setting(settings.get("autoDeepEnrichment")); let ui_scale = normalize_ui_scale(settings.get("uiScale")); let open_devtools_on_startup = normalize_boolean_setting(settings.get("openDevToolsOnStartup")); let audio_input_enabled = normalize_boolean_setting(settings.get("audioInputEnabled")); @@ -291,6 +299,7 @@ fn normalize_settings(settings: &mut SettingsMap) { settings.insert("rerankModel".into(), json!(rerank_model)); settings.insert("enrichmentModel".into(), json!(enrichment_model)); settings.insert("transcriptionModel".into(), json!(transcription_model)); + settings.insert("autoDeepEnrichment".into(), json!(auto_deep_enrichment)); settings.insert("uiScale".into(), json!(ui_scale)); settings.insert( "openDevToolsOnStartup".into(), diff --git a/src/GeneralSettings.jsx b/src/GeneralSettings.jsx index e1d7a33..93eea15 100644 --- a/src/GeneralSettings.jsx +++ b/src/GeneralSettings.jsx @@ -15,6 +15,7 @@ const ENRICHMENT_MODEL_KEY = 'enrichmentModel'; const MODEL_KEY = 'chatModel'; const VISION_MODEL_KEY = 'visionModel'; const TRANSCRIPTION_MODEL_KEY = 'transcriptionModel'; +const AUTO_DEEP_ENRICHMENT_KEY = 'autoDeepEnrichment'; const DEFAULT_AUDIO_INPUT_DEVICE_ID = ''; const DEFAULT_AUDIO_INPUT_LANGUAGE = ''; const DEFAULT_BACKEND_API_URL = 'http://127.0.0.1:8000'; @@ -90,6 +91,7 @@ export default function GeneralSettings({ const [selectedModel, setSelectedModel] = useState(''); const [visionModel, setVisionModel] = useState(''); const [transcriptionModel, setTranscriptionModel] = useState(DEFAULT_TRANSCRIPTION_MODEL); + const [autoDeepEnrichment, setAutoDeepEnrichment] = useState(true); const [audioInputDeviceId, setAudioInputDeviceId] = useState(DEFAULT_AUDIO_INPUT_DEVICE_ID); const [audioInputLanguage, setAudioInputLanguage] = useState(DEFAULT_AUDIO_INPUT_LANGUAGE); const [audioInputDevices, setAudioInputDevices] = useState([]); @@ -127,6 +129,7 @@ export default function GeneralSettings({ setSelectedModel(settings.chatModel || ''); setVisionModel(settings.visionModel || settings.chatModel || ''); setTranscriptionModel(settings.transcriptionModel || DEFAULT_TRANSCRIPTION_MODEL); + setAutoDeepEnrichment(settings.autoDeepEnrichment !== false); setAudioInputDeviceId( typeof settings.audioInputDeviceId === 'string' ? settings.audioInputDeviceId @@ -416,6 +419,12 @@ export default function GeneralSettings({ } }; + const handleAutoDeepEnrichmentToggle = () => { + const nextValue = !autoDeepEnrichment; + setAutoDeepEnrichment(nextValue); + desktopApi.setSetting(AUTO_DEEP_ENRICHMENT_KEY, nextValue); + }; + const refreshAudioDevices = async ({ requestAccess = false } = {}) => { if (!audioInputSupported) { return; @@ -850,6 +859,20 @@ export default function GeneralSettings({ if (panel === 'Advanced') { return (
+
+

Automatic Deep Enrichment

+ +

+ After standard metadata is generated, Heimgeist automatically queues deep enrichment to add generated Q&A. Individual content can still be switched back to Standard. +

+

Purge Databases

diff --git a/src/desktop/desktopApi.js b/src/desktop/desktopApi.js index 41f70bb..a75e6e9 100644 --- a/src/desktop/desktopApi.js +++ b/src/desktop/desktopApi.js @@ -6,6 +6,7 @@ const DEFAULT_SETTINGS = Object.freeze({ embedModel: 'nomic-embed-text:latest', rerankModel: 'nomic-embed-text:latest', transcriptionModel: 'base', + autoDeepEnrichment: true, colorScheme: 'Default', uiScale: 1, openDevToolsOnStartup: false,