From fd8474845057aeb0b2e41a60282c1be5fce5d8df Mon Sep 17 00:00:00 2001
From: Victor Giers <mail@victorgiers.com>
Date: Mon, 15 Jun 2026 01:21:05 +0200
Subject: [PATCH] Improve RAG index building and local rag processing -
 Implement robust chunking for oversized text in index_builder. - Prevent job
 preparation in local_rag if source data contains failed sync items.

---
 backend/local_rag.py         | 10 +++++++++-
 backend/rag/index_builder.py | 28 +++++++++++++++++++++++++++-
 2 files changed, 36 insertions(+), 2 deletions(-)

diff --git a/backend/local_rag.py b/backend/local_rag.py
index 0fe70cb..2100dd6 100644
--- a/backend/local_rag.py
+++ b/backend/local_rag.py
@@ -1109,7 +1109,15 @@ async def list_libraries():
             continue
         try:
             data = _read_json(meta)
-            if _pipeline_meta(data).get("pending_prepare_signature") and not _has_active_job(data["slug"]):
+            has_failed_item = any(
+                str(entry.get("sync_status") or "") == "failed"
+                for entry in data.get("files", [])
+            )
+            if (
+                _pipeline_meta(data).get("pending_prepare_signature")
+                and not _has_active_job(data["slug"])
+                and not has_failed_item
+            ):
                 await _ensure_prepare_job(data["slug"])
             libraries.append(library_payload(read_library(data["slug"])))
         except Exception:
diff --git a/backend/rag/index_builder.py b/backend/rag/index_builder.py
index edcea3e..37fc17f 100644
--- a/backend/rag/index_builder.py
+++ b/backend/rag/index_builder.py
@@ -110,10 +110,16 @@ def chunk_text(txt: str, target_chars: int = 2500, overlap_chars: int = 200) ->
     paras = [p.strip() for p in (txt or "").split("\n\n") if p.strip()]
     if not paras:
         if txt.strip():
-            yield txt.strip()
+            yield from _split_oversized_text(txt.strip(), target_chars, overlap_chars)
         return
     buf, size = [], 0
     for p in paras:
+        if len(p) > target_chars:
+            if buf:
+                yield "\n\n".join(buf)
+                buf, size = [], 0
+            yield from _split_oversized_text(p, target_chars, overlap_chars)
+            continue
         if size + len(p) + 2 > target_chars and buf:
             chunk = "\n\n".join(buf)
             yield chunk
@@ -127,6 +133,26 @@ def chunk_text(txt: str, target_chars: int = 2500, overlap_chars: int = 200) ->
     if buf:
         yield "\n\n".join(buf)
 
+def _split_oversized_text(txt: str, target_chars: int, overlap_chars: int) -> Iterable[str]:
+    clean = (txt or "").strip()
+    target = max(400, int(target_chars))
+    overlap = min(max(0, int(overlap_chars)), target // 3)
+    start = 0
+    while start < len(clean):
+        limit = min(len(clean), start + target)
+        end = limit
+        if limit < len(clean):
+            search_from = start + max(1, target // 2)
+            whitespace = max(clean.rfind(" ", search_from, limit), clean.rfind("\n", search_from, limit))
+            if whitespace > start:
+                end = whitespace
+        chunk = clean[start:end].strip()
+        if chunk:
+            yield chunk
+        if end >= len(clean):
+            break
+        start = max(start + 1, end - overlap)
+
 def clamp_int(value: int, low: int, high: int) -> int:
     return max(low, min(high, value))