From d0e5c9300c832e12ed9352d5bcd5182abc4bccf2 Mon Sep 17 00:00:00 2001 From: Victor Giers Date: Mon, 15 Jun 2026 03:51:19 +0200 Subject: [PATCH] Feature: Add full support for handling, indexing, and saving chat message content to the knowledge base. --- backend/database.py | 17 ++++++++- backend/local_rag.py | 89 ++++++++++++++++++++++++++++++++++++++++++-- backend/main.py | 77 +++++++++++++++++++++++++++++++++++++- backend/models.py | 2 + backend/schemas.py | 7 ++++ 5 files changed, 185 insertions(+), 7 deletions(-) diff --git a/backend/database.py b/backend/database.py index 5412d1a..823f99d 100644 --- a/backend/database.py +++ b/backend/database.py @@ -2,6 +2,7 @@ from sqlalchemy import text from sqlalchemy import create_engine from sqlalchemy.orm import DeclarativeBase, sessionmaker +import uuid from .paths import database_path, sqlite_database_url @@ -32,11 +33,25 @@ class Base(DeclarativeBase): def ensure_sources_column(engine): try: - with engine.connect() as conn: + with engine.begin() as conn: cols = [row[1] for row in conn.execute(text("PRAGMA table_info(chat_messages)"))] + if "message_id" not in cols: + conn.execute(text("ALTER TABLE chat_messages ADD COLUMN message_id TEXT")) if "sources_json" not in cols: conn.execute(text("ALTER TABLE chat_messages ADD COLUMN sources_json TEXT DEFAULT '[]'")) if "attachments_json" not in cols: conn.execute(text("ALTER TABLE chat_messages ADD COLUMN attachments_json TEXT DEFAULT '[]'")) + missing_ids = conn.execute( + text("SELECT id FROM chat_messages WHERE message_id IS NULL OR message_id = ''") + ).fetchall() + for row in missing_ids: + conn.execute( + text("UPDATE chat_messages SET message_id = :message_id WHERE id = :id"), + {"message_id": str(uuid.uuid4()), "id": row[0]}, + ) + conn.execute(text( + "CREATE UNIQUE INDEX IF NOT EXISTS ix_chat_messages_message_id " + "ON chat_messages (message_id)" + )) except Exception as e: print("[db] ensure_sources_column error:", e) diff --git a/backend/local_rag.py b/backend/local_rag.py index 5bfa774..195d35d 100644 --- a/backend/local_rag.py +++ b/backend/local_rag.py @@ -233,7 +233,7 @@ def _source_signature(files: List[Dict[str, Any]]) -> Optional[str]: "rel": entry.get("rel") or "", "size": int(entry.get("size") or 0), } - if entry.get("managed") or entry.get("kind") in {"text", "website"}: + if entry.get("managed") or entry.get("kind") in {"text", "website", "chat_message"}: payload.update({ "item_id": entry.get("item_id") or "", "kind": entry.get("kind") or "file", @@ -1394,10 +1394,15 @@ def get_library_item(slug: str, item_id: str): raise HTTPException(status_code=404, detail="Content item not found") payload = dict(entry) - if entry.get("kind") in {"text", "website"}: + if entry.get("kind") in {"text", "website", "chat_message"}: path = Path(str(entry.get("path") or "")) payload["content"] = path.read_text(encoding="utf-8") if path.exists() else "" - payload["content_origin"] = "stored_snapshot" if entry.get("kind") == "website" else "stored_text" + if entry.get("kind") == "website": + payload["content_origin"] = "stored_snapshot" + elif entry.get("kind") == "chat_message": + payload["content_origin"] = "stored_chat_message" + else: + payload["content_origin"] = "stored_text" payload["content_truncated"] = False else: content, truncated = _read_indexed_file_text(slug, entry) @@ -1443,11 +1448,87 @@ async def create_library_text(slug: str, req: CreateTextRequest): return {"item": entry, "job_id": job_id, "library": library_payload(data)} +async def save_chat_message_snapshot( + slug: str, + *, + title: str, + content: str, + source_message_id: str, + source_session_id: str, + source_session_title: str, + source_role: str, + source_created_at: Optional[str], + sources: Optional[List[str]] = None, + saved_by: str = "user", +) -> Dict[str, Any]: + data = read_library(slug) + existing = next( + ( + entry for entry in data.get("files", []) + if entry.get("kind") == "chat_message" + and entry.get("source_message_id") == source_message_id + ), + None, + ) + if existing: + return { + "item": existing, + "job_id": None, + "already_saved": True, + "library": library_payload(data), + } + + clean_title = str(title or "").strip() + clean_content = str(content or "").strip() + if not clean_title: + raise HTTPException(status_code=400, detail="A title is required.") + if not clean_content: + raise HTTPException(status_code=400, detail="Message content is required.") + + item_id = uuid.uuid4().hex + source_path = _managed_source_path(slug, item_id) + rel = _managed_stage_rel(item_id) + _write_text_atomic(source_path, clean_content) + _ensure_stage_link(slug, source_path, rel) + timestamp = now_iso() + entry = { + "item_id": item_id, + "kind": "chat_message", + "title": clean_title[:240], + "name": clean_title[:240], + "path": str(source_path), + "rel": rel, + "sha256": _sha256_file(source_path), + "size": source_path.stat().st_size, + "added_at": timestamp, + "updated_at": timestamp, + "sync_status": "pending", + "enrich_enabled": False, + "metadata": {"status": "pending"}, + "managed": True, + "source_role": source_role, + "source_message_id": source_message_id, + "source_session_id": source_session_id, + "source_session_title": source_session_title, + "source_created_at": source_created_at, + "sources": list(sources or []), + "saved_by": saved_by, + } + data.setdefault("files", []).append(entry) + job_id = await _save_library_change(slug, data) + return { + "item": entry, + "job_id": job_id, + "already_saved": False, + "library": library_payload(data), + } + + @router.patch("/libraries/{slug}/texts/{item_id}") async def update_library_text(slug: str, item_id: str, req: UpdateTextRequest): data = read_library(slug) entry = _find_item(data, item_id) - if not entry or entry.get("kind") != "text": + if not entry or entry.get("kind") not in {"text", "chat_message"}: raise HTTPException(status_code=404, detail="Text item not found") title = str(req.title or "").strip() diff --git a/backend/main.py b/backend/main.py index 364d138..680b360 100644 --- a/backend/main.py +++ b/backend/main.py @@ -14,7 +14,7 @@ import tempfile from pathlib import Path from . import models, schemas from .database import Base, engine, SessionLocal, ensure_sources_column -from .local_rag import router as local_rag_router +from .local_rag import router as local_rag_router, save_chat_message_snapshot from .ollama_admin import inspect_ollama_startup, prepare_startup_models, pull_local_model, start_local_ollama from .ollama_client import ( list_model_catalog as ollama_list_model_catalog, @@ -517,7 +517,12 @@ def _row_to_history_message(row: models.ChatMessage) -> dict: sources = [] attachments = _load_message_attachments(getattr(row, "attachments_json", None)) - payload = {"role": row.role, "content": row.content, "sources": sources} + payload = { + "message_id": row.message_id, + "role": row.role, + "content": row.content, + "sources": sources, + } if attachments: payload["attachments"] = [_attachment_history_payload(attachment) for attachment in attachments] return payload @@ -736,6 +741,74 @@ def history(session_id: str, db: Session = Depends(get_db)): msgs = [_row_to_history_message(r) for r in rows] return {"messages": msgs} + +def _default_knowledge_title(content: str, role: str) -> str: + clean = re.sub(r"\s+", " ", str(content or "")).strip() + if len(clean) > 90: + clean = clean[:87].rstrip() + "..." + prefix = "User note" if role == "user" else "Saved answer" + return f"{prefix}: {clean}" if clean else prefix + + +@app.post("/libraries/{slug}/chat-messages") +async def save_message_to_knowledge( + slug: str, + req: schemas.SaveMessageToKnowledgeRequest, + db: Session = Depends(get_db), +): + row = ( + db.query(models.ChatMessage) + .filter(models.ChatMessage.message_id == str(req.message_id or "").strip()) + .first() + ) + if not row: + raise HTTPException(status_code=404, detail="Chat message not found.") + + session = db.query(models.ChatSession).filter(models.ChatSession.id == row.session_pk).first() + if not session: + raise HTTPException(status_code=404, detail="Chat session not found.") + + sources = [] + try: + sources = json.loads(row.sources_json or "[]") + except Exception: + sources = [] + + previous_user = None + if row.role == "assistant": + previous_user = ( + db.query(models.ChatMessage) + .filter( + models.ChatMessage.session_pk == row.session_pk, + models.ChatMessage.role == "user", + models.ChatMessage.id < row.id, + ) + .order_by(models.ChatMessage.id.desc()) + .first() + ) + + default_content = row.content + if row.role == "assistant" and previous_user: + default_content = ( + f"User question:\n{previous_user.content.strip()}\n\n" + f"Assistant response:\n{row.content.strip()}" + ) + + content = str(req.content if req.content is not None else default_content).strip() + title = str(req.title or _default_knowledge_title(row.content, row.role)).strip() + return await save_chat_message_snapshot( + slug, + title=title, + content=content, + source_message_id=row.message_id, + source_session_id=session.session_id, + source_session_title=sanitize_chat_title(session.name), + source_role=row.role, + source_created_at=row.created_at.isoformat() if row.created_at else None, + sources=[str(source) for source in sources if str(source).strip()], + saved_by="user", + ) + @app.post("/chat") async def chat(req: schemas.ChatRequest, db: Session = Depends(get_db)): # Find or create session diff --git a/backend/models.py b/backend/models.py index 5f4384e..8e352b7 100644 --- a/backend/models.py +++ b/backend/models.py @@ -1,6 +1,7 @@ from sqlalchemy import Column, Integer, String, Text, DateTime, ForeignKey from sqlalchemy.orm import relationship from datetime import datetime +import uuid from .database import Base class ChatSession(Base): @@ -17,6 +18,7 @@ class ChatMessage(Base): __tablename__ = 'chat_messages' id = Column(Integer, primary_key=True, index=True) + message_id = Column(String(36), unique=True, index=True, nullable=False, default=lambda: str(uuid.uuid4())) session_pk = Column(Integer, ForeignKey('chat_sessions.id'), nullable=False) role = Column(String(16), nullable=False) # 'user' | 'assistant' content = Column(Text, nullable=False) diff --git a/backend/schemas.py b/backend/schemas.py index 69079fb..824dff4 100644 --- a/backend/schemas.py +++ b/backend/schemas.py @@ -13,6 +13,7 @@ class ChatAttachment(BaseModel): text: Optional[str] = None class Message(BaseModel): + message_id: Optional[str] = None role: str content: str sources: Optional[List[str]] = None @@ -35,6 +36,12 @@ class ChatResponse(BaseModel): class HistoryResponse(BaseModel): messages: List[Message] + +class SaveMessageToKnowledgeRequest(BaseModel): + message_id: str + title: Optional[str] = None + content: Optional[str] = None + class GenerateTitleRequest(BaseModel): session_id: str message: str