diff --git a/README.md b/README.md index 3a32944..7d72228 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # llm_web -Local LLM chat web page powered by Ollama and `qwen2.5:7b`. +Local LLM chat web page powered by Ollama. ## Run @@ -23,3 +23,26 @@ http://:8787 ``` Chat history is stored locally in `data/chats.json`. The `data/*.json` files are ignored by Git. + +## Models + +The web UI can switch between installed Ollama models: + +- `qwen2.5:7b`: default, best speed/quality balance on this notebook +- `llama3.1:8b`: available for comparison +- `qwen2.5:14b`: test-only; can be slow or memory hungry on 16 GB RAM +- `qwen2.5:3b`: fallback model + +Recent quick benchmark: + +```text +qwen2.5:7b 18.0s +llama3.1:8b 32.4s +qwen2.5:14b 82.7s +``` + +## RAG + +RAG is enabled by default. The server indexes local `.md`, `.txt`, `.py`, `.js`, `.html`, `.css`, and `.ps1` files, excluding `.git`, `.venv`, `__pycache__`, and `data`. + +Use the `Reindex files` button after adding or editing reference files. diff --git a/app.js b/app.js index 2005f37..a618ff6 100644 --- a/app.js +++ b/app.js @@ -5,11 +5,16 @@ const sendButton = document.querySelector("#sendButton"); const newChatButton = document.querySelector("#newChat"); const historyEl = document.querySelector("#history"); const chatTitleEl = document.querySelector("#chatTitle"); +const modelSelect = document.querySelector("#modelSelect"); +const ragToggle = document.querySelector("#ragToggle"); +const ragStatus = document.querySelector("#ragStatus"); +const reindexButton = document.querySelector("#reindexButton"); +const metaLine = document.querySelector("#metaLine"); let activeChatId = null; let messages = []; -function addMessage(role, content) { +function addMessage(role, content, meta = "") { const article = document.createElement("article"); article.className = `message ${role}`; @@ -21,6 +26,13 @@ function addMessage(role, content) { bubble.className = "bubble"; bubble.textContent = content; + if (meta) { + const metaEl = document.createElement("div"); + metaEl.className = "message-meta"; + metaEl.textContent = meta; + bubble.append(document.createElement("br"), metaEl); + } + article.append(avatar, bubble); messagesEl.append(article); messagesEl.scrollTop = messagesEl.scrollHeight; @@ -31,7 +43,7 @@ function renderMessages() { messagesEl.innerHTML = ""; if (!messages.length) { - addMessage("assistant", "안녕하세요. 이 노트북에서 실행되는 로컬 LLM입니다."); + addMessage("assistant", "Local LLM is ready. Ask in Korean or English."); return; } @@ -42,6 +54,32 @@ function setTitle(title) { chatTitleEl.textContent = title || "Local LLM"; } +function selectedModel() { + return modelSelect.value || "qwen2.5:7b"; +} + +async function loadModels() { + const response = await fetch("/api/models"); + const data = await response.json(); + modelSelect.innerHTML = ""; + + data.models.forEach((model) => { + const option = document.createElement("option"); + option.value = model; + option.textContent = model; + if (model === data.defaultModel) { + option.selected = true; + } + modelSelect.append(option); + }); +} + +async function loadRagStatus() { + const response = await fetch("/api/rag/status"); + const data = await response.json(); + ragStatus.textContent = `RAG: ${data.files} files, ${data.chunks} chunks`; +} + async function loadHistory() { const response = await fetch("/api/chats"); const data = await response.json(); @@ -62,6 +100,8 @@ async function loadChat(chatId) { const data = await response.json(); activeChatId = data.chat.id; messages = data.chat.messages || []; + modelSelect.value = data.chat.model || selectedModel(); + ragToggle.checked = data.chat.useRag !== false; setTitle(data.chat.title); renderMessages(); await loadHistory(); @@ -91,6 +131,18 @@ promptInput.addEventListener("keydown", (event) => { newChatButton.addEventListener("click", resetChat); +reindexButton.addEventListener("click", async () => { + reindexButton.disabled = true; + ragStatus.textContent = "RAG: indexing"; + try { + const response = await fetch("/api/rag/reindex", { method: "POST" }); + const data = await response.json(); + ragStatus.textContent = `RAG: ${data.files} files, ${data.chunks} chunks`; + } finally { + reindexButton.disabled = false; + } +}); + form.addEventListener("submit", async (event) => { event.preventDefault(); @@ -105,32 +157,49 @@ form.addEventListener("submit", async (event) => { messages.push({ role: "user", content }); sendButton.disabled = true; - const pending = addMessage("assistant", "생각 중..."); + const pending = addMessage("assistant", "Thinking..."); try { const response = await fetch("/api/chat", { method: "POST", headers: { "Content-Type": "application/json" }, - body: JSON.stringify({ chatId: activeChatId, messages }), + body: JSON.stringify({ + chatId: activeChatId, + messages, + model: selectedModel(), + useRag: ragToggle.checked, + }), }); const data = await response.json(); if (!response.ok) { - throw new Error(data.error || "응답 생성 실패"); + throw new Error(data.error || "Response failed"); } + const sourceText = data.sources?.length + ? `sources: ${data.sources.map((item) => item.source).join(", ")}` + : "no sources"; pending.textContent = data.reply; + + const meta = document.createElement("div"); + meta.className = "message-meta"; + meta.textContent = `${data.model} / ${(data.elapsedMs / 1000).toFixed(1)}s / ${sourceText}`; + pending.append(document.createElement("br"), meta); + activeChatId = data.chatId; messages = data.chat.messages || [...messages, { role: "assistant", content: data.reply }]; setTitle(data.chat.title); + metaLine.textContent = `${data.model} with RAG ${ragToggle.checked ? "on" : "off"}`; await loadHistory(); } catch (error) { - pending.textContent = `오류: ${error.message}`; + pending.textContent = `Error: ${error.message}`; } finally { sendButton.disabled = false; promptInput.focus(); } }); +loadModels(); +loadRagStatus(); renderMessages(); loadHistory(); diff --git a/benchmark_models.py b/benchmark_models.py new file mode 100644 index 0000000..376f4a6 --- /dev/null +++ b/benchmark_models.py @@ -0,0 +1,35 @@ +from __future__ import annotations + +import time + +import ollama + + +MODELS = ["qwen2.5:7b", "llama3.1:8b", "qwen2.5:14b"] +PROMPT = ( + "Answer in Korean in exactly three bullets. Compare your usefulness for " + "local coding assistance, Korean Q&A, and careful reasoning." +) + + +def main() -> None: + for model in MODELS: + started = time.perf_counter() + try: + response = ollama.chat( + model=model, + messages=[ + {"role": "system", "content": "Be concise and practical."}, + {"role": "user", "content": PROMPT}, + ], + ) + elapsed = time.perf_counter() - started + text = response["message"]["content"].strip().replace("\n", " ") + print(f"{model}\t{elapsed:.1f}s\t{text[:180]}") + except Exception as exc: + elapsed = time.perf_counter() - started + print(f"{model}\t{elapsed:.1f}s\tERROR: {exc}") + + +if __name__ == "__main__": + main() diff --git a/index.html b/index.html index 28320b8..8ba87c4 100644 --- a/index.html +++ b/index.html @@ -8,24 +8,31 @@
Local LLM - Ollama on this notebook + Ollama on this notebook
- - + +
diff --git a/server.py b/server.py index a8cbd0a..f4b5368 100644 --- a/server.py +++ b/server.py @@ -1,11 +1,16 @@ from __future__ import annotations import json +import math +import re import socket +import time import uuid +from collections import Counter from datetime import datetime, timezone from http.server import SimpleHTTPRequestHandler, ThreadingHTTPServer from pathlib import Path +from urllib.parse import urlparse import ollama @@ -13,9 +18,15 @@ import ollama ROOT = Path(__file__).parent DATA_DIR = ROOT / "data" CHATS_FILE = DATA_DIR / "chats.json" -MODEL = "qwen2.5:7b" +DEFAULT_MODEL = "qwen2.5:7b" +AVAILABLE_MODELS = ["qwen2.5:7b", "llama3.1:8b", "qwen2.5:14b", "qwen2.5:3b"] HOST = "0.0.0.0" PORT = 8787 +RAG_EXTENSIONS = {".md", ".txt", ".py", ".js", ".html", ".css", ".ps1"} +RAG_EXCLUDED_DIRS = {".git", ".venv", "__pycache__", "data"} + +rag_chunks: list[dict] = [] +rag_indexed_at = "" def now_iso() -> str: @@ -41,8 +52,114 @@ def chat_title(messages: list[dict]) -> str: for message in messages: if message.get("role") == "user" and message.get("content"): title = " ".join(str(message["content"]).split()) - return title[:40] or "새 대화" - return "새 대화" + return title[:40] or "New chat" + return "New chat" + + +def installed_models() -> list[str]: + try: + response = ollama.list() + except Exception: + return AVAILABLE_MODELS + + names: list[str] = [] + for model in response.get("models", []): + if isinstance(model, dict): + name = model.get("name") or model.get("model") + else: + name = getattr(model, "name", None) or getattr(model, "model", None) + if name: + names.append(name) + + ordered = [name for name in AVAILABLE_MODELS if name in names] + ordered.extend(name for name in names if name not in ordered) + return ordered or AVAILABLE_MODELS + + +def tokenize(text: str) -> list[str]: + return [token for token in re.findall(r"\w+", text.lower(), flags=re.UNICODE) if len(token) > 1] + + +def iter_rag_files() -> list[Path]: + files: list[Path] = [] + for path in ROOT.rglob("*"): + if any(part in RAG_EXCLUDED_DIRS for part in path.relative_to(ROOT).parts): + continue + if path.is_file() and path.suffix.lower() in RAG_EXTENSIONS: + files.append(path) + return sorted(files) + + +def chunk_text(path: Path, text: str, size: int = 1800, overlap: int = 250) -> list[dict]: + chunks: list[dict] = [] + start = 0 + rel = path.relative_to(ROOT).as_posix() + while start < len(text): + end = min(start + size, len(text)) + chunk = text[start:end].strip() + if chunk: + chunks.append({"source": rel, "text": chunk, "tokens": Counter(tokenize(chunk))}) + if end == len(text): + break + start = max(0, end - overlap) + return chunks + + +def build_rag_index() -> dict: + global rag_chunks, rag_indexed_at + + chunks: list[dict] = [] + for path in iter_rag_files(): + try: + text = path.read_text(encoding="utf-8", errors="replace") + except OSError: + continue + chunks.extend(chunk_text(path, text)) + + rag_chunks = chunks + rag_indexed_at = now_iso() + return rag_status() + + +def rag_status() -> dict: + return { + "chunks": len(rag_chunks), + "indexedAt": rag_indexed_at, + "files": len({chunk["source"] for chunk in rag_chunks}), + } + + +def search_rag(query: str, limit: int = 5) -> list[dict]: + if not rag_chunks: + build_rag_index() + + query_tokens = Counter(tokenize(query)) + if not query_tokens: + return [] + + scored: list[tuple[float, dict]] = [] + total = max(1, len(rag_chunks)) + doc_freq = Counter() + for chunk in rag_chunks: + for token in chunk["tokens"]: + doc_freq[token] += 1 + + for chunk in rag_chunks: + score = 0.0 + chunk_tokens = chunk["tokens"] + for token, query_count in query_tokens.items(): + if token not in chunk_tokens: + continue + idf = math.log((total + 1) / (doc_freq[token] + 1)) + 1 + score += min(query_count, chunk_tokens[token]) * idf + if score: + scored.append((score, chunk)) + + scored.sort(key=lambda item: item[0], reverse=True) + return [ + {"source": chunk["source"], "text": chunk["text"], "score": round(score, 3)} + for score, chunk in scored[:limit] + ] class Handler(SimpleHTTPRequestHandler): @@ -50,21 +167,35 @@ class Handler(SimpleHTTPRequestHandler): super().__init__(*args, directory=str(ROOT), **kwargs) def do_GET(self) -> None: - if self.path == "/api/chats": + path = urlparse(self.path).path + + if path == "/api/models": + self._send_json(200, {"defaultModel": DEFAULT_MODEL, "models": installed_models()}) + return + + if path == "/api/rag/status": + if not rag_chunks: + build_rag_index() + self._send_json(200, rag_status()) + return + + if path == "/api/chats": store = read_store() chats = [ { "id": chat["id"], - "title": chat.get("title", "새 대화"), + "title": chat.get("title", "New chat"), "updatedAt": chat.get("updatedAt", ""), + "model": chat.get("model", DEFAULT_MODEL), + "useRag": chat.get("useRag", True), } for chat in store.get("chats", []) ] self._send_json(200, {"chats": chats}) return - if self.path.startswith("/api/chats/"): - chat_id = self.path.rsplit("/", 1)[-1] + if path.startswith("/api/chats/"): + chat_id = path.rsplit("/", 1)[-1] store = read_store() chat = next((item for item in store.get("chats", []) if item["id"] == chat_id), None) if not chat: @@ -77,7 +208,12 @@ class Handler(SimpleHTTPRequestHandler): super().do_GET() def do_POST(self) -> None: - if self.path != "/api/chat": + path = urlparse(self.path).path + if path == "/api/rag/reindex": + self._send_json(200, build_rag_index()) + return + + if path != "/api/chat": self.send_error(404) return @@ -85,38 +221,59 @@ class Handler(SimpleHTTPRequestHandler): data = json.loads(self.rfile.read(length) or b"{}") chat_id = data.get("chatId") messages = data.get("messages", []) + model = data.get("model") or DEFAULT_MODEL + use_rag = bool(data.get("useRag", True)) try: + sources = search_rag(messages[-1]["content"]) if use_rag and messages else [] + system_prompt = ( + "You are a concise local assistant. Reply in Korean unless the user asks otherwise. " + "If reference context is provided, use it and mention source filenames when useful. " + "If the context is not enough, say so clearly." + ) + if sources: + context = "\n\n".join( + f"[{index + 1}] {item['source']}\n{item['text']}" for index, item in enumerate(sources) + ) + system_prompt += f"\n\nReference context:\n{context}" + + started = time.perf_counter() response = ollama.chat( - model=MODEL, + model=model, messages=[ - { - "role": "system", - "content": "You are a concise local assistant. Reply in Korean unless the user asks otherwise.", - }, + {"role": "system", "content": system_prompt}, *messages, ], ) + elapsed_ms = round((time.perf_counter() - started) * 1000) reply = response["message"]["content"] saved_messages = [*messages, {"role": "assistant", "content": reply}] - chat = self._save_chat(chat_id, saved_messages) - payload = {"chatId": chat["id"], "reply": reply, "chat": chat} + chat = self._save_chat(chat_id, saved_messages, model, use_rag) + payload = { + "chatId": chat["id"], + "reply": reply, + "chat": chat, + "model": model, + "elapsedMs": elapsed_ms, + "sources": [{"source": item["source"], "score": item["score"]} for item in sources], + } self._send_json(200, payload) except Exception as exc: self._send_json(500, {"error": str(exc)}) def do_DELETE(self) -> None: - if not self.path.startswith("/api/chats/"): + path = urlparse(self.path).path + if not path.startswith("/api/chats/"): self.send_error(404) return - chat_id = self.path.rsplit("/", 1)[-1] + chat_id = path.rsplit("/", 1)[-1] store = read_store() store["chats"] = [item for item in store.get("chats", []) if item["id"] != chat_id] write_store(store) self._send_json(200, {"ok": True}) - def _save_chat(self, chat_id: str | None, messages: list[dict]) -> dict: + def _save_chat(self, chat_id: str | None, messages: list[dict], model: str, use_rag: bool) -> dict: store = read_store() chats = store.setdefault("chats", []) chat = next((item for item in chats if item["id"] == chat_id), None) @@ -132,12 +289,14 @@ class Handler(SimpleHTTPRequestHandler): chat["title"] = chat_title(messages) chat["messages"] = messages + chat["model"] = model + chat["useRag"] = use_rag chat["updatedAt"] = now_iso() chats.sort(key=lambda item: item.get("updatedAt", ""), reverse=True) write_store(store) return chat - def _send_json(self, status: int, payload: dict[str, str]) -> None: + def _send_json(self, status: int, payload: dict) -> None: body = json.dumps(payload, ensure_ascii=False).encode("utf-8") self.send_response(status) self.send_header("content-type", "application/json; charset=utf-8") @@ -147,6 +306,7 @@ class Handler(SimpleHTTPRequestHandler): def main() -> None: + build_rag_index() server = ThreadingHTTPServer((HOST, PORT), Handler) local_ip = socket.gethostbyname(socket.gethostname()) print(f"LLM web is running locally at http://127.0.0.1:{PORT}") diff --git a/styles.css b/styles.css index bced13f..caf9469 100644 --- a/styles.css +++ b/styles.css @@ -34,7 +34,9 @@ body { } .new-chat, -.history-item { +.history-item, +.secondary, +.select { width: 100%; min-height: 42px; border: 1px solid #3a3a3a; @@ -47,8 +49,43 @@ body { cursor: pointer; } +.select { + appearance: none; + background: var(--sidebar-soft); +} + +.secondary { + text-align: center; + background: var(--sidebar-soft); +} + +.secondary:disabled { + cursor: wait; + opacity: 0.6; +} + +.control-label, +.toggle, +.rag-status, +.message-meta { + color: #b8b8b8; + font-size: 12px; +} + +.toggle { + min-height: 32px; + display: flex; + align-items: center; + gap: 8px; +} + +.rag-status { + padding: 0 4px 8px; +} + .history { flex: 1; + overflow: auto; } .history-item.active { @@ -125,6 +162,12 @@ body { overflow-wrap: anywhere; } +.message-meta { + display: inline-block; + margin-top: 8px; + color: var(--muted); +} + .composer { position: fixed; left: 260px;