From 5deea5664eb3d5d99af33c10aa2a6014d85b4390 Mon Sep 17 00:00:00 2001 From: shdrs Date: Mon, 1 Jun 2026 20:19:37 +0800 Subject: [PATCH 001/974] Disable scroll-snap on landing page --- docs/index.html | 14 +++++++++++--- static/landing.html | 14 +++++++++++--- 2 files changed, 22 insertions(+), 6 deletions(-) diff --git a/docs/index.html b/docs/index.html index 8c6a21d..00b37d5 100644 --- a/docs/index.html +++ b/docs/index.html @@ -25,9 +25,17 @@ --radius: 8px; } * { box-sizing: border-box; } - html { scroll-behavior: smooth; scroll-snap-type: y mandatory; scroll-padding-top: 60px; } - /* Each section is a full-viewport "page" with its content centered, so only - one shows at a time and the snap is obvious. */ + html { scroll-behavior: smooth; scroll-padding-top: 60px; } + /* REMOVED: "scroll-snap-type: y mandatory" + The idea was: >>Each section is a full-viewport "page" with its content centered, + so only one shows at a time and the snap is obvious.<< + + PROBLEM: sections easily grow taller than 100vh IRL + This cause forced jumps mid-read. It's intrusive UX. + + Preserved: CSS snap-points to avoid destroying code meta-data + Less intrusive version: "scroll-snap-type: y proximity" + For now: fully removed (bad UX)*/ .hero, section { scroll-snap-align: start; min-height: 100vh; display: flex; flex-direction: column; justify-content: center; diff --git a/static/landing.html b/static/landing.html index f983786..e1f12f7 100644 --- a/static/landing.html +++ b/static/landing.html @@ -25,9 +25,17 @@ --radius: 8px; } * { box-sizing: border-box; } - html { scroll-behavior: smooth; scroll-snap-type: y mandatory; scroll-padding-top: 60px; } - /* Each section is a full-viewport "page" with its content centered, so only - one shows at a time and the snap is obvious. */ + html { scroll-behavior: smooth; scroll-padding-top: 60px; } + /* REMOVED: "scroll-snap-type: y mandatory" + The idea was: >>Each section is a full-viewport "page" with its content centered, + so only one shows at a time and the snap is obvious.<< + + PROBLEM: sections easily grow taller than 100vh IRL + This cause forced jumps mid-read. It's intrusive UX. + + Preserved: CSS snap-points to avoid destroying code meta-data + Less intrusive version: "scroll-snap-type: y proximity" + For now: fully removed (bad UX)*/ .hero, section { scroll-snap-align: start; min-height: 100vh; display: flex; flex-direction: column; justify-content: center; From 88c9f1fa747657e854772e374576273633439809 Mon Sep 17 00:00:00 2001 From: joi-lightyears Date: Fri, 5 Jun 2026 13:17:14 +0700 Subject: [PATCH 002/974] fix(memory): let manual add specify memory category Add a category selector on the Brain Add tab and include it in the /api/memory/add JSON payload instead of always defaulting to fact. Fixes #2784 --- static/index.html | 1 + static/js/memory.js | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/static/index.html b/static/index.html index c5f3828..9a612ab 100644 --- a/static/index.html +++ b/static/index.html @@ -307,6 +307,7 @@ Add a memory — e.g. 'I prefer concise replies' +
diff --git a/static/js/memory.js b/static/js/memory.js index e0f064e..6f3e570 100644 --- a/static/js/memory.js +++ b/static/js/memory.js @@ -18,6 +18,26 @@ let selectedIds = new Set(); const MEMORY_CATEGORIES = ['fact', 'identity', 'preference', 'contact', 'project', 'goal', 'task']; +function _ensureNewMemoryCategorySelect() { + const sel = document.getElementById('new-memory-category'); + if (!sel || sel.dataset.wired === '1') return; + sel.dataset.wired = '1'; + MEMORY_CATEGORIES.forEach(cat => { + const opt = document.createElement('option'); + opt.value = cat; + opt.textContent = cat; + if (cat === 'fact') opt.selected = true; + sel.appendChild(opt); + }); +} + +function _readNewMemoryCategory() { + _ensureNewMemoryCategorySelect(); + const sel = document.getElementById('new-memory-category'); + const cat = sel?.value || 'fact'; + return MEMORY_CATEGORIES.includes(cat) ? cat : 'fact'; +} + let _memoryDragWired = false; function _wireMemoryDrag() { if (_memoryDragWired) return; @@ -274,6 +294,7 @@ async function syncPrefToggle(elementId, prefKey, onMsg, offMsg, dimBelow = true } export async function loadMemories() { + _ensureNewMemoryCategorySelect(); try { const response = await fetch(`${window.location.origin}/api/memory`); @@ -977,6 +998,7 @@ export function updateMemoryCount() { export async function addNewMemory() { const input = document.getElementById('new-memory-input'); const text = input.value.trim(); + const category = _readNewMemoryCategory(); if (!text) { showError('Memory text cannot be empty'); @@ -991,6 +1013,7 @@ export async function addNewMemory() { }, body: JSON.stringify({ text: text, + category: category, }) }); From 0f8d12363ad7df43520db97be12ccdfcf4b0eb55 Mon Sep 17 00:00:00 2001 From: nsgds <161509862+nsgds@users.noreply.github.com> Date: Fri, 5 Jun 2026 19:04:33 +0800 Subject: [PATCH 003/974] fix(images): render agent-generated images in chat (#2809) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * fix(images): render agent-generated images in chat When a chat model calls generate_image mid-conversation (agentic flow), the image does not display — it survives only as a URL the model echoes in prose. generate_image runs as a text-only MCP server, so result['image_url'] is never populated and the existing buildImageBubble render path never fires. Promote the image URL out of the tool's stdout in tool_execution so the agent loop's existing forwarding renders it via buildImageBubble — deterministically, no dependence on the model echoing the URL. Backend-only; reuses dev's image bubble, forwarding, and the tool's existing parseable output. Co-Authored-By: Claude Opus 4.8 (1M context) * feat(images): fully-qualified, valid generated-image links The chat model often mangled the generated-image URL it echoed in prose (relative path, or copying the 'image_url:' label into the link href). Build a fully-qualified link by prefixing the existing app_public_url setting (empty default keeps relative paths), and present it as a clean 'Direct link:' the model can echo verbatim (the frontend auto-links bare https URLs). One file; independent of how the image is rendered. Co-Authored-By: Claude Opus 4.8 (1M context) * test(images): cover _promote_image_fields; make exit-code guard self-contained Adds the unit tests requested in review on #2809: absolute URL, relative URL, no URL (result unchanged), and non-zero exit_code (not promoted). Moves the dict/exit_code==0 guard from the call site into _promote_image_fields so the function is self-contained and the failure case is unit-testable; call-site behavior is unchanged. Co-Authored-By: Claude Opus 4.8 (1M context) --------- Co-authored-by: Claude Opus 4.8 (1M context) --- mcp_servers/image_gen_server.py | 14 ++++++-- src/tool_execution.py | 32 +++++++++++++++++ tests/test_promote_image_fields.py | 57 ++++++++++++++++++++++++++++++ 3 files changed, 101 insertions(+), 2 deletions(-) create mode 100644 tests/test_promote_image_fields.py diff --git a/mcp_servers/image_gen_server.py b/mcp_servers/image_gen_server.py index 872ccd6..4607b08 100644 --- a/mcp_servers/image_gen_server.py +++ b/mcp_servers/image_gen_server.py @@ -115,6 +115,10 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: img = images[0] image_url = None + # Prefix the instance's public base URL (existing app_public_url setting) so the + # link is fully-qualified and clickable when the model echoes it. Empty = relative + # same-origin path (unchanged default). + _pub_base = (get_setting("app_public_url", "") or "").rstrip("/") if img.get("b64_json"): img_dir = Path("data/generated_images") @@ -122,7 +126,7 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: filename = f"{uuid.uuid4().hex[:12]}.png" img_path = img_dir / filename img_path.write_bytes(base64.b64decode(img["b64_json"])) - image_url = f"/api/generated-image/{filename}" + image_url = f"{_pub_base}/api/generated-image/{filename}" # Save to gallery try: @@ -146,7 +150,13 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]: else: return [TextContent(type="text", text="Error: Unexpected image API response format")] - result = f"Generated image for: {prompt[:100]}\nimage_url: {image_url}\nmodel: {model_id}\nsize: {size}" + # "Direct link:" rather than an "image_url:" label — small models copied the + # label token ("image_url") into the link href, producing a broken link. + result = ( + f"Generated image for: {prompt[:100]}\n" + f"Direct link: {image_url}\n" + f"model: {model_id}\nsize: {size}" + ) return [TextContent(type="text", text=result)] except httpx.TimeoutException: diff --git a/src/tool_execution.py b/src/tool_execution.py index 8e44c34..9af6cce 100644 --- a/src/tool_execution.py +++ b/src/tool_execution.py @@ -13,6 +13,7 @@ import json import logging import os import pathlib +import re import sys import time from typing import Any, Awaitable, Callable, Dict, Optional, Tuple @@ -594,9 +595,40 @@ async def _call_mcp_tool( if fallback: return fallback + # generate_image runs as a text-only MCP tool, so the saved image URL never + # reaches the agent loop's structured forwarding (which renders the image via + # buildImageBubble on result["image_url"]). Lift it out of the tool's stdout so + # the image renders deterministically — no dependence on the model echoing the + # URL into its prose (which it mangles/hallucinates). + if tool == "generate_image": + _promote_image_fields(result) + return result +def _promote_image_fields(result: Dict) -> None: + """Lift the image URL (+ prompt/model/size) from a successful generate_image MCP + text result into structured fields the agent loop already forwards to + buildImageBubble. Only acts on a dict result with exit_code 0; matches the + generated-image URL by pattern (absolute or relative) so it's robust to the + result's wording.""" + if not isinstance(result, dict) or result.get("exit_code") != 0: + return + out = result.get("stdout") or "" + m = re.search(r'(?:https?://[^\s)\]]+)?/api/generated-image/[A-Za-z0-9._-]+', out) + if not m: + return + result["image_url"] = m.group(0).strip() + for field, pat in ( + ("image_prompt", r'^Generated image for:\s*(.+)$'), + ("image_model", r'^model:\s*(.+)$'), + ("image_size", r'^size:\s*(.+)$'), + ): + fm = re.search(pat, out, re.M) + if fm: + result[field] = fm.group(1).strip() + + _BG_MARKERS = {"#!bg", "#bg", "# bg", "#background", "# background", "@background", "# @background"} diff --git a/tests/test_promote_image_fields.py b/tests/test_promote_image_fields.py new file mode 100644 index 0000000..1cf4cb0 --- /dev/null +++ b/tests/test_promote_image_fields.py @@ -0,0 +1,57 @@ +"""Unit tests for `_promote_image_fields` (PR #2809). + +`generate_image` is a text-only MCP tool, so the saved image URL never reaches +the agent loop's structured forwarding (which renders the image via +`buildImageBubble` on `result["image_url"]`). `_promote_image_fields` lifts the +URL — plus prompt/model/size — out of the tool's stdout into structured fields so +the image renders deterministically, without relying on the model echoing the URL +into prose. These cases cover the absolute-URL, relative-URL, no-URL, and +non-success-exit paths. +""" +from src.tool_execution import _promote_image_fields + + +def _result(stdout, exit_code=0): + return {"exit_code": exit_code, "stdout": stdout} + + +def test_absolute_url_promoted_with_fields(): + """An absolute https URL in stdout is lifted into image_url, along with the + prompt/model/size lines.""" + r = _result( + "Generated image for: a red fox in snow\n" + "Direct link: https://odysseus.example.com/api/generated-image/abc123.png\n" + "model: qwen-image\n" + "size: 1024x1024" + ) + _promote_image_fields(r) + assert r["image_url"] == "https://odysseus.example.com/api/generated-image/abc123.png" + assert r["image_prompt"] == "a red fox in snow" + assert r["image_model"] == "qwen-image" + assert r["image_size"] == "1024x1024" + + +def test_relative_url_promoted(): + """A relative /api/generated-image/... path (no host) is still matched.""" + r = _result( + "Generated image for: a cat\n" + "Direct link: /api/generated-image/def456.png" + ) + _promote_image_fields(r) + assert r["image_url"] == "/api/generated-image/def456.png" + assert r["image_prompt"] == "a cat" + + +def test_no_url_leaves_result_unchanged(): + """No generated-image URL anywhere -> no image_url key is added.""" + r = _result("Generated image for: a dog\n(no link produced)") + _promote_image_fields(r) + assert "image_url" not in r + assert "image_prompt" not in r + + +def test_nonzero_exit_not_promoted(): + """A non-success result is never promoted, even if stdout contains a URL.""" + r = _result("https://host/api/generated-image/zzz.png", exit_code=1) + _promote_image_fields(r) + assert "image_url" not in r From 2a1febdeef2b26e21508a413d05596651438c449 Mon Sep 17 00:00:00 2001 From: Ocean Bennett <204957658+undergroundrap@users.noreply.github.com> Date: Fri, 5 Jun 2026 07:13:13 -0400 Subject: [PATCH 004/974] fix(actions): scope scheduled model resolution to owner (#2773) --- src/builtin_actions.py | 14 +- tests/test_builtin_actions_owner_scope.py | 154 ++++++++++++++++++++++ 2 files changed, 161 insertions(+), 7 deletions(-) create mode 100644 tests/test_builtin_actions_owner_scope.py diff --git a/src/builtin_actions.py b/src/builtin_actions.py index d532603..b168700 100644 --- a/src/builtin_actions.py +++ b/src/builtin_actions.py @@ -593,9 +593,9 @@ async def action_classify_events(owner: str, **kwargs) -> Tuple[str, bool]: if not events: return "No upcoming events to classify", True - llm_url, llm_model, llm_headers = resolve_endpoint("utility") + llm_url, llm_model, llm_headers = resolve_endpoint("utility", owner=owner) if not llm_url: - llm_url, llm_model, llm_headers = resolve_endpoint("default") + llm_url, llm_model, llm_headers = resolve_endpoint("default", owner=owner) llm_available = bool(llm_url and llm_model) # Pull user memories so the LLM has personal context (relationships, @@ -867,9 +867,9 @@ async def action_learn_sender_signatures(owner: str, **kwargs) -> Tuple[str, boo if not eligible: return "All sender sigs already cached (or no eligible senders)", True - url, model, headers = resolve_endpoint("utility") + url, model, headers = resolve_endpoint("utility", owner=owner) if not url or not model: - url, model, headers = resolve_endpoint("default") + url, model, headers = resolve_endpoint("default", owner=owner) if not url or not model: return "No LLM endpoint available", False @@ -1480,12 +1480,12 @@ async def action_check_email_urgency(owner: str, **kwargs) -> Tuple[str, bool]: # ── 1. Resolve LLM candidates (utility primary + utility fallbacks; fall # through to default chat as a last resort). - url, model, headers = resolve_endpoint("utility") + url, model, headers = resolve_endpoint("utility", owner=owner) if not url or not model: - url, model, headers = resolve_endpoint("default") + url, model, headers = resolve_endpoint("default", owner=owner) if not url or not model: return "No LLM endpoint available", False - candidates = [(url, model, headers)] + resolve_utility_fallback_candidates() + candidates = [(url, model, headers)] + resolve_utility_fallback_candidates(owner=owner) # ── 2. Enumerate enabled accounts. Match this task's owner AND fall # back to the legacy "unowned account whose imap_user / from_address diff --git a/tests/test_builtin_actions_owner_scope.py b/tests/test_builtin_actions_owner_scope.py new file mode 100644 index 0000000..446aba8 --- /dev/null +++ b/tests/test_builtin_actions_owner_scope.py @@ -0,0 +1,154 @@ +"""Regression tests for owner-scoped model resolution in scheduled actions.""" + +from datetime import datetime +from types import SimpleNamespace + +import pytest + + +class _Column: + def __eq__(self, _other): + return True + + def __ne__(self, _other): + return True + + def __ge__(self, _other): + return True + + def __le__(self, _other): + return True + + +class _Query: + def __init__(self, rows): + self._rows = rows + + def filter(self, *_args, **_kwargs): + return self + + def limit(self, _limit): + return self + + def all(self): + return list(self._rows) + + +class _Db: + def __init__(self, rows_by_model): + self._rows_by_model = rows_by_model + self.commits = 0 + self.closed = False + + def query(self, model): + return _Query(self._rows_by_model.get(model, [])) + + def commit(self): + self.commits += 1 + + def close(self): + self.closed = True + + +def _resolver_spy(monkeypatch, utility_result=("", "", {}), default_result=("http://llm", "model", {})): + from src import endpoint_resolver + + calls = [] + fallback_calls = [] + + def fake_resolve(kind, *args, **kwargs): + calls.append((kind, kwargs.get("owner"))) + return utility_result if kind == "utility" else default_result + + def fake_fallbacks(*args, **kwargs): + fallback_calls.append(kwargs.get("owner")) + return [] + + monkeypatch.setattr(endpoint_resolver, "resolve_endpoint", fake_resolve) + monkeypatch.setattr(endpoint_resolver, "resolve_utility_fallback_candidates", fake_fallbacks) + return calls, fallback_calls + + +@pytest.mark.asyncio +async def test_classify_events_resolves_llm_for_task_owner(monkeypatch): + from core import database + from src.builtin_actions import action_classify_events + + class FakeCalendarEvent: + dtstart = _Column() + status = _Column() + + event = SimpleNamespace( + summary="Demo presentation", + event_type="work", + importance="high", + color=None, + dtstart=datetime(2026, 1, 1, 9, 0, 0), + location="", + ) + db = _Db({FakeCalendarEvent: [event]}) + calls, _fallback_calls = _resolver_spy(monkeypatch, utility_result=("http://llm", "model", {})) + + monkeypatch.setattr(database, "CalendarEvent", FakeCalendarEvent) + monkeypatch.setattr(database, "SessionLocal", lambda: db) + + message, ok = await action_classify_events("alice") + + assert ok is True + assert "Scanned 1 upcoming event" in message + assert calls == [("utility", "alice")] + assert db.closed is True + + +@pytest.mark.asyncio +async def test_learn_sender_signatures_resolves_llm_for_task_owner(monkeypatch): + from routes import email_helpers + from src.builtin_actions import action_learn_sender_signatures + + class FakeImap: + def select(self, *_args, **_kwargs): + return "OK", [] + + def search(self, *_args, **_kwargs): + return "OK", [b"1 2 3"] + + def fetch(self, _uid, _query): + return "OK", [(None, b"From: Writer \r\n\r\n")] + + def logout(self): + return None + + calls, _fallback_calls = _resolver_spy(monkeypatch, utility_result=("", "", {}), default_result=("", "", {})) + monkeypatch.setattr(email_helpers, "_imap_connect", lambda _account_id=None: FakeImap()) + + message, ok = await action_learn_sender_signatures("alice") + + assert ok is False + assert message == "No LLM endpoint available" + assert calls == [("utility", "alice"), ("default", "alice")] + + +@pytest.mark.asyncio +async def test_check_email_urgency_resolves_llm_candidates_for_task_owner(monkeypatch, tmp_path): + from core import database + from src.builtin_actions import TaskNoop, action_check_email_urgency + + class FakeEmailAccount: + enabled = _Column() + owner = _Column() + imap_user = _Column() + from_address = _Column() + + db = _Db({FakeEmailAccount: []}) + calls, fallback_calls = _resolver_spy(monkeypatch, utility_result=("http://llm", "model", {})) + + monkeypatch.chdir(tmp_path) + monkeypatch.setattr(database, "EmailAccount", FakeEmailAccount) + monkeypatch.setattr(database, "SessionLocal", lambda: db) + + with pytest.raises(TaskNoop, match="no email accounts configured"): + await action_check_email_urgency("alice") + + assert calls == [("utility", "alice")] + assert fallback_calls == ["alice"] + assert db.closed is True From 688194113b5d341c698fa315bf323ac4c7f1d67b Mon Sep 17 00:00:00 2001 From: Vykos Date: Fri, 5 Jun 2026 13:15:23 +0200 Subject: [PATCH 005/974] Constrain upload paths to upload root (#2825) --- routes/upload_routes.py | 83 +++++++++++++--------- tests/test_upload_routes_owner_scope.py | 94 +++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 33 deletions(-) diff --git a/routes/upload_routes.py b/routes/upload_routes.py index 4f55b50..f348453 100644 --- a/routes/upload_routes.py +++ b/routes/upload_routes.py @@ -13,9 +13,43 @@ from src.upload_handler import count_recent_uploads logger = logging.getLogger(__name__) router = APIRouter(prefix="/api/upload", tags=["upload"]) +UPLOAD_RESPONSE_HEADERS = {"X-Content-Type-Options": "nosniff"} def setup_upload_routes(upload_handler): """Setup upload routes with the provided handler""" + + def _upload_root() -> str: + from src.constants import UPLOAD_DIR + return os.path.realpath(getattr(upload_handler, "upload_dir", UPLOAD_DIR)) + + def _path_inside_upload_dir(path: str) -> bool: + try: + return os.path.commonpath([_upload_root(), os.path.realpath(path)]) == _upload_root() + except Exception: + return False + + def _resolve_upload_path(file_id: str) -> str: + from src.constants import UPLOAD_DIR + upload_root = getattr(upload_handler, "upload_dir", UPLOAD_DIR) + direct = os.path.join(upload_root, file_id) + if os.path.lexists(direct): + if not _path_inside_upload_dir(direct): + raise HTTPException(403, "Access denied") + if os.path.isfile(direct): + return direct + raise HTTPException(404, "File not found") + + for root, _dirs, files in os.walk(upload_root, followlinks=False): + if file_id not in files: + continue + path = os.path.join(root, file_id) + if not _path_inside_upload_dir(path): + raise HTTPException(403, "Access denied") + if os.path.isfile(path): + return path + raise HTTPException(404, "File not found") + + raise HTTPException(404, "File not found") @router.post("") async def api_upload(request: Request, files: List[UploadFile] = File(...)): @@ -91,23 +125,11 @@ def setup_upload_routes(upload_handler): client isn't downloading the full-resolution photo just to show it tiny.""" if not upload_handler.validate_upload_id(file_id): raise HTTPException(400, "Invalid file ID") - # Search upload directories for the file - from src.constants import UPLOAD_DIR import mimetypes as _mt - path = os.path.join(UPLOAD_DIR, file_id) - if not os.path.exists(path): - for root, dirs, files in os.walk(UPLOAD_DIR): - if file_id in files: - path = os.path.join(root, file_id) - break - else: - raise HTTPException(404, "File not found") - if not upload_handler.inside_base_dir(path): - raise HTTPException(403, "Access denied") # Look up original filename and owner from uploads.json original_name = file_id info = None - uploads_db = os.path.join(UPLOAD_DIR, "uploads.json") + uploads_db = os.path.join(_upload_root(), "uploads.json") if os.path.exists(uploads_db): with open(uploads_db, encoding="utf-8") as f: db = json.load(f) @@ -123,13 +145,14 @@ def setup_upload_routes(upload_handler): raise HTTPException(403, "Access denied") if file_owner != current_user and not auth_mgr.is_admin(current_user): raise HTTPException(404, "File not found") - mime = _mt.guess_type(path)[0] or "application/octet-stream" + path = _resolve_upload_path(file_id) + mime = (info or {}).get("mime") or _mt.guess_type(path)[0] or "application/octet-stream" from fastapi.responses import FileResponse # Downscaled thumbnail for image previews — generated once and cached. if thumb and mime.startswith("image/"): try: from PIL import Image, ImageOps - thumb_dir = os.path.join(UPLOAD_DIR, ".thumbs") + thumb_dir = os.path.join(_upload_root(), ".thumbs") os.makedirs(thumb_dir, exist_ok=True) thumb_path = os.path.join(thumb_dir, file_id + ".jpg") if (not os.path.exists(thumb_path) @@ -145,17 +168,21 @@ def setup_upload_routes(upload_handler): if im.mode not in ("RGB", "L"): im = im.convert("RGB") im.save(thumb_path, "JPEG", quality=80) - return FileResponse(thumb_path, media_type="image/jpeg") + return FileResponse(thumb_path, media_type="image/jpeg", headers=UPLOAD_RESPONSE_HEADERS) except Exception as e: logger.warning(f"Thumbnail generation failed for {file_id}: {e}") # Fall through to the full image. - return FileResponse(path, media_type=mime, filename=original_name) + return FileResponse( + path, + media_type=mime, + filename=original_name, + headers=UPLOAD_RESPONSE_HEADERS, + ) def _load_upload_info(file_id: str): """Look up the uploads.json record for a file_id, with owner/auth checks.""" - from src.constants import UPLOAD_DIR info = None - uploads_db = os.path.join(UPLOAD_DIR, "uploads.json") + uploads_db = os.path.join(_upload_root(), "uploads.json") if os.path.exists(uploads_db): with open(uploads_db, encoding="utf-8") as f: db = json.load(f) @@ -163,8 +190,7 @@ def setup_upload_routes(upload_handler): return info def _vision_cache_path(file_id: str) -> str: - from src.constants import UPLOAD_DIR - cache_dir = os.path.join(UPLOAD_DIR, ".vision") + cache_dir = os.path.join(_upload_root(), ".vision") os.makedirs(cache_dir, exist_ok=True) return os.path.join(cache_dir, file_id + ".txt") @@ -175,17 +201,6 @@ def setup_upload_routes(upload_handler): subsequent loads are instant. Pass force=1 to recompute.""" if not upload_handler.validate_upload_id(file_id): raise HTTPException(400, "Invalid file ID") - from src.constants import UPLOAD_DIR - path = os.path.join(UPLOAD_DIR, file_id) - if not os.path.exists(path): - for root, dirs, files in os.walk(UPLOAD_DIR): - if file_id in files: - path = os.path.join(root, file_id) - break - else: - raise HTTPException(404, "File not found") - if not upload_handler.inside_base_dir(path): - raise HTTPException(403, "Access denied") info = _load_upload_info(file_id) auth_mgr = getattr(request.app.state, "auth_manager", None) auth_configured = bool(auth_mgr and auth_mgr.is_configured) @@ -196,8 +211,9 @@ def setup_upload_routes(upload_handler): raise HTTPException(403, "Access denied") if file_owner != current_user and not auth_mgr.is_admin(current_user): raise HTTPException(404, "File not found") + path = _resolve_upload_path(file_id) import mimetypes as _mt - mime = _mt.guess_type(path)[0] or "" + mime = (info or {}).get("mime") or _mt.guess_type(path)[0] or "" if not mime.startswith("image/"): raise HTTPException(400, "Not an image") cache_path = _vision_cache_path(file_id) @@ -238,6 +254,7 @@ def setup_upload_routes(upload_handler): raise HTTPException(403, "Access denied") if file_owner != current_user and not auth_mgr.is_admin(current_user): raise HTTPException(404, "File not found") + _resolve_upload_path(file_id) body = await request.json() text = (body or {}).get("text", "") if not isinstance(text, str): diff --git a/tests/test_upload_routes_owner_scope.py b/tests/test_upload_routes_owner_scope.py index 497c583..a2647f5 100644 --- a/tests/test_upload_routes_owner_scope.py +++ b/tests/test_upload_routes_owner_scope.py @@ -1,6 +1,7 @@ import asyncio import builtins import json +import os from types import SimpleNamespace import pytest @@ -90,6 +91,35 @@ def _guard_cache_open(monkeypatch, cache_path, blocked_modes): monkeypatch.setattr(builtins, "open", guarded_open) +def _add_upload_row(upload_dir, row): + db_path = upload_dir / "uploads.json" + index = json.loads(db_path.read_text(encoding="utf-8")) + index[f"{row.get('owner')}:{row['id']}"] = row + db_path.write_text(json.dumps(index), encoding="utf-8") + + +def _add_upload_symlink(upload_dir, file_id, target_path, owner="alice"): + dated = upload_dir / "2026" / "06" / "02" + link_path = dated / file_id + try: + os.symlink(target_path, link_path) + except (AttributeError, NotImplementedError, OSError) as exc: + pytest.skip(f"symlinks unavailable: {exc}") + _add_upload_row( + upload_dir, + { + "id": file_id, + "path": str(link_path), + "mime": "image/png", + "size": target_path.stat().st_size, + "name": "escape.png", + "original_name": "escape.png", + "owner": owner, + }, + ) + return link_path + + def test_download_file_denies_anonymous_when_auth_is_configured(tmp_path, monkeypatch): handler, alice_id, _bob_id, _upload_dir = _make_upload_store(tmp_path, monkeypatch) download_file = _upload_endpoints(handler, monkeypatch)["download_file"] @@ -120,6 +150,7 @@ def test_download_file_allows_same_owner(tmp_path, monkeypatch): assert response.path.endswith(alice_id) assert response.media_type == "image/png" + assert response.headers["X-Content-Type-Options"] == "nosniff" def test_download_file_allows_admin_to_read_other_owner_upload(tmp_path, monkeypatch): @@ -137,6 +168,44 @@ def test_download_file_allows_admin_to_read_other_owner_upload(tmp_path, monkeyp assert response.media_type == "image/png" +def test_download_file_rejects_upload_symlink_escape(tmp_path, monkeypatch): + handler, _alice_id, _bob_id, upload_dir = _make_upload_store(tmp_path, monkeypatch) + download_file = _upload_endpoints(handler, monkeypatch)["download_file"] + escape_id = "c" * 32 + ".png" + outside = tmp_path / "outside-upload-root.png" + outside.write_bytes(b"outside upload root") + _add_upload_symlink(upload_dir, escape_id, outside) + + with pytest.raises(HTTPException) as exc: + asyncio.run( + download_file( + _Request(user="alice", auth_manager=_AuthManager()), + escape_id, + ) + ) + + assert exc.value.status_code == 403 + + +def test_download_file_keeps_owner_gate_before_path_resolution(tmp_path, monkeypatch): + handler, _alice_id, _bob_id, upload_dir = _make_upload_store(tmp_path, monkeypatch) + download_file = _upload_endpoints(handler, monkeypatch)["download_file"] + bob_escape_id = "d" * 32 + ".png" + outside = tmp_path / "bob-outside-upload-root.png" + outside.write_bytes(b"bob outside upload root") + _add_upload_symlink(upload_dir, bob_escape_id, outside, owner="bob") + + with pytest.raises(HTTPException) as exc: + asyncio.run( + download_file( + _Request(user="alice", auth_manager=_AuthManager()), + bob_escape_id, + ) + ) + + assert exc.value.status_code == 404 + + def test_get_vision_text_denies_cross_owner_before_cache_read(tmp_path, monkeypatch): handler, _alice_id, bob_id, upload_dir = _make_upload_store(tmp_path, monkeypatch) get_vision_text = _upload_endpoints(handler, monkeypatch)["get_vision_text"] @@ -178,6 +247,31 @@ def test_get_vision_text_denies_cross_owner_before_image_analysis(tmp_path, monk assert exc.value.status_code == 404 +def test_get_vision_text_rejects_upload_symlink_escape_before_analysis(tmp_path, monkeypatch): + handler, _alice_id, _bob_id, upload_dir = _make_upload_store(tmp_path, monkeypatch) + get_vision_text = _upload_endpoints(handler, monkeypatch)["get_vision_text"] + escape_id = "e" * 32 + ".png" + outside = tmp_path / "vision-outside-upload-root.png" + outside.write_bytes(b"outside upload root") + _add_upload_symlink(upload_dir, escape_id, outside) + + def fail_analysis(_path): + raise AssertionError("upload root gate should run before image analysis") + + monkeypatch.setattr("src.document_processor.analyze_image_with_vl", fail_analysis) + + with pytest.raises(HTTPException) as exc: + asyncio.run( + get_vision_text( + _Request(user="alice", auth_manager=_AuthManager()), + escape_id, + force=1, + ) + ) + + assert exc.value.status_code == 403 + + def test_put_vision_text_denies_cross_owner_before_cache_write(tmp_path, monkeypatch): handler, _alice_id, bob_id, upload_dir = _make_upload_store(tmp_path, monkeypatch) put_vision_text = _upload_endpoints(handler, monkeypatch)["put_vision_text"] From 0b0d747f1c045119b7cad8d6567d797f554cef81 Mon Sep 17 00:00:00 2001 From: Vykos Date: Fri, 5 Jun 2026 13:17:43 +0200 Subject: [PATCH 006/974] Constrain signature uploads to PNG data (#2844) --- routes/signature_routes.py | 60 +++++++++---- static/js/signature.js | 2 +- tests/test_signature_route_hardening.py | 104 +++++++++++++++++++++++ tests/test_signature_settings_dom_xss.py | 2 +- 4 files changed, 150 insertions(+), 18 deletions(-) create mode 100644 tests/test_signature_route_hardening.py diff --git a/routes/signature_routes.py b/routes/signature_routes.py index b60bb75..b758a69 100644 --- a/routes/signature_routes.py +++ b/routes/signature_routes.py @@ -21,10 +21,44 @@ from src.auth_helpers import get_current_user logger = logging.getLogger(__name__) -_DATA_URL_RE = re.compile( - r'^data:image/(?Ppng|jpeg|jpg);base64,(?P.+)$', - re.IGNORECASE | re.DOTALL, -) +_DATA_URL_RE = re.compile(r"^data:image/png;base64,(?P.+)$", re.IGNORECASE | re.DOTALL) +_ANY_IMAGE_DATA_URL_RE = re.compile(r"^data:image/[^;]+;base64,", re.IGNORECASE) +_PNG_MAGIC = b"\x89PNG\r\n\x1a\n" +_MAX_SIGNATURE_BYTES = 2 * 1024 * 1024 +_MAX_SIGNATURE_B64 = ((_MAX_SIGNATURE_BYTES + 2) // 3) * 4 +_MAX_SIGNATURE_DIMENSION = 4096 + + +def _normalize_signature_png(raw: str) -> str: + raw = (raw or "").strip() + m = _DATA_URL_RE.match(raw) + if m: + b64 = m.group("data") + elif _ANY_IMAGE_DATA_URL_RE.match(raw): + raise HTTPException(400, "Signature data must be a PNG image") + else: + b64 = raw + if len(b64) > _MAX_SIGNATURE_B64: + raise HTTPException(400, "Signature PNG is too large") + try: + payload = base64.b64decode(b64, validate=True) + except Exception: + raise HTTPException(400, "Signature data must be base64-encoded PNG bytes") + if not payload: + raise HTTPException(400, "Signature PNG is empty") + if len(payload) > _MAX_SIGNATURE_BYTES: + raise HTTPException(400, "Signature PNG is too large") + if not payload.startswith(_PNG_MAGIC): + raise HTTPException(400, "Signature data must be a PNG image") + return base64.b64encode(payload).decode("ascii") + + +def _signature_dimension(value: Optional[int]) -> Optional[int]: + if value is None: + return None + if not isinstance(value, int) or value < 1 or value > _MAX_SIGNATURE_DIMENSION: + raise HTTPException(400, "Signature dimensions are invalid") + return value class SignatureCreate(BaseModel): @@ -67,24 +101,18 @@ def setup_signature_routes() -> APIRouter: @router.post("/api/signatures") async def create_signature(request: Request, req: SignatureCreate) -> Dict[str, Any]: user = get_current_user(request) - raw = (req.data or "").strip() - m = _DATA_URL_RE.match(raw) - b64 = m.group("data") if m else raw - try: - payload = base64.b64decode(b64, validate=True) - if not payload: - raise ValueError("empty payload") - except Exception: - raise HTTPException(400, "Signature data must be base64-encoded PNG bytes") + b64 = _normalize_signature_png(req.data) + width = _signature_dimension(req.width) + height = _signature_dimension(req.height) sig = Signature( id=str(uuid.uuid4()), owner=user, name=(req.name or "Signature").strip() or "Signature", data_png=b64, - width=req.width, - height=req.height, - svg=req.svg, + width=width, + height=height, + svg=None, ) db = SessionLocal() try: diff --git a/static/js/signature.js b/static/js/signature.js index 94f8dfe..3b5bc0f 100644 --- a/static/js/signature.js +++ b/static/js/signature.js @@ -25,7 +25,7 @@ function _esc(s) { function _safeSignatureDataUrl(raw) { const value = String(raw || '').trim(); - return /^data:image\/(?:png|jpe?g);base64,[a-z0-9+/=\s]+$/i.test(value) ? value : ''; + return /^data:image\/png;base64,[a-z0-9+/=\s]+$/i.test(value) ? value : ''; } // Last signature the user picked or created in this session. Lets the export diff --git a/tests/test_signature_route_hardening.py b/tests/test_signature_route_hardening.py new file mode 100644 index 0000000..f66c7a2 --- /dev/null +++ b/tests/test_signature_route_hardening.py @@ -0,0 +1,104 @@ +import asyncio +import base64 +from types import SimpleNamespace +from unittest.mock import MagicMock + +import pytest +from fastapi import HTTPException + +from routes import signature_routes + + +_PNG_BYTES = b"\x89PNG\r\n\x1a\nsignature-bytes" +_PNG_B64 = base64.b64encode(_PNG_BYTES).decode("ascii") + + +class _SignatureRecord: + def __init__(self, **kwargs): + self.__dict__.update(kwargs) + self.created_at = None + + +class _FakeDb: + def __init__(self): + self.added = None + self.add = MagicMock(side_effect=self._add) + self.commit = MagicMock() + self.refresh = MagicMock() + self.rollback = MagicMock() + self.close = MagicMock() + + def _add(self, sig): + self.added = sig + + +def _request(user="alice"): + return SimpleNamespace(state=SimpleNamespace(current_user=user)) + + +def _route_endpoint(path, method): + router = signature_routes.setup_signature_routes() + for route in router.routes: + if route.path == path and method in route.methods: + return route.endpoint + raise AssertionError(f"route not found: {method} {path}") + + +def test_signature_png_normalization_accepts_data_url_and_raw_base64(): + data_url = f"data:image/png;base64,{_PNG_B64}" + + assert signature_routes._normalize_signature_png(data_url) == _PNG_B64 + assert signature_routes._normalize_signature_png(_PNG_B64) == _PNG_B64 + + +@pytest.mark.parametrize( + "raw", + [ + "", + "not base64!!!", + base64.b64encode(b"not a png").decode("ascii"), + "data:image/jpeg;base64," + base64.b64encode(b"\xff\xd8jpeg").decode("ascii"), + "A" * (signature_routes._MAX_SIGNATURE_B64 + 4), + ], +) +def test_signature_png_normalization_rejects_invalid_inputs(raw): + with pytest.raises(HTTPException) as exc: + signature_routes._normalize_signature_png(raw) + + assert exc.value.status_code == 400 + + +@pytest.mark.parametrize("value", [0, -1, signature_routes._MAX_SIGNATURE_DIMENSION + 1, "20"]) +def test_signature_dimensions_are_bounded(value): + with pytest.raises(HTTPException) as exc: + signature_routes._signature_dimension(value) + + assert exc.value.status_code == 400 + + +def test_create_signature_stores_normalized_png_and_drops_svg(monkeypatch): + db = _FakeDb() + monkeypatch.setattr(signature_routes, "SessionLocal", lambda: db) + monkeypatch.setattr(signature_routes, "Signature", _SignatureRecord) + create_signature = _route_endpoint("/api/signatures", "POST") + + response = asyncio.run(create_signature( + _request(), + signature_routes.SignatureCreate( + name=" Full signature ", + data=f"data:image/png;base64,{_PNG_B64}", + width=320, + height=80, + svg='', + ), + )) + + assert db.added.owner == "alice" + assert db.added.name == "Full signature" + assert db.added.data_png == _PNG_B64 + assert db.added.width == 320 + assert db.added.height == 80 + assert db.added.svg is None + assert response["data_url"] == f"data:image/png;base64,{_PNG_B64}" + db.commit.assert_called_once() + db.close.assert_called_once() diff --git a/tests/test_signature_settings_dom_xss.py b/tests/test_signature_settings_dom_xss.py index daa3388..c6cf348 100644 --- a/tests/test_signature_settings_dom_xss.py +++ b/tests/test_signature_settings_dom_xss.py @@ -10,7 +10,7 @@ def test_signature_picker_allows_only_raster_data_urls(): src = (_REPO / "static" / "js" / "signature.js").read_text(encoding="utf-8") assert "function _safeSignatureDataUrl(raw)" in src - assert r"^data:image\/(?:png|jpe?g);base64," in src + assert r"^data:image\/png;base64," in src assert '' in src assert 'dataUrl: s.data_url' not in src From 6d64055328cdd8a1608470ed174116de4dd06871 Mon Sep 17 00:00:00 2001 From: Vykos Date: Fri, 5 Jun 2026 13:20:02 +0200 Subject: [PATCH 007/974] Constrain research handler JSON paths (#2846) --- src/research_handler.py | 57 ++++++++++--- .../test_research_handler_path_confinement.py | 83 +++++++++++++++++++ 2 files changed, 130 insertions(+), 10 deletions(-) create mode 100644 tests/test_research_handler_path_confinement.py diff --git a/src/research_handler.py b/src/research_handler.py index bec9695..4a721cd 100644 --- a/src/research_handler.py +++ b/src/research_handler.py @@ -20,6 +20,7 @@ from src.research_utils import strip_thinking, is_low_quality logger = logging.getLogger(__name__) RESEARCH_DATA_DIR = Path("data/deep_research") +_RESEARCH_SESSION_ID_RE = re.compile(r"^[A-Za-z0-9-]{1,128}$") def _bounded_int(value, *, default: int, minimum: int, maximum: int) -> int: @@ -48,6 +49,18 @@ def _format_probe_failure(model: str, exc: Exception) -> str: return f"Cannot reach model '{model}' — check that the endpoint is running and accessible." +def _research_json_path(session_id: str) -> Optional[Path]: + if not isinstance(session_id, str) or not _RESEARCH_SESSION_ID_RE.fullmatch(session_id): + return None + root = RESEARCH_DATA_DIR.resolve() + path = (RESEARCH_DATA_DIR / f"{session_id}.json").resolve() + try: + path.relative_to(root) + except ValueError: + return None + return path + + class ResearchHandler: """Handles research service operations with iterative deep research.""" @@ -232,6 +245,9 @@ class ResearchHandler: max_rounds is the safety cap; the AI's _should_stop decision (after min_rounds) terminates the loop earlier in normal operation. """ + if _research_json_path(session_id) is None: + raise ValueError("Invalid research session_id") + # Resolve the hard wall-clock timeout from settings when the caller # didn't pin one. Local / edge models routinely need more than the # old 600s default to finish a deep-research synthesis. A setting of @@ -368,7 +384,9 @@ class ResearchHandler: result["avg_duration"] = round(avg, 1) return result # Check disk for completed research (skip consumed results) - path = RESEARCH_DATA_DIR / f"{session_id}.json" + path = _research_json_path(session_id) + if path is None: + return None if path.exists(): try: data = json.loads(path.read_text(encoding="utf-8")) @@ -407,7 +425,9 @@ class ResearchHandler: if entry["status"] in ("done", "error", "cancelled"): return entry.get("result") # Check disk (skip consumed results) - path = RESEARCH_DATA_DIR / f"{session_id}.json" + path = _research_json_path(session_id) + if path is None: + return None if path.exists(): try: data = json.loads(path.read_text(encoding="utf-8")) @@ -429,7 +449,9 @@ class ResearchHandler: if researcher and researcher.findings: return self._extract_sources(researcher.findings) # Check disk - path = RESEARCH_DATA_DIR / f"{session_id}.json" + path = _research_json_path(session_id) + if path is None: + return None if path.exists(): try: data = json.loads(path.read_text(encoding="utf-8")) @@ -446,7 +468,9 @@ class ResearchHandler: if researcher and researcher.findings: return self._extract_raw_findings(researcher.findings) # Check disk - path = RESEARCH_DATA_DIR / f"{session_id}.json" + path = _research_json_path(session_id) + if path is None: + return None if path.exists(): try: data = json.loads(path.read_text(encoding="utf-8")) @@ -521,7 +545,9 @@ class ResearchHandler: Keeps the JSON on disk so visual reports can be generated later. """ self._active_tasks.pop(session_id, None) - path = RESEARCH_DATA_DIR / f"{session_id}.json" + path = _research_json_path(session_id) + if path is None: + return if path.exists(): try: data = json.loads(path.read_text(encoding="utf-8")) @@ -533,6 +559,10 @@ class ResearchHandler: def _save_result(self, session_id: str, entry: dict): """Persist completed research result to disk.""" try: + path = _research_json_path(session_id) + if path is None: + logger.error("Refusing to save research result for invalid session_id: %r", session_id) + return # Extract and cache sources + raw findings sources = [] raw_findings = [] @@ -542,7 +572,6 @@ class ResearchHandler: raw_findings = self._extract_raw_findings(researcher.findings) entry["sources"] = sources - path = RESEARCH_DATA_DIR / f"{session_id}.json" data = { "query": entry["query"], "status": entry["status"], @@ -569,7 +598,9 @@ class ResearchHandler: def _get_session_json(self, session_id: str) -> Optional[dict]: """Load the saved research JSON for a session, if it exists.""" - path = RESEARCH_DATA_DIR / f"{session_id}.json" + path = _research_json_path(session_id) + if path is None: + return None if path.exists(): try: return json.loads(path.read_text(encoding="utf-8")) @@ -579,7 +610,9 @@ class ResearchHandler: def get_report_html(self, session_id: str) -> Optional[str]: """Generate the visual HTML report for a session (always fresh from JSON).""" - json_path = RESEARCH_DATA_DIR / f"{session_id}.json" + json_path = _research_json_path(session_id) + if json_path is None: + return None if not json_path.exists(): logger.warning(f"No JSON found for visual report: {json_path}") return None @@ -606,7 +639,9 @@ class ResearchHandler: def hide_image(self, session_id: str, image_url: str) -> bool: """Add image_url to the persisted hidden_images list for a research.""" - path = RESEARCH_DATA_DIR / f"{session_id}.json" + path = _research_json_path(session_id) + if path is None: + return False if not path.exists(): return False try: @@ -624,7 +659,9 @@ class ResearchHandler: def unhide_all_images(self, session_id: str) -> bool: """Clear the hidden_images list for a research.""" - path = RESEARCH_DATA_DIR / f"{session_id}.json" + path = _research_json_path(session_id) + if path is None: + return False if not path.exists(): return False try: diff --git a/tests/test_research_handler_path_confinement.py b/tests/test_research_handler_path_confinement.py new file mode 100644 index 0000000..5682a52 --- /dev/null +++ b/tests/test_research_handler_path_confinement.py @@ -0,0 +1,83 @@ +import json + +import pytest + +from src import research_handler +from src.research_handler import ResearchHandler + + +def _handler(): + handler = ResearchHandler.__new__(ResearchHandler) + handler._active_tasks = {} + return handler + + +def test_research_json_path_allows_safe_ids(tmp_path, monkeypatch): + data_dir = tmp_path / "deep_research" + monkeypatch.setattr(research_handler, "RESEARCH_DATA_DIR", data_dir) + + path = research_handler._research_json_path("rp-abc123") + + assert path == (data_dir / "rp-abc123.json").resolve() + + +@pytest.mark.parametrize("session_id", ["../escape", "..", "rp/test", "rp_test", "", None]) +def test_research_json_path_rejects_invalid_ids(tmp_path, monkeypatch, session_id): + monkeypatch.setattr(research_handler, "RESEARCH_DATA_DIR", tmp_path / "deep_research") + + assert research_handler._research_json_path(session_id) is None + + +def test_research_json_path_rejects_symlink_escape(tmp_path, monkeypatch): + data_dir = tmp_path / "deep_research" + outside = tmp_path / "outside" + data_dir.mkdir() + outside.mkdir() + monkeypatch.setattr(research_handler, "RESEARCH_DATA_DIR", data_dir) + link = data_dir / "rp-abc123.json" + target = outside / "rp-abc123.json" + target.write_text("{}", encoding="utf-8") + try: + link.symlink_to(target) + except (AttributeError, NotImplementedError, OSError) as exc: + pytest.skip(f"symlinks unavailable: {exc}") + + assert research_handler._research_json_path("rp-abc123") is None + + +def test_handler_disk_read_methods_reject_invalid_ids(tmp_path, monkeypatch): + outside = tmp_path / "escape.json" + outside.write_text(json.dumps({"result": "secret"}), encoding="utf-8") + monkeypatch.setattr(research_handler, "RESEARCH_DATA_DIR", tmp_path / "deep_research") + handler = _handler() + + assert handler.get_status("../escape") is None + assert handler.get_result("../escape") is None + assert handler.get_sources("../escape") is None + assert handler.get_raw_findings("../escape") is None + assert handler._get_session_json("../escape") is None + assert handler.get_report_html("../escape") is None + + +def test_handler_mutations_reject_invalid_ids_without_touching_outside_files(tmp_path, monkeypatch): + outside = tmp_path / "escape.json" + outside.write_text(json.dumps({"result": "secret", "hidden_images": ["x"]}), encoding="utf-8") + monkeypatch.setattr(research_handler, "RESEARCH_DATA_DIR", tmp_path / "deep_research") + handler = _handler() + + assert handler.hide_image("../escape", "https://example.com/image.png") is False + assert handler.unhide_all_images("../escape") is False + handler.clear_result("../escape") + handler._save_result("../escape", {"query": "q", "status": "done", "result": "r", "started_at": 1}) + + assert json.loads(outside.read_text(encoding="utf-8")) == { + "result": "secret", + "hidden_images": ["x"], + } + + +def test_start_research_rejects_invalid_session_id(): + handler = _handler() + + with pytest.raises(ValueError): + handler.start_research("../escape", "q", "http://localhost", "model") From 370ae5d45173fe219deac290ccb28d402103eff2 Mon Sep 17 00:00:00 2001 From: Vykos Date: Fri, 5 Jun 2026 13:22:21 +0200 Subject: [PATCH 008/974] Harden DAV outbound URL validation (#2819) --- routes/contacts_routes.py | 64 +++++++++++---- src/caldav_sync.py | 50 +++++++++++- src/caldav_writeback.py | 6 ++ tests/test_caldav_url_hardening.py | 47 ++++++++++- tests/test_caldav_url_nonstring.py | 13 ++- tests/test_caldav_writeback.py | 102 ++++++++++++++++++++++++ tests/test_contacts_carddav_security.py | 66 +++++++++++++++ 7 files changed, 326 insertions(+), 22 deletions(-) create mode 100644 tests/test_contacts_carddav_security.py diff --git a/routes/contacts_routes.py b/routes/contacts_routes.py index 409184f..8a90cf4 100644 --- a/routes/contacts_routes.py +++ b/routes/contacts_routes.py @@ -11,14 +11,17 @@ import uuid import json import csv import io +import os import httpx from pathlib import Path from datetime import datetime -from fastapi import APIRouter, Query, Depends, Response +from urllib.parse import urljoin, urlparse, urlunparse + +from fastapi import APIRouter, Query, Depends, Response, HTTPException from typing import List, Dict, Optional -from src.auth_helpers import require_user from core.middleware import require_admin +from src.url_safety import check_outbound_url logger = logging.getLogger(__name__) @@ -53,6 +56,21 @@ def _carddav_configured(cfg: Optional[Dict] = None) -> bool: return bool((cfg.get("url") or "").strip()) +def _validate_carddav_url(url: str) -> str: + cleaned = (url if isinstance(url, str) else "").strip().rstrip("/") + ok, reason = check_outbound_url( + cleaned, + block_private=os.getenv("CARDDAV_BLOCK_PRIVATE_IPS", "false").lower() == "true", + ) + if not ok: + raise ValueError(f"Rejected CardDAV URL: {reason}") + return cleaned + + +def _carddav_base_url(cfg: Dict) -> str: + return _validate_carddav_url(cfg.get("url") or "") + + def _normalize_contact(contact: Dict) -> Dict: emails = [] for e in contact.get("emails") or ([] if not contact.get("email") else [contact.get("email")]): @@ -219,14 +237,18 @@ _contact_cache = {"contacts": [], "fetched_at": None} def _abs_url(href: str) -> str: """Combine a multistatus (an absolute path like /user/contacts/x.vcf) with the configured CardDAV server origin so we - get a fully-qualified URL to PUT/DELETE. If href is already absolute - (http...), return it as-is.""" - from urllib.parse import urlparse, urlunparse - if href.startswith("http://") or href.startswith("https://"): - return href + get a fully-qualified URL to PUT/DELETE. Absolute hrefs are accepted only + for the configured origin; a cross-origin href is treated as a path on the + configured server so a malicious CardDAV response cannot redirect later + writes/deletes to cloud metadata or another host.""" cfg = _get_carddav_config() - p = urlparse(cfg["url"]) - return urlunparse((p.scheme, p.netloc, href, "", "", "")) + base = _carddav_base_url(cfg) + base_p = urlparse(base) + joined = urljoin(base.rstrip("/") + "/", href or "") + joined_p = urlparse(joined) + if (joined_p.scheme, joined_p.netloc) != (base_p.scheme, base_p.netloc): + joined = urlunparse((base_p.scheme, base_p.netloc, joined_p.path or "/", "", joined_p.query, "")) + return _validate_carddav_url(joined) # CardDAV REPORT body — pull every card's etag + raw vCard in ONE request, @@ -297,6 +319,7 @@ def _fetch_contacts(force=False): return contacts try: + cfg["url"] = _carddav_base_url(cfg) auth = None if cfg["username"]: auth = (cfg["username"], cfg["password"]) @@ -353,8 +376,8 @@ def _create_contact(name: str, email: str) -> bool: contact_uid = str(uuid.uuid4()) vcard = _build_vcard(name, email, contact_uid) - url = cfg["url"].rstrip("/") + "/" + contact_uid + ".vcf" try: + url = _carddav_base_url(cfg) + "/" + contact_uid + ".vcf" auth = None if cfg["username"]: auth = (cfg["username"], cfg["password"]) @@ -382,7 +405,7 @@ def _vcard_url(uid: str) -> str: escape the collection and target an arbitrary CardDAV resource.""" from urllib.parse import quote cfg = _get_carddav_config() - return cfg["url"].rstrip("/") + "/" + quote(uid, safe="") + ".vcf" + return _carddav_base_url(cfg) + "/" + quote(uid, safe="") + ".vcf" def _import_vcards(text: str) -> Dict: @@ -413,6 +436,11 @@ def _import_vcards(text: str) -> Dict: if imported: _save_local_contacts(contacts) return {"imported": imported, "failed": 0, "total": len(parsed)} + try: + base_url = _carddav_base_url(cfg) + except ValueError as e: + logger.warning("CardDAV import URL rejected: %s", e) + return {"imported": 0, "failed": 0, "total": 0, "error": str(e)} auth = (cfg["username"], cfg["password"]) if cfg["username"] else None # Split into individual cards. re.split drops the BEGIN line, so we # re-add it. Normalize CRLF. @@ -441,7 +469,7 @@ def _import_vcards(text: str) -> Dict: elif not re.search(r"^VERSION:", block, re.MULTILINE): block = block.replace("BEGIN:VCARD", "BEGIN:VCARD\nVERSION:4.0", 1) vcard = block.replace("\n", "\r\n") + "\r\n" - url = cfg["url"].rstrip("/") + "/" + quote(uid, safe="") + ".vcf" + url = base_url + "/" + quote(uid, safe="") + ".vcf" try: r = httpx.put( url, data=vcard.encode("utf-8"), @@ -601,8 +629,8 @@ def _update_contact(uid: str, name: str, emails: List[str], phones: List[str]) - vcard = _build_vcard(name, "", uid=uid, emails=emails, phones=phones) # Use the real resource href (handles externally-created contacts whose # filename != UID); falls back to the .vcf guess. - url = _resolve_resource_url(uid) try: + url = _resolve_resource_url(uid) auth = (cfg["username"], cfg["password"]) if cfg["username"] else None r = httpx.put( url, @@ -630,8 +658,8 @@ def _delete_contact(uid: str) -> bool: _save_local_contacts(remaining) return True - url = _resolve_resource_url(uid) try: + url = _resolve_resource_url(uid) auth = (cfg["username"], cfg["password"]) if cfg["username"] else None r = httpx.delete(url, auth=auth, timeout=10) if r.status_code in (200, 204): @@ -747,7 +775,13 @@ def setup_contacts_routes(): settings = _load_settings() for key in ("carddav_url", "carddav_username", "carddav_password"): if key in data: - settings[key] = data[key] + if key == "carddav_url" and str(data[key] or "").strip(): + try: + settings[key] = _validate_carddav_url(data[key]) + except ValueError as e: + raise HTTPException(400, str(e)) + else: + settings[key] = data[key] _save_settings(settings) # Force re-fetch _contact_cache["fetched_at"] = None diff --git a/src/caldav_sync.py b/src/caldav_sync.py index 663c0bd..b139dbb 100644 --- a/src/caldav_sync.py +++ b/src/caldav_sync.py @@ -27,6 +27,7 @@ import hashlib import ipaddress import logging import os +import socket import uuid from datetime import date, datetime, timedelta, timezone from urllib.parse import urlparse, urlunparse @@ -50,15 +51,55 @@ def _private_caldav_allowed() -> bool: return os.environ.get("ODYSSEUS_ALLOW_PRIVATE_CALDAV", "0").lower() in {"1", "true", "yes"} +def _validate_caldav_address(addr: ipaddress._BaseAddress) -> None: + if isinstance(addr, ipaddress.IPv6Address) and addr.ipv4_mapped is not None: + addr = addr.ipv4_mapped + if ( + addr.is_loopback + or addr.is_link_local + or addr.is_multicast + or addr.is_unspecified + or addr.is_reserved + ): + raise ValueError("CalDAV URL host is not allowed") + if addr.is_private and not _private_caldav_allowed(): + raise ValueError("Private CalDAV IPs require ODYSSEUS_ALLOW_PRIVATE_CALDAV=1") + + def _validate_caldav_ip(host: str) -> None: try: ip = ipaddress.ip_address(host.strip("[]")) except ValueError: return - if ip.is_loopback or ip.is_link_local or ip.is_multicast or ip.is_unspecified: - raise ValueError("CalDAV URL host is not allowed") - if ip.is_private and not _private_caldav_allowed(): - raise ValueError("Private CalDAV IPs require ODYSSEUS_ALLOW_PRIVATE_CALDAV=1") + _validate_caldav_address(ip) + + +def _resolve_caldav_host_ips(host: str) -> list[ipaddress._BaseAddress]: + addrs: list[ipaddress._BaseAddress] = [] + for family, _, _, _, sockaddr in socket.getaddrinfo(host, None): + if family not in (socket.AF_INET, socket.AF_INET6): + continue + try: + addrs.append(ipaddress.ip_address(sockaddr[0].split("%", 1)[0])) + except ValueError: + continue + return addrs + + +def _validate_caldav_hostname(host: str) -> None: + try: + ipaddress.ip_address(host.strip("[]")) + return + except ValueError: + pass + try: + addrs = _resolve_caldav_host_ips(host) + except OSError: + raise ValueError("CalDAV URL host does not resolve") + if not addrs: + raise ValueError("CalDAV URL host does not resolve") + for addr in addrs: + _validate_caldav_address(addr) def validate_caldav_url(raw_url: str) -> str: @@ -83,6 +124,7 @@ def validate_caldav_url(raw_url: str) -> str: if host in _BLOCKED_HOSTS or host.endswith(".localhost"): raise ValueError("CalDAV URL host is not allowed") _validate_caldav_ip(host) + _validate_caldav_hostname(host) return urlunparse(parsed._replace(fragment="")).rstrip("/") diff --git a/src/caldav_writeback.py b/src/caldav_writeback.py index 1b6d6cc..e5bc46d 100644 --- a/src/caldav_writeback.py +++ b/src/caldav_writeback.py @@ -167,6 +167,12 @@ async def writeback_event(owner: str, calendar_source: str, calendar_id: str, pw = decrypt(cfg.get("password") or "") if not (url and user and pw): return {"skipped": "caldav not configured"} + from src.caldav_sync import validate_caldav_url + try: + url = validate_caldav_url(url) + except ValueError as e: + logger.warning("CalDAV write-back URL rejected: %s", e) + return {"ok": False, "error": str(e)[:200]} result = await asyncio.to_thread(_writeback_blocking, calendar_id, ev, delete, url, user, pw) if not result.get("ok"): logger.warning("CalDAV write-back did not apply: %s", result.get("error") or result) diff --git a/tests/test_caldav_url_hardening.py b/tests/test_caldav_url_hardening.py index 40b1f34..39de9a9 100644 --- a/tests/test_caldav_url_hardening.py +++ b/tests/test_caldav_url_hardening.py @@ -1,4 +1,5 @@ import asyncio +import ipaddress import sys import types from pathlib import Path @@ -8,7 +9,12 @@ import pytest from src import caldav_sync -def test_validate_caldav_url_normalizes_safe_url(): +def test_validate_caldav_url_normalizes_safe_url(monkeypatch): + monkeypatch.setattr( + caldav_sync, + "_resolve_caldav_host_ips", + lambda host: [ipaddress.ip_address("93.184.216.34")], + ) assert ( caldav_sync.validate_caldav_url(" https://calendar.example.com/dav/ ") == "https://calendar.example.com/dav" @@ -42,7 +48,46 @@ def test_validate_caldav_url_blocks_private_ips_unless_explicitly_allowed(monkey assert caldav_sync.validate_caldav_url("http://10.0.0.5:5232/dav") == "http://10.0.0.5:5232/dav" +def test_validate_caldav_url_blocks_dns_to_private(monkeypatch): + monkeypatch.delenv("ODYSSEUS_ALLOW_PRIVATE_CALDAV", raising=False) + monkeypatch.setattr( + caldav_sync, + "_resolve_caldav_host_ips", + lambda host: [ipaddress.ip_address("10.0.0.5")], + ) + + with pytest.raises(ValueError, match="Private CalDAV IPs require"): + caldav_sync.validate_caldav_url("https://calendar.example.com/dav") + + +def test_validate_caldav_url_blocks_dns_to_link_local_even_when_private_allowed(monkeypatch): + monkeypatch.setenv("ODYSSEUS_ALLOW_PRIVATE_CALDAV", "1") + monkeypatch.setattr( + caldav_sync, + "_resolve_caldav_host_ips", + lambda host: [ipaddress.ip_address("169.254.169.254")], + ) + + with pytest.raises(ValueError, match="host is not allowed"): + caldav_sync.validate_caldav_url("https://calendar.example.com/dav") + + +def test_validate_caldav_url_fails_closed_when_hostname_does_not_resolve(monkeypatch): + def _no_dns(host): + raise OSError("no such host") + + monkeypatch.setattr(caldav_sync, "_resolve_caldav_host_ips", _no_dns) + + with pytest.raises(ValueError, match="host does not resolve"): + caldav_sync.validate_caldav_url("https://calendar.example.com/dav") + + def test_sync_caldav_decrypts_stored_password_and_validates_url(monkeypatch): + monkeypatch.setattr( + caldav_sync, + "_resolve_caldav_host_ips", + lambda host: [ipaddress.ip_address("93.184.216.34")], + ) prefs_mod = types.ModuleType("routes.prefs_routes") prefs_mod._load_for_user = lambda owner: { "caldav": { diff --git a/tests/test_caldav_url_nonstring.py b/tests/test_caldav_url_nonstring.py index a9d8f3f..db50b8c 100644 --- a/tests/test_caldav_url_nonstring.py +++ b/tests/test_caldav_url_nonstring.py @@ -5,9 +5,13 @@ It did `(raw_url or "").strip()`, so a non-string scalar (e.g. an int from a mis-typed config) reached `.strip()` and raised TypeError instead of the function\'s own ValueError. """ +import ipaddress + import pytest -from src.caldav_sync import validate_caldav_url +from src import caldav_sync + +validate_caldav_url = caldav_sync.validate_caldav_url def test_non_string_raises_valueerror_not_typeerror(): @@ -17,6 +21,11 @@ def test_non_string_raises_valueerror_not_typeerror(): validate_caldav_url(None) -def test_valid_url_passes(): +def test_valid_url_passes(monkeypatch): + monkeypatch.setattr( + caldav_sync, + "_resolve_caldav_host_ips", + lambda host: [ipaddress.ip_address("93.184.216.34")], + ) out = validate_caldav_url("https://dav.example.com/calendars/") assert "example.com" in out diff --git a/tests/test_caldav_writeback.py b/tests/test_caldav_writeback.py index c501ad1..f636712 100644 --- a/tests/test_caldav_writeback.py +++ b/tests/test_caldav_writeback.py @@ -5,6 +5,9 @@ iCalendar serialization, hash-based remote-calendar discovery, and the create/update/delete orchestration. """ +import asyncio +import sys +import types from datetime import datetime from src.caldav_writeback import ( @@ -123,3 +126,102 @@ def test_push_missing_uid_reports_input_error_before_remote_lookup(): res = push_event([cal], CAL_ID, _ev(uid="")) assert res["ok"] is False and "uid" in res["error"] assert cal._existing.saved is False + + +def test_writeback_validates_saved_url_before_remote_call(monkeypatch): + import src.caldav_sync as sync + import src.caldav_writeback as wb + + prefs_mod = types.ModuleType("routes.prefs_routes") + prefs_mod._load_for_user = lambda owner: { + "caldav": { + "url": " https://dav.example.com/calendars/home/ ", + "username": owner, + "password": "enc:pw", + } + } + secret_mod = types.ModuleType("src.secret_storage") + secret_mod.decrypt = lambda value: "plain-password" + monkeypatch.setitem(sys.modules, "routes.prefs_routes", prefs_mod) + monkeypatch.setitem(sys.modules, "src.secret_storage", secret_mod) + + captured = {} + + def fake_validate(url): + captured["validated_url"] = url + return "https://dav.example.com/calendars/home" + + def fake_writeback_blocking(local_cal_id, ev, delete, url, username, password): + captured.update( + { + "local_cal_id": local_cal_id, + "delete": delete, + "url": url, + "username": username, + "password": password, + } + ) + return {"ok": True} + + async def inline_to_thread(func, *args, **kwargs): + return func(*args, **kwargs) + + monkeypatch.setattr(sync, "validate_caldav_url", fake_validate) + monkeypatch.setattr(wb, "_writeback_blocking", fake_writeback_blocking) + monkeypatch.setattr(wb.asyncio, "to_thread", inline_to_thread) + + result = asyncio.run( + wb.writeback_event("alice", "caldav", "caldav-123", {"uid": "evt-1"}) + ) + + assert result == {"ok": True} + assert captured == { + "validated_url": "https://dav.example.com/calendars/home/", + "local_cal_id": "caldav-123", + "delete": False, + "url": "https://dav.example.com/calendars/home", + "username": "alice", + "password": "plain-password", + } + + +def test_writeback_rejects_unsafe_saved_url_before_remote_call(monkeypatch): + import src.caldav_sync as sync + import src.caldav_writeback as wb + + prefs_mod = types.ModuleType("routes.prefs_routes") + prefs_mod._load_for_user = lambda owner: { + "caldav": { + "url": "http://evil.example/latest/meta-data", + "username": owner, + "password": "enc:pw", + } + } + secret_mod = types.ModuleType("src.secret_storage") + secret_mod.decrypt = lambda value: "plain-password" + monkeypatch.setitem(sys.modules, "routes.prefs_routes", prefs_mod) + monkeypatch.setitem(sys.modules, "src.secret_storage", secret_mod) + + called = False + + def fake_validate(_url): + raise ValueError("CalDAV URL host is not allowed") + + def fake_writeback_blocking(*_args, **_kwargs): + nonlocal called + called = True + return {"ok": True} + + async def inline_to_thread(func, *args, **kwargs): + return func(*args, **kwargs) + + monkeypatch.setattr(sync, "validate_caldav_url", fake_validate) + monkeypatch.setattr(wb, "_writeback_blocking", fake_writeback_blocking) + monkeypatch.setattr(wb.asyncio, "to_thread", inline_to_thread) + + result = asyncio.run( + wb.writeback_event("alice", "caldav", "caldav-123", {"uid": "evt-1"}) + ) + + assert result == {"ok": False, "error": "CalDAV URL host is not allowed"} + assert called is False diff --git a/tests/test_contacts_carddav_security.py b/tests/test_contacts_carddav_security.py new file mode 100644 index 0000000..8a20af0 --- /dev/null +++ b/tests/test_contacts_carddav_security.py @@ -0,0 +1,66 @@ +"""CardDAV outbound URL hardening tests.""" + +import pytest + +import routes.contacts_routes as contacts + + +def test_validate_carddav_url_blocks_metadata_targets(monkeypatch): + monkeypatch.setattr( + contacts, + "check_outbound_url", + lambda url, *, block_private=False: (False, "link-local address blocked"), + ) + + with pytest.raises(ValueError, match="link-local"): + contacts._validate_carddav_url("http://169.254.169.254/latest/meta-data") + + +def test_validate_carddav_url_rejects_non_string(monkeypatch): + monkeypatch.setattr( + contacts, + "check_outbound_url", + lambda url, *, block_private=False: (False, "URL is required"), + ) + + with pytest.raises(ValueError, match="URL is required"): + contacts._validate_carddav_url(12345) + + +def test_abs_url_pins_cross_origin_href_to_configured_carddav_origin(monkeypatch): + monkeypatch.setattr( + contacts, + "_get_carddav_config", + lambda: {"url": "https://dav.example.com/addressbooks/alice", "username": "", "password": ""}, + ) + monkeypatch.setattr( + contacts, + "check_outbound_url", + lambda url, *, block_private=False: (True, "ok"), + ) + + assert ( + contacts._abs_url("http://169.254.169.254/latest/meta-data") + == "https://dav.example.com/latest/meta-data" + ) + + +def test_vcard_url_validates_base_and_quotes_uid(monkeypatch): + seen = [] + monkeypatch.setattr( + contacts, + "_get_carddav_config", + lambda: {"url": "https://dav.example.com/addressbooks/alice/", "username": "", "password": ""}, + ) + + def _safe(url, *, block_private=False): + seen.append((url, block_private)) + return True, "ok" + + monkeypatch.setattr(contacts, "check_outbound_url", _safe) + + assert ( + contacts._vcard_url("uid/../../escape") + == "https://dav.example.com/addressbooks/alice/uid%2F..%2F..%2Fescape.vcf" + ) + assert seen == [("https://dav.example.com/addressbooks/alice", False)] From 301d1109b59fdda2bad19df32ce82efa1c6cf4fe Mon Sep 17 00:00:00 2001 From: Alexandre Teixeira <111787685+alteixeira20@users.noreply.github.com> Date: Fri, 5 Jun 2026 12:27:44 +0100 Subject: [PATCH 009/974] refactor(tests): centralize fake database import-state cleanup Test-only refactor continuing #2523. Centralizes the repeated guarded fake core.database/src.database import-state cleanup into a focused helper. --- tests/helpers/import_state.py | 33 +++++ tests/test_calendar_rrule.py | 15 +-- ...test_document_close_clears_active_route.py | 16 +-- tests/test_helpers_import_state.py | 113 +++++++++++++++++- tests/test_sqlite_foreign_keys.py | 16 +-- tests/test_task_scheduler_session_delivery.py | 15 +-- tests/test_topic_analyzer.py | 16 +-- 7 files changed, 155 insertions(+), 69 deletions(-) diff --git a/tests/helpers/import_state.py b/tests/helpers/import_state.py index 35059bf..c27ab7a 100644 --- a/tests/helpers/import_state.py +++ b/tests/helpers/import_state.py @@ -8,6 +8,10 @@ had before the block — present, absent, or carrying a parent-package attribute Use ``clear_module`` to drop a single module from both ``sys.modules`` and its parent-package attribute (e.g. before forcing a fresh import inside the block). +Use ``clear_fake_database_modules`` to evict a *stubbed* ``core.database`` (and +its companion ``src.database``) that another test left in import state, without +touching a real ``core.database`` loaded from disk. + Background: importing ``routes.session_routes`` also sets ``session_routes`` on the parent ``routes`` package object. A ``from routes import session_routes`` or ``import routes.session_routes as X`` statement resolves through that parent @@ -60,6 +64,35 @@ def clear_module(dotted_name): _restore_one(dotted_name, _ABSENT, _ABSENT) +def clear_fake_database_modules(): + """Evict a *stubbed* ``core.database`` (and ``src.database``) from import state. + + Test-only. Some tests install a fake ``core.database`` — a stub module with + no on-disk ``__file__`` — into ``sys.modules`` and onto the ``core`` package. + A later test that needs the real database module must evict that stub first, + or its ``import core.database`` resolves to the fake. + + This is deliberately conservative and mirrors the per-file helpers it + replaces: + + * It acts only when ``core.database`` is a fake/stub, detected by a missing + string ``__file__``. A real ``core.database`` loaded from disk is left + untouched, as is the case where nothing is cached. + * When it does act, it also drops the cached ``src.database`` entry. + * It removes the ``core.database`` parent-package attribute only when that + attribute is the same fake object being evicted. + """ + parent = sys.modules.get("core") + attr = getattr(parent, "database", None) if parent is not None else None + mod = sys.modules.get("core.database") or attr + if mod is None or isinstance(getattr(mod, "__file__", None), str): + return + sys.modules.pop("core.database", None) + sys.modules.pop("src.database", None) + if parent is not None and attr is mod: + delattr(parent, "database") + + @contextmanager def preserve_import_state(*module_names): """Save and restore sys.modules entries and parent-package attributes. diff --git a/tests/test_calendar_rrule.py b/tests/test_calendar_rrule.py index c49f142..18d6eaa 100644 --- a/tests/test_calendar_rrule.py +++ b/tests/test_calendar_rrule.py @@ -15,20 +15,9 @@ from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker from sqlalchemy.pool import NullPool +from tests.helpers.import_state import clear_fake_database_modules -def _drop_fake_core_database(): - parent = sys.modules.get("core") - attr = getattr(parent, "database", None) if parent is not None else None - mod = sys.modules.get("core.database") or attr - if mod is None or isinstance(getattr(mod, "__file__", None), str): - return - sys.modules.pop("core.database", None) - sys.modules.pop("src.database", None) - if parent is not None and attr is mod: - delattr(parent, "database") - - -_drop_fake_core_database() +clear_fake_database_modules() import core.database as cdb from core.database import CalendarEvent diff --git a/tests/test_document_close_clears_active_route.py b/tests/test_document_close_clears_active_route.py index 5428d4f..dbd84e5 100644 --- a/tests/test_document_close_clears_active_route.py +++ b/tests/test_document_close_clears_active_route.py @@ -13,7 +13,6 @@ while completing reliably everywhere. """ import tempfile -import sys import uuid from types import SimpleNamespace @@ -22,20 +21,9 @@ from sqlalchemy.orm import sessionmaker from sqlalchemy.pool import NullPool from unittest.mock import MagicMock +from tests.helpers.import_state import clear_fake_database_modules -def _drop_fake_core_database(): - parent = sys.modules.get("core") - attr = getattr(parent, "database", None) if parent is not None else None - mod = sys.modules.get("core.database") or attr - if mod is None or isinstance(getattr(mod, "__file__", None), str): - return - sys.modules.pop("core.database", None) - sys.modules.pop("src.database", None) - if parent is not None and attr is mod: - delattr(parent, "database") - - -_drop_fake_core_database() +clear_fake_database_modules() import core.database as cdb import routes.document_routes as droutes diff --git a/tests/test_helpers_import_state.py b/tests/test_helpers_import_state.py index d9f9254..3e7d7a7 100644 --- a/tests/test_helpers_import_state.py +++ b/tests/test_helpers_import_state.py @@ -4,10 +4,18 @@ import types import pytest -from tests.helpers.import_state import clear_module, preserve_import_state +from tests.helpers.import_state import ( + clear_fake_database_modules, + clear_module, + preserve_import_state, +) _SENTINEL = "tests._import_state_test_sentinel" +# Names touched by clear_fake_database_modules — snapshot/restore these so the +# tests never leak into the real core/src packages. +_DB_NAMES = ("core", "core.database", "src", "src.database") + def test_absent_module_is_removed_after_block(): assert _SENTINEL not in sys.modules @@ -139,3 +147,106 @@ def test_parent_attr_restored_correctly_when_parent_also_preserved(): finally: sys.modules.pop("_fake_istate_parent", None) sys.modules.pop("_fake_istate_parent.child", None) + + +def test_clear_fake_database_removes_stub_core_database(): + with preserve_import_state(*_DB_NAMES): + fake_core = types.ModuleType("core") + fake_db = types.ModuleType("core.database") # no __file__ => a stub + fake_core.database = fake_db + sys.modules["core"] = fake_core + sys.modules["core.database"] = fake_db + + clear_fake_database_modules() + + assert "core.database" not in sys.modules + assert not hasattr(fake_core, "database") + + +def test_clear_fake_database_preserves_real_core_database(): + with preserve_import_state(*_DB_NAMES): + fake_core = types.ModuleType("core") + real_db = types.ModuleType("core.database") + real_db.__file__ = "/somewhere/core/database.py" # looks on-disk + fake_core.database = real_db + sys.modules["core"] = fake_core + sys.modules["core.database"] = real_db + + clear_fake_database_modules() + + assert sys.modules["core.database"] is real_db + assert fake_core.database is real_db + + +def test_clear_fake_database_drops_src_database_when_core_is_fake(): + with preserve_import_state(*_DB_NAMES): + fake_core = types.ModuleType("core") + fake_db = types.ModuleType("core.database") + fake_core.database = fake_db + sys.modules["core"] = fake_core + sys.modules["core.database"] = fake_db + sys.modules["src.database"] = types.ModuleType("src.database") + + clear_fake_database_modules() + + assert "src.database" not in sys.modules + + +def test_clear_fake_database_leaves_src_database_when_core_is_real(): + with preserve_import_state(*_DB_NAMES): + fake_core = types.ModuleType("core") + real_db = types.ModuleType("core.database") + real_db.__file__ = "/somewhere/core/database.py" + fake_core.database = real_db + sys.modules["core"] = fake_core + sys.modules["core.database"] = real_db + src_db = types.ModuleType("src.database") + sys.modules["src.database"] = src_db + + clear_fake_database_modules() + + assert sys.modules["src.database"] is src_db + + +def test_clear_fake_database_keeps_parent_attr_pointing_elsewhere(): + """When the cached core.database is a stub but the `database` attr on the + core package points at a *different* object, the attr is left intact — + only the same fake object is unlinked.""" + with preserve_import_state(*_DB_NAMES): + fake_core = types.ModuleType("core") + cached_fake = types.ModuleType("core.database") # the stub in sys.modules + other = types.ModuleType("core.database") # parent attr points here + fake_core.database = other + sys.modules["core"] = fake_core + sys.modules["core.database"] = cached_fake + + clear_fake_database_modules() + + assert "core.database" not in sys.modules + assert fake_core.database is other + + +def test_clear_fake_database_uses_parent_attr_when_not_in_sys_modules(): + """A stub reachable only via the core package's `database` attribute (not in + sys.modules) is still detected and unlinked from the parent.""" + with preserve_import_state(*_DB_NAMES): + sys.modules.pop("core.database", None) + fake_core = types.ModuleType("core") + fake_db = types.ModuleType("core.database") + fake_core.database = fake_db + sys.modules["core"] = fake_core + + clear_fake_database_modules() + + assert not hasattr(fake_core, "database") + + +def test_clear_fake_database_noop_when_nothing_cached(): + with preserve_import_state(*_DB_NAMES): + sys.modules.pop("core.database", None) + fake_core = types.ModuleType("core") # no `database` attr + sys.modules["core"] = fake_core + + clear_fake_database_modules() # must not raise + + assert "core.database" not in sys.modules diff --git a/tests/test_sqlite_foreign_keys.py b/tests/test_sqlite_foreign_keys.py index dcf5642..0983009 100644 --- a/tests/test_sqlite_foreign_keys.py +++ b/tests/test_sqlite_foreign_keys.py @@ -1,22 +1,10 @@ import pytest -import sys from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker +from tests.helpers.import_state import clear_fake_database_modules -def _drop_fake_core_database(): - parent = sys.modules.get("core") - attr = getattr(parent, "database", None) if parent is not None else None - mod = sys.modules.get("core.database") or attr - if mod is None or isinstance(getattr(mod, "__file__", None), str): - return - sys.modules.pop("core.database", None) - sys.modules.pop("src.database", None) - if parent is not None and attr is mod: - delattr(parent, "database") - - -_drop_fake_core_database() +clear_fake_database_modules() from core.database import Base, Session, ChatMessage from datetime import datetime diff --git a/tests/test_task_scheduler_session_delivery.py b/tests/test_task_scheduler_session_delivery.py index 4f35cb3..a08f670 100644 --- a/tests/test_task_scheduler_session_delivery.py +++ b/tests/test_task_scheduler_session_delivery.py @@ -12,20 +12,9 @@ if not isinstance(sqlalchemy, _types.ModuleType): from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker +from tests.helpers.import_state import clear_fake_database_modules -def _drop_fake_core_database(): - parent = sys.modules.get("core") - attr = getattr(parent, "database", None) if parent is not None else None - mod = sys.modules.get("core.database") or attr - if mod is None or isinstance(getattr(mod, "__file__", None), str): - return - sys.modules.pop("core.database", None) - sys.modules.pop("src.database", None) - if parent is not None and attr is mod: - delattr(parent, "database") - - -_drop_fake_core_database() +clear_fake_database_modules() import core.database as cdb from core.database import Base, Session as DbSession diff --git a/tests/test_topic_analyzer.py b/tests/test_topic_analyzer.py index c47d14e..f9cca19 100644 --- a/tests/test_topic_analyzer.py +++ b/tests/test_topic_analyzer.py @@ -1,24 +1,12 @@ """Tests for topic keyword matching (src/topic_analyzer.py).""" -import sys from types import SimpleNamespace import pytest from sqlalchemy import create_engine from sqlalchemy.orm import sessionmaker +from tests.helpers.import_state import clear_fake_database_modules -def _drop_fake_core_database(): - parent = sys.modules.get("core") - attr = getattr(parent, "database", None) if parent is not None else None - mod = sys.modules.get("core.database") or attr - if mod is None or isinstance(getattr(mod, "__file__", None), str): - return - sys.modules.pop("core.database", None) - sys.modules.pop("src.database", None) - if parent is not None and attr is mod: - delattr(parent, "database") - - -_drop_fake_core_database() +clear_fake_database_modules() from core.database import Base, Session as DbSession, ChatMessage as DbChatMessage from core.session_manager import SessionManager From 452a94fb1b0ec117160f1472de9d297fa4c887c9 Mon Sep 17 00:00:00 2001 From: Alexandre Teixeira <111787685+alteixeira20@users.noreply.github.com> Date: Fri, 5 Jun 2026 13:23:46 +0100 Subject: [PATCH 010/974] refactor(tests): centralize fake endpoint resolver cleanup Test-only refactor continuing #2523. Centralizes the final repeated fake src.endpoint_resolver cleanup pattern into a focused import-state helper. --- tests/helpers/import_state.py | 49 ++++++++ tests/test_chat_image_routing.py | 9 +- tests/test_endpoint_probing.py | 7 +- tests/test_helpers_import_state.py | 174 +++++++++++++++++++++++++++++ tests/test_model_routes.py | 11 +- 5 files changed, 234 insertions(+), 16 deletions(-) diff --git a/tests/helpers/import_state.py b/tests/helpers/import_state.py index c27ab7a..0eea62d 100644 --- a/tests/helpers/import_state.py +++ b/tests/helpers/import_state.py @@ -12,6 +12,11 @@ Use ``clear_fake_database_modules`` to evict a *stubbed* ``core.database`` (and its companion ``src.database``) that another test left in import state, without touching a real ``core.database`` loaded from disk. +Use ``clear_fake_endpoint_resolver_modules`` to evict a *stubbed* +``src.endpoint_resolver`` (and the route modules that imported it) that another +test left in import state, without touching a real ``src.endpoint_resolver`` +loaded from disk. + Background: importing ``routes.session_routes`` also sets ``session_routes`` on the parent ``routes`` package object. A ``from routes import session_routes`` or ``import routes.session_routes as X`` statement resolves through that parent @@ -93,6 +98,50 @@ def clear_fake_database_modules(): delattr(parent, "database") +def clear_fake_endpoint_resolver_modules(*extra_modules): + """Evict a *stubbed* ``src.endpoint_resolver`` (and dependent route modules). + + Test-only. Several route tests need the *real* ``src.endpoint_resolver`` URL + helpers, but another test may have installed a fake — a stub module with no + on-disk ``__file__`` — into ``sys.modules`` and onto the ``src`` package + during collection. The route modules (``routes.model_routes`` and any extras + passed in, e.g. ``routes.chat_routes``) get cached against that fake on first + import, so they must be evicted too. + + Conservative, mirroring ``clear_fake_database_modules`` and the per-file + guards it replaces: + + * It acts only when ``src.endpoint_resolver`` is a fake/stub, detected by a + falsy ``__file__`` (missing, ``None``, or empty string) — exactly the + truthiness check the old inline guards used. A real resolver loaded from + disk carries a truthy ``__file__`` and is left untouched, as is the case + where nothing is cached. When the resolver is real, the dependent route + modules are left untouched too. + * When it does act, it drops ``routes.model_routes`` plus every name in + ``extra_modules``. + * It removes the ``src.endpoint_resolver`` parent-package attribute only when + that attribute is the same fake object being evicted. + + Behavior delta vs. the old bare ``sys.modules.pop(...)`` guards: dependent + modules are dropped via :func:`clear_module`, which also clears the parent + ``routes`` package attribute (e.g. ``routes.model_routes``), not just the + ``sys.modules`` entry. This prevents a stale parent attribute from shadowing + the fresh import — the same parent-attr handling the rest of this helper + family already applies. + """ + parent = sys.modules.get("src") + attr = getattr(parent, "endpoint_resolver", None) if parent is not None else None + mod = sys.modules.get("src.endpoint_resolver") or attr + if mod is None or getattr(mod, "__file__", None): + return + sys.modules.pop("src.endpoint_resolver", None) + if parent is not None and attr is mod: + delattr(parent, "endpoint_resolver") + clear_module("routes.model_routes") + for name in extra_modules: + clear_module(name) + + @contextmanager def preserve_import_state(*module_names): """Save and restore sys.modules entries and parent-package attributes. diff --git a/tests/test_chat_image_routing.py b/tests/test_chat_image_routing.py index 92b8476..21e06f5 100644 --- a/tests/test_chat_image_routing.py +++ b/tests/test_chat_image_routing.py @@ -1,12 +1,9 @@ import json -import sys from types import SimpleNamespace -_endpoint_resolver = sys.modules.get("src.endpoint_resolver") -if _endpoint_resolver is not None and not getattr(_endpoint_resolver, "__file__", None): - sys.modules.pop("src.endpoint_resolver", None) - sys.modules.pop("routes.model_routes", None) - sys.modules.pop("routes.chat_routes", None) +from tests.helpers.import_state import clear_fake_endpoint_resolver_modules + +clear_fake_endpoint_resolver_modules("routes.chat_routes") from routes import chat_routes diff --git a/tests/test_endpoint_probing.py b/tests/test_endpoint_probing.py index a9e7554..0206ebf 100644 --- a/tests/test_endpoint_probing.py +++ b/tests/test_endpoint_probing.py @@ -25,12 +25,11 @@ from unittest.mock import MagicMock import httpx import pytest +from tests.helpers.import_state import clear_fake_endpoint_resolver_modules + # Match test_model_routes.py: if another test stubbed src.endpoint_resolver # during collection, drop the stub so the real URL helpers load here. -_endpoint_resolver = sys.modules.get("src.endpoint_resolver") -if _endpoint_resolver is not None and not getattr(_endpoint_resolver, "__file__", None): - sys.modules.pop("src.endpoint_resolver", None) - sys.modules.pop("routes.model_routes", None) +clear_fake_endpoint_resolver_modules() if "core.database" not in sys.modules: _core_db = types.ModuleType("core.database") diff --git a/tests/test_helpers_import_state.py b/tests/test_helpers_import_state.py index 3e7d7a7..fdf4067 100644 --- a/tests/test_helpers_import_state.py +++ b/tests/test_helpers_import_state.py @@ -6,6 +6,7 @@ import pytest from tests.helpers.import_state import ( clear_fake_database_modules, + clear_fake_endpoint_resolver_modules, clear_module, preserve_import_state, ) @@ -16,6 +17,16 @@ _SENTINEL = "tests._import_state_test_sentinel" # tests never leak into the real core/src packages. _DB_NAMES = ("core", "core.database", "src", "src.database") +# Names touched by clear_fake_endpoint_resolver_modules — snapshot/restore these +# so the tests never leak into the real src/routes packages. +_RESOLVER_NAMES = ( + "src", + "src.endpoint_resolver", + "routes", + "routes.model_routes", + "routes.chat_routes", +) + def test_absent_module_is_removed_after_block(): assert _SENTINEL not in sys.modules @@ -250,3 +261,166 @@ def test_clear_fake_database_noop_when_nothing_cached(): clear_fake_database_modules() # must not raise assert "core.database" not in sys.modules + + +def test_clear_fake_resolver_removes_stub_endpoint_resolver(): + with preserve_import_state(*_RESOLVER_NAMES): + fake_src = types.ModuleType("src") + fake_resolver = types.ModuleType("src.endpoint_resolver") # no __file__ => stub + fake_src.endpoint_resolver = fake_resolver + sys.modules["src"] = fake_src + sys.modules["src.endpoint_resolver"] = fake_resolver + + clear_fake_endpoint_resolver_modules() + + assert "src.endpoint_resolver" not in sys.modules + assert not hasattr(fake_src, "endpoint_resolver") + + +def test_clear_fake_resolver_preserves_real_endpoint_resolver(): + with preserve_import_state(*_RESOLVER_NAMES): + fake_src = types.ModuleType("src") + real_resolver = types.ModuleType("src.endpoint_resolver") + real_resolver.__file__ = "/somewhere/src/endpoint_resolver.py" # looks on-disk + fake_src.endpoint_resolver = real_resolver + sys.modules["src"] = fake_src + sys.modules["src.endpoint_resolver"] = real_resolver + + clear_fake_endpoint_resolver_modules() + + assert sys.modules["src.endpoint_resolver"] is real_resolver + assert fake_src.endpoint_resolver is real_resolver + + +def test_clear_fake_resolver_evicts_empty_file_resolver(): + """A resolver with __file__ = "" is a stub under the old truthiness guard, so + it (and its dependents) must be evicted, not preserved.""" + with preserve_import_state(*_RESOLVER_NAMES): + fake_src = types.ModuleType("src") + empty_resolver = types.ModuleType("src.endpoint_resolver") + empty_resolver.__file__ = "" # falsy => stub + fake_src.endpoint_resolver = empty_resolver + sys.modules["src"] = fake_src + sys.modules["src.endpoint_resolver"] = empty_resolver + model_routes = types.ModuleType("routes.model_routes") + sys.modules["routes.model_routes"] = model_routes + + clear_fake_endpoint_resolver_modules() + + assert "src.endpoint_resolver" not in sys.modules + assert not hasattr(fake_src, "endpoint_resolver") + assert "routes.model_routes" not in sys.modules + + +def test_clear_fake_resolver_removes_model_routes_when_resolver_fake(): + """model_routes is dropped, and its parent `routes` attr is cleared too — + the behavior delta over the old bare sys.modules.pop() guards.""" + with preserve_import_state(*_RESOLVER_NAMES): + fake_src = types.ModuleType("src") + fake_resolver = types.ModuleType("src.endpoint_resolver") + fake_src.endpoint_resolver = fake_resolver + sys.modules["src"] = fake_src + sys.modules["src.endpoint_resolver"] = fake_resolver + + fake_routes = types.ModuleType("routes") + model_routes = types.ModuleType("routes.model_routes") + fake_routes.model_routes = model_routes + sys.modules["routes"] = fake_routes + sys.modules["routes.model_routes"] = model_routes + + clear_fake_endpoint_resolver_modules() + + assert "routes.model_routes" not in sys.modules + assert not hasattr(fake_routes, "model_routes") + + +def test_clear_fake_resolver_removes_extra_modules_when_resolver_fake(): + with preserve_import_state(*_RESOLVER_NAMES): + fake_src = types.ModuleType("src") + fake_resolver = types.ModuleType("src.endpoint_resolver") + fake_src.endpoint_resolver = fake_resolver + sys.modules["src"] = fake_src + sys.modules["src.endpoint_resolver"] = fake_resolver + + fake_routes = types.ModuleType("routes") + chat_routes = types.ModuleType("routes.chat_routes") + fake_routes.chat_routes = chat_routes + sys.modules["routes"] = fake_routes + sys.modules["routes.chat_routes"] = chat_routes + + clear_fake_endpoint_resolver_modules("routes.chat_routes") + + assert "routes.chat_routes" not in sys.modules + assert not hasattr(fake_routes, "chat_routes") + + +def test_clear_fake_resolver_keeps_dependents_when_resolver_real(): + with preserve_import_state(*_RESOLVER_NAMES): + fake_src = types.ModuleType("src") + real_resolver = types.ModuleType("src.endpoint_resolver") + real_resolver.__file__ = "/somewhere/src/endpoint_resolver.py" + fake_src.endpoint_resolver = real_resolver + sys.modules["src"] = fake_src + sys.modules["src.endpoint_resolver"] = real_resolver + + model_routes = types.ModuleType("routes.model_routes") + chat_routes = types.ModuleType("routes.chat_routes") + sys.modules["routes.model_routes"] = model_routes + sys.modules["routes.chat_routes"] = chat_routes + + clear_fake_endpoint_resolver_modules("routes.chat_routes") + + assert sys.modules["routes.model_routes"] is model_routes + assert sys.modules["routes.chat_routes"] is chat_routes + + +def test_clear_fake_resolver_noop_when_nothing_cached(): + with preserve_import_state(*_RESOLVER_NAMES): + sys.modules.pop("src.endpoint_resolver", None) + fake_src = types.ModuleType("src") # no endpoint_resolver attr + sys.modules["src"] = fake_src + model_routes = types.ModuleType("routes.model_routes") + sys.modules["routes.model_routes"] = model_routes + + clear_fake_endpoint_resolver_modules() # must not raise + + assert "src.endpoint_resolver" not in sys.modules + # dependents are left alone when the resolver was never cached + assert sys.modules["routes.model_routes"] is model_routes + + +def test_clear_fake_resolver_keeps_parent_attr_pointing_elsewhere(): + """When the cached src.endpoint_resolver is a stub but the `endpoint_resolver` + attr on the src package points at a *different* object, the attr is left + intact — only the same fake object is unlinked.""" + with preserve_import_state(*_RESOLVER_NAMES): + fake_src = types.ModuleType("src") + cached_fake = types.ModuleType("src.endpoint_resolver") # the stub in sys.modules + other = types.ModuleType("src.endpoint_resolver") # parent attr points here + fake_src.endpoint_resolver = other + sys.modules["src"] = fake_src + sys.modules["src.endpoint_resolver"] = cached_fake + + clear_fake_endpoint_resolver_modules() + + assert "src.endpoint_resolver" not in sys.modules + assert fake_src.endpoint_resolver is other + + +def test_clear_fake_resolver_uses_parent_attr_when_not_in_sys_modules(): + """A stub reachable only via the src package's `endpoint_resolver` attribute + (not in sys.modules) is still detected, unlinked, and triggers dependent + eviction.""" + with preserve_import_state(*_RESOLVER_NAMES): + sys.modules.pop("src.endpoint_resolver", None) + fake_src = types.ModuleType("src") + fake_resolver = types.ModuleType("src.endpoint_resolver") + fake_src.endpoint_resolver = fake_resolver + sys.modules["src"] = fake_src + model_routes = types.ModuleType("routes.model_routes") + sys.modules["routes.model_routes"] = model_routes + + clear_fake_endpoint_resolver_modules() + + assert not hasattr(fake_src, "endpoint_resolver") + assert "routes.model_routes" not in sys.modules diff --git a/tests/test_model_routes.py b/tests/test_model_routes.py index ec435ac..4d68546 100644 --- a/tests/test_model_routes.py +++ b/tests/test_model_routes.py @@ -11,12 +11,11 @@ from types import SimpleNamespace import httpx import pytest -_endpoint_resolver = sys.modules.get("src.endpoint_resolver") -if _endpoint_resolver is not None and not getattr(_endpoint_resolver, "__file__", None): - # Other tests stub this module during collection. These helper tests need - # the real URL normalization helpers so Anthropic /v1 handling is covered. - sys.modules.pop("src.endpoint_resolver", None) - sys.modules.pop("routes.model_routes", None) +from tests.helpers.import_state import clear_fake_endpoint_resolver_modules + +# Other tests stub this module during collection. These helper tests need +# the real URL normalization helpers so Anthropic /v1 handling is covered. +clear_fake_endpoint_resolver_modules() if "core.database" not in sys.modules: _core_db = types.ModuleType("core.database") From b5c45326e46c41b841a551778b1016368c9043b5 Mon Sep 17 00:00:00 2001 From: the_peaceful <54059317+pancake37@users.noreply.github.com> Date: Fri, 5 Jun 2026 14:41:07 +0200 Subject: [PATCH 011/974] Fix Windows Cookbook background tasks, exit statuses, and empty SSH logs wrapper (#1389) This commit consolidates all Windows Cookbook background fixes into a single comprehensive commit based on the latest main branch. Key fixes included: 1. React looksSuccessful Mismatch: Append 'DOWNLOAD_OK' for pip install commands in routes/cookbook_routes.py. 2. Local Windows SSH Wrapper & Log Directory Mismatch: Bypassed ssh wrappers and dynamically selected odysseus-tmux logs for local tasks in static/js/cookbookRunning.js. 3. WSL Bash Filtration: Filtered out the WSL bash stub at C:\Windows\System32\bash.exe in core/platform_compat.py. 4. Drive-Colon Path Normalization: Replaced .as_posix() with git_bash_path() in routes/shell_routes.py and src/bg_jobs.py. 5. GGUF-Only Hardware Fitting: Restructured local Windows recommendations to rank GGUF only in services/hwfit/fit.py. 6. Safe Win32 Process Liveness Probe: Replaced os.kill(pid, 0) with a safe Win32 API probe using GetExitCodeProcess in core/platform_compat.py. 7. Prebuilt llama-cpp-python Wheels: Supply the CPU extra index during compilation failure fallback. 8. Enforce UTF-8 log encoding: Set PYTHONIOENCODING=utf-8 on Windows bootstrap runners. 9. Fix Linux Llama.cpp Build script syntax error in routes/cookbook_helpers.py. 10. Page Reload Status Check: Run sys.executable instead of 'python3' to bypass Microsoft Store execution stubs on local Windows hosts. 11. Llama.cpp serve build bypass: Bypassed cmake compilation checks on local Windows and verified python bindings directly. 12. Serve Command Path Validation: Masked safe GGUF path printf subshells '' inside the serve command validator. 13. CPU Mismatch Diagnostics: Intercepted AVX2-lacking '0xc000001d' (Illegal Instruction) crashes in static/js/cookbook-diagnosis.js and guided users to Ollama. 14. Windows Pytest stability: Fixed stub import leakage in test files. --- core/platform_compat.py | 15 ++++ routes/cookbook_helpers.py | 17 ++++- routes/cookbook_routes.py | 118 ++++++++++++++++++++----------- routes/shell_routes.py | 9 ++- src/bg_jobs.py | 3 +- static/js/cookbook-diagnosis.js | 9 +++ static/js/cookbook.js | 15 +++- static/js/cookbookRunning.js | 7 +- tests/test_chat_image_routing.py | 6 ++ tests/test_cookbook_helpers.py | 21 +++++- 10 files changed, 166 insertions(+), 54 deletions(-) diff --git a/core/platform_compat.py b/core/platform_compat.py index e2339ad..f2160d9 100644 --- a/core/platform_compat.py +++ b/core/platform_compat.py @@ -180,6 +180,21 @@ def _is_windows_bash_stub(path: str) -> bool: ) +def git_bash_path(path: str | Path) -> str: + """Convert a path to POSIX style suitable for Git Bash on Windows. + + Transforms drive letters (e.g., 'C:\\path') to POSIX '/c/path', + and uses forward slashes. + """ + p = Path(path) + p_str = p.as_posix() + if IS_WINDOWS and len(p_str) >= 2 and p_str[1] == ":": + drive = p_str[0].lower() + return f"/{drive}{p_str[2:]}" + return p_str + + + def find_bash() -> Optional[str]: """Locate a real ``bash`` interpreter, or None. diff --git a/routes/cookbook_helpers.py b/routes/cookbook_helpers.py index 8fbaa9e..a20d78a 100644 --- a/routes/cookbook_helpers.py +++ b/routes/cookbook_helpers.py @@ -206,12 +206,16 @@ def _pip_install_fallback_chain(package: str, *, python_cmd: str = "python3 -m p exit code is preserved (no ``| tail`` masking) and the last 5 lines of pip output appear in the Cookbook log on failure. """ + from core.platform_compat import IS_WINDOWS upgrade_flag = " -U" if upgrade else "" # Shell-quote the package spec: an extras spec like ``llama-cpp-python[server]`` # contains brackets that bash would treat as a glob, so it must be quoted # before being embedded in the install command. Plain names (e.g. # ``huggingface_hub``) are returned unchanged by ``shlex.quote``. pkg = shlex.quote(package) + if IS_WINDOWS and "llama-cpp-python" in package: + pkg += " --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu" + base = _pip_install_attempt(f"{python_cmd} install -q{upgrade_flag} {pkg}") user = _pip_install_attempt(f"{python_cmd} install --user --break-system-packages -q{upgrade_flag} {pkg}") # Derive the python executable for the venv detection check. @@ -525,6 +529,7 @@ def _validate_serve_cmd(v: str | None) -> str | None: # Backticks and raw newlines are never legitimate here. if any(c in v for c in ("`", "\n", "\r")): raise HTTPException(400, "Invalid characters in cmd") + # Known GGUF launcher prelude → validate the serve invocation(s) it guards. m = _GGUF_PRELUDE_RE.match(v) if m: @@ -533,9 +538,19 @@ def _validate_serve_cmd(v: str | None) -> str | None: for part in rest.split("||"): _check_serve_binary(part.strip()) return v + # Otherwise: a single invocation — no shell metacharacters allowed. + # Temporarily replace safe $(printf %s ...) expressions with a placeholder + # to avoid triggering the metacharacter/command-injection checks. + cleaned_v = v + printf_matches = list(re.finditer(r"\$\(\s*printf\s+%s\s+([^\n()]*?)\)", v)) + for match in printf_matches: + inner = match.group(1) + if not any(c in inner for c in (";", "&&", "||", "$(", "`")): + cleaned_v = cleaned_v.replace(match.group(0), "/placeholder/safe/path.gguf") + # (`$(` was the original intent; bare `$` is fine for shell-safe paths.) - if any(c in v for c in (";", "&&", "||", "$(")): + if any(c in cleaned_v for c in (";", "&&", "||", "$(")): raise HTTPException(400, "Invalid characters in cmd") _check_serve_binary(v) return v diff --git a/routes/cookbook_routes.py b/routes/cookbook_routes.py index af5ff1d..2b86b6e 100644 --- a/routes/cookbook_routes.py +++ b/routes/cookbook_routes.py @@ -22,6 +22,7 @@ from core.platform_compat import ( IS_WINDOWS, detached_popen_kwargs, find_bash, + git_bash_path, kill_process_tree, pid_alive, safe_chmod, @@ -175,6 +176,7 @@ def setup_cookbook_routes() -> APIRouter: safe_chmod(key_path.with_suffix(".pub"), 0o644) return {"ok": True, "public_key": _read_cookbook_public_key()} + def _needs_binary(cmd: str, binary: str) -> bool: return bool(re.search(rf"(^|[\s;&|()]){re.escape(binary)}($|[\s;&|()])", cmd or "")) @@ -235,8 +237,8 @@ def setup_cookbook_routes() -> APIRouter: # POSIX form + shell-quoting so drive paths / spaces survive. inner = TMUX_LOG_DIR / f"{session_id}_run.sh" inner.write_text("\n".join(bash_lines) + "\n", encoding="utf-8") - lp = shlex.quote(log_path.as_posix()) - ip = shlex.quote(inner.as_posix()) + lp = shlex.quote(git_bash_path(log_path)) + ip = shlex.quote(git_bash_path(inner)) script_path = TMUX_LOG_DIR / f"{session_id}.sh" script_path.write_text( f"bash {ip} > {lp} 2>&1\n", @@ -352,6 +354,8 @@ def setup_cookbook_routes() -> APIRouter: ps_lines = [] ps_lines.append('$sessionDir = "$env:TEMP\\odysseus-sessions"') ps_lines.append('New-Item -ItemType Directory -Force -Path $sessionDir | Out-Null') + ps_lines.append('$env:PYTHONIOENCODING = "utf-8"') + ps_lines.append('$env:PYTHONUTF8 = "1"') if req.hf_token: ps_lines.append(f"$env:HF_TOKEN = '{_ps_squote(req.hf_token)}'") if req.env_prefix: @@ -851,6 +855,16 @@ def setup_cookbook_routes() -> APIRouter: in_venv=sys.prefix != sys.base_prefix, ) is_pip_install = bool(req.cmd and "pip install" in req.cmd) + remote = req.remote_host + is_windows = req.platform == "windows" + local_windows = IS_WINDOWS and not remote + if is_windows or local_windows: + if req.cmd.startswith("python3 "): + req.cmd = "python " + req.cmd[len("python3 "):] + if is_pip_install and ("llama-cpp-python" in req.cmd or "llama_cpp" in req.cmd) and (is_windows or local_windows): + if "--extra-index-url" not in req.cmd: + req.cmd += " --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu" + if is_pip_install: # Keep big dependency wheel builds (vLLM, …) off the home filesystem's # pip cache so they don't fail mid-build with "No space left" (#1219) @@ -908,6 +922,8 @@ def setup_cookbook_routes() -> APIRouter: ps_lines = [] ps_lines.append('$sessionDir = "$env:TEMP\\odysseus-sessions"') ps_lines.append('New-Item -ItemType Directory -Force -Path $sessionDir | Out-Null') + ps_lines.append('$env:PYTHONIOENCODING = "utf-8"') + ps_lines.append('$env:PYTHONUTF8 = "1"') if req.hf_token: ps_lines.append(f"$env:HF_TOKEN = '{_ps_squote(req.hf_token)}'") if req.gpus: @@ -926,7 +942,7 @@ def setup_cookbook_routes() -> APIRouter: ps_lines.append('try { python -c "import llama_cpp" 2>$null } catch {}') ps_lines.append('if ($LASTEXITCODE -ne 0) {') ps_lines.append(' Write-Host "Installing llama-cpp-python..."') - ps_lines.append(' python -m pip install llama-cpp-python[server]') + ps_lines.append(' python -m pip install llama-cpp-python[server] --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu') ps_lines.append('}') elif "vllm" in req.cmd: ps_lines.append('Write-Host "ERROR: vLLM is not supported on Windows. Use Ollama or llama.cpp instead."') @@ -1001,45 +1017,57 @@ def setup_cookbook_routes() -> APIRouter: # ollama is found (otherwise macOS falls back to a slow source build). # /opt/homebrew = Apple Silicon, /usr/local = Intel; harmless on Linux. runner_lines.append('export PATH="$HOME/.local/bin:$HOME/bin:$HOME/llama.cpp/build/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"') - runner_lines.append('if [ -d /data/data/com.termux ]; then') - runner_lines.append(' # Termux: no native build — use the Python bindings (CPU).') - runner_lines.append(' if ! python3 -c "import llama_cpp" 2>/dev/null; then') - runner_lines.append(' pkg install -y cmake 2>/dev/null') - runner_lines.append(' pip install numpy diskcache jinja2 2>/dev/null') - runner_lines.append(' CMAKE_ARGS="-DGGML_BLAS=OFF -DGGML_LLAMAFILE=OFF" pip install \'llama-cpp-python[server]\' --no-build-isolation --no-cache-dir 2>&1 || true') - runner_lines.append(' fi') - runner_lines.append('elif ! command -v llama-server &>/dev/null; then') - runner_lines.append(' echo "Native llama-server not found — building from source (one-time, may take a few minutes)..."') - runner_lines.append(' mkdir -p ~/bin') - runner_lines.append(' cd ~ && [ -d llama.cpp ] || git clone --depth 1 https://github.com/ggml-org/llama.cpp') - # Build with the right accelerator: Metal on macOS (llama.cpp - # enables it automatically, no flag), CUDA on Linux when present, - # else a plain CPU build. nproc is Linux-only — fall back to - # `sysctl hw.ncpu` on macOS. (Tip: `brew install llama.cpp` ships - # a prebuilt llama-server and skips this whole source build.) - runner_lines.append(' NPROC="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)"') - runner_lines.append(' if [ "$(uname -s)" = "Darwin" ]; then') - runner_lines.append(' command -v cmake >/dev/null 2>&1 || echo "WARNING: cmake not found — install it with: brew install cmake (or: brew install llama.cpp for a prebuilt llama-server)."') - # Start from a clean cache: a prior failed configure (e.g. a CUDA - # attempt) poisons build/CMakeCache.txt, so a plain `cmake -B build` - # would reuse the bad settings and fail again. CMAKE_BUILD_TYPE is - # explicit so the binary is optimized (Metal auto-enables on macOS). - runner_lines.append(' cd ~/llama.cpp && rm -rf build && cmake -B build -DCMAKE_BUILD_TYPE=Release \\') - runner_lines.append(' && cmake --build build -j"$NPROC" --target llama-server \\') - runner_lines.append(' && ln -sf ~/llama.cpp/build/bin/llama-server ~/bin/llama-server') - runner_lines.append(' else') - _append_llama_cpp_linux_accel_build_lines(runner_lines) - runner_lines.append(' fi') - runner_lines.append(' # If the native build failed, fall back to the Python bindings.') - runner_lines.append(' if ! command -v llama-server &>/dev/null && ! python3 -c "import llama_cpp" 2>/dev/null; then') - runner_lines.append(' echo "llama-server build failed — installing Python bindings as fallback..."') - runner_lines.append(f" {_pip_install_fallback_chain('llama-cpp-python[server]', python_cmd='pip')} || true") - runner_lines.append(' fi') - runner_lines.append(' if ! command -v llama-server &>/dev/null && ! python3 -c "import llama_cpp" 2>/dev/null; then') - runner_lines.append(' echo "ERROR: llama.cpp serving is not available after install/build attempts."') - runner_lines.append(' ODYSSEUS_PREFLIGHT_EXIT=127') - runner_lines.append(' fi') - runner_lines.append('fi') + if local_windows: + # LOCAL Windows: no native source compilation (no cmake/compiler on Git Bash). + # Just check python bindings (using native `python` binary) and fall back to pip install. + runner_lines.append('if ! command -v llama-server &>/dev/null && ! python -c "import llama_cpp" 2>/dev/null; then') + runner_lines.append(' echo "llama-server not found — installing Python bindings..."') + runner_lines.append(f" {_pip_install_fallback_chain('llama-cpp-python[server]', python_cmd='python')} || true") + runner_lines.append('fi') + runner_lines.append('if ! command -v llama-server &>/dev/null && ! python -c "import llama_cpp" 2>/dev/null; then') + runner_lines.append(' echo "ERROR: llama.cpp serving is not available after install attempts."') + runner_lines.append(' ODYSSEUS_PREFLIGHT_EXIT=127') + runner_lines.append('fi') + else: + runner_lines.append('if [ -d /data/data/com.termux ]; then') + runner_lines.append(' # Termux: no native build — use the Python bindings (CPU).') + runner_lines.append(' if ! python3 -c "import llama_cpp" 2>/dev/null; then') + runner_lines.append(' pkg install -y cmake 2>/dev/null') + runner_lines.append(' pip install numpy diskcache jinja2 2>/dev/null') + runner_lines.append(' CMAKE_ARGS="-DGGML_BLAS=OFF -DGGML_LLAMAFILE=OFF" pip install \'llama-cpp-python[server]\' --no-build-isolation --no-cache-dir 2>&1 || true') + runner_lines.append(' fi') + runner_lines.append('elif ! command -v llama-server &>/dev/null; then') + runner_lines.append(' echo "Native llama-server not found — building from source (one-time, may take a few minutes)..."') + runner_lines.append(' mkdir -p ~/bin') + runner_lines.append(' cd ~ && [ -d llama.cpp ] || git clone --depth 1 https://github.com/ggml-org/llama.cpp') + # Build with the right accelerator: Metal on macOS (llama.cpp + # enables it automatically, no flag), CUDA on Linux when present, + # else a plain CPU build. nproc is Linux-only — fall back to + # `sysctl hw.ncpu` on macOS. (Tip: `brew install llama.cpp` ships + # a prebuilt llama-server and skips this whole source build.) + runner_lines.append(' NPROC="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)"') + runner_lines.append(' if [ "$(uname -s)" = "Darwin" ]; then') + runner_lines.append(' command -v cmake >/dev/null 2>&1 || echo "WARNING: cmake not found — install it with: brew install cmake (or: brew install llama.cpp for a prebuilt llama-server)."') + # Start from a clean cache: a prior failed configure (e.g. a CUDA + # attempt) poisons build/CMakeCache.txt, so a plain `cmake -B build` + # would reuse the bad settings and fail again. CMAKE_BUILD_TYPE is + # explicit so the binary is optimized (Metal auto-enables on macOS). + runner_lines.append(' cd ~/llama.cpp && rm -rf build && cmake -B build -DCMAKE_BUILD_TYPE=Release \\') + runner_lines.append(' && cmake --build build -j"$NPROC" --target llama-server \\') + runner_lines.append(' && ln -sf ~/llama.cpp/build/bin/llama-server ~/bin/llama-server') + runner_lines.append(' else') + _append_llama_cpp_linux_accel_build_lines(runner_lines) + runner_lines.append(' fi') + # If the native build failed, fall back to the Python bindings. + runner_lines.append(' if ! command -v llama-server &>/dev/null && ! python3 -c "import llama_cpp" 2>/dev/null; then') + runner_lines.append(' echo "llama-server build failed — installing Python bindings as fallback..."') + runner_lines.append(f" {_pip_install_fallback_chain('llama-cpp-python[server]', python_cmd='pip')} || true") + runner_lines.append(' fi') + runner_lines.append(' if ! command -v llama-server &>/dev/null && ! python3 -c "import llama_cpp" 2>/dev/null; then') + runner_lines.append(' echo "ERROR: llama.cpp serving is not available after install/build attempts."') + runner_lines.append(' ODYSSEUS_PREFLIGHT_EXIT=127') + runner_lines.append(' fi') + runner_lines.append('fi') elif "ollama" in req.cmd: handled_ollama_serve = True _ollama_default_host = "0.0.0.0" if remote else "127.0.0.1" @@ -2076,7 +2104,11 @@ def setup_cookbook_routes() -> APIRouter: "inc=os.path.isdir(blobs) and any(x.endswith('.incomplete') for x in os.listdir(blobs));" "sys.exit(0 if ok and not inc else 1)" ) - cmd = ["python3", "-c", py, repo_id] + if not remote_host: + import sys + cmd = [sys.executable, "-c", py, repo_id] + else: + cmd = ["python3", "-c", py, repo_id] try: if remote_host: ssh_base = ["ssh"] diff --git a/routes/shell_routes.py b/routes/shell_routes.py index 3be54ab..9f99678 100644 --- a/routes/shell_routes.py +++ b/routes/shell_routes.py @@ -37,6 +37,7 @@ from core.platform_compat import ( IS_WINDOWS, detached_popen_kwargs, find_bash, + git_bash_path, ) @@ -368,8 +369,12 @@ async def _create_shell(command: str, **kwargs): POSIX: /bin/sh via create_subprocess_shell (unchanged behaviour). Windows: prefer a real bash (Git Bash/WSL) so bash-syntax commands behave the same as on Linux; fall back to cmd.exe when no bash is installed. + Powershell commands are executed directly via cmd.exe /c to avoid quoting + and env variable expansion errors under Git Bash. """ if IS_WINDOWS: + if command.strip().lower().startswith("powershell"): + return await asyncio.create_subprocess_shell(command, **kwargs) bash = find_bash() if bash: return await asyncio.create_subprocess_exec(bash, "-c", command, **kwargs) @@ -672,8 +677,8 @@ async def _generate_win_detached(cmd: str, request: Request): if bash: script_path = TMUX_LOG_DIR / f"{session_id}.sh" script_path.write_text( - f"{cmd} > {shlex.quote(str(log_path))} 2>&1\n" - f"echo $? > {shlex.quote(str(exit_path))}\n", + f"{cmd} > {shlex.quote(git_bash_path(log_path))} 2>&1\n" + f"echo $? > {shlex.quote(git_bash_path(exit_path))}\n", encoding="utf-8", ) argv = [bash, str(script_path)] diff --git a/src/bg_jobs.py b/src/bg_jobs.py index 587851b..c103dfd 100644 --- a/src/bg_jobs.py +++ b/src/bg_jobs.py @@ -33,6 +33,7 @@ from core.atomic_io import atomic_write_json from core.platform_compat import ( detached_popen_kwargs, find_bash, + git_bash_path, kill_process_tree, pid_alive, ) @@ -106,7 +107,7 @@ def launch(command: str, session_id: str, cwd: Optional[str] = None, # handles drive paths and spaces correctly. cmd_path = _JOBS_DIR / f"{job_id}.cmd.sh" cmd_path.write_text(command + "\n", encoding="utf-8") - lp, xp, cp = (shlex.quote(p.as_posix()) for p in (log_path, exit_path, cmd_path)) + lp, xp, cp = (shlex.quote(git_bash_path(p)) for p in (log_path, exit_path, cmd_path)) script_path = _JOBS_DIR / f"{job_id}.sh" script_path.write_text( f"bash {cp} > {lp} 2>&1\n" diff --git a/static/js/cookbook-diagnosis.js b/static/js/cookbook-diagnosis.js index af90d99..19512ab 100644 --- a/static/js/cookbook-diagnosis.js +++ b/static/js/cookbook-diagnosis.js @@ -426,6 +426,15 @@ export const ERROR_PATTERNS = [ { label: 'Copy install command', action: () => _copyText('pip install "llama-cpp-python[server]"') }, ], }, + { + pattern: /Windows Error 0xc000001d|Illegal instruction|0xc000001d/i, + message: 'AVX2 Instruction Set Mismatch: the precompiled llama-cpp-python wheel requires CPU features (AVX2/FMA) that your processor or virtual machine lacks.', + suggestion: 'Suggested action: switch this serve config to Ollama (highly recommended, has dynamic CPU fallbacks), or choose a remote Linux GPU server.', + fixes: [ + { label: 'Switch to Ollama', action: (panel) => _openServeEditFromDiagnosis(panel, { backend: 'ollama' }) }, + { label: 'Choose remote server', action: (panel) => _openServeEditFromDiagnosis(panel) }, + ], + }, { pattern: /CUDA Toolkit not found|Unable to find cudart library|missing:\s*CUDA_CUDART/i, message: 'llama.cpp found nvcc, but the CUDA runtime library is missing.', diff --git a/static/js/cookbook.js b/static/js/cookbook.js index 358d664..edcbab3 100644 --- a/static/js/cookbook.js +++ b/static/js/cookbook.js @@ -161,8 +161,17 @@ function _getPort(hostOrTask) { /** Get platform for a given host (or task object). Returns 'windows', 'termux', 'linux', or '' */ export function _getPlatform(hostOrTask) { - if (!hostOrTask) return _envState.platform || ''; - if (typeof hostOrTask === 'object') return hostOrTask.platform || _getPlatform(hostOrTask.remoteHost); + const isWinBrowser = (window.navigator.userAgent || window.navigator.platform || '').toLowerCase().includes('win'); + if (!hostOrTask || hostOrTask === 'local') { + return _envState.platform || (isWinBrowser ? 'windows' : ''); + } + if (typeof hostOrTask === 'object') { + const h = hostOrTask.remoteHost; + if (!h || h === 'local') { + return hostOrTask.platform || _envState.platform || (isWinBrowser ? 'windows' : ''); + } + return hostOrTask.platform || _getPlatform(h); + } const srv = _envState.servers.find(s => s.host === hostOrTask); return srv?.platform || ''; } @@ -637,7 +646,7 @@ async function _fetchDependencies() { const data = await resp.json(); const pkgs = data.packages || []; if (!pkgs.length) { list.innerHTML = '
No packages found
'; return; } - const _winUnsupported = new Set(['diffusers', 'hf_transfer', 'vllm', 'rembg', 'gfpgan']); + const _winUnsupported = new Set(['vllm', 'rembg', 'gfpgan']); const _statusTag = (pkg, isLocal, isSystemDep, winBlocked) => { if (winBlocked) return `N/A`; diff --git a/static/js/cookbookRunning.js b/static/js/cookbookRunning.js index 30d78f8..186004c 100644 --- a/static/js/cookbookRunning.js +++ b/static/js/cookbookRunning.js @@ -2738,6 +2738,7 @@ async function _reconnectTask(el, task) { _updateTask(task.sessionId, { status: 'done', _doneConfirmAt: null, _lastStatusFlipAt: Date.now() }); const _el = document.querySelector(`.cookbook-task[data-task-id="${task.sessionId}"]`); if (_el) { + _clearDiagnosis(_el); _el.dataset.status = 'done'; const _badge = _el.querySelector('.cookbook-task-status'); if (_badge) { _badge.textContent = _statusLabel('done', task.type); _badge.className = 'cookbook-task-status cookbook-task-done'; } @@ -2804,13 +2805,14 @@ async function _reconnectTask(el, task) { const curProgress = computeProgressSignal(_bytes, _dlAgg, lastPct, snapshot); const _fetchPctMatches = [...snapshot.matchAll(/Fetching\s+\d+\s+files:\s*(\d+)%/g)]; const _fetchPct = _fetchPctMatches.length ? parseInt(_fetchPctMatches[_fetchPctMatches.length - 1][1]) : null; + const isPipDep = !!(task.payload && task.payload._dep); const _startupStalled = !_bytes && ((_dlAgg === 0) || (_fetchPct === 0)) && curProgress === '0'; const _STALE_TIMEOUT = _startupStalled ? STARTUP_STALE_PROGRESS_MS : STALE_PROGRESS_MS; if (!el._lastProgress) { el._lastProgress = curProgress; el._lastProgressTime = Date.now(); } if (curProgress !== el._lastProgress) { el._lastProgress = curProgress; el._lastProgressTime = Date.now(); - } else if (Date.now() - (el._lastProgressTime || 0) > _STALE_TIMEOUT && task._autoRestarted) { + } else if (!isPipDep && Date.now() - (el._lastProgressTime || 0) > _STALE_TIMEOUT && task._autoRestarted) { const mins = Math.floor((Date.now() - (el._lastProgressTime || 0)) / 60000); // Already auto-restarted once and stalled again — make the badge a // one-click retry (resumes from the cached partial files) so the @@ -2823,7 +2825,7 @@ async function _reconnectTask(el, task) { badge._retryBound = true; badge.addEventListener('click', (e) => { e.stopPropagation(); _retryTask(el, task); }); } - } else if (Date.now() - (el._lastProgressTime || 0) > _STALE_TIMEOUT && !task._autoRestarted) { + } else if (!isPipDep && Date.now() - (el._lastProgressTime || 0) > _STALE_TIMEOUT && !task._autoRestarted) { task._autoRestarted = true; _updateTask(task.sessionId, { _autoRestarted: true }); badge.textContent = _startupStalled ? '0% stall — retrying' : 'stale — restarting'; @@ -2975,6 +2977,7 @@ async function _reconnectTask(el, task) { break; } if (snapshot.includes('DOWNLOAD_OK') || (snapshot.includes('/snapshots/') && completed >= totalFiles && totalFiles > 0)) { + _clearDiagnosis(el); _dlRetryCount.delete(task.payload?.repo_id || task.name); badge.textContent = _statusLabel('done', task.type); badge.className = 'cookbook-task-status cookbook-task-done'; diff --git a/tests/test_chat_image_routing.py b/tests/test_chat_image_routing.py index 21e06f5..14f8744 100644 --- a/tests/test_chat_image_routing.py +++ b/tests/test_chat_image_routing.py @@ -1,3 +1,9 @@ +import sys +for mod_name in ["src.endpoint_resolver", "src.database", "core.database"]: + _mod = sys.modules.get(mod_name) + if _mod is not None and not getattr(_mod, "__file__", None): + sys.modules.pop(mod_name, None) + import json from types import SimpleNamespace diff --git a/tests/test_cookbook_helpers.py b/tests/test_cookbook_helpers.py index 84e91ba..033823e 100644 --- a/tests/test_cookbook_helpers.py +++ b/tests/test_cookbook_helpers.py @@ -238,6 +238,8 @@ def test_pip_install_attempt_failure_propagates_real_exit_code(): """Run the generated snippet against a deliberately broken pip install to confirm the subshell exits with pip's non-zero status.""" snippet = _pip_install_attempt("python3 -m pip install __nonexistent_package_12345__") + if sys.platform == "win32": + snippet = snippet.replace("$", "\\$") result = subprocess.run( ["bash", "-c", snippet], capture_output=True, @@ -250,6 +252,8 @@ def test_pip_install_attempt_failure_propagates_real_exit_code(): def test_pip_install_attempt_success_exits_zero(): """When pip succeeds, the subshell should exit 0.""" snippet = _pip_install_attempt("python3 -c 'pass'") + if sys.platform == "win32": + snippet = snippet.replace("$", "\\$") result = subprocess.run( ["bash", "-c", snippet], capture_output=True, @@ -262,6 +266,8 @@ def test_pip_install_attempt_success_exits_zero(): def test_pip_install_attempt_surfaces_stderr_on_failure(): """On failure, the last 5 lines of pip output should appear in stdout.""" snippet = _pip_install_attempt("python3 -m pip install __nonexistent_package_12345__") + if sys.platform == "win32": + snippet = snippet.replace("$", "\\$") result = subprocess.run( ["bash", "-c", snippet], capture_output=True, @@ -354,6 +360,15 @@ def test_validate_serve_cmd_accepts_llama_advanced_controls(): assert _validate_serve_cmd(cmd) == cmd +def test_validate_serve_cmd_accepts_windows_printf_format(): + cmd = ( + "python -m llama_cpp.server --model " + "\"$(printf %s ${HOME}'/.cache/huggingface/hub/models--unsloth--Qwen3.5-2B-GGUF/snapshots/f6d5376be1edb4d416d56da11e5397a961aca8ae/Qwen3.5-2B-Q4_K_M.gguf')\" " + "--host 0.0.0.0 --port 8000 --n_gpu_layers 99 --n_ctx 32768 --flash_attn true --type_k q4_0 --type_v q4_0" + ) + assert _validate_serve_cmd(cmd) == cmd + + def test_ollama_serve_defaults_to_loopback_bind(): assert _ollama_bind_from_cmd("ollama serve") == ("127.0.0.1", "11434") assert _ollama_bind_from_cmd("ollama run qwen2.5:0.5b") == ("127.0.0.1", "11434") @@ -481,11 +496,13 @@ def test_llama_cpp_rebuild_cmd_clears_cached_build_paths(): def test_llama_cpp_rebuild_cmd_runs_clean_on_a_fresh_home(tmp_path): """The command should succeed even when neither path exists yet.""" import os + from core.platform_compat import find_bash, git_bash_path + bash = find_bash() or "bash" env = dict(os.environ) - env["HOME"] = str(tmp_path) + env["HOME"] = git_bash_path(tmp_path) result = subprocess.run( - ["bash", "-c", _llama_cpp_rebuild_cmd()], + [bash, "-c", _llama_cpp_rebuild_cmd()], capture_output=True, text=True, env=env, timeout=10, ) From ec8fbf5d8f2429428fbaefdb27b3654db2d97dac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yi=C4=9Fit=20Egemen?= Date: Fri, 5 Jun 2026 14:47:24 +0200 Subject: [PATCH 012/974] Add support for EMBEDDING_API_KEY (#2691) * feat: support for embedding API key * feat: encrypt and decrypt embedding API key * test: add unit tests for EmbeddingClient authorization header behavior --- .env.example | 3 +++ docker-compose.gpu-amd.yml | 1 + docker-compose.gpu-nvidia.yml | 1 + docker-compose.yml | 1 + routes/embedding_routes.py | 10 +++++++- src/embeddings.py | 9 +++++-- tests/test_embeddings.py | 46 +++++++++++++++++++++++++++++++++++ 7 files changed, 68 insertions(+), 3 deletions(-) create mode 100644 tests/test_embeddings.py diff --git a/.env.example b/.env.example index f282880..39c90b3 100644 --- a/.env.example +++ b/.env.example @@ -112,6 +112,9 @@ SEARXNG_INSTANCE=http://localhost:8080 # Default: http://{LLM_HOST}:11434/v1/embeddings (ollama) # EMBEDDING_URL=http://localhost:11434/v1/embeddings +# Embedding API key (if there's one) +# EMBEDDING_API_KEY=embedding_api_key_here + # Embedding model name (must be available at the endpoint above) # EMBEDDING_MODEL=all-minilm:l6-v2 diff --git a/docker-compose.gpu-amd.yml b/docker-compose.gpu-amd.yml index 47e0c85..6d87cb6 100644 --- a/docker-compose.gpu-amd.yml +++ b/docker-compose.gpu-amd.yml @@ -52,6 +52,7 @@ services: - SECURE_COOKIES=${SECURE_COOKIES:-false} - EMBEDDING_URL=${EMBEDDING_URL:-} - EMBEDDING_MODEL=${EMBEDDING_MODEL:-} + - EMBEDDING_API_KEY=${EMBEDDING_API_KEY:-} - FASTEMBED_MODEL=${FASTEMBED_MODEL:-sentence-transformers/all-MiniLM-L6-v2} - FASTEMBED_CACHE_PATH=${FASTEMBED_CACHE_PATH:-} - CLEANUP_INTERVAL_HOURS=${CLEANUP_INTERVAL_HOURS:-24} diff --git a/docker-compose.gpu-nvidia.yml b/docker-compose.gpu-nvidia.yml index 36ca10e..f61d22a 100644 --- a/docker-compose.gpu-nvidia.yml +++ b/docker-compose.gpu-nvidia.yml @@ -51,6 +51,7 @@ services: - SECURE_COOKIES=${SECURE_COOKIES:-false} - EMBEDDING_URL=${EMBEDDING_URL:-} - EMBEDDING_MODEL=${EMBEDDING_MODEL:-} + - EMBEDDING_API_KEY=${EMBEDDING_API_KEY:-} - FASTEMBED_MODEL=${FASTEMBED_MODEL:-sentence-transformers/all-MiniLM-L6-v2} - FASTEMBED_CACHE_PATH=${FASTEMBED_CACHE_PATH:-} - CLEANUP_INTERVAL_HOURS=${CLEANUP_INTERVAL_HOURS:-24} diff --git a/docker-compose.yml b/docker-compose.yml index f3a8dcc..b5b3fd9 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -40,6 +40,7 @@ services: - SECURE_COOKIES=${SECURE_COOKIES:-false} - EMBEDDING_URL=${EMBEDDING_URL:-} - EMBEDDING_MODEL=${EMBEDDING_MODEL:-} + - EMBEDDING_API_KEY=${EMBEDDING_API_KEY:-} - FASTEMBED_MODEL=${FASTEMBED_MODEL:-sentence-transformers/all-MiniLM-L6-v2} - FASTEMBED_CACHE_PATH=${FASTEMBED_CACHE_PATH:-} - CLEANUP_INTERVAL_HOURS=${CLEANUP_INTERVAL_HOURS:-24} diff --git a/routes/embedding_routes.py b/routes/embedding_routes.py index a5ef4c0..c6f0645 100644 --- a/routes/embedding_routes.py +++ b/routes/embedding_routes.py @@ -258,7 +258,7 @@ def setup_embedding_routes(): } @router.post("/endpoint") - def set_endpoint(url: str = Form(...), model: str = Form("")): + def set_endpoint(url: str = Form(...), model: str = Form(""), api_key: str = Form("")): """Save a custom embedding endpoint URL.""" url = url.strip() if not url: @@ -282,6 +282,7 @@ def setup_embedding_routes(): resp = httpx.post( url, json={"input": ["test"], "model": model or "test"}, + headers={"Authorization": f"Bearer {api_key}"} if api_key else {}, timeout=10, ) resp.raise_for_status() @@ -292,10 +293,16 @@ def setup_embedding_routes(): data = {"url": url} if model: data["model"] = model + if api_key: + from src.secret_storage import encrypt + data["api_key"] = encrypt(api_key) + _save_custom_endpoint(data) os.environ["EMBEDDING_URL"] = url if model: os.environ["EMBEDDING_MODEL"] = model + if api_key: + os.environ["EMBEDDING_API_KEY"] = api_key # Reset the RAG singleton so it picks up the new endpoint import src.rag_singleton as _rs @@ -329,6 +336,7 @@ def setup_embedding_routes(): # Remove from environment os.environ.pop("EMBEDDING_URL", None) os.environ.pop("EMBEDDING_MODEL", None) + os.environ.pop("EMBEDDING_API_KEY", None) # Reset the RAG singleton so it falls back to fastembed import src.rag_singleton as _rs diff --git a/src/embeddings.py b/src/embeddings.py index 67cfd86..f2d0c59 100644 --- a/src/embeddings.py +++ b/src/embeddings.py @@ -38,12 +38,13 @@ _DEFAULT_FASTEMBED_MODEL = "sentence-transformers/all-MiniLM-L6-v2" class EmbeddingClient: """Drop-in replacement for SentenceTransformer.encode() using an HTTP API.""" - def __init__(self, url: Optional[str] = None, model: Optional[str] = None): + def __init__(self, url: Optional[str] = None, model: Optional[str] = None, api_key: Optional[str] = None): self.url = url or os.getenv( "EMBEDDING_URL", f"http://{os.getenv('LLM_HOST', 'localhost')}:11434/v1/embeddings", ) self.model = model or os.getenv("EMBEDDING_MODEL", _DEFAULT_MODEL) + self.api_key = api_key or os.getenv("EMBEDDING_API_KEY") self._dim: Optional[int] = None # Short connect timeout so a DOWN embedding endpoint (e.g. Ollama not # running on :11434) fast-fails to the local FastEmbed fallback instead @@ -74,6 +75,7 @@ class EmbeddingClient: batch = texts[i : i + 64] resp = self._client.post( self.url, + headers={"Authorization": f"Bearer {self.api_key}"} if self.api_key else {}, json={"input": batch, "model": self.model}, ) resp.raise_for_status() @@ -222,11 +224,14 @@ def get_embedding_client(): if persisted.get("url"): url = persisted["url"] model = persisted.get("model", "") + api_key = persisted.get("api_key", "") # Also set in env so other code sees it os.environ["EMBEDDING_URL"] = url if model: os.environ["EMBEDDING_MODEL"] = model - + if api_key: + from src.secret_storage import decrypt + os.environ["EMBEDDING_API_KEY"] = decrypt(api_key) # Try the HTTP embedding API — unless we already found it down this process # (avoids paying the connect timeout again on every RAG/memory/tool probe). if not _http_embed_down: diff --git a/tests/test_embeddings.py b/tests/test_embeddings.py new file mode 100644 index 0000000..a32fb1e --- /dev/null +++ b/tests/test_embeddings.py @@ -0,0 +1,46 @@ +"""Tests for embeddings.py""" +from unittest.mock import MagicMock, patch +from src.embeddings import EmbeddingClient + + +class TestEmbeddingClient: + _MOCK_RESPONSE = { + "data": [{"embedding": [0.1], "index": 0}], + } + + def _make_mock_resp(self): + resp = MagicMock() + resp.status_code = 200 + resp.json.return_value = self._MOCK_RESPONSE + resp.raise_for_status = MagicMock() + return resp + + @patch("src.embeddings.httpx.Client") + def test_bearer_header_sent_when_api_key_set(self, mock_httpx): + """ + Test that the EmbeddingClient sends the Authorization header with the correct value when api_key is set. + """ + mock_httpx.return_value.post.return_value = self._make_mock_resp() + + client = EmbeddingClient( + url="http://test:11434/v1/embeddings", + model="all-minilm:l6-v2", + api_key="secret-key", + ) + client.encode(["x"]) + + headers = mock_httpx.return_value.post.call_args.kwargs["headers"] + assert headers.get("Authorization") == "Bearer secret-key" + + @patch("src.embeddings.httpx.Client") + def test_no_bearer_header_when_api_key_none(self, mock_httpx): + """ + Test that the EmbeddingClient does not send the Authorization header when api_key is None. + """ + mock_httpx.return_value.post.return_value = self._make_mock_resp() + + client = EmbeddingClient(url="http://test:11434/v1/embeddings") + client.encode(["x"]) + + headers = mock_httpx.return_value.post.call_args.kwargs["headers"] + assert "Authorization" not in headers From bec594904d49554c346fb0a121e0baef388a4730 Mon Sep 17 00:00:00 2001 From: Zen0-99 Date: Fri, 5 Jun 2026 13:53:33 +0100 Subject: [PATCH 013/974] Fix/windows llama cpp serve and test upstream (#2669) * fix: code runner base64, Windows serve paths, endpoint cache clear, copy-log guards, model-picker remove-recent * Revert model-picker 'remove from recent' feature and remove stray PR_DRAFT.md --- routes/cookbook_helpers.py | 9 +++++++ routes/cookbook_routes.py | 41 +++++++++++++++++++++--------- routes/shell_routes.py | 8 ++++-- static/js/codeRunner.js | 8 ++++-- static/js/cookbook-hwfit.js | 8 ++++++ static/js/cookbookRunning.js | 20 ++++++++++++++- static/js/cookbookServe.js | 48 ++++++++++++++++++++++++++++++------ 7 files changed, 119 insertions(+), 23 deletions(-) diff --git a/routes/cookbook_helpers.py b/routes/cookbook_helpers.py index a20d78a..298a336 100644 --- a/routes/cookbook_helpers.py +++ b/routes/cookbook_helpers.py @@ -342,6 +342,15 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None) -> str: " if f.is_file(): nf += 1; sz += f.stat().st_size", " if f.name.endswith('.incomplete'): ic = True", " snap = os.path.join(cache, d, 'snapshots')", + " # Windows HF cache stores files directly in snapshots/; blobs/ may be empty.", + " # Fallback: scan snapshots for real files when blobs yielded nothing.", + " if sz == 0 and os.path.isdir(snap):", + " for sd in os.listdir(snap):", + " sf = os.path.join(snap, sd)", + " if not os.path.isdir(sf): continue", + " for f in os.scandir(sf):", + " if f.is_file(): nf += 1; sz += f.stat().st_size", + " if f.name.endswith('.incomplete'): ic = True", " is_diffusion = False; gguf_files = []", " if os.path.isdir(snap):", " for sd in os.listdir(snap):", diff --git a/routes/cookbook_routes.py b/routes/cookbook_routes.py index 2b86b6e..04ad055 100644 --- a/routes/cookbook_routes.py +++ b/routes/cookbook_routes.py @@ -799,6 +799,10 @@ def setup_cookbook_routes() -> APIRouter: existing.name = display_name if supports_tools is not None: existing.supports_tools = supports_tools + # Wipe stale model lists so the picker re-probes and discovers + # the newly-served model instead of showing the old one. + existing.cached_models = None + existing.hidden_models = None db.commit() logger.info(f"Updated existing local model endpoint: {base_url}") return existing.id @@ -1089,13 +1093,23 @@ def setup_cookbook_routes() -> APIRouter: runner_lines.append(' ODYSSEUS_OLLAMA_PORT="$_ody_try_port"') runner_lines.append(' break') runner_lines.append(' fi') - runner_lines.append(' exec 3<&-; exec 3>&-') - runner_lines.append('done') + runner_lines.append(' echo "[odysseus] Ollama API ready on port ${ODYSSEUS_OLLAMA_PORT}: ${ODYSSEUS_OLLAMA_URL}"') + runner_lines.append(' echo "[odysseus] This task is monitoring an existing Ollama server; stopping it here will not stop an external Docker/system service."') + if local_windows: + # Windows detached process has no TTY; exec bash -i crashes. + # Keep the monitoring task alive with a sleep loop. + runner_lines.append(' while true; do sleep 60; done') + else: + runner_lines.append(' exec bash -i') + runner_lines.append('fi') runner_lines.append('if ! command -v ollama &>/dev/null; then') runner_lines.append(' echo "ERROR: Ollama not found on this server. Install it from https://ollama.com/download or `curl -fsSL https://ollama.com/install.sh | sh`."') runner_lines.append(' echo') runner_lines.append(' echo "=== Process exited with code 127 ==="') - runner_lines.append(' exec bash -i') + if local_windows: + runner_lines.append(' exit 127') + else: + runner_lines.append(' exec bash -i') runner_lines.append('fi') runner_lines.append('ODYSSEUS_OLLAMA_URL="http://${ODYSSEUS_OLLAMA_HOST}:${ODYSSEUS_OLLAMA_PORT}"') if remote and _ollama_host in ("0.0.0.0", "::"): @@ -1103,10 +1117,13 @@ def setup_cookbook_routes() -> APIRouter: runner_lines.append('echo "[odysseus] Ollama has no built-in authentication; expose this only on a trusted LAN/VPN or provide an explicit OLLAMA_HOST with your own access controls."') runner_lines.append('echo "Starting ollama server on ${ODYSSEUS_OLLAMA_HOST}:${ODYSSEUS_OLLAMA_PORT}..."') runner_lines.append('OLLAMA_HOST="${ODYSSEUS_OLLAMA_HOST}:${ODYSSEUS_OLLAMA_PORT}" ollama serve') - runner_lines.append('_ody_exit=$?') - runner_lines.append('echo') - runner_lines.append('echo "=== Process exited with code ${_ody_exit} ==="') - runner_lines.append('exec bash -i') + if local_windows: + _append_serve_exit_code_lines(runner_lines, keep_shell_open=False) + else: + runner_lines.append('_ody_exit=$?') + runner_lines.append('echo') + runner_lines.append('echo "=== Process exited with code ${_ody_exit} ==="') + runner_lines.append('exec bash -i') elif "vllm serve" in req.cmd: # vLLM is CUDA/ROCm-only and does not run on macOS at all. runner_lines.append('if [ "$(uname -s)" = "Darwin" ]; then') @@ -2104,11 +2121,13 @@ def setup_cookbook_routes() -> APIRouter: "inc=os.path.isdir(blobs) and any(x.endswith('.incomplete') for x in os.listdir(blobs));" "sys.exit(0 if ok and not inc else 1)" ) - if not remote_host: - import sys - cmd = [sys.executable, "-c", py, repo_id] - else: + if remote_host: cmd = ["python3", "-c", py, repo_id] + else: + # Local Windows: python3 can hit the Microsoft Store stub. Use the + # real Python Odysseus is running under (guaranteed to exist). + import sys as _sys_local + cmd = [_sys_local.executable, "-c", py, repo_id] try: if remote_host: ssh_base = ["ssh"] diff --git a/routes/shell_routes.py b/routes/shell_routes.py index 9f99678..e8077f6 100644 --- a/routes/shell_routes.py +++ b/routes/shell_routes.py @@ -373,7 +373,11 @@ async def _create_shell(command: str, **kwargs): and env variable expansion errors under Git Bash. """ if IS_WINDOWS: - if command.strip().lower().startswith("powershell"): + # PowerShell commands (used by the frontend for Windows log-file polling + # and session management) must run directly — passing them through + # bash -c mangles $env:VAR syntax and breaks the command. + cmd_trim = command.strip() + if cmd_trim.startswith("powershell") or cmd_trim.startswith("cmd "): return await asyncio.create_subprocess_shell(command, **kwargs) bash = find_bash() if bash: @@ -758,7 +762,7 @@ def setup_shell_routes() -> APIRouter: return {"stdout": "", "stderr": "No command provided", "exit_code": 1} logger.info("User shell exec requested: length=%d", len(cmd)) - result = await _exec_shell(cmd, timeout=EXEC_TIMEOUT) + result = await _exec_shell(cmd, timeout=req.timeout if req.timeout is not None else EXEC_TIMEOUT) return result @router.post("/api/shell/stream") diff --git a/static/js/codeRunner.js b/static/js/codeRunner.js index d0336b9..bd333a8 100644 --- a/static/js/codeRunner.js +++ b/static/js/codeRunner.js @@ -310,11 +310,15 @@ try { */ export async function runServer(code, panel, lang) { showLoading(panel, 'Running on server...'); + // Base64-encode the script so newlines survive the shell quoting intact. + // JSON.stringify turns \n into literal \\n which python3 -c sees as backslash-n; + // base64 avoids every quoting/escaping pitfall. + const b64 = btoa(unescape(encodeURIComponent(code))); var command; if (lang === 'python' || lang === 'py') { - command = 'python3 -c ' + JSON.stringify(code); + command = `python3 -c "import base64; exec(base64.b64decode('${b64}').decode('utf-8'))"`; } else { - command = 'bash -c ' + JSON.stringify(code); + command = `python3 -c "import base64, subprocess, sys; sys.exit(subprocess.run(['bash','-c',base64.b64decode('${b64}').decode('utf-8')]).returncode)"`; } try { var res = await fetch('/api/shell/exec', { diff --git a/static/js/cookbook-hwfit.js b/static/js/cookbook-hwfit.js index 161b6f3..7d57d1c 100644 --- a/static/js/cookbook-hwfit.js +++ b/static/js/cookbook-hwfit.js @@ -443,6 +443,9 @@ export async function _hwfitFetch(fresh = false) { if (_cached) { _hwfitCache = _cached; _hwfitRenderHw(hw, _cached.system); + if (!remoteHost && _cached.system && _cached.system.platform) { + _envState.platform = _cached.system.platform; + } _hwfitRenderList(list, _applyEngineFilter(_cached.models)); } else { // Show spinner while scanning — stack the spinner above a text label @@ -578,6 +581,11 @@ export async function _hwfitFetch(fresh = false) { } _hwfitCache = data; _hwfitRenderHw(hw, data.system); + // Propagate local platform from hardware probe so _isWindows(task) works + // for local tasks (menu items, shell commands, etc.). + if (!remoteHost && data.system && data.system.platform) { + _envState.platform = data.system.platform; + } // Sort client-side by the active column so the highest↔lowest toggle is // deterministic (the previous array .reverse() didn't reliably flip). // 1st click on a column = highest first; clicking it again = lowest first. diff --git a/static/js/cookbookRunning.js b/static/js/cookbookRunning.js index 186004c..4254309 100644 --- a/static/js/cookbookRunning.js +++ b/static/js/cookbookRunning.js @@ -1862,7 +1862,17 @@ export function _renderRunningTab() { const startNow = el.querySelector('.cookbook-task-start-now'); if (startNow) startNow.style.display = (task.type === 'download' && task.status === 'queued') ? '' : 'none'; const terminalDiag = _terminalServeDiagnosis(task, el.querySelector('.cookbook-output-pre')?.textContent || task.output || ''); - if (terminalDiag) _showDiagnosis(el, terminalDiag, el.querySelector('.cookbook-output-pre')?.textContent || task.output || ''); + if (terminalDiag) { + _showDiagnosis(el, terminalDiag, el.querySelector('.cookbook-output-pre')?.textContent || task.output || ''); + } else { + const existingDiag = el.querySelector('.cookbook-diagnosis'); + // Keep diagnosis for failed tasks even if output was cleared and we + // can no longer re-derive the exact message — removing it would hide + // the crash reason from the user. + if (existingDiag && !['stopped', 'error', 'crashed', 'failed'].includes(task.status)) { + existingDiag.remove(); + } + } } if (!task) { if (el._uptimeInterval) { clearInterval(el._uptimeInterval); el._uptimeInterval = null; } @@ -2201,6 +2211,10 @@ export function _renderRunningTab() { items.push({ label: 'Copy last 50 lines', action: 'copy-log', custom: () => { const out = (el.querySelector('.cookbook-output-pre')?.textContent || task.output || ''); const last = out.split('\n').slice(-50).join('\n'); + if (!last.trim()) { + uiModule.showToast('No log content available yet'); + return; + } _copyText(last); uiModule.showToast('Copied last 50 lines'); }}); @@ -2437,6 +2451,10 @@ export function _renderRunningTab() { el.querySelector('.cookbook-output-copy').addEventListener('click', (e) => { e.stopPropagation(); const text = el.querySelector('.cookbook-output-pre')?.textContent || ''; + if (!text.trim()) { + uiModule.showToast('No log content available yet'); + return; + } _copyText(text).then(() => { const btn = el.querySelector('.cookbook-output-copy'); const origHTML = btn.innerHTML; diff --git a/static/js/cookbookServe.js b/static/js/cookbookServe.js index c27ac38..69a912c 100644 --- a/static/js/cookbookServe.js +++ b/static/js/cookbookServe.js @@ -242,6 +242,21 @@ function _shellPathExpr(path) { function _selectedGgufExpr(model, repo, relPath) { const rel = String(relPath || '').replace(/^\/+/, ''); if (!rel) return ''; + if (_isWindows()) { + // PowerShell: plain path — no bash $() syntax (backend validator rejects + // $( ) in non-prelude commands, and PowerShell doesn't have printf). + const relW = rel.replace(/\//g, '\\'); + if (model.is_local_dir && model.path) { + const base = String(model.path || '').replace(/\/+$/, '').replace(/\//g, '\\'); + return `${base}\\${repo.replace(/\//g, '\\')}\\${relW}`; + } + if (model.path) { + const base = String(model.path || '').replace(/\/+$/, '').replace(/\//g, '\\'); + return `${base}\\models--${repo.replace(/\//g, '--')}\\snapshots\\${relW}`; + } + const cacheRepo = repo.replace(/\//g, '--'); + return `$env:USERPROFILE\\.cache\\huggingface\\hub\\models--${cacheRepo}\\snapshots\\${relW}`; + } if (model.is_local_dir && model.path) { const base = String(model.path || '').replace(/\/+$/, ''); return `$(printf %s ${_shellPathExpr(`${base}/${repo}/${rel}`)})`; @@ -255,6 +270,15 @@ function _selectedGgufExpr(model, repo, relPath) { } function _ggufSearchDirExpr(model, repo) { + if (_isWindows()) { + if (model.is_local_dir && model.path) { + return `${String(model.path || '').replace(/\/+$/, '').replace(/\//g, '\\')}\\${repo.replace(/\//g, '\\')}`; + } + if (model.path) { + return `${String(model.path || '').replace(/\/+$/, '').replace(/\//g, '\\')}\\models--${repo.replace(/\//g, '--')}\\snapshots`; + } + return `$env:USERPROFILE\\.cache\\huggingface\\hub\\models--${repo.replace(/\//g, '--')}\\snapshots`; + } if (model.is_local_dir && model.path) return _shellQuote(`${String(model.path || '').replace(/\/+$/, '')}/${repo}`); if (model.path) return _shellQuote(`${String(model.path || '').replace(/\/+$/, '')}/models--${repo.replace(/\//g, '--')}/snapshots`); return `"$HOME/.cache/huggingface/hub/models--${repo.replace(/\//g, '--')}/snapshots"`; @@ -800,17 +824,27 @@ function _rerenderCachedModels() { // model the file lives under "/" — search there just like we // search the HF snapshots dir, so serving a GGUF from a custom dir works // instead of handing llama.cpp a directory (which fails). - const _ldir = m.path ? _shellQuote(`${m.path}/${repo}`) : '""'; - f._gguf_path = selectedGguf - ? _selectedGgufExpr(m, repo, selectedGguf.rel_path) - : m.is_local_dir && m.path - ? `$({ find ${_ldir} -name '*-00001-of-*.gguf' 2>/dev/null | sort; find ${_ldir} -name '*.gguf' 2>/dev/null | sort; } | head -1)` - : `$({ find ${dir} -name '*-00001-of-*.gguf' 2>/dev/null | sort; find ${dir} -name '*.gguf' 2>/dev/null | sort; } | head -1)`; + const _ldir = m.path + ? (_isWindows() ? `${m.path.replace(/\//g, '\\')}\\${repo.replace(/\//g, '\\')}` : _shellQuote(`${m.path}/${repo}`)) + : (_isWindows() ? '' : '""'); + if (selectedGguf) { + f._gguf_path = _selectedGgufExpr(m, repo, selectedGguf.rel_path); + } else if (_isWindows()) { + // Windows fallback: no bash $() available; validator rejects it. + // Return empty so the serve fails with a clear message. + f._gguf_path = ''; + } else if (m.is_local_dir && m.path) { + f._gguf_path = `$({ find ${_ldir} -name '*-00001-of-*.gguf' 2>/dev/null | sort; find ${_ldir} -name '*.gguf' 2>/dev/null | sort; } | head -1)`; + } else { + f._gguf_path = `$({ find ${dir} -name '*-00001-of-*.gguf' 2>/dev/null | sort; find ${dir} -name '*.gguf' 2>/dev/null | sort; } | head -1)`; + } // Vision: auto-find the mmproj (CLIP/projector) file in the same dir. // Resolved at runtime so the toggle just works if an mmproj-*.gguf is // present (downloaded alongside the model). Empty if none → cmd omits it. const _vsearchdir = (m.is_local_dir && m.path) ? _ldir : dir; - f._mmproj_path = `$(find ${_vsearchdir} -iname 'mmproj*.gguf' 2>/dev/null | sort | head -1)`; + f._mmproj_path = _isWindows() + ? (_vsearchdir ? `${_vsearchdir}\\mmproj*.gguf` : '') + : `$(find ${_vsearchdir} -iname 'mmproj*.gguf' 2>/dev/null | sort | head -1)`; } if (f.reasoning_parser) { const _rpEl2 = panel.querySelector('[data-field="reasoning_parser"]'); From 747d005645589e888d7f0ebd2b8bcab1c6b19b51 Mon Sep 17 00:00:00 2001 From: nubs Date: Fri, 5 Jun 2026 13:01:01 +0000 Subject: [PATCH 014/974] fix(gallery): validate target album owner on image PATCH + owner-scope album count/cover (#2755) --- routes/gallery_routes.py | 23 ++++++++++--- tests/test_gallery_album_owner_scope.py | 45 +++++++++++++++++++++++++ 2 files changed, 63 insertions(+), 5 deletions(-) create mode 100644 tests/test_gallery_album_owner_scope.py diff --git a/routes/gallery_routes.py b/routes/gallery_routes.py index d8d52b2..ce6f627 100644 --- a/routes/gallery_routes.py +++ b/routes/gallery_routes.py @@ -526,18 +526,24 @@ def setup_gallery_routes() -> APIRouter: albums = q.order_by(GalleryAlbum.created_at.desc()).all() result = [] for a in albums: - count = db.query(GalleryImage).filter( + _count_q = db.query(GalleryImage).filter( GalleryImage.album_id == a.id, GalleryImage.is_active == True - ).count() + ) + if user: + _count_q = _count_q.filter(GalleryImage.owner == user) + count = _count_q.count() cover_url = None if a.cover_id: cover = db.query(GalleryImage).filter(GalleryImage.id == a.cover_id).first() if cover: cover_url = f"/api/generated-image/{cover.filename}" elif count > 0: - first = db.query(GalleryImage).filter( + _cover_q = db.query(GalleryImage).filter( GalleryImage.album_id == a.id, GalleryImage.is_active == True - ).order_by(GalleryImage.created_at.desc()).first() + ) + if user: + _cover_q = _cover_q.filter(GalleryImage.owner == user) + first = _cover_q.order_by(GalleryImage.created_at.desc()).first() if first: cover_url = f"/api/generated-image/{first.filename}" result.append({ @@ -670,7 +676,14 @@ def setup_gallery_routes() -> APIRouter: if req.favorite is not None: img.favorite = req.favorite if req.album_id is not None: - img.album_id = req.album_id if req.album_id else None + if req.album_id: + # Validate the target album belongs to the caller before + # moving the image into it — mirrors add_to_album, so you + # cannot file your image into another user's album. + _get_or_404_album(db, req.album_id, user) + img.album_id = req.album_id + else: + img.album_id = None db.commit() db.refresh(img) return _image_to_dict(img) diff --git a/tests/test_gallery_album_owner_scope.py b/tests/test_gallery_album_owner_scope.py new file mode 100644 index 0000000..eafc0a1 --- /dev/null +++ b/tests/test_gallery_album_owner_scope.py @@ -0,0 +1,45 @@ +"""Issue #2754 — gallery owner-scoping. + +`patch_gallery_image` must validate that the *target album* belongs to the caller +before moving an image into it (otherwise user B can file B's image into user A's +album), and `list_albums` must owner-scope the per-album count + cover-fallback +queries. The gallery route handlers are closures, so — matching the AST-assertion +convention of test_gallery_image_privileges.py — we assert the guards are present +in the source. +""" +import ast +from pathlib import Path + + +def _function_sources(): + source = Path("routes/gallery_routes.py").read_text(encoding="utf-8") + tree = ast.parse(source) + return { + node.name: ast.get_source_segment(source, node) or "" + for node in ast.walk(tree) + if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef)) + } + + +def test_patch_validates_target_album_ownership(): + fns = _function_sources() + body = fns["patch_gallery_image"] + assert "req.album_id" in body + # The target album must be ownership-validated (via the same helper the + # sibling mutators use) before the image is reassigned to it. + assert "_get_or_404_album(db, req.album_id, user)" in body + + +def test_list_albums_count_and_cover_are_owner_scoped(): + fns = _function_sources() + body = fns["list_albums"] + # Both the per-album image count and the cover-fallback query must owner-scope + # by GalleryImage.owner (the album list itself already filters by owner). + assert body.count("GalleryImage.owner == user") >= 2 + + +def test_get_or_404_album_enforces_owner(): + # Guard the precedent we rely on: the helper rejects another user's album. + fns = _function_sources() + helper = fns["_get_or_404_album"] + assert "album.owner != user" in helper From e9ff6cde779883155e4ab4fe1f84bb0905338516 Mon Sep 17 00:00:00 2001 From: Alexandre Teixeira <111787685+alteixeira20@users.noreply.github.com> Date: Fri, 5 Jun 2026 14:04:10 +0100 Subject: [PATCH 015/974] docs(tests): document helper conventions Documentation-only PR continuing #2523. Adds tests/README.md to document helper conventions, validation expectations, and the next test-suite refactor phase. --- tests/README.md | 106 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 106 insertions(+) create mode 100644 tests/README.md diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..03633ae --- /dev/null +++ b/tests/README.md @@ -0,0 +1,106 @@ +# Test Suite Notes + +## Purpose + +This file documents the shared test helpers and the review expectations that go +with them. The suite is being refactored incrementally, so this is a working +reference for that effort — not a claim that the suite is already fully +organized. Read it before adding a new helper or before reviewing a PR that +touches `tests/helpers/`. + +## Core principles + +- Keep PRs small and homogeneous: one kind of change per PR. +- Prefer explicit local setup over hidden global fixtures. +- Avoid expanding the root `conftest.py` unless absolutely necessary. +- Do not mix file moves with logic changes in the same PR. +- Do not weaken tests with `skip`/`xfail` just to make CI pass. +- Validate the focused files you changed, plus any neighboring or + order-sensitive groups they interact with. + +## Helper conventions + +The helpers below live under `tests/helpers/`. They exist to remove repeated +boilerplate that already appeared across multiple tests. Reach for one only when +your test matches its intended use; do not stretch a helper to cover a new case. + +### `tests.helpers.cli_loader.load_script` + +Use when a test needs to import a script under `scripts/` without repeating +`SourceFileLoader` / `importlib.util` boilerplate. + +- Intended for script/CLI tests that load a single file from `scripts/`. +- Not for arbitrary package imports — use a normal `import` for those. +- When migrating an existing test to it, keep the existing stubs and assertions + unchanged. Any `sys.modules` stubs the script needs at import time must still + be injected (e.g. via `monkeypatch`) before calling `load_script`. + +### `tests.helpers.import_state.clear_module` + +Use when a test must drop one cached module and its parent-package attribute +before a fresh import. + +- Clears `sys.modules[name]`. +- Clears the parent-package attribute when present. +- Good replacement for local `sys.modules.pop(...)` + `delattr(parent, child)` + blocks. + +### `tests.helpers.import_state.preserve_import_state` + +Use when a test temporarily installs stubs into `sys.modules` and needs +deterministic cleanup afterward. + +- Context manager: restores both `sys.modules` entries and parent-package + attributes on exit (normal or exception). +- Useful around module-level stubs or temporary imports. +- Prefer narrow, explicit module names over broad ones. + +### `tests.helpers.import_state.clear_fake_database_modules` + +Use only for the guarded fake/stub database cleanup pattern. + +- Preserves a real-looking `core.database` (one with a string `__file__`). +- Removes a fake/stub `core.database` and the related `src.database` state. +- Do not use as a general database reset fixture. + +### `tests.helpers.import_state.clear_fake_endpoint_resolver_modules` + +Use only for the guarded fake/stub `src.endpoint_resolver` cleanup pattern. + +- Preserves real resolver modules (those with a truthy `__file__`). +- Evicts fake/stub resolver modules and the dependent route modules that were + cached against them. +- Accepts explicit extra dependent module names to evict alongside the defaults. + +## What not to abstract yet + +Some remaining patterns should stay as-is for now rather than being forced into +helpers: + +- Large mixed files such as security/review regression files. +- Setup-oriented `sys.modules` stub installers. +- One-off custom module patching. +- DB/session/route setup, until it has been audited separately. + +## Validation expectations + +Run validation locally before opening or approving a PR. Practical checks: + +- `git diff --check` — catch whitespace and conflict-marker errors. +- `python3 -m py_compile ` — confirm changed files compile. +- Focused `pytest` on the changed test files. +- `pytest` on neighboring or order-sensitive test groups that share import + state with the changed files. +- `grep` for the old boilerplate when replacing it, to confirm no stragglers + remain. +- A fresh audit worktree when changing the helpers themselves, so stale + `__pycache__` or import state cannot mask a regression. + +## Current roadmap + +1. Import-state cleanup — complete. +2. Document helper conventions (this file). +3. Audit fake DB / `SessionLocal` / route setup duplication. +4. Add tiny helpers only when the repeated semantics are clear. +5. Start low-risk file moves only after helper conventions are documented. +6. Avoid moving high-risk security/route regression files first. From 05f047b188aa1b023d05d9fe583409b4a3b121cc Mon Sep 17 00:00:00 2001 From: Wes Huber Date: Fri, 5 Jun 2026 06:05:30 -0700 Subject: [PATCH 016/974] fix: prevent document link click from resetting active session (#2055) * fix: prevent document link click from resetting active session Clicking a #document- link in chat caused the session to reset because of two issues: 1. chatRenderer.js: clicking on the text inside an yields a Text node target whose .closest() is undefined, so preventDefault never fires and the browser performs a default hash-navigation 2. sessions.js: the hashchange handler treated the entity hash (document-) as a session lookup, found no match, and the subsequent loadSessions created a new default-model chat Fix: walk past Text nodes before calling .closest(), and skip entity-prefixed hashes in the hashchange handler. Fixes #2035 Co-Authored-By: Claude Opus 4.6 (1M context) * fix(documents): move isOpen=true after container check in openPanel isOpen was set to true before the #chat-container existence check. If the container was missing during a race, the function returned early but isOpen stayed true, preventing the panel from ever reopening on subsequent calls. Move isOpen=true to after the container guard so a failed open doesn't leave the flag stuck. Co-Authored-By: Claude Opus 4.6 (1M context) --------- Co-authored-by: Claude Opus 4.6 (1M context) --- static/js/chatRenderer.js | 7 ++++++- static/js/document.js | 6 +++--- static/js/sessions.js | 6 +++++- 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/static/js/chatRenderer.js b/static/js/chatRenderer.js index 63c5650..e8aa9de 100644 --- a/static/js/chatRenderer.js +++ b/static/js/chatRenderer.js @@ -1005,7 +1005,12 @@ document.addEventListener('click', function(e) { // matching module via a dynamic import (avoids circular deps — // sessions.js itself imports chatRenderer.js). document.addEventListener('click', function(e) { - const a = e.target && e.target.closest && e.target.closest('a[href]'); + // Walk past Text nodes — clicking link text yields a Text node target + // whose .closest is undefined, so preventDefault never fires and the + // browser performs a default hash-navigation that resets the session. + let _t = e.target; + while (_t && _t.nodeType === Node.TEXT_NODE) _t = _t.parentElement; + const a = _t && _t.closest && _t.closest('a[href]'); if (!a) return; const href = a.getAttribute('href') || ''; if (!href.startsWith('#')) return; diff --git a/static/js/document.js b/static/js/document.js index 87ad298..ec9d797 100644 --- a/static/js/document.js +++ b/static/js/document.js @@ -3728,6 +3728,9 @@ import * as Modals from './modalManager.js'; _minimizedDocId = null; Modals.unregister('doc-panel'); } + const container = document.getElementById('chat-container'); + if (!container) return; + isOpen = true; // Doc was opened last → it goes in front of the email windows (clears the // email-front flag; the doc/email z-index alternation lives in CSS). @@ -3735,9 +3738,6 @@ import * as Modals from './modalManager.js'; _ensureAgentMode(); _markDocVisibleState(_lastSessionId, 'open'); - const container = document.getElementById('chat-container'); - if (!container) return; - document.body.classList.add('doc-view'); // Sync toggle button state diff --git a/static/js/sessions.js b/static/js/sessions.js index dab25a1..23310d3 100644 --- a/static/js/sessions.js +++ b/static/js/sessions.js @@ -1999,9 +1999,13 @@ export function initDragSort() { }); } -// Hash-based routing: navigate between sessions with browser back/forward +// Hash-based routing: navigate between sessions with browser back/forward. +// Skip entity-prefixed hashes (document-, note-, etc.) — those are handled +// by their own click handlers in chatRenderer.js and must not trigger +// session navigation (which would reset the active chat). window.addEventListener('hashchange', () => { const hashId = window.location.hash.replace('#', ''); + if (/^(document|note|image|email|event|task|skill|research)-/.test(hashId)) return; if (hashId && hashId !== currentSessionId) { const target = sessions.find(s => s.id === hashId && !s.archived); if (target) selectSession(hashId); From d4ff7fce8160fd4b782090be49034800b865d7c3 Mon Sep 17 00:00:00 2001 From: Ernest Hysa <59969602+ErnestHysa@users.noreply.github.com> Date: Fri, 5 Jun 2026 14:12:33 +0100 Subject: [PATCH 017/974] fix(gallery): add auth check to /api/image/sharpen endpoint (#2761) Every other image-processing endpoint (denoise, upscale, remove-bg, enhance-face, inpaint, harmonize) calls require_privilege(request, "can_generate_images"). The sharpen endpoint was missing this check, allowing unauthenticated users to trigger CPU-intensive image processing. --- routes/gallery_routes.py | 1 + 1 file changed, 1 insertion(+) diff --git a/routes/gallery_routes.py b/routes/gallery_routes.py index fdac5a4..eb40565 100644 --- a/routes/gallery_routes.py +++ b/routes/gallery_routes.py @@ -1316,6 +1316,7 @@ def setup_gallery_routes() -> APIRouter: @router.post("/api/image/sharpen") async def sharpen_image(request: Request): """Apply unsharp-mask sharpening to an image.""" + require_privilege(request, "can_generate_images") body = await request.json() image_b64 = body.get("image") amount = body.get("amount", 50) / 100.0 From f5c9095222d7f44250c1c7a30a118b655f8b7e83 Mon Sep 17 00:00:00 2001 From: Ernest Hysa <59969602+ErnestHysa@users.noreply.github.com> Date: Fri, 5 Jun 2026 14:12:40 +0100 Subject: [PATCH 018/974] fix(document): add 404 guard to version list/get endpoints (#2762) list_versions and get_version used a soft 'if doc:' guard that skipped ownership verification when the Document row was missing (e.g. after hard delete). Orphaned DocumentVersion rows would be returned to any caller without auth. Now raises 404 when the parent document is gone, matching the pattern already used in restore_version. --- routes/document_routes.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/routes/document_routes.py b/routes/document_routes.py index 03661b2..aef2a5f 100644 --- a/routes/document_routes.py +++ b/routes/document_routes.py @@ -663,8 +663,9 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter: try: # Verify ownership before listing versions doc = db.query(Document).filter(Document.id == doc_id).first() - if doc: - _verify_doc_owner(db, doc, user) + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) versions = db.query(DocumentVersion).filter( DocumentVersion.document_id == doc_id ).order_by(DocumentVersion.version_number.desc()).all() @@ -687,8 +688,9 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter: try: # Verify ownership doc = db.query(Document).filter(Document.id == doc_id).first() - if doc: - _verify_doc_owner(db, doc, user) + if not doc: + raise HTTPException(404, "Document not found") + _verify_doc_owner(db, doc, user) ver = db.query(DocumentVersion).filter( DocumentVersion.document_id == doc_id, DocumentVersion.version_number == num, From 3738df3b93cb9ed02fba20ad285c6ec86e274272 Mon Sep 17 00:00:00 2001 From: Ernest Hysa <59969602+ErnestHysa@users.noreply.github.com> Date: Fri, 5 Jun 2026 14:12:47 +0100 Subject: [PATCH 019/974] fix(tasks): validate then_task_id belongs to same owner on create/update (#2764) then_task_id was stored without checking the target task's owner. A user could chain their task to execute any other user's task on success via the scheduler's _run_chained path. Now verifies the target task exists and belongs to the requesting user before storing. --- routes/task_routes.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/routes/task_routes.py b/routes/task_routes.py index 6604923..38513b6 100644 --- a/routes/task_routes.py +++ b/routes/task_routes.py @@ -497,6 +497,15 @@ def setup_task_routes(task_scheduler) -> APIRouter: else bool(req.notifications_enabled) if req.notifications_enabled is not None else True ) + # Validate chained task belongs to same owner + if req.then_task_id: + chain_target = db.query(ScheduledTask).filter( + ScheduledTask.id == req.then_task_id + ).first() + if not chain_target: + raise HTTPException(400, "Chained task not found") + if chain_target.owner != user: + raise HTTPException(403, "Cannot chain to another user's task") task = ScheduledTask( id=task_id, owner=user, @@ -671,6 +680,14 @@ def setup_task_routes(task_scheduler) -> APIRouter: if req.trigger_count is not None: task.trigger_count = req.trigger_count if req.then_task_id is not None: + if req.then_task_id: + chain_target = db.query(ScheduledTask).filter( + ScheduledTask.id == req.then_task_id + ).first() + if not chain_target: + raise HTTPException(400, "Chained task not found") + if chain_target.owner != user: + raise HTTPException(403, "Cannot chain to another user's task") task.then_task_id = req.then_task_id or None if req.notifications_enabled is not None: task.notifications_enabled = bool(req.notifications_enabled) From 73673258199b353f9b3e04da9b37ae95077e2c8b Mon Sep 17 00:00:00 2001 From: Ernest Hysa <59969602+ErnestHysa@users.noreply.github.com> Date: Fri, 5 Jun 2026 14:12:54 +0100 Subject: [PATCH 020/974] fix(caldav): include owner in calendar ID hash to prevent PK collision (#2765) _stable_cal_id hashed only the remote URL, producing the same calendar ID for all users syncing the same CalDAV endpoint. The second user would get an IntegrityError on the primary key. Now includes owner in the hash so each user gets a distinct calendar row. --- src/caldav_sync.py | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/caldav_sync.py b/src/caldav_sync.py index 663c0bd..f875b7c 100644 --- a/src/caldav_sync.py +++ b/src/caldav_sync.py @@ -86,10 +86,12 @@ def validate_caldav_url(raw_url: str) -> str: return urlunparse(parsed._replace(fragment="")).rstrip("/") -def _stable_cal_id(remote_url: str) -> str: +def _stable_cal_id(remote_url: str, owner: str = "") -> str: """Deterministic local id for a remote CalDAV calendar — same URL - always maps to the same local row across restarts and re-syncs.""" - h = hashlib.sha256(remote_url.encode("utf-8")).hexdigest()[:24] + always maps to the same local row across restarts and re-syncs. + Owner is included in the hash to prevent PK collisions when multiple + users sync the same CalDAV endpoint.""" + h = hashlib.sha256(f"{owner}:{remote_url}".encode("utf-8")).hexdigest()[:24] return f"caldav-{h}" @@ -170,7 +172,7 @@ def _sync_blocking(owner: str, url: str, username: str, password: str) -> dict: for remote_cal in calendars: try: remote_url = str(remote_cal.url) - cal_id = _stable_cal_id(remote_url) + cal_id = _stable_cal_id(remote_url, owner) display_name = (remote_cal.name or "").strip() or "CalDAV" local_cal = db.query(CalendarCal).filter( From 8159733c6cea6920000b9db0676870e02fbf2816 Mon Sep 17 00:00:00 2001 From: L1 <148907002+davieduard0x01@users.noreply.github.com> Date: Fri, 5 Jun 2026 10:18:16 -0300 Subject: [PATCH 021/974] fix(caldav): pull Google Calendar events from the events collection, not the /user principal (#2531) * fix(caldav): pull Google Calendar events from the events collection, not the /user principal Google serves its CalDAV principal at .../caldav/v2//user but events live under .../caldav/v2//events. The caldav library's principal->home-set discovery does not reliably enumerate calendars from Google's /user endpoint, so _sync_blocking fell into its 'treat the URL as a single calendar' fallback and ran every calendar-query REPORT against the principal URL. /user holds no VEVENTs, so the REPORT returned a clean but empty 200 for every date range: auth succeeded, the calendar stayed empty (Apple Calendar works because iCloud exposes standard discovery at the pasted URL). Add _google_caldav_events_url() to map a recognised Google principal URL to its events collection, and route both discovery-less fallbacks through _open_url_as_calendar() so Google syncs hit /events while other servers' URLs are used unchanged. Fixes #2507 * fix(caldav): also map Google's legacy www.google.com/calendar/dav principal URL Some Google accounts authenticate against the older CalDAV endpoint (https://www.google.com/calendar/dav//user) rather than the newer apidata.googleusercontent.com/caldav/v2 form (reported on #2507). Both have the same principal-vs-events split, so map the legacy /user URL to its /events collection as well. The legacy branch is gated on the /calendar/dav/ path so an unrelated www.google.com URL ending in /user is left untouched. --- src/caldav_sync.py | 50 ++++++- tests/test_caldav_google_principal_url.py | 162 ++++++++++++++++++++++ 2 files changed, 210 insertions(+), 2 deletions(-) create mode 100644 tests/test_caldav_google_principal_url.py diff --git a/src/caldav_sync.py b/src/caldav_sync.py index b139dbb..578e370 100644 --- a/src/caldav_sync.py +++ b/src/caldav_sync.py @@ -166,6 +166,52 @@ def _find_existing_event(db, pending, uid_val, calendar_id): ).first() +def _google_caldav_events_url(url: str) -> str | None: + """Map a Google CalDAV *principal* URL to its event-collection URL. + + Google serves the principal at ``…/user`` but events live under ``…/events`` + — the ``/user`` resource holds no VEVENTs. The `caldav` library's + principal→home-set discovery does not reliably enumerate calendars from + Google's ``/user`` endpoint, so the sync falls into the "treat the URL as a + single calendar" fallback below. Pointed at ``/user`` that fallback issues + every calendar-query REPORT against the principal, which returns a clean but + empty 200 for all date ranges — the calendar shows no events even though + auth succeeded (issue #2507). + + Both Google CalDAV endpoint forms are handled, since some accounts only + authenticate against one of them: + - newer: ``https://apidata.googleusercontent.com/caldav/v2//user`` + - legacy: ``https://www.google.com/calendar/dav//user`` + + Returns the events URL for a recognised Google principal URL, else None so + the caller keeps the original URL unchanged. + """ + parts = urlparse(url) + host = (parts.hostname or "").lower() + path = parts.path.rstrip("/") + if not path.endswith("/user"): + return None + is_google = ( + host.endswith("googleusercontent.com") # newer /caldav/v2 form + or (host in ("www.google.com", "google.com") and "/calendar/dav/" in path) # legacy form + ) + if not is_google: + return None + new_path = path[: -len("/user")] + "/events" + return urlunparse(parts._replace(path=new_path)) + + +def _open_url_as_calendar(client, url: str): + """Open ``url`` as a single calendar collection. + + Used when principal discovery yields no calendars. Google's principal URL + is not an event collection, so map it to the events URL first + (see ``_google_caldav_events_url``); other servers' URLs are used as-is. + """ + target = _google_caldav_events_url(url) or url + return client.calendar(url=target) + + def _sync_blocking(owner: str, url: str, username: str, password: str) -> dict: """The actual sync — synchronous, intended to run in a threadpool. Returns counts: {calendars, events, deleted, errors}.""" @@ -192,14 +238,14 @@ def _sync_blocking(owner: str, url: str, username: str, password: str) -> dict: except Exception as e: logger.info(f"CalDAV principal discovery failed, trying URL as calendar: {e}") try: - calendars = [client.calendar(url=url)] + calendars = [_open_url_as_calendar(client, url)] except Exception as e2: result["errors"].append(f"Could not open URL as calendar: {e2}") return result if not calendars: try: - calendars = [client.calendar(url=url)] + calendars = [_open_url_as_calendar(client, url)] except Exception as e: result["errors"].append(f"No calendars and URL fallback failed: {e}") return result diff --git a/tests/test_caldav_google_principal_url.py b/tests/test_caldav_google_principal_url.py new file mode 100644 index 0000000..ce9cefe --- /dev/null +++ b/tests/test_caldav_google_principal_url.py @@ -0,0 +1,162 @@ +"""Google Calendar over CalDAV must surface events, not come back empty (#2507). + +Google's CalDAV principal lives at ``.../caldav/v2//user`` but events are +served from ``.../caldav/v2//events``. When the `caldav` library's +principal discovery yields no calendars for Google's ``/user`` endpoint, +``_sync_blocking`` fell back to ``client.calendar(url=url)`` — i.e. it queried +the principal URL itself, which returns a clean but empty 200 for every date +range. Auth succeeded, the calendar stayed empty. + +These tests inject a fake ``caldav`` module that mimics Google's behaviour +(principal discovery returns no calendars; the ``/user`` collection holds no +events; the ``/events`` collection holds one VEVENT) and assert the sync now +maps the principal URL to its events collection and pulls the event. No live +Google account is required. +""" +import sys +import tempfile +import types +from datetime import datetime, timedelta + +import pytest +from sqlalchemy import create_engine +from sqlalchemy.orm import sessionmaker +from sqlalchemy.pool import NullPool + +import core.database as cdb +from core.database import CalendarCal, CalendarEvent +from src import caldav_sync + +_TMPDB = tempfile.NamedTemporaryFile(suffix=".db", delete=False) +_ENGINE = create_engine( + f"sqlite:///{_TMPDB.name}", + connect_args={"check_same_thread": False}, + poolclass=NullPool, +) +cdb.Base.metadata.create_all(_ENGINE) +_TS = sessionmaker(bind=_ENGINE, autoflush=False, autocommit=False) + +_GOOGLE_PRINCIPAL = "https://apidata.googleusercontent.com/caldav/v2/me@gmail.com/user" +_GOOGLE_EVENTS = "https://apidata.googleusercontent.com/caldav/v2/me@gmail.com/events" + + +def _ics_one_event(): + # An event inside the sync window (now-90d .. now+365d). + dt = datetime.utcnow() + timedelta(days=2) + stamp = dt.strftime("%Y%m%dT%H%M%SZ") + return ( + "BEGIN:VCALENDAR\r\n" + "VERSION:2.0\r\n" + "BEGIN:VEVENT\r\n" + "UID:evt-1@google\r\n" + f"DTSTART:{stamp}\r\n" + f"DTEND:{stamp}\r\n" + "SUMMARY:Standup\r\n" + "END:VEVENT\r\n" + "END:VCALENDAR\r\n" + ) + + +class _FakeObj: + def __init__(self, data): + self.data = data + + +class _FakeCalendar: + def __init__(self, url): + self.url = url + self.name = "Primary" + + def date_search(self, start, end, expand=False): + # Google's /user principal holds no events; the /events collection does. + if str(self.url).rstrip("/").endswith("/events"): + return [_FakeObj(_ics_one_event())] + return [] + + +class _FakePrincipal: + def calendars(self): + # Simulate Google's /user endpoint yielding no calendars from discovery. + return [] + + +class _FakeClient: + def __init__(self, url=None, username=None, password=None): + self.url = url + + def principal(self): + return _FakePrincipal() + + def calendar(self, url=None): + return _FakeCalendar(url) + + +def _install_fake_caldav(monkeypatch): + fake = types.ModuleType("caldav") + fake.DAVClient = _FakeClient + err = types.ModuleType("caldav.lib.error") + + class AuthorizationError(Exception): + pass + + class NotFoundError(Exception): + pass + + err.AuthorizationError = AuthorizationError + err.NotFoundError = NotFoundError + lib = types.ModuleType("caldav.lib") + lib.error = err + fake.lib = lib + monkeypatch.setitem(sys.modules, "caldav", fake) + monkeypatch.setitem(sys.modules, "caldav.lib", lib) + monkeypatch.setitem(sys.modules, "caldav.lib.error", err) + monkeypatch.setattr(caldav_sync, "SessionLocal", _TS, raising=False) + monkeypatch.setattr(cdb, "SessionLocal", _TS, raising=False) + + +def _clear_db(): + db = _TS() + try: + db.query(CalendarEvent).delete() + db.query(CalendarCal).delete() + db.commit() + finally: + db.close() + + +def test_maps_google_principal_url_to_events_collection(): + assert caldav_sync._google_caldav_events_url(_GOOGLE_PRINCIPAL) == _GOOGLE_EVENTS + # Trailing slash tolerated. + assert caldav_sync._google_caldav_events_url(_GOOGLE_PRINCIPAL + "/") == _GOOGLE_EVENTS + # Non-Google or non-principal URLs are left untouched (None => caller keeps URL). + assert caldav_sync._google_caldav_events_url("https://calendar.example.com/dav") is None + assert caldav_sync._google_caldav_events_url(_GOOGLE_EVENTS) is None + + +def test_maps_legacy_google_calendar_dav_url(): + # Google's older endpoint (some accounts authenticate only against this one). + legacy_user = "https://www.google.com/calendar/dav/me@gmail.com/user" + legacy_events = "https://www.google.com/calendar/dav/me@gmail.com/events" + assert caldav_sync._google_caldav_events_url(legacy_user) == legacy_events + assert caldav_sync._google_caldav_events_url(legacy_user + "/") == legacy_events + # A non-CalDAV www.google.com /user path must NOT be rewritten. + assert caldav_sync._google_caldav_events_url("https://www.google.com/accounts/user") is None + + +def test_google_sync_pulls_events_instead_of_empty(monkeypatch): + _install_fake_caldav(monkeypatch) + _clear_db() + + result = caldav_sync._sync_blocking("alice", _GOOGLE_PRINCIPAL, "me@gmail.com", "app-pw") + + # The fix routes discovery-less Google sync to the /events collection, so + # the VEVENT is pulled. Pre-fix this queried /user and returned 0 events. + assert result["events"] == 1, result + assert not result["errors"], result["errors"] + + db = _TS() + try: + ev = db.query(CalendarEvent).filter(CalendarEvent.uid == "evt-1@google").first() + assert ev is not None and ev.summary == "Standup" + finally: + db.close() From 8354948a1cfa5afba3fc90ce02b9435405ba5f37 Mon Sep 17 00:00:00 2001 From: nubs Date: Fri, 5 Jun 2026 13:22:08 +0000 Subject: [PATCH 022/974] fix(llm): route harmony thinking streams (#2449) --- src/llm_core.py | 243 ++++++++++++++++++++++++------- tests/test_llm_core_reasoning.py | 34 +++++ 2 files changed, 224 insertions(+), 53 deletions(-) diff --git a/src/llm_core.py b/src/llm_core.py index 7dcf380..f8664eb 100644 --- a/src/llm_core.py +++ b/src/llm_core.py @@ -6,8 +6,9 @@ import json import logging import hashlib import threading +import re from fastapi import HTTPException -from typing import Optional, Dict, List +from typing import Optional, Dict, List, Tuple from src.model_context import get_context_length, DEFAULT_CONTEXT from urllib.parse import urlparse @@ -66,6 +67,103 @@ _host_fails: Dict[str, int] = {} _host_health_lock = threading.Lock() _model_activity: Dict[str, float] = {} +_HARMONY_MARKER_RE = re.compile( + r"<\|channel\|>(analysis|final)" + r"|<\|start\|>(?:assistant|system|user|tool)?" + r"|<\|message\|>" + r"|<\|end\|>" + r"|<\|return\|>" + r"|<\|call\|>" +) +_HARMONY_MARKERS = ( + "<|channel|>analysis", + "<|channel|>final", + "<|start|>assistant", + "<|start|>system", + "<|start|>user", + "<|start|>tool", + "<|start|>", + "<|message|>", + "<|end|>", + "<|return|>", + "<|call|>", +) +_HARMONY_MAX_MARKER_LEN = max(len(marker) for marker in _HARMONY_MARKERS) + + +def _harmony_suffix_hold_len(text: str) -> int: + """Return how many trailing chars could be the start of a harmony marker.""" + limit = min(len(text), _HARMONY_MAX_MARKER_LEN - 1) + for n in range(limit, 0, -1): + suffix = text[-n:] + if any(marker.startswith(suffix) for marker in _HARMONY_MARKERS): + return n + return 0 + + +class _HarmonyStreamRouter: + """Route OpenAI harmony analysis/final channels without leaking markers.""" + + def __init__(self) -> None: + self._buf = "" + self._seen_harmony = False + self._channel: Optional[str] = None + self._in_message = False + + def feed(self, text: str) -> List[Tuple[str, bool]]: + if not text: + return [] + self._buf += text + return self._drain(final=False) + + def flush(self) -> List[Tuple[str, bool]]: + return self._drain(final=True) + + def _append_text(self, out: List[Tuple[str, bool]], text: str) -> None: + if not text: + return + if not self._seen_harmony: + out.append((text, False)) + return + if self._in_message: + out.append((text, self._channel == "analysis")) + + def _handle_marker(self, match: re.Match[str]) -> None: + marker = match.group(0) + self._seen_harmony = True + if marker.startswith("<|channel|>"): + self._channel = match.group(1) + self._in_message = False + elif marker == "<|message|>": + self._in_message = True + else: + self._in_message = False + if marker in {"<|end|>", "<|return|>", "<|call|>"}: + self._channel = None + + def _drain(self, *, final: bool) -> List[Tuple[str, bool]]: + out: List[Tuple[str, bool]] = [] + while True: + match = _HARMONY_MARKER_RE.search(self._buf) + if not match: + break + self._append_text(out, self._buf[:match.start()]) + self._handle_marker(match) + self._buf = self._buf[match.end():] + + hold = 0 if final else _harmony_suffix_hold_len(self._buf) + emit = self._buf if hold == 0 else self._buf[:-hold] + self._buf = "" if hold == 0 else self._buf[-hold:] + self._append_text(out, emit) + return out + + +def _stream_delta_event(text: str, *, thinking: bool = False) -> str: + payload = {"delta": text} + if thinking: + payload["thinking"] = True + return f"data: {json.dumps(payload)}\n\n" + def _model_activity_key(url: str, model: str) -> str: return f"{(url or '').strip()}|{(model or '').strip()}" @@ -1217,6 +1315,7 @@ async def stream_llm(url: str, model: str, messages: List[Dict], temperature: fl # ── Native Ollama streaming ── if provider == "ollama": _ollama_tool_calls: List[Dict] = [] + _harmony_router = _HarmonyStreamRouter() try: client = _get_http_client() async with client.stream('POST', target_url, json=payload, headers=h, timeout=stream_timeout) as r: @@ -1236,10 +1335,11 @@ async def stream_llm(url: str, model: str, messages: List[Dict], temperature: fl message = j.get("message") or {} thinking = message.get("thinking") or "" if thinking: - yield f'data: {json.dumps({"delta": thinking, "thinking": True})}\n\n' + yield _stream_delta_event(thinking, thinking=True) content = message.get("content") or "" if content: - yield f'data: {json.dumps({"delta": content})}\n\n' + for part, is_thinking in _harmony_router.feed(content): + yield _stream_delta_event(part, thinking=is_thinking) for tc in message.get("tool_calls") or []: fn = tc.get("function") or {} if fn.get("name"): @@ -1249,12 +1349,16 @@ async def stream_llm(url: str, model: str, messages: List[Dict], temperature: fl "arguments": json.dumps(fn.get("arguments") or {}), }) if j.get("done"): + for part, is_thinking in _harmony_router.flush(): + yield _stream_delta_event(part, thinking=is_thinking) if _ollama_tool_calls: yield f'data: {json.dumps({"type": "tool_calls", "calls": _ollama_tool_calls})}\n\n' if j.get("prompt_eval_count") is not None or j.get("eval_count") is not None: yield f'data: {json.dumps({"type": "usage", "data": {"input_tokens": j.get("prompt_eval_count", 0), "output_tokens": j.get("eval_count", 0)}})}\n\n' yield "data: [DONE]\n\n" return + for part, is_thinking in _harmony_router.flush(): + yield _stream_delta_event(part, thinking=is_thinking) yield "data: [DONE]\n\n" except (httpx.ConnectError, httpx.ConnectTimeout) as e: _cooled = _mark_host_dead(target_url) @@ -1387,6 +1491,8 @@ async def stream_llm(url: str, model: str, messages: List[Dict], temperature: fl _first_content_sent = False _in_think_tag = False # True while consuming … content _think_open_stripped = False # opening tag already removed + _harmony_router = _HarmonyStreamRouter() + _harmony_active = False # sticky: gpt-oss harmony <|channel|> stream detected def _emit_tool_calls(): """Build the tool_calls event string if any were accumulated.""" @@ -1395,6 +1501,22 @@ async def stream_llm(url: str, model: str, messages: List[Dict], temperature: fl calls = [_tc_acc[i] for i in sorted(_tc_acc)] return f'data: {json.dumps({"type": "tool_calls", "calls": calls})}\n\n' + def _format_routed_content(parts: List[Tuple[str, bool]]) -> List[str]: + nonlocal _first_content_sent + events = [] + for part, is_thinking in parts: + if is_thinking: + events.append(_stream_delta_event(part, thinking=True)) + continue + # Some thinking backends start normal content with a stray closing + # tag. Repair only that shape; do not wrap every first token for + # model families like MiniMax, which often stream ordinary answers. + if _thinking_model and not _first_content_sent and part.lstrip().lower().startswith("… in content stream. - # Covers Qwen3-derived models (Qwopus, QwQ forks) whose - # names don't match _THINKING_MODEL_PATTERNS but still - # emit literal markup via llama.cpp --jinja. - if not _first_content_sent and not _thinking_model and not _in_think_tag and stripped.lower().startswith("") - if close_idx != -1: - # Split: up-to- → thinking, remainder → content - think_part = content[:close_idx] - if not _think_open_stripped: - # Strip the opening from the first chunk. - # Use a dedicated flag — _first_content_sent stays False - # throughout the think block, so it must not be reused. - tag_end = think_part.lower().find(">") - if tag_end != -1: - think_part = think_part[tag_end + 1:] - _think_open_stripped = True - regular_part = content[close_idx + len(""):] - _in_think_tag = False - if think_part: - yield f'data: {json.dumps({"delta": think_part, "thinking": True})}\n\n' - if regular_part: - _first_content_sent = True - yield f'data: {json.dumps({"delta": regular_part})}\n\n' - else: - # Still inside : route to thinking channel - if not _think_open_stripped: - # Strip the opening tag (first chunk only) - tag_end = stripped.lower().find(">") - if tag_end != -1: - content = stripped[tag_end + 1:] - _think_open_stripped = True - if content: - yield f'data: {json.dumps({"delta": content, "thinking": True})}\n\n' + # gpt-oss harmony format (<|channel|>analysis/final): route via the harmony + # stream router. Sticky once the first marker appears — distinct from the + # path below (handled in the else, preserving #2588 behaviour). + if _harmony_active or "<|" in content: + _harmony_active = True + for event in _format_routed_content(_harmony_router.feed(content)): + yield event else: - # Some thinking backends start normal content with a - # stray closing tag. Repair only that shape; do not - # wrap every first token for model families like - # MiniMax, which often stream ordinary answers. - if _thinking_model and not _first_content_sent and stripped.lower().startswith("… in content stream. + # Covers Qwen3-derived models (Qwopus, QwQ forks) whose + # names don't match _THINKING_MODEL_PATTERNS but still + # emit literal markup via llama.cpp --jinja. + if not _first_content_sent and not _thinking_model and not _in_think_tag and stripped.lower().startswith("") + if close_idx != -1: + # Split: up-to- → thinking, remainder → content + think_part = content[:close_idx] + if not _think_open_stripped: + # Strip the opening from the first chunk. + # Use a dedicated flag — _first_content_sent stays False + # throughout the think block, so it must not be reused. + tag_end = think_part.lower().find(">") + if tag_end != -1: + think_part = think_part[tag_end + 1:] + _think_open_stripped = True + regular_part = content[close_idx + len(""):] + _in_think_tag = False + if think_part: + yield f'data: {json.dumps({"delta": think_part, "thinking": True})}\n\n' + if regular_part: + _first_content_sent = True + yield f'data: {json.dumps({"delta": regular_part})}\n\n' + else: + # Still inside : route to thinking channel + if not _think_open_stripped: + # Strip the opening tag (first chunk only) + tag_end = stripped.lower().find(">") + if tag_end != -1: + content = stripped[tag_end + 1:] + _think_open_stripped = True + if content: + yield f'data: {json.dumps({"delta": content, "thinking": True})}\n\n' + else: + # Some thinking backends start normal content with a + # stray closing tag. Repair only that shape; do not + # wrap every first token for model families like + # MiniMax, which often stream ordinary answers. + if _thinking_model and not _first_content_sent and stripped.lower().startswith(""), f"expected repair prefix, got: {first!r}" + + +def test_thinking_field_emits_thinking_chunk(monkeypatch): + deltas = _run_stream( + "gpt-oss:20b", + [ + 'data: {"choices":[{"delta":{"thinking":"checking files"}}]}', + 'data: {"choices":[{"delta":{"content":"visible answer"}}]}', + "data: [DONE]", + ], + monkeypatch, + ) + assert any(d.get("thinking") and d["delta"] == "checking files" for d in deltas), deltas + assert any((not d.get("thinking")) and d["delta"] == "visible answer" for d in deltas), deltas + +def test_harmony_analysis_channel_routes_to_thinking(monkeypatch): + deltas = _run_stream( + "gpt-oss:20b", + [ + 'data: {"choices":[{"delta":{"content":"<|channel|>ana"}}]}', + 'data: {"choices":[{"delta":{"content":"lysis<|message|>We need to inspect."}}]}', + 'data: {"choices":[{"delta":{"content":"<|end|><|channel|>final<|message|>Here "}}]}', + 'data: {"choices":[{"delta":{"content":"are the files.<|end|>"}}]}', + "data: [DONE]", + ], + monkeypatch, + ) + thinking = "".join(d["delta"] for d in deltas if d.get("thinking")) + answer = "".join(d["delta"] for d in deltas if not d.get("thinking")) + + assert thinking == "We need to inspect." + assert answer == "Here are the files." + assert "<|channel|>" not in thinking + answer + assert "<|message|>" not in thinking + answer From 6973c5427c07c36493bd031da414f6e72a7a32cf Mon Sep 17 00:00:00 2001 From: nubs Date: Fri, 5 Jun 2026 13:56:54 +0000 Subject: [PATCH 023/974] fix(model-context): count tool_calls in estimate_tokens so compaction sees real size (#2751) --- src/model_context.py | 22 ++++++++++- tests/test_estimate_tokens_tool_calls.py | 47 ++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 tests/test_estimate_tokens_tool_calls.py diff --git a/src/model_context.py b/src/model_context.py index 3a445fe..c71d76f 100644 --- a/src/model_context.py +++ b/src/model_context.py @@ -357,7 +357,11 @@ def estimate_tokens(messages: List[Dict]) -> int: Uses chars * 0.3 which is closer to real BPE tokenizer output than the commonly-cited chars/4 (which underestimates by ~20-30%). - Also adds ~4 tokens per message for role/formatting overhead. + Also adds ~4 tokens per message for role/formatting overhead, and counts + assistant tool_calls (name + arguments) — a tool-only turn carries + content=None with the real payload in tool_calls, so ignoring them made the + estimate (and the compaction/trim gates that rely on it) blind to large + tool arguments. """ total = 0 for msg in messages: @@ -369,4 +373,20 @@ def estimate_tokens(messages: List[Dict]) -> int: for item in content: if isinstance(item, dict) and item.get("type") == "text": total += int(len(item.get("text", "")) * 0.3) + # Tool calls carry real payload too: a tool-only assistant turn is stored + # with content=None and the actual args (e.g. a create_document body) in + # tool_calls[].function.arguments. Ignoring them made large tool arguments + # read as ~0 tokens, so the compaction/trim gates missed genuine overflow. + tool_calls = msg.get("tool_calls") + if isinstance(tool_calls, list): + for tc in tool_calls: + if not isinstance(tc, dict): + continue + fn = tc.get("function") if isinstance(tc.get("function"), dict) else tc + name = fn.get("name", "") or "" + args = fn.get("arguments", "") or "" + if not isinstance(args, str): + args = str(args) # some shapes store arguments as a dict + total += 4 # per tool-call overhead (id, type, wrapper) + total += int((len(str(name)) + len(args)) * 0.3) return total diff --git a/tests/test_estimate_tokens_tool_calls.py b/tests/test_estimate_tokens_tool_calls.py new file mode 100644 index 0000000..39c890f --- /dev/null +++ b/tests/test_estimate_tokens_tool_calls.py @@ -0,0 +1,47 @@ +"""Issue #2748 — estimate_tokens must count assistant tool_calls (name + arguments). + +A tool-only assistant turn is stored with content=None and the real payload (e.g. +a large create_document body) in tool_calls[].function.arguments. Before this fix +estimate_tokens ignored tool_calls, so such a turn counted as ~4 tokens and the +compaction/trim gates that rely on estimate_tokens silently missed real context +overflow, letting the upstream call 400 with 'context length exceeded'. +""" + +from src.model_context import estimate_tokens + + +def test_tool_call_arguments_are_counted(): + big = "x" * 40000 # ~ a large create_document body + msg = { + "role": "assistant", + "content": None, + "tool_calls": [ + {"id": "c1", "type": "function", + "function": {"name": "create_document", "arguments": big}}, + ], + } + est = estimate_tokens([msg]) + # ~40k chars * 0.3 ≈ 12000, vs the old ~4 that ignored tool_calls entirely. + assert est > 10000, est + + +def test_content_only_message_is_unchanged(): + # No tool_calls -> identical to the previous behaviour (content*0.3 + overhead). + msg = {"role": "user", "content": "x" * 100} + assert estimate_tokens([msg]) == 4 + int(100 * 0.3) + + +def test_dict_arguments_are_handled(): + # Some shapes store arguments as a dict rather than a JSON string. + msg = { + "role": "assistant", + "content": None, + "tool_calls": [{"function": {"name": "f", "arguments": {"path": "x" * 1000}}}], + } + assert estimate_tokens([msg]) > 200 + + +def test_empty_and_malformed_tool_calls_are_safe(): + # tool_calls=None and non-dict entries must not raise and must not inflate. + assert estimate_tokens([{"role": "assistant", "content": "hi", "tool_calls": None}]) == 4 + int(2 * 0.3) + assert estimate_tokens([{"role": "assistant", "content": None, "tool_calls": ["bad", 5]}]) == 4 From c9d0c6db18fd485423841a0aa74b132de6b8b240 Mon Sep 17 00:00:00 2001 From: ooovenenoso <120500656+ooovenenoso@users.noreply.github.com> Date: Fri, 5 Jun 2026 10:00:20 -0400 Subject: [PATCH 024/974] fix: quote IMAP mailbox arguments (#2170) * fix: quote IMAP mailbox arguments * fix: quote MCP move destinations --------- Co-authored-by: Kevin <120500656+oooindefatigable@users.noreply.github.com> --- mcp_servers/email_server.py | 37 ++++++---- routes/document_routes.py | 4 +- routes/email_pollers.py | 4 +- tests/test_imap_mailbox_quoting.py | 111 +++++++++++++++++++++++++++++ 4 files changed, 138 insertions(+), 18 deletions(-) create mode 100644 tests/test_imap_mailbox_quoting.py diff --git a/mcp_servers/email_server.py b/mcp_servers/email_server.py index 9382624..ba75dd0 100644 --- a/mcp_servers/email_server.py +++ b/mcp_servers/email_server.py @@ -38,6 +38,11 @@ def _b(value) -> bytes: return str(value).encode() +def _q(name: str) -> str: + """Quote an IMAP mailbox name for commands that take mailbox args.""" + return '"' + (name or "").replace("\\", "\\\\").replace('"', '\\"') + '"' + + def _uid_fetch_rows(data) -> list: return [d for d in (data or []) if isinstance(d, bytes) and b"UID " in d] @@ -419,7 +424,7 @@ def _list_emails(folder="INBOX", max_results=20, unresponded_only=False, account selects mailbox (None = default). """ conn = _imap_connect(account) - select_status, _ = conn.select(folder, readonly=True) + select_status, _ = conn.select(_q(folder), readonly=True) if select_status != "OK": conn.logout() raise ValueError(f"IMAP folder not found: {folder}") @@ -542,7 +547,7 @@ def _search_emails(query, folders=None, max_results=20, account=None): try: for folder in folders: try: - status, _ = conn.select(folder, readonly=True) + status, _ = conn.select(_q(folder), readonly=True) if status != "OK": continue status, data = conn.uid("SEARCH", None, search_cmd) @@ -653,7 +658,7 @@ def _read_email(uid=None, message_id=None, folder="INBOX", account=None): """Read full email content by UID or message-ID. account = mailbox selector.""" cfg = _load_config(account) conn = _imap_connect(account) - conn.select(folder, readonly=True) + conn.select(_q(folder), readonly=True) if message_id and not uid: status, data = conn.uid("SEARCH", None, f'(HEADER Message-ID "{message_id}")') @@ -827,7 +832,7 @@ def _send_email(to, subject, body, in_reply_to=None, references=None, cc=None, b imap = _imap_connect(send_account) try: sent_folder = _detect_sent_folder(imap) - append_st, append_data = imap.append(sent_folder, "\\Seen", None, msg.as_bytes()) + append_st, append_data = imap.append(_q(sent_folder), "\\Seen", None, msg.as_bytes()) if append_st == "OK" and append_data: m = re.search(rb"APPENDUID\s+\d+\s+(\d+)", append_data[0] or b"") if m: @@ -854,7 +859,7 @@ def _send_email(to, subject, body, in_reply_to=None, references=None, cc=None, b def _reply_to_email(uid, body, folder="INBOX", reply_all=False, account=None): """Reply to an existing email by UID. Threads via In-Reply-To/References.""" conn = _imap_connect(account) - conn.select(folder, readonly=True) + conn.select(_q(folder), readonly=True) status, msg_data = conn.uid("FETCH", _b(uid), "(BODY.PEEK[])") conn.logout() if status != "OK" or not msg_data or not msg_data[0]: @@ -896,7 +901,7 @@ def _reply_to_email(uid, body, folder="INBOX", reply_all=False, account=None): def _set_flag(uid, folder, flag, add=True, account=None): """Add or remove an IMAP flag (e.g. \\Seen, \\Answered, \\Deleted).""" conn = _imap_connect(account) - conn.select(folder) + conn.select(_q(folder)) op = "+FLAGS" if add else "-FLAGS" try: status, data = conn.uid("STORE", _b(uid), op, flag) @@ -918,7 +923,7 @@ def _bulk_set_flag(uids, folder, flag, add=True, account=None): conn = _imap_connect(account) touched = [] try: - conn.select(folder) + conn.select(_q(folder)) op = "+FLAGS" if add else "-FLAGS" msg_set = ",".join(str(u) for u in uids) try: @@ -945,7 +950,7 @@ def _bulk_move(uids, source_folder, dest_folder, account=None, role: str = ""): conn = _imap_connect(account) moved = 0 try: - conn.select(source_folder) + conn.select(_q(source_folder)) dest_folder = _resolve_folder(conn, dest_folder, role or _folder_role_from_name(dest_folder)) msg_set = ",".join(str(u) for u in uids) try: @@ -956,10 +961,11 @@ def _bulk_move(uids, source_folder, dest_folder, account=None, role: str = ""): if not existing: return 0 moved = len(existing) - status, _ = conn.uid("MOVE", _b(msg_set), dest_folder) + dest_arg = _q(dest_folder) + status, _ = conn.uid("MOVE", _b(msg_set), dest_arg) if status != "OK": # Fallback: UID copy + flag-delete + expunge - status, _ = conn.uid("COPY", _b(msg_set), dest_folder) + status, _ = conn.uid("COPY", _b(msg_set), dest_arg) if status != "OK": return 0 status, _ = conn.uid("STORE", _b(msg_set), "+FLAGS", "\\Deleted") @@ -976,7 +982,7 @@ def _search_uids(folder="INBOX", criteria="UNSEEN", account=None): ALL, ANSWERED). Used to resolve selectors like all_unread → uids.""" conn = _imap_connect(account) try: - conn.select(folder, readonly=True) + conn.select(_q(folder), readonly=True) status, data = conn.uid("SEARCH", None, criteria) if status != "OK" or not data or not data[0]: return [] @@ -988,7 +994,7 @@ def _search_uids(folder="INBOX", criteria="UNSEEN", account=None): def _move_message(uid, source_folder, dest_folder, account=None, role: str = ""): """Move a message between folders. Tries IMAP MOVE, falls back to copy+delete.""" conn = _imap_connect(account) - conn.select(source_folder) + conn.select(_q(source_folder)) try: dest_folder = _resolve_folder(conn, dest_folder, role or _folder_role_from_name(dest_folder)) try: @@ -998,11 +1004,12 @@ def _move_message(uid, source_folder, dest_folder, account=None, role: str = "") existing = _uid_fetch_rows(data) if status != "OK" or not existing: return False - status, _ = conn.uid("MOVE", _b(uid), dest_folder) + dest_arg = _q(dest_folder) + status, _ = conn.uid("MOVE", _b(uid), dest_arg) if status == "OK": return True # Fallback: UID copy + delete - status, _ = conn.uid("COPY", _b(uid), dest_folder) + status, _ = conn.uid("COPY", _b(uid), dest_arg) if status != "OK": return False status, _ = conn.uid("STORE", _b(uid), "+FLAGS", "\\Deleted") @@ -1032,7 +1039,7 @@ def _archive_email(uid, folder="INBOX", account=None): def _download_attachment(uid, index, folder="INBOX", account=None): """Extract a specific attachment to disk and return its local path.""" conn = _imap_connect(account) - conn.select(folder, readonly=True) + conn.select(_q(folder), readonly=True) status, msg_data = conn.uid("FETCH", _b(uid), "(BODY.PEEK[])") conn.logout() if status != "OK": diff --git a/routes/document_routes.py b/routes/document_routes.py index 03661b2..e2b5621 100644 --- a/routes/document_routes.py +++ b/routes/document_routes.py @@ -1629,9 +1629,11 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter: # context (To/Subject/In-Reply-To/References). try: from routes.email_routes import _imap, _decode_header + from routes.email_helpers import _q except Exception: _imap = None _decode_header = lambda x: x or "" + _q = lambda x: x or "" to_addr = "" from_name = "" @@ -1641,7 +1643,7 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter: if _imap: try: with _imap(doc.source_email_account_id or None) as conn: - conn.select(doc.source_email_folder, readonly=True) + conn.select(_q(doc.source_email_folder), readonly=True) status, data = conn.fetch(doc.source_email_uid.encode(), "(RFC822.HEADER)") if status == "OK" and data and data[0]: raw_hdr = data[0][1] diff --git a/routes/email_pollers.py b/routes/email_pollers.py index 04ffb0a..146db0e 100644 --- a/routes/email_pollers.py +++ b/routes/email_pollers.py @@ -210,7 +210,7 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None if auto_cal: for sent_name in ("Sent", "INBOX/Sent", "Sent Items", "[Gmail]/Sent Mail"): try: - st, _ = conn.select(sent_name, readonly=True) + st, _ = conn.select(_q(sent_name), readonly=True) if st == "OK": folders_to_scan.append(sent_name) break @@ -1046,7 +1046,7 @@ def _scheduled_poll_once() -> dict: try: with _imap(row_account_id, owner=row_owner) as imap: sent_folder = _detect_sent_folder(imap) - imap.append(sent_folder, "\\Seen", None, outer.as_bytes()) + imap.append(_q(sent_folder), "\\Seen", None, outer.as_bytes()) except Exception as e: logger.warning(f"Failed to append scheduled {sid} to Sent: {e}") diff --git a/tests/test_imap_mailbox_quoting.py b/tests/test_imap_mailbox_quoting.py new file mode 100644 index 0000000..7c5bb16 --- /dev/null +++ b/tests/test_imap_mailbox_quoting.py @@ -0,0 +1,111 @@ +"""Regression coverage for IMAP mailbox names that contain spaces. + +imaplib does not quote mailbox arguments for SELECT/APPEND/MOVE/COPY, so callers +must quote names such as "[Gmail]/All Mail" or "Sent Items" themselves. +""" + +from pathlib import Path + +import pytest + +pytest.importorskip("mcp") + +import mcp_servers.email_server as es + + +class FakeListConn: + def __init__(self): + self.calls = [] + + def select(self, folder, readonly=False): + self.calls.append(("select", folder, readonly)) + return "OK", [] + + def uid(self, command, *args): + self.calls.append(("uid", command, *args)) + if command == "SEARCH": + return "OK", [b""] + return "OK", [] + + def logout(self): + self.calls.append(("logout",)) + + +class FakeMoveConn: + def __init__(self): + self.calls = [] + + def list(self): + self.calls.append(("list",)) + return "OK", [] + + def select(self, folder, readonly=False): + self.calls.append(("select", folder, readonly)) + return "OK", [] + + def uid(self, command, *args): + self.calls.append(("uid", command, *args)) + if command == "FETCH": + return "OK", [b"1 (UID 123)"] + if command == "MOVE": + return "NO", [] + return "OK", [] + + def expunge(self): + self.calls.append(("expunge",)) + + def logout(self): + self.calls.append(("logout",)) + + +def test_mcp_list_emails_quotes_spaced_folder_on_select(monkeypatch): + conn = FakeListConn() + monkeypatch.setattr(es, "_imap_connect", lambda account=None: conn) + + assert es._list_emails(folder="Sent Items") == [] + + assert conn.calls[0] == ("select", '"Sent Items"', True) + + +def test_mcp_quote_helper_handles_spaced_and_quoted_mailboxes(): + assert es._q("Sent Items") == '"Sent Items"' + assert es._q('[Gmail]/All Mail') == '"[Gmail]/All Mail"' + assert es._q('Label "Needs Reply"') == '"Label \\"Needs Reply\\""' + + +def test_known_imap_mailbox_call_sites_are_quoted(): + mcp = Path("mcp_servers/email_server.py").read_text() + assert "conn.select(folder" not in mcp + assert "conn.select(source_folder" not in mcp + assert "imap.append(sent_folder" not in mcp + assert 'conn.uid("MOVE", _b(msg_set), dest_folder)' not in mcp + assert 'conn.uid("COPY", _b(msg_set), dest_folder)' not in mcp + assert 'conn.uid("MOVE", _b(uid), dest_folder)' not in mcp + assert 'conn.uid("COPY", _b(uid), dest_folder)' not in mcp + + pollers = Path("routes/email_pollers.py").read_text() + assert "conn.select(sent_name" not in pollers + assert "imap.append(sent_folder" not in pollers + + document_routes = Path("routes/document_routes.py").read_text() + assert "conn.select(doc.source_email_folder" not in document_routes + + +def test_mcp_move_message_quotes_destination_for_move_and_fallback_copy(monkeypatch): + conn = FakeMoveConn() + monkeypatch.setattr(es, "_imap_connect", lambda account=None: conn) + + assert es._move_message("123", "INBOX", "[Gmail]/All Mail") is True + + assert ("uid", "MOVE", b"123", '"[Gmail]/All Mail"') in conn.calls + assert ("uid", "COPY", b"123", '"[Gmail]/All Mail"') in conn.calls + + +def test_mcp_bulk_move_quotes_destination_for_move_and_fallback_copy(monkeypatch): + conn = FakeMoveConn() + monkeypatch.setattr(es, "_imap_connect", lambda account=None: conn) + + assert es._bulk_move(["123"], "INBOX", "[Gmail]/All Mail") == 1 + + assert ("uid", "MOVE", b"123", '"[Gmail]/All Mail"') in conn.calls + assert ("uid", "COPY", b"123", '"[Gmail]/All Mail"') in conn.calls From 4bfe0c690a420b6083a4024a5a9c2f53c335b0a2 Mon Sep 17 00:00:00 2001 From: ooovenenoso <120500656+ooovenenoso@users.noreply.github.com> Date: Fri, 5 Jun 2026 10:05:14 -0400 Subject: [PATCH 025/974] fix(calendar): cap RRULE expansion (#2902) --- routes/calendar_routes.py | 32 ++++++++++++++++++++++++------- tests/test_calendar_recurrence.py | 17 ++++++++++++++++ 2 files changed, 42 insertions(+), 7 deletions(-) diff --git a/routes/calendar_routes.py b/routes/calendar_routes.py index 7e1523a..d1b621a 100644 --- a/routes/calendar_routes.py +++ b/routes/calendar_routes.py @@ -460,6 +460,9 @@ def _event_to_dict(ev: CalendarEvent) -> dict: # ── Recurrence expansion ── +_RRULE_EXPANSION_LIMIT = 1000 + + def _expand_rrule( ev: CalendarEvent, start: datetime, end: datetime ) -> List[dict]: @@ -482,6 +485,7 @@ def _expand_rrule( d = _event_to_dict(ev) d["is_recurrence"] = False d["series_uid"] = ev.uid + d["truncated"] = False return [d] # Parse the rrule, applying it to the base dtstart. @@ -507,6 +511,7 @@ def _expand_rrule( d = _event_to_dict(ev) d["is_recurrence"] = False d["series_uid"] = ev.uid + d["truncated"] = False # Malformed RRULE rows are fetched by the recurring SQL branch # with only dtstart < end_dt — the base event may not actually # overlap the window. Only return if it does. @@ -519,22 +524,26 @@ def _expand_rrule( # (matching non-recurring overlap semantics: dtstart < end AND # dtend > start). expand_start = start - duration - occurrences = rule.between(expand_start, end, inc=True) - if not occurrences: - return [] - results = [] + truncated = False base = _event_to_dict(ev) - for occ_start in occurrences: + for occ_start in rule.xafter(expand_start, inc=True): + if occ_start >= end: + break + occ_end = occ_start + duration # Overlap filter: occurrence must intersect [start, end). # This enforces exclusive-end semantics (occ_start >= end is # excluded) and includes multi-day crossings (occ_end > start). - if occ_start >= end or occ_end <= start: + if occ_end <= start: continue + if len(results) >= _RRULE_EXPANSION_LIMIT: + truncated = True + break + # Build the compound uid: {base_uid}::{date} or ::{datetime} if ev.all_day: occ_uid = f"{ev.uid}::{occ_start.strftime('%Y-%m-%d')}" @@ -545,6 +554,7 @@ def _expand_rrule( d["uid"] = occ_uid d["series_uid"] = ev.uid d["is_recurrence"] = True + d["truncated"] = False if ev.all_day: d["dtstart"] = occ_start.strftime("%Y-%m-%d") @@ -557,6 +567,10 @@ def _expand_rrule( results.append(d) + if truncated: + for d in results: + d["truncated"] = True + return results @@ -786,8 +800,12 @@ def setup_calendar_routes() -> APIRouter: expanded.extend(_expand_rrule(e, start_dt, end_dt)) # Sort by occurrence start time for consistent frontend ordering. + truncated = any(e.get("truncated") for e in expanded) expanded.sort(key=lambda d: d["dtstart"]) - return {"events": expanded} + response: dict = {"events": expanded} + if truncated: + response["truncated"] = True + return response except HTTPException: raise except Exception as e: diff --git a/tests/test_calendar_recurrence.py b/tests/test_calendar_recurrence.py index cc80656..bc78127 100644 --- a/tests/test_calendar_recurrence.py +++ b/tests/test_calendar_recurrence.py @@ -319,3 +319,20 @@ def test_expand_metadata_inheritance(): assert r["importance"] == "critical" assert r["event_type"] == "work" assert r["location"] == "Room 42" + + +def test_expand_daily_rrule_large_window_is_capped_and_marked_truncated(): + """Wide recurring windows must not materialize unbounded occurrence lists.""" + cal = _import_calendar_helpers() + ev = _make_event( + uid="evt-daily-cap", + dtstart=datetime(2020, 1, 1, 9, 0), + dtend=datetime(2020, 1, 1, 10, 0), + rrule="FREQ=DAILY", + ) + + results = cal._expand_rrule(ev, datetime(2020, 1, 1), datetime(2030, 1, 1)) + + assert len(results) == cal._RRULE_EXPANSION_LIMIT + assert results[-1]["uid"] == "evt-daily-cap::2022-09-26T09:00" + assert all(r["truncated"] is True for r in results) From 01f127881186363830b18c0d86a1d6ddb49f27b1 Mon Sep 17 00:00:00 2001 From: Greg Stevenson Date: Fri, 5 Jun 2026 15:11:08 +0100 Subject: [PATCH 026/974] fix: Settings now correctly displays CalDAV integrations when more than one isconfigured (#2901) * fix(calendar): expose source in calendar list and add per-calendar delete - GET /api/calendar/calendars now includes source field so the frontend can distinguish CalDAV collections from local calendars - Add DELETE /api/calendar/calendars/{cal_id} to remove a specific calendar and its events by owner-scoped ID * fix(settings): show all CalDAV calendars in integrations list Previously one card was shown for the CalDAV server connection regardless of how many calendar collections had been synced. The Calendars page showed them all; Settings did not. - Fetch /api/calendar/calendars alongside existing requests and render one card per source=caldav collection, falling back to the single server-level card if nothing has synced yet - Delete now targets the specific calendar by ID rather than clearing the whole server config - Confirm dialog shows the calendar name so the user can verify before removing --- routes/calendar_routes.py | 24 +++++++++++++++++++++++- static/js/settings.js | 27 +++++++++++++++++++++------ 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/routes/calendar_routes.py b/routes/calendar_routes.py index d1b621a..b8bb1e9 100644 --- a/routes/calendar_routes.py +++ b/routes/calendar_routes.py @@ -729,6 +729,28 @@ def setup_calendar_routes() -> APIRouter: from src.caldav_sync import sync_caldav return await sync_caldav(owner) + @router.delete("/calendars/{cal_id}") + async def delete_calendar(cal_id: str, request: Request): + owner = _require_user(request) + db = SessionLocal() + try: + cal = db.query(CalendarCal).filter( + CalendarCal.id == cal_id, + CalendarCal.owner == owner, + ).first() + if not cal: + raise HTTPException(404, "Calendar not found") + db.delete(cal) + db.commit() + return {"ok": True} + except HTTPException: + raise + except Exception as e: + logger.error("Failed to delete calendar %s: %s", cal_id, e) + raise HTTPException(500, "Failed to delete calendar") + finally: + db.close() + @router.get("/calendars") async def list_calendars(request: Request): owner = _require_user(request) @@ -737,7 +759,7 @@ def setup_calendar_routes() -> APIRouter: _ensure_default_calendar(db, owner) cals = db.query(CalendarCal).filter(CalendarCal.owner == owner).all() return {"calendars": [ - {"name": c.name, "href": c.id, "color": c.color} + {"name": c.name, "href": c.id, "color": c.color, "source": c.source} for c in cals ]} except HTTPException: diff --git a/static/js/settings.js b/static/js/settings.js index 8269bb6..068cd80 100644 --- a/static/js/settings.js +++ b/static/js/settings.js @@ -3197,7 +3197,7 @@ async function initUnifiedIntegrations() { } async function fetchAll() { - const [apiRes, calRes, cardRes, contactsRes, emailAccountsRes, mcpRes, vaultRes, tokenRes] = await Promise.all([ + const [apiRes, calRes, cardRes, contactsRes, emailAccountsRes, mcpRes, vaultRes, tokenRes, calendarsRes] = await Promise.all([ fetch('/api/auth/integrations', { credentials: 'same-origin' }).then(r => r.ok ? r.json() : { integrations: [] }).catch(() => ({ integrations: [] })), fetch('/api/calendar/config', { credentials: 'same-origin' }).then(r => r.ok ? r.json() : {}).catch(() => ({})), fetch('/api/contacts/config', { credentials: 'same-origin' }).then(r => r.ok ? r.json() : {}).catch(() => ({})), @@ -3206,14 +3206,21 @@ async function initUnifiedIntegrations() { fetch('/api/mcp/servers', { credentials: 'same-origin' }).then(r => r.ok ? r.json() : []).catch(() => []), fetch('/api/vault/config', { credentials: 'same-origin' }).then(r => r.ok ? r.json() : {}).catch(() => ({})), fetch('/api/tokens', { credentials: 'same-origin' }).then(r => r.ok ? r.json() : []).catch(() => []), + fetch('/api/calendar/calendars', { credentials: 'same-origin' }).then(r => r.ok ? r.json() : { calendars: [] }).catch(() => ({ calendars: [] })), ]); const items = []; // API integrations for (const intg of (apiRes.integrations || [])) { items.push({ type: 'api', id: intg.id, name: intg.name || 'Unnamed', detail: intg.base_url || '', enabled: intg.enabled !== false, data: intg }); } - // CalDAV - if (calRes.url) { + // CalDAV — one card per synced calendar collection; fall back to the + // server-level entry if calendars haven't been synced yet. + const caldavCals = (calendarsRes.calendars || []).filter(c => c.source === 'caldav'); + if (caldavCals.length > 0) { + for (const cal of caldavCals) { + items.push({ type: 'caldav', id: cal.href, name: cal.name, detail: calRes.url || 'CalDAV', enabled: true, data: { ...cal, serverData: calRes } }); + } + } else if (calRes.url) { items.push({ type: 'caldav', id: '__caldav__', name: 'Calendar (CalDAV)', detail: calRes.url, enabled: true, data: calRes }); } // Contacts import first, then the optional CardDAV sync account. @@ -3283,7 +3290,7 @@ async function initUnifiedIntegrations() {
${item.detail || ''}
${statusDot} - `; @@ -3321,12 +3328,20 @@ async function initUnifiedIntegrations() { listEl.querySelectorAll('.intg-del-btn').forEach(btn => { btn.addEventListener('click', async (e) => { e.stopPropagation(); - if (!await window.styledConfirm('Remove this integration?', { confirmText: 'Remove', danger: true })) return; + const intgName = btn.dataset.intgName || 'this integration'; + if (!await window.styledConfirm(`Remove "${intgName}"?`, { confirmText: 'Remove', danger: true })) return; const type = btn.dataset.intgType; const id = btn.dataset.intgId; try { if (type === 'api') await fetch(`/api/auth/integrations/${id}`, { method: 'DELETE', credentials: 'same-origin' }); - else if (type === 'caldav') await fetch('/api/calendar/config', { method: 'POST', credentials: 'same-origin', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ url: '', username: '', password: '' }) }); + else if (type === 'caldav') { + if (id === '__caldav__') { + // Fallback card: server configured but never synced — clear credentials + await fetch('/api/calendar/config', { method: 'POST', credentials: 'same-origin', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ url: '', username: '', password: '' }) }); + } else { + await fetch(`/api/calendar/calendars/${id}`, { method: 'DELETE', credentials: 'same-origin' }); + } + } else if (type === 'contacts') { await fetch('/api/contacts/clear', { method: 'DELETE', credentials: 'same-origin' }); } From 2e207fc31584235e4360b475dba8fca67911b2b6 Mon Sep 17 00:00:00 2001 From: nubs Date: Fri, 5 Jun 2026 14:25:05 +0000 Subject: [PATCH 027/974] fix(notes): track + remove the select-mode Esc keydown listener so it doesn't leak per open (#2792) --- static/js/notes.js | 19 ++++++++++++-- tests/test_notes_select_esc_listener_js.py | 30 ++++++++++++++++++++++ 2 files changed, 47 insertions(+), 2 deletions(-) create mode 100644 tests/test_notes_select_esc_listener_js.py diff --git a/static/js/notes.js b/static/js/notes.js index 039b310..e64e503 100644 --- a/static/js/notes.js +++ b/static/js/notes.js @@ -31,6 +31,9 @@ let _reminderTimer = null; // (previously leaked one per openPanel; on multi-open sessions this // stacked dozens of identical handlers). let _notesKeydownHandler = null; +// Capture-phase "Esc cancels select mode" listener on document — tracked so it +// is removed on close instead of leaking +1 per panel open/close cycle. +let _notesSelectEscHandler = null; const REMINDER_FIRED_KEY = 'odysseus-notes-reminder-fired'; // Note IDs already shown with the entry-glow once. Re-set when the user // reschedules the reminder so the new firing glows again on next open. @@ -54,6 +57,10 @@ function _forceCloseNotesPanel() { document.removeEventListener('keydown', _notesKeydownHandler); _notesKeydownHandler = null; } + if (_notesSelectEscHandler) { + document.removeEventListener('keydown', _notesSelectEscHandler, true); + _notesSelectEscHandler = null; + } if (_reminderTimer) { clearInterval(_reminderTimer); _reminderTimer = null; @@ -1270,13 +1277,17 @@ export function openPanel() { // than a *-bulk-cancel button, so the global Esc-cancel handler in // keyboard-shortcuts.js can't reach it — handle it here. Capture phase // + stopPropagation so Esc cancels select instead of closing the panel. - document.addEventListener('keydown', (e) => { + if (_notesSelectEscHandler) { + document.removeEventListener('keydown', _notesSelectEscHandler, true); + } + _notesSelectEscHandler = (e) => { if (e.key === 'Escape' && _selectMode) { e.preventDefault(); e.stopPropagation(); _exitSelectMode(); } - }, true); + }; + document.addEventListener('keydown', _notesSelectEscHandler, true); document.getElementById('notes-select-all').addEventListener('change', (e) => { if (e.target.checked) _notes.forEach(n => _selectedIds.add(n.id)); else _selectedIds.clear(); @@ -1580,6 +1591,10 @@ export function closePanel(direction) { document.removeEventListener('keydown', _notesKeydownHandler); _notesKeydownHandler = null; } + if (_notesSelectEscHandler) { + document.removeEventListener('keydown', _notesSelectEscHandler, true); + _notesSelectEscHandler = null; + } if (_reminderTimer) { clearInterval(_reminderTimer); _reminderTimer = null; diff --git a/tests/test_notes_select_esc_listener_js.py b/tests/test_notes_select_esc_listener_js.py new file mode 100644 index 0000000..dedc612 --- /dev/null +++ b/tests/test_notes_select_esc_listener_js.py @@ -0,0 +1,30 @@ +"""Issue #2791 — the Notes panel's capture-phase "Esc cancels select mode" +keydown listener must be tracked and removed on close, not leaked anonymously on +every open/close cycle. + +notes.js is a browser ES module with a heavy import chain (can't be node-imported +in isolation), so — per the repo's convention for DOM-coupled guards (cf. the +document.js diff-discard and memory.js filter-guard tests) — this asserts the +tracked-handler pattern in source. +""" +from pathlib import Path + +SRC = Path("static/js/notes.js").read_text(encoding="utf-8") + + +def test_select_esc_listener_is_tracked_not_anonymous(): + assert "let _notesSelectEscHandler = null;" in SRC + # added via the tracked module-level var in capture phase + assert "document.addEventListener('keydown', _notesSelectEscHandler, true);" in SRC + + +def test_select_esc_listener_removed_with_matching_capture_flag(): + # remove-before-add in openPanel + removal in both close paths => >= 3, + # each with the `true` capture flag (a removal without it would not match). + removals = SRC.count("document.removeEventListener('keydown', _notesSelectEscHandler, true);") + assert removals >= 3, removals + + +def test_old_anonymous_capture_listener_is_gone(): + # the leak was an inline anonymous capture listener; it must no longer exist. + assert "addEventListener('keydown', (e) => {\n if (e.key === 'Escape' && _selectMode)" not in SRC From 8ce945d3385f827db00c70a3491048d33e508f41 Mon Sep 17 00:00:00 2001 From: Kenny Van de Maele Date: Fri, 5 Jun 2026 16:32:25 +0200 Subject: [PATCH 028/974] feat: Add plan mode to the chat agent (#638) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: Add plan mode to the chat agent Adds a plan mode: the agent investigates read-only, proposes a checklist, and waits for approval before changing anything. On approval it runs with full tools and checks items off as it goes. Enforcement reuses the existing disabled_tools gate. Includes a slash command: `/plan [on|off]` (and `/toggle plan`) to flip the plan toggle from the chat input. - src/tool_security.py, src/mcp_manager.py: read-only allowlist (tools + MCP). - src/agent_loop.py, routes/chat_routes.py: union the disabled set, prepend the plan directive, force agent mode. - static/: plan toggle pill, Approve & Run, dockable plan window, task-list checkboxes, and the /plan slash command. - tests/test_plan_mode.py. * Plan mode: persistent re-referenceable plan + agent write-back Three improvements so a long plan survives a weak model and stays in reach: 1. Re-reference the plan (out-of-context fix). On the execution turn the frontend sends the approved checklist back (`approved_plan`); the backend pins it as a top-of-context `## ACTIVE PLAN` system note (kept by the context trimmer), so the agent can always re-read the plan instead of losing the thread on a long run. New `build_active_plan_note()` (unit-tested). 2. Re-open / dock the plan anytime. The plan checklist is stored per-session (localStorage). When a plan exists, the plan-mode button opens a small menu ("Show plan" / "Plan mode: On/Off") that re-opens the side-dockable plan window — so it can stay docked while the agent works. The window live-refreshes as the plan changes. 3. Agent write-back: new `update_plan` tool. The agent calls it to tick steps `- [x]` after finishing them, or to revise steps when the user asks. Marker tool (no I/O) → `plan_update` SSE event → the stored plan + docked window update live. The ACTIVE PLAN note instructs the agent to use it. Backend: src/agent_loop.py (param + pin + note builder + emit + prompt blurb), src/tool_execution.py (update_plan handler), routes/chat_routes.py (parse `approved_plan`, relay `plan_update`), registration in tool_schemas / agent_tools / tool_index (always-available, not admin-gated). Frontend: static/js/chat.js (plan store, send `approved_plan`, handle `plan_update`, capture restated checklists), static/app.js (plan-button menu), static/js/planWindow.js (`isPlanWindowOpen`), static/js/storage.js (PLAN key). Tests: tests/test_plan_mode.py (plan-note), tests/test_update_plan_tool.py. * Plan mode: drop bash/python, rely on read-only discovery tools Shell can mutate (write files, hit the network) and can't be constrained to read-only at the tool layer, so plan mode no longer relies on a prompt to keep it well-behaved — bash/python are removed from the read-only allowlist and added to the fail-closed block set. Discovery is covered by the dedicated read-only tools (read_file, grep, glob, ls) instead. Rewrites the plan-mode directive to state shell is disabled and lists the available read-only tools positively. Addresses review feedback on #638. Co-Authored-By: Claude Opus 4.8 * Comment: note _MCP_READONLY_VERBS are prefixes not whole words Clarifies that entries like "summar" are intentional stems matched via startswith (covers summarise/summarize/summary), not typos. Addresses review feedback on #638. Co-Authored-By: Claude Opus 4.8 * Plan mode: clarify why gating inverts the allowlist into a denylist Rename _PLAN_MODE_FALLBACK_BLOCK -> _PLAN_MODE_KNOWN_MUTATORS and rewrite the comments. The tool gate is a denylist (disabled_tools); plan mode's policy is an allowlist, so it returns the inverse (all known tool names minus the allowlist). The static mutator set is a backstop for the schema-derived name list, which misses XML-only tools and can fail to import. Addresses review feedback on #638. Co-Authored-By: Claude Opus 4.8 * Plan mode: stop hardcoding the read-only tool list in the directive The model is already shown its available (read-only) tools by _assemble_prompt, which removes every disabled tool. Enumerating them again in the directive only duplicated that list and would drift as tools change. Point at the tools listed below instead. Addresses review feedback on #638. --- routes/chat_routes.py | 22 +++++++ src/agent_loop.py | 95 ++++++++++++++++++++++++++++- src/agent_tools.py | 2 +- src/mcp_manager.py | 66 +++++++++++++++++++- src/tool_execution.py | 35 +++++++++++ src/tool_index.py | 3 + src/tool_schemas.py | 14 +++++ src/tool_security.py | 95 +++++++++++++++++++++++++++++ static/app.js | 80 ++++++++++++++++++++++++ static/index.html | 7 +++ static/js/chat.js | 108 +++++++++++++++++++++++++++++++++ static/js/markdown.js | 18 ++++-- static/js/planWindow.js | 79 ++++++++++++++++++++++++ static/js/slashCommands.js | 24 ++++++++ static/js/storage.js | 3 +- static/style.css | 98 ++++++++++++++++++++++++++++++ tests/test_plan_mode.py | 104 +++++++++++++++++++++++++++++++ tests/test_update_plan_tool.py | 46 ++++++++++++++ 18 files changed, 891 insertions(+), 8 deletions(-) create mode 100644 static/js/planWindow.js create mode 100644 tests/test_plan_mode.py create mode 100644 tests/test_update_plan_tool.py diff --git a/routes/chat_routes.py b/routes/chat_routes.py index cd5e4e6..a4de453 100644 --- a/routes/chat_routes.py +++ b/routes/chat_routes.py @@ -394,6 +394,7 @@ def setup_chat_routes( search_context = form_data.get("search_context") # pre-fetched web search results (compare mode) compare_mode = str(form_data.get("compare_mode", "")).lower() == "true" incognito = str(form_data.get("incognito", "")).lower() == "true" + plan_mode = str(form_data.get("plan_mode", "")).lower() == "true" chat_mode = str(form_data.get("mode", "")).lower() # 'chat' or 'agent' # Workspace: confine the agent's file/shell tools to this folder. Validate # it's a real directory; ignore (no confinement) otherwise. @@ -401,6 +402,17 @@ def setup_chat_routes( if workspace: _ws_real = os.path.realpath(os.path.expanduser(workspace)) workspace = _ws_real if os.path.isdir(_ws_real) else "" + # Plan mode is a modifier on agent mode — it only makes sense with tools. + if plan_mode: + chat_mode = "agent" + # An approved plan being EXECUTED: the frontend sends the checklist back + # on each turn so we can pin it in context. This way a long plan on a + # weak model survives history truncation — the agent can always re-read + # the plan. Ignored while still proposing (plan_mode on). Capped so a + # huge plan can't blow the prompt. + approved_plan = "" + if not plan_mode: + approved_plan = (form_data.get("approved_plan") or "").strip()[:8192] # Did the USER explicitly pick agent mode? (vs. us auto-escalating # below). Skill extraction should only learn from real agent sessions, # not chats we quietly promoted for a notes/calendar intent. @@ -659,6 +671,13 @@ def setup_chat_routes( if chat_mode == 'chat': disabled_tools.update({"bash", "python", "read_file", "write_file", "web_search", "web_fetch", "search_chats", "manage_tasks"}) + # Plan mode: investigate read-only, propose a plan, don't mutate. Block + # every tool not on the read-only allowlist. (stream_agent_loop enforces + # this again + drops MCP, so this is belt-and-suspenders.) + if plan_mode: + from src.tool_security import plan_mode_disabled_tools + disabled_tools.update(plan_mode_disabled_tools()) + async def stream_with_save() -> AsyncGenerator[str, None]: # _effective_mode is read-only here; closure captures it from # the outer scope. (Was `nonlocal` but never reassigned.) @@ -1015,6 +1034,8 @@ def setup_chat_routes( owner=_user, fallbacks=_fallback_candidates, workspace=workspace or None, + plan_mode=plan_mode, + approved_plan=approved_plan or None, ): if chunk.startswith("data: ") and not chunk.startswith("data: [DONE]"): try: @@ -1036,6 +1057,7 @@ def setup_chat_routes( "doc_update", "doc_suggestions", "ui_control", "rounds_exhausted", "ask_user", + "plan_update", ): if data.get("type") == "agent_step": _agent_rounds = max(_agent_rounds, data.get("round", 1)) diff --git a/src/agent_loop.py b/src/agent_loop.py index a74c95e..84870db 100644 --- a/src/agent_loop.py +++ b/src/agent_loop.py @@ -19,7 +19,7 @@ from src.llm_core import stream_llm, stream_llm_with_fallback, _is_ollama_native from src.model_context import estimate_tokens from src.settings import get_setting from src.prompt_security import untrusted_context_message -from src.tool_security import blocked_tools_for_owner +from src.tool_security import blocked_tools_for_owner, plan_mode_disabled_tools from src.agent_tools import ( parse_tool_blocks, strip_tool_blocks, @@ -336,6 +336,7 @@ If the user asks for a reminder/alarm before the event, pass `reminder_minutes` "pipeline": "- ```pipeline``` — Run a multi-step AI pipeline. Args (JSON) with ordered steps, each specifying a model and prompt. Use for complex workflows.", "ui_control": "- ```ui_control``` — Control the UI: toggle tools on/off, OPEN PANELS, open email reply drafts, switch models, change themes. Commands: `toggle on/off` (names: bash/shell, web/search, research, incognito, document_editor/documents), `open_panel ` (panels: documents, gallery, email, sessions, notes, memories/brain, skills, settings, cookbook), `open_email_reply ` (opens an email compose document, does NOT send), `set_mode agent/chat`, `switch_model `, `set_theme `, `create_theme ` (optional key=val for advanced colors AND background effects: bgPattern=, bgEffectColor=#RRGGBB, bgEffectIntensity=, bgEffectSize=, frosted=true|false). \"open documents\" / \"open library\" / \"show gallery\" / \"open inbox\" / \"open notes\" / \"open cookbook\" all map to `open_panel `. Theme presets: dark, light, midnight, paper, cyberpunk, retrowave, forest, ocean, ume, copper, terminal, organs, lavender, gpt, claude, cute.", "ask_user": "- ```ask_user``` — Ask the user a multiple-choice question when the task is genuinely ambiguous and the answer changes what you do next (pick an approach, confirm an assumption, choose a target). Args (JSON): {\"question\": \"...\", \"options\": [{\"label\": \"...\", \"description\": \"...\"?}, ...], \"multi\": false?}. 2-6 options. The user gets clickable buttons; calling this ENDS your turn and their choice comes back as your next message. Prefer sensible defaults — only ask when you truly can't proceed well without their input.", + "update_plan": "- ```update_plan``` — While executing an approved plan, write the plan back: tick steps done or revise them. Args (JSON): {\"plan\": \"- [x] done step\\n- [ ] next step\"}. Always pass the COMPLETE checklist, not a diff. Call it after finishing each step (mark it `- [x]`) and whenever the user asks to change the plan. The user's docked plan window updates live. Does nothing if there's no active plan.", "list_served_models": "- ```list_served_models``` — Show what the Cookbook (LLM-serving subsystem) is currently running. NO args. Use this for ANY 'what's running' / 'what's serving' / 'show my cookbook' / 'is anything up' query. DO NOT shell out (`ps aux`, `docker ps`, etc.) — this tool is the source of truth. Failed serve tasks include recent logs plus diagnosis/retry suggestions; use those suggestions to call `serve_model` again with an adjusted command when appropriate.", "stop_served_model": "- ```stop_served_model``` — Stop a running model server. Args (JSON): {\"session_id\": \"\"}. Use for 'kill my cookbook' / 'stop the model' / 'shut down vLLM'.", "tail_serve_output": "- ```tail_serve_output``` — Read the actual tmux stderr/traceback of a CURRENTLY failing cookbook task. Args (JSON): {\"session_id\": \"\", \"tail\": 150?}. **Use ONLY after** you just launched something via `serve_model` AND `list_served_models` reports YOUR new task as `crashed`/`error`. DO NOT use it on old stopped/completed download tasks (they're historical noise — won't predict whether a new launch succeeds). DO NOT call it before launching a fresh attempt. When you do call it, bump `tail` to 400+ only if the visible error references 'see root cause above'.", @@ -1372,6 +1373,53 @@ def _empty_response_fallback( return _error_msg, f'data: {json.dumps({"delta": _error_msg})}\n\n' +PLAN_MODE_DIRECTIVE = ( + "## PLAN MODE — OVERRIDES EVERYTHING ELSE BELOW\n" + "You are in PLAN MODE. Your ONLY job this turn is to PROPOSE a plan. You have " + "NOT done anything yet. Do NOT claim you created, wrote, ran, sent, or changed " + "anything — that would be a lie.\n" + "\n" + "ABSOLUTE RULE — DO NOT MUTATE ANYTHING. Every write/state-changing tool, " + "including the shell (`bash`/`python`), is disabled this turn and will be " + "rejected — only read-only tools remain available. Use the read-only tools " + "listed below (read files, search code, browse the project, web lookups) to " + "ground the plan. If the task is 'write a file', your plan is to DESCRIBE " + "writing it — you do NOT write it now.\n" + "\n" + "OUTPUT: present the plan as a GitHub-style checklist, one concrete step per line:\n" + "- [ ] first action you will take once approved\n" + "- [ ] next action\n" + "Each item = one concrete action (file to create/edit, command to run, side " + "effect). Do not execute. Do not end with 'Done' or anything implying the work " + "is finished. End your turn with the checklist." +) + + +def build_active_plan_note(approved_plan: str) -> str: + """System note that pins an approved plan during execution. + + Sent back by the frontend each turn so a long plan on a weak model survives + history truncation — the agent can always re-read it. Returns "" for empty + input. + """ + if not approved_plan or not approved_plan.strip(): + return "" + return ( + "## ACTIVE PLAN (approved — execute this)\n" + "You are executing a plan the user already approved. THE FULL PLAN IS " + "BELOW — it is always provided here every turn. Do NOT say you lost it, " + "and do NOT look for it in tasks, notes, memory, files, or the API; just " + "read it below. Work through it IN ORDER. After finishing each step, call " + "the `update_plan` tool with the full checklist and that step marked " + "`- [x]` so progress stays visible in the user's plan window. If the user " + "asks to change the plan, call `update_plan` with the revised checklist. " + "Do the next unchecked item until all are done. Do not skip, reorder, or " + "invent steps; if a step is genuinely impossible, say so and stop.\n\n" + "Current plan:\n" + + approved_plan.strip() + ) + + async def stream_agent_loop( endpoint_url: str, model: str, @@ -1390,6 +1438,8 @@ async def stream_agent_loop( relevant_tools: Optional[Set[str]] = None, fallbacks: Optional[List[tuple]] = None, workspace: Optional[str] = None, + plan_mode: bool = False, + approved_plan: Optional[str] = None, _is_teacher_run: bool = False, ) -> AsyncGenerator[str, None]: """Streaming agent loop generator. @@ -1413,6 +1463,13 @@ async def stream_agent_loop( # public/non-admin users rather than trying to enumerate every tool. mcp_mgr = None + if plan_mode: + # Plan mode: investigate read-only, propose a plan, don't execute. The + # route also unions the read-only-disabled set, but enforce here too so + # the loop is safe regardless of caller. MCP stays available but is + # filtered to read-only tools below (after the disabled map is loaded). + disabled_tools.update(plan_mode_disabled_tools()) + _t0 = time.time() _needs_admin = _detect_admin_intent(messages) _last_user = _extract_last_user_message(messages) @@ -1420,6 +1477,13 @@ async def stream_agent_loop( # not just the latest message, so short follow-ups don't drop just-used tools. _retrieval_query = _recent_context_for_retrieval(messages) or _last_user _mcp_disabled_map = _load_mcp_disabled_map() if mcp_mgr else {} + if plan_mode and mcp_mgr: + # Allow read-only MCP tools to investigate, block write/unknown ones: + # hide them from the schemas AND reject them at runtime by qualified name. + _mcp_block_map, _mcp_block_q = mcp_mgr.plan_mode_blocked_mcp() + for _sid, _names in _mcp_block_map.items(): + _mcp_disabled_map.setdefault(_sid, set()).update(_names) + disabled_tools.update(_mcp_block_q) prep_timings["request_setup"] = time.time() - _t0 # RAG-based tool selection: retrieve relevant tools for this query. @@ -1577,6 +1641,27 @@ async def stream_agent_loop( else: messages.insert(0, {"role": "system", "content": _ws_note}) logger.info("[workspace] active for this turn: %s", workspace) + if plan_mode: + # Steer the model to investigate-then-propose. Hard tool gating handles + # every write path except shell; this directive is what keeps the + # intentionally-allowed bash/python read-only, so it must DOMINATE. Put + # it at the very TOP of the system prompt (the base prompt is large and + # action-oriented — appending buried it, and small models ignored it). + if messages and messages[0].get("role") == "system": + messages[0]["content"] = PLAN_MODE_DIRECTIVE + "\n\n" + (messages[0].get("content") or "") + else: + messages.insert(0, {"role": "system", "content": PLAN_MODE_DIRECTIVE}) + elif approved_plan and approved_plan.strip(): + # EXECUTING an approved plan. Pin the checklist as a top-of-context + # system note so a long plan on a weak model survives history + # truncation — the agent can always re-read the plan instead of losing + # the thread. (The first system message is kept by the context trimmer.) + _plan_note = build_active_plan_note(approved_plan) + if messages and messages[0].get("role") == "system": + messages[0]["content"] = _plan_note + "\n\n" + (messages[0].get("content") or "") + else: + messages.insert(0, {"role": "system", "content": _plan_note}) + logger.info("[plan] pinned approved plan (%d chars) for execution turn", len(approved_plan)) prep_timings["prompt_build"] = time.time() - _t2 _t3 = time.time() @@ -2287,6 +2372,14 @@ async def stream_agent_loop( ) _awaiting_user = True + # update_plan: agent wrote back to the plan (ticked a step / revised). + # Push it to the frontend so the stored plan + docked window update + # live. Does NOT end the turn — the agent keeps working. + if "plan_update" in result: + yield ( + f'data: {json.dumps({"type": "plan_update", "data": result["plan_update"]})}\n\n' + ) + # Build output for frontend tool bubble. # Document tools get a short summary — content goes to the editor panel. output_text = "" diff --git a/src/agent_tools.py b/src/agent_tools.py index c7c2a36..41f0411 100644 --- a/src/agent_tools.py +++ b/src/agent_tools.py @@ -34,7 +34,7 @@ TOOL_TAGS = {"bash", "python", "web_search", "web_fetch", "read_file", "write_fi "send_to_session", "pipeline", "manage_session", "manage_memory", "list_models", - "ui_control", "generate_image", "ask_user", + "ui_control", "generate_image", "ask_user", "update_plan", "manage_tasks", "api_call", "ask_teacher", "manage_skills", "suggest_document", "manage_endpoints", "manage_mcp", "manage_webhooks", diff --git a/src/mcp_manager.py b/src/mcp_manager.py index 03bcf18..29fdede 100644 --- a/src/mcp_manager.py +++ b/src/mcp_manager.py @@ -9,7 +9,7 @@ import json import logging import os import re -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Set, Tuple logger = logging.getLogger(__name__) @@ -90,6 +90,44 @@ def _format_mcp_params(input_schema: Any) -> str: return hint +# Tool-name prefixes that denote a read-only/inspection operation. Used to +# classify MCP tools for plan mode when the server provides no readOnlyHint. +# These are PREFIXES, not whole words (matched via str.startswith below), so a +# stem like "summar" intentionally covers "summarise"/"summarize"/"summary". +_MCP_READONLY_VERBS = ( + "list", "get", "read", "search", "fetch", "query", "find", "describe", + "show", "view", "lookup", "count", "status", "info", "inspect", "summar", +) + + +def mcp_tool_is_readonly(tool: Dict) -> bool: + """Classify an MCP tool as safe (non-mutating) for plan mode. + + Prefer the server's own annotations (readOnlyHint / destructiveHint). When + absent, fall back to a tool-name verb heuristic, and FAIL CLOSED (treat as + write) for anything that doesn't clearly read — plan mode must not run a + write tool just because its intent is ambiguous. + """ + ann = tool.get("annotations") + # annotations may be a dict or a pydantic model + read_hint = None + destructive = None + if ann is not None: + if isinstance(ann, dict): + read_hint = ann.get("readOnlyHint") + destructive = ann.get("destructiveHint") + else: + read_hint = getattr(ann, "readOnlyHint", None) + destructive = getattr(ann, "destructiveHint", None) + if read_hint is True: + return True + if read_hint is False or destructive is True: + return False + # No usable hint — heuristic on the tool name's leading verb. + name = (tool.get("name") or "").lower() + return name.startswith(_MCP_READONLY_VERBS) + + class McpManager: """Manages MCP server connections and tool routing.""" @@ -170,6 +208,10 @@ class McpManager: "name": tool.name, "description": tool.description or "", "input_schema": tool.inputSchema if hasattr(tool, 'inputSchema') else {}, + # MCP tool annotations (readOnlyHint / destructiveHint) drive + # plan-mode read-only gating. Absent on many servers, so we + # fall back to a name heuristic in mcp_tool_is_readonly(). + "annotations": getattr(tool, 'annotations', None), }) self._sessions[server_id] = session @@ -227,6 +269,10 @@ class McpManager: "name": tool.name, "description": tool.description or "", "input_schema": tool.inputSchema if hasattr(tool, 'inputSchema') else {}, + # MCP tool annotations (readOnlyHint / destructiveHint) drive + # plan-mode read-only gating. Absent on many servers, so we + # fall back to a name heuristic in mcp_tool_is_readonly(). + "annotations": getattr(tool, 'annotations', None), }) self._sessions[server_id] = session @@ -537,6 +583,24 @@ class McpManager: }) return result + def plan_mode_blocked_mcp(self) -> Tuple[Dict[str, Set[str]], Set[str]]: + """Plan mode: block every MCP tool that isn't clearly read-only. + + Returns (disabled_map, qualified_names): + - disabled_map: {server_id: {tool_name, ...}} to hide write tools from + the prompt/schemas (merged into the existing mcp_disabled_map). + - qualified_names: {"mcp____", ...} for runtime rejection + in execute_tool_block (which matches the qualified name). + """ + disabled_map: Dict[str, Set[str]] = {} + qualified: Set[str] = set() + for server_id, tools in self._tools.items(): + for tool in tools: + if not mcp_tool_is_readonly(tool): + disabled_map.setdefault(server_id, set()).add(tool["name"]) + qualified.add(f"mcp__{server_id}__{tool['name']}") + return disabled_map, qualified + def is_builtin(self, server_id: str) -> bool: """Check if a server is a built-in (auto-registered) server.""" return server_id.startswith("builtin_") or server_id in { diff --git a/src/tool_execution.py b/src/tool_execution.py index 9af6cce..40bca42 100644 --- a/src/tool_execution.py +++ b/src/tool_execution.py @@ -1263,6 +1263,41 @@ async def execute_tool_block( logger.info("Tool executed: %s (%d options, multi=%s)", desc, len(options), multi) return desc, result + # update_plan: the agent writes back to the active plan — tick an item done + # or revise steps (e.g. when the user asks to change something). Pure UI + # marker: returns a `plan_update` payload the agent loop turns into a + # `plan_update` SSE event; the frontend replaces the stored plan and refreshes + # the docked plan window. Does NOT end the turn. + if tool == "update_plan": + import json as _json + raw = (content or "").strip() + plan = "" + try: + parsed = _json.loads(raw) if raw else {} + except (ValueError, TypeError): + parsed = {} + if isinstance(parsed, dict) and parsed.get("plan"): + plan = str(parsed.get("plan", "")).strip() + else: + # Plain-string call (raw checklist) or JSON without a usable `plan`. + plan = raw + if not plan: + return "update_plan: invalid", { + "error": "update_plan needs a non-empty `plan` (the full updated checklist as markdown).", + "exit_code": 1, + } + plan = plan[:8192] + done = plan.count("- [x]") + plan.count("- [X]") + total = done + plan.count("- [ ]") + desc = f"update_plan: {done}/{total} done" if total else "update_plan" + result = { + "plan_update": {"plan": plan}, + "output": f"Plan updated ({done}/{total} steps complete)." if total else "Plan updated.", + "exit_code": 0, + } + logger.info("Tool executed: %s", desc) + return desc, result + # Background execution: a `bash` block whose first line is the `#!bg` # marker runs DETACHED — returns a job id immediately so the chat stream # isn't held open for a multi-minute install/ffmpeg/download. The always-on diff --git a/src/tool_index.py b/src/tool_index.py index c6eea86..a56fa05 100644 --- a/src/tool_index.py +++ b/src/tool_index.py @@ -55,6 +55,8 @@ ALWAYS_AVAILABLE = frozenset({ # Ask the user a multiple-choice question for a decision/clarification. # Always reachable so the agent can pause and ask at any point. "ask_user", + # Write back to the active plan (tick steps done / revise) during execution. + "update_plan", }) # Tools that the Personal Assistant always has access to during scheduled @@ -115,6 +117,7 @@ BUILTIN_TOOL_DESCRIPTIONS: Dict[str, str] = { "send_to_session": "Send a message to another chat. Cross-chat communication.", "search_chats": "Search through chat history across all sessions.", "ask_user": "Ask the user a multiple-choice question to get a decision or clarification. Use this when the task is genuinely ambiguous and the answer changes what you do next — pick between approaches, confirm an assumption, choose among options — instead of guessing. Provide a clear `question` and 2-6 `options` (each with a short `label`, optional `description`). Calling this ENDS your turn: the user sees clickable buttons and their choice arrives as your next message. Don't use it for things you can decide from context or sensible defaults, or for irreversible-action confirmation if a dedicated flow exists.", + "update_plan": "Write back to the ACTIVE PLAN while executing an approved plan: mark steps done or revise them. After finishing a step call this with the full checklist and that step marked done; when the user asks to change the plan call it with the revised checklist. Always pass the COMPLETE markdown checklist (`- [ ]` / `- [x]`), not a diff. The user's docked plan window updates live. No effect when there is no active plan.", "ui_control": "Control the UI and toggle tools on/off. Use this to turn off / turn on / disable / enable individual tools and features: shell (bash), search (web), research, browser, documents, incognito. Open panels (documents library, gallery, email inbox, sessions, notes, memories/brain, skills, settings, cookbook) via `open_panel `. Use `open_email_reply reply` to open an email reply draft document without sending. Also switches between chat/agent modes, changes the current model, and applies/creates themes.", "list_email_accounts": "List configured email accounts and default status. Use before reading or sending mail when the user mentions Gmail, work mail, custom domain mail, another mailbox, or asks to compare/check multiple inboxes.", "list_emails": "List emails for a folder/account, newest first, including read messages by default. Shows subject, sender, date, UID, account, and AI summary. Check inbox, find emails needing replies. Supports account from list_email_accounts for Gmail/work/custom mailboxes. For last/latest/newest email, use max_results=1 and unread_only=false.", diff --git a/src/tool_schemas.py b/src/tool_schemas.py index 7c6a639..3138d60 100644 --- a/src/tool_schemas.py +++ b/src/tool_schemas.py @@ -474,6 +474,20 @@ FUNCTION_TOOL_SCHEMAS = [ } } }, + { + "type": "function", + "function": { + "name": "update_plan", + "description": "Write back to the ACTIVE PLAN: mark steps done or revise them. Use this while executing an approved plan — after you finish a step, call update_plan with the full checklist and that step marked `- [x]`; when the user asks to change the plan, call it with the revised checklist. The user's docked plan window updates live. Pass the COMPLETE checklist every time (not a diff). No effect if there is no active plan.", + "parameters": { + "type": "object", + "properties": { + "plan": {"type": "string", "description": "The full updated plan as a GitHub-style markdown checklist — one step per line, `- [ ]` for pending and `- [x]` for done. Always send the whole list."} + }, + "required": ["plan"] + } + } + }, { "type": "function", "function": { diff --git a/src/tool_security.py b/src/tool_security.py index 8ffa50f..82d2c3d 100644 --- a/src/tool_security.py +++ b/src/tool_security.py @@ -51,6 +51,101 @@ NON_ADMIN_BLOCKED_TOOLS = { } +# Plan mode: the agent may investigate but must not mutate anything. Only these +# read-only/inspection tools stay enabled; everything else (writes, sends, +# manage_*, model serving, MCP, etc.) is blocked. Allowlist rather than blocklist +# so any newly added tool defaults to BLOCKED in plan mode — fail safe. +# +# bash/python are deliberately NOT here: the shell can mutate (write files, hit +# the network) and can't be constrained to read-only at the tool layer, so plan +# mode blocks it outright rather than relying on a prompt to keep it well-behaved. +# Code/file discovery is covered by the dedicated read-only tools below +# (read_file, grep, glob, ls) instead of freestyle shell. +PLAN_MODE_READONLY_TOOLS = { + "read_file", + "grep", + "glob", + "ls", + "web_search", + "web_fetch", + "search_chats", + "list_models", + "list_sessions", + "list_emails", + "read_email", + "list_served_models", + "list_downloads", + "list_cached_models", + "search_hf_models", + "list_serve_presets", + "list_cookbook_servers", + "resolve_contact", + "chat_with_model", + "ask_teacher", +} + + +# The agent's tool gate is a DENYLIST: execute_tool_block blocks any tool whose +# name is in `disabled_tools`. Plan mode's policy is the opposite — an allowlist +# (PLAN_MODE_READONLY_TOOLS). To apply an allowlist through a denylist, plan mode +# returns the inverse: every known tool name minus the allowlist. +# +# Known tool names come from FUNCTION_TOOL_SCHEMAS, but that source is imperfect: +# some tools are only XML-invocable (e.g. manage_notes, generate_image) and never +# appear there, and the import can fail outright. Either gap would drop a mutating +# tool from the subtraction and silently leave it enabled. This set is the static +# backstop for both: union it in so known mutators are always subtracted, and so a +# failed import still blocks them (fail closed, never open). Only mutators belong +# here — read-only tools are covered by the allowlist. Keep in sync when adding +# new mutating tools. +_PLAN_MODE_KNOWN_MUTATORS = { + "write_file", "create_document", "edit_document", "update_document", + "suggest_document", "manage_documents", "create_session", "manage_session", + "send_to_session", "pipeline", "manage_memory", "manage_skills", + "manage_tasks", "manage_notes", "manage_endpoints", "manage_mcp", + "manage_webhooks", "manage_tokens", "manage_settings", "manage_contact", + "manage_calendar", "api_call", "app_api", "ui_control", + "send_email", "reply_to_email", "bulk_email", "delete_email", + "archive_email", "mark_email_read", "download_model", "serve_model", + "stop_served_model", "cancel_download", "adopt_served_model", "serve_preset", + "generate_image", "edit_image", "trigger_research", "manage_research", + # Shell is never read-only-safe; block it explicitly so it stays out of plan + # mode even if the schema list fails to load. + "bash", "python", +} + + +def plan_mode_disabled_tools() -> Set[str]: + """Tool names to add to the denylist in plan mode. + + Plan mode allows only PLAN_MODE_READONLY_TOOLS. The gate is a denylist, so + return the inverse: every known tool name minus the allowlist. Known names + come from the function-tool schemas, backstopped by _PLAN_MODE_KNOWN_MUTATORS + (see above) so XML-only tools and a failed schema import can't leave a mutator + enabled. MCP tools are handled separately — the loop drops the MCP manager + entirely in plan mode.""" + try: + # agent_tools / tool_parsing / tool_schemas form a mutually-circular + # cluster that only resolves cleanly when entered via agent_tools. + # Import it first so the lazy schema import works even from a cold + # import (e.g. tests) — not just after the app has wired everything up. + import src.agent_tools # noqa: F401 + from src.tool_schemas import FUNCTION_TOOL_SCHEMAS + + all_names = { + (t.get("function") or {}).get("name") + for t in FUNCTION_TOOL_SCHEMAS + } + all_names.discard(None) + except Exception as exc: + logger.warning("Unable to load tool schemas for plan-mode gating: %s", exc) + all_names = set() + # Subtract the allowlist from all known tool names (schema-derived plus the + # static mutator backstop). Fail closed: if the schema import failed above, + # the backstop alone still blocks known mutators. + return (all_names | _PLAN_MODE_KNOWN_MUTATORS) - PLAN_MODE_READONLY_TOOLS + + def is_public_blocked_tool(tool_name: Optional[str]) -> bool: """Return True when a non-admin/public user must not execute this tool. diff --git a/static/app.js b/static/app.js index 08ab121..5621ef7 100644 --- a/static/app.js +++ b/static/app.js @@ -1555,6 +1555,7 @@ function initializeEventListeners() { const MODE_TOOLS = [ { btnId: 'web-toggle-btn', checkboxId: 'web-toggle', stateKey: 'web' }, { btnId: 'bash-toggle-btn', checkboxId: 'bash-toggle', stateKey: 'bash' }, + { btnId: 'plan-toggle-btn', checkboxId: 'plan-toggle', stateKey: 'plan' }, ]; function _modeKey(stateKey, mode) { return `${stateKey}_${mode}`; } @@ -1563,6 +1564,9 @@ function initializeEventListeners() { const state = loadToggleState(); const key = _modeKey(stateKey, mode); if (Object.prototype.hasOwnProperty.call(state, key)) return !!state[key]; + // Plan mode is opt-in: never default it on, otherwise every agent turn + // would be forced into planning. + if (stateKey === 'plan') return false; return mode === 'agent'; // default: ON in agent, OFF in chat } @@ -1575,6 +1579,7 @@ function initializeEventListeners() { const TOOL_TOGGLE_TOAST_LABELS = { web: 'Web search', bash: 'Shell', + plan: 'Plan mode', }; function showToolToggleToast(stateKey, active) { @@ -1688,6 +1693,81 @@ function initializeEventListeners() { } setupToggle('web-toggle-btn', 'web-toggle', 'web'); setupToggle('bash-toggle-btn', 'bash-toggle', 'bash'); + try { workspaceModule.initWorkspace(); } catch (_) {} + setupToggle('plan-toggle-btn', 'plan-toggle', 'plan'); + + // Set plan mode on/off directly (checkbox + button state + saved pref) WITHOUT + // going through the button's click handler — used by the plan menu and by the + // "Approve & Run" flow. Going through .click() would hit the plan-menu + // intercept below (a stored plan re-opens the menu instead of toggling), which + // is exactly the bug that left approved plans stuck in plan mode. + function _setPlanMode(on) { + const btn = el('plan-toggle-btn'); + const chk = el('plan-toggle'); + const mode = (loadToggleState().mode) || 'chat'; + if (chk) chk.checked = !!on; + if (btn) { btn.classList.toggle('active', !!on); btn.setAttribute('aria-pressed', String(!!on)); } + saveToolPref('plan', mode, !!on); + } + window._setPlanMode = _setPlanMode; + + // ── Plan-button menu ── + // When a plan exists for this chat, clicking the plan button opens a small + // menu (Show plan / Plan mode on-off) instead of plain-toggling — so the plan + // window can be re-opened and docked at any time while the agent works. With + // no plan, the button behaves as before (one-click toggle). + (function initPlanMenu() { + const planBtn = el('plan-toggle-btn'); + if (!planBtn) return; + const _hasPlan = () => { try { return !!(window._getStoredPlan && window._getStoredPlan()); } catch (_) { return false; } }; + const _close = () => { const m = document.getElementById('plan-menu'); if (m) m.remove(); }; + function _open() { + _close(); + const planChk = el('plan-toggle'); + const on = !!(planChk && planChk.checked); + const menu = document.createElement('div'); + menu.id = 'plan-menu'; + menu.className = 'overflow-menu plan-menu'; + menu.innerHTML = + '' + + ''; + document.body.appendChild(menu); + const r = planBtn.getBoundingClientRect(); + menu.style.position = 'fixed'; + menu.style.left = Math.round(r.left) + 'px'; + menu.style.top = Math.round(r.top - menu.offsetHeight - 6) + 'px'; + menu.querySelector('[data-act="show"]').addEventListener('click', () => { + _close(); + const txt = window._getStoredPlan ? window._getStoredPlan() : ''; + if (txt && window.planWindowModule) window.planWindowModule.openPlanWindow(txt, null); + }); + menu.querySelector('[data-act="toggle"]').addEventListener('click', () => { + _close(); + _setPlanMode(!on); // flip state directly (no click → no menu re-open) + }); + // Dismiss on any outside click (capture so it beats other handlers) / Escape. + setTimeout(() => { + const off = (e) => { + if (!menu.contains(e.target) && e.target !== planBtn) { + _close(); document.removeEventListener('click', off, true); document.removeEventListener('keydown', esc, true); + } + }; + const esc = (e) => { if (e.key === 'Escape') { _close(); document.removeEventListener('click', off, true); document.removeEventListener('keydown', esc, true); } }; + document.addEventListener('click', off, true); + document.addEventListener('keydown', esc, true); + }, 0); + } + planBtn.addEventListener('click', (e) => { + // With a stored plan, the button opens the menu (Show plan / toggle). + // Without one, it falls through to the normal one-click toggle. + if (_hasPlan()) { e.preventDefault(); e.stopImmediatePropagation(); _open(); } + }, true); // capture phase: intercept before setupToggle's bubble handler + })(); + try { workspaceModule.initWorkspace(); } catch (_) {} // Document editor toggle (special: uses module panel, not a checkbox) diff --git a/static/index.html b/static/index.html index 3916cca..22cdfda 100644 --- a/static/index.html +++ b/static/index.html @@ -1076,6 +1076,12 @@ + + + +
+
+ +
+ `; + document.body.appendChild(_modal); + _modal.querySelector('#plan-window-close').addEventListener('click', closePlanWindow); + _modal.querySelector('#plan-window-approve').addEventListener('click', () => { + const cb = _onApprove; + closePlanWindow(); + if (typeof cb === 'function') cb(); + }); + // Draggable + side-dockable, same one-call helper as the other windows. + const content = _modal.querySelector('.modal-content'); + const header = _modal.querySelector('.modal-header'); + if (content && header) makeWindowDraggable(_modal, { content, header }); + return _modal; +} + +/** + * Open the plan window with rendered markdown and an approve callback. + * @param {string} planMarkdown - the agent's proposed plan (raw markdown) + * @param {Function} onApprove - called when the user clicks Approve & Run + */ +export function openPlanWindow(planMarkdown, onApprove) { + const modal = _getModal(); + _onApprove = onApprove || null; + const body = modal.querySelector('#plan-window-body'); + if (body) { + body.innerHTML = markdownModule.processWithThinking( + markdownModule.squashOutsideCode(planMarkdown || '') + ); + if (window.hljs) body.querySelectorAll('pre code').forEach((b) => window.hljs.highlightElement(b)); + } + const approveBtn = modal.querySelector('#plan-window-approve'); + if (approveBtn) approveBtn.style.display = onApprove ? '' : 'none'; + // Title reflects state: still awaiting approval (approve callback present) vs + // already approved and being executed. + const title = modal.querySelector('#plan-window-title'); + if (title) title.textContent = onApprove ? 'Proposed plan' : 'Approved plan'; + modal.style.display = 'flex'; + if (uiModule && uiModule.scrollHistory) { try { uiModule.scrollHistory(); } catch (_) {} } +} + +export function closePlanWindow() { + if (_modal) _modal.style.display = 'none'; +} + +/** True when the plan window is currently visible (for live-refresh on progress). */ +export function isPlanWindowOpen() { + return !!(_modal && _modal.style.display !== 'none'); +} + +export default { openPlanWindow, closePlanWindow, isPlanWindowOpen }; diff --git a/static/js/slashCommands.js b/static/js/slashCommands.js index 0f3a720..1a11454 100644 --- a/static/js/slashCommands.js +++ b/static/js/slashCommands.js @@ -1170,6 +1170,22 @@ async function _cmdWorkspace(args, ctx) { slashReply('Usage: /workspace · set /path · clear · pick'); return true; } +// Plan mode: drive the real toggle pill (#plan-toggle-btn) so its per-mode +// persistence/UI logic runs. Only meaningful in agent mode. +async function _cmdTogglePlan(args, ctx) { + const btn = document.getElementById('plan-toggle-btn'); + const chk = document.getElementById('plan-toggle'); + if (!btn || btn.style.display === 'none' || btn.offsetParent === null) { + slashReply('Plan mode is only available in agent mode — switch to Agent first.'); + return true; + } + const cur = !!(chk && chk.checked); + const v = (args[0] || '').toLowerCase(); + const target = v === 'on' ? true : v === 'off' ? false : !cur; + if (target !== cur) btn.click(); + slashReply(`Plan mode: ${target ? 'on' : 'off'}`); + return true; +} async function _cmdToggleShow(args, ctx) { const name = (args[0] || '').toLowerCase(); @@ -5489,6 +5505,7 @@ const COMMANDS = { 'bash': { handler: _cmdToggleBash, alias: ['b','shell'], help: 'Toggle bash/shell', usage: '/toggle bash' }, 'research': { handler: _cmdToggleResearch, alias: ['r'], help: 'Toggle deep research', usage: '/toggle research' }, 'doc': { handler: _cmdToggleDoc, alias: [], help: 'Toggle document editor', usage: '/toggle doc' }, + 'plan': { handler: _cmdTogglePlan, alias: ['p'], help: 'Toggle plan mode (agent)', usage: '/toggle plan' }, 'sidebar': { handler: _cmdToggleSidebar, alias: ['sb'], help: 'Cycle sidebar (full/mini/off)', usage: '/toggle sidebar [1|2|3]' }, '_show': { handler: _cmdToggleShow, alias: [], help: 'Show all toggle states', usage: '/toggle' } } @@ -5501,6 +5518,13 @@ const COMMANDS = { noUserBubble: true, usage: '/workspace [set | clear | pick]', }, + plan: { + alias: [], + category: 'Quick toggles', + help: 'Toggle plan mode (agent)', + handler: _cmdTogglePlan, + usage: '/plan [on|off]', + }, memory: { alias: ['m'], category: 'Memory', diff --git a/static/js/storage.js b/static/js/storage.js index 7ff9c6b..06b4d54 100644 --- a/static/js/storage.js +++ b/static/js/storage.js @@ -24,7 +24,8 @@ export const KEYS = { SECTION_ORDER: 'sidebar-section-order', ADMIN_LAST_TAB: 'admin-last-tab', DENSITY: 'odysseus-density', - WORKSPACE: 'odysseus-workspace' + WORKSPACE: 'odysseus-workspace', + PLAN: 'odysseus-plan' }; /** diff --git a/static/style.css b/static/style.css index 8243a0b..2c79b51 100644 --- a/static/style.css +++ b/static/style.css @@ -2305,6 +2305,104 @@ body.bg-pattern-sparkles { color: var(--fg); background: color-mix(in srgb, var(--fg) 9%, transparent); } + /* Plan mode: "Approve & Run" affordance under a proposed plan */ + .plan-approve-bar { + margin: 8px 0 2px; + } + .plan-approve-btn { + font: inherit; + font-size: 13px; + font-weight: 600; + padding: 6px 14px; + border-radius: 8px; + cursor: pointer; + color: var(--accent); + background: color-mix(in srgb, var(--accent) 12%, transparent); + border: 1px solid var(--accent); + transition: background 0.15s, transform 0.1s; + } + .plan-approve-btn:hover { + background: color-mix(in srgb, var(--accent) 22%, transparent); + } + .plan-approve-btn:active { + transform: scale(0.97); + } + .plan-approve-bar { + display: flex; + gap: 8px; + align-items: center; + } + .plan-open-btn { + font: inherit; + font-size: 13px; + padding: 6px 12px; + border-radius: 8px; + cursor: pointer; + color: var(--fg); + background: color-mix(in srgb, var(--fg) 8%, transparent); + border: 1px solid color-mix(in srgb, var(--fg) 22%, transparent); + transition: background 0.15s; + } + .plan-open-btn:hover { + background: color-mix(in srgb, var(--fg) 15%, transparent); + } + /* GitHub-style task lists (- [ ] / - [x]) — used by plan-mode checklists */ + li.task-item { + list-style: none; + margin-left: -1.2em; + display: flex; + align-items: flex-start; + gap: 8px; + } + li.task-item .task-check { + flex: 0 0 auto; + width: 15px; + height: 15px; + margin-top: 3px; + border-radius: 4px; + border: 1.5px solid color-mix(in srgb, var(--fg) 45%, transparent); + box-sizing: border-box; + position: relative; + } + li.task-item.task-done .task-check { + background: var(--accent); + border-color: var(--accent); + } + li.task-item.task-done .task-check::after { + content: ''; + position: absolute; + left: 4px; + top: 1px; + width: 4px; + height: 8px; + border: solid var(--bg); + border-width: 0 2px 2px 0; + transform: rotate(45deg); + } + li.task-item.task-done .task-text { + opacity: 0.6; + text-decoration: line-through; + } + /* Plan window: a draggable/dockable modal (shares .modal framework) */ + .plan-window-content { + width: 520px; + max-width: 92vw; + max-height: 80vh; + display: flex; + flex-direction: column; + } + .plan-window-body { + overflow-y: auto; + padding: 14px 18px; + flex: 1 1 auto; + line-height: 1.55; + } + .plan-window-footer { + padding: 10px 18px; + border-top: 1px solid color-mix(in srgb, var(--fg) 12%, transparent); + display: flex; + justify-content: flex-end; + } /* While the menu is open the chevron stays in its highlighted state — don't run the opacity fade transition so we never flash from 0.5 → hover-1.0 → drop-back. The state holds steady. */ diff --git a/tests/test_plan_mode.py b/tests/test_plan_mode.py new file mode 100644 index 0000000..cfca831 --- /dev/null +++ b/tests/test_plan_mode.py @@ -0,0 +1,104 @@ +"""Plan mode gating regression tests. + +Plan mode restricts the agent to read-only/inspection tools so it can investigate +and propose a plan without mutating anything. These pin the security-relevant +contract: + +- The read-only allowlist contains only inspection tools (no writes/sends/manage_*). +- `plan_mode_disabled_tools()` blocks every mutating tool and never blocks an + allowlisted one. +- It fails CLOSED: if the tool-schema list can't be loaded, it still blocks a + known-mutating set rather than returning nothing (which would allow mutations). + +Pure-function tests — no FastAPI app boot, no DB. +""" + +from src.tool_security import ( + PLAN_MODE_READONLY_TOOLS, + _PLAN_MODE_KNOWN_MUTATORS, + plan_mode_disabled_tools, +) + + +def test_allowlist_has_no_obvious_mutating_tools(): + # Sanity: the read-only allowlist must not contain mutating/external tools. + mutating_markers = ("write_", "send_", "manage_", "create_", "edit_", "delete_") + for name in PLAN_MODE_READONLY_TOOLS: + assert not name.startswith(mutating_markers), f"{name} should not be read-only" + + +def test_plan_mode_blocks_mutating_tools(): + disabled = plan_mode_disabled_tools() + # A representative spread of mutating/external tools must be blocked. + for name in ( + "write_file", "send_email", "reply_to_email", "manage_memory", + "manage_settings", "create_document", "edit_document", "download_model", + "generate_image", "trigger_research", + ): + assert name in disabled, f"{name} must be blocked in plan mode" + + +def test_plan_mode_allows_readonly_tools(): + disabled = plan_mode_disabled_tools() + # Read-only investigation tools stay enabled, including the discovery tools + # (grep/glob/ls) that replace freestyle shell. + for name in ("read_file", "grep", "glob", "ls", "web_search", "web_fetch", "search_chats"): + assert name not in disabled, f"{name} should be usable in plan mode" + + +def test_plan_mode_blocks_shell(): + # bash/python can mutate and can't be constrained read-only, so plan mode + # must block them (the whole point of dropping shell from plan mode). + disabled = plan_mode_disabled_tools() + for name in ("bash", "python"): + assert name in disabled, f"{name} must be blocked in plan mode" + + +def test_disabled_never_intersects_allowlist(): + assert plan_mode_disabled_tools() & PLAN_MODE_READONLY_TOOLS == set() + + +def test_mcp_readonly_classification(): + from src.mcp_manager import mcp_tool_is_readonly as ro + # Server-provided hints win over the name heuristic. + assert ro({"name": "zap", "annotations": {"readOnlyHint": True}}) is True + assert ro({"name": "list_things", "annotations": {"readOnlyHint": False}}) is False + assert ro({"name": "get_x", "annotations": {"destructiveHint": True}}) is False + # No hint → leading-verb heuristic, fail closed for ambiguous names. + assert ro({"name": "list_files"}) is True + assert ro({"name": "search_docs"}) is True + assert ro({"name": "send_message"}) is False + assert ro({"name": "frobnicate"}) is False + + +def test_fail_closed_fallback_blocks_mutations(monkeypatch): + # If the schema list can't load, we must still block (fail closed), not + # return an empty set that would silently allow every mutating tool. + import src.tool_security as ts + + def _boom(): + raise ImportError("simulated circular import failure") + + # Force the dynamic path to fail by making the lazy import explode. + monkeypatch.setitem( + __import__("sys").modules, "src.agent_tools", None + ) + disabled = ts.plan_mode_disabled_tools() + assert disabled, "plan mode must never fail open (empty disabled set)" + assert "write_file" in disabled + assert "send_email" in disabled + assert disabled == set(_PLAN_MODE_KNOWN_MUTATORS) + + +def test_active_plan_note_pins_checklist(): + """The approved-plan note re-grounds execution so a long plan survives + history truncation (the agent can always re-read it).""" + from src.agent_loop import build_active_plan_note + plan = "- [ ] step one\n- [ ] step two" + note = build_active_plan_note(plan) + assert "ACTIVE PLAN" in note + assert plan in note # the actual checklist is embedded + assert "IN ORDER" in note # execution guidance present + # Empty input → no note (so we never inject a blank pin). + assert build_active_plan_note("") == "" + assert build_active_plan_note(" ") == "" diff --git a/tests/test_update_plan_tool.py b/tests/test_update_plan_tool.py new file mode 100644 index 0000000..cac58b2 --- /dev/null +++ b/tests/test_update_plan_tool.py @@ -0,0 +1,46 @@ +"""`update_plan` — the agent writes back to the active plan (tick done / revise). + +Pure UI-control marker: `execute_tool_block` returns a `plan_update` payload the +agent loop turns into a `plan_update` SSE event; the frontend replaces the stored +plan and refreshes the docked plan window. No I/O, does not end the turn. +""" +import asyncio +import json + +from src.agent_tools import ToolBlock, TOOL_TAGS # import first to avoid circular +from src.tool_execution import execute_tool_block +from src.tool_index import ALWAYS_AVAILABLE, BUILTIN_TOOL_DESCRIPTIONS +from src.tool_security import is_public_blocked_tool + + +def _run(content): + return asyncio.run(execute_tool_block(ToolBlock("update_plan", content))) + + +def test_valid_plan_returns_marker_and_counts(): + plan = "- [x] step one\n- [ ] step two\n- [ ] step three" + desc, result = _run(json.dumps({"plan": plan})) + assert result.get("exit_code") == 0 + assert result["plan_update"]["plan"] == plan + assert "1/3" in result["output"] # 1 done of 3 + + +def test_plain_string_accepted(): + plan = "- [ ] a\n- [x] b" + _, result = _run(plan) + assert result["plan_update"]["plan"] == plan + + +def test_empty_rejected(): + _, result = _run(json.dumps({"plan": " "})) + assert "error" in result and result.get("exit_code") == 1 + + +def test_registered_everywhere(): + assert "update_plan" in TOOL_TAGS + assert "update_plan" in ALWAYS_AVAILABLE + assert "update_plan" in BUILTIN_TOOL_DESCRIPTIONS + from src.tool_schemas import FUNCTION_TOOL_SCHEMAS + assert "update_plan" in {s["function"]["name"] for s in FUNCTION_TOOL_SCHEMAS} + # Not admin/public-gated — any user can drive their own plan. + assert is_public_blocked_tool("update_plan") is False From 977daf064377c41e3db50fc89ec21c133515e4e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enes=20=C3=96z?= <118854526+en970@users.noreply.github.com> Date: Fri, 5 Jun 2026 18:07:08 +0300 Subject: [PATCH 029/974] Improve edge-docked window behavior (#2779) * Make edge-docked windows resizable Add draggable resize seams for left and right docked windows. Keep the main chat area from getting too narrow and remember each window's dock width. * Show emoji shortcodes as icons by default Keep text-only emoji mode opt-in so model output like :blush: goes through the normal emoji renderer. * Fix dock resize seams and left dock layout Hide the resize seam when another floating modal is open, and keep the left-docked window from covering the chat area. * Keep narrow modal tabs usable * Fix split layout with both edge docks * Fix left snap after right dock * Enable left edge snap for all windows * Tighten dock resize handle observers * Use edge docking for settings window --- static/app.js | 10 +- static/js/modalSnap.js | 335 +++++++++++++++++++++++++++++++++++++--- static/js/settings.js | 2 +- static/js/windowDrag.js | 10 +- static/style.css | 120 ++++++++++++-- 5 files changed, 434 insertions(+), 43 deletions(-) diff --git a/static/app.js b/static/app.js index 5621ef7..be94aef 100644 --- a/static/app.js +++ b/static/app.js @@ -2497,7 +2497,7 @@ function initializeEventListeners() { }; // Keys hidden by default on first run (no localStorage yet) - const UI_VIS_DEFAULT_OFF = new Set(['models-section', 'rag-toggle-btn']); + const UI_VIS_DEFAULT_OFF = new Set(['models-section', 'rag-toggle-btn', 'text-emojis']); // Keys that need admin to toggle off (reserved for future use) const UI_VIS_ADMIN_ONLY = new Set([]); @@ -2525,11 +2525,9 @@ function initializeEventListeners() { document.querySelectorAll('.section[draggable]').forEach(el => { el.setAttribute('draggable', dragEnabled ? 'true' : 'false'); }); - // Text-only emojis toggle. Default is ON (the checkbox defaults to - // checked because text-emojis isn't in UI_VIS_DEFAULT_OFF), so treat - // an absent value as enabled — otherwise the toggle looked on at - // startup but the effect only activated after the user flipped it. - applyTextEmojis(state['text-emojis'] !== false); + // Text-only emojis toggle. Default is OFF so model-emitted shortcodes + // like `:blush:` render through the normal monochrome emoji path. + applyTextEmojis(state['text-emojis'] === true); // Hide thinking sections toggle (show-thinking: checked=show, unchecked=hide) document.body.classList.toggle('hide-thinking', state['show-thinking'] === false); } diff --git a/static/js/modalSnap.js b/static/js/modalSnap.js index f3085be..e7cce55 100644 --- a/static/js/modalSnap.js +++ b/static/js/modalSnap.js @@ -5,8 +5,8 @@ // emailLibrary.js / documentLibrary.js / galleryEditor.js). While docked: // - the modal-content lives at `right: 0; top: 0; bottom: 0` with a // viewport-fraction width -// - body gets `right-dock-active` + `--right-dock-w` so the chat / -// doc panel / notes pane underneath reserves room via padding-right +// - body gets `right-dock-active` + `--right-dock-w` so the workspace +// underneath reserves room for the fixed side panel // - if the remaining chat width would drop under 380px, the wide // sidebar auto-collapses to the icon rail (mirrors notes-view UX) // @@ -21,6 +21,14 @@ const SNAP_PX = 60; const UNSNAP_PX = 80; const MIN_CHAT_WIDTH = 380; const EMAIL_DOC_SPLIT_WIDTH_KEY = 'odysseus-email-doc-split-width'; +const EDGE_DOCK_WIDTH_KEY_PREFIX = 'odysseus-edge-dock-width'; +const MIN_EDGE_DOCK_WIDTH = 320; + +let _edgeDockHandlePositioner = null; + +function _positionEdgeDockResizeHandles() { + try { _edgeDockHandlePositioner && _edgeDockHandlePositioner(); } catch (_) {} +} function _dockClassForSide(side) { return side === 'left' ? 'modal-left-docked' : 'modal-right-docked'; @@ -48,6 +56,7 @@ export function clearDockSide(side, owner = null) { if (side === 'left') { try { window._restoreSidebarIfRouteCollapsed?.(); } catch (_) {} } + _positionEdgeDockResizeHandles(); } // Default dock width: ~38% of viewport, clamped to a reasonable band. @@ -55,6 +64,78 @@ function _defaultDockWidth() { return Math.min(640, Math.max(420, Math.round(window.innerWidth * 0.38))); } +function _dockWidthStorageKey(modal, content, side) { + const id = modal?.id || content?.id || content?.dataset?.modalId || ''; + return id ? `${EDGE_DOCK_WIDTH_KEY_PREFIX}:${side}:${id}` : null; +} + +function _storedDockWidth(modal, content, side) { + const key = _dockWidthStorageKey(modal, content, side); + if (!key) return null; + try { + const n = parseFloat(localStorage.getItem(key) || ''); + return Number.isFinite(n) && n > 0 ? n : null; + } catch (_) { + return null; + } +} + +function _saveDockWidth(modal, content, side, width) { + const key = _dockWidthStorageKey(modal, content, side); + if (!key) return; + try { localStorage.setItem(key, String(Math.round(width))); } catch (_) {} +} + +function _minEdgeDockWidth() { + return window.innerWidth < 900 ? 280 : MIN_EDGE_DOCK_WIDTH; +} + +function _activeDockWidth(side) { + if (side !== 'left' && side !== 'right') return 0; + const cls = side === 'left' ? 'left-dock-active' : 'right-dock-active'; + if (!document.body.classList.contains(cls)) return 0; + const prop = side === 'left' ? '--left-dock-w' : '--right-dock-w'; + const raw = getComputedStyle(document.documentElement).getPropertyValue(prop); + const n = parseFloat(raw || ''); + return Number.isFinite(n) && n > 0 ? n : 0; +} + +function _clampDockWidthToSpace(width, min, max) { + const floor = Math.min(min, Math.max(220, Math.round(max))); + const ceiling = Math.max(floor, Math.round(max)); + return Math.min(ceiling, Math.max(floor, Math.round(width))); +} + +function _clampRightDockWidth(width) { + const min = _minEdgeDockWidth(); + const navRight = _leftNavRight(); + const leftDockW = _activeDockWidth('left'); + const maxByChat = window.innerWidth - navRight - leftDockW - MIN_CHAT_WIDTH; + const max = Math.min(Math.round(window.innerWidth * 0.82), maxByChat); + return _clampDockWidthToSpace(width, min, max); +} + +function _clampLeftDockWidth(width, left = _leftNavRight()) { + const min = _minEdgeDockWidth(); + const rightDockW = _activeDockWidth('right'); + const available = Math.max(0, window.innerWidth - left - rightDockW); + const max = Math.min(Math.round(available * 0.82), available - MIN_CHAT_WIDTH); + return _clampDockWidthToSpace(width, min, max); +} + +function _resolveRightDockWidth(modal, content) { + return _clampRightDockWidth(content?._userDockWidth || _storedDockWidth(modal, content, 'right') || _defaultDockWidth()); +} + +function _resolveLeftDockWidth(content, left = _leftNavRight()) { + return _clampLeftDockWidth(content?._userDockWidth || _storedDockWidth(content?._dockOwner, content, 'left') || _resolveEmailDocSplitWidth(content, left), left); +} + +function _isEmailDockOwner(owner) { + const id = owner?.id || ''; + return id === 'email-lib-modal' || id.startsWith('email-reader-') || owner?.classList?.contains('email-window-modal'); +} + function _showSnapHint(on, side = 'right') { const cls = side === 'left' ? 'modal-snap-hint-left' : 'modal-snap-hint-right'; let hint = document.querySelector('.' + cls); @@ -85,7 +166,7 @@ function _shouldAutoCollapseSidebar(dockW) { const rl = (rail && window.getComputedStyle(rail).display !== 'none') ? rail.getBoundingClientRect().width : 0; - const remaining = window.innerWidth - sb - rl - dockW; + const remaining = window.innerWidth - sb - rl - _activeDockWidth('left') - dockW; return remaining < MIN_CHAT_WIDTH; } @@ -154,7 +235,7 @@ function _applyEmailDocSplitGeometry(left, emailWidth) { if (!docPane || window.innerWidth <= 768) return; docPane.style.setProperty('position', 'fixed', 'important'); docPane.style.setProperty('left', `${x}px`, 'important'); - docPane.style.setProperty('right', '0px', 'important'); + docPane.style.setProperty('right', 'var(--right-dock-w, 0px)', 'important'); docPane.style.setProperty('top', '0px', 'important'); docPane.style.setProperty('bottom', '0px', 'important'); docPane.style.setProperty('width', 'auto', 'important'); @@ -196,7 +277,9 @@ function _resolveEmailDocSplitWidth(content, left) { function _anchorLeftDock(content) { if (!content || content._dockSide !== 'left') return; const left = _leftNavRight(); - const w = _resolveEmailDocSplitWidth(content, left); + const w = document.body.classList.contains('doc-view') + ? _resolveEmailDocSplitWidth(content, left) + : _resolveLeftDockWidth(content, left); content.style.left = left + 'px'; content.style.width = w + 'px'; content.style.maxWidth = w + 'px'; @@ -205,14 +288,17 @@ function _anchorLeftDock(content) { // the doc-pane becomes position:fixed starting at the email's right edge. // No flex/max-width fighting; the doc just owns the right side from the // email's right edge to the viewport edge — they touch flush, no gap. - const docOpen = document.body.classList.contains('doc-view'); + const docOpen = document.body.classList.contains('doc-view') && _isEmailDockOwner(content._dockOwner); if (docOpen) { if (!document.body.classList.contains('email-doc-split-active')) { document.body.classList.add('email-doc-split-active'); } + document.documentElement.style.setProperty('--left-dock-w', '0px'); _applyEmailDocSplitGeometry(left, w); } else if (document.body.classList.contains('email-doc-split-active')) { _clearEmailDocSplitGeometry(); + } else { + document.documentElement.style.setProperty('--left-dock-w', w + 'px'); } } @@ -316,19 +402,21 @@ function _applyDockInternal(modal, side, dockClass) { content.style.margin = '0'; let w; if (side === 'left') { - // Email-style left dock: collapse the sidebar to the icon rail, then - // OVERLAY the window beside the rail, covering the chat area. We anchor - // at the rail's right edge (so it sits to the RIGHT of the rail — not - // left of the sidebar) and DON'T reserve body padding (so it covers the - // chat rather than pushing it), leaving the right side free for the doc. + // Left dock: collapse the sidebar to the icon rail, then pin the window + // beside the rail. Normal left docks reserve their width so chat shrinks; + // the email+document split keeps its existing overlay geometry. _collapseSidebarToRail(); content._preDockSnapshot.collapsedSidebar = true; content.style.right = 'auto'; content._dockSide = 'left'; + content._dockOwner = modal; _anchorLeftDock(content); w = parseFloat(content.style.width) || 0; document.body.classList.add('left-dock-active'); - document.documentElement.style.setProperty('--left-dock-w', '0px'); // overlay, no push + document.documentElement.style.setProperty( + '--left-dock-w', + document.body.classList.contains('email-doc-split-active') ? '0px' : w + 'px', + ); // Re-anchor the email when the sidebar is toggled (expanded/collapsed) so // the nav slides the window over instead of growing on top of it. Also // re-anchor when the document editor pane appears/disappears (signaled by @@ -406,7 +494,7 @@ function _applyDockInternal(modal, side, dockClass) { }; } } else { - w = _defaultDockWidth(); + w = _resolveRightDockWidth(modal, content); content.style.left = 'auto'; content.style.right = '0'; content.style.width = w + 'px'; @@ -419,6 +507,8 @@ function _applyDockInternal(modal, side, dockClass) { } } content._dockSide = side; + content._dockOwner = modal; + _positionEdgeDockResizeHandles(); // Watch for the docked modal disappearing (removed from DOM or hidden // via .hidden class) and clean up the body padding + sidebar in that // case. Without this, closing a docked window leaves a phantom strip @@ -498,7 +588,9 @@ function _onDockedModalGone(modal, dockClass) { } delete _c._preDockSnapshot; delete _c._dockSide; + delete _c._dockOwner; } + _positionEdgeDockResizeHandles(); } function _expandSidebarFromRail() { @@ -526,6 +618,7 @@ export function clearRightDock(modal, cx, cy, dockClass) { _clearEmailDocSplitGeometry(); } delete content._dockSide; + delete content._dockOwner; _disconnectLeftDockObservers(content); const snap = content._preDockSnapshot; // Re-expand the wide sidebar if we collapsed it — but only if the @@ -571,6 +664,7 @@ export function clearRightDock(modal, cx, cy, dockClass) { content.style.top = (typeof targetTop === 'number') ? targetTop + 'px' : targetTop; delete content._preDockSnapshot; delete content._dockSuspended; + _positionEdgeDockResizeHandles(); } // Temporarily release a docked modal's body push (chat returns to full @@ -604,6 +698,7 @@ export function suspendDock(modal) { modal.classList.remove('email-snap-left'); _clearEmailDocSplitGeometry(); delete content._dockSide; + delete content._dockOwner; delete content._dockSuspended; return null; } @@ -614,6 +709,7 @@ export function suspendDock(modal) { _expandSidebarFromRail(); } content._dockSuspended = side; + _positionEdgeDockResizeHandles(); return side; } @@ -641,15 +737,11 @@ export function makeRightDockController(modal, dockClass = 'modal-right-docked') return makeEdgeDockController(modal, 'right', dockClass); } -// Read live rail+sidebar width — used as the LEFT "edge" for snap -// detection, since the visible left boundary the user can drag to is -// the nav, not x=0 (the rail covers 0..48 and the wide sidebar covers -// 0..~290 when open). +// Read the current visible left-nav edge for snap detection. Use measured +// geometry instead of CSS vars because the sidebar can auto-collapse during a +// dock operation while --sidebar-w is still settling. function _leftNavWidth() { - const rs = getComputedStyle(document.documentElement); - const rail = parseInt(rs.getPropertyValue('--icon-rail-w') || '48', 10) || 0; - const sb = parseInt(rs.getPropertyValue('--sidebar-w') || '0', 10) || 0; - return rail + sb; + return _leftNavRight(); } // Generic edge-snap controller. `side` is 'left' or 'right'. Same pattern @@ -692,6 +784,207 @@ export function makeEdgeDockController(modal, side = 'right', dockClass) { }; } +(function _initEdgeDockResizeHandles() { + if (typeof document === 'undefined') return; + if (!document.body) { + document.addEventListener('DOMContentLoaded', _initEdgeDockResizeHandles, { once: true }); + return; + } + + const handles = { + left: document.createElement('div'), + right: document.createElement('div'), + }; + const _setStyle = (el, prop, value) => { + if (el.style[prop] !== value) el.style[prop] = value; + }; + const _hideHandle = (handle) => _setStyle(handle, 'display', 'none'); + + for (const side of ['left', 'right']) { + const handle = handles[side]; + handle.className = `edge-dock-resize-handle edge-dock-resize-handle-${side}`; + handle.style.position = 'fixed'; + handle.style.top = '0'; + handle.style.bottom = '0'; + handle.style.width = '10px'; + handle.style.cursor = 'col-resize'; + handle.style.background = 'linear-gradient(to right, transparent 0 3px, color-mix(in srgb, var(--accent, var(--red)) 35%, transparent) 3px 7px, transparent 7px 10px)'; + handle.style.pointerEvents = 'auto'; + handle.style.touchAction = 'none'; + handle.style.display = 'none'; + handle.title = 'Drag to resize docked window'; + document.body.appendChild(handle); + } + + const _isUsableDockOwner = (owner) => { + if (!owner || !owner.isConnected) return false; + if (owner.classList?.contains('hidden')) return false; + if (owner.style?.display === 'none') return false; + const nodes = _resolveDockNodes(owner); + const content = nodes?.content; + if (!content || !content.isConnected) return false; + if (content.classList?.contains('hidden')) return false; + if (content.style?.display === 'none') return false; + const r = content.getBoundingClientRect(); + return r.width > 0 && r.height > 0; + }; + + const _activeDockOwner = (side) => { + const cls = _dockClassForSide(side); + const all = Array.from(document.querySelectorAll(`.${cls}`)); + for (const owner of all.reverse()) { + if (_isUsableDockOwner(owner)) return owner; + } + return null; + }; + + const _zIndexFor = (el, fallback = 250) => { + const raw = el ? window.getComputedStyle(el).zIndex : ''; + const n = parseInt(raw, 10); + return Number.isFinite(n) ? n : fallback; + }; + + const _hasVisibleFloatingModal = (owner) => { + const all = Array.from(document.querySelectorAll('.modal:not(.hidden):not(.modal-minimized)')); + return all.some((modal) => { + if (!modal || modal === owner) return false; + if (owner?.contains?.(modal) || modal.contains?.(owner)) return false; + if (modal.classList.contains('modal-left-docked') + || modal.classList.contains('modal-right-docked') + || modal.classList.contains('email-snap-left')) return false; + if (modal.style.display === 'none') return false; + const content = _resolveDockNodes(modal)?.content; + const r = content?.getBoundingClientRect?.(); + return !!r && r.width > 0 && r.height > 0; + }); + }; + + const _setWidth = (owner, side, clientX) => { + const nodes = _resolveDockNodes(owner); + const content = nodes?.content; + if (!content) return 0; + let w = 0; + if (side === 'right') { + w = _clampRightDockWidth(window.innerWidth - clientX); + content._userDockWidth = w; + content.style.left = 'auto'; + content.style.right = '0'; + content.style.width = w + 'px'; + content.style.maxWidth = w + 'px'; + document.body.classList.add('right-dock-active'); + document.documentElement.style.setProperty('--right-dock-w', w + 'px'); + if (_shouldAutoCollapseSidebar(w)) { + _collapseSidebarToRail(); + if (content._preDockSnapshot) content._preDockSnapshot.collapsedSidebar = true; + } + } else { + const left = _leftNavRight(); + w = _clampLeftDockWidth(clientX - left, left); + content._userDockWidth = w; + content._emailDocSplitUserW = w; + content.style.left = left + 'px'; + content.style.right = 'auto'; + content.style.width = w + 'px'; + content.style.maxWidth = w + 'px'; + document.body.classList.add('left-dock-active'); + document.documentElement.style.setProperty( + '--left-dock-w', + document.body.classList.contains('email-doc-split-active') ? '0px' : w + 'px', + ); + } + _positionEdgeDockResizeHandles(); + return w; + }; + + _edgeDockHandlePositioner = () => { + const splitOwnsLeftSeam = document.body.classList.contains('email-doc-split-active') + && document.body.classList.contains('doc-view') + && window.innerWidth > 768; + for (const side of ['left', 'right']) { + const handle = handles[side]; + if (window.innerWidth <= 768 || (side === 'left' && splitOwnsLeftSeam)) { + _hideHandle(handle); + continue; + } + const owner = _activeDockOwner(side); + const content = owner && _resolveDockNodes(owner)?.content; + if (!content) { + _hideHandle(handle); + continue; + } + if (_hasVisibleFloatingModal(owner)) { + _hideHandle(handle); + continue; + } + const r = content.getBoundingClientRect(); + const x = side === 'right' ? r.left : r.right; + if (!Number.isFinite(x) || x <= 0 || x >= window.innerWidth) { + _hideHandle(handle); + continue; + } + _setStyle(handle, 'display', 'block'); + _setStyle(handle, 'left', (x - 5) + 'px'); + _setStyle(handle, 'zIndex', String(_zIndexFor(owner) + 1)); + } + }; + + for (const side of ['left', 'right']) { + const handle = handles[side]; + handle.addEventListener('pointerdown', (e) => { + if (handle.style.display === 'none') return; + const owner = _activeDockOwner(side); + if (!owner) return; + e.preventDefault(); + e.stopPropagation(); + handle.setPointerCapture?.(e.pointerId); + const nodes = _resolveDockNodes(owner); + const content = nodes?.content; + const prevCursor = document.body.style.cursor; + const prevUserSelect = document.body.style.userSelect; + document.body.style.cursor = 'col-resize'; + document.body.style.userSelect = 'none'; + document.body.classList.add('edge-dock-resizing'); + _setWidth(owner, side, e.clientX); + const onMove = (ev) => { + ev.preventDefault(); + _setWidth(owner, side, ev.clientX); + }; + const onUp = (ev) => { + try { handle.releasePointerCapture?.(e.pointerId); } catch (_) {} + document.removeEventListener('pointermove', onMove, true); + document.removeEventListener('pointerup', onUp, true); + document.removeEventListener('pointercancel', onUp, true); + document.body.classList.remove('edge-dock-resizing'); + document.body.style.cursor = prevCursor; + document.body.style.userSelect = prevUserSelect; + const finalW = side === 'right' + ? parseFloat(document.documentElement.style.getPropertyValue('--right-dock-w')) || content?.getBoundingClientRect?.().width || 0 + : content?.getBoundingClientRect?.().width || 0; + if (finalW) _saveDockWidth(owner, content, side, finalW); + ev.preventDefault(); + }; + document.addEventListener('pointermove', onMove, true); + document.addEventListener('pointerup', onUp, true); + document.addEventListener('pointercancel', onUp, true); + }); + } + + new MutationObserver(_positionEdgeDockResizeHandles).observe(document.body, { attributes: true, attributeFilter: ['class'] }); + new MutationObserver(_positionEdgeDockResizeHandles).observe(document.documentElement, { attributes: true, attributeFilter: ['style'] }); + let raf = 0; + const schedulePosition = () => { + if (raf) return; + raf = requestAnimationFrame(() => { + raf = 0; + _positionEdgeDockResizeHandles(); + }); + }; + new MutationObserver(schedulePosition).observe(document.body, { childList: true }); + window.addEventListener('resize', _positionEdgeDockResizeHandles); + window.addEventListener('odysseus:modal-opened', _positionEdgeDockResizeHandles); + _positionEdgeDockResizeHandles(); +})(); + (function _initSplitSeamIndicator() { if (typeof document === 'undefined') return; const stripe = document.createElement('div'); diff --git a/static/js/settings.js b/static/js/settings.js index 068cd80..f9e7655 100644 --- a/static/js/settings.js +++ b/static/js/settings.js @@ -53,7 +53,7 @@ function initDrag() { content, header, skipSelector: 'button, input, select, .theme-opacity-wrap', - enableDock: false, + enableDock: true, }); } diff --git a/static/js/windowDrag.js b/static/js/windowDrag.js index 7c16a53..5e7cb0c 100644 --- a/static/js/windowDrag.js +++ b/static/js/windowDrag.js @@ -93,11 +93,11 @@ export function makeWindowDraggable(modal, options = {}) { } const rightDock = enableDock ? makeEdgeDockController(modal, 'right') : null; - // Left dock is opt-in (enableLeftDock). For most windows it's off — the - // sidebar lives on the left, so a left dock collides with it. The email - // window enables it so you can park the message on the left and read it - // while replying in the document on the right. - const leftDock = (enableDock && options.enableLeftDock) ? makeEdgeDockController(modal, 'left') : null; + // Left dock is enabled by default too. modalSnap collapses the wide sidebar + // and anchors the panel beside the icon rail, so it no longer collides with + // the navigation. Callers can still pass enableLeftDock:false for a special + // modal that should only dock right. + const leftDock = (enableDock && options.enableLeftDock !== false) ? makeEdgeDockController(modal, 'left') : null; // Per-drag state, reset on mousedown. let dragging = false; diff --git a/static/style.css b/static/style.css index 2c79b51..c7a2163 100644 --- a/static/style.css +++ b/static/style.css @@ -97,9 +97,9 @@ html, body { overflow-x: hidden; height: 100%; margin: 0; overscroll-behavior: n body { background-color: var(--bg); color: var(--fg); - /* Animate the dock push BOTH ways. Keeping the transition on the base body - (not on .right/left-dock-active) means removing the class on undock also - animates padding back to 0 — otherwise the chat snapped back instantly. */ + /* Keep the base padding transition for older layout paths that still adjust + the body directly. Edge docks reserve workspace room on the flex panes + below so left + right docks can coexist without skewing the whole body. */ transition: padding-left 160ms cubic-bezier(0.22, 0.61, 0.36, 1), padding-right 160ms cubic-bezier(0.22, 0.61, 0.36, 1); font-family: var(--font-family, 'Fira Code', monospace); @@ -1773,6 +1773,8 @@ body.bg-pattern-sparkles { min-width:0; margin-top:8px; margin-bottom: 0; + transition: margin-left 160ms cubic-bezier(0.22, 0.61, 0.36, 1), + margin-right 160ms cubic-bezier(0.22, 0.61, 0.36, 1); } .chat-meta { font-size:12px; color:color-mix(in srgb, var(--fg) 60%, transparent); margin-bottom:6px; } .chat-history { @@ -4939,6 +4941,15 @@ body.bg-pattern-sparkles { pointer-events:auto; animation: modal-enter 0.25s ease-out both; } + .memory-modal-content, + .tasks-modal-content, + .preset-modal-content, + #cookbook-modal .modal-content, + #theme-popup, + .doclib-modal-content, + .gallery-modal-content { + container-type: inline-size; + } .modal-header { display:flex; justify-content:space-between; align-items:center; margin-bottom:6px; cursor:grab; user-select:none; @@ -14843,7 +14854,7 @@ body:has(.doc-version-panel:not(.hidden)) .hamburger-btn { body.email-doc-split-active.doc-view .doc-editor-pane { position: fixed !important; left: var(--email-doc-split-right-x, 420px) !important; - right: 0 !important; + right: var(--right-dock-w, 0px) !important; top: 0 !important; bottom: 0 !important; width: auto !important; @@ -14864,15 +14875,21 @@ body [data-act="from-sender"] { display: none !important; } -/* Snap-to-right docking. A modal dragged to the right edge becomes a - docked side panel (mirrors Notes/Doc panels). Body reserves space via - padding-right so the chat / notes / doc panel underneath shrinks to - fit instead of being hidden behind the panel. */ +/* Edge docking. Docked panels are fixed to the viewport edge; the workspace + panes reserve room with margins so left + right docks can be active at the + same time without skewing the entire body box. */ body.right-dock-active { - padding-right: var(--right-dock-w, 0px); + padding-right: 0; } body.left-dock-active { - padding-left: var(--left-dock-w, 0px); + padding-left: 0; +} +body.left-dock-active:not(.email-doc-split-active) .chat-container { + margin-left: var(--left-dock-w, 0px); +} +body.right-dock-active .chat-container, +body.right-dock-active:not(.email-doc-split-active) .doc-editor-pane { + margin-right: var(--right-dock-w, 0px); } .modal.modal-right-docked { align-items: stretch; @@ -23192,6 +23209,89 @@ input.settings-select::placeholder { color: color-mix(in srgb, var(--fg) 35%, tr opacity: 1; border-bottom-color: var(--red); } + +/* Narrow modal tab strips should stay on one row. Resized docked windows can + be much narrower than the viewport, so this cannot live only in mobile media + queries. */ +.cookbook-tabs, +.memory-tabs, +.admin-tabs, +.lib-tabs, +.gallery-tabs, +.preset-tabs { + flex-wrap: nowrap !important; + overflow-x: auto !important; + overflow-y: hidden; + -webkit-overflow-scrolling: touch; + overscroll-behavior-x: contain; + scrollbar-width: none; +} +.cookbook-tabs::-webkit-scrollbar, +.memory-tabs::-webkit-scrollbar, +.admin-tabs::-webkit-scrollbar, +.lib-tabs::-webkit-scrollbar, +.gallery-tabs::-webkit-scrollbar, +.preset-tabs::-webkit-scrollbar { + display: none; +} +.cookbook-tabs > *, +.memory-tabs > *, +.admin-tabs > *, +.lib-tabs > *, +.gallery-tabs > *, +.preset-tabs > * { + flex: 0 0 auto; +} +.cookbook-tab, +.memory-tab, +.admin-tab, +.lib-tab, +.gallery-tab, +.preset-tab { + display: inline-flex; + align-items: center; + justify-content: center; + white-space: nowrap; + line-height: 1; +} +.gallery-tab { + gap: 6px; +} + +@container (max-width: 360px) { + .cookbook-tab:has(svg), + .memory-tab:has(svg), + .admin-tab:has(svg), + .lib-tab:has(svg), + .gallery-tab:has(svg), + .preset-tab:has(svg) { + width: 34px; + min-width: 34px; + padding-left: 0; + padding-right: 0; + font-size: 0; + } + + .cookbook-tab:has(svg) svg, + .memory-tab:has(svg) svg, + .admin-tab:has(svg) svg, + .lib-tab:has(svg) svg, + .gallery-tab:has(svg) svg, + .preset-tab:has(svg) svg { + width: 14px; + height: 14px; + margin-right: 0 !important; + vertical-align: middle !important; + } + + .memory-tab:has(svg) .memory-count, + .gallery-tab:has(svg) .gallery-tab-label, + .gallery-tab:has(svg) .gallery-tab-close, + .cookbook-tab:has(svg) .cookbook-tab-count, + .preset-tab:has(svg) .preset-count { + display: none !important; + } +} /* Icon + label layout inside each tab. */ .gallery-tab { display: inline-flex; From b448119919c953abc7484694307053844bd34615 Mon Sep 17 00:00:00 2001 From: Giulio Zelante Date: Fri, 5 Jun 2026 19:48:23 +0200 Subject: [PATCH 030/974] feat(skills): import SKILL.md bundles from public GitHub URLs (#2576) * feat(skills): import SKILL.md bundles from public GitHub URLs Supports GitHub tree/blob/raw links and skills.sh pages that resolve to GitHub. Installs SKILL.md plus sibling text assets under data/skills/imported/. Co-authored-by: Cursor * fix(skills): admin-gate URL import and validate redirect hosts - require_admin on POST /api/skills/import-from-url (matches other skill admin routes) - reject cross-host redirects after httpx follow_redirects - test for redirect host validation Co-authored-by: Cursor * fix(skills): match Brain Add panel import/submit button styles - Skill URL Import: theme-io-btn + download icon (same as memory Import) - Add Skill submit: confirm-btn confirm-btn-primary Co-authored-by: Cursor * fix(skills): allow api.github.com during directory import Real imports hit the GitHub contents API after redirects; whitelist api.github.com and add regression tests. Shrink Import button with flex:none. Co-authored-by: Cursor * fix(skills): align skill Import button with URL input row Match memory-add-input height (28px) in memory-add-row and center the download icon with flexbox instead of vertical-align hacks. Co-authored-by: Cursor * fix(skills): cancel modal-body margin on skill Import button The skill Import button sits in .memory-add-row beside an input; the global .modal-body button { margin-top: 6px } rule only affected buttons, pushing Import down and misaligning the download icon. Reset margin-top and match Memory Import SVG markup at 28px row height. Co-authored-by: Cursor * fix(skills): surface GitHub API errors on URL import Pass through GitHub response messages (especially 403 rate limits) as SkillImportError instead of a generic download failure. Co-authored-by: Cursor --------- Co-authored-by: Cursor --- routes/skills_routes.py | 36 ++++ services/memory/skill_importer.py | 283 ++++++++++++++++++++++++++++++ services/memory/skills.py | 48 +++++ static/index.html | 12 +- static/js/skills.js | 33 ++++ static/style.css | 9 + tests/test_skill_importer.py | 178 +++++++++++++++++++ 7 files changed, 597 insertions(+), 2 deletions(-) create mode 100644 services/memory/skill_importer.py create mode 100644 tests/test_skill_importer.py diff --git a/routes/skills_routes.py b/routes/skills_routes.py index 6894a13..705502e 100644 --- a/routes/skills_routes.py +++ b/routes/skills_routes.py @@ -11,6 +11,8 @@ import logging import re from typing import List, Optional +import httpx + from fastapi import APIRouter, HTTPException, Request from pydantic import BaseModel, Field @@ -51,6 +53,10 @@ class SkillAddRequest(BaseModel): steps: List[str] = Field(default_factory=list) +class SkillImportUrlRequest(BaseModel): + url: str = Field(..., min_length=8, max_length=2000) + + class SkillUpdateRequest(BaseModel): name: Optional[str] = None description: Optional[str] = None @@ -1203,6 +1209,36 @@ def setup_skills_routes(skills_manager: SkillsManager) -> APIRouter: save_settings(settings) return {"ok": True, "name": name, "is_overridden": False} + @router.post("/import-from-url") + async def import_skill_from_url(request: Request, body: SkillImportUrlRequest): + """Install a SKILL.md bundle from a public GitHub URL (skills.sh links supported).""" + require_admin(request) + user = _owner(request) + from services.memory.skill_importer import ( + SkillImportError, + fetch_skill_bundle, + ) + + try: + files, _src = fetch_skill_bundle(body.url.strip()) + entry = skills_manager.import_bundle_from_files( + files, + owner=user, + source_url=body.url.strip(), + ) + except SkillImportError as e: + raise HTTPException(400, str(e)) from e + except httpx.HTTPError as e: + logger.warning("skill import fetch failed: %s", e) + detail = str(e).strip() or "Could not download skill from URL" + raise HTTPException(502, detail) from e + except Exception as e: + logger.error("skill import failed: %s", e) + raise HTTPException(500, "Skill import failed") from e + + _fire_skill_added(user) + return {"ok": True, "skill": entry, "files": len(files)} + @router.post("/add") async def add_skill(request: Request, body: SkillAddRequest): user = _owner(request) diff --git a/services/memory/skill_importer.py b/services/memory/skill_importer.py new file mode 100644 index 0000000..65f4b21 --- /dev/null +++ b/services/memory/skill_importer.py @@ -0,0 +1,283 @@ +"""Import SKILL.md bundles from public GitHub (or skills.sh → GitHub) URLs.""" +from __future__ import annotations + +import logging +import os +import re +from dataclasses import dataclass +from typing import Dict, List, Optional, Tuple +from urllib.parse import quote, urlparse + +import httpx + +from src.url_safety import check_outbound_url + +logger = logging.getLogger(__name__) + +MAX_FILES = 64 +MAX_TOTAL_BYTES = 2_000_000 +MAX_FILE_BYTES = 400_000 +ALLOWED_SUFFIXES = ( + ".md", ".txt", ".json", ".yaml", ".yml", ".py", ".sh", ".toml", + ".js", ".ts", ".css", ".html", ".xml", ".csv", +) +TEXT_NAMES = {"skill.md", "license", "license.md", "readme.md"} +_GITHUB_HOSTS = frozenset({ + "github.com", "www.github.com", "api.github.com", "raw.githubusercontent.com", +}) + + +def _github_host(url: str) -> str: + return (urlparse(str(url)).hostname or "").lower() + + +def _assert_github_url(url: str, *, context: str = "URL") -> None: + host = _github_host(url) + if host not in _GITHUB_HOSTS: + raise SkillImportError( + f"{context} must stay on GitHub (got {host or 'unknown host'})" + ) + + +@dataclass +class ResolvedSource: + owner: str + repo: str + ref: str + path: str # directory or file path inside repo (no leading slash) + + +class SkillImportError(ValueError): + pass + + +def _safe_relpath(rel: str) -> str: + rel = (rel or "").replace("\\", "/").strip().lstrip("/") + if not rel or rel.startswith("..") or "/../" in f"/{rel}/": + raise SkillImportError(f"unsafe path: {rel!r}") + parts = [p for p in rel.split("/") if p and p != "."] + if any(p == ".." for p in parts): + raise SkillImportError(f"unsafe path: {rel!r}") + return "/".join(parts) + + +def _is_text_file(name: str) -> bool: + low = name.lower() + if low in TEXT_NAMES: + return True + return any(low.endswith(s) for s in ALLOWED_SUFFIXES) + + +def parse_skill_source(url: str) -> ResolvedSource: + """Normalize skills.sh / GitHub web URLs into owner/repo/ref/path.""" + raw = (url or "").strip() + if not raw: + raise SkillImportError("URL is required") + + # skills.sh often links to GitHub; try to unwrap ?url= or redirect target later. + if "skills.sh" in raw and "github.com" not in raw: + ok, reason = check_outbound_url(raw) + if not ok: + raise SkillImportError(reason) + with httpx.Client(follow_redirects=True, timeout=20.0) as client: + r = client.get(raw) + if r.status_code >= 400: + raise _github_response_error(r) + final = str(r.url) + _assert_github_url(final, context="redirect target") + # Page may embed a github link; prefer final URL if redirected. + if "github.com" in final: + raw = final + else: + m = re.search(r"https?://github\.com/[^\s\"')]+", r.text or "") + if m: + raw = m.group(0).rstrip(".,)") + + parsed = urlparse(raw) + host = _github_host(raw) + if host not in _GITHUB_HOSTS: + raise SkillImportError( + "Only GitHub URLs are supported (https://github.com/... or raw.githubusercontent.com/...)" + ) + + if host == "raw.githubusercontent.com": + # /owner/repo/ref/path/to/file + bits = [p for p in parsed.path.split("/") if p] + if len(bits) < 4: + raise SkillImportError("Invalid raw GitHub URL") + owner, repo, ref = bits[0], bits[1], bits[2] + path = "/".join(bits[3:]) + return ResolvedSource(owner=owner, repo=repo, ref=ref, path=path) + + bits = [p for p in parsed.path.split("/") if p] + if len(bits) < 2: + raise SkillImportError("Invalid GitHub URL") + owner, repo = bits[0], bits[1] + ref = "main" + path = "" + + if len(bits) >= 4 and bits[2] in ("tree", "blob"): + ref = bits[3] + path = "/".join(bits[4:]) + elif len(bits) == 2: + path = "" + else: + raise SkillImportError("GitHub URL must include /tree//... or /blob//...") + + return ResolvedSource(owner=owner, repo=repo, ref=ref, path=path) + + +def _raw_url(src: ResolvedSource, rel_path: str) -> str: + rel = _safe_relpath(rel_path) + return f"https://raw.githubusercontent.com/{src.owner}/{src.repo}/{quote(src.ref, safe='')}/{quote(rel, safe='/')}" + + +def _api_contents_url(src: ResolvedSource, rel_path: str = "") -> str: + rel = _safe_relpath(rel_path) if rel_path else "" + base = f"https://api.github.com/repos/{src.owner}/{src.repo}/contents" + if rel: + base += f"/{quote(rel, safe='/')}" + return f"{base}?ref={quote(src.ref, safe='')}" + + +def _github_response_error(response: httpx.Response) -> SkillImportError: + """Turn a failed GitHub HTTP response into a user-visible import error.""" + status = response.status_code + detail = "" + try: + body = response.json() + if isinstance(body, dict): + detail = str(body.get("message") or "").strip() + except Exception: + detail = (response.text or "").strip()[:200] + + low = detail.lower() + if status == 403 and "rate limit" in low: + return SkillImportError( + "GitHub API rate limit exceeded — try again in a bit" + + (f" ({detail})" if detail else "") + ) + if status == 404: + return SkillImportError("path not found on GitHub") + if detail: + return SkillImportError(f"GitHub request failed ({status}): {detail}") + return SkillImportError(f"GitHub request failed ({status})") + + +def _fetch_bytes(url: str) -> bytes: + ok, reason = check_outbound_url(url) + if not ok: + raise SkillImportError(reason) + with httpx.Client(follow_redirects=True, timeout=30.0) as client: + r = client.get(url, headers={"Accept": "application/vnd.github+json"}) + if r.status_code >= 400: + raise _github_response_error(r) + _assert_github_url(str(r.url), context="redirect target") + if len(r.content) > MAX_FILE_BYTES: + raise SkillImportError(f"file too large: {url}") + return r.content + + +def _fetch_text(url: str) -> str: + data = _fetch_bytes(url) + try: + return data.decode("utf-8") + except UnicodeDecodeError as e: + raise SkillImportError(f"non-text file: {url}") from e + + +def _list_github_dir(src: ResolvedSource, rel_dir: str, out: Dict[str, str], *, depth: int = 0) -> None: + if depth > 4 or len(out) >= MAX_FILES: + return + url = _api_contents_url(src, rel_dir) + ok, reason = check_outbound_url(url) + if not ok: + raise SkillImportError(reason) + with httpx.Client(follow_redirects=True, timeout=30.0) as client: + r = client.get(url, headers={"Accept": "application/vnd.github+json"}) + if r.status_code >= 400: + raise _github_response_error(r) + _assert_github_url(str(r.url), context="redirect target") + entries = r.json() + if not isinstance(entries, list): + raise SkillImportError("expected a directory on GitHub") + total = sum(len(v.encode("utf-8")) for v in out.values()) + for ent in entries: + if len(out) >= MAX_FILES or total >= MAX_TOTAL_BYTES: + break + if not isinstance(ent, dict): + continue + name = ent.get("name") or "" + ent_type = ent.get("type") + rel = _safe_relpath(f"{rel_dir}/{name}" if rel_dir else name) + if ent_type == "dir": + _list_github_dir(src, rel, out, depth=depth + 1) + total = sum(len(v.encode("utf-8")) for v in out.values()) + continue + if ent_type != "file" or not _is_text_file(name): + continue + dl = ent.get("download_url") + if not dl: + continue + _assert_github_url(dl, context="download URL") + text = _fetch_text(dl) + total += len(text.encode("utf-8")) + if total > MAX_TOTAL_BYTES: + raise SkillImportError("skill bundle exceeds size limit") + out[rel] = text + + +def fetch_skill_bundle(url: str) -> Tuple[Dict[str, str], ResolvedSource]: + """Download SKILL.md and sibling text assets. Returns relative_path → content.""" + src = parse_skill_source(url) + files: Dict[str, str] = {} + + path = _safe_relpath(src.path) if src.path else "" + if path.lower().endswith("skill.md"): + files[path] = _fetch_text(_raw_url(src, path)) + parent = "/".join(path.split("/")[:-1]) + if parent: + try: + _list_github_dir(src, parent, files) + except SkillImportError: + pass + return files, src + + if path: + try: + _fetch_text(_raw_url(src, f"{path}/SKILL.md")) + _list_github_dir(src, path, files) + return files, src + except Exception: + pass + try: + text = _fetch_text(_raw_url(src, path)) + if path.lower().endswith(".md"): + files[path] = text + return files, src + except Exception: + pass + _list_github_dir(src, path, files) + else: + _list_github_dir(src, "", files) + + if not any(p.lower().endswith("skill.md") for p in files): + # Flat repo root with SKILL.md only + try: + files["SKILL.md"] = _fetch_text(_raw_url(src, "SKILL.md")) + except Exception as e: + raise SkillImportError( + "No SKILL.md found — link to a skill folder or SKILL.md on GitHub" + ) from e + return files, src + + +def pick_skill_md(files: Dict[str, str]) -> Tuple[str, str]: + for rel, content in files.items(): + if rel.lower().endswith("skill.md"): + return rel, content + raise SkillImportError("bundle has no SKILL.md") + + +def default_category_from_source(src: ResolvedSource) -> str: + return "imported" diff --git a/services/memory/skills.py b/services/memory/skills.py index 87f74d5..9cfe801 100644 --- a/services/memory/skills.py +++ b/services/memory/skills.py @@ -381,6 +381,54 @@ class SkillsManager: return sk.to_dict() + def import_bundle_from_files( + self, + files: Dict[str, str], + *, + owner: Optional[str] = None, + source_url: str = "", + category: str = "imported", + ) -> Dict: + """Install a fetched skill bundle (relative path → text) under skills/.""" + from .skill_importer import SkillImportError, pick_skill_md, _safe_relpath + from core.atomic_io import atomic_write_text + + if not files: + raise SkillImportError("empty bundle") + _rel, skill_md = pick_skill_md(files) + sk = Skill.from_markdown(skill_md) + nm = slugify(sk.name or _rel.split("/")[-2] or "skill") + cat = slugify(category or sk.category or "imported", fallback="imported") + + existing = {s["name"] for s in self.load_all()} + base = nm + i = 2 + while nm in existing: + nm = f"{base}-{i}" + i += 1 + + skill_dir = self._skill_dir(cat, nm) + os.makedirs(skill_dir, exist_ok=True) + + # Preserve bundle layout (templates/, references/, etc.) under the skill dir. + for rel, content in files.items(): + safe = _safe_relpath(rel) + dest = os.path.join(skill_dir, safe) + os.makedirs(os.path.dirname(dest), exist_ok=True) + atomic_write_text(dest, content) + + sk.name = nm + sk.category = cat + sk.owner = owner + sk.source = "imported" + if source_url: + extra = (sk.body_extra or "").strip() + note = f"Imported from {source_url}" + sk.body_extra = f"{extra}\n\n{note}".strip() if extra else note + atomic_write_text(self._skill_file(cat, nm), sk.to_markdown()) + sk.path = self._skill_file(cat, nm) + return sk.to_dict() + def update_skill(self, skill_id: str, updates: Dict, owner: Optional[str] = None) -> bool: """`skill_id` is the slug name. Allows updating any field plus renames if `name` changes (file is moved on disk). diff --git a/static/index.html b/static/index.html index 22cdfda..3d5bad5 100644 --- a/static/index.html +++ b/static/index.html @@ -314,7 +314,15 @@

Add Skill

-

Create a skill by hand — title, what it solves, and an approach.

+

Import a skill from GitHub or skills.sh (folder with SKILL.md and optional templates).

+
+
+ + Import URL — e.g. GitHub tree link to a skill folder +
+ +
+

Or create a skill by hand — title, what it solves, and an approach.

Title — short name, e.g. “build-vllm-wheel” @@ -332,7 +340,7 @@ Tags — comma-separated, e.g. python, build, vllm
- +
diff --git a/static/js/skills.js b/static/js/skills.js index afb7475..f9c522a 100644 --- a/static/js/skills.js +++ b/static/js/skills.js @@ -1818,6 +1818,35 @@ async function _showSkillSource(name) { }); } +async function importSkillFromUrl() { + const input = document.getElementById('skill-import-url'); + const url = (input?.value || '').trim(); + if (!url) { + uiModule.showError('Paste a GitHub or skills.sh URL first'); + return; + } + const btn = document.getElementById('skill-import-url-btn'); + if (btn) btn.disabled = true; + try { + const res = await fetch(`${API}/api/skills/import-from-url`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ url }), + }); + const data = await res.json().catch(() => ({})); + if (!res.ok) throw new Error(data.detail || data.error || `HTTP ${res.status}`); + if (input) input.value = ''; + await loadSkills(); + const name = data.skill?.name || 'skill'; + uiModule.showToast(`Imported ${name} (${data.files || 1} file(s))`); + if (name) openSkill(name); + } catch (err) { + uiModule.showError('Import failed: ' + err.message); + } finally { + if (btn) btn.disabled = false; + } +} + async function addSkill() { const name = document.getElementById('new-skill-name')?.value.trim() || document.getElementById('new-skill-title')?.value.trim(); @@ -1866,6 +1895,10 @@ async function addSkill() { } document.addEventListener('DOMContentLoaded', () => { + document.getElementById('skill-import-url-btn')?.addEventListener('click', importSkillFromUrl); + document.getElementById('skill-import-url')?.addEventListener('keydown', (e) => { + if (e.key === 'Enter') importSkillFromUrl(); + }); document.getElementById('add-skill-btn')?.addEventListener('click', addSkill); document.getElementById('skills-search')?.addEventListener('input', renderSkillsList); document.getElementById('skills-sort')?.addEventListener('change', (e) => { diff --git a/static/style.css b/static/style.css index c7a2163..60d2d47 100644 --- a/static/style.css +++ b/static/style.css @@ -10126,6 +10126,15 @@ details a:hover { height: 32px; } +/* Skill Import beside URL field — match input height; cancel modal-body button margin. */ +.memory-add-row .theme-io-btn { + flex: none; + height: 28px; + box-sizing: border-box; + margin-top: 0; + padding: 5px 10px; +} + .memory-add-input { flex: 1; height: 28px; diff --git a/tests/test_skill_importer.py b/tests/test_skill_importer.py new file mode 100644 index 0000000..eecca61 --- /dev/null +++ b/tests/test_skill_importer.py @@ -0,0 +1,178 @@ +"""Skill URL importer — GitHub path parsing.""" +import pytest + +from services.memory.skill_importer import ( + ResolvedSource, + SkillImportError, + _assert_github_url, + _fetch_bytes, + _list_github_dir, + parse_skill_source, +) + + +def test_parse_github_blob_skill_md(): + src = parse_skill_source( + "https://github.com/anthropics/skills/blob/main/skills/pdf/SKILL.md" + ) + assert src.owner == "anthropics" + assert src.repo == "skills" + assert src.ref == "main" + assert src.path.endswith("skills/pdf/SKILL.md") + + +def test_parse_github_tree_directory(): + src = parse_skill_source( + "https://github.com/example/my-skills/tree/develop/caveman-skill" + ) + assert src.owner == "example" + assert src.repo == "my-skills" + assert src.ref == "develop" + assert src.path == "caveman-skill" + + +def test_parse_raw_github(): + src = parse_skill_source( + "https://raw.githubusercontent.com/o/r/main/path/SKILL.md" + ) + assert src.owner == "o" + assert src.repo == "r" + assert src.ref == "main" + assert src.path == "path/SKILL.md" + + +def test_rejects_non_github(): + with pytest.raises(SkillImportError): + parse_skill_source("https://example.com/skill.md") + + +def test_fetch_bytes_rejects_cross_host_redirect(monkeypatch): + class _Resp: + url = "https://evil.example/secret" + status_code = 200 + content = b"x" + + def raise_for_status(self): + return None + + class _Client: + def __init__(self, *args, **kwargs): + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def get(self, url, headers=None): + return _Resp() + + monkeypatch.setattr("services.memory.skill_importer.httpx.Client", _Client) + monkeypatch.setattr( + "services.memory.skill_importer.check_outbound_url", + lambda url: (True, ""), + ) + with pytest.raises(SkillImportError, match="redirect target"): + _fetch_bytes("https://raw.githubusercontent.com/o/r/main/SKILL.md") + + +def test_assert_github_url_allows_api_host(): + _assert_github_url( + "https://api.github.com/repos/o/r/contents?ref=main", + context="redirect target", + ) + + +def test_list_github_dir_accepts_api_github_response(monkeypatch): + monkeypatch.setattr( + "services.memory.skill_importer._fetch_text", + lambda url: "# skill\n", + ) + monkeypatch.setattr( + "services.memory.skill_importer.check_outbound_url", + lambda url: (True, ""), + ) + + class _Resp: + url = "https://api.github.com/repos/o/r/contents?ref=main" + status_code = 200 + + def raise_for_status(self): + return None + + def json(self): + return [{ + "name": "SKILL.md", + "type": "file", + "download_url": "https://raw.githubusercontent.com/o/r/main/SKILL.md", + }] + + class _Client: + def __init__(self, *args, **kwargs): + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def get(self, url, headers=None): + return _Resp() + + monkeypatch.setattr("services.memory.skill_importer.httpx.Client", _Client) + + out = {} + src = ResolvedSource(owner="o", repo="r", ref="main", path="") + _list_github_dir(src, "", out) + assert "SKILL.md" in out + + +def _mock_httpx_client(monkeypatch, response): + class _Client: + def __init__(self, *args, **kwargs): + pass + + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def get(self, url, headers=None): + return response + + monkeypatch.setattr("services.memory.skill_importer.httpx.Client", _Client) + monkeypatch.setattr( + "services.memory.skill_importer.check_outbound_url", + lambda url: (True, ""), + ) + + +def test_list_github_dir_surfaces_rate_limit(monkeypatch): + class _Resp: + url = "https://api.github.com/repos/o/r/contents?ref=main" + status_code = 403 + + def json(self): + return {"message": "API rate limit exceeded for 203.0.113.1"} + + _mock_httpx_client(monkeypatch, _Resp()) + src = ResolvedSource(owner="o", repo="r", ref="main", path="") + with pytest.raises(SkillImportError, match="rate limit"): + _list_github_dir(src, "", {}) + + +def test_fetch_bytes_surfaces_github_error_detail(monkeypatch): + class _Resp: + url = "https://raw.githubusercontent.com/o/r/main/SKILL.md" + status_code = 403 + content = b"" + + def json(self): + return {"message": "Forbidden"} + + _mock_httpx_client(monkeypatch, _Resp()) + with pytest.raises(SkillImportError, match="GitHub request failed \\(403\\): Forbidden"): + _fetch_bytes("https://raw.githubusercontent.com/o/r/main/SKILL.md") From fa9f62b44c8978af8530d1eb5998cea4dbef5754 Mon Sep 17 00:00:00 2001 From: nubs Date: Fri, 5 Jun 2026 18:23:38 +0000 Subject: [PATCH 031/974] fix(compactor): shrink oversized tool_calls arguments so trim_for_context can fit a tool-only turn (#2949) --- src/context_compactor.py | 51 +++++++++++++-- tests/test_compact_truncate_tool_call_args.py | 62 +++++++++++++++++++ 2 files changed, 108 insertions(+), 5 deletions(-) create mode 100644 tests/test_compact_truncate_tool_call_args.py diff --git a/src/context_compactor.py b/src/context_compactor.py index c70ed0b..7da5242 100644 --- a/src/context_compactor.py +++ b/src/context_compactor.py @@ -5,6 +5,7 @@ Auto-compacts conversation history when approaching context window limits. Summarizes older messages via the same LLM, preserving key context. """ +import json import logging from typing import Any, Dict, List, Optional @@ -146,15 +147,53 @@ def _truncate_text_to_token_budget(text: str, token_budget: int) -> str: return text[:head_len].rstrip() + notice + "\n\n" + text[-tail_len:].lstrip() +def _truncate_tool_call_args(msg: Dict[str, Any], token_budget: int) -> Dict[str, Any]: + """Shrink oversized assistant ``tool_calls`` arguments to fit ``token_budget``. + + A tool-only turn persists ``content=None`` with its whole payload in + ``tool_calls[].function.arguments`` (e.g. a large create_document body), which + the text-content truncation can't reach — so the message could stay over + budget and the upstream call would 400. Replace each argument string that + overflows its share of the budget with a small valid-JSON placeholder, + preserving ``id``/``type``/``function.name`` so tool/result pairing and + provider validation are unaffected. Returns msg unchanged when there is + nothing oversized. + """ + tool_calls = msg.get("tool_calls") + if not isinstance(tool_calls, list) or not tool_calls: + return msg + # Budget left after whatever content survived (estimate_tokens counts tool + # arguments too, so measure content alone here). + content_tokens = estimate_tokens([{"role": msg.get("role", "assistant"), "content": msg.get("content")}]) + per_call = max(16, (max(0, token_budget - content_tokens)) // len(tool_calls)) + new_calls = [] + changed = False + for tc in tool_calls: + fn = tc.get("function") if isinstance(tc, dict) else None + args = fn.get("arguments") if isinstance(fn, dict) else None + if isinstance(args, str) and int(len(args) * 0.3) > per_call: + new_fn = dict(fn) + new_fn["arguments"] = json.dumps({"_truncated_for_context": len(args)}) + new_tc = dict(tc) + new_tc["function"] = new_fn + new_calls.append(new_tc) + changed = True + else: + new_calls.append(tc) + if not changed: + return msg + out = dict(msg) + out["tool_calls"] = new_calls + return out + + def _truncate_message_to_token_budget(msg: Dict[str, Any], token_budget: int) -> Dict[str, Any]: - """Return a copy of msg whose text content fits inside token_budget.""" + """Return a copy of msg whose text content (and tool-call args) fit token_budget.""" out = dict(msg) content = out.get("content", "") if isinstance(content, str): out["content"] = _truncate_text_to_token_budget(content, token_budget) - return out - - if isinstance(content, list): + elif isinstance(content, list): remaining = token_budget new_content = [] for item in content: @@ -168,7 +207,9 @@ def _truncate_message_to_token_budget(msg: Dict[str, Any], token_budget: int) -> new_content.append(cloned) remaining -= _message_text_token_estimate(truncated) out["content"] = new_content - return out + # A tool-only turn (content=None) carries its payload in tool_calls args, + # which the branches above can't shrink — handle it so the message can fit. + return _truncate_tool_call_args(out, token_budget) def trim_for_context(messages: List[Dict], context_length: int, reserve_tokens: int = 512) -> List[Dict]: diff --git a/tests/test_compact_truncate_tool_call_args.py b/tests/test_compact_truncate_tool_call_args.py new file mode 100644 index 0000000..cc081b9 --- /dev/null +++ b/tests/test_compact_truncate_tool_call_args.py @@ -0,0 +1,62 @@ +"""Issue #2947 — _truncate_message_to_token_budget must shrink oversized tool_calls +arguments, not just text content. + +A tool-only assistant turn persists content=None with its whole payload in +tool_calls[].function.arguments. The text-content truncation can't reach it, so +trim_for_context's last-resort message shrink left the message over budget and the +upstream call 400'd. This pins that oversized args are bounded (so the message +fits) while id/type/function.name are preserved, and that small args / plain text +are untouched. +""" +import json +import sys +from unittest.mock import MagicMock + +import pytest + +for mod in [ + 'sqlalchemy', 'sqlalchemy.orm', 'sqlalchemy.ext', 'sqlalchemy.ext.declarative', + 'sqlalchemy.ext.hybrid', 'sqlalchemy.sql', 'sqlalchemy.sql.expression', + 'src.database', + 'core.models', 'core.database', +]: + if mod not in sys.modules: + sys.modules[mod] = MagicMock() + +from src.context_compactor import _truncate_message_to_token_budget # noqa: E402 +from src.model_context import estimate_tokens # noqa: E402 + + +def _tool_msg(arg_len): + return { + "role": "assistant", + "content": None, + "tool_calls": [{ + "id": "c1", "type": "function", + "function": {"name": "create_document", "arguments": "x" * arg_len}, + }], + } + + +def test_oversized_tool_call_args_are_truncated_to_fit_budget(): + budget = 200 + out = _truncate_message_to_token_budget(_tool_msg(40000), budget) + # The message now fits the budget (before the fix it stayed ~12k tokens). + assert estimate_tokens([out]) <= budget, estimate_tokens([out]) + tc = out["tool_calls"][0] + # Structure preserved so tool/result pairing + provider validation still hold. + assert tc["id"] == "c1" and tc["type"] == "function" + assert tc["function"]["name"] == "create_document" + # Arguments remain valid JSON, just bounded. + parsed = json.loads(tc["function"]["arguments"]) + assert parsed.get("_truncated_for_context") == 40000 + + +def test_small_tool_call_args_are_left_untouched(): + out = _truncate_message_to_token_budget(_tool_msg(20), 500) + assert out["tool_calls"][0]["function"]["arguments"] == "x" * 20 + + +def test_plain_text_content_still_truncates(): + out = _truncate_message_to_token_budget({"role": "user", "content": "y" * 40000}, 200) + assert len(out["content"]) < 2000 # truncated, not left at 40k From 545e6925650b9722258d642d706f0802b9f65d47 Mon Sep 17 00:00:00 2001 From: ghreprimand Date: Fri, 5 Jun 2026 13:27:10 -0500 Subject: [PATCH 032/974] fix(auth): distinguish empty model allowlists (#2938) Co-authored-by: ghreprimand <203024559+ghreprimand@users.noreply.github.com> --- core/auth.py | 2 ++ routes/chat_helpers.py | 8 +++-- static/js/admin.js | 26 +++++++++------- tests/test_chat_helpers.py | 64 +++++++++++++++++++++++++++++++++++++- 4 files changed, 85 insertions(+), 15 deletions(-) diff --git a/core/auth.py b/core/auth.py index 3c7669d..ed083b0 100644 --- a/core/auth.py +++ b/core/auth.py @@ -30,10 +30,12 @@ DEFAULT_PRIVILEGES = { "can_manage_memory": True, "max_messages_per_day": 0, "allowed_models": [], + "allowed_models_restricted": False, } # Admins get everything ADMIN_PRIVILEGES = {k: (True if isinstance(v, bool) else (0 if isinstance(v, int) else [])) for k, v in DEFAULT_PRIVILEGES.items()} +ADMIN_PRIVILEGES["allowed_models_restricted"] = False DEFAULT_AUTH_PATH = os.path.join( Path(__file__).parent.parent, "data", "auth.json" diff --git a/routes/chat_helpers.py b/routes/chat_helpers.py index 0929b69..c62d345 100644 --- a/routes/chat_helpers.py +++ b/routes/chat_helpers.py @@ -75,7 +75,7 @@ def _enforce_chat_privileges(request, sess) -> None: allowlist, or HTTPException(429) if the user has hit their daily message cap. No-op for unauthenticated callers or when auth_manager is absent (single-user mode). Admins receive ADMIN_PRIVILEGES from get_privileges, - which means empty allowed_models / zero cap → no-op for them. + which means unrestricted allowed_models / zero cap -> no-op for them. """ try: user = get_current_user(request) @@ -88,8 +88,10 @@ def _enforce_chat_privileges(request, sess) -> None: return privs = auth_manager.get_privileges(user) or {} - allowed = privs.get("allowed_models") or [] - if allowed and sess.model and sess.model not in allowed: + allowed_raw = privs.get("allowed_models") + allowed = allowed_raw if isinstance(allowed_raw, list) else [] + restricted = bool(privs.get("allowed_models_restricted")) or bool(allowed) + if restricted and sess.model and sess.model not in allowed: raise HTTPException(403, f"Your account is not allowed to use model '{sess.model}'.") cap = int(privs.get("max_messages_per_day") or 0) diff --git a/static/js/admin.js b/static/js/admin.js index 5019096..5211bf6 100644 --- a/static/js/admin.js +++ b/static/js/admin.js @@ -87,8 +87,11 @@ async function loadUsers() { `; // Allowed models — checkbox list - const allowedSet = new Set((u.privileges && u.privileges.allowed_models) || []); - const allEmpty = allowedSet.size === 0; + const allowedModels = Array.isArray(u.privileges && u.privileges.allowed_models) + ? u.privileges.allowed_models + : []; + const allowedSet = new Set(allowedModels); + const modelsRestricted = !!(u.privileges && u.privileges.allowed_models_restricted); html += `
Allowed models @@ -97,7 +100,7 @@ async function loadUsers() { None
-
${allEmpty ? 'All models allowed (no restrictions)' : allowedSet.size + ' model(s) allowed'}
+
${!modelsRestricted ? 'All models allowed (no restrictions)' : (allowedSet.size === 0 ? 'No models allowed' : allowedSet.size + ' model(s) allowed')}
Loading models...
@@ -119,7 +122,7 @@ async function loadUsers() { // Load models list on first expand if (!_modelsLoaded && !privPanel.classList.contains('hidden')) { _modelsLoaded = true; - _loadModelsForUser(u.username, allowedSet, privPanel); + _loadModelsForUser(u.username, allowedSet, modelsRestricted, privPanel); } }); @@ -199,7 +202,7 @@ async function loadUsers() { } catch (e) { list.innerHTML = '
Failed to load users
'; } } -async function _loadModelsForUser(username, allowedSet, privPanel) { +async function _loadModelsForUser(username, allowedSet, modelsRestricted, privPanel) { const listEl = privPanel.querySelector(`.priv-models-list[data-user="${username}"]`); if (!listEl) return; try { @@ -216,9 +219,9 @@ async function _loadModelsForUser(username, allowedSet, privPanel) { listEl.innerHTML = 'No models available'; return; } - const allEmpty = allowedSet.size === 0; + let restricted = modelsRestricted; listEl.innerHTML = sortModelObjects(allModels).map(m => { - const checked = allEmpty || allowedSet.has(m.mid) ? 'checked' : ''; + const checked = !restricted || allowedSet.has(m.mid) ? 'checked' : ''; return `