MrSphay/odysseus
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'upstream/dev' into dev
All checks were successful
Container Image / build-and-push (push) Successful in 10m32s
Details

Browse Source 
# Conflicts:
#	README.md
This commit is contained in:
MrSphay
2026-07-07 23:48:48 +02:00
parent 9c7b365668 c67deaa60a
commit 5a550c26f6
822 changed files with 129876 additions and 21462 deletions
Download Patch File Download Diff File Expand all files Collapse all files

10
.dockerignore
View File

@@ -10,6 +10,16 @@ dist/
build/
.env
.env.bak.*
# Secrets: keep plaintext and every transient secrets.env variant out of
# the build context. If an encrypted secrets.env is used, it is mounted
# at runtime — never baked into the image. Mirrored in .gitignore.
secrets.env
secrets.env.*
secrets.env~
.secrets.env.swp
.secrets.env.swo
**/#secrets.env#
!secrets.env.example
/data/
/logs/
.git/

52
.env.example
View File

@@ -56,6 +56,13 @@ SEARXNG_INSTANCE=http://localhost:8080
# SQLite database path (default: sqlite:///./data/app.db)
# DATABASE_URL=sqlite:///./data/app.db
# ============================================================
# Data directory
# ============================================================
# Move everything that lives under data/ - settings, sessions, database, auth,
# cache, uploads, etc. - to another path:
# ODYSSEUS_DATA_DIR=C:\path\to\dir
# ============================================================
# Auth & Security
# ============================================================
@@ -112,6 +119,9 @@ SEARXNG_INSTANCE=http://localhost:8080
# Default: http://{LLM_HOST}:11434/v1/embeddings (ollama)
# EMBEDDING_URL=http://localhost:11434/v1/embeddings
# Embedding API key (if there's one)
# EMBEDDING_API_KEY=embedding_api_key_here
# Embedding model name (must be available at the endpoint above)
# EMBEDDING_MODEL=all-minilm:l6-v2
@@ -144,6 +154,41 @@ SEARXNG_INSTANCE=http://localhost:8080
# if you intentionally want scheduled scripts to run remotely.
# ODYSSEUS_SCRIPT_HOST=localhost
# Chat / agent attachment size cap in bytes (default: 10 MB).
# Raise this for local installs that need larger PDFs or text documents.
# Example: 52428800 = 50 MB.
# ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=10485760
# Other per-feature upload size caps in bytes. All are validated and optional;
# defaults shown. An invalid value (non-integer or < 1) fails fast at startup.
# ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=104857600 # gallery image upload (100 MB)
# ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES=26214400 # gallery transform input (25 MB)
# ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=10485760 # memory import file (10 MB)
# ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=26214400 # personal document upload (25 MB)
# ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=26214400 # email compose attachment (25 MB)
# ODYSSEUS_STT_MAX_AUDIO_BYTES=26214400 # speech-to-text audio (25 MB)
# ODYSSEUS_ICS_MAX_BYTES=10485760 # calendar .ics import (10 MB)
# ============================================================
# Host Docker access (explicit opt-in)
# ============================================================
# Default Docker Compose does not mount /var/run/docker.sock. Existing
# Ollama, vLLM, and other OpenAI-compatible endpoints remain usable without it.
#
# Enable this only for intentional Cookbook/local Docker-daemon management.
# Raw socket access is high-trust and can grant broad control over the host
# Docker daemon. Set DOCKER_GID to the host docker group's numeric GID.
# Put these values in .env, or export them before running docker compose.
# COMPOSE_FILE=docker-compose.yml:docker/host-docker.yml
# DOCKER_GID=963
# docker/host-docker.yml sets this inside the container. Keep it paired
# with the socket overlay; setting it alone is not sufficient.
# ODYSSEUS_ENABLE_HOST_DOCKER=true
#
# Host Docker access can be combined with one GPU overlay:
# COMPOSE_FILE=docker-compose.yml:docker/gpu.nvidia.yml:docker/host-docker.yml
# COMPOSE_FILE=docker-compose.yml:docker/gpu.amd.yml:docker/host-docker.yml
# ============================================================
# GPU support (Docker Compose)
# ============================================================
@@ -165,3 +210,10 @@ SEARXNG_INSTANCE=http://localhost:8080
# These overlays only expose the GPU devices. The slim Odysseus image
# still needs CUDA/ROCm userspace via Cookbook -> Dependencies (vLLM,
# llama-cpp-python, etc.) before models can actually serve on GPU.
# ============================================================
# Storage Paths (Docker Compose)
# ============================================================
# APP_DATA_DIR=./data
# APP_LOGS_DIR=./logs

9
.github/CODEOWNERS vendored Normal file
View File

@@ -0,0 +1,9 @@
# Code owners.
#
# Intentionally empty for now. The catch-all rule that mapped every path to a
# single owner froze all merges the moment "Require review from Code Owners"
# was enabled, because no other maintainer's approval could satisfy the gate.
# A per-area ownership map (security/auth, CI, frontend, agent internals, with
# multiple named owners per line) is being worked out in issue #593; once
# agreed it replaces this file. Until then, required reviews and the security
# CI gate (docs/security-ci.md) remain in force via branch protection.

2
.github/ISSUE_TEMPLATE/bug_report.yml vendored
View File

@@ -23,7 +23,7 @@ body:
required: true
- label: This is **not** a security vulnerability. (Vulnerabilities go to [GitHub Security Advisories](https://github.com/pewdiepie-archdaemon/odysseus/security/advisories/new) — see [SECURITY.md](https://github.com/pewdiepie-archdaemon/odysseus/blob/main/SECURITY.md).)
required: true
- label: I am running the latest code from `main`.
- label: I am running the latest code from the `dev` branch (the default branch you get on clone, where fixes land first) and the bug still reproduces there. Please `git pull` the latest `dev` before filing.
required: true
- type: dropdown

48
.github/dependabot.yml vendored Normal file
View File

@@ -0,0 +1,48 @@
# Dependabot keeps dependencies and pinned action versions current.
#
# Why this matters for security: every workflow in this repo pins its GitHub
# Actions to an exact commit (a SHA), which is safe but freezes them in time.
# Dependabot opens a small, reviewable pull request whenever a newer version
# exists -- for Python packages, npm packages, the Docker base image, and the
# pinned Actions themselves -- so staying patched does not require manual work.
# Updates are grouped so a week's bumps arrive as one PR per ecosystem, not a
# flood of separate ones.
version: 2
updates:
# Python dependencies (requirements.txt + requirements-optional.txt).
- package-ecosystem: pip
directory: "/"
schedule:
interval: weekly
open-pull-requests-limit: 5
groups:
python:
patterns: ["*"]
# Frontend / tooling npm packages (package.json).
- package-ecosystem: npm
directory: "/"
schedule:
interval: weekly
open-pull-requests-limit: 5
groups:
npm:
patterns: ["*"]
# The pinned action SHAs used across .github/workflows.
- package-ecosystem: github-actions
directory: "/"
schedule:
interval: weekly
open-pull-requests-limit: 5
groups:
actions:
patterns: ["*"]
# The Docker base image in the Dockerfile.
- package-ecosystem: docker
directory: "/"
schedule:
interval: weekly
open-pull-requests-limit: 5

123
.github/pull_request_review_template.md vendored Normal file
View File

@@ -0,0 +1,123 @@
# Pull Request Review Template
Use this shape as a copyable reference for substantive PR reviews; GitHub does
not auto-apply this file to review comments. Omit sections that do not add
useful signal. Lead with confirmed findings; keep speculative notes out of the
public review unless they are framed as a concrete open question.
## Small PR Path
For narrow docs, typo, test-only, or obvious local fixes, a short review is
enough:
```md
LGTM after checking:
- scope:
- validation:
- residual risk:
```
Use the fuller structure below for larger, risky, multi-finding, or
security-sensitive reviews.
## Findings
**<sub><sub>![P2 Badge](https://img.shields.io/badge/P2-yellow?style=flat)</sub></sub> issue (test): Short issue title**
- **Problem:** Concrete broken flow, contract, input, or risk.
- **Impact:** Why this matters to users, CI, maintainers, data, security, or scale.
- **Ask:** Smallest practical correction or decision the author should make.
- **Location:** `path:line`
## Open Questions
- **question (scope, non-blocking): Short author question** Ask the concrete
intent, scope, or tradeoff question.
## Validation
- Ran:
- Not run:
- Residual risk:
## PR Hygiene
- Target/template/checks:
- Related, duplicate, or superseding context:
## No Findings Variant
```md
## Findings
none confirmed
## Validation
- Ran:
- Not run:
- Residual risk:
```
## Legend
- **Findings:** Verified, author-actionable issues that should be fixed or
consciously accepted before merge.
- **Priority badges:** The shields.io badges below are optional formatting for
priority labels. Plain `P0`, `P1`, `P2`, or `P3` text is also acceptable when
an external image dependency is undesirable or may not render.
- **P0:** `![P0 Badge](https://img.shields.io/badge/P0-red?style=flat)` -
release-blocking or actively dangerous.
- **P1:** `![P1 Badge](https://img.shields.io/badge/P1-orange?style=flat)` -
serious bug, security risk, data-loss risk, or broken primary flow.
- **P2:** `![P2 Badge](https://img.shields.io/badge/P2-yellow?style=flat)` -
meaningful correctness, test, maintainability, or edge-case issue.
- **P3:** `![P3 Badge](https://img.shields.io/badge/P3-lightgrey?style=flat)` -
minor polish or low-risk cleanup.
- **Intent labels:**
- **`issue`:** A confirmed defect, regression, broken contract, or concrete
risk.
- **`suggestion`:** A non-blocking improvement that would make the PR clearer,
safer, or easier to maintain.
- **`nit`:** A tiny, non-blocking cleanup or style note. Use it only when the
author can safely ignore it without changing the review outcome.
- **`question`:** A real author-facing clarification about intent, scope, or
tradeoffs. Do not use questions to hide an issue that should be stated
directly.
- **`LGTM`:** "Looks good to me." Use only when the review found no blocking
issues, or when any remaining notes are clearly optional.
- **Decorations:** Optional labels in parentheses that clarify the finding type,
scope, or merge impact.
- **`security`:** Auth, authorization, ownership, secrets, SSRF, injection,
unsafe external input, or other trust-boundary concerns.
- **`test`:** Missing, failing, misleading, brittle, or insufficient tests.
- **`scope`:** PR scope, feature boundaries, unrelated churn, or work that
should be split into a separate issue or PR.
- **`ci`:** CI configuration, workflow failures, flaky checks, or validation
signal quality.
- **`api`:** Route, request/response, public function, schema, persistence, or
integration contract changes.
- **`docs`:** User-facing docs, contributor docs, examples, or comments that
need to change with the code.
- **`non-blocking`:** Useful feedback that should not prevent merge by
itself.
- **Finding fields:**
- **Problem:** What is wrong, what contract is ambiguous, or what risk the PR
introduces.
- **Impact:** Why the problem matters in practical terms.
- **Ask:** The smallest concrete fix, test, or decision requested from the PR
author.
- **Location:** The most useful repo-relative file and line reference for the
finding, using `path:line`.
- **Optional sections:**
- **Open Questions:** Genuine scope or intent questions; omit when there are
no real questions.
- **Validation:** What the reviewer ran, what was intentionally not run, and
what risk remains after review.
- **PR Hygiene:** Target-branch, template, CI/check, duplicate, related-work,
or superseding-PR notes.
- **`none confirmed`:** Use only when no review-worthy findings were confirmed;
still list validation gaps or residual risk when relevant.

11
.github/scripts/check-pr-description.js vendored
View File

@@ -103,14 +103,21 @@ module.exports = async ({ github, context, core }) => {
async function swapLabel(num, add, remove) {
if (await labelExists(add)) {
await github.rest.issues.addLabels({ owner, repo, issue_number: num, labels: [add] });
try {
await github.rest.issues.addLabels({ owner, repo, issue_number: num, labels: [add] });
} catch (e) {
// Fail soft on a token that can't write labels so a label permission
// problem never masks the actual description verdict.
if (e.status !== 403) throw e;
core.warning(`Could not add "${add}" — token lacks label write here; skipping.`);
}
} else {
core.warning(`Label "${add}" does not exist in the repo — skipping. Create it once to enable labelling.`);
}
try {
await github.rest.issues.removeLabel({ owner, repo, issue_number: num, name: remove });
} catch (e) {
if (e.status !== 404 && e.status !== 410) throw e;
if (e.status !== 404 && e.status !== 410 && e.status !== 403) throw e;
}
}

124
.github/scripts/focused_test_guidance.py vendored Normal file
View File

@@ -0,0 +1,124 @@
#!/usr/bin/env python3
"""Report focused pytest guidance for changed paths under tests/."""
from __future__ import annotations
import argparse
import os
import shlex
import subprocess
import sys
from collections.abc import Iterable
from pathlib import PurePosixPath
def parse_paths(raw_paths: bytes) -> list[str]:
"""Decode the NUL-delimited output of ``git diff --name-only -z``."""
return [os.fsdecode(path) for path in raw_paths.split(b"\0") if path]
def changed_paths_from_merge_base(base_sha: str, head_sha: str) -> list[str]:
"""Return changed ``tests/`` paths using GitHub PR three-dot semantics.
GitHub PR changed files are based on the merge base and the PR head, not a
direct endpoint diff between the current base branch tip and the PR head.
Using the direct endpoint diff can include files changed only on the base
branch when the PR branch is stale.
"""
merge_base = subprocess.check_output(
["git", "merge-base", base_sha, head_sha],
stderr=subprocess.DEVNULL,
).strip()
raw_paths = subprocess.check_output(
[
"git",
"diff",
"--name-only",
"--diff-filter=ACMRT",
"-z",
os.fsdecode(merge_base),
head_sha,
"--",
"tests/",
],
)
return parse_paths(raw_paths)
def select_test_paths(paths: Iterable[str]) -> list[str]:
"""Return unique, repository-relative paths contained by tests/."""
selected: set[str] = set()
for raw_path in paths:
path = PurePosixPath(raw_path)
if path.is_absolute() or ".." in path.parts:
continue
parts = tuple(part for part in path.parts if part != ".")
if len(parts) >= 2 and parts[0] == "tests":
selected.add(PurePosixPath(*parts).as_posix())
return sorted(selected)
def is_pytest_file(path: str) -> bool:
"""Return whether a changed path follows this repository's pytest naming."""
name = PurePosixPath(path).name
return name.endswith(".py") and (
name.startswith("test_") or name.endswith("_test.py")
)
def pytest_command(paths: Iterable[str]) -> str:
"""Build a copyable pytest command for changed runnable test files."""
command = ["python3", "-m", "pytest", "-q", *paths]
return shlex.join(command)
def format_report(paths: Iterable[str]) -> str:
"""Format focused guidance for CI logs and the workflow summary."""
changed_paths = select_test_paths(paths)
runnable_paths = [path for path in changed_paths if is_pytest_file(path)]
lines = ["## Focused test guidance (report-only)", ""]
if not changed_paths:
lines.append("No changed paths under `tests/`.")
else:
lines.extend(["Changed paths under `tests/`:", ""])
lines.extend(f"- `{path}`" for path in changed_paths)
lines.extend(["", "Suggested focused validation:", ""])
if runnable_paths:
lines.append(f"```sh\n{pytest_command(runnable_paths)}\n```")
else:
lines.append("No directly runnable pytest files changed.")
lines.extend(
[
"",
"This guidance does not infer tests from source changes. "
"Existing blocking CI remains the source of truth.",
]
)
return "\n".join(lines)
def _parse_args(argv: list[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Report focused pytest guidance for changed tests/ paths.",
)
parser.add_argument("--base-sha", help="Pull request base commit SHA.")
parser.add_argument("--head-sha", help="Pull request head commit SHA.")
return parser.parse_args(argv)
def main(argv: list[str] | None = None) -> int:
args = _parse_args(sys.argv[1:] if argv is None else argv)
if bool(args.base_sha) != bool(args.head_sha):
raise SystemExit("--base-sha and --head-sha must be provided together")
if args.base_sha and args.head_sha:
paths = changed_paths_from_merge_base(args.base_sha, args.head_sha)
else:
paths = parse_paths(sys.stdin.buffer.read())
print(format_report(paths))
return 0
if __name__ == "__main__":
raise SystemExit(main())

100
.github/workflows/ci.yml vendored
View File

@@ -15,12 +15,68 @@ concurrency:
cancel-in-progress: true
jobs:
focused-test-guidance:
name: Focused test guidance (report-only)
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
continue-on-error: true
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
fetch-depth: 0
persist-credentials: false
- name: Report changed test paths
env:
BASE_SHA: ${{ github.event.pull_request.base.sha }}
HEAD_SHA: ${{ github.event.pull_request.head.sha }}
run: |
report_file="$RUNNER_TEMP/focused-test-guidance.md"
publish_report() {
cat "$report_file"
if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then
cat "$report_file" >> "$GITHUB_STEP_SUMMARY" || true
fi
return 0
}
report_unavailable() {
{
printf '%s\n\n' '## Focused test guidance unavailable (report-only)'
printf '%s\n\n' "$1"
printf '%s\n' 'Existing blocking CI remains the source of truth.'
} > "$report_file"
publish_report
exit 0
}
if [ -z "$BASE_SHA" ] || [ -z "$HEAD_SHA" ]; then
report_unavailable "Pull request base/head metadata is missing."
fi
if ! git cat-file -e "${BASE_SHA}^{commit}" 2>/dev/null; then
report_unavailable "The pull request base commit is unavailable locally."
fi
if ! git cat-file -e "${HEAD_SHA}^{commit}" 2>/dev/null; then
report_unavailable "The pull request head commit is unavailable locally."
fi
if ! python3 .github/scripts/focused_test_guidance.py \
--base-sha "$BASE_SHA" \
--head-sha "$HEAD_SHA" > "$report_file"; then
report_unavailable "The focused test guidance helper could not produce a report."
fi
publish_report
python-syntax:
name: Python syntax (compileall)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: "3.11"
# Byte-compile sources — catches syntax errors without installing deps.
@@ -30,8 +86,10 @@ jobs:
name: JS syntax (node --check)
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e # v6.4.0
with:
node-version: "20"
# Syntax-check our own JS (skip vendored libs in static/lib).
@@ -50,11 +108,41 @@ jobs:
# ROADMAP "fresh install smoke tests" item; make this required once green.
continue-on-error: true
steps:
- uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
- uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
fetch-depth: 0
persist-credentials: false
# Detect whether this PR only touches documentation files.
# If so, skip the expensive pytest run while still reporting a passing check.
- name: Check for docs-only changes
id: docs-check
run: |
if [ "${{ github.event_name }}" = "pull_request" ]; then
BASE="${{ github.event.pull_request.base.sha }}"
HEAD="${{ github.event.pull_request.head.sha }}"
else
BASE="${{ github.event.before }}"
HEAD="${{ github.sha }}"
fi
# List all changed files; if every file matches docs/markdown patterns, skip pytest.
changed=$(git diff --name-only "$BASE" "$HEAD" 2>/dev/null || git diff --name-only HEAD~1 HEAD)
non_docs=$(echo "$changed" | grep -Ev '^(docs/|.*\.md$|\.github/[^/]+\.md$)' || true)
if [ -z "$non_docs" ]; then
echo "docs_only=true" >> "$GITHUB_OUTPUT"
echo "Docs-only change detected — skipping pytest."
else
echo "docs_only=false" >> "$GITHUB_OUTPUT"
fi
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
if: steps.docs-check.outputs.docs_only != 'true'
with:
python-version: "3.11"
cache: pip
- run: pip install -r requirements.txt
if: steps.docs-check.outputs.docs_only != 'true'
- run: mkdir -p data # sqlite DB lives at ./data/app.db
if: steps.docs-check.outputs.docs_only != 'true'
- run: python -m pytest -q
if: steps.docs-check.outputs.docs_only != 'true'

52
.github/workflows/container-scan.yml vendored Normal file
View File

@@ -0,0 +1,52 @@
# Container security: Dockerfile lint
#
# Purpose: the Docker image is how most people run Odysseus, so it is part of
# the attack surface. hadolint lints the Dockerfile for mistakes and insecure
# patterns (running as root longer than needed, unpinned base image, bad apt
# usage). Blocking.
#
# The image vulnerability scan (Trivy, advisory) lives in its own file,
# container-trivy.yml. Keeping it separate lets that advisory scan be
# path-filtered and held to a read-only token on pull requests without
# weakening this blocking gate, which must always report so a required check
# never hangs.
#
# Note: a separate open PR (#120) proposes a local `scripts/scan_image.py`.
# This job is complementary -- it is a CI gate, not a script a contributor has
# to remember to run.
name: Container scan
on:
pull_request:
push:
branches: [main]
workflow_dispatch:
permissions: {}
concurrency:
group: container-scan-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
hadolint:
name: hadolint (Dockerfile lint)
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Lint Dockerfile
uses: hadolint/hadolint-action@2332a7b74a6de0dda2e2221d575162eba76ba5e5 # v3.3.0
with:
dockerfile: Dockerfile
# DL3008: pinning apt package versions is impractical on a -slim base
# image. Debian purges old package versions from its repos, so a
# pinned version breaks future rebuilds. The base image itself is
# what should be pinned (tracked by Dependabot's docker ecosystem).
ignore: DL3008

125
.github/workflows/container-trivy.yml vendored Normal file
View File

@@ -0,0 +1,125 @@
# Container image vulnerability scan (advisory)
#
# Trivy builds the application image and scans it for known-vulnerable OS and
# Python packages. Advisory only -- it reports findings to the repo's Security
# tab without blocking a merge, because the image inevitably contains
# already-known CVEs in upstream packages that are not this project's bug.
#
# Split from the Dockerfile lint (container-scan.yml) for two reasons:
#
# - Least privilege. The image build runs Dockerfile instructions, which on a
# pull request are attacker-influenceable. That path (the `scan` job) is
# held to a read-only token and never publishes results. Only `publish`,
# which runs on push to main (curated, fast-forwarded from reviewed dev),
# gets security-events:write to upload SARIF.
# - Cost. Docs-only changes do not rebuild the image (paths-ignore below),
# matching docker-publish.yml. hadolint stays on the broad trigger in
# container-scan.yml so the blocking gate always reports.
name: Container scan (Trivy)
on:
pull_request:
paths-ignore:
- '**.md'
- 'docs/**'
- '.github/ISSUE_TEMPLATE/**'
push:
branches: [main]
paths-ignore:
- '**.md'
- 'docs/**'
- '.github/ISSUE_TEMPLATE/**'
workflow_dispatch:
permissions: {}
concurrency:
group: container-trivy-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
# Pull requests and manual runs: build and scan under a read-only token.
# The build executes PR-supplied Dockerfile instructions, so this job must
# not hold any write scope, and it does not upload to the Security tab.
scan:
name: Trivy (image scan, advisory)
if: github.event_name != 'push'
runs-on: ubuntu-latest
# Advisory: a CVE in an upstream package must not block a PR.
continue-on-error: true
permissions:
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Set up Buildx
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
# Build without pushing so a broken Dockerfile is caught here, and the
# exact image we ship is what gets scanned.
- name: Build image
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
with:
context: .
push: false
load: true
tags: odysseus:ci
- name: Scan image with Trivy
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
with:
image-ref: odysseus:ci
format: table
ignore-unfixed: true
env:
# Pin the vuln DB source to GHCR to avoid rate-limited Docker Hub
# mirrors that flake on shared runners.
TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db:2
# Push to main only: build, scan, and publish SARIF to the Security tab.
# This is the only path that runs trusted code, so it is the only one granted
# security-events:write.
publish:
name: Trivy (image scan + SARIF upload)
if: github.event_name == 'push'
runs-on: ubuntu-latest
continue-on-error: true
permissions:
contents: read
security-events: write # upload SARIF to the Security tab
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Set up Buildx
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
- name: Build image
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
with:
context: .
push: false
load: true
tags: odysseus:ci
- name: Scan image with Trivy
uses: aquasecurity/trivy-action@ed142fd0673e97e23eac54620cfb913e5ce36c25 # v0.36.0
with:
image-ref: odysseus:ci
format: sarif
output: trivy-results.sarif
ignore-unfixed: true
env:
TRIVY_DB_REPOSITORY: ghcr.io/aquasecurity/trivy-db:2
- name: Upload Trivy results
uses: github/codeql-action/upload-sarif@8aad20d150bbac5944a9f9d289da16a4b0d87c1e # v4.36.2
with:
sarif_file: trivy-results.sarif
category: trivy-image

71
.github/workflows/dependency-review.yml vendored Normal file
View File

@@ -0,0 +1,71 @@
# Supply-chain review
#
# Purpose: defend against "side-chain" / supply-chain attacks -- a pull request
# that adds (or bumps) a dependency to a version with a known vulnerability or a
# disallowed license. Two layers:
#
# - dependency-review: runs ONLY on pull requests. It compares the
# dependencies before and after the PR and blocks the merge if the change
# pulls in a package with a known security advisory. This is the gate.
# - pip-audit: scans the project's current Python requirements against the
# advisory database. Advisory only (it never blocks a merge), because it can
# flag a pre-existing issue in an already-shipped dependency.
name: Dependency review
on:
pull_request:
push:
branches: [main]
workflow_dispatch:
# Default-deny token; jobs grant only read access.
permissions: {}
concurrency:
group: dependency-review-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
dependency-review:
name: dependency-review (PR gate)
# Only meaningful on a pull request -- it needs a base..head diff to review.
if: github.event_name == 'pull_request'
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Review dependency changes
uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0
with:
# Fail the PR on any newly introduced moderate-or-worse advisory.
fail-on-severity: moderate
pip-audit:
name: pip-audit (advisory)
runs-on: ubuntu-latest
# Advisory: report known-vulnerable Python deps without blocking the merge.
continue-on-error: true
permissions:
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: '3.12'
- name: Run pip-audit on requirements
run: |
set -euo pipefail
pip install pip-audit==2.10.0
pip-audit -r requirements.txt -r requirements-optional.txt --strict

140
.github/workflows/docker-publish.yml vendored Normal file
View File

@@ -0,0 +1,140 @@
name: ci / docker publish
# Build the Odysseus image and publish to GHCR.
# push to main -> :latest, :X.Y.Z (curated release; main is fast-forwarded at releases)
# push to dev -> :dev, :X.Y.Z-dev.<sha> (rolling dev + an immutable, traceable pin)
# Multi-arch (linux/amd64 + linux/arm64): each arch builds on its own native
# runner and pushes by digest, then a merge job stitches the digests into one
# manifest list and applies the tags (faster + cleaner than QEMU emulation).
# Registry: ghcr.io/<owner>/<repo>.
on:
push:
branches: [dev, main]
paths-ignore:
- '**.md'
- 'docs/**'
- '.github/ISSUE_TEMPLATE/**'
concurrency:
group: docker-publish-${{ github.ref }}
cancel-in-progress: true
permissions:
contents: read
env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}
jobs:
build:
name: build (${{ matrix.arch }})
runs-on: ${{ matrix.runner }}
permissions:
contents: read
packages: write
strategy:
fail-fast: false
matrix:
include:
- platform: linux/amd64
arch: amd64
runner: ubuntu-latest
- platform: linux/arm64
arch: arm64
runner: ubuntu-24.04-arm
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Set up Buildx
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
- name: Log in to GHCR
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Build and push by digest
id: build
uses: docker/build-push-action@f9f3042f7e2789586610d6e8b85c8f03e5195baf # v7.2.0
with:
context: .
platforms: ${{ matrix.platform }}
outputs: type=image,name=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }},push-by-digest=true,name-canonical=true,push=true
cache-from: type=gha,scope=${{ matrix.arch }}
cache-to: type=gha,mode=max,scope=${{ matrix.arch }}
- name: Export digest
run: |
mkdir -p /tmp/digests
digest="${{ steps.build.outputs.digest }}"
touch "/tmp/digests/${digest#sha256:}"
- name: Upload digest
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: digest-${{ matrix.arch }}
path: /tmp/digests/*
if-no-files-found: error
retention-days: 1
merge:
name: merge manifest + tag
runs-on: ubuntu-latest
needs: build
permissions:
contents: read
packages: write
steps:
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Read APP_VERSION + short sha
id: ver
run: |
v=$(grep -E '^APP_VERSION' src/constants.py | head -1 | sed -E 's/.*"([^"]+)".*/\1/')
[ -n "$v" ] || { echo "APP_VERSION not found"; exit 1; }
echo "version=$v" >> "$GITHUB_OUTPUT"
echo "short=${GITHUB_SHA::7}" >> "$GITHUB_OUTPUT"
- name: Download digests
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
with:
path: /tmp/digests
pattern: digest-*
merge-multiple: true
- name: Set up Buildx
uses: docker/setup-buildx-action@d7f5e7f509e45cec5c76c4d5afdd7de93d0b3df5 # v4.1.0
- name: Log in to GHCR
uses: docker/login-action@650006c6eb7dba73a995cc03b0b2d7f5ca915bee # v4.2.0
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Compute tags
id: meta
uses: docker/metadata-action@80c7e94dd9b9319bd5eb7a0e0fe9291e23a2a2e9 # v6.1.0
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
tags: |
type=raw,value=latest,enable=${{ github.ref == 'refs/heads/main' }}
type=raw,value=${{ steps.ver.outputs.version }},enable=${{ github.ref == 'refs/heads/main' }}
type=raw,value=dev,enable=${{ github.ref == 'refs/heads/dev' }}
type=raw,value=${{ steps.ver.outputs.version }}-dev.${{ steps.ver.outputs.short }},enable=${{ github.ref == 'refs/heads/dev' }}
- name: Create manifest list + push tags
working-directory: /tmp/digests
run: |
tags=$(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON")
digests=$(printf "${REGISTRY}/${IMAGE_NAME}@sha256:%s " *)
# word-splitting is intended: $tags and $digests each expand to multiple args
# shellcheck disable=SC2086
docker buildx imagetools create $tags $digests
env:
REGISTRY: ${{ env.REGISTRY }}
IMAGE_NAME: ${{ env.IMAGE_NAME }}
- name: Inspect
run: |
if [ "$GITHUB_REF" = "refs/heads/main" ]; then ref=latest; else ref=dev; fi
docker buildx imagetools inspect "${REGISTRY}/${IMAGE_NAME}:${ref}"
env:
REGISTRY: ${{ env.REGISTRY }}
IMAGE_NAME: ${{ env.IMAGE_NAME }}

5
.github/workflows/issue-description-check.yml vendored
View File

@@ -14,10 +14,11 @@ jobs:
# Skip bots (Dependabot, release-drafter, etc.)
if: ${{ github.event.issue.user.type != 'Bot' }}
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
sparse-checkout: .github/scripts
persist-credentials: false
- uses: actions/github-script@v7
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: return require('./.github/scripts/check-issue-description.js')({github, context, core})

105
.github/workflows/pr-description-check.yml vendored
View File

@@ -1,28 +1,109 @@
name: ci / PR description check
name: ci / PR checks
on:
pull_request_target:
types: [opened, edited, synchronize, reopened]
# pull_request_target runs in the base-repo context (has secrets) so the check
# works on fork PRs. Safe here: the checkout pins to the base branch (no fork
# code runs) and the scripts only read context.payload and call the GitHub API.
pull_request_target: # zizmor: ignore[dangerous-triggers]
types: [opened, edited, synchronize, reopened, ready_for_review]
# pull_request_target runs in the base-repo context (has secrets).
# The checkout below pins to the base branch so no fork code is executed.
# The script only reads context.payload and calls the GitHub API.
permissions:
issues: write
pull-requests: write
# Default-deny at the workflow level; each job opts into only the scopes it needs.
# Note: modifying a PR's labels/comments needs pull-requests:write even though the
# REST path is under /issues/{n}/...; issues:write alone returns 403 on PRs.
permissions: {}
jobs:
check-description:
name: Check PR description
runs-on: ubuntu-latest
# Skip bots — they open PRs programmatically and have their own process.
permissions:
contents: read
pull-requests: write
issues: write
# Skip bots: they open PRs programmatically and have their own process.
if: github.event.pull_request.user.type != 'Bot'
steps:
- uses: actions/checkout@v4
- uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
ref: ${{ github.base_ref }}
sparse-checkout: .github/scripts
persist-credentials: false
- uses: actions/github-script@v7
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: return require('./.github/scripts/check-pr-description.js')({github, context, core})
check-title:
name: Check PR title (Conventional Commits)
runs-on: ubuntu-latest
permissions: {}
# Skip bots: they open PRs programmatically and have their own process.
if: github.event.pull_request.user.type != 'Bot'
steps:
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
const title = context.payload.pull_request.title || "";
// Conventional Commits: type(optional-scope)(optional !): summary
const re = /^(feat|fix|docs|style|refactor|perf|test|build|ci|chore|revert)(\([\w .\/-]+\))?!?: .+/;
if (!re.test(title)) {
core.setFailed(
`PR title is not in Conventional Commits format:\n "${title}"\n\n` +
`Expected: type(scope): summary\n` +
`Example: fix(search): handle empty query\n` +
`Types: feat, fix, docs, style, refactor, perf, test, build, ci, chore, revert.`
);
} else {
core.info(`PR title OK: ${title}`);
}
check-mergeable:
name: Flag unmergeable PRs
runs-on: ubuntu-latest
permissions:
pull-requests: write
issues: write
# Skip bots: they open PRs programmatically and have their own process.
if: github.event.pull_request.user.type != 'Bot'
steps:
- uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
with:
script: |
const repo = { owner: context.repo.owner, repo: context.repo.repo };
const number = context.payload.pull_request.number;
const READY = "ready for review";
const CONFLICT = "merge conflict";
// Ensure the conflict label exists (red). Ignore if already present.
try {
await github.rest.issues.getLabel({ ...repo, name: CONFLICT });
} catch {
await github.rest.issues.createLabel({
...repo, name: CONFLICT, color: "B60205",
description: "Conflicts with the base branch; needs a rebase before review.",
}).catch(() => {});
}
// mergeable is computed asynchronously and is often null right after
// an event, so poll a few times until GitHub has resolved it.
let pr = null;
for (let i = 0; i < 5; i++) {
const { data } = await github.rest.pulls.get({ ...repo, pull_number: number });
if (data.mergeable !== null) { pr = data; break; }
await new Promise(r => setTimeout(r, 3000));
}
if (!pr || pr.draft) return;
const labels = pr.labels.map(l => l.name);
if (pr.mergeable === false) {
if (labels.includes(READY)) {
await github.rest.issues.removeLabel({ ...repo, issue_number: number, name: READY }).catch(() => {});
}
if (!labels.includes(CONFLICT)) {
await github.rest.issues.addLabels({ ...repo, issue_number: number, labels: [CONFLICT] });
}
} else if (pr.mergeable === true) {
if (labels.includes(CONFLICT)) {
await github.rest.issues.removeLabel({ ...repo, issue_number: number, name: CONFLICT }).catch(() => {});
}
}

60
.github/workflows/secret-scan.yml vendored Normal file
View File

@@ -0,0 +1,60 @@
# Secret scanning
#
# Purpose: stop credentials (API keys, tokens, passwords, private keys) from
# ever living in the Git history. Odysseus deliberately keeps real secrets in
# files that are gitignored (.env, data/), but a slip in a future commit -- or a
# malicious pull request that sneaks one in -- would otherwise go unnoticed.
# This job reads the repository and the full commit history and fails if it
# finds anything that looks like a secret.
#
# It runs the official gitleaks BINARY directly (pinned to an exact version and
# verified against the project's published SHA-256 checksum) rather than the
# gitleaks GitHub Action, because the Action asks for a paid license on
# organization-owned repos. The binary is free and behaves identically.
name: Secret scan
on:
pull_request:
push:
branches: [main]
workflow_dispatch:
# Start with zero permissions; the single job opts back in to read-only.
permissions: {}
concurrency:
group: secret-scan-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
gitleaks:
name: gitleaks
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
# Full history so a secret committed in an earlier commit (and later
# deleted) is still caught -- deletion does not remove it from Git.
fetch-depth: 0
persist-credentials: false
# Pinned version + checksum so a tampered release binary cannot run here.
# Bump VERSION/SHA256 together; the checksum comes from the matching
# gitleaks_<version>_checksums.txt on the GitHub release.
- name: Run gitleaks (pinned, checksum-verified)
env:
GITLEAKS_VERSION: 8.30.1
GITLEAKS_SHA256: 551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb
run: |
set -euo pipefail
TARBALL="gitleaks_${GITLEAKS_VERSION}_linux_x64.tar.gz"
curl -fsSL -o "${TARBALL}" \
"https://github.com/gitleaks/gitleaks/releases/download/v${GITLEAKS_VERSION}/${TARBALL}"
echo "${GITLEAKS_SHA256} ${TARBALL}" | sha256sum -c -
tar -xzf "${TARBALL}" gitleaks
# Scan the whole history. Findings print to the log and fail the job.
./gitleaks git --no-banner --redact --verbose .

80
.github/workflows/workflow-security.yml vendored Normal file
View File

@@ -0,0 +1,80 @@
# Workflow security (CI that audits the CI)
#
# Purpose: the GitHub Actions workflows themselves are an attack surface. A
# poorly written workflow can leak the repository token, run attacker-supplied
# code from a pull request, or pull in a tampered third-party action. These two
# tools check every workflow file in this repo for those mistakes:
#
# - actionlint: catches workflow syntax errors and shell-script bugs inside
# `run:` steps before they reach main.
# - zizmor: a security linter for Actions. Flags template-injection holes,
# unpinned actions, credential persistence, and over-broad token
# permissions -- exactly the patterns the rest of this CI is built to avoid.
#
# Add this early: it then audits every workflow added after it.
name: Workflow security
on:
pull_request:
push:
branches: [main]
workflow_dispatch:
# Default-deny token; each job grants only read access to the code.
permissions: {}
concurrency:
group: workflow-security-${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
actionlint:
name: actionlint
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
# Pinned version + checksum so a tampered binary cannot run here.
- name: Run actionlint (pinned, checksum-verified)
env:
ACTIONLINT_VERSION: 1.7.12
ACTIONLINT_SHA256: 8aca8db96f1b94770f1b0d72b6dddcb1ebb8123cb3712530b08cc387b349a3d8
run: |
set -euo pipefail
TARBALL="actionlint_${ACTIONLINT_VERSION}_linux_amd64.tar.gz"
curl -fsSL -o "${TARBALL}" \
"https://github.com/rhysd/actionlint/releases/download/v${ACTIONLINT_VERSION}/${TARBALL}"
echo "${ACTIONLINT_SHA256} ${TARBALL}" | sha256sum -c -
tar -xzf "${TARBALL}" actionlint
./actionlint -color
zizmor:
name: zizmor (Actions SAST)
runs-on: ubuntu-latest
permissions:
contents: read
steps:
- name: Checkout repository
uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
with:
python-version: '3.12'
# Pinned zizmor release. --offline keeps the audit hermetic (no network
# calls about the actions it inspects); --min-severity=low surfaces
# everything so nothing slips through under the gate.
- name: Run zizmor
run: |
set -euo pipefail
pip install zizmor==1.25.2
zizmor --offline --min-severity=low .github/workflows/

13
.gitignore vendored
View File

@@ -14,6 +14,15 @@ venv/
.env
.env.bak.*
!.env.example
# Local uv lockfile (optional, per-platform — see "Faster installs with uv" in README)
requirements.lock
# SOPS workflow — encrypted `secrets.env` is intentionally committable,
# but every variant (plaintext, manual decrypt copy, editor backup)
# must stay out of git. Mirrored in .dockerignore so the same artifacts
# also cannot enter image build layers.
secrets.env.*
!secrets.env.example
# Data — all user data stays local
data/
@@ -61,6 +70,9 @@ output.txt.txt
*.tiff
*.pdf
# …except shipped static assets
!static/icons/*.png
# …except shipped demo assets in docs/ that the README links to.
!docs/*.jpg
!docs/*.jpeg
@@ -89,3 +101,4 @@ docs/windows-port/
compound.config.json
*.error.log
_scratch/
/odysseus/

1
ACKNOWLEDGMENTS.md
View File

@@ -86,6 +86,7 @@ Bundled in `static/fonts/`:
| [Fira Code](https://github.com/tonsky/FiraCode) | SIL Open Font License 1.1 | Nikita Prokopov & contributors |
| [Inter](https://github.com/rsms/inter) | SIL Open Font License 1.1 | Rasmus Andersson |
| [GohuFont](https://font.gohu.org/) (`fonts/custom/GohuFont.ttf`) | WTFPL | Hugo Chargois |
| [OpenDyslexic](https://opendyslexic.org/) (`fonts/OpenDyslexic-{Regular,Bold}.woff2`) | SIL Open Font License 1.1 ([`licenses/OpenDyslexic-OFL.txt`](licenses/OpenDyslexic-OFL.txt)) | Abbie Gonzalez |
## Python dependencies

14
CONTRIBUTING.md
View File

@@ -37,7 +37,7 @@ Manual development uses Python 3.11+:
python3 -m venv venv
source venv/bin/activate
pip install -r requirements.txt
python -m uvicorn app:app --host 0.0.0.0 --port 7000
python -m uvicorn app:app --host 127.0.0.1 --port 7000
```
Windows is not actively tested. Docker on Linux or a Linux/macOS manual install is the safer path for now.
@@ -94,6 +94,18 @@ Before submitting any change that affects what the app looks like — buttons, i
If you are unsure whether a change is "visual," it is. Default to attaching a screenshot.
## Code conventions
Don't hardcode values that the project already exposes through a constant or a helper. Hardcoded literals drift out of sync, break on non-default deployments, and reintroduce bugs we've already fixed.
- **Filesystem paths:** never build writable paths from `Path(__file__)...` into the source tree, hardcode `/app/...`, or use a relative `"data/..."` string. Every persisted file and directory has a named constant in `src/constants.py` (for example `AUTH_FILE`, `USER_PREFS_FILE`, `SETTINGS_FILE`, `TTS_CACHE_DIR`, `CHROMA_DIR`). Import and use that named constant; do not re-derive the path locally with `os.path.join(DATA_DIR, "x.json")` or `DATA_DIR / "x.json"`. `DATA_DIR` is the single place that reads `ODYSSEUS_DATA_DIR`, so use it directly only for dynamic paths that have no fixed name (for example per-owner files). If a data file or directory has no constant yet, add one to `src/constants.py`. The source tree is read-only in Docker and `/app/...` does not exist on native runs; guard directory creation so an unwritable path degrades gracefully instead of crashing at import.
- **Internal API / loopback URLs:** don't hardcode `http://localhost:7000`. Use `internal_api_base()` from `src.constants` (it honors `ODYSSEUS_INTERNAL_BASE` / `APP_PORT`).
- **Ports, limits, model lists, and similar:** reuse the existing constant if one exists; if it doesn't and the value is used in more than one place, add a constant rather than copying the literal.
If you need a value that has no constant or helper yet, add it to `src/constants.py` (the single source of truth for paths and config; `core/constants.py` only re-exports it for backward compatibility) and import it, rather than repeating a literal across files.
**Commits:** use [Conventional Commits](https://www.conventionalcommits.org), `type(scope): summary` (e.g. `fix(search): ...`, `feat(notes): ...`, `docs(contributing): ...`). Common types: `fix`, `feat`, `refactor`, `docs`, `test`, `chore`, `ci`. Keep the subject short and imperative; put the "why" in the body when it isn't obvious.
## Issue Reports
For bugs, include:

63
Dockerfile
View File

@@ -1,4 +1,15 @@
FROM python:3.12-slim
# ---- builder: patch + build wheels for Real-ESRGAN's broken-on-3.14 deps ----
# basicsr/gfpgan/facexlib read their version via exec()+locals()['__version__'],
# which raises KeyError on Python 3.13+ (PEP 667). Build patched wheels here so
# the final image / Cookbook never has to compile the broken sdists. See
# docker/build-realesrgan-wheels.sh for the full rationale.
FROM python:3.14-slim AS realesrgan-wheels
RUN apt-get update && apt-get install -y --no-install-recommends curl \
&& rm -rf /var/lib/apt/lists/*
COPY docker/build-realesrgan-wheels.sh /usr/local/bin/build-realesrgan-wheels.sh
RUN bash /usr/local/bin/build-realesrgan-wheels.sh /wheels
FROM python:3.14-slim
# System deps. tmux is required by Cookbook for background downloads/serves.
# openssh-client is required for Cookbook remote server tests, setup, probes,
@@ -18,8 +29,44 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
tmux \
openssh-client \
gosu \
libgl1 \
libglib2.0-0t64 \
libxcb1 \
libmagic1 \
&& rm -rf /var/lib/apt/lists/*
# libgl1/libglib2.0-0t64/libxcb1 are runtime shared libs (libGL.so.1,
# libglib-2.0/libgthread, libxcb.so.1) that opencv-python (cv2) loads. The
# slim base omits them, so the Cookbook "install realesrgan" path imports cv2
# and dies with `libxcb.so.1: cannot open shared object file` despite a clean
# pip install. Using full opencv-python (not -headless) because basicsr/gfpgan/
# facexlib/realesrgan all depend on the `opencv-python` distribution by name.
#
# libmagic1 is the shared lib (libmagic.so.1) that python-magic dlopens for
# content-based MIME sniffing in src/upload_handler.py. We install both here
# (libmagic1 + the python-magic wrapper, below) rather than in requirements.txt
# because python-magic resolves libmagic at import time: where the lib is
# absent the import can block or raise, so keeping it image-only avoids
# regressing pip/venv installs on hosts without libmagic. Debian always has the
# lib here, so the import is instant and detection actually works.
# Docker CLI (client only — daemon stays on the host via the
# /var/run/docker.sock mount). The Debian `docker.io` package ships
# dockerd but not the client binary on slim, so grab the static client
# tarball from download.docker.com instead.
ARG DOCKER_CLI_VERSION=27.5.1
RUN ARCH="$(dpkg --print-architecture)" \
&& case "$ARCH" in \
amd64) DARCH=x86_64 ;; \
arm64) DARCH=aarch64 ;; \
*) echo "unsupported arch $ARCH"; exit 1 ;; \
esac \
&& curl -fsSL "https://download.docker.com/linux/static/stable/${DARCH}/docker-${DOCKER_CLI_VERSION}.tgz" \
-o /tmp/docker.tgz \
&& tar -xzf /tmp/docker.tgz -C /tmp \
&& install -m 0755 /tmp/docker/docker /usr/local/bin/docker \
&& rm -rf /tmp/docker /tmp/docker.tgz
WORKDIR /app
# Install Python deps first (layer cache). Optional extras (PyMuPDF AGPL, etc.)
@@ -29,6 +76,20 @@ COPY requirements.txt requirements-optional.txt ./
RUN pip install --no-cache-dir -r requirements.txt \
&& if [ "$INSTALL_OPTIONAL" = "true" ]; then pip install --no-cache-dir -r requirements-optional.txt; fi
# python-magic powers content-based MIME sniffing in src/upload_handler.py.
# Image-only (not in requirements.txt) because it needs the libmagic1 system
# lib installed above; see the apt note near the top of this stage.
RUN pip install --no-cache-dir python-magic==0.4.27
# Pre-install the patched basicsr/gfpgan/facexlib wheels built in the
# realesrgan-wheels stage (--no-deps keeps the image lean — torch & friends are
# pulled only when realesrgan is actually installed). With these dists already
# satisfied, the Cookbook's plain `pip install realesrgan` resolves them from
# wheels instead of rebuilding the sdists that fail on Python 3.14.
COPY --from=realesrgan-wheels /wheels/ /tmp/odysseus-wheels/
RUN pip install --no-cache-dir --no-deps /tmp/odysseus-wheels/*.whl \
&& rm -rf /tmp/odysseus-wheels
# Copy app code
COPY . .

248
LICENSE
View File

@@ -1,21 +1,235 @@
MIT License
GNU AFFERO GENERAL PUBLIC LICENSE
Version 3, 19 November 2007
Copyright (c) 2025 Odysseus Contributors
Copyright (C) 2007 Free Software Foundation, Inc. <http://fsf.org/>
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
Everyone is permitted to copy and distribute verbatim copies of this license document, but changing it is not allowed.
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
Preamble
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
The GNU Affero General Public License is a free, copyleft license for software and other kinds of works, specifically designed to ensure cooperation with the community in the case of network server software.
The licenses for most software and other practical works are designed to take away your freedom to share and change the works. By contrast, our General Public Licenses are intended to guarantee your freedom to share and change all versions of a program--to make sure it remains free software for all its users.
When we speak of free software, we are referring to freedom, not price. Our General Public Licenses are designed to make sure that you have the freedom to distribute copies of free software (and charge for them if you wish), that you receive source code or can get it if you want it, that you can change the software or use pieces of it in new free programs, and that you know you can do these things.
Developers that use our General Public Licenses protect your rights with two steps: (1) assert copyright on the software, and (2) offer you this License which gives you legal permission to copy, distribute and/or modify the software.
A secondary benefit of defending all users' freedom is that improvements made in alternate versions of the program, if they receive widespread use, become available for other developers to incorporate. Many developers of free software are heartened and encouraged by the resulting cooperation. However, in the case of software used on network servers, this result may fail to come about. The GNU General Public License permits making a modified version and letting the public access it on a server without ever releasing its source code to the public.
The GNU Affero General Public License is designed specifically to ensure that, in such cases, the modified source code becomes available to the community. It requires the operator of a network server to provide the source code of the modified version running there to the users of that server. Therefore, public use of a modified version, on a publicly accessible server, gives the public access to the source code of the modified version.
An older license, called the Affero General Public License and published by Affero, was designed to accomplish similar goals. This is a different license, not a version of the Affero GPL, but Affero has released a new version of the Affero GPL which permits relicensing under this license.
The precise terms and conditions for copying, distribution and modification follow.
TERMS AND CONDITIONS
0. Definitions.
"This License" refers to version 3 of the GNU Affero General Public License.
"Copyright" also means copyright-like laws that apply to other kinds of works, such as semiconductor masks.
"The Program" refers to any copyrightable work licensed under this License. Each licensee is addressed as "you". "Licensees" and "recipients" may be individuals or organizations.
To "modify" a work means to copy from or adapt all or part of the work in a fashion requiring copyright permission, other than the making of an exact copy. The resulting work is called a "modified version" of the earlier work or a work "based on" the earlier work.
A "covered work" means either the unmodified Program or a work based on the Program.
To "propagate" a work means to do anything with it that, without permission, would make you directly or secondarily liable for infringement under applicable copyright law, except executing it on a computer or modifying a private copy. Propagation includes copying, distribution (with or without modification), making available to the public, and in some countries other activities as well.
To "convey" a work means any kind of propagation that enables other parties to make or receive copies. Mere interaction with a user through a computer network, with no transfer of a copy, is not conveying.
An interactive user interface displays "Appropriate Legal Notices" to the extent that it includes a convenient and prominently visible feature that (1) displays an appropriate copyright notice, and (2) tells the user that there is no warranty for the work (except to the extent that warranties are provided), that licensees may convey the work under this License, and how to view a copy of this License. If the interface presents a list of user commands or options, such as a menu, a prominent item in the list meets this criterion.
1. Source Code.
The "source code" for a work means the preferred form of the work for making modifications to it. "Object code" means any non-source form of a work.
A "Standard Interface" means an interface that either is an official standard defined by a recognized standards body, or, in the case of interfaces specified for a particular programming language, one that is widely used among developers working in that language.
The "System Libraries" of an executable work include anything, other than the work as a whole, that (a) is included in the normal form of packaging a Major Component, but which is not part of that Major Component, and (b) serves only to enable use of the work with that Major Component, or to implement a Standard Interface for which an implementation is available to the public in source code form. A "Major Component", in this context, means a major essential component (kernel, window system, and so on) of the specific operating system (if any) on which the executable work runs, or a compiler used to produce the work, or an object code interpreter used to run it.
The "Corresponding Source" for a work in object code form means all the source code needed to generate, install, and (for an executable work) run the object code and to modify the work, including scripts to control those activities. However, it does not include the work's System Libraries, or general-purpose tools or generally available free programs which are used unmodified in performing those activities but which are not part of the work. For example, Corresponding Source includes interface definition files associated with source files for the work, and the source code for shared libraries and dynamically linked subprograms that the work is specifically designed to require, such as by intimate data communication or control flow between those
subprograms and other parts of the work.
The Corresponding Source need not include anything that users can regenerate automatically from other parts of the Corresponding Source.
The Corresponding Source for a work in source code form is that same work.
2. Basic Permissions.
All rights granted under this License are granted for the term of copyright on the Program, and are irrevocable provided the stated conditions are met. This License explicitly affirms your unlimited permission to run the unmodified Program. The output from running a covered work is covered by this License only if the output, given its content, constitutes a covered work. This License acknowledges your rights of fair use or other equivalent, as provided by copyright law.
You may make, run and propagate covered works that you do not convey, without conditions so long as your license otherwise remains in force. You may convey covered works to others for the sole purpose of having them make modifications exclusively for you, or provide you with facilities for running those works, provided that you comply with the terms of this License in conveying all material for which you do not control copyright. Those thus making or running the covered works for you must do so exclusively on your behalf, under your direction and control, on terms that prohibit them from making any copies of your copyrighted material outside their relationship with you.
Conveying under any other circumstances is permitted solely under the conditions stated below. Sublicensing is not allowed; section 10 makes it unnecessary.
3. Protecting Users' Legal Rights From Anti-Circumvention Law.
No covered work shall be deemed part of an effective technological measure under any applicable law fulfilling obligations under article 11 of the WIPO copyright treaty adopted on 20 December 1996, or similar laws prohibiting or restricting circumvention of such measures.
When you convey a covered work, you waive any legal power to forbid circumvention of technological measures to the extent such circumvention is effected by exercising rights under this License with respect to the covered work, and you disclaim any intention to limit operation or modification of the work as a means of enforcing, against the work's users, your or third parties' legal rights to forbid circumvention of technological measures.
4. Conveying Verbatim Copies.
You may convey verbatim copies of the Program's source code as you receive it, in any medium, provided that you conspicuously and appropriately publish on each copy an appropriate copyright notice; keep intact all notices stating that this License and any non-permissive terms added in accord with section 7 apply to the code; keep intact all notices of the absence of any warranty; and give all recipients a copy of this License along with the Program.
You may charge any price or no price for each copy that you convey, and you may offer support or warranty protection for a fee.
5. Conveying Modified Source Versions.
You may convey a work based on the Program, or the modifications to produce it from the Program, in the form of source code under the terms of section 4, provided that you also meet all of these conditions:
a) The work must carry prominent notices stating that you modified it, and giving a relevant date.
b) The work must carry prominent notices stating that it is released under this License and any conditions added under section 7. This requirement modifies the requirement in section 4 to "keep intact all notices".
c) You must license the entire work, as a whole, under this License to anyone who comes into possession of a copy. This License will therefore apply, along with any applicable section 7 additional terms, to the whole of the work, and all its parts, regardless of how they are packaged. This License gives no permission to license the work in any other way, but it does not invalidate such permission if you have separately received it.
d) If the work has interactive user interfaces, each must display Appropriate Legal Notices; however, if the Program has interactive interfaces that do not display Appropriate Legal Notices, your work need not make them do so.
A compilation of a covered work with other separate and independent works, which are not by their nature extensions of the covered work, and which are not combined with it such as to form a larger program, in or on a volume of a storage or distribution medium, is called an "aggregate" if the compilation and its resulting copyright are not used to limit the access or legal rights of the compilation's users beyond what the individual works permit. Inclusion of a covered work in an aggregate does not cause this License to apply to the other parts of the aggregate.
6. Conveying Non-Source Forms.
You may convey a covered work in object code form under the terms of sections 4 and 5, provided that you also convey the machine-readable Corresponding Source under the terms of this License, in one of these ways:
a) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by the Corresponding Source fixed on a durable physical medium customarily used for software interchange.
b) Convey the object code in, or embodied in, a physical product (including a physical distribution medium), accompanied by a written offer, valid for at least three years and valid for as long as you offer spare parts or customer support for that product model, to give anyone who possesses the object code either (1) a copy of the Corresponding Source for all the software in the product that is covered by this License, on a durable physical medium customarily used for software interchange, for a price no more than your reasonable cost of physically performing this conveying of source, or (2) access to copy the Corresponding Source from a network server at no charge.
c) Convey individual copies of the object code with a copy of the written offer to provide the Corresponding Source. This alternative is allowed only occasionally and noncommercially, and only if you received the object code with such an offer, in accord with subsection 6b.
d) Convey the object code by offering access from a designated place (gratis or for a charge), and offer equivalent access to the Corresponding Source in the same way through the same place at no further charge. You need not require recipients to copy the Corresponding Source along with the object code. If the place to copy the object code is a network server, the Corresponding Source may be on a different server (operated by you or a third party) that supports equivalent copying facilities, provided you maintain clear directions next to the object code saying where to find the Corresponding Source. Regardless of what server hosts the Corresponding Source, you remain obligated to ensure that it is available for as long as needed to satisfy these requirements.
e) Convey the object code using peer-to-peer transmission, provided you inform other peers where the object code and Corresponding Source of the work are being offered to the general public at no charge under subsection 6d.
A separable portion of the object code, whose source code is excluded from the Corresponding Source as a System Library, need not be included in conveying the object code work.
A "User Product" is either (1) a "consumer product", which means any tangible personal property which is normally used for personal, family, or household purposes, or (2) anything designed or sold for incorporation into a dwelling. In determining whether a product is a consumer product, doubtful cases shall be resolved in favor of coverage. For a particular product received by a particular user, "normally used" refers to a typical or common use of that class of product, regardless of the status of the particular user or of the way in which the particular user actually uses, or expects or is expected to use, the product. A product is a consumer product regardless of whether the product has substantial commercial, industrial or non-consumer uses, unless such uses represent the only significant mode of use of the product.
"Installation Information" for a User Product means any methods, procedures, authorization keys, or other information required to install and execute modified versions of a covered work in that User Product from a modified version of its Corresponding Source. The information must suffice to ensure that the continued functioning of the modified object code is in no case prevented or interfered with solely because modification has been made.
If you convey an object code work under this section in, or with, or specifically for use in, a User Product, and the conveying occurs as part of a transaction in which the right of possession and use of the User Product is transferred to the recipient in perpetuity or for a fixed term (regardless of how the transaction is characterized), the Corresponding Source conveyed under this section must be accompanied by the Installation Information. But this requirement does not apply if neither you nor any third party retains the ability to install modified object code on the User Product (for example, the work has been installed in ROM).
The requirement to provide Installation Information does not include a requirement to continue to provide support service, warranty, or updates for a work that has been modified or installed by the recipient, or for the User Product in which it has been modified or installed. Access to a network may be denied when the modification itself materially and adversely affects the operation of the network or violates the rules and protocols for communication across the network.
Corresponding Source conveyed, and Installation Information provided, in accord with this section must be in a format that is publicly documented (and with an implementation available to the public in source code form), and must require no special password or key for unpacking, reading or copying.
7. Additional Terms.
"Additional permissions" are terms that supplement the terms of this License by making exceptions from one or more of its conditions. Additional permissions that are applicable to the entire Program shall be treated as though they were included in this License, to the extent that they are valid under applicable law. If additional permissions apply only to part of the Program, that part may be used separately under those permissions, but the entire Program remains governed by this License without regard to the additional permissions.
When you convey a copy of a covered work, you may at your option remove any additional permissions from that copy, or from any part of it. (Additional permissions may be written to require their own removal in certain cases when you modify the work.) You may place additional permissions on material, added by you to a covered work, for which you have or can give appropriate copyright permission.
Notwithstanding any other provision of this License, for material you add to a covered work, you may (if authorized by the copyright holders of that material) supplement the terms of this License with terms:
a) Disclaiming warranty or limiting liability differently from the terms of sections 15 and 16 of this License; or
b) Requiring preservation of specified reasonable legal notices or author attributions in that material or in the Appropriate Legal Notices displayed by works containing it; or
c) Prohibiting misrepresentation of the origin of that material, or requiring that modified versions of such material be marked in reasonable ways as different from the original version; or
d) Limiting the use for publicity purposes of names of licensors or authors of the material; or
e) Declining to grant rights under trademark law for use of some trade names, trademarks, or service marks; or
f) Requiring indemnification of licensors and authors of that material by anyone who conveys the material (or modified versions of it) with contractual assumptions of liability to the recipient, for any liability that these contractual assumptions directly impose on those licensors and authors.
All other non-permissive additional terms are considered "further restrictions" within the meaning of section 10. If the Program as you received it, or any part of it, contains a notice stating that it is governed by this License along with a term that is a further restriction, you may remove that term. If a license document contains a further restriction but permits relicensing or conveying under this License, you may add to a covered work material governed by the terms of that license document, provided that the further restriction does not survive such relicensing or conveying.
If you add terms to a covered work in accord with this section, you must place, in the relevant source files, a statement of the additional terms that apply to those files, or a notice indicating where to find the applicable terms.
Additional terms, permissive or non-permissive, may be stated in the form of a separately written license, or stated as exceptions; the above requirements apply either way.
8. Termination.
You may not propagate or modify a covered work except as expressly provided under this License. Any attempt otherwise to propagate or modify it is void, and will automatically terminate your rights under this License (including any patent licenses granted under the third paragraph of section 11).
However, if you cease all violation of this License, then your license from a particular copyright holder is reinstated (a) provisionally, unless and until the copyright holder explicitly and finally terminates your license, and (b) permanently, if the copyright holder fails to notify you of the violation by some reasonable means prior to 60 days after the cessation.
Moreover, your license from a particular copyright holder is reinstated permanently if the copyright holder notifies you of the violation by some reasonable means, this is the first time you have received notice of violation of this License (for any work) from that copyright holder, and you cure the violation prior to 30 days after your receipt of the notice.
Termination of your rights under this section does not terminate the licenses of parties who have received copies or rights from you under this License. If your rights have been terminated and not permanently reinstated, you do not qualify to receive new licenses for the same material under section 10.
9. Acceptance Not Required for Having Copies.
You are not required to accept this License in order to receive or run a copy of the Program. Ancillary propagation of a covered work occurring solely as a consequence of using peer-to-peer transmission to receive a copy likewise does not require acceptance. However, nothing other than this License grants you permission to propagate or modify any covered work. These actions infringe copyright if you do not accept this License. Therefore, by modifying or propagating a covered work, you indicate your acceptance of this License to do so.
10. Automatic Licensing of Downstream Recipients.
Each time you convey a covered work, the recipient automatically receives a license from the original licensors, to run, modify and propagate that work, subject to this License. You are not responsible for enforcing compliance by third parties with this License.
An "entity transaction" is a transaction transferring control of an organization, or substantially all assets of one, or subdividing an organization, or merging organizations. If propagation of a covered work results from an entity transaction, each party to that transaction who receives a copy of the work also receives whatever licenses to the work the party's predecessor in interest had or could give under the previous paragraph, plus a right to possession of the Corresponding Source of the work from the predecessor in interest, if the predecessor has it or can get it with reasonable efforts.
You may not impose any further restrictions on the exercise of the rights granted or affirmed under this License. For example, you may not impose a license fee, royalty, or other charge for exercise of rights granted under this License, and you may not initiate litigation (including a cross-claim or counterclaim in a lawsuit) alleging that any patent claim is infringed by making, using, selling, offering for sale, or importing the Program or any portion of it.
11. Patents.
A "contributor" is a copyright holder who authorizes use under this License of the Program or a work on which the Program is based. The work thus licensed is called the contributor's "contributor version".
A contributor's "essential patent claims" are all patent claims owned or controlled by the contributor, whether already acquired or hereafter acquired, that would be infringed by some manner, permitted by this License, of making, using, or selling its contributor version, but do not include claims that would be infringed only as a consequence of further modification of the contributor version. For purposes of this definition, "control" includes the right to grant patent sublicenses in a manner consistent with the requirements of this License.
Each contributor grants you a non-exclusive, worldwide, royalty-free patent license under the contributor's essential patent claims, to make, use, sell, offer for sale, import and otherwise run, modify and propagate the contents of its contributor version.
In the following three paragraphs, a "patent license" is any express agreement or commitment, however denominated, not to enforce a patent (such as an express permission to practice a patent or covenant not to sue for patent infringement). To "grant" such a patent license to a party means to make such an agreement or commitment not to enforce a patent against the party.
If you convey a covered work, knowingly relying on a patent license, and the Corresponding Source of the work is not available for anyone to copy, free of charge and under the terms of this License, through a publicly available network server or other readily accessible means, then you must either (1) cause the Corresponding Source to be so available, or (2) arrange to deprive yourself of the benefit of the patent license for this particular work, or (3) arrange, in a manner consistent with the requirements of this License, to extend the patent
license to downstream recipients. "Knowingly relying" means you have actual knowledge that, but for the patent license, your conveying the covered work in a country, or your recipient's use of the covered work in a country, would infringe one or more identifiable patents in that country that you have reason to believe are valid.
If, pursuant to or in connection with a single transaction or arrangement, you convey, or propagate by procuring conveyance of, a covered work, and grant a patent license to some of the parties receiving the covered work authorizing them to use, propagate, modify or convey a specific copy of the covered work, then the patent license you grant is automatically extended to all recipients of the covered work and works based on it.
A patent license is "discriminatory" if it does not include within the scope of its coverage, prohibits the exercise of, or is conditioned on the non-exercise of one or more of the rights that are specifically granted under this License. You may not convey a covered work if you are a party to an arrangement with a third party that is in the business of distributing software, under which you make payment to the third party based on the extent of your activity of conveying the work, and under which the third party grants, to any of the parties who would receive the covered work from you, a discriminatory patent license (a) in connection with copies of the covered work conveyed by you (or copies made from those copies), or (b) primarily for and in connection with specific products or compilations that contain the covered work, unless you entered into that arrangement, or that patent license was granted, prior to 28 March 2007.
Nothing in this License shall be construed as excluding or limiting any implied license or other defenses to infringement that may otherwise be available to you under applicable patent law.
12. No Surrender of Others' Freedom.
If conditions are imposed on you (whether by court order, agreement or otherwise) that contradict the conditions of this License, they do not excuse you from the conditions of this License. If you cannot convey a covered work so as to satisfy simultaneously your obligations under this License and any other pertinent obligations, then as a consequence you may
not convey it at all. For example, if you agree to terms that obligate you to collect a royalty for further conveying from those to whom you convey the Program, the only way you could satisfy both those terms and this License would be to refrain entirely from conveying the Program.
13. Remote Network Interaction; Use with the GNU General Public License.
Notwithstanding any other provision of this License, if you modify the Program, your modified version must prominently offer all users interacting with it remotely through a computer network (if your version supports such interaction) an opportunity to receive the Corresponding Source of your version by providing access to the Corresponding Source from a network server at no charge, through some standard or customary means of facilitating copying of software. This Corresponding Source shall include the Corresponding Source for any work covered by version 3 of the GNU General Public License that is incorporated pursuant to the following paragraph.
Notwithstanding any other provision of this License, you have permission to link or combine any covered work with a work licensed under version 3 of the GNU General Public License into a single combined work, and to convey the resulting work. The terms of this License will continue to apply to the part which is the covered work, but the work with which it is combined will remain governed by version 3 of the GNU General Public License.
14. Revised Versions of this License.
The Free Software Foundation may publish revised and/or new versions of the GNU Affero General Public License from time to time. Such new versions will be similar in spirit to the present version, but may differ in detail to address new problems or concerns.
Each version is given a distinguishing version number. If the Program specifies that a certain numbered version of the GNU Affero General Public License "or any later version" applies to it, you have the option of following the terms and conditions either of that numbered version or of any later version published by the Free Software Foundation. If the Program does not specify a version number of the GNU Affero General Public License, you may choose any version ever published by the Free Software Foundation.
If the Program specifies that a proxy can decide which future versions of the GNU Affero General Public License can be used, that proxy's public statement of acceptance of a version permanently authorizes you to choose that version for the Program.
Later license versions may give you additional or different permissions. However, no additional obligations are imposed on any author or copyright holder as a result of your choosing to follow a later version.
15. Disclaimer of Warranty.
THERE IS NO WARRANTY FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, REPAIR OR CORRECTION.
16. Limitation of Liability.
IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MODIFIES AND/OR CONVEYS THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGES.
17. Interpretation of Sections 15 and 16.
If the disclaimer of warranty and limitation of liability provided above cannot be given local legal effect according to their terms, reviewing courts shall apply local law that most closely approximates an absolute waiver of all civil liability in connection with the Program, unless a warranty or assumption of liability accompanies a copy of the Program in return for a fee.
END OF TERMS AND CONDITIONS
How to Apply These Terms to Your New Programs
If you develop a new program, and you want it to be of the greatest possible use to the public, the best way to achieve this is to make it free software which everyone can redistribute and change under these terms.
To do so, attach the following notices to the program. It is safest to attach them to the start of each source file to most effectively state the exclusion of warranty; and each file should have at least the "copyright" line and a pointer to where the full notice is found.
<one line to give the program's name and a brief idea of what it does.>
Copyright (C) <year> <name of author>
This program is free software: you can redistribute it and/or modify it under the terms of the GNU Affero General Public License as published by the Free Software Foundation, either version 3 of the License, or (at your option) any later version.
This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Affero General Public License for more details.
You should have received a copy of the GNU Affero General Public License along with this program. If not, see <http://www.gnu.org/licenses/>.
Also add information on how to contact you by electronic and paper mail.
If your software can interact with users remotely through a computer network, you should also make sure that it provides a way for users to get its source. For example, if your program is a web application, its interface could display a "Source" link that leads users to an archive of the code. There are many ways you could offer source, and different solutions will be better for different programs; see section 13 for the specific requirements.
You should also get your employer (if you work as a programmer) or school, if any, to sign a "copyright disclaimer" for the program, if necessary. For more information on this, and how to apply and follow the GNU AGPL, see <http://www.gnu.org/licenses/>.

45
Odysseus.spec Normal file
View File

@@ -0,0 +1,45 @@
# -*- mode: python ; coding: utf-8 -*-
a = Analysis(
['launcher.py'],
pathex=[],
binaries=[],
datas=[('static', 'static'), ('scripts', 'scripts'), ('mcp_servers', 'mcp_servers'), ('services/hwfit/data', 'services/hwfit/data'), ('config', 'config'), ('.env.example', '.env.example')],
hiddenimports=[],
hookspath=[],
hooksconfig={},
runtime_hooks=[],
excludes=[],
noarchive=False,
optimize=0,
)
pyz = PYZ(a.pure)
exe = EXE(
pyz,
a.scripts,
[],
exclude_binaries=True,
name='Odysseus',
debug=False,
bootloader_ignore_signals=False,
strip=False,
upx=True,
console=False,
disable_windowed_traceback=False,
argv_emulation=False,
target_arch=None,
codesign_identity=None,
entitlements_file=None,
icon=['static\\icon.ico'],
)
coll = COLLECT(
exe,
a.binaries,
a.datas,
strip=False,
upx=True,
upx_exclude=[],
name='Odysseus',
)

451
README.md
View File

@@ -1,75 +1,40 @@
# Odysseus
<p align="center">
<img src="docs/odysseus-wordmark.png" alt="Odysseus" width="238">
</p>
```
───────────────────────────────────────────────
⊹ ࣪ ˖ ૮( ˶ᵔ ᵕ ᵔ˶ )っ Odysseus vers. 1.0
───────────────────────────────────────────────
```
<p align="center">
A self-hosted AI workspace for chat, agents, research, documents, email, notes, calendar, and local model workflows.
</p>
![Odysseus](docs/odysseus.jpg)
<p align="center">
<a href="#quick-start">Quick Start</a> ·
<a href="docs/setup.md">Setup Guide</a> ·
<a href="CONTRIBUTING.md">Contributing</a> ·
<a href="ROADMAP.md">Roadmap</a>
</p>
A self-hosted AI workspace -- meant to be the self-hosted version of the UI experience you get from ChatGPT and Claude. But with more jank and fun. Running on your own hardware, with your own data -- local-first, privacy-first, and no trojan.
<p align="center">
<a href="https://repology.org/project/odysseus-ai/versions"><img src="https://repology.org/badge/vertical-allrepos/odysseus-ai.svg" alt="Packaging status"></a>
</p>
## Features
- **Chat** -- chat with any local model or API; adding them is super simple.<br>　<sub>vLLM · llama.cpp · Ollama · OpenRouter · OpenAI · GitHub Copilot</sub>
- **Agent** -- hand it tools and let it run the whole task itself.<br>　<sub>built on [opencode](https://github.com/anomalyco/opencode) · MCP · web · files · shell · skills · memory</sub>
- **Cookbook** -- Scans your hardware, recommends models, click to download and serve.. easy!<br>　<sub>built on [llmfit](https://github.com/AlexsJones/llmfit) · VRAM-aware · GGUF / FP8 / AWQ · fit scoring · vLLM / llama.cpp serving</sub>
- **Deep Research** -- multi-step runs that gather, read, and synthesize sources into a nice visual report.<br>　<sub>adapted from [Tongyi DeepResearch](https://github.com/Alibaba-NLP/DeepResearch)</sub>
- **Compare** -- a fun tool to compare models side by side. Test completely blind, no bias!<br>　<sub>multi-model · blind test · synthesis</sub>
- **Documents** -- YOU write the text, AI is there to assist, not the opposite.<br>　<sub>multi-tab editor · markdown · HTML · CSV · syntax highlighting · AI edits · suggestions</sub>
- **Memory / Skills** -- Persistent memory and skills, your agent evolves over time as it better understands you and your tasks!<br>　<sub>ChromaDB · fastembed (ONNX) · vector + keyword retrieval · import/export</sub>
- **Email** -- IMAP/SMTP inbox with AI triage built in: urgency reminders, auto-tag, auto-summary, auto-reply drafts, auto-spam.<br>　<sub>IMAP · SMTP · per-account routing · CalDAV-aware</sub>
- **Notes & Tasks** -- Quick notes with reminders, a todo list, and scheduled tasks the agent can act on.<br>　<sub>note pings · checklist · cron-style tasks · ntfy / browser / email channels</sub>
- **Calendar** -- Local-first calendar with CalDAV sync to Radicale / Nextcloud / Apple / Fastmail.<br>　<sub>CalDAV pull · .ics import/export · per-calendar colors · agent-aware</sub>
- **Works on mobile** -- looks and runs great on your phone, not just desktop.<br>　<sub>responsive · installable (PWA) · touch gestures</sub>
- **Extras** -- more to explore, happy if you give it a go!<br>　<sub>image editor · theme editor · file uploads (vision + PDF) · web search · presets · sessions · 2FA</sub>
<p align="center">
<img src="docs/odysseus-browser.jpg" alt="Odysseus interface">
</p>
## Demo
A full, hover-to-play tour lives on the landing page (`docs/index.html`).
<details>
<summary>Screenshots / clips</summary>
### Chat & Agents
![Chat & Agents](docs/chat.gif)
### Deep Research
![Deep Research](docs/research.gif)
### Compare
![Compare](docs/compare.gif)
### Documents
![Documents](docs/document.gif)
### Notes & Tasks
![Notes & Tasks](docs/notes.gif)
</details>
---
## Quick Start
Defaults work out of the box: clone, run, then configure models/search/email
inside **Settings**. Only edit `.env` for deployment-level overrides like
`APP_BIND`, `APP_PORT`, `AUTH_ENABLED`, `DATABASE_URL`, or a pre-seeded admin password.
> `dev` is the default branch and gets the newest changes first. Use [`main`](https://github.com/pewdiepie-archdaemon/odysseus/tree/main) if you want the more curated branch.
On first setup, Odysseus creates an admin account (`admin` unless
`ODYSSEUS_ADMIN_USER` is set) and prints a temporary password in the terminal.
For Docker installs, the same line is in `docker compose logs odysseus`.
Use that for the first login, then change it in **Settings**.
Contributing? See [CONTRIBUTING.md](CONTRIBUTING.md) for setup, testing, and
pull request guidelines.
### Docker (recommended)
```bash
git clone https://github.com/pewdiepie-archdaemon/odysseus.git
cd odysseus
cp .env.example .env # optional, but recommended for explicit defaults
cp .env.example .env
docker compose up -d --build
```
To include optional extras in the image (PDF viewer, Office extraction; includes AGPL PyMuPDF), build with `docker compose build --build-arg INSTALL_OPTIONAL=true` before `up`.
Open `http://localhost:7000` when the containers are healthy. Docker Compose
binds the web UI to `127.0.0.1` by default. If the port is taken, set
`APP_PORT=7001` in `.env` and recreate the container. Set `APP_BIND=0.0.0.0`
only when you intentionally want LAN/reverse-proxy access.
Open `http://localhost:7000` when the containers are healthy. The first admin password is printed in `docker compose logs odysseus`.
### Dockge / Gitea Registry Image
Use this when you want Dockge to pull the prebuilt image from the Gitea
@@ -214,356 +179,30 @@ PUID=1000
PGID=1000
```
### Native Linux / macOS
```bash
git clone https://github.com/pewdiepie-archdaemon/odysseus.git
cd odysseus
python3 -m venv venv
source venv/bin/activate
pip install -r requirements.txt
python setup.py
python -m uvicorn app:app --host 127.0.0.1 --port 7000
```
Requirements: Python 3.11+. Cookbook also needs `tmux` for background model
downloads and serves. The app itself is lightweight; local model serving is the
heavy part and depends on the model, runtime, GPU, and VRAM, so small hosts can
connect to API or remote model servers instead. Use `--host 0.0.0.0` only when you intentionally want LAN/reverse-proxy access.
Native installs, GPU notes, Windows/macOS instructions, HTTPS, and configuration live in the [setup guide](docs/setup.md).
### Apple Silicon
Docker on macOS cannot use the Metal GPU. For GPU-accelerated Cookbook on an
M-series Mac, run Odysseus natively:
## Features
```bash
git clone https://github.com/pewdiepie-archdaemon/odysseus.git
cd odysseus
./start-macos.sh
```
- **Chat + Agents** — local/API models, tools, MCP, files, shell, skills, and memory.
- **Cookbook** — hardware-aware model recommendations, downloads, and serving.
- **Deep Research** — multi-step web research with source reading and report generation.
- **Compare** — blind side-by-side model testing and synthesis.
- **Documents** — writing-first editor with AI edits, suggestions, Markdown, HTML, CSV, and syntax highlighting.
- **Email** — IMAP/SMTP inbox with triage, tags, summaries, reminders, and reply drafts.
- **Notes, Tasks + Calendar** — reminders, todos, scheduled agent tasks, and CalDAV sync.
- **Extras** — gallery/image editor, themes, uploads, web search, presets, sessions, and 2FA.
It launches at `http://127.0.0.1:7860`. To expose it to your phone over a trusted LAN/VPN such as Tailscale, bind all interfaces:
## Demo
```bash
ODYSSEUS_HOST=0.0.0.0 ./start-macos.sh
# then open http://<tailscale-ip>:7860
```
The script also reads `.env` at startup, so `APP_BIND=0.0.0.0` and `APP_PORT`
set there are picked up automatically without a command-line override each run.
Keep `AUTH_ENABLED=true` (the default) before binding outside loopback. Do not
expose this port directly to the public internet. To build a clickable app wrapper:
```bash
./build-macos-app.sh
```
<details>
<summary>Cookbook, GPU, Ollama, and troubleshooting notes</summary>
**Docker bundled services.** Compose starts Odysseus, ChromaDB, SearXNG, and
ntfy. Odysseus and the bundled service ports bind to `127.0.0.1` by default, so
they are reachable from the host but not exposed to your LAN/public internet
unless you opt in.
**Cookbook storage in Docker.** Downloads live in `./data/huggingface`
(`~/.cache/huggingface` in the container). Cookbook-installed Python CLIs and
serve engines live in `./data/local` (`~/.local` in the container), so they
survive container recreation.
**Remote servers.** In **Cookbook -> Settings -> Servers**, generate the
Odysseus SSH key and add the public key to the remote server's
`~/.ssh/authorized_keys`. From the host you can also run:
```bash
ssh-copy-id -i data/ssh/id_ed25519.pub user@server
```
**Docker GPU overlays.** CPU-only users can skip this section. Cookbook can
only detect GPUs that Docker exposes to the container — if the host runtime or
device passthrough is not configured, Cookbook sees the iGPU, another card, or
CPU instead of your intended GPU.
For NVIDIA, `scripts/check-docker-gpu.sh` diagnoses GPU passthrough and can
optionally install the host runtime or update `.env`.
```bash
# Read-only diagnostic (default — installs nothing, never edits .env):
scripts/check-docker-gpu.sh
# Print OS-specific install commands without running them:
scripts/check-docker-gpu.sh --print-install-commands
# Install NVIDIA Container Toolkit on Ubuntu/Debian (requires sudo):
scripts/check-docker-gpu.sh --install-nvidia-toolkit
# Write COMPOSE_FILE to .env (only when GPU passthrough is confirmed working):
scripts/check-docker-gpu.sh --enable-nvidia-overlay
# Full assisted setup — install toolkit, then enable overlay if passthrough works:
scripts/check-docker-gpu.sh --install-nvidia-toolkit --enable-nvidia-overlay
```
Safety notes:
- The app never installs host GPU runtime automatically.
- The app never edits `.env` automatically.
- `.env` is only modified when `--enable-nvidia-overlay` is explicitly passed,
and only after GPU passthrough succeeds. `--yes` skips prompts but does not
bypass the passthrough gate.
- `.env.bak.*` backups created by `--enable-nvidia-overlay` are ignored by
Git and the Docker build context.
To enable manually without the script, add this to `.env`:
```bash
COMPOSE_FILE=docker-compose.yml:docker/gpu.nvidia.yml
```
**AMD / ROCm.** AMD setup is read-only diagnostic plus manual `.env` edit. Run:
```bash
scripts/check-docker-amd-gpu.sh
```
Then add the reported values to `.env`, replacing `RENDER_GID` with your host's
numeric render group id:
```bash
COMPOSE_FILE=docker-compose.yml:docker/gpu.amd.yml
RENDER_GID=989
```
For NVIDIA/AMD GPU support, also read the comments in the selected overlay file: docker/gpu.nvidia.yml or docker/gpu.amd.yml.
**Stack-management UIs (Portainer, Coolify, Dockhand, etc.).** These tools
often accept only a single Compose file and do not reliably honor `COMPOSE_FILE`
or multiple `-f` overlays. CLI users should keep using the `COMPOSE_FILE`
overlay workflow above. For stack UIs, point the stack at one of the standalone
files instead, which bundle the base stack plus the GPU settings:
- `docker-compose.gpu-nvidia.yml` — still requires the NVIDIA Container Toolkit
on the host.
- `docker-compose.gpu-amd.yml` — still requires host ROCm/kfd/DRI setup, the
`video`/`render` group membership, and `RENDER_GID` when needed.
The base `docker-compose.yml` plus the `docker/gpu.*.yml` overlays remain the
source of truth; the standalone files mirror them for single-file deployments.
Verify after enabling either overlay:
```bash
docker compose exec odysseus nvidia-smi -L # NVIDIA
docker compose exec odysseus sh -lc 'test -e /dev/kfd && test -d /dev/dri && ls -l /dev/kfd /dev/dri/renderD*' # AMD
```
> **GPU passthrough ≠ llama.cpp CUDA.** `nvidia-smi` passing inside the
> container confirms Docker GPU access, but llama.cpp also needs `cudart` and
> the CUDA Toolkit at runtime. If Cookbook logs show `Unable to find cudart
> library`, `Could NOT find CUDAToolkit`, `CUDA Toolkit not found`, or
> tensors/layers assigned to CPU, that is a Cookbook/llama.cpp build issue —
> not a Docker passthrough failure. Re-install the serve engine via
> **Cookbook → Dependencies** to get a CUDA-enabled build.
>
> The same split applies to AMD/ROCm: seeing `/dev/kfd` and `/dev/dri` inside
> the container confirms device passthrough, not ROCm userspace or a
> ROCm-enabled vLLM/llama.cpp build. `rocm-smi` and `rocminfo` are not expected
> inside the slim Odysseus image.
**Ollama with Docker.** If Ollama runs on the host, add this endpoint in
Settings:
```text
http://host.docker.internal:11434/v1
```
Ollama must listen outside its own loopback interface:
```bash
OLLAMA_HOST=0.0.0.0:11434 ollama serve
```
This connects Odysseus in Docker to an Ollama server that is already running on
your host machine; it does not start Ollama inside the container.
`host.docker.internal` is Docker's hostname for the host machine from inside the
container. Cookbook **Serve** is a separate workflow for serving downloaded
models through Odysseus/llama.cpp, so Windows users with an existing Ollama
install usually only need to add the endpoint in Settings.
**Useful checks.**
```bash
docker compose ps
docker compose logs --tail=120 odysseus
docker compose logs odysseus | grep -E 'ChromaDB|MemoryVectorStore|DEGRADED'
```
**macOS details.** `start-macos.sh` installs Homebrew deps, creates the venv,
runs setup, and starts uvicorn on port `7860` because AirPlay often holds
`7000`. It uses llama.cpp/Ollama for Metal. vLLM/SGLang are CUDA/ROCm-only and
do not run on macOS. MLX-only models are not served by Odysseus.
</details>
### Native Windows
**One-command launcher** (creates the venv, installs deps, runs setup, starts the
server; safe to re-run):
```powershell
git clone https://github.com/pewdiepie-archdaemon/odysseus.git
cd odysseus
powershell -ExecutionPolicy Bypass -File .\launch-windows.ps1
```
Or do it by hand:
```powershell
git clone https://github.com/pewdiepie-archdaemon/odysseus.git
cd odysseus
py -3.11 -m venv venv
venv\Scripts\Activate.ps1
pip install -r requirements.txt
python setup.py
python -m uvicorn app:app --host 127.0.0.1 --port 7000
```
If `python` points at an older interpreter, use `py -3.12` (or another installed
3.11+ version) for the venv step.
**Requirements:** Python 3.11+. The core app (chat, agent, memory, documents,
email, calendar, deep research) runs fully native. For full **Cookbook** background
model downloads and the agent shell tool, also install
[Git for Windows](https://git-scm.com/download/win) (provides `bash.exe`).
Local GPU *serving* of vLLM/SGLang needs Linux/WSL2; for a local model on Windows,
[Ollama](https://ollama.com/download) is the easiest path — point Odysseus at
`http://localhost:11434/v1` in Settings.
Open `http://localhost:7000`, log in with the generated admin password,
and configure everything else inside **Settings**.
## Troubleshooting & Advanced Setup
### `chromadb-client` conflicts with embedded ChromaDB
If `chromadb-client` (the lightweight HTTP-only package) is installed alongside the full `chromadb` package, Odysseus starts but ChromaDB silently falls back to HTTP-only mode and fails.
**Fix:** uninstall `chromadb-client` and force-reinstall the full package:
```bash
./venv/bin/pip uninstall chromadb-client -y
./venv/bin/pip install --force-reinstall chromadb
```
### HTTPS + LAN/Tailscale exposure
To expose Odysseus on a local network or Tailscale with HTTPS:
1. Change the bind address to `0.0.0.0` in `.env` (`APP_BIND=0.0.0.0` or `ODYSSEUS_HOST=0.0.0.0`).
2. Generate a locally-trusted cert for your LAN/Tailscale IPs using [mkcert](https://github.com/FiloSottile/mkcert):
```bash
mkcert -install
mkcert -cert-file cert.pem -key-file key.pem 192.168.1.100 tailscale-ip
```
3. Run `uvicorn` with the generated certs:
```bash
python -m uvicorn app:app --host 0.0.0.0 --port 7000 --ssl-certfile=cert.pem --ssl-keyfile=key.pem
```
4. Install the `mkcert` CA on any other device you want to access Odysseus from (e.g., for iOS, email the `rootCA.pem` to yourself, install the profile, and trust it in Certificate Trust Settings).
### Optional Dependencies
`requirements-optional.txt` contains packages that unlock extra features. It is not installed by default.
| Package | Feature unlocked |
|---------|-----------------|
| `faster-whisper` | Local speech-to-text (microphone -> text) via the "local" STT provider. |
| `duckduckgo-search` | DuckDuckGo as a search provider option. |
| `PyMuPDF` | PDF page rendering in the side viewer panel and form-filling. (Note: AGPL-3.0) |
| `markitdown` | Office/EPUB document text extraction (converts .docx/.xlsx/.pptx/.xls/.epub to Markdown). |
## Security Notes
Odysseus is a self-hosted workspace with powerful local tools: shell access, file uploads, model downloads, web research, email/calendar integrations, and API tokens. Treat it like an admin console.
- Keep `AUTH_ENABLED=true` for any network-accessible deployment.
- Keep `LOCALHOST_BYPASS=false` outside local development.
- Use `SECURE_COOKIES=true` when Odysseus is served through HTTPS by a trusted reverse proxy or private access gateway.
- Do not expose it directly to the public internet without HTTPS and a trusted reverse proxy or private access layer.
- Keep `.env`, `data/`, `logs/`, databases, uploads, generated media, backups, auth/session files, API keys, and model/provider tokens out of Git and private shares. They are ignored by default.
- Review `data/auth.json` after first boot: disable open signup unless you intentionally want it, make only your own account admin, and keep demo/test accounts non-admin.
- Non-admin users do not get shell/Python/file read/write by default, and admin-only routes/tools such as MCP management, API tokens, webhooks, model/cookbook serving, backup/vault, and app settings are admin-gated. Other features are controlled by per-user privileges, so review each user's privileges before exposing a deployment.
- Rotate any API keys or tokens that were ever pasted into a shared chat, demo, screenshot, or log.
- If you enable API tokens or webhooks, create separate tokens per integration and delete unused ones.
- Prefer binding manual development runs to `127.0.0.1`; bind to `0.0.0.0` only when you intentionally want LAN/reverse-proxy access.
- Keep ChromaDB, SearXNG, ntfy, Ollama, vLLM, llama.cpp, databases, and raw model/provider APIs internal-only. Expose only the authenticated Odysseus web/API entrypoint through your trusted proxy or private access layer.
- Before publishing a fork, run `git status --short` and confirm no private files from `.env`, `data/`, `logs/`, uploads, backups, or local databases are staged.
### Private or proxied deployments
Odysseus serves plain HTTP on its app port. Docker Compose binds Odysseus and the bundled services to `127.0.0.1` by default, so a typical production/private setup is:
1. Keep Odysseus on localhost, for example `127.0.0.1:7000`.
2. Terminate HTTPS at a trusted reverse proxy or private access gateway.
3. Put the authenticated Odysseus web/API entrypoint behind that layer.
4. Keep raw service and model ports internal-only.
Cloudflare Access, Tailscale, Caddy, nginx, and Traefik can all fit this pattern; none are required by Odysseus. If your access layer reaches Odysseus on the same host, proxy to `http://127.0.0.1:7000` and keep `AUTH_ENABLED=true`, `LOCALHOST_BYPASS=false`, and `SECURE_COOKIES=true`.
Common internal-only ports from the default docs/compose setup:
| Port | Service |
|---|---|
| `7000` | Odysseus raw app port |
| `8080` | SearXNG |
| `8091` | ntfy |
| `8100` | ChromaDB host port for manual/compose access |
| `11434` | Ollama |
| `8000-8020` | Common local model/provider APIs |
A full hover-to-play tour lives on the landing page: [`docs/index.html`](docs/index.html).
## Contributing
Help is welcome. The best entry points are fresh-install testing, provider setup
bugs, mobile/editor polish, docs, and small focused refactors. See
[ROADMAP.md](ROADMAP.md) for the current help-wanted list.
## Configuration
Most setup is done inside the app with `/setup` or **Settings**. Use `.env`
for deployment-level defaults and secrets you want present before first boot.
Key settings:
Help is welcome. The best entry points are fresh-install testing, provider setup bugs, mobile/editor polish, docs, and small focused refactors. See [CONTRIBUTING.md](CONTRIBUTING.md) and [ROADMAP.md](ROADMAP.md).
| Variable | Default | Description |
|---|---|---|
| `LLM_HOST` | `localhost` | Your LLM server (e.g. `llm-host.local:8000`) |
| `LLM_HOSTS` | -- | Comma-separated list for model discovery |
| `OPENAI_API_KEY` | -- | Optional OpenAI key. Prefer adding providers in the app unless pre-seeding. |
| `SEARXNG_INSTANCE` | `http://localhost:8080` | SearXNG URL. Docker overrides this to `http://searxng:8080`. |
| `SEARXNG_SECRET` | generated on first Docker boot | Optional SearXNG cookie/CSRF secret. Leave blank unless you need to pin it. |
| `APP_BIND` | `127.0.0.1` | Docker Compose host bind address for the web UI. Use `0.0.0.0` only for intentional LAN/reverse-proxy access. |
| `APP_PORT` | `7000` | Docker Compose host port for the web UI. |
| `AUTH_ENABLED` | `true` | Enable/disable login |
| `LOCALHOST_BYPASS` | `false` | Development-only auth bypass for loopback requests. Keep false for shared/network deployments. |
| `SECURE_COOKIES` | `false` | Set true when serving Odysseus through HTTPS at a trusted proxy or private access gateway. |
| `DATABASE_URL` | `sqlite:///./data/app.db` | Database connection string |
| `CHROMADB_HOST` | `localhost` | ChromaDB host for vector memory. Docker overrides this to `chromadb`. |
| `CHROMADB_PORT` | `8100` | ChromaDB port for manual host runs. Docker overrides this to `8000`. |
| `EMBEDDING_URL` | -- | OpenAI-compatible embeddings endpoint |
## Security
### Built-in MCP servers (optional setup)
Odysseus auto-registers a few built-in MCP servers at startup. The npx-based ones (currently the browser server, `@playwright/mcp`) only start when their npm package is already in the local npx cache. If a package isn't cached, that server is skipped with a startup log message explaining what to do, so a fresh install does not block on a multi-minute npm download or hang if Playwright system deps are missing.
To enable the browser MCP (page navigation, screenshots, vision), run once:
```bash
npx -y @playwright/mcp@latest --version
```
That installs `@playwright/mcp` plus Playwright (~300MB total). Restart Odysseus and the server will register at startup.
## Architecture
```
app.py # FastAPI entry point
core/ auth, database, middleware, constants
src/ llm_core, agent_loop, agent_tools, chat_processor, search/
routes/ chat, session, document, memory, model … endpoints
services/ docs, memory, search, hwfit (Cookbook) …
static/ index.html + app.js + style.css + js/ (modular front-end)
docs/ landing page (index.html) + preview clips
```
## Data
All user data lives in `data/` (gitignored): `app.db` (sessions, messages, documents),
`memory.json`, `presets.json`, `uploads/`, `personal_docs/`, `chroma/`, `settings.json`.
Odysseus is a self-hosted workspace with powerful local tools. Keep auth enabled, keep private data out of Git, and do not expose raw model/service ports publicly. Deployment details are in the [setup guide](docs/setup.md#security-notes).
## Star History
@@ -576,19 +215,5 @@ All user data lives in `data/` (gitignored): `app.db` (sessions, messages, docum
</a>
## License
MIT -- see [LICENSE](LICENSE) and [ACKNOWLEDGMENTS.md](ACKNOWLEDGMENTS.md).
```
|
|||
|||||
| | | |||||||
)_) )_) )_) ~|~
)___))___))___)\ |
)____)____)_____)\\|
_____|____|____|_____\\\__
\ /
~^~^~~^~^~~^~^~~^~^~~^~^~~^~^~~^~^~~^~^~
~^~ all aboard! ~^~
~^~^~~^~^~~^~^~~^~^~~^~^~~^~^~~^~^~~^~^~
```
AGPL-3.0-or-later -- see [LICENSE](LICENSE) and [ACKNOWLEDGMENTS.md](ACKNOWLEDGMENTS.md).

348
app.py
View File

@@ -1,6 +1,18 @@
# app.py — slim orchestrator
import mimetypes
import os
import sys
import asyncio
import time
# On Windows, asyncio.create_subprocess_exec/shell require the ProactorEventLoop.
# When started via `python -m uvicorn` from a terminal, uvicorn sets this
# automatically. But the VS Code debugger (and other non-uvicorn entrypoints)
# use the default SelectorEventLoop, which raises NotImplementedError on any
# subprocess call. Force ProactorEventLoop here so the right loop is always
# used, regardless of how the process is launched.
if sys.platform == "win32":
asyncio.set_event_loop_policy(asyncio.WindowsProactorEventLoopPolicy())
def register_static_mime_types() -> None:
@@ -38,24 +50,25 @@ load_dotenv(encoding="utf-8-sig")
import asyncio
import logging
import secrets
from datetime import datetime
from datetime import datetime, timezone
from typing import Dict
from contextlib import asynccontextmanager
from fastapi import FastAPI, Request, HTTPException
from fastapi.responses import JSONResponse, FileResponse, HTMLResponse
from fastapi.responses import JSONResponse, FileResponse
from fastapi.middleware.cors import CORSMiddleware
from fastapi.staticfiles import StaticFiles
from starlette.middleware.base import BaseHTTPMiddleware
from starlette.middleware.gzip import GZipMiddleware
# Core imports
from core.constants import (
BASE_DIR, STATIC_DIR, SESSIONS_FILE,
REQUEST_TIMEOUT, OPENAI_API_KEY,
REQUEST_TIMEOUT, OPENAI_API_KEY, AUTH_FILE,
)
from core.database import SessionLocal, ApiToken
from core.middleware import SecurityHeadersMiddleware
from core.auth import AuthManager
from core.middleware import SecurityHeadersMiddleware, is_cors_preflight
from core.auth import AuthManager, normalize_known_username
from core.exceptions import (
SessionNotFoundError, InvalidFileUploadError,
LLMServiceError, WebSearchError,
@@ -63,15 +76,42 @@ from core.exceptions import (
import bcrypt as _bcrypt
from src.app_helpers import abs_join
from src.app_helpers import abs_join, serve_html_with_nonce
from src.generated_images import GENERATED_IMAGE_HEADERS, resolve_generated_image_path
from starlette.responses import RedirectResponse
# ========= LOGGING =========
logging.basicConfig(
level=logging.INFO,
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
)
import logging.handlers
from core.constants import DATA_DIR
_root_logger = logging.getLogger()
_root_logger.setLevel(logging.INFO)
_formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s')
# Clear existing handlers to avoid duplicates
for _h in list(_root_logger.handlers):
_root_logger.removeHandler(_h)
_console_h = logging.StreamHandler()
_console_h.setFormatter(_formatter)
_root_logger.addHandler(_console_h)
try:
_log_dir = os.path.join(DATA_DIR, "logs")
os.makedirs(_log_dir, exist_ok=True)
_log_file = os.path.join(_log_dir, "app.log")
# RotatingFileHandler is not multi-process safe (e.g. if uvicorn is run with --workers N).
# Odysseus is single-process by convention, so this is acceptable, but be aware that
# concurrent log rotation issues can arise if multiple workers are configured.
_file_h = logging.handlers.RotatingFileHandler(
_log_file, maxBytes=5 * 1024 * 1024, backupCount=3, encoding="utf-8"
)
_file_h.setFormatter(_formatter)
_root_logger.addHandler(_file_h)
except Exception as e:
_root_logger.warning(f"Failed to initialize file logging handler (falling back to console-only): {e}")
logger = logging.getLogger(__name__)
# ========= APP =========
@@ -85,12 +125,13 @@ app = FastAPI(
)
# ========= CORS =========
CORS_ALLOW_METHODS = ["GET", "POST", "PUT", "PATCH", "DELETE"]
allowed_origins = os.getenv("ALLOWED_ORIGINS", "http://localhost,http://127.0.0.1").split(",")
app.add_middleware(
CORSMiddleware,
allow_origins=allowed_origins,
allow_credentials=True,
allow_methods=["GET", "POST", "PUT", "DELETE"],
allow_methods=CORS_ALLOW_METHODS,
allow_headers=[
"Accept",
"Authorization",
@@ -104,6 +145,16 @@ app.add_middleware(
],
)
# ========= RESPONSE COMPRESSION (gzip) =========
# The frontend's text assets (style.css, index.html, the JS bundles) shipped
# uncompressed on every cold load. gzip cuts CSS/JS/HTML by ~75-85% on the wire
# with no behavioural change. Starlette's GZipMiddleware excludes
# `text/event-stream` by default, so the SSE streams (chat, shell, research,
# model-probe — all served with media_type="text/event-stream") are never
# compressed or buffered; only complete bodies over minimum_size are. The
# security-header middleware composes cleanly on top.
app.add_middleware(GZipMiddleware, minimum_size=1024, compresslevel=6)
# ========= SECURITY HEADERS MIDDLEWARE =========
app.add_middleware(SecurityHeadersMiddleware)
@@ -129,6 +180,7 @@ _TIMEOUT_EXEMPT_PREFIXES = (
"/api/cookbook/setup", # remote pacman/apt installs
"/api/upload", # large files
"/api/image", # diffusion proxies (inpaint/harmonize/upscale/etc.) — own 120s httpx timeout
"/api/memory/audit", # retains own 120s LLM inactivity timeout
)
@@ -146,7 +198,50 @@ class _RequestTimeoutMiddleware(_BaseHTTPMiddleware):
)
class _InteractiveActivityMiddleware(_BaseHTTPMiddleware):
async def dispatch(self, request, call_next):
from src.interactive_gate import should_track_interactive_request, track_interactive_request
path = request.url.path or ""
if not should_track_interactive_request(path, request.method):
return await call_next(request)
async def _stop_background():
try:
await task_scheduler.stop_background_tasks_for_foreground(reason=f"foreground request {request.method} {path}")
except Exception:
logging.getLogger("app.foreground_gate").debug("foreground task stop failed", exc_info=True)
asyncio.create_task(_stop_background())
async with track_interactive_request(path, request.method):
return await call_next(request)
class _SlowRequestLogMiddleware(_BaseHTTPMiddleware):
async def dispatch(self, request, call_next):
start = time.perf_counter()
status = 500
try:
response = await call_next(request)
status = getattr(response, "status_code", 0) or 0
return response
finally:
elapsed = time.perf_counter() - start
try:
threshold = float(os.getenv("ODYSSEUS_SLOW_REQUEST_LOG_SECONDS", "0.75") or "0.75")
except Exception:
threshold = 0.75
if elapsed >= threshold:
logging.getLogger("app.slow_request").warning(
"slow_request method=%s path=%s status=%s elapsed=%.3fs",
request.method,
request.url.path,
status,
elapsed,
)
app.add_middleware(_RequestTimeoutMiddleware)
app.add_middleware(_InteractiveActivityMiddleware)
app.add_middleware(_SlowRequestLogMiddleware)
# ========= AUTH =========
from routes.auth_routes import setup_auth_routes, SESSION_COOKIE
@@ -217,8 +312,16 @@ if AUTH_ENABLED:
try:
rows = db.query(ApiToken).filter(ApiToken.is_active == True).all()
for r in rows:
owner_key = normalize_known_username(auth_manager.users, getattr(r, "owner", None))
if not owner_key:
logger.warning(
"Ignoring active API token '%s' for unknown auth user '%s'",
getattr(r, "id", ""),
getattr(r, "owner", None),
)
continue
scopes = [s.strip() for s in (getattr(r, "scopes", "") or "chat").split(",") if s.strip()]
new_map[r.token_prefix].append((r.id, r.token_hash, getattr(r, "owner", None), scopes))
new_map[r.token_prefix].append((r.id, r.token_hash, owner_key, scopes))
finally:
db.close()
_token_cache.clear()
@@ -253,6 +356,15 @@ if AUTH_ENABLED:
class AuthMiddleware(BaseHTTPMiddleware):
async def dispatch(self, request: Request, call_next):
path = request.url.path
# A genuine CORS preflight (OPTIONS + Access-Control-Request-Method)
# carries no credentials by design and must reach CORSMiddleware to be
# answered. AuthMiddleware is the outermost middleware, so gating the
# preflight on auth 401s it before CORS can respond -- which blocks
# every cross-origin browser/WebView client before the real request
# is sent. Let real preflights through (only OPTIONS w/ the ACRM
# header; never a credentialed request).
if is_cors_preflight(request.method, request.headers):
return await call_next(request)
if _is_auth_exempt(path):
return await call_next(request)
# In-process internal-tool token bypass. Used by the agent
@@ -260,7 +372,7 @@ if AUTH_ENABLED:
# (no admin cookie available in that context). Restricted to
# loopback clients + matching token to keep it locked down.
try:
from core.middleware import INTERNAL_TOOL_HEADER, INTERNAL_TOOL_TOKEN as _ITT
from core.middleware import INTERNAL_TOOL_HEADER, INTERNAL_TOOL_TOKEN as _ITT, INTERNAL_TOOL_USER
_hdr = request.headers.get(INTERNAL_TOOL_HEADER)
if _hdr and secrets.compare_digest(_hdr, _ITT) and _is_trusted_loopback(request):
# Impersonation: when the agent's loopback call sets
@@ -272,11 +384,11 @@ if AUTH_ENABLED:
if _impersonate and _impersonate in getattr(_auth_mgr, "users", {}):
request.state.current_user = _impersonate
else:
request.state.current_user = "internal-tool"
request.state.current_user = INTERNAL_TOOL_USER
request.state.api_token = False
return await call_next(request)
except Exception:
pass
except Exception as _e:
logger.warning("Internal tool auth header check failed", exc_info=_e)
# Allow DIRECT localhost requests (internal service calls from
# heartbeats etc.). Tunnel/proxy-forwarded requests are excluded by
# _is_trusted_loopback so LOCALHOST_BYPASS can't be abused over a
@@ -329,11 +441,10 @@ if AUTH_ENABLED:
_db.close()
try:
await _asyncio.to_thread(_do)
except Exception:
pass
except Exception as _e:
logger.debug("Failed to update token last_used_at", exc_info=_e)
_asyncio.create_task(_touch_last_used(matched_id))
# Keep bearer-token callers out of normal cookie/user
# routes. API-aware routes can read api_token_owner.
request.state.current_user = "api"
request.state.api_token = True
request.state.api_token_id = matched_id
@@ -382,7 +493,7 @@ class _RevalidatingStatic(StaticFiles):
return resp
app.mount("/static", _RevalidatingStatic(directory="static"), name="static")
app.mount("/static", _RevalidatingStatic(directory=STATIC_DIR), name="static")
# ========= GENERATED IMAGES =========
@app.get("/api/generated-image/{filename}")
@@ -408,8 +519,8 @@ async def serve_generated_image(filename: str, request: Request):
_db.close()
except HTTPException:
raise
except Exception:
pass
except Exception as _e:
logger.warning("Image ownership verification failed for %r", filename, exc_info=_e)
ext = filename.rsplit('.', 1)[-1].lower()
mime = {
"png": "image/png", "jpg": "image/jpeg", "jpeg": "image/jpeg",
@@ -463,14 +574,21 @@ components = initialize_managers(BASE_DIR, rag_manager)
session_manager = components["session_manager"]
from src.assistant_log import set_session_manager as _set_asst_sm
_set_asst_sm(session_manager)
# Set the global session manager singleton (used by core.models.Session.add_message)
from core.models import set_session_manager_instance
set_session_manager_instance(session_manager)
app.state.session_manager = session_manager
memory_manager = components["memory_manager"]
memory_vector = components.get("memory_vector")
upload_handler = components["upload_handler"]
app.state.upload_handler = upload_handler
personal_docs_mgr = components["personal_docs_manager"]
app.state.personal_docs_manager = personal_docs_mgr
api_key_manager = components["api_key_manager"]
preset_manager = components["preset_manager"]
chat_processor = components["chat_processor"]
research_handler = components["research_handler"]
app.state.research_handler = research_handler
chat_handler = components["chat_handler"]
model_discovery = components["model_discovery"]
skills_manager = components["skills_manager"]
@@ -509,6 +627,20 @@ webhook_manager = WebhookManager(api_key_manager=api_key_manager)
auth_router = setup_auth_routes(auth_manager)
app.include_router(auth_router)
@app.post("/api/activity/heartbeat")
async def activity_heartbeat():
from src.interactive_gate import mark_browser_activity
await mark_browser_activity()
async def _stop_background():
try:
await task_scheduler.stop_background_tasks_for_foreground(reason="browser heartbeat")
except Exception:
logging.getLogger("app.foreground_gate").debug("heartbeat task stop failed", exc_info=True)
asyncio.create_task(_stop_background())
return {"ok": True}
# Uploads
from routes.upload_routes import setup_upload_routes
upload_router, upload_cleanup_func = setup_upload_routes(upload_handler)
@@ -520,9 +652,6 @@ upload_cleanup_task = None
from routes.emoji_routes import setup_emoji_routes
app.include_router(setup_emoji_routes())
from routes.workspace_routes import setup_workspace_routes
app.include_router(setup_workspace_routes())
# Sessions
from routes.session_routes import setup_session_routes
session_config = {"REQUEST_TIMEOUT": REQUEST_TIMEOUT, "OPENAI_API_KEY": OPENAI_API_KEY, "SESSIONS_FILE": SESSIONS_FILE}
@@ -533,7 +662,7 @@ from routes.admin_wipe_routes import setup_admin_wipe_routes
app.include_router(setup_admin_wipe_routes(session_manager))
# Memory
from routes.memory_routes import setup_memory_routes
from routes.memory.memory_routes import setup_memory_routes
memory_router = setup_memory_routes(memory_manager, session_manager, memory_vector=memory_vector)
app.include_router(memory_router)
from routes.skills_routes import setup_skills_routes
@@ -550,11 +679,11 @@ app.include_router(setup_chat_routes(
))
# Research (background deep-research tasks)
from routes.research_routes import setup_research_routes
from routes.research.research_routes import setup_research_routes
app.include_router(setup_research_routes(research_handler, session_manager=session_manager))
# History
from routes.history_routes import setup_history_routes
from routes.history.history_routes import setup_history_routes
app.include_router(setup_history_routes(session_manager))
# Search
@@ -567,7 +696,7 @@ app.include_router(setup_preset_routes(preset_manager))
# Diagnostics
from routes.diagnostics_routes import setup_diagnostics_routes
app.include_router(setup_diagnostics_routes(rag_manager, rag_available, research_handler))
app.include_router(setup_diagnostics_routes(rag_manager, rag_available, research_handler, memory_vector))
# Cleanup
from routes.cleanup_routes import setup_cleanup_routes
@@ -589,6 +718,10 @@ app.include_router(setup_model_routes(model_discovery))
from routes.copilot_routes import setup_copilot_routes
app.include_router(setup_copilot_routes())
# ChatGPT Subscription device-flow login
from routes.chatgpt_subscription_routes import setup_chatgpt_subscription_routes
app.include_router(setup_chatgpt_subscription_routes())
# TTS
from routes.tts_routes import setup_tts_routes
app.include_router(setup_tts_routes(tts_service))
@@ -610,7 +743,7 @@ from routes.signature_routes import setup_signature_routes
app.include_router(setup_signature_routes())
# Gallery (image library)
from routes.gallery_routes import setup_gallery_routes
from routes.gallery.gallery_routes import setup_gallery_routes
app.include_router(setup_gallery_routes())
# Persisted image-editor drafts (server-backed projects)
@@ -641,6 +774,9 @@ app.include_router(setup_shell_routes())
from routes.cookbook_routes import setup_cookbook_routes
app.include_router(setup_cookbook_routes())
from routes.workspace_routes import setup_workspace_routes
app.include_router(setup_workspace_routes())
# Hardware model fitting (cookbook "What Fits?" tab)
from routes.hwfit_routes import setup_hwfit_routes
app.include_router(setup_hwfit_routes())
@@ -715,7 +851,7 @@ from routes.vault_routes import setup_vault_routes
app.include_router(setup_vault_routes())
# Contacts (CardDAV)
from routes.contacts_routes import setup_contacts_routes
from routes.contacts.contacts_routes import setup_contacts_routes
app.include_router(setup_contacts_routes())
from companion import setup_companion_routes
@@ -723,23 +859,17 @@ app.include_router(setup_companion_routes())
# ========= ROUTES (kept in app.py) =========
def _serve_html_with_nonce(request: Request, file_path: str) -> HTMLResponse:
"""Read an HTML file and inject the CSP nonce into inline <script> tags."""
with open(file_path, "r", encoding="utf-8") as f:
html = f.read()
nonce = getattr(request.state, "csp_nonce", "")
html = html.replace("{{CSP_NONCE}}", nonce)
return HTMLResponse(html)
@app.get("/")
async def serve_index(request: Request):
static_path = abs_join(BASE_DIR, "static/index.html")
if os.path.exists(static_path):
return _serve_html_with_nonce(request, static_path)
root_path = abs_join(BASE_DIR, "index.html")
if os.path.exists(root_path):
return _serve_html_with_nonce(request, root_path)
raise HTTPException(404, "index.html not found")
return serve_html_with_nonce(request, static_path)
# No static bundle — fall back to a root-level index.html if one is shipped.
# If neither exists, serve_html_with_nonce logs it and returns a generic 500:
# a missing index.html is a broken deployment (server fault), not a client
# "not found". This keeps the app-shell route consistent with the other
# bundled-template routes instead of mislabelling the fault as a 404.
return serve_html_with_nonce(request, abs_join(BASE_DIR, "index.html"))
@app.get("/notes")
async def serve_notes(request: Request):
@@ -780,11 +910,13 @@ async def serve_library(request: Request):
@app.get("/backgrounds")
async def serve_backgrounds(request: Request):
"""Sandbox page for prototyping background effects. No auth required."""
return _serve_html_with_nonce(request, abs_join(BASE_DIR, "static/backgrounds.html"))
return serve_html_with_nonce(request, abs_join(BASE_DIR, "static/backgrounds.html"))
@app.get("/login")
async def serve_login(request: Request):
return _serve_html_with_nonce(request, abs_join(BASE_DIR, "static/login.html"))
if not AUTH_ENABLED:
return RedirectResponse(url="/", status_code=302)
return serve_html_with_nonce(request, abs_join(BASE_DIR, "static/login.html"))
@app.get("/api/version")
async def get_version():
@@ -793,7 +925,35 @@ async def get_version():
@app.get("/api/health")
async def health_check() -> Dict[str, str]:
return {"status": "healthy", "timestamp": datetime.utcnow().isoformat()}
return {"status": "healthy", "timestamp": datetime.now(timezone.utc).isoformat()}
@app.post("/api/client-perf")
async def client_perf(request: Request):
"""Low-volume frontend timing reports for stalls that happen before SSE logs."""
try:
data = await request.json()
except Exception:
data = {}
try:
kind = str(data.get("type") or "client").replace("\n", " ")[:80]
total_ms = float(data.get("total_ms") or 0)
stages = data.get("stages") if isinstance(data.get("stages"), list) else []
stage_txt = " ".join(
f"{str(s.get('name') or '')[:40]}={float(s.get('delta_ms') or 0):.0f}ms"
for s in stages[:20]
if isinstance(s, dict)
)
extra = str(data.get("extra") or "").replace("\n", " ")[:200]
logging.getLogger("app.client_perf").warning(
"client_perf type=%s total=%.0fms %s%s",
kind,
total_ms,
stage_txt,
f" extra={extra}" if extra else "",
)
except Exception:
logging.getLogger("app.client_perf").debug("client_perf log failed", exc_info=True)
return {"ok": True}
@app.get("/api/ready")
async def readiness_check() -> JSONResponse:
@@ -891,59 +1051,66 @@ async def _startup_event():
_startup_tasks.append(asyncio.create_task(_startup_mcp_connections()))
# Pre-warm the RAG tool index off the request path. Loading the local
# embedding model + opening ChromaDB + indexing the built-in tools is a
# one-time ~1-3s cost that otherwise lands on the user's FIRST message
# (showing up as a big `tool_selection` time). Doing it here makes the
# first turn as fast as subsequent ones (warm embed ≈ a few ms).
async def _warmup_tool_index():
try:
from src.tool_index import get_tool_index
idx = await asyncio.to_thread(get_tool_index)
if idx:
await asyncio.to_thread(idx.get_tools_for_query, "warmup", 8)
logger.info("[startup] Tool index pre-warmed")
except Exception as e:
logger.warning(f"Tool index warmup failed (non-critical): {type(e).__name__}: {e}")
# Startup warmups are opt-in. They make later requests a little warmer, but
# they also compete with the first seconds of real UI use on slow or busy
# machines. Default to clear/idle startup and let requests warm what they use.
_startup_warmups_enabled = str(os.getenv("ODYSSEUS_STARTUP_WARMUPS", "")).lower() in {"1", "true", "yes", "on"}
if _startup_warmups_enabled:
async def _warmup_tool_index():
try:
from src.tool_index import get_tool_index
idx = await asyncio.to_thread(get_tool_index)
if idx:
await asyncio.to_thread(idx.get_tools_for_query, "warmup", 8)
logger.info("[startup] Tool index pre-warmed")
except Exception as e:
logger.warning(f"Tool index warmup failed (non-critical): {type(e).__name__}: {e}")
_startup_tasks.append(asyncio.create_task(_warmup_tool_index()))
# Warmup: ping all known LLM endpoints to prime connections
async def _warmup_endpoints():
try:
import httpx
endpoints = model_discovery.get_endpoints() if model_discovery else []
for ep in endpoints[:5]:
url = ep.get("url", "").replace("/chat/completions", "/models")
if url:
_startup_tasks.append(asyncio.create_task(_warmup_tool_index()))
async def _warmup_endpoints():
try:
import httpx
urls = (
await asyncio.to_thread(model_discovery.warmup_ping_urls)
if model_discovery else []
)
for url in urls:
try:
async with httpx.AsyncClient(timeout=5.0) as client:
await client.get(url)
logger.info(f"Warmup ping OK: {url}")
except Exception as e:
logger.debug(f"Warmup ping failed for endpoint: {e}")
except Exception as e:
logger.debug(f"Warmup ping skipped: {e}")
_startup_tasks.append(asyncio.create_task(_warmup_endpoints()))
# Keep-alive: ping endpoints every 60 seconds to prevent cold starts
async def _keepalive_loop():
while True:
try:
await asyncio.sleep(60)
await _warmup_endpoints()
except Exception as e:
logger.warning(f"Keepalive loop error: {e}")
await asyncio.sleep(300) # Back off on error
logger.debug(f"Warmup ping skipped: {e}")
_startup_tasks.append(asyncio.create_task(_keepalive_loop()))
_startup_tasks.append(asyncio.create_task(_warmup_endpoints()))
else:
logger.info("Startup warmups disabled (set ODYSSEUS_STARTUP_WARMUPS=1 to enable)")
# Keep-alive is opt-in. The ping path performs model discovery, and when
# stale LAN endpoints are configured it can add periodic backend pressure
# that delays unrelated UI requests such as Notes/Documents.
_keepalive_enabled = str(os.getenv("ODYSSEUS_MODEL_KEEPALIVE", "")).lower() in {"1", "true", "yes", "on"}
if _keepalive_enabled:
async def _keepalive_loop():
while True:
try:
await asyncio.sleep(60)
await _warmup_endpoints()
except Exception as e:
logger.warning(f"Keepalive loop error: {e}")
await asyncio.sleep(300) # Back off on error
_startup_tasks.append(asyncio.create_task(_keepalive_loop()))
async def _ensure_default_tasks():
# Create/reconcile default automation tasks + personal assistant for every user.
owners = set()
try:
import json as _json
auth_path = "data/auth.json"
auth_path = AUTH_FILE
with open(auth_path, encoding="utf-8") as f:
users = _json.load(f).get("users", {})
owners.update(users.keys())
@@ -990,7 +1157,7 @@ async def _startup_event():
# does not make an existing library look empty after auth/account changes.
try:
import json as _json
auth_path = "data/auth.json"
auth_path = AUTH_FILE
with open(auth_path, encoding="utf-8") as f:
users = _json.load(f).get("users", {})
primary_owner = None
@@ -1098,3 +1265,12 @@ async def _shutdown_event():
except Exception as e:
logger.warning(f"MCP shutdown error: {e}")
logger.info("Application shutdown complete")
if __name__ == "__main__":
import uvicorn
bind_host = os.getenv("APP_BIND", "127.0.0.1")
bind_port = int(os.getenv("APP_PORT", "7000"))
uvicorn.run(app, host=bind_host, port=bind_port, log_level="info")

72
build-windows-portable.ps1 Normal file
View File

@@ -0,0 +1,72 @@
#Requires -Version 5.1
<#
Build a portable Windows distribution for Odysseus.
Output layout:
dist\Odysseus\Odysseus.exe
dist\Odysseus\static\...
dist\Odysseus\scripts\...
dist\Odysseus\mcp_servers\...
dist\Odysseus\services\hwfit\data\...
The app then keeps using its normal filesystem layout when frozen.
Usage:
powershell -ExecutionPolicy Bypass -File .\build-windows-portable.ps1
#>
$ErrorActionPreference = "Stop"
Set-Location -Path $PSScriptRoot
function Write-Step($msg) { Write-Host ""; Write-Host ("==> " + $msg) -ForegroundColor Cyan }
function Fail($msg) {
Write-Host ""
Write-Host ("ERROR: " + $msg) -ForegroundColor Red
exit 1
}
Write-Step "Checking for Python"
$pyExe = $null
if (Test-Path ".\.venv\Scripts\python.exe") {
$pyExe = (Resolve-Path ".\.venv\Scripts\python.exe").Path
} else {
foreach ($c in @("py", "python")) {
$cmd = Get-Command $c -ErrorAction SilentlyContinue
if ($cmd) { $pyExe = $cmd.Source; break }
}
if ($pyExe -like "*WindowsApps*python.exe") {
$pyCmd = Get-Command py -ErrorAction SilentlyContinue
if ($pyCmd) {
$pyExe = $pyCmd.Source
}
}
}
if (-not $pyExe) {
Fail "Python not found on PATH. Install Python 3.11+ first."
}
Write-Host ("Using Python: " + $pyExe)
Write-Step "Installing build dependencies"
& $pyExe -m pip install --upgrade pip --quiet
& $pyExe -m pip install -r requirements.txt pyinstaller pystray Pillow
if ($LASTEXITCODE -ne 0) { Fail "Dependency install failed." }
Write-Step "Building portable exe bundle"
Remove-Item -Recurse -Force build, dist -ErrorAction SilentlyContinue
$dataArgs = @(
"--add-data", "static;static",
"--add-data", "scripts;scripts",
"--add-data", "mcp_servers;mcp_servers",
"--add-data", "services/hwfit/data;services/hwfit/data",
"--add-data", "config;config",
"--add-data", ".env.example;.env.example"
)
& $pyExe -m PyInstaller --noconfirm --clean --onedir --noconsole --icon=static/icon.ico --name Odysseus @dataArgs launcher.py
if ($LASTEXITCODE -ne 0) { Fail "PyInstaller build failed." }
Write-Host ""
Write-Host "Build complete." -ForegroundColor Green
Write-Host "Portable app folder: $PSScriptRoot\dist\Odysseus" -ForegroundColor Green
Write-Host "Distribute the whole folder (or zip it) so static assets and scripts stay with the exe." -ForegroundColor Green

4
companion/pairing.py
View File

@@ -14,6 +14,8 @@ import uuid
import bcrypt
from src.constants import AUTH_FILE
PAIRING_VERSION = 1
COMPANION_SCOPE = "chat"
@@ -61,7 +63,7 @@ def lan_ip_candidates() -> list[str]:
def find_admin_user() -> str | None:
"""Resolve an admin username from data/auth.json (schema uses is_admin),
falling back to the first user."""
auth_path = os.path.join("data", "auth.json")
auth_path = AUTH_FILE
try:
with open(auth_path, "r", encoding="utf-8") as f:
data = json.load(f)

20
companion/routes.py
View File

@@ -5,8 +5,9 @@ offers and pair to it, without duplicating any LLM logic.
Auth is enforced globally by AuthMiddleware (app.py), so reaching a handler here
means the caller is authenticated by either a cookie session or a Bearer `ody_`
API token. The read endpoints (ping/info/models) accept either; the pairing
endpoints are admin-cookie only.
API token. Ping/info accept either credential type, models requires a chat-
scoped API token for bearer callers, and the pairing endpoints are admin-cookie
only.
Pairing CSRF posture: minting happens ONLY on POST. The session cookie is
SameSite=Lax (routes/auth_routes.py), which a browser does not send on a
@@ -18,7 +19,7 @@ on a GET would be unsafe (Lax cookies ride top-level GET navigations), so GET
import html
from fastapi import APIRouter, Request
from fastapi import APIRouter, HTTPException, Request
from fastapi.responses import HTMLResponse
from core.middleware import require_admin
@@ -52,6 +53,18 @@ def owner_can_see(row_owner, owner) -> bool:
return row_owner is None or row_owner == owner
def require_models_scope(request: Request) -> None:
"""Require the companion chat scope for bearer-token model inventory."""
if not getattr(request.state, "api_token", False):
return
scopes = getattr(request.state, "api_token_scopes", None) or []
if isinstance(scopes, str):
scopes = [scope.strip() for scope in scopes.split(",")]
scope_set = {str(scope).strip() for scope in scopes if str(scope).strip()}
if _pairing.COMPANION_SCOPE not in scope_set:
raise HTTPException(403, "API token requires chat scope")
def mint_pairing_token(owner: str, invalidate=None) -> tuple[str, str]:
"""Mint a pairing token AND invalidate the auth middleware's in-memory token
cache, so the new token is accepted on the very next request without a server
@@ -103,6 +116,7 @@ def setup_companion_routes() -> APIRouter:
rows -- the same rule as owner_filter. Read-only; never returns api_key
material.
"""
require_models_scope(request)
import json as _json
from core.database import SessionLocal, ModelEndpoint

2
core/atomic_io.py
View File

@@ -34,6 +34,8 @@ def atomic_write_json(path: str, data: Any, *, indent: Optional[int] = None) ->
def atomic_write_text(path: str, text: str) -> None:
if not isinstance(text, str):
raise TypeError("atomic_write_text expects a string")
os.makedirs(os.path.dirname(path) or ".", exist_ok=True)
tmp = f"{path}.tmp.{os.getpid()}"
with open(tmp, "w", encoding="utf-8") as f:

208
core/auth.py
View File

@@ -3,6 +3,7 @@ Authentication module — multi-user password hashing, session tokens, config pe
Config stored in data/auth.json. Uses bcrypt directly.
"""
import enum
import json
import os
import secrets
@@ -19,6 +20,7 @@ logger = logging.getLogger(__name__)
from core.atomic_io import atomic_write_json as _atomic_write_json # noqa: E402
from core.middleware import INTERNAL_TOOL_USER # noqa: E402
DEFAULT_PRIVILEGES = {
"can_use_agent": True,
@@ -30,14 +32,24 @@ DEFAULT_PRIVILEGES = {
"can_manage_memory": True,
"max_messages_per_day": 0,
"allowed_models": [],
"allowed_models_restricted": False,
# Explicit "block every model" sentinel. An empty `allowed_models` list is
# ambiguous — it's also what gets sent when the admin clicks "[All]" — so
# we need a dedicated flag to express "this user may use no models at all"
# distinctly from "this user has no restriction".
"block_all_models": False,
}
# Admins get everything
ADMIN_PRIVILEGES = {k: (True if isinstance(v, bool) else (0 if isinstance(v, int) else [])) for k, v in DEFAULT_PRIVILEGES.items()}
ADMIN_PRIVILEGES["allowed_models_restricted"] = False
# Admins must never be blocked from using models — the generic dict
# comprehension above flips every boolean default to True, which would be
# backwards for this sentinel.
ADMIN_PRIVILEGES["block_all_models"] = False
DEFAULT_AUTH_PATH = os.path.join(
Path(__file__).parent.parent, "data", "auth.json"
)
from src.constants import AUTH_FILE, PASSWORD_MIN_LENGTH
DEFAULT_AUTH_PATH = AUTH_FILE
TOKEN_TTL = 60 * 60 * 24 * 7 # 7 days
# Usernames the auth + middleware layer reserve as internal "synthetic owner"
@@ -54,7 +66,15 @@ TOKEN_TTL = 60 * 60 * 24 * 7 # 7 days
# of those names would be denied an assistant and inconsistently owner-scoped.
# Refuse to create or rename into any of them so the sentinels can't be
# impersonated. (Keep this in sync with that synthetic-owner set.)
RESERVED_USERNAMES = frozenset({"internal-tool", "api", "demo", "system"})
RESERVED_USERNAMES = frozenset({INTERNAL_TOOL_USER, "api", "demo", "system"})
def normalize_known_username(users: Dict[str, Any], username: str | None) -> Optional[str]:
"""Return a normalized username only when it exists in the auth user map."""
key = str(username or "").strip().lower()
if not key or key not in users:
return None
return key
def _hash_password(password: str) -> str:
@@ -65,6 +85,15 @@ def _verify_password(password: str, hashed: str) -> bool:
return bcrypt.checkpw(password.encode("utf-8"), hashed.encode("utf-8"))
class SetAdminResult(enum.Enum):
"""Outcome of AuthManager.set_admin, so callers can map each case to a
precise response instead of guessing from a bare bool."""
OK = "ok"
USER_NOT_FOUND = "user_not_found"
NOT_AUTHORIZED = "not_authorized" # requester is not an admin
LAST_ADMIN = "last_admin" # would remove the last remaining admin
class AuthManager:
"""Manages multi-user password + session-token auth system."""
@@ -86,6 +115,7 @@ class AuthManager:
self._load()
self._load_sessions()
self._migrate_single_user()
self._drop_reserved_loaded_users()
self._migrate_legacy_admin_role()
def _load(self):
@@ -138,20 +168,52 @@ class AuthManager:
def _migrate_single_user(self):
"""Migrate old single-user format to multi-user format."""
if "password_hash" in self._config and "users" not in self._config:
old_user = self._config.get("username", "admin")
old_user = str(self._config.get("username", "admin") or "admin").strip().lower()
if old_user in RESERVED_USERNAMES:
logger.warning(
"Migrating legacy single-user reserved username '%s' to 'admin'",
old_user,
)
old_user = "admin"
old_hash = self._config["password_hash"]
self._config = {
"users": {
old_user: {
"password_hash": old_hash,
"created": time.time(),
"is_admin": True,
with self._config_lock:
self._config = {
"users": {
old_user: {
"password_hash": old_hash,
"created": time.time(),
"is_admin": True,
}
}
}
}
self._save()
self._save()
logger.info(f"Migrated single-user auth to multi-user (admin: {old_user})")
def _drop_reserved_loaded_users(self):
"""Fail closed for legacy/manual auth rows that collide with sentinels."""
users = self._config.get("users")
if not isinstance(users, dict):
return
normalized = {}
removed = []
for username, data in users.items():
key = str(username or "").strip().lower()
if not key:
continue
if key in RESERVED_USERNAMES:
removed.append(key)
continue
normalized[key] = data
if removed or normalized != users:
with self._config_lock:
self._config["users"] = normalized
self._save()
if removed:
logger.warning(
"Removed reserved username(s) from auth config: %s",
", ".join(sorted(set(removed))),
)
def _migrate_legacy_admin_role(self):
"""Normalize setup.py's old role='admin' marker to is_admin=True."""
changed = False
@@ -184,6 +246,15 @@ class AuthManager:
def is_configured(self) -> bool:
return len(self.users) > 0
def policy(self) -> dict:
"""Return public auth policy constants for the frontend."""
return {
"password_min_length": PASSWORD_MIN_LENGTH,
"reserved_usernames": sorted(RESERVED_USERNAMES),
"signup_enabled": self.signup_enabled,
"session_days": TOKEN_TTL // 86400,
}
# ------------------------------------------------------------------
# Account management
# ------------------------------------------------------------------
@@ -234,6 +305,22 @@ class AuthManager:
return False
if not self.users.get(requesting_user, {}).get("is_admin"):
return False
# Revoke API bearer tokens before removing the auth row. The bearer
# path authenticates from ApiToken rows and does not require the
# owner to still exist, so a successful delete must not leave active
# rows behind. If the token store is unavailable, fail closed and
# keep the user/session state intact so the admin can retry.
try:
from core.database import get_db_session, ApiToken
with get_db_session() as db:
removed_tokens = db.query(ApiToken).filter(ApiToken.owner == username).delete()
if removed_tokens:
logger.info(
f"Revoked {removed_tokens} API token(s) owned by deleted user '{username}'"
)
except Exception:
logger.warning(f"Failed to revoke API tokens for deleted user '{username}'")
return False
del self._config["users"][username]
self._save()
# Purge all sessions belonging to this user. validate_token doesn't
@@ -248,18 +335,6 @@ class AuthManager:
revoked += 1
if revoked:
self._save_sessions()
# Also revoke API bearer tokens owned by this user. The bearer auth
# path authenticates straight against ApiToken rows and never
# re-checks that the owner still exists, so leaving the rows behind
# would let a deleted user keep full API access indefinitely.
try:
from core.database import get_db_session, ApiToken
with get_db_session() as db:
removed = db.query(ApiToken).filter(ApiToken.owner == username).delete()
if removed:
logger.info(f"Revoked {removed} API token(s) owned by deleted user '{username}'")
except Exception:
logger.warning(f"Failed to revoke API tokens for deleted user '{username}'")
logger.info(f"Deleted user '{username}' (by {requesting_user}); revoked {revoked} active session(s)")
return True
@@ -334,6 +409,69 @@ class AuthManager:
logger.info(f"Updated privileges for '{username}': {current}")
return True
def set_admin(self, username: str, is_admin: bool,
requesting_user: str) -> SetAdminResult:
"""Promote/demote an existing user to/from admin. Admin only.
Refuses to remove the last remaining admin so the instance can never
be locked out of admin access; self-demotion is allowed as long as
another admin remains. Admin status is re-checked live on every
request, so unlike delete/rename no session or token revocation is
needed — a demoted admin simply fails the next is_admin() gate.
Promotion stashes the user's current privilege map and demotion
restores it, so a temporary admin stint can't silently broaden a
user's non-admin access; users without a stash (created as admin,
or promoted before stashing existed) demote to DEFAULT_PRIVILEGES.
Counting admins and flipping the flag happen in one critical section
so two concurrent demotions can't race the admin count to zero.
"""
username = (username or "").strip().lower()
requesting_user = (requesting_user or "").strip().lower()
is_admin = bool(is_admin)
with self._config_lock:
target = self._config.get("users", {}).get(username)
if target is None:
return SetAdminResult.USER_NOT_FOUND
if not self.users.get(requesting_user, {}).get("is_admin"):
return SetAdminResult.NOT_AUTHORIZED
currently_admin = bool(target.get("is_admin"))
if currently_admin == is_admin:
return SetAdminResult.OK # no-op; leave privileges untouched
if currently_admin and not is_admin:
admin_count = sum(1 for d in self.users.values() if d.get("is_admin"))
if admin_count <= 1:
return SetAdminResult.LAST_ADMIN
# Write order matters for lock-free readers: get_privileges()
# reads without _config_lock and trusts is_admin, so the admin
# flag must be flipped while the stored map is safe to expose —
# before writing admin privileges on promote, after restoring
# the pre-admin map on demote.
if is_admin:
target["is_admin"] = True
# Stash the pre-admin map so a later demotion can restore it.
# While is_admin is set the stored map is inert: get_privileges
# short-circuits to ADMIN_PRIVILEGES and set_privileges refuses
# admins, so only set_admin ever touches the stash.
target["privileges_before_admin"] = dict(
target.get("privileges") or DEFAULT_PRIVILEGES
)
target["privileges"] = dict(ADMIN_PRIVILEGES)
else:
# Restore the stashed pre-admin map. Fall back to defaults for
# users created as admins (their stored map is ADMIN_PRIVILEGES,
# which must not leak past demotion — e.g. can_use_bash) and
# for admins promoted before the stash existed.
target["privileges"] = dict(
target.pop("privileges_before_admin", None)
or DEFAULT_PRIVILEGES
)
target["is_admin"] = False
self._save()
logger.info("Set is_admin=%s for '%s' (by '%s')", is_admin, username, requesting_user)
return SetAdminResult.OK
def change_password(self, username: str, current_password: str, new_password: str) -> bool:
username = username.strip().lower()
if username not in self.users:
@@ -445,12 +583,22 @@ class AuthManager:
username = username.strip().lower()
if not self.verify_password(username, password):
return None
return self.create_session_trusted(username)
def create_session_trusted(self, username: str) -> Optional[str]:
"""Issue a session token for an already-verified user.
Call only after verify_password (and TOTP if enabled) have passed."""
username = username.strip().lower()
token = secrets.token_hex(32)
with self._sessions_lock:
self._sessions[token] = {
"username": username,
"expiry": time.time() + TOKEN_TTL,
}
with self._config_lock:
if username not in self.users:
logger.warning("Refused to issue session for missing user '%s'", username)
return None
with self._sessions_lock:
self._sessions[token] = {
"username": username,
"expiry": time.time() + TOKEN_TTL,
}
self._save_sessions()
return token

50
core/constants.py
View File

@@ -1,40 +1,12 @@
# src/constants.py
"""Application-wide constants and configuration values."""
import os
# core/constants.py
"""Backward-compatible shim — the single source of truth is src/constants.py.
APP_VERSION = "0.9.1"
# Base paths
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) + "/"
STATIC_DIR = os.path.join(BASE_DIR, "static")
DATA_DIR = os.path.join(BASE_DIR, "data")
# Data file paths
SESSIONS_FILE = os.path.join(DATA_DIR, "sessions.json")
MEMORY_FILE = os.path.join(DATA_DIR, "memory.json")
MEMORY_DOC = os.path.join(DATA_DIR, "memory_doc.md")
PERSONAL_DIR = os.path.join(DATA_DIR, "personal_docs")
RUNBOOK_DIR = os.path.join(PERSONAL_DIR, "runbook")
UPLOAD_DIR = os.path.join(DATA_DIR, "uploads")
FEATURES_FILE = os.path.join(DATA_DIR, "features.json")
SETTINGS_FILE = os.path.join(DATA_DIR, "settings.json")
# API Configuration
MAX_CONTEXT_MESSAGES = 90
REQUEST_TIMEOUT = 20
OPENAI_COMPAT_PATH = "/v1/chat/completions"
# Environment variables with defaults
DEFAULT_HOST = os.getenv("LLM_HOST", "localhost")
LLM_HOSTS = [h.strip() for h in os.getenv("LLM_HOSTS", "").split(",") if h.strip()]
OPENAI_API_KEY = os.getenv("OPENAI_API_KEY")
SEARXNG_INSTANCE = os.getenv('SEARXNG_INSTANCE', 'http://localhost:8080')
# Cleanup configuration
CLEANUP_ENABLED = os.getenv("CLEANUP_ENABLED", "True").lower() == "true"
CLEANUP_INTERVAL_HOURS = int(os.getenv("CLEANUP_INTERVAL_HOURS", "24"))
# Default parameters
DEFAULT_TEMPERATURE = 1.0
DEFAULT_MAX_TOKENS = 0
Historically there were two copies of this module (this one lagged behind at
APP_VERSION 0.9.1 and was missing the consolidated tool-output constants). To
kill the drift, this now simply re-exports everything from src.constants so
there is exactly one place that defines paths and reads ODYSSEUS_DATA_DIR.
internal_api_base() also lives in src.constants now and is re-exported here so
existing `from core.constants import internal_api_base` callers keep working.
"""
from src.constants import * # noqa: F401,F403
from src.constants import internal_api_base # noqa: F401 (explicit: functions aren't covered by some linters' * checks)

486
core/database.py
View File

@@ -2,12 +2,15 @@ import os
import logging
import sqlite3
from datetime import datetime, timezone
from pathlib import Path
from sqlalchemy import event, create_engine, Column, String, Text, Boolean, DateTime, Integer, ForeignKey, JSON, Index, func, text
from sqlalchemy.engine import Engine
from sqlalchemy.types import TypeDecorator
from sqlalchemy.ext.declarative import declarative_base, declared_attr
from sqlalchemy.orm import relationship, sessionmaker, backref
from src.runtime_paths import get_app_root
logger = logging.getLogger(__name__)
# Create base class for declarative models
@@ -29,8 +32,26 @@ class TimestampMixin:
def updated_at(cls):
return Column(DateTime, default=utcnow_naive, onupdate=utcnow_naive, nullable=False)
# Get database URL from environment, default to SQLite
DATABASE_URL = os.getenv("DATABASE_URL", "sqlite:///./data/app.db")
# Ensure the writable data directory exists before SQLite connects.
from src.constants import DATA_DIR, AUTH_FILE, MEMORY_FILE, USER_PREFS_FILE, SETTINGS_FILE
Path(DATA_DIR).mkdir(parents=True, exist_ok=True)
def _default_database_url() -> str:
return f"sqlite:///{Path(DATA_DIR) / 'app.db'}"
def _normalize_sqlite_url(url: str) -> str:
if not url.startswith("sqlite:///"):
return url
db_path = url.replace("sqlite:///", "", 1)
if db_path == ":memory:" or os.path.isabs(db_path):
return url
return f"sqlite:///{(Path(get_app_root()) / db_path).resolve().as_posix()}"
# Get database URL from environment, default to SQLite in DATA_DIR
DATABASE_URL = _normalize_sqlite_url(os.getenv("DATABASE_URL", _default_database_url()))
# Create engine
engine = create_engine(
@@ -255,6 +276,7 @@ class GalleryImage(TimestampMixin, Base):
id = Column(String, primary_key=True, index=True)
filename = Column(String, nullable=False, unique=True)
prompt = Column(Text, nullable=False, default="")
caption = Column(Text, nullable=True, default="")
model = Column(String, nullable=True)
size = Column(String, nullable=True)
quality = Column(String, nullable=True)
@@ -323,6 +345,13 @@ class EmailAccount(TimestampMixin, Base):
smtp_password = Column(String, default="")
from_address = Column(String, default="")
display_name = Column(String, nullable=True) # "Hriday Ranka" — used in From: header
# OAuth2 (Google / Google Workspace). Tokens stored encrypted via secret_storage.
oauth_provider = Column(String, nullable=True) # "google" or None
oauth_access_token = Column(String, nullable=True) # encrypted
oauth_refresh_token = Column(String, nullable=True) # encrypted
oauth_token_expiry = Column(String, nullable=True) # unix timestamp string
__table_args__ = (
Index('ix_email_accounts_owner_default', 'owner', 'is_default'),
@@ -360,6 +389,24 @@ class ModelEndpoint(TimestampMixin, Base):
# is the historical default. When non-null, the model picker only shows
# the endpoint to that user (admins always see everything).
owner = Column(String, nullable=True, index=True)
# Optional OAuth/session-backed credential row. Used by subscription-backed
# providers that need refresh tokens instead of a static API key.
provider_auth_id = Column(String, nullable=True, index=True)
class ProviderAuthSession(TimestampMixin, Base):
"""Encrypted OAuth/session credentials for refresh-aware model providers."""
__tablename__ = "provider_auth_sessions"
id = Column(String, primary_key=True, index=True)
provider = Column(String, nullable=False, index=True)
owner = Column(String, nullable=True, index=True)
label = Column(String, nullable=True)
base_url = Column(String, nullable=False)
access_token = Column(EncryptedText, nullable=True)
refresh_token = Column(EncryptedText, nullable=True)
last_refresh = Column(DateTime, nullable=True)
auth_mode = Column(String, nullable=True)
class McpServer(TimestampMixin, Base):
"""Admin-configured MCP (Model Context Protocol) tool servers."""
@@ -669,6 +716,7 @@ def _migrate_add_last_message_at_column():
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
conn = None
try:
conn = sqlite3.connect(db_path)
cursor = conn.execute("PRAGMA table_info(sessions)")
@@ -694,10 +742,14 @@ def _migrate_add_last_message_at_column():
"ON sessions(archived, last_message_at)"
)
conn.commit()
conn.close()
logging.getLogger(__name__).info("Migrated: added + backfilled 'last_message_at' on sessions")
except Exception as e:
logging.getLogger(__name__).warning(f"last_message_at migration failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
def _migrate_add_document_archived_column():
"""Add `archived` to documents (soft-archive flag). Guarded + idempotent."""
@@ -705,6 +757,7 @@ def _migrate_add_document_archived_column():
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
conn = None
try:
conn = sqlite3.connect(db_path)
cursor = conn.execute("PRAGMA table_info(documents)")
@@ -713,9 +766,13 @@ def _migrate_add_document_archived_column():
conn.execute("ALTER TABLE documents ADD COLUMN archived BOOLEAN DEFAULT 0")
conn.commit()
logging.getLogger(__name__).info("Migrated: added 'archived' to documents")
conn.close()
except Exception as e:
logging.getLogger(__name__).warning(f"documents.archived migration failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
def _migrate_add_owner_column():
@@ -724,6 +781,7 @@ def _migrate_add_owner_column():
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
conn = None
try:
conn = sqlite3.connect(db_path)
cursor = conn.execute("PRAGMA table_info(sessions)")
@@ -733,9 +791,13 @@ def _migrate_add_owner_column():
conn.execute("CREATE INDEX IF NOT EXISTS ix_sessions_owner ON sessions(owner)")
conn.commit()
logging.getLogger(__name__).info("Migrated: added 'owner' column to sessions")
conn.close()
except Exception as e:
logging.getLogger(__name__).warning(f"Migration check failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
def _migrate_model_endpoints():
"""Recreate model_endpoints table if schema changed (url->base_url)."""
@@ -743,6 +805,7 @@ def _migrate_model_endpoints():
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
conn = None
try:
conn = sqlite3.connect(db_path)
cursor = conn.execute("PRAGMA table_info(model_endpoints)")
@@ -751,9 +814,13 @@ def _migrate_model_endpoints():
conn.execute("DROP TABLE IF EXISTS model_endpoints")
conn.commit()
logging.getLogger(__name__).info("Migrated: dropped old model_endpoints table (schema change)")
conn.close()
except Exception as e:
logging.getLogger(__name__).warning(f"model_endpoints migration check failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
def _migrate_add_hidden_models_column():
"""Add hidden_models column to model_endpoints if it doesn't exist."""
@@ -761,6 +828,7 @@ def _migrate_add_hidden_models_column():
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
conn = None
try:
conn = sqlite3.connect(db_path)
cursor = conn.execute("PRAGMA table_info(model_endpoints)")
@@ -769,9 +837,13 @@ def _migrate_add_hidden_models_column():
conn.execute("ALTER TABLE model_endpoints ADD COLUMN hidden_models TEXT")
conn.commit()
logging.getLogger(__name__).info("Migrated: added 'hidden_models' column to model_endpoints")
conn.close()
except Exception as e:
logging.getLogger(__name__).warning(f"hidden_models migration failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
def _migrate_add_model_endpoint_owner_column():
"""Add owner column to model_endpoints if it doesn't exist.
@@ -786,6 +858,7 @@ def _migrate_add_model_endpoint_owner_column():
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
conn = None
try:
conn = sqlite3.connect(db_path)
cursor = conn.execute("PRAGMA table_info(model_endpoints)")
@@ -795,9 +868,38 @@ def _migrate_add_model_endpoint_owner_column():
conn.execute("CREATE INDEX IF NOT EXISTS ix_model_endpoints_owner ON model_endpoints(owner)")
conn.commit()
logging.getLogger(__name__).info("Migrated: added 'owner' column + index to model_endpoints")
conn.close()
except Exception as e:
logging.getLogger(__name__).warning(f"model_endpoints.owner migration failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
def _migrate_add_provider_auth_id_column():
"""Add provider_auth_id column to model_endpoints if it doesn't exist."""
import sqlite3
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
conn = None
try:
conn = sqlite3.connect(db_path)
cursor = conn.execute("PRAGMA table_info(model_endpoints)")
columns = [row[1] for row in cursor.fetchall()]
if columns and "provider_auth_id" not in columns:
conn.execute("ALTER TABLE model_endpoints ADD COLUMN provider_auth_id VARCHAR")
conn.execute("CREATE INDEX IF NOT EXISTS ix_model_endpoints_provider_auth_id ON model_endpoints(provider_auth_id)")
conn.commit()
logging.getLogger(__name__).info("Migrated: added 'provider_auth_id' column + index to model_endpoints")
except Exception as e:
logging.getLogger(__name__).warning(f"model_endpoints.provider_auth_id migration failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
def _migrate_add_model_type_column():
@@ -806,6 +908,7 @@ def _migrate_add_model_type_column():
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
conn = None
try:
conn = sqlite3.connect(db_path)
cursor = conn.execute("PRAGMA table_info(model_endpoints)")
@@ -814,9 +917,13 @@ def _migrate_add_model_type_column():
conn.execute("ALTER TABLE model_endpoints ADD COLUMN model_type TEXT DEFAULT 'llm'")
conn.commit()
logging.getLogger(__name__).info("Migrated: added 'model_type' column to model_endpoints")
conn.close()
except Exception as e:
logging.getLogger(__name__).warning(f"model_type migration failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
def _migrate_add_model_endpoint_refresh_columns():
"""Add endpoint classification / refresh policy columns if missing."""
@@ -824,6 +931,7 @@ def _migrate_add_model_endpoint_refresh_columns():
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
conn = None
try:
conn = sqlite3.connect(db_path)
cursor = conn.execute("PRAGMA table_info(model_endpoints)")
@@ -837,9 +945,13 @@ def _migrate_add_model_endpoint_refresh_columns():
if columns and "model_refresh_timeout" not in columns:
conn.execute("ALTER TABLE model_endpoints ADD COLUMN model_refresh_timeout INTEGER")
conn.commit()
conn.close()
except Exception as e:
logging.getLogger(__name__).warning(f"model_endpoints refresh-policy migration failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
def _migrate_add_task_run_model_column():
"""Add model column to task_runs if it doesn't exist (records which model ran)."""
@@ -847,6 +959,7 @@ def _migrate_add_task_run_model_column():
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
conn = None
try:
conn = sqlite3.connect(db_path)
cursor = conn.execute("PRAGMA table_info(task_runs)")
@@ -855,9 +968,13 @@ def _migrate_add_task_run_model_column():
conn.execute("ALTER TABLE task_runs ADD COLUMN model TEXT")
conn.commit()
logging.getLogger(__name__).info("Migrated: added 'model' column to task_runs")
conn.close()
except Exception as e:
logging.getLogger(__name__).warning(f"task_runs model migration failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
def _migrate_add_supports_tools_column():
"""Add supports_tools column to model_endpoints if it doesn't exist."""
@@ -865,6 +982,7 @@ def _migrate_add_supports_tools_column():
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
conn = None
try:
conn = sqlite3.connect(db_path)
cursor = conn.execute("PRAGMA table_info(model_endpoints)")
@@ -873,9 +991,13 @@ def _migrate_add_supports_tools_column():
conn.execute("ALTER TABLE model_endpoints ADD COLUMN supports_tools BOOLEAN")
conn.commit()
logging.getLogger(__name__).info("Migrated: added 'supports_tools' column to model_endpoints")
conn.close()
except Exception as e:
logging.getLogger(__name__).warning(f"supports_tools migration failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
def _migrate_add_cached_models_column():
@@ -884,6 +1006,7 @@ def _migrate_add_cached_models_column():
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
conn = None
try:
conn = sqlite3.connect(db_path)
cursor = conn.execute("PRAGMA table_info(model_endpoints)")
@@ -891,9 +1014,13 @@ def _migrate_add_cached_models_column():
if columns and "cached_models" not in columns:
conn.execute("ALTER TABLE model_endpoints ADD COLUMN cached_models TEXT")
conn.commit()
conn.close()
except Exception as e:
logging.getLogger(__name__).warning(f"cached_models migration failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
def _migrate_add_pinned_models_column():
"""Add pinned_models column to model_endpoints if it doesn't exist."""
@@ -901,6 +1028,7 @@ def _migrate_add_pinned_models_column():
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
conn = None
try:
conn = sqlite3.connect(db_path)
cursor = conn.execute("PRAGMA table_info(model_endpoints)")
@@ -909,9 +1037,13 @@ def _migrate_add_pinned_models_column():
conn.execute("ALTER TABLE model_endpoints ADD COLUMN pinned_models TEXT")
conn.commit()
logging.getLogger(__name__).info("Migrated: added 'pinned_models' column to model_endpoints")
conn.close()
except Exception as e:
logging.getLogger(__name__).warning(f"pinned_models migration failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
def _migrate_add_notes_sort_order():
"""Add sort_order, image_url, repeat columns to notes if they don't exist."""
@@ -919,6 +1051,7 @@ def _migrate_add_notes_sort_order():
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
conn = None
try:
conn = sqlite3.connect(db_path)
cursor = conn.execute("PRAGMA table_info(notes)")
@@ -936,9 +1069,13 @@ def _migrate_add_notes_sort_order():
if columns and "agent_session_id" not in columns:
conn.execute("ALTER TABLE notes ADD COLUMN agent_session_id TEXT")
conn.commit()
conn.close()
except Exception as e:
logging.getLogger(__name__).warning(f"notes migration failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
def _migrate_add_mode_column():
"""Add mode column to sessions table if it doesn't exist."""
@@ -946,6 +1083,7 @@ def _migrate_add_mode_column():
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
conn = None
try:
conn = sqlite3.connect(db_path)
cursor = conn.execute("PRAGMA table_info(sessions)")
@@ -954,9 +1092,13 @@ def _migrate_add_mode_column():
conn.execute("ALTER TABLE sessions ADD COLUMN mode TEXT")
conn.commit()
logging.getLogger(__name__).info("Migrated: added 'mode' column to sessions")
conn.close()
except Exception as e:
logging.getLogger(__name__).warning(f"Migration check for mode failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
def _migrate_add_folder_column():
"""Add folder column to sessions table if it doesn't exist."""
@@ -964,6 +1106,7 @@ def _migrate_add_folder_column():
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
conn = None
try:
conn = sqlite3.connect(db_path)
cursor = conn.execute("PRAGMA table_info(sessions)")
@@ -972,9 +1115,13 @@ def _migrate_add_folder_column():
conn.execute("ALTER TABLE sessions ADD COLUMN folder TEXT")
conn.commit()
logging.getLogger(__name__).info("Migrated: added 'folder' column to sessions")
conn.close()
except Exception as e:
logging.getLogger(__name__).warning(f"Migration check for folder failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
def _migrate_add_token_columns():
"""Add cumulative token tracking columns to sessions table."""
@@ -982,6 +1129,7 @@ def _migrate_add_token_columns():
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
conn = None
try:
conn = sqlite3.connect(db_path)
cursor = conn.execute("PRAGMA table_info(sessions)")
@@ -991,9 +1139,13 @@ def _migrate_add_token_columns():
conn.execute("ALTER TABLE sessions ADD COLUMN total_output_tokens INTEGER DEFAULT 0")
conn.commit()
logging.getLogger(__name__).info("Migrated: added token tracking columns to sessions")
conn.close()
except Exception as e:
logging.getLogger(__name__).warning(f"Migration check for token columns failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
def _migrate_add_owner_to_table(table_name: str, index_name: str):
"""Generic helper: add owner TEXT column + index to a table if missing."""
@@ -1001,6 +1153,7 @@ def _migrate_add_owner_to_table(table_name: str, index_name: str):
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
conn = None
try:
conn = sqlite3.connect(db_path)
cursor = conn.execute(f"PRAGMA table_info({table_name})")
@@ -1010,9 +1163,13 @@ def _migrate_add_owner_to_table(table_name: str, index_name: str):
conn.execute(f"CREATE INDEX IF NOT EXISTS {index_name} ON {table_name}(owner)")
conn.commit()
logging.getLogger(__name__).info(f"Migrated: added 'owner' column to {table_name}")
conn.close()
except Exception as e:
logging.getLogger(__name__).warning(f"Migration owner column for {table_name} failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
def _migrate_add_multiuser_owner_columns():
"""Add owner column to memories, gallery_images, user_tools, comparisons."""
@@ -1026,6 +1183,29 @@ def _migrate_add_multiuser_owner_columns():
_migrate_add_owner_to_table("documents", "ix_documents_owner")
def _migrate_add_gallery_caption_column():
"""Add OCR/vision caption storage for gallery images."""
import sqlite3
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
conn = None
try:
conn = sqlite3.connect(db_path)
columns = [row[1] for row in conn.execute("PRAGMA table_info(gallery_images)").fetchall()]
if columns and "caption" not in columns:
conn.execute("ALTER TABLE gallery_images ADD COLUMN caption TEXT DEFAULT ''")
conn.commit()
logging.getLogger(__name__).info("Migrated: added caption column to gallery_images")
except Exception as e:
logging.getLogger(__name__).warning(f"Migration gallery caption column failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
def _migrate_add_api_token_scopes_column():
"""Add API token scopes for existing installs.
@@ -1037,6 +1217,7 @@ def _migrate_add_api_token_scopes_column():
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
conn = None
try:
conn = sqlite3.connect(db_path)
columns = [row[1] for row in conn.execute("PRAGMA table_info(api_tokens)").fetchall()]
@@ -1045,9 +1226,13 @@ def _migrate_add_api_token_scopes_column():
conn.execute("UPDATE api_tokens SET scopes = 'chat' WHERE scopes IS NULL OR scopes = ''")
conn.commit()
logging.getLogger(__name__).info("Migrated: added scopes column to api_tokens")
conn.close()
except Exception as e:
logging.getLogger(__name__).warning(f"api_tokens.scopes migration failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
def _migrate_assign_legacy_owner():
"""Assign all null-owner data to the first (admin) user.
@@ -1065,7 +1250,7 @@ def _migrate_assign_legacy_owner():
# fell through to "first user" every time.
auth_path = os.path.join(os.path.dirname(DATABASE_URL.replace("sqlite:///", "")), "auth.json")
if not os.path.isabs(auth_path):
auth_path = os.path.join("data", "auth.json")
auth_path = AUTH_FILE
admin_user = None
try:
with open(auth_path, "r", encoding="utf-8") as f:
@@ -1089,6 +1274,7 @@ def _migrate_assign_legacy_owner():
return
logger = logging.getLogger(__name__)
conn = None
try:
conn = sqlite3.connect(db_path)
# Every table with an `owner` column. New tables added later will be
@@ -1113,12 +1299,16 @@ def _migrate_assign_legacy_owner():
except Exception as e:
logger.warning(f"Legacy owner assignment for {table} failed: {e}")
conn.commit()
conn.close()
except Exception as e:
logger.warning(f"Legacy owner migration failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
# Also migrate memory.json
mem_path = os.path.join("data", "memory.json")
mem_path = MEMORY_FILE
try:
if os.path.exists(mem_path):
with open(mem_path, "r", encoding="utf-8") as f:
@@ -1136,7 +1326,7 @@ def _migrate_assign_legacy_owner():
logger.warning(f"memory.json legacy migration failed: {e}")
# Also migrate user_prefs.json to per-user format
prefs_path = os.path.join("data", "user_prefs.json")
prefs_path = USER_PREFS_FILE
try:
if os.path.exists(prefs_path):
with open(prefs_path, "r", encoding="utf-8") as f:
@@ -1288,6 +1478,25 @@ def _migrate_add_task_automation_columns():
except Exception as e:
logging.getLogger(__name__).warning(f"task automation migration: {e}")
def _migrate_add_email_oauth_columns():
"""Add Google OAuth and display_name columns to email_accounts if missing."""
try:
with engine.connect() as conn:
cols = [r[1] for r in conn.execute(text("PRAGMA table_info(email_accounts)"))]
for col, typedef in [
("oauth_provider", "TEXT"),
("oauth_access_token", "TEXT"),
("oauth_refresh_token", "TEXT"),
("oauth_token_expiry", "TEXT"),
("display_name", "TEXT"),
]:
if col not in cols:
conn.execute(text(f"ALTER TABLE email_accounts ADD COLUMN {col} {typedef}"))
conn.commit()
except Exception as e:
logging.getLogger(__name__).warning(f"email oauth columns migration: {e}")
def _migrate_add_oauth_config():
"""Add oauth_config column to mcp_servers table if missing."""
try:
@@ -1458,7 +1667,12 @@ class CalendarCal(TimestampMixin, Base):
owner = Column(String, nullable=True, index=True)
name = Column(String, nullable=False)
color = Column(String, default="#5b8abf")
source = Column(String, default="local") # "local" or "timetree"
source = Column(String, default="local") # "local" or "caldav"
# UUID of the CalDAV account in user prefs that owns this calendar.
# NULL for local calendars and for CalDAV calendars created before
# multi-account support was added (treated as "use any configured account").
account_id = Column(String, nullable=True, index=True)
caldav_base_url = Column(String, nullable=True)
events = relationship("CalendarEvent", back_populates="calendar", cascade="all, delete-orphan")
@@ -1480,6 +1694,7 @@ class CalendarEvent(TimestampMixin, Base):
# `Z`-suffix on serialization so the frontend interprets correctly.
is_utc = Column(Boolean, default=False, nullable=False)
rrule = Column(String, default="")
recurrence_exdates = Column(Text, default="") # JSON list of skipped occurrence starts
color = Column(String, nullable=True) # per-event color override
status = Column(String, default="confirmed") # confirmed, cancelled
importance = Column(String, default="normal") # low | normal | high | critical
@@ -1489,10 +1704,27 @@ class CalendarEvent(TimestampMixin, Base):
# vanishes upstream). NULL/local = created locally (agent, email triage, or
# a UI event whose write-back failed) and must NOT be pruned by the sync.
origin = Column(String, nullable=True, index=True)
remote_href = Column(String, nullable=True) # CalDAV object URL for updates/deletes
remote_etag = Column(String, nullable=True) # Last seen CalDAV ETag, when available
caldav_sync_pending = Column(String, nullable=True) # create | update | delete retry marker
calendar = relationship("CalendarCal", back_populates="events")
class CalendarDeletedEvent(TimestampMixin, Base):
"""Hidden CalDAV delete tombstone retained until remote delete succeeds."""
__tablename__ = "caldav_deleted_events"
uid = Column(String, primary_key=True, index=True)
owner = Column(String, nullable=True, index=True)
calendar_id = Column(String, nullable=True, index=True)
remote_href = Column(String, nullable=True)
remote_etag = Column(String, nullable=True)
caldav_base_url = Column(String, nullable=True)
summary = Column(String, nullable=True)
last_error = Column(Text, nullable=True)
class Integration(TimestampMixin, Base):
"""An external service connection (email, RSS, webhook, etc.)."""
__tablename__ = "integrations"
@@ -1526,7 +1758,7 @@ def _migrate_seed_email_account():
import json as _json
import uuid as _uuid
from pathlib import Path
settings_file = Path("data/settings.json")
settings_file = Path(SETTINGS_FILE)
if not settings_file.exists():
return
try:
@@ -1594,6 +1826,7 @@ def init_db():
_migrate_add_model_type_column()
_migrate_add_model_endpoint_refresh_columns()
_migrate_add_model_endpoint_owner_column()
_migrate_add_provider_auth_id_column()
_migrate_add_supports_tools_column()
_migrate_add_task_run_model_column()
_migrate_add_owner_column()
@@ -1603,12 +1836,14 @@ def init_db():
_migrate_add_token_columns()
_migrate_add_mode_column()
_migrate_add_multiuser_owner_columns()
_migrate_add_gallery_caption_column()
_migrate_add_api_token_scopes_column()
_migrate_backfill_document_owner_from_session()
_migrate_assign_legacy_owner()
_migrate_add_tidy_verdict()
_migrate_add_doc_source_email_cols()
_migrate_add_oauth_config()
_migrate_add_email_oauth_columns()
_migrate_add_task_automation_columns()
_migrate_add_disabled_tools()
_migrate_add_mcp_oauth_tokens_column()
@@ -1622,9 +1857,107 @@ def init_db():
_migrate_add_calendar_metadata()
_migrate_add_calendar_is_utc()
_migrate_add_calendar_origin()
_migrate_add_calendar_account_id()
_migrate_add_caldav_sync_columns()
_migrate_add_calendar_recurrence_exdates()
_migrate_chat_messages_fts()
_migrate_encrypt_email_passwords()
_migrate_encrypt_signatures()
_migrate_encrypt_endpoint_keys()
_migrate_backfill_task_folders()
def _migrate_backfill_task_folders():
"""Backfill folder='Tasks' on pre-existing task/research sessions.
Sessions created by the task scheduler (LLM tasks, action tasks, research
runs) now set folder='Tasks' at creation time. This migration tags any
older sessions that predate that assignment. Idempotent — only touches
rows where folder is NULL or empty and the title matches known prefixes.
"""
try:
with engine.connect() as conn:
cols = [r[1] for r in conn.execute(text("PRAGMA table_info(sessions)"))]
if "folder" not in cols:
return
res = conn.execute(text(
"UPDATE sessions SET folder = 'Tasks' "
"WHERE (folder IS NULL OR folder = '') "
"AND (name LIKE '[Task] %' OR name LIKE '[Research] %')"
))
conn.commit()
if res.rowcount:
logging.getLogger(__name__).info(
f"Backfilled folder='Tasks' on {res.rowcount} task/research sessions")
except Exception as e:
logging.getLogger(__name__).warning(f"task folder backfill: {e}")
def _migrate_chat_messages_fts():
"""Create and backfill the session transcript FTS index for SQLite."""
if not DATABASE_URL.startswith("sqlite"):
return
db_path = DATABASE_URL.replace("sqlite:///", "")
if db_path == ":memory:":
return
conn = None
try:
conn = sqlite3.connect(db_path)
try:
conn.execute("CREATE VIRTUAL TABLE IF NOT EXISTS temp._odysseus_fts5_probe USING fts5(content)")
conn.execute("DROP TABLE IF EXISTS temp._odysseus_fts5_probe")
except Exception as e:
logging.getLogger(__name__).warning(f"chat_messages FTS migration skipped; FTS5 unavailable: {e}")
return
conn.executescript(
"""
CREATE VIRTUAL TABLE IF NOT EXISTS chat_messages_fts USING fts5(
content,
message_id UNINDEXED,
session_id UNINDEXED,
role UNINDEXED
);
CREATE TRIGGER IF NOT EXISTS chat_messages_fts_ai
AFTER INSERT ON chat_messages BEGIN
INSERT INTO chat_messages_fts(content, message_id, session_id, role)
VALUES (COALESCE(new.content, ''), new.id, new.session_id, new.role);
END;
CREATE TRIGGER IF NOT EXISTS chat_messages_fts_ad
AFTER DELETE ON chat_messages BEGIN
DELETE FROM chat_messages_fts WHERE message_id = old.id;
END;
CREATE TRIGGER IF NOT EXISTS chat_messages_fts_au
AFTER UPDATE ON chat_messages BEGIN
DELETE FROM chat_messages_fts WHERE message_id = old.id;
INSERT INTO chat_messages_fts(content, message_id, session_id, role)
VALUES (COALESCE(new.content, ''), new.id, new.session_id, new.role);
END;
"""
)
conn.execute(
"""
INSERT INTO chat_messages_fts(content, message_id, session_id, role)
SELECT COALESCE(cm.content, ''), cm.id, cm.session_id, cm.role
FROM chat_messages cm
WHERE NOT EXISTS (
SELECT 1 FROM chat_messages_fts fts
WHERE fts.message_id = cm.id
)
"""
)
conn.commit()
except Exception as e:
logging.getLogger(__name__).warning(f"chat_messages FTS migration failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
def _migrate_add_email_smtp_security():
@@ -1633,6 +1966,7 @@ def _migrate_add_email_smtp_security():
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
conn = None
try:
conn = sqlite3.connect(db_path)
cursor = conn.execute("PRAGMA table_info(email_accounts)")
@@ -1648,9 +1982,13 @@ def _migrate_add_email_smtp_security():
)
conn.commit()
logging.getLogger(__name__).info("Migrated: added smtp_security column to email_accounts")
conn.close()
except Exception as e:
logging.getLogger(__name__).warning(f"smtp_security migration skipped: {e}")
finally:
try:
conn.close()
except Exception:
pass
def _migrate_encrypt_endpoint_keys():
@@ -1751,6 +2089,7 @@ def _migrate_add_calendar_is_utc():
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
conn = None
try:
conn = sqlite3.connect(db_path)
cursor = conn.execute("PRAGMA table_info(calendar_events)")
@@ -1759,9 +2098,13 @@ def _migrate_add_calendar_is_utc():
conn.execute("ALTER TABLE calendar_events ADD COLUMN is_utc BOOLEAN DEFAULT 0 NOT NULL")
conn.commit()
logging.getLogger(__name__).info("Migrated: added 'is_utc' column to calendar_events")
conn.close()
except Exception as e:
logging.getLogger(__name__).warning(f"is_utc migration failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
def _migrate_add_calendar_origin():
@@ -1772,6 +2115,7 @@ def _migrate_add_calendar_origin():
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
conn = None
try:
conn = sqlite3.connect(db_path)
cursor = conn.execute("PRAGMA table_info(calendar_events)")
@@ -1781,9 +2125,64 @@ def _migrate_add_calendar_origin():
conn.execute("CREATE INDEX IF NOT EXISTS ix_calendar_events_origin ON calendar_events(origin)")
conn.commit()
logging.getLogger(__name__).info("Migrated: added 'origin' column to calendar_events")
conn.close()
except Exception as e:
logging.getLogger(__name__).warning(f"calendar_events.origin migration failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
def _migrate_add_calendar_account_id():
"""Add `account_id` to calendars so each CalDAV-backed calendar knows which
credential set (from caldav_accounts in user prefs) owns it. Idempotent."""
import sqlite3
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
conn = None
try:
conn = sqlite3.connect(db_path)
cursor = conn.execute("PRAGMA table_info(calendars)")
columns = [row[1] for row in cursor.fetchall()]
if columns and "account_id" not in columns:
conn.execute("ALTER TABLE calendars ADD COLUMN account_id TEXT")
conn.execute("CREATE INDEX IF NOT EXISTS ix_calendars_account_id ON calendars(account_id)")
conn.commit()
logging.getLogger(__name__).info("Migrated: added 'account_id' column to calendars")
except Exception as e:
logging.getLogger(__name__).warning(f"calendars.account_id migration failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
def _migrate_add_caldav_sync_columns():
"""Add remote CalDAV metadata used for bidirectional sync."""
import sqlite3
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
try:
conn = sqlite3.connect(db_path)
ev_columns = [row[1] for row in conn.execute("PRAGMA table_info(calendar_events)").fetchall()]
if ev_columns and "remote_href" not in ev_columns:
conn.execute("ALTER TABLE calendar_events ADD COLUMN remote_href TEXT")
if ev_columns and "remote_etag" not in ev_columns:
conn.execute("ALTER TABLE calendar_events ADD COLUMN remote_etag TEXT")
if ev_columns and "caldav_sync_pending" not in ev_columns:
conn.execute("ALTER TABLE calendar_events ADD COLUMN caldav_sync_pending TEXT")
cal_columns = [row[1] for row in conn.execute("PRAGMA table_info(calendars)").fetchall()]
if cal_columns and "caldav_base_url" not in cal_columns:
conn.execute("ALTER TABLE calendars ADD COLUMN caldav_base_url TEXT")
conn.commit()
conn.close()
except Exception as e:
logging.getLogger(__name__).warning(f"CalDAV sync metadata migration failed: {e}")
def _migrate_add_calendar_metadata():
@@ -1792,6 +2191,7 @@ def _migrate_add_calendar_metadata():
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
conn = None
try:
conn = sqlite3.connect(db_path)
cursor = conn.execute("PRAGMA table_info(calendar_events)")
@@ -1803,9 +2203,35 @@ def _migrate_add_calendar_metadata():
if columns and "last_pinged" not in columns:
conn.execute("ALTER TABLE calendar_events ADD COLUMN last_pinged DATETIME")
conn.commit()
conn.close()
except Exception as e:
logging.getLogger(__name__).warning(f"calendar_events migration failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
def _migrate_add_calendar_recurrence_exdates():
"""Add skipped recurrence occurrences for deleting one instance of a series."""
import sqlite3
db_path = DATABASE_URL.replace("sqlite:///", "")
if not os.path.exists(db_path):
return
conn = None
try:
conn = sqlite3.connect(db_path)
columns = [row[1] for row in conn.execute("PRAGMA table_info(calendar_events)").fetchall()]
if columns and "recurrence_exdates" not in columns:
conn.execute("ALTER TABLE calendar_events ADD COLUMN recurrence_exdates TEXT DEFAULT ''")
conn.commit()
except Exception as e:
logging.getLogger(__name__).warning(f"calendar_events recurrence_exdates migration failed: {e}")
finally:
try:
conn.close()
except Exception:
pass
def get_db():
"""

2
core/exceptions.py
View File

@@ -1,4 +1,4 @@
# src/exceptions.py
# core/exceptions.py
"""Custom exceptions for the application."""
class SessionNotFoundError(Exception):

27
core/log_safety.py Normal file
View File

@@ -0,0 +1,27 @@
"""Helpers for keeping sensitive data out of logs.
Endpoint URLs configured by admins can embed credentials in the userinfo
(``https://user:pass@host``) or query string (``?api_key=...``). Logging them
raw leaks those secrets, so route/diagnostic logs run URLs through
``redact_url`` first. Reconstructing the URL without userinfo/query/fragment
also doubles as a sanitizer barrier for CodeQL's clear-text-logging query.
"""
from urllib.parse import urlparse, urlunparse
def redact_url(url: str) -> str:
"""Return a URL safe for logs by removing userinfo and query/fragment.
Keeps scheme, host, port and path so logs stay useful for debugging.
"""
try:
parsed = urlparse(url or "")
host = parsed.hostname or ""
if ":" in host: # IPv6 literal — re-bracket so host:port stays unambiguous
host = f"[{host}]"
if parsed.port:
host = f"{host}:{parsed.port}"
return urlunparse((parsed.scheme, host, parsed.path, "", "", ""))
except Exception:
return "<endpoint>"

37
core/middleware.py
View File

@@ -15,6 +15,17 @@ from starlette.responses import Response
# same value from this module. Never persisted or exposed externally.
INTERNAL_TOOL_TOKEN = os.environ.get("ODYSSEUS_INTERNAL_TOKEN") or secrets.token_hex(32)
INTERNAL_TOOL_HEADER = "X-Odysseus-Internal-Token"
# Pseudo-username on in-process tool-loopback requests; require_admin trusts it and it is reserved.
INTERNAL_TOOL_USER = "internal-tool"
def is_cors_preflight(method: str, headers) -> bool:
"""True for a genuine CORS preflight: an OPTIONS request carrying the
Access-Control-Request-Method header. Such requests are credential-less by
design and must reach CORSMiddleware to be answered -- gating them on auth
401s the preflight and breaks every cross-origin browser/WebView client.
Pure so it can be unit-tested without standing up the app."""
return method == "OPTIONS" and "access-control-request-method" in headers
def require_admin(request: Request):
@@ -30,7 +41,7 @@ def require_admin(request: Request):
hdr = request.headers.get(INTERNAL_TOOL_HEADER)
if hdr and secrets.compare_digest(hdr, INTERNAL_TOOL_TOKEN):
return
if getattr(request.state, "current_user", None) == "internal-tool":
if getattr(request.state, "current_user", None) == INTERNAL_TOOL_USER:
return
except Exception:
pass
@@ -56,13 +67,23 @@ class SecurityHeadersMiddleware(BaseHTTPMiddleware):
response = await call_next(request)
path = request.url.path
# Tool render endpoints are served inside iframes — allow framing by self
# Tool render endpoints
is_tool_render = path.startswith("/api/tools/") and path.endswith("/render")
# Document library PDF preview endpoint
is_document_pdf_preview = path.startswith("/api/document/") and path.endswith("/render-pdf")
# Visual report pages are self-contained HTML — need inline scripts + external images
is_report = path.startswith("/api/research/report/")
response.headers["X-Content-Type-Options"] = "nosniff"
response.headers["Referrer-Policy"] = "no-referrer"
response.headers["Permissions-Policy"] = "camera=(), microphone=(self), geolocation=()"
is_https = (
request.url.scheme == "https"
or request.headers.get("X-Forwarded-Proto") == "https"
)
if is_https:
response.headers["Strict-Transport-Security"] = "max-age=31536000; includeSubDomains"
if is_report:
response.headers["Content-Security-Policy"] = (
@@ -75,10 +96,14 @@ class SecurityHeadersMiddleware(BaseHTTPMiddleware):
"frame-ancestors 'none'"
)
elif is_tool_render:
# Tool iframe content: skip all framing headers — the iframe's
# sandbox="allow-scripts" attribute provides isolation.
# Don't overwrite the route's own restrictive CSP either.
# Skip framing headers for tools.
pass
elif is_document_pdf_preview:
response.headers["X-Frame-Options"] = "SAMEORIGIN"
response.headers["Content-Security-Policy"] = (
"default-src 'none'; "
"frame-ancestors 'self'"
)
else:
response.headers["X-Frame-Options"] = "DENY"
# NOTE: `style-src 'unsafe-inline'` is intentionally retained.
@@ -92,7 +117,7 @@ class SecurityHeadersMiddleware(BaseHTTPMiddleware):
f"script-src 'self' 'nonce-{nonce}' https://cdn.jsdelivr.net; "
"style-src 'self' 'unsafe-inline' https://cdn.jsdelivr.net; "
"font-src 'self' https://cdn.jsdelivr.net; "
"img-src 'self' data: blob:; "
"img-src 'self' data: blob: https:; "
"media-src 'self' blob:; "
"connect-src 'self'; "
"frame-src 'self'; "

61
core/models.py
View File

@@ -11,14 +11,24 @@ from typing import Dict, List, Any, Optional, TYPE_CHECKING
if TYPE_CHECKING:
from .session_manager import SessionManager
# Module-level session manager reference (set at app startup)
_session_manager: Optional["SessionManager"] = None
# Module-level session manager singleton (single source of truth)
_SESSION_MANAGER_INSTANCE: Optional["SessionManager"] = None
def set_session_manager(manager: "SessionManager"):
"""Set the global session manager reference."""
global _session_manager
_session_manager = manager
def set_session_manager_instance(manager: "SessionManager"):
"""Set the global SessionManager singleton."""
global _SESSION_MANAGER_INSTANCE
_SESSION_MANAGER_INSTANCE = manager
def get_session_manager_instance() -> Optional["SessionManager"]:
"""Get the global SessionManager singleton."""
return _SESSION_MANAGER_INSTANCE
# Keep legacy name for backward compatibility
set_session_manager = set_session_manager_instance
get_session_manager = get_session_manager_instance
@dataclass
@@ -42,7 +52,17 @@ class ChatMessage:
@dataclass
class Session:
"""A chat session — pure data container."""
"""A chat session — pure data container.
``.history`` is the authoritative mutable message list. Callers may
read, append, pop, or reassign it directly — these changes take
effect immediately. ``_history`` remains a compatibility alias that
always resolves to the authoritative ``history`` list.
Each session gets its own unique history list at construction time
(the dataclass default is never shared between instances).
"""
id: str
name: str
endpoint_url: str
@@ -56,24 +76,35 @@ class Session:
message_count: int = 0
def __post_init__(self):
if self.history is None:
self.history = []
if self.headers is None:
self.headers = {}
# Ensure each session gets its OWN list (not the shared dataclass default)
if self.history is None:
self.history = []
@property
def _history(self) -> List[ChatMessage]:
"""Compatibility alias for callers that still reference ``_history``."""
return self.history
@_history.setter
def _history(self, messages: List[ChatMessage]):
self.history = messages
def add_message(self, message: ChatMessage):
"""
Add a message to this session.
Delegates to SessionManager for persistence if available,
otherwise just appends to history.
Appends to the authoritative history list and increments
message_count. Delegates to SessionManager for persistence
if available.
"""
self.history.append(message)
self.message_count = len(self.history)
# Delegate to session manager for persistence
if _session_manager:
_session_manager._persist_message(self.id, message)
if _SESSION_MANAGER_INSTANCE:
_SESSION_MANAGER_INSTANCE._persist_message(self.id, message)
def get_context_messages(self) -> List[Dict[str, Any]]:
"""Get messages in format for LLM API.
@@ -94,3 +125,7 @@ class Session:
def get(self, key: str, default=None):
"""Dict-like access for compatibility."""
return getattr(self, key, default)
def __getitem__(self, key: str):
"""Allow session['field'] syntax."""
return getattr(self, key)

214
core/platform_compat.py
View File

@@ -18,10 +18,22 @@ import ntpath
import shutil
import subprocess
from pathlib import Path
import sys
from typing import List, Optional
import platform
IS_WINDOWS = os.name == "nt"
IS_POSIX = not IS_WINDOWS
# Allows APFEL support and ARM-native binary recommendations on Apple Silicon Macs.
IS_APPLE_SILICON = (
IS_POSIX
and platform.system() == "Darwin"
and platform.machine().lower()
in {
"arm64",
"aarch64",
}
)
# ── File permissions ────────────────────────────────────────────────────────
@@ -53,9 +65,8 @@ def detached_popen_kwargs() -> dict:
and is detached from any console.
"""
if IS_WINDOWS:
flags = (
getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0x00000200)
| getattr(subprocess, "DETACHED_PROCESS", 0x00000008)
flags = getattr(subprocess, "CREATE_NEW_PROCESS_GROUP", 0x00000200) | getattr(
subprocess, "DETACHED_PROCESS", 0x00000008
)
return {"creationflags": flags}
return {"start_new_session": True}
@@ -150,6 +161,29 @@ _WINDOWS_BASH_RELATIVE_PATHS = (
("usr", "bin", "bash.exe"),
)
# Paths to add to the remote SSH probe command to find tools like nvidia-smi that may not be on PATH.
_SSH_PATH_MEMBERS = (
"/usr/bin",
"/usr/local/bin",
"/usr/local/cuda/bin",
"/usr/lib/wsl/lib"
)
# Fallback locations for nvidia-smi on WSL and other Linux distros where it may not be on PATH.
NVIDIA_PATH_CANDIDATES = (
"/usr/bin/nvidia-smi",
"/usr/local/bin/nvidia-smi",
"/usr/local/cuda/bin/nvidia-smi",
"/usr/lib/wsl/lib/nvidia-smi",
)
def _ssh_path_override() -> str:
"""Build the PATH export snippet used for remote SSH shell probes."""
return f"export PATH=\"$PATH:{':'.join(_SSH_PATH_MEMBERS)}\"; "
SSH_PATH_OVERRIDE = _ssh_path_override()
def _windows_bash_fallbacks() -> List[str]:
roots: List[str] = []
@@ -157,6 +191,8 @@ def _windows_bash_fallbacks() -> List[str]:
base = os.environ.get(env_name)
if base:
roots.append(ntpath.join(base, "Git"))
if env_name == "LocalAppData":
roots.append(ntpath.join(base, "Programs", "Git"))
roots.extend(_WINDOWS_BASH_DEFAULT_ROOTS)
paths: List[str] = []
@@ -180,6 +216,21 @@ def _is_windows_bash_stub(path: str) -> bool:
)
def git_bash_path(path: str | Path) -> str:
"""Convert a path to POSIX style suitable for Git Bash on Windows.
Transforms drive letters (e.g., 'C:\\path') to POSIX '/c/path',
and uses forward slashes.
"""
p = Path(path)
p_str = p.as_posix()
if IS_WINDOWS and len(p_str) >= 2 and p_str[1] == ":":
drive = p_str[0].lower()
return f"/{drive}{p_str[2:]}"
return p_str
def find_bash() -> Optional[str]:
"""Locate a real ``bash`` interpreter, or None.
@@ -242,3 +293,160 @@ def run_script_argv(script_path) -> List[str]:
comspec = os.environ.get("ComSpec", "cmd.exe")
return [comspec, "/c", str(script_path)]
return ["sh", str(script_path)]
def is_wsl() -> bool:
"""True if running inside Windows Subsystem for Linux (WSL)."""
import sys
if sys.platform.startswith("linux") or os.name == "posix":
try:
with open("/proc/version", "r", encoding="utf-8", errors="ignore") as f:
if "microsoft" in f.read().lower():
return True
except Exception:
pass
return False
def translate_path(path_str: str) -> str:
"""Translate a path (possibly a Windows path) to the current OS format.
Particularly handles Windows paths (e.g. C:\\foo or C:/foo) when running
under WSL, translating them to /mnt/c/foo.
Also handles standard path normalization to avoid string breakages.
"""
if not path_str:
return path_str
if is_wsl():
path_str = path_str.replace("\\", "/")
import re
m = re.match(r"^([a-zA-Z]):(.*)", path_str)
if m:
drive = m.group(1).lower()
rest = m.group(2)
if not rest.startswith("/"):
rest = "/" + rest
return f"/mnt/{drive}{rest}"
try:
return str(Path(path_str).resolve())
except Exception:
return path_str
def get_wsl_windows_user_profile() -> Optional[str]:
"""Retrieve the Windows host User Profile path from inside WSL."""
if not is_wsl():
return None
try:
r = run_wsl_windows_powershell("Write-Output $env:USERPROFILE", timeout=5)
if r.returncode == 0 and r.stdout.strip():
return translate_path(r.stdout.strip())
except Exception:
pass
try:
users_dir = "/mnt/c/Users"
if os.path.isdir(users_dir):
for entry in os.listdir(users_dir):
if entry not in ("All Users", "Default", "Default User", "desktop.ini", "Public"):
path = os.path.join(users_dir, entry)
if os.path.isdir(path):
return path
except Exception:
pass
return None
def _ssh_exec_argv(
remote: str,
ssh_port: str | None,
*,
remote_cmd: str | None = None,
connect_timeout: int | None = None,
strict_host_key_checking: bool | None = None,
) -> list[str]:
"""Build a consistent ssh argv for remote command execution."""
remote_value = str(remote or "").strip()
remote_host = remote_value.rsplit("@", 1)[-1]
if not remote_value or remote_value.startswith("-") or not remote_host or remote_host.startswith("-"):
raise ValueError("Invalid SSH remote host")
argv = ["ssh"]
if connect_timeout is not None:
argv.extend(["-o", f"ConnectTimeout={int(connect_timeout)}"])
if strict_host_key_checking is not None:
argv.extend(
[
"-o",
"StrictHostKeyChecking=yes"
if strict_host_key_checking
else "StrictHostKeyChecking=no",
]
)
if ssh_port and ssh_port != "22":
argv.extend(["-p", str(ssh_port)])
argv.append(remote)
if remote_cmd is not None:
argv.append(remote_cmd)
return argv
def run_ssh_command(
remote: str,
ssh_port: str | None,
remote_cmd: str,
*,
timeout: float,
connect_timeout: int | None = None,
strict_host_key_checking: bool | None = None,
text: bool = True,
) -> subprocess.CompletedProcess:
"""Run an ssh command with centralized timeout and stderr/stdout capture."""
return subprocess.run(
_ssh_exec_argv(
remote,
ssh_port,
remote_cmd=remote_cmd,
connect_timeout=connect_timeout,
strict_host_key_checking=strict_host_key_checking,
),
timeout=timeout,
capture_output=True,
text=text,
)
def _windows_powershell_argv(
command: str,
*,
no_profile: bool = True,
non_interactive: bool = True,
) -> List[str]:
argv: List[str] = ["powershell.exe"]
if no_profile:
argv.append("-NoProfile")
if non_interactive:
argv.append("-NonInteractive")
argv.extend(["-Command", command])
return argv
def run_wsl_windows_powershell(
command: str,
*,
timeout: float = 5,
) -> subprocess.CompletedProcess[str]:
"""Run a PowerShell command on the Windows host from WSL.
Raises ``RuntimeError`` when called outside WSL.
"""
if not is_wsl():
raise RuntimeError("run_wsl_windows_powershell is only supported in WSL")
return subprocess.run(
_windows_powershell_argv(command),
capture_output=True,
text=True,
timeout=timeout,
)

62
core/session_manager.py
View File

@@ -17,6 +17,9 @@ from typing import Dict, Optional
from .database import Session as DbSession, ChatMessage as DbChatMessage, Document as DbDocument, SessionLocal, utcnow_naive
from .models import Session, ChatMessage
# Re-export singleton accessors from models for convenience
from .models import set_session_manager_instance, get_session_manager_instance
logger = logging.getLogger(__name__)
@@ -37,7 +40,18 @@ def _parse_msg_content(raw):
if isinstance(raw, str) and raw.startswith('[{') and '"type"' in raw:
try:
parsed = json.loads(raw)
if isinstance(parsed, list) and all(isinstance(p, dict) for p in parsed):
# Only treat as serialized multimodal content when EVERY element is
# a dict whose "type" is a recognized content-block kind. Otherwise a
# plain text message that merely *looks* like a JSON array of objects
# (e.g. a user pasting an API schema/sample with a "type" field) was
# silently parsed back into a list, destroying the original string.
_BLOCK_TYPES = {
"text", "image", "image_url", "audio", "input_audio",
"input_image", "document", "file",
}
if (isinstance(parsed, list) and parsed
and all(isinstance(p, dict) and p.get("type") in _BLOCK_TYPES
for p in parsed)):
return parsed
except (json.JSONDecodeError, ValueError):
pass
@@ -188,12 +202,17 @@ class SessionManager:
"""
Add a message to a session and persist to database.
Updates the authoritative history list and persists through this
manager directly so tests and temporary managers do not depend on the
process-wide session-manager singleton.
Args:
session_id: Session ID
message: ChatMessage to add
"""
session = self.get_session(session_id)
session.history.append(message)
session._history = session.history
session.message_count = len(session.history)
self._persist_message(session_id, message)
@@ -232,7 +251,10 @@ class SessionManager:
)
db.add(db_message)
db_session.message_count = len(self.sessions.get(session_id, {}).history) if session_id in self.sessions else 0
if session_id in self.sessions:
db_session.message_count = len(self.sessions[session_id].history)
else:
db_session.message_count = 0
_now = datetime.now(timezone.utc)
db_session.last_accessed = _now
# Clean "last conversation" timestamp — only bumped here on a
@@ -283,6 +305,7 @@ class SessionManager:
# Update in-memory
session.history = session.history[:keep_count]
session._history = session.history
logger.info(f"Truncated session {session_id} to {keep_count} messages")
return True
@@ -333,6 +356,7 @@ class SessionManager:
db.commit()
session.history = list(messages)
session._history = session.history
session.message_count = len(messages)
logger.info("Replaced session %s history with %d messages", session_id, len(messages))
return True
@@ -608,24 +632,52 @@ class SessionManager:
def save_sessions(self):
"""No-op for DB compatibility."""
def ensure_task_session(self, session_id: str, name: str, endpoint_url: str, model: str, owner: str = None, task: object = None) -> Session:
"""Create a task session if it doesn't exist, or return the existing one.
Unlike create_session, this checks the cache first and does NOT
overwrite an existing in-memory session. The task scheduler must
use this instead of direct dict assignment.
"""
if session_id in self.sessions:
return self.sessions[session_id]
session = self.create_session(session_id, name, endpoint_url, model, owner=owner)
if task is not None:
task.session_id = session_id
return session
# ------------------------------------------------------------------
# Cleanup
# ------------------------------------------------------------------
def cleanup_empty_sessions(self, auto_archive_days: int = 30) -> dict:
"""Clean up empty and old sessions."""
def cleanup_empty_sessions(self, auto_archive_days: int = 30, min_age_hours: int = 1) -> dict:
"""Clean up empty and old sessions.
Args:
auto_archive_days: Age in days before non-important sessions are archived.
min_age_hours: Minimum age in hours before an empty session can be deleted.
Prevents deleting sessions that were just created.
"""
db = SessionLocal()
stats = {'deleted_empty': 0, 'archived_old': 0, 'total_checked': 0}
try:
all_sessions = db.query(DbSession).all()
cutoff_date = utcnow_naive() - timedelta(days=auto_archive_days)
min_age = utcnow_naive() - timedelta(hours=min_age_hours)
for db_session in all_sessions:
stats['total_checked'] += 1
# Delete empty sessions
# Delete empty sessions only if older than min_age_hours
if db_session.message_count == 0:
if db_session.created_at is not None:
created = db_session.created_at
if created.tzinfo is None:
created = created.replace(tzinfo=timezone.utc)
if created > min_age:
continue # Too young to delete
if db_session.id in self.sessions:
del self.sessions[db_session.id]
db.delete(db_session)

19
docker-compose.gpu-amd.yml
View File

@@ -16,18 +16,18 @@ services:
ports:
- "${APP_BIND:-127.0.0.1}:${APP_PORT:-7000}:7000"
volumes:
- ./data:/app/data:z
- ./logs:/app/logs:z
- ${APP_DATA_DIR:-./data}:/app/data:z
- ${APP_LOGS_DIR:-./logs}:/app/logs:z
# Cookbook remote-server SSH identity. Odysseus can generate a key here;
# add the shown public key to each remote server's authorized_keys.
- ./data/ssh:/app/.ssh:z
- ${APP_DATA_DIR:-./data}/ssh:/app/.ssh:z
# Cookbook local model cache. Inside Docker, "Local" means the Odysseus
# container, so persist its HuggingFace cache under ./data/huggingface.
- ./data/huggingface:/app/.cache/huggingface:z
- ${APP_DATA_DIR:-./data}/huggingface:/app/.cache/huggingface:z
# Cookbook-installed Python CLIs/packages (vLLM, llama-cpp-python, etc.)
# land under /app/.local for the odysseus user. Persist them so a
# container recreate does not silently remove installed serve engines.
- ./data/local:/app/.local:z
- ${APP_DATA_DIR:-./data}/local:/app/.local:z
extra_hosts:
# Lets the container reach local services on the Docker host, including
# Ollama at http://host.docker.internal:11434.
@@ -52,12 +52,21 @@ services:
- SECURE_COOKIES=${SECURE_COOKIES:-false}
- EMBEDDING_URL=${EMBEDDING_URL:-}
- EMBEDDING_MODEL=${EMBEDDING_MODEL:-}
- EMBEDDING_API_KEY=${EMBEDDING_API_KEY:-}
- FASTEMBED_MODEL=${FASTEMBED_MODEL:-sentence-transformers/all-MiniLM-L6-v2}
- FASTEMBED_CACHE_PATH=${FASTEMBED_CACHE_PATH:-}
- CLEANUP_INTERVAL_HOURS=${CLEANUP_INTERVAL_HOURS:-24}
- ODYSSEUS_INPROCESS_POLLERS=${ODYSSEUS_INPROCESS_POLLERS:-1}
- ODYSSEUS_INPROCESS_TASKS=${ODYSSEUS_INPROCESS_TASKS:-1}
- ODYSSEUS_SCRIPT_HOST=${ODYSSEUS_SCRIPT_HOST:-localhost}
- ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=${ODYSSEUS_CHAT_UPLOAD_MAX_BYTES:-10485760}
- ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES:-104857600}
- ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES:-26214400}
- ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=${ODYSSEUS_MEMORY_IMPORT_MAX_BYTES:-10485760}
- ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=${ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES:-26214400}
- ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400}
- ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400}
- ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760}
- DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-}
- GOOGLE_API_KEY=${GOOGLE_API_KEY:-}
- GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-}

19
docker-compose.gpu-nvidia.yml
View File

@@ -15,18 +15,18 @@ services:
ports:
- "${APP_BIND:-127.0.0.1}:${APP_PORT:-7000}:7000"
volumes:
- ./data:/app/data:z
- ./logs:/app/logs:z
- ${APP_DATA_DIR:-./data}:/app/data:z
- ${APP_LOGS_DIR:-./logs}:/app/logs:z
# Cookbook remote-server SSH identity. Odysseus can generate a key here;
# add the shown public key to each remote server's authorized_keys.
- ./data/ssh:/app/.ssh:z
- ${APP_DATA_DIR:-./data}/ssh:/app/.ssh:z
# Cookbook local model cache. Inside Docker, "Local" means the Odysseus
# container, so persist its HuggingFace cache under ./data/huggingface.
- ./data/huggingface:/app/.cache/huggingface:z
- ${APP_DATA_DIR:-./data}/huggingface:/app/.cache/huggingface:z
# Cookbook-installed Python CLIs/packages (vLLM, llama-cpp-python, etc.)
# land under /app/.local for the odysseus user. Persist them so a
# container recreate does not silently remove installed serve engines.
- ./data/local:/app/.local:z
- ${APP_DATA_DIR:-./data}/local:/app/.local:z
extra_hosts:
# Lets the container reach local services on the Docker host, including
# Ollama at http://host.docker.internal:11434.
@@ -51,12 +51,21 @@ services:
- SECURE_COOKIES=${SECURE_COOKIES:-false}
- EMBEDDING_URL=${EMBEDDING_URL:-}
- EMBEDDING_MODEL=${EMBEDDING_MODEL:-}
- EMBEDDING_API_KEY=${EMBEDDING_API_KEY:-}
- FASTEMBED_MODEL=${FASTEMBED_MODEL:-sentence-transformers/all-MiniLM-L6-v2}
- FASTEMBED_CACHE_PATH=${FASTEMBED_CACHE_PATH:-}
- CLEANUP_INTERVAL_HOURS=${CLEANUP_INTERVAL_HOURS:-24}
- ODYSSEUS_INPROCESS_POLLERS=${ODYSSEUS_INPROCESS_POLLERS:-1}
- ODYSSEUS_INPROCESS_TASKS=${ODYSSEUS_INPROCESS_TASKS:-1}
- ODYSSEUS_SCRIPT_HOST=${ODYSSEUS_SCRIPT_HOST:-localhost}
- ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=${ODYSSEUS_CHAT_UPLOAD_MAX_BYTES:-10485760}
- ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES:-104857600}
- ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES:-26214400}
- ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=${ODYSSEUS_MEMORY_IMPORT_MAX_BYTES:-10485760}
- ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=${ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES:-26214400}
- ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400}
- ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400}
- ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760}
- DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-}
- GOOGLE_API_KEY=${GOOGLE_API_KEY:-}
- GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-}

19
docker-compose.yml
View File

@@ -4,18 +4,18 @@ services:
ports:
- "${APP_BIND:-127.0.0.1}:${APP_PORT:-7000}:7000"
volumes:
- ./data:/app/data:z
- ./logs:/app/logs:z
- ${APP_DATA_DIR:-./data}:/app/data:z
- ${APP_LOGS_DIR:-./logs}:/app/logs:z
# Cookbook remote-server SSH identity. Odysseus can generate a key here;
# add the shown public key to each remote server's authorized_keys.
- ./data/ssh:/app/.ssh:z
- ${APP_DATA_DIR:-./data}/ssh:/app/.ssh:z
# Cookbook local model cache. Inside Docker, "Local" means the Odysseus
# container, so persist its HuggingFace cache under ./data/huggingface.
- ./data/huggingface:/app/.cache/huggingface:z
- ${APP_DATA_DIR:-./data}/huggingface:/app/.cache/huggingface:z
# Cookbook-installed Python CLIs/packages (vLLM, llama-cpp-python, etc.)
# land under /app/.local for the odysseus user. Persist them so a
# container recreate does not silently remove installed serve engines.
- ./data/local:/app/.local:z
- ${APP_DATA_DIR:-./data}/local:/app/.local:z
extra_hosts:
# Lets the container reach local services on the Docker host, including
# Ollama at http://host.docker.internal:11434.
@@ -40,12 +40,21 @@ services:
- SECURE_COOKIES=${SECURE_COOKIES:-false}
- EMBEDDING_URL=${EMBEDDING_URL:-}
- EMBEDDING_MODEL=${EMBEDDING_MODEL:-}
- EMBEDDING_API_KEY=${EMBEDDING_API_KEY:-}
- FASTEMBED_MODEL=${FASTEMBED_MODEL:-sentence-transformers/all-MiniLM-L6-v2}
- FASTEMBED_CACHE_PATH=${FASTEMBED_CACHE_PATH:-}
- CLEANUP_INTERVAL_HOURS=${CLEANUP_INTERVAL_HOURS:-24}
- ODYSSEUS_INPROCESS_POLLERS=${ODYSSEUS_INPROCESS_POLLERS:-1}
- ODYSSEUS_INPROCESS_TASKS=${ODYSSEUS_INPROCESS_TASKS:-1}
- ODYSSEUS_SCRIPT_HOST=${ODYSSEUS_SCRIPT_HOST:-localhost}
- ODYSSEUS_CHAT_UPLOAD_MAX_BYTES=${ODYSSEUS_CHAT_UPLOAD_MAX_BYTES:-10485760}
- ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES:-104857600}
- ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES=${ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES:-26214400}
- ODYSSEUS_MEMORY_IMPORT_MAX_BYTES=${ODYSSEUS_MEMORY_IMPORT_MAX_BYTES:-10485760}
- ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES=${ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES:-26214400}
- ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES=${ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES:-26214400}
- ODYSSEUS_STT_MAX_AUDIO_BYTES=${ODYSSEUS_STT_MAX_AUDIO_BYTES:-26214400}
- ODYSSEUS_ICS_MAX_BYTES=${ODYSSEUS_ICS_MAX_BYTES:-10485760}
- DATA_BRAVE_API_KEY=${DATA_BRAVE_API_KEY:-}
- GOOGLE_API_KEY=${GOOGLE_API_KEY:-}
- GOOGLE_PSE_CX=${GOOGLE_PSE_CX:-}

70
docker/build-realesrgan-wheels.sh Executable file
View File

@@ -0,0 +1,70 @@
#!/usr/bin/env bash
# Build patched wheels for Real-ESRGAN's unmaintained dependencies.
#
# basicsr / gfpgan / facexlib (xinntao, last released 2022) read their version
# in setup.py with:
#
# exec(compile(f.read(), version_file, 'exec'))
# return locals()['__version__']
#
# Python 3.13+ implements PEP 667: locals() inside a function returns an
# independent snapshot that exec() can no longer mutate, so the read raises
# `KeyError: '__version__'` and the sdist build fails. That is why the Cookbook
# "install realesrgan" button dies on the python:3.14 image. The packages have
# no fixed release, so we patch get_version() to exec into an explicit namespace
# dict (works on every Python) and build wheels from the patched source.
#
# Usage: build-realesrgan-wheels.sh [OUTPUT_DIR] (default: /wheels)
set -euo pipefail
OUT="${1:-/wheels}"
mkdir -p "$OUT"
work="$(mktemp -d)"
trap 'rm -rf "$work"' EXIT
cd "$work"
# Pinned to the versions Real-ESRGAN 0.3.0 resolves to.
SPECS="basicsr==1.4.2 gfpgan==1.3.8 facexlib==0.3.0"
for spec in $SPECS; do
name="${spec%%==*}"
ver="${spec##*==}"
# pip download builds metadata (and trips the same bug), so fetch the raw
# sdist URL from the PyPI JSON API instead.
url="$(python - "$name" "$ver" <<'PY'
import json, sys, urllib.request
name, ver = sys.argv[1], sys.argv[2]
data = json.load(urllib.request.urlopen(f"https://pypi.org/pypi/{name}/{ver}/json"))
for f in data["urls"]:
if f["packagetype"] == "sdist":
print(f["url"]); break
else:
sys.exit(f"no sdist found for {name}=={ver}")
PY
)"
echo ">> fetching ${name} ${ver}: ${url}"
curl -fsSL "$url" -o "${name}.tar.gz"
tar xzf "${name}.tar.gz"
done
echo ">> patching get_version()"
python - <<'PY'
import pathlib
old_exec = "exec(compile(f.read(), version_file, 'exec'))"
new_exec = "_ver_ns = {}\n exec(compile(f.read(), version_file, 'exec'), _ver_ns)"
old_ret = "return locals()['__version__']"
new_ret = "return _ver_ns['__version__']"
patched = 0
for setup in pathlib.Path(".").glob("*/setup.py"):
s = setup.read_text()
if old_exec in s and old_ret in s:
setup.write_text(s.replace(old_exec, new_exec).replace(old_ret, new_ret))
print(" patched", setup)
patched += 1
assert patched == 3, f"expected to patch 3 setup.py files, patched {patched}"
PY
echo ">> building wheels into ${OUT}"
pip wheel --no-deps -w "$OUT" ./basicsr-* ./gfpgan-* ./facexlib-*
ls -l "$OUT"

93
docker/entrypoint.sh
View File

@@ -13,6 +13,8 @@ set -e
PUID="${PUID:-1000}"
PGID="${PGID:-1000}"
GOSU_BIN="$(command -v gosu)"
PYTHON_BIN="$(command -v python)"
# Reuse an existing matching group/user if the host's UID/GID already
# corresponds to one in /etc/passwd (e.g. when the image is rebuilt
@@ -24,26 +26,78 @@ if ! getent passwd "$PUID" >/dev/null 2>&1; then
useradd -u "$PUID" -g "$PGID" -M -s /bin/sh -d /app odysseus
fi
# Repair ownership on every writable path the app touches at runtime.
#
# Bind-mounted dirs (/app/data, /app/logs) are the obvious ones, but
# the app ALSO writes inside the image's own source tree at runtime:
# - services/cache/{search,content}/* (search cache LRU)
# - services/search_analytics.json
# - services/search_engine_error.log
# - services/tts cache, etc.
# These dirs were created as root during `docker build`, so dropping
# to PUID:PGID would otherwise crash on the first import that tries
# to mkdir them. Chown the whole /app tree — fast (<1s on this size)
# and idempotent via the `-not -uid` filter so we only touch files
# that need fixing.
for dir in /app /app/data /app/logs; do
ODY_USER="$(getent passwd "$PUID" | cut -d: -f1)"
[ -z "$ODY_USER" ] && ODY_USER=odysseus
# Docker-socket group plumbing for the explicit host-Docker overlay. When
# opted in, the socket is owned by root:<host docker gid>. Add the app user
# to that group and later call gosu by username so supplementary groups are
# retained.
DOCKER_SOCK="${DOCKER_SOCK:-/var/run/docker.sock}"
if [ "${ODYSSEUS_ENABLE_HOST_DOCKER:-}" = "true" ] && [ -S "$DOCKER_SOCK" ]; then
SOCK_GID="$(stat -c '%g' "$DOCKER_SOCK" 2>/dev/null || echo '')"
if [ -n "$SOCK_GID" ] && [ "$SOCK_GID" != "0" ]; then
if ! getent group "$SOCK_GID" >/dev/null 2>&1; then
groupadd -g "$SOCK_GID" docker_host || true
fi
SOCK_GROUP="$(getent group "$SOCK_GID" | cut -d: -f1)"
if [ -n "$SOCK_GROUP" ]; then
usermod -aG "$SOCK_GROUP" "$ODY_USER" 2>/dev/null || true
fi
fi
fi
mount_root_for() {
awk -v target="$1" '$5 == target { print $4; exit }' /proc/self/mountinfo 2>/dev/null || true
}
is_broad_mount_root() {
case "$1" in
/|/home|/srv|/var|/usr|/opt|/tmp|/mnt|/media)
return 0
;;
esac
return 1
}
repair_tree_ownership() {
dir="$1"
if [ -d "$dir" ]; then
# `find ... -not -uid` keeps this O(touched-files), not
# O(everything), so terabyte-sized maildirs don't slow startup.
find "$dir" -not -uid "$PUID" -print0 2>/dev/null \
find "$dir" -xdev -not -uid "$PUID" -print0 2>/dev/null \
| xargs -0 -r chown "$PUID:$PGID" 2>/dev/null || true
fi
}
repair_app_tree_ownership() {
if [ -d /app ]; then
find /app -xdev \
\( -path /app/data -o -path /app/logs -o -path /app/.ssh -o -path /app/.cache -o -path /app/.local \) -prune \
-o -not -uid "$PUID" -print0 2>/dev/null \
| xargs -0 -r chown "$PUID:$PGID" 2>/dev/null || true
fi
}
repair_bind_mount_ownership() {
dir="$1"
if [ ! -d "$dir" ]; then
return
fi
mount_root="$(mount_root_for "$dir")"
if is_broad_mount_root "$mount_root"; then
echo "Skipping recursive ownership repair for $dir because it maps to broad host path $mount_root" >&2
chown "$PUID:$PGID" "$dir" 2>/dev/null || true
return
fi
repair_tree_ownership "$dir"
}
# Repair image-owned writable paths without walking into bind-mounted host
# trees, then repair the app-owned mount roots separately.
repair_app_tree_ownership
for dir in /app/data /app/logs /app/.ssh /app/.cache/huggingface /app/.local; do
repair_bind_mount_ownership "$dir"
done
# Cookbook installs vllm/etc. via `pip install --user`, which pulls
@@ -70,6 +124,7 @@ for cu in \
break
fi
done
# Disable the FlashInfer JIT sampler unconditionally — it is sampler-only
# and has no impact on the attention path, but requires nvcc + matching
# CUDA headers at startup. Without this, vLLM crashes with "Could not find
@@ -83,9 +138,9 @@ export PATH="/app/.local/bin:$PATH"
# Run first-time setup as the app user so data/ files get the right ownership.
# setup.py is idempotent — skips auth.json / .env if they already exist.
# || true so a setup failure never prevents the container from starting.
gosu "$PUID:$PGID" python /app/setup.py || true
"$GOSU_BIN" "$ODY_USER" "$PYTHON_BIN" /app/setup.py || true
# Drop root and run the actual app. `gosu` is preferred over `su` /
# `sudo` because it cleans up the process tree (no extra shell layer)
# so signals (SIGTERM from `docker stop`) reach uvicorn directly.
exec gosu "$PUID:$PGID" "$@"
exec "$GOSU_BIN" "$ODY_USER" "$@"

12
docker/host-docker.yml Normal file
View File

@@ -0,0 +1,12 @@
# High-trust host Docker access. Enable only when local Docker-daemon
# management from Cookbook is required and you accept that raw socket access
# grants broad control over the host Docker daemon.
# COMPOSE_FILE=docker-compose.yml:docker/host-docker.yml
# DOCKER_GID=<numeric host Docker group id>
services:
odysseus:
volumes:
- /var/run/docker.sock:/var/run/docker.sock
group_add: ["${DOCKER_GID:-963}"]
environment:
- ODYSSEUS_ENABLE_HOST_DOCKER=true

194
docs/agent-migration.md Normal file
View File

@@ -0,0 +1,194 @@
# Agent migration manifests
Odysseus should be able to learn from another agent without blindly trusting
that agent's whole state. The safe migration path is:
```text
source agent export -> source adapter -> agent-migration.v1 manifest -> preview -> apply
```
The manifest is intentionally source-neutral. OpenClaw, Hermes, a folder of
Markdown notes, or any other agent can have its own adapter, but Odysseus only
needs to understand the normalized manifest.
## Why not import everything as memory?
Durable memory should stay compact and useful. Long notes, logs, session
transcripts, and project archives are useful context, but they are not all
memories. A good migration keeps two layers separate:
- **Archive documents** preserve source material for search, reading, and later
extraction.
- **Memory candidates** are short facts or preferences that can be reviewed
before being saved into Odysseus memory.
This keeps Odysseus' existing memory-review flow intact while giving it better
source material to review.
## Manifest shape
`agent-migration.v1` is a JSON object:
```json
{
"schema_version": "agent-migration.v1",
"generated_at": "2026-06-06T00:00:00Z",
"source": {
"name": "example-agent",
"kind": "generic"
},
"summary": {
"item_count": 3,
"counts_by_kind": {
"memory": 1,
"skill": 1,
"conversation_thread": 1,
"archive_document": 1
},
"warning_count": 0
},
"items": [],
"warnings": []
}
```
Each item has a stable `id`, a `kind`, source metadata, and enough content for a
future importer to preview it before applying.
Supported item kinds in the first pass:
- `memory` — a candidate memory with `text`, `category`, `source`, and
provenance metadata.
- `skill` — a `SKILL.md` file with content and parsed frontmatter metadata.
- `conversation_thread` — a normalized transcript thread from an exported chat
history. Message content is optional; adapters can preserve only thread
metadata, message counts, timestamps, and hashes when a manifest should stay
small or avoid embedding private transcript text.
- `archive_document` — long-form source material. Content is optional; adapters
can preserve only path/hash/size metadata when a manifest should stay small.
## Build a manifest
Use the read-only helper:
```bash
python3 scripts/agent_migration_manifest.py \
--source-name old-agent \
--source-kind generic \
--memory-json /path/to/memories.json \
--skills-dir /path/to/skills \
--conversation-json /path/to/conversations.json \
--archive /path/to/notes \
--output /tmp/agent-migration.json
```
The helper does not write to `data/`, call an LLM, import Odysseus modules, or
modify the source. It only writes JSON.
Memory JSON may be:
```json
[
"A plain memory string",
{
"text": "A categorized memory",
"category": "preference",
"source": "old-agent"
}
]
```
or an object containing a list under `memories`, `memory`, `items`, or `data`.
Skills are scanned recursively for `SKILL.md`:
```bash
python3 scripts/agent_migration_manifest.py \
--source-name hermes \
--source-kind hermes \
--skills-dir ~/.hermes/skills \
--output /tmp/hermes-skills-manifest.json
```
Archive documents are metadata-only by default. To embed text content:
```bash
python3 scripts/agent_migration_manifest.py \
--source-name notes-export \
--archive /path/to/markdown-notes \
--include-archive-content \
--output /tmp/notes-manifest.json
```
Conversation exports are also metadata-only by default:
```bash
python3 scripts/agent_migration_manifest.py \
--source-name chatgpt-export \
--source-kind chatgpt \
--conversation-json /path/to/conversations.json \
--output /tmp/chatgpt-conversations-manifest.json
```
The first pass supports generic conversation JSON such as:
```json
[
{
"id": "thread-1",
"title": "Project plan",
"messages": [
{"role": "user", "content": "Can we design this?"},
{"role": "assistant", "content": "Yes, start with a narrow slice."}
]
}
]
```
It also recognizes ChatGPT-style `mapping` exports from `conversations.json`.
To embed normalized messages:
```bash
python3 scripts/agent_migration_manifest.py \
--source-name chatgpt-export \
--source-kind chatgpt \
--conversation-json /path/to/conversations.json \
--include-conversation-content \
--max-conversation-messages 2000 \
--output /tmp/chatgpt-conversations-with-content.json
```
Content embedding is explicit because exported chat histories can be huge and
private. A future source-specific adapter can add ZIP traversal, attachment
metadata, and provider-specific project/workspace fields while still emitting
the same `conversation_thread` manifest item.
## Recommended apply behavior
A future Odysseus importer should treat the manifest as untrusted user-provided
data and apply it in stages:
1. Show a dry-run summary with counts, warnings, duplicates, and sample items.
2. Back up current `data/` state before writing anything.
3. Import archive documents as documents or another searchable source, not as
memory.
4. Import conversation threads as searchable archived context first, with
citations back to the source thread. Do not turn whole transcripts into
memory.
5. Show memory candidates for review before saving through the normal memory
path.
6. Import skills only after name/category conflict checks.
7. Skip secrets by default. Credentials need explicit, provider-specific flows.
## What belongs in source adapters?
Adapters can be source-specific. The core manifest should not be.
For example, an OpenClaw adapter may know about OpenClaw's workspace files. A
Hermes adapter may know about `~/.hermes/config.yaml` and `~/.hermes/skills`.
A ChatGPT adapter may know about `conversations.json`, uploaded-file metadata,
and image attachment directories. A Claude adapter may know about Claude's
export shape and project boundaries. A generic adapter may only know about
memory JSON, conversation JSON, `SKILL.md`, and Markdown folders.
Nonstandard folders should be adapter details, not required Odysseus concepts.

129
docs/backup-restore.md Normal file
View File

@@ -0,0 +1,129 @@
# Backup & Restore
Odysseus keeps all of your state in the `data/` directory — the SQLite database
(`app.db`), the Fernet encryption key (`data/.app_key`), the vault, memory, RAG
indexes, personal documents, and uploads. The `scripts/odysseus-backup` tool
snapshots that directory into a single gzip tarball and restores it later.
Snapshots are safe to take while the app is running: SQLite databases are copied
through SQLite's own `.backup` API rather than a raw file copy, so an in-flight
write can't corrupt the snapshot.
> **A snapshot contains your secrets.** The tarball includes the Fernet
> encryption key (`data/.app_key`), the vault, sessions, and any stored
> provider/API tokens — so treat it like a password. Store backups somewhere
> private, never commit them to Git, and prefer an encrypted destination when
> copying them offsite.
## Quick start
Run the tool from the repository root:
```bash
# Create a snapshot → backups/odysseus-backup-<YYYYMMDD-HHMMSS>.tar.gz
./scripts/odysseus-backup snapshot
# List existing snapshots (most recent first)
./scripts/odysseus-backup list
# Check a tarball's integrity without extracting it
./scripts/odysseus-backup verify backups/odysseus-backup-20260101-120000.tar.gz
# Restore (destructive — see the warning below)
./scripts/odysseus-backup restore backups/odysseus-backup-20260101-120000.tar.gz --yes
```
The script depends only on the Python standard library, so any `python3` on your
`PATH` will run it — you don't need the app's virtualenv active.
Every command prints a JSON result. Add `--pretty` for indented output.
## Commands
### `snapshot`
Writes a `tar.gz` of `data/` to `backups/<timestamp>.tar.gz`.
| Flag | Effect |
| --- | --- |
| `--out PATH` | Write to a specific path instead of the default `backups/` location. Must be **outside** `data/`. |
| `--include-research` | Include `data/deep_research/` (skipped by default — research runs are large). |
| `--include-attachments` | Include `data/mail-attachments/` (skipped by default — cached IMAP extractions, re-derivable). |
By default the snapshot includes everything under `data/` **except**
`deep_research/` and `mail-attachments/`. Personal uploads and documents are
included.
```bash
# Snapshot straight to a mounted NAS path
./scripts/odysseus-backup snapshot --out /mnt/nas/odysseus-$(date +%F).tar.gz
# Full snapshot including research runs and mail attachments
./scripts/odysseus-backup snapshot --include-research --include-attachments
```
### `list`
Lists the tarballs in `backups/`, most recent first, with size and modification
time.
### `verify PATH`
Opens the tarball read-only and walks every member to confirm it is intact and
safe to restore. Nothing is extracted. Use this before relying on an old backup
or after copying one across machines.
### `restore PATH --yes`
Overwrites `data/` from a tarball.
> **Restore is destructive.** It replaces the current `data/` directory. `--yes`
> is required so a mistyped command can't wipe your live state.
Restore is not a blind delete: before extracting, the tool **renames your current
`data/` to `data.before-restore-<timestamp>`** in the repository root. If a
restore turns out to be wrong, your previous state is still there — delete the
restored `data/` and rename the stashed directory back. The restore path is also
validated entry-by-entry: archives containing absolute paths, `..` segments,
symlinks, or anything outside `data/` are rejected.
## Scheduling offsite backups
The tarball output composes cleanly with cron and any copy tool. For example, a
nightly snapshot copied offsite:
```cron
0 3 * * * cd /path/to/odysseus && ./scripts/odysseus-backup snapshot --out "/mnt/nas/odysseus-$(date +\%F).tar.gz"
```
Swap the `--out` target for `scp`, `rclone`, `s3cmd`, or similar to push the
snapshot to remote storage.
## Docker vs native installs
The tool reads `data/` and writes `backups/` relative to the repository root, so
where you run it matters:
- **Native installs** — run it from the repo root as shown above. `data/` and
`backups/` are both in the repo directory.
- **Docker** — `docker-compose.yml` bind-mounts the host's `./data` to
`/app/data`, so the live data is also present on the host. **Run the tool on
the host** from the repo root; the snapshot reads the bind-mounted `./data` and
writes to `./backups` on the host. Running it *inside* the container is not
recommended, because `backups/` is not a mounted volume and the tarball would
be lost when the container is recreated.
> **ChromaDB caveat (Docker only).** In the Docker setup, ChromaDB stores its
> vectors in a separate Compose-managed volume (declared as `chromadb-data`),
> **not** under `./data`. `odysseus-backup` therefore does not capture the Docker
> ChromaDB store. Back it up separately if you need it. Compose prefixes the
> volume with the project name, so find the real name first
> (`docker volume ls | grep chromadb`), then archive it — for example:
>
> ```bash
> docker run --rm -v <project>_chromadb-data:/data -v "$PWD":/backup \
> alpine tar czf /backup/chromadb.tar.gz -C /data .
> ```
>
> On native installs ChromaDB lives at `data/chroma/` and is included in the
> snapshot normally.

BIN
docs/chat.gif
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 3.0 MiB

BIN
docs/compare.gif
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 3.4 MiB

BIN
docs/document.gif
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 1.1 MiB

17
docs/email-outlook.md Normal file
View File

@@ -0,0 +1,17 @@
# Outlook / Office 365 email accounts
Odysseus email accounts currently use IMAP and SMTP with username/password
authentication. That works for providers that still allow app passwords or
mailbox passwords for IMAP/SMTP.
Microsoft disables basic authentication for Outlook and Microsoft 365 in most
modern accounts and tenants. If you try to add an Outlook account with a normal
password, Microsoft may return errors such as:
- `IMAP: AUTHENTICATE failed`
- `SMTP: 535 5.7.139 Authentication unsuccessful, basic authentication is disabled`
This is expected. Odysseus does not support Microsoft OAuth or Graph Mail yet,
so Outlook / Office 365 accounts cannot currently be added through the password
form. Use another email provider with app-password support, or track the future
Microsoft Graph OAuth integration.

13
docs/index.html
View File

@@ -25,9 +25,16 @@
--radius: 8px;
}
* { box-sizing: border-box; }
html { scroll-behavior: smooth; scroll-snap-type: y proximity; scroll-padding-top: 60px; }
/* Each section is a full-viewport "page" with its content centered, so only
one shows at a time and the snap is obvious. */
html { scroll-behavior: smooth; scroll-padding-top: 60px; }
/* REMOVED: "scroll-snap-type: y proximity"
The idea was: >>Each section is a full-viewport "page" with its content centered,
so only one shows at a time and the snap is obvious.<<
PROBLEM: sections easily grow taller than 100vh IRL
This cause forced jumps mid-read. It's intrusive UX.
The landing-page is not a PowerPoint presentation!
Preserved: CSS snap-points to avoid destroying code meta-data*/
.hero, section {
scroll-snap-align: start; min-height: 100vh;
display: flex; flex-direction: column; justify-content: center;

BIN
docs/notes.gif
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 1003 KiB

BIN
docs/odysseus-browser.jpg Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 185 KiB

BIN
docs/odysseus-wordmark.png Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 16 KiB

BIN
docs/odysseus.jpg
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 45 KiB

After

Width:  |  Height:  |  Size: 79 KiB

BIN
docs/research.gif
View File

Binary file not shown.
Side by Side

Before

Width:  |  Height:  |  Size: 2.5 MiB

107
docs/security-ci.md Normal file
View File

@@ -0,0 +1,107 @@
# Security CI guide
This project runs a set of automated security checks on pull requests and
selected branch pushes. This page explains what each one does, whether it can
block a merge, and the few one-time settings you should turn on to get the full
benefit.
## What runs, and why
Most checks live in files under `.github/workflows/`. CodeQL is configured
through GitHub's code scanning default setup, so it appears as a dynamic GitHub
workflow instead of a checked-in workflow file. They run automatically; you do
not start them.
| Check | What it protects against | Blocks a merge? |
|---|---|---|
| **Secret scan** (gitleaks) | An API key, token, or password being committed by mistake or on purpose | Yes |
| **Workflow security** (actionlint + zizmor) | A broken or insecure automation file that could leak the repo's access token | Yes |
| **Dependency review** | A pull request that adds a software library with a known security hole | Yes |
| **pip-audit** | Known security holes in the Python libraries already used | No (advisory) |
| **Container scan: hadolint** | Mistakes and insecure patterns in the `Dockerfile` | Yes |
| **Container scan: Trivy** | Known security holes in the Docker image | No (advisory) |
| **CodeQL** | Real bugs in the app's own code: injection, auth mistakes, path traversal | No (advisory) |
"Blocks a merge" means a red X appears on the pull request and, once you enable
the setting below, the **Merge** button is disabled until it is fixed.
"Advisory" means it reports problems into the repository's **Security** tab so
you can review them on your own schedule, but it never stops a merge. These are
advisory on purpose: they often flag long-standing issues in other people's
libraries, not something a given pull request introduced.
## Where results appear
- **Checks tab of a pull request**: the pass/fail of each check. A green tick is
good; a red X needs attention.
- **Security tab of the repository**: detailed findings from the advisory
scanners (Trivy and CodeQL). This is your dashboard.
## If a check fails
- **Secret scan failed**: a real credential may have been committed. Treat it as
leaked: rotate (regenerate) that key or token immediately, then remove it from
the file. Do not just delete the commit; assume it was seen.
- **Dependency review failed**: the pull request adds a library with a known
vulnerability. Ask the contributor to use a patched version, or decline the
change.
- **hadolint / workflow security failed**: the contributor changed the
`Dockerfile` or an automation file in a way the linter rejects. Ask them to
address the message shown in the failed check.
## One-time settings to turn on
These two settings unlock the full value. You only do them once.
### 1. Require the blocking checks before merging
This makes the **Merge** button refuse to work until the gating checks pass.
1. Go to the repository on GitHub.
2. Click **Settings** (top right of the repo).
3. In the left sidebar, click **Branches**.
4. Under **Branch protection rules**, click **Add branch ruleset** (or **Add
rule**), and set the branch name pattern to `dev` (this is the branch all
pull requests target; `main` is fast-forwarded at releases).
5. Enable **Require status checks to pass before merging**.
6. In the search box that appears, add these checks by name:
- `Python syntax (compileall)`
- `JS syntax (node --check)`
- `gitleaks`
- `actionlint`
- `zizmor (Actions SAST)`
- `hadolint (Dockerfile lint)`
- `dependency-review (PR gate)`
The first two come from the correctness CI (`ci.yml`); the rest are this
security suite. Leave pytest, pip-audit, Trivy, and CodeQL unchecked so they
stay advisory.
7. Also enable **Require a pull request before merging** and **Require review
from Code Owners** (this uses the `.github/CODEOWNERS` file so every change
needs your sign-off).
8. Click **Create** / **Save changes**.
Note: a check name only appears in the list after it has run at least once, so
let the workflows run on one pull request first, then add them here.
### 2. Turn on the Security tab features
1. **Settings -> Code security** (or **Code security and analysis**).
2. Turn on **Dependency graph** (usually on by default for public repos) -- this
powers Dependency review and Dependabot.
3. Turn on **Dependabot alerts** and **Dependabot security updates**.
4. Under **Code scanning**, use **Set up -> Default** for CodeQL. GitHub then
runs CodeQL as a dynamic workflow without the fork-token limitations that
affect checked-in advanced workflows.
Do not also add a checked-in CodeQL workflow while default setup is enabled:
GitHub rejects advanced CodeQL uploads when default setup is active. If the
project later needs an advanced CodeQL workflow, disable default setup first
and keep only one CodeQL publishing path active.
## Keeping it current
`.github/dependabot.yml` opens small weekly pull requests to update Python and
npm packages, the Docker base image, and the pinned automation actions
themselves. Review and merge those like any other pull request; they keep the
project patched without manual tracking.

475
docs/setup.md Normal file
View File

@@ -0,0 +1,475 @@
# Odysseus Setup Guide
This page keeps the detailed install, deployment, troubleshooting, and configuration notes out of the front README.
## Quick Start
> **Branch note:** `dev` is the default branch and contains the latest development changes, but it may be unstable. For the more stable curated branch, use [`main`](https://github.com/pewdiepie-archdaemon/odysseus/tree/main).
Defaults work out of the box: clone, run, then configure models/search/email
inside **Settings**. Only edit `.env` for deployment-level overrides like
`APP_BIND`, `APP_PORT`, `AUTH_ENABLED`, `DATABASE_URL`, or a pre-seeded admin password.
On first setup, Odysseus creates an admin account (`admin` unless
`ODYSSEUS_ADMIN_USER` is set) and prints a temporary password in the terminal.
For Docker installs, the same line is in `docker compose logs odysseus`.
Use that for the first login, then change it in **Settings**.
Contributing? See [CONTRIBUTING.md](../CONTRIBUTING.md) for setup, testing, and
pull request guidelines.
### Docker (recommended)
```bash
git clone https://github.com/pewdiepie-archdaemon/odysseus.git
cd odysseus
cp .env.example .env # optional, but recommended for explicit defaults
docker compose up -d --build
```
To include optional extras in the image (PDF viewer, Office extraction; includes AGPL PyMuPDF), build with `docker compose build --build-arg INSTALL_OPTIONAL=true` before `up`.
Open `http://localhost:7000` when the containers are healthy. Docker Compose
binds the web UI to `127.0.0.1` by default. If the port is taken, set
`APP_PORT=7001` in `.env` and recreate the container. Set `APP_BIND=0.0.0.0`
only when you intentionally want LAN/reverse-proxy access.
> **On Apple Silicon (M-series) Macs:** Docker can't reach the Metal GPU, so
> Cookbook serves local models on CPU only. For GPU-accelerated model serving,
> run natively instead — see [Apple Silicon](#apple-silicon) below.
### Native Linux / macOS
```bash
git clone https://github.com/pewdiepie-archdaemon/odysseus.git
cd odysseus
python3 -m venv venv
source venv/bin/activate
pip install -r requirements.txt
python setup.py
python -m uvicorn app:app --host 127.0.0.1 --port 7000
```
Requirements: Python 3.11+. Cookbook also needs `tmux` for background model
downloads and serves. The app itself is lightweight; local model serving is the
heavy part and depends on the model, runtime, GPU, and VRAM, so small hosts can
connect to API or remote model servers instead. Use `--host 0.0.0.0` only when you intentionally want LAN/reverse-proxy access.
### Apple Silicon
Docker on macOS cannot use the Metal GPU. For GPU-accelerated Cookbook on an
M-series Mac, run Odysseus natively:
```bash
git clone https://github.com/pewdiepie-archdaemon/odysseus.git
cd odysseus
./start-macos.sh
```
It launches at `http://127.0.0.1:7860`. To expose it to your phone over a trusted LAN/VPN such as Tailscale, bind all interfaces:
```bash
ODYSSEUS_HOST=0.0.0.0 ./start-macos.sh
# then open http://<tailscale-ip>:7860
```
The script also reads `.env` at startup, so `APP_BIND=0.0.0.0` and `APP_PORT`
set there are picked up automatically without a command-line override each run.
Keep `AUTH_ENABLED=true` (the default) before binding outside loopback. Do not
expose this port directly to the public internet. To build a clickable app wrapper:
```bash
./build-macos-app.sh
```
<details>
<summary>Cookbook, GPU, Ollama, and troubleshooting notes</summary>
**Docker bundled services.** Compose starts Odysseus, ChromaDB, SearXNG, and
ntfy. Odysseus and the bundled service ports bind to `127.0.0.1` by default, so
they are reachable from the host but not exposed to your LAN/public internet
unless you opt in.
**Cookbook storage in Docker.** Downloads live in `./data/huggingface`
(`~/.cache/huggingface` in the container). Cookbook-installed Python CLIs and
serve engines live in `./data/local` (`~/.local` in the container), so they
survive container recreation.
**Remote servers.** In **Cookbook -> Settings -> Servers**, generate the
Odysseus SSH key and add the public key to the remote server's
`~/.ssh/authorized_keys`. From the host you can also run:
```bash
ssh-copy-id -i data/ssh/id_ed25519.pub user@server
```
**Host Docker access (explicit opt-in).** Default Docker Compose intentionally
does not mount `/var/run/docker.sock`. You can still connect Odysseus to
existing Ollama, vLLM, and other OpenAI-compatible endpoints without Docker
socket access.
Cookbook/local Docker-daemon management requires the opt-in overlay below. Raw
Docker socket access is high-trust because it can effectively grant broad
control over the host Docker daemon. Remote server Docker workflows over SSH
remain preferred.
Place these values in `.env`, or export them in the shell before running
`docker compose`:
```bash
COMPOSE_FILE=docker-compose.yml:docker/host-docker.yml
DOCKER_GID=<host docker group gid>
```
Combine host Docker access with a GPU overlay when both are intentionally
required:
```bash
COMPOSE_FILE=docker-compose.yml:docker/gpu.nvidia.yml:docker/host-docker.yml
# or
COMPOSE_FILE=docker-compose.yml:docker/gpu.amd.yml:docker/host-docker.yml
```
**Docker GPU overlays.** CPU-only users can skip this section. Cookbook can
only detect GPUs that Docker exposes to the container — if the host runtime or
device passthrough is not configured, Cookbook sees the iGPU, another card, or
CPU instead of your intended GPU.
For NVIDIA, `scripts/check-docker-gpu.sh` diagnoses GPU passthrough and can
optionally install the host runtime or update `.env`.
```bash
# Read-only diagnostic (default — installs nothing, never edits .env):
scripts/check-docker-gpu.sh
# Print OS-specific install commands without running them:
scripts/check-docker-gpu.sh --print-install-commands
# Install NVIDIA Container Toolkit on Ubuntu/Debian (requires sudo):
scripts/check-docker-gpu.sh --install-nvidia-toolkit
# Write COMPOSE_FILE to .env (only when GPU passthrough is confirmed working):
scripts/check-docker-gpu.sh --enable-nvidia-overlay
# Full assisted setup — install toolkit, then enable overlay if passthrough works:
scripts/check-docker-gpu.sh --install-nvidia-toolkit --enable-nvidia-overlay
```
Safety notes:
- The app never installs host GPU runtime automatically.
- The app never edits `.env` automatically.
- `.env` is only modified when `--enable-nvidia-overlay` is explicitly passed,
and only after GPU passthrough succeeds. `--yes` skips prompts but does not
bypass the passthrough gate.
- `.env.bak.*` backups created by `--enable-nvidia-overlay` are ignored by
Git and the Docker build context.
To enable manually without the script, add this to `.env`:
```bash
COMPOSE_FILE=docker-compose.yml:docker/gpu.nvidia.yml
```
**AMD / ROCm.** AMD setup is read-only diagnostic plus manual `.env` edit. Run:
```bash
scripts/check-docker-amd-gpu.sh
```
Then add the reported values to `.env`, replacing `RENDER_GID` with your host's
numeric render group id:
```bash
COMPOSE_FILE=docker-compose.yml:docker/gpu.amd.yml
RENDER_GID=989
```
For NVIDIA/AMD GPU support, also read the comments in the selected overlay file: docker/gpu.nvidia.yml or docker/gpu.amd.yml.
**Stack-management UIs (Portainer, Coolify, Dockhand, etc.).** These tools
often accept only a single Compose file and do not reliably honor `COMPOSE_FILE`
or multiple `-f` overlays. CLI users should keep using the `COMPOSE_FILE`
overlay workflow above. For stack UIs, point the stack at one of the standalone
files instead, which bundle the base stack plus the GPU settings:
- `docker-compose.gpu-nvidia.yml` — still requires the NVIDIA Container Toolkit
on the host.
- `docker-compose.gpu-amd.yml` — still requires host ROCm/kfd/DRI setup, the
`video`/`render` group membership, and `RENDER_GID` when needed.
The base `docker-compose.yml` plus the `docker/gpu.*.yml` overlays remain the
source of truth; the standalone files mirror them for single-file deployments.
Verify after enabling either overlay:
```bash
docker compose exec odysseus nvidia-smi -L # NVIDIA
docker compose exec odysseus sh -lc 'test -e /dev/kfd && test -d /dev/dri && ls -l /dev/kfd /dev/dri/renderD*' # AMD
```
> **GPU passthrough ≠ llama.cpp CUDA.** `nvidia-smi` passing inside the
> container confirms Docker GPU access, but llama.cpp also needs `cudart` and
> the CUDA Toolkit at runtime. If Cookbook logs show `Unable to find cudart
> library`, `Could NOT find CUDAToolkit`, `CUDA Toolkit not found`, or
> tensors/layers assigned to CPU, that is a Cookbook/llama.cpp build issue —
> not a Docker passthrough failure. Reinstall the serve engine via
> **Cookbook → Dependencies** to get a CUDA-enabled build.
>
> The same split applies to AMD/ROCm: seeing `/dev/kfd` and `/dev/dri` inside
> the container confirms device passthrough, not ROCm userspace or a
> ROCm-enabled vLLM/llama.cpp build. `rocm-smi` and `rocminfo` are not expected
> inside the slim Odysseus image.
**Ollama with Docker.** If Ollama runs on the host, add this endpoint in
Settings:
```text
http://host.docker.internal:11434/v1
```
Ollama must listen outside its own loopback interface:
```bash
OLLAMA_HOST=0.0.0.0:11434 ollama serve
```
This connects Odysseus in Docker to an Ollama server that is already running on
your host machine; it does not start Ollama inside the container.
`host.docker.internal` is Docker's hostname for the host machine from inside the
container. Cookbook **Serve** is a separate workflow for serving downloaded
models through Odysseus/llama.cpp, so Windows users with an existing Ollama
install usually only need to add the endpoint in Settings.
**Useful checks.**
```bash
docker compose ps
docker compose logs --tail=120 odysseus
docker compose logs odysseus | grep -E 'ChromaDB|MemoryVectorStore|DEGRADED'
```
**macOS details.** `start-macos.sh` installs Homebrew deps, creates the venv,
runs setup, and starts uvicorn on port `7860` because AirPlay often holds
`7000`. It uses llama.cpp/Ollama for Metal. vLLM/SGLang are CUDA/ROCm-only and
do not run on macOS. MLX-only models are not served by Odysseus.
</details>
### Native Windows
**One-command launcher** (creates the venv, installs deps, runs setup, starts the
server; safe to re-run):
```powershell
git clone https://github.com/pewdiepie-archdaemon/odysseus.git
cd odysseus
powershell -ExecutionPolicy Bypass -File .\launch-windows.ps1
```
Or do it by hand:
```powershell
git clone https://github.com/pewdiepie-archdaemon/odysseus.git
cd odysseus
py -3.11 -m venv venv
venv\Scripts\Activate.ps1
pip install -r requirements.txt
python setup.py
python -m uvicorn app:app --host 127.0.0.1 --port 7000
```
If `python` points at an older interpreter, use `py -3.12` (or another installed
3.11+ version) for the venv step.
**Exposing on a LAN/Tailscale (Windows):** the launcher binds to `127.0.0.1` and
does **not** read `APP_BIND` / `ODYSSEUS_HOST` from `.env`, so editing `.env`
alone leaves the native Windows server on loopback. Pass the launcher's
`-BindHost` flag instead:
```powershell
powershell -ExecutionPolicy Bypass -File .\launch-windows.ps1 -BindHost 0.0.0.0
```
The manual `uvicorn` command takes the same address as `--host 0.0.0.0`. Bind
outside loopback only for a trusted LAN/VPN such as Tailscale: keep
`AUTH_ENABLED=true` and do not expose the port directly to the public internet.
**Requirements:** Python 3.11+. The core app (chat, agent, memory, documents,
email, calendar, deep research) runs fully native. For full **Cookbook** background
model downloads and the agent shell tool, also install
[Git for Windows](https://git-scm.com/download/win) (provides `bash.exe`).
Local GPU *serving* of vLLM/SGLang needs Linux/WSL2; for a local model on Windows,
[Ollama](https://ollama.com/download) is the easiest path — point Odysseus at
`http://localhost:11434/v1` in Settings.
Open `http://localhost:7000`, log in with the generated admin password,
and configure everything else inside **Settings**.
## Troubleshooting & Advanced Setup
### `chromadb-client` conflicts with embedded ChromaDB
If `chromadb-client` (the lightweight HTTP-only package) is installed alongside the full `chromadb` package, Odysseus starts but ChromaDB silently falls back to HTTP-only mode and fails.
**Fix:** uninstall `chromadb-client` and force-reinstall the full package:
```bash
./venv/bin/pip uninstall chromadb-client -y
./venv/bin/pip install --force-reinstall chromadb
```
### HTTPS + LAN/Tailscale exposure
To expose Odysseus on a local network or Tailscale with HTTPS:
1. Change the bind address to `0.0.0.0` in `.env` (`APP_BIND=0.0.0.0` or `ODYSSEUS_HOST=0.0.0.0`).
2. Generate a locally-trusted cert for your LAN/Tailscale IPs using [mkcert](https://github.com/FiloSottile/mkcert):
```bash
mkcert -install
mkcert -cert-file cert.pem -key-file key.pem 192.168.1.100 tailscale-ip
```
3. Run `uvicorn` with the generated certs:
```bash
python -m uvicorn app:app --host 0.0.0.0 --port 7000 --ssl-certfile=cert.pem --ssl-keyfile=key.pem
```
4. Install the `mkcert` CA on any other device you want to access Odysseus from (e.g., for iOS, email the `rootCA.pem` to yourself, install the profile, and trust it in Certificate Trust Settings).
### Common self-host traps (30-second fixes)
A grab-bag of small gotchas that otherwise turn into long debugging sessions.
- **`AUTH_ENABLED=false` is ignored / you're still forced to log in (Windows).** If you edited `.env` in Notepad it may have saved a UTF-8 **BOM**, turning the first key into `AUTH_ENABLED` so it is never matched. Odysseus loads `.env` with `encoding="utf-8-sig"` to tolerate a leading BOM, but the safe fix is to re-save `.env` as **UTF-8 without BOM** (VS Code: *Save with Encoding → UTF-8*).
- **macOS: the app isn't at `http://localhost:7000`.** macOS AirPlay Receiver usually holds port `7000`, so the macOS start script serves on **`7860`** instead — open `http://localhost:7860`. To use `7000`, free it (System Settings → General → AirDrop & Handoff → turn off *AirPlay Receiver*) and set `APP_PORT=7000`.
- **Copy buttons do nothing over a plain-HTTP Tailscale/LAN URL.** Browsers only expose the clipboard API (`navigator.clipboard`) on **secure origins** — HTTPS, or `localhost`. Over `http://100.x.y.z:7860` it is blocked. Serve over HTTPS (see *HTTPS + LAN/Tailscale exposure* above); `localhost` is exempt, so copy still works on the host itself.
- **Self-hosted ntfy reminders don't reach your phone.** Two things: (1) the bundled ntfy binds to loopback by default — to reach it from your phone set `NTFY_BIND` to your host/Tailscale IP and `NTFY_BASE_URL` to the same server URL in `.env`, then recreate the ntfy container (see the `NTFY_*` block in `.env.example`); (2) in the ntfy **Android** app, subscribe to the topic with **Instant delivery** enabled — non-`ntfy.sh` servers don't get instant push otherwise.
- **Local mail (Dovecot) login fails: "Plaintext authentication disallowed on non-encrypted connections."** Your IMAP/SMTP server is refusing cleartext auth over an unencrypted link. Prefer enabling TLS on the mail server; on a trusted LAN only, you can allow cleartext (Dovecot: `disable_plaintext_auth = no`).
- **Calendar/contacts (Radicale) won't sync.** Point Odysseus at the **full collection URL** with its trailing slash — e.g. `http://host:5232/<user>/<collection-id>/` — not just the server root. Radicale shows this address for each calendar/address book in its web UI.
### Optional Dependencies
`requirements-optional.txt` contains packages that unlock extra features. It is not installed by default.
| Package | Feature unlocked |
|---------|-----------------|
| `faster-whisper` | Local speech-to-text (microphone -> text) via the "local" STT provider. |
| `ddgs` | DuckDuckGo as a search provider option. |
| `PyMuPDF` | PDF page rendering in the side viewer panel and form-filling. (Note: AGPL-3.0) |
| `markitdown` | Office/EPUB document text extraction (converts .docx/.xlsx/.pptx/.xls/.epub to Markdown). |
### Faster, reproducible installs with uv (optional)
[uv](https://docs.astral.sh/uv/) works as a drop-in replacement for the
venv + pip steps in the native install guides, no project changes are needed but this change results in faster installs along with a lockfile for reproducible environments. After [installing `uv`](https://docs.astral.sh/uv/getting-started/installation/), use:
```bash
uv venv venv --python 3.13
uv pip install -r requirements.txt
# then continue as usual: python setup.py, uvicorn, ...
```
`requirements.txt` is intentionally unpinned, so two installs at different times can produce different package versions. If you want a reproducible environment (e.g. across your own machines, or to roll back after a bad upgrade), snapshot and restore exact versions with:
```bash
uv pip compile requirements.txt -o requirements.lock # snapshot current resolution
uv pip sync requirements.lock # reproduce it exactly later
```
`requirements.lock` is gitignored and platform-specific (compile it on the OS you deploy to). Regenerate it deliberately when you want to take upgrades. The plain `uv pip install -r requirements.txt` keeps following the unpinned requirements like pip does.
### Outlook / Office 365 email
Odysseus email accounts currently use IMAP/SMTP username-password auth. Outlook
and Microsoft 365 generally require OAuth instead, so normal Microsoft mailbox
passwords will fail. See [docs/email-outlook.md](docs/email-outlook.md) for the
current limitation and the planned integration direction.
## Security Notes
Odysseus is a self-hosted workspace with powerful local tools: shell access, file uploads, model downloads, web research, email/calendar integrations, and API tokens. Treat it like an admin console.
- Keep `AUTH_ENABLED=true` for any network-accessible deployment.
- Keep `LOCALHOST_BYPASS=false` outside local development.
- Use `SECURE_COOKIES=true` when Odysseus is served through HTTPS by a trusted reverse proxy or private access gateway.
- Do not expose it directly to the public internet without HTTPS and a trusted reverse proxy or private access layer.
- Keep `.env`, `data/`, `logs/`, databases, uploads, generated media, backups, auth/session files, API keys, and model/provider tokens out of Git and private shares. They are ignored by default.
- Review `data/auth.json` after first boot: disable open signup unless you intentionally want it, make only your own account admin, and keep demo/test accounts non-admin.
- Non-admin users do not get shell/Python/file read/write by default, and admin-only routes/tools such as MCP management, API tokens, webhooks, model/cookbook serving, backup/vault, and app settings are admin-gated. Other features are controlled by per-user privileges, so review each user's privileges before exposing a deployment.
- Rotate any API keys or tokens that were ever pasted into a shared chat, demo, screenshot, or log.
- If you enable API tokens or webhooks, create separate tokens per integration and delete unused ones.
- Prefer binding manual development runs to `127.0.0.1`; bind to `0.0.0.0` only when you intentionally want LAN/reverse-proxy access.
- Keep ChromaDB, SearXNG, ntfy, Ollama, vLLM, llama.cpp, databases, and raw model/provider APIs internal-only. Expose only the authenticated Odysseus web/API entrypoint through your trusted proxy or private access layer.
- Before publishing a fork, run `git status --short` and confirm no private files from `.env`, `data/`, `logs/`, uploads, backups, or local databases are staged.
### Private or proxied deployments
Odysseus serves plain HTTP on its app port. Docker Compose binds Odysseus and the bundled services to `127.0.0.1` by default, so a typical production/private setup is:
1. Keep Odysseus on localhost, for example `127.0.0.1:7000`.
2. Terminate HTTPS at a trusted reverse proxy or private access gateway.
3. Put the authenticated Odysseus web/API entrypoint behind that layer.
4. Keep raw service and model ports internal-only.
Cloudflare Access, Tailscale, Caddy, nginx, and Traefik can all fit this pattern; none are required by Odysseus. If your access layer reaches Odysseus on the same host, proxy to `http://127.0.0.1:7000` and keep `AUTH_ENABLED=true`, `LOCALHOST_BYPASS=false`, and `SECURE_COOKIES=true`.
`ALLOWED_ORIGINS` lists exact permitted origins for cross-origin browser/API clients; ordinary same-origin reverse-proxy access usually does not need a special CORS entry.
Common internal-only ports from the default docs/compose setup:
| Port | Service |
|---|---|
| `7000` | Odysseus raw app port |
| `8080` | SearXNG |
| `8091` | ntfy |
| `8100` | ChromaDB host port for manual/compose access |
| `11434` | Ollama |
| `8000-8020` | Common local model/provider APIs |
## Configuration
Most setup is done inside the app with `/setup` or **Settings**. Use `.env`
for deployment-level defaults and secrets you want present before first boot.
Key settings:
| Variable | Default | Description |
|---|---|---|
| `LLM_HOST` | `localhost` | Your LLM server (e.g. `llm-host.local:8000`) |
| `LLM_HOSTS` | -- | Comma-separated list for model discovery |
| `OPENAI_API_KEY` | -- | Optional OpenAI key. Prefer adding providers in the app unless pre-seeding. |
| `SEARXNG_INSTANCE` | `http://localhost:8080` | SearXNG URL. Docker overrides this to `http://searxng:8080`. |
| `SEARXNG_SECRET` | generated on first Docker boot | Optional SearXNG cookie/CSRF secret. Leave blank unless you need to pin it. |
| `APP_BIND` | `127.0.0.1` | Docker Compose host bind address for the web UI. Use `0.0.0.0` only for intentional LAN/reverse-proxy access. |
| `APP_PORT` | `7000` | Docker Compose host port for the web UI. |
| `APP_DATA_DIR` | `./data` | Docker Compose host directory for application data volumes. |
| `APP_LOGS_DIR` | `./logs` | Docker Compose host directory for application logs. |
| `AUTH_ENABLED` | `true` | Enable/disable login |
| `LOCALHOST_BYPASS` | `false` | Development-only auth bypass for loopback requests. Keep false for shared/network deployments. |
| `ALLOWED_ORIGINS` | `http://localhost,http://127.0.0.1` | Comma-separated exact permitted origins for cross-origin browser/API clients. |
| `SECURE_COOKIES` | `false` | Set true when serving Odysseus through HTTPS at a trusted proxy or private access gateway. |
| `DATABASE_URL` | `sqlite:///./data/app.db` | Database connection string |
| `CHROMADB_HOST` | `localhost` | ChromaDB host for vector memory. Docker overrides this to `chromadb`. |
| `CHROMADB_PORT` | `8100` | ChromaDB port for manual host runs. Docker overrides this to `8000`. |
| `EMBEDDING_URL` | -- | OpenAI-compatible embeddings endpoint |
| `ODYSSEUS_CHAT_UPLOAD_MAX_BYTES` | `10485760` | Chat/agent attachment cap in bytes. Raise for larger local PDFs or text documents. |
| `ODYSSEUS_GALLERY_UPLOAD_MAX_BYTES` | `104857600` | Gallery image upload cap in bytes (100 MB). |
| `ODYSSEUS_GALLERY_TRANSFORM_UPLOAD_MAX_BYTES` | `26214400` | Gallery transform input cap in bytes (25 MB). |
| `ODYSSEUS_MEMORY_IMPORT_MAX_BYTES` | `10485760` | Memory import file cap in bytes (10 MB). |
| `ODYSSEUS_PERSONAL_UPLOAD_MAX_BYTES` | `26214400` | Personal document upload cap in bytes (25 MB). |
| `ODYSSEUS_EMAIL_COMPOSE_UPLOAD_MAX_BYTES` | `26214400` | Email compose attachment cap in bytes (25 MB). |
| `ODYSSEUS_STT_MAX_AUDIO_BYTES` | `26214400` | Speech-to-text audio cap in bytes (25 MB). |
| `ODYSSEUS_ICS_MAX_BYTES` | `10485760` | Calendar `.ics` import cap in bytes (10 MB). |
All upload-limit vars are validated (must be a positive integer) and optional; an invalid value fails fast at startup.
### Built-in MCP servers (optional setup)
Odysseus auto-registers a few built-in MCP servers at startup. The npx-based ones (currently the browser server, `@playwright/mcp`) only start when their npm package is already in the local npx cache. If a package isn't cached, that server is skipped with a startup log message explaining what to do, so a fresh install does not block on a multi-minute npm download or hang if Playwright system deps are missing.
To enable the browser MCP (page navigation, screenshots, vision), run once:
```bash
npx -y @playwright/mcp@latest --version
```
That installs `@playwright/mcp` plus Playwright (~300MB total). Restart Odysseus and the server will register at startup.
## Architecture
```
app.py # FastAPI entry point
core/ auth, database, middleware, constants
src/ llm_core, agent_loop, agent_tools, chat_processor, search/
routes/ chat, session, document, memory, model … endpoints
services/ docs, memory, search, hwfit (Cookbook) …
static/ index.html + app.js + style.css + js/ (modular front-end)
docs/ landing page (index.html) + preview clips
```
## Data
All user data lives in `data/` (gitignored): `app.db` (sessions, messages, documents),
`memory.json`, `presets.json`, `uploads/`, `personal_docs/`, `chroma/`, `settings.json`.
To back up or restore everything in `data/`, see the
[Backup & Restore guide](backup-restore.md).

1
integrations/claude/skills/odysseus/SKILL.md
View File

@@ -102,6 +102,7 @@ python3 ~/.claude/skills/odysseus/scripts/odysseus_api.py POST /api/codex/memory
## Email draft + send
- Prefer `POST /api/codex/emails/draft-document` for agent-written email replies. It creates an editable Odysseus Document with `language: "email"` and does not touch IMAP/send.
- `POST /api/codex/emails/draft` — body matches `SendEmailRequest` (`to`, `cc`, `bcc`, `subject`, `body`, `body_html`, `attachments`, `account_id`, `in_reply_to`, `references`). Requires `email:draft` (or `email:send`).
- `POST /api/codex/emails/send` — same body. Requires `email:send`. Never send without explicit user instruction.

32
integrations/claude/skills/odysseus/scripts/odysseus_api.py
View File

@@ -17,6 +17,11 @@ def _usage() -> int:
print(" odysseus_api.py todos add TITLE", file=sys.stderr)
print(" odysseus_api.py emails list [limit]", file=sys.stderr)
print(" odysseus_api.py emails read UID", file=sys.stderr)
print(" odysseus_api.py emails draft-doc JSON_PAYLOAD", file=sys.stderr)
print(" odysseus_api.py documents list [limit]", file=sys.stderr)
print(" odysseus_api.py documents read DOC_ID", file=sys.stderr)
print(" odysseus_api.py documents create JSON_PAYLOAD", file=sys.stderr)
print(" odysseus_api.py documents delete DOC_ID", file=sys.stderr)
print(" odysseus_api.py cookbook tasks", file=sys.stderr)
print(" odysseus_api.py cookbook servers", file=sys.stderr)
print(" odysseus_api.py cookbook cached [HOST]", file=sys.stderr)
@@ -79,6 +84,33 @@ def main() -> int:
method = "GET"
path = f"/api/codex/emails/{sys.argv[3]}"
body = None
elif action in ("draft-doc", "draft_document") and len(sys.argv) >= 4:
method = "POST"
path = "/api/codex/emails/draft-document"
body = " ".join(sys.argv[3:])
else:
return _usage()
elif command in ("documents", "docs"):
if len(sys.argv) < 3:
return _usage()
action = sys.argv[2].lower()
if action == "list":
method = "GET"
limit = sys.argv[3] if len(sys.argv) >= 4 else "50"
path = f"/api/codex/documents?limit={limit}"
body = None
elif action == "read" and len(sys.argv) >= 4:
method = "GET"
path = f"/api/codex/documents/{sys.argv[3]}"
body = None
elif action == "create" and len(sys.argv) >= 4:
method = "POST"
path = "/api/codex/documents"
body = " ".join(sys.argv[3:])
elif action == "delete" and len(sys.argv) >= 4:
method = "DELETE"
path = f"/api/codex/documents/{sys.argv[3]}"
body = None
else:
return _usage()
elif command == "cookbook":

32
integrations/codex/scripts/odysseus_api.py
View File

@@ -17,6 +17,11 @@ def _usage() -> int:
print(" odysseus_api.py todos add TITLE", file=sys.stderr)
print(" odysseus_api.py emails list [limit]", file=sys.stderr)
print(" odysseus_api.py emails read UID", file=sys.stderr)
print(" odysseus_api.py emails draft-doc JSON_PAYLOAD", file=sys.stderr)
print(" odysseus_api.py documents list [limit]", file=sys.stderr)
print(" odysseus_api.py documents read DOC_ID", file=sys.stderr)
print(" odysseus_api.py documents create JSON_PAYLOAD", file=sys.stderr)
print(" odysseus_api.py documents delete DOC_ID", file=sys.stderr)
print(" odysseus_api.py cookbook tasks", file=sys.stderr)
print(" odysseus_api.py cookbook servers", file=sys.stderr)
print(" odysseus_api.py cookbook cached [HOST]", file=sys.stderr)
@@ -79,6 +84,33 @@ def main() -> int:
method = "GET"
path = f"/api/codex/emails/{sys.argv[3]}"
body = None
elif action in ("draft-doc", "draft_document") and len(sys.argv) >= 4:
method = "POST"
path = "/api/codex/emails/draft-document"
body = " ".join(sys.argv[3:])
else:
return _usage()
elif command in ("documents", "docs"):
if len(sys.argv) < 3:
return _usage()
action = sys.argv[2].lower()
if action == "list":
method = "GET"
limit = sys.argv[3] if len(sys.argv) >= 4 else "50"
path = f"/api/codex/documents?limit={limit}"
body = None
elif action == "read" and len(sys.argv) >= 4:
method = "GET"
path = f"/api/codex/documents/{sys.argv[3]}"
body = None
elif action == "create" and len(sys.argv) >= 4:
method = "POST"
path = "/api/codex/documents"
body = " ".join(sys.argv[3:])
elif action == "delete" and len(sys.argv) >= 4:
method = "DELETE"
path = f"/api/codex/documents/{sys.argv[3]}"
body = None
else:
return _usage()
elif command == "cookbook":

1
integrations/codex/skills/odysseus/SKILL.md
View File

@@ -102,6 +102,7 @@ python3 integrations/codex/scripts/odysseus_api.py POST /api/codex/memory '{"tex
## Email draft + send
- Prefer `POST /api/codex/emails/draft-document` for Codex-written email replies. It creates an editable Odysseus Document with `language: "email"` and does not touch IMAP/send.
- `POST /api/codex/emails/draft` — body matches `SendEmailRequest` (`to`, `cc`, `bcc`, `subject`, `body`, `body_html`, `attachments`, `account_id`, `in_reply_to`, `references`). Requires `email:draft` (or `email:send`).
- `POST /api/codex/emails/send` — same body. Requires `email:send`. Never send without explicit user instruction.

39
launch-windows.ps1
View File

@@ -30,14 +30,26 @@ function Fail($msg) {
exit 1
}
function Test-WindowsBashStub($path) {
if (-not $path) { return $false }
$lowered = $path.ToLowerInvariant()
foreach ($stub in @("system32\bash.exe", "sysnative\bash.exe", "windowsapps\bash.exe")) {
if ($lowered.Contains($stub)) { return $true }
}
return $false
}
function Find-GitBash {
$cmd = Get-Command bash -ErrorAction SilentlyContinue
if ($cmd) { return $cmd.Source }
if ($cmd -and -not (Test-WindowsBashStub $cmd.Source)) { return $cmd.Source }
$roots = @()
foreach ($name in @("ProgramFiles", "ProgramW6432", "ProgramFiles(x86)", "LocalAppData")) {
$base = [Environment]::GetEnvironmentVariable($name)
if ($base) { $roots += (Join-Path $base "Git") }
if ($base) {
$roots += (Join-Path $base "Git")
if ($name -eq "LocalAppData") { $roots += (Join-Path $base "Programs\Git") }
}
}
$roots += @("C:\Program Files\Git", "C:\Program Files (x86)\Git")
@@ -93,6 +105,14 @@ if (-not $pyExe) {
}
}
if ($pyExe -like "*WindowsApps*python.exe") {
$pyCmd = Get-Command py -ErrorAction SilentlyContinue
if ($pyCmd) {
$pyExe = $pyCmd.Source
$pyArgs = @("-3.11")
}
}
if (-not $pyExe) {
Fail "Couldn't find Python 3.11+ for Windows setup. Install Python 3.11+ (or open the Python launcher with 'py -3.11') from https://www.python.org/downloads/, then re-run this script."
}
@@ -129,7 +149,20 @@ if (-not (Find-GitBash)) {
Write-Host " https://git-scm.com/download/win" -ForegroundColor Yellow
}
# 6. Start the server (use `python -m uvicorn` - bare `uvicorn` may not be on PATH)
# 6. Point CUDA_PATH at a real CUDA toolkit so GPU llama-cpp-python can import.
$cudaBase = "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA"
if (Test-Path $cudaBase) {
$cudaBest = Get-ChildItem $cudaBase -Directory -ErrorAction SilentlyContinue |
Where-Object { Test-Path (Join-Path $_.FullName "bin") } |
Sort-Object { try { [version]($_.Name -replace "^v", "") } catch { [version]"0.0" } } -Descending |
Select-Object -First 1
if ($cudaBest) {
$env:CUDA_PATH = $cudaBest.FullName
Write-Host ("Using CUDA_PATH = " + $cudaBest.FullName) -ForegroundColor Cyan
}
}
# 7. Start the server (use `python -m uvicorn` - bare `uvicorn` may not be on PATH)
Write-Step ("Starting Odysseus at http://{0}:{1}" -f $BindHost, $Port)
Write-Host "Press Ctrl+C to stop."
Write-Host ""

142
launcher.py Normal file
View File

@@ -0,0 +1,142 @@
# launcher.py
"""Dedicated entrypoint for the standalone Windows portable launcher.
Handles:
- Immediate GUI splash screen creation using tkinter.
- Suppressing console stream crashes in windowed GUI mode via NullWriter.
- Spawning system tray icon via pystray and Pillow (lazy-loaded).
- Auto-opening default browser pointing to the running backend.
- Launching the FastAPI server (importing and running app.py).
"""
import os
import sys
import threading
import time
import webbrowser
# Define a dummy NullWriter to suppress standard stream crashes (isatty etc.) in GUI mode
class NullWriter:
def write(self, text):
pass
def flush(self):
pass
def isatty(self):
return False
if sys.stdout is None:
sys.stdout = NullWriter()
if sys.stderr is None:
sys.stderr = NullWriter()
splash_root = None
# If running from a frozen PyInstaller bundle, launch the splash screen IMMEDIATELY
if getattr(sys, 'frozen', False):
import tkinter as tk
def show_splash_instantly():
global splash_root
try:
splash_root = tk.Tk()
splash_root.title("Odysseus")
splash_root.overrideredirect(True)
splash_root.configure(bg="#1a1c23")
# Accented borders
splash_root.config(highlightbackground="#e06c75", highlightcolor="#e06c75", highlightthickness=1)
w, h = 360, 160
ws = splash_root.winfo_screenwidth()
hs = splash_root.winfo_screenheight()
x = (ws - w) // 2
y = (hs - h) // 2
splash_root.geometry(f"{w}x{h}+{x}+{y}")
tk.Label(splash_root, text="⛵ Odysseus", font=("Segoe UI", 22, "bold"), bg="#1a1c23", fg="#e06c75").pack(pady=(22, 2))
tk.Label(splash_root, text="Launching background services...", font=("Segoe UI", 10), bg="#1a1c23", fg="#d1d4e0").pack(pady=2)
tk.Label(splash_root, text="Please wait, this will take a few seconds.", font=("Segoe UI", 8, "italic"), bg="#1a1c23", fg="#5c6370").pack(pady=(12, 0))
splash_root.attributes("-topmost", True)
splash_root.mainloop()
except Exception:
pass
# Launch the GUI splash screen immediately on a background thread
threading.Thread(target=show_splash_instantly, daemon=True).start()
def create_tray_image():
# Generate a beautiful 64x64 icon matching Odysseus brand red accent (#e06c75)
from PIL import Image, ImageDraw
image = Image.new('RGBA', (64, 64), (0, 0, 0, 0))
dc = ImageDraw.Draw(image)
accent_red = (224, 108, 117, 255)
light_red = (224, 108, 117, 150)
# Draw premium sailing boat
dc.polygon([(32, 10), (32, 45), (12, 45)], fill=accent_red)
dc.polygon([(32, 18), (32, 45), (48, 45)], fill=light_red)
dc.polygon([(8, 48), (56, 48), (44, 56), (20, 56)], fill=accent_red)
return image
def on_open_browser(icon, item, url):
webbrowser.open(url)
def on_exit(icon, item):
icon.stop()
os._exit(0)
def setup_system_tray(url):
try:
import pystray
icon_img = create_tray_image()
menu = (
pystray.MenuItem('Open Odysseus', lambda icon, item: on_open_browser(icon, item, url), default=True),
pystray.MenuItem('Exit', on_exit)
)
tray_icon = pystray.Icon(
"Odysseus",
icon_img,
"Odysseus",
menu
)
tray_icon.run()
except Exception:
pass
def open_browser(url):
# Allow uvicorn and app lifecycles to complete warmups
time.sleep(3.5)
# Safely close the splash screen
try:
global splash_root
if splash_root:
splash_root.after(0, splash_root.destroy)
except Exception:
pass
webbrowser.open(url)
if __name__ == "__main__":
import uvicorn
# Import the FastAPI app from app.py
from app import app
bind_host = os.getenv("APP_BIND", "127.0.0.1")
bind_port = int(os.getenv("APP_PORT", "7000"))
url = f"http://{bind_host}:{bind_port}"
if getattr(sys, 'frozen', False):
# Start browser manager thread
threading.Thread(target=open_browser, args=(url,), daemon=True).start()
# Start system tray manager thread
threading.Thread(target=setup_system_tray, args=(url,), daemon=True).start()
uvicorn.run(app, host=bind_host, port=bind_port, log_level="info")

94
licenses/OpenDyslexic-OFL.txt Normal file
View File

@@ -0,0 +1,94 @@
Copyright (c) 2019-07-29, Abbie Gonzalez (https://abbiecod.es|support@abbiecod.es),
with Reserved Font Name OpenDyslexic.
Copyright (c) 12/2012 - 2019
This Font Software is licensed under the SIL Open Font License, Version 1.1.
This license is copied below, and is also available with a FAQ at:
http://scripts.sil.org/OFL
-----------------------------------------------------------
SIL OPEN FONT LICENSE Version 1.1 - 26 February 2007
-----------------------------------------------------------
PREAMBLE
The goals of the Open Font License (OFL) are to stimulate worldwide
development of collaborative font projects, to support the font creation
efforts of academic and linguistic communities, and to provide a free and
open framework in which fonts may be shared and improved in partnership
with others.
The OFL allows the licensed fonts to be used, studied, modified and
redistributed freely as long as they are not sold by themselves. The
fonts, including any derivative works, can be bundled, embedded,
redistributed and/or sold with any software provided that any reserved
names are not used by derivative works. The fonts and derivatives,
however, cannot be released under any other type of license. The
requirement for fonts to remain under this license does not apply
to any document created using the fonts or their derivatives.
DEFINITIONS
"Font Software" refers to the set of files released by the Copyright
Holder(s) under this license and clearly marked as such. This may
include source files, build scripts and documentation.
"Reserved Font Name" refers to any names specified as such after the
copyright statement(s).
"Original Version" refers to the collection of Font Software components as
distributed by the Copyright Holder(s).
"Modified Version" refers to any derivative made by adding to, deleting,
or substituting -- in part or in whole -- any of the components of the
Original Version, by changing formats or by porting the Font Software to a
new environment.
"Author" refers to any designer, engineer, programmer, technical
writer or other person who contributed to the Font Software.
PERMISSION & CONDITIONS
Permission is hereby granted, free of charge, to any person obtaining
a copy of the Font Software, to use, study, copy, merge, embed, modify,
redistribute, and sell modified and unmodified copies of the Font
Software, subject to the following conditions:
1) Neither the Font Software nor any of its individual components,
in Original or Modified Versions, may be sold by itself.
2) Original or Modified Versions of the Font Software may be bundled,
redistributed and/or sold with any software, provided that each copy
contains the above copyright notice and this license. These can be
included either as stand-alone text files, human-readable headers or
in the appropriate machine-readable metadata fields within text or
binary files as long as those fields can be easily viewed by the user.
3) No Modified Version of the Font Software may use the Reserved Font
Name(s) unless explicit written permission is granted by the corresponding
Copyright Holder. This restriction only applies to the primary font name as
presented to the users.
4) The name(s) of the Copyright Holder(s) or the Author(s) of the Font
Software shall not be used to promote, endorse or advertise any
Modified Version, except to acknowledge the contribution(s) of the
Copyright Holder(s) and the Author(s) or with their explicit written
permission.
5) The Font Software, modified or unmodified, in part or in whole,
must be distributed entirely under this license, and must not be
distributed under any other license. The requirement for fonts to
remain under this license does not apply to any document created
using the Font Software.
TERMINATION
This license becomes null and void if any of the above conditions are
not met.
DISCLAIMER
THE FONT SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO ANY WARRANTIES OF
MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
OF COPYRIGHT, PATENT, TRADEMARK, OR OTHER RIGHT. IN NO EVENT SHALL THE
COPYRIGHT HOLDER BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
INCLUDING ANY GENERAL, SPECIAL, INDIRECT, INCIDENTAL, OR CONSEQUENTIAL
DAMAGES, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
FROM, OUT OF THE USE OR INABILITY TO USE THE FONT SOFTWARE OR FROM
OTHER DEALINGS IN THE FONT SOFTWARE.

22
mcp_servers/_common.py
View File

@@ -1,22 +0,0 @@
"""
_common.py
Shared constants and helpers for built-in MCP servers.
"""
MAX_OUTPUT_CHARS = 10_000
MAX_READ_CHARS = 20_000
SHELL_TIMEOUT = 60
PYTHON_TIMEOUT = 30
SEARCH_TIMEOUT = 30
def truncate(text: str, limit: int = MAX_OUTPUT_CHARS) -> str:
"""Truncate text to *limit* characters with a suffix note."""
if not isinstance(text, str):
# Tool output is occasionally None or a non-string; len(None) would
# raise. Coerce so this shared helper never crashes a tool response.
text = "" if text is None else str(text)
if len(text) > limit:
return text[:limit] + f"\n... (truncated, {len(text)} chars total)"
return text

1220
mcp_servers/email_server.py
View File

File diff suppressed because it is too large Load Diff

22
mcp_servers/image_gen_server.py
View File

@@ -16,6 +16,8 @@ from mcp.types import Tool, TextContent
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from src.constants import GENERATED_IMAGES_DIR
server = Server("image_gen")
@@ -71,7 +73,7 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
if not model_spec:
for candidate in ("gpt-image-1.5", "gpt-image-1", "dall-e-3"):
try:
_resolve_model(candidate)
await asyncio.to_thread(_resolve_model, candidate)
model_spec = candidate
break
except ValueError:
@@ -79,7 +81,7 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
if not model_spec:
return [TextContent(type="text", text="Error: No image model found. Configure one in Admin.")]
url, model_id, headers = _resolve_model(model_spec)
url, model_id, headers = await asyncio.to_thread(_resolve_model, model_spec)
is_gpt_image = "gpt-image" in model_id.lower()
base_url = url.replace("/chat/completions", "").replace("/v1/messages", "").rstrip("/")
@@ -115,14 +117,18 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
img = images[0]
image_url = None
# Prefix the instance's public base URL (existing app_public_url setting) so the
# link is fully-qualified and clickable when the model echoes it. Empty = relative
# same-origin path (unchanged default).
_pub_base = (get_setting("app_public_url", "") or "").rstrip("/")
if img.get("b64_json"):
img_dir = Path("data/generated_images")
img_dir = Path(GENERATED_IMAGES_DIR)
img_dir.mkdir(parents=True, exist_ok=True)
filename = f"{uuid.uuid4().hex[:12]}.png"
img_path = img_dir / filename
img_path.write_bytes(base64.b64decode(img["b64_json"]))
image_url = f"/api/generated-image/{filename}"
image_url = f"{_pub_base}/api/generated-image/{filename}"
# Save to gallery
try:
@@ -146,7 +152,13 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
else:
return [TextContent(type="text", text="Error: Unexpected image API response format")]
result = f"Generated image for: {prompt[:100]}\nimage_url: {image_url}\nmodel: {model_id}\nsize: {size}"
# "Direct link:" rather than an "image_url:" label — small models copied the
# label token ("image_url") into the link href, producing a broken link.
result = (
f"Generated image for: {prompt[:100]}\n"
f"Direct link: {image_url}\n"
f"model: {model_id}\nsize: {size}"
)
return [TextContent(type="text", text=result)]
except httpx.TimeoutException:

124
mcp_servers/memory_server.py
View File

@@ -6,6 +6,7 @@ Imports MemoryManager and MemoryVectorStore from the Odysseus codebase.
"""
import asyncio
import os
import sys
import time
from pathlib import Path
@@ -23,6 +24,55 @@ _memory_manager = None
_memory_vector = None
_initialized = False
_OWNER_ENV_KEYS = ("ODYSSEUS_MCP_MEMORY_OWNER", "ODYSSEUS_MEMORY_OWNER")
_OWNER_SCOPE_ERROR = (
"Error: Memory MCP owner is not configured for an owner-scoped memory store. "
"Set ODYSSEUS_MCP_MEMORY_OWNER for this server or use the owner-aware native memory tool."
)
def _configured_owner() -> str | None:
for key in _OWNER_ENV_KEYS:
owner = os.environ.get(key, "").strip()
if owner:
return owner
return None
def _entry_owner(entry: dict) -> str | None:
owner = entry.get("owner")
if owner is None:
return None
owner_text = str(owner).strip()
return owner_text or None
def _owner_scoped_store(entries: list[dict]) -> bool:
return any(_entry_owner(entry) for entry in entries if isinstance(entry, dict))
def _scope_entries() -> tuple[str | None, list[dict], list[dict], str | None]:
"""Return configured owner, all entries, visible entries, and optional error."""
entries = _memory_manager.load_all()
owner = _configured_owner()
if owner is None and _owner_scoped_store(entries):
return None, entries, [], _OWNER_SCOPE_ERROR
if owner is None:
visible = [
entry for entry in entries
if isinstance(entry, dict) and _entry_owner(entry) is None
]
else:
visible = [
entry for entry in entries
if isinstance(entry, dict) and _entry_owner(entry) == owner
]
return owner, entries, visible, None
def _text_result(text: str) -> list[TextContent]:
return [TextContent(type="text", text=text)]
def _ensure_init():
"""Lazy-init memory managers on first use."""
@@ -75,43 +125,46 @@ async def list_tools() -> list[Tool]:
@server.call_tool()
async def call_tool(name: str, arguments: dict) -> list[TextContent]:
if name != "manage_memory":
return [TextContent(type="text", text=f"Unknown tool: {name}")]
return _text_result(f"Unknown tool: {name}")
_ensure_init()
if not _memory_manager:
return [TextContent(type="text", text="Error: Memory manager not available")]
return _text_result("Error: Memory manager not available")
action = arguments.get("action", "")
if action == "list":
category_filter = arguments.get("category", "")
memories = _memory_manager.load()
_owner, _all_memories, memories, scope_error = _scope_entries()
if scope_error:
return _text_result(scope_error)
if category_filter:
memories = [m for m in memories if m.get("category", "").lower() == category_filter.lower()]
if not memories:
msg = "No memories found"
if category_filter:
msg += f" in category '{category_filter}'"
return [TextContent(type="text", text=msg + ".")]
return _text_result(msg + ".")
lines = [f"Found {len(memories)} memory entries:\n"]
for m in memories[:100]:
for m in memories:
cat = m.get("category", "fact")
mid = m.get("id", "?")[:8]
text = m.get("text", "")
if len(text) > 150:
text = text[:150] + "..."
lines.append(f"- [{cat}] `{mid}` — {text}")
if len(memories) > 100:
lines.append(f"... and {len(memories) - 100} more")
return [TextContent(type="text", text="\n".join(lines))]
return _text_result("\n".join(lines))
elif action == "add":
text = arguments.get("text", "")
category = arguments.get("category", "fact")
if not text:
return [TextContent(type="text", text="Error: Memory text cannot be empty")]
entry = _memory_manager.add_entry(text, source="ai_agent", category=category)
memories = _memory_manager.load_all()
return _text_result("Error: Memory text cannot be empty")
owner, memories, _visible, scope_error = _scope_entries()
if scope_error:
return _text_result(scope_error)
entry = _memory_manager.add_entry(text, source="ai_agent", category=category, owner=owner)
memories.append(entry)
_memory_manager.save(memories)
if _memory_vector and _memory_vector.healthy:
@@ -119,25 +172,28 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
_memory_vector.add(entry["id"], text)
except Exception:
pass
return [TextContent(type="text", text=f"Memory added: [{category}] {text} (id: {entry['id'][:8]})")]
return _text_result(f"Memory added: [{category}] {text} (id: {entry['id'][:8]})")
elif action == "edit":
memory_id = arguments.get("memory_id", "")
new_text = arguments.get("text", "")
if not memory_id or not new_text:
return [TextContent(type="text", text="Error: edit needs memory_id and text")]
memories = _memory_manager.load_all()
found = False
return _text_result("Error: edit needs memory_id and text")
_owner, memories, visible, scope_error = _scope_entries()
if scope_error:
return _text_result(scope_error)
full_id = None
for m in memories:
for m in visible:
if m.get("id", "").startswith(memory_id):
m["text"] = new_text
m["timestamp"] = int(time.time())
found = True
full_id = m["id"]
break
if not found:
return [TextContent(type="text", text=f"Error: Memory '{memory_id}' not found")]
if not full_id:
return _text_result(f"Error: Memory '{memory_id}' not found")
for m in memories:
if m.get("id") == full_id:
m["text"] = new_text
m["timestamp"] = int(time.time())
break
_memory_manager.save(memories)
if _memory_vector and _memory_vector.healthy and full_id:
try:
@@ -145,24 +201,26 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
_memory_vector.add(full_id, new_text)
except Exception:
pass
return [TextContent(type="text", text=f"Memory updated: {new_text}")]
return _text_result(f"Memory updated: {new_text}")
elif action == "delete":
memory_id = arguments.get("memory_id", "")
if not memory_id:
return [TextContent(type="text", text="Error: delete needs memory_id")]
memories = _memory_manager.load_all()
return _text_result("Error: delete needs memory_id")
_owner, memories, visible, scope_error = _scope_entries()
if scope_error:
return _text_result(scope_error)
full_id = None
deleted_text = ""
deleted_category = ""
for m in memories:
for m in visible:
if m.get("id", "").startswith(memory_id):
full_id = m["id"]
deleted_text = m.get("text", "")
deleted_category = m.get("category", "")
break
if not full_id:
return [TextContent(type="text", text=f"Error: Memory '{memory_id}' not found")]
return _text_result(f"Error: Memory '{memory_id}' not found")
memories = [m for m in memories if m.get("id") != full_id]
_memory_manager.save(memories)
if _memory_vector and _memory_vector.healthy and full_id:
@@ -172,30 +230,32 @@ async def call_tool(name: str, arguments: dict) -> list[TextContent]:
pass
cat = f"[{deleted_category}] " if deleted_category else ""
snippet = deleted_text if len(deleted_text) <= 120 else deleted_text[:117] + "..."
return [TextContent(type="text", text=f"Memory deleted: {cat}{snippet} (id: {memory_id})")]
return _text_result(f"Memory deleted: {cat}{snippet} (id: {memory_id})")
elif action == "search":
query = arguments.get("text", "")
if not query:
return [TextContent(type="text", text="Error: search needs text (query)")]
memories = _memory_manager.load()
return _text_result("Error: search needs text (query)")
_owner, _all_memories, memories, scope_error = _scope_entries()
if scope_error:
return _text_result(scope_error)
if hasattr(_memory_manager, 'get_relevant_memories'):
results = _memory_manager.get_relevant_memories(query, memories, threshold=0.05, max_items=20)
else:
query_lower = query.lower()
results = [m for m in memories if query_lower in m.get("text", "").lower()][:20]
if not results:
return [TextContent(type="text", text=f"No memories found matching '{query}'.")]
return _text_result(f"No memories found matching '{query}'.")
lines = [f"Found {len(results)} matching memories:\n"]
for m in results:
cat = m.get("category", "fact")
mid = m.get("id", "?")[:8]
text = m.get("text", "")
lines.append(f"- [{cat}] `{mid}` — {text}")
return [TextContent(type="text", text="\n".join(lines))]
return _text_result("\n".join(lines))
else:
return [TextContent(type="text", text=f"Error: Unknown action '{action}'. Use: list, add, edit, delete, search")]
return _text_result(f"Error: Unknown action '{action}'. Use: list, add, edit, delete, search")
async def run():

85
package-lock.json generated
View File

@@ -1,93 +1,22 @@
{
"name": "odysseus-ui",
"name": "odysseus",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"dependencies": {
"@anthropic-ai/sdk": "^0.98.0"
},
"devDependencies": {
"@antithesishq/bombadil": "^0.3.2"
}
},
"node_modules/@anthropic-ai/sdk": {
"version": "0.98.0",
"resolved": "https://registry.npmjs.org/@anthropic-ai/sdk/-/sdk-0.98.0.tgz",
"integrity": "sha512-N7aXtCvC5g6T1Y4V29lJjceu/zTkVkIZF0jdBvagr0TRFHuKeImffalGWEfqZKrvjH+IQbzJWw6TmSmUzrlMgg==",
"license": "MIT",
"dependencies": {
"json-schema-to-ts": "^3.1.1",
"standardwebhooks": "^1.0.0"
},
"bin": {
"anthropic-ai-sdk": "bin/cli"
},
"peerDependencies": {
"zod": "^3.25.0 || ^4.0.0"
},
"peerDependenciesMeta": {
"zod": {
"optional": true
}
"@antithesishq/bombadil": "^0.6.1"
}
},
"node_modules/@antithesishq/bombadil": {
"version": "0.3.2",
"resolved": "https://registry.npmjs.org/@antithesishq/bombadil/-/bombadil-0.3.2.tgz",
"integrity": "sha512-ATy1w9ZY5gbny1H8DFc7rxZitT7DLLLFDiGcRZe+8TQiUrV5tLO+IJGOVNNLp3RpCqjZqSsxGiKoQsx31ipV1g==",
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/@antithesishq/bombadil/-/bombadil-0.6.1.tgz",
"integrity": "sha512-d1iufG3MI7gSMSiSmMeNdcMW+qR0yQXL2zdkVynC3n3DYgFJYlYXKUQzygmqU12m4RWlR5iOdQU1hsx5UT6+IA==",
"dev": true,
"license": "MIT"
},
"node_modules/@babel/runtime": {
"version": "7.29.7",
"resolved": "https://registry.npmjs.org/@babel/runtime/-/runtime-7.29.7.tgz",
"integrity": "sha512-Nq8OhGWiZIZGV6hLHoyAKLLcJihP/xFeBMGJoUrxTX2psI8dCifzLhZISFb+VWS3wFMRDmCGw5R+dOySCqPLhw==",
"license": "MIT",
"engines": {
"node": ">=6.9.0"
"bin": {
"bombadil": "bin/bombadil.js"
}
},
"node_modules/@stablelib/base64": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/@stablelib/base64/-/base64-1.0.1.tgz",
"integrity": "sha512-1bnPQqSxSuc3Ii6MhBysoWCg58j97aUjuCSZrGSmDxNqtytIi0k8utUenAwTZN4V5mXXYGsVUI9zeBqy+jBOSQ==",
"license": "MIT"
},
"node_modules/fast-sha256": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/fast-sha256/-/fast-sha256-1.3.0.tgz",
"integrity": "sha512-n11RGP/lrWEFI/bWdygLxhI+pVeo1ZYIVwvvPkW7azl/rOy+F3HYRZ2K5zeE9mmkhQppyv9sQFx0JM9UabnpPQ==",
"license": "Unlicense"
},
"node_modules/json-schema-to-ts": {
"version": "3.1.1",
"resolved": "https://registry.npmjs.org/json-schema-to-ts/-/json-schema-to-ts-3.1.1.tgz",
"integrity": "sha512-+DWg8jCJG2TEnpy7kOm/7/AxaYoaRbjVB4LFZLySZlWn8exGs3A4OLJR966cVvU26N7X9TWxl+Jsw7dzAqKT6g==",
"license": "MIT",
"dependencies": {
"@babel/runtime": "^7.18.3",
"ts-algebra": "^2.0.0"
},
"engines": {
"node": ">=16"
}
},
"node_modules/standardwebhooks": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/standardwebhooks/-/standardwebhooks-1.0.0.tgz",
"integrity": "sha512-BbHGOQK9olHPMvQNHWul6MYlrRTAOKn03rOe4A8O3CLWhNf4YHBqq2HJKKC+sfqpxiBY52pNeesD6jIiLDz8jg==",
"license": "MIT",
"dependencies": {
"@stablelib/base64": "^1.0.0",
"fast-sha256": "^1.3.0"
}
},
"node_modules/ts-algebra": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ts-algebra/-/ts-algebra-2.0.0.tgz",
"integrity": "sha512-FPAhNPFMrkwz76P7cdjdmiShwMynZYN6SgOujD1urY4oNm80Ou9oMdmbR45LotcKOXoy7wSmHkRFE6Mxbrhefw==",
"license": "MIT"
}
}
}

5
package.json
View File

@@ -4,9 +4,6 @@
"url": "https://github.com/pewdiepie-archdaemon/odysseus.git"
},
"devDependencies": {
"@antithesishq/bombadil": "^0.3.2"
},
"dependencies": {
"@anthropic-ai/sdk": "^0.98.0"
"@antithesishq/bombadil": "^0.6.1"
}
}

19
pyproject.toml
View File

@@ -1,3 +1,22 @@
[tool.pytest.ini_options]
testpaths = ["tests"]
asyncio_mode = "auto"
# Test-taxonomy markers added at collection time by tests/conftest.py. The
# stable area_* markers are declared here; the dynamic sub_<filename-token>
# markers are registered before collection by pytest_configure in
# tests/conftest.py, so unknown-mark warnings still flag genuine typos outside
# the taxonomy. See tests/_taxonomy.py and tests/README.md.
markers = [
"area_security: tests covering auth, owner-scope, SSRF, XSS, confinement, redaction",
"area_routes: tests covering HTTP route / API behavior",
"area_services: tests covering service-layer behavior (llm, cookbook, email, calendar, ...)",
"area_cli: tests covering CLI / script behavior",
"area_js: JavaScript / Node-backed tests",
"area_helpers: self-tests for the shared test helpers in tests/helpers/",
"area_unit: pure parser / utility tests that do not clearly belong elsewhere",
"area_uncategorized: tests not yet matched by the taxonomy (fallback)",
# Fast-lane marker (issue #3443). Opt-in and orthogonal to the area_*/sub_*
# taxonomy. The fast lane runs `not slow`; mark a test slow only with
# duration evidence (see tests/run_focus.py --durations and tests/README.md).
"slow: opt-in marker for known-slow tests; excluded by the fast lane (not slow)",
]

4
requirements-optional.txt
View File

@@ -15,7 +15,7 @@ faster-whisper
# DuckDuckGo as a search provider option.
# Install if you want DDG in the search-provider dropdown.
# Alternatives: SearXNG, Brave, Tavily, Serper, Google PSE.
duckduckgo-search
ddgs
# PDF form-filling feature (fillable AcroForm detection, field extraction,
# value/annotation/signature stamping, page rendering for the form overlay).
@@ -33,4 +33,4 @@ PyMuPDF
# magika (onnxruntime), already a core dep via fastembed. We avoid the
# [all]/Azure/audio extras (cloud + heavy). Pinned to a release >30 days old per
# the dependency-age discussion in issue #485.
markitdown[docx,pptx,xlsx,xls]==0.1.5
markitdown[docx,pptx,xlsx,xls]==0.1.6

9
requirements.txt
View File

@@ -3,8 +3,9 @@ uvicorn
python-multipart
python-dotenv
httpx
pydantic>=2.0
pydantic-settings>=2.0
httpcore>=1.0,<2.0
pydantic>=2.13.4
pydantic-settings>=2.14.1
SQLAlchemy
pypdf
beautifulsoup4
@@ -43,3 +44,7 @@ qrcode[pil]
croniter
pytest
pytest-asyncio
# starlette.testclient prefers httpx2 since Starlette 1.2.0 and warns on every
# TestClient import when only classic httpx is present. Runtime code keeps
# using `httpx` above; this is test-client only.
httpx2

31
routes/_validators.py Normal file
View File

@@ -0,0 +1,31 @@
import re
from fastapi import HTTPException
_REMOTE_HOST_RE = re.compile(
r"^(?:[A-Za-z0-9][A-Za-z0-9._-]*@)?[A-Za-z0-9][A-Za-z0-9._-]*$"
)
_SSH_PORT_RE = re.compile(r"^\d{1,5}$")
def validate_remote_host(v: str | None) -> str | None:
if v is None or v == "":
return None
if not _REMOTE_HOST_RE.match(v):
raise HTTPException(
400,
"Invalid remote_host — must be host or user@host, no SSH option syntax",
)
return v
def validate_ssh_port(v: str | None) -> str | None:
if v is None or v == "":
return None
if not _SSH_PORT_RE.fullmatch(str(v)):
raise HTTPException(400, "Invalid ssh_port")
port = int(v)
if port < 1 or port > 65535:
raise HTTPException(400, "Invalid ssh_port")
return str(port)

10
routes/admin_wipe_routes.py
View File

@@ -31,7 +31,7 @@ from core.database import (
CalendarEvent,
CalendarCal,
)
from src.constants import DATA_DIR
from src.constants import DATA_DIR, SKILLS_DIR, SKILLS_FILE, GALLERY_DIR, GALLERY_UPLOADS_DIR
logger = logging.getLogger(__name__)
@@ -107,7 +107,7 @@ def setup_admin_wipe_routes(session_manager):
# Skills live as SKILL.md files under data/skills/. Drop
# the entire directory; the SkillsManager re-creates the
# tree on next write.
skills_dir = os.path.join(DATA_DIR, "skills")
skills_dir = SKILLS_DIR
count = 0
if os.path.isdir(skills_dir):
# Count SKILL.md files for the response — quick walk.
@@ -115,7 +115,7 @@ def setup_admin_wipe_routes(session_manager):
count += sum(1 for f in files if f == "SKILL.md")
_rmtree_quiet(skills_dir)
# Legacy fallback file
legacy = os.path.join(DATA_DIR, "skills.json")
legacy = SKILLS_FILE
if os.path.exists(legacy):
try:
os.remove(legacy)
@@ -151,8 +151,8 @@ def setup_admin_wipe_routes(session_manager):
db.query(GalleryAlbum).delete()
db.commit()
# Also drop the upload dir so disk doesn't keep orphans.
_rmtree_quiet(os.path.join(DATA_DIR, "gallery"))
_rmtree_quiet(os.path.join(DATA_DIR, "gallery_uploads"))
_rmtree_quiet(GALLERY_DIR)
_rmtree_quiet(GALLERY_UPLOADS_DIR)
return {"status": "deleted", "kind": kind, "count": count}
if kind == "calendar":

33
routes/api_token_routes.py
View File

@@ -25,10 +25,13 @@ ALLOWED_SCOPES = {
"calendar:write",
"memory:read",
"memory:write",
"cookbook:read",
"cookbook:launch",
}
TOKEN_PROFILES = {
"chat": ["chat"],
"codex_todos": ["todos:read", "todos:write"],
"codex_documents": ["documents:read", "documents:write"],
"codex_email_drafts": ["email:read", "email:draft", "documents:read", "documents:write"],
}
@@ -65,6 +68,7 @@ def _normalize_scopes(scopes: str | list[str] | None = None, profile: str | None
ensure_before("calendar:write", "calendar:read")
ensure_before("memory:write", "memory:read")
ensure_before("email:draft", "email:read")
ensure_before("cookbook:launch", "cookbook:read")
return normalized or [DEFAULT_SCOPES]
@@ -151,26 +155,39 @@ def setup_api_token_routes() -> APIRouter:
@router.patch("/tokens/{token_id}")
async def update_token(request: Request, token_id: str):
require_admin(request)
current_user = get_current_user(request)
try:
payload = await request.json()
except Exception:
payload = {}
scope_list = _normalize_scopes(payload.get("scopes"))
scopes_value = ",".join(scope_list)
if not isinstance(payload, dict):
payload = {}
with get_db_session() as db:
token = db.query(ApiToken).filter(ApiToken.id == token_id).first()
if not token:
raise HTTPException(404, "Token not found")
if current_user and token.owner != current_user:
raise HTTPException(403, "Not your token")
if isinstance(payload.get("name"), str) and payload["name"].strip():
token.name = payload["name"].strip()[:MAX_NAME_LEN]
token.scopes = scopes_value
# Only touch scopes when the caller actually sent them. A partial
# update such as a rename ({"name": ...} with no "scopes" key) must
# not silently reset the token to the default scope — that dropped
# every previously granted scope.
if "scopes" in payload:
token.scopes = ",".join(_normalize_scopes(payload.get("scopes")))
db.add(token)
current_scopes = [
s.strip()
for s in (getattr(token, "scopes", "") or DEFAULT_SCOPES).split(",")
if s.strip()
]
response = {
"id": token_id,
"name": getattr(token, "name", ""),
"owner": getattr(token, "owner", None),
"token_prefix": getattr(token, "token_prefix", ""),
"scopes": scope_list,
"scopes": current_scopes,
}
_invalidate_cache(request)
return response
@@ -178,10 +195,14 @@ def setup_api_token_routes() -> APIRouter:
@router.delete("/tokens/{token_id}")
def delete_token(request: Request, token_id: str):
require_admin(request)
current_user = get_current_user(request)
with get_db_session() as db:
deleted = db.query(ApiToken).filter(ApiToken.id == token_id).delete()
if not deleted:
token = db.query(ApiToken).filter(ApiToken.id == token_id).first()
if not token:
raise HTTPException(404, "Token not found")
if current_user and token.owner != current_user:
raise HTTPException(403, "Not your token")
db.delete(token)
_invalidate_cache(request)
return {"status": "deleted"}

5
routes/assistant_routes.py
View File

@@ -16,6 +16,7 @@ from pydantic import BaseModel
from core.database import SessionLocal, CrewMember, ScheduledTask
from src.auth_helpers import get_current_user
from core.auth import RESERVED_USERNAMES
from src.task_scheduler import compute_next_run
@@ -89,11 +90,11 @@ def setup_assistant_routes(task_scheduler) -> APIRouter:
# check-in tasks seeded. Hitting any /assistant route under one of these
# used to seed a full CrewMember + Morning/Midday/Evening tasks under that
# owner, which then double-fired alongside the real user's check-ins.
_SYNTHETIC_OWNERS = frozenset({"internal-tool", "api", "demo", "system", ""})
# RESERVED_USERNAMES covers the same set; the `not owner` guard handles "".
async def _get_or_create(owner: str) -> CrewMember:
"""Return the per-owner assistant CrewMember, creating it on demand."""
if not owner or owner in _SYNTHETIC_OWNERS:
if not owner or owner in RESERVED_USERNAMES:
raise HTTPException(status_code=400, detail=f"Cannot seed assistant for {owner!r}")
db = SessionLocal()
try:

281
routes/auth_routes.py
View File

@@ -7,7 +7,13 @@ import asyncio
import logging
import os
from core.auth import AuthManager
import json
import re
from pathlib import Path
from core.atomic_io import atomic_write_json, atomic_write_text
from core.auth import AuthManager, RESERVED_USERNAMES, SetAdminResult, TOKEN_TTL
from src.constants import DEEP_RESEARCH_DIR, MEMORY_FILE, PASSWORD_MIN_LENGTH, SKILLS_DIR
from src.rate_limiter import RateLimiter
from src.settings_scrub import scrub_settings
from src.settings import (
@@ -67,6 +73,11 @@ class DeleteUserRequest(BaseModel):
class RenameUserRequest(BaseModel):
username: str
class SetAdminRequest(BaseModel):
is_admin: bool
class SetOpenRegistrationRequest(BaseModel):
enabled: bool
@@ -91,8 +102,12 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
raise HTTPException(429, "Too many requests — try again later")
if auth_manager.is_configured:
raise HTTPException(400, "Already configured")
if len(body.password) < 8:
raise HTTPException(400, "Password must be at least 8 characters")
if len(body.password) < PASSWORD_MIN_LENGTH:
raise HTTPException(400, f"Password must be at least {PASSWORD_MIN_LENGTH} characters")
if len(body.username.strip()) < 1:
raise HTTPException(400, "Username is required")
if body.username.lower() in RESERVED_USERNAMES:
raise HTTPException(403, "Username is reserved")
ok = await asyncio.to_thread(auth_manager.setup, body.username, body.password)
if not ok:
raise HTTPException(500, "Setup failed")
@@ -107,10 +122,12 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
raise HTTPException(400, "Run setup first")
if not auth_manager.signup_enabled:
raise HTTPException(403, "Registration is disabled. Ask an admin for an account.")
if len(body.password) < 8:
raise HTTPException(400, "Password must be at least 8 characters")
if len(body.password) < PASSWORD_MIN_LENGTH:
raise HTTPException(400, f"Password must be at least {PASSWORD_MIN_LENGTH} characters")
if len(body.username.strip()) < 1:
raise HTTPException(400, "Username is required")
if body.username.lower() in RESERVED_USERNAMES:
raise HTTPException(403, "Username is reserved")
ok = await asyncio.to_thread(auth_manager.create_user, body.username, body.password, is_admin=False)
if not ok:
raise HTTPException(409, "Username already taken")
@@ -131,8 +148,8 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
return {"ok": False, "requires_totp": True, "username": username}
if not auth_manager.totp_verify(username, body.totp_code):
raise HTTPException(401, "Invalid 2FA code")
# All checks passed — create session
token = await asyncio.to_thread(auth_manager.create_session, username, body.password)
# All checks passed — create session (password already verified above)
token = await asyncio.to_thread(auth_manager.create_session_trusted, username)
if not token:
raise HTTPException(401, "Invalid credentials")
cookie_kwargs = dict(
@@ -144,7 +161,7 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
path="/",
)
if body.remember:
cookie_kwargs["max_age"] = 60 * 60 * 24 * 7 # 7 days
cookie_kwargs["max_age"] = TOKEN_TTL
response.set_cookie(**cookie_kwargs)
return {"ok": True, "username": username}
@@ -173,13 +190,18 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
pass
return result
@router.get("/policy")
async def auth_policy():
"""Return public auth policy constants for the frontend."""
return auth_manager.policy()
@router.post("/change-password")
async def change_password(body: ChangePasswordRequest, request: Request):
user = _get_current_user(request)
if not user:
raise HTTPException(401, "Not authenticated")
if len(body.new_password) < 8:
raise HTTPException(400, "Password must be at least 8 characters")
if len(body.new_password) < PASSWORD_MIN_LENGTH:
raise HTTPException(400, f"Password must be at least {PASSWORD_MIN_LENGTH} characters")
current_token = request.cookies.get(SESSION_COOKIE)
ok = await asyncio.to_thread(auth_manager.change_password, user, body.current_password, body.new_password)
if not ok:
@@ -259,8 +281,12 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
user = _get_current_user(request)
if not user or not auth_manager.is_admin(user):
raise HTTPException(403, "Admin only")
if len(body.password) < 8:
raise HTTPException(400, "Password must be at least 8 characters")
if len(body.password) < PASSWORD_MIN_LENGTH:
raise HTTPException(400, f"Password must be at least {PASSWORD_MIN_LENGTH} characters")
if len(body.username.strip()) < 1:
raise HTTPException(400, "Username is required")
if body.username.lower() in RESERVED_USERNAMES:
raise HTTPException(403, "Username is reserved")
ok = auth_manager.create_user(body.username, body.password, body.is_admin)
if not ok:
raise HTTPException(409, "Username already taken")
@@ -293,9 +319,30 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
if new_username in auth_manager.users:
raise HTTPException(409, "Username already taken")
# Gate on auth first. Every mutation below is contingent on this
# succeeding — doing it last meant a rejected rename (e.g. reserved
# username) left file-backed owner fields already rewritten with no
# way to roll them back.
ok = auth_manager.rename_user(old_username, new_username, user)
if not ok:
raise HTTPException(400, "Cannot rename user")
def _rollback_auth_rename() -> bool:
# On self-rename the admin session has already moved to the new
# username, so the rollback must authenticate as the new user.
rollback_user = new_username if user == old_username else user
try:
return bool(auth_manager.rename_user(new_username, old_username, rollback_user))
except Exception as rollback_err:
logger.error(
"Failed to roll back auth rename %s -> %s after owner migration failure: %s",
new_username, old_username, rollback_err,
)
return False
# Usernames are ownership keys for user data. Rename the common
# owner-scoped DB rows before changing auth so the account keeps
# access to its sessions, docs, email accounts, tasks, etc.
# owner-scoped DB rows so the account keeps access to its sessions,
# docs, email accounts, tasks, etc.
try:
from sqlalchemy import func
from core.database import Base, SessionLocal
@@ -318,6 +365,11 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
db.close()
except Exception as e:
logger.error("Failed to rename owner references %s -> %s: %s", old_username, new_username, e)
if not _rollback_auth_rename():
logger.error(
"Auth rename %s -> %s could not be rolled back after owner migration failure",
old_username, new_username,
)
raise HTTPException(500, "Failed to rename user data")
# Per-user prefs are JSON-backed, not SQL-backed.
@@ -337,9 +389,133 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
except Exception as e:
logger.warning("Failed to rename user prefs %s -> %s: %s", old_username, new_username, e)
ok = auth_manager.rename_user(old_username, new_username, user)
if not ok:
raise HTTPException(400, "Cannot rename user")
# In-flight deep-research tasks live in the process-local
# ResearchHandler registry. They are not covered by the persisted JSON
# migration above, but the research routes filter and cancel by this
# owner field while the job is running. Do this before sweeping
# completed JSON files so a job that finishes during the rename saves
# with the new owner or is caught by the disk sweep below.
try:
rh = getattr(request.app.state, "research_handler", None)
rename_owner = getattr(rh, "rename_owner", None)
if callable(rename_owner):
rename_owner(old_username, new_username)
except Exception as e:
logger.warning("Failed to rename active research tasks %s -> %s: %s", old_username, new_username, e)
# deep_research: each completed report is a standalone JSON file with
# an `owner` field. research_routes filters by d.get("owner") == user,
# so a stale owner makes every report invisible to the renamed user.
try:
dr_dir = Path(DEEP_RESEARCH_DIR)
if dr_dir.is_dir():
for p in dr_dir.glob("*.json"):
try:
d = json.loads(p.read_text(encoding="utf-8"))
if str(d.get("owner", "")).strip().lower() == old_username:
d["owner"] = new_username
atomic_write_json(str(p), d)
except Exception as err:
logger.warning("Failed to update research owner in %s: %s", p.name, err)
except Exception as e:
logger.warning("Failed to rename research owner references %s -> %s: %s", old_username, new_username, e)
# memory.json: a flat JSON array where each entry carries an `owner`
# field. memory_manager.load(owner=user) filters on it, so stale
# entries disappear from the memory panel.
try:
if os.path.isfile(MEMORY_FILE):
with open(MEMORY_FILE, encoding="utf-8") as fh:
entries = json.loads(fh.read())
if isinstance(entries, list):
changed = False
for entry in entries:
if isinstance(entry, dict) and str(entry.get("owner", "")).strip().lower() == old_username:
entry["owner"] = new_username
changed = True
if changed:
atomic_write_json(MEMORY_FILE, entries)
except Exception as e:
logger.warning("Failed to rename memory.json owner references %s -> %s: %s", old_username, new_username, e)
# uploads.json: upload rows use owner metadata for access checks and
# owner-prefixed index keys for dedupe. Rename both so attachments keep
# resolving after the account username changes.
try:
upload_handler = getattr(request.app.state, "upload_handler", None)
rename_owner = getattr(upload_handler, "rename_owner", None)
if callable(rename_owner):
rename_owner(old_username, new_username)
except Exception as e:
logger.warning("Failed to rename upload owner references %s -> %s: %s", old_username, new_username, e)
# direct personal RAG uploads live in per-owner directories and the
# vector metadata also carries the username used for owner-filtered
# search. Keep both in sync with the auth rename.
try:
from routes.personal_routes import rename_personal_upload_owner
personal_docs_manager = getattr(request.app.state, "personal_docs_manager", None)
if personal_docs_manager is not None:
rag_manager = getattr(personal_docs_manager, "rag_manager", None)
rename_personal_upload_owner(
old_username,
new_username,
personal_docs_manager=personal_docs_manager,
rag_manager=rag_manager,
)
except Exception as e:
logger.warning("Failed to rename personal RAG upload owner references %s -> %s: %s", old_username, new_username, e)
# skills: SKILL.md frontmatter carries owner: <username>; the usage
# sidecar (_usage.json) keys entries as owner::skill-name. Both must
# be updated or the renamed user's Skills panel goes empty.
try:
skills_root = Path(SKILLS_DIR)
if skills_root.is_dir():
_owner_re = re.compile(
r'(?m)^(owner:\s*)' + re.escape(old_username) + r'\s*$',
re.IGNORECASE,
)
for p in skills_root.rglob("SKILL.md"):
try:
text = p.read_text(encoding="utf-8")
new_text = _owner_re.sub(r'\g<1>' + new_username, text)
if new_text != text:
atomic_write_text(str(p), new_text)
except Exception as err:
logger.warning("Failed to update skill owner in %s: %s", p, err)
usage_path = skills_root / "_usage.json"
if usage_path.is_file():
try:
usage = json.loads(usage_path.read_text(encoding="utf-8"))
if isinstance(usage, dict):
new_usage = {}
changed = False
for k, v in usage.items():
owner_part, sep, skill_part = k.partition("::")
if sep and owner_part.lower() == old_username:
new_usage[new_username + "::" + skill_part] = v
changed = True
else:
new_usage[k] = v
if changed:
atomic_write_json(str(usage_path), new_usage)
except Exception as err:
logger.warning("Failed to update skills usage keys %s -> %s: %s", old_username, new_username, err)
except Exception as e:
logger.warning("Failed to rename skills owner references %s -> %s: %s", old_username, new_username, e)
# The in-memory session cache (session_manager.sessions) stores each
# session's owner at load time. Without this patch the renamed user's
# sessions are invisible on the next /api/sessions call because
# get_sessions_for_user does an exact `s.owner == username` comparison
# against stale in-memory values.
sm = getattr(request.app.state, "session_manager", None)
if sm is not None:
for sess in list(getattr(sm, "sessions", {}).values()):
if str(getattr(sess, "owner", None) or "").strip().lower() == old_username:
sess.owner = new_username
# The owner-rename loop above updated ApiToken.owner in the DB, but the
# bearer-token cache still maps each token to the OLD owner. Without
# refreshing it, the renamed user's API tokens resolve to the old (now
@@ -350,6 +526,31 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
invalidator()
return {"ok": True, "username": new_username, "renamed_self": old_username == user}
@router.put("/users/{username}/admin")
async def set_user_admin(username: str, body: SetAdminRequest, request: Request):
"""Promote/demote a user to/from admin. Admin only.
The last remaining admin can't be demoted (no lockout). Self-demotion
is allowed while another admin exists; the `self` flag tells the UI to
reload the acting user into the normal-user view.
"""
user = _get_current_user(request)
if not user or not auth_manager.is_admin(user):
raise HTTPException(403, "Admin only")
result = auth_manager.set_admin(username, body.is_admin, user)
if result is SetAdminResult.USER_NOT_FOUND:
raise HTTPException(404, "User not found")
if result is SetAdminResult.NOT_AUTHORIZED:
raise HTTPException(403, "Admin only")
if result is SetAdminResult.LAST_ADMIN:
raise HTTPException(400, "Cannot demote the last admin")
target = (username or "").strip().lower()
return {
"ok": True,
"is_admin": body.is_admin,
"self": target == (user or "").strip().lower(),
}
@router.post("/signup-toggle", deprecated=True)
async def toggle_signup(request: Request):
"""
@@ -380,7 +581,23 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
user = _get_current_user(request)
if not user or not auth_manager.is_admin(user):
raise HTTPException(403, "Admin only")
ok = auth_manager.delete_user(body.username, user)
def _invalidate_api_token_cache():
try:
invalidator = getattr(request.app.state, "invalidate_token_cache", None)
if invalidator:
invalidator()
except Exception:
pass
try:
ok = auth_manager.delete_user(body.username, user)
except Exception:
# delete_user can touch ApiToken rows before a later auth-store write
# fails. Dirty the bearer cache anyway so a partial token purge does
# not leave already-cached tokens authenticating until restart.
_invalidate_api_token_cache()
raise
if not ok:
raise HTTPException(400, "Cannot delete user")
# delete_user removes the user's ApiToken rows, but the bearer-auth
@@ -388,12 +605,7 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
# rebuilds when flagged dirty. Without this, a deleted user's already
# cached token keeps authenticating until some other token op or a
# restart clears the cache. Mirror what the token routes do.
try:
invalidator = getattr(request.app.state, "invalidate_token_cache", None)
if invalidator:
invalidator()
except Exception:
pass
_invalidate_api_token_cache()
return {"ok": True}
# ---- Feature visibility (admin-managed) ----
@@ -585,6 +797,27 @@ def setup_auth_routes(auth_manager: AuthManager) -> APIRouter:
hint = " If this is Docker Compose ntfy, set NTFY_BIND to that host/Tailscale IP and NTFY_BASE_URL to the same server URL in .env, then recreate ntfy."
return {"ok": False, "message": f"ntfy publish to {full_url} failed: {e}.{hint}"[:500]}
if preset == "discord_webhook":
import httpx
webhook_url = (integ.get("base_url") or "").strip()
if not webhook_url:
return {"ok": False, "message": "No webhook URL set — paste the full Discord webhook URL into the Base URL field."}
payload = {
"embeds": [{
"title": "Odysseus connectivity test",
"description": "If you see this, your Discord Webhook integration is wired up correctly.",
"color": 5793266,
}]
}
try:
async with httpx.AsyncClient(timeout=8.0) as client:
r = await client.post(webhook_url, json=payload)
if r.is_success:
return {"ok": True, "message": "Test embed sent — check your Discord channel to confirm it arrived."}
return {"ok": False, "message": f"Discord returned HTTP {r.status_code}: {r.text[:200]}"}
except Exception as e:
return {"ok": False, "message": f"Request failed: {e}"[:400]}
# All other presets: GET against a known health endpoint.
# Fall back to detecting from name if preset is missing.
health_paths = {

74
routes/backup_routes.py
View File

@@ -101,24 +101,74 @@ def setup_backup_routes(memory_manager, preset_manager, skills_manager) -> APIRo
# ── Skills ──
if "skills" in body and isinstance(body["skills"], list):
existing = skills_manager.load_all()
existing_ids = {s.get("id") for s in existing}
existing_titles = {s.get("title", "").strip().lower() for s in existing}
# Dedup against THIS user's own skills only. Using every tenant's
# rows (load_all) meant a skill whose id/name/title matched any
# other user's was silently skipped, so the importing user lost
# their own data — same cross-tenant bug fixed for memories above.
# The full store is still saved back below.
own = [s for s in existing if s.get("owner") == user]
existing_names = {s.get("name") for s in own if s.get("name")}
existing_ids = {s.get("id") for s in own if s.get("id")}
existing_titles = {
(s.get("title") or s.get("description") or "").strip().lower()
for s in own
}
added = 0
for skill in body["skills"]:
if not isinstance(skill, dict) or not skill.get("title"):
if not isinstance(skill, dict):
continue
# Skip if same id or same title already exists
if skill.get("id") in existing_ids:
title = (
skill.get("title") or skill.get("description")
or skill.get("name") or ""
).strip()
if not title:
continue
if skill["title"].strip().lower() in existing_titles:
sid = skill.get("id") or skill.get("name")
if sid and sid in existing_ids:
continue
if user and not skill.get("owner"):
skill["owner"] = user
existing.append(skill)
existing_ids.add(skill.get("id"))
existing_titles.add(skill["title"].strip().lower())
nm = skill.get("name")
if nm and nm in existing_names:
continue
if title.lower() in existing_titles:
continue
owner = skill.get("owner")
if user and not owner:
owner = user
# Skills live on disk as SKILL.md files; the old JSON-era
# skills_manager.save() no longer exists. Write each new skill
# via add_skill (source="user" skips auto-dedup — this is an
# explicit backup restore).
result = skills_manager.add_skill(
title=title,
name=skill.get("name"),
description=skill.get("description"),
problem=skill.get("problem", ""),
solution=skill.get("solution", ""),
steps=skill.get("steps"),
tags=skill.get("tags"),
source="user",
teacher_model=skill.get("teacher_model"),
confidence=skill.get("confidence", 0.8),
owner=owner,
category=skill.get("category", "general"),
when_to_use=skill.get("when_to_use"),
procedure=skill.get("procedure"),
pitfalls=skill.get("pitfalls"),
verification=skill.get("verification"),
platforms=skill.get("platforms"),
requires_toolsets=skill.get("requires_toolsets"),
fallback_for_toolsets=skill.get("fallback_for_toolsets"),
status=skill.get("status", "draft"),
version=skill.get("version", "1.0.0"),
)
if result.get("_deduped"):
continue
if result.get("name"):
existing_names.add(result["name"])
if result.get("id"):
existing_ids.add(result["id"])
existing_titles.add(title.lower())
added += 1
skills_manager.save(existing)
imported.append(f"{added} skills")
# ── Presets ──

500
routes/calendar_routes.py
View File

@@ -1,6 +1,7 @@
"""Calendar routes — local SQLite-backed calendar CRUD."""
import logging
import json
import re
import uuid
from datetime import datetime, date, timedelta
@@ -11,9 +12,9 @@ from pydantic import BaseModel
from sqlalchemy import or_, and_
from dateutil.rrule import rrulestr
from core.database import SessionLocal, CalendarCal, CalendarEvent
from core.database import SessionLocal, CalendarCal, CalendarDeletedEvent, CalendarEvent
from src.auth_helpers import require_user
from src.upload_limits import read_upload_limited
from src.upload_limits import read_upload_limited, ICS_MAX_BYTES
logger = logging.getLogger(__name__)
@@ -34,6 +35,24 @@ def _ics_naive_dtstart(dt):
return datetime(dt.year, dt.month, dt.day)
return dt
def _ensure_positive_duration(start_dt, end_dt, all_day):
"""Clamp an imported event's end so it has a positive duration.
Some .ics exporters write a single-day all-day event with DTEND equal to
DTSTART (treating DTEND as inclusive rather than the RFC 5545 exclusive
bound). Stored verbatim that produces a zero-duration row, which the
list_events overlap filter (dtstart < end AND dtend > start) silently
drops — the event never appears on the calendar even though the web UI
would otherwise show it. Normalize a non-positive end to the same default
span used when DTEND is absent: one day for all-day events, one hour
otherwise.
"""
if end_dt <= start_dt:
return start_dt + (timedelta(days=1) if all_day else timedelta(hours=1))
return end_dt
# Single-user fallback identity. Used only when:
# 1. The app is configured for single-user (no auth middleware), AND
# 2. The request didn't resolve to an authenticated user.
@@ -126,6 +145,54 @@ def _resolve_base_uid(uid: str) -> str:
raise ValueError("malformed compound UID: missing base before ::")
return base
async def _push_caldav_event_after_commit(owner: str, uid: str, action: str):
"""Best-effort CalDAV write-through. Local writes stay authoritative if
the remote server is unreachable; pending flags let /sync retry later."""
try:
result = {"ok": True}
if action == "create":
from src.caldav_sync import push_event_create
result = await push_event_create(owner, uid)
elif action == "update":
from src.caldav_sync import push_event_update
result = await push_event_update(owner, uid)
elif action == "delete":
from src.caldav_sync import push_event_delete
result = await push_event_delete(owner, uid)
if result and not result.get("ok") and not result.get("skipped"):
raise RuntimeError(result.get("error") or result)
except Exception as e:
logger.warning("CalDAV %s push failed for uid=%s: %s", action, uid, e)
if action in {"create", "update"}:
db = SessionLocal()
try:
ev = _get_or_404_event(db, uid, owner)
ev.caldav_sync_pending = action
db.commit()
except Exception:
db.rollback()
finally:
db.close()
def _record_caldav_delete_tombstone(db, ev: CalendarEvent, owner: str) -> None:
if not (ev.calendar and ev.calendar.source == "caldav"):
return
tombstone = db.query(CalendarDeletedEvent).filter(
CalendarDeletedEvent.uid == ev.uid,
CalendarDeletedEvent.owner == owner,
).first()
if not tombstone:
tombstone = CalendarDeletedEvent(uid=ev.uid, owner=owner)
db.add(tombstone)
tombstone.calendar_id = ev.calendar_id
tombstone.remote_href = ev.remote_href
tombstone.remote_etag = ev.remote_etag
tombstone.caldav_base_url = getattr(ev.calendar, "caldav_base_url", None)
tombstone.summary = ev.summary or ""
tombstone.last_error = None
# ── Pydantic models ──
class EventCreate(BaseModel):
@@ -258,6 +325,17 @@ def parse_due_for_user(s: str) -> str:
if t is not None:
return base.replace(hour=t[0], minute=t[1]).isoformat()
# Time-first: "3pm today", "11pm today", "9am tomorrow"
m = _re.match(r'^(.+?)\s+(today|tonight|tomorrow|tmrw|yesterday)$', lower)
if m:
time_part, word = m.group(1).strip(), m.group(2)
base = today
if word in ("tomorrow", "tmrw"): base = today + _td(days=1)
elif word == "yesterday": base = today - _td(days=1)
t = _parse_time(time_part)
if t is not None:
return base.replace(hour=t[0], minute=t[1]).isoformat()
m = _re.match(r'^in\s+(\d+)\s*(hour|hr|minute|min|day)s?\s*$', lower)
if m:
n = int(m.group(1)); unit = m.group(2)
@@ -375,6 +453,20 @@ def _parse_dt(s: str) -> datetime:
if t is not None:
return base.replace(hour=t[0], minute=t[1])
# time-first: "3pm today", "9am tomorrow", "11pm tonight"
# (parity with parse_due_for_user, which handles these via the same form)
m = _re.match(r'^(.+?)\s+(today|tonight|tomorrow|tmrw|yesterday)$', lower)
if m:
time_part, word = m.group(1).strip(), m.group(2)
base = today
if word in ("tomorrow", "tmrw"):
base = today + timedelta(days=1)
elif word == "yesterday":
base = today - timedelta(days=1)
t = _parse_time(time_part)
if t is not None:
return base.replace(hour=t[0], minute=t[1])
# next <weekday> [at] TIME
weekdays = ["monday", "tuesday", "wednesday", "thursday", "friday", "saturday", "sunday"]
m = _re.match(r'^next\s+(\w+)(?:\s+at)?\s*(.*)$', lower)
@@ -450,6 +542,7 @@ def _event_to_dict(ev: CalendarEvent) -> dict:
"description": ev.description or "",
"location": ev.location or "",
"rrule": ev.rrule or "",
"recurrence_exdates": _recurrence_exdates(ev),
"calendar": ev.calendar.name if ev.calendar else "",
"calendar_href": ev.calendar_id,
"color": ev.color or (ev.calendar.color if ev.calendar else ""),
@@ -460,6 +553,31 @@ def _event_to_dict(ev: CalendarEvent) -> dict:
# ── Recurrence expansion ──
_RRULE_EXPANSION_LIMIT = 1000
def _recurrence_exdates(ev: CalendarEvent) -> list[str]:
raw = getattr(ev, "recurrence_exdates", "") or ""
if not raw:
return []
try:
values = json.loads(raw)
except Exception:
return []
if not isinstance(values, list):
return []
return [str(v) for v in values if isinstance(v, str) and v.strip()]
def _occurrence_exdate_key(uid: str, ev: CalendarEvent) -> str:
if "::" not in uid:
return ""
suffix = uid.split("::", 1)[1]
if ev.all_day:
return suffix[:10]
return suffix[:16]
def _expand_rrule(
ev: CalendarEvent, start: datetime, end: datetime
) -> List[dict]:
@@ -482,6 +600,7 @@ def _expand_rrule(
d = _event_to_dict(ev)
d["is_recurrence"] = False
d["series_uid"] = ev.uid
d["truncated"] = False
return [d]
# Parse the rrule, applying it to the base dtstart.
@@ -507,6 +626,7 @@ def _expand_rrule(
d = _event_to_dict(ev)
d["is_recurrence"] = False
d["series_uid"] = ev.uid
d["truncated"] = False
# Malformed RRULE rows are fetched by the recurring SQL branch
# with only dtstart < end_dt — the base event may not actually
# overlap the window. Only return if it does.
@@ -519,32 +639,43 @@ def _expand_rrule(
# (matching non-recurring overlap semantics: dtstart < end AND
# dtend > start).
expand_start = start - duration
occurrences = rule.between(expand_start, end, inc=True)
if not occurrences:
return []
results = []
truncated = False
base = _event_to_dict(ev)
exdates = set(_recurrence_exdates(ev))
for occ_start in rule.xafter(expand_start, inc=True):
if occ_start >= end:
break
for occ_start in occurrences:
occ_end = occ_start + duration
# Overlap filter: occurrence must intersect [start, end).
# This enforces exclusive-end semantics (occ_start >= end is
# excluded) and includes multi-day crossings (occ_end > start).
if occ_start >= end or occ_end <= start:
if occ_end <= start:
continue
if len(results) >= _RRULE_EXPANSION_LIMIT:
truncated = True
break
# Build the compound uid: {base_uid}::{date} or ::{datetime}
if ev.all_day:
occ_uid = f"{ev.uid}::{occ_start.strftime('%Y-%m-%d')}"
exdate_key = occ_start.strftime("%Y-%m-%d")
else:
occ_uid = f"{ev.uid}::{occ_start.strftime('%Y-%m-%dT%H:%M')}"
exdate_key = occ_start.strftime("%Y-%m-%dT%H:%M")
if exdate_key in exdates:
continue
d = dict(base)
d["uid"] = occ_uid
d["series_uid"] = ev.uid
d["is_recurrence"] = True
d["truncated"] = False
if ev.all_day:
d["dtstart"] = occ_start.strftime("%Y-%m-%d")
@@ -557,6 +688,10 @@ def _expand_rrule(
results.append(d)
if truncated:
for d in results:
d["truncated"] = True
return results
@@ -565,72 +700,178 @@ def _expand_rrule(
def setup_calendar_routes() -> APIRouter:
router = APIRouter(prefix="/api/calendar", tags=["calendar"])
# CalDAV connect form (Integrations → Calendar). Storage is local
# SQLite; sync (src/caldav_sync.py) pulls remote events into it on
# calendar open and periodically via the scheduler.
# ── CalDAV multi-account helpers ─────────────────────────────────────────
def _get_caldav_accounts(owner: str) -> list:
from src.caldav_sync import _load_caldav_accounts
return _load_caldav_accounts(owner)
def _save_caldav_accounts(owner: str, accounts: list) -> None:
from routes.prefs_routes import _load_for_user, _save_for_user
prefs = _load_for_user(owner) or {}
prefs["caldav_accounts"] = accounts
prefs.pop("caldav", None)
_save_for_user(owner, prefs)
# ── CalDAV config routes (backward-compat single-account API) ────────────
@router.get("/config")
async def get_config(request: Request):
"""Legacy single-account endpoint — returns the first configured account."""
owner = _require_user(request)
from routes.prefs_routes import _load_for_user
cfg = (_load_for_user(owner) or {}).get("caldav", {}) or {}
caldav_password = cfg.get("password") or ""
if caldav_password:
accounts = _get_caldav_accounts(owner)
if not accounts:
return {"url": "", "username": "", "password": "", "has_password": False, "local": True}
first = accounts[0]
pw = first.get("password") or ""
has_pw = False
if pw:
try:
from src.secret_storage import decrypt
caldav_password = decrypt(caldav_password)
has_pw = bool(decrypt(pw))
except Exception:
pass
# Surface url+username but never hand the password back to the
# client — saved-state UI shouldn't leak the credential.
has_pw = bool(pw)
return {
"url": cfg.get("url", "") or "",
"username": cfg.get("username", "") or "",
"url": first.get("url", "") or "",
"username": first.get("username", "") or "",
"password": "",
"has_password": bool(caldav_password),
"local": not bool(cfg.get("url")),
"has_password": has_pw,
"local": not bool(first.get("url")),
}
@router.post("/config")
async def save_config(request: Request):
"""Legacy single-account endpoint — upserts the first account."""
owner = _require_user(request)
from routes.prefs_routes import _load_for_user, _save_for_user
try:
body = await request.json()
except Exception:
body = {}
prefs = _load_for_user(owner) or {}
cfg = dict(prefs.get("caldav") or {})
# Empty url => clear the whole entry (treat as "remove integration").
accounts = _get_caldav_accounts(owner)
if not (body.get("url") or "").strip():
prefs.pop("caldav", None)
_save_for_user(owner, prefs)
_save_caldav_accounts(owner, [])
return {"ok": True, "cleared": True}
from src.caldav_sync import validate_caldav_url
try:
cfg["url"] = validate_caldav_url(body.get("url", ""))
validated_url = validate_caldav_url(body.get("url", ""))
except ValueError as e:
raise HTTPException(400, str(e))
cfg["username"] = (body.get("username") or "").strip()
# Preserve the stored password when the client sends an empty
# one (edit form re-submitted without re-typing the password).
# cfg already holds the existing (already-encrypted) password from
# prefs, so we only touch it when a new password is supplied —
# re-encrypting the stored value would double-encrypt it.
if accounts:
acc = dict(accounts[0])
else:
import uuid as _uuid
acc = {"id": str(_uuid.uuid4()), "label": "CalDAV"}
acc["url"] = validated_url
acc["username"] = (body.get("username") or "").strip()
if body.get("password"):
from src.secret_storage import encrypt
cfg["password"] = encrypt(body["password"])
prefs["caldav"] = cfg
_save_for_user(owner, prefs)
acc["password"] = encrypt(body["password"])
new_accounts = [acc] + (accounts[1:] if len(accounts) > 1 else [])
_save_caldav_accounts(owner, new_accounts)
return {"ok": True}
# ── CalDAV multi-account CRUD ─────────────────────────────────────────────
@router.get("/config/accounts")
async def list_caldav_accounts(request: Request):
"""Return all configured CalDAV accounts (passwords never returned)."""
owner = _require_user(request)
accounts = _get_caldav_accounts(owner)
safe = []
for acc in accounts:
pw = acc.get("password") or ""
has_pw = False
if pw:
try:
from src.secret_storage import decrypt
has_pw = bool(decrypt(pw))
except Exception:
has_pw = bool(pw)
safe.append({
"id": acc.get("id", ""),
"label": acc.get("label", "") or acc.get("url", ""),
"url": acc.get("url", "") or "",
"username": acc.get("username", "") or "",
"has_password": has_pw,
})
return {"accounts": safe}
@router.post("/config/accounts")
async def add_caldav_account(request: Request):
"""Add a new CalDAV account."""
import uuid as _uuid
owner = _require_user(request)
try:
body = await request.json()
except Exception:
body = {}
from src.caldav_sync import validate_caldav_url
try:
url = validate_caldav_url(body.get("url", ""))
except ValueError as e:
raise HTTPException(400, str(e))
if not body.get("password"):
raise HTTPException(400, "Password is required")
from src.secret_storage import encrypt
new_acc = {
"id": str(_uuid.uuid4()),
"label": (body.get("label") or "").strip() or "CalDAV",
"url": url,
"username": (body.get("username") or "").strip(),
"password": encrypt(body["password"]),
}
accounts = _get_caldav_accounts(owner)
accounts.append(new_acc)
_save_caldav_accounts(owner, accounts)
return {"ok": True, "id": new_acc["id"]}
@router.put("/config/accounts/{account_id}")
async def update_caldav_account(account_id: str, request: Request):
"""Update an existing CalDAV account by id."""
owner = _require_user(request)
try:
body = await request.json()
except Exception:
body = {}
accounts = _get_caldav_accounts(owner)
idx = next((i for i, a in enumerate(accounts) if a.get("id") == account_id), None)
if idx is None:
raise HTTPException(404, "Account not found")
acc = dict(accounts[idx])
if body.get("url"):
from src.caldav_sync import validate_caldav_url
try:
acc["url"] = validate_caldav_url(body["url"])
except ValueError as e:
raise HTTPException(400, str(e))
if body.get("label") is not None:
acc["label"] = (body.get("label") or "").strip() or "CalDAV"
if body.get("username") is not None:
acc["username"] = (body.get("username") or "").strip()
if body.get("password"):
from src.secret_storage import encrypt
acc["password"] = encrypt(body["password"])
accounts[idx] = acc
_save_caldav_accounts(owner, accounts)
return {"ok": True}
@router.delete("/config/accounts/{account_id}")
async def delete_caldav_account(account_id: str, request: Request):
"""Remove a CalDAV account by id."""
owner = _require_user(request)
accounts = _get_caldav_accounts(owner)
new_accounts = [a for a in accounts if a.get("id") != account_id]
if len(new_accounts) == len(accounts):
raise HTTPException(404, "Account not found")
_save_caldav_accounts(owner, new_accounts)
return {"ok": True}
@router.post("/test")
async def test_connection(request: Request):
"""Actually probe the configured CalDAV server with a PROPFIND
request (the same handshake every CalDAV client uses). Accepts
an optional {url, username, password} body so the user can test
a configuration BEFORE saving it; falls back to the stored
creds otherwise. Returns {ok, error?} with a useful message on
failure (status code, auth issue, network error)."""
"""Probe a CalDAV server with a PROPFIND. Accepts an optional body:
{url, username, password} to test before saving, or {account_id} to
test an already-saved account. Falls back to the first saved account
when nothing is provided."""
owner = _require_user(request)
try:
body = await request.json()
@@ -640,19 +881,24 @@ def setup_calendar_routes() -> APIRouter:
user = (body.get("username") or "").strip()
pw = body.get("password") or ""
if not (url and user and pw):
# Fall back to saved settings for this user.
from routes.prefs_routes import _load_for_user
cfg = (_load_for_user(owner) or {}).get("caldav", {}) or {}
url = url or (cfg.get("url") or "")
user = user or (cfg.get("username") or "")
if not pw:
pw = cfg.get("password") or ""
if pw:
try:
from src.secret_storage import decrypt
pw = decrypt(pw)
except Exception:
pass
# Look up a saved account: by id if supplied, else first account.
accounts = _get_caldav_accounts(owner)
acc = None
if body.get("account_id"):
acc = next((a for a in accounts if a.get("id") == body["account_id"]), None)
if acc is None and accounts:
acc = accounts[0]
if acc:
url = url or (acc.get("url") or "")
user = user or (acc.get("username") or "")
if not pw:
pw = acc.get("password") or ""
if pw:
try:
from src.secret_storage import decrypt
pw = decrypt(pw)
except Exception:
pass
if not (url and user and pw):
return {"ok": False, "error": "Missing URL, username, or password"}
from src.caldav_sync import validate_caldav_url
@@ -707,13 +953,34 @@ def setup_calendar_routes() -> APIRouter:
return {"ok": False, "error": str(e)[:200]}
@router.post("/sync")
async def sync_caldav_endpoint(request: Request):
"""Pull events from the configured CalDAV server into local DB.
async def sync_caldav_endpoint(request: Request, direction: str = "pull"):
"""Sync events with the configured CalDAV server.
Returns counts + any per-calendar errors. Called by the frontend
on calendar open and by the periodic scheduler loop."""
owner = _require_user(request)
from src.caldav_sync import sync_caldav
return await sync_caldav(owner)
from src.caldav_sync import sync_caldav_direction
return await sync_caldav_direction(owner, direction)
@router.delete("/calendars/{cal_id}")
async def delete_calendar(request: Request, cal_id: str):
owner = _require_user(request)
db = SessionLocal()
try:
cal = _get_or_404_calendar(db, cal_id, owner)
db.query(CalendarEvent).filter(CalendarEvent.calendar_id == cal_id).delete()
db.delete(cal)
db.commit()
return {"ok": True}
except HTTPException:
raise
except Exception as e:
db.rollback()
logger.error("Failed to delete calendar %s: %s", cal_id, e)
raise HTTPException(500, "Failed to delete calendar")
finally:
db.close()
@router.get("/calendars")
async def list_calendars(request: Request):
@@ -723,7 +990,7 @@ def setup_calendar_routes() -> APIRouter:
_ensure_default_calendar(db, owner)
cals = db.query(CalendarCal).filter(CalendarCal.owner == owner).all()
return {"calendars": [
{"name": c.name, "href": c.id, "color": c.color}
{"name": c.name, "href": c.id, "color": c.color, "source": c.source}
for c in cals
]}
except HTTPException:
@@ -786,8 +1053,12 @@ def setup_calendar_routes() -> APIRouter:
expanded.extend(_expand_rrule(e, start_dt, end_dt))
# Sort by occurrence start time for consistent frontend ordering.
truncated = any(e.get("truncated") for e in expanded)
expanded.sort(key=lambda d: d["dtstart"])
return {"events": expanded}
response: dict = {"events": expanded}
if truncated:
response["truncated"] = True
return response
except HTTPException:
raise
except Exception as e:
@@ -841,19 +1112,12 @@ def setup_calendar_routes() -> APIRouter:
is_utc=_is_utc and not data.all_day,
rrule=data.rrule or "",
color=data.color or None,
caldav_sync_pending="create" if cal.source == "caldav" else None,
)
db.add(ev)
db.commit()
if cal.source == "caldav":
# Push the new event to the remote so it appears on the user's
# other devices — the sync is otherwise pull-only (#800).
from src.caldav_writeback import writeback_event
await writeback_event(owner, cal.source, cal.id, {
"uid": uid, "summary": data.summary, "description": data.description,
"location": data.location, "dtstart": dtstart, "dtend": dtend,
"all_day": data.all_day, "is_utc": _is_utc and not data.all_day,
"rrule": data.rrule or "",
})
await _push_caldav_event_after_commit(owner, uid, "create")
return {"ok": True, "uid": uid}
except HTTPException:
raise
@@ -899,15 +1163,12 @@ def setup_calendar_routes() -> APIRouter:
ev.rrule = data.rrule
if data.color is not None:
ev.color = data.color if data.color else None
is_caldav = ev.calendar and ev.calendar.source == "caldav"
if is_caldav:
ev.caldav_sync_pending = "update"
db.commit()
cal = db.query(CalendarCal).filter(CalendarCal.id == ev.calendar_id).first()
if cal and cal.source == "caldav":
from src.caldav_writeback import writeback_event
await writeback_event(owner, cal.source, cal.id, {
"uid": ev.uid, "summary": ev.summary, "description": ev.description,
"location": ev.location, "dtstart": ev.dtstart, "dtend": ev.dtend,
"all_day": ev.all_day, "is_utc": ev.is_utc, "rrule": ev.rrule or "",
})
if is_caldav:
await _push_caldav_event_after_commit(owner, base_uid, "update")
return {"ok": True}
except HTTPException:
raise
@@ -919,7 +1180,7 @@ def setup_calendar_routes() -> APIRouter:
db.close()
@router.delete("/events/{uid}")
async def delete_event(request: Request, uid: str):
async def delete_event(request: Request, uid: str, scope: str = "series"):
owner = _require_user(request)
try:
base_uid = _resolve_base_uid(uid)
@@ -928,15 +1189,28 @@ def setup_calendar_routes() -> APIRouter:
db = SessionLocal()
try:
ev = _get_or_404_event(db, base_uid, owner)
# Capture what the remote push needs BEFORE the row is gone.
_cal = db.query(CalendarCal).filter(CalendarCal.id == ev.calendar_id).first()
_is_caldav = bool(_cal and _cal.source == "caldav")
_cal_id, _ev_uid = ev.calendar_id, ev.uid
is_occurrence_delete = scope in {"occurrence", "instance"} and "::" in uid and bool(ev.rrule)
is_caldav = ev.calendar and ev.calendar.source == "caldav"
if is_occurrence_delete:
key = _occurrence_exdate_key(uid, ev)
if not key:
raise HTTPException(400, "Invalid recurring occurrence uid")
exdates = _recurrence_exdates(ev)
if key not in exdates:
exdates.append(key)
ev.recurrence_exdates = json.dumps(sorted(exdates))
if is_caldav:
ev.caldav_sync_pending = "update"
db.commit()
if is_caldav:
await _push_caldav_event_after_commit(owner, base_uid, "update")
return {"ok": True, "scope": "occurrence", "exdate": key}
if is_caldav:
_record_caldav_delete_tombstone(db, ev, owner)
db.delete(ev)
db.commit()
if _is_caldav:
from src.caldav_writeback import writeback_event
await writeback_event(owner, "caldav", _cal_id, {"uid": _ev_uid}, delete=True)
if is_caldav:
await _push_caldav_event_after_commit(owner, base_uid, "delete")
return {"ok": True}
except HTTPException:
raise
@@ -990,27 +1264,10 @@ def setup_calendar_routes() -> APIRouter:
finally:
db.close()
@router.delete("/calendars/{cal_id}")
async def delete_calendar(request: Request, cal_id: str):
owner = _require_user(request)
db = SessionLocal()
try:
cal = _get_or_404_calendar(db, cal_id, owner)
db.query(CalendarEvent).filter(CalendarEvent.calendar_id == cal_id).delete()
db.delete(cal)
db.commit()
return {"ok": True}
except HTTPException:
raise
except Exception as e:
db.rollback()
return {"error": str(e)}
finally:
db.close()
# 10 MB hard cap on ICS upload. Loading the whole file into memory is
# unavoidable with python-icalendar, so an unbounded upload would OOM.
_ICS_MAX_BYTES = 10 * 1024 * 1024
# Hard cap on ICS upload (ICS_MAX_BYTES, default 10 MB). Loading the whole
# file into memory is unavoidable with python-icalendar, so an unbounded
# upload would OOM.
@router.post("/import")
async def import_ics(request: Request, file: UploadFile = File(...), calendar_name: str = ""):
@@ -1020,7 +1277,7 @@ def setup_calendar_routes() -> APIRouter:
owner = _require_user(request)
db = SessionLocal()
try:
content = await read_upload_limited(file, _ICS_MAX_BYTES, "ICS file")
content = await read_upload_limited(file, ICS_MAX_BYTES, "ICS file")
try:
cal_data = iCal.from_ical(content)
except Exception as e:
@@ -1046,7 +1303,7 @@ def setup_calendar_routes() -> APIRouter:
db.commit()
db.refresh(target_cal)
imported = skipped = 0
imported = skipped = repaired = 0
for comp in cal_data.walk():
if comp.name != "VEVENT":
continue
@@ -1082,6 +1339,18 @@ def setup_calendar_routes() -> APIRouter:
.first()
)
if existing:
# An import predating the clamp below may have stored
# this same event with a non-positive duration, which
# the list_events overlap filter hides. Re-importing
# lands here and would skip without touching that row,
# so the event would stay invisible. Backfill the clamp
# onto the stored row before skipping it.
fixed_end = _ensure_positive_duration(
existing.dtstart, existing.dtend, bool(existing.all_day)
)
if fixed_end != existing.dtend:
existing.dtend = fixed_end
repaired += 1
skipped += 1
continue
@@ -1115,6 +1384,8 @@ def setup_calendar_routes() -> APIRouter:
else:
end_dt = start_dt + timedelta(hours=1)
end_dt = _ensure_positive_duration(start_dt, end_dt, all_day)
ev = CalendarEvent(
uid=uid_val,
calendar_id=target_cal.id,
@@ -1135,6 +1406,7 @@ def setup_calendar_routes() -> APIRouter:
"ok": True,
"imported": imported,
"skipped": skipped,
"repaired": repaired,
"calendar": cal_display,
"calendar_id": target_cal.id,
}
@@ -1217,7 +1489,7 @@ def setup_calendar_routes() -> APIRouter:
"tomorrow", "next Tuesday", "in 30 minutes" resolve correctly.
Uses the "utility" endpoint (small / fast model) to keep latency low.
"""
_require_user(request)
owner = _require_user(request)
from src.endpoint_resolver import resolve_endpoint
from src.llm_core import llm_call_async
from src.text_helpers import strip_think
@@ -1243,9 +1515,9 @@ def setup_calendar_routes() -> APIRouter:
if tz_hint:
set_user_tz_name(tz_hint)
url, model, headers = resolve_endpoint("utility")
url, model, headers = resolve_endpoint("utility", owner=owner or None)
if not url:
url, model, headers = resolve_endpoint("default")
url, model, headers = resolve_endpoint("default", owner=owner or None)
if not url or not model:
return {"ok": False, "error": "No LLM endpoint configured"}

443
routes/chat_helpers.py
View File

@@ -14,7 +14,8 @@ from core.database import Session as DBSession, ModelEndpoint
from src.llm_core import normalize_model_id
from src.endpoint_resolver import normalize_base
from src.context_compactor import maybe_compact, trim_for_context
from src.auth_helpers import get_current_user
from src.model_context import estimate_tokens
from src.auth_helpers import effective_user
from src.prompt_security import untrusted_context_message
from routes.prefs_routes import _load_for_user as load_prefs_for_user
@@ -22,6 +23,47 @@ from fastapi import HTTPException
logger = logging.getLogger(__name__)
_CASUAL_OPENING_RE = re.compile(
r"^\s*(?:h+i+|hey+|hello+|yo+|sup+|what'?s up|wass?up|hiya|howdy|"
r"lol|lmao|haha+|hehe+|thanks?|thank you|ty|idk|dunno|meh|bruh|bro)\b(?P<tail>.*)$",
re.IGNORECASE,
)
_CASUAL_BLOCKLIST_RE = re.compile(
r"\b(?:cookbook|serve|serving|launch|start|vllm|sglang|llama\.?cpp|ollama|"
r"download|model|email|document|doc|note|calendar|task|search|web|research|"
r"file|folder|repo|git|settings?|endpoint|api|token|mcp)\b",
re.IGNORECASE,
)
def _is_casual_low_signal(text: str) -> bool:
"""Short greetings/slang should not pull memory, skills, RAG, or docs."""
s = str(text or "").strip()
m = _CASUAL_OPENING_RE.match(s)
if not m:
return False
tail = m.group("tail") or ""
if _CASUAL_BLOCKLIST_RE.search(tail):
return False
tail_words = re.findall(r"[A-Za-z0-9_'-]+", tail)
return len(tail_words) <= 2
# Strong references to in-flight fire-and-forget tasks scheduled from this
# module. asyncio only keeps weak references to tasks created via
# create_task, so without this the GC can collect a task mid-execution and
# the background work (extraction, auto-naming) silently never runs.
# Mirrors WebhookManager._spawn_tracked from src/webhook_manager.py.
_BG_TASKS: set[asyncio.Task] = set()
def _spawn_bg(coro) -> asyncio.Task:
"""Schedule a background task and hold a strong reference until it finishes."""
task = asyncio.create_task(coro)
_BG_TASKS.add(task)
task.add_done_callback(_BG_TASKS.discard)
return task
# ── Data containers ────────────────────────────────────────────────────── #
@@ -58,11 +100,19 @@ class ChatContext:
uprefs: dict
preset: PresetInfo
preprocessed: PreprocessedMessage
context_trimmed: bool = False
context_messages_before_trim: int = 0
context_messages_after_trim: int = 0
context_tokens_before_trim: int = 0
context_tokens_after_trim: int = 0
# Documents auto-created server-side during preprocess (e.g. when an
# attached fillable PDF gets rendered into a markdown editor doc).
# The chat route emits a doc_update SSE event for each before streaming
# begins, so the editor pane switches to the new doc immediately.
auto_opened_docs: list = field(default_factory=list)
# Uploads attached to this user turn, resolved and owner-checked for the
# agent's private context. This is not emitted to the browser.
uploaded_files: list = field(default_factory=list)
# ── Helpers ────────────────────────────────────────────────────────────── #
@@ -75,10 +125,10 @@ def _enforce_chat_privileges(request, sess) -> None:
allowlist, or HTTPException(429) if the user has hit their daily message
cap. No-op for unauthenticated callers or when auth_manager is absent
(single-user mode). Admins receive ADMIN_PRIVILEGES from get_privileges,
which means empty allowed_models / zero cap → no-op for them.
which means unrestricted allowed_models / zero cap -> no-op for them.
"""
try:
user = get_current_user(request)
user = effective_user(request)
except Exception:
user = None
if not user:
@@ -88,8 +138,18 @@ def _enforce_chat_privileges(request, sess) -> None:
return
privs = auth_manager.get_privileges(user) or {}
allowed = privs.get("allowed_models") or []
if allowed and sess.model and sess.model not in allowed:
# Explicit "block everything" sentinel takes precedence over the
# allowlist — it's the only way to distinguish "user clicked [None]"
# (block all) from "user clicked [All]" (no restriction), since both
# otherwise produce an empty `allowed_models` list.
if privs.get("block_all_models"):
raise HTTPException(403, f"Your account is not allowed to use model '{sess.model}'.")
allowed_raw = privs.get("allowed_models")
allowed = allowed_raw if isinstance(allowed_raw, list) else []
restricted = bool(privs.get("allowed_models_restricted")) or bool(allowed)
if restricted and sess.model and sess.model not in allowed:
raise HTTPException(403, f"Your account is not allowed to use model '{sess.model}'.")
cap = int(privs.get("max_messages_per_day") or 0)
@@ -150,7 +210,7 @@ async def auto_name_session(session_manager, sess):
owner = getattr(sess, "owner", None)
t_url, t_model, t_headers = resolve_task_endpoint(
sess.endpoint_url, sess.model, sess.headers, owner=owner,
sess.endpoint_url, sess.model, sess.headers, owner=owner
)
if not t_model:
logger.debug("[auto-name] No model provided, skipping")
@@ -194,14 +254,26 @@ def try_fallback_endpoint(sess, session_id: str) -> dict | None:
Returns {"model": ..., "endpoint_url": ..., "endpoint_name": ...} or None.
"""
import requests as _req
from src.endpoint_resolver import build_chat_url, build_headers, build_models_url, normalize_base
from src.endpoint_resolver import (
build_chat_url,
build_headers,
build_models_url,
normalize_base,
resolve_endpoint_runtime,
)
from src.chatgpt_subscription import is_chatgpt_subscription_base
current_url = sess.endpoint_url or ""
owner = getattr(sess, "owner", None)
db = SessionLocal()
try:
endpoints = db.query(ModelEndpoint).filter(
q = db.query(ModelEndpoint).filter(
ModelEndpoint.is_enabled == True
).all()
)
if owner:
from src.auth_helpers import owner_filter
q = owner_filter(q, ModelEndpoint, owner)
endpoints = q.all()
finally:
db.close()
@@ -210,26 +282,33 @@ def try_fallback_endpoint(sess, session_id: str) -> dict | None:
# Skip current endpoint
if current_url and base in current_url:
continue
# Quick ping
ping_url = build_models_url(base)
headers = build_headers(ep.api_key, base)
try:
r = _req.get(ping_url, headers=headers, timeout=5)
r.raise_for_status()
data = r.json()
models = [m.get("id") for m in (data.get("data") or []) if m.get("id")]
if not models:
models = [
m.get("name") or m.get("model")
for m in (data.get("models") or [])
if m.get("name") or m.get("model")
]
base, api_key = resolve_endpoint_runtime(ep, owner=owner)
except Exception:
continue
ping_url = build_models_url(base)
headers = build_headers(api_key, base)
try:
if ping_url:
r = _req.get(ping_url, headers=headers, timeout=5)
r.raise_for_status()
data = r.json()
models = [m.get("id") for m in (data.get("data") or []) if m.get("id")]
if not models:
models = [
m.get("name") or m.get("model")
for m in (data.get("models") or [])
if m.get("name") or m.get("model")
]
else:
models = json.loads(ep.cached_models or "[]")
if not models:
continue
# Found a working endpoint — update session
new_model = models[0]
chat_url = build_chat_url(base)
new_headers = build_headers(ep.api_key, base)
new_headers = build_headers(api_key, base)
persisted_headers = {} if is_chatgpt_subscription_base(base) else new_headers
sess.model = new_model
sess.endpoint_url = chat_url
@@ -241,7 +320,7 @@ def try_fallback_endpoint(sess, session_id: str) -> dict | None:
_db.query(DBSession).filter(DBSession.id == session_id).update({
"model": new_model,
"endpoint_url": chat_url,
"headers": json.dumps(new_headers),
"headers": persisted_headers,
})
_db.commit()
finally:
@@ -275,11 +354,16 @@ def extract_preset(chat_handler, preset_id) -> PresetInfo:
async def preprocess(
chat_handler, message, att_ids, sess,
auto_opened_docs: Optional[list] = None,
allow_tool_preprocessing: bool = True,
) -> PreprocessedMessage:
"""Run chat_handler.preprocess_message and wrap the result."""
enhanced, user_content, text_ctx, yt_transcripts, att_meta = (
await chat_handler.preprocess_message(
message, att_ids, sess, auto_opened_docs=auto_opened_docs
message,
att_ids,
sess,
auto_opened_docs=auto_opened_docs,
allow_tool_preprocessing=allow_tool_preprocessing,
)
)
return PreprocessedMessage(
@@ -291,6 +375,59 @@ async def preprocess(
)
def build_uploaded_file_manifest(att_ids: list, upload_handler, owner: Optional[str]) -> list[dict]:
"""Resolve current-turn upload IDs into a small tool-facing manifest.
The chat UI already sends attachment ids, and preprocessing inlines as much
text as fits. Agent mode still needs a discoverable bridge for files whose
content was truncated/omitted or when the model chooses file tools. Only
owner-authorized uploads are included, and paths must remain inside the
configured upload directory.
"""
if not att_ids or not upload_handler or not hasattr(upload_handler, "resolve_upload"):
return []
def _read_file_can_open(path: str) -> bool:
try:
from src.tool_execution import _resolve_tool_path
return _resolve_tool_path(path) == os.path.realpath(path)
except Exception:
return False
manifest: list[dict] = []
for att_id in att_ids:
try:
info = upload_handler.resolve_upload(str(att_id), owner=owner)
except Exception:
logger.debug("Failed to resolve upload %r for agent manifest", att_id, exc_info=True)
continue
if not isinstance(info, dict):
continue
path = info.get("path")
if path:
try:
inside = True
if hasattr(upload_handler, "_inside_upload_dir"):
inside = bool(upload_handler._inside_upload_dir(path))
elif hasattr(upload_handler, "inside_base_dir"):
inside = bool(upload_handler.inside_base_dir(path))
if not inside or not os.path.exists(path) or not _read_file_can_open(path):
path = None
except Exception:
path = None
manifest.append({
"id": info.get("id") or str(att_id),
"name": info.get("name") or info.get("original_name") or str(att_id),
"mime": info.get("mime", ""),
"size": info.get("size", 0),
"path": path,
})
return manifest
def add_user_message(sess, chat_handler, preprocessed: PreprocessedMessage, incognito: bool = False):
"""Add user message to session history and update session name.
In incognito mode, still add to in-memory history (for conversation context)
@@ -304,11 +441,11 @@ def add_user_message(sess, chat_handler, preprocessed: PreprocessedMessage, inco
def fire_message_event(request, webhook_manager, session_id: str, sess, message: str, compare_mode: bool = False):
"""Fire webhook and event_bus events for a new user message."""
if webhook_manager and not compare_mode:
asyncio.create_task(webhook_manager.fire("chat.message", {
webhook_manager.fire_and_forget("chat.message", {
"session_id": session_id, "model": sess.model, "message": message[:2000],
}))
})
from src.event_bus import fire_event
user = get_current_user(request)
user = effective_user(request)
fire_event("message_sent", user)
@@ -329,16 +466,26 @@ def _session_url_matches_endpoint(session_url: str, endpoint_base: str) -> bool:
return False
def _has_auth_keys(headers) -> bool:
"""True if a headers dict carries an Authorization/x-api-key entry."""
return isinstance(headers, dict) and any(
k.lower() in ('authorization', 'x-api-key') for k in headers
)
def resolve_session_auth(sess, session_id: str, owner: Optional[str] = None):
"""Ensure session has auth headers — resolve from endpoint DB if missing."""
has_auth = sess.headers and isinstance(sess.headers, dict) and any(
k.lower() in ('authorization', 'x-api-key') for k in sess.headers
)
if has_auth:
try:
from src.chatgpt_subscription import is_chatgpt_subscription_base
is_chatgpt_subscription = is_chatgpt_subscription_base(getattr(sess, "endpoint_url", "") or "")
except Exception:
is_chatgpt_subscription = False
has_auth = _has_auth_keys(sess.headers)
if has_auth and not is_chatgpt_subscription:
return
try:
from src.endpoint_resolver import build_headers, normalize_base
from src.endpoint_resolver import build_headers, resolve_endpoint_runtime
db = SessionLocal()
try:
target_url = getattr(sess, "endpoint_url", "") or ""
@@ -354,10 +501,30 @@ def resolve_session_auth(sess, session_id: str, owner: Optional[str] = None):
for ep in q.all():
if not _session_url_matches_endpoint(target_url, ep.base_url or ""):
continue
if not ep.api_key:
try:
base, api_key = resolve_endpoint_runtime(ep, owner=owner)
except Exception as e:
logger.warning("Failed to resolve provider auth for session %s: %s", session_id, e)
return
if not api_key:
# No usable key (e.g. ChatGPT Subscription needs re-auth).
return
sess.headers = build_headers(api_key, base)
if is_chatgpt_subscription:
# The bearer is short-lived and re-resolved per request, so it
# stays request-local and is never written to the plaintext
# sessions.headers column. Proactively strip any bearer an
# older code path may have persisted so it does not linger.
stale_q = db.query(DBSession).filter(DBSession.id == session_id)
if owner:
stale_q = stale_q.filter(DBSession.owner == owner)
stored = stale_q.first()
if stored is not None and _has_auth_keys(stored.headers):
stale_q.update({"headers": {}})
db.commit()
logger.info(f"Cleared persisted ChatGPT Subscription bearer from session {session_id}")
logger.debug(f"Resolved request-local ChatGPT Subscription auth for session {session_id}")
return
base = normalize_base(ep.base_url or "")
sess.headers = build_headers(ep.api_key, base)
update_q = db.query(DBSession).filter(DBSession.id == session_id)
if owner:
update_q = update_q.filter(DBSession.owner == owner)
@@ -401,7 +568,12 @@ def _normalize_model_id_from_cache(sess) -> Optional[str]:
db = SessionLocal()
try:
endpoints = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True).all()
q = db.query(ModelEndpoint).filter(ModelEndpoint.is_enabled == True)
owner = getattr(sess, "owner", None)
if owner:
from src.auth_helpers import owner_filter
q = owner_filter(q, ModelEndpoint, owner)
endpoints = q.all()
for ep in endpoints:
try:
if normalize_base(getattr(ep, "base_url", "") or "") != session_base:
@@ -428,6 +600,29 @@ def _normalize_model_id_from_cache(sess) -> Optional[str]:
return None
def _session_is_research_spinoff(sess) -> bool:
"""True if this session was created via research "Discuss" spin-off.
Detected by the primer system message the spin-off endpoint seeds into
history (metadata ``research_spinoff_from``). Such sessions are grounded
on the seeded report, so global memory + personal-doc RAG injection is
suppressed for them (the report is the sole knowledge base). Handles both
ChatMessage objects and plain dicts.
"""
for m in getattr(sess, "history", []) or []:
role = getattr(m, "role", None)
if role is None and isinstance(m, dict):
role = m.get("role")
if role != "system":
continue
md = getattr(m, "metadata", None)
if md is None and isinstance(m, dict):
md = m.get("metadata")
if (md or {}).get("research_spinoff_from"):
return True
return False
async def build_chat_context(
sess,
request,
@@ -448,6 +643,7 @@ async def build_chat_context(
webhook_manager=None,
use_enhanced_message: bool = False,
agent_mode: bool = False,
allow_tool_preprocessing: bool = True,
) -> ChatContext:
"""Build the full context (preface + messages) for an LLM call.
@@ -465,6 +661,7 @@ async def build_chat_context(
preprocessed = await preprocess(
chat_handler, message, att_ids or [], sess,
auto_opened_docs=auto_opened_docs,
allow_tool_preprocessing=allow_tool_preprocessing,
)
# Add user message to history
@@ -474,27 +671,48 @@ async def build_chat_context(
if not incognito:
fire_message_event(request, webhook_manager, session_id, sess, message, compare_mode)
# Resolve user prefs
user = get_current_user(request)
# Resolve owner-scoped prefs/context. Browser requests keep the cookie user;
# bearer-token chat requests use the token owner instead of the "api" sentinel.
user = effective_user(request)
uprefs = load_prefs_for_user(user)
uploaded_files = build_uploaded_file_manifest(
att_ids or [],
getattr(chat_handler, "upload_handler", None),
getattr(sess, "owner", None),
)
casual_low_signal = _is_casual_low_signal(message)
# Memory enabled?
mem_enabled = not incognito and not no_memory and uprefs.get("memory_enabled", True)
# Skills injection respects its own enable toggle (mirrors memory_enabled).
# When off, the "Available skills" index is not added to the prompt.
skills_enabled = not incognito and uprefs.get("skills_enabled", True)
if not allow_tool_preprocessing:
mem_enabled = False
skills_enabled = False
if casual_low_signal:
mem_enabled = False
skills_enabled = False
logger.debug(
"Memory enabled=%s for user=%s (incognito=%s, no_memory=%s, pref=%s)",
mem_enabled, user, incognito, no_memory, uprefs.get("memory_enabled", "NOT_SET"),
)
# Research-spinoff ("Discuss") sessions are grounded on the seeded report:
# the primer system message IS the knowledge base. Injecting global memory
# or personal-doc RAG on every turn pulls in keyword-matched but off-topic
# facts ("wrong data") and competes with the report, so suppress both here.
is_research_spinoff = _session_is_research_spinoff(sess)
if is_research_spinoff:
mem_enabled = False
# Use RAG?
use_rag_val = (str(use_rag).lower() != "false") if use_rag is not None else True
if incognito:
if incognito or not allow_tool_preprocessing or is_research_spinoff or casual_low_signal:
use_rag_val = False
# If pre-fetched search context was provided (compare mode), skip live web search
skip_web = bool(search_context)
skip_web = bool(search_context) or not allow_tool_preprocessing or casual_low_signal
# Build context preface
# The stream path uses enhanced_message (with CoT/preprocessing applied),
@@ -513,7 +731,7 @@ async def build_chat_context(
incognito=incognito,
use_skills=skills_enabled,
)
if use_rag is not None:
if use_rag is not None or is_research_spinoff or casual_low_signal:
_preface_kwargs["use_rag"] = use_rag_val
preface, rag_sources, web_sources = chat_processor.build_context_preface(**_preface_kwargs)
@@ -521,7 +739,7 @@ async def build_chat_context(
used_memories = getattr(chat_processor, '_last_used_memories', [])
# Inject pre-fetched search context (compare mode)
if search_context:
if search_context and allow_tool_preprocessing and not casual_low_signal:
preface.append(untrusted_context_message("prefetched search context", search_context))
# YouTube transcripts
@@ -530,18 +748,47 @@ async def build_chat_context(
# Normalize model ID. Prefer cached endpoint models so group chat does not
# re-hit slow local /models endpoints on every participant turn.
norm = _normalize_model_id_from_cache(sess) or normalize_model_id(sess.endpoint_url, sess.model)
norm = _normalize_model_id_from_cache(sess) or normalize_model_id(
sess.endpoint_url,
sess.model,
owner=getattr(sess, "owner", None),
)
if norm:
sess.model = norm
# Build messages
messages = preface + sess.get_context_messages()
# Current date/time — injected as a standalone *user*-role context message
# placed immediately before the latest user turn, NOT folded into the
# system prompt. Its text changes every minute, and local OpenAI-compatible
# backends (llama.cpp / LM Studio) key their KV-cache prefix off the
# system message byte-for-byte; mixing ever-changing timestamp text into
# it would invalidate the cached prefix on every request (issue #2927).
# Placing it at the tail also keeps it out of the stable
# preface+history prefix, so that prefix stays byte-identical turn over
# turn (modulo the genuinely new history entries) and the cache survives.
if not agent_mode:
try:
from src.user_time import current_datetime_context_message
_dt_msg = current_datetime_context_message()
if messages and messages[-1].get("role") == "user":
messages.insert(len(messages) - 1, _dt_msg)
else:
messages.append(_dt_msg)
except Exception:
logger.debug("Failed to add current date/time context", exc_info=True)
# Auto-compact
messages, context_length, was_compacted = await maybe_compact(
sess, sess.endpoint_url, sess.model, messages, sess.headers,
sess, sess.endpoint_url, sess.model, messages, sess.headers, owner=user,
)
_before_trim_messages = len(messages)
_before_trim_tokens = estimate_tokens(messages)
messages = trim_for_context(messages, context_length)
_after_trim_messages = len(messages)
_after_trim_tokens = estimate_tokens(messages)
_context_trimmed = _after_trim_messages < _before_trim_messages or _after_trim_tokens < _before_trim_tokens
return ChatContext(
preface=preface,
@@ -555,7 +802,13 @@ async def build_chat_context(
uprefs=uprefs,
preset=preset,
preprocessed=preprocessed,
context_trimmed=_context_trimmed,
context_messages_before_trim=_before_trim_messages,
context_messages_after_trim=_after_trim_messages,
context_tokens_before_trim=_before_trim_tokens,
context_tokens_after_trim=_after_trim_tokens,
auto_opened_docs=auto_opened_docs,
uploaded_files=uploaded_files,
)
@@ -772,7 +1025,19 @@ def save_assistant_response(
):
"""Add assistant response to session history. In incognito mode, keeps in-memory context but skips DB persistence."""
md = dict(last_metrics) if last_metrics else {}
md["model"] = sess.model
def _model_value(value) -> str:
if value is None:
return ""
if not isinstance(value, str):
value = str(value)
return value.strip()
requested_model = _model_value(md.get("requested_model") or md.get("selected_model") or getattr(sess, "model", ""))
actual_model = _model_value(md.get("model") or md.get("actual_model") or requested_model)
if requested_model:
md["requested_model"] = requested_model
if actual_model:
md["model"] = actual_model
if character_name:
md["character_name"] = character_name
if web_sources:
@@ -821,6 +1086,54 @@ def save_assistant_response(
return None
def _is_session_stream_active(session_id: str) -> bool:
"""Best-effort check for "is a chat completion currently streaming for
this session?" — used to keep background extraction from overlapping a
main completion and competing for the local backend's processing slots
(issue #2927). Lazily imports the route module's live registry to avoid
a circular import (chat_routes imports this module at load time)."""
try:
from routes import chat_routes as _cr
return session_id in getattr(_cr, "_active_streams", {})
except Exception:
return False
async def _run_extraction_jobs_sequentially(session_id: str, jobs: list, max_wait_s: float = 120.0):
"""Run queued background-extraction coroutines one at a time, only once
no chat completion is actively streaming for this session.
As diagnosed in issue #2927, firing memory/skill extraction concurrently
with the main chat completion (or with each other) makes them compete for
the local backend's limited processing slots, evicting the main
conversation's cached KV-cache checkpoint and forcing a full prompt
re-evaluation on the next turn. Waiting for the stream to go idle and then
running the jobs strictly in sequence keeps at most one "side" request in
flight against the backend at any time, and never alongside the user's
own conversation.
"""
# Wait for the triggering turn's own stream to finish winding down (it
# almost always already has by the time this task gets scheduled — this
# is a small safety margin, not the primary mechanism).
waited = 0.0
poll = 0.25
while _is_session_stream_active(session_id) and waited < max_wait_s:
await asyncio.sleep(poll)
waited += poll
for name, job in jobs:
# Re-check before each job: a fast follow-up message from the user
# may have started a new stream for this session while we waited.
waited = 0.0
while _is_session_stream_active(session_id) and waited < max_wait_s:
await asyncio.sleep(poll)
waited += poll
try:
await job
except Exception:
logger.warning("[bg-extract] %s extraction job failed for session %s", name, session_id, exc_info=True)
def run_post_response_tasks(
sess,
session_manager,
@@ -841,21 +1154,37 @@ def run_post_response_tasks(
skills_manager=None,
owner: str = None,
extract_skills: bool = True,
allow_background_extraction: bool = True,
):
"""Fire background tasks after a completed response: memory extraction, webhooks, auto-name, skill extraction."""
"""Fire background tasks after a completed response: memory extraction, webhooks, auto-name, skill extraction.
Memory/skill extraction are queued to run *sequentially*, after the main
completion stream for this session has fully wound down — never
concurrently with it or with each other. As diagnosed in issue #2927,
firing these "side" LLM calls in parallel with the main chat completion
makes them compete for the local backend's limited processing slots
(llama.cpp defaults to 4), evicting the main conversation's cached
checkpoint and forcing a full prompt re-evaluation on the next turn. By
the time this function runs the main response is already saved, but the
extraction calls themselves are still async — queuing them through
``_queue_background_extraction`` keeps them from overlapping the *next*
turn's request too.
"""
_extraction_jobs: list = []
# Memory extraction — only every 4th message pair to avoid excess LLM calls
_msg_count = len(sess.history) if hasattr(sess, 'history') else 0
_should_extract = (_msg_count >= 4) and (_msg_count % 4 == 0)
if not incognito and not compare_mode and _should_extract and uprefs.get("auto_memory", True):
if allow_background_extraction and not incognito and not compare_mode and _should_extract and uprefs.get("auto_memory", True):
from services.memory.memory_extractor import extract_and_store
from src.task_endpoint import resolve_task_endpoint
t_url, t_model, t_headers = resolve_task_endpoint(
sess.endpoint_url, sess.model, sess.headers, owner=owner,
)
asyncio.create_task(extract_and_store(
_extraction_jobs.append(("memory", extract_and_store(
sess, memory_manager, memory_vector,
t_url, t_model, t_headers,
))
)))
# Skill extraction from complex agent runs. Only when the user actually
# chose agent mode — not a chat we auto-escalated for a notes/calendar
@@ -873,6 +1202,7 @@ def run_post_response_tasks(
)
if (
extract_skills
and allow_background_extraction
and auto_skills_enabled
and not incognito
and not compare_mode
@@ -890,12 +1220,15 @@ def run_post_response_tasks(
sess.endpoint_url, sess.model, sess.headers, owner=owner,
)
logger.debug("[skill-extract] dispatching extractor (model=%s)", s_model)
asyncio.create_task(maybe_extract_skill(
_extraction_jobs.append(("skill", maybe_extract_skill(
sess, skills_manager,
s_url, s_model, s_headers,
agent_rounds, agent_tool_calls,
owner=owner,
))
)))
if _extraction_jobs:
_spawn_bg(_run_extraction_jobs_sequentially(session_id, _extraction_jobs))
# Token accumulation
if last_metrics:
@@ -903,11 +1236,11 @@ def run_post_response_tasks(
# Webhook
if webhook_manager and not compare_mode:
asyncio.create_task(webhook_manager.fire("chat.completed", {
webhook_manager.fire_and_forget("chat.completed", {
"session_id": session_id, "model": sess.model,
"user_message": message, "response": full_response[:2000],
}))
})
# Auto-name
if needs_auto_name(sess.name):
asyncio.create_task(auto_name_session(session_manager, sess))
_spawn_bg(auto_name_session(session_manager, sess))

672
routes/chat_routes.py
View File

File diff suppressed because it is too large Load Diff

170
routes/chatgpt_subscription_routes.py Normal file
View File

@@ -0,0 +1,170 @@
"""ChatGPT Subscription device-flow setup routes."""
import json
import logging
import uuid
from typing import Dict, Optional
from fastapi import HTTPException, Request
from core.database import ModelEndpoint, ProviderAuthSession, SessionLocal, utcnow_naive
from routes.device_flow import (
DeviceFlowPoll,
DeviceFlowStart,
PendingDeviceFlowStore,
create_device_flow_router,
)
from src.auth_helpers import get_current_user
from src import chatgpt_subscription
logger = logging.getLogger(__name__)
_DEVICE_FLOW_STORE = PendingDeviceFlowStore()
def _provision_endpoint(tokens: Dict, owner: Optional[str]) -> Dict:
access_token = tokens.get("access_token")
refresh_token = tokens.get("refresh_token")
if not access_token or not refresh_token:
raise ValueError("ChatGPT token response was missing access_token or refresh_token")
base = chatgpt_subscription.DEFAULT_CHATGPT_SUBSCRIPTION_BASE_URL
models = chatgpt_subscription.fetch_available_models(access_token)
if not models:
raise ValueError("ChatGPT Subscription connected, but no usable Codex models were discovered for this account.")
db = SessionLocal()
try:
auth = (
db.query(ProviderAuthSession)
.filter(
ProviderAuthSession.provider == chatgpt_subscription.CHATGPT_SUBSCRIPTION_PROVIDER,
ProviderAuthSession.owner == owner,
)
.first()
)
if auth is None:
auth = ProviderAuthSession(
id=str(uuid.uuid4())[:8],
provider=chatgpt_subscription.CHATGPT_SUBSCRIPTION_PROVIDER,
owner=owner,
label="ChatGPT Subscription",
base_url=base,
auth_mode="chatgpt",
)
db.add(auth)
auth.base_url = base
auth.access_token = access_token
auth.refresh_token = refresh_token
auth.last_refresh = utcnow_naive()
auth.auth_mode = "chatgpt"
ep = (
db.query(ModelEndpoint)
.filter(
ModelEndpoint.base_url == base,
ModelEndpoint.provider_auth_id == auth.id,
ModelEndpoint.owner == owner,
)
.first()
)
if ep is None:
ep = ModelEndpoint(
id=str(uuid.uuid4())[:8],
name="ChatGPT Subscription",
base_url=base,
model_type="llm",
endpoint_kind="api",
owner=owner,
)
db.add(ep)
ep.name = "ChatGPT Subscription"
ep.base_url = base
ep.api_key = None
ep.provider_auth_id = auth.id
ep.is_enabled = True
ep.supports_tools = False
ep.model_type = "llm"
ep.endpoint_kind = "api"
ep.model_refresh_mode = "manual"
ep.cached_models = json.dumps(models)
db.commit()
result = {
"id": ep.id,
"name": ep.name,
"base_url": ep.base_url,
"models": models,
}
finally:
db.close()
try:
from routes.model_routes import _invalidate_models_cache
_invalidate_models_cache()
except Exception:
pass
return result
def _start_device_flow(request: Request, _form) -> DeviceFlowStart:
try:
data = chatgpt_subscription.request_device_code()
except Exception as exc:
raise chatgpt_subscription.to_http_exception(exc)
device_auth_id = data.get("device_auth_id")
user_code = data.get("user_code")
if not device_auth_id or not user_code:
raise HTTPException(502, "ChatGPT did not return a complete device code")
verification_uri = data.get("verification_uri") or f"{chatgpt_subscription.CHATGPT_OAUTH_ISSUER}/codex/device"
return DeviceFlowStart(
pending={
"device_auth_id": device_auth_id,
"user_code": user_code,
"owner": get_current_user(request) or None,
},
response={
"user_code": user_code,
"verification_uri": verification_uri,
},
interval=int(data.get("interval") or 5),
expires_in=int(data.get("expires_in") or 900),
)
def _poll_device_flow(_request: Request, pending: Dict) -> DeviceFlowPoll:
try:
data = chatgpt_subscription.poll_device_auth(pending["device_auth_id"], pending["user_code"])
except Exception as exc:
logger.debug("ChatGPT device poll failed: %s", exc)
return DeviceFlowPoll.pending(str(exc))
authorization_code = data.get("authorization_code")
code_verifier = data.get("code_verifier")
if authorization_code and code_verifier:
try:
tokens = chatgpt_subscription.exchange_authorization_code(authorization_code, code_verifier)
result = _provision_endpoint(tokens, pending["owner"])
except Exception as exc:
logger.exception("ChatGPT Subscription endpoint provisioning failed")
raise chatgpt_subscription.to_http_exception(exc)
return DeviceFlowPoll.authorized(result)
err = data.get("error") or data.get("status")
if err in ("authorization_pending", "pending", None):
return DeviceFlowPoll.pending()
if err == "slow_down":
return DeviceFlowPoll.slow_down(int(data.get("interval") or 0) or None)
if err in ("expired_token", "access_denied", "denied"):
return DeviceFlowPoll.failed(err)
return DeviceFlowPoll.pending(err or "unknown")
def setup_chatgpt_subscription_routes():
return create_device_flow_router(
prefix="/api/chatgpt-subscription",
tags=["chatgpt-subscription"],
store=_DEVICE_FLOW_STORE,
start_flow=_start_device_flow,
poll_flow=_poll_device_flow,
)

176
routes/codex_routes.py
View File

@@ -15,8 +15,11 @@ from typing import Any
from fastapi import APIRouter, BackgroundTasks, Body, HTTPException, Request
from fastapi.responses import StreamingResponse
from src.auth_helpers import require_user
from core.middleware import require_admin
from src.auth_helpers import require_authenticated_request, require_user
from src.tool_implementations import do_manage_notes
from src.constants import COOKBOOK_STATE_FILE
from routes._validators import validate_remote_host, validate_ssh_port
COOKBOOK_READ_SCOPES = {"cookbook:read", "cookbook:launch"}
@@ -35,13 +38,34 @@ DOCS_WRITE_SCOPES = {"documents:write"}
WRITE_ACTIONS = {"add", "create", "new", "save", "remind", "update", "delete", "toggle_item", "remove", "remove_item"}
def _ssh_prefix_for_task(task: dict) -> tuple[str, str]:
"""Resolve a cookbook task's stored SSH target into ``(host, port_flag)``.
``host`` is ``""`` for a local task. ``remoteHost`` / ``sshPort`` come from
cookbook_state.json and get interpolated into an ``ssh`` command string, so
validate them the same way the cookbook routes do. A tampered entry with
shell metacharacters in ``remoteHost`` is rejected with 400 rather than
injected.
"""
raw_host = task.get("remoteHost")
raw_port = task.get("sshPort")
host_value = str(raw_host).strip() if raw_host is not None else None
port_value = str(raw_port).strip() if raw_port is not None else None
host = validate_remote_host(host_value or None) or ""
ssh_port = validate_ssh_port(port_value or None) or ""
port_flag = f"-p {ssh_port} " if ssh_port and ssh_port != "22" else ""
return host, port_flag
async def _as_owner(request: Request, owner: str, fn, *args, **kwargs):
"""Run an existing route handler with request.state.current_user temporarily
set to ``owner`` so its internal get_current_user/require_user calls see
the scope-gated owner (not the "api" pseudo-user the bearer middleware sets).
Restores the original value when done. Works for sync and async handlers."""
orig = getattr(request.state, "current_user", None)
orig_api_token = getattr(request.state, "api_token", None)
request.state.current_user = owner
request.state.api_token = False
try:
result = fn(*args, **kwargs)
if asyncio.iscoroutine(result):
@@ -49,6 +73,13 @@ async def _as_owner(request: Request, owner: str, fn, *args, **kwargs):
return result
finally:
request.state.current_user = orig
if orig_api_token is None:
try:
delattr(request.state, "api_token")
except AttributeError:
pass
else:
request.state.api_token = orig_api_token
def _scope_owner(request: Request, allowed: set[str]) -> str:
@@ -65,6 +96,34 @@ def _scope_owner(request: Request, allowed: set[str]) -> str:
return require_user(request)
def _scope_owner_all(request: Request, required: set[str]) -> str:
"""Return owner only when an API token has every required scope."""
if getattr(request.state, "api_token", False):
scopes = set(getattr(request.state, "api_token_scopes", []) or [])
missing = required - scopes
if missing:
raise HTTPException(403, f"API token missing required scope: {' and '.join(sorted(missing))}")
owner = getattr(request.state, "api_token_owner", None)
if not owner:
raise HTTPException(403, "API token has no owner")
return owner
return require_user(request)
def _require_cookbook_scope(request: Request, allowed: set[str]) -> str:
"""Authorize a Codex cookbook route.
For API-token callers, enforce the given scope set.
For cookie-session callers, additionally require admin privileges
because cookbook surfaces expose host topology, task logs, tmux
commands, and model-serving controls.
"""
owner = _scope_owner(request, allowed)
if not getattr(request.state, "api_token", False):
require_admin(request)
return owner
def _find_endpoint(router: APIRouter | None, method: str, path: str):
if router is None:
return None
@@ -74,6 +133,18 @@ def _find_endpoint(router: APIRouter | None, method: str, path: str):
return None
def _clamp_pagination(offset: Any, limit: Any, *, default_limit: int = 50, max_limit: int = 50) -> tuple[int, int]:
try:
parsed_offset = int(0 if offset in (None, "") else offset)
except (TypeError, ValueError):
raise HTTPException(400, "Invalid offset")
try:
parsed_limit = int(default_limit if limit in (None, "") else limit)
except (TypeError, ValueError):
raise HTTPException(400, "Invalid limit")
return max(0, parsed_offset), max(1, min(parsed_limit, max_limit))
def setup_codex_routes(
email_router: APIRouter | None = None,
memory_router: APIRouter | None = None,
@@ -112,7 +183,7 @@ def setup_codex_routes(
"read": scoped(EMAIL_READ_SCOPES),
"draft": scoped(EMAIL_DRAFT_SCOPES),
"send": scoped(EMAIL_SEND_SCOPES),
"actions": ["list", "read", "draft", "send"],
"actions": ["list", "read", "draft_document", "draft", "send"],
},
"memory": {
"read": scoped(MEMORY_READ_SCOPES),
@@ -146,7 +217,7 @@ def setup_codex_routes(
@router.get("/plugin.zip")
def plugin_zip(request: Request):
require_user(request)
require_authenticated_request(request)
root = Path(__file__).resolve().parent.parent / "integrations" / "codex"
if not root.exists():
raise HTTPException(404, "Codex plugin bundle not found")
@@ -236,6 +307,59 @@ def setup_codex_routes(
# Both handlers in routes/email_routes.py already accept `owner=` via
# FastAPI Depends, so we call them directly without patching state.
def _email_draft_document_content(body: dict[str, Any]) -> str:
def clean(v: Any) -> str:
if isinstance(v, list):
return ", ".join(str(x).strip() for x in v if str(x).strip())
return str(v or "").strip()
to = clean(body.get("to"))
cc = clean(body.get("cc"))
bcc = clean(body.get("bcc"))
subject = clean(body.get("subject"))
in_reply_to = clean(body.get("in_reply_to"))
references = clean(body.get("references"))
body_text = str(body.get("body") or body.get("body_html") or "").strip()
lines = [
f"To: {to}",
]
if cc:
lines.append(f"Cc: {cc}")
if bcc:
lines.append(f"Bcc: {bcc}")
lines.append(f"Subject: {subject}")
if in_reply_to:
lines.append(f"In-Reply-To: {in_reply_to}")
if references:
lines.append(f"References: {references}")
lines.extend(["---", body_text])
return "\n".join(lines).rstrip() + "\n"
@router.post("/emails/draft-document")
async def codex_email_draft_document(request: Request, body: dict[str, Any] = Body(default_factory=dict)):
owner = _scope_owner(request, EMAIL_DRAFT_SCOPES)
docs_owner = _scope_owner_all(request, DOCS_WRITE_SCOPES)
if docs_owner != owner:
raise HTTPException(403, "API token owner mismatch")
if documents_create_endpoint is None:
raise HTTPException(503, "Documents integration is not available")
from routes.document_routes import DocumentCreate
subject = str(body.get("subject") or "Email draft").strip() or "Email draft"
title = str(body.get("title") or subject).strip() or "Email draft"
req = DocumentCreate(
session_id=body.get("session_id"),
title=title,
language="email",
content=_email_draft_document_content(body),
)
result = await _as_owner(request, owner, documents_create_endpoint, request, req)
if isinstance(result, dict):
result = dict(result)
result["draft_type"] = "document"
result["send_required_confirmation"] = True
return result
@router.post("/emails/draft")
async def codex_email_draft(request: Request, body: dict[str, Any] = Body(default_factory=dict)):
owner = _scope_owner(request, EMAIL_DRAFT_SCOPES)
@@ -328,10 +452,18 @@ def setup_codex_routes(
owner = _scope_owner(request, DOCS_READ_SCOPES)
if documents_library_endpoint is None:
raise HTTPException(503, "Documents integration is not available")
return await _as_owner(
offset, limit = _clamp_pagination(offset, limit)
result = await _as_owner(
request, owner, documents_library_endpoint,
request, search, language, sort, offset, limit, archived,
)
if isinstance(result, dict):
docs = result.get("documents")
total = result.get("total")
if isinstance(docs, list) and isinstance(total, int):
next_offset = offset + len(docs)
result["next_offset"] = next_offset if next_offset < total else None
return result
@router.get("/documents/{doc_id}")
async def codex_documents_get(request: Request, doc_id: str):
@@ -415,8 +547,8 @@ def setup_codex_routes(
def _read_cookbook_state() -> dict:
from pathlib import Path as _Path
import os as _os, json as _json
p = _Path(_os.environ.get("DATA_DIR", "data")) / "cookbook_state.json"
import json as _json
p = _Path(COOKBOOK_STATE_FILE)
if not p.exists():
return {}
try:
@@ -435,14 +567,14 @@ def setup_codex_routes(
@router.get("/cookbook/tasks")
async def codex_cookbook_tasks(request: Request):
_scope_owner(request, COOKBOOK_READ_SCOPES)
_require_cookbook_scope(request, COOKBOOK_READ_SCOPES)
state = _read_cookbook_state()
tasks = state.get("tasks") or []
return {"tasks": [_redact_task(t) for t in tasks]}
@router.get("/cookbook/servers")
async def codex_cookbook_servers(request: Request):
_scope_owner(request, COOKBOOK_READ_SCOPES)
_require_cookbook_scope(request, COOKBOOK_READ_SCOPES)
state = _read_cookbook_state()
servers = state.get("env", {}).get("servers") or []
# Strip ssh creds / passwords; keep only what's needed to pick a host.
@@ -461,7 +593,7 @@ def setup_codex_routes(
@router.get("/cookbook/output/{session_id}")
async def codex_cookbook_output(request: Request, session_id: str, tail: int = 400):
_scope_owner(request, COOKBOOK_READ_SCOPES)
_require_cookbook_scope(request, COOKBOOK_READ_SCOPES)
# Defensive: session_id must be the tmux-style id we issue
# (`serve-XXXX` / `cookbook-XXXX` / `queue-XXXX`); anything else
# would let the agent run arbitrary `tmux capture-pane` targets.
@@ -476,8 +608,7 @@ def setup_codex_routes(
task = next((t for t in tasks if t.get("sessionId") == session_id), None)
if task is None:
raise HTTPException(404, "task not found")
host = (task.get("remoteHost") or "").strip()
ssh_port = (task.get("sshPort") or "").strip()
host, port_flag = _ssh_prefix_for_task(task)
# Prefer the persisted log file over the tmux pane. The pane gets
# overwritten by the post-crash neofetch banner + bash prompt the
# moment vllm exits; the log file is the raw stdout/stderr and
@@ -489,7 +620,6 @@ def setup_codex_routes(
f"else tmux capture-pane -t {session_id} -p -S -{tail}; fi"
)
if host:
port_flag = f"-p {ssh_port} " if ssh_port and ssh_port != "22" else ""
import shlex
cmd = f"ssh {port_flag}{host} {shlex.quote(inner)}"
else:
@@ -505,7 +635,7 @@ def setup_codex_routes(
@router.post("/cookbook/serve")
async def codex_cookbook_serve(request: Request, body: dict[str, Any] = Body(default_factory=dict)):
_scope_owner(request, COOKBOOK_LAUNCH_SCOPES)
_require_cookbook_scope(request, COOKBOOK_LAUNCH_SCOPES)
# Wraps /api/model/serve with the SAME validation the UI uses.
# _validate_serve_cmd (called inside model_serve) rejects shell
# metachars and requires the leading binary to be in the
@@ -544,17 +674,15 @@ def setup_codex_routes(
@router.post("/cookbook/stop/{session_id}")
async def codex_cookbook_stop(request: Request, session_id: str):
_scope_owner(request, COOKBOOK_LAUNCH_SCOPES)
_require_cookbook_scope(request, COOKBOOK_LAUNCH_SCOPES)
import re as _re
if not _re.fullmatch(r"[a-zA-Z0-9_-]+", session_id):
raise HTTPException(400, "Invalid session id")
state = _read_cookbook_state()
tasks = state.get("tasks") or []
task = next((t for t in tasks if t.get("sessionId") == session_id), None)
host = ((task or {}).get("remoteHost") or "").strip()
ssh_port = ((task or {}).get("sshPort") or "").strip()
host, port_flag = _ssh_prefix_for_task(task or {})
if host:
port_flag = f"-p {ssh_port} " if ssh_port and ssh_port != "22" else ""
cmd = f"ssh {port_flag}{host} \"tmux kill-session -t {session_id}\""
else:
cmd = f"tmux kill-session -t {session_id}"
@@ -566,7 +694,7 @@ def setup_codex_routes(
"""List cached models on a configured server (or local if host is omitted).
Mirrors `list_cached_models` from the chat agent so external agents have
the same inventory view before deciding what to serve/download."""
_scope_owner(request, COOKBOOK_READ_SCOPES)
_require_cookbook_scope(request, COOKBOOK_READ_SCOPES)
# Hit /api/model/cached internally, with the same modelDirs the chat
# agent's list_cached_models would resolve from cookbook state.
state = _read_cookbook_state()
@@ -628,7 +756,7 @@ def setup_codex_routes(
"""List saved serve presets (model + host + port + launch cmd).
Counterpart to `list_serve_presets`. Use BEFORE composing a `serve`
body — the user's saved preset usually has the working cmd already."""
_scope_owner(request, COOKBOOK_READ_SCOPES)
_require_cookbook_scope(request, COOKBOOK_READ_SCOPES)
state = _read_cookbook_state()
presets = state.get("presets") or []
out = []
@@ -648,7 +776,7 @@ def setup_codex_routes(
async def codex_cookbook_serve_preset(request: Request, name: str):
"""Launch a saved preset by name. Reuses the working cmd + host the
user already saved, avoiding the cmd-allowlist trial-and-error loop."""
_scope_owner(request, COOKBOOK_LAUNCH_SCOPES)
_require_cookbook_scope(request, COOKBOOK_LAUNCH_SCOPES)
import re as _re
if not _re.fullmatch(r"[A-Za-z0-9 _.:@\-]+", name):
raise HTTPException(400, "Invalid preset name")
@@ -700,11 +828,11 @@ def setup_codex_routes(
cookbook tracking. Needed when serve_model rejects a cmd and the
agent falls back to direct ssh — without adoption the session is
invisible to the UI. Body: {tmux_session, model, host?, port?}."""
_scope_owner(request, COOKBOOK_LAUNCH_SCOPES)
_require_cookbook_scope(request, COOKBOOK_LAUNCH_SCOPES)
norm = dict(body or {})
sess = (norm.get("tmux_session") or norm.get("session_id") or "").strip()
model = (norm.get("model") or norm.get("repo_id") or "").strip()
host = (norm.get("host") or norm.get("remote_host") or "").strip()
host = validate_remote_host((norm.get("host") or norm.get("remote_host") or "").strip() or None) or ""
port = norm.get("port") or 8000
import re as _re
if not sess or not _re.fullmatch(r"[a-zA-Z0-9_-]+", sess):
@@ -724,7 +852,7 @@ def setup_codex_routes(
import time as _t, json as _json
from core.atomic_io import atomic_write_json
from pathlib import Path as _Path
cookbook_state_path = _Path("/app/data/cookbook_state.json")
cookbook_state_path = _Path(COOKBOOK_STATE_FILE)
try:
state = _json.loads(cookbook_state_path.read_text(encoding="utf-8"))
except Exception:
@@ -762,7 +890,7 @@ def setup_claude_routes() -> APIRouter:
@router.get("/plugin.zip")
def plugin_zip(request: Request):
require_user(request)
require_authenticated_request(request)
# Only ship the skills/ subtree so extracting at ~/.claude/ doesn't dump
# README.md or other bundle metadata into the user's claude config dir.
skills_root = Path(__file__).resolve().parent.parent / "integrations" / "claude" / "skills"

132
routes/compare_routes.py
View File

@@ -12,6 +12,7 @@ import logging
from core.database import Comparison, SessionLocal
from core.session_manager import SessionManager
from src.auth_helpers import get_current_user
from routes.session_routes import _reject_raw_endpoint_url_for_non_admin
logger = logging.getLogger(__name__)
@@ -38,6 +39,24 @@ def _owned_endpoint_by_url(db, base_url, owner):
return owner_filter(q, ModelEndpoint, owner).first()
def _owned_endpoint_by_id(db, endpoint_id, owner):
"""ModelEndpoint whose id == `endpoint_id` and is VISIBLE to `owner` (their
own rows + legacy null-owner "shared" rows); None otherwise.
Preferred over _owned_endpoint_by_url for credential resolution: two visible
endpoints can share the same base_url but hold DIFFERENT api_keys (e.g. two
accounts on the same provider). A base_url-only match returns whichever row
sorts first, so it can copy the WRONG owner-scoped key into the [CMP] session.
An id pins the exact registered endpoint, so /api/compare/start prefers it and
only falls back to URL matching for legacy / admin raw-URL callers. Owner
scoping is identical to _owned_endpoint_by_url (a null/empty owner is a no-op).
"""
from core.database import ModelEndpoint
from src.auth_helpers import owner_filter
q = db.query(ModelEndpoint).filter(ModelEndpoint.id == endpoint_id)
return owner_filter(q, ModelEndpoint, owner).first()
class RecordVoteRequest(BaseModel):
prompt: str
models: List[str]
@@ -54,8 +73,10 @@ def setup_compare_routes(session_manager: SessionManager):
prompt: str = Form(...),
model_a: str = Form(...),
model_b: str = Form(...),
endpoint_a: str = Form(...),
endpoint_b: str = Form(...),
endpoint_a: str = Form(""),
endpoint_b: str = Form(""),
endpoint_a_id: str = Form(""),
endpoint_b_id: str = Form(""),
is_blind: str = Form("true"),
):
"""Create two ephemeral sessions and a comparison record.
@@ -63,10 +84,10 @@ def setup_compare_routes(session_manager: SessionManager):
Returns the comparison ID and the two session IDs so the client
can fire two independent SSE streams to /api/chat_stream.
"""
user = getattr(request.state, 'current_user', None)
comp_id = str(uuid.uuid4())
sid_a = str(uuid.uuid4())
sid_b = str(uuid.uuid4())
user = getattr(request.state, 'current_user', None)
# Blind mapping: randomly assign left/right
blind = str(is_blind).lower() == "true"
@@ -87,31 +108,94 @@ def setup_compare_routes(session_manager: SessionManager):
# de-anonymizing the comparison before the user votes (issue #1285).
slot_name = {session_left: "Model A", session_right: "Model B"}
# Create ephemeral sessions (prefixed [CMP])
for sid, model, endpoint in [(sid_a, model_a, endpoint_a), (sid_b, model_b, endpoint_b)]:
# SECURITY: resolve and validate BOTH endpoints before creating any
# session. Compare copies a registered endpoint's Authorization header
# into the [CMP] session, so validating one endpoint while creating its
# session, then rejecting the other, would leave a partial compare
# session behind with that header attached. Doing all the owner-scope
# resolution + raw-URL rejection up front means a 403 on either endpoint
# aborts the whole request with nothing created and no header copied.
from src.endpoint_resolver import build_chat_url, build_headers, normalize_base
resolved = []
db = SessionLocal()
try:
for sid, model, endpoint, endpoint_id in [
(sid_a, model_a, endpoint_a, endpoint_a_id),
(sid_b, model_b, endpoint_b, endpoint_b_id),
]:
# Prefer an explicit endpoint id: it pins the EXACT registered
# endpoint (and its api_key), even when two endpoints visible to
# the caller share a base_url with different keys — a URL-only
# match would copy whichever row sorts first, i.e. possibly the
# wrong key. Fall back to URL resolution only for legacy / admin
# raw-URL callers that don't send an id.
eid = endpoint_id.strip() if isinstance(endpoint_id, str) else ""
if eid:
ep = _owned_endpoint_by_id(db, eid, user)
if ep is None:
# An id the caller can't see (wrong owner / deleted) must
# NOT silently fall back to a same-URL row with a different
# key — that's exactly the mix-up ids exist to prevent.
raise HTTPException(404, "Model endpoint not found")
# The id already resolved the endpoint; ignore any raw URL the
# caller also sent and dial the stored config instead.
endpoint = ep.base_url
elif not endpoint:
raise HTTPException(
422, "endpoint_a/endpoint_b or endpoint_a_id/endpoint_b_id is required"
)
else:
# Resolve the supplied URL to a ModelEndpoint the caller owns
# (their own rows + legacy null-owner shared rows), scoped so a
# comparison can't borrow another user's private endpoint key.
base = normalize_base(endpoint)
ep = _owned_endpoint_by_url(db, base, user)
# Reject *unregistered* raw URLs for signed-in non-admins; a
# matched registered endpoint supplies an id so the caller can
# still compare endpoints they own. Blanket-rejecting here (the
# earlier `endpoint_id=None` call) locked non-admins out of
# compare entirely, since compare resolves endpoints by URL with
# no endpoint_id. Mirrors the gallery inpaint/harmonize checks.
# Raised here (phase 1), before any session exists.
_reject_raw_endpoint_url_for_non_admin(
request, user, str(ep.id) if ep is not None else None, endpoint
)
# Bind the [CMP] session to the RESOLVED endpoint, not the raw
# caller-supplied string. When the URL matches a registered
# endpoint visible to the caller, use that row's own normalized
# base URL (the same value owner scoping + endpoint validation
# already vetted) so the session dials exactly where the stored
# config points. The raw `endpoint` only survives for callers
# allowed to pass one — admins / single-user mode, where
# `_reject_raw_endpoint_url_for_non_admin` is a no-op and `ep`
# is None. Mirrors the registered-endpoint path in session_routes.
session_endpoint_url = (
build_chat_url(normalize_base(ep.base_url)) if ep is not None else endpoint
)
# Headers come only from a matched endpoint's key; None when
# `ep` is None (raw admin URL or no match), so a comparison can
# never inherit another user's key/headers.
headers = build_headers(ep.api_key, ep.base_url) if (ep and ep.api_key) else None
resolved.append((sid, model, session_endpoint_url, headers))
finally:
db.close()
# Both endpoints validated — only now create the ephemeral [CMP]
# sessions and copy any resolved headers.
for sid, model, session_endpoint_url, headers in resolved:
name = f"[CMP] {slot_name[sid]}" if blind else f"[CMP] {model.split('/')[-1]}"
session_manager.create_session(
session_id=sid,
name=name,
endpoint_url=endpoint,
endpoint_url=session_endpoint_url,
model=model,
rag=False,
owner=user,
)
# Copy API key from endpoint config
db = SessionLocal()
try:
from src.endpoint_resolver import build_headers, normalize_base
# Find matching endpoint by URL, scoped to the caller so a
# comparison can't borrow another user's private endpoint key.
base = normalize_base(endpoint)
ep = _owned_endpoint_by_url(db, base, user)
if ep and ep.api_key:
s = session_manager.sessions.get(sid)
if s:
s.headers = build_headers(ep.api_key, ep.base_url)
finally:
db.close()
if headers:
s = session_manager.sessions.get(sid)
if s:
s.headers = headers
# Store comparison record
db = SessionLocal()
@@ -121,8 +205,12 @@ def setup_compare_routes(session_manager: SessionManager):
prompt=prompt,
model_a=model_a,
model_b=model_b,
endpoint_a=endpoint_a,
endpoint_b=endpoint_b,
# Record the URL the session actually dials. For URL callers this
# is their raw input; for id-only callers (empty endpoint_a/_b)
# fall back to the resolved endpoint URL so the column stays
# meaningful and non-null. resolved is in [a, b] order.
endpoint_a=endpoint_a or resolved[0][2],
endpoint_b=endpoint_b or resolved[1][2],
is_blind=blind,
blind_mapping=json.dumps(mapping),
owner=user,

5
routes/contacts/__init__.py Normal file
View File

@@ -0,0 +1,5 @@
"""Contacts route domain package (slice 2e, #4082/#4071).
Contains contacts_routes.py, migrated from the flat routes/ directory.
Backward-compat shim at routes/contacts_routes.py re-exports from here.
"""

916
routes/contacts/contacts_routes.py Normal file
View File

@@ -0,0 +1,916 @@
"""
contacts_routes.py
CardDAV contacts integration. Reads from local Radicale, supports
search and adding new contacts.
"""
import re
import logging
import uuid
import json
import csv
import io
import os
import inspect
import httpx
from pathlib import Path
from datetime import datetime
from urllib.parse import urljoin, urlparse, urlunparse
from core.log_safety import redact_url
from fastapi import APIRouter, Query, Depends, Response, HTTPException
from typing import List, Dict, Optional
from core.middleware import require_admin
from src.url_safety import check_outbound_url
logger = logging.getLogger(__name__)
from src.constants import DATA_DIR as _DATA_DIR, SETTINGS_FILE as _SETTINGS_FILE, CONTACTS_FILE as _CONTACTS_FILE
DATA_DIR = Path(_DATA_DIR)
SETTINGS_FILE = Path(_SETTINGS_FILE)
LOCAL_CONTACTS_FILE = Path(_CONTACTS_FILE)
def _load_settings():
if SETTINGS_FILE.exists():
return json.loads(SETTINGS_FILE.read_text(encoding="utf-8"))
return {}
def _save_settings(settings):
from core.atomic_io import atomic_write_json
atomic_write_json(str(SETTINGS_FILE), settings, indent=2)
def _get_carddav_config():
import os
settings = _load_settings()
password = settings.get("carddav_password", os.environ.get("CARDDAV_PASSWORD", ""))
if password and "carddav_password" in settings:
from src.secret_storage import decrypt
password = decrypt(password)
return {
"url": settings.get("carddav_url", os.environ.get("CARDDAV_URL", "")),
"username": settings.get("carddav_username", os.environ.get("CARDDAV_USERNAME", "")),
"password": password,
}
def _carddav_configured(cfg: Optional[Dict] = None) -> bool:
cfg = cfg or _get_carddav_config()
return bool((cfg.get("url") or "").strip())
def _validate_carddav_url(url: str) -> str:
cleaned = (url if isinstance(url, str) else "").strip().rstrip("/")
ok, reason = check_outbound_url(
cleaned,
block_private=os.getenv("CARDDAV_BLOCK_PRIVATE_IPS", "false").lower() == "true",
)
if not ok:
raise ValueError(f"Rejected CardDAV URL: {reason}")
return cleaned
def _carddav_base_url(cfg: Dict) -> str:
return _validate_carddav_url(cfg.get("url") or "")
def _normalize_contact(contact: Dict) -> Dict:
emails = []
for e in contact.get("emails") or ([] if not contact.get("email") else [contact.get("email")]):
e = str(e or "").strip()
if e and e not in emails:
emails.append(e)
phones = []
for p in contact.get("phones") or ([] if not contact.get("phone") else [contact.get("phone")]):
p = str(p or "").strip()
if p and p not in phones:
phones.append(p)
name = str(contact.get("name") or "").strip()
if not name and emails:
name = emails[0].split("@")[0]
address = str(contact.get("address") or "").strip()
return {
"uid": str(contact.get("uid") or uuid.uuid4()),
"name": name,
"emails": emails,
"phones": phones,
"address": address,
}
def _load_local_contacts() -> List[Dict]:
try:
if not LOCAL_CONTACTS_FILE.exists():
return []
data = json.loads(LOCAL_CONTACTS_FILE.read_text(encoding="utf-8"))
rows = data.get("contacts", data) if isinstance(data, dict) else data
return [_normalize_contact(c) for c in (rows or []) if isinstance(c, dict)]
except Exception as e:
logger.error(f"Failed to load local contacts: {e}")
return []
def _save_local_contacts(contacts: List[Dict]) -> None:
from core.atomic_io import atomic_write_json
DATA_DIR.mkdir(parents=True, exist_ok=True)
atomic_write_json(str(LOCAL_CONTACTS_FILE), {"contacts": [_normalize_contact(c) for c in contacts]}, indent=2)
_contact_cache["contacts"] = [_normalize_contact(c) for c in contacts]
_contact_cache["fetched_at"] = datetime.utcnow()
# ── vCard parsing ──
def _vunesc(value: str) -> str:
"""Reverse _vesc() — turn escaped vCard text back into the raw value.
Order matters: handle \\n/\\, /\\; first, backslash-unescape last."""
if not value:
return value
out = []
i = 0
while i < len(value):
ch = value[i]
if ch == "\\" and i + 1 < len(value):
nxt = value[i + 1]
if nxt in ("n", "N"):
out.append("\n")
elif nxt in (",", ";", "\\"):
out.append(nxt)
else:
out.append(nxt)
i += 2
else:
out.append(ch)
i += 1
return "".join(out)
def _parse_vcards(text: str) -> List[Dict]:
"""Parse a stream of vCards into dicts with name, email, phone."""
# Unfold RFC 6350 3.2 line folding first: a CRLF/LF followed by a single
# space or tab is a continuation of the previous logical line. Real
# CardDAV servers (Radicale, iCloud, Apple/Google) fold long EMAIL / FN /
# PHOTO lines, and splitting on raw newlines without unfolding dropped the
# continuation (e.g. "...@example\n .com" lost the ".com"), truncating the
# email/name.
text = re.sub(r"\r\n[ \t]", "", text or "")
text = re.sub(r"\n[ \t]", "", text)
contacts = []
for block in re.split(r"BEGIN:VCARD", text):
if not block.strip():
continue
contact = {"name": "", "emails": [], "phones": [], "uid": "", "address": ""}
for line in block.split("\n"):
line = line.strip()
# Strip an optional RFC 6350 group prefix (e.g. "item1.EMAIL;...")
# that Apple Contacts / iCloud / many CardDAV servers emit by
# default — without this the property-name checks below miss those
# lines and silently drop the email / phone. The group token only
# precedes the property name, so it is safe to strip for matching
# and value extraction, and a no-op for non-grouped lines.
name_part = re.sub(r"^[A-Za-z0-9-]+\.", "", line, count=1)
if name_part.startswith("FN:") or name_part.startswith("FN;"):
contact["name"] = _vunesc(name_part.split(":", 1)[1]) if ":" in name_part else ""
elif name_part.startswith("EMAIL"):
# Handle EMAIL:foo@bar OR EMAIL;TYPE=...:foo@bar OR EMAIL;PREF=1:foo@bar
if ":" in name_part:
email_addr = _vunesc(name_part.split(":", 1)[1])
if email_addr and email_addr not in contact["emails"]:
contact["emails"].append(email_addr)
elif name_part.startswith("TEL"):
if ":" in name_part:
phone = _vunesc(name_part.split(":", 1)[1])
if phone and phone not in contact["phones"]:
contact["phones"].append(phone)
elif name_part.startswith("ADR"):
# vCard ADR is 7 semicolon-separated components:
# post-office-box;extended-address;street;locality;region;postal-code;country.
# Recover a human-readable string by joining non-empty
# components with ", ".
if ":" in name_part:
raw = name_part.split(":", 1)[1]
parts = [_vunesc(p).strip() for p in raw.split(";")]
contact["address"] = ", ".join(p for p in parts if p)
elif name_part.startswith("UID:"):
contact["uid"] = _vunesc(name_part[4:])
if contact["name"] or contact["emails"]:
contacts.append(contact)
return contacts
def _vesc(value: str) -> str:
"""Escape a vCard property VALUE per RFC 6350 §3.4: backslash, comma,
semicolon, and newlines. Without this, a name like 'Sekisui House,Ltd'
or any value containing a newline produces a malformed vCard (broken
N/FN fields) or could inject arbitrary properties."""
return (
(value or "")
.replace("\\", "\\\\")
.replace("\n", "\\n")
.replace("\r", "")
.replace(",", "\\,")
.replace(";", "\\;")
)
def _build_vcard(name: str, email: str, uid: Optional[str] = None,
emails: Optional[List[str]] = None,
phones: Optional[List[str]] = None,
address: Optional[str] = None) -> str:
"""Build a vCard. Accepts either a single `email` (legacy callers) or
full `emails`/`phones` lists (edit path). The first email is marked
PREF=1. All values are RFC-6350-escaped."""
if not uid:
uid = str(uuid.uuid4())
# Normalize email lists — `email` arg is a convenience for single-email
# creation; `emails` (if given) is authoritative.
email_list = [e.strip() for e in (emails if emails is not None else ([email] if email else [])) if e and e.strip()]
phone_list = [p.strip() for p in (phones or []) if p and p.strip()]
# Try to split name into first/last
parts = name.strip().split()
if len(parts) >= 2:
first = parts[0]
last = " ".join(parts[1:])
else:
first = name
last = ""
# N field is structured (5 components separated by ';') — escape each
# component individually so a comma in the name doesn't split it.
n_field = f"{_vesc(last)};{_vesc(first)};;;"
lines = [
"BEGIN:VCARD",
"VERSION:4.0",
f"UID:{_vesc(uid)}",
f"FN:{_vesc(name)}",
f"N:{n_field}",
]
for i, em in enumerate(email_list):
# First email is the preferred one.
lines.append(f"EMAIL;PREF=1:{_vesc(em)}" if i == 0 else f"EMAIL:{_vesc(em)}")
for ph in phone_list:
lines.append(f"TEL:{_vesc(ph)}")
# Address: stuff the whole human-readable string into the street
# component of ADR. vCard ADR has 7 semicolon-separated components:
# post-office-box;extended-address;street;locality;region;postal-code;country.
addr = (address or "").strip()
if addr:
lines.append(f"ADR:;;{_vesc(addr)};;;;")
lines.append("END:VCARD")
return "\r\n".join(lines) + "\r\n"
# ── In-memory cache ──
_contact_cache = {"contacts": [], "fetched_at": None}
def _abs_url(href: str) -> str:
"""Combine a multistatus <href> (an absolute path like
/user/contacts/x.vcf) with the configured CardDAV server origin so we
get a fully-qualified URL to PUT/DELETE. Absolute hrefs are accepted only
for the configured origin; a cross-origin href is treated as a path on the
configured server so a malicious CardDAV response cannot redirect later
writes/deletes to cloud metadata or another host."""
cfg = _get_carddav_config()
base = _carddav_base_url(cfg)
base_p = urlparse(base)
joined = urljoin(base.rstrip("/") + "/", href or "")
joined_p = urlparse(joined)
if (joined_p.scheme, joined_p.netloc) != (base_p.scheme, base_p.netloc):
joined = urlunparse((base_p.scheme, base_p.netloc, joined_p.path or "/", "", joined_p.query, ""))
return _validate_carddav_url(joined)
# CardDAV REPORT body — pull every card's etag + raw vCard in ONE request,
# alongside the resource href. Lets us map each contact's UID to the real
# server resource path (which is NOT always <uid>.vcf for contacts created
# by other clients).
_ADDRESSBOOK_QUERY = (
'<?xml version="1.0" encoding="utf-8"?>'
'<C:addressbook-query xmlns:D="DAV:" xmlns:C="urn:ietf:params:xml:ns:carddav">'
'<D:prop><D:getetag/><C:address-data/></D:prop>'
'<C:filter/>'
'</C:addressbook-query>'
)
def _fetch_via_report(cfg, auth):
"""Try a CardDAV REPORT addressbook-query — returns contacts WITH an
`href` field, or None if the server doesn't support it / errors."""
from defusedxml import ElementTree as ET
try:
r = httpx.request(
"REPORT", cfg["url"],
content=_ADDRESSBOOK_QUERY.encode("utf-8"),
headers={"Content-Type": "application/xml; charset=utf-8", "Depth": "1"},
auth=auth, timeout=10,
)
if r.status_code not in (207, 200):
return None
root = ET.fromstring(r.text)
ns = {"D": "DAV:", "C": "urn:ietf:params:xml:ns:carddav"}
out = []
for resp in root.findall("D:response", ns):
href_el = resp.find("D:href", ns)
data_el = resp.find(".//C:address-data", ns)
if href_el is None or data_el is None or not (data_el.text or "").strip():
continue
parsed = _parse_vcards(data_el.text)
if not parsed:
continue
c = parsed[0]
c["href"] = href_el.text.strip()
out.append(c)
# If the REPORT parsed to ZERO contacts, don't trust it — some
# CardDAV servers treat an empty <filter/> as "match nothing" and
# return a valid-but-empty 207. Return None so the caller falls
# back to the plain GET (which lists everything). A genuinely empty
# address book just costs one extra GET that also returns nothing.
if not out:
return None
return out
except Exception as e:
logger.warning(f"CardDAV REPORT failed, falling back to GET: {e}")
return None
def _fetch_contacts(force=False):
"""Fetch all contacts. Uses CardDAV when configured, otherwise local JSON."""
if not force and _contact_cache["fetched_at"]:
age = (datetime.utcnow() - _contact_cache["fetched_at"]).total_seconds()
if age < 60:
return _contact_cache["contacts"]
cfg = _get_carddav_config()
if not _carddav_configured(cfg):
contacts = _load_local_contacts()
_contact_cache["contacts"] = contacts
_contact_cache["fetched_at"] = datetime.utcnow()
return contacts
try:
cfg["url"] = _carddav_base_url(cfg)
auth = None
if cfg["username"]:
auth = (cfg["username"], cfg["password"])
# Preferred path: REPORT gives us hrefs for reliable edit/delete.
contacts = _fetch_via_report(cfg, auth)
if contacts is None:
# Fallback: plain GET, concatenated vCards, no hrefs.
r = httpx.get(cfg["url"], auth=auth, timeout=10)
if r.status_code != 200:
logger.warning(f"CardDAV returned {r.status_code}")
return _contact_cache["contacts"]
contacts = _parse_vcards(r.text)
_contact_cache["contacts"] = contacts
_contact_cache["fetched_at"] = datetime.utcnow()
return contacts
except Exception as e:
logger.error(f"Failed to fetch contacts: {e}")
return _contact_cache["contacts"]
def _resolve_resource_url(uid: str) -> str:
"""Map a contact UID to its real CardDAV resource URL. Uses the href
captured during fetch when available (handles contacts whose filename
!= UID); falls back to the <uid>.vcf guess for app-created contacts or
when no href is known."""
def _lookup():
for c in _contact_cache.get("contacts", []):
if c.get("uid") == uid and c.get("href"):
return _abs_url(c["href"])
return None
found = _lookup()
if found:
return found
# Not in cache (or no href) — refresh once and retry before guessing.
try:
_fetch_contacts(force=True)
except Exception:
pass
return _lookup() or _vcard_url(uid)
def _create_contact(name: str, email: str = "", address: str = "", phones: Optional[List[str]] = None) -> bool:
"""Add a new contact via CardDAV or local contacts."""
email = (email or "").strip()
phone_list = [str(p or "").strip() for p in (phones or []) if str(p or "").strip()]
cfg = _get_carddav_config()
if not _carddav_configured(cfg):
contacts = _load_local_contacts()
email_l = email.lower()
for c in contacts:
if email_l and email_l in [e.lower() for e in c.get("emails", [])]:
return True
if phone_list and any(p in (c.get("phones") or []) for p in phone_list):
return True
contacts.append(_normalize_contact({
"name": name,
"emails": [email] if email else [],
"phones": phone_list,
"address": address,
}))
_save_local_contacts(contacts)
return True
contact_uid = str(uuid.uuid4())
vcard = _build_vcard(name, email, contact_uid, address=address, phones=phone_list)
try:
url = _carddav_base_url(cfg) + "/" + contact_uid + ".vcf"
auth = None
if cfg["username"]:
auth = (cfg["username"], cfg["password"])
r = httpx.put(
url,
data=vcard.encode("utf-8"),
headers={"Content-Type": "text/vcard; charset=utf-8"},
auth=auth,
timeout=10,
)
if r.status_code in (200, 201, 204):
# Invalidate cache
_contact_cache["fetched_at"] = None
return True
logger.warning(f"CardDAV PUT returned {r.status_code}: {r.text[:200]}")
return False
except Exception as e:
logger.error(f"Failed to create contact: {e}")
return False
def _vcard_url(uid: str) -> str:
"""The CardDAV resource URL for a given contact UID. The uid is URL-
encoded so a value containing '/', '..' or other path chars can't
escape the collection and target an arbitrary CardDAV resource."""
from urllib.parse import quote
cfg = _get_carddav_config()
return _carddav_base_url(cfg) + "/" + quote(uid, safe="") + ".vcf"
def _import_vcards(text: str) -> Dict:
"""Import a (possibly multi-card) .vcf blob. Each card is PUT to the
CardDAV server PRESERVING its full original content (ADR/ORG/photo/
etc.) — we don't rebuild it, just ensure it has VERSION + UID and
normalize line endings. Returns {imported, failed, total}."""
from urllib.parse import quote
cfg = _get_carddav_config()
if not cfg.get("url"):
parsed = _parse_vcards(text)
contacts = _load_local_contacts()
existing = {
e.lower()
for c in contacts
for e in (c.get("emails") or [])
if e
}
imported = 0
for c in parsed:
emails = [e for e in (c.get("emails") or []) if e]
if emails and any(e.lower() in existing for e in emails):
continue
contacts.append(_normalize_contact(c))
for e in emails:
existing.add(e.lower())
imported += 1
if imported:
_save_local_contacts(contacts)
return {"imported": imported, "failed": 0, "total": len(parsed)}
try:
base_url = _carddav_base_url(cfg)
except ValueError as e:
logger.warning("CardDAV import URL rejected: %s", e)
return {"imported": 0, "failed": 0, "total": 0, "error": str(e)}
auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
# Split into individual cards. re.split drops the BEGIN line, so we
# re-add it. Normalize CRLF.
raw = (text or "").replace("\r\n", "\n").replace("\r", "\n")
blocks = []
for chunk in raw.split("BEGIN:VCARD"):
chunk = chunk.strip()
if not chunk:
continue
# Trim anything after END:VCARD (defensive).
end = chunk.upper().find("END:VCARD")
body = chunk[: end + len("END:VCARD")] if end != -1 else chunk
blocks.append("BEGIN:VCARD\n" + body)
imported = 0
failed = 0
for block in blocks:
# Extract or assign a UID.
m = re.search(r"^UID:(.+)$", block, re.MULTILINE)
uid = (m.group(1).strip() if m else "") or str(uuid.uuid4())
if not m:
# Inject a UID right after the VERSION line (or after BEGIN).
if re.search(r"^VERSION:", block, re.MULTILINE):
block = re.sub(r"(^VERSION:.*$)", r"\1\nUID:" + uid, block, count=1, flags=re.MULTILINE)
else:
block = block.replace("BEGIN:VCARD", f"BEGIN:VCARD\nVERSION:4.0\nUID:{uid}", 1)
elif not re.search(r"^VERSION:", block, re.MULTILINE):
block = block.replace("BEGIN:VCARD", "BEGIN:VCARD\nVERSION:4.0", 1)
vcard = block.replace("\n", "\r\n") + "\r\n"
url = base_url + "/" + quote(uid, safe="") + ".vcf"
try:
r = httpx.put(
url, data=vcard.encode("utf-8"),
headers={"Content-Type": "text/vcard; charset=utf-8"},
auth=auth, timeout=15,
)
if r.status_code in (200, 201, 204):
imported += 1
else:
failed += 1
logger.warning(f"Import PUT {uid} returned {r.status_code}: {r.text[:120]}")
except Exception as e:
failed += 1
logger.error(f"Import PUT {uid} failed: {e}")
if imported:
_contact_cache["fetched_at"] = None
return {"imported": imported, "failed": failed, "total": len(blocks)}
def _import_csv_contacts(text: str) -> Dict:
"""Import contacts from CSV. Supports common headers:
name/full_name/display_name, email/email_address/e-mail, phone/tel.
Falls back to first columns as name,email,phone when no headers exist."""
raw = (text or "").strip()
if not raw:
return {"imported": 0, "failed": 0, "total": 0, "error": "No CSV data found"}
try:
sample = raw[:2048]
dialect = csv.Sniffer().sniff(sample)
except Exception:
dialect = csv.excel
stream = io.StringIO(raw)
try:
has_header = csv.Sniffer().has_header(raw[:2048])
except Exception:
has_header = True
rows = []
if has_header:
reader = csv.DictReader(stream, dialect=dialect)
for row in reader:
lowered = {str(k or "").strip().lower(): (v or "").strip() for k, v in row.items()}
name = (
lowered.get("name") or lowered.get("full name") or lowered.get("full_name")
or lowered.get("display name") or lowered.get("display_name")
or lowered.get("fn") or ""
)
email = (
lowered.get("email") or lowered.get("email address")
or lowered.get("email_address") or lowered.get("e-mail")
or lowered.get("mail") or ""
)
phone = lowered.get("phone") or lowered.get("telephone") or lowered.get("tel") or ""
rows.append((name, email, phone))
else:
stream.seek(0)
reader = csv.reader(stream, dialect=dialect)
for row in reader:
cols = [(c or "").strip() for c in row]
if not any(cols):
continue
rows.append((
cols[0] if len(cols) > 0 else "",
cols[1] if len(cols) > 1 else "",
cols[2] if len(cols) > 2 else "",
))
imported = 0
failed = 0
total = 0
existing_emails = {
e.lower()
for c in _fetch_contacts()
for e in (c.get("emails") or [])
if e
}
for name, email, phone in rows:
email = (email or "").strip()
name = (name or "").strip() or (email.split("@")[0] if email else "")
if not email:
continue
total += 1
if email.lower() in existing_emails:
continue
ok = _create_contact(name, email)
if ok:
imported += 1
existing_emails.add(email.lower())
# If the CSV had a phone number, rewrite the just-created row
# through the richer update path so phone lands in CardDAV too.
if phone:
try:
contacts = _fetch_contacts(force=True)
created = next((c for c in contacts if email.lower() in [e.lower() for e in c.get("emails", [])]), None)
if created and created.get("uid"):
_update_contact(created["uid"], name, [email], [phone])
except Exception:
pass
else:
failed += 1
if imported:
_contact_cache["fetched_at"] = None
return {"imported": imported, "failed": failed, "total": total}
def _contacts_to_vcf(contacts: List[Dict]) -> str:
return "".join(
_build_vcard(
c.get("name") or ((c.get("emails") or [""])[0].split("@")[0] if c.get("emails") else "Contact"),
"",
uid=c.get("uid") or str(uuid.uuid4()),
emails=c.get("emails") or [],
phones=c.get("phones") or [],
)
for c in contacts
)
def _contacts_to_csv(contacts: List[Dict]) -> str:
out = io.StringIO()
writer = csv.writer(out)
writer.writerow(["name", "email", "phone"])
for c in contacts:
emails = c.get("emails") or [""]
phones = c.get("phones") or [""]
max_len = max(len(emails), len(phones), 1)
for i in range(max_len):
writer.writerow([
c.get("name") or "",
emails[i] if i < len(emails) else "",
phones[i] if i < len(phones) else "",
])
return out.getvalue()
def _update_contact(uid: str, name: str, emails: List[str], phones: List[str], address: str = "") -> bool:
"""Rewrite an existing contact via CardDAV or local contacts."""
cfg = _get_carddav_config()
if not _carddav_configured(cfg):
contacts = _load_local_contacts()
found = False
out = []
for c in contacts:
if c.get("uid") == uid:
# Preserve existing address when caller passes "" (only
# updating name/emails/phones, not touching address).
addr = address if address else c.get("address", "")
out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": addr}))
found = True
else:
out.append(c)
if not found:
out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones, "address": address}))
_save_local_contacts(out)
return True
vcard = _build_vcard(name, "", uid=uid, emails=emails, phones=phones, address=address)
# Use the real resource href (handles externally-created contacts whose
# filename != UID); falls back to the <uid>.vcf guess.
try:
url = _resolve_resource_url(uid)
auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
r = httpx.put(
url,
data=vcard.encode("utf-8"),
headers={"Content-Type": "text/vcard; charset=utf-8"},
auth=auth,
timeout=10,
)
if r.status_code in (200, 201, 204):
_contact_cache["fetched_at"] = None
return True
logger.warning(f"CardDAV update PUT returned {r.status_code}: {r.text[:200]}")
return False
except Exception as e:
logger.error(f"Failed to update contact: {e}")
return False
def _delete_contact(uid: str) -> bool:
"""Delete a contact via CardDAV or local contacts."""
cfg = _get_carddav_config()
if not _carddav_configured(cfg):
contacts = _load_local_contacts()
remaining = [c for c in contacts if c.get("uid") != uid]
_save_local_contacts(remaining)
return True
try:
url = _resolve_resource_url(uid)
auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
r = httpx.delete(url, auth=auth, timeout=10)
if r.status_code in (200, 204, 404):
# Invalidate cache so the next fetch sees the server truth.
_contact_cache["fetched_at"] = None
# Verify: force a fresh fetch and check the UID is actually gone.
# A 404 on the guessed URL ({uid}.vcf) can mean the contact
# lives at a different resource URL — the DELETE missed it but
# we'd silently report success. This check catches that.
fresh = _fetch_contacts(force=True)
still_there = any(c.get("uid") == uid for c in fresh)
if still_there:
logger.warning(
f"CardDAV DELETE reported success for {uid} "
f"but UID still present after re-fetch — "
f"resource URL may differ from {redact_url(url)}"
)
return False
if r.status_code == 404:
logger.info(f"CardDAV DELETE 404 for {uid} — already gone")
return True
logger.warning(f"CardDAV DELETE returned {r.status_code}: {r.text[:200]}")
return False
except Exception as e:
logger.error(f"Failed to delete contact: {e}")
return False
# ── Routes ──
def setup_contacts_routes():
router = APIRouter(prefix="/api/contacts", tags=["contacts"])
@router.get("/list")
async def list_contacts(_admin: str = Depends(require_admin)):
"""List all contacts."""
contacts = _fetch_contacts()
return {"contacts": contacts, "count": len(contacts)}
@router.get("/search")
async def search_contacts(q: str = Query(""), _admin: str = Depends(require_admin)):
"""Search contacts by name or email. Returns up to 10 matches."""
contacts = _fetch_contacts()
if not q:
return {"results": []}
q_lower = q.lower()
results = []
for c in contacts:
if q_lower in c["name"].lower():
results.append(c)
continue
for em in c["emails"]:
if q_lower in em.lower():
results.append(c)
break
return {"results": results[:10]}
@router.post("/add")
async def add_contact(data: dict, _admin: str = Depends(require_admin)):
"""Add a new contact."""
name = (data.get("name") or "").strip()
email = (data.get("email") or "").strip()
phone = (data.get("phone") or "").strip()
phones = [str(p or "").strip() for p in (data.get("phones") or []) if str(p or "").strip()]
if phone and phone not in phones:
phones.insert(0, phone)
address = (data.get("address") or "").strip()
if not name and email:
name = email.split("@")[0]
if not name and not email and not phones and not address:
return {"success": False, "error": "Name, email, phone, or address required"}
if not name:
name = email.split("@")[0] if email else (phones[0] if phones else "Contact")
contacts = _fetch_contacts()
for c in contacts:
if email and email.lower() in [e.lower() for e in c.get("emails", [])]:
return {"success": True, "message": "Already exists", "contact": c}
if phones and any(p in (c.get("phones") or []) for p in phones):
return {"success": True, "message": "Already exists", "contact": c}
create_params = inspect.signature(_create_contact).parameters
if "phones" in create_params:
ok = _create_contact(name, email, address, phones=phones)
elif len(create_params) >= 3:
ok = _create_contact(name, email, address)
else:
ok = _create_contact(name, email)
# If a phone was provided, do an immediate update to thread it
# through (the simple _create_contact signature only takes name +
# email + address; phones happen via update).
if ok and phones and "phones" not in create_params:
try:
fresh = _fetch_contacts(force=True)
created = next((c for c in fresh if name == c.get("name") and (not email or email in c.get("emails", []))), None)
if created:
_update_contact(
created["uid"], name,
created.get("emails", []),
phones,
address,
)
except Exception:
pass
return {"success": ok}
@router.post("/import")
async def import_vcf(data: dict, _admin: str = Depends(require_admin)):
"""Import contacts from .vcf or CSV. Body: {"vcf": "..."} or {"csv": "..."}."""
# Coerce defensively: a non-string vcf/text/csv (e.g. a number or list
# in the JSON body) would otherwise reach .strip() and 500 with an
# AttributeError instead of degrading to a clean "no data" response.
text = str(data.get("vcf") or data.get("text") or "")
csv_text = str(data.get("csv") or "")
if text.strip():
if "BEGIN:VCARD" not in text.upper():
return {"success": False, "error": "No vCard data found"}
result = _import_vcards(text)
elif csv_text.strip():
result = _import_csv_contacts(csv_text)
else:
return {"success": False, "error": "No contact data found"}
result["success"] = result.get("imported", 0) > 0
return result
@router.get("/export")
async def export_contacts(
format: str = Query("vcf", pattern="^(vcf|csv)$"),
_admin: str = Depends(require_admin),
):
"""Export all contacts as vCard or CSV."""
contacts = _fetch_contacts(force=True)
if format == "csv":
content = _contacts_to_csv(contacts)
media_type = "text/csv; charset=utf-8"
filename = "odysseus-contacts.csv"
else:
content = _contacts_to_vcf(contacts)
media_type = "text/vcard; charset=utf-8"
filename = "odysseus-contacts.vcf"
return Response(
content=content,
media_type=media_type,
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
)
@router.get("/config")
async def get_config(_admin: str = Depends(require_admin)):
cfg = _get_carddav_config()
# Mask password
if cfg["password"]:
cfg["password"] = "***"
return cfg
@router.put("/config")
async def update_config(data: dict, _admin: str = Depends(require_admin)):
settings = _load_settings()
for key in ("carddav_url", "carddav_username", "carddav_password"):
if key in data:
if key == "carddav_url" and str(data[key] or "").strip():
try:
settings[key] = _validate_carddav_url(data[key])
except ValueError as e:
raise HTTPException(400, str(e))
else:
value = data[key]
if key == "carddav_password" and value:
from src.secret_storage import encrypt
value = encrypt(value)
settings[key] = value
_save_settings(settings)
# Force re-fetch
_contact_cache["fetched_at"] = None
return {"success": True}
@router.delete("/clear")
async def clear_contacts(_admin: str = Depends(require_admin)):
"""Clear all local contacts. If CardDAV is configured, only clears the local fallback cache."""
_save_local_contacts([])
return {"success": True}
# NOTE: the /{uid} routes are declared LAST so the literal paths above
# (/list, /search, /add, /config) win — otherwise PUT /config would
# match PUT /{uid} with uid="config".
@router.put("/{uid}")
async def edit_contact(uid: str, data: dict, _admin: str = Depends(require_admin)):
"""Edit an existing contact — name / emails / phones / address."""
name = (data.get("name") or "").strip()
emails = data.get("emails")
phones = data.get("phones")
if emails is None and data.get("email"):
emails = [data["email"]]
emails = [e.strip() for e in (emails or []) if e and e.strip()]
phones = [p.strip() for p in (phones or []) if p and p.strip()]
address = (data.get("address") or "").strip()
if not name and not emails and not address:
return {"success": False, "error": "Name, email, or address required"}
if not name and emails:
name = emails[0].split("@")[0]
ok = _update_contact(uid, name, emails, phones, address)
return {"success": ok}
@router.delete("/{uid}")
async def delete_contact(uid: str, _admin: str = Depends(require_admin)):
"""Delete a contact by UID."""
if not uid:
return {"success": False, "error": "UID required"}
ok = _delete_contact(uid)
return {"success": ok}
return router

793
routes/contacts_routes.py
View File

@@ -1,790 +1,13 @@
"""
contacts_routes.py
"""Backward-compat shim — canonical location is routes/contacts/contacts_routes.py.
CardDAV contacts integration. Reads from local Radicale, supports
search and adding new contacts.
This module is replaced in ``sys.modules`` by the canonical module object so
that ``import routes.contacts_routes``, ``from routes.contacts_routes import X``,
``importlib.import_module("routes.contacts_routes")``, and string-targeted
monkeypatches all operate on the same object the application actually uses.
"""
import re
import logging
import uuid
import json
import csv
import io
import httpx
from pathlib import Path
from datetime import datetime
from fastapi import APIRouter, Query, Depends, Response
from typing import List, Dict, Optional
import sys as _sys
from src.auth_helpers import require_user
from core.middleware import require_admin
from routes.contacts import contacts_routes as _canonical # noqa: F401
logger = logging.getLogger(__name__)
DATA_DIR = Path(__file__).resolve().parent.parent / "data"
SETTINGS_FILE = DATA_DIR / "settings.json"
LOCAL_CONTACTS_FILE = DATA_DIR / "contacts.json"
def _load_settings():
if SETTINGS_FILE.exists():
return json.loads(SETTINGS_FILE.read_text(encoding="utf-8"))
return {}
def _save_settings(settings):
from core.atomic_io import atomic_write_json
atomic_write_json(str(SETTINGS_FILE), settings, indent=2)
def _get_carddav_config():
import os
settings = _load_settings()
return {
"url": settings.get("carddav_url", os.environ.get("CARDDAV_URL", "")),
"username": settings.get("carddav_username", os.environ.get("CARDDAV_USERNAME", "")),
"password": settings.get("carddav_password", os.environ.get("CARDDAV_PASSWORD", "")),
}
def _carddav_configured(cfg: Optional[Dict] = None) -> bool:
cfg = cfg or _get_carddav_config()
return bool((cfg.get("url") or "").strip())
def _normalize_contact(contact: Dict) -> Dict:
emails = []
for e in contact.get("emails") or ([] if not contact.get("email") else [contact.get("email")]):
e = str(e or "").strip()
if e and e not in emails:
emails.append(e)
phones = []
for p in contact.get("phones") or ([] if not contact.get("phone") else [contact.get("phone")]):
p = str(p or "").strip()
if p and p not in phones:
phones.append(p)
name = str(contact.get("name") or "").strip()
if not name and emails:
name = emails[0].split("@")[0]
return {
"uid": str(contact.get("uid") or uuid.uuid4()),
"name": name,
"emails": emails,
"phones": phones,
}
def _load_local_contacts() -> List[Dict]:
try:
if not LOCAL_CONTACTS_FILE.exists():
return []
data = json.loads(LOCAL_CONTACTS_FILE.read_text(encoding="utf-8"))
rows = data.get("contacts", data) if isinstance(data, dict) else data
return [_normalize_contact(c) for c in (rows or []) if isinstance(c, dict)]
except Exception as e:
logger.error(f"Failed to load local contacts: {e}")
return []
def _save_local_contacts(contacts: List[Dict]) -> None:
from core.atomic_io import atomic_write_json
DATA_DIR.mkdir(parents=True, exist_ok=True)
atomic_write_json(str(LOCAL_CONTACTS_FILE), {"contacts": [_normalize_contact(c) for c in contacts]}, indent=2)
_contact_cache["contacts"] = [_normalize_contact(c) for c in contacts]
_contact_cache["fetched_at"] = datetime.utcnow()
# ── vCard parsing ──
def _vunesc(value: str) -> str:
"""Reverse _vesc() — turn escaped vCard text back into the raw value.
Order matters: handle \\n/\\, /\\; first, backslash-unescape last."""
if not value:
return value
out = []
i = 0
while i < len(value):
ch = value[i]
if ch == "\\" and i + 1 < len(value):
nxt = value[i + 1]
if nxt in ("n", "N"):
out.append("\n")
elif nxt in (",", ";", "\\"):
out.append(nxt)
else:
out.append(nxt)
i += 2
else:
out.append(ch)
i += 1
return "".join(out)
def _parse_vcards(text: str) -> List[Dict]:
"""Parse a stream of vCards into dicts with name, email, phone."""
contacts = []
for block in re.split(r"BEGIN:VCARD", text):
if not block.strip():
continue
contact = {"name": "", "emails": [], "phones": [], "uid": ""}
for line in block.split("\n"):
line = line.strip()
# Strip an optional RFC 6350 group prefix (e.g. "item1.EMAIL;...")
# that Apple Contacts / iCloud / many CardDAV servers emit by
# default — without this the property-name checks below miss those
# lines and silently drop the email / phone. The group token only
# precedes the property name, so it is safe to strip for matching
# and value extraction, and a no-op for non-grouped lines.
name_part = re.sub(r"^[A-Za-z0-9-]+\.", "", line, count=1)
if name_part.startswith("FN:") or name_part.startswith("FN;"):
contact["name"] = _vunesc(name_part.split(":", 1)[1]) if ":" in name_part else ""
elif name_part.startswith("EMAIL"):
# Handle EMAIL:foo@bar OR EMAIL;TYPE=...:foo@bar OR EMAIL;PREF=1:foo@bar
if ":" in name_part:
email_addr = _vunesc(name_part.split(":", 1)[1])
if email_addr and email_addr not in contact["emails"]:
contact["emails"].append(email_addr)
elif name_part.startswith("TEL"):
if ":" in name_part:
phone = _vunesc(name_part.split(":", 1)[1])
if phone and phone not in contact["phones"]:
contact["phones"].append(phone)
elif name_part.startswith("UID:"):
contact["uid"] = _vunesc(name_part[4:])
if contact["name"] or contact["emails"]:
contacts.append(contact)
return contacts
def _vesc(value: str) -> str:
"""Escape a vCard property VALUE per RFC 6350 §3.4: backslash, comma,
semicolon, and newlines. Without this, a name like 'Sekisui House,Ltd'
or any value containing a newline produces a malformed vCard (broken
N/FN fields) or could inject arbitrary properties."""
return (
(value or "")
.replace("\\", "\\\\")
.replace("\n", "\\n")
.replace("\r", "")
.replace(",", "\\,")
.replace(";", "\\;")
)
def _build_vcard(name: str, email: str, uid: Optional[str] = None,
emails: Optional[List[str]] = None,
phones: Optional[List[str]] = None) -> str:
"""Build a vCard. Accepts either a single `email` (legacy callers) or
full `emails`/`phones` lists (edit path). The first email is marked
PREF=1. All values are RFC-6350-escaped."""
if not uid:
uid = str(uuid.uuid4())
# Normalize email lists — `email` arg is a convenience for single-email
# creation; `emails` (if given) is authoritative.
email_list = [e.strip() for e in (emails if emails is not None else ([email] if email else [])) if e and e.strip()]
phone_list = [p.strip() for p in (phones or []) if p and p.strip()]
# Try to split name into first/last
parts = name.strip().split()
if len(parts) >= 2:
first = parts[0]
last = " ".join(parts[1:])
else:
first = name
last = ""
# N field is structured (5 components separated by ';') — escape each
# component individually so a comma in the name doesn't split it.
n_field = f"{_vesc(last)};{_vesc(first)};;;"
lines = [
"BEGIN:VCARD",
"VERSION:4.0",
f"UID:{_vesc(uid)}",
f"FN:{_vesc(name)}",
f"N:{n_field}",
]
for i, em in enumerate(email_list):
# First email is the preferred one.
lines.append(f"EMAIL;PREF=1:{_vesc(em)}" if i == 0 else f"EMAIL:{_vesc(em)}")
for ph in phone_list:
lines.append(f"TEL:{_vesc(ph)}")
lines.append("END:VCARD")
return "\r\n".join(lines) + "\r\n"
# ── In-memory cache ──
_contact_cache = {"contacts": [], "fetched_at": None}
def _abs_url(href: str) -> str:
"""Combine a multistatus <href> (an absolute path like
/user/contacts/x.vcf) with the configured CardDAV server origin so we
get a fully-qualified URL to PUT/DELETE. If href is already absolute
(http...), return it as-is."""
from urllib.parse import urlparse, urlunparse
if href.startswith("http://") or href.startswith("https://"):
return href
cfg = _get_carddav_config()
p = urlparse(cfg["url"])
return urlunparse((p.scheme, p.netloc, href, "", "", ""))
# CardDAV REPORT body — pull every card's etag + raw vCard in ONE request,
# alongside the resource href. Lets us map each contact's UID to the real
# server resource path (which is NOT always <uid>.vcf for contacts created
# by other clients).
_ADDRESSBOOK_QUERY = (
'<?xml version="1.0" encoding="utf-8"?>'
'<C:addressbook-query xmlns:D="DAV:" xmlns:C="urn:ietf:params:xml:ns:carddav">'
'<D:prop><D:getetag/><C:address-data/></D:prop>'
'<C:filter/>'
'</C:addressbook-query>'
)
def _fetch_via_report(cfg, auth):
"""Try a CardDAV REPORT addressbook-query — returns contacts WITH an
`href` field, or None if the server doesn't support it / errors."""
from defusedxml import ElementTree as ET
try:
r = httpx.request(
"REPORT", cfg["url"],
content=_ADDRESSBOOK_QUERY.encode("utf-8"),
headers={"Content-Type": "application/xml; charset=utf-8", "Depth": "1"},
auth=auth, timeout=10,
)
if r.status_code not in (207, 200):
return None
root = ET.fromstring(r.text)
ns = {"D": "DAV:", "C": "urn:ietf:params:xml:ns:carddav"}
out = []
for resp in root.findall("D:response", ns):
href_el = resp.find("D:href", ns)
data_el = resp.find(".//C:address-data", ns)
if href_el is None or data_el is None or not (data_el.text or "").strip():
continue
parsed = _parse_vcards(data_el.text)
if not parsed:
continue
c = parsed[0]
c["href"] = href_el.text.strip()
out.append(c)
# If the REPORT parsed to ZERO contacts, don't trust it — some
# CardDAV servers treat an empty <filter/> as "match nothing" and
# return a valid-but-empty 207. Return None so the caller falls
# back to the plain GET (which lists everything). A genuinely empty
# address book just costs one extra GET that also returns nothing.
if not out:
return None
return out
except Exception as e:
logger.warning(f"CardDAV REPORT failed, falling back to GET: {e}")
return None
def _fetch_contacts(force=False):
"""Fetch all contacts. Uses CardDAV when configured, otherwise local JSON."""
if not force and _contact_cache["fetched_at"]:
age = (datetime.utcnow() - _contact_cache["fetched_at"]).total_seconds()
if age < 60:
return _contact_cache["contacts"]
cfg = _get_carddav_config()
if not _carddav_configured(cfg):
contacts = _load_local_contacts()
_contact_cache["contacts"] = contacts
_contact_cache["fetched_at"] = datetime.utcnow()
return contacts
try:
auth = None
if cfg["username"]:
auth = (cfg["username"], cfg["password"])
# Preferred path: REPORT gives us hrefs for reliable edit/delete.
contacts = _fetch_via_report(cfg, auth)
if contacts is None:
# Fallback: plain GET, concatenated vCards, no hrefs.
r = httpx.get(cfg["url"], auth=auth, timeout=10)
if r.status_code != 200:
logger.warning(f"CardDAV returned {r.status_code}")
return _contact_cache["contacts"]
contacts = _parse_vcards(r.text)
_contact_cache["contacts"] = contacts
_contact_cache["fetched_at"] = datetime.utcnow()
return contacts
except Exception as e:
logger.error(f"Failed to fetch contacts: {e}")
return _contact_cache["contacts"]
def _resolve_resource_url(uid: str) -> str:
"""Map a contact UID to its real CardDAV resource URL. Uses the href
captured during fetch when available (handles contacts whose filename
!= UID); falls back to the <uid>.vcf guess for app-created contacts or
when no href is known."""
def _lookup():
for c in _contact_cache.get("contacts", []):
if c.get("uid") == uid and c.get("href"):
return _abs_url(c["href"])
return None
found = _lookup()
if found:
return found
# Not in cache (or no href) — refresh once and retry before guessing.
try:
_fetch_contacts(force=True)
except Exception:
pass
return _lookup() or _vcard_url(uid)
def _create_contact(name: str, email: str) -> bool:
"""Add a new contact via CardDAV or local contacts."""
cfg = _get_carddav_config()
if not _carddav_configured(cfg):
contacts = _load_local_contacts()
email_l = (email or "").strip().lower()
for c in contacts:
if email_l and email_l in [e.lower() for e in c.get("emails", [])]:
return True
contacts.append(_normalize_contact({"name": name, "emails": [email]}))
_save_local_contacts(contacts)
return True
contact_uid = str(uuid.uuid4())
vcard = _build_vcard(name, email, contact_uid)
url = cfg["url"].rstrip("/") + "/" + contact_uid + ".vcf"
try:
auth = None
if cfg["username"]:
auth = (cfg["username"], cfg["password"])
r = httpx.put(
url,
data=vcard.encode("utf-8"),
headers={"Content-Type": "text/vcard; charset=utf-8"},
auth=auth,
timeout=10,
)
if r.status_code in (200, 201, 204):
# Invalidate cache
_contact_cache["fetched_at"] = None
return True
logger.warning(f"CardDAV PUT returned {r.status_code}: {r.text[:200]}")
return False
except Exception as e:
logger.error(f"Failed to create contact: {e}")
return False
def _vcard_url(uid: str) -> str:
"""The CardDAV resource URL for a given contact UID. The uid is URL-
encoded so a value containing '/', '..' or other path chars can't
escape the collection and target an arbitrary CardDAV resource."""
from urllib.parse import quote
cfg = _get_carddav_config()
return cfg["url"].rstrip("/") + "/" + quote(uid, safe="") + ".vcf"
def _import_vcards(text: str) -> Dict:
"""Import a (possibly multi-card) .vcf blob. Each card is PUT to the
CardDAV server PRESERVING its full original content (ADR/ORG/photo/
etc.) — we don't rebuild it, just ensure it has VERSION + UID and
normalize line endings. Returns {imported, failed, total}."""
from urllib.parse import quote
cfg = _get_carddav_config()
if not cfg.get("url"):
parsed = _parse_vcards(text)
contacts = _load_local_contacts()
existing = {
e.lower()
for c in contacts
for e in (c.get("emails") or [])
if e
}
imported = 0
for c in parsed:
emails = [e for e in (c.get("emails") or []) if e]
if emails and any(e.lower() in existing for e in emails):
continue
contacts.append(_normalize_contact(c))
for e in emails:
existing.add(e.lower())
imported += 1
if imported:
_save_local_contacts(contacts)
return {"imported": imported, "failed": 0, "total": len(parsed)}
auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
# Split into individual cards. re.split drops the BEGIN line, so we
# re-add it. Normalize CRLF.
raw = (text or "").replace("\r\n", "\n").replace("\r", "\n")
blocks = []
for chunk in raw.split("BEGIN:VCARD"):
chunk = chunk.strip()
if not chunk:
continue
# Trim anything after END:VCARD (defensive).
end = chunk.upper().find("END:VCARD")
body = chunk[: end + len("END:VCARD")] if end != -1 else chunk
blocks.append("BEGIN:VCARD\n" + body)
imported = 0
failed = 0
for block in blocks:
# Extract or assign a UID.
m = re.search(r"^UID:(.+)$", block, re.MULTILINE)
uid = (m.group(1).strip() if m else "") or str(uuid.uuid4())
if not m:
# Inject a UID right after the VERSION line (or after BEGIN).
if re.search(r"^VERSION:", block, re.MULTILINE):
block = re.sub(r"(^VERSION:.*$)", r"\1\nUID:" + uid, block, count=1, flags=re.MULTILINE)
else:
block = block.replace("BEGIN:VCARD", f"BEGIN:VCARD\nVERSION:4.0\nUID:{uid}", 1)
elif not re.search(r"^VERSION:", block, re.MULTILINE):
block = block.replace("BEGIN:VCARD", "BEGIN:VCARD\nVERSION:4.0", 1)
vcard = block.replace("\n", "\r\n") + "\r\n"
url = cfg["url"].rstrip("/") + "/" + quote(uid, safe="") + ".vcf"
try:
r = httpx.put(
url, data=vcard.encode("utf-8"),
headers={"Content-Type": "text/vcard; charset=utf-8"},
auth=auth, timeout=15,
)
if r.status_code in (200, 201, 204):
imported += 1
else:
failed += 1
logger.warning(f"Import PUT {uid} returned {r.status_code}: {r.text[:120]}")
except Exception as e:
failed += 1
logger.error(f"Import PUT {uid} failed: {e}")
if imported:
_contact_cache["fetched_at"] = None
return {"imported": imported, "failed": failed, "total": len(blocks)}
def _import_csv_contacts(text: str) -> Dict:
"""Import contacts from CSV. Supports common headers:
name/full_name/display_name, email/email_address/e-mail, phone/tel.
Falls back to first columns as name,email,phone when no headers exist."""
raw = (text or "").strip()
if not raw:
return {"imported": 0, "failed": 0, "total": 0, "error": "No CSV data found"}
try:
sample = raw[:2048]
dialect = csv.Sniffer().sniff(sample)
except Exception:
dialect = csv.excel
stream = io.StringIO(raw)
try:
has_header = csv.Sniffer().has_header(raw[:2048])
except Exception:
has_header = True
rows = []
if has_header:
reader = csv.DictReader(stream, dialect=dialect)
for row in reader:
lowered = {str(k or "").strip().lower(): (v or "").strip() for k, v in row.items()}
name = (
lowered.get("name") or lowered.get("full name") or lowered.get("full_name")
or lowered.get("display name") or lowered.get("display_name")
or lowered.get("fn") or ""
)
email = (
lowered.get("email") or lowered.get("email address")
or lowered.get("email_address") or lowered.get("e-mail")
or lowered.get("mail") or ""
)
phone = lowered.get("phone") or lowered.get("telephone") or lowered.get("tel") or ""
rows.append((name, email, phone))
else:
stream.seek(0)
reader = csv.reader(stream, dialect=dialect)
for row in reader:
cols = [(c or "").strip() for c in row]
if not any(cols):
continue
rows.append((
cols[0] if len(cols) > 0 else "",
cols[1] if len(cols) > 1 else "",
cols[2] if len(cols) > 2 else "",
))
imported = 0
failed = 0
total = 0
existing_emails = {
e.lower()
for c in _fetch_contacts()
for e in (c.get("emails") or [])
if e
}
for name, email, phone in rows:
email = (email or "").strip()
name = (name or "").strip() or (email.split("@")[0] if email else "")
if not email:
continue
total += 1
if email.lower() in existing_emails:
continue
ok = _create_contact(name, email)
if ok:
imported += 1
existing_emails.add(email.lower())
# If the CSV had a phone number, rewrite the just-created row
# through the richer update path so phone lands in CardDAV too.
if phone:
try:
contacts = _fetch_contacts(force=True)
created = next((c for c in contacts if email.lower() in [e.lower() for e in c.get("emails", [])]), None)
if created and created.get("uid"):
_update_contact(created["uid"], name, [email], [phone])
except Exception:
pass
else:
failed += 1
if imported:
_contact_cache["fetched_at"] = None
return {"imported": imported, "failed": failed, "total": total}
def _contacts_to_vcf(contacts: List[Dict]) -> str:
return "".join(
_build_vcard(
c.get("name") or ((c.get("emails") or [""])[0].split("@")[0] if c.get("emails") else "Contact"),
"",
uid=c.get("uid") or str(uuid.uuid4()),
emails=c.get("emails") or [],
phones=c.get("phones") or [],
)
for c in contacts
)
def _contacts_to_csv(contacts: List[Dict]) -> str:
out = io.StringIO()
writer = csv.writer(out)
writer.writerow(["name", "email", "phone"])
for c in contacts:
emails = c.get("emails") or [""]
phones = c.get("phones") or [""]
max_len = max(len(emails), len(phones), 1)
for i in range(max_len):
writer.writerow([
c.get("name") or "",
emails[i] if i < len(emails) else "",
phones[i] if i < len(phones) else "",
])
return out.getvalue()
def _update_contact(uid: str, name: str, emails: List[str], phones: List[str]) -> bool:
"""Rewrite an existing contact via CardDAV or local contacts."""
cfg = _get_carddav_config()
if not _carddav_configured(cfg):
contacts = _load_local_contacts()
found = False
out = []
for c in contacts:
if c.get("uid") == uid:
out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones}))
found = True
else:
out.append(c)
if not found:
out.append(_normalize_contact({"uid": uid, "name": name, "emails": emails, "phones": phones}))
_save_local_contacts(out)
return True
vcard = _build_vcard(name, "", uid=uid, emails=emails, phones=phones)
# Use the real resource href (handles externally-created contacts whose
# filename != UID); falls back to the <uid>.vcf guess.
url = _resolve_resource_url(uid)
try:
auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
r = httpx.put(
url,
data=vcard.encode("utf-8"),
headers={"Content-Type": "text/vcard; charset=utf-8"},
auth=auth,
timeout=10,
)
if r.status_code in (200, 201, 204):
_contact_cache["fetched_at"] = None
return True
logger.warning(f"CardDAV update PUT returned {r.status_code}: {r.text[:200]}")
return False
except Exception as e:
logger.error(f"Failed to update contact: {e}")
return False
def _delete_contact(uid: str) -> bool:
"""Delete a contact via CardDAV or local contacts."""
cfg = _get_carddav_config()
if not _carddav_configured(cfg):
contacts = _load_local_contacts()
remaining = [c for c in contacts if c.get("uid") != uid]
_save_local_contacts(remaining)
return True
url = _resolve_resource_url(uid)
try:
auth = (cfg["username"], cfg["password"]) if cfg["username"] else None
r = httpx.delete(url, auth=auth, timeout=10)
if r.status_code in (200, 204):
_contact_cache["fetched_at"] = None
return True
if r.status_code == 404:
# Resource not found at the resolved URL. With href resolution
# this should be rare (genuinely already deleted). Invalidate
# the cache and report success so the UI doesn't keep a ghost.
logger.info(f"CardDAV DELETE 404 for {uid} — treating as already gone")
_contact_cache["fetched_at"] = None
return True
logger.warning(f"CardDAV DELETE returned {r.status_code}: {r.text[:200]}")
return False
except Exception as e:
logger.error(f"Failed to delete contact: {e}")
return False
# ── Routes ──
def setup_contacts_routes():
router = APIRouter(prefix="/api/contacts", tags=["contacts"])
@router.get("/list")
async def list_contacts(_admin: str = Depends(require_admin)):
"""List all contacts."""
contacts = _fetch_contacts()
return {"contacts": contacts, "count": len(contacts)}
@router.get("/search")
async def search_contacts(q: str = Query(""), _admin: str = Depends(require_admin)):
"""Search contacts by name or email. Returns up to 10 matches."""
contacts = _fetch_contacts()
if not q:
return {"results": []}
q_lower = q.lower()
results = []
for c in contacts:
if q_lower in c["name"].lower():
results.append(c)
continue
for em in c["emails"]:
if q_lower in em.lower():
results.append(c)
break
return {"results": results[:10]}
@router.post("/add")
async def add_contact(data: dict, _admin: str = Depends(require_admin)):
"""Add a new contact."""
name = (data.get("name") or "").strip()
email = (data.get("email") or "").strip()
if not email:
return {"success": False, "error": "Email required"}
# Check if already exists
contacts = _fetch_contacts()
for c in contacts:
if email.lower() in [e.lower() for e in c["emails"]]:
return {"success": True, "message": "Already exists", "contact": c}
if not name:
name = email.split("@")[0]
ok = _create_contact(name, email)
return {"success": ok}
@router.post("/import")
async def import_vcf(data: dict, _admin: str = Depends(require_admin)):
"""Import contacts from .vcf or CSV. Body: {"vcf": "..."} or {"csv": "..."}."""
text = data.get("vcf") or data.get("text") or ""
csv_text = data.get("csv") or ""
if text.strip():
if "BEGIN:VCARD" not in text.upper():
return {"success": False, "error": "No vCard data found"}
result = _import_vcards(text)
elif csv_text.strip():
result = _import_csv_contacts(csv_text)
else:
return {"success": False, "error": "No contact data found"}
result["success"] = result.get("imported", 0) > 0
return result
@router.get("/export")
async def export_contacts(
format: str = Query("vcf", pattern="^(vcf|csv)$"),
_admin: str = Depends(require_admin),
):
"""Export all contacts as vCard or CSV."""
contacts = _fetch_contacts(force=True)
if format == "csv":
content = _contacts_to_csv(contacts)
media_type = "text/csv; charset=utf-8"
filename = "odysseus-contacts.csv"
else:
content = _contacts_to_vcf(contacts)
media_type = "text/vcard; charset=utf-8"
filename = "odysseus-contacts.vcf"
return Response(
content=content,
media_type=media_type,
headers={"Content-Disposition": f'attachment; filename="{filename}"'},
)
@router.get("/config")
async def get_config(_admin: str = Depends(require_admin)):
cfg = _get_carddav_config()
# Mask password
if cfg["password"]:
cfg["password"] = "***"
return cfg
@router.put("/config")
async def update_config(data: dict, _admin: str = Depends(require_admin)):
settings = _load_settings()
for key in ("carddav_url", "carddav_username", "carddav_password"):
if key in data:
settings[key] = data[key]
_save_settings(settings)
# Force re-fetch
_contact_cache["fetched_at"] = None
return {"success": True}
@router.delete("/clear")
async def clear_contacts(_admin: str = Depends(require_admin)):
"""Clear all local contacts. If CardDAV is configured, only clears the local fallback cache."""
_save_local_contacts([])
return {"success": True}
# NOTE: the /{uid} routes are declared LAST so the literal paths above
# (/list, /search, /add, /config) win — otherwise PUT /config would
# match PUT /{uid} with uid="config".
@router.put("/{uid}")
async def edit_contact(uid: str, data: dict, _admin: str = Depends(require_admin)):
"""Edit an existing contact — name / emails / phones."""
name = (data.get("name") or "").strip()
emails = data.get("emails")
phones = data.get("phones")
if emails is None and data.get("email"):
emails = [data["email"]]
emails = [e.strip() for e in (emails or []) if e and e.strip()]
phones = [p.strip() for p in (phones or []) if p and p.strip()]
if not name and not emails:
return {"success": False, "error": "Name or email required"}
if not name and emails:
name = emails[0].split("@")[0]
ok = _update_contact(uid, name, emails, phones)
return {"success": ok}
@router.delete("/{uid}")
async def delete_contact(uid: str, _admin: str = Depends(require_admin)):
"""Delete a contact by UID."""
if not uid:
return {"success": False, "error": "UID required"}
ok = _delete_contact(uid)
return {"success": ok}
return router
_sys.modules[__name__] = _canonical

593
routes/cookbook_helpers.py
View File

@@ -1,16 +1,21 @@
"""cookbook_helpers.py — validators + small helpers shared by the cookbook routes.
Extracted from cookbook_routes.py; the routes module imports the symbols it needs."""
import json
import logging
import ntpath
import os
import posixpath
import re
import shlex
from pathlib import Path
from fastapi import HTTPException
from pydantic import BaseModel
from routes._validators import validate_remote_host, validate_ssh_port
from core.platform_compat import _ssh_exec_argv
logger = logging.getLogger(__name__)
@@ -28,20 +33,24 @@ _LOCAL_MODEL_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")
_OLLAMA_MODEL_ID_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/-]{0,200}$")
# Include pattern is a glob: allow typical safe glyphs only.
_INCLUDE_RE = re.compile(r"^[A-Za-z0-9._\-*?/\[\]]+$")
# Remote host: user@host (optionally with :port-free hostname parts).
_REMOTE_HOST_RE = re.compile(r"^[A-Za-z0-9._-]+@[A-Za-z0-9._-]+$")
# HF tokens and API tokens are url-safe base64-like.
_TOKEN_RE = re.compile(r"^[A-Za-z0-9._~+/=-]+$")
# Session IDs we mint look like "cookbook-deadbeef" or "serve-deadbeef".
# Anything beyond plain alphanumerics + dash + underscore could break out
# of the shell/PowerShell contexts the value lands in.
_SESSION_ID_RE = re.compile(r"^[A-Za-z0-9_-]{1,64}$")
_SSH_PORT_RE = re.compile(r"^\d{1,5}$")
_GPU_LIST_RE = re.compile(r"^\d+(?:,\d+)*$")
# A download target directory. Absolute or ~-relative path; safe path glyphs
# only (no quotes, shell metacharacters, or spaces) since it lands in a shell
# command. A leading ~ is expanded to $HOME at command-build time.
_LOCAL_DIR_RE = re.compile(r"^~?/[A-Za-z0-9._/-]*$|^~$")
# only (no quotes or shell metacharacters). Spaces are allowed because command
# builders pass the value through quoted shell/Python contexts. The character
# class uses ``\w`` — Unicode word characters under Python 3's default str
# matching — so non-ASCII folder names pass validation too: Cyrillic, accented
# Latin, CJK, e.g. ``/Volumes/Модели`` or ``D:\AI Models\Модели``. This stays
# shell-safe: none of ``; & | ` $ '' "" () {}`` newlines etc. are in ``[\w. -]``,
# so injection vectors remain rejected. A leading ~ is expanded to $HOME at
# command-build time. (Drive letters stay ASCII: ``[A-Za-z]:``.)
_LOCAL_DIR_RE = re.compile(r"^~?(?:/[\w. -]*)+$|^~$")
_WINDOWS_LOCAL_DIR_RE = re.compile(r"^[A-Za-z]:[\\/](?:[\w. -]+(?:[\\/][\w. -]+)*[\\/]?)?$")
_WINDOWS_DRIVE_PATH_RE = re.compile(r"^[A-Za-z]:[\\/]")
@@ -75,14 +84,6 @@ def _validate_include(v: str | None) -> str | None:
return v
def _validate_remote_host(v: str | None) -> str | None:
if v is None or v == "":
return None
if not _REMOTE_HOST_RE.match(v):
raise HTTPException(400, "Invalid remote_host — must be user@host, no SSH option syntax")
return v
def _validate_token(v: str | None) -> str | None:
if v is None or v == "":
return None
@@ -91,26 +92,43 @@ def _validate_token(v: str | None) -> str | None:
return v
def load_stored_hf_token(*, state_path: Path | str | None = None) -> str:
"""Return the decrypted HF token from cookbook_state.json, else env fallback."""
path = Path(state_path) if state_path else Path(os.environ.get("DATA_DIR", "data")) / "cookbook_state.json"
token = ""
if path.exists():
try:
state = json.loads(path.read_text(encoding="utf-8"))
env = state.get("env") if isinstance(state, dict) else {}
if isinstance(env, dict) and env.get("hfToken"):
from src.secret_storage import decrypt
token = decrypt(env.get("hfToken") or "")
except Exception:
token = ""
if not token:
token = (os.environ.get("HF_TOKEN") or os.environ.get("HUGGING_FACE_HUB_TOKEN") or "").strip()
return token
def _validate_local_dir(v: str | None) -> str | None:
if v is None or v == "":
return None
if len(v) >= 2 and v[0] == v[-1] and v[0] in {"'", '"'}:
v = v[1:-1]
v = v.rstrip("/") or "/"
if not _LOCAL_DIR_RE.match(v):
raise HTTPException(400, "Invalid local_dir — must be an absolute or ~ path with no spaces or shell metacharacters")
if not (_LOCAL_DIR_RE.match(v) or _WINDOWS_LOCAL_DIR_RE.match(v)):
raise HTTPException(400, "Invalid local_dir — must be an absolute or ~ path with no shell metacharacters")
# Reject path segments that start with '-' (option injection). '-' is in the
# allowlist, so a dir like ``/models/-rf`` or ``D:\models\-rf`` could be read
# as a CLI flag by hf/etc. — and quoting does NOT stop a value from being
# parsed as an option. This is the one residual that command-build-time
# quoting can't cover, so the guard lives here, keeping the safety wholly
# inside the validator rather than relying on consumers.
if any(seg.startswith("-") for seg in re.split(r"[\\/]", v) if seg):
raise HTTPException(400, "Invalid local_dir — path segments cannot start with '-'")
return v
def _validate_ssh_port(v: str | None) -> str | None:
if v is None or v == "":
return None
if not _SSH_PORT_RE.fullmatch(str(v)):
raise HTTPException(400, "Invalid ssh_port")
port = int(v)
if port < 1 or port > 65535:
raise HTTPException(400, "Invalid ssh_port")
return str(port)
def _validate_gpus(v: str | None) -> str | None:
if v is None or v == "":
return None
@@ -122,7 +140,7 @@ def _validate_gpus(v: str | None) -> str | None:
def _shell_path(p: str) -> str:
"""Render a validated path for a double-quoted shell context, expanding a
leading ~ to $HOME (single quotes wouldn't expand it). Safe because
_validate_local_dir already restricts the charset."""
_validate_local_dir already rejects quotes and shell metacharacters."""
if p == "~":
return '"$HOME"'
if p.startswith("~/"):
@@ -195,6 +213,20 @@ def _pip_install_attempt(pip_cmd: str) -> str:
)
def _pip_command(python_cmd: str) -> str:
"""Return a pip command for either a pip executable or a Python executable."""
cmd = python_cmd.strip()
if " -m pip" in cmd or cmd in {"pip", "pip3"}:
return python_cmd
if cmd in {"python", "python3", "python.exe"} or cmd.endswith(("/python", "/python3", "\\python.exe")):
return f"{python_cmd} -m pip"
return python_cmd
def _pip_break_system_packages_check(pip_cmd: str) -> str:
return f"{pip_cmd} install --help 2>/dev/null | grep -q -- --break-system-packages"
def _pip_install_fallback_chain(package: str, *, python_cmd: str = "python3 -m pip", upgrade: bool = False) -> str:
"""Build a bash pip install fallback chain that surfaces errors.
@@ -206,33 +238,44 @@ def _pip_install_fallback_chain(package: str, *, python_cmd: str = "python3 -m p
exit code is preserved (no ``| tail`` masking) and the last 5 lines of
pip output appear in the Cookbook log on failure.
"""
from core.platform_compat import IS_WINDOWS
upgrade_flag = " -U" if upgrade else ""
# Shell-quote the package spec: an extras spec like ``llama-cpp-python[server]``
# contains brackets that bash would treat as a glob, so it must be quoted
# before being embedded in the install command. Plain names (e.g.
# ``huggingface_hub``) are returned unchanged by ``shlex.quote``.
pkg = shlex.quote(package)
base = _pip_install_attempt(f"{python_cmd} install -q{upgrade_flag} {pkg}")
user = _pip_install_attempt(f"{python_cmd} install --user --break-system-packages -q{upgrade_flag} {pkg}")
# llama-cpp-python source builds are brittle on older distro pip/packaging
# stacks (common on WSL images). Prefer the prebuilt wheel index whenever
# this package is requested so dependency-install tasks are reliable.
if "llama-cpp-python" in package:
pkg += " --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu"
pip_cmd = _pip_command(python_cmd)
base = _pip_install_attempt(f"{pip_cmd} install -q{upgrade_flag} {pkg}")
user = _pip_install_attempt(f"{pip_cmd} install --user -q{upgrade_flag} {pkg}")
user_break_system = _pip_install_attempt(f"{pip_cmd} install --user --break-system-packages -q{upgrade_flag} {pkg}")
user_fallback = f"( {user} || {{ {_pip_break_system_packages_check(pip_cmd)} && {user_break_system}; }} )"
# Derive the python executable for the venv detection check.
# Must use the same interpreter that pip belongs to; hardcoding
# python3 breaks when pip lives in a venv that only has "python".
if " -m pip" in python_cmd:
python_exe = python_cmd.replace(" -m pip", "")
elif python_cmd.strip() == "pip":
if " -m pip" in pip_cmd:
python_exe = pip_cmd.replace(" -m pip", "")
elif pip_cmd.strip() == "pip":
python_exe = "python"
elif python_cmd.strip() == "pip3":
elif pip_cmd.strip() == "pip3":
python_exe = "python3"
else:
python_exe = "python3"
venv_check = f'{python_exe} -c "import sys; sys.exit(0 if sys.prefix != sys.base_prefix else 1)"'
# Negated: `! venv_check` succeeds (exit 0) when NOT in a venv → `&&` tries
# --user. When IN a venv `! venv_check` fails → `&&` skips --user and the
# Negated: `! venv_check` succeeds (exit 0) when NOT in a venv -> `&&` tries
# --user. When IN a venv `! venv_check` fails -> `&&` skips --user and the
# group exits non-zero, propagating the base-install failure instead of
# masking it as success (the `|| { venv_check || … }` shape from #903
# swallowed the exit code because venv_check's exit-0 became the group's
# result).
return f"{base} || {{ ! {venv_check} && {user}; }}"
# result). `--break-system-packages` is only attempted when the active pip
# supports it; older pip versions abort with "no such option" otherwise.
return f"{base} || {{ ! {venv_check} && {user_fallback}; }}"
def _venv_safe_local_pip_install_cmd(cmd: str, *, local: bool, in_venv: bool) -> str:
@@ -263,6 +306,55 @@ def _venv_safe_local_pip_install_cmd(cmd: str, *, local: bool, in_venv: bool) ->
return shlex.join(stripped)
def _pip_install_command_without_break_system_packages(cmd: str) -> str:
try:
parts = shlex.split(cmd)
except ValueError:
return cmd
stripped = [part for part in parts if part != "--break-system-packages"]
return shlex.join(stripped)
def _pip_install_help_check_from_cmd(cmd: str) -> str | None:
try:
parts = shlex.split(cmd)
except ValueError:
return None
try:
install_index = parts.index("install")
except ValueError:
return None
if install_index <= 0:
return None
pip_prefix = parts[:install_index]
return f"{shlex.join(pip_prefix + ['install', '--help'])} 2>/dev/null | grep -q -- --break-system-packages"
def _append_pip_install_runner_lines(runner_lines: list[str], cmd: str) -> None:
"""Append a pip install command, guarding --break-system-packages support.
The Dependencies UI may submit ``python3 -m pip install --user
--break-system-packages ...`` for non-venv installs. That flag is useful on
PEP-668-locked distros, but older pip (including Ubuntu 22.04's apt pip in
the NVIDIA CUDA base image) aborts with "no such option". Branch at runner
time so stale browser JS and remote targets are handled by the server too.
"""
if "--break-system-packages" not in (cmd or ""):
runner_lines.append(cmd)
return
help_check = _pip_install_help_check_from_cmd(cmd)
without_break = _pip_install_command_without_break_system_packages(cmd)
if not help_check or without_break == cmd:
runner_lines.append(cmd)
return
runner_lines.append(f"if {help_check}; then")
runner_lines.append(f" {cmd}")
runner_lines.append("else")
runner_lines.append(' echo "[odysseus] pip does not support --break-system-packages; installing without it."')
runner_lines.append(f" {without_break}")
runner_lines.append("fi")
def _user_shell_path_bootstrap() -> list[str]:
return [
'ODYSSEUS_USER_SHELL="${SHELL:-}"',
@@ -270,12 +362,20 @@ def _user_shell_path_bootstrap() -> list[str]:
' ODYSSEUS_USER_PATH="$("$ODYSSEUS_USER_SHELL" -ic \'printf "__ODYSSEUS_PATH__%s\\n" "$PATH"\' 2>/dev/null | sed -n \'s/^__ODYSSEUS_PATH__//p\' | tail -n 1 || true)"',
' if [ -n "$ODYSSEUS_USER_PATH" ]; then export PATH="$ODYSSEUS_USER_PATH:$PATH"; fi',
'fi',
'command -v python3 >/dev/null 2>&1 || python3() { python "$@"; }',
# Windows can expose python3 as a Microsoft Store App Execution Alias
# under WindowsApps. Git Bash sees that stub as present, but it exits
# before running Python. A Windows venv usually has python.exe, not
# python3.exe, so treat a missing or WindowsApps python3 as absent.
'_odys_py3="$(command -v python3 2>/dev/null || true)"',
'case "$_odys_py3" in ""|*[Ww]indows[Aa]pps*) python3() { python "$@"; } ;; esac',
'command -v python >/dev/null 2>&1 || python() { python3 "$@"; }',
]
def _cached_model_scan_script(model_dirs: list[str] | None = None) -> str:
"""Build the standalone Python scanner used by /api/model/cached."""
def _cached_model_scan_script(model_dirs: list[str] | None = None, add_hf_cache: str | None = None) -> str:
"""Build the standalone Python scanner used by /api/model/cached.
Allows for an additional HuggingFace cache path to be scanned (i.e. Windows HF cache for local WSL envs.)
"""
lines = [
"import json, os, re, shutil, subprocess, urllib.request",
"models = []",
@@ -306,6 +406,7 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None) -> str:
" for root, dirs, fns in safe_walk(base):",
" for fn in sorted(fns):",
" if not fn.lower().endswith('.gguf'): continue",
" if fn.startswith('._'): continue # macOS AppleDouble sidecar, not a real GGUF",
" fp = os.path.join(root, fn)",
" try: size = os.path.getsize(fp)",
" except Exception: size = 0",
@@ -338,6 +439,30 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None) -> str:
" if f.is_file(): nf += 1; sz += f.stat().st_size",
" if f.name.endswith('.incomplete'): ic = True",
" snap = os.path.join(cache, d, 'snapshots')",
" def snapshot_size():",
" total, count, incomplete = 0, 0, False",
" seen_real = set()",
" for sd in os.listdir(snap):",
" sf = os.path.join(snap, sd)",
" if not os.path.isdir(sf): continue",
" for root, dirs, fns in safe_walk(sf):",
" for fn in fns:",
" fp = os.path.join(root, fn)",
" if fn.endswith('.incomplete'): incomplete = True",
" try:",
" real = os.path.realpath(fp)",
" if real in seen_real: continue",
" seen_real.add(real)",
" total += os.path.getsize(real)",
" count += 1",
" except Exception:",
" pass",
" return total, count, incomplete",
" # Some HF caches (macOS/MLX/Xet-style) keep blobs elsewhere or expose",
" # snapshot symlinks only. Size snapshots too when blob accounting is empty.",
" if sz == 0 and os.path.isdir(snap):",
" sz2, nf2, ic2 = snapshot_size()",
" sz, nf, ic = sz2, nf2, ic or ic2",
" is_diffusion = False; gguf_files = []",
" if os.path.isdir(snap):",
" for sd in os.listdir(snap):",
@@ -346,7 +471,33 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None) -> str:
" if os.path.exists(os.path.join(sf, 'model_index.json')): is_diffusion = True",
" for f in collect_ggufs(sf): f['rel_path'] = sd + '/' + f['rel_path']; gguf_files.append(f)",
" models.append({'repo_id':rid,'size_bytes':sz,'nb_files':nf,'has_incomplete':ic,'path':cache,'is_diffusion':is_diffusion,'is_gguf':bool(gguf_files),'gguf_files':gguf_files})",
"def hf_cache_paths():",
" candidates = []",
" def add(p):",
" if not p: return",
" p = os.path.expanduser(p)",
" if p not in candidates: candidates.append(p)",
" add(os.environ.get('HUGGINGFACE_HUB_CACHE'))",
" hf_home = os.environ.get('HF_HOME')",
" if hf_home: add(os.path.join(hf_home, 'hub'))",
" add('~/.cache/huggingface/hub')",
" # Docker images mount ./data/huggingface at /app/.cache/huggingface.",
" # When HOME is /root, expanduser() misses that persisted cache.",
" add('/app/.cache/huggingface/hub')",
f" add({add_hf_cache!r})" if add_hf_cache else "",
" return candidates",
"def normalize_model_dir(p):",
" p = os.path.expanduser((p or '').strip())",
" if not p: return p",
" if os.path.isdir(p) or os.path.isabs(p): return p",
" # Users often paste Linux absolute paths without the leading slash.",
" # Treat home/<user>/... as /home/<user>/... so remote scans work.",
" if p.startswith(('home/', 'mnt/', 'media/', 'data/', 'opt/', 'srv/', 'var/')):",
" prefixed = '/' + p",
" if os.path.isdir(prefixed): return prefixed",
" return p",
"def scan_dir(p):",
" p = normalize_model_dir(p)",
" if not os.path.isdir(p) or not safe_path(p): return",
" for d in sorted(os.listdir(p)):",
" if d.startswith('.'): continue",
@@ -380,6 +531,8 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None) -> str:
" if u.startswith('KB'): return int(n * 1024)",
" return int(n)",
"def scan_ollama():",
" if any(m.get('is_ollama') for m in models): return",
" if os.name == 'nt' and not os.environ.get('ODYSSEUS_ALLOW_OLLAMA_CLI_SCAN'): return",
" if not shutil.which('ollama'): return",
" try:",
" p = subprocess.run(['ollama', 'list'], stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, timeout=6)",
@@ -409,12 +562,12 @@ def _cached_model_scan_script(model_dirs: list[str] | None = None) -> str:
" seen.add(name)",
" models.append({'repo_id':name,'size_bytes':size_bytes,'nb_files':1,'has_incomplete':False,'path':'ollama','backend':'ollama','is_ollama':True})",
" return",
"scan_hf(os.path.expanduser('~/.cache/huggingface/hub'))",
"scan_ollama()",
"for _hf_cache in hf_cache_paths(): scan_hf(_hf_cache)",
"scan_ollama_api()",
"scan_ollama()",
]
for model_dir in model_dirs or []:
lines.append(f"scan_dir(os.path.expanduser({model_dir!r}))")
lines.append(f"scan_dir({model_dir!r})")
lines.append("print(json.dumps(models))")
return "\n".join(lines) + "\n"
@@ -431,10 +584,22 @@ def _bash_squote(v: str) -> str:
return v.replace("'", "'\\''")
# Shown by generated runner scripts when the ollama binary is missing on the
# target host. Must stay free of backticks/$( ) and be emitted single-quoted:
# an earlier version wrapped the install one-liner in backticks inside a
# double-quoted echo, which bash executed as command substitution and ran the
# system-wide installer (including on remote SSH hosts) instead of printing
# the hint.
OLLAMA_MISSING_HINT = (
"ERROR: Ollama not found on this server. Install it from "
"https://ollama.com/download or run: curl -fsSL https://ollama.com/install.sh | sh"
)
# Allow-list of binaries permitted as the leading token of `req.cmd` for /api/model/serve.
# Anything else is rejected before the cmd is interpolated into a tmux/PowerShell wrapper.
_SERVE_CMD_ALLOWLIST = {
"vllm", "llama-server", "llama_server", "llama.cpp", "ollama",
"vllm", "llama-server", "llama-server.exe", "llama_server", "llama.cpp", "ollama",
"python", "python3",
"sglang", "lmdeploy",
"node", "npx",
@@ -450,9 +615,49 @@ _SERVE_CMD_ALLOWLIST = {
_GGUF_PRELUDE_RE = re.compile(
r'^MODEL_FILE=\$\([^\n]*?\)\s*&&\s*\{[^{}]*\}\s*\|\|\s*\{[^{}]*\}\s*&&\s*'
)
_SAFE_SUBSHELL_TEXT = r"[^'\n;&|`$()<>]+"
_SAFE_SUBSHELL_DQ_HOME_PATH = r'"\$HOME/[^"\n;&|`()<>]*"'
_SAFE_PRINTF_SUBSHELL_RE = re.compile(
rf"^\$\(printf[ \t]+%s[ \t]+(?:'{_SAFE_SUBSHELL_TEXT}'|\$\{{HOME\}}'/{_SAFE_SUBSHELL_TEXT}')\)$"
)
_SAFE_FIND_MMPROJ_SUBSHELL_RE = re.compile(
rf"^\$\(find[ \t]+(?:'{_SAFE_SUBSHELL_TEXT}'|{_SAFE_SUBSHELL_DQ_HOME_PATH}|{_SAFE_SUBSHELL_TEXT})"
r"[ \t]+-iname[ \t]+'mmproj\*\.gguf'"
r"(?:[ \t]+2>/dev/null)?[ \t]*\|[ \t]*sort[ \t]*\|[ \t]*head[ \t]+-1\)$"
)
_OLLAMA_HOST_ASSIGNMENT_RE = re.compile(r"(?:^|\s)OLLAMA_HOST=([^\s]+)")
_OLLAMA_BIND_RE = re.compile(r"^\[([^\]]+)\]:(\d+)$|^([^:]+):(\d+)$")
_OLLAMA_BIND_HOST_RE = re.compile(r"^[A-Za-z0-9._:-]+$")
_LLAMA_CPP_PYTHON_GGML_TYPES = {
"f32": "0",
"f16": "1",
"q4_0": "2",
"q4_1": "3",
"q5_0": "6",
"q5_1": "7",
"q8_0": "8",
"q8_1": "9",
"q2_k": "10",
"q3_k": "11",
"q4_k": "12",
"q5_k": "13",
"q6_k": "14",
"q8_k": "15",
"iq2_xxs": "16",
"iq2_xs": "17",
"iq3_xxs": "18",
"iq1_s": "19",
"iq4_nl": "20",
"iq3_s": "21",
"iq2_s": "22",
"iq4_xs": "23",
"mxfp4": "39",
"nvfp4": "40",
"q1_0": "41",
}
_LLAMA_CPP_PYTHON_TYPE_FLAG_RE = re.compile(
r"(?P<flag>--type_[kv])(?P<sep>\s+|=)(?P<quote>['\"]?)(?P<value>[A-Za-z0-9_]+)(?P=quote)"
)
def _ollama_bind_from_cmd(cmd: str | None, *, default_host: str = "127.0.0.1") -> tuple[str, str]:
@@ -484,6 +689,22 @@ def _ollama_bind_from_cmd(cmd: str | None, *, default_host: str = "127.0.0.1") -
return f"[{host}]" if bracketed_host else host, port
def _normalize_llama_cpp_python_cache_types(cmd: str | None) -> str | None:
"""Map llama.cpp KV cache type names to llama-cpp-python's integer enum."""
if not cmd or "llama_cpp.server" not in cmd:
return cmd
def repl(match: re.Match[str]) -> str:
value = match.group("value")
mapped = _LLAMA_CPP_PYTHON_GGML_TYPES.get(value.lower())
if not mapped:
return match.group(0)
quote = match.group("quote")
return f"{match.group('flag')}{match.group('sep')}{quote}{mapped}{quote}"
return _LLAMA_CPP_PYTHON_TYPE_FLAG_RE.sub(repl, cmd)
def _check_serve_binary(seg: str) -> None:
"""Validate that a single command segment starts with an allowlisted binary
(after skipping leading env-var assignments like `CUDA_VISIBLE_DEVICES=0`)."""
@@ -504,6 +725,13 @@ def _check_serve_binary(seg: str) -> None:
)
def _is_safe_serve_subshell(subshell: str) -> bool:
return bool(
_SAFE_PRINTF_SUBSHELL_RE.fullmatch(subshell)
or _SAFE_FIND_MMPROJ_SUBSHELL_RE.fullmatch(subshell)
)
def _validate_serve_cmd(v: str | None) -> str | None:
"""Reject serve commands that aren't in the allowlist or contain shell metachars.
@@ -525,6 +753,7 @@ def _validate_serve_cmd(v: str | None) -> str | None:
# Backticks and raw newlines are never legitimate here.
if any(c in v for c in ("`", "\n", "\r")):
raise HTTPException(400, "Invalid characters in cmd")
# Known GGUF launcher prelude → validate the serve invocation(s) it guards.
m = _GGUF_PRELUDE_RE.match(v)
if m:
@@ -533,9 +762,19 @@ def _validate_serve_cmd(v: str | None) -> str | None:
for part in rest.split("||"):
_check_serve_binary(part.strip())
return v
# Otherwise: a single invocation — no shell metacharacters allowed.
# Otherwise: a single invocation — no shell metacharacters allowed. Replace
# only the exact command substitutions emitted by the Cookbook UI:
# $(printf %s 'safe-path') and the mmproj lookup
# $(find <path> -iname 'mmproj*.gguf' 2>/dev/null | sort | head -1).
def _replace_safe_subshell(match: re.Match[str]) -> str:
subshell = match.group(0)
return "/placeholder/safe/path" if _is_safe_serve_subshell(subshell) else subshell
cleaned_v = re.sub(r"\$\([^()]*\)", _replace_safe_subshell, v)
# (`$(` was the original intent; bare `$` is fine for shell-safe paths.)
if any(c in v for c in (";", "&&", "||", "$(")):
if any(c in cleaned_v for c in (";", "&&", "||", "$(")):
raise HTTPException(400, "Invalid characters in cmd")
_check_serve_binary(v)
return v
@@ -602,24 +841,149 @@ def _append_llama_cpp_linux_accel_build_lines(runner_lines: list[str]) -> None:
to hard-wire CUDA on Linux. That made ROCm hosts attempt a CUDA configure and
fail with "CUDA Toolkit not found" instead of building with HIP.
"""
# Try a prebuilt binary from llama.cpp's GitHub releases FIRST — no
# cmake/build-essential/git/CUDA-headers needed at all. The from-source
# build below stays as a fallback (custom flags, esoteric arch, no
# internet, etc). 30 seconds vs 5+ minutes of compile, and removes
# every OS-package dep from the launch path. Sets _odysseus_have_prebuilt=1
# on success; the existing build-tier if/elif chain below is gated on
# that variable so we never compile twice or shadow the prebuilt symlink.
runner_lines.append(' _odysseus_have_prebuilt=""')
runner_lines.append(' _odysseus_arch="$(uname -m)"')
runner_lines.append(' _odysseus_prebuilt_url=""')
runner_lines.append(' if command -v curl >/dev/null 2>&1 && [ "$_odysseus_arch" = "x86_64" ]; then')
runner_lines.append(' _odysseus_pat=""')
runner_lines.append(' _odysseus_has_nv_inline() { command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L 2>/dev/null | grep -q "GPU "; }')
runner_lines.append(' _odysseus_has_vk_inline() { ldconfig -p 2>/dev/null | grep -q "libvulkan\\.so" || command -v vulkaninfo >/dev/null 2>&1 || [ -e /usr/lib/x86_64-linux-gnu/libvulkan.so.1 ]; }')
runner_lines.append(' _odysseus_has_vkdev_inline() { ls /dev/dri/renderD* >/dev/null 2>&1 || (lspci 2>/dev/null | grep -Ei \'VGA|3D|Display\' | grep -Eiq \'AMD|ATI|Radeon\'); }')
runner_lines.append(' if _odysseus_has_nv_inline; then')
runner_lines.append(' _odysseus_pat="ubuntu.*cuda"')
runner_lines.append(' elif _odysseus_has_vkdev_inline && _odysseus_has_vk_inline; then')
runner_lines.append(' _odysseus_pat="ubuntu.*vulkan"')
runner_lines.append(' else')
runner_lines.append(' _odysseus_pat="ubuntu-x64\\\\.zip"')
runner_lines.append(' fi')
runner_lines.append(' _odysseus_prebuilt_url="$(curl -fsSL --max-time 15 https://api.github.com/repos/ggml-org/llama.cpp/releases/latest 2>/dev/null | grep \'"browser_download_url"\' | cut -d\'"\' -f4 | grep -iE "$_odysseus_pat" | grep -iv "arm\\|aarch64" | head -1)"')
runner_lines.append(' fi')
# Accept any of unzip / bsdtar / python3 -m zipfile as the extractor.
# python3 is essentially always present on modern Linux, so this lets
# the prebuilt path work on minimal Ubuntu installs that lack `unzip`.
runner_lines.append(' if [ -n "$_odysseus_prebuilt_url" ] && (command -v unzip >/dev/null 2>&1 || command -v bsdtar >/dev/null 2>&1 || command -v python3 >/dev/null 2>&1); then')
runner_lines.append(' echo "[odysseus] Found prebuilt llama-server: $_odysseus_prebuilt_url"')
runner_lines.append(' mkdir -p ~/bin "$HOME/.cache/odysseus/llama-cpp-prebuilt" && cd "$HOME/.cache/odysseus/llama-cpp-prebuilt"')
runner_lines.append(' rm -f llama-cpp.zip')
runner_lines.append(' if curl -fsSL --max-time 120 "$_odysseus_prebuilt_url" -o llama-cpp.zip && [ -s llama-cpp.zip ]; then')
runner_lines.append(' rm -rf build && mkdir -p build')
runner_lines.append(' if command -v unzip >/dev/null 2>&1; then unzip -qq -o llama-cpp.zip -d build; elif command -v bsdtar >/dev/null 2>&1; then bsdtar -xf llama-cpp.zip -C build; else python3 -c "import zipfile; zipfile.ZipFile(\\"llama-cpp.zip\\").extractall(\\"build\\")"; fi')
runner_lines.append(' _odysseus_extracted="$(find build -type f -name llama-server 2>/dev/null | head -1)"')
runner_lines.append(' if [ -n "$_odysseus_extracted" ]; then')
runner_lines.append(' chmod +x "$_odysseus_extracted"')
runner_lines.append(' ln -sf "$_odysseus_extracted" ~/bin/llama-server')
runner_lines.append(' _odysseus_libdir="$(dirname "$_odysseus_extracted")"')
runner_lines.append(' mkdir -p ~/.config && echo "export LD_LIBRARY_PATH=\\"$_odysseus_libdir:\\${LD_LIBRARY_PATH:-}\\"" > ~/.config/odysseus-llama-cpp-env')
runner_lines.append(' _odysseus_have_prebuilt=1')
runner_lines.append(' echo "[odysseus] Prebuilt llama-server installed at $_odysseus_extracted"')
runner_lines.append(' fi')
runner_lines.append(' fi')
runner_lines.append(' [ -z "$_odysseus_have_prebuilt" ] && echo "[odysseus] Prebuilt download/extract failed — falling back to from-source build."')
runner_lines.append(' elif [ -z "$_odysseus_prebuilt_url" ]; then')
runner_lines.append(' echo "[odysseus] No matching prebuilt llama-server for this host (arch=$_odysseus_arch) — will build from source."')
runner_lines.append(' fi')
runner_lines.append(' if [ -z "$_odysseus_have_prebuilt" ]; then')
# Detect pip-installed nvcc (from vLLM/nvidia CUDA wheels) and put it on PATH
# so cmake's CUDA configure can find it. We keep this after the ROCm/HIP
# check — a machine with both stacks should honor the native HIP toolchain on
# AMD hosts instead of accidentally preferring a stray nvcc wheel.
runner_lines.append(' for _cudir in ~/.local/lib/python*/site-packages/nvidia/cu13 ~/.local/lib/python*/site-packages/nvidia/cu12 ~/.local/lib/python*/site-packages/nvidia/cuda_nvcc; do')
runner_lines.append(' [ -x "$_cudir/bin/nvcc" ] && export CUDA_HOME="$_cudir" && export PATH="$_cudir/bin:$PATH" && break')
runner_lines.append(' done')
# so cmake's CUDA configure can find it — BUT only when actual NVIDIA
# hardware is present. On AMD/Intel hosts the pip nvcc is a misleading
# leftover (no libcudart, no GPU it could target) and would otherwise
# send the build down the CUDA branch and fail with "CUDA Toolkit not
# found" instead of trying Vulkan.
runner_lines.append(' _odysseus_has_nvidia_hw() {')
runner_lines.append(' command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L 2>/dev/null | grep -q "GPU " && return 0')
runner_lines.append(' ls /dev/nvidia* >/dev/null 2>&1 && return 0')
runner_lines.append(' lspci 2>/dev/null | grep -iE \'VGA|3D|Display\' | grep -iq nvidia && return 0')
runner_lines.append(' return 1')
runner_lines.append(' }')
runner_lines.append(' if _odysseus_has_nvidia_hw; then')
runner_lines.append(' for _cudir in ~/.local/lib/python*/site-packages/nvidia/cu13 ~/.local/lib/python*/site-packages/nvidia/cu12 ~/.local/lib/python*/site-packages/nvidia/cuda_nvcc; do')
runner_lines.append(' [ -x "$_cudir/bin/nvcc" ] && export CUDA_HOME="$_cudir" && export PATH="$_cudir/bin:$PATH" && break')
runner_lines.append(' done')
runner_lines.append(' fi')
# rm -rf build so a prior poisoned CMakeCache.txt (e.g. from a failed CUDA
# or HIP attempt) doesn't cause the next configure to reuse stale settings.
runner_lines.append(' cd ~/llama.cpp && rm -rf build')
runner_lines.append(' mkdir -p ~/bin')
# Try to install cmake / build-essential / git automatically before the
# build, but ONLY via passwordless sudo (`sudo -n`) — interactive sudo
# would hang a tmux-backgrounded serve task waiting for a password. If
# sudo asks for a password the install is skipped silently and the
# diagnosis pattern (cookbook_routes.py / cookbook_helpers.py) surfaces
# an explicit "install cmake" suggestion in the Cookbook diagnosis
# toolbar after the inevitable build failure.
runner_lines.append(' _odysseus_apt_bootstrap() {')
runner_lines.append(' local _missing=""')
runner_lines.append(' command -v cmake >/dev/null 2>&1 || _missing="$_missing cmake"')
runner_lines.append(' command -v g++ >/dev/null 2>&1 || command -v gcc >/dev/null 2>&1 || _missing="$_missing build-essential"')
runner_lines.append(' command -v git >/dev/null 2>&1 || _missing="$_missing git"')
runner_lines.append(' [ -z "$_missing" ] && return 0')
runner_lines.append(' if command -v apt-get >/dev/null 2>&1 && sudo -n true 2>/dev/null; then')
runner_lines.append(' echo "[odysseus] Auto-installing missing build deps via apt:$_missing"')
runner_lines.append(' sudo -n env DEBIAN_FRONTEND=noninteractive apt-get update -qq 2>&1 | tail -3')
runner_lines.append(' sudo -n env DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends $_missing 2>&1 | tail -5 || true')
runner_lines.append(' elif command -v pacman >/dev/null 2>&1 && sudo -n true 2>/dev/null; then')
runner_lines.append(' echo "[odysseus] Auto-installing missing build deps via pacman:$_missing"')
runner_lines.append(' local _pacpkgs="$(echo "$_missing" | sed -e \'s/build-essential/base-devel/g\')"')
runner_lines.append(' sudo -n pacman -Sy --needed --noconfirm $_pacpkgs 2>&1 | tail -5 || true')
runner_lines.append(' elif command -v dnf >/dev/null 2>&1 && sudo -n true 2>/dev/null; then')
runner_lines.append(' echo "[odysseus] Auto-installing missing build deps via dnf:$_missing"')
runner_lines.append(' local _dnfpkgs="$(echo "$_missing" | sed -e \'s/build-essential/gcc gcc-c++ make/g\')"')
runner_lines.append(' sudo -n dnf install -y $_dnfpkgs 2>&1 | tail -5 || true')
runner_lines.append(' else')
runner_lines.append(' echo "[odysseus] WARNING: missing build deps ($_missing) — passwordless sudo is unavailable, cannot auto-install. Cookbook Diagnosis will explain the fix after the build fails."')
runner_lines.append(' fi')
runner_lines.append(' }')
runner_lines.append(' _odysseus_apt_bootstrap')
runner_lines.append(' _odysseus_missing_build_deps=""')
runner_lines.append(' command -v cmake >/dev/null 2>&1 || _odysseus_missing_build_deps="$_odysseus_missing_build_deps cmake"')
runner_lines.append(' command -v git >/dev/null 2>&1 || _odysseus_missing_build_deps="$_odysseus_missing_build_deps git"')
runner_lines.append(' command -v g++ >/dev/null 2>&1 || command -v gcc >/dev/null 2>&1 || _odysseus_missing_build_deps="$_odysseus_missing_build_deps build-essential"')
runner_lines.append(' if [ -n "$_odysseus_missing_build_deps" ]; then')
runner_lines.append(' echo "ERROR: llama.cpp source build needs missing packages:$_odysseus_missing_build_deps"')
runner_lines.append(' if command -v apt-get >/dev/null 2>&1; then')
runner_lines.append(' echo "Install on this host: sudo apt-get update && sudo apt-get install -y cmake build-essential git"')
runner_lines.append(' elif command -v pacman >/dev/null 2>&1; then')
runner_lines.append(' echo "Install on this host: sudo pacman -Sy --needed cmake base-devel git"')
runner_lines.append(' elif command -v dnf >/dev/null 2>&1; then')
runner_lines.append(' echo "Install on this host: sudo dnf install -y cmake gcc gcc-c++ make git"')
runner_lines.append(' fi')
runner_lines.append(' echo "Alternative: install a native llama-server on PATH, then relaunch."')
runner_lines.append(' ODYSSEUS_PREFLIGHT_EXIT=127')
runner_lines.append(' fi')
runner_lines.append(' cd ~/llama.cpp')
runner_lines.append(' _odysseus_has_vulkan() {')
runner_lines.append(' ldconfig -p 2>/dev/null | grep -q \'libvulkan\\.so\' && return 0')
runner_lines.append(' [ -e /usr/lib/libvulkan.so.1 ] && return 0')
runner_lines.append(' [ -e /usr/lib/x86_64-linux-gnu/libvulkan.so.1 ] && return 0')
runner_lines.append(' command -v vulkaninfo >/dev/null 2>&1 && return 0')
runner_lines.append(' return 1')
runner_lines.append(' }')
runner_lines.append(' _odysseus_has_vulkan_device() {')
runner_lines.append(' ls /dev/dri/renderD* >/dev/null 2>&1 && return 0')
runner_lines.append(' lspci 2>/dev/null | grep -Ei \'VGA|3D|Display\' | grep -Eiq \'AMD|ATI|Radeon\' && return 0')
runner_lines.append(' return 1')
runner_lines.append(' }')
# Backend preference: native ROCm/HIP > native CUDA > Vulkan > CPU.
# Vulkan is a portable fallback that works on AMD when ROCm isn't
# installed (e.g. Strix Halo) and on any vendor's discrete GPU, but
# it's ~30-40% slower than native HIP/CUDA for LLM inference — only
# pick it when no native toolchain is present.
runner_lines.append(' if command -v hipconfig &>/dev/null || [ -d /opt/rocm ] || [ -n "$ROCM_PATH" ] || [ -n "$HIP_PATH" ]; then')
runner_lines.append(' rm -rf build')
runner_lines.append(' if command -v hipconfig &>/dev/null; then')
runner_lines.append(' export HIPCXX="${HIPCXX:-$(hipconfig -l)/clang}"')
runner_lines.append(' export HIP_PATH="${HIP_PATH:-$(hipconfig -R)}"')
runner_lines.append(' fi')
runner_lines.append(' echo "[odysseus] ROCm/HIP detected — building llama-server with HIP support..."')
runner_lines.append(' cmake -B build -DCMAKE_BUILD_TYPE=Release -DGGML_HIP=ON && cmake --build build -j"$NPROC" --target llama-server && ln -sf ~/llama.cpp/build/bin/llama-server ~/bin/llama-server')
runner_lines.append(' elif command -v nvcc &>/dev/null; then')
runner_lines.append(' elif command -v nvcc &>/dev/null && _odysseus_has_nvidia_hw; then')
runner_lines.append(' rm -rf build')
# nvcc alone is not sufficient — pip-installed CUDA wheels or incomplete
# tooling can expose nvcc without shipping libcudart, causing cmake to fail
# mid-build with "CUDA runtime library not found". Check cudart explicitly
@@ -643,36 +1007,56 @@ def _append_llama_cpp_linux_accel_build_lines(runner_lines: list[str]) -> None:
runner_lines.append(' echo "[odysseus] Ensure libcudart is installed (e.g. cuda-runtime package) and visible via ldconfig or CUDA_HOME."')
runner_lines.append(' cmake -B build -DCMAKE_BUILD_TYPE=Release && cmake --build build -j"$NPROC" --target llama-server && ln -sf ~/llama.cpp/build/bin/llama-server ~/bin/llama-server')
runner_lines.append(' fi')
runner_lines.append(' elif _odysseus_has_vulkan_device && _odysseus_has_vulkan; then')
runner_lines.append(' echo "[odysseus] Vulkan-capable GPU detected (no ROCm/CUDA toolchain installed) — building llama-server with Vulkan support..."')
runner_lines.append(' rm -rf build-vulkan')
runner_lines.append(' cmake -B build-vulkan -DCMAKE_BUILD_TYPE=Release -DGGML_VULKAN=ON && cmake --build build-vulkan -j"$NPROC" --target llama-server && ln -sf ~/llama.cpp/build-vulkan/bin/llama-server ~/bin/llama-server')
runner_lines.append(' else')
runner_lines.append(' echo "[odysseus] WARNING: no HIP/CUDA toolchain found — building llama-server for CPU only."')
runner_lines.append(' echo "[odysseus] WARNING: no HIP/CUDA/Vulkan toolchain found — building llama-server for CPU only."')
runner_lines.append(' echo "[odysseus] GPU inference will not be available for this llama.cpp build."')
runner_lines.append(' echo "[odysseus] Install ROCm for AMD GPUs or vLLM/CUDA tooling for NVIDIA, then re-launch this serve task."')
runner_lines.append(' echo "[odysseus] Install Vulkan (libvulkan-dev) / ROCm for AMD GPUs or CUDA tooling for NVIDIA, then re-launch this serve task."')
runner_lines.append(' rm -rf build')
runner_lines.append(' cmake -B build -DCMAKE_BUILD_TYPE=Release && cmake --build build -j"$NPROC" --target llama-server && ln -sf ~/llama.cpp/build/bin/llama-server ~/bin/llama-server')
runner_lines.append(' fi')
runner_lines.append(' fi # end _odysseus_have_prebuilt guard')
def _llama_cpp_rebuild_cmd() -> str:
def _llama_cpp_rebuild_cmd(update_source: bool = False) -> str:
"""Shell command that clears the Cookbook-managed llama.cpp build.
Removes the cached ``llama-server`` symlink and the ``~/llama.cpp/build``
Removes the cached ``llama-server`` symlink and the ``~/llama.cpp/build*``
directory so the next llama.cpp serve recompiles from source, picking up a
CUDA or HIP toolchain if one is now available. The serve bootstrap only
builds when ``llama-server`` is missing from PATH, so without this an
existing CPU-only build is reused forever. It deliberately installs and
downloads nothing; the rebuild itself happens on the next serve.
existing CPU-only build is reused forever. When ``update_source`` is true,
the command also fast-forwards the Cookbook-managed ``~/llama.cpp`` checkout
if it exists. The rebuild itself happens on the next serve.
"""
update_cmd = ''
if update_source:
update_cmd = (
'if [ -d "$HOME/llama.cpp/.git" ]; then '
'git -C "$HOME/llama.cpp" pull --ff-only --depth 1 || '
'echo "[odysseus] WARNING: llama.cpp source update failed; clearing cached build anyway."; '
'elif command -v git >/dev/null 2>&1; then '
'git clone --depth 1 https://github.com/ggml-org/llama.cpp "$HOME/llama.cpp" || '
'echo "[odysseus] WARNING: llama.cpp clone failed; clearing cached build anyway."; '
'fi && '
)
return (
'mkdir -p "$HOME/bin" && '
f'{update_cmd}'
'rm -f "$HOME/bin/llama-server" && '
'rm -rf "$HOME/llama.cpp/build" && '
'rm -rf "$HOME/llama.cpp/build" "$HOME/llama.cpp/build-vulkan" && '
'echo "[odysseus] Cleared the cached llama.cpp build. '
'Re-launch the serve task to rebuild llama-server from source '
'(CUDA or HIP will be used if a toolchain is now available)."'
'(Vulkan, HIP, or CUDA will be used if a matching toolchain is now available)."'
)
class ModelDownloadRequest(BaseModel):
repo_id: str
backend: str | None = None # "hf" (default) or "ollama"
include: str | None = None # glob pattern e.g. "*Q4_K_M*"
hf_token: str | None = None
env_prefix: str | None = None # e.g. "source ~/venv/bin/activate"
@@ -914,13 +1298,57 @@ def _diagnose_serve_output(text: str) -> dict | None:
"vLLM is not installed or not in PATH on this server.",
[{"label": "install vLLM in Cookbook Dependencies", "op": "dependency", "package": "vllm"}],
),
(
r"sgl_kernel[\s\S]*(Python\.h|libnuma\.so\.1|common_ops|libnvrtc\.so)|"
r"(Python\.h|libnuma\.so\.1|common_ops|libnvrtc\.so)[\s\S]*sgl_kernel|"
r"Could not load any common_ops library|"
r"Please ensure sgl_kernel is properly installed",
"SGLang native kernel/runtime is missing or mismatched on this server.",
[
{"label": "repair sglang-kernel in this Python environment", "op": "dependency", "package": "sglang-kernel"},
{"label": "install OS packages: libnuma-dev python3.12-dev build-essential", "op": "manual"},
{"label": "if libnvrtc is still missing, install the matching CUDA/NVRTC runtime on this host", "op": "manual"},
],
),
(
r"sglang.*command not found|No module named sglang|SGLang is not installed",
"SGLang is not installed or not in PATH on this server.",
[{"label": "install SGLang in Cookbook Dependencies", "op": "dependency", "package": "sglang[all]"}],
),
(
r"llama-server.*command not found|llama\.cpp.*not found|No module named.*llama_cpp|No module named 'starlette_context'|git: command not found|cmake: command not found",
r"No module named ['\"]?mlx_lm|mlx_lm.*command not found|MLX is not installed|MLX LM is not installed",
"MLX LM is not installed on this server.",
[{"label": "install mlx-lm in Cookbook Dependencies", "op": "dependency", "package": "mlx-lm"}],
),
(
r"Unable to quantize model of type <class ['\"]mlx_lm\.models\.switch_layers\.QuantizedSwitchLinear['\"]>|QuantizedSwitchLinear",
"MLX-LM tried to quantize an already-quantized DeepSeek switch layer.",
[
{"label": "relaunch from the cached local Hugging Face snapshot path on this Mac", "op": "manual"},
{"label": "Odysseus now rewrites MLX repo-id launches to a cached snapshot when one exists", "op": "manual"},
],
),
# System build deps come BEFORE the generic llama.cpp catch-all so
# cmake / build-essential / git missing → a specific OS-package
# remediation instead of "install llama-cpp-python[server]" (which
# itself fails to compile when cmake is absent).
(
r"cmake: command not found|cmake.*not found.*[Cc]ould not",
"cmake is required to build llama.cpp from source but isn't installed on this server.",
[{"label": "install build deps for llama.cpp (apt: cmake build-essential git / pacman: cmake base-devel git / dnf: cmake gcc-c++ make git / brew: cmake git)", "op": "dependency", "package": "llama-cpp-python[server]"}],
),
(
r"^(make|g\+\+|gcc): command not found|Could not find C\+\+ compiler",
"A C/C++ compiler (build-essential) is required to build llama.cpp from source.",
[{"label": "install build deps for llama.cpp on this server", "op": "dependency", "package": "llama-cpp-python[server]"}],
),
(
r"^git: command not found",
"git is required to clone the llama.cpp source tree.",
[{"label": "install build deps for llama.cpp on this server", "op": "dependency", "package": "llama-cpp-python[server]"}],
),
(
r"llama-server.*command not found|llama\.cpp.*not found|No module named.*llama_cpp|No module named 'starlette_context'",
"llama.cpp / llama-cpp-python dependencies are missing.",
[{"label": "install llama.cpp dependencies or llama-cpp-python[server]", "op": "dependency", "package": "llama-cpp-python[server]"}],
),
@@ -951,3 +1379,40 @@ def _diagnose_serve_output(text: str) -> dict | None:
"suggestions": [{"label": "inspect traceback and retry with adjusted backend/settings", "op": "manual"}],
}
return None
async def run_ssh_command_async(
remote: str,
ssh_port: str | None,
remote_cmd: str,
*,
timeout: float,
connect_timeout: int | None = None,
strict_host_key_checking: bool | None = None,
stdin_data: bytes | None = None,
) -> tuple[int, bytes, bytes]:
"""Run an ssh command with centralized timeout and stderr/stdout capture.
Async version of core.platform_compat.run_ssh_command_sync.
"""
import asyncio
proc = await asyncio.create_subprocess_exec(
*_ssh_exec_argv(
remote,
ssh_port,
remote_cmd=remote_cmd,
connect_timeout=connect_timeout,
strict_host_key_checking=strict_host_key_checking,
),
stdin=asyncio.subprocess.PIPE if stdin_data is not None else None,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
try:
stdout, stderr = await asyncio.wait_for(
proc.communicate(input=stdin_data), timeout=timeout
)
except asyncio.TimeoutError:
proc.kill()
await proc.communicate()
raise
return proc.returncode or 0, stdout, stderr

75
routes/cookbook_output.py Normal file
View File

@@ -0,0 +1,75 @@
"""Pure helpers for shaping cookbook task output for the status response.
Kept dependency-free (no FastAPI / SQLAlchemy imports) so the behavior can be
unit-tested without standing up the whole app.
"""
import re
_FETCHING_ZERO_FILES_RE = re.compile(r"Fetching\s+0\s+files", re.IGNORECASE)
# Probe scripts for the dead-session download check, run as
# `python3 -c <PROBE> <repo_id> <cache_root>` (locally or over SSH).
# cache_root is the task's custom download dir, '' for the default HF cache.
# It has to be passed explicitly: the download runner exports
# HF_HOME=<local_dir>, so that task's cache lives under <local_dir>/hub, and
# the probe process's own environment knows nothing about it.
HF_CACHE_COMPLETE_PROBE = (
"import os,sys;"
"repo=sys.argv[1];"
"root=os.path.expanduser(sys.argv[2]) if len(sys.argv)>2 and sys.argv[2] else '';"
"base=os.path.join(root,'hub') if root else (os.environ.get('HUGGINGFACE_HUB_CACHE') or os.path.join(os.environ.get('HF_HOME', os.path.expanduser('~/.cache/huggingface')), 'hub'));"
"d=os.path.join(base,'models--'+repo.replace('/','--'));"
"snap=os.path.join(d,'snapshots');"
"ok=os.path.isdir(snap) and any(os.path.isdir(os.path.join(snap,x)) and os.listdir(os.path.join(snap,x)) for x in os.listdir(snap));"
"inc=False;"
"blobs=os.path.join(d,'blobs');"
"inc=os.path.isdir(blobs) and any(x.endswith('.incomplete') for x in os.listdir(blobs));"
"sys.exit(0 if ok and not inc else 1)"
)
HF_CACHE_INCOMPLETE_PROBE = (
"import os,sys;"
"repo=sys.argv[1];"
"root=os.path.expanduser(sys.argv[2]) if len(sys.argv)>2 and sys.argv[2] else '';"
"base=os.path.join(root,'hub') if root else (os.environ.get('HUGGINGFACE_HUB_CACHE') or os.path.join(os.environ.get('HF_HOME', os.path.expanduser('~/.cache/huggingface')), 'hub'));"
"d=os.path.join(base,'models--'+repo.replace('/','--'));"
"blobs=os.path.join(d,'blobs');"
"inc=os.path.isdir(blobs) and any(x.endswith('.incomplete') for x in os.listdir(blobs));"
"sys.exit(0 if inc else 1)"
)
def classify_dead_download(full_snapshot: str):
"""Resolve a dead download session's status from its runner markers.
The runner prints DOWNLOAD_OK only after exiting 0 (and DOWNLOAD_FAILED
otherwise), so the markers stay trustworthy after the tmux pane is gone.
Returns (status, zero_files), or None when the snapshot carries no marker
and the caller has to fall back to the cache probe. Same precedence as
the live-session branch: DOWNLOAD_OK wins, except a "Fetching 0 files"
run is an error (nothing matched the include/quant pattern).
"""
if not full_snapshot:
return None
if "DOWNLOAD_OK" in full_snapshot:
if _FETCHING_ZERO_FILES_RE.search(full_snapshot):
return ("error", True)
return ("completed", False)
if "DOWNLOAD_FAILED" in full_snapshot:
return ("error", False)
return None
def error_aware_output_tail(full_snapshot: str, status: str) -> str:
"""Return the trailing slice of a task log for the status response.
Failed tasks return the last 50 lines so the "Copy last 50 lines" action
surfaces the actual error context (stack traces, build output). Running and
other non-error tasks keep the cheaper 12-line tail to limit the payload on
the 10s polling interval.
"""
if not full_snapshot:
return ""
tail_lines = 50 if status == "error" else 12
return "\n".join(full_snapshot.splitlines()[-tail_lines:])

2693
routes/cookbook_routes.py
View File

File diff suppressed because it is too large Load Diff

184
routes/copilot_routes.py
View File

@@ -20,39 +20,26 @@ All routes are admin-gated (endpoint/provider management is an admin action).
"""
import json
import time
import uuid
import logging
import threading
from typing import Dict, Optional
import httpx
from fastapi import APIRouter, Request, Form, HTTPException
from fastapi import HTTPException, Request
from core.database import SessionLocal, ModelEndpoint
from core.middleware import require_admin
from routes.device_flow import (
DeviceFlowPoll,
DeviceFlowStart,
PendingDeviceFlowStore,
create_device_flow_router,
)
from src.auth_helpers import get_current_user
from src import copilot
logger = logging.getLogger(__name__)
# Pending device-flow logins, keyed by an opaque poll_id. The device_code is a
# bearer-like secret, so it lives here (server memory) rather than in the
# browser. Entries expire with the GitHub device code.
#
# NOTE: this is per-process state. The device flow assumes a single worker
# (Odysseus' default): with multiple uvicorn workers, the poll request can land
# on a worker that never saw the start, returning "Unknown or expired login
# session". Move this to a shared store (DB/Redis) if running multi-worker.
_PENDING: Dict[str, Dict] = {}
_PENDING_LOCK = threading.Lock()
def _prune_expired() -> None:
now = time.time()
with _PENDING_LOCK:
for k in [k for k, v in _PENDING.items() if v.get("expires_at", 0) < now]:
_PENDING.pop(k, None)
_DEVICE_FLOW_STORE = PendingDeviceFlowStore()
def _provision_endpoint(token: str, base: str, owner: Optional[str]) -> Dict:
@@ -112,112 +99,75 @@ def _provision_endpoint(token: str, base: str, owner: Optional[str]) -> Dict:
return result
def setup_copilot_routes() -> APIRouter:
router = APIRouter(prefix="/api/copilot", tags=["copilot"])
def _start_device_flow(request: Request, form) -> DeviceFlowStart:
host = copilot.GITHUB_HOST
ent = str(form.get("enterprise_url") or "").strip()
if ent:
host = copilot.normalize_domain(ent)
try:
data = copilot.request_device_code(host)
except httpx.HTTPStatusError as e:
status = e.response.status_code if e.response is not None else "unknown"
raise HTTPException(502, f"GitHub device-code request failed (HTTP {status})")
except Exception as e:
raise HTTPException(502, f"GitHub device-code request failed: {e}")
@router.post("/device/start")
def device_start(request: Request, enterprise_url: str = Form("")):
require_admin(request)
_prune_expired()
host = copilot.GITHUB_HOST
ent = (enterprise_url or "").strip()
if ent:
host = copilot.normalize_domain(ent)
try:
data = copilot.request_device_code(host)
except httpx.HTTPStatusError as e:
status = e.response.status_code if e.response is not None else "unknown"
raise HTTPException(502, f"GitHub device-code request failed (HTTP {status})")
except Exception as e:
raise HTTPException(502, f"GitHub device-code request failed: {e}")
device_code = data.get("device_code")
if not device_code:
raise HTTPException(502, "GitHub did not return a device code")
device_code = data.get("device_code")
if not device_code:
raise HTTPException(502, "GitHub did not return a device code")
interval = int(data.get("interval") or 5)
expires_in = int(data.get("expires_in") or 900)
poll_id = uuid.uuid4().hex
with _PENDING_LOCK:
_PENDING[poll_id] = {
"device_code": device_code,
"host": host,
"enterprise_url": ent,
"interval": interval,
"owner": get_current_user(request) or None,
"expires_at": time.time() + expires_in,
"next_poll_at": 0.0,
}
# verification_uri_complete embeds the user code, so the browser tab we
# open lands the user straight on GitHub's "Authorize" screen with the
# code pre-filled — one click, no manual code entry.
return {
"poll_id": poll_id,
# verification_uri_complete embeds the user code, so the browser tab we
# open lands the user straight on GitHub's "Authorize" screen with the
# code pre-filled — one click, no manual code entry.
return DeviceFlowStart(
pending={
"device_code": device_code,
"host": host,
"enterprise_url": ent,
"owner": get_current_user(request) or None,
},
response={
"user_code": data.get("user_code"),
"verification_uri": data.get("verification_uri"),
"verification_uri_complete": data.get("verification_uri_complete"),
"interval": interval,
"expires_in": expires_in,
}
},
interval=int(data.get("interval") or 5),
expires_in=int(data.get("expires_in") or 900),
)
@router.post("/device/poll")
def device_poll(request: Request, poll_id: str = Form(...)):
require_admin(request)
_prune_expired()
with _PENDING_LOCK:
pending = _PENDING.get(poll_id)
if not pending:
raise HTTPException(404, "Unknown or expired login session")
# Enforce GitHub's polling interval server-side so a chatty client
# can't trip slow_down.
now = time.time()
if now < pending.get("next_poll_at", 0):
return {"status": "pending"}
def _poll_device_flow(_request: Request, pending: Dict) -> DeviceFlowPoll:
try:
data = copilot.poll_access_token(pending["host"], pending["device_code"])
except Exception as e:
return DeviceFlowPoll.pending(f"poll error: {e}")
token = data.get("access_token")
if token:
base = copilot.enterprise_base(pending["enterprise_url"]) if pending["enterprise_url"] else copilot.COPILOT_BASE
try:
data = copilot.poll_access_token(pending["host"], pending["device_code"])
result = _provision_endpoint(token, base, pending["owner"])
except Exception as e:
return {"status": "pending", "detail": f"poll error: {e}"}
logger.exception("Copilot endpoint provisioning failed")
raise HTTPException(500, f"Login succeeded but provisioning failed: {e}")
return DeviceFlowPoll.authorized(result)
token = data.get("access_token")
if token:
base = copilot.enterprise_base(pending["enterprise_url"]) if pending["enterprise_url"] else copilot.COPILOT_BASE
try:
result = _provision_endpoint(token, base, pending["owner"])
except Exception as e:
logger.exception("Copilot endpoint provisioning failed")
with _PENDING_LOCK:
_PENDING.pop(poll_id, None)
raise HTTPException(500, f"Login succeeded but provisioning failed: {e}")
with _PENDING_LOCK:
_PENDING.pop(poll_id, None)
return {"status": "authorized", "endpoint": result}
err = data.get("error")
if err == "authorization_pending":
return DeviceFlowPoll.pending()
if err == "slow_down":
return DeviceFlowPoll.slow_down(int(data.get("interval") or 0) or None)
if err in ("expired_token", "access_denied"):
return DeviceFlowPoll.failed(err)
# Unknown error — surface but keep the session for another try.
return DeviceFlowPoll.pending(err or "unknown")
err = data.get("error")
if err == "authorization_pending":
with _PENDING_LOCK:
if poll_id in _PENDING:
_PENDING[poll_id]["next_poll_at"] = now + pending["interval"]
return {"status": "pending"}
if err == "slow_down":
new_interval = int(data.get("interval") or (pending["interval"] + 5))
with _PENDING_LOCK:
if poll_id in _PENDING:
_PENDING[poll_id]["interval"] = new_interval
_PENDING[poll_id]["next_poll_at"] = now + new_interval
return {"status": "pending"}
if err in ("expired_token", "access_denied"):
with _PENDING_LOCK:
_PENDING.pop(poll_id, None)
return {"status": "failed", "error": err}
# Unknown error — surface but keep the session for another try.
return {"status": "pending", "detail": err or "unknown"}
@router.post("/device/cancel")
def device_cancel(request: Request, poll_id: str = Form(...)):
require_admin(request)
with _PENDING_LOCK:
_PENDING.pop(poll_id, None)
return {"status": "cancelled"}
return router
def setup_copilot_routes():
return create_device_flow_router(
prefix="/api/copilot",
tags=["copilot"],
store=_DEVICE_FLOW_STORE,
start_flow=_start_device_flow,
poll_flow=_poll_device_flow,
)

193
routes/device_flow.py Normal file
View File

@@ -0,0 +1,193 @@
"""Shared OAuth/device-flow route scaffolding for provider setup."""
from __future__ import annotations
import inspect
import threading
import time
import uuid
from dataclasses import dataclass
from typing import Any, Callable, Iterable, Mapping, Optional
from fastapi import APIRouter, Form, HTTPException, Request
from core.middleware import require_admin
@dataclass(frozen=True)
class DeviceFlowStart:
"""Provider-specific start result consumed by the shared route wrapper."""
pending: Mapping[str, Any]
response: Mapping[str, Any]
interval: int = 5
expires_in: int = 900
@dataclass(frozen=True)
class DeviceFlowPoll:
"""Normalized provider poll outcome."""
status: str
endpoint: Optional[Mapping[str, Any]] = None
error: Optional[str] = None
detail: Optional[str] = None
interval: Optional[int] = None
@classmethod
def pending(cls, detail: Optional[str] = None) -> "DeviceFlowPoll":
return cls(status="pending", detail=detail)
@classmethod
def slow_down(cls, interval: Optional[int] = None, detail: Optional[str] = None) -> "DeviceFlowPoll":
return cls(status="slow_down", interval=interval, detail=detail)
@classmethod
def authorized(cls, endpoint: Mapping[str, Any]) -> "DeviceFlowPoll":
return cls(status="authorized", endpoint=endpoint)
@classmethod
def failed(cls, error: str) -> "DeviceFlowPoll":
return cls(status="failed", error=error)
class PendingDeviceFlowStore:
"""Thread-safe in-memory pending device-flow store.
Device codes and provider-side secrets stay inside this process. Each entry
stores provider payload separately from poll metadata so provider callbacks
only receive the fields they created.
"""
def __init__(self, *, time_func: Callable[[], float] = time.time):
self._pending: dict[str, dict[str, Any]] = {}
self._lock = threading.Lock()
self._time = time_func
def _now(self) -> float:
return float(self._time())
def prune_expired(self) -> None:
now = self._now()
with self._lock:
for key in [k for k, v in self._pending.items() if v.get("expires_at", 0) < now]:
self._pending.pop(key, None)
def add(self, payload: Mapping[str, Any], *, interval: int, expires_in: int) -> str:
self.prune_expired()
poll_id = uuid.uuid4().hex
with self._lock:
self._pending[poll_id] = {
"payload": dict(payload),
"interval": max(int(interval or 5), 1),
"expires_at": self._now() + max(int(expires_in or 900), 1),
"next_poll_at": 0.0,
}
return poll_id
def get_payload(self, poll_id: str) -> Optional[dict[str, Any]]:
self.prune_expired()
with self._lock:
entry = self._pending.get(poll_id)
if entry is None:
return None
return dict(entry.get("payload") or {})
def is_throttled(self, poll_id: str) -> bool:
with self._lock:
entry = self._pending.get(poll_id)
return bool(entry and self._now() < float(entry.get("next_poll_at") or 0))
def schedule_next(self, poll_id: str) -> None:
now = self._now()
with self._lock:
entry = self._pending.get(poll_id)
if entry is not None:
entry["next_poll_at"] = now + int(entry.get("interval") or 5)
def slow_down(self, poll_id: str, interval: Optional[int] = None) -> None:
now = self._now()
with self._lock:
entry = self._pending.get(poll_id)
if entry is not None:
new_interval = int(interval or (int(entry.get("interval") or 5) + 5))
entry["interval"] = max(new_interval, 1)
entry["next_poll_at"] = now + entry["interval"]
def pop(self, poll_id: str) -> None:
with self._lock:
self._pending.pop(poll_id, None)
async def _maybe_await(value: Any) -> Any:
if inspect.isawaitable(value):
return await value
return value
def _pending_response(detail: Optional[str] = None) -> dict[str, Any]:
response: dict[str, Any] = {"status": "pending"}
if detail:
response["detail"] = detail
return response
def create_device_flow_router(
*,
prefix: str,
tags: Iterable[str],
store: PendingDeviceFlowStore,
start_flow: Callable[[Request, Mapping[str, Any]], DeviceFlowStart],
poll_flow: Callable[[Request, Mapping[str, Any]], DeviceFlowPoll],
) -> APIRouter:
"""Create standard `/device/start|poll|cancel` routes for a provider."""
router = APIRouter(prefix=prefix, tags=list(tags))
@router.post("/device/start")
async def device_start(request: Request):
require_admin(request)
form = await request.form()
start = await _maybe_await(start_flow(request, form))
interval = int(start.interval or 5)
expires_in = int(start.expires_in or 900)
poll_id = store.add(start.pending, interval=interval, expires_in=expires_in)
response = dict(start.response)
response.update({"poll_id": poll_id, "interval": interval, "expires_in": expires_in})
return response
@router.post("/device/poll")
async def device_poll(request: Request, poll_id: str = Form(...)):
require_admin(request)
payload = store.get_payload(poll_id)
if payload is None:
raise HTTPException(404, "Unknown or expired login session")
if store.is_throttled(poll_id):
return {"status": "pending"}
try:
outcome = await _maybe_await(poll_flow(request, payload))
except Exception:
store.pop(poll_id)
raise
if outcome.status == "authorized":
store.pop(poll_id)
return {"status": "authorized", "endpoint": dict(outcome.endpoint or {})}
if outcome.status == "failed":
store.pop(poll_id)
return {"status": "failed", "error": outcome.error or "denied"}
if outcome.status == "slow_down":
store.slow_down(poll_id, outcome.interval)
return _pending_response(outcome.detail)
store.schedule_next(poll_id)
return _pending_response(outcome.detail)
@router.post("/device/cancel")
def device_cancel(request: Request, poll_id: str = Form(...)):
require_admin(request)
store.pop(poll_id)
return {"status": "cancelled"}
return router

36
routes/diagnostics_routes.py
View File

@@ -1,12 +1,13 @@
"""Diagnostics routes — /api/db/stats, /api/rag/stats, /api/test/youtube, /api/test-research."""
import logging
import os
from typing import Dict, Any
from fastapi import APIRouter, HTTPException, Form, Request
from services.youtube.youtube_handler import extract_youtube_id, extract_transcript_async
from core.constants import DEFAULT_HOST
from core.constants import DEFAULT_HOST, DATA_DIR
from core.middleware import require_admin
logger = logging.getLogger(__name__)
@@ -16,9 +17,42 @@ def setup_diagnostics_routes(
rag_manager,
rag_available: bool,
research_handler,
memory_vector=None,
) -> APIRouter:
router = APIRouter(tags=["diagnostics"])
@router.get("/api/diagnostics/services")
async def get_service_health(request: Request) -> Dict[str, Any]:
"""Consolidated degraded-state report for ChromaDB, SearXNG, email,
ntfy, and provider endpoints. Non-intrusive probes — safe to poll."""
require_admin(request)
from src.service_health import collect_service_health
return await collect_service_health(rag_manager, memory_vector)
@router.get("/api/diagnostics/logs")
async def get_diagnostics_logs(request: Request, limit: int = 200) -> Dict[str, Any]:
require_admin(request)
limit = max(1, min(limit, 1000))
try:
log_file = os.path.join(DATA_DIR, "logs", "app.log")
if not os.path.exists(log_file):
return {"status": "success", "logs": []}
# Safe tail read of the log file (max 5MB via rotation)
with open(log_file, "r", encoding="utf-8", errors="ignore") as f:
lines = f.readlines()
tail_lines = lines[-limit:] if len(lines) > limit else lines
tail_lines = [line.rstrip('\r\n') for line in tail_lines]
return {
"status": "success",
"logs": tail_lines
}
except Exception as e:
logger.error(f"Diagnostics logs retrieval error: {e}")
raise HTTPException(500, f"Failed to retrieve logs: {str(e)}")
@router.get("/api/db/stats")
async def get_database_stats(request: Request) -> Dict[str, Any]:
require_admin(request)

10
routes/document_helpers.py
View File

@@ -12,6 +12,7 @@ from pydantic import BaseModel
from core.database import Document, DocumentVersion
from core.database import Session as DbSession
from src.auth_helpers import _auth_disabled
from src.upload_handler import UploadHandler
logger = logging.getLogger(__name__)
@@ -28,6 +29,7 @@ class DocumentCreate(BaseModel):
class DocumentUpdate(BaseModel):
content: str
summary: Optional[str] = None
force_version: bool = False
class DocumentPatch(BaseModel):
title: Optional[str] = None
@@ -78,6 +80,8 @@ def _verify_doc_owner(db, doc: Document, user: str):
the session join for any not-yet-backfilled legacy row.
"""
if user is None:
if _auth_disabled():
return # Single-user / no-auth mode: allow access
raise HTTPException(403, "Authentication required")
if doc.owner is not None:
if doc.owner != user:
@@ -102,8 +106,10 @@ def _owner_session_filter(q, user):
The owner backfill runs in init_db before the app serves requests, so
by the time this filter is live there are no NULL-owner rows to leak;
we therefore match the owner strictly."""
if user is None:
we therefore match the owner strictly for authenticated callers."""
if not user:
if user == "" or _auth_disabled():
return q
return q.filter(False)
return q.filter(Document.owner == user)

240
routes/document_routes.py
View File

@@ -7,14 +7,24 @@ from typing import Dict, Any, List, Optional
from fastapi import APIRouter, HTTPException, Query, Request, UploadFile, File, Form
from sqlalchemy import func
from sqlalchemy import case, func, or_
from core.database import SessionLocal, Document, DocumentVersion
from core.database import Session as DbSession
from src.auth_helpers import get_current_user
from src.auth_helpers import get_current_user, _auth_disabled
from src.constants import MAIL_ATTACHMENTS_DIR
logger = logging.getLogger(__name__)
def _get_session_or_404(db, session_id: str, user: Optional[str]):
session = db.query(DbSession).filter(DbSession.id == session_id).first()
if not session:
raise HTTPException(404, "Session not found")
if user and session.owner != user:
raise HTTPException(404, "Session not found")
return session
def _aggregate_language_facets(lang_rows):
"""Sum document counts per display language for the library facet.
@@ -30,6 +40,31 @@ def _aggregate_language_facets(lang_rows):
return out
def _library_language_for_document(doc: Document) -> str:
"""Return the display language used by the document library.
PDF documents are stored as markdown wrappers so the editor can preserve
extracted text, form fields, and annotations. The library should still
identify them as PDFs instead of exposing that internal wrapper format.
"""
from src.pdf_form_doc import find_source_upload_id
if find_source_upload_id(doc.current_content or ""):
return "pdf"
return doc.language or "text"
def _email_source_key(content: str) -> tuple[str, str]:
"""Return the source email identity embedded in an email draft document."""
import re
text = content or ""
uid_m = re.search(r"(?im)^X-Source-UID:\s*(.+?)\s*$", text)
folder_m = re.search(r"(?im)^X-Source-Folder:\s*(.+?)\s*$", text)
uid = (uid_m.group(1).strip() if uid_m else "")
folder = (folder_m.group(1).strip() if folder_m else "INBOX")
return uid, folder
from routes.document_helpers import (
DocumentCreate, DocumentUpdate, DocumentPatch,
@@ -69,20 +104,12 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
# the doc is owner-stamped, so it lives in the library on its own.
session = None
if req.session_id:
session = db.query(DbSession).filter(DbSession.id == req.session_id).first()
if not session:
raise HTTPException(404, "Session not found")
# Match the lenient ownership model the rest of the app uses
# (see _owner_filter): only block when an AUTHENTICATED user is
# writing into a DIFFERENT user's session. In single-user /
# unconfigured / localhost-bypass mode the middleware leaves
# current_user unset (None), and those sessions are already
# served freely everywhere else.
if user and session.owner and session.owner != user:
raise HTTPException(403, "Cannot create document in another user's session")
doc_id = str(uuid.uuid4())
ver_id = str(uuid.uuid4())
# unconfigured / localhost-bypass mode, falsey users preserve
# the existing lenient path.
session = _get_session_or_404(db, req.session_id, user)
# If no language was supplied (e.g. cloning a doc whose language
# was never set), detect it from the content rather than storing
@@ -90,15 +117,56 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
# to markdown for prose.
language = req.language
if not language:
from src.tool_implementations import _looks_like_email_document, _sniff_doc_language
from src.agent_tools.document_tools import _looks_like_email_document, _sniff_doc_language, _coerce_email_document_content
language = _sniff_doc_language(req.content)
else:
from src.tool_implementations import _looks_like_email_document
from src.agent_tools.document_tools import _looks_like_email_document, _coerce_email_document_content
if _looks_like_email_document(req.content, req.title):
language = "email"
_assert_pdf_marker_upload_owned(request, req.content, user, upload_handler)
# Reply drafts are keyed to the source email. If a UI/tool path tries
# to create a second draft for the same email in the same chat,
# update the existing draft instead so quoted thread history stays
# attached to the visible document.
if language == "email" and req.session_id:
source_uid, source_folder = _email_source_key(req.content)
if source_uid:
candidates = (
db.query(Document)
.filter(Document.session_id == req.session_id)
.filter(Document.is_active == True)
.filter(Document.language == "email")
.order_by(Document.updated_at.desc())
.limit(25)
.all()
)
for existing in candidates:
old_uid, old_folder = _email_source_key(existing.current_content or "")
if old_uid != source_uid or old_folder != source_folder:
continue
merged = _coerce_email_document_content(existing.current_content or "", req.content)
if existing.current_content != merged:
new_ver = (existing.version_count or 1) + 1
existing.current_content = merged
existing.title = req.title or existing.title
existing.version_count = new_ver
db.add(DocumentVersion(
id=str(uuid.uuid4()),
document_id=existing.id,
version_number=new_ver,
content=merged,
summary="Updated existing email draft",
source="user",
))
db.commit()
db.refresh(existing)
return _doc_to_dict(existing)
doc_id = str(uuid.uuid4())
ver_id = str(uuid.uuid4())
doc = Document(
id=doc_id,
session_id=req.session_id,
@@ -171,11 +239,7 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
if session_id:
db = SessionLocal()
try:
sess = db.query(DbSession).filter(DbSession.id == session_id).first()
if not sess:
raise HTTPException(404, "Session not found")
if user and sess.owner and sess.owner != user:
raise HTTPException(403, "Cannot import into another user's session")
_get_session_or_404(db, session_id, user)
finally:
db.close()
@@ -198,7 +262,7 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
title = os.path.splitext(meta.get("original_name") or meta.get("name") or upload_id)[0]
try:
body_text = strip_pdf_content_marker(_process_pdf(pdf_path))
body_text = strip_pdf_content_marker(_process_pdf(pdf_path, owner=user))
except Exception:
body_text = None
@@ -260,18 +324,29 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
db = SessionLocal()
try:
from sqlalchemy import or_
pdf_marker_cond = or_(
Document.current_content.like('%<!-- pdf_source upload_id="%'),
Document.current_content.like('%<!-- pdf_form_source upload_id="%'),
)
library_language_expr = case(
(pdf_marker_cond, "pdf"),
(Document.language.is_(None), "text"),
else_=Document.language,
)
# Archived view shows ONLY archived docs; the default view excludes
# them (NULL = legacy rows that predate the column = not archived).
_arch_cond = (Document.archived == True) if archived else or_(
Document.archived == False, Document.archived.is_(None))
# Language facet counts (owner-filtered)
# Language facet counts (owner-filtered). PDF documents are stored
# as markdown wrappers, so group by the library display language
# instead of the raw stored language.
lang_q = (
db.query(Document.language, func.count(Document.id))
db.query(library_language_expr, func.count(Document.id))
.outerjoin(DbSession, Document.session_id == DbSession.id)
.filter(Document.is_active == True).filter(_arch_cond)
)
lang_q = _owner_session_filter(lang_q, user)
lang_rows = lang_q.group_by(Document.language).all()
lang_rows = lang_q.group_by(library_language_expr).all()
languages = _aggregate_language_facets(lang_rows)
# Session count (owner-filtered)
@@ -303,12 +378,17 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
Document.title.ilike(term) | Document.current_content.ilike(term)
)
# Language filter
# Language filter. "pdf" is a display language derived from the
# source marker; "markdown" excludes those wrappers.
if language:
if language == "text":
q = q.filter((Document.language == None) | (Document.language == "text"))
elif language == "pdf":
q = q.filter(pdf_marker_cond)
else:
q = q.filter(Document.language == language)
if language == "markdown":
q = q.filter(~pdf_marker_cond)
# Total before pagination
total = q.count()
@@ -332,7 +412,7 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
"session_id": doc.session_id,
"session_name": session_name,
"title": doc.title,
"language": doc.language or "text",
"language": _library_language_for_document(doc),
"preview": (doc.current_content or "")[:500],
"version_count": doc.version_count,
"created_at": (doc.created_at.isoformat() + "Z") if doc.created_at else None,
@@ -358,19 +438,19 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
db = SessionLocal()
try:
if not user:
raise HTTPException(403, "Authentication required")
session = db.query(DbSession).filter(DbSession.id == session_id).first()
if not _auth_disabled():
raise HTTPException(403, "Authentication required")
# v2 review HIGH-9: raise 403 explicitly when the caller
# can't see this session, instead of returning [] which the
# UI treats identically to "no docs" and silently masks
# auth failures.
if not session:
raise HTTPException(404, "Session not found")
if user and session.owner and session.owner != user:
raise HTTPException(403, "Access denied")
docs = db.query(Document).filter(
_get_session_or_404(db, session_id, user)
q = db.query(Document).filter(
Document.session_id == session_id
).order_by(Document.created_at.desc()).all()
)
if user:
q = q.filter(or_(Document.owner == user, Document.owner.is_(None)))
docs = q.order_by(Document.created_at.desc()).all()
return [_doc_to_dict(d) for d in docs]
finally:
db.close()
@@ -437,7 +517,7 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
raise HTTPException(404, "Source PDF could not be located")
try:
body_text = strip_pdf_content_marker(_process_pdf(pdf_path))
body_text = strip_pdf_content_marker(_process_pdf(pdf_path, owner=user))
except Exception as e:
logger.error(f"extract_pdf_text failed for {pdf_path}: {e}")
raise HTTPException(500, f"Extraction failed: {e}")
@@ -474,7 +554,8 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
user = get_current_user(request)
try:
data = await request.json()
except Exception:
except Exception as e:
logger.warning("Failed to parse export request body, defaulting to empty", exc_info=e)
data = {}
ids = data.get("ids") or []
if not ids:
@@ -539,11 +620,23 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
raise HTTPException(404, "Document not found")
_verify_doc_owner(db, doc, user)
# Skip if content is identical
if doc.current_content == req.content:
incoming_content = req.content
from src.agent_tools.document_tools import _coerce_email_document_content, _looks_like_email_document
is_email_doc = (
(doc.language or "").lower() == "email"
or _looks_like_email_document(doc.current_content or "", doc.title or "")
or _looks_like_email_document(req.content or "", doc.title or "")
)
if is_email_doc:
incoming_content = _coerce_email_document_content(doc.current_content or "", req.content)
doc.language = "email"
# Skip if content is identical unless the caller explicitly wants
# a checkpoint version from the current editor state.
if doc.current_content == incoming_content and not req.force_version:
return _doc_to_dict(doc)
_assert_pdf_marker_upload_owned(request, req.content, user, upload_handler)
_assert_pdf_marker_upload_owned(request, incoming_content, user, upload_handler)
# Check if we can coalesce with the latest version
latest_ver = db.query(DocumentVersion).filter(
@@ -552,14 +645,14 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
now = datetime.now(timezone.utc)
coalesced = False
if latest_ver and latest_ver.source == "user":
if latest_ver and latest_ver.source == "user" and not req.force_version:
ver_time = latest_ver.created_at
if ver_time.tzinfo is None:
ver_time = ver_time.replace(tzinfo=timezone.utc)
age = (now - ver_time).total_seconds()
if age < VERSION_COALESCE_SECONDS:
# Update the existing version in-place
latest_ver.content = req.content
latest_ver.content = incoming_content
latest_ver.created_at = now
if req.summary:
latest_ver.summary = req.summary
@@ -571,14 +664,14 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
id=str(uuid.uuid4()),
document_id=doc_id,
version_number=new_ver,
content=req.content,
content=incoming_content,
summary=req.summary or "Manual edit",
source="user",
)
doc.version_count = new_ver
db.add(ver)
doc.current_content = req.content
doc.current_content = incoming_content
db.commit()
db.refresh(doc)
return _doc_to_dict(doc)
@@ -606,16 +699,18 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
doc.language = req.language
if req.session_id is not None:
# Empty string = unlink from session
if req.session_id:
_get_session_or_404(db, req.session_id, user)
doc.session_id = req.session_id if req.session_id else None
if not req.session_id:
# Tab closed / doc detached from its session — drop the
# in-memory active-doc pointer so the last-resort injection
# path doesn't re-surface this doc in a later chat (#1160).
try:
from src.tool_implementations import clear_active_document
from src.agent_tools.document_tools import clear_active_document
clear_active_document(doc_id)
except Exception:
pass
except Exception as e:
logger.warning("Failed to clear active document %r on detach", doc_id, exc_info=e)
db.commit()
db.refresh(doc)
return _doc_to_dict(doc)
@@ -641,7 +736,7 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
# Closed/deleted — drop the in-memory active-doc pointer so it isn't
# re-injected into a later, unrelated chat (#1160).
try:
from src.tool_implementations import clear_active_document
from src.agent_tools.document_tools import clear_active_document
clear_active_document(doc_id)
except Exception:
pass
@@ -663,8 +758,9 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
try:
# Verify ownership before listing versions
doc = db.query(Document).filter(Document.id == doc_id).first()
if doc:
_verify_doc_owner(db, doc, user)
if not doc:
raise HTTPException(404, "Document not found")
_verify_doc_owner(db, doc, user)
versions = db.query(DocumentVersion).filter(
DocumentVersion.document_id == doc_id
).order_by(DocumentVersion.version_number.desc()).all()
@@ -687,8 +783,9 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
try:
# Verify ownership
doc = db.query(Document).filter(Document.id == doc_id).first()
if doc:
_verify_doc_owner(db, doc, user)
if not doc:
raise HTTPException(404, "Document not found")
_verify_doc_owner(db, doc, user)
ver = db.query(DocumentVersion).filter(
DocumentVersion.document_id == doc_id,
DocumentVersion.version_number == num,
@@ -764,10 +861,26 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
from src.document_actions import _JUNK_TITLES
to_delete = []
now = datetime.now(timezone.utc)
for doc in docs:
created = doc.created_at
if created and created.tzinfo is None:
created = created.replace(tzinfo=timezone.utc)
# Skip freshly created documents to avoid deleting them while the user is actively editing
if created and (now - created).total_seconds() < 900: # 15 minutes
continue
content = (doc.current_content or "").strip()
title_raw = (doc.title or "").strip()
title = title_raw.lower()
is_fresh_empty = (
not content
and created is not None
and (now - created).total_seconds() < 1800
)
if is_fresh_empty:
continue
# Strip markdown noise to get a "real" character count
stripped = _re.sub(r"^#{1,6}\s+", "", content, flags=_re.MULTILINE)
@@ -802,10 +915,6 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
to_delete.append(doc); deleted += 1; continue
if title in _JUNK_TITLES:
to_delete.append(doc); deleted += 1; continue
if real_len < 30:
to_delete.append(doc); deleted += 1; continue
if "\n" not in content and real_len < 50:
to_delete.append(doc); deleted += 1; continue
# Fix empty or placeholder titles on survivors
if not title_raw or title_raw == "Untitled":
@@ -853,10 +962,10 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
from src.llm_core import llm_call_async
user = get_current_user(request)
url, model, headers = resolve_task_endpoint()
url, model, headers = resolve_task_endpoint(owner=user or None)
if not url or not model:
# Fall back to default endpoint
url, model, headers = resolve_endpoint("default")
url, model, headers = resolve_endpoint("default", owner=user or None)
if not url or not model:
raise HTTPException(500, "No endpoint configured for AI tidy")
@@ -1156,7 +1265,7 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
settings = _load_vl_settings()
vl_model = settings.get("vision_model", "")
try:
url, model_id, headers = _resolve_vl_model(vl_model)
url, model_id, headers = _resolve_vl_model(vl_model, owner=user)
except Exception as e:
raise HTTPException(503, f"No vision model available: {e}")
@@ -1298,6 +1407,12 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
if not pdf_path:
raise HTTPException(404, f"Source PDF {upload_id} not found")
# Fail fast with a clear 503 if the optional PyMuPDF dependency
# is missing — fill_fields/stamp_annotations will otherwise
# raise RuntimeError deep inside and bubble out as a 500.
# Mirrors the convention in _load_pdf_viewer_fitz above.
_load_pdf_viewer_fitz()
values = parse_markdown_to_values(doc.current_content or "")
out_path = tempfile.NamedTemporaryFile(suffix=".pdf", delete=False).name
_to_unlink.append(out_path)
@@ -1510,10 +1625,7 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
# don't import from a routes file (cycle-prone). Same env override
# as email_routes (ODYSSEUS_MAIL_ATTACHMENTS_DIR).
from pathlib import Path as _Path
import os as _os
_DATA_DIR = _Path(__file__).resolve().parent.parent / "data"
_BASE = _os.environ.get("ODYSSEUS_MAIL_ATTACHMENTS_DIR", str(_DATA_DIR / "mail-attachments"))
_COMPOSE_DIR = _Path(_BASE) / "_compose"
_COMPOSE_DIR = _Path(MAIL_ATTACHMENTS_DIR) / "_compose"
_COMPOSE_DIR.mkdir(parents=True, exist_ok=True)
user = get_current_user(request)
@@ -1629,9 +1741,11 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
# context (To/Subject/In-Reply-To/References).
try:
from routes.email_routes import _imap, _decode_header
from routes.email_helpers import _q
except Exception:
_imap = None
_decode_header = lambda x: x or ""
_q = lambda x: x or ""
to_addr = ""
from_name = ""
@@ -1641,7 +1755,7 @@ def setup_document_routes(session_manager, upload_handler=None) -> APIRouter:
if _imap:
try:
with _imap(doc.source_email_account_id or None) as conn:
conn.select(doc.source_email_folder, readonly=True)
conn.select(_q(doc.source_email_folder), readonly=True)
status, data = conn.fetch(doc.source_email_uid.encode(), "(RFC822.HEADER)")
if status == "OK" and data and data[0]:
raw_hdr = data[0][1]

598
routes/email_helpers.py
View File

@@ -13,6 +13,8 @@ and `email_pollers.py` (the background loops):
"""
import os
import base64
import time
import imaplib
import smtplib
import email as email_mod
@@ -38,6 +40,116 @@ from src.secret_storage import decrypt as _decrypt
logger = logging.getLogger(__name__)
class EmailNotConfiguredError(RuntimeError):
"""Raised when an IMAP operation is attempted on an account that has no
inbox configured (e.g. a send-only / SMTP-only account).
Subclasses RuntimeError so existing broad ``except Exception`` handlers
keep working; callers that want to treat "no inbox" as an empty result
rather than a failure can catch this type specifically.
"""
def _xoauth2_raw(user: str, access_token: str) -> str:
"""The SASL XOAUTH2 initial-response string (unencoded).
Both smtplib.SMTP.auth() and imaplib.IMAP4.authenticate() base64-encode
the value their callback returns, so callers pass this raw form — never
pre-encoded — to avoid double base64.
"""
return f"user={user}\x01auth=Bearer {access_token}\x01\x01"
def _xoauth2_bytes(user: str, access_token: str) -> bytes:
"""Raw XOAUTH2 bytes for imaplib's authenticate() callback."""
return _xoauth2_raw(user, access_token).encode()
def make_oauth_state(account_id: str, owner: str) -> str:
"""Return an HMAC-signed, base64-encoded OAuth state token.
Encodes account_id + owner + a random nonce, signed with the app secret
so the callback can validate that the flow was initiated by an
authenticated, owning user (CSRF / state-forgery protection).
"""
import hmac as _hmac, hashlib as _hl, secrets as _sec
from src.secret_storage import _load_or_create_key
nonce = _sec.token_hex(16)
payload = json.dumps({"a": account_id, "o": owner, "n": nonce}, separators=(",", ":"))
sig = _hmac.new(_load_or_create_key(), payload.encode(), _hl.sha256).hexdigest()
return base64.urlsafe_b64encode(f"{payload}|{sig}".encode()).decode()
def verify_oauth_state(state: str) -> dict | None:
"""Verify an OAuth state token's HMAC signature.
Returns the decoded payload dict ({"a", "o", "n"}) on success, or None if
the token is malformed, tampered, or signed with a different key.
"""
import hmac as _hmac, hashlib as _hl
from src.secret_storage import _load_or_create_key
try:
decoded = base64.urlsafe_b64decode(state.encode()).decode()
payload, sig = decoded.rsplit("|", 1)
expected = _hmac.new(_load_or_create_key(), payload.encode(), _hl.sha256).hexdigest()
if not _hmac.compare_digest(sig, expected):
return None
return json.loads(payload)
except Exception:
return None
def _refresh_google_token(account_id: str) -> str | None:
"""Exchange the stored refresh token for a new access token and persist it."""
import httpx
from core.database import SessionLocal as _SL, EmailAccount as _EA
from src.secret_storage import encrypt as _enc, decrypt as _dec
client_id = os.environ.get("GOOGLE_OAUTH_CLIENT_ID", "")
client_secret = os.environ.get("GOOGLE_OAUTH_CLIENT_SECRET", "")
if not client_id or not client_secret:
return None
db = _SL()
try:
row = db.get(_EA, account_id)
if not row or not row.oauth_refresh_token:
return None
refresh_token = _dec(row.oauth_refresh_token or "")
if not refresh_token:
return None
resp = httpx.post("https://oauth2.googleapis.com/token", data={
"client_id": client_id,
"client_secret": client_secret,
"refresh_token": refresh_token,
"grant_type": "refresh_token",
}, timeout=10)
resp.raise_for_status()
data = resp.json()
access_token = data["access_token"]
row.oauth_access_token = _enc(access_token)
row.oauth_token_expiry = str(int(time.time()) + data.get("expires_in", 3600))
db.commit()
return access_token
except Exception:
logger.warning(f"Google token refresh failed for account {account_id}")
return None
finally:
db.close()
def _get_valid_google_token(account_id: str, cfg: dict) -> str | None:
"""Return a valid Google access token, refreshing if expired or missing."""
from src.secret_storage import decrypt as _dec
access_token = _dec(cfg.get("oauth_access_token") or "")
expiry_str = cfg.get("oauth_token_expiry") or ""
if access_token and expiry_str:
try:
if int(expiry_str) - 60 > time.time():
return access_token
except (ValueError, TypeError):
pass
return _refresh_google_token(account_id)
def _smtp_security_mode(cfg: dict) -> str:
raw = str(cfg.get("smtp_security") or "").strip().lower()
if raw in {"ssl", "starttls", "none"}:
@@ -54,23 +166,64 @@ def _send_smtp_message(cfg: dict, from_addr: str, recipients: list[str], message
port = int(cfg.get("smtp_port") or 465)
user = cfg.get("smtp_user") or ""
password = cfg.get("smtp_password") or ""
def _auth_smtp(smtp):
if cfg.get("oauth_provider") == "google":
token = _get_valid_google_token(cfg.get("account_id"), cfg)
if not token:
raise RuntimeError("Google OAuth token unavailable — reconnect the account")
smtp.ehlo()
smtp.auth("XOAUTH2", lambda challenge=None: _xoauth2_raw(user, token), initial_response_ok=True)
elif user and password:
smtp.login(user, password)
security = _smtp_security_mode(cfg)
if security == "ssl":
with smtplib.SMTP_SSL(host, port, timeout=timeout) as smtp:
if user and password:
smtp.login(user, password)
_auth_smtp(smtp)
smtp.sendmail(from_addr, recipients, message)
return
with smtplib.SMTP(host, port, timeout=timeout) as smtp:
if security == "starttls":
smtp.starttls()
if user and password:
smtp.login(user, password)
_auth_smtp(smtp)
smtp.sendmail(from_addr, recipients, message)
def _friendly_email_auth_error(protocol: str, host: str, error: object) -> str:
"""Return a clearer setup error for known provider auth policies."""
raw = str(error or "")
lower = raw.lower()
host_lower = (host or "").lower()
microsoft_host = any(
marker in host_lower
for marker in (
"outlook.office365.com",
"smtp.office365.com",
"office365.com",
"outlook.com",
"hotmail.com",
"live.com",
)
)
microsoft_basic_auth_failure = (
"5.7.139" in lower
or "basic authentication is disabled" in lower
or ("authenticate failed" in lower and microsoft_host)
or ("authentication unsuccessful" in lower and microsoft_host)
)
if microsoft_basic_auth_failure:
return (
"Microsoft no longer accepts normal mailbox passwords for "
"Outlook/Office 365 IMAP/SMTP in most accounts. Odysseus "
"does not support Microsoft OAuth/Graph mail yet, so Outlook "
"accounts cannot be added with this password form."
)
return raw[:200]
def _strip_think(text: str) -> str:
"""Email-flavored think strip — thin wrapper over the central helper.
@@ -82,8 +235,9 @@ def _strip_think(text: str) -> str:
"""
if not text:
return ""
from src.text_helpers import strip_think as _central, _THINK_CLOSED_RE, _THINK_OPEN_RE, _THINK_TAG_RE
had_think = bool(_THINK_CLOSED_RE.search(text) or _THINK_OPEN_RE.search(text) or _THINK_TAG_RE.search(text))
from src.text_helpers import strip_think as _central, _THINK_TAG_RE
# Single linear tag check; the old closed/open `.search()` calls could ReDoS.
had_think = bool(_THINK_TAG_RE.search(text))
return _central(text, prose=had_think, prompt_echo=True)
@@ -195,7 +349,7 @@ def _assert_owns_account(account_id: str, owner: str) -> None:
row = db.query(_EA).filter(_EA.id == account_id).first()
if row is None:
raise HTTPException(404, "Account not found")
if row.owner and row.owner != owner:
if not _account_visible_to_owner(row, owner):
# Treat as 404 (not 403) so we don't leak existence.
raise HTTPException(404, "Account not found")
finally:
@@ -208,6 +362,26 @@ def _assert_owns_account(account_id: str, owner: str) -> None:
logger.error(f"Account-owner check failed: {e}")
raise HTTPException(503, "Account check failed")
def _account_visible_to_owner(row, owner: str) -> bool:
"""Whether an authenticated `owner` may act on this EmailAccount row.
Mirrors the SQL predicate in `_get_email_config`'s
`_owner_or_matching_legacy_account`: a caller sees an account they own, or a
legacy owner-less account (owner NULL/"") only when its own mailbox
(`imap_user` / `from_address`) is the caller's. `email_accounts` is the one
owner-scoped table deliberately left out of the legacy-owner migration
backfill, so ownerless rows persist on multi-user deploys — making this the
gate that keeps one tenant off another's imported mailbox and its decrypted
IMAP/SMTP credentials."""
row_owner = getattr(row, "owner", None) or ""
if row_owner:
return row_owner == owner
return owner in {
getattr(row, "imap_user", None) or "",
getattr(row, "from_address", None) or "",
}
def _q(name: str) -> str:
"""Quote an IMAP mailbox name. Defensive: escapes `\\` and `"` and wraps
in double quotes so user-supplied folder names with spaces or quotes can't
@@ -254,26 +428,35 @@ def _cleanup_compose_uploads(tokens) -> None:
pass
DATA_DIR = Path(__file__).resolve().parent.parent / "data"
SETTINGS_FILE = DATA_DIR / "settings.json"
from src.constants import DATA_DIR as _DATA_DIR, MAIL_ATTACHMENTS_DIR, SETTINGS_FILE as _SETTINGS_FILE, SCHEDULED_EMAILS_DB
DATA_DIR = Path(_DATA_DIR)
SETTINGS_FILE = Path(_SETTINGS_FILE)
# Override at deploy time via ODYSSEUS_MAIL_ATTACHMENTS_DIR. Defaults to a
# subdir of the install's data/ tree so the app works out-of-the-box without
# a hardcoded /home/<user>/ path.
ATTACHMENTS_DIR = Path(os.environ.get("ODYSSEUS_MAIL_ATTACHMENTS_DIR", str(DATA_DIR / "mail-attachments")))
ATTACHMENTS_DIR = Path(MAIL_ATTACHMENTS_DIR)
ATTACHMENTS_DIR.mkdir(parents=True, exist_ok=True)
COMPOSE_UPLOADS_DIR = ATTACHMENTS_DIR / "_compose"
COMPOSE_UPLOADS_DIR.mkdir(parents=True, exist_ok=True)
SCHEDULED_DB = DATA_DIR / "scheduled_emails.db"
SCHEDULED_DB = Path(SCHEDULED_EMAILS_DB)
OWNER_SCOPED_EMAIL_CACHE_TABLES = {
"email_summaries",
"email_ai_replies",
"email_translations",
"email_calendar_extractions",
"email_urgency_alerts",
"sender_signatures",
}
def email_translation_body_hash(body: str) -> str:
import hashlib as _hashlib
normalized = (body or "").strip()
return _hashlib.sha256(normalized.encode("utf-8", errors="ignore")).hexdigest()
def _email_cache_owner_clause(owner: str = "") -> tuple[str, tuple[str, ...]]:
owner = (owner or "").strip()
if owner:
@@ -281,14 +464,34 @@ def _email_cache_owner_clause(owner: str = "") -> tuple[str, tuple[str, ...]]:
return "(owner = '' OR owner IS NULL)", ()
def _ensure_owner_scoped_email_cache_table(conn, table: str, create_sql: str, columns: list[str]):
def _ensure_owner_scoped_email_cache_table(
conn,
table: str,
create_sql: str,
columns: list[str],
pk_columns: list[str] | None = None,
):
"""Rebuild legacy Message-ID-only cache tables with owner in the PK."""
desired_pk_cols = pk_columns or ["message_id", "owner"]
conn.execute(create_sql)
try:
info = conn.execute(f"PRAGMA table_info({table})").fetchall()
cols = [r[1] for r in info]
pk_cols = [r[1] for r in sorted((r for r in info if r[5]), key=lambda r: r[5])]
if "owner" in cols and pk_cols == ["message_id", "owner"]:
for col in columns:
if col not in cols:
if col == "owner":
conn.execute(f"ALTER TABLE {table} ADD COLUMN owner TEXT DEFAULT ''")
elif col in {"event_uids"}:
conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} TEXT DEFAULT '[]'")
elif col.startswith("has_") or col.endswith("_created") or col.endswith("_count"):
conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} INTEGER DEFAULT 0")
elif col == "created_at":
conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} TEXT DEFAULT ''")
else:
conn.execute(f"ALTER TABLE {table} ADD COLUMN {col} TEXT")
cols.append(col)
if "owner" in cols and pk_cols == desired_pk_cols:
return
conn.execute(f"ALTER TABLE {table} RENAME TO {table}__old")
@@ -308,6 +511,55 @@ def _ensure_owner_scoped_email_cache_table(conn, table: str, create_sql: str, co
_lg.getLogger(__name__).warning(f"{table} owner-migration skipped: {_mig_e}")
def _ensure_sender_signatures_table(conn):
"""Create/migrate learned sender signatures to an owner-scoped cache."""
create_sql = """
CREATE TABLE IF NOT EXISTS sender_signatures (
from_address TEXT,
owner TEXT DEFAULT '',
signature_text TEXT,
sample_count INTEGER,
last_built_at TEXT NOT NULL,
model_used TEXT,
source TEXT,
PRIMARY KEY (from_address, owner)
)
"""
conn.execute(create_sql)
try:
info = conn.execute("PRAGMA table_info(sender_signatures)").fetchall()
cols = [r[1] for r in info]
pk_cols = [r[1] for r in sorted((r for r in info if r[5]), key=lambda r: r[5])]
if "owner" in cols and pk_cols == ["from_address", "owner"]:
return
conn.execute("ALTER TABLE sender_signatures RENAME TO sender_signatures__old")
conn.execute(create_sql)
old_cols = [r[1] for r in conn.execute("PRAGMA table_info(sender_signatures__old)").fetchall()]
copy_cols = [
c for c in (
"from_address",
"signature_text",
"sample_count",
"last_built_at",
"model_used",
"source",
)
if c in old_cols
]
source_owner = "COALESCE(owner, '')" if "owner" in old_cols else "''"
conn.execute(
f"INSERT OR IGNORE INTO sender_signatures "
f"({', '.join([*copy_cols, 'owner'])}) "
f"SELECT {', '.join([*copy_cols, source_owner])} "
f"FROM sender_signatures__old"
)
conn.execute("DROP TABLE sender_signatures__old")
except Exception as _mig_e:
import logging as _lg
_lg.getLogger(__name__).warning(f"sender_signatures owner-migration skipped: {_mig_e}")
def attachment_extract_dir(folder: str, uid: str) -> Path:
"""Containment-safe extraction directory for an attachment.
@@ -372,6 +624,25 @@ def _init_scheduled_db():
PRIMARY KEY (message_id, owner)
)
""", ["message_id", "owner", "uid", "folder", "reply", "model_used", "created_at"])
_ensure_owner_scoped_email_cache_table(conn, "email_translations", """
CREATE TABLE IF NOT EXISTS email_translations (
body_hash TEXT,
owner TEXT DEFAULT '',
target_language TEXT DEFAULT 'English',
uid TEXT,
folder TEXT,
subject TEXT,
sender TEXT,
translation TEXT,
same_language INTEGER DEFAULT 0,
model_used TEXT,
created_at TEXT NOT NULL,
PRIMARY KEY (body_hash, owner, target_language)
)
""", [
"body_hash", "owner", "target_language", "uid", "folder", "subject", "sender",
"translation", "same_language", "model_used", "created_at",
], ["body_hash", "owner", "target_language"])
# Email tags / spam classification cache. SECURITY: keyed by
# (message_id, owner) because Message-IDs are GLOBAL (a newsletter goes
# to many users with the same Message-ID). Without owner-scoping, a
@@ -381,6 +652,7 @@ def _init_scheduled_db():
CREATE TABLE IF NOT EXISTS email_tags (
message_id TEXT,
owner TEXT DEFAULT '',
account_id TEXT DEFAULT '',
uid TEXT,
folder TEXT,
subject TEXT,
@@ -391,7 +663,7 @@ def _init_scheduled_db():
moved_to TEXT,
model_used TEXT,
created_at TEXT NOT NULL,
PRIMARY KEY (message_id, owner)
PRIMARY KEY (message_id, owner, account_id)
)
""")
# Backfill migration: older installs created the table with
@@ -399,28 +671,35 @@ def _init_scheduled_db():
# promote it into the PK by rebuild-copy-swap (SQLite can't ALTER PK).
try:
_cols = [r[1] for r in conn.execute("PRAGMA table_info(email_tags)")]
_pk_cols = [r[1] for r in sorted(conn.execute("PRAGMA table_info(email_tags)").fetchall(), key=lambda row: row[5] or 99) if r[5]]
if "owner" not in _cols:
# Add the column first so reads/writes don't break mid-migration.
conn.execute("ALTER TABLE email_tags ADD COLUMN owner TEXT DEFAULT ''")
# Rebuild with composite PK. Existing rows get owner='' (legacy
# single-user); the urgency scanner will overwrite as it
# re-classifies. No data loss.
_cols.append("owner")
if "account_id" not in _cols:
conn.execute("ALTER TABLE email_tags ADD COLUMN account_id TEXT DEFAULT ''")
_cols.append("account_id")
if _pk_cols != ["message_id", "owner", "account_id"]:
# Rebuild with account-aware composite PK. Existing rows get
# account_id='' and are still readable as legacy fallback rows;
# fresh task runs write exact account ids and no longer block each
# other when two accounts share a Message-ID.
conn.execute("""
CREATE TABLE IF NOT EXISTS email_tags__new (
message_id TEXT,
owner TEXT DEFAULT '',
account_id TEXT DEFAULT '',
uid TEXT, folder TEXT, subject TEXT, sender TEXT,
tags TEXT, spam_verdict INTEGER DEFAULT 0,
spam_reason TEXT, moved_to TEXT, model_used TEXT,
created_at TEXT NOT NULL,
PRIMARY KEY (message_id, owner)
PRIMARY KEY (message_id, owner, account_id)
)
""")
conn.execute("""
INSERT OR IGNORE INTO email_tags__new
(message_id, owner, uid, folder, subject, sender, tags,
(message_id, owner, account_id, uid, folder, subject, sender, tags,
spam_verdict, spam_reason, moved_to, model_used, created_at)
SELECT message_id, COALESCE(owner, ''), uid, folder, subject,
SELECT message_id, COALESCE(owner, ''), COALESCE(account_id, ''), uid, folder, subject,
sender, tags, spam_verdict, spam_reason, moved_to,
model_used, created_at
FROM email_tags
@@ -436,11 +715,12 @@ def _init_scheduled_db():
message_id TEXT,
owner TEXT DEFAULT '',
uid TEXT,
event_uids TEXT DEFAULT '[]',
events_created INTEGER DEFAULT 0,
created_at TEXT NOT NULL,
PRIMARY KEY (message_id, owner)
)
""", ["message_id", "owner", "uid", "events_created", "created_at"])
""", ["message_id", "owner", "uid", "event_uids", "events_created", "created_at"])
_ensure_owner_scoped_email_cache_table(conn, "email_urgency_alerts", """
CREATE TABLE IF NOT EXISTS email_urgency_alerts (
message_id TEXT,
@@ -466,6 +746,64 @@ def _init_scheduled_db():
PRIMARY KEY (owner, account_key, folder, message_key)
)
""")
conn.execute("""
CREATE TABLE IF NOT EXISTS email_message_index (
owner TEXT NOT NULL DEFAULT '',
account_key TEXT NOT NULL DEFAULT '',
folder TEXT NOT NULL,
uid TEXT NOT NULL,
message_id TEXT,
subject TEXT,
from_name TEXT,
from_address TEXT,
to_text TEXT,
cc_text TEXT,
date_iso TEXT,
date_display TEXT,
date_epoch REAL DEFAULT 0,
size INTEGER DEFAULT 0,
flags TEXT DEFAULT '',
has_attachments INTEGER DEFAULT 0,
updated_at TEXT NOT NULL,
PRIMARY KEY (owner, account_key, folder, uid)
)
""")
conn.execute("""
CREATE INDEX IF NOT EXISTS ix_email_message_index_folder_date
ON email_message_index(owner, account_key, folder, date_epoch DESC)
""")
conn.execute("""
CREATE INDEX IF NOT EXISTS ix_email_message_index_message_id
ON email_message_index(owner, account_key, message_id)
""")
conn.execute("""
CREATE TABLE IF NOT EXISTS email_body_preview_cache (
owner TEXT NOT NULL DEFAULT '',
account_key TEXT NOT NULL DEFAULT '',
folder TEXT NOT NULL,
uid TEXT NOT NULL,
message_id TEXT,
payload_json TEXT NOT NULL,
updated_at TEXT NOT NULL,
PRIMARY KEY (owner, account_key, folder, uid)
)
""")
conn.execute("""
CREATE INDEX IF NOT EXISTS ix_email_body_preview_message_id
ON email_body_preview_cache(owner, account_key, message_id)
""")
conn.execute("""
CREATE TABLE IF NOT EXISTS email_attachment_metadata_cache (
owner TEXT NOT NULL DEFAULT '',
account_key TEXT NOT NULL DEFAULT '',
folder TEXT NOT NULL,
uid TEXT NOT NULL,
message_id TEXT,
attachments_json TEXT NOT NULL,
updated_at TEXT NOT NULL,
PRIMARY KEY (owner, account_key, folder, uid)
)
""")
# Boundary cache — LLM-detected sig/quote start positions in the body.
# Stored as char offsets (-1 = no boundary found). Once cached, the
# client uses these to fold without ever re-calling the LLM.
@@ -526,20 +864,10 @@ def _init_scheduled_db():
conn.execute("ALTER TABLE email_boundaries ADD COLUMN turns_json TEXT")
except Exception:
pass
# Per-sender signature cache. Populated by `learn_sender_signatures`
# action: the LLM extracts the common trailing block across N emails
# from each sender; the renderer folds it consistently for every
# future email from that address.
conn.execute("""
CREATE TABLE IF NOT EXISTS sender_signatures (
from_address TEXT PRIMARY KEY,
signature_text TEXT,
sample_count INTEGER,
last_built_at TEXT NOT NULL,
model_used TEXT,
source TEXT
)
""")
# Per-sender signature cache. Populated by `learn_sender_signatures`.
# Message sender addresses are global, so signatures must be scoped to the
# mailbox owner before `/read` returns them to the renderer.
_ensure_sender_signatures_table(conn)
conn.commit()
conn.close()
@@ -595,12 +923,13 @@ def _get_email_config(account_id: str | None = None, owner: str = "") -> dict:
try:
if account_id:
row = db.query(_EA).filter(_EA.id == account_id, _EA.enabled == True).first() # noqa: E712
# If the resolved row belongs to a different owner, treat as
# If the resolved row isn't visible to this owner, treat as
# not-found rather than silently serving it. This is a defense
# in depth — `require_owner` already calls `_assert_owns_account`
# for query-param account_ids, but other callers (cookbook
# rules, scheduled poller) may not.
if row is not None and owner and row.owner and row.owner != owner:
# rules, scheduled poller) may not. Ownerless legacy rows are
# only visible on a mailbox match, same as the fallback below.
if row is not None and owner and not _account_visible_to_owner(row, owner):
row = None
# Fallback path — restrict to this owner's accounts so we don't
# leak another user's default mailbox to an unconfigured user.
@@ -628,10 +957,16 @@ def _get_email_config(account_id: str | None = None, owner: str = "") -> dict:
"imap_password": _decrypt(row.imap_password or ""),
"imap_starttls": bool(row.imap_starttls),
"from_address": row.from_address or row.imap_user or "",
"oauth_provider": row.oauth_provider or "",
"oauth_access_token": row.oauth_access_token or "",
"oauth_refresh_token": row.oauth_refresh_token or "",
"oauth_token_expiry": row.oauth_token_expiry or "",
"display_name": row.display_name or "",
}
if not (cfg["smtp_host"] and cfg["smtp_user"] and cfg["smtp_password"]):
is_oauth = bool(cfg.get("oauth_provider"))
if not is_oauth and not (cfg["smtp_host"] and cfg["smtp_user"] and cfg["smtp_password"]):
logger.warning(f"SMTP not configured for account {row.name!r}")
if not (cfg["imap_host"] and cfg["imap_user"] and cfg["imap_password"]):
if not is_oauth and not (cfg["imap_host"] and cfg["imap_user"] and cfg["imap_password"]):
logger.warning(f"IMAP not configured for account {row.name!r}")
return cfg
finally:
@@ -705,7 +1040,16 @@ def _open_imap_connection(host: str, port: int, *, starttls: bool, timeout: int
port = int(port or 993)
if starttls:
conn = imaplib.IMAP4(host, port, timeout=timeout)
conn.starttls()
try:
conn.starttls()
except Exception:
# Don't leak the open plain socket if the STARTTLS upgrade is
# rejected; close it before propagating. (#3174)
try:
conn.shutdown()
except Exception:
pass
raise
elif port == 993:
conn = imaplib.IMAP4_SSL(host, port, timeout=timeout)
else:
@@ -714,13 +1058,29 @@ def _open_imap_connection(host: str, port: int, *, starttls: bool, timeout: int
conn.sock.settimeout(timeout)
except Exception:
pass
# Raise the IMAP line-length limit from the default 1 MB to 50 MB so that
# large mailboxes (tens of thousands of messages) don't crash with
# "got more than 1000000 bytes" on UID SEARCH ALL. (#2883)
imaplib._MAXLINE = 50_000_000
return conn
def _imap_connect(account_id: str | None = None, owner: str = ""):
def _imap_connect(account_id: str | None = None, owner: str = "",
timeout: int = _IMAP_TIMEOUT_SECONDS):
# SECURITY: passing `owner` scopes the fallback config lookup so a brand
# new user doesn't get connected against another user's default mailbox
# when they have no account configured.
#
# `timeout` is overridable so short-lived callers (e.g. the service-health
# probe) can impose a tighter budget than the default IMAP timeout.
cfg = _get_email_config(account_id, owner=owner)
# Send-only (SMTP-only) account: no IMAP host means there is no inbox to
# read. Bail out with a clear, typed error instead of handing an empty
# host to imaplib — IMAP4("", 993) silently dials localhost:993 and fails
# with a confusing "[Errno 111] Connection refused" on every inbox poll.
if not cfg.get("imap_host"):
raise EmailNotConfiguredError(
f"IMAP is not configured for account {cfg.get('account_name') or 'default'!r}"
)
# Connection mode:
# STARTTLS on → plain + upgrade
# STARTTLS off + port 993 → implicit SSL (IMAPS)
@@ -732,9 +1092,27 @@ def _imap_connect(account_id: str | None = None, owner: str = ""):
cfg["imap_host"],
cfg["imap_port"],
starttls=bool(cfg.get("imap_starttls")),
timeout=_IMAP_TIMEOUT_SECONDS,
timeout=timeout,
)
conn.login(cfg["imap_user"], cfg["imap_password"])
try:
if cfg.get("oauth_provider") == "google":
token = _get_valid_google_token(cfg.get("account_id"), cfg)
if not token:
raise RuntimeError("Google OAuth token unavailable — reconnect the account in Settings → Integrations")
conn.authenticate("XOAUTH2", lambda x: _xoauth2_bytes(cfg["imap_user"], token))
else:
conn.login(cfg["imap_user"], cfg["imap_password"])
except Exception:
# A failed AUTHENTICATE (e.g. an Office 365 app password on an
# MFA-enabled tenant, #3174, or an expired/revoked OAuth token)
# otherwise orphans the already-connected socket; close it before
# propagating so a misconfigured account can't leak one descriptor
# per retry / background poller pass.
try:
conn.shutdown()
except Exception:
pass
raise
return conn
@@ -798,20 +1176,28 @@ def _imap(account_id: str | None = None, owner: str = ""):
def _decode_header(raw):
if not raw:
return ""
parts = email.header.decode_header(raw)
decoded = []
for data, charset in parts:
if isinstance(data, bytes):
try:
decoded.append(data.decode(charset or "utf-8", errors="replace"))
except (LookupError, ValueError):
# Unknown/invalid MIME charset (e.g. a malformed or spam header
# like =?x-unknown-charset?B?...?=). errors="replace" only covers
# byte-decode errors, not codec lookup, so fall back to utf-8.
decoded.append(data.decode("utf-8", errors="replace"))
else:
decoded.append(data)
return " ".join(decoded)
try:
# make_header concatenates per RFC 2047: no spurious space between an
# encoded-word and adjacent plain text (plain runs keep their own
# whitespace), and the whitespace between two adjacent encoded-words is
# dropped. The old " ".join produced "Re: Jose"-style double spaces on
# every non-ASCII subject or sender.
return str(email.header.make_header(email.header.decode_header(raw)))
except Exception:
# Malformed header or unknown/invalid MIME charset (e.g. a spam header
# like =?x-unknown-charset?B?...?=) makes make_header raise LookupError;
# fall back to a lossy per-part decode. errors="replace" only covers
# byte-decode errors, not codec lookup, hence the explicit utf-8 retry.
decoded = []
for data, charset in email.header.decode_header(raw):
if isinstance(data, bytes):
try:
decoded.append(data.decode(charset or "utf-8", errors="replace"))
except (LookupError, ValueError):
decoded.append(data.decode("utf-8", errors="replace"))
else:
decoded.append(data)
return "".join(decoded)
def _detect_sent_folder(conn):
@@ -908,10 +1294,15 @@ def _imap_move(uid, dest, src="INBOX", account_id: str | None = None, owner: str
try:
c = _imap_connect(account_id, owner=owner)
c.select(_q(src))
status, _ = c.copy(uid, _q(dest))
# Callers pass a real IMAP UID (from conn.uid("SEARCH", ...)). copy()
# and store() operate on message SEQUENCE NUMBERS, so addressing them
# with a UID moved/deleted the wrong message (or silently no-oped when
# the UID exceeded the message count). Use the UID commands, matching
# the move/delete path in email_routes.py.
status, _ = c.uid("COPY", uid, _q(dest))
if status != "OK":
return False
c.store(uid, "+FLAGS", "\\Deleted")
c.uid("STORE", uid, "+FLAGS", "\\Deleted")
c.expunge()
return True
except Exception as e:
@@ -1000,56 +1391,95 @@ def _list_attachments_from_msg(msg):
return attachments
idx = 0
for part in msg.walk():
if part.is_multipart():
continue
cd = str(part.get("Content-Disposition", ""))
ct = part.get_content_type()
is_attached_email = ct == "message/rfc822" and ("attachment" in cd.lower() or part.get_filename())
if part.is_multipart() and not is_attached_email:
continue
# Skip text/html body parts (only consider real attachments)
if ct in ("text/plain", "text/html") and "attachment" not in cd:
continue
filename = part.get_filename()
if filename:
filename = _decode_header(filename)
if ct == "message/rfc822" and not re.search(r"\.[A-Za-z0-9]{1,8}$", filename):
filename = f"{filename}.eml"
else:
# Inline images, etc. - generate a name
ext = ct.split("/")[-1] if "/" in ct else "bin"
ext = "eml" if ct == "message/rfc822" else (ct.split("/")[-1] if "/" in ct else "bin")
filename = f"attachment_{idx}.{ext}"
payload = part.get_payload(decode=True)
size = len(payload) if payload else 0
if payload is None and ct == "message/rfc822":
try:
payload = part.as_bytes()
except Exception:
payload = b""
size = len(payload) if payload is not None else 0
content_id = (part.get("Content-ID") or "").strip().strip("<>")
attachments.append({
"index": idx,
"filename": filename,
"content_type": ct,
"size": size,
"is_inline": "inline" in cd.lower(),
"content_id": content_id,
})
idx += 1
return attachments
def _is_likely_signature_image_attachment(att: dict) -> bool:
"""Match the reader's inline signature/logo image filter."""
filename = str((att or {}).get("filename") or "").lower()
if not re.search(r"\.(png|jpe?g|gif|bmp|svg|webp)$", filename):
return False
size = int((att or {}).get("size") or 0)
if re.search(r"^image\d{3,}\.(png|jpe?g|gif)$", filename):
return True
if re.search(r"^(signature|logo|sig|footer|banner)[-_\d]*\.(png|jpe?g|gif|svg)$", filename):
return True
return 0 < size < 30 * 1024
def _has_visible_attachments(msg) -> bool:
"""Return True only for attachments the reader will render as chips."""
return any(
not _is_likely_signature_image_attachment(att)
for att in _list_attachments_from_msg(msg)
)
def _extract_attachment_to_disk(msg, index, target_dir):
"""Extract a specific attachment to disk and return the file path."""
if not msg.is_multipart():
return None
idx = 0
for part in msg.walk():
if part.is_multipart():
continue
cd = str(part.get("Content-Disposition", ""))
ct = part.get_content_type()
is_attached_email = ct == "message/rfc822" and ("attachment" in cd.lower() or part.get_filename())
if part.is_multipart() and not is_attached_email:
continue
if ct in ("text/plain", "text/html") and "attachment" not in cd:
continue
if idx == index:
filename = part.get_filename()
if filename:
filename = _decode_header(filename)
if ct == "message/rfc822" and not re.search(r"\.[A-Za-z0-9]{1,8}$", filename):
filename = f"{filename}.eml"
else:
ext = ct.split("/")[-1] if "/" in ct else "bin"
ext = "eml" if ct == "message/rfc822" else (ct.split("/")[-1] if "/" in ct else "bin")
filename = f"attachment_{idx}.{ext}"
# Sanitize
safe_name = re.sub(r"[^\w\s\-.]", "_", filename).strip()
payload = part.get_payload(decode=True)
if not payload:
if payload is None and ct == "message/rfc822":
try:
payload = part.as_bytes()
except Exception:
payload = b""
if payload is None:
return None
target_dir.mkdir(parents=True, exist_ok=True)
filepath = target_dir / safe_name
@@ -1136,13 +1566,9 @@ def _fetch_sender_thread_context(sender_addr: str,
if exclude_uid:
seen_uids.add((exclude_folder or "INBOX", str(exclude_uid)))
conn = None
try:
conn = _imap_connect(account_id, owner=owner)
except Exception as e:
logger.warning(f"sender-thread-context: imap connect failed: {e}")
return ""
try:
for folder in ["INBOX", "Sent", "Archive", "Drafts"]:
if len(blocks) >= limit:
break
@@ -1209,11 +1635,14 @@ def _fetch_sender_thread_context(sender_addr: str,
if atts_text:
lines.append(atts_text)
blocks.append("\n".join(lines))
except Exception as e:
logger.warning(f"sender-thread-context: imap failed: {e}")
finally:
try: conn.close()
except Exception: pass
try: conn.logout()
except Exception: pass
if conn:
try: conn.close()
except Exception: pass
try: conn.logout()
except Exception: pass
if not blocks:
return ""
@@ -1316,6 +1745,7 @@ def _pre_retrieve_context(
if not terms_list:
return context_snippets, terms_list
ctx_conn = None
try:
ctx_conn = _imap_connect(account_id, owner=owner)
for folder in ["INBOX", "Sent", "Archive", "Drafts"]:
@@ -1352,12 +1782,12 @@ def _pre_retrieve_context(
except Exception as _e:
logger.warning(f" search {folder} {term!r} failed: {_e}")
continue
try:
ctx_conn.logout()
except Exception:
pass
except Exception as _e:
logger.warning(f"IMAP context search failed: {_e}")
finally:
if ctx_conn:
try: ctx_conn.logout()
except Exception: pass
try:
from routes.contacts_routes import _fetch_contacts
@@ -1431,6 +1861,10 @@ class SendEmailRequest(BaseModel):
attachments: Optional[List[str]] = None
# Which account to send from. None = default account.
account_id: Optional[str] = None
# Source message for replies. When present, /send marks this exact message
# answered after successful delivery so it leaves undone/reply-soon views.
source_uid: Optional[str] = None
source_folder: Optional[str] = None
# Internal marker for Odysseus-generated mail (e.g. reminder, scheduled).
odysseus_kind: Optional[str] = None
# If true, /send waits for SMTP + Sent append and returns the sent UID.

361
routes/email_pollers.py
View File

@@ -29,7 +29,7 @@ from datetime import datetime
from email.mime.text import MIMEText
from email.mime.multipart import MIMEMultipart
from src.llm_core import llm_call_async
from src.task_endpoint import resolve_task_candidates, task_llm_call_async
from routes.email_helpers import (
_strip_think, _extract_reply, _apply_email_style_mechanics, _load_settings, _save_settings, _get_email_config,
@@ -44,6 +44,46 @@ from routes.email_helpers import (
logger = logging.getLogger(__name__)
# Recovers a `[{"action": ...}, ...]` JSON array from raw LLM output when the
# fenced-block strip leaves nothing usable. Runs on model output influenced by
# untrusted email bodies, so it must not backtrack: the object content class is
# `[^{}]` (brace-delimited, greedy) rather than the old `[^[\]]*?` lazy runs,
# which exploded exponentially on inputs like `[{"action"},{` + `}},{{` * N
# (CodeQL py/redos #198).
_CAL_ACTION_ARRAY_RE = re.compile(
r'\[\s*\{[^{}]*"action"[^{}]*\}\s*(?:,\s*\{[^{}]*\}\s*)*\]',
re.DOTALL,
)
def _extract_json_array_from_text(text: str):
"""Return the last valid JSON array embedded in model output, if any."""
if not text:
return None
cleaned = re.sub(r"^```(?:json)?\s*|\s*```$", "", text.strip(), flags=re.MULTILINE).strip()
decoder = json.JSONDecoder()
try:
parsed = decoder.decode(cleaned)
if isinstance(parsed, list):
return parsed
except Exception:
pass
# Models often explain themselves and finish with `[]` or `[{"action":...}]`.
# Scan every array opener and keep the last complete JSON array, rather than
# using a greedy regex that can swallow prose containing square brackets.
last = None
for idx, ch in enumerate(cleaned):
if ch != "[":
continue
try:
parsed, _end = decoder.raw_decode(cleaned[idx:])
except Exception:
continue
if isinstance(parsed, list):
last = parsed
return last
def _owner_for_email_account(account_id: str | None) -> str:
if not account_id:
@@ -77,6 +117,8 @@ async def _run_auto_summarize_once(do_summary: bool = True, do_reply: bool = Tru
do_tag: bool = False, do_spam: bool = False,
do_calendar: bool = False,
days_back: int = 1,
account_id: str | None = None,
max_process: int | None = None,
progress_cb=None) -> str:
"""One iteration of the email scan. Temporarily flips settings flags
so the existing background-loop logic runs exactly once for the requested ops."""
@@ -91,7 +133,12 @@ async def _run_auto_summarize_once(do_summary: bool = True, do_reply: bool = Tru
settings["email_auto_calendar"] = bool(do_calendar)
_save_settings(settings)
try:
return await _auto_summarize_pass(days_back=days_back, progress_cb=progress_cb)
return await _auto_summarize_pass(
days_back=days_back,
account_id=account_id,
max_process=max_process,
progress_cb=progress_cb,
)
finally:
s2 = _load_settings()
for k, v in prev.items():
@@ -129,7 +176,7 @@ def _latest_inbox_fallback_uids(conn, reconnect):
return [], reconnect()
async def _auto_summarize_pass(days_back: int = 1, account_id: str | None = None, progress_cb=None) -> str:
async def _auto_summarize_pass(days_back: int = 1, account_id: str | None = None, max_process: int | None = None, progress_cb=None) -> str:
"""Single pass of the auto-summarize/reply scan.
When account_id is None, iterates over every enabled account in
@@ -156,28 +203,41 @@ async def _auto_summarize_pass(days_back: int = 1, account_id: str | None = None
names = {}
if len(ids) <= 1:
# Single-account (or zero rows — fallback to legacy settings.json lookup)
return await _auto_summarize_pass_single(days_back=days_back, account_id=(ids[0] if ids else None), progress_cb=progress_cb)
return await _auto_summarize_pass_single(
days_back=days_back,
account_id=(ids[0] if ids else None),
max_process=max_process,
progress_cb=progress_cb,
)
outs = []
for idx, aid in enumerate(ids, start=1):
try:
await _emit_progress(progress_cb, f"{names.get(aid, aid[:8])}: starting ({idx}/{len(ids)})")
result = await _auto_summarize_pass_single(days_back=days_back, account_id=aid, progress_cb=progress_cb)
result = await _auto_summarize_pass_single(
days_back=days_back,
account_id=aid,
max_process=max_process,
progress_cb=progress_cb,
)
outs.append(f"[{names.get(aid, aid[:8])}] {result}")
except Exception as e:
logger.warning(f"auto-summarize pass failed for account {aid}: {e}")
outs.append(f"[{names.get(aid, aid[:8])}] error: {e}")
return "\n".join(outs)
return await _auto_summarize_pass_single(days_back=days_back, account_id=account_id, progress_cb=progress_cb)
return await _auto_summarize_pass_single(
days_back=days_back,
account_id=account_id,
max_process=max_process,
progress_cb=progress_cb,
)
async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None = None, progress_cb=None) -> str:
async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None = None, max_process: int | None = None, progress_cb=None) -> str:
"""Single pass of the auto-summarize/reply scan for ONE account.
Reads current settings flags."""
import asyncio
import sqlite3 as _sql3
import requests as _req
from src.endpoint_resolver import resolve_endpoint
from src.llm_core import _uses_max_completion_tokens, _restricts_temperature
from src.llm_core import _uses_max_completion_tokens
settings = _load_settings()
auto_sum = settings.get("email_auto_summarize", False)
@@ -210,7 +270,7 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
if auto_cal:
for sent_name in ("Sent", "INBOX/Sent", "Sent Items", "[Gmail]/Sent Mail"):
try:
st, _ = conn.select(sent_name, readonly=True)
st, _ = conn.select(_q(sent_name), readonly=True)
if st == "OK":
folders_to_scan.append(sent_name)
break
@@ -254,9 +314,15 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
).fetchall()}
if auto_tag or auto_spam:
if account_owner:
_tag_existing = {r[0] for r in _c.execute("SELECT message_id FROM email_tags WHERE owner=?", (account_owner,)).fetchall()}
_tag_existing = {r[0] for r in _c.execute(
"SELECT message_id FROM email_tags WHERE owner=? AND (account_id=? OR account_id='' OR account_id IS NULL)",
(account_owner, account_id or ""),
).fetchall()}
else:
_tag_existing = {r[0] for r in _c.execute("SELECT message_id FROM email_tags WHERE owner='' OR owner IS NULL").fetchall()}
_tag_existing = {r[0] for r in _c.execute(
"SELECT message_id FROM email_tags WHERE (owner='' OR owner IS NULL) AND (account_id=? OR account_id='' OR account_id IS NULL)",
(account_id or "",),
).fetchall()}
else:
_tag_existing = set()
_cal_existing = {r[0] for r in _c.execute(
@@ -285,11 +351,10 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
if auto_spam and not spam_folder:
logger.warning("Auto-spam enabled but no Junk/Spam folder detected — will classify but not move")
url, model, headers = resolve_endpoint("utility", owner=account_owner)
if not url:
url, model, headers = resolve_endpoint("default", owner=account_owner)
if not url or not model:
task_candidates = resolve_task_candidates(owner=account_owner)
if not task_candidates:
return "No model configured"
url, model, headers = task_candidates[0]
writing_style = settings.get("email_writing_style", "")
processed = 0
@@ -303,7 +368,14 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
_reply_failed = 0
_detail_lines = []
_current_folder = "INBOX"
_max_process = 5
# Calendar extraction is sequential and each row can involve a model
# call plus a calendar write. Keep the scheduled calendar-only pass
# below the 5-minute action budget instead of timing out mid-run.
_default_max_process = 3 if (auto_cal and not auto_sum and not auto_reply and not auto_tag and not auto_spam) else 5
try:
_max_process = max(1, int(max_process)) if max_process is not None else _default_max_process
except Exception:
_max_process = _default_max_process
for _entry in uid_list:
if processed >= _max_process:
break
@@ -395,48 +467,30 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
req_headers.update(headers)
if need_sum:
tok_key = "max_completion_tokens" if _uses_max_completion_tokens(model) else "max_tokens"
payload = {
"model": model,
"messages": [
{"role": "system", "content": "You are an email summarizer. Format: 1-3 short bullet points (use '- '). Cover: main point, action items, deadlines. If the email has attachments (marked '--- ATTACHMENTS ---'), USE THEIR CONTENTS — pull out invoice totals, deadlines, key clauses, any concrete numbers/dates in PDFs/docs, and reflect them in the bullets. Be terse.\n\nOUTPUT FORMAT: Put ONLY the bullet points between these exact markers, each on its own line:\n<<<SUMMARY>>>\n- ...\n<<<END>>>\nAny reasoning or planning must come BEFORE <<<SUMMARY>>> (ideally inside <think>...</think>). Only the text between the markers is kept."},
{"role": "user", "content": f"From: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}\n\n---\n\nSummarize the email. Output the bullets between <<<SUMMARY>>> and <<<END>>>."},
],
tok_key: 16384,
"temperature": 0.3,
"stream": False,
}
# Reasoning models (o1/o3/o4/gpt-5) reject an explicit temperature.
if _restricts_temperature(model):
payload.pop("temperature", None)
try:
# Use to_thread so this sync HTTP call doesn't freeze
# the entire event loop while the LLM thinks (240s).
resp = await asyncio.to_thread(
_req.post, url, json=payload, headers=req_headers, timeout=240
summary = await task_llm_call_async(
messages=[
{"role": "system", "content": "You are an email summarizer. Format: 1-3 short bullet points (use '- '). Cover: main point, action items, deadlines. If the email has attachments (marked '--- ATTACHMENTS ---'), USE THEIR CONTENTS — pull out invoice totals, deadlines, key clauses, any concrete numbers/dates in PDFs/docs, and reflect them in the bullets. Be terse.\n\nOUTPUT FORMAT: Put ONLY the bullet points between these exact markers, each on its own line:\n<<<SUMMARY>>>\n- ...\n<<<END>>>\nAny reasoning or planning must come BEFORE <<<SUMMARY>>> (ideally inside <think>...</think>). Only the text between the markers is kept."},
{"role": "user", "content": f"From: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}\n\n---\n\nSummarize the email. Output the bullets between <<<SUMMARY>>> and <<<END>>>."},
],
fallback_url=url, fallback_model=model, fallback_headers=headers,
owner=account_owner or None,
temperature=0.3, max_tokens=16384, timeout=240,
)
if resp.ok:
rdata = resp.json()
m = (rdata.get("choices") or [{}])[0].get("message", {})
summary = (m.get("content") or "").strip()
summary = _extract_reply(summary)
if not summary:
rc = (m.get("reasoning_content") or "").strip()
bullets = [ln.strip() for ln in rc.split("\n") if re.match(r"^[-•*]\s+|^\d+[.)]\s+", ln.strip())]
summary = "\n".join(bullets) if bullets else ""
if summary:
_c = _sql3.connect(SCHEDULED_DB)
_c.execute("""
INSERT OR REPLACE INTO email_summaries
(message_id, owner, uid, folder, subject, sender, summary, model_used, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""", (message_id, account_owner or "", uid.decode() if isinstance(uid, bytes) else str(uid), _folder, subject, sender, summary, model, datetime.utcnow().isoformat()))
_c.commit()
_c.close()
_sum_existing.add(message_id)
_summaries_created += 1
_uid_text = uid.decode() if isinstance(uid, bytes) else str(uid)
_detail_lines.append(f"summary · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}")
summary = _extract_reply((summary or "").strip())
if summary:
_c = _sql3.connect(SCHEDULED_DB)
_c.execute("""
INSERT OR REPLACE INTO email_summaries
(message_id, owner, uid, folder, subject, sender, summary, model_used, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)
""", (message_id, account_owner or "", uid.decode() if isinstance(uid, bytes) else str(uid), _folder, subject, sender, summary, model, datetime.utcnow().isoformat()))
_c.commit()
_c.close()
_sum_existing.add(message_id)
_summaries_created += 1
_uid_text = uid.decode() if isinstance(uid, bytes) else str(uid)
_detail_lines.append(f"summary · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}")
except Exception as e:
_uid_text = uid.decode() if isinstance(uid, bytes) else str(uid)
_detail_lines.append(f"summary failed · {_folder}#{_uid_text} · {subject or '(no subject)'} — {sender or '(unknown sender)'}")
@@ -457,14 +511,14 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
if context_snippets:
sys_prompt += "\n\nRELEVANT CONTEXT FROM PAST EMAILS AND CONTACTS:\n" + "\n\n---\n\n".join(context_snippets[:5])
try:
reply = await llm_call_async(
url=url, model=model,
reply = await task_llm_call_async(
messages=[
{"role": "system", "content": sys_prompt},
{"role": "user", "content": f"Original email:\nFrom: {sender}\nSubject: {subject}\n\n{body_for_llm[:12000]}\n\nDraft a reply. Return only the reply body text."},
],
temperature=0.7, max_tokens=1024,
headers=req_headers, timeout=90,
fallback_url=url, fallback_model=model, fallback_headers=headers,
owner=account_owner or None,
temperature=0.7, max_tokens=1024, timeout=90,
)
reply = _apply_email_style_mechanics(_extract_reply(reply or ""))
if reply:
@@ -491,6 +545,8 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
# ── Calendar event extraction (independent of reply drafting) ──
if need_cal:
_cal_run_count = 0
_cal_event_uids = []
_cal_parse_ok = False
try:
# Pull a snapshot of upcoming events so the LLM can decide
# create vs update vs cancel based on what already exists.
@@ -499,8 +555,7 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
_existing_summary = get_upcoming_events(_acct_owner, horizon_days=60, limit=40)
existing_json = json.dumps(_existing_summary)
is_sent = _folder.lower().startswith("sent") or "sent" in _folder.lower()
cal_extract = await llm_call_async(
url=url, model=model,
cal_extract = await task_llm_call_async(
messages=[
{"role": "system", "content": (
"You are a calendar assistant. The user receives emails AND sends replies "
@@ -551,21 +606,22 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
f"{body[:4000]}"
)},
],
temperature=0.1, max_tokens=16384,
headers=req_headers, timeout=180,
fallback_url=url, fallback_model=model, fallback_headers=headers,
owner=account_owner or None,
temperature=0.1, max_tokens=16384, timeout=75,
)
_raw_original = cal_extract or ""
cal_extract = _strip_think(_raw_original)
cal_extract = re.sub(r"^```(?:json)?\s*|\s*```$", "", cal_extract, flags=re.MULTILINE).strip()
if not cal_extract and _raw_original:
matches = list(re.finditer(r'\[\s*\{[^[\]]*?"action"[^[\]]*?\}\s*(?:,\s*\{[^[\]]*?\}\s*)*\]', _raw_original, re.DOTALL))
matches = list(_CAL_ACTION_ARRAY_RE.finditer(_raw_original))
if matches:
cal_extract = matches[-1].group()
logger.info(f"[cal-extract] uid={uid.decode() if isinstance(uid, bytes) else uid} folder={_folder} subj={subject[:50]!r} raw_len={len(cal_extract)} orig_len={len(_raw_original)} raw={cal_extract[:800]!r}")
jm = re.search(r'\[.*\]', cal_extract, re.DOTALL)
if jm:
ops = _extract_json_array_from_text(cal_extract)
if ops is not None:
try:
ops = json.loads(jm.group())
_cal_parse_ok = True
logger.info(f"[cal-extract] parsed {len(ops)} op(s)")
if isinstance(ops, list) and ops:
from src.tool_implementations import do_manage_calendar
@@ -595,6 +651,8 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
r = await do_manage_calendar(json.dumps(args), owner=_acct_owner)
if r.get("exit_code", 0) == 0:
logger.info(f"[cal-extract] Updated event uid={cuid} → {op.get('title')} {op['date']}")
if cuid and cuid not in _cal_event_uids:
_cal_event_uids.append(cuid)
_cal_run_count += 1
else:
logger.warning(f"[cal-extract] update failed: {r.get('error')}")
@@ -675,28 +733,43 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
r = await do_manage_calendar(cal_args, owner=_acct_owner)
if r.get("exit_code", 0) == 0:
logger.info(f"[cal-extract] Created event: {op['title']} on {op['date']}")
_created_uid = (r.get("uid") or "").strip()
if _created_uid and _created_uid not in _cal_event_uids:
_cal_event_uids.append(_created_uid)
_events_created += 1
_cal_run_count += 1
else:
logger.warning(f"[cal-extract] create failed: {r.get('error')} args={cal_args[:200]}")
except Exception as je:
logger.warning(f"[cal-extract] JSON parse failed: {je} on raw={cal_extract[:200]!r}")
else:
logger.warning(f"[cal-extract] no JSON array found on raw={cal_extract[:200]!r}")
except Exception as e:
logger.warning(f"[cal-extract] Meeting extraction LLM call failed for uid={uid}: {e}")
# Record we processed this email so we don't re-LLM next run
try:
_cc = _sql3.connect(SCHEDULED_DB)
_cc.execute(
"INSERT OR REPLACE INTO email_calendar_extractions "
"(message_id, owner, uid, events_created, created_at) VALUES (?, ?, ?, ?, ?)",
(message_id, account_owner or "", uid.decode() if isinstance(uid, bytes) else str(uid),
_cal_run_count, datetime.utcnow().isoformat())
)
_cc.commit()
_cc.close()
_cal_existing.add(message_id)
except Exception as ce:
logger.debug(f"Could not cache calendar extraction: {ce}")
else:
# Record successfully parsed results so we don't re-LLM
# no-op emails. Transient LLM failures are retried on
# the next poll run.
try:
if _cal_parse_ok:
_cc = _sql3.connect(SCHEDULED_DB)
_cc.execute(
"INSERT OR REPLACE INTO email_calendar_extractions "
"(message_id, owner, uid, event_uids, events_created, created_at) VALUES (?, ?, ?, ?, ?, ?)",
(
message_id,
account_owner or "",
uid.decode() if isinstance(uid, bytes) else str(uid),
json.dumps(_cal_event_uids),
_cal_run_count,
datetime.utcnow().isoformat(),
),
)
_cc.commit()
_cc.close()
_cal_existing.add(message_id)
except Exception as ce:
logger.debug(f"Could not cache calendar extraction: {ce}")
if need_urgent:
try:
@@ -728,9 +801,11 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
"temperature": 0,
tok_key: 200,
}
urg_raw = await llm_call_async(
url=url, model=model, messages=payload["messages"],
temperature=0, max_tokens=200, headers=req_headers, timeout=60,
urg_raw = await task_llm_call_async(
messages=payload["messages"],
fallback_url=url, fallback_model=model, fallback_headers=headers,
owner=account_owner or None,
temperature=0, max_tokens=200, timeout=60,
)
urg_raw = _strip_think(urg_raw or "")
urg_raw = re.sub(r"^```(?:json)?\s*|\s*```$", "", urg_raw, flags=re.MULTILINE).strip()
@@ -831,8 +906,13 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
class_sys = (
"Classify the email. Return ONLY a JSON object, no prose, no markdown fences. "
"Schema: {\"tags\": [\"tag1\"], \"spam\": false, \"reason\": \"short\"}. "
"Pick 1-2 tags from: work, personal, finance, bills, receipt, travel, "
"newsletter, promo, notification, security, social, shopping, calendar.\n\n"
"Pick 1-3 tags from: work, personal, urgent, action-needed, finance, bills, "
"receipt, legal, travel, newsletter, promo, notification, security, social, "
"shopping, calendar, support.\n\n"
"Use work for professional/company/client/operations messages. "
"Use personal for friends/family/private-life messages. "
"Use urgent for real time-sensitive consequences. "
"Use action-needed when the user likely needs to reply, pay, sign, book, or decide.\n\n"
"Set spam=true for ANY of:\n"
"- Phishing, scams, chain mail, deceptive offers\n"
"- Marketing/promotional blasts (\"special offer\", \"limited time\", discount codes)\n"
@@ -849,70 +929,55 @@ async def _auto_summarize_pass_single(days_back: int = 1, account_id: str | None
"If it's a mass-mailed generic update with no personal CTA, mark spam=true even if from a legitimate service. "
"Reason should be 5-10 words."
)
tok_key = "max_completion_tokens" if _uses_max_completion_tokens(model) else "max_tokens"
payload = {
"model": model,
"messages": [
raw_out = await task_llm_call_async(
messages=[
{"role": "system", "content": class_sys},
{"role": "user", "content": f"From: {sender}\nSubject: {subject}\n\n{body[:4000]}"},
],
tok_key: 512,
"temperature": 0.1,
"stream": False,
}
# Reasoning models (o1/o3/o4/gpt-5) reject an explicit temperature.
if _restricts_temperature(model):
payload.pop("temperature", None)
# to_thread keeps the event loop responsive during the LLM call
resp = await asyncio.to_thread(
_req.post, url, json=payload, headers=req_headers, timeout=120
fallback_url=url, fallback_model=model, fallback_headers=headers,
owner=account_owner or None,
temperature=0.1, max_tokens=512, timeout=120,
)
if not resp.ok:
logger.warning(f"Auto-classify {uid.decode() if isinstance(uid, bytes) else str(uid)} HTTP {resp.status_code}: {resp.text[:200]}")
else:
rdata = resp.json()
m = (rdata.get("choices") or [{}])[0].get("message", {})
raw_out = (m.get("content") or "").strip()
raw_out = _strip_think(raw_out)
raw_out = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw_out, flags=re.MULTILINE).strip()
jm = re.search(r'\{.*\}', raw_out, re.DOTALL)
parsed = None
if jm:
try:
parsed = json.loads(jm.group(0))
except Exception:
parsed = None
if parsed is not None:
_ALLOWED_TAGS = {"work","personal","finance","bills","receipt","travel",
"newsletter","marketing","notification","security","social",
"shopping","calendar"}
raw_tags = parsed.get("tags") or []
if isinstance(raw_tags, str):
raw_tags = [raw_tags]
tags = [t.strip().lower().replace("_", "-") for t in raw_tags if isinstance(t, str)]
tags = ["marketing" if t == "promo" else t for t in tags]
tags = [t for t in tags if t in _ALLOWED_TAGS][:2]
is_spam = bool(parsed.get("spam"))
spam_reason = str(parsed.get("reason") or "")[:200]
raw_out = _strip_think((raw_out or "").strip())
raw_out = re.sub(r"^```(?:json)?\s*|\s*```$", "", raw_out, flags=re.MULTILINE).strip()
jm = re.search(r'\{.*\}', raw_out, re.DOTALL)
parsed = None
if jm:
try:
parsed = json.loads(jm.group(0))
except Exception:
parsed = None
if parsed is not None:
_ALLOWED_TAGS = {"work","personal","urgent","action-needed","finance","bills",
"receipt","legal","travel","newsletter","marketing","notification",
"security","social","shopping","calendar","support"}
raw_tags = parsed.get("tags") or []
if isinstance(raw_tags, str):
raw_tags = [raw_tags]
tags = [t.strip().lower().replace("_", "-") for t in raw_tags if isinstance(t, str)]
tags = ["marketing" if t == "promo" else t for t in tags]
tags = [t for t in tags if t in _ALLOWED_TAGS][:3]
is_spam = bool(parsed.get("spam"))
spam_reason = str(parsed.get("reason") or "")[:200]
moved_to = ""
if is_spam and auto_spam and spam_folder:
if _imap_move(uid, spam_folder, account_id=account_id, owner=account_owner):
moved_to = spam_folder
logger.info(f"Auto-spam moved uid={uid.decode() if isinstance(uid, bytes) else str(uid)} to {spam_folder}: {spam_reason}")
moved_to = ""
if is_spam and auto_spam and spam_folder:
if _imap_move(uid, spam_folder, account_id=account_id, owner=account_owner):
moved_to = spam_folder
logger.info(f"Auto-spam moved uid={uid.decode() if isinstance(uid, bytes) else str(uid)} to {spam_folder}: {spam_reason}")
_c = _sql3.connect(SCHEDULED_DB)
_c.execute("""
INSERT OR REPLACE INTO email_tags
(message_id, owner, uid, folder, subject, sender, tags, spam_verdict,
spam_reason, moved_to, model_used, created_at)
VALUES (?, ?, ?, 'INBOX', ?, ?, ?, ?, ?, ?, ?, ?)
""", (message_id, account_owner or "", uid.decode() if isinstance(uid, bytes) else str(uid), subject, sender,
json.dumps(tags), 1 if is_spam else 0,
spam_reason, moved_to, model, datetime.utcnow().isoformat()))
_c.commit()
_c.close()
_tag_existing.add(message_id)
_c = _sql3.connect(SCHEDULED_DB)
_c.execute("""
INSERT OR REPLACE INTO email_tags
(message_id, owner, account_id, uid, folder, subject, sender, tags, spam_verdict,
spam_reason, moved_to, model_used, created_at)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
""", (message_id, account_owner or "", account_id or "", uid.decode() if isinstance(uid, bytes) else str(uid), _folder, subject, sender,
json.dumps(tags), 1 if is_spam else 0,
spam_reason, moved_to, model, datetime.utcnow().isoformat()))
_c.commit()
_c.close()
_tag_existing.add(message_id)
except Exception as e:
logger.warning(f"Auto-classify {uid} failed: {e}")
@@ -1046,7 +1111,7 @@ def _scheduled_poll_once() -> dict:
try:
with _imap(row_account_id, owner=row_owner) as imap:
sent_folder = _detect_sent_folder(imap)
imap.append(sent_folder, "\\Seen", None, outer.as_bytes())
imap.append(_q(sent_folder), "\\Seen", None, outer.as_bytes())
except Exception as e:
logger.warning(f"Failed to append scheduled {sid} to Sent: {e}")

2533
routes/email_routes.py
View File

File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More