odysseus/routes/cookbook_routes.py

"""Cookbook routes — model download, serve, cache scanning, and cookbook state sync."""

import asyncio
import json
import logging
import os
import re
import shlex
import shutil
import subprocess
import sys
import time
import urllib.request
import uuid
from pathlib import Path

from fastapi import APIRouter, HTTPException, Request, Depends

from src.auth_helpers import require_user
from src.constants import COOKBOOK_STATE_FILE
from pydantic import BaseModel

from core.middleware import require_admin
from routes._validators import validate_remote_host, validate_ssh_port
from core.platform_compat import (
    IS_WINDOWS,
    detached_popen_kwargs,
    find_bash,
    kill_process_tree,
    pid_alive,
    safe_chmod,
    which_tool,
)
from routes.shell_routes import TMUX_LOG_DIR
from src.host_docker_access import (
    HOST_DOCKER_ACCESS_HINT,
    HOST_DOCKER_SOCKET_PATH,
    host_docker_access_enabled,
    local_docker_available,
    running_in_container,
)
from routes.cookbook_output import (
    error_aware_output_tail, classify_dead_download,
    HF_CACHE_COMPLETE_PROBE, HF_CACHE_INCOMPLETE_PROBE,
)

logger = logging.getLogger(__name__)

from routes.cookbook_helpers import (
    _SESSION_ID_RE, _validate_repo_id, _validate_serve_model_id, _validate_include, _validate_token,
    _validate_local_dir, _validate_gpus, _shell_path,
    _ps_squote, _bash_squote, _validate_serve_cmd, _parse_serve_phase, OLLAMA_MISSING_HINT,
    _safe_env_prefix, _local_tooling_path_export, _append_serve_preflight_exit_lines,
    _append_serve_exit_code_lines, _append_llama_cpp_linux_accel_build_lines, _cached_model_scan_script,
    load_stored_hf_token,
    _append_vllm_linux_preflight_lines, _ollama_bind_from_cmd, _pip_install_fallback_chain,
    _pip_install_no_cache, _user_shell_path_bootstrap, _venv_safe_local_pip_install_cmd,
    _diagnose_serve_output, run_ssh_command_async,
    _ollama_bind_from_cmd, _pip_install_fallback_chain, _pip_install_no_cache,
    _user_shell_path_bootstrap, _venv_safe_local_pip_install_cmd,
    _append_pip_install_runner_lines, _pip_install_command_without_break_system_packages,
    _normalize_llama_cpp_python_cache_types,
    ModelDownloadRequest, ServeRequest,
)

_HF_TOKEN_STATUS_SNIPPET = (
    'if [ -n "$HF_TOKEN" ]; then '
    'echo "[odysseus] HF token: applied"; '
    'else '
    'echo "[odysseus] HF token: NOT SET — gated/private models will be denied. '
    'Add one in Odysseus Cookbook -> Settings -> HuggingFace Token."; '
    'fi'
)


def _venv_root_from_serve_cmd(cmd: str) -> str:
    """Best-effort venv root from an absolute venv python in a serve command."""
    try:
        parts = shlex.split(cmd or "")
    except Exception:
        parts = (cmd or "").split()
    for part in parts:
        if re.search(r"/bin/python(?:3(?:\.\d+)?)?$", part or ""):
            return re.sub(r"/bin/python(?:3(?:\.\d+)?)?$", "", part)
    return ""


def _append_venv_nvidia_library_path_lines(lines: list[str], *, cmd: str = "") -> None:
    """Expose NVIDIA CUDA runtime wheels bundled inside the active venv.

    SGLang/vLLM wheels can depend on CUDA libraries shipped as Python packages
    under site-packages/nvidia. Activating the venv puts Python packages on
    sys.path, but the dynamic loader still cannot find libraries such as
    libnvrtc.so.13 unless those package lib dirs are on LD_LIBRARY_PATH.
    """
    venv_root = _venv_root_from_serve_cmd(cmd)
    lines.append(f'_ODY_VENV_FOR_LIBS="${{VIRTUAL_ENV:-{_bash_squote(venv_root)}}}"')
    lines.append('if [ -n "$_ODY_VENV_FOR_LIBS" ] && [ -d "$_ODY_VENV_FOR_LIBS" ]; then')
    lines.append('  for _ody_nvlib in "$_ODY_VENV_FOR_LIBS"/lib/python*/site-packages/nvidia/cu13/lib "$_ODY_VENV_FOR_LIBS"/lib/python*/site-packages/nvidia/cu12/lib "$_ODY_VENV_FOR_LIBS"/lib/python*/site-packages/nvidia/cuda_nvrtc/lib "$_ODY_VENV_FOR_LIBS"/lib/python*/site-packages/nvidia/cuda_runtime/lib "$_ODY_VENV_FOR_LIBS"/lib/python*/site-packages/nvidia/cublas/lib "$_ODY_VENV_FOR_LIBS"/lib/python*/site-packages/nvidia/cudnn/lib; do')
    lines.append('    [ -d "$_ody_nvlib" ] && export LD_LIBRARY_PATH="$_ody_nvlib:${LD_LIBRARY_PATH:-}"')
    lines.append('  done')
    lines.append('fi')


def _serve_port_from_cmd(cmd: str) -> str:
    m = re.search(r"--port(?:=|\s+)(\d+)", cmd or "")
    return m.group(1) if m else ""


def _append_openai_port_preflight_lines(lines: list[str], *, cmd: str, expected_model: str) -> None:
    port = _serve_port_from_cmd(cmd)
    if not port:
        return
    lines.append(f"ODYSSEUS_SERVE_PORT='{_bash_squote(port)}'")
    lines.append(f"ODYSSEUS_EXPECTED_MODEL='{_bash_squote(expected_model)}'")
    lines.append("if [ -n \"$ODYSSEUS_SERVE_PORT\" ]; then")
    lines.append("  python3 - \"$ODYSSEUS_SERVE_PORT\" \"$ODYSSEUS_EXPECTED_MODEL\" <<'PY'")
    lines.append("import json, sys, urllib.request")
    lines.append("port = sys.argv[1]")
    lines.append("expected = (sys.argv[2] or '').strip()")
    lines.append("url = f'http://127.0.0.1:{port}/v1/models'")
    lines.append("try:")
    lines.append("    with urllib.request.urlopen(url, timeout=1.5) as r:")
    lines.append("        data = json.loads(r.read().decode('utf-8', 'replace') or '{}')")
    lines.append("except Exception:")
    lines.append("    raise SystemExit(0)")
    lines.append("models = [str(x.get('id') or '') for x in data.get('data', []) if isinstance(x, dict)]")
    lines.append("def base(s): return s.lower().split('/')[-1]")
    lines.append("match = bool(expected) and any((m.lower() == expected.lower() or base(m) == base(expected) or base(expected) in m.lower() or base(m) in expected.lower()) for m in models)")
    lines.append("print(f'ERROR: Port {port} is already serving {models or [\"unknown\"]}.')")
    lines.append("if expected and not match:")
    lines.append("    print(f'ERROR: Cookbook was about to launch {expected}, but this port is occupied by a different model. Stop the old server or choose another port.')")
    lines.append("else:")
    lines.append("    print('ERROR: Stop the existing server or choose another port before launching a duplicate serve.')")
    lines.append("raise SystemExit(98)")
    lines.append("PY")
    lines.append("  _ody_port_ec=$?")
    lines.append("  if [ \"$_ody_port_ec\" -ne 0 ]; then ODYSSEUS_PREFLIGHT_EXIT=\"$_ody_port_ec\"; fi")
    lines.append("fi")

_OLLAMA_SIDECAR_CONTAINERS = {"ollama-test", "ollama-rocm"}
_UNSAFE_DOCKER_EXEC_CHARS = frozenset(";&|<>$`\r\n")
_SAFE_OLLAMA_MODEL_TOKEN_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._:/-]*$")
_SAFE_OLLAMA_FILE_TOKEN_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._-]*$")


def _is_generated_ollama_docker_exec_cmd(cmd: str | None) -> bool:
    """Match only the fixed Docker exec shapes generated by Cookbook."""
    if not cmd or any(char in cmd for char in _UNSAFE_DOCKER_EXEC_CHARS):
        return False
    try:
        parts = shlex.split(cmd)
    except ValueError:
        return False
    if len(parts) < 4 or parts[:2] != ["docker", "exec"]:
        return False
    container, executable = parts[2:4]
    if container not in _OLLAMA_SIDECAR_CONTAINERS:
        return False
    if container == "ollama-rocm" and executable == "ollama":
        return (
            len(parts) == 6
            and parts[4] == "show"
            and _SAFE_OLLAMA_MODEL_TOKEN_RE.fullmatch(parts[5]) is not None
        )
    if container != "ollama-test" or executable != "ollama-import":
        return False
    if len(parts) not in {7, 8}:
        return False
    model, name, context_size = parts[4:7]
    return (
        _SAFE_OLLAMA_MODEL_TOKEN_RE.fullmatch(model) is not None
        and _SAFE_OLLAMA_FILE_TOKEN_RE.fullmatch(name) is not None
        and re.fullmatch(r"[0-9]+", context_size) is not None
        and (
            len(parts) == 7
            or _SAFE_OLLAMA_FILE_TOKEN_RE.fullmatch(parts[7]) is not None
        )
    )


def _missing_binary_message(
    binary: str,
    target: str,
    *,
    local_host_docker_blocked: bool = False,
) -> str:
    if binary == "tmux":
        return (
            f"tmux is required for Cookbook background downloads/serves on {target}. "
            "Install it with your OS package manager, or run Cookbook server setup for that server."
        )
    if binary == "docker":
        if local_host_docker_blocked:
            return HOST_DOCKER_ACCESS_HINT
        return (
            f"Docker is required by this Cookbook launch command on {target}, but the docker CLI was not found. "
            "Install Docker and make sure this user can run `docker`, then retry."
        )
    return f"{binary} is required on {target}, but it was not found."


async def _remote_binary_available(
    remote: str,
    ssh_port: str | None,
    binary: str,
    *,
    windows: bool = False,
) -> bool:
    port = ssh_port or ""
    port_args = ["-p", port] if port and port != "22" else []
    if windows:
        check = f'powershell -NoProfile -Command "if (Get-Command {binary} -ErrorAction SilentlyContinue) {{ exit 0 }} else {{ exit 127 }}"'
    else:
        check = f'PATH="$HOME/.local/bin:$HOME/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"; command -v {shlex.quote(binary)} >/dev/null 2>&1'
    try:
        proc = await asyncio.create_subprocess_exec(
            "ssh",
            "-o",
            "ConnectTimeout=6",
            "-o",
            "StrictHostKeyChecking=no",
            *port_args,
            remote,
            check,
            stdout=asyncio.subprocess.PIPE,
            stderr=asyncio.subprocess.PIPE,
        )
        await asyncio.wait_for(proc.communicate(), timeout=10)
        return proc.returncode == 0
    except Exception:
        return False


def _remote_posix_path_prefix() -> str:
    return 'PATH="$HOME/.local/bin:$HOME/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"; '


def _remote_tmux_command(*args: str) -> str:
    """Shell command for remote tmux when non-login SSH has a thin PATH."""
    tmux = (
        'ODYSSEUS_TMUX="$(command -v tmux '
        '|| command -v /opt/homebrew/bin/tmux '
        '|| command -v /usr/local/bin/tmux '
        '|| command -v /usr/bin/tmux '
        '|| true)"; '
        'if [ -z "$ODYSSEUS_TMUX" ]; then echo "tmux not found" >&2; exit 127; fi; '
    )
    quoted = " ".join(shlex.quote(str(arg)) for arg in args)
    return f'{_remote_posix_path_prefix()}{tmux}"$ODYSSEUS_TMUX" {quoted}'


def _remote_tmux_launch_command(session_id: str, runner: str) -> str:
    """Shell command that chmods a runner and starts it in remote tmux."""
    tmux = (
        'ODYSSEUS_TMUX="$(command -v tmux '
        '|| command -v /opt/homebrew/bin/tmux '
        '|| command -v /usr/local/bin/tmux '
        '|| command -v /usr/bin/tmux '
        '|| true)"; '
        'if [ -z "$ODYSSEUS_TMUX" ]; then echo "tmux not found" >&2; exit 127; fi; '
    )
    sid = shlex.quote(str(session_id))
    runner_q = shlex.quote(str(runner))
    runner_exec = shlex.quote(f"./{runner}")
    return (
        f'{_remote_posix_path_prefix()}{tmux}'
        f'chmod +x {runner_q} && '
        f'"$ODYSSEUS_TMUX" set-option -g history-limit 100000 2>/dev/null; '
        f'"$ODYSSEUS_TMUX" new-session -d -s {sid} {runner_exec}'
    )


async def _binary_available(
    binary: str,
    remote: str | None,
    ssh_port: str | None,
    *,
    windows: bool = False,
    in_container: bool | None = None,
    environ=None,
    socket_path: str = HOST_DOCKER_SOCKET_PATH,
) -> bool:
    if remote:
        return await _remote_binary_available(
            remote,
            ssh_port,
            binary,
            windows=windows,
        )
    cli_available = shutil.which(binary) is not None
    if binary != "docker":
        return cli_available
    return local_docker_available(
        cli_available=cli_available,
        in_container=in_container,
        environ=environ,
        socket_path=socket_path,
    )



def _local_ollama_docker_fallback_available(
    *,
    in_container: bool | None = None,
    environ: dict[str, str] | None = None,
    socket_path: str = HOST_DOCKER_SOCKET_PATH,
) -> bool:
    return local_docker_available(
        cli_available=shutil.which("docker") is not None,
        in_container=in_container,
        environ=environ,
        socket_path=socket_path,
    )


def _local_ollama_docker_access_blocked(
    *,
    in_container: bool | None = None,
    environ: dict[str, str] | None = None,
    socket_path: str = HOST_DOCKER_SOCKET_PATH,
) -> bool:
    containerized = running_in_container() if in_container is None else in_container
    if not containerized or shutil.which("docker") is None:
        return False
    return not _local_ollama_docker_fallback_available(
        in_container=containerized,
        environ=environ,
        socket_path=socket_path,
    )


def _append_local_ollama_download_command_lines(
    lines: list[str],
    ollama_cmd: str,
    *,
    docker_fallback_available: bool,
    docker_fallback_blocked: bool,
) -> None:
    lines.append('if command -v ollama >/dev/null 2>&1; then')
    lines.append(f'  ODYSSEUS_OLLAMA_PULL_CMD={shlex.quote(ollama_cmd)}')
    if docker_fallback_available:
        lines.append('elif command -v docker >/dev/null 2>&1; then')
        lines.append("  ODYSSEUS_OLLAMA_CONTAINER=\"$(docker ps --format '{{.Names}}' 2>/dev/null | grep -E '^(ollama-rocm|ollama-test)$' | head -1)\"")
        lines.append('  if [ -n "$ODYSSEUS_OLLAMA_CONTAINER" ]; then')
        lines.append(f'    ODYSSEUS_OLLAMA_PULL_CMD={shlex.quote("docker exec ${ODYSSEUS_OLLAMA_CONTAINER} " + ollama_cmd)}')
        lines.append('  fi')
    elif docker_fallback_blocked:
        hint = shlex.quote("ERROR: " + HOST_DOCKER_ACCESS_HINT)
        lines.append('else')
        lines.append(f"  printf '%s\\n' {hint}; exit 127")
    lines.append('fi')
    lines.append('if [ -z "$ODYSSEUS_OLLAMA_PULL_CMD" ]; then echo "ERROR: Ollama not found on this server. Install Ollama or start an ollama-rocm/ollama-test container."; exit 127; fi')


def setup_cookbook_routes() -> APIRouter:
    router = APIRouter(tags=["cookbook"])
    _cookbook_state_path = Path(COOKBOOK_STATE_FILE)
    _state_get_cache = {"ts": 0.0, "mtime": 0.0, "value": None}
    _tasks_status_cache = {"ts": 0.0, "value": None}
    _tasks_status_inflight = {"task": None}

    def _mask_secret(value: str) -> str:
        if not value:
            return ""
        if len(value) <= 8:
            return "stored"
        return f"{value[:4]}...{value[-4:]}"

    def _client_host_platform() -> str:
        return "windows" if IS_WINDOWS else ""

    def _decrypt_secret(value: str | None) -> str:
        if not value:
            return ""
        from src.secret_storage import decrypt
        return decrypt(value)

    def _encrypt_secret(value: str) -> str:
        from src.secret_storage import encrypt
        return encrypt(value)

    def _strip_task_secrets(state):
        tasks = state.get("tasks") if isinstance(state, dict) else None
        if isinstance(tasks, list):
            for task in tasks:
                if isinstance(task, dict) and isinstance(task.get("payload"), dict):
                    task["payload"].pop("hf_token", None)
        return state

    def _diagnose_serve_output(text: str) -> dict | None:
        """Server-side mirror of the Cookbook UI's common serve diagnoses.

        The browser uses cookbook-diagnosis.js for clickable fixes. This gives
        the agent/tool path the same structured signal so it can retry with an
        adjusted command instead of guessing from raw tmux output.
        """
        if not text:
            return None
        tail = text[-6000:]
        patterns = [
            (
                r"No available memory for the cache blocks|Available KV cache memory:.*-",
                "No GPU memory left for KV cache after loading model.",
                [
                    {"label": "retry with GPU memory utilization 0.95", "op": "replace", "flag": "--gpu-memory-utilization", "value": "0.95"},
                    {"label": "retry with context 2048", "op": "replace", "flag": "--max-model-len", "value": "2048"},
                ],
            ),
            (
                r"CUDA out of memory|torch\.cuda\.OutOfMemoryError|CUDA error: out of memory|warming up sampler|max_num_seqs.*gpu_memory_utilization",
                "GPU ran out of memory during startup or warmup.",
                [
                    {"label": "retry with context 4096", "op": "replace", "flag": "--max-model-len", "value": "4096"},
                    {"label": "retry with GPU memory utilization 0.80", "op": "replace", "flag": "--gpu-memory-utilization", "value": "0.80"},
                    {"label": "retry with --enforce-eager", "op": "append", "arg": "--enforce-eager"},
                ],
            ),
            (
                r"not divisib|must be divisible|attention heads.*divisible",
                "Tensor parallel size is incompatible with the model.",
                [
                    {"label": "retry with tensor parallel size 1", "op": "replace", "flag": "--tensor-parallel-size", "value": "1"},
                    {"label": "retry with tensor parallel size 2", "op": "replace", "flag": "--tensor-parallel-size", "value": "2"},
                ],
            ),
            (
                r"KV cache.*too (small|large)|max_model_len.*exceeds|maximum.*context",
                "Context length is too large for available GPU memory.",
                [
                    {"label": "retry with context 8192", "op": "replace", "flag": "--max-model-len", "value": "8192"},
                    {"label": "retry with context 4096", "op": "replace", "flag": "--max-model-len", "value": "4096"},
                ],
            ),
            (
                r"enable-auto-tool-choice requires --tool-call-parser",
                "Auto tool choice requires an explicit tool call parser.",
                [{"label": "retry with Hermes tool parser", "op": "append", "arg": "--tool-call-parser hermes"}],
            ),
            (
                r"Please pass.*trust.remote.code=True|contains custom code which must be executed to correctly load|does not recognize this architecture|model type.*but Transformers does not",
                "Model requires custom code or newer model support.",
                [{"label": "retry with --trust-remote-code", "op": "append", "arg": "--trust-remote-code"}],
            ),
            (
                r"Either a revision or a version must be specified|transformers\.integrations\.hub_kernels|kernels/layer",
                "vLLM/Transformers kernel package mismatch.",
                [{"label": "update vLLM, Transformers, and kernels on this server", "op": "dependency", "package": "vllm transformers kernels"}],
            ),
            (
                r"Address already in use|bind.*address.*in use",
                "Port is already in use.",
                [{"label": "retry on port 8001", "op": "replace", "flag": "--port", "value": "8001"}],
            ),
            (
                r"No CUDA GPUs are available|no GPU.*found|CUDA_VISIBLE_DEVICES.*invalid",
                "No GPUs are visible to the serve process.",
                [{"label": "clear Cookbook GPU selection or choose available GPUs", "op": "settings", "field": "gpus", "value": ""}],
            ),
            (
                r"Failed to infer device type|NVML Shared Library Not Found|No module named 'amdsmi'|platform is not available",
                "vLLM could not find a supported GPU (CUDA or ROCm). "
                "This machine may have integrated or unsupported graphics only.",
                [
                    {"label": "switch to llama.cpp (CPU/Metal, works without a discrete GPU)", "op": "manual"},
                    {"label": "switch to Ollama (CPU/Metal, works without a discrete GPU)", "op": "manual"},
                ],
            ),
            (
                r"vllm.*command not found|No module named vllm|ERROR: vLLM is not installed",
                "vLLM is not installed or not in PATH on this server.",
                [{"label": "install vLLM in Cookbook Dependencies", "op": "dependency", "package": "vllm"}],
            ),
            (
                r"sgl_kernel[\s\S]*(Python\.h|libnuma\.so\.1|common_ops|libnvrtc\.so)|"
                r"(Python\.h|libnuma\.so\.1|common_ops|libnvrtc\.so)[\s\S]*sgl_kernel|"
                r"Could not load any common_ops library|"
                r"Please ensure sgl_kernel is properly installed",
                "SGLang native kernel/runtime is missing or mismatched on this server.",
                [
                    {"label": "repair sglang-kernel in this Python environment", "op": "dependency", "package": "sglang-kernel"},
                    {"label": "if libnvrtc is still missing, install the matching CUDA/NVRTC runtime on this host", "op": "manual"},
                ],
            ),
            (
                r"sglang.*command not found|No module named sglang|SGLang is not installed",
                "SGLang is not installed or not in PATH on this server.",
                [{"label": "install SGLang in Cookbook Dependencies", "op": "dependency", "package": "sglang[all]"}],
            ),
            (
                r"No module named ['\"]?mlx_lm|mlx_lm.*command not found|MLX is not installed|MLX LM is not installed",
                "MLX LM is not installed on this server.",
                [{"label": "install mlx-lm in Cookbook Dependencies", "op": "dependency", "package": "mlx-lm"}],
            ),
            (
                r"Unable to quantize model of type <class ['\"]mlx_lm\.models\.switch_layers\.QuantizedSwitchLinear['\"]>|QuantizedSwitchLinear",
                "MLX-LM tried to quantize an already-quantized DeepSeek switch layer.",
                [
                    {"label": "relaunch from the cached local Hugging Face snapshot path on this Mac", "op": "manual"},
                    {"label": "Odysseus now rewrites MLX repo-id launches to a cached snapshot when one exists", "op": "manual"},
                ],
            ),
            # System build deps come BEFORE the generic llama.cpp catch-all
            # so cmake / build-essential / git missing → a specific OS-package
            # remediation instead of "install llama-cpp-python[server]" (which
            # itself fails to compile when cmake is absent).
            (
                r"cmake: command not found|cmake.*not found.*[Cc]ould not",
                "cmake is required to build llama.cpp from source but isn't installed on this server.",
                [{"label": "install build deps for llama.cpp (apt: cmake build-essential git / pacman: cmake base-devel git / dnf: cmake gcc-c++ make git / brew: cmake git)", "op": "dependency", "package": "llama-cpp-python[server]"}],
            ),
            (
                r"^(make|g\+\+|gcc): command not found|Could not find C\+\+ compiler",
                "A C/C++ compiler (build-essential) is required to build llama.cpp from source.",
                [{"label": "install build deps for llama.cpp on this server", "op": "dependency", "package": "llama-cpp-python[server]"}],
            ),
            (
                r"^git: command not found",
                "git is required to clone the llama.cpp source tree.",
                [{"label": "install build deps for llama.cpp on this server", "op": "dependency", "package": "llama-cpp-python[server]"}],
            ),
            (
                r"llama-server.*command not found|llama\.cpp.*not found|No module named.*llama_cpp|No module named 'starlette_context'",
                "llama.cpp / llama-cpp-python dependencies are missing.",
                [{"label": "install llama.cpp dependencies or llama-cpp-python[server]", "op": "dependency", "package": "llama-cpp-python[server]"}],
            ),
            (
                r"No GGUF found on this host|no \.gguf file|No GGUF file found",
                "No GGUF file found for this model on this host. The llama.cpp backend needs a .gguf file.",
                [{"label": "download a GGUF build of this model (repo name usually ends in -GGUF, file like Q4_K_M.gguf)", "op": "manual"}],
            ),
            (
                r"No module named 'torch'|No module named torch|No module named 'diffusers'|No module named diffusers",
                "Diffusion serving requires PyTorch and diffusers.",
                [{"label": "install diffusers[torch] in Cookbook Dependencies", "op": "dependency", "package": "diffusers[torch]"}],
            ),
            (
                r"403 Forbidden|401 Unauthorized|Access to model.*is restricted|gated repo|not in the authorized list|awaiting a review",
                "Model access is gated or unauthorized.",
                [{"label": "set HF token and request model access on HuggingFace", "op": "manual"}],
            ),
        ]
        for pattern, message, suggestions in patterns:
            if re.search(pattern, tail, re.I):
                return {"message": message, "suggestions": suggestions}
        if re.search(r"Traceback \(most recent call last\)", tail, re.I) and not re.search(
            r"Application startup complete|GET /v1/|Uvicorn running on", tail, re.I
        ):
            return {
                "message": "Python traceback detected during serve startup.",
                "suggestions": [{"label": "inspect traceback and retry with adjusted backend/settings", "op": "manual"}],
            }
        return None

    def _state_for_client(state):
        """Return cookbook state without raw secrets for browser clients."""
        _strip_task_secrets(state)
        env = state.get("env") if isinstance(state, dict) else None
        if isinstance(state, dict) and not isinstance(env, dict):
            env = {}
            state["env"] = env
        if isinstance(env, dict):
            token = _decrypt_secret(env.get("hfToken"))
            env.pop("hfToken", None)
            env["hfTokenConfigured"] = bool(token)
            env["hfTokenMasked"] = _mask_secret(token)
            env["hostPlatform"] = _client_host_platform()
        return state

    def _state_for_storage(state, on_disk=None):
        """Encrypt cookbook secrets before writing state to disk."""
        _strip_task_secrets(state)
        env = state.get("env") if isinstance(state, dict) else None
        disk_env = on_disk.get("env") if isinstance(on_disk, dict) and isinstance(on_disk.get("env"), dict) else {}
        if isinstance(env, dict):
            incoming = env.get("hfToken")
            if incoming:
                _validate_token(incoming)
                env["hfToken"] = _encrypt_secret(incoming)
            elif disk_env.get("hfToken"):
                env["hfToken"] = disk_env["hfToken"]
            else:
                env.pop("hfToken", None)
            env.pop("hfTokenMasked", None)
            env.pop("hfTokenConfigured", None)
            env.pop("hostPlatform", None)
        return state

    def _load_stored_hf_token() -> str:
        return load_stored_hf_token(state_path=_cookbook_state_path)

    def _normalize_minimax_m3_vllm_cmd(cmd: str) -> str:
        """Patch MiniMax M3 vLLM launches into the known-good local form.

        The browser form can be stale or omit advanced-only fields. MiniMax M3
        is sensitive to several flags: using the HF repo id with block-size 128
        fails KV-cache setup, and FlashInfer sampler JIT fails on this host's
        system nvcc. Normalize server-side before writing the tmux runner.
        """
        cmd_lower = (cmd or "").lower()
        if not cmd or "vllm serve" not in cmd_lower or "minimax" not in cmd_lower or "m3" not in cmd_lower:
            return cmd
        try:
            parts = shlex.split(cmd)
        except ValueError:
            return cmd
        if "serve" not in parts:
            return cmd

        env_re = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*=")
        env_parts = [p for p in parts if env_re.match(p)]
        body = [p for p in parts if not env_re.match(p)]
        try:
            serve_i = body.index("serve")
        except ValueError:
            return cmd
        if serve_i + 1 >= len(body):
            return cmd

        repo_id = "cyankiwi/MiniMax-M3-AWQ-INT4"
        snapshot = (
            "/home/pewds/.cache/huggingface/hub/"
            "models--cyankiwi--MiniMax-M3-AWQ-INT4/"
            "snapshots/4082acbbec1236d21828d55b6bb0fe02ade4ab5b"
        )
        if body[serve_i + 1] == repo_id:
            body[serve_i + 1] = snapshot

        def add_env(key: str, value: str) -> None:
            if not any(p.startswith(f"{key}=") for p in env_parts):
                env_parts.append(f"{key}={value}")

        def has_flag(flag: str) -> bool:
            return any(p == flag or p.startswith(flag + "=") for p in body)

        def set_flag(flag: str, value: str) -> None:
            for i, part in enumerate(body):
                if part == flag:
                    if i + 1 < len(body):
                        body[i + 1] = value
                    else:
                        body.append(value)
                    return
                if part.startswith(flag + "="):
                    body[i] = f"{flag}={value}"
                    return
            body.extend([flag, value])

        def add_bool(flag: str) -> None:
            if not has_flag(flag):
                body.append(flag)

        add_env("VLLM_TARGET_DEVICE", "cuda")
        add_env("VLLM_USE_FLASHINFER_SAMPLER", "0")
        set_flag("--served-model-name", repo_id)
        set_flag("--tool-call-parser", "minimax_m3")
        set_flag("--reasoning-parser", "minimax_m3")
        set_flag("--attention-backend", "TRITON_ATTN")
        set_flag("--block-size", "128")
        add_bool("--language-model-only")
        add_bool("--disable-custom-all-reduce")
        add_bool("--enable-expert-parallel")
        return shlex.join(env_parts + body)

    def _normalize_deepseek_v4_sglang_cmd(cmd: str) -> str:
        """Patch stale DeepSeek-V4 SGLang commands into the safer local form.

        The browser command builder already emits these flags, but saved presets,
        running-row retries, and old tabs can still submit a pre-fix command to
        /api/model/serve. Normalize server-side so the tmux runner does not keep
        relaunching DeepSeek-V4 with the known CUDA-graph crash shape.
        """
        cmd_lower = (cmd or "").lower()
        if (
            not cmd
            or "sglang.launch_server" not in cmd_lower
            or "deepseek-v4" not in cmd_lower
        ):
            return cmd
        try:
            parts = shlex.split(cmd)
        except ValueError:
            return cmd

        env_re = re.compile(r"^[A-Za-z_][A-Za-z0-9_]*=")
        env_parts = [p for p in parts if env_re.match(p)]
        body = [p for p in parts if not env_re.match(p)]

        def add_env(key: str, value: str) -> None:
            if not any(p.startswith(f"{key}=") for p in env_parts):
                env_parts.append(f"{key}={value}")

        def has_flag(flag: str) -> bool:
            return any(p == flag or p.startswith(flag + "=") for p in body)

        def flag_value(flag: str) -> str | None:
            for i, part in enumerate(body):
                if part == flag:
                    return body[i + 1] if i + 1 < len(body) else ""
                if part.startswith(flag + "="):
                    return part.split("=", 1)[1]
            return None

        def set_flag(flag: str, value: str) -> None:
            for i, part in enumerate(body):
                if part == flag:
                    if i + 1 < len(body):
                        body[i + 1] = value
                    else:
                        body.append(value)
                    return
                if part.startswith(flag + "="):
                    body[i] = f"{flag}={value}"
                    return
            body.extend([flag, value])

        def remove_flag(flag: str) -> None:
            i = 0
            while i < len(body):
                part = body[i]
                if part == flag:
                    del body[i:i + 2]
                    continue
                if part.startswith(flag + "="):
                    del body[i]
                    continue
                i += 1

        add_env("SGLANG_DSV4_COMPRESS_STATE_DTYPE", "bf16")
        mem_fraction = flag_value("--mem-fraction-static")
        try:
            mem_fraction_num = float(mem_fraction) if mem_fraction not in (None, "") else None
        except (TypeError, ValueError):
            mem_fraction_num = None
        if mem_fraction in (None, "", "0.90", "0.9") or (
            mem_fraction_num is not None and mem_fraction_num < 0.76
        ):
            set_flag("--mem-fraction-static", "0.80")
        if not has_flag("--reasoning-parser"):
            set_flag("--reasoning-parser", "deepseek-v4")
        if not has_flag("--tool-call-parser"):
            set_flag("--tool-call-parser", "deepseekv4")
        if not has_flag("--cuda-graph-backend-decode"):
            remove_flag("--cuda-graph-max-bs-decode")
            set_flag("--cuda-graph-backend-decode", "disabled")
        return shlex.join(env_parts + body)

    def _cookbook_ssh_dir() -> Path:
        # The Docker image keeps cookbook keys under /app/.ssh; that path only
        # exists inside the container. On Windows (and any non-container host)
        # fall back to the user profile's ~/.ssh, which OpenSSH on Win10+ uses.
        if not IS_WINDOWS:
            app_ssh = Path("/app/.ssh")
            if Path("/app").exists():
                return app_ssh
        return Path.home() / ".ssh"

    def _cookbook_ssh_key_path() -> Path:
        return _cookbook_ssh_dir() / "id_ed25519"

    def _ssh_known_host_name(host: str) -> str:
        """Return the host part OpenSSH stores in known_hosts.

        Cookbook accepts `user@host` for convenience, but known_hosts entries
        are keyed by host, not username.
        """
        return (host or "").rsplit("@", 1)[-1]

    def _known_hosts_targets(host: str, ssh_port: str | None = None) -> list[str]:
        name = _ssh_known_host_name(host)
        targets = [name]
        if ssh_port and ssh_port != "22":
            targets.insert(0, f"[{name}]:{ssh_port}")
        return [t for t in targets if t]

    def _ssh_host_key_changed(stderr_txt: str) -> bool:
        text = stderr_txt or ""
        return (
            "REMOTE HOST IDENTIFICATION HAS CHANGED" in text
            or "Host key verification failed" in text and "Offending" in text
        )

    async def _repair_cookbook_known_host(host: str, ssh_port: str | None = None) -> tuple[bool, str]:
        """Refresh Odysseus' own known_hosts entry for a validated Cookbook host.

        This is intentionally scoped to Cookbook SSH targets and only called
        after OpenSSH reports a changed host key. It fixes container-local
        known_hosts drift without asking the user to run ssh-keygen manually.
        """
        known_hosts = _cookbook_ssh_dir() / "known_hosts"
        known_hosts.parent.mkdir(parents=True, exist_ok=True)
        known_hosts.touch(mode=0o600, exist_ok=True)
        safe_chmod(known_hosts, 0o600)

        ssh_keygen = which_tool("ssh-keygen") or "ssh-keygen"
        ssh_keyscan = which_tool("ssh-keyscan") or "ssh-keyscan"
        removed_chunks: list[str] = []
        for target in _known_hosts_targets(host, ssh_port):
            proc = await asyncio.create_subprocess_exec(
                ssh_keygen,
                "-f",
                str(known_hosts),
                "-R",
                target,
                stdout=asyncio.subprocess.PIPE,
                stderr=asyncio.subprocess.PIPE,
            )
            stdout, stderr = await proc.communicate()
            removed_chunks.append((stdout or stderr).decode("utf-8", errors="replace").strip())

        scan_args = [ssh_keyscan, "-H", "-t", "ed25519,ecdsa,rsa"]
        if ssh_port and ssh_port != "22":
            scan_args.extend(["-p", ssh_port])
        scan_args.append(_ssh_known_host_name(host))
        proc = await asyncio.create_subprocess_exec(
            *scan_args,
            stdout=asyncio.subprocess.PIPE,
            stderr=asyncio.subprocess.PIPE,
        )
        try:
            stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=8)
        except asyncio.TimeoutError:
            proc.kill()
            await proc.communicate()
            return False, "ssh-keyscan timed out while refreshing known_hosts"
        if proc.returncode != 0 or not stdout.strip():
            detail = (stderr or stdout).decode("utf-8", errors="replace").strip()
            return False, detail or "ssh-keyscan returned no host keys"
        with known_hosts.open("ab") as f:
            if known_hosts.stat().st_size > 0:
                f.write(b"\n")
            f.write(stdout.strip() + b"\n")
        safe_chmod(known_hosts, 0o600)
        return True, "\n".join(chunk for chunk in removed_chunks if chunk) or "known_hosts refreshed"

    def _read_cookbook_public_key() -> str:
        pub = _cookbook_ssh_key_path().with_suffix(".pub")
        if not pub.exists():
            return ""
        return pub.read_text(encoding="utf-8", errors="replace").strip()

    def _server_env_prefix_for_download(remote_host: str | None) -> str | None:
        """Recover a server venv/conda activation for stale download clients.

        Older browser bundles could submit /api/model/download without
        env_prefix even when the selected server profile had an envPath. The
        remote runner would then use system python and exit before hf download.
        Resolve the server profile by host here so downloads remain correct even
        if the user has a cached JS bundle.
        """
        if not remote_host or not _cookbook_state_path.exists():
            return None
        try:
            state = json.loads(_cookbook_state_path.read_text(encoding="utf-8"))
        except Exception:
            return None
        env_state = state.get("env") if isinstance(state, dict) else {}
        servers = env_state.get("servers") if isinstance(env_state, dict) else []
        if not isinstance(servers, list):
            return None
        selected = None
        for server in servers:
            if isinstance(server, dict) and (server.get("host") or "").strip() == remote_host:
                selected = server
                break
        if not selected:
            return None
        env = (selected.get("env") or "none").strip().lower()
        env_path = (selected.get("envPath") or "").strip()
        if not env_path:
            return None
        if env == "venv" or (env in {"", "none"} and re.search(r"(?:^|/)(?:\.?venv|env)(?:/|$)|/bin/activate$", env_path, re.I)):
            activate = env_path if env_path.endswith("/bin/activate") else env_path.rstrip("/") + "/bin/activate"
            return "source " + shlex.quote(activate)
        if env == "conda":
            return 'eval "$(conda shell.bash hook)" && conda activate ' + shlex.quote(env_path)
        return None

    @router.get("/api/cookbook/ssh-key")
    async def get_cookbook_ssh_key(request: Request):
        require_admin(request)
        public_key = _read_cookbook_public_key()
        return {
            "configured": bool(public_key),
            "public_key": public_key,
        }

    @router.post("/api/cookbook/ssh-key")
    async def generate_cookbook_ssh_key(request: Request):
        require_admin(request)
        ssh_dir = _cookbook_ssh_dir()
        key_path = _cookbook_ssh_key_path()
        ssh_dir.mkdir(parents=True, exist_ok=True)
        # safe_chmod no-ops on Windows (~/.ssh is already ACL-restricted to the
        # user profile); applies 0o700 on POSIX.
        safe_chmod(ssh_dir, 0o700)
        if not key_path.exists():
            # ssh-keygen ships with the OpenSSH client on Win10+; resolve it via
            # which_tool so the .exe is found even when PATHEXT is unusual.
            ssh_keygen = which_tool("ssh-keygen") or "ssh-keygen"
            proc = await asyncio.create_subprocess_exec(
                ssh_keygen, "-t", "ed25519", "-N", "", "-C", "odysseus-cookbook", "-f", str(key_path),
                stdout=asyncio.subprocess.PIPE,
                stderr=asyncio.subprocess.PIPE,
            )
            stdout, stderr = await proc.communicate()
            if proc.returncode != 0:
                detail = (stderr or stdout).decode("utf-8", errors="replace").strip()[-500:]
                return {"ok": False, "error": detail or "Failed to generate SSH key"}
        safe_chmod(key_path, 0o600)
        safe_chmod(key_path.with_suffix(".pub"), 0o644)
        return {"ok": True, "public_key": _read_cookbook_public_key()}

    class CookbookSshTestRequest(BaseModel):
        host: str
        ssh_port: str | None = None

    @router.post("/api/cookbook/test-ssh")
    async def test_cookbook_ssh(request: Request, req: CookbookSshTestRequest):
        """Test a configured Cookbook SSH target without using generic shell exec."""
        require_admin(request)
        host = validate_remote_host(req.host)
        ssh_port = validate_ssh_port(req.ssh_port)
        try:
            code, stdout, stderr = await run_ssh_command_async(
                host,
                ssh_port,
                "echo ok",
                timeout=8,
                connect_timeout=5,
                strict_host_key_checking=False,
            )
        except asyncio.TimeoutError:
            return {"stdout": "", "stderr": "SSH test timed out", "exit_code": 124}
        except Exception as e:
            return {"stdout": "", "stderr": str(e), "exit_code": -1}
        return {
            "stdout": stdout.decode("utf-8", errors="replace"),
            "stderr": stderr.decode("utf-8", errors="replace"),
            "exit_code": code,
        }

    def _needs_binary(cmd: str, binary: str) -> bool:
        return bool(re.search(rf"(^|[\s;&|()]){re.escape(binary)}($|[\s;&|()])", cmd or ""))

    def _launch_local_detached(session_id: str, bash_lines: list[str]) -> dict:
        """Windows-native stand-in for a LOCAL tmux session (tmux doesn't exist
        on Windows). Mirrors shell_routes._generate_win_detached / bg_jobs.launch:
        runs the wrapper detached so it survives a browser/SSE disconnect (the
        whole point of the tmux feature for long downloads/serves), writing a
        <session>.log the status poller tails and a <session>.pid for liveness.

        `bash_lines` is the same bash wrapper used on POSIX. Prefers Git Bash
        for full command-syntax parity; falls back to a cmd.exe wrapper that
        runs the script through whatever bash is reachable, else best-effort
        directly (simple commands only). Returns the launched job record."""
        log_path = TMUX_LOG_DIR / f"{session_id}.log"
        pid_path = TMUX_LOG_DIR / f"{session_id}.pid"
        bash = find_bash()
        if bash:
            # Run the existing bash wrapper verbatim through Git Bash, redirecting
            # all output to the log the poller reads. Paths handed to bash use
            # POSIX form + shell-quoting so drive paths / spaces survive.
            inner = TMUX_LOG_DIR / f"{session_id}_run.sh"
            pp = shlex.quote(pid_path.as_posix())
            inner.write_text(
                f"printf '%s\\n' \"$$\" > {pp}\n" + "\n".join(bash_lines) + "\n",
                encoding="utf-8",
            )
            lp = shlex.quote(log_path.as_posix())
            ip = shlex.quote(inner.as_posix())
            script_path = TMUX_LOG_DIR / f"{session_id}.sh"
            script_path.write_text(
                f"bash {ip} > {lp} 2>&1\n",
                encoding="utf-8",
            )
            argv = [bash, str(script_path)]
        else:
            # No bash on this Windows host: the bash wrapper can't run. Fall back
            # to a cmd.exe wrapper that just records a clear error to the log so
            # the UI surfaces "install Git Bash" instead of silently hanging.
            script_path = TMUX_LOG_DIR / f"{session_id}.cmd"
            script_path.write_text(
                "@echo off\r\n"
                f'echo Cookbook LOCAL execution on Windows needs Git Bash ^(bash.exe^) on PATH. > "{log_path}" 2>&1\r\n'
                f'echo Install Git for Windows, then retry. >> "{log_path}"\r\n',
                encoding="utf-8",
            )
            argv = [os.environ.get("ComSpec", "cmd.exe"), "/c", str(script_path)]
        env = os.environ.copy()
        env["PYTHONUTF8"] = "1"
        env["PYTHONIOENCODING"] = "utf-8"
        proc = subprocess.Popen(
            argv,
            stdout=subprocess.DEVNULL,
            stderr=subprocess.DEVNULL,
            stdin=subprocess.DEVNULL,
            env=env,
            **detached_popen_kwargs(),
        )
        pid_path.write_text(str(proc.pid), encoding="utf-8")
        return {"pid": proc.pid, "log_path": str(log_path)}

    @router.post("/api/model/download")
    async def model_download(request: Request, req: ModelDownloadRequest):
        """Download a HuggingFace model in a tmux session.
        Uses `hf download` CLI directly — runs in tmux via `script -qc`
        for real TTY progress, streams ANSI-stripped output via log file."""
        require_admin(request)
        # Defence-in-depth: even though this endpoint is admin-gated, refuse
        # values that would land in shell contexts with metacharacters.
        backend = (req.backend or "").strip().lower()
        is_ollama_download = backend == "ollama" or ("/" not in req.repo_id and ":" in req.repo_id)
        if is_ollama_download:
            _validate_serve_model_id(req.repo_id)
            req.include = None
            req.local_dir = None
        else:
            _validate_repo_id(req.repo_id)
            _validate_include(req.include)
        validate_remote_host(req.remote_host)
        req.ssh_port = validate_ssh_port(req.ssh_port)
        req.local_dir = _validate_local_dir(req.local_dir)
        req.hf_token = "" if is_ollama_download else (req.hf_token or _load_stored_hf_token())
        _validate_token(req.hf_token)
        if req.remote_host and not req.env_prefix:
            req.env_prefix = _server_env_prefix_for_download(req.remote_host)
        TMUX_LOG_DIR.mkdir(parents=True, exist_ok=True)
        session_id = f"cookbook-{uuid.uuid4().hex[:8]}"
        wrapper_script = TMUX_LOG_DIR / f"{session_id}.sh"

        # Custom download dir: point the HF cache at <dir>/hub via env vars
        # (HF_HOME + HUGGINGFACE_HUB_CACHE) instead of --local-dir. local_dir
        # produces a flat layout (<dir>/<name>/<file>) and the local-dir
        # bookkeeping files (.cache/huggingface/.gitignore.lock), and it
        # also breaks robust resume on flaky transfers — the blob-based hub
        # cache survives SSL ReadError mid-stream by reusing <sha>.incomplete,
        # local_dir does not. See issue #2722.
        _dl_hf_home_shell = _shell_path(req.local_dir.rstrip("/")) if req.local_dir else None
        _dl_pyarg = ""  # snapshot_download honors the env vars too — no kwarg needed

        # Build the hf download command. Redirection to suppress the interactive
        # "update available? [Y/n]" prompt is added per-platform further down
        # (< /dev/null on bash, $null | on PowerShell).
        hf_download_args = f"download {shlex.quote(req.repo_id)}"
        if req.include:
            hf_download_args += f" --include {shlex.quote(req.include)}"
        hf_cmd = f"hf {hf_download_args}"
        ollama_cmd = f"ollama pull {shlex.quote(req.repo_id)}"

        # Build the shell wrapper — runs hf download directly in tmux (which is a TTY)
        # No script/tee needed — we'll use tmux capture-pane to read output
        lines = ["#!/bin/bash"]
        lines.extend(_user_shell_path_bootstrap())
        if req.hf_token:
            lines.append(f"export HF_TOKEN='{_bash_squote(req.hf_token)}'")
        if _dl_hf_home_shell and not is_ollama_download:
            # Make hf download / snapshot_download honor the chosen dir via the
            # standard HF cache (gives us the models--org--name/blobs/... layout
            # with resumable .incomplete blobs).
            lines.append(f"export HF_HOME={_dl_hf_home_shell}")
            lines.append(f"export HUGGINGFACE_HUB_CACHE={_dl_hf_home_shell}/hub")
            lines.append(f"export HF_HUB_CACHE={_dl_hf_home_shell}/hub")
        # Ensure pip-user scripts (e.g. hf CLI installed via --user) are on PATH
        lines.append('export PATH="$HOME/.local/bin:$HOME/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"')
        # When Odysseus runs from a venv (e.g. native macOS install), put its bin
        # on PATH so the tmux shell finds the bundled `hf`/`python3` without an
        # activated venv. Local bash runs only — meaningless over SSH.
        if not req.remote_host:
            lines.append(_local_tooling_path_export(sys.executable))
        # Best-effort install hf CLI (always). hf_transfer (Rust parallel downloader)
        # is fast but flaky on large files — it tends to crash near the end at high
        # throughput. Retries set disable_hf_transfer to fall back to the plain,
        # slower-but-reliable downloader (resumes cleanly from the .incomplete files).
        # Use `python3 -m pip` not `pip` — macOS has no bare `pip` command.
        if is_ollama_download:
            _append_local_ollama_download_command_lines(
                lines,
                ollama_cmd,
                docker_fallback_available=_local_ollama_docker_fallback_available(),
                docker_fallback_blocked=_local_ollama_docker_access_blocked(),
            )
        else:
            lines.append(f"command -v hf >/dev/null 2>&1 || {_pip_install_fallback_chain('huggingface_hub', upgrade=True)}")
            if req.disable_hf_transfer:
                lines.append("export HF_HUB_ENABLE_HF_TRANSFER=0")
                lines.append("export HF_HUB_DOWNLOAD_MAX_WORKERS=4")
            else:
                lines.append(f"python3 -c 'import hf_transfer' 2>/dev/null || {_pip_install_fallback_chain('hf_transfer')}")
                lines.append("python3 -c 'import hf_transfer' 2>/dev/null && export HF_HUB_ENABLE_HF_TRANSFER=1")
                lines.append("export HF_HUB_DOWNLOAD_MAX_WORKERS=8")

        remote = req.remote_host  # None for local
        is_windows = req.platform == "windows"
        # LOCAL execution on a native-Windows host never uses tmux (it uses the
        # detached-process path below), regardless of the UI-supplied platform.
        local_windows = IS_WINDOWS and not remote
        logger.info(f"Download request: repo={req.repo_id}, remote={remote}, ssh_port={req.ssh_port}, platform={req.platform}")

        if not is_windows and not local_windows and not await _binary_available("tmux", remote, req.ssh_port):
            return {
                "ok": False,
                "error": _missing_binary_message("tmux", remote or "local server"),
                "session_id": session_id,
            }

        if remote and is_windows:
            # ── Windows remote: generate .ps1 runner, use Start-Process for background ──
            remote_runner = f".{session_id}_run.ps1"
            ps_lines = []
            ps_lines.append('$sessionDir = "$env:TEMP\\odysseus-sessions"')
            ps_lines.append('New-Item -ItemType Directory -Force -Path $sessionDir | Out-Null')
            if req.hf_token:
                ps_lines.append(f"$env:HF_TOKEN = '{_ps_squote(req.hf_token)}'")
            if req.local_dir and not is_ollama_download:
                # Mirror the bash branch — point the HF cache at the user's dir
                # via env vars instead of --local-dir, so resume works on flaky
                # transfers (issue #2722).
                _dl_ps = _ps_squote(req.local_dir.rstrip("/"))
                ps_lines.append(f"$env:HF_HOME = '{_dl_ps}'")
                ps_lines.append(f"$env:HUGGINGFACE_HUB_CACHE = '{_dl_ps}/hub'")
                ps_lines.append(f"$env:HF_HUB_CACHE = '{_dl_ps}/hub'")
            if req.env_prefix:
                ps_lines.append(_safe_env_prefix(req.env_prefix))
            if is_ollama_download:
                ps_lines.append('if (-not (Get-Command ollama -ErrorAction SilentlyContinue)) { Write-Host "ERROR: Ollama not found. Install from https://ollama.com/download/windows"; exit 127 }')
                ps_lines.append(f"$null | ollama pull '{_ps_squote(req.repo_id)}'")
                ps_lines.append('if ($LASTEXITCODE -eq 0) { Write-Host ""; Write-Host "DOWNLOAD_OK" } else { Write-Host ""; Write-Host "DOWNLOAD_FAILED (exit $LASTEXITCODE)" }')
            else:
                # Try hf CLI, fall back to Python huggingface_hub, then auto-install
                ps_lines.append('try {{')
                ps_lines.append('  $hfPath = Get-Command hf -ErrorAction SilentlyContinue')
                ps_lines.append('  if ($hfPath) {{')
                # Pipe $null to stdin to suppress interactive "update available? [Y/n]" prompt
                ps_lines.append(f'    $null | {hf_cmd}')
                ps_lines.append('  }} else {{')
                ps_lines.append('    python -c "import huggingface_hub" 2>$null')
                ps_lines.append('    if ($LASTEXITCODE -eq 0) {{')
                ps_lines.append('      Write-Host "hf CLI not found, using Python huggingface_hub..."')
                ps_lines.append('      python -m pip install -q hf_transfer 2>$null')
                ps_lines.append('      $env:HF_HUB_ENABLE_HF_TRANSFER = "1"')
                ps_lines.append(f"      python -c \"import os; from huggingface_hub import snapshot_download; snapshot_download('{req.repo_id}'{_dl_pyarg}, max_workers=8)\"")
                ps_lines.append('    }} else {{')
                ps_lines.append('      Write-Host "Installing huggingface-hub..."')
                ps_lines.append('      python -m pip install -q huggingface-hub hf_transfer')
                ps_lines.append('      $env:HF_HUB_ENABLE_HF_TRANSFER = "1"')
                ps_lines.append(f"      python -c \"import os; from huggingface_hub import snapshot_download; snapshot_download('{req.repo_id}'{_dl_pyarg}, max_workers=8)\"")
                ps_lines.append('    }}')
                ps_lines.append('  }}')
                ps_lines.append('  if ($LASTEXITCODE -eq 0) {{ Write-Host ""; Write-Host "DOWNLOAD_OK" }}')
                ps_lines.append('  else {{ Write-Host ""; Write-Host "DOWNLOAD_FAILED (exit $LASTEXITCODE)" }}')
                ps_lines.append('}} catch {{')
                ps_lines.append('  Write-Host ""; Write-Host "DOWNLOAD_FAILED ($_)"')
                ps_lines.append('}}')
            ps_lines.append(f'Remove-Item -Force "$HOME\\{remote_runner}" -ErrorAction SilentlyContinue')
            runner_path = TMUX_LOG_DIR / f"{session_id}_run.ps1"
            runner_path.write_text("\r\n".join(ps_lines) + "\r\n", encoding="utf-8")

            # scp the .ps1 script, then launch it as a detached process with log + pid files
            _port = req.ssh_port
            _Pf = f"-P {_port} " if _port and _port != "22" else ""
            _pf = f"-p {_port} " if _port and _port != "22" else ""
            # Start-Process creates a fully detached process that survives SSH disconnect
            launch_ps = (
                "$sd = \\\"$env:TEMP\\odysseus-sessions\\\"; "
                f"Start-Process powershell -ArgumentList '-ExecutionPolicy','Bypass','-File','$HOME\\{remote_runner}' "
                f"-RedirectStandardOutput \\\"$sd\\{session_id}.log\\\" "
                f"-RedirectStandardError \\\"$sd\\{session_id}.err.log\\\" "
                f"-NoNewWindow -PassThru | ForEach-Object {{ $_.Id | Out-File \\\"$sd\\{session_id}.pid\\\" }}"
            )
            setup_cmd = (
                f"scp -O {_Pf}-q '{runner_path}' {remote}:{remote_runner} && "
                f'ssh {_pf}{remote} "powershell -Command \\"{launch_ps}\\""'
            )

        elif remote:
            # ── Linux/Termux remote: create tmux session ON the remote host ──
            remote_runner = f".{session_id}_run.sh"
            runner_lines = ["#!/bin/bash"]
            runner_lines.extend(_user_shell_path_bootstrap())
            runner_lines.append("# Auto-detect environment")
            runner_lines.append("deactivate 2>/dev/null; hash -r")
            if req.hf_token:
                runner_lines.append(f"export HF_TOKEN='{_bash_squote(req.hf_token)}'")
            if _dl_hf_home_shell and not is_ollama_download:
                runner_lines.append(f"export HF_HOME={_dl_hf_home_shell}")
                runner_lines.append(f"export HUGGINGFACE_HUB_CACHE={_dl_hf_home_shell}/hub")
                runner_lines.append(f"export HF_HUB_CACHE={_dl_hf_home_shell}/hub")
            if req.env_prefix:
                runner_lines.append(_safe_env_prefix(req.env_prefix))
            else:
                # Fallback: find a venv with hf CLI, or install huggingface-hub
                runner_lines.append(
                    'for p in ~/vllm-env ~/venv ~/.venv; do '
                    'if [ -f "$p/bin/activate" ]; then source "$p/bin/activate"; break; fi; '
                    'done'
                )
            # Ensure pip-user scripts (e.g. hf CLI installed via --user) are on PATH
            runner_lines.append('export PATH="$HOME/.local/bin:$HOME/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"')
            runner_lines.append('ODYSSEUS_PY="$(command -v python3 || command -v python || true)"')
            runner_lines.append('if [ -z "$ODYSSEUS_PY" ]; then echo "ERROR: python3/python not found on this server."; exit 127; fi')
            # Install hf CLI + optional hf_transfer best-effort. Retries disable
            # hf_transfer because the Rust parallel path is fast but has been
            # flaky near the end of very large multi-file downloads.
            # Use --break-system-packages on PEP-668 systems (Arch, newer Debian) so it doesn't bail.
            if is_ollama_download:
                runner_lines.append('if command -v ollama >/dev/null 2>&1; then')
                runner_lines.append(f'  ODYSSEUS_OLLAMA_PULL_CMD={shlex.quote(ollama_cmd)}')
                runner_lines.append('elif command -v docker >/dev/null 2>&1; then')
                runner_lines.append('  ODYSSEUS_OLLAMA_CONTAINER="$(docker ps --format \'{{.Names}}\' 2>/dev/null | grep -E \'^(ollama-rocm|ollama-test)$\' | head -1)"')
                runner_lines.append('  if [ -n "$ODYSSEUS_OLLAMA_CONTAINER" ]; then')
                runner_lines.append(f'    ODYSSEUS_OLLAMA_PULL_CMD={shlex.quote("docker exec ${ODYSSEUS_OLLAMA_CONTAINER} " + ollama_cmd)}')
                runner_lines.append('  fi')
                runner_lines.append('fi')
                runner_lines.append('if [ -z "$ODYSSEUS_OLLAMA_PULL_CMD" ]; then echo "ERROR: Ollama not found on this server. Install Ollama or start an ollama-rocm/ollama-test container."; exit 127; fi')
            else:
                hf_hub_install = _pip_install_fallback_chain(
                    "huggingface_hub",
                    python_cmd='"$ODYSSEUS_PY" -m pip',
                    upgrade=True,
                )
                runner_lines.append(f"command -v hf >/dev/null 2>&1 || command -v huggingface-cli >/dev/null 2>&1 || {hf_hub_install}")
                runner_lines.append('hash -r 2>/dev/null || true')
                runner_lines.append('ODYSSEUS_HF_CLI="$(command -v hf || command -v huggingface-cli || true)"')
                runner_lines.append('if [ -z "$ODYSSEUS_HF_CLI" ]; then echo "ERROR: HF CLI not found after installing huggingface_hub."; exit 127; fi')
                if req.disable_hf_transfer:
                    runner_lines.append("export HF_HUB_ENABLE_HF_TRANSFER=0")
                    runner_lines.append("export HF_HUB_DOWNLOAD_MAX_WORKERS=4")
                else:
                    hf_transfer_install = _pip_install_fallback_chain(
                        "hf_transfer",
                        python_cmd='"$ODYSSEUS_PY" -m pip',
                    )
                    runner_lines.append(f"\"$ODYSSEUS_PY\" -c 'import hf_transfer' 2>/dev/null || {hf_transfer_install}")
                    runner_lines.append("\"$ODYSSEUS_PY\" -c 'import hf_transfer' 2>/dev/null && export HF_HUB_ENABLE_HF_TRANSFER=1")
                    runner_lines.append("export HF_HUB_DOWNLOAD_MAX_WORKERS=8")
                # Surface whether the HF token actually reached THIS server, so a gated
                # download's "not authorized" failure can be told apart from a missing
                # token (the token is masked — we only print applied / not-set).
                runner_lines.append(_HF_TOKEN_STATUS_SNIPPET)
            # Wrap the download in a retry loop. Large HF/Ollama transfers can
            # hit transient network failures; both backends resume cached partials.
            mw = 4 if req.disable_hf_transfer else 8
            runner_lines.append('_max_retries=10; _attempt=0; _ec=0')
            runner_lines.append('while [ $_attempt -lt $_max_retries ]; do')
            runner_lines.append('  _attempt=$((_attempt+1))')
            if is_ollama_download:
                runner_lines.append('  eval "$ODYSSEUS_OLLAMA_PULL_CMD" < /dev/null')
            else:
                runner_lines.append(f'  "$ODYSSEUS_HF_CLI" {hf_download_args} < /dev/null')
            runner_lines.append('  _ec=$?')
            runner_lines.append('  if [ $_ec -eq 0 ]; then break; fi')
            runner_lines.append('  if [ $_attempt -lt $_max_retries ]; then')
            runner_lines.append('    echo ""; echo "Download attempt $_attempt failed (exit $_ec) — retrying in 30s..."')
            runner_lines.append('    sleep 30')
            runner_lines.append('  fi')
            runner_lines.append('done')
            runner_lines.append('if [ $_ec -eq 0 ]; then echo ""; echo "DOWNLOAD_OK"; else echo ""; echo "DOWNLOAD_FAILED (exit $_ec after $_attempt attempts)"; fi')
            runner_lines.append(f"rm -f {remote_runner}")
            runner_lines.append('exec "${SHELL:-/bin/bash}"')
            runner_path = TMUX_LOG_DIR / f"{session_id}_run.sh"
            runner_path.write_text("\n".join(runner_lines) + "\n", encoding="utf-8")
            # Local temp file is scp'd then chmod'd on the remote; the local bit
            # is irrelevant (no-op on Windows).
            safe_chmod(runner_path, 0o755)

            # scp the runner script, then create tmux session on the remote
            _port = req.ssh_port
            _pf = f"-P {_port} " if _port and _port != "22" else ""
            _spf = f"-p {_port} " if _port and _port != "22" else ""
            setup_cmd = (
                f"scp -O {_pf}-q '{runner_path}' {remote}:{remote_runner} && "
                f"ssh {_spf}{remote} {shlex.quote(_remote_tmux_launch_command(session_id, remote_runner))}"
            )
        else:
            # Local: run hf download in the background (tmux on POSIX, a detached
            # process + logfile on Windows where tmux doesn't exist).
            if req.env_prefix:
                lines.append(_safe_env_prefix(req.env_prefix))
            else:
                lines.append("deactivate 2>/dev/null; hash -r")
            # Show whether the HF token reached this run (masked) — tells a gated
            # "not authorized" failure apart from a missing token.
            if not is_ollama_download:
                lines.append(_HF_TOKEN_STATUS_SNIPPET)
            # Retry loop — same rationale as the remote-bash path. Issue #2722.
            _hf_invoke = 'eval "$ODYSSEUS_OLLAMA_PULL_CMD" < /dev/null' if is_ollama_download else (hf_cmd if IS_WINDOWS else f"{hf_cmd} < /dev/null")
            lines.append('_max_retries=10; _attempt=0; _ec=0')
            lines.append('while [ $_attempt -lt $_max_retries ]; do')
            lines.append('  _attempt=$((_attempt+1))')
            lines.append(f'  {_hf_invoke}')
            lines.append('  _ec=$?')
            lines.append('  if [ $_ec -eq 0 ]; then break; fi')
            lines.append('  if [ $_attempt -lt $_max_retries ]; then')
            lines.append('    echo ""; echo "Download attempt $_attempt failed (exit $_ec) — retrying in 30s..."')
            lines.append('    sleep 30')
            lines.append('  fi')
            lines.append('done')
            lines.append('if [ $_ec -eq 0 ]; then echo ""; echo "DOWNLOAD_OK"; else echo ""; echo "DOWNLOAD_FAILED (exit $_ec after $_attempt attempts)"; fi')
            if not IS_WINDOWS:
                lines.append(f"rm -f '{wrapper_script}'")
                lines.append('exec "${SHELL:-/bin/bash}"')
                wrapper_script.write_text("\n".join(lines) + "\n", encoding="utf-8")
                wrapper_script.chmod(0o755)
            setup_cmd = None if IS_WINDOWS else f"tmux set-option -g history-limit 100000 2>/dev/null; tmux new-session -d -s {session_id} {shlex.quote(str(wrapper_script))}"

        logger.info(f"Model download: {req.repo_id} (backend={'ollama' if is_ollama_download else 'hf'}, include={req.include}, session={session_id}, remote={remote})")
        logger.info(f"Download setup_cmd: {setup_cmd}")

        if setup_cmd is None:
            # LOCAL Windows: launch the bash wrapper detached; no tmux setup_cmd.
            try:
                _launch_local_detached(session_id, lines)
            except Exception as e:
                logger.error(f"Local detached download launch failed: {e}")
                return {"ok": False, "error": str(e), "session_id": session_id}
        else:
            proc = await asyncio.create_subprocess_shell(
                setup_cmd,
                stdout=asyncio.subprocess.PIPE,
                stderr=asyncio.subprocess.PIPE,
            )
            await proc.wait()

            if proc.returncode != 0:
                stderr = (await proc.stderr.read()).decode(errors="replace")
                logger.error(f"Download failed (rc={proc.returncode}): {stderr}")
                return {"ok": False, "error": stderr, "session_id": session_id}

        # Log to assistant
        try:
            from src.assistant_log import log_to_assistant
            from src.auth_helpers import get_current_user
            owner = get_current_user(request)
            log_to_assistant(
                owner,
                f"Started downloading {req.repo_id} to {remote or 'local'}",
                category="Download",
            )
        except Exception:
            pass

        return {"ok": True, "session_id": session_id, "remote": remote or "local"}

    @router.get("/api/model/cached")
    async def model_cached(request: Request, host: str | None = None, model_dir: str | None = None, ssh_port: str | None = None, platform: str | None = None):
        """List cached models. Scans HF cache + optional model directory."""
        require_admin(request)
        # Validate shell-bound inputs, matching the sibling list_gpus endpoint —
        # `host`/`ssh_port` are interpolated into an ssh command below, so an
        # unvalidated value (e.g. "x'; rm -rf ~ #") would be command injection.
        host = validate_remote_host(host)
        ssh_port = validate_ssh_port(ssh_port)
        TMUX_LOG_DIR.mkdir(parents=True, exist_ok=True)

        model_dirs = []
        if model_dir:
            for d in model_dir.split(','):
                d = d.strip()
                if d:
                    if d.startswith(("home/", "mnt/", "media/", "data/", "opt/", "srv/", "var/")):
                        d = "/" + d
                    model_dirs.append(d)
        paths_code = _cached_model_scan_script(model_dirs)

        scan_py = TMUX_LOG_DIR / "scan_cache.py"
        scan_py.write_text(paths_code, encoding="utf-8")

        async def _run_cached_scan_once():
            if host:
                _ssh_opts = "-o BatchMode=yes -o ConnectTimeout=8 -o ServerAliveInterval=4 -o ServerAliveCountMax=1 "
                _pf = f"-p {ssh_port} " if ssh_port and ssh_port != "22" else ""
                if platform == "windows":
                    # Windows: use 'python' and pipe via stdin with double-quote wrapping
                    cmd = f'ssh {_ssh_opts}{_pf}{host} "python -" < \'{scan_py}\''
                else:
                    cmd = f"ssh {_ssh_opts}{_pf}{host} 'python3 -' < '{scan_py}'"
                proc = await asyncio.create_subprocess_shell(
                    cmd,
                    stdout=asyncio.subprocess.PIPE,
                    stderr=asyncio.subprocess.PIPE,
                    cwd=str(Path.home()),
                )
            else:
                # LOCAL scan: use sys.executable (the venv Python Odysseus is already
                # running under) — it's guaranteed real Python on all platforms.
                # Falling back to which_tool on Windows risks hitting the Microsoft
                # Store stub alias for "python3"/"python", which prints
                # "Python was not found; run without arguments to install from the
                # Microsoft Store" and exits 9009, producing empty stdout and a
                # JSON parse error. sys.executable bypasses PATH entirely.
                local_py = sys.executable or (
                    which_tool("python3") or which_tool("python")
                    or which_tool("py") or "python"
                )
                proc = await asyncio.create_subprocess_exec(
                    local_py, str(scan_py),
                    stdout=asyncio.subprocess.PIPE,
                    stderr=asyncio.subprocess.PIPE,
                    cwd=str(Path.home()),
                )
            return await asyncio.wait_for(proc.communicate(), timeout=60), proc.returncode

        (stdout_b, stderr_b), returncode = await _run_cached_scan_once()
        stderr_txt = stderr_b.decode(errors="replace").strip()
        stdout_txt = stdout_b.decode(errors="replace").strip()
        if host and returncode != 0 and _ssh_host_key_changed(stderr_txt):
            ok, detail = await _repair_cookbook_known_host(host, ssh_port)
            if ok:
                logger.info("Repaired Cookbook known_hosts for %s after host-key-change scan failure", host)
                (stdout_b, stderr_b), returncode = await _run_cached_scan_once()
                stderr_txt = stderr_b.decode(errors="replace").strip()
                stdout_txt = stdout_b.decode(errors="replace").strip()
            else:
                logger.warning("Failed to repair Cookbook known_hosts for %s: %s", host, detail[:300])
        if returncode != 0:
            msg = stderr_txt or f"Cached model scan failed with exit code {returncode}"
            logger.warning(f"Cached model scan failed host={host or 'local'} rc={returncode}: {msg[:500]}")
            return {"models": [], "host": host or "local", "error": msg}

        models = []
        try:
            raw = json.loads(stdout_txt)
            for m in raw:
                size_gb = m["size_bytes"] / (1024 ** 3)
                if size_gb >= 1:
                    size_str = f"{size_gb:.1f} GB"
                else:
                    size_str = f"{m['size_bytes'] / (1024**2):.0f} MB"
                entry = {
                    "repo_id": m["repo_id"],
                    "size": size_str,
                    "nb_files": m["nb_files"],
                    "has_incomplete": m["has_incomplete"],
                    "status": "downloading" if m["has_incomplete"] else "ready",
                    "path": m.get("path", ""),
                    "is_diffusion": m.get("is_diffusion", False),
                }
                if m.get("is_local_dir"):
                    entry["is_local_dir"] = True
                if m.get("is_gguf"):
                    entry["is_gguf"] = True
                if m.get("backend"):
                    entry["backend"] = m.get("backend")
                if m.get("is_ollama"):
                    entry["is_ollama"] = True
                if isinstance(m.get("gguf_files"), list):
                    entry["gguf_files"] = m["gguf_files"]
                models.append(entry)
        except Exception as e:
            logger.warning(f"Failed to parse cached models host={host or 'local'}: {e}")
            if stderr_txt:
                logger.warning(f"stderr: {stderr_txt[:500]}")
            msg = stderr_txt or stdout_txt[:500] or str(e)
            return {"models": [], "host": host or "local", "error": msg}

        return {"models": models, "host": host or "local"}

    def _auto_register_image_endpoint(req: ServeRequest, remote: str | None) -> str | None:
        """Register a diffusion model as an image endpoint so it appears in the model selector."""
        import re
        from core.database import SessionLocal, ModelEndpoint

        # Parse port from command (--port NNNN), default 8100 for diffusion_server
        port_match = re.search(r'--port\s+(\d+)', req.cmd)
        port = int(port_match.group(1)) if port_match else 8100

        # Determine host
        if remote:
            # SSH alias — use as hostname (Tailscale resolves it later)
            host = remote.split("@")[-1] if "@" in remote else remote
        else:
            host = "localhost"

        base_url = f"http://{host}:{port}/v1"

        # Friendly display name from repo_id
        short_name = req.repo_id.split("/")[-1] if "/" in req.repo_id else req.repo_id
        display_name = f"{short_name} (image)"

        db = SessionLocal()
        try:
            # Check for existing endpoint with same base_url — update it
            existing = db.query(ModelEndpoint).filter(ModelEndpoint.base_url == base_url).first()
            if existing:
                existing.is_enabled = True
                existing.model_type = "image"
                existing.name = display_name
                db.commit()
                logger.info(f"Updated existing image endpoint: {base_url}")
                return existing.id

            ep_id = f"img-{uuid.uuid4().hex[:8]}"
            ep = ModelEndpoint(
                id=ep_id,
                name=display_name,
                base_url=base_url,
                api_key=None,
                is_enabled=True,
                model_type="image",
            )
            db.add(ep)
            db.commit()
            logger.info(f"Auto-registered image endpoint: {display_name} @ {base_url}")
            return ep_id
        except Exception as e:
            logger.error(f"Failed to auto-register image endpoint: {e}")
            db.rollback()
            return None
        finally:
            db.close()

    def _pick_free_port_for_ollama(
        remote: str | None, ssh_port: str | None, start_port: int, max_offset: int
    ) -> int | None:
        """Return the first free port in [start_port, start_port+max_offset] on
        the target host. Used to pick a real bind for `ollama serve` so we
        don't reattach to an external systemd ollama (or other listener) the
        Cookbook Stop button can't kill."""
        import socket
        if remote:
            # Probe over SSH. Bash's /dev/tcp gives a portable "is anything
            # listening" check without requiring ss/netstat/nmap.
            ssh_base = ["ssh", "-o", "ConnectTimeout=4", "-o", "StrictHostKeyChecking=no"]
            if ssh_port and str(ssh_port) != "22":
                try:
                    ssh_port = validate_ssh_port(ssh_port)
                except HTTPException:
                    return None
                ssh_base.extend(["-p", str(ssh_port)])
            try:
                host_arg = validate_remote_host(remote)
            except HTTPException:
                return None
            if not host_arg:
                return None
            probe_ports = " ".join(str(start_port + i) for i in range(max_offset + 1))
            script = (
                f"for p in {probe_ports}; do "
                "if ! (exec 3<>/dev/tcp/127.0.0.1/$p) 2>/dev/null; then "
                "echo $p; exit 0; fi; exec 3<&-; exec 3>&-; done; exit 1"
            )
            try:
                import subprocess
                r = subprocess.run(
                    ssh_base + [host_arg, script],
                    capture_output=True, text=True, timeout=8,
                )
                if r.returncode == 0:
                    out = (r.stdout or "").strip().splitlines()
                    if out and out[0].isdigit():
                        return int(out[0])
            except Exception:
                return None
            return None
        # Local: just try to connect.
        for off in range(max_offset + 1):
            p = start_port + off
            with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
                s.settimeout(0.25)
                try:
                    s.connect(("127.0.0.1", p))
                except (ConnectionRefusedError, socket.timeout, OSError):
                    return p
        return None

    async def _serve_crash_watchdog(
        endpoint_id: str,
        session_id: str,
        remote: str | None,
        ssh_port: str | None,
        is_windows: bool,
    ) -> None:
        """Drop a freshly-registered endpoint when the cookbook serve dies early.

        The runner script always emits ``=== Process exited with code N ===``
        when the launched cmd terminates (success or failure). We poll the
        tmux pane periodically; on a non-zero exit detected within the watch
        window, the endpoint row is deleted so the picker doesn't keep a
        dead model around. A zero exit (rare for a long-running serve, but
        possible for fast-failing builds that the runner reports as code 0)
        and "missing exit marker" both leave the endpoint alone — that's
        the loading-but-not-yet-bound state, which the probe-marks-offline
        logic already handles.

        Times are picked to outlast realistic vLLM load times (Qwen3.5-122B
        takes ~3 min to load) without burning resources on a stuck-forever
        wait. After the last check, the watchdog gives up — the picker's
        per-endpoint probe takes over from there.
        """
        # Cumulative wait points: 25 s, 60 s, 2 min, 5 min.
        _waits = [25, 35, 60, 180]
        # Tmux capture-pane equivalent of the polling path used elsewhere in
        # this file. Build it once and reuse on each tick. Skip the watchdog
        # entirely on native-Windows local runs (no tmux). The Windows
        # detached-process path writes its log to a known file and has its
        # own lifecycle tracking; punting here keeps the code simple.
        local_win = is_windows and not remote
        if local_win:
            return
        if remote:
            ssh_args = ["ssh"]
            if ssh_port and ssh_port != "22":
                ssh_args.extend(["-p", str(ssh_port)])
            capture_cmd = ssh_args + [remote, _remote_tmux_command("capture-pane", "-t", session_id, "-p", "-S", "-2000")]
        else:
            capture_cmd = ["tmux", "capture-pane", "-t", session_id, "-p", "-S", "-2000"]

        _exit_re = re.compile(r"=== Process exited with code (-?\d+) ===")
        for wait_s in _waits:
            await asyncio.sleep(wait_s)
            try:
                proc = await asyncio.create_subprocess_exec(
                    *capture_cmd,
                    stdout=asyncio.subprocess.PIPE,
                    stderr=asyncio.subprocess.DEVNULL,
                )
                stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=8)
                output = stdout.decode("utf-8", errors="replace")
            except Exception as e:
                logger.debug(f"crash-watchdog: capture-pane failed (will retry): {e!r}")
                continue
            # Last occurrence wins — a serve that exits/restarts under the
            # runner's "exec bash -i" trail will emit multiple markers; the
            # most-recent code is the one that matters.
            matches = list(_exit_re.finditer(output))
            if not matches:
                continue
            try:
                exit_code = int(matches[-1].group(1))
            except (ValueError, IndexError):
                continue
            if exit_code == 0:
                # Exit 0 on a long-running serve is unusual (a normal "loaded
                # then ready" path keeps the process alive) but it happens for
                # commands like "ollama pull" the user might launch through
                # the same form. Don't drop the endpoint on a clean exit;
                # let the probe layer mark it offline if nothing's listening.
                logger.info(f"crash-watchdog: serve {session_id} exited cleanly (0); leaving endpoint {endpoint_id}")
                return
            # Non-zero exit — drop the endpoint.
            try:
                from core.database import SessionLocal as _SL, ModelEndpoint as _ME
                db = _SL()
                try:
                    ep = db.query(_ME).filter(_ME.id == endpoint_id).first()
                    if ep:
                        # A scheduled serve can leave old non-zero exit markers
                        # in tmux scrollback while the current OpenAI endpoint is
                        # actually alive. Verify reachability before deleting the
                        # endpoint row; otherwise chats fall back even though the
                        # served model is ready.
                        try:
                            probe_url = ep.base_url.rstrip("/") + "/models"
                            with urllib.request.urlopen(probe_url, timeout=3) as resp:
                                if 200 <= getattr(resp, "status", 0) < 300:
                                    logger.info(
                                        f"crash-watchdog: serve {session_id} has exit marker {exit_code} "
                                        f"but endpoint {ep.id} is reachable; leaving it registered"
                                    )
                                    return
                        except Exception:
                            pass
                        logger.info(
                            f"crash-watchdog: dropping endpoint {endpoint_id} "
                            f"({ep.name} @ {ep.base_url}) — serve exited {exit_code}"
                        )
                        db.delete(ep)
                        db.commit()
                finally:
                    db.close()
            except Exception as e:
                logger.warning(f"crash-watchdog: endpoint cleanup failed: {e!r}")
            return
        logger.debug(f"crash-watchdog: no exit marker for {session_id} within window; leaving endpoint {endpoint_id}")

    def _auto_register_llm_endpoint(req: ServeRequest, remote: str | None) -> str | None:
        """Register a freshly-served LLM as a model endpoint so it appears in the
        model picker without a manual /setup step — the text-model sibling of
        _auto_register_image_endpoint.

        Cookbook serve commands launch an OpenAI-compatible server (llama.cpp's
        llama-server, vLLM, SGLang, or Ollama) on a known port. We point an
        endpoint at that server's /v1; the picker auto-discovers the model id by
        probing /v1/models and dims the endpoint until the server is reachable,
        so registering immediately (before the server finishes loading) is safe.
        """
        logger.info(
            f"_auto_register_llm_endpoint: ENTRY repo_id={req.repo_id!r} "
            f"remote={remote!r} cmd_prefix={req.cmd[:80]!r}"
        )
        import re
        from core.database import SessionLocal, ModelEndpoint

        # Port: ordered fallbacks so we match whatever the user actually
        # asked for, not a hardcoded default:
        #   1. explicit `--port N`  (vllm / sglang / llama-server)
        #   2. `OLLAMA_HOST=host:port`  (the way Ollama specifies its bind)
        #   3. fallback by backend (11434 ollama / 8080 llama.cpp)
        # Previously the OLLAMA_HOST form was silently ignored and we
        # registered every Ollama endpoint at 11434 — even if the user
        # set OLLAMA_HOST=0.0.0.0:11435 to avoid colliding with an
        # existing systemd Ollama, the registered endpoint pointed at
        # the OLD port and showed as offline.
        port_match = re.search(r'--port\s+(\d+)', req.cmd)
        ollama_host_match = re.search(r'OLLAMA_HOST=[^\s]*?:(\d+)', req.cmd)
        if port_match:
            port = int(port_match.group(1))
        elif ollama_host_match:
            port = int(ollama_host_match.group(1))
        elif "ollama" in req.cmd:
            port = 11434
        else:
            port = 8080  # llama.cpp's llama-server default — the Apple Silicon path

        # Determine host. The cookbook tmux for `local=true` serves runs INSIDE
        # the odysseus container — so the right URL for the in-container
        # backend to reach it is `localhost`, NOT `host.docker.internal`
        # (the latter points at the docker HOST, which doesn't have a server
        # on that port). The previous host.docker.internal fallback only made
        # sense for /setup-added external services like systemd Ollama on the
        # host — and those go through manual setup, not this auto-register
        # code path. For remote serves we still use the SSH host alias.
        if remote:
            host = remote.split("@")[-1] if "@" in remote else remote
        elif re.search(r"\bdocker\s+exec\s+(?:ollama-rocm|ollama-test)\b", req.cmd or ""):
            host = "host.docker.internal"
        else:
            host = "localhost"

        base_url = f"http://{host}:{port}/v1"

        short_name = req.repo_id.split("/")[-1] if "/" in req.repo_id else req.repo_id
        display_name = short_name or "Local model"
        is_mlx_deepseek_v4 = (
            "mlx_lm.server" in (req.cmd or "")
            and "deepseek-v4" in ((req.repo_id or "") + " " + (req.cmd or "")).lower()
        )
        mlx_shim_model_id = ""
        if is_mlx_deepseek_v4 and short_name:
            home_match = re.search(r"((?:/Users|/home)/[^/\s'\"]+)", req.cmd or "")
            remote_home = home_match.group(1) if home_match else ""
            if remote_home:
                mlx_shim_model_id = f"{remote_home}/.cache/odysseus/mlx-shims/{short_name}"

        # If the serve command opts models into OpenAI tool-calling, record it so
        # agent_loop trusts emitted tool_calls instead of the name heuristic.
        is_ollama_endpoint = "ollama" in (req.cmd or "").lower()
        supports_tools = True if "--enable-auto-tool-choice" in req.cmd else None
        # Pin the model the user launched for every Cookbook-created LLM
        # endpoint, not just Ollama. Some OpenAI-compatible servers report a
        # deployment alias from /v1/models, and a stale server can answer on the
        # same port while the new launch failed. Keeping the requested model id
        # pinned makes the picker reflect the actual launch intent.
        pinned_models = [mlx_shim_model_id] if mlx_shim_model_id else ([req.repo_id] if req.repo_id else [])

        db = SessionLocal()
        try:
            # Reuse an endpoint already pointed at this URL instead of duplicating.
            existing = db.query(ModelEndpoint).filter(ModelEndpoint.base_url == base_url).first()
            if existing:
                existing.is_enabled = True
                existing.model_type = "llm"
                existing.name = display_name
                existing.endpoint_kind = "local"
                existing.model_refresh_mode = "auto"
                if pinned_models:
                    try:
                        existing_pinned = json.loads(existing.pinned_models or "[]")
                    except Exception:
                        existing_pinned = []
                    merged_pinned = []
                    for mid in [*existing_pinned, *pinned_models]:
                        if mid and mid not in merged_pinned:
                            merged_pinned.append(mid)
                    existing.pinned_models = json.dumps(merged_pinned) if merged_pinned else None
                if is_ollama_endpoint:
                    existing.endpoint_kind = "ollama"
                    if pinned_models:
                        existing.cached_models = json.dumps(pinned_models)
                if supports_tools is not None:
                    existing.supports_tools = supports_tools
                db.commit()
                logger.info(f"Updated existing local model endpoint: {base_url}")
                # Re-probe so cached_models matches what the server actually
                # serves right now (the URL may have stayed the same but the
                # model behind it changed across launches).
                try:
                    if mlx_shim_model_id:
                        existing.cached_models = json.dumps([mlx_shim_model_id])
                        existing.pinned_models = json.dumps([mlx_shim_model_id])
                        db.commit()
                    else:
                        from routes.model_routes import _probe_endpoint
                        import json as _json2
                        probed = _probe_endpoint(base_url, existing.api_key, timeout=5)
                        if probed:
                            existing.cached_models = _json2.dumps(probed)
                            db.commit()
                except Exception as _pe:
                    logger.warning(f"Re-probe failed for {base_url}: {_pe!r}")
                # Sweep stale dupes: other endpoints with the same display name
                # at DIFFERENT URLs (likely failed earlier-attempt ports) get
                # deleted so the picker doesn't show an offline ghost next to
                # the working one. Only sweeps endpoints whose id starts with
                # `local-` so we never touch a user's hand-added DeepSeek/OpenAI/
                # etc. entry with a coincidentally matching name.
                stale = (db.query(ModelEndpoint)
                         .filter(ModelEndpoint.name == display_name)
                         .filter(ModelEndpoint.base_url != base_url)
                         .filter(ModelEndpoint.id.like("local-%"))
                         .all())
                for s in stale:
                    logger.info(f"Sweeping stale local endpoint {s.id} ({s.base_url})")
                    db.delete(s)
                if stale:
                    db.commit()
                return existing.id

            ep_id = f"local-{uuid.uuid4().hex[:8]}"
            ep = ModelEndpoint(
                id=ep_id,
                name=display_name,
                base_url=base_url,
                api_key=None,
                is_enabled=True,
                model_type="llm",
                endpoint_kind="ollama" if is_ollama_endpoint else "local",
                model_refresh_mode="auto",
                cached_models=json.dumps(pinned_models) if pinned_models else None,
                pinned_models=json.dumps(pinned_models) if pinned_models else None,
                supports_tools=supports_tools,
            )
            db.add(ep)
            db.commit()
            logger.info(f"Auto-registered local model endpoint: {display_name} @ {base_url}")
            # Same sweep on first-register path: drop any pre-existing local-*
            # endpoints with this display name pointed elsewhere.
            stale = (db.query(ModelEndpoint)
                     .filter(ModelEndpoint.name == display_name)
                     .filter(ModelEndpoint.id != ep_id)
                     .filter(ModelEndpoint.id.like("local-%"))
                     .all())
            for s in stale:
                logger.info(f"Sweeping stale local endpoint {s.id} ({s.base_url})")
                db.delete(s)
            if stale:
                db.commit()
            # Probe /v1/models NOW and write cached_models so the chat
            # picker actually shows the model on the next /api/models
            # call. Without this immediate probe, the endpoint has empty
            # cached_models until the next background refresh fires (up
            # to a minute later) and the picker shows nothing — even
            # though the endpoint is in the DB and the server is up.
            try:
                if mlx_shim_model_id:
                    ep.cached_models = json.dumps([mlx_shim_model_id])
                    ep.pinned_models = json.dumps([mlx_shim_model_id])
                    db.commit()
                    logger.info(f"Auto-register: pinned MLX DeepSeek-V4 shim model @ {base_url}")
                else:
                    from routes.model_routes import _probe_endpoint
                    import json as _json2
                    probed = _probe_endpoint(base_url, None, timeout=5)
                    if probed:
                        ep.cached_models = _json2.dumps(probed)
                        db.commit()
                        logger.info(f"Auto-register: probed {len(probed)} models @ {base_url}")
            except Exception as _pe:
                logger.warning(f"Auto-register: probe-after-create failed for {base_url}: {_pe!r}")
            return ep_id
        except Exception as e:
            logger.error(f"Failed to auto-register local model endpoint: {e}")
            db.rollback()
            return None
        finally:
            db.close()

    @router.post("/api/model/serve")
    async def model_serve(request: Request, req: ServeRequest):
        """Launch a model server in a tmux session (or PowerShell background process on Windows).

        `repo_id` is dual-purpose: a HuggingFace repo (`<org>/<name>`) for
        model-serve commands, a cached local-model id (the folder name reported
        by `/api/model/cached`) for models scanned from a custom model dir, OR a
        bare pip package name when the cmd is a `python -m pip install …`. We
        keep strict validation, but serving local cached models must not require
        a fake org/name wrapper.
        """
        require_admin(request)
        # Defence-in-depth: reject values that could break out of shell contexts.
        validate_remote_host(req.remote_host)
        req.ssh_port = validate_ssh_port(req.ssh_port)
        req.gpus = _validate_gpus(req.gpus)
        req.hf_token = req.hf_token or _load_stored_hf_token()
        _validate_token(req.hf_token)
        # Cookbook emits two fixed Docker exec forms for its Ollama sidecars.
        # Keep Docker out of the general allowlist: only these parsed shapes may
        # proceed to the target-aware Docker availability/opt-in preflight.
        if _is_generated_ollama_docker_exec_cmd(req.cmd):
            req.cmd = req.cmd.strip()
        else:
            # Normalize away backslash-newline continuations (multi-line pasted
            # serve commands) so the cleaned single-line command is what gets
            # written into the runner script and used for engine auto-detection.
            # `_validate_serve_cmd` returns None for empty input; coerce to "" so
            # downstream `"engine" in req.cmd` checks cannot raise TypeError.
            req.cmd = _validate_serve_cmd(req.cmd) or ""
        req.cmd = _normalize_llama_cpp_python_cache_types(req.cmd) or ""
        req.cmd = _normalize_minimax_m3_vllm_cmd(req.cmd)
        req.cmd = _normalize_deepseek_v4_sglang_cmd(req.cmd)
        req.cmd = _venv_safe_local_pip_install_cmd(
            req.cmd,
            local=not bool(req.remote_host),
            in_venv=sys.prefix != sys.base_prefix,
        )
        is_pip_install = bool(req.cmd and "pip install" in req.cmd)
        if is_pip_install:
            # Keep big dependency wheel builds (vLLM, …) off the home filesystem's
            # pip cache so they don't fail mid-build with "No space left" (#1219)
            # and leave the dep installed-but-unusable (#1459).
            req.cmd = _pip_install_no_cache(req.cmd)
            # Accept common aliases and enforce server extras for llama-cpp so
            # `python -m llama_cpp.server` has all runtime dependencies.
            # CRITICAL: the lookbehind / lookahead must also exclude `/` so
            # the regex DOESN'T mangle a URL path like
            #   https://abetlen.github.io/llama-cpp-python/whl/cu124
            # The previous regex turned that URL into
            #   https://abetlen.github.io/llama-cpp-python[server]/whl/cu124
            # which pip then couldn't resolve → silent fallback to source
            # build of the .tar.gz → CPU-only binary (because CMAKE_ARGS
            # isn't set), defeating the entire purpose of the CUDA index.
            req.cmd = re.sub(r"(?<![A-Za-z0-9_.\-/])llama_cpp(?![A-Za-z0-9_.\-/])", "llama-cpp-python[server]", req.cmd)
            req.cmd = re.sub(r"(?<![A-Za-z0-9_.\-/])llama-cpp-python(?![\[/])", "llama-cpp-python[server]", req.cmd)
            if "llama-cpp-python" in req.cmd and "--extra-index-url" not in req.cmd:
                req.cmd += " --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cpu"
            # PEP-508-style package spec — letters, digits, `.-_` for the
            # name; `[` `]` for extras; `<>=!~,` for version specifiers.
            # v2 review HIGH-14: tightened from the previous regex which
            # also allowed spaces and `+`, both of which can be abused to
            # introduce extra shell tokens once interpolated into the
            # serve command. We now use `re.fullmatch` and drop space/`+`.
            if not req.repo_id or not re.fullmatch(
                r"[A-Za-z0-9][A-Za-z0-9._\-\[\]<>=!,~]{0,200}", req.repo_id
            ):
                raise HTTPException(400, "Invalid pip package name")
        else:
            _validate_serve_model_id(req.repo_id)
        TMUX_LOG_DIR.mkdir(parents=True, exist_ok=True)
        session_id = f"serve-{uuid.uuid4().hex[:8]}"
        remote = req.remote_host
        is_windows = req.platform == "windows"

        # Ollama: if the user didn't pin a port, resolve the actual port we'll
        # bind to here (before runner construction) by probing the target host.
        # Otherwise the runner script picks one at runtime and `_auto_register`
        # below still registers the stale 11434 default — which on a host with
        # a systemd ollama lands on the wrong (unreachable-from-docker) service.
        # Match "ollama serve" as a phrase (with optional flags after), not
        # any substring containing "ollama" — otherwise commands like
        # `docker exec ollama-test ollama-import …` get wrapped as if they
        # were native `ollama serve`, prepending OLLAMA_HOST=… and then
        # running the ollama-not-found preflight which exits 127.
        if re.search(r"\bollama\s+serve\b", req.cmd) and "OLLAMA_HOST=" not in req.cmd:
            _ollama_bind_host = "0.0.0.0" if remote else "127.0.0.1"
            _ollama_chosen_port = _pick_free_port_for_ollama(
                remote, req.ssh_port, start_port=11434, max_offset=10,
            )
            if _ollama_chosen_port:
                req.cmd = f"OLLAMA_HOST={_ollama_bind_host}:{_ollama_chosen_port} {req.cmd}"
        # LOCAL execution on a native-Windows host never uses tmux (detached
        # process path below), regardless of the UI-supplied platform.
        local_windows = IS_WINDOWS and not remote
        if is_windows and remote and "diffusion_server.py" in req.cmd:
            raise HTTPException(
                400,
                "Remote Windows Diffusers serving is not supported yet; use local Windows or a Linux remote server.",
            )

        if not is_windows and not local_windows and not await _binary_available("tmux", remote, req.ssh_port):
            return {
                "ok": False,
                "error": _missing_binary_message("tmux", remote or "local server"),
                "session_id": session_id,
            }
        if _needs_binary(req.cmd, "docker") and not await _binary_available("docker", remote, req.ssh_port, windows=is_windows):
            local_host_docker_blocked = (
                not remote
                and running_in_container()
                and not host_docker_access_enabled()
            )
            return {
                "ok": False,
                "error": _missing_binary_message(
                    "docker",
                    remote or "local server",
                    local_host_docker_blocked=local_host_docker_blocked,
                ),
                "session_id": session_id,
            }

        if is_windows and remote:
            # ── Windows remote: generate .ps1 serve runner ──
            remote_runner = f".{session_id}_run.ps1"
            ps_lines = []
            ps_lines.append('$sessionDir = "$env:TEMP\\odysseus-sessions"')
            ps_lines.append('New-Item -ItemType Directory -Force -Path $sessionDir | Out-Null')
            if req.hf_token:
                ps_lines.append(f"$env:HF_TOKEN = '{_ps_squote(req.hf_token)}'")
            if req.gpus:
                ps_lines.append(f"$env:CUDA_VISIBLE_DEVICES = '{req.gpus}'")
            if req.env_prefix:
                ps_lines.append(_safe_env_prefix(req.env_prefix))
            # Auto-install ollama if the command uses it
            if "ollama" in req.cmd:
                ps_lines.append('# Check if ollama is available')
                ps_lines.append('if (-not (Get-Command ollama -ErrorAction SilentlyContinue)) {')
                ps_lines.append('  Write-Host "Ollama not found. Please install from https://ollama.com/download/windows"')
                ps_lines.append('  exit 1')
                ps_lines.append('}')
            elif "llama_cpp" in req.cmd or "llama-server" in req.cmd:
                ps_lines.append('# Auto-install llama-cpp-python if missing')
                ps_lines.append('try { python -c "import llama_cpp" 2>$null } catch {}')
                ps_lines.append('if ($LASTEXITCODE -ne 0) {')
                ps_lines.append('  Write-Host "Installing llama-cpp-python..."')
                ps_lines.append('  python -m pip install llama-cpp-python[server]')
                ps_lines.append('}')
            elif "vllm" in req.cmd:
                ps_lines.append('Write-Host "ERROR: vLLM is not supported on Windows. Use Ollama or llama.cpp instead."')
                ps_lines.append('exit 1')
            ps_lines.append(req.cmd)
            if is_pip_install:
                ps_lines.append('if ($LASTEXITCODE -eq 0) { Write-Host ""; Write-Host "DOWNLOAD_OK" }')
            ps_lines.append('Write-Host ""')
            ps_lines.append('Write-Host "=== Process exited with code $LASTEXITCODE ==="')
            runner_path = TMUX_LOG_DIR / f"{session_id}_run.ps1"
            runner_path.write_text("\r\n".join(ps_lines) + "\r\n", encoding="utf-8")

            _port = req.ssh_port
            _Pf = f"-P {_port} " if _port and _port != "22" else ""
            _pf = f"-p {_port} " if _port and _port != "22" else ""
            launch_ps = (
                "$sd = \\\"$env:TEMP\\odysseus-sessions\\\"; "
                f"Start-Process powershell -ArgumentList '-ExecutionPolicy','Bypass','-File','$HOME\\{remote_runner}' "
                f"-RedirectStandardOutput \\\"$sd\\{session_id}.log\\\" "
                f"-RedirectStandardError \\\"$sd\\{session_id}.err.log\\\" "
                f"-NoNewWindow -PassThru | ForEach-Object {{ $_.Id | Out-File \\\"$sd\\{session_id}.pid\\\" }}"
            )
            setup_cmd = (
                f"scp -O {_Pf}-q '{runner_path}' {remote}:{remote_runner} && "
                f'ssh {_pf}{remote} "powershell -Command \\"{launch_ps}\\""'
            )
        else:
            # ── Linux/Termux: bash + tmux (existing flow) ──
            runner_lines = ["#!/bin/bash"]
            # Mirror every line of stdout+stderr into a persistent log file
            # on the host running the serve. This is the file tail_serve_output
            # reads when the tmux pane has been overwritten by the post-crash
            # bash prompt — without it, the agent's diagnostic tool sees the
            # neofetch banner instead of the actual Python traceback.
            # We save the original fds to 3/4 so we can RESTORE them before
            # `exec ${SHELL}` at the end of the script. Without that restore,
            # the post-crash interactive shell's neofetch banner ALSO gets
            # teed into the log file and `tail -N` returns ONLY the banner —
            # the actual traceback ends up earlier than the tail window.
            runner_lines.append("mkdir -p /tmp/odysseus-tmux 2>/dev/null || true")
            runner_lines.append("exec 3>&1 4>&2")
            runner_lines.append(
                f"exec > >(tee -a /tmp/odysseus-tmux/{session_id}.log) 2>&1"
            )
            runner_lines.extend(_user_shell_path_bootstrap())
            runner_lines.append('ODYSSEUS_PREFLIGHT_EXIT=""')
            # Put Odysseus's own venv bin on PATH (local runs only) so the serve
            # shell resolves the bundled python3/hf, mirroring the download flow.
            if not remote:
                runner_lines.append(_local_tooling_path_export(sys.executable))
                if local_windows:
                    # Detached Git Bash runs do not always inherit recently edited
                    # user PATH entries from the already-running Odysseus process.
                    runner_lines.append('export PATH="$HOME/bin:$HOME/llama.cpp/build-cuda/bin/Release:$HOME/llama.cpp/build/bin/Release:$HOME/llama.cpp/build/bin/Debug:$HOME/llama.cpp/build/bin:$PATH"')
            runner_lines.append("export FLASHINFER_DISABLE_VERSION_CHECK=1")
            if req.hf_token:
                runner_lines.append(f"export HF_TOKEN='{_bash_squote(req.hf_token)}'")
            if req.gpus:
                runner_lines.append(f"export CUDA_VISIBLE_DEVICES='{req.gpus}'")
            if req.env_prefix:
                runner_lines.append(_safe_env_prefix(req.env_prefix))
            else:
                runner_lines.append("deactivate 2>/dev/null; hash -r")
            _append_venv_nvidia_library_path_lines(runner_lines, cmd=req.cmd)
            if "sglang.launch_server" in req.cmd or "mlx_lm.server" in req.cmd or re.search(r"\bvllm\s+serve\b", req.cmd or ""):
                _append_openai_port_preflight_lines(runner_lines, cmd=req.cmd, expected_model=req.repo_id)
            # Show whether the HF token reached this server (masked) — a gated
            # model vLLM has to download will be denied without it.
            runner_lines.append(_HF_TOKEN_STATUS_SNIPPET)
            handled_ollama_serve = False
            # Auto-install inference engine if missing
            local_windows_llama_cmd = local_windows and ("llama_cpp" in req.cmd or "llama-server" in req.cmd)
            if ("llama_cpp" in req.cmd or "llama-server" in req.cmd) and not local_windows_llama_cmd:
                # Prefer the NATIVE llama-server binary — its minja templating
                # renders modern GGUF chat templates that the Python bindings'
                # Jinja2 rejects (do_tojson ensure_ascii). Build it once from
                # source if missing; keep llama-cpp-python only as a fallback.
                runner_lines.append('# Ensure a llama.cpp server (prefer native llama-server)')
                # Include the Homebrew bin dirs so a brew-installed llama-server /
                # ollama is found (otherwise macOS falls back to a slow source build).
                # /opt/homebrew = Apple Silicon, /usr/local = Intel; harmless on Linux.
                runner_lines.append('export PATH="$HOME/.local/bin:$HOME/bin:$HOME/llama.cpp/build/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"')
                runner_lines.append('if [ -d /data/data/com.termux ]; then')
                runner_lines.append('  # Termux: no native build — use the Python bindings (CPU).')
                runner_lines.append('  if ! python3 -c "import llama_cpp" 2>/dev/null; then')
                runner_lines.append('    pkg install -y cmake 2>/dev/null')
                runner_lines.append('    pip install numpy diskcache jinja2 2>/dev/null')
                runner_lines.append('    CMAKE_ARGS="-DGGML_BLAS=OFF -DGGML_LLAMAFILE=OFF" pip install \'llama-cpp-python[server]\' --no-build-isolation --no-cache-dir 2>&1 || true')
                runner_lines.append('  fi')
                runner_lines.append('elif ! command -v llama-server &>/dev/null; then')
                runner_lines.append('  echo "Native llama-server not found — building from source (one-time, may take a few minutes)..."')
                runner_lines.append('  mkdir -p ~/bin')
                runner_lines.append('  cd ~ && [ -d llama.cpp ] || git clone --depth 1 https://github.com/ggml-org/llama.cpp')
                # Build with the right accelerator: Metal on macOS (llama.cpp
                # enables it automatically, no flag), CUDA on Linux when present,
                # else a plain CPU build. nproc is Linux-only — fall back to
                # `sysctl hw.ncpu` on macOS. (Tip: `brew install llama.cpp` ships
                # a prebuilt llama-server and skips this whole source build.)
                runner_lines.append('  NPROC="$(nproc 2>/dev/null || sysctl -n hw.ncpu 2>/dev/null || echo 4)"')
                runner_lines.append('  if [ "$(uname -s)" = "Darwin" ]; then')
                runner_lines.append('    command -v cmake >/dev/null 2>&1 || echo "WARNING: cmake not found — install it with: brew install cmake (or: brew install llama.cpp for a prebuilt llama-server)."')
                # Start from a clean cache: a prior failed configure (e.g. a CUDA
                # attempt) poisons build/CMakeCache.txt, so a plain `cmake -B build`
                # would reuse the bad settings and fail again. CMAKE_BUILD_TYPE is
                # explicit so the binary is optimized (Metal auto-enables on macOS).
                runner_lines.append('    cd ~/llama.cpp && rm -rf build && cmake -B build -DCMAKE_BUILD_TYPE=Release \\')
                runner_lines.append('      && cmake --build build -j"$NPROC" --target llama-server \\')
                runner_lines.append('      && ln -sf ~/llama.cpp/build/bin/llama-server ~/bin/llama-server')
                runner_lines.append('  else')
                _append_llama_cpp_linux_accel_build_lines(runner_lines)
                runner_lines.append('  fi')
                # Source the env file the prebuilt-download path writes so
                # LD_LIBRARY_PATH includes the directory holding libllama.so
                # and friends. No-op when prebuilt wasn't used.
                runner_lines.append('  [ -r ~/.config/odysseus-llama-cpp-env ] && . ~/.config/odysseus-llama-cpp-env')
                # Auto-upgrade pip llama-cpp-python to the CUDA-enabled
                # wheel when (a) NVIDIA hardware is present and (b) the
                # currently-installed wheel is CPU-only. Without this the
                # user gets the Python server happily running at 3 tok/s
                # because pip's default index ships CPU-only wheels.
                # Forward-compat: cu124 wheels work on driver/runtime
                # 12.4+ including the cu13.x line.
                runner_lines.append('  if command -v nvidia-smi >/dev/null 2>&1 && nvidia-smi -L 2>/dev/null | grep -q "GPU " && python3 -c "import llama_cpp" 2>/dev/null; then')
                runner_lines.append('    if ! python3 -c "import llama_cpp; import sys; sys.exit(0 if llama_cpp.llama_supports_gpu_offload() else 1)" 2>/dev/null; then')
                runner_lines.append('      echo "[odysseus] NVIDIA detected but installed llama-cpp-python is CPU-only — reinstalling with CUDA wheel index for GPU offload..."')
                runner_lines.append('      python3 -m pip install --user --break-system-packages --force-reinstall --no-cache-dir "llama-cpp-python[server]" --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu124 2>&1 | tail -8 || echo "[odysseus] WARNING: CUDA wheel reinstall failed — Python server will stay CPU-only (slow). Manual fix: pip install --user --force-reinstall \'llama-cpp-python[server]\' --extra-index-url https://abetlen.github.io/llama-cpp-python/whl/cu124"')
                runner_lines.append('      if python3 -c "import llama_cpp; import sys; sys.exit(0 if llama_cpp.llama_supports_gpu_offload() else 1)" 2>/dev/null; then')
                runner_lines.append('        echo "[odysseus] llama-cpp-python now supports GPU offload."')
                runner_lines.append('      fi')
                runner_lines.append('    fi')
                runner_lines.append('  fi')
                # SHORT-CIRCUIT before the build/pip fallback: if the
                # native binary is missing but llama_cpp Python is already
                # installed, drop a wrapper at ~/bin/llama-server that
                # translates llama-server CLI args to llama_cpp.server's
                # underscore-style flags. The user's serve command stays
                # `llama-server ...` and "just works" — no build, no cmake,
                # no second install. This is the path that unblocks every
                # remote where pip-installed llama-cpp-python is already
                # working but Cookbook used to insist on a native binary.
                runner_lines.append('  if ! command -v llama-server >/dev/null 2>&1 && python3 -c "import llama_cpp" 2>/dev/null; then')
                runner_lines.append('    mkdir -p ~/bin')
                runner_lines.append('    cat > ~/bin/llama-server <<\'_ODY_LLAMA_SHIM_EOF\'')
                runner_lines.append('#!/usr/bin/env bash')
                runner_lines.append('# Auto-generated by Odysseus Cookbook: a `llama-server` lookalike')
                runner_lines.append('# that translates the native CLI to `python -m llama_cpp.server`.')
                runner_lines.append('# Lets cookbook-generated launch commands run unchanged on hosts')
                runner_lines.append('# where only the pip llama-cpp-python package is installed.')
                runner_lines.append('ARGS=()')
                runner_lines.append('while [ $# -gt 0 ]; do')
                runner_lines.append('  case "$1" in')
                runner_lines.append('    -ngl|--gpu-layers|--n-gpu-layers) ARGS+=(--n_gpu_layers "$2"); shift 2 ;;')
                runner_lines.append('    -c|--ctx-size) ARGS+=(--n_ctx "$2"); shift 2 ;;')
                runner_lines.append('    -b|--batch-size) ARGS+=(--n_batch "$2"); shift 2 ;;')
                runner_lines.append('    -ub|--ubatch-size) shift 2 ;;  # llama-cpp-python has no separate ubatch')
                runner_lines.append('    --flash-attn) ARGS+=(--flash_attn true); shift 2 ;;')
                runner_lines.append('    --cache-type-k) ARGS+=(--type_k "$2"); shift 2 ;;')
                runner_lines.append('    --cache-type-v) ARGS+=(--type_v "$2"); shift 2 ;;')
                runner_lines.append('    --n-cpu-moe) ARGS+=(--n_cpu_moe "$2"); shift 2 ;;')
                runner_lines.append('    --mmproj) ARGS+=(--clip_model_path "$2"); shift 2 ;;')
                runner_lines.append('    --image-max-tokens) shift 2 ;;  # native-only')
                runner_lines.append('    --no-mmap) ARGS+=(--no_mmap true); shift ;;')
                runner_lines.append('    --no-warmup) shift ;;  # native-only')
                runner_lines.append('    --chat-template) ARGS+=(--chat_format "$2"); shift 2 ;;')
                runner_lines.append('    --fit|--split-mode|--tensor-split|--main-gpu|--parallel) shift 2 ;;  # native-only')
                runner_lines.append('    --mlock) ARGS+=(--use_mlock true); shift ;;')
                runner_lines.append('    *) ARGS+=("$1"); shift ;;')
                runner_lines.append('  esac')
                runner_lines.append('done')
                runner_lines.append('exec python3 -m llama_cpp.server "${ARGS[@]}"')
                runner_lines.append('_ODY_LLAMA_SHIM_EOF')
                runner_lines.append('    chmod +x ~/bin/llama-server')
                runner_lines.append('    echo "[odysseus] Created llama-server shim → python -m llama_cpp.server (no native binary needed)"')
                runner_lines.append('  fi')
                runner_lines.append('  # If the native build failed, fall back to the Python bindings.')
                runner_lines.append('  if ! command -v llama-server &>/dev/null && ! python3 -c "import llama_cpp" 2>/dev/null; then')
                runner_lines.append('    echo "llama-server build failed — installing Python bindings as fallback..."')
                runner_lines.append(f"    {_pip_install_fallback_chain('llama-cpp-python[server]', python_cmd='pip')} || true")
                runner_lines.append('  fi')
                runner_lines.append('  if ! command -v llama-server &>/dev/null && ! python3 -c "import llama_cpp" 2>/dev/null; then')
                runner_lines.append('    echo "ERROR: llama.cpp serving is not available after install/build attempts."')
                runner_lines.append('    ODYSSEUS_PREFLIGHT_EXIT=127')
                runner_lines.append('  fi')
                runner_lines.append('fi')
            elif re.search(r"\bollama\s+serve\b", req.cmd):
                handled_ollama_serve = True
                _ollama_default_host = "0.0.0.0" if remote else "127.0.0.1"
                _ollama_host, _ollama_port = _ollama_bind_from_cmd(
                    req.cmd,
                    default_host=_ollama_default_host,
                )
                # Always launch a fresh ollama under tmux so Stop reliably
                # kills it. If the requested port is busy (e.g. a systemd
                # ollama on 11434), scan upward for a free one rather than
                # silently reattaching to an external service that Stop
                # can't reach.
                runner_lines.append(f'ODYSSEUS_OLLAMA_HOST={_bash_squote(_ollama_host)}')
                runner_lines.append(f'ODYSSEUS_OLLAMA_PORT="{_ollama_port}"')
                runner_lines.append('for _ody_off in 0 1 2 3 4 5 6 7 8 9; do')
                runner_lines.append('  _ody_try_port=$((ODYSSEUS_OLLAMA_PORT + _ody_off))')
                runner_lines.append('  if ! (exec 3<>/dev/tcp/127.0.0.1/$_ody_try_port) 2>/dev/null; then')
                runner_lines.append('    exec 3<&-; exec 3>&-')
                runner_lines.append('    ODYSSEUS_OLLAMA_PORT="$_ody_try_port"')
                runner_lines.append('    break')
                runner_lines.append('  fi')
                runner_lines.append('  exec 3<&-; exec 3>&-')
                runner_lines.append('done')
                runner_lines.append('if ! command -v ollama &>/dev/null; then')
                # Single-quoted on purpose: backticks inside a double-quoted
                # echo are command substitution, and this line used to run the
                # curl|sh installer on the target host instead of printing it.
                runner_lines.append(f"  echo '{_bash_squote(OLLAMA_MISSING_HINT)}'")
                runner_lines.append('  echo')
                runner_lines.append('  echo "=== Process exited with code 127 ==="')
                runner_lines.append('  exec bash -i')
                runner_lines.append('fi')
                runner_lines.append('ODYSSEUS_OLLAMA_URL="http://${ODYSSEUS_OLLAMA_HOST}:${ODYSSEUS_OLLAMA_PORT}"')
                if remote and _ollama_host in ("0.0.0.0", "::"):
                    runner_lines.append('echo "[odysseus] WARNING: remote Ollama will bind to ${ODYSSEUS_OLLAMA_HOST}:${ODYSSEUS_OLLAMA_PORT} so Odysseus can reach it from this host."')
                    runner_lines.append('echo "[odysseus] Ollama has no built-in authentication; expose this only on a trusted LAN/VPN or provide an explicit OLLAMA_HOST with your own access controls."')
                runner_lines.append('echo "Starting ollama server on ${ODYSSEUS_OLLAMA_HOST}:${ODYSSEUS_OLLAMA_PORT}..."')
                runner_lines.append('OLLAMA_HOST="${ODYSSEUS_OLLAMA_HOST}:${ODYSSEUS_OLLAMA_PORT}" ollama serve')
                runner_lines.append('_ody_exit=$?')
                runner_lines.append('echo')
                runner_lines.append('echo "=== Process exited with code ${_ody_exit} ==="')
                runner_lines.append('exec bash -i')
            elif "vllm serve" in req.cmd:
                # vLLM is CUDA/ROCm-only and does not run on macOS at all.
                runner_lines.append('if [ "$(uname -s)" = "Darwin" ]; then')
                runner_lines.append('  echo "ERROR: vLLM does not run on macOS. Use Ollama or llama.cpp (Metal) instead."')
                runner_lines.append('  ODYSSEUS_PREFLIGHT_EXIT=1')
                runner_lines.append('fi')
                # Put ~/.local/bin on PATH first — without a venv, vllm installs
                # there via --user and the non-login serve shell otherwise can't
                # find the `vllm` CLI ("command not found"). Mirrors llama.cpp above.
                runner_lines.append('export PATH="$HOME/.local/bin:$PATH"')
                runner_lines.append('if ! command -v vllm &>/dev/null; then')
                runner_lines.append('  echo "ERROR: vLLM is not installed."')
                runner_lines.append('  ODYSSEUS_PREFLIGHT_EXIT=127')
                runner_lines.append('fi')
                runner_lines.append(f"ODYSSEUS_SERVE_CMD='{_bash_squote(req.cmd)}'")
                runner_lines.append('if [ -z "$ODYSSEUS_PREFLIGHT_EXIT" ]; then')
                runner_lines.append('  ODYSSEUS_VLLM_HELP_CMD="$(python3 - "$ODYSSEUS_SERVE_CMD" <<\'PY\'')
                runner_lines.append('import shlex, sys')
                runner_lines.append('parts = shlex.split(sys.argv[1])')
                runner_lines.append('try:')
                runner_lines.append('    serve_i = parts.index("serve")')
                runner_lines.append('except ValueError:')
                runner_lines.append('    print("vllm serve --help")')
                runner_lines.append('else:')
                runner_lines.append('    print(shlex.join(parts[:serve_i + 1] + ["--help"]))')
                runner_lines.append('PY')
                runner_lines.append(')"')
                runner_lines.append('  ODYSSEUS_VLLM_SUPPORTS_SWAP=0')
                runner_lines.append('  if eval "$ODYSSEUS_VLLM_HELP_CMD" 2>&1 | grep -q -- "--swap-space"; then ODYSSEUS_VLLM_SUPPORTS_SWAP=1; fi')
                runner_lines.append('fi')
                runner_lines.append('if [ -z "$ODYSSEUS_PREFLIGHT_EXIT" ] && [ "${ODYSSEUS_VLLM_SUPPORTS_SWAP:-0}" = "1" ] && ! printf "%s" "$ODYSSEUS_SERVE_CMD" | grep -q -- "--swap-space"; then')
                runner_lines.append('  echo "[odysseus] Setting vLLM --swap-space 0 so the runtime does not reserve CPU swap per GPU."')
                runner_lines.append('  ODYSSEUS_SERVE_CMD="${ODYSSEUS_SERVE_CMD} --swap-space 0"')
                runner_lines.append('fi')
                runner_lines.append('if [ -z "$ODYSSEUS_PREFLIGHT_EXIT" ] && [ "${ODYSSEUS_VLLM_SUPPORTS_SWAP:-0}" != "1" ]; then')
                runner_lines.append('  if printf "%s" "$ODYSSEUS_SERVE_CMD" | grep -q -- "--swap-space"; then')
                runner_lines.append('    echo "[odysseus] vLLM serve does not expose --swap-space; removing the flag and patching the runtime default to 0."')
                runner_lines.append('    ODYSSEUS_SERVE_CMD="$(python3 - "$ODYSSEUS_SERVE_CMD" <<\'PY\'')
                runner_lines.append('import shlex, sys')
                runner_lines.append('parts = shlex.split(sys.argv[1])')
                runner_lines.append('out = []')
                runner_lines.append('skip = False')
                runner_lines.append('for part in parts:')
                runner_lines.append('    if skip:')
                runner_lines.append('        skip = False')
                runner_lines.append('        continue')
                runner_lines.append('    if part == "--swap-space":')
                runner_lines.append('        skip = True')
                runner_lines.append('        continue')
                runner_lines.append('    if part.startswith("--swap-space="):')
                runner_lines.append('        continue')
                runner_lines.append('    out.append(part)')
                runner_lines.append('print(shlex.join(out))')
                runner_lines.append('PY')
                runner_lines.append(')"')
                runner_lines.append('  fi')
                runner_lines.append('  ODYSSEUS_SERVE_CMD="$(python3 - "$ODYSSEUS_SERVE_CMD" <<\'PY\'')
                runner_lines.append('import shlex, sys')
                runner_lines.append('parts = shlex.split(sys.argv[1])')
                runner_lines.append('patch = r"""import inspect, sys')
                runner_lines.append('from vllm.engine.arg_utils import EngineArgs, AsyncEngineArgs')
                runner_lines.append('def _odysseus_swap0(cls):')
                runner_lines.append('    params = list(inspect.signature(cls).parameters)')
                runner_lines.append('    if "swap_space" not in params:')
                runner_lines.append('        return')
                runner_lines.append('    idx = params.index("swap_space")')
                runner_lines.append('    defaults = list(cls.__init__.__defaults__ or ())')
                runner_lines.append('    if idx < len(defaults):')
                runner_lines.append('        defaults[idx] = 0')
                runner_lines.append('        cls.__init__.__defaults__ = tuple(defaults)')
                runner_lines.append('    fields = getattr(cls, "__dataclass_fields__", {})')
                runner_lines.append('    if "swap_space" in fields:')
                runner_lines.append('        fields["swap_space"].default = 0')
                runner_lines.append('_odysseus_swap0(EngineArgs)')
                runner_lines.append('_odysseus_swap0(AsyncEngineArgs)')
                runner_lines.append('try:')
                runner_lines.append('    from vllm.config import CacheConfig')
                runner_lines.append('    CacheConfig.swap_space = 0')
                runner_lines.append('except Exception:')
                runner_lines.append('    pass')
                runner_lines.append('_orig_create_engine_config = EngineArgs.create_engine_config')
                runner_lines.append('def _odysseus_create_engine_config(self, *args, **kwargs):')
                runner_lines.append('    self.swap_space = 0')
                runner_lines.append('    return _orig_create_engine_config(self, *args, **kwargs)')
                runner_lines.append('EngineArgs.create_engine_config = _odysseus_create_engine_config')
                runner_lines.append('AsyncEngineArgs.create_engine_config = _odysseus_create_engine_config')
                runner_lines.append('from vllm.entrypoints.cli.main import main')
                runner_lines.append('sys.exit(main())"""')
                runner_lines.append('try:')
                runner_lines.append('    serve_i = parts.index("serve")')
                runner_lines.append('except ValueError:')
                runner_lines.append('    print(shlex.join(parts))')
                runner_lines.append('else:')
                runner_lines.append('    exe_i = serve_i - 1')
                runner_lines.append('    exe = parts[exe_i] if exe_i >= 0 else "vllm"')
                runner_lines.append('    py = "python3"')
                runner_lines.append('    if exe.endswith("/bin/vllm"):')
                runner_lines.append('        py = exe[:-len("/bin/vllm")] + "/bin/python"')
                runner_lines.append('    parts[exe_i:serve_i] = [py, "-c", patch]')
                runner_lines.append('    print(shlex.join(parts))')
                runner_lines.append('PY')
                runner_lines.append(')"')
                runner_lines.append('  echo "[odysseus] Patched vLLM internal swap_space default to 0 for this runtime."')
                runner_lines.append('fi')
            elif "sglang.launch_server" in req.cmd:
                runner_lines.append('export PATH="$HOME/.local/bin:$PATH"')
                runner_lines.append('if ! command -v sglang &>/dev/null; then')
                runner_lines.append('  echo "ERROR: SGLang is not installed."')
                runner_lines.append('  ODYSSEUS_PREFLIGHT_EXIT=127')
                runner_lines.append('elif ! ODYSSEUS_SGLANG_IMPORT_ERROR="$(python3 -c "import sglang" 2>&1)"; then')
                runner_lines.append('  echo "ERROR: SGLang is installed but failed to import."')
                runner_lines.append('  printf "%s\\n" "$ODYSSEUS_SGLANG_IMPORT_ERROR"')
                runner_lines.append('  ODYSSEUS_PREFLIGHT_EXIT=127')
                runner_lines.append('fi')
            elif "mlx_lm.server" in req.cmd:
                runner_lines.append('export PATH="$HOME/.local/bin:/opt/homebrew/bin:/usr/local/bin:$PATH"')
                runner_lines.append('if ! ODYSSEUS_MLX_IMPORT_ERROR="$(python3 -c "import mlx_lm" 2>&1)"; then')
                runner_lines.append('  echo "ERROR: MLX LM is not installed."')
                runner_lines.append('  printf "%s\\n" "$ODYSSEUS_MLX_IMPORT_ERROR"')
                runner_lines.append('  ODYSSEUS_PREFLIGHT_EXIT=127')
                runner_lines.append('fi')
                runner_lines.append(f"ODYSSEUS_SERVE_CMD='{_bash_squote(req.cmd)}'")
                runner_lines.append('if [ -z "$ODYSSEUS_PREFLIGHT_EXIT" ]; then')
                runner_lines.append('  ODYSSEUS_MLX_CMD_PY="$(python3 - "$ODYSSEUS_SERVE_CMD" <<\'PY\'')
                runner_lines.append('import shlex, sys')
                runner_lines.append('parts = shlex.split(sys.argv[1])')
                runner_lines.append('py = "python3"')
                runner_lines.append('for i, part in enumerate(parts):')
                runner_lines.append('    if part.endswith("/bin/python") or part.endswith("/bin/python3") or "/bin/python3." in part:')
                runner_lines.append('        py = part')
                runner_lines.append('        break')
                runner_lines.append('print(py)')
                runner_lines.append('PY')
                runner_lines.append(')"')
                runner_lines.append('  ODYSSEUS_SERVE_CMD="$("$ODYSSEUS_MLX_CMD_PY" - "$ODYSSEUS_SERVE_CMD" <<\'PY\'')
                runner_lines.append('import json, os, shlex, sys')
                runner_lines.append('from pathlib import Path')
                runner_lines.append('parts = shlex.split(sys.argv[1])')
                runner_lines.append('try:')
                runner_lines.append('    i = parts.index("--model")')
                runner_lines.append('    model = parts[i + 1]')
                runner_lines.append('except Exception:')
                runner_lines.append('    print(shlex.join(parts)); raise SystemExit')
                runner_lines.append('if "/" in model and not model.startswith("/") and model.startswith("mlx-community/"):')
                runner_lines.append('    roots = []')
                runner_lines.append('    def add(p):')
                runner_lines.append('        if not p: return')
                runner_lines.append('        p = os.path.expanduser(p)')
                runner_lines.append('        if p not in roots: roots.append(p)')
                runner_lines.append('    add(os.environ.get("HUGGINGFACE_HUB_CACHE"))')
                runner_lines.append('    hf_home = os.environ.get("HF_HOME")')
                runner_lines.append('    if hf_home: add(os.path.join(hf_home, "hub"))')
                runner_lines.append('    add("~/.cache/huggingface/hub")')
                runner_lines.append('    cache_name = "models--" + model.replace("/", "--")')
                runner_lines.append('    best = ""')
                runner_lines.append('    best_mtime = -1.0')
                runner_lines.append('    for root in roots:')
                runner_lines.append('        snap_root = os.path.join(root, cache_name, "snapshots")')
                runner_lines.append('        if not os.path.isdir(snap_root): continue')
                runner_lines.append('        for name in os.listdir(snap_root):')
                runner_lines.append('            path = os.path.join(snap_root, name)')
                runner_lines.append('            if not os.path.isdir(path): continue')
                runner_lines.append('            if not os.path.exists(os.path.join(path, "config.json")): continue')
                runner_lines.append('            try: mtime = os.path.getmtime(path)')
                runner_lines.append('            except OSError: mtime = 0')
                runner_lines.append('            if mtime > best_mtime:')
                runner_lines.append('                best, best_mtime = path, mtime')
                runner_lines.append('    if best:')
                runner_lines.append('        print("[odysseus] MLX using cached snapshot:", best, file=sys.stderr)')
                runner_lines.append('        launch_model = best')
                runner_lines.append('        if "deepseek-v4" in model.lower():')
                runner_lines.append('            try:')
                runner_lines.append('                import mlx_lm.models.deepseek_v4 as dsv4')
                runner_lines.append('                import mlx_lm.utils as mlx_utils')
                runner_lines.append('                utils_path = Path(mlx_utils.__file__)')
                runner_lines.append('                utils_text = utils_path.read_text()')
                runner_lines.append('                utils_needle = \'        def class_predicate(p, m):\\n            # Handle custom per layer quantizations\\n            if p in config["quantization"]:\\n                return config["quantization"][p]\\n            if not hasattr(m, "to_quantized"):\\n                return False\\n            return f"{p}.scales" in weights\\n\'')
                runner_lines.append('                utils_repl = \'        def class_predicate(p, m):\\n            # Odysseus: DeepSeek-V4 MXFP4 switch layers may already be quantized.\\n            if type(m).__name__ == "QuantizedSwitchLinear":\\n                return False\\n            # Handle custom per layer quantizations\\n            if p in config["quantization"]:\\n                return config["quantization"][p]\\n            if not hasattr(m, "to_quantized"):\\n                return False\\n            return f"{p}.scales" in weights\\n\'')
                runner_lines.append('                if utils_repl not in utils_text and utils_needle in utils_text:')
                runner_lines.append('                    bak = utils_path.with_suffix(utils_path.suffix + ".odysseus_bak")')
                runner_lines.append('                    if not bak.exists(): bak.write_text(utils_text)')
                runner_lines.append('                    utils_path.write_text(utils_text.replace(utils_needle, utils_repl))')
                runner_lines.append('                    print("[odysseus] Patched MLX-LM QuantizedSwitchLinear double-quantization guard.", file=sys.stderr)')
                runner_lines.append('                dsv4_path = Path(dsv4.__file__)')
                runner_lines.append('                dsv4_text = dsv4_path.read_text()')
                runner_lines.append('                dsv4_needle = \'            for sub in ("attn", "ffn"):\\n                for p in ("fn", "base", "scale"):\\n                    nk = nk.replace(f".hc_{sub}_{p}", f".hc_{sub}.{p}")\\n            for wo, wn in w_remap.items():\\n\'')
                runner_lines.append('                dsv4_repl = \'            for sub in ("attn", "ffn"):\\n                for p in ("fn", "base", "scale"):\\n                    nk = nk.replace(f".hc_{sub}_{p}", f".hc_{sub}.{p}")\\n            # Odysseus: normalize alternate hyper-connection key aliases.\\n            nk = nk.replace(".attn_hc.", ".hc_attn.")\\n            nk = nk.replace(".ffn_hc.", ".hc_ffn.")\\n            for wo, wn in w_remap.items():\\n\'')
                runner_lines.append('                if dsv4_repl not in dsv4_text and dsv4_needle in dsv4_text:')
                runner_lines.append('                    bak = dsv4_path.with_suffix(dsv4_path.suffix + ".odysseus_bak")')
                runner_lines.append('                    if not bak.exists(): bak.write_text(dsv4_text)')
                runner_lines.append('                    dsv4_path.write_text(dsv4_text.replace(dsv4_needle, dsv4_repl))')
                runner_lines.append('                    print("[odysseus] Patched MLX-LM DeepSeek-V4 hyper-connection key aliases.", file=sys.stderr)')
                runner_lines.append('            except Exception as e:')
                runner_lines.append('                print("[odysseus] WARNING: failed to apply MLX DeepSeek-V4 compatibility patch:", e, file=sys.stderr)')
                runner_lines.append('            try:')
                runner_lines.append('                src = Path(best)')
                runner_lines.append('                shim = Path.home() / ".cache" / "odysseus" / "mlx-shims" / src.name')
                runner_lines.append('                if len(src.name) > 20:')
                runner_lines.append('                    shim = Path.home() / ".cache" / "odysseus" / "mlx-shims" / model.split("/")[-1]')
                runner_lines.append('                shim.mkdir(parents=True, exist_ok=True)')
                runner_lines.append('                for child in src.iterdir():')
                runner_lines.append('                    target = shim / child.name')
                runner_lines.append('                    if child.name == "tokenizer_config.json":')
                runner_lines.append('                        continue')
                runner_lines.append('                    if target.exists() or target.is_symlink():')
                runner_lines.append('                        continue')
                runner_lines.append('                    target.symlink_to(child)')
                runner_lines.append('                tc_path = src / "tokenizer_config.json"')
                runner_lines.append('                if tc_path.exists():')
                runner_lines.append('                    tc = json.loads(tc_path.read_text())')
                runner_lines.append('                    if tc.get("tool_parser_type") == "deepseek_v4":')
                runner_lines.append('                        tc.pop("tool_parser_type", None)')
                runner_lines.append('                    (shim / "tokenizer_config.json").write_text(json.dumps(tc, indent=2, ensure_ascii=False))')
                runner_lines.append('                    launch_model = str(shim)')
                runner_lines.append('                    print("[odysseus] MLX DeepSeek-V4 using sanitized shim:", launch_model, file=sys.stderr)')
                runner_lines.append('            except Exception as e:')
                runner_lines.append('                print("[odysseus] WARNING: failed to create MLX DeepSeek-V4 shim:", e, file=sys.stderr)')
                runner_lines.append('        parts[i + 1] = launch_model')
                runner_lines.append('    else:')
                runner_lines.append('        print("[odysseus] MLX cached snapshot not found for:", model, file=sys.stderr)')
                runner_lines.append('print(shlex.join(parts))')
                runner_lines.append('PY')
                runner_lines.append(')"')
                runner_lines.append('fi')
            elif "scripts/diffusion_server.py" in req.cmd or ".diffusion_server.py" in req.cmd:
                runner_lines.append('export PATH="$HOME/.local/bin:$PATH"')
                runner_lines.append('if ! ODYSSEUS_DIFFUSION_IMPORT_ERROR="$(python3 -c "import torch, diffusers" 2>&1)"; then')
                runner_lines.append('  echo "ERROR: Diffusion serving requires PyTorch + diffusers."')
                runner_lines.append('  printf "%s\\n" "$ODYSSEUS_DIFFUSION_IMPORT_ERROR"')
                runner_lines.append('  ODYSSEUS_PREFLIGHT_EXIT=127')
                runner_lines.append('fi')

            handled_ollama_sidecar_probe = False
            if (not handled_ollama_serve
                and re.search(r"\bdocker\s+exec\s+(?:ollama-rocm|ollama-test)\s+ollama\s+show\b", req.cmd or "")):
                handled_ollama_sidecar_probe = True
                _append_serve_preflight_exit_lines(
                    runner_lines,
                    keep_shell_open=not local_windows,
                )
                runner_lines.append(req.cmd)
                runner_lines.append('_ody_exit=$?')
                runner_lines.append('echo')
                runner_lines.append('echo "=== Process exited with code ${_ody_exit} ==="')
                runner_lines.append('if [ "$_ody_exit" -eq 0 ]; then')
                runner_lines.append('  echo "[odysseus] Ollama sidecar model is available; keeping Cookbook task attached to the persistent Ollama daemon."')
                runner_lines.append('  while true; do sleep 3600; done')
                runner_lines.append('fi')
                runner_lines.append('exec bash -i')

            if not handled_ollama_serve and not handled_ollama_sidecar_probe:
                _append_serve_preflight_exit_lines(
                    runner_lines,
                    keep_shell_open=not local_windows,
                )
                if "vllm serve" in req.cmd or "mlx_lm.server" in req.cmd:
                    runner_lines.append('eval "$ODYSSEUS_SERVE_CMD"')
                elif is_pip_install:
                    if not is_windows and (req.platform or "").lower() in {"darwin", "macos"}:
                        req.cmd = _pip_install_command_without_break_system_packages(req.cmd)
                    _append_pip_install_runner_lines(runner_lines, req.cmd)
                else:
                    runner_lines.append(req.cmd)
                if local_windows:
                    # Detached background process — no interactive shell to keep open.
                    # Print the exit marker the status poller looks for, then stop.
                    _append_serve_exit_code_lines(
                        runner_lines,
                        keep_shell_open=False,
                        is_pip_install=is_pip_install,
                    )
                else:
                    # Keep shell open after exit so user can see errors
                    _append_serve_exit_code_lines(
                        runner_lines,
                        keep_shell_open=True,
                        is_pip_install=is_pip_install,
                    )

            runner_path = TMUX_LOG_DIR / f"{session_id}_run.sh"
            runner_path.write_text("\n".join(runner_lines) + "\n", encoding="utf-8")
            # chmod is a no-op on Windows; bash on Windows runs the script
            # regardless of the executable bit.
            safe_chmod(runner_path, 0o755)

            if local_windows:
                # LOCAL Windows: launch the bash runner detached (tmux replacement).
                setup_cmd = None
            elif remote:
                remote_runner = f".{session_id}_run.sh"
                # If command references scripts/, scp those too
                scp_extras = ""
                _port = req.ssh_port
                _Pf = f"-P {_port} " if _port and _port != "22" else ""
                _pf = f"-p {_port} " if _port and _port != "22" else ""
                if "scripts/diffusion_server.py" in req.cmd:
                    from core.constants import BASE_DIR
                    diff_script = Path(BASE_DIR) / "scripts" / "diffusion_server.py"
                    if diff_script.exists():
                        scp_extras = f"scp -O {_Pf}-q '{diff_script}' {remote}:.diffusion_server.py && "
                        runner_path.write_text(
                            runner_path.read_text(encoding="utf-8").replace(
                                "scripts/diffusion_server.py", ".diffusion_server.py"
                            ),
                            encoding="utf-8",
                        )
                setup_cmd = (
                    f"{scp_extras}"
                    f"scp -O {_Pf}-q '{runner_path}' {remote}:{remote_runner} && "
                    f"ssh {_pf}{remote} {shlex.quote(_remote_tmux_launch_command(session_id, remote_runner))}"
                )
            else:
                setup_cmd = f"tmux set-option -g history-limit 100000 2>/dev/null; tmux new-session -d -s {session_id} {shlex.quote(str(runner_path))}"

        if setup_cmd is None:
            # LOCAL Windows: launch the bash runner detached; no tmux setup_cmd.
            try:
                _launch_local_detached(session_id, runner_lines)
            except Exception as e:
                logger.error(f"Local detached serve launch failed: {e}")
                return {"ok": False, "error": str(e), "session_id": session_id}
        else:
            proc = await asyncio.create_subprocess_shell(
                setup_cmd,
                stdout=asyncio.subprocess.PIPE,
                stderr=asyncio.subprocess.PIPE,
            )
            await proc.wait()

            if proc.returncode != 0:
                stderr = (await proc.stderr.read()).decode(errors="replace")
                return {"ok": False, "error": stderr, "session_id": session_id}

        # Auto-register a model endpoint so the served model shows up in the model
        # picker with no manual /setup step. Diffusion models get an image
        # endpoint; any other real model serve (i.e. not a pip-install task) gets
        # a local LLM endpoint pointed at its /v1.
        endpoint_id = None
        is_diffusion = "diffusion_server.py" in req.cmd
        if is_diffusion:
            endpoint_id = _auto_register_image_endpoint(req, remote)
        elif not is_pip_install:
            endpoint_id = _auto_register_llm_endpoint(req, remote)

        # Crash watchdog: the auto-register above writes the endpoint row
        # IMMEDIATELY (before the server has even bound its port) so the
        # picker shows the model as it warms up. When the serve process
        # crashes right at startup (missing module, bad cmd, port collision,
        # ModuleNotFoundError on llama_cpp, etc.), the endpoint is left
        # dangling — every subsequent chat returns 503 or an empty response.
        # Schedule a background task to read the tmux output for the
        # "=== Process exited with code N ===" marker the runner emits;
        # if N != 0 within the watch window, delete the endpoint we just
        # created. Skipped for diffusion (different image-endpoint cleanup
        # path) and pip-install tasks (no endpoint to drop).
        if endpoint_id and not is_diffusion and not is_pip_install:
            asyncio.create_task(_serve_crash_watchdog(
                endpoint_id=endpoint_id,
                session_id=session_id,
                remote=remote,
                ssh_port=req.ssh_port,
                is_windows=is_windows,
            ))

        # Log to assistant
        try:
            from src.assistant_log import log_to_assistant
            from src.auth_helpers import get_current_user
            owner = get_current_user(request)
            short = req.repo_id.split("/")[-1] if "/" in req.repo_id else req.repo_id
            log_to_assistant(
                owner,
                f"Started serving {short} on {remote or 'local'}",
                category="Serve",
            )
        except Exception:
            pass

        return {"ok": True, "session_id": session_id, "remote": remote or "local",
                "endpoint_id": endpoint_id}

    # ── Server setup (install deps on remote) ──

    class SetupRequest(BaseModel):
        host: str
        ssh_port: str | None = None

    @router.post("/api/cookbook/setup")
    async def server_setup(request: Request, req: SetupRequest):
        """Install required dependencies on a remote server via SSH."""
        require_admin(request)
        host = validate_remote_host(req.host)
        if not host:
            raise HTTPException(400, "host is required")
        port = req.ssh_port
        port = validate_ssh_port(port)
        pf = f"-p {port} " if port and port != "22" else ""

        # Detect platform: Windows first (echo %OS% → Windows_NT), then Termux, then Linux
        detect_cmd = f'ssh {pf}{host} "echo %OS%"'
        platform = "linux"
        try:
            proc = await asyncio.create_subprocess_shell(
                detect_cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
            )
            stdout, _ = await asyncio.wait_for(proc.communicate(), timeout=10)
            out = stdout.decode().strip()
            if "Windows_NT" in out:
                platform = "windows"
            else:
                # Check for Termux
                detect_cmd2 = f"ssh {pf}{host} 'test -d /data/data/com.termux && echo termux || echo linux'"
                proc2 = await asyncio.create_subprocess_shell(
                    detect_cmd2, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
                )
                stdout2, _ = await asyncio.wait_for(proc2.communicate(), timeout=10)
                platform = stdout2.decode().strip()
        except Exception:
            platform = "linux"

        if platform == "windows":
            # Windows setup: ensure Python + pip + huggingface-hub via PowerShell
            # Also create the session directory for background tasks
            setup_script = (
                'powershell -Command "'
                "New-Item -ItemType Directory -Force -Path $env:TEMP\\odysseus-sessions | Out-Null; "
                "try { python --version } catch { Write-Host 'ERROR: Python not found — install from python.org'; exit 1 }; "
                "python -m pip install -q huggingface-hub 2>$null; "
                "python -c \\\"from huggingface_hub import snapshot_download; print('OK')\\\""
                '"'
            )
            cmd = f'ssh {pf}{host} {setup_script}'
        elif platform == "termux":
            setup_script = (
                "pkg install -y python tmux 2>/dev/null; "
                "pip install --no-deps -q huggingface-hub 2>/dev/null; "
                "pip install -q filelock fsspec packaging pyyaml tqdm typer httpx requests 2>/dev/null; "
                "python3 -c 'from huggingface_hub import snapshot_download; print(\"OK\")'"
            )
            cmd = f"ssh {pf}{host} '{setup_script}'"
        else:
            # Linux: auto-install tmux (via whichever package manager is available)
            # and huggingface_hub + hf_transfer (falling back to --user/--break-system-packages
            # on PEP-668 locked distros like Arch / newer Debian).
            setup_script = (
                # Install tmux if missing — try common package managers; skip if no sudo
                "if ! command -v tmux >/dev/null 2>&1; then "
                "  if command -v apt-get >/dev/null 2>&1; then sudo -n apt-get install -y tmux 2>/dev/null; "
                "  elif command -v pacman >/dev/null 2>&1; then sudo -n pacman -S --noconfirm tmux 2>/dev/null; "
                "  elif command -v dnf >/dev/null 2>&1; then sudo -n dnf install -y tmux 2>/dev/null; "
                "  elif command -v apk >/dev/null 2>&1; then sudo -n apk add --no-interactive tmux 2>/dev/null; "
                "  elif command -v zypper >/dev/null 2>&1; then sudo -n zypper --non-interactive install tmux 2>/dev/null; "
                "  fi; "
                "fi; "
                "command -v tmux >/dev/null 2>&1 || echo 'WARNING: tmux missing and auto-install failed (need passwordless sudo). Install manually.'; "
                # Install Python bits. Try system install first; fall back to --user --break-system-packages on PEP 668 systems.
                "pip install -q huggingface_hub hf_transfer 2>/dev/null || "
                "pip install --user --break-system-packages -q huggingface_hub hf_transfer 2>/dev/null || "
                "pip3 install --user --break-system-packages -q huggingface_hub hf_transfer 2>/dev/null; "
                "python3 -c 'from huggingface_hub import snapshot_download; print(\"OK\")'"
            )
            cmd = f"ssh {pf}{host} '{setup_script}'"

        try:
            proc = await asyncio.create_subprocess_shell(
                cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
            )
            stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=120)
            output = stdout.decode() + stderr.decode()
            ok = "OK" in output
            return {"ok": ok, "output": output.strip(), "platform": platform}
        except asyncio.TimeoutError:
            return {"ok": False, "error": "Setup timed out (120s)", "platform": platform}
        except Exception as e:
            return {"ok": False, "error": str(e), "platform": platform}

    # ── GPU availability probe ──

    async def _run_nvidia_smi(query: str, host: str | None, ssh_port: str | None, timeout: int = 8):
        """Run nvidia-smi locally or over SSH. Returns (stdout, error_or_None)."""
        if host:
            pf = f"-p {ssh_port} " if ssh_port and ssh_port != "22" else ""
            cmd = f"ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no {pf}{host} '{query}'"
            proc = await asyncio.create_subprocess_shell(
                cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
            )
        else:
            proc = await asyncio.create_subprocess_exec(
                *shlex.split(query),
                stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
            )
        try:
            stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout)
        except asyncio.TimeoutError:
            proc.kill()
            return None, "nvidia-smi timed out"
        if proc.returncode != 0:
            err = (stderr.decode("utf-8", errors="replace") or "").strip()[:200]
            return None, err or "nvidia-smi failed"
        return stdout.decode("utf-8", errors="replace"), None

    async def _run_gpu_shell(cmd_text: str, host: str | None, ssh_port: str | None, timeout: int = 8):
        """Run a small GPU probe shell command locally or over SSH."""
        if host:
            pf = f"-p {ssh_port} " if ssh_port and ssh_port != "22" else ""
            quoted_cmd = shlex.quote(cmd_text)
            remote_cmd = (
                f"if command -v sh >/dev/null 2>&1; then sh -lc {quoted_cmd}; "
                f"elif command -v bash >/dev/null 2>&1; then bash -lc {quoted_cmd}; "
                f"elif command -v zsh >/dev/null 2>&1; then zsh -lc {quoted_cmd}; "
                "else echo 'No POSIX shell found for GPU probe' >&2; exit 127; fi"
            )
            cmd = f"ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no {pf}{host} {shlex.quote(remote_cmd)}"
            proc = await asyncio.create_subprocess_shell(
                cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
            )
        else:
            proc = await asyncio.create_subprocess_shell(
                cmd_text, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
            )
        try:
            stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout)
        except asyncio.TimeoutError:
            proc.kill()
            return None, "GPU probe timed out"
        if proc.returncode != 0:
            err = (stderr.decode("utf-8", errors="replace") or "").strip()[:200]
            return None, err or f"GPU probe failed ({proc.returncode})"
        return stdout.decode("utf-8", errors="replace"), None

    async def _gpu_read_file(path: str, host: str | None, ssh_port: str | None) -> str | None:
        out, err = await _run_gpu_shell(f"cat {shlex.quote(path)} 2>/dev/null", host, ssh_port, timeout=4)
        if err is not None or out is None:
            return None
        return out.strip()

    async def _probe_gpu_device_processes(host: str | None, ssh_port: str | None) -> list[dict]:
        pid_cmd = (
            "{ command -v lsof >/dev/null 2>&1 && "
            "lsof -w -t /dev/kfd /dev/dri/renderD* 2>/dev/null || true; "
            "command -v fuser >/dev/null 2>&1 && "
            "fuser /dev/kfd /dev/dri/renderD* 2>/dev/null || true; } "
            "| tr ' ' '\\n' | sed '/^[0-9][0-9]*$/!d' | sort -n -u"
        )
        out, err = await _run_gpu_shell(pid_cmd, host, ssh_port, timeout=5)
        if err is not None or not out:
            return []
        processes = []
        seen = set()
        for raw in out.splitlines():
            try:
                pid = int(raw.strip())
            except ValueError:
                continue
            if pid in seen:
                continue
            seen.add(pid)
            name_out, _ = await _run_gpu_shell(f"ps -p {pid} -o comm= 2>/dev/null", host, ssh_port, timeout=3)
            name = (name_out or "").strip().splitlines()[0] if (name_out or "").strip() else "process"
            processes.append({"pid": pid, "name": name[:80], "used_mb": 0})
        return processes

    async def _probe_amd_sysfs(host: str | None, ssh_port: str | None) -> list[dict]:
        out, err = await _run_gpu_shell("ls -1 /sys/class/drm 2>/dev/null", host, ssh_port, timeout=4)
        if err is not None or not out:
            return []
        # Pick the runtime label up-front so each GPU dict gets the
        # right `backend`. AMD silicon can be driven by ROCm/HIP (native)
        # OR Vulkan (mesa RADV). Reporting "rocm" on a host where no
        # ROCm toolchain is installed misleads the frontend env-var
        # prefix logic — it would emit `HIP_VISIBLE_DEVICES=` for a
        # Vulkan-only stack, which is a silent no-op at best.
        rt_out, _ = await _run_gpu_shell(
            'command -v rocminfo >/dev/null 2>&1 && echo rocm '
            '|| (command -v hipconfig >/dev/null 2>&1 && echo rocm) '
            '|| (command -v vulkaninfo >/dev/null 2>&1 && echo vulkan) '
            '|| echo unknown',
            host, ssh_port, timeout=4,
        )
        _amd_runtime = (rt_out or "").strip().splitlines()[-1:][0].strip() if rt_out else "rocm"
        if _amd_runtime not in ("rocm", "vulkan"):
            # Default to rocm so existing ROCm-installed hosts keep
            # working; "unknown" only happens when neither toolchain is
            # detected (e.g. minimal sysfs read on a fresh box).
            _amd_runtime = "rocm"
        gpus = []
        for entry in out.split():
            if not entry.startswith("card") or "-" in entry:
                continue
            base = f"/sys/class/drm/{entry}/device"
            vendor = await _gpu_read_file(f"{base}/vendor", host, ssh_port)
            if vendor != "0x1002":
                continue
            vram_raw = await _gpu_read_file(f"{base}/mem_info_vram_total", host, ssh_port)
            vis_raw = await _gpu_read_file(f"{base}/mem_info_vis_vram_total", host, ssh_port)
            gtt_raw = await _gpu_read_file(f"{base}/mem_info_gtt_total", host, ssh_port)
            vram_bytes = int(vram_raw) if vram_raw and vram_raw.isdigit() else 0
            vis_bytes = int(vis_raw) if vis_raw and vis_raw.isdigit() else 0
            gtt_bytes = int(gtt_raw) if gtt_raw and gtt_raw.isdigit() else 0
            total_bytes = max(vram_bytes, vis_bytes)
            used_attr = "mem_info_vis_vram_used" if vis_bytes and vis_bytes >= vram_bytes else "mem_info_vram_used"
            unified = bool(vis_bytes and vis_bytes >= vram_bytes)
            if total_bytes <= 0:
                total_bytes = gtt_bytes
                used_attr = "mem_info_gtt_used"
                unified = True
            if total_bytes <= 0:
                continue
            used_raw = await _gpu_read_file(f"{base}/{used_attr}", host, ssh_port)
            used_bytes = int(used_raw) if used_raw and used_raw.isdigit() else 0
            name = await _gpu_read_file(f"{base}/product_name", host, ssh_port)
            if not name:
                device = await _gpu_read_file(f"{base}/device", host, ssh_port)
                name = f"AMD GPU {device or entry}"
            total_mb = max(0, int(total_bytes / (1024 * 1024)))
            used_mb = max(0, min(total_mb, int(used_bytes / (1024 * 1024))))
            free_mb = max(0, total_mb - used_mb)
            # GTT = the system-RAM pool the GPU pages into when VRAM is full.
            # On a discrete card a large gtt_used means the model spilled past
            # VRAM into RAM over PCIe — much slower. Surface it so the UI can
            # warn "spilling to RAM" instead of the user wondering why it's slow.
            gtt_used_raw = await _gpu_read_file(f"{base}/mem_info_gtt_used", host, ssh_port)
            gtt_used_mb = max(0, int(int(gtt_used_raw) / (1024 * 1024))) if (gtt_used_raw and gtt_used_raw.isdigit()) else 0
            gpus.append({
                "index": len(gpus), "name": name, "uuid": entry,
                "free_mb": free_mb, "total_mb": total_mb, "used_mb": used_mb,
                "gtt_used_mb": gtt_used_mb,
                "util_pct": 0, "busy": bool(total_mb and (free_mb / total_mb) < 0.85),
                "processes": [], "backend": _amd_runtime, "source": "amd-sysfs",
                "unified_memory": unified,
            })
        if gpus:
            processes = await _probe_gpu_device_processes(host, ssh_port)
            if processes:
                gpus[0]["processes"] = processes
                gpus[0]["busy"] = True
        return gpus

    async def _probe_apple_unified_memory(host: str | None, ssh_port: str | None) -> dict | None:
        """Best-effort Apple Silicon unified-memory probe, local or over SSH."""
        cmd = (
            "uname -s; uname -m; "
            "sysctl -n hw.memsize 2>/dev/null || true; "
            "vm_stat 2>/dev/null | awk '"
            "/page size of/ {gsub(/[^0-9]/, \"\", $8); page=$8} "
            "/Pages free/ {gsub(/[^0-9]/, \"\", $3); free=$3} "
            "/Pages inactive/ {gsub(/[^0-9]/, \"\", $3); inactive=$3} "
            "/Pages speculative/ {gsub(/[^0-9]/, \"\", $3); speculative=$3} "
            "/Pages purgeable/ {gsub(/[^0-9]/, \"\", $3); purgeable=$3} "
            "END {if (!page) page=16384; print page, free+inactive+speculative+purgeable}'"
        )
        out, err = await _run_gpu_shell(cmd, host, ssh_port, timeout=6)
        if err is not None or not out:
            return None
        lines = [ln.strip() for ln in out.splitlines() if ln.strip()]
        if len(lines) < 4:
            return None
        if lines[0] != "Darwin" or lines[1] not in {"arm64", "arm64e"}:
            return None
        try:
            total_bytes = int(lines[2])
            page_parts = lines[3].split()
            page_size = int(page_parts[0])
            available_pages = int(page_parts[1])
        except (ValueError, IndexError):
            return None
        if total_bytes <= 0:
            return None
        total_mb = int(total_bytes / (1024 * 1024))
        free_mb = max(0, min(total_mb, int((page_size * available_pages) / (1024 * 1024))))
        used_mb = max(0, total_mb - free_mb)
        return {
            "ok": True,
            "gpus": [{
                "index": 0,
                "name": "Apple Silicon unified memory",
                "uuid": "apple-metal-0",
                "free_mb": free_mb,
                "total_mb": total_mb,
                "used_mb": used_mb,
                "util_pct": 0,
                "busy": bool(total_mb and (free_mb / total_mb) < 0.2),
                "processes": [],
                "backend": "metal",
                "source": "apple-vm-stat",
                "unified_memory": True,
            }],
            "backend": "metal",
            "source": "apple-vm-stat",
        }

    @router.get("/api/cookbook/gpus")
    async def list_gpus(request: Request, host: str | None = None, ssh_port: str | None = None):
        """Probe GPU memory/process state locally or via SSH.

        Probe order:
            1. NVIDIA via nvidia-smi
            2. AMD/ROCm and unified-memory APUs via /sys/class/drm
            3. Generic GPU device holders via /dev/kfd and /dev/dri/renderD*

        Returned shape:
            { "ok": True, "gpus": [
                {"index": 0, "name": "...", "free_mb": int, "total_mb": int,
                 "used_mb": int, "util_pct": int, "busy": bool,
                 "uuid": "GPU-...",
                 "processes": [{"pid": int, "name": str, "used_mb": int}, ...]
                }, ...
            ]}
        `busy` is True when free_mb/total_mb < 0.5.
        """
        require_admin(request)
        host = validate_remote_host(host)
        ssh_port = validate_ssh_port(ssh_port)
        gpu_query = "nvidia-smi --query-gpu=index,name,memory.free,memory.total,memory.used,utilization.gpu,uuid --format=csv,noheader,nounits"
        nvidia_error = None
        try:
            gpu_out, err = await _run_nvidia_smi(gpu_query, host, ssh_port)
            if err is not None:
                nvidia_error = err
                gpu_out = ""
        except FileNotFoundError:
            nvidia_error = "nvidia-smi not found"
            gpu_out = ""
        except Exception as e:
            nvidia_error = str(e)[:200]
            gpu_out = ""

        gpus = []
        uuid_to_idx: dict[str, int] = {}
        for line in (gpu_out or "").strip().splitlines():
            parts = [p.strip() for p in line.split(",")]
            if len(parts) < 7:
                continue
            try:
                idx = int(parts[0])
                name = parts[1]
                free_mb = int(float(parts[2]))
                total_mb = int(float(parts[3]))
                used_mb = int(float(parts[4]))
                util_pct = int(float(parts[5]))
                gpu_uuid = parts[6]
            except (ValueError, IndexError):
                continue
            busy = total_mb > 0 and (free_mb / total_mb) < 0.5
            uuid_to_idx[gpu_uuid] = idx
            gpus.append({
                "index": idx, "name": name, "uuid": gpu_uuid,
                "free_mb": free_mb, "total_mb": total_mb,
                "used_mb": used_mb, "util_pct": util_pct,
                "busy": busy, "processes": [],
            })

        # Best-effort process listing — skip silently if it fails
        proc_query = "nvidia-smi --query-compute-apps=pid,gpu_uuid,process_name,used_memory --format=csv,noheader,nounits"
        try:
            proc_out, proc_err = await _run_nvidia_smi(proc_query, host, ssh_port, timeout=5)
            if proc_err is None and proc_out:
                gpus_by_idx = {g["index"]: g for g in gpus}
                for line in proc_out.strip().splitlines():
                    parts = [p.strip() for p in line.split(",")]
                    if len(parts) < 4:
                        continue
                    try:
                        pid = int(parts[0])
                        pname = parts[2]
                        pmem = int(float(parts[3]))
                    except (ValueError, IndexError):
                        continue
                    idx = uuid_to_idx.get(parts[1])
                    if idx is None or idx not in gpus_by_idx:
                        continue
                    gpus_by_idx[idx]["processes"].append({
                        "pid": pid, "name": pname, "used_mb": pmem,
                    })
        except Exception:
            pass

        if gpus:
            return {"ok": True, "gpus": gpus, "backend": "cuda", "source": "nvidia-smi"}

        # Local Apple Silicon / Metal fallback. macOS has no nvidia-smi and no
        # Linux /sys/class/drm tree, but services.hwfit.hardware already knows
        # how to size the shared unified-memory GPU budget. Keep this route in
        # sync so Cookbook's GPU picker doesn't show "nvidia-smi not found" on
        # native Mac launches.
        if not host and sys.platform == "darwin":
            try:
                from services.hwfit.hardware import detect_system
                info = detect_system(fresh=True)
                backend = str(info.get("backend") or "").lower()
                if backend in {"metal", "mps", "apple"} and info.get("gpu_count", 0) > 0:
                    total_mb = int(float(info.get("gpu_vram_gb") or info.get("total_ram_gb") or 0) * 1024)
                    free_mb = int(float(info.get("available_ram_gb") or 0) * 1024)
                    if total_mb and (free_mb <= 0 or free_mb > total_mb):
                        free_mb = total_mb
                    used_mb = max(0, total_mb - max(0, free_mb))
                    return {
                        "ok": True,
                        "gpus": [{
                            "index": 0,
                            "name": info.get("gpu_name") or info.get("cpu_name") or "Apple Silicon GPU",
                            "uuid": "apple-metal-0",
                            "free_mb": max(0, free_mb),
                            "total_mb": max(0, total_mb),
                            "used_mb": used_mb,
                            "util_pct": 0,
                            "busy": bool(total_mb and (free_mb / total_mb) < 0.5),
                            "processes": [],
                            "backend": "metal",
                            "source": "apple-metal",
                            "unified_memory": True,
                        }],
                        "backend": "metal",
                        "source": "apple-metal",
                        "fallback_from": "nvidia-smi",
                        "nvidia_error": nvidia_error,
                    }
            except Exception as e:
                logger.warning("Apple Metal GPU fallback failed: %s", e)

        apple_gpus = await _probe_apple_unified_memory(host, ssh_port)
        if apple_gpus:
            apple_gpus["fallback_from"] = "nvidia-smi"
            apple_gpus["nvidia_error"] = nvidia_error
            return apple_gpus

        amd_gpus = await _probe_amd_sysfs(host, ssh_port)
        if amd_gpus:
            # The per-GPU dict already carries the runtime label picked by
            # _probe_amd_sysfs (rocm vs vulkan); mirror that into the
            # wrapper so the frontend can read `data.backend` directly
            # without scanning the list.
            _amd_wrap_backend = str(amd_gpus[0].get("backend") or "rocm")
            return {
                "ok": True,
                "gpus": amd_gpus,
                "backend": _amd_wrap_backend,
                "source": "amd-sysfs",
                "fallback_from": "nvidia-smi",
                "nvidia_error": nvidia_error,
            }

        processes = await _probe_gpu_device_processes(host, ssh_port)
        if processes:
            return {
                "ok": True,
                "gpus": [{
                    "index": 0, "name": "GPU device holders", "uuid": "dev-dri",
                    "free_mb": 0, "total_mb": 0, "used_mb": 0, "util_pct": 0,
                    "busy": True, "processes": processes,
                    "backend": "generic", "source": "gpu-devices",
                }],
                "backend": "generic",
                "source": "gpu-devices",
                "fallback_from": "nvidia-smi",
                "nvidia_error": nvidia_error,
            }

        return {"ok": False, "error": nvidia_error or "No GPU memory probe available", "gpus": []}

    class KillPidRequest(BaseModel):
        pid: int
        host: str | None = None
        ssh_port: str | None = None
        signal: str = "TERM"  # TERM (graceful) or KILL (force)

    @router.post("/api/cookbook/kill-pid")
    async def kill_pid(request: Request, req: KillPidRequest):
        """Kill a PID that's holding GPU memory.

        Admin-gated. Validates PID is positive int, signal is TERM/KILL, and
        forbids low PIDs (<100) to avoid accidentally signalling init/system
        daemons. Uses `kill -<sig> <pid>` locally or over SSH.
        """
        require_admin(request)
        if req.pid < 100:
            raise HTTPException(400, f"Refusing to signal PID {req.pid} (<100, likely system process)")
        sig = (req.signal or "TERM").upper()
        if sig not in ("TERM", "KILL", "INT"):
            raise HTTPException(400, "signal must be TERM, KILL, or INT")
        host = validate_remote_host(req.host)
        req.ssh_port = validate_ssh_port(req.ssh_port)
        kill_cmd = f"kill -{sig} {req.pid}"
        try:
            if host:
                pf = f"-p {req.ssh_port} " if req.ssh_port and req.ssh_port != "22" else ""
                cmd = f"ssh -o ConnectTimeout=5 -o StrictHostKeyChecking=no {pf}{host} '{kill_cmd}'"
                proc = await asyncio.create_subprocess_shell(
                    cmd, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE
                )
            elif IS_WINDOWS:
                # No `kill` binary / POSIX signals on Windows. taskkill /F /T tears
                # down the PID and its children. There's no graceful-vs-force
                # distinction, so TERM/KILL/INT all map to the same forced kill.
                # NB: never use os.kill(pid, 0) to probe here — on Windows that
                # routes to TerminateProcess and would kill the process.
                if not pid_alive(req.pid):
                    return {"ok": False, "error": f"PID {req.pid} is not running"}
                await asyncio.to_thread(kill_process_tree, req.pid)
                return {"ok": True, "pid": req.pid, "signal": sig}
            else:
                proc = await asyncio.create_subprocess_exec(
                    "kill", f"-{sig}", str(req.pid),
                    stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE,
                )
            stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=5)
            if proc.returncode != 0:
                err = (stderr.decode("utf-8", errors="replace") or "").strip()[:200]
                return {"ok": False, "error": err or f"kill returned {proc.returncode}"}
            return {"ok": True, "pid": req.pid, "signal": sig}
        except asyncio.TimeoutError:
            return {"ok": False, "error": "kill command timed out"}
        except Exception as e:
            return {"ok": False, "error": str(e)[:200]}

    # ── Cookbook state persistence (cross-device sync) ──

    @router.get("/api/cookbook/state")
    async def get_cookbook_state(request: Request):
        """Load saved cookbook state (tasks, servers, presets, settings)."""
        require_admin(request)
        now = time.monotonic()
        try:
            mtime = _cookbook_state_path.stat().st_mtime if _cookbook_state_path.exists() else 0.0
        except Exception:
            mtime = 0.0
        cached = _state_get_cache.get("value")
        if cached is not None and _state_get_cache.get("mtime") == mtime and now - float(_state_get_cache.get("ts") or 0) < 1.5:
            return cached
        if _cookbook_state_path.exists():
            try:
                state = json.loads(_cookbook_state_path.read_text(encoding="utf-8"))
                saved_tasks = state.get("tasks", [])
                tasks = saved_tasks if isinstance(saved_tasks, list) else list(saved_tasks.values()) if isinstance(saved_tasks, dict) else []
                client_state = _state_for_client(state)
                _state_get_cache.update({"ts": now, "mtime": mtime, "value": client_state})
                return client_state
            except Exception:
                client_state = _state_for_client({})
                _state_get_cache.update({"ts": now, "mtime": mtime, "value": client_state})
                return client_state
        client_state = _state_for_client({})
        _state_get_cache.update({"ts": now, "mtime": mtime, "value": client_state})
        return client_state

    @router.post("/api/cookbook/state")
    async def save_cookbook_state(request: Request):
        """Save cookbook state for cross-device sync.

        Admin-gated because cookbook state is read back into shell-quoting
        contexts when polling tmux session status (see status handler).

        Merge guard: the UI debounces a `_syncToServer` POST every few
        seconds with whatever localStorage has. The agent's tool layer
        writes server-side tasks (e.g. `download_model` registering a
        task). Without a merge, every UI sync wipes the agent's recent
        additions. We preserve any on-disk task that the incoming body
        omits but was added in the last RACE_WINDOW seconds — that's a
        race, not an intentional delete.
        """
        require_admin(request)
        RACE_WINDOW_MS = 60_000
        try:
            from core.atomic_io import atomic_write_json
            data = await request.json()
            if not isinstance(data, dict):
                data = {}
            try:
                if _cookbook_state_path.exists():
                    on_disk = json.loads(_cookbook_state_path.read_text(encoding="utf-8"))
                else:
                    on_disk = {}
            except Exception:
                on_disk = {}
            # Anti-wipe guard for env servers. The UI debounces a
            # sync of whatever is in memory; if it fires before the state has
            # hydrated from GET /state (a load-time race) or during a render
            # glitch, `env.servers` would be empty and silently overwrite the
            # saved servers on disk. Never let an empty/absent incoming
            # env.servers clobber a populated on-disk one — preserve the disk
            # values while still accepting the rest of the incoming env.
            disk_env = on_disk.get("env") if isinstance(on_disk, dict) and isinstance(on_disk.get("env"), dict) else None
            if disk_env:
                inc_env = data.get("env") if isinstance(data.get("env"), dict) else None
                if inc_env is None:
                    data["env"] = disk_env
                    logger.warning("cookbook state POST: incoming body had no env; preserved on-disk env (anti-wipe guard)")
                elif disk_env.get("servers") and not inc_env.get("servers"):
                    inc_env["servers"] = disk_env["servers"]
                    logger.warning("cookbook state POST: incoming env.servers empty; preserved on-disk servers (anti-wipe guard)")

            disk_tasks = on_disk.get("tasks") or [] if isinstance(on_disk, dict) else []
            incoming_tasks = data.get("tasks") if isinstance(data.get("tasks"), list) else []
            incoming_removed = data.get("removedTasks") if isinstance(data.get("removedTasks"), dict) else {}
            disk_removed = on_disk.get("removedTasks") if isinstance(on_disk, dict) and isinstance(on_disk.get("removedTasks"), dict) else {}
            removed_tasks = {**disk_removed, **incoming_removed}
            data["removedTasks"] = removed_tasks
            removed_ids = set(removed_tasks.keys())
            if removed_ids:
                incoming_tasks = [
                    t for t in incoming_tasks
                    if not (isinstance(t, dict) and t.get("sessionId") in removed_ids)
                ]
                data["tasks"] = incoming_tasks
            # Anti-poisoning guard: a stale browser tab can keep POSTing a
            # download task as status='done' from before the strict-finish
            # fix landed, undoing any server-side correction. For each
            # incoming "done" download, override to "running" if the last
            # shard pattern says N<total AND no DOWNLOAD_OK/DOWNLOAD_FAILED/
            # /snapshots/ sentinel is in the output.
            import re as _re_dl
            for _it in incoming_tasks:
                if (not isinstance(_it, dict)) or _it.get("type") != "download" or _it.get("status") != "done":
                    continue
                _out = _it.get("output") or ""
                if ("DOWNLOAD_OK" in _out) or ("DOWNLOAD_FAILED" in _out) or ("/snapshots/" in _out):
                    continue
                _shards = _re_dl.findall(r"model-(\d+)-of-(\d+)\.safetensors", _out)
                if _shards:
                    _n, _tot = _shards[-1]
                    if int(_n) < int(_tot):
                        logger.info(f"cookbook state POST: rejecting stale done for {_it.get('sessionId')} "
                                    f"(last shard {_n}/{_tot}, no DOWNLOAD_OK)")
                        _it["status"] = "running"
                else:
                    _completed = _out.count("Download complete")
                    _starts = _out.count("Downloading '")
                    if _starts > _completed:
                        logger.info(f"cookbook state POST: rejecting stale done for {_it.get('sessionId')} "
                                    f"({_completed}/{_starts} files complete, no DOWNLOAD_OK)")
                        _it["status"] = "running"
            incoming_ids = {t.get("sessionId") for t in incoming_tasks if isinstance(t, dict) and t.get("sessionId")}
            import time as _t
            now_ms = int(_t.time() * 1000)
            preserved = []
            for t in disk_tasks:
                if not isinstance(t, dict):
                    continue
                sid = t.get("sessionId")
                if not sid or sid in incoming_ids:
                    continue  # client's version wins
                if sid in removed_ids:
                    continue  # intentional cross-device clear/remove
                ts = t.get("ts") or 0
                if isinstance(ts, (int, float)) and (now_ms - ts) <= RACE_WINDOW_MS:
                    preserved.append(t)
            if preserved:
                logger.info(f"cookbook state POST: preserving {len(preserved)} recent task(s) "
                            f"not in incoming body (race guard): "
                            f"{[t.get('sessionId') for t in preserved]}")
                data["tasks"] = incoming_tasks + preserved
            storage_state = _state_for_storage(data, on_disk)
            if storage_state == on_disk:
                return {"ok": True, "preserved": len(preserved), "unchanged": True}
            atomic_write_json(str(_cookbook_state_path), storage_state, indent=2)
            try:
                mtime = _cookbook_state_path.stat().st_mtime
                _state_get_cache.update({
                    "ts": time.monotonic(),
                    "mtime": mtime,
                    "value": _state_for_client(storage_state),
                })
            except Exception:
                pass
            return {"ok": True, "preserved": len(preserved)}
        except Exception as e:
            return {"ok": False, "error": str(e)}

    @router.get("/api/cookbook/hf-latest")
    async def hf_latest(vram_gb: float = 0, limit: int = 10, pipeline: str = "text-generation", owner: str = Depends(require_user)):
        """Fetch latest HuggingFace models, filtered by what fits in available VRAM.

        vram_gb: total available VRAM in GB. 0 = no filter (return everything).
        limit:   how many models to return (default 10).
        pipeline: HF pipeline_tag filter (text-generation, text-to-image, etc.).
        """
        import re
        import httpx

        # Fetch a larger pool so we have enough to filter from (we drop ~80%)
        pool_size = max(limit * 15, 100)
        url = (
            "https://huggingface.co/api/models"
            f"?sort=trendingScore&direction=-1&limit={pool_size}&filter={pipeline}"
        )
        try:
            async with httpx.AsyncClient(timeout=15) as client:
                resp = await client.get(url)
                if resp.status_code != 200:
                    return {"models": [], "error": f"HF API HTTP {resp.status_code}"}
                raw = resp.json()
        except Exception as e:
            return {"models": [], "error": str(e)}

        # Estimate VRAM from the model id. Looks for patterns like "7B", "70B", "1.5B" etc.
        # Returns approx VRAM in GB at fp16 (params*2). Caller adjusts for quant.
        def _est_vram_fp16(repo_id: str) -> float | None:
            m = re.search(r'[-_/](\d+(?:\.\d+)?)\s*[Bb](?![a-zA-Z])', repo_id)
            if not m:
                return None
            params_b = float(m.group(1))
            return params_b * 2.0  # fp16 baseline

        # Detect quantization from repo_id / tags. Returns a multiplier on fp16 size.
        def _quant_factor(repo_id: str, tags: list) -> float:
            text = (repo_id + " " + " ".join(tags or [])).lower()
            if "fp4" in text or "nf4" in text or "int4" in text or "4bit" in text or "q4" in text or "awq" in text or "gptq" in text:
                return 0.25
            if "int8" in text or "8bit" in text or "q8" in text or "fp8" in text:
                return 0.5
            if "bf16" in text or "fp16" in text:
                return 1.0
            return 1.0  # default fp16

        # Exclude adapters, LoRAs, datasets, GGUF-only repos, and other non-runnable artifacts
        EXCLUDE_TAG_SUBSTRINGS = (
            "lora", "adapter", "peft", "qlora",
            "dataset", "embeddings",
            "merge", "control-lora",
            "diffusion-lora", "stable-diffusion-lora",
            "text-classification", "token-classification",
            "feature-extraction", "sentence-similarity",
        )
        EXCLUDE_NAME_SUBSTRINGS = (
            "lora", "adapter", "peft", "qlora",
            "embedding", "embed-",
            "dataset",
        )

        def _is_excluded(repo_id: str, tags: list) -> bool:
            text = repo_id.lower()
            for s in EXCLUDE_NAME_SUBSTRINGS:
                if s in text:
                    return True
            tag_text = " ".join(t.lower() for t in (tags or []))
            for s in EXCLUDE_TAG_SUBSTRINGS:
                if s in tag_text:
                    return True
            return False

        out = []
        for entry in raw:
            repo_id = entry.get("modelId") or entry.get("id") or ""
            if not repo_id:
                continue
            tags = entry.get("tags") or []
            pipeline_tag = entry.get("pipeline_tag") or ""

            # Hard filter: only the requested pipeline (HF's filter param is loose)
            if pipeline and pipeline_tag and pipeline_tag != pipeline:
                continue
            # Skip adapters, LoRAs, datasets, etc.
            if _is_excluded(repo_id, tags):
                continue

            est_fp16 = _est_vram_fp16(repo_id)
            quant_mult = _quant_factor(repo_id, tags)
            est_vram = (est_fp16 * quant_mult) if est_fp16 else None
            # Add 30% headroom for KV cache, activations, etc.
            needed_vram = (est_vram * 1.3) if est_vram else None

            if vram_gb > 0:
                if needed_vram is None:
                    # The "trending models that fit" list must be conservative:
                    # if we cannot estimate size from the repo id/tags, do not
                    # present it as runnable on this hardware.
                    continue
                if needed_vram > vram_gb:
                    continue

            out.append({
                "repo_id": repo_id,
                "downloads": entry.get("downloads", 0),
                "likes": entry.get("likes", 0),
                "createdAt": entry.get("createdAt", ""),
                "tags": tags[:5],  # trim
                "pipeline_tag": pipeline_tag,
                "est_vram_gb": round(est_vram, 1) if est_vram else None,
                "needed_vram_gb": round(needed_vram, 1) if needed_vram else None,
            })
            if len(out) >= limit:
                break

        return {"models": out}

    # Rate-limit for the orphan-tmux adoption sweep. Five-minute interval so SSH
    # work is genuinely sparse even on an actively-polled cookbook page.
    _last_orphan_sweep_ts = [0.0]
    _ORPHAN_SWEEP_MIN_INTERVAL_S = 300.0
    # Concurrency guard so two requests racing don't both spawn a sweep.
    _orphan_sweep_inflight = [False]

    def _maybe_sweep_orphans(tasks: list, state: dict) -> None:
        """Scan each configured cookbook server for `serve-*` tmux sessions
        the cookbook doesn't know about and adopt them into state.tasks.

        Heavy SSH work runs in a background thread via asyncio.to_thread so
        it never blocks the request that triggered it. Was previously
        disabled because the sync implementation pegged uvicorn CPU during
        active cookbook polling — re-enabled now with the work pushed off
        the event loop and a slower (60s) cadence.
        """
        import time as _time
        now = _time.monotonic()
        if _orphan_sweep_inflight[0]:
            return
        if now - _last_orphan_sweep_ts[0] < _ORPHAN_SWEEP_MIN_INTERVAL_S:
            return
        _last_orphan_sweep_ts[0] = now
        _orphan_sweep_inflight[0] = True
        # Snapshot inputs so the worker doesn't race with state mutations.
        try:
            tasks_snap = list(tasks or [])
        except Exception:
            tasks_snap = []
        state_snap = state if isinstance(state, dict) else {}

        # Caller is _cookbook_tasks_status_sync (sync context, no event
        # loop). Use a plain background thread — no asyncio needed.
        import threading
        def _run_sweep() -> None:
            try:
                _sync_sweep_orphans(tasks_snap, state_snap)
            except Exception as _e:
                logger.warning(f"orphan sweep thread failed: {_e!r}")
            finally:
                _orphan_sweep_inflight[0] = False
        try:
            threading.Thread(target=_run_sweep, daemon=True, name="orphan-sweep").start()
        except Exception as _e:
            logger.warning(f"orphan sweep thread spawn failed: {_e!r}")
            _orphan_sweep_inflight[0] = False
        return

    def _sync_sweep_orphans(tasks: list, state: dict) -> None:
        """The actual sync sweep — never call this on the event loop."""
        import subprocess
        env = state.get("env") if isinstance(state, dict) else {}
        servers = env.get("servers") if isinstance(env, dict) else []
        logger.info(f"orphan sweep starting: {len(servers) if isinstance(servers, list) else 0} server(s), known_sids={len([t for t in tasks if isinstance(t, dict) and t.get('sessionId')])}")
        if not isinstance(servers, list):
            return

        known_sids = {
            t.get("sessionId") for t in tasks
            if isinstance(t, dict) and t.get("sessionId")
        }

        adopted_any = False
        for srv in servers:
            if not isinstance(srv, dict):
                continue
            host = (srv.get("host") or "").strip()
            if not host:
                continue  # local-only entry; the /proc scan handles it
            try:
                host = validate_remote_host(host)
            except HTTPException:
                continue
            sport = str(srv.get("port") or "").strip()
            ssh_base = ["ssh", "-o", "ConnectTimeout=4", "-o", "StrictHostKeyChecking=no"]
            if sport and sport != "22":
                try:
                    sport = validate_ssh_port(sport)
                except HTTPException:
                    continue
                if sport != "22":
                    ssh_base.extend(["-p", sport])

            try:
                ls = subprocess.run(
                    ssh_base + [host, _remote_tmux_command("ls")],
                    timeout=6, capture_output=True, text=True,
                )
            except Exception:
                continue
            for line in (ls.stdout or "").splitlines():
                sid = line.split(":", 1)[0].strip()
                if not sid or not _SESSION_ID_RE.match(sid):
                    continue
                if sid in known_sids:
                    continue
                try:
                    cap = subprocess.run(
                        ssh_base + [host, _remote_tmux_command("capture-pane", "-t", sid, "-p", "-S", "-300")],
                        timeout=6, capture_output=True, text=True,
                    )
                    pane = cap.stdout or ""
                except Exception:
                    pane = ""

                if sid.startswith("cookbook-"):
                    repo_id = ""
                    try:
                        script = subprocess.run(
                            ssh_base + [host, "cat", f".{sid}_run.sh"],
                            timeout=6, capture_output=True, text=True,
                        )
                        script_text = script.stdout or ""
                    except Exception:
                        script_text = ""
                    m_repo = re.search(r"repo_id\s*=\s*['\"]([^'\"]+/[^'\"]+)['\"]", script_text)
                    if not m_repo:
                        m_repo = re.search(r"snapshot_download\(\s*repo_id\s*=\s*['\"]([^'\"]+/[^'\"]+)['\"]", script_text)
                    if not m_repo:
                        m_repo = re.search(r"(?:https://huggingface\.co/)?([A-Za-z0-9_.-]+/[A-Za-z0-9_.-]+)", script_text)
                    if not m_repo and pane:
                        m_cache = re.search(r"models--([A-Za-z0-9_.-]+)--([A-Za-z0-9_.-]+)", pane)
                        if m_cache:
                            repo_id = f"{m_cache.group(1)}/{m_cache.group(2)}"
                    repo_id = m_repo.group(1) if m_repo else f"adopted:{sid}"
                    adopted_download_done = "DOWNLOAD_OK" in pane
                    if repo_id.startswith("adopted:") and pane:
                        m_cache = re.search(r"models--([A-Za-z0-9_.-]+)--([A-Za-z0-9_.-]+)", pane)
                        if m_cache:
                            repo_id = f"{m_cache.group(1)}/{m_cache.group(2)}"
                    import time as _t2
                    tasks.append({
                        "id": sid,
                        "sessionId": sid,
                        "name": repo_id.split("/")[-1] if "/" in repo_id else repo_id,
                        "type": "download",
                        "status": "completed" if adopted_download_done else "running",
                        "progress": "Download complete" if adopted_download_done else "",
                        "output": (pane or f"Auto-adopted from orphan tmux download session on {host}.")[-5000:],
                        "ts": int(_t2.time() * 1000),
                        "completedAt": int(_t2.time() * 1000) if adopted_download_done else None,
                        "payload": {
                            "repo_id": repo_id,
                            "remote_host": host,
                            "_cmd": "(orphan tmux download - original launch cmd recovered from tmux/session only)",
                        },
                        "remoteHost": host,
                        "sshPort": sport,
                        "platform": "linux",
                        "_cacheComplete": adopted_download_done,
                        "_adoptedExternally": True,
                    })
                    known_sids.add(sid)
                    adopted_any = True
                    logger.info(f"auto-adopted orphan download tmux session {sid!r} on {host}")
                    continue
                # Adopt any session whose pane is currently running a
                # known model-server process (checked below). The earlier
                # prefix gate (serve-/cookbook-) dropped legitimate
                # serves whenever tmux fell back to numeric IDs, leaving
                # them invisible in the Cookbook UI — so the user could
                # neither see nor stop them.
                # Skip zombie / idle-shell sessions. A tmux session left
                # over from a crashed vllm just shows a bash prompt —
                # adopting it would pollute the UI with "running" tasks
                # that aren't actually serving anything. pane_current_command
                # is the foreground process in the pane right now; only
                # real model serves leave a python/vllm/etc. process there.
                try:
                    pc = subprocess.run(
                        ssh_base + [host, "tmux", "list-panes", "-t", sid,
                                    "-F", "#{pane_current_command}"],
                        timeout=4, capture_output=True, text=True,
                    )
                    cur = (pc.stdout or "").strip().splitlines()
                except Exception:
                    cur = []
                LIVE_PROCS = {"python", "python3", "vllm", "llama-server",
                              "llama_cpp_main", "sglang", "mlx_lm", "lmdeploy",
                              "ollama", "node", "uvicorn"}
                if not any(c in LIVE_PROCS for c in cur):
                    continue
                # Try to recover a plausible repo_id + port from the
                # pane buffer. Cheap heuristic — if we can't, register
                # with placeholder fields; the UI still shows it.
                import re as _re_orphan
                # vLLM banner: "model   /path/...". Falls back to the
                # raw vllm-serve command if the banner already scrolled.
                m_model = _re_orphan.search(r"model\s+(\S+)", pane)
                model = m_model.group(1) if m_model else ""
                if not model:
                    m_serve = _re_orphan.search(r"vllm\s+serve\s+(\S+)", pane)
                    model = m_serve.group(1) if m_serve else f"adopted:{sid}"
                m_port = _re_orphan.search(r"--port\s+(\d+)", pane)
                port = int(m_port.group(1)) if m_port else 0

                import time as _t2
                tasks.append({
                    "id": sid,
                    "sessionId": sid,
                    "name": model.split("/")[-1] if "/" in model else model,
                    "type": "serve",
                    "status": "running",
                    "output": f"Auto-adopted from orphan tmux session on {host}. "
                              "Open the task to see live output.",
                    "ts": int(_t2.time() * 1000),
                    "payload": {
                        "repo_id": model,
                        "remote_host": host,
                        "_cmd": "(orphan tmux session — original launch cmd unknown)",
                        "port": port,
                    },
                    "remoteHost": host,
                    "sshPort": sport,
                    "platform": "linux",
                    "_serveReady": False,
                    "_endpointAdded": False,
                    "_adoptedExternally": True,
                })
                known_sids.add(sid)
                adopted_any = True
                logger.info(f"auto-adopted orphan tmux session {sid!r} on {host}")

        if adopted_any:
            try:
                from core.atomic_io import atomic_write_json
                state["tasks"] = tasks
                atomic_write_json(_cookbook_state_path, state)
            except Exception as e:
                logger.warning(f"orphan sweep: state write failed: {e}")

    @router.get("/api/cookbook/hf-gguf-files")
    async def hf_gguf_files(repo_id: str, owner: str = Depends(require_user)):
        """List GGUF files in a HuggingFace repo for the direct-download picker."""
        import httpx

        repo_id = _validate_repo_id(repo_id)
        url = f"https://huggingface.co/api/models/{repo_id}"
        try:
            headers = {}
            token = _load_stored_hf_token()
            if token:
                headers["Authorization"] = f"Bearer {token}"
            async with httpx.AsyncClient(timeout=15, follow_redirects=True) as client:
                resp = await client.get(url, headers=headers)
                if resp.status_code != 200:
                    return {"ok": False, "files": [], "error": f"HF API HTTP {resp.status_code}"}
                data = resp.json()
        except Exception:
            logger.exception("HF GGUF file scan failed for %s", repo)
            return {"ok": False, "files": [], "error": "HF API request failed"}
        files = [
            str(s.get("rfilename") or "")
            for s in data.get("siblings", [])
            if str(s.get("rfilename") or "").lower().endswith(".gguf")
        ]
        return {"ok": True, "repo_id": repo_id, "files": files}

    # In-memory cache for the Ollama library scrape. ollama.com is a public
    # site, but it doesn't expose a stable JSON listing — we fetch the HTML
    # search page and regex out the model cards. Cached for 1 h so a busy
    # cookbook view doesn't hammer the site on every render.
    _ollama_library_cache: dict = {"models": [], "fetched_at": 0.0, "error": None}

    _OLLAMA_FALLBACK_LIBRARY = [
        {"name": "qwen2.5", "description": "Qwen2.5 series — strong general/coding model from Alibaba.", "sizes": ["0.5b", "1.5b", "3b", "7b", "14b", "32b", "72b"]},
        {"name": "qwen2.5-coder", "description": "Code-specialized Qwen2.5 family.", "sizes": ["0.5b", "1.5b", "3b", "7b", "14b", "32b"]},
        {"name": "qwen3", "description": "Qwen3 — newer Alibaba family with hybrid reasoning.", "sizes": ["0.6b", "1.7b", "4b", "8b", "14b", "32b"]},
        {"name": "llama3.2", "description": "Meta Llama 3.2 instruct (and tiny / vision variants).", "sizes": ["1b", "3b", "11b", "90b"]},
        {"name": "llama3.1", "description": "Meta Llama 3.1 instruct.", "sizes": ["8b", "70b", "405b"]},
        {"name": "llama3.3", "description": "Meta Llama 3.3 70B instruct.", "sizes": ["70b"]},
        {"name": "gemma3", "description": "Google Gemma 3 — multimodal capable open-weights.", "sizes": ["1b", "4b", "12b", "27b"]},
        {"name": "gemma2", "description": "Google Gemma 2 instruct.", "sizes": ["2b", "9b", "27b"]},
        {"name": "mistral", "description": "Mistral 7B instruct — small, fast generalist.", "sizes": ["7b"]},
        {"name": "mistral-nemo", "description": "Mistral NeMo 12B instruct.", "sizes": ["12b"]},
        {"name": "mistral-small", "description": "Mistral Small 22B / 24B instruct.", "sizes": ["22b", "24b"]},
        {"name": "mixtral", "description": "Mistral MoE 8x7B / 8x22B.", "sizes": ["8x7b", "8x22b"]},
        {"name": "phi3", "description": "Microsoft Phi-3 small / medium.", "sizes": ["mini", "medium"]},
        {"name": "phi4", "description": "Microsoft Phi-4 14B.", "sizes": ["14b"]},
        {"name": "deepseek-r1", "description": "DeepSeek R1 reasoning model (distilled variants).", "sizes": ["1.5b", "7b", "8b", "14b", "32b", "70b"]},
        {"name": "deepseek-v3", "description": "DeepSeek V3 MoE 671B (huge — needs serious VRAM).", "sizes": ["671b"]},
        {"name": "codellama", "description": "Meta Code Llama instruct family.", "sizes": ["7b", "13b", "34b", "70b"]},
        {"name": "starcoder2", "description": "BigCode StarCoder2 — code completion.", "sizes": ["3b", "7b", "15b"]},
        {"name": "deepseek-coder-v2", "description": "DeepSeek Coder V2 — code MoE.", "sizes": ["16b", "236b"]},
        {"name": "nomic-embed-text", "description": "Embedding model — text vector encoder.", "sizes": ["latest"]},
        {"name": "mxbai-embed-large", "description": "Embedding model — Mixedbread large.", "sizes": ["latest"]},
        {"name": "llava", "description": "LLaVA multimodal vision-language model.", "sizes": ["7b", "13b", "34b"]},
        {"name": "minicpm-v", "description": "MiniCPM-V multimodal.", "sizes": ["8b"]},
        {"name": "command-r", "description": "Cohere Command R — RAG-oriented.", "sizes": ["35b"]},
        {"name": "command-r-plus", "description": "Cohere Command R+ — larger RAG model.", "sizes": ["104b"]},
        {"name": "qwq", "description": "Qwen QwQ reasoning preview.", "sizes": ["32b"]},
        {"name": "smollm2", "description": "HuggingFaceTB SmolLM2 — tiny capable models.", "sizes": ["135m", "360m", "1.7b"]},
        {"name": "granite3.1-dense", "description": "IBM Granite 3.1 dense instruct.", "sizes": ["2b", "8b"]},
        {"name": "nemotron", "description": "NVIDIA Nemotron 70B.", "sizes": ["70b"]},
        {"name": "olmo2", "description": "AI2 OLMo 2 open-weights.", "sizes": ["7b", "13b"]},
    ]

    @router.get("/api/cookbook/ollama/library")
    async def ollama_library(refresh: int = 0, request: Request = None, owner: str = Depends(require_user)):
        """List popular Ollama library models for the Browse picker.

        Tries a 1-hour-cached fetch of ollama.com/library, falls back to a
        curated hard-coded list so the picker always renders something."""
        import time as _time
        import httpx as _httpx
        TTL = 3600.0
        now = _time.time()
        if refresh or (now - _ollama_library_cache["fetched_at"]) > TTL or not _ollama_library_cache["models"]:
            models: list[dict] = []
            err = None
            try:
                async with _httpx.AsyncClient(timeout=8, follow_redirects=True) as client:
                    resp = await client.get(
                        "https://ollama.com/search?sort=popular",
                        headers={"User-Agent": "odysseus-cookbook/1.0"},
                    )
                if resp.status_code == 200:
                    html = resp.text
                    # ollama.com renders each model card as a single anchor:
                    #   <a href="/library/<name>" class="group w-full"> … </a>
                    # The description + sizes live inside that anchor. Pull
                    # the whole block then extract pieces individually.
                    block_re = re.compile(
                        r'<a[^>]*href="/library/([A-Za-z0-9._-]+)"[^>]*>(.*?)</a>',
                        re.DOTALL,
                    )
                    desc_re = re.compile(r'<p[^>]*>([^<]{4,400})</p>', re.DOTALL)
                    # Size tags on ollama.com cards look like "0.5b", "14b",
                    # "8x7b", "27b". Pulled from short <span>-wrapped chips.
                    size_re = re.compile(r'>\s*(\d+(?:\.\d+)?(?:x\d+)?[bBmM])\s*<')
                    seen: set[str] = set()
                    for bm in block_re.finditer(html):
                        name = bm.group(1).strip()
                        if name in seen:
                            continue
                        seen.add(name)
                        body = bm.group(2)
                        dm = desc_re.search(body)
                        desc = (dm.group(1).strip() if dm else "").replace("\n", " ")
                        sizes_raw = size_re.findall(body)
                        # Dedup sizes preserving order
                        sizes: list[str] = []
                        for s in sizes_raw:
                            s_low = s.lower()
                            if s_low not in sizes:
                                sizes.append(s_low)
                        models.append({"name": name, "description": desc, "sizes": sizes})
                        if len(models) >= 80:
                            break
                else:
                    err = f"HTTP {resp.status_code}"
            except Exception as e:
                err = str(e)[:160]
            # Merge curated fallback so classics (qwen2.5, llama3, deepseek-r1,
            # …) stay reachable even when ollama.com's front page is dominated
            # by brand-new releases the user might not be looking for.
            live_names = {m["name"] for m in models}
            for fb in _OLLAMA_FALLBACK_LIBRARY:
                if fb["name"] not in live_names:
                    models.append(fb)
            if not models:
                models = list(_OLLAMA_FALLBACK_LIBRARY)
                if err is None:
                    err = "parsed 0 results — using fallback list"
            _ollama_library_cache["models"] = models
            _ollama_library_cache["fetched_at"] = now
            _ollama_library_cache["error"] = err
        return {
            "models": _ollama_library_cache["models"],
            "fetched_at": _ollama_library_cache["fetched_at"],
            "error": _ollama_library_cache["error"],
        }

    # ── vLLM recipe scraper ─────────────────────────────────────────────
    # Fetches the official YAML recipe for a model from vllm-project/recipes
    # and normalizes it into a small JSON the frontend can consume. Cached
    # per-repo so the GitHub raw endpoint isn't hammered.
    _vllm_recipe_cache: dict[str, tuple[float, dict | None]] = {}
    # Manifest of all <org>/<model> ids that have a recipe in the upstream
    # repo. Cheap to fetch (one Git Tree API call), so we cache the whole
    # set for ~12h. Per-row "does this model have a recipe?" lookups hit
    # this set instead of doing 912 individual recipe fetches.
    _vllm_recipe_manifest: dict = {"fetched_at": 0.0, "models": set(), "error": ""}

    @router.get("/api/cookbook/vllm-recipe-manifest")
    async def vllm_recipe_manifest(refresh: int = 0):
        """Return the set of <org>/<model> ids known to have a vLLM recipe.
        One GitHub Tree API call, 12h cache. The frontend uses this to badge
        rows in the model list before the user expands them."""
        import time as _time
        import httpx as _httpx
        TTL = 12 * 3600.0
        now = _time.time()
        if (
            refresh
            or (now - _vllm_recipe_manifest["fetched_at"]) > TTL
            or not _vllm_recipe_manifest["models"]
        ):
            url = (
                "https://api.github.com/repos/vllm-project/recipes/"
                "git/trees/main?recursive=1"
            )
            def _fetch_sync() -> tuple[int, dict | None, str]:
                try:
                    headers = {"Accept": "application/vnd.github+json"}
                    with _httpx.Client(timeout=10.0, follow_redirects=True) as client:
                        r = client.get(url, headers=headers)
                        if r.status_code != 200:
                            return r.status_code, None, r.text[:200]
                        return 200, r.json(), ""
                except Exception as e:
                    return 0, None, f"fetch error: {e}"
            status, data, err = await asyncio.to_thread(_fetch_sync)
            if status == 200 and isinstance(data, dict):
                models: set[str] = set()
                for entry in data.get("tree") or []:
                    path = (entry or {}).get("path") or ""
                    if not path.startswith("models/") or not path.endswith(".yaml"):
                        continue
                    # path = "models/<org>/<model>.yaml" → "<org>/<model>"
                    body = path[len("models/"):-len(".yaml")]
                    if "/" in body:
                        models.add(body)
                _vllm_recipe_manifest["models"] = models
                _vllm_recipe_manifest["fetched_at"] = now
                _vllm_recipe_manifest["error"] = ""
            else:
                _vllm_recipe_manifest["error"] = (
                    f"HTTP {status}: {err}" if status else err
                )
                # Don't clobber a stale-but-usable list on transient failures.
                if not _vllm_recipe_manifest["models"]:
                    return {
                        "models": [],
                        "count": 0,
                        "error": _vllm_recipe_manifest["error"],
                    }
        return {
            "models": sorted(_vllm_recipe_manifest["models"]),
            "count": len(_vllm_recipe_manifest["models"]),
            "fetched_at": _vllm_recipe_manifest["fetched_at"],
            "error": _vllm_recipe_manifest["error"],
        }

    @router.get("/api/cookbook/vllm-recipe")
    async def vllm_recipe(repo: str, refresh: int = 0):
        """Return the vLLM official recipe for a HuggingFace repo, if one
        exists at vllm-project/recipes. `repo` is the full HF id like
        'MiniMaxAI/MiniMax-M2'. Cached 6h."""
        import time as _time
        import httpx as _httpx
        import yaml as _yaml

        TTL = 6 * 3600.0
        now = _time.time()
        repo = (repo or "").strip().strip("/")
        if "/" not in repo:
            return {"exists": False, "error": "repo must be <org>/<model>"}

        cached = _vllm_recipe_cache.get(repo)
        if cached and not refresh and (now - cached[0]) < TTL:
            return cached[1] or {"exists": False, "cached": True}

        url = (
            f"https://raw.githubusercontent.com/vllm-project/recipes/"
            f"main/models/{repo}.yaml"
        )

        def _fetch_sync() -> tuple[int, str]:
            try:
                with _httpx.Client(timeout=8.0, follow_redirects=True) as client:
                    r = client.get(url)
                    return r.status_code, r.text
            except Exception as e:
                return 0, f"fetch error: {e}"

        status, text = await asyncio.to_thread(_fetch_sync)
        if status == 404:
            _vllm_recipe_cache[repo] = (now, {"exists": False})
            return {"exists": False}
        if status != 200:
            return {"exists": False, "error": f"HTTP {status}", "transient": True}

        try:
            doc = _yaml.safe_load(text) or {}
        except Exception as e:
            return {"exists": False, "error": f"yaml parse: {e}"}

        meta = doc.get("meta") or {}
        model = doc.get("model") or {}
        features = doc.get("features") or {}
        deps = doc.get("dependencies") or []
        variants = doc.get("variants") or {}
        hw_overrides = doc.get("hardware_overrides") or {}
        strat_overrides = doc.get("strategy_overrides") or {}

        # Tool-call + reasoning parsers, as flat arg arrays, so the frontend
        # can drop them straight into the launch command.
        tool_calling = features.get("tool_calling") or {}
        reasoning = features.get("reasoning") or {}

        normalized = {
            "exists": True,
            "source_url": url,
            "title": meta.get("title") or "",
            "provider": meta.get("provider") or "",
            "description": meta.get("description") or "",
            "date_updated": str(meta.get("date_updated") or ""),
            "hardware_support": meta.get("hardware") or {},
            "model_id": model.get("model_id") or repo,
            "min_vllm_version": model.get("min_vllm_version") or "",
            "architecture": model.get("architecture") or "",
            "parameter_count": model.get("parameter_count") or "",
            "active_parameters": model.get("active_parameters") or "",
            "context_length": model.get("context_length") or 0,
            "base_args": list(model.get("base_args") or []),
            "base_env": dict(model.get("base_env") or {}),
            "tool_calling": {
                "description": tool_calling.get("description") or "",
                "args": list(tool_calling.get("args") or []),
            } if tool_calling else None,
            "reasoning": {
                "description": reasoning.get("description") or "",
                "args": list(reasoning.get("args") or []),
            } if reasoning else None,
            "dependencies": [
                {
                    "note": (d.get("note") or "").strip(),
                    "command": (d.get("command") or "").strip(),
                    "optional": bool(d.get("optional", False)),
                }
                for d in deps if isinstance(d, dict)
            ],
            "variants": {
                k: {
                    "model_id": v.get("model_id") or model.get("model_id") or repo,
                    "precision": v.get("precision") or "",
                    "vram_minimum_gb": v.get("vram_minimum_gb") or 0,
                    "description": v.get("description") or "",
                    "extra_args": list(v.get("extra_args") or []),
                    "extra_env": dict(v.get("extra_env") or {}),
                }
                for k, v in variants.items() if isinstance(v, dict)
            },
            "hardware_overrides": {
                hw: {
                    "extra_args": list((ov or {}).get("extra_args") or []),
                    "extra_env": dict((ov or {}).get("extra_env") or {}),
                }
                for hw, ov in hw_overrides.items() if isinstance(ov, dict)
            },
            "strategy_overrides": {
                strat: dict(ov or {})
                for strat, ov in strat_overrides.items() if isinstance(ov, dict)
            },
            "compatible_strategies": list(doc.get("compatible_strategies") or []),
        }
        _vllm_recipe_cache[repo] = (now, normalized)
        return normalized

    @router.get("/api/cookbook/tasks/status")
    async def cookbook_tasks_status(request: Request):
        """Check status of all active cookbook tmux sessions.

        Critical: every subprocess.run inside this handler is a sync blocking
        call that — when this was a plain async def — froze the entire server
        event loop. Now the whole body runs in a worker thread via
        asyncio.to_thread so other requests stay responsive."""
        require_admin(request)
        now = time.monotonic()
        cached = _tasks_status_cache.get("value")
        if cached is not None and now - float(_tasks_status_cache.get("ts") or 0) < 2.0:
            return cached
        inflight = _tasks_status_inflight.get("task")
        if inflight and not inflight.done():
            return await inflight

        async def _compute():
            data = await asyncio.to_thread(_cookbook_tasks_status_sync)
            _tasks_status_cache.update({"ts": time.monotonic(), "value": data})
            return data

        task = asyncio.create_task(_compute())
        _tasks_status_inflight["task"] = task
        try:
            return await task
        finally:
            if _tasks_status_inflight.get("task") is task:
                _tasks_status_inflight["task"] = None

    def _cookbook_tasks_status_sync():
        import subprocess

        def _pick_download_progress(lines: list[str]) -> str:
            """Pick the most useful live HF progress line from a tmux pane."""
            if not lines:
                return ""
            downloading_lines = [l for l in lines if l.startswith("Downloading")]
            if downloading_lines:
                return downloading_lines[-1]
            progress_lines = [
                l for l in lines
                if re.search(r"\b(?:100|[1-9]?\d)%", l)
                and (
                    "<" in l
                    or "it/s" in l
                    or "B/s" in l
                    or "safetensors" in l
                    or ".gguf" in l.lower()
                )
            ]
            if progress_lines:
                return progress_lines[-1]
            return lines[-1]

        def _download_cache_complete(repo_id: str, remote_host: str = "", ssh_port: str = "", cache_root: str = "") -> bool:
            """Best-effort check for a completed HF cache entry.

            tmux output can stop at a stale progress line if the pane/session
            disappears before Cookbook captures the final DOWNLOAD_OK marker.
            In that case, trust the cache shape: a snapshot directory with files
            and no *.incomplete blobs means HuggingFace finished materializing the
            model. cache_root is the task's custom download dir — the runner
            pointed HF_HOME there, so the cache lives under <cache_root>/hub,
            not wherever this probe's environment says.
            """
            if not repo_id or "/" not in repo_id:
                return False
            cmd = ["python3", "-c", HF_CACHE_COMPLETE_PROBE, repo_id, cache_root or ""]
            try:
                if remote_host:
                    ssh_base = ["ssh"]
                    if ssh_port and ssh_port != "22":
                        ssh_base.extend(["-p", str(ssh_port)])
                    shell_cmd = " ".join(shlex.quote(x) for x in cmd)
                    proc = subprocess.run(ssh_base + [remote_host, shell_cmd], timeout=12, capture_output=True)
                else:
                    proc = subprocess.run(cmd, timeout=12, capture_output=True)
                return proc.returncode == 0
            except Exception:
                return False

        def _download_cache_incomplete(repo_id: str, remote_host: str = "", ssh_port: str = "", cache_root: str = "") -> bool:
            """Best-effort check for resumable HF partial blobs.

            A lost SSH/tmux session can leave a real download still incomplete.
            Treat any *.incomplete blob as stronger evidence than stale
            "100%" lines in the captured pane output.
            """
            if not repo_id or "/" not in repo_id:
                return False
            cmd = ["python3", "-c", HF_CACHE_INCOMPLETE_PROBE, repo_id, cache_root or ""]
            try:
                if remote_host:
                    ssh_base = ["ssh"]
                    if ssh_port and ssh_port != "22":
                        ssh_base.extend(["-p", str(ssh_port)])
                    shell_cmd = " ".join(shlex.quote(x) for x in cmd)
                    proc = subprocess.run(ssh_base + [remote_host, shell_cmd], timeout=12, capture_output=True)
                else:
                    proc = subprocess.run(cmd, timeout=12, capture_output=True)
                return proc.returncode == 0
            except Exception:
                return False

        # Load saved tasks from cookbook state
        tasks = []
        state = {}
        if _cookbook_state_path.exists():
            try:
                state = json.loads(_cookbook_state_path.read_text(encoding="utf-8"))
                saved_tasks = state.get("tasks", [])
                if isinstance(saved_tasks, list):
                    tasks = saved_tasks
                elif isinstance(saved_tasks, dict):
                    tasks = list(saved_tasks.values())
            except Exception:
                pass

        # Orphan-tmux auto-adoption sweep. When the agent (or anyone)
        # SSH-launches a `serve-*` tmux session — usually because
        # serve_model rejected `source ... && vllm ...` or because of a
        # manual relaunch via tmux send-keys — that session is invisible
        # to the cookbook UI even though it's a live model server. The
        # sweep finds those orphans on each configured remote host and
        # writes them into state.tasks with _adoptedExternally=True, so
        # they show up in the UI on the next poll without anyone having
        # to remember to call adopt_served_model. Rate-limited via the
        # module-level _last_orphan_sweep so we don't SSH every 3s.
        try:
            _maybe_sweep_orphans(tasks, state)
        except Exception as _sweep_e:
            logger.warning(f"orphan sweep failed (non-fatal): {_sweep_e!r}")

        results = []
        for task in tasks:
            session_id = task.get("sessionId", "")
            if not session_id:
                continue
            remote = task.get("remoteHost", "")
            task_type = task.get("type", "download")  # "download" or "serve"
            # Field name varies depending on whether the task was added
            # via the download flow (`repoId`), the serve flow (`modelId`),
            # or the UI-side serve preset (which uses `name` + `payload.repo_id`).
            _payload = task.get("payload") or {}
            model = (
                task.get("modelId")
                or task.get("repoId")
                or task.get("name")
                or _payload.get("repo_id")
                or _payload.get("modelId")
                or ""
            )
            task_platform = task.get("platform", "")

            # Check if session is alive + capture output
            _tport = task.get("sshPort", "")
            # Defense-in-depth: cookbook state is admin-writable but the values
            # land in shell-interpolated commands below. Reject anything that
            # isn't a benign session-id / hostname / port.
            if not _SESSION_ID_RE.match(session_id):
                logger.warning(f"Skipping task with unsafe session_id: {session_id!r}")
                continue
            if remote:
                try:
                    remote = validate_remote_host(remote)
                except HTTPException:
                    logger.warning(f"Skipping task with unsafe remoteHost: {remote!r}")
                    continue
            if _tport:
                try:
                    _tport = validate_ssh_port(str(_tport))
                except HTTPException:
                    logger.warning(f"Skipping task with unsafe sshPort: {_tport!r}")
                    continue
            if task_platform == "windows" and remote:
                # Windows: check PID file + Get-Process, read log tail
                sd = "$env:TEMP\\odysseus-sessions"
                ssh_base = ["ssh"]
                if _tport and _tport != "22":
                    ssh_base.extend(["-p", str(_tport)])
                check_cmd = ssh_base + [
                    remote,
                    "powershell",
                    "-Command",
                    f"$pid = Get-Content \"{sd}\\{session_id}.pid\" -ErrorAction SilentlyContinue; "
                    "if ($pid) {{ Get-Process -Id $pid -ErrorAction SilentlyContinue | Out-Null; if ($?) {{ exit 0 }} else {{ exit 1 }} }} else {{ exit 1 }}"
                ]
                capture_cmd = ssh_base + [
                    remote,
                    "powershell",
                    "-Command",
                    f"Get-Content \"{sd}\\{session_id}.log\" -Tail 10 -ErrorAction SilentlyContinue",
                ]
            elif remote:
                ssh_base = ["ssh"]
                if _tport and _tport != "22":
                    ssh_base.extend(["-p", str(_tport)])
                check_cmd = ssh_base + [remote, _remote_tmux_command("has-session", "-t", session_id)]
                # Capture 500 lines (was 50) so a Python traceback survives
                # the post-crash neofetch banner + bash prompt that otherwise
                # fills the visible tail. Without this, output_tail ends up
                # as just "Locale: C / Ubuntu_Odysseus ❯" and the agent
                # can't diagnose the actual error.
                capture_cmd = ssh_base + [remote, _remote_tmux_command("capture-pane", "-t", session_id, "-p", "-S", "-500")]
            elif IS_WINDOWS:
                # LOCAL Windows task: launched as a detached process (no tmux).
                # Liveness comes from the <session>.pid file, output from the
                # <session>.log file the wrapper redirects into. No subprocess.
                check_cmd = None
                capture_cmd = None
            else:
                check_cmd = ["tmux", "has-session", "-t", session_id]
                capture_cmd = ["tmux", "capture-pane", "-t", session_id, "-p", "-S", "-500"]

            local_win_task = (not remote) and IS_WINDOWS

            progress_text = ""
            full_snapshot = (task.get("output") or "")[-12000:] if task_type == "serve" else ""

            if local_win_task:
                # File-based liveness + output for the detached-process model.
                pid_path = TMUX_LOG_DIR / f"{session_id}.pid"
                log_path = TMUX_LOG_DIR / f"{session_id}.log"
                task_pid = None
                try:
                    task_pid = int(pid_path.read_text(encoding="utf-8").strip())
                except Exception:
                    task_pid = None
                is_alive = pid_alive(task_pid)
                try:
                    if log_path.exists():
                        full_snapshot = log_path.read_text(
                            encoding="utf-8", errors="replace"
                        ).strip()[-12000:]
                        lines = [l.strip() for l in full_snapshot.split('\n') if l.strip()]
                        progress_text = _pick_download_progress(lines)
                except Exception:
                    pass
            else:
                # Skip the live SSH check entirely for tasks already in a
                # terminal state — they won't change, and 10s timeouts
                # stacked per task were the dominant cost of this whole
                # status endpoint (3+ minute stalls with ~8 accumulated
                # stopped tasks). The agent's `list_served_models` call
                # was blocking the chat stream every time.
                _task_status = (task.get("status") or "").lower()
                _persisted_serve_ready = (
                    task_type == "serve"
                    and bool(full_snapshot)
                    and _parse_serve_phase(full_snapshot, task_type).get("status") == "ready"
                )
                if _task_status in {"stopped", "done", "completed",
                                    "crashed", "error", "failed",
                                    "ended", "killed"} and not _persisted_serve_ready:
                    is_alive = False
                    # Keep the persisted output_tail for the UI — it's
                    # what the agent uses to diagnose past failures.
                    full_snapshot = (task.get("output") or "")[-12000:]
                else:
                    try:
                        alive = subprocess.run(check_cmd, timeout=4, capture_output=True)
                        is_alive = alive.returncode == 0
                    except Exception:
                        is_alive = False

                    # Capture last lines for progress. Prefer the "Downloading" line
                    # (real aggregate bytes) over "Fetching N files" (whole-file count that
                    # lags with hf_transfer). Falls back to the true last line otherwise.
                    if is_alive:
                        try:
                            cap = subprocess.run(capture_cmd, timeout=4, capture_output=True, text=True)
                            if cap.returncode == 0:
                                full_snapshot = cap.stdout.strip()
                                lines = [l.strip() for l in full_snapshot.split('\n') if l.strip()]
                                progress_text = _pick_download_progress(lines)
                        except Exception:
                            pass

            # Determine status. For the local-Windows detached model the log file
            # persists after the process exits, so a finished download still has a
            # snapshot to classify (DOWNLOAD_OK / exit marker) — evaluate it even
            # when the PID is gone instead of blindly reporting "stopped".
            download_zero_files = False
            exit_code = None
            status = "unknown"
            download_has_ok = task_type == "download" and "DOWNLOAD_OK" in full_snapshot
            download_has_failed = task_type == "download" and "DOWNLOAD_FAILED" in full_snapshot
            download_has_incomplete_evidence = (
                task_type == "download"
                and (
                    ".incomplete" in full_snapshot
                    or bool(re.search(r'model-\d+-of-\d+\.[A-Za-z0-9_.-]+:\s+(?:[0-9]|[1-8][0-9])%', full_snapshot))
                    or _download_cache_incomplete(_payload.get("repo_id") or model, remote, str(_tport or ""), _payload.get("local_dir") or "")
                )
            )
            if is_alive or (local_win_task and full_snapshot):
                lower = full_snapshot.lower()
                exit_match = re.search(r"=== process exited with code\s+(-?\d+)", full_snapshot, re.I)
                has_exit = exit_match is not None
                exit_code = int(exit_match.group(1)) if exit_match else None
                has_error = "error" in lower or "failed" in lower or "traceback" in lower
                if has_exit and task_type == "serve":
                    # Serve tasks that exit are always errors — they should run indefinitely
                    status = "error"
                elif has_exit and task_type == "download":
                    # Dependency installs are tracked as download tasks but only
                    # emit the generic runner exit marker, not HF download markers.
                    if download_has_incomplete_evidence and not download_has_ok:
                        status = "running" if is_alive else "stopped"
                    else:
                        status = "completed" if exit_code == 0 else "error"
                elif has_exit and "unrecognized arguments" in lower:
                    status = "error"
                elif has_error and not ("application startup complete" in lower):
                    status = "error"
                elif task_type == "download" and download_has_ok:
                    if re.search(r"Fetching\s+0\s+files", full_snapshot, re.IGNORECASE):
                        status = "error"
                        download_zero_files = True
                    else:
                        status = "completed"
                elif task_type == "download" and download_has_failed:
                    status = "error"
                elif task_type == "download" and download_has_incomplete_evidence:
                    status = "running" if is_alive else "stopped"
                elif "application startup complete" in lower:
                    status = "ready"
                elif not is_alive:
                    # local-Windows: process gone, log has no success/ready marker.
                    status = "stopped"
                else:
                    status = "running"
            else:
                # Session is dead — check if it completed or crashed. The
                # runner markers in the retained output are conclusive
                # (DOWNLOAD_OK only prints after exit 0), so check them before
                # the cache probe, which can't see ollama pulls at all.
                marker = classify_dead_download(full_snapshot) if task_type == "download" else None
                if marker is not None:
                    status, download_zero_files = marker
                    if status == "completed" and not progress_text:
                        progress_text = "Download complete"
                elif (
                    task_type == "download"
                    and not download_has_incomplete_evidence
                    and _download_cache_complete(_payload.get("repo_id") or model, remote, str(_tport or ""), _payload.get("local_dir") or "")
                ):
                    status = "completed"
                    if not progress_text:
                        progress_text = "Download complete"
                    if not full_snapshot:
                        full_snapshot = "DOWNLOAD_OK"
                else:
                    status = "stopped"

            # Parse structured phase info — single source of truth for the UI
            phase_info = _parse_serve_phase(full_snapshot, task_type) if (task_type == "serve" and full_snapshot) else {}
            if phase_info.get("status") == "ready" and is_alive:
                status = "ready"
            serve_phase = phase_info.get("phase", "")
            diagnosis = _diagnose_serve_output(full_snapshot) if task_type == "serve" and full_snapshot else None
            if diagnosis and status in {"running", "unknown", "stopped"} and phase_info.get("status") != "ready":
                status = "error"
            if download_zero_files:
                diagnosis = {"message": "No matching files were downloaded. The model repo or filename/quant pattern may be wrong (for example a ':Q4_K_M' tag that does not exist in the repo). Check the repo and the include/quant pattern."}
            output_tail = error_aware_output_tail(full_snapshot, status)

            results.append({
                "session_id": session_id,
                "type": task_type,
                "model": model.split("/")[-1] if "/" in model else model,
                "status": status,
                "progress": serve_phase if task_type == "serve" else progress_text[:120],
                "phase": serve_phase,
                "diagnosis": diagnosis,
                "output_tail": output_tail,
                "exit_code": exit_code,
                "cmd": _payload.get("_cmd") or "",
                "tps": phase_info.get("tps"),
                "reqs": phase_info.get("reqs"),
                "pct": phase_info.get("pct"),
                "remote": remote or "local",
            })

        return {"tasks": results}

    return router