intelligence-terminal/server.mjs

#!/usr/bin/env node
// Crucix Intelligence Engine — Dev Server
// Serves the Jarvis dashboard, runs sweep cycle, pushes live updates via SSE

import express from 'express';
import { readFileSync, writeFileSync, mkdirSync, existsSync } from 'fs';
import { dirname, join } from 'path';
import { fileURLToPath } from 'url';
import { exec } from 'child_process';
import config from './crucix.config.mjs';
import { getLocale, currentLanguage, getSupportedLocales } from './lib/i18n.mjs';
import { fullBriefing } from './apis/briefing.mjs';
import { synthesize, generateIdeas } from './dashboard/inject.mjs';
import { MemoryManager } from './lib/delta/index.mjs';
import { createLLMProvider } from './lib/llm/index.mjs';
import { generateLLMIdeas } from './lib/llm/ideas.mjs';
import { TelegramChatAssistant, buildTelegramChatContext } from './lib/llm/telegram-chat.mjs';
import { TerminalAgent } from './lib/agent/terminal-agent.mjs';
import { createTerminalToolRegistry } from './lib/agent/terminal-tools.mjs';
import { TelegramAlerter } from './lib/alerts/telegram.mjs';
import { DiscordAlerter } from './lib/alerts/discord.mjs';
import { getFetchMetrics } from './apis/utils/fetch.mjs';
import { IntelligenceStore } from './lib/intelligence-store.mjs';
import { formatStaleAlert, shouldSendStaleAlert } from './lib/stale-alerts.mjs';
import { evaluateScenarios } from './lib/scenarios.mjs';

const __dirname = dirname(fileURLToPath(import.meta.url));
const ROOT = __dirname;
const RUNS_DIR = join(ROOT, 'runs');
const MEMORY_DIR = join(RUNS_DIR, 'memory');

// Ensure directories exist
for (const dir of [RUNS_DIR, MEMORY_DIR, join(MEMORY_DIR, 'cold')]) {
  if (!existsSync(dir)) mkdirSync(dir, { recursive: true });
}

// === State ===
let currentData = null;    // Current synthesized dashboard data
let lastSweepTime = null;  // Timestamp of last sweep
let lastSuccessfulSweepTime = null;
let lastSweepError = null;
let bootstrapDataLoaded = false;
let sweepStartedAt = null; // Timestamp when current/last sweep started
let sweepInProgress = false;
const startTime = Date.now();
const sseClients = new Set();
const terminalActionBuckets = new Map();
const staleAlertState = {};

// === Delta/Memory ===
const memory = new MemoryManager(RUNS_DIR);
const intelligenceStore = new IntelligenceStore(join(RUNS_DIR, 'intelligence.db'));
await intelligenceStore.init();

// === LLM + Telegram + Discord ===
const llmProvider = createLLMProvider(config.llm);
const telegramAlerter = new TelegramAlerter(config.telegram);
const discordAlerter = new DiscordAlerter(config.discord || {});
const terminalToolRegistry = createTerminalToolRegistry({
  getData: () => currentData,
  getHealth: () => buildHealth(),
  getDelta: () => memory.getLastDelta(),
  buildBrief,
  intelligenceStore,
  triggerSweep: () => runSweepCycle().catch(error => console.error('[Agent] Confirmed sweep failed:', error.message)),
  isSweepInProgress: () => sweepInProgress,
  telegramAlerter,
});
const terminalAgent = new TerminalAgent({
  provider: llmProvider,
  registry: terminalToolRegistry,
  maxSteps: config.telegram.agentMaxSteps,
  maxTokens: config.telegram.aiMaxTokens,
  timeoutMs: config.telegram.aiTimeoutMs,
  confirmationTtlMs: config.telegram.agentConfirmationTtlSeconds * 1000,
  proactiveCooldownMs: config.telegram.agentProactiveCooldownMinutes * 60 * 1000,
});
const telegramChatAssistant = new TelegramChatAssistant({
  provider: llmProvider,
  agent: config.telegram.agentEnabled ? terminalAgent : null,
  getContext: () => buildTelegramChatContext(currentData, buildHealth()),
  historyMessages: config.telegram.aiHistoryMessages,
  maxInputChars: config.telegram.aiMaxInputChars,
  maxTokens: config.telegram.aiMaxTokens,
  timeoutMs: config.telegram.aiTimeoutMs,
});

if (llmProvider) console.log(`[Crucix] LLM enabled: ${llmProvider.name} (${llmProvider.model})`);
else if (config.llm.provider) console.warn(`[Crucix] LLM provider "${config.llm.provider}" is not configured; LLM features disabled`);
if (telegramAlerter.isConfigured) {
  console.log('[Crucix] Telegram alerts enabled');

  // ─── Two-Way Bot Commands ───────────────────────────────────────────────

  telegramAlerter.onCommand('/status', async () => {
    const uptime = Math.floor((Date.now() - startTime) / 1000);
    const h = Math.floor(uptime / 3600);
    const m = Math.floor((uptime % 3600) / 60);
    const sourcesOk = currentData?.meta?.sourcesOk || 0;
    const sourcesTotal = currentData?.meta?.sourcesQueried || 0;
    const sourcesFailed = currentData?.meta?.sourcesFailed || 0;
    const llmStatus = llmProvider?.isConfigured ? `✅ ${llmProvider.name}` : '❌ Disabled';
    const nextSweep = lastSweepTime
      ? new Date(new Date(lastSweepTime).getTime() + config.refreshIntervalMinutes * 60000).toLocaleTimeString()
      : 'pending';

    return [
      `🖥️ *CRUCIX STATUS*`,
      ``,
      `Uptime: ${h}h ${m}m`,
      `Last sweep: ${lastSweepTime ? new Date(lastSweepTime).toLocaleTimeString() + ' UTC' : 'never'}`,
      `Next sweep: ${nextSweep} UTC`,
      `Sweep in progress: ${sweepInProgress ? '🔄 Yes' : '⏸️ No'}`,
      `Sources: ${sourcesOk}/${sourcesTotal} OK${sourcesFailed > 0 ? ` (${sourcesFailed} failed)` : ''}`,
      `LLM: ${llmStatus}`,
      `AI chat: ${config.telegram.aiChatEnabled && telegramChatAssistant.isConfigured ? 'enabled' : 'disabled'}`,
      `Tool agent: ${config.telegram.agentEnabled && terminalAgent.isConfigured ? 'enabled' : 'disabled'} (${terminalAgent.listTools().length} tools)`,
      `SSE clients: ${sseClients.size}`,
      `Dashboard: http://localhost:${config.port}`,
    ].join('\n');
  });

  telegramAlerter.onCommand('/sweep', async () => {
    if (sweepInProgress) return '🔄 Sweep already in progress. Please wait.';
    // Fire and forget — don't block the bot response
    runSweepCycle().catch(err => console.error('[Crucix] Manual sweep failed:', err.message));
    return '🚀 Manual sweep triggered. You\'ll receive alerts if anything significant is detected.';
  });

  telegramAlerter.onCommand('/brief', async () => {
    if (!currentData) return '⏳ No data yet — waiting for first sweep to complete.';
    return buildBrief(currentData);

    const tg = currentData.tg || {};
    const energy = currentData.energy || {};
    const metals = currentData.metals || {};
    const delta = memory.getLastDelta();
    const ideas = (currentData.ideas || []).slice(0, 3);

    const sections = [
      `📋 *CRUCIX BRIEF*`,
      `_${new Date().toISOString().replace('T', ' ').substring(0, 19)} UTC_`,
      ``,
    ];

    // Delta direction
    if (delta?.summary) {
      const dirEmoji = { 'risk-off': '📉', 'risk-on': '📈', 'mixed': '↔️' }[delta.summary.direction] || '↔️';
      sections.push(`${dirEmoji} Direction: *${delta.summary.direction.toUpperCase()}* | ${delta.summary.totalChanges} changes, ${delta.summary.criticalChanges} critical`);
      sections.push('');
    }

    // Key metrics
    const vix = currentData.fred?.find(f => f.id === 'VIXCLS');
    const hy = currentData.fred?.find(f => f.id === 'BAMLH0A0HYM2');
    if (vix || energy.wti || metals.gold || metals.silver) {
      sections.push(`📊 VIX: ${vix?.value || '--'} | WTI: $${energy.wti || '--'} | Brent: $${energy.brent || '--'}`);
      sections.push(`   Gold: $${metals.gold || '--'} | Silver: $${metals.silver || '--'}${hy ? ` | HY Spread: ${hy.value}` : ''}`);
      sections.push(`   NatGas: $${energy.natgas || '--'}`);
      sections.push('');
    }

    // OSINT
    if (tg.urgent?.length > 0) {
      sections.push(`📡 OSINT: ${tg.urgent.length} urgent signals, ${tg.posts || 0} total posts`);
      // Top 2 urgent
      for (const p of tg.urgent.slice(0, 2)) {
        sections.push(`  • ${(p.text || '').substring(0, 80)}`);
      }
      sections.push('');
    }

    // Top ideas
    if (ideas.length > 0) {
      sections.push(`💡 *Top Ideas:*`);
      for (const idea of ideas) {
        sections.push(`  ${idea.type === 'long' ? '📈' : idea.type === 'hedge' ? '🛡️' : '👁️'} ${idea.title}`);
      }
    }

    return sections.join('\n');
  });

  const answerTelegramQuestion = async (question, msg) => {
    if (!config.telegram.aiChatEnabled) {
      return { text: 'AI chat is disabled by TELEGRAM_AI_CHAT_ENABLED.', parseMode: null };
    }
    const chatId = msg?.chat?.id || config.telegram.chatId;
    const result = await telegramChatAssistant.replyDetailed(question, { chatId });
    const tools = [...new Set((result.trace || []).filter(item => item.status === 'ok').map(item => item.tool))];
    const traceSuffix = tools.length ? `\n\nTools used: ${tools.join(', ')}` : '';
    if (result.pendingAction) {
      const action = result.pendingAction;
      return {
        text: `${result.answer}\nAction: ${action.tool}\nReason: ${action.rationale || 'requested by agent'}\nExpires: ${action.expiresAt}`,
        parseMode: null,
        replyMarkup: {
          inline_keyboard: [[
            { text: 'Confirm', callback_data: `agent_confirm:${action.id}` },
            { text: 'Cancel', callback_data: `agent_cancel:${action.id}` },
          ]],
        },
      };
    }
    return { text: `${result.answer}${traceSuffix}`, parseMode: null };
  };

  telegramAlerter.onMessage((text, msg) => answerTelegramQuestion(text, msg));

  telegramAlerter.onCommand('/ask', async (args, _messageId, msg) => {
    if (!args.trim()) return { text: 'Usage: /ask <question>', parseMode: null };
    return answerTelegramQuestion(args, msg);
  });

  telegramAlerter.onCommand('/reset', async (_args, _messageId, msg) => {
    telegramChatAssistant.reset(msg?.chat?.id || config.telegram.chatId);
    return { text: 'AI conversation history cleared.', parseMode: null };
  });

  telegramAlerter.onCommand('/tools', async () => ({
    text: terminalAgent.listTools().map(tool => `${tool.mutating ? '[confirm]' : '[read]'} ${tool.name}: ${tool.description}`).join('\n'),
    parseMode: null,
  }));

  telegramAlerter.onCommand('/trace', async (_args, _messageId, msg) => {
    const trace = terminalAgent.getLastTrace(msg?.chat?.id || config.telegram.chatId);
    return {
      text: trace.length
        ? trace.map(item => `${item.status}: ${item.tool} (${item.durationMs}ms)${item.rationale ? ` - ${item.rationale}` : ''}`).join('\n')
        : 'No tool trace is available for this chat.',
      parseMode: null,
    };
  });

  const confirmAgentAction = async (id, chatId) => {
    const result = await terminalAgent.confirm(id, chatId);
    return { text: result.message, parseMode: null };
  };
  const cancelAgentAction = (id, chatId) => ({
    text: terminalAgent.cancel(id, chatId) ? 'Pending action cancelled.' : 'Pending action is unknown, expired, or belongs to another chat.',
    parseMode: null,
  });

  telegramAlerter.onCommand('/confirm', async (args, _messageId, msg) => confirmAgentAction(args.trim(), msg?.chat?.id || config.telegram.chatId));
  telegramAlerter.onCommand('/cancel', async (args, _messageId, msg) => cancelAgentAction(args.trim(), msg?.chat?.id || config.telegram.chatId));
  telegramAlerter.onCallback(async (data, query) => {
    const [operation, id] = String(data).split(':', 2);
    const chatId = query.message?.chat?.id || config.telegram.chatId;
    if (operation === 'agent_confirm') return confirmAgentAction(id, chatId);
    if (operation === 'agent_cancel') return cancelAgentAction(id, chatId);
    return { text: 'Unknown agent action.', parseMode: null };
  });

  telegramAlerter.onCommand('/portfolio', async () => {
    return '📊 Portfolio integration requires Alpaca MCP connection.\nUse the Crucix dashboard or Claude agent for portfolio queries.';
  });

  // Start polling for bot commands
  telegramAlerter.startPolling(config.telegram.botPollingInterval);
}

// === Discord Bot ===
if (discordAlerter.isConfigured) {
  console.log('[Crucix] Discord bot enabled');

  // Reuse the same command handlers as Telegram (DRY)
  discordAlerter.onCommand('status', async () => {
    const uptime = Math.floor((Date.now() - startTime) / 1000);
    const h = Math.floor(uptime / 3600);
    const m = Math.floor((uptime % 3600) / 60);
    const sourcesOk = currentData?.meta?.sourcesOk || 0;
    const sourcesTotal = currentData?.meta?.sourcesQueried || 0;
    const sourcesFailed = currentData?.meta?.sourcesFailed || 0;
    const llmStatus = llmProvider?.isConfigured ? `✅ ${llmProvider.name}` : '❌ Disabled';
    const nextSweep = lastSweepTime
      ? new Date(new Date(lastSweepTime).getTime() + config.refreshIntervalMinutes * 60000).toLocaleTimeString()
      : 'pending';

    return [
      `**🖥️ CRUCIX STATUS**\n`,
      `Uptime: ${h}h ${m}m`,
      `Last sweep: ${lastSweepTime ? new Date(lastSweepTime).toLocaleTimeString() + ' UTC' : 'never'}`,
      `Next sweep: ${nextSweep} UTC`,
      `Sweep in progress: ${sweepInProgress ? '🔄 Yes' : '⏸️ No'}`,
      `Sources: ${sourcesOk}/${sourcesTotal} OK${sourcesFailed > 0 ? ` (${sourcesFailed} failed)` : ''}`,
      `LLM: ${llmStatus}`,
      `SSE clients: ${sseClients.size}`,
      `Dashboard: http://localhost:${config.port}`,
    ].join('\n');
  });

  discordAlerter.onCommand('sweep', async () => {
    if (sweepInProgress) return '🔄 Sweep already in progress. Please wait.';
    runSweepCycle().catch(err => console.error('[Crucix] Manual sweep failed:', err.message));
    return '🚀 Manual sweep triggered. You\'ll receive alerts if anything significant is detected.';
  });

  discordAlerter.onCommand('brief', async () => {
    if (!currentData) return '⏳ No data yet — waiting for first sweep to complete.';
    return buildBrief(currentData);

    const tg = currentData.tg || {};
    const energy = currentData.energy || {};
    const metals = currentData.metals || {};
    const delta = memory.getLastDelta();
    const ideas = (currentData.ideas || []).slice(0, 3);

    const sections = [`**📋 CRUCIX BRIEF**\n_${new Date().toISOString().replace('T', ' ').substring(0, 19)} UTC_\n`];

    if (delta?.summary) {
      const dirEmoji = { 'risk-off': '📉', 'risk-on': '📈', 'mixed': '↔️' }[delta.summary.direction] || '↔️';
      sections.push(`${dirEmoji} Direction: **${delta.summary.direction.toUpperCase()}** | ${delta.summary.totalChanges} changes, ${delta.summary.criticalChanges} critical\n`);
    }

    const vix = currentData.fred?.find(f => f.id === 'VIXCLS');
    const hy = currentData.fred?.find(f => f.id === 'BAMLH0A0HYM2');
    if (vix || energy.wti || metals.gold || metals.silver) {
      sections.push(`📊 VIX: ${vix?.value || '--'} | WTI: $${energy.wti || '--'} | Brent: $${energy.brent || '--'}`);
      sections.push(`   Gold: $${metals.gold || '--'} | Silver: $${metals.silver || '--'}${hy ? ` | HY Spread: ${hy.value}` : ''}`);
      sections.push(`   NatGas: $${energy.natgas || '--'}`);
      sections.push('');
    }

    if (tg.urgent?.length > 0) {
      sections.push(`📡 OSINT: ${tg.urgent.length} urgent signals, ${tg.posts || 0} total posts`);
      for (const p of tg.urgent.slice(0, 2)) {
        sections.push(`  • ${(p.text || '').substring(0, 80)}`);
      }
      sections.push('');
    }

    if (ideas.length > 0) {
      sections.push(`**💡 Top Ideas:**`);
      for (const idea of ideas) {
        sections.push(`  ${idea.type === 'long' ? '📈' : idea.type === 'hedge' ? '🛡️' : '👁️'} ${idea.title}`);
      }
    }

    return sections.join('\n');
  });

  discordAlerter.onCommand('portfolio', async () => {
    return '📊 Portfolio integration requires Alpaca MCP connection.\nUse the Crucix dashboard or Claude agent for portfolio queries.';
  });

  // Start the Discord bot (non-blocking — connection happens async)
  discordAlerter.start().catch(err => {
    console.error('[Crucix] Discord bot startup failed (non-fatal):', err.message);
  });
}

// === Express Server ===
const app = express();
app.use(express.static(join(ROOT, 'dashboard/public')));

// Serve loading page until first sweep completes, then the dashboard with injected locale
app.get('/', (req, res) => {
  if (!currentData) {
    res.sendFile(join(ROOT, 'dashboard/public/loading.html'));
  } else {
    const htmlPath = join(ROOT, 'dashboard/public/jarvis.html');
    let html = readFileSync(htmlPath, 'utf-8');
    
    // Inject locale data into the HTML
    const locale = getLocale();
    const localeScript = `<script>window.__CRUCIX_LOCALE__ = ${JSON.stringify(locale).replace(/<\/script>/gi, '<\\/script>')};</script>`;
    html = html.replace('</head>', `${localeScript}\n</head>`);
    
    res.type('html').send(html);
  }
});

// API: current data
app.get('/api/data', (req, res) => {
  if (!currentData) return res.status(503).json({ error: 'No data yet — first sweep in progress' });
  res.json(currentData);
});

// API: health check
app.get('/api/health', (req, res) => {
  const health = buildHealth();
  const httpStatus = health.status === 'error' ? 500 : health.status === 'starting' ? 503 : 200;
  res.status(httpStatus).json(health);
});

// API: network/source/LLM metrics
app.get('/api/metrics', (req, res) => {
  res.json({
    generatedAt: new Date().toISOString(),
    fetch: getFetchMetrics(),
    sources: currentData?.sourceHealth || currentData?.health || [],
    news: currentData?.newsMeta || {},
    llm: getLLMStatus(),
    memory: intelligenceStore.status(),
  });
});

app.get('/api/memory/search', (req, res) => {
  const guard = authorizeTerminalAction(req, res, 'memory:search');
  if (!guard.ok) return;
  auditTerminalAction(req, 'memory:search', 'ok');
  res.json(intelligenceStore.queryMemory({
    q: req.query.q || '',
    limit: req.query.limit || 25,
  }));
});

app.get('/api/memory/predictions', (req, res) => {
  const guard = authorizeTerminalAction(req, res, 'memory:predictions');
  if (!guard.ok) return;
  auditTerminalAction(req, 'memory:predictions', 'ok');
  res.json(intelligenceStore.listPredictions({
    state: req.query.state || null,
    limit: req.query.limit || 25,
  }));
});

app.post('/api/sweep', express.json(), (req, res) => {
  const guard = authorizeTerminalAction(req, res, 'sweep');
  if (!guard.ok) return;
  triggerSweepAction(req, res, 'sweep');
});

app.post('/api/action', express.json(), (req, res) => {
  const action = String(req.body?.action || req.body?.command || '').trim().toLowerCase();
  const guard = authorizeTerminalAction(req, res, action || 'unknown');
  if (!guard.ok) return;

  if (action === 'status') {
    auditTerminalAction(req, 'status', 'ok');
    return res.json({ ok: true, action, status: 'ok', health: buildHealth() });
  }

  if (action === 'brief') {
    if (!currentData) {
      auditTerminalAction(req, 'brief', 'rejected', 'no_data');
      return res.status(503).json({ ok: false, action, error: 'No data yet - first sweep in progress' });
    }
    auditTerminalAction(req, 'brief', 'ok');
    const brief = buildBrief(currentData);
    return res.json({ ok: true, action, status: 'ok', brief, text: brief });
  }

  if (action === 'memory') {
    auditTerminalAction(req, 'memory', 'ok');
    return res.json({
      ok: true,
      action,
      memory: intelligenceStore.status(),
      recentEvents: intelligenceStore.queryMemory({ q: req.body?.q || '', limit: 8 }).results,
      predictions: intelligenceStore.listPredictions({ limit: 8 }).predictions,
    });
  }

  if (action === 'sweep') return triggerSweepAction(req, res, 'action:sweep');

  auditTerminalAction(req, action || 'unknown', 'rejected', 'unknown_action');
  return res.status(400).json({ ok: false, error: 'Unknown action', allowed: ['status', 'brief', 'memory', 'sweep'], actions: ['status', 'brief', 'memory', 'sweep'] });
});

// API: available locales
app.get('/api/locales', (req, res) => {
  res.json({
    current: currentLanguage,
    supported: getSupportedLocales(),
  });
});

// SSE: live updates
app.get('/events', (req, res) => {
  res.writeHead(200, {
    'Content-Type': 'text/event-stream',
    'Cache-Control': 'no-cache',
    'Connection': 'keep-alive',
    'Access-Control-Allow-Origin': '*',
    'X-Accel-Buffering': 'no',
  });
  res.write('retry: 10000\n');
  res.write('data: {"type":"connected"}\n\n');
  const heartbeatMs = Math.max(5000, config.sseHeartbeatIntervalMs || 25000);
  const heartbeat = setInterval(() => {
    try {
      res.write(`: heartbeat ${new Date().toISOString()}\n\n`);
    } catch {
      clearInterval(heartbeat);
      sseClients.delete(res);
    }
  }, heartbeatMs);
  sseClients.add(res);
  req.on('close', () => {
    clearInterval(heartbeat);
    sseClients.delete(res);
  });
});

function broadcast(data) {
  const msg = `data: ${JSON.stringify(data)}\n\n`;
  for (const client of sseClients) {
    try { client.write(msg); } catch { sseClients.delete(client); }
  }
}

function requestIp(req) {
  return req.ip || req.socket?.remoteAddress || 'unknown';
}

function isLocalRequest(req) {
  const remote = requestIp(req);
  return remote === '::1'
    || remote === '127.0.0.1'
    || remote === '::ffff:127.0.0.1'
    || remote.startsWith('127.')
    || remote === 'localhost';
}

function sameOriginPost(req) {
  const origin = req.get('origin');
  if (!origin) return true;
  try {
    const originUrl = new URL(origin);
    const host = req.get('host');
    return host && originUrl.host === host;
  } catch {
    return false;
  }
}

function actionToken(req) {
  return req.get('x-crucix-token') || req.body?.token || null;
}

function auditTerminalAction(req, action, outcome, detail = null) {
  const suffix = detail ? ` detail=${detail}` : '';
  console.log(`[Crucix][audit] terminal_action action=${action || 'unknown'} outcome=${outcome} ip=${requestIp(req)}${suffix}`);
}

function rateLimitTerminalAction(req, action) {
  const now = Date.now();
  const windowMs = Math.max(1000, config.terminalActionRateLimitWindowMs || 60_000);
  const max = Math.max(1, config.terminalActionRateLimitMax || 10);
  const key = `${requestIp(req)}:${action}`;
  const bucket = terminalActionBuckets.get(key);
  if (!bucket || now > bucket.resetAt) {
    terminalActionBuckets.set(key, { count: 1, resetAt: now + windowMs });
    return { ok: true };
  }
  bucket.count += 1;
  if (bucket.count > max) {
    return { ok: false, retryAfterSeconds: Math.ceil((bucket.resetAt - now) / 1000) };
  }
  return { ok: true };
}

function authorizeTerminalAction(req, res, action) {
  const rate = rateLimitTerminalAction(req, action);
  if (!rate.ok) {
    auditTerminalAction(req, action, 'rejected', 'rate_limited');
    res.set('Retry-After', String(rate.retryAfterSeconds));
    res.status(429).json({ error: 'Too many terminal actions', retryAfterSeconds: rate.retryAfterSeconds });
    return { ok: false };
  }

  if (!sameOriginPost(req)) {
    auditTerminalAction(req, action, 'rejected', 'csrf_origin');
    res.status(403).json({ error: 'Origin mismatch' });
    return { ok: false };
  }

  const local = isLocalRequest(req);
  const token = actionToken(req);
  if (!config.terminalActionsEnabled) {
    auditTerminalAction(req, action, 'rejected', 'disabled');
    res.status(403).json({ error: 'Terminal actions are disabled' });
    return { ok: false };
  }

  if (config.sweepToken) {
    if (token !== config.sweepToken) {
      auditTerminalAction(req, action, 'rejected', 'invalid_token');
      res.status(401).json({ error: 'Invalid terminal action token' });
      return { ok: false };
    }
    return { ok: true };
  }

  if (!local) {
    auditTerminalAction(req, action, 'rejected', 'missing_token');
    res.status(403).json({ error: 'Terminal actions are local-only unless SWEEP_TOKEN is set' });
    return { ok: false };
  }

  return { ok: true };
}

function triggerSweepAction(req, res, auditAction) {
  if (sweepInProgress) {
    auditTerminalAction(req, auditAction, 'rejected', 'already_running');
    return res.status(409).json({ ok: true, status: 'already_running', sweepStartedAt });
  }
  auditTerminalAction(req, auditAction, 'accepted');
  runSweepCycle().catch(err => console.error('[Crucix] API-triggered sweep failed:', err.message));
  return res.status(202).json({ ok: true, status: 'accepted' });
}

function dataAgeMs() {
  const ts = currentData?.meta?.timestamp || lastSuccessfulSweepTime || lastSweepTime;
  const ms = ts ? Date.now() - new Date(ts).getTime() : null;
  return Number.isFinite(ms) ? ms : null;
}

function getLLMStatus() {
  if (!config.llm.provider) return { state: 'disabled' };
  if (!llmProvider) return { state: 'misconfigured', provider: config.llm.provider };
  return typeof llmProvider.status === 'object'
    ? llmProvider.status
    : { state: llmProvider.isConfigured ? 'configured' : 'misconfigured', provider: llmProvider.name, model: llmProvider.model };
}

function buildHealth() {
  const ageMs = dataAgeMs();
  const stale = ageMs != null && ageMs > config.staleDataMaxAgeMinutes * 60 * 1000;
  const sourcesFailed = currentData?.meta?.sourcesFailed || 0;
  const sourcesDegraded = currentData?.meta?.sourcesDegraded || 0;
  const status = lastSweepError
    ? 'error'
    : !currentData
      ? 'starting'
      : stale
        ? 'stale'
        : (sourcesFailed > 0 || sourcesDegraded > 0)
          ? 'degraded'
          : 'healthy';
  return {
    status,
    uptime: Math.floor((Date.now() - startTime) / 1000),
    lastSweep: lastSweepTime,
    lastSuccessfulSweep: lastSuccessfulSweepTime,
    nextSweep: lastSweepTime
      ? new Date(new Date(lastSweepTime).getTime() + config.refreshIntervalMinutes * 60000).toISOString()
      : null,
    dataAgeSeconds: ageMs == null ? null : Math.floor(ageMs / 1000),
    stale,
    bootstrapDataLoaded,
    sweepInProgress,
    sweepStartedAt,
    lastSweepError,
    sourcesOk: currentData?.meta?.sourcesOk || 0,
    sourcesDegraded,
    sourcesFailed,
    sourceHealth: currentData?.sourceHealth || currentData?.health || [],
    llm: getLLMStatus(),
    telegramEnabled: !!(config.telegram.botToken && config.telegram.chatId),
    telegramAiChat: {
      enabled: Boolean(config.telegram.aiChatEnabled && telegramChatAssistant.isConfigured),
      historyMessages: config.telegram.aiHistoryMessages,
      maxInputChars: config.telegram.aiMaxInputChars,
    },
    telegramAgent: {
      enabled: Boolean(config.telegram.agentEnabled && terminalAgent.isConfigured),
      tools: terminalAgent.listTools().length,
      maxSteps: config.telegram.agentMaxSteps,
      proactive: Boolean(config.telegram.agentEnabled && config.telegram.agentProactiveEnabled),
    },
    discordEnabled: !!(config.discord?.botToken || config.discord?.webhookUrl),
    terminalActionsEnabled: config.terminalActionsEnabled,
    terminalActionsTokenRequired: !!config.sweepToken,
    refreshIntervalMinutes: config.refreshIntervalMinutes,
    language: currentLanguage,
    memory: intelligenceStore.status(),
  };
}

async function notifyIfDataStale(context = 'scheduled sweep') {
  const health = buildHealth();
  const decision = shouldSendStaleAlert(health, staleAlertState, {
    cooldownMs: config.staleAlertCooldownMinutes * 60 * 1000,
  });
  if (!decision.send) return false;

  const dashboardUrl = config.dashboardUrl || `http://localhost:${config.port}`;
  const message = formatStaleAlert(health, { dashboardUrl, context });
  const sends = [];
  if (telegramAlerter.isConfigured) sends.push(telegramAlerter.sendMessage(message));
  if (discordAlerter.isConfigured) sends.push(discordAlerter.sendAlert(message));

  if (sends.length === 0) {
    console.warn('[Crucix] Data is stale but no operator alert channel is configured');
    return false;
  }

  const results = await Promise.allSettled(sends);
  const sent = results.some(r => r.status === 'fulfilled' && (r.value === true || r.value?.ok === true));
  if (sent) console.warn('[Crucix] Operator stale-data alert sent');
  else console.warn('[Crucix] Operator stale-data alert attempted but no channel accepted it');
  return sent;
}

function buildBrief(data) {
  const verbosity = config.telegram.briefVerbosity || 'standard';
  const delta = memory.getLastDelta();
  const sourceRows = (data.sourceHealth || data.health || []).slice(0, verbosity === 'audit' ? 12 : 6);
  const degraded = sourceRows.filter(s => s.status && s.status !== 'ok');
  const evidence = [
    ...(data.newsFeed || []).filter(n => n.url).slice(0, 4),
    ...(data.news || []).filter(n => n.url).slice(0, 4),
  ].slice(0, verbosity === 'compact' ? 3 : 6);
  const ideas = (data.ideas || []).slice(0, verbosity === 'compact' ? 2 : 4);
  const vix = data.fred?.find(f => f.id === 'VIXCLS');
  const id = `evt-${Buffer.from(`${data.meta?.timestamp || Date.now()}-${delta?.summary?.direction || 'mixed'}`).toString('base64url').slice(0, 10)}`;

  const lines = [
    '*CRUCIX BRIEF*',
    `_${new Date().toISOString().replace('T', ' ').substring(0, 19)} UTC_`,
    `Event ID: \`${id}\``,
    '',
    `Direction: *${(delta?.summary?.direction || 'mixed').toUpperCase()}* | Changes: ${delta?.summary?.totalChanges || 0} | Critical: ${delta?.summary?.criticalChanges || 0}`,
    `VIX: ${vix?.value || '--'} | WTI: $${data.energy?.wti || '--'} | Gold: $${data.metals?.gold || '--'} | Sources: ${data.meta?.sourcesOk || 0}/${data.meta?.sourcesQueried || 0} OK`,
    '',
    '*Source Integrity*',
    degraded.length
      ? degraded.map(s => `- ${s.name || s.n}: ${s.status || 'degraded'}${s.error || s.message ? ` (${String(s.error || s.message).slice(0, 80)})` : ''}`).join('\n')
      : '- Strong: no degraded source in the sampled health set',
    '',
    '*Top Evidence*',
    evidence.length
      ? evidence.map((n, idx) => `${idx + 1}. ${n.source || 'source'}: ${n.headline || n.title || 'link'}\n${n.url}`).join('\n')
      : '- No direct links available in the current sweep',
  ];

  if (ideas.length) {
    lines.push('', '*Why This Matters*');
    for (const idea of ideas) lines.push(`- ${idea.title}: ${(idea.rationale || idea.text || '').slice(0, 140)}`);
  }
  const scenarioChanges = data.scenarios?.changed || [];
  if (scenarioChanges.length) {
    lines.push('', '*Scenario Watchlist*');
    for (const scenario of scenarioChanges.slice(0, 4)) {
      lines.push(`- ${scenario.name}: ${scenario.state.toUpperCase()} (${scenario.confidence}% confidence)`);
    }
  }
  lines.push('', '*What To Do Next*', '- Open the dashboard, verify the evidence links, and compare source health before acting.');
  return lines.join('\n');
}

// === Sweep Cycle ===
async function runSweepCycle() {
  if (sweepInProgress) {
    console.log('[Crucix] Sweep already in progress, skipping');
    return;
  }

  sweepInProgress = true;
  sweepStartedAt = new Date().toISOString();
  broadcast({ type: 'sweep_start', timestamp: sweepStartedAt });
  console.log(`\n${'='.repeat(60)}`);
  console.log(`[Crucix] Starting sweep at ${new Date().toLocaleTimeString()}`);
  console.log(`${'='.repeat(60)}`);

  try {
    lastSweepError = null;
    // 1. Run the full briefing sweep
    const rawData = await fullBriefing();

    // 2. Save to runs/latest.json
    writeFileSync(join(RUNS_DIR, 'latest.json'), JSON.stringify(rawData, null, 2));
    lastSweepTime = new Date().toISOString();

    // 3. Synthesize into dashboard format
    console.log('[Crucix] Synthesizing dashboard data...');
    const synthesized = await synthesize(rawData);
    synthesized.meta = {
      ...synthesized.meta,
      generatedAt: new Date().toISOString(),
      stale: false,
      bootstrap: false,
    };
    synthesized.sourceHealth = Object.entries(rawData.timing || {}).map(([name, info]) => ({
      name,
      status: info.status || 'unknown',
      ms: info.ms || 0,
      error: info.error || rawData.errors?.find(e => e.name === name)?.error || null,
    }));

    // 4. Delta computation + memory
    const delta = memory.addRun(synthesized);
    synthesized.delta = delta;
    synthesized.scenarios = evaluateScenarios(synthesized, delta, RUNS_DIR);

    // 5. LLM-powered trade ideas (LLM-only feature) — isolated so failures don't kill sweep
    if (llmProvider?.isConfigured) {
      try {
        console.log('[Crucix] Generating LLM trade ideas...');
        const previousIdeas = memory.getLastRun()?.ideas || [];
        const llmIdeas = await generateLLMIdeas(llmProvider, synthesized, delta, previousIdeas);
        if (llmIdeas) {
          synthesized.ideas = llmIdeas;
          synthesized.ideasSource = 'llm';
          console.log(`[Crucix] LLM generated ${llmIdeas.length} ideas`);
        } else {
          synthesized.ideas = [];
          synthesized.ideasSource = 'llm-failed';
        }
      } catch (llmErr) {
        console.error('[Crucix] LLM ideas failed (non-fatal):', llmErr.message);
        synthesized.ideas = [];
        synthesized.ideasSource = 'llm-failed';
      }
    } else {
      synthesized.ideas = [];
      synthesized.ideasSource = 'disabled';
    }

    // 6. Alert evaluation — Telegram + Discord (LLM with rule-based fallback, multi-tier, semantic dedup)
    if (delta?.summary?.totalChanges > 0) {
      if (telegramAlerter.isConfigured) {
        if (config.telegram.agentEnabled && config.telegram.agentProactiveEnabled && shouldRunProactiveAgent(delta)) {
          runProactiveAgent(synthesized, delta).catch(err => {
            console.error('[Agent] Proactive analysis failed, using rule fallback:', err.message);
            telegramAlerter.evaluateAndAlert(null, delta, memory).catch(fallbackError => {
              console.error('[Crucix] Telegram alert fallback error:', fallbackError.message);
            });
          });
        } else {
          telegramAlerter.evaluateAndAlert(llmProvider, delta, memory).catch(err => {
            console.error('[Crucix] Telegram alert error:', err.message);
          });
        }
      }
      if (discordAlerter.isConfigured) {
        discordAlerter.evaluateAndAlert(llmProvider, delta, memory).catch(err => {
          console.error('[Crucix] Discord alert error:', err.message);
        });
      }
    }

    // Prune old alerted signals
    memory.pruneAlertedSignals();

    currentData = synthesized;
    lastSuccessfulSweepTime = lastSweepTime;
    intelligenceStore.recordRun(currentData, delta);

    // 6. Push to all connected browsers
    broadcast({ type: 'update', data: currentData });

    console.log(`[Crucix] Sweep complete — ${currentData.meta.sourcesOk}/${currentData.meta.sourcesQueried} sources OK`);
    console.log(`[Crucix] ${currentData.ideas.length} ideas (${synthesized.ideasSource}) | ${currentData.news.length} news | ${currentData.newsFeed.length} feed items`);
    if (delta?.summary) console.log(`[Crucix] Delta: ${delta.summary.totalChanges} changes, ${delta.summary.criticalChanges} critical, direction: ${delta.summary.direction}`);
    console.log(`[Crucix] Next sweep at ${new Date(Date.now() + config.refreshIntervalMinutes * 60000).toLocaleTimeString()}`);

  } catch (err) {
    console.error('[Crucix] Sweep failed:', err.message);
    lastSweepError = err.message;
    broadcast({ type: 'sweep_error', error: err.message });
  } finally {
    sweepInProgress = false;
    await notifyIfDataStale(lastSweepError ? 'failed sweep' : 'completed sweep').catch(err => {
      console.error('[Crucix] Stale-data operator alert failed:', err.message);
    });
  }
}

function shouldRunProactiveAgent(delta) {
  return (delta?.summary?.criticalChanges || 0) > 0
    || (delta?.summary?.totalChanges || 0) >= config.telegram.agentProactiveMinChanges;
}

async function runProactiveAgent(data, delta) {
  if (telegramAlerter.getMuteStatus().muted) return false;
  const prompt = `Evaluate the latest sweep for a proactive operator notification. Cross-check material changes with source health, evidence, scenarios, memory, and predictions as needed. Do not call mutating tools. Delta summary: ${JSON.stringify(delta?.summary || {})}`;
  const result = await terminalAgent.analyzeProactively(prompt, {
    context: buildTelegramChatContext(data, buildHealth()),
    runtime: { data, delta },
  });
  if (result.pendingAction) return false;
  if (!result.notify) {
    return telegramAlerter.evaluateAndAlert(null, delta, memory);
  }
  const evidence = result.evidence?.length ? `\nEvidence:\n${result.evidence.map(item => `- ${item}`).join('\n')}` : '';
  const tools = [...new Set((result.trace || []).filter(item => item.status === 'ok').map(item => item.tool))];
  const trace = tools.length ? `\nTools: ${tools.join(', ')}` : '';
  const sent = await telegramAlerter.sendMessage(`[AGENT ${String(result.priority || 'routine').toUpperCase()}]\n${result.answer}${evidence}${trace}`, { parseMode: null });
  return sent.ok;
}

// === Startup ===
async function start() {
  const port = config.port;

  console.log(`
  ╔══════════════════════════════════════════════╗
  ║           CRUCIX INTELLIGENCE ENGINE         ║
  ║          Local Palantir · 26 Sources         ║
  ╠══════════════════════════════════════════════╣
  ║  Dashboard:  http://localhost:${port}${' '.repeat(14 - String(port).length)}║
  ║  Health:     http://localhost:${port}/api/health${' '.repeat(4 - String(port).length)}║
  ║  Refresh:    Every ${config.refreshIntervalMinutes} min${' '.repeat(20 - String(config.refreshIntervalMinutes).length)}║
  ║  LLM:        ${(config.llm.provider || 'disabled').padEnd(31)}║
  ║  Telegram:   ${config.telegram.botToken ? 'enabled' : 'disabled'}${' '.repeat(config.telegram.botToken ? 24 : 23)}║
  ║  Discord:    ${config.discord?.botToken ? 'enabled' : config.discord?.webhookUrl ? 'webhook only' : 'disabled'}${' '.repeat(config.discord?.botToken ? 24 : config.discord?.webhookUrl ? 20 : 23)}║
  ╚══════════════════════════════════════════════╝
  `);

  const server = app.listen(port);

  server.on('error', (err) => {
    if (err.code === 'EADDRINUSE') {
      console.error(`\n[Crucix] FATAL: Port ${port} is already in use!`);
      console.error(`[Crucix] A previous Crucix instance may still be running.`);
      console.error(`[Crucix] Fix:  taskkill /F /IM node.exe   (Windows)`);
      console.error(`[Crucix]       kill $(lsof -ti:${port})   (macOS/Linux)`);
      console.error(`[Crucix] Or change PORT in .env\n`);
    } else {
      console.error(`[Crucix] Server error:`, err.stack || err.message);
    }
    process.exit(1);
  });

  server.on('listening', async () => {
    console.log(`[Crucix] Server running on http://localhost:${port}`);

    if (config.autoOpenBrowser) {
      const openCmd = process.platform === 'win32' ? 'cmd /c start ""' :
                      process.platform === 'darwin' ? 'open' : 'xdg-open';
      exec(`${openCmd} "http://localhost:${port}"`, (err) => {
        if (err) console.log('[Crucix] Could not auto-open browser:', err.message);
      });
    } else {
      console.log('[Crucix] Auto-open browser disabled (AUTO_OPEN_BROWSER=false)');
    }

    // Try to load existing data first for instant display (await so dashboard shows immediately)
    try {
      const existing = JSON.parse(readFileSync(join(RUNS_DIR, 'latest.json'), 'utf8'));
      const data = await synthesize(existing);
      data.meta = {
        ...data.meta,
        generatedAt: new Date().toISOString(),
        stale: true,
        bootstrap: true,
        bootstrapReason: 'Loaded from runs/latest.json while initial sweep starts',
      };
      data.sourceHealth = Object.entries(existing.timing || {}).map(([name, info]) => ({
        name,
        status: info.status || 'bootstrap',
        ms: info.ms || 0,
        error: info.error || null,
      }));
      currentData = data;
      bootstrapDataLoaded = true;
      console.log('[Crucix] Loaded existing data from runs/latest.json as stale bootstrap; initial sweep will refresh it');
      broadcast({ type: 'update', data: currentData });
    } catch {
      console.log('[Crucix] No existing data found — first sweep required');
    }

    // Run first sweep (refreshes data in background)
    console.log('[Crucix] Running initial sweep...');
    runSweepCycle().catch(err => {
      console.error('[Crucix] Initial sweep failed:', err.message || err);
    });

    // Schedule recurring sweeps
    setInterval(runSweepCycle, config.refreshIntervalMinutes * 60 * 1000);
  });
}

// Graceful error handling — log full stack traces for diagnosis
process.on('unhandledRejection', (err) => {
  console.error('[Crucix] Unhandled rejection:', err?.stack || err?.message || err);
});
process.on('uncaughtException', (err) => {
  console.error('[Crucix] Uncaught exception:', err?.stack || err?.message || err);
});

start().catch(err => {
  console.error('[Crucix] FATAL — Server failed to start:', err?.stack || err?.message || err);
  process.exit(1);
});