Merge production readiness tooling into test

deploy/ai-health.mjs

+import { createRequire } from 'node:module';
+import { resolve } from 'node:path';
+import { fileURLToPath } from 'node:url';
+
+export function queueHealthy(stats) {
+  return (
+    Number.isFinite(stats.oldestSeconds) &&
+    stats.oldestSeconds <= 300 &&
+    Number.isInteger(stats.expiredLeases ?? 0) &&
+    (stats.expiredLeases ?? 0) === 0
+  );
+}
+export async function probeJson(url, expected, headers = {}) {
+  const response = await fetch(url, { headers, signal: AbortSignal.timeout(5000) });
+  if (!response.ok) throw Error('http');
+  const body = await response.json();
+  if (!expected(body)) throw Error('body');
+  return body;
+}
+export async function main() {
+  const checks = {};
+  const run = async (name, fn) => {
+    try {
+      checks[name] = { ok: true, ...(await fn()) };
+    } catch {
+      checks[name] = { ok: false };
+    }
+  };
+  await run('ai', async () => {
+    await probeJson(
+      'http://127.0.0.1:3104/health/ready',
+      (b) => b.status === 'ok' && b.db === 'up',
+    );
+  });
+  await run('web', async () => {
+    const r = await fetch('http://ai-web:3103/login', { signal: AbortSignal.timeout(5000) });
+    if (!r.ok) throw Error('web');
+  });
+  await run('pac', async () => {
+    await probeJson(process.env.AI_PAC_BASE_URL + '/health', (b) => b.status === 'ok');
+  });
+  await run('ingress', async () => {
+    const stats = await probeJson(
+      'http://channel-ingress:3106/status',
+      (b) => Number.isInteger(b.pending) && Number.isFinite(b.oldestSeconds),
+      { Authorization: 'Bearer ' + process.env.AI_INGRESS_TOKEN },
+    );
+    return {
+      ok: queueHealthy(stats),
+      pending: stats.pending,
+      oldestSeconds: Math.round(stats.oldestSeconds),
+      retrying: stats.retrying,
+    };
+  });
+  await run('outbox', async () => {
+    const require = createRequire(`${process.cwd()}/package.json`);
+    const { PrismaClient } = require('./generated/prisma');
+    const url = new URL(process.env.AI_DATABASE_URL);
+    url.searchParams.set('connection_limit', '1');
+    url.searchParams.set('connect_timeout', '5');
+    const db = new PrismaClient({ datasources: { db: { url: url.toString() } } });
+    try {
+      await db.$executeRawUnsafe("SET statement_timeout='5s'");
+      // 只查已有部分索引中的待处理记录，未来预约排程不算积压。
+      const [s] = await db.$queryRawUnsafe(`SELECT
+        (SELECT count(*)::int FROM app.outbox WHERE status='pending' AND scheduled_at<=now()) AS pending,
+        COALESCE((SELECT extract(epoch FROM now()-min(scheduled_at))::float8 FROM app.outbox WHERE status='pending' AND scheduled_at<=now()),0) AS "oldestSeconds",
+        (SELECT count(*)::int FROM app.outbox WHERE status='processing' AND lease_until<now()-interval '1 minute') AS "expiredLeases"`);
+      return { ...s, ok: queueHealthy(s), oldestSeconds: Math.round(s.oldestSeconds) };
+    } finally {
+      await db.$disconnect();
+    }
+  });
+  const report = {
+    event: 'ai_ops_health',
+    at: new Date().toISOString(),
+    status: Object.values(checks).every((c) => c.ok) ? 'ok' : 'degraded',
+    checks,
+  };
+  console.log(JSON.stringify(report));
+  return report.status === 'ok' ? 0 : 1;
+}
+if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url))
+  process.exitCode = await main();

deploy/check-ai-health.sh

+#!/usr/bin/env bash
+# 应用机单次业务巡检；由 cron/运维监控调用，不依赖开发电脑或 Codex。
+# 不自动重启、不重试消息；退出 0=正常，1=异常。所有输出均不含凭据、患者信息。
+set -euo pipefail
+cd "$(dirname "${BASH_SOURCE[0]}")/.."
+command -v timeout >/dev/null || { echo '{"event":"ai_ops_health","status":"degraded","reason":"timeout_command_missing"}'; exit 1; }
+if ! output=$(timeout 45s docker exec -i pac-ai-ai-service-1 node --input-type=module < <(cat deploy/ai-health.mjs; echo 'process.exitCode=await main();') 2>/dev/null); then
+  if [[ "$output" == *'"event":"ai_ops_health"'* ]]; then printf '%s\n' "$output";
+  else echo '{"event":"ai_ops_health","status":"degraded","reason":"probe_unavailable"}'; fi
+  exit 1
+fi
+printf '%s\n' "$output"

deploy/deploy-ai.sh

@@ -24,6 +24,7 @@ main() {
   docker network inspect pac_default >/dev/null
   export AI_IMAGE_TAG="$(git rev-parse --short HEAD)-$(date -u +%Y%m%dT%H%M%SZ)"
   local compose=(docker compose -p pac-ai -f docker-compose.ai.yml)
+  [[ "${COMPOSE_MANAGED:-0}" != 1 ]] || compose+=(-f docker-compose.ai.managed.yml)
   "${compose[@]}" config --quiet
   # 先构建、预检；失败时旧容器继续运行。串行构建，限制对同机 PAC 的影响。
   "${compose[@]}" build ai-service
@@ -51,7 +52,7 @@ main() {
     actual=$(docker inspect "$id" --format '{{.Image}}')
     [[ "$actual" == "$expected" ]] || { echo "$service 镜像校验失败"; return 1; }
   done
-  "${compose[@]}" exec -T ai-service node -e "fetch(process.env.AI_PAC_BASE_URL+'/health',{signal:AbortSignal.timeout(5000)}).then(async r=>{const v=await r.json();if(!r.ok||v.status!=='ok')process.exit(1)}).catch(()=>process.exit(1))"
+  "${compose[@]}" exec -T --interactive=false ai-service node -e "fetch(process.env.AI_PAC_BASE_URL+'/health',{signal:AbortSignal.timeout(5000)}).then(async r=>{const v=await r.json();if(!r.ok||v.status!=='ok')process.exit(1)}).catch(()=>process.exit(1))"
   printf '%s\n' "$AI_IMAGE_TAG" > "$state/current-image-tag"
   echo "FridayAI 部署完成：${AI_IMAGE_TAG}；Web 3103 / API 3104；PAC 未重启"
 }

deploy/deploy-ingress.sh

@@ -2,7 +2,9 @@
 # 独立发布中转；日常 AI 部署不调用这个脚本。
 set -euo pipefail
 target=${1:-}
-[[ "$target" == staging || "$target" == production ]] || { echo '用法：bash deploy/deploy-ingress.sh staging|production'; exit 1; }
+option=${2:-}
+[[ -z "$option" || "$option" == --check ]] || { echo '未知选项'; exit 1; }
+[[ "$target" == staging || "$target" == production ]] || { echo '用法：bash deploy/deploy-ingress.sh staging|production [--check]'; exit 1; }
 cd "$(dirname "${BASH_SOURCE[0]}")/.."
 branch=test
 [[ "$target" == production ]] && branch=main
@@ -15,9 +17,11 @@ untracked=$(git ls-files --others --exclude-standard | grep -vE '(^|/)\.env([.-]
 docker network inspect pac_default >/dev/null
 export INGRESS_IMAGE_TAG="$(git rev-parse --short HEAD)-$(date -u +%Y%m%dT%H%M%SZ)"
 compose=(docker compose -p pac-ingress -f docker-compose.ingress.yml)
+[[ "${COMPOSE_MANAGED:-0}" != 1 ]] || compose+=(-f docker-compose.ingress.managed.yml)
 "${compose[@]}" config --quiet
 "${compose[@]}" build channel-ingress
-"${compose[@]}" run --rm --no-deps -T --interactive=false channel-ingress node -e "try{require('./dist/config').readConfig()}catch{console.error('中转配置无效');process.exit(1)}"
+"${compose[@]}" run --rm --no-deps -T --interactive=false channel-ingress node -e "try{const c=require('./dist/config').readConfig();if('$target'==='production'&&(c.localAccounts.length||c.localToken))throw Error('production local routing')}catch{console.error('中转配置无效');process.exit(1)}"
+if [[ "$option" == --check ]]; then echo "中转构建与配置检查通过：${INGRESS_IMAGE_TAG}（未迁移或重启）"; exit 0; fi
 # 独立 schema 的幂等 DDL；不停止 AI，不执行 AI 的迁移。
 "${compose[@]}" run --rm --no-deps -T --interactive=false ingress-migrate
 # 切换配置前先检查积压归属；失败时旧中转保持运行。
@@ -29,4 +33,4 @@ actual=$(docker inspect "$id" --format '{{.Image}}')
 [[ "$expected" == "$actual" ]] || { echo '中转镜像校验失败'; exit 1; }
 mkdir -p "$(git rev-parse --git-common-dir)/ingress-deploy"
 printf '%s\n' "$INGRESS_IMAGE_TAG" > "$(git rev-parse --git-common-dir)/ingress-deploy/current-image-tag"
-echo "中转部署完成：$INGRESS_IMAGE_TAG；3106；未修改 Nginx、未停止旧调试中转"
+echo "中转部署完成：${INGRESS_IMAGE_TAG}；3106；未修改 Nginx、未停止旧调试中转"

deploy/health-watch.sh

@@ -21,6 +21,8 @@
 #
 # 环境变量:
 #   WATCH_URL             被监控服务的 base(默认 https://pac.friday.tech),脚本自己拼 /health
+#   WATCH_PATH            默认 /health；FridayAI 使用 /health/ready
+#   WATCH_REQUIRE_DB      FridayAI 设 1，同时要求 db=up
 #   WATCH_NAME            告警里显示的名字 + 状态文件名(默认 pac-prod)
 #   ALERT_WEBHOOK_URL     企微群机器人 webhook。⛔ 不设 = 只写日志不推送(等于没装)
 #   WATCH_FAIL_THRESHOLD  连续失败几次才告警(默认 3)。× cron 间隔 = 真实容忍时长
@@ -52,6 +54,8 @@ set -uo pipefail   # ⛔ 不加 -e:探测失败是正常业务路径,不是脚
 
 WATCH_URL="${WATCH_URL:-https://pac.friday.tech}"
 WATCH_NAME="${WATCH_NAME:-pac-prod}"
+WATCH_PATH="${WATCH_PATH:-/health}"
+WATCH_REQUIRE_DB="${WATCH_REQUIRE_DB:-0}"
 WATCH_FAIL_THRESHOLD="${WATCH_FAIL_THRESHOLD:-3}"
 WATCH_REPEAT_EVERY="${WATCH_REPEAT_EVERY:-12}"
 WATCH_TIMEOUT="${WATCH_TIMEOUT:-10}"
@@ -86,12 +90,13 @@ fi
 # 返回 0 = 健康。stdout 是给人看的诊断串(状态码 + body 截断)。
 probe() {
   local base="$1" out code body
-  out=$(curl -sS -m "$WATCH_TIMEOUT" -w $'\n%{http_code}' "$base/health" 2>&1) || out=$'\n000'
+  out=$(curl -sS -m "$WATCH_TIMEOUT" -w $'\n%{http_code}' "$base$WATCH_PATH" 2>&1) || out=$'\n000'
   code=${out##*$'\n'}
   body=${out%$'\n'*}
   body=${body//$'\n'/ }
   printf 'http=%s body=%s' "$code" "${body:0:200}"
-  [[ "$code" == "200" && "$body" == *'"status":"ok"'* ]]
+  [[ "$code" == "200" && "$body" == *'"status":"ok"'* ]] &&
+    { [[ "$WATCH_REQUIRE_DB" != 1 ]] || [[ "$body" == *'"db":"up"'* ]]; }
 }
 
 # ── 状态读写 ───────────────────────────────────────────────────────────
@@ -124,8 +129,8 @@ json_escape() {
 
 push() { # $1=icon $2=color $3=标题 $4=正文
   local content resp payload
-  content=$(printf '%s **<font color="%s">[PAC 外部探针] %s</font>**\n%s\n> 目标: %s/health\n> 探针机: %s\n> time: %s' \
-    "$1" "$2" "$3" "$4" "$WATCH_URL" "$(hostname)" "$(ts)")
+  content=$(printf '%s **<font color="%s">[PAC 外部探针] %s</font>**\n%s\n> 目标: %s\n> 探针机: %s\n> time: %s' \
+    "$1" "$2" "$3" "$4" "$WATCH_URL$WATCH_PATH" "$(hostname)" "$(ts)")
   if [[ "$WATCH_DRY_RUN" == "1" ]]; then
     printf '%s [%s] [DRY-RUN 不推送] ↓↓↓\n%s\n' "$(ts)" "$WATCH_NAME" "$content"
     return 0

deploy/prepare-pac-indexes.cjs

+#!/usr/bin/env node
+// 在 PAC 运行容器内执行；不改应用版本，不代替 Prisma 迁移登记。
+const { createRequire } = require('node:module');
+const { setTimeout: delay } = require('node:timers/promises');
+const requireApp = createRequire(`${process.cwd()}/package.json`);
+const indexes = [
+  [
+    'patient_transactions_patient_id_clinic_id_idx',
+    'patient_transactions',
+    ['patient_id', 'clinic_id'],
+  ],
+  [
+    'patient_facts_host_id_tenant_id_type_occurred_at_idx',
+    'patient_facts',
+    ['host_id', 'tenant_id', 'type', 'occurred_at'],
+  ],
+  [
+    'patient_facts_host_id_tenant_id_type_planned_for_idx',
+    'patient_facts',
+    ['host_id', 'tenant_id', 'type', 'planned_for'],
+  ],
+];
+function validIndex(row, table, columns) {
+  return (
+    row &&
+    row.indisvalid &&
+    row.indisready &&
+    row.table_name === table &&
+    row.method === 'btree' &&
+    !row.predicate &&
+    !row.expressions &&
+    !row.indisunique &&
+    JSON.stringify(row.columns) === JSON.stringify(columns)
+  );
+}
+module.exports = { indexes, validIndex };
+async function main() {
+  const apply = process.argv.includes('--apply');
+  const { PrismaClient } = requireApp('@prisma/client');
+  const url = new URL(process.env.DATABASE_URL);
+  url.searchParams.set('connection_limit', '1');
+  url.searchParams.set('application_name', 'pac-index-preparation');
+  const db = new PrismaClient({ datasources: { db: { url: url.toString() } } });
+  const observer = new PrismaClient({ datasources: { db: { url: url.toString() } } });
+  const log = (event, data = {}) =>
+    console.log(
+      JSON.stringify({ at: new Date().toISOString(), event, ...data }, (_, v) =>
+        typeof v === 'bigint' ? String(v) : v,
+      ),
+    );
+  const inspect = async (name) =>
+    (
+      await db.$queryRawUnsafe(
+        `
+    SELECT i.indisvalid,i.indisready,i.indisunique,t.relname AS table_name,a.amname AS method,
+      pg_get_expr(i.indpred,i.indrelid) AS predicate,pg_get_expr(i.indexprs,i.indrelid) AS expressions,
+      ARRAY(SELECT pg_get_indexdef(i.indexrelid,k,true) FROM generate_series(1,i.indnatts) k) AS columns,
+      pg_size_pretty(pg_relation_size(c.oid)) AS size
+    FROM pg_class c JOIN pg_namespace n ON n.oid=c.relnamespace
+    JOIN pg_index i ON i.indexrelid=c.oid JOIN pg_class t ON t.oid=i.indrelid
+    JOIN pg_am a ON a.oid=c.relam
+    WHERE n.nspname='public' AND c.relname=$1`,
+        name,
+      )
+    )[0];
+  const busy = async () => {
+    // 摄入在 ClickHouse 查询/JS 解析期间，PostgreSQL 可能完全空闲；不能只看 pg_stat_activity。
+    const [sync] = await observer.$queryRawUnsafe(
+      "SELECT EXISTS(SELECT 1 FROM sync_logs WHERE status='running' OR ended_at>now()-interval '15 minutes') AS busy",
+    );
+    if (sync.busy) return true;
+    try {
+      const response = await fetch('http://127.0.0.1:3101/health', {
+        signal: AbortSignal.timeout(1000),
+      });
+      if (!response.ok || (await response.json()).status !== 'ok') return true;
+    } catch {
+      return true;
+    }
+    const [r] = await observer.$queryRawUnsafe(`SELECT count(*)::int AS count FROM pg_stat_activity
+      WHERE datname=current_database() AND application_name <> 'pac-index-preparation'
+      AND backend_type='client backend' AND
+      ((state='active' AND query_start<now()-interval '5 seconds') OR xact_start<now()-interval '1 minute')`);
+    const builds = await observer.$queryRawUnsafe('SELECT pid FROM pg_stat_progress_create_index');
+    return r.count > 0 || builds.length > 0;
+  };
+  let locked = false;
+  let missing = 0;
+  try {
+    if (apply) {
+      [{ locked }] = await db.$queryRawUnsafe(
+        "SELECT pg_try_advisory_lock(hashtext('pac:index-preparation')) AS locked",
+      );
+      if (!locked) throw Error('已有索引准备任务，停止');
+      await db.$executeRawUnsafe("SET maintenance_work_mem='64MB'");
+      await db.$executeRawUnsafe('SET max_parallel_maintenance_workers=0');
+      await db.$executeRawUnsafe("SET lock_timeout='5s'");
+      await db.$executeRawUnsafe("SET statement_timeout='45min'");
+    }
+    const deadline = Date.now() + 6 * 60 * 60 * 1000;
+    for (const [name, table, columns] of indexes) {
+      const old = await inspect(name);
+      if (old) {
+        if (!validIndex(old, table, columns))
+          throw Error(`索引无效或定义不符：${name}；人工检查后再处理`);
+        log('index_valid', { name, size: old.size });
+        continue;
+      }
+      log('index_missing', { name });
+      missing++;
+      if (!apply) continue;
+      let idle = 0;
+      while (idle < 6) {
+        if (Date.now() > deadline) throw Error('等待空闲超过六小时，未启动后续索引');
+        idle = (await busy()) ? 0 : idle + 1;
+        log('idle_sample', { name, consecutive: idle });
+        if (idle < 6) await delay(30000);
+      }
+      const [{ pid }] = await db.$queryRawUnsafe('SELECT pg_backend_pid() AS pid');
+      log('index_start', { name, pid });
+      let done = false,
+        failure;
+      const work = db
+        .$executeRawUnsafe(
+          `CREATE INDEX CONCURRENTLY "${name}" ON "public"."${table}" (${columns.map((c) => '"' + c + '"').join(',')})`,
+        )
+        .catch((e) => {
+          failure = e;
+        })
+        .finally(() => {
+          done = true;
+        });
+      let healthFailures = 0;
+      while (!done) {
+        await Promise.race([work, delay(30000)]);
+        if (done) break;
+        const rows = await observer.$queryRawUnsafe(
+          'SELECT phase,blocks_total,blocks_done FROM pg_stat_progress_create_index WHERE pid=$1',
+          pid,
+        );
+        log('index_progress', { name, progress: rows });
+        try {
+          const r = await fetch('http://127.0.0.1:3101/health', {
+            signal: AbortSignal.timeout(5000),
+          });
+          if (!r.ok || (await r.json()).status !== 'ok') throw Error('health');
+          healthFailures = 0;
+        } catch {
+          healthFailures++;
+        }
+        if (healthFailures >= 2) {
+          await observer.$queryRawUnsafe('SELECT pg_cancel_backend($1::int)', pid);
+          log('index_cancelled', { name, reason: 'PAC health failed twice' });
+          await work;
+          throw Error('PAC 健康检查失败，已取消本任务索引；不得自动重试');
+        }
+      }
+      await work;
+      if (failure)
+        throw Error(`索引创建失败 ${name} (${failure.code || 'database error'})；检查有效性后处理`);
+      const current = await inspect(name);
+      if (!validIndex(current, table, columns)) throw Error(`索引验证失败：${name}`);
+      log('index_valid', { name, size: current.size });
+    }
+    if (!apply && missing > 0) throw Error(`缺少 ${missing} 个索引`);
+    log(apply ? 'complete' : 'check_complete');
+  } finally {
+    if (locked)
+      await db
+        .$queryRawUnsafe("SELECT pg_advisory_unlock(hashtext('pac:index-preparation'))")
+        .catch(() => {});
+    await db.$disconnect();
+    await observer.$disconnect();
+  }
+}
+if (require.main === module)
+  main().catch((e) => {
+    console.error(JSON.stringify({ event: 'failed', message: e.message }));
+    process.exitCode = 1;
+  });

deploy/production/friday-ai-database.sql

+-- 运维使用 RDS 管理账号首次执行；不要连接 pac 库安装 AI 扩展。
+-- 密码使用 DMS 安全界面或 psql \password friday_ai 设置，不放进 SQL 文件。
+CREATE ROLE friday_ai LOGIN;
+CREATE DATABASE friday_ai OWNER friday_ai TEMPLATE template0
+  ENCODING 'UTF8' LC_COLLATE 'C' LC_CTYPE 'en_US.utf8';
+ALTER DATABASE friday_ai SET timezone TO 'UTC';
+-- 切换连接到 friday_ai 后再执行以下两句。
+-- CREATE EXTENSION IF NOT EXISTS pgcrypto;
+-- CREATE EXTENSION IF NOT EXISTS btree_gist;

deploy/production/friday-ai-gateway.conf.example

+# 在现有生产网关安装。替换 AI_DOMAIN、RELAY_DOMAIN、证书路径后 nginx -t。
+# 前提：只有此网关可访问应用机 3103/3104/3106；真实来源仅一层代理。
+server {
+    listen 80;
+    server_name AI_DOMAIN RELAY_DOMAIN;
+    return 301 https://$host$request_uri;
+}
+server {
+    listen 443 ssl;
+    server_name AI_DOMAIN;
+    ssl_certificate /REQUIRED/AI/fullchain.pem;
+    ssl_certificate_key /REQUIRED/AI/privkey.pem;
+    client_max_body_size 25m;
+    proxy_http_version 1.1;
+    proxy_set_header Connection "";
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $remote_addr;
+    proxy_set_header X-Forwarded-Proto $scheme;
+    proxy_connect_timeout 10s;
+    proxy_send_timeout 300s;
+    proxy_read_timeout 3600s;
+    location /api/ {
+        proxy_pass http://172.16.4.239:3104;
+        proxy_buffering off;
+        proxy_cache off;
+    }
+    location = /health { proxy_pass http://172.16.4.239:3104; access_log off; }
+    location = /health/ready { proxy_pass http://172.16.4.239:3104; access_log off; }
+    location / { proxy_pass http://172.16.4.239:3103; }
+}
+server {
+    listen 443 ssl;
+    server_name RELAY_DOMAIN;
+    ssl_certificate /REQUIRED/RELAY/fullchain.pem;
+    ssl_certificate_key /REQUIRED/RELAY/privkey.pem;
+    location = /hook/ecom {
+        limit_except POST { deny all; }
+        client_max_body_size 20m;
+        proxy_pass http://172.16.4.239:3106;
+        proxy_set_header Host $host;
+        proxy_set_header X-Real-IP $remote_addr;
+        proxy_set_header X-Forwarded-For $remote_addr;
+        proxy_connect_timeout 2s;
+        proxy_read_timeout 8s;
+    }
+    location / { return 404; }
+}

deploy/production/friday-ai.cron

+# 安装到 /etc/cron.d/friday-ai-health；仅在 AI 和中转完成部署后启用。
+# 运维采集 JSON 日志：连续 3 次 degraded 或超过 5 分钟无新日志须告警。
+*/2 * * * * root flock -n /run/friday-ai-health.lock /bin/bash /data/pac/deploy/check-ai-health.sh >> /var/log/friday-ai-health.log 2>&1

deploy/production/friday-ai.logrotate

+/var/log/friday-ai-health.log {
+    daily
+    rotate 14
+    compress
+    missingok
+    notifempty
+    create 0600 root root
+}

deploy/tests/ai-health.test.mjs

+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { queueHealthy, probeJson } from '../ai-health.mjs';
+import { createServer } from 'node:http';
+test('过期租约或五分钟积压报告异常，缺失指标不能冒充正常', () => {
+  assert.equal(queueHealthy({ oldestSeconds: 0, expiredLeases: 0 }), true);
+  for (const x of [{ oldestSeconds: 301 }, { oldestSeconds: 0, expiredLeases: 1 }, {}])
+    assert.equal(queueHealthy(x), false);
+});
+test('健康检查识别 HTTP 200 信封错误和数据库不可用', async () => {
+  const server = createServer((req, res) => {
+    res.end(
+      req.url === '/ok'
+        ? ' {"status":"ok","db":"up"} '
+        : req.url === '/db'
+          ? ' {"status":"ok","db":"down"} '
+          : '{"code":10004,"msg":"not found"}',
+    );
+  });
+  await new Promise((r) => server.listen(0, '127.0.0.1', r));
+  try {
+    const url = `http://127.0.0.1:${server.address().port}`;
+    const expected = (b) => b.status === 'ok' && b.db === 'up';
+    await probeJson(url + '/ok', expected);
+    await assert.rejects(probeJson(url + '/db', expected));
+    await assert.rejects(probeJson(url + '/missing', expected));
+  } finally {
+    await new Promise((r) => server.close(r));
+  }
+});

deploy/tests/deploy-ai.test.mjs

@@ -7,9 +7,18 @@ import { spawnSync } from 'node:child_process';
 
 const source = resolve('deploy/deploy-ai.sh');
 // 用假 Docker 注入失败，验证真实脚本不会迁移/切换不完整版本，不触碰本地或远端容器。
-function runDeploy({ fail = '', check = false, branch = 'test', mismatch = false, piped = false } = {}) {
+function runDeploy({
+  fail = '',
+  check = false,
+  branch = 'test',
+  mismatch = false,
+  piped = false,
+  managed = false,
+  target = 'staging',
+} = {}) {
   const root = mkdtempSync(join(tmpdir(), 'ai-deploy-flow-'));
-  const bin = join(root, 'bin'), log = join(root, 'commands.log');
+  const bin = join(root, 'bin'),
+    log = join(root, 'commands.log');
   for (const dir of [bin, join(root, '.git'), join(root, 'deploy')]) mkdirSync(dir);
   copyFileSync(source, join(root, 'deploy/deploy-ai.sh'));
   writeFileSync(log, '');
@@ -21,7 +30,7 @@ function runDeploy({ fail = '', check = false, branch = 'test', mismatch = false
       'rev-parse --git-common-dir') echo .git ;;
     esac`,
     flock: 'exit 0',
-    docker: String.raw`if [[ "$*" == *' run '* && "$*" != *'--interactive=false'* ]]; then cat >/dev/null; fi
+    docker: String.raw`if [[ ( "$*" == *' run '* || "$*" == *' exec '* ) && "$*" != *'--interactive=false'* ]]; then cat >/dev/null; fi
       if [[ -n "$DEPLOY_TEST_FAIL" && "$*" == *"$DEPLOY_TEST_FAIL"* ]]; then exit 43; fi
       case "$*" in
         *' ps -q ai-service') echo test-service ;;
@@ -34,16 +43,36 @@ function runDeploy({ fail = '', check = false, branch = 'test', mismatch = false
   };
   try {
     for (const [name, body] of Object.entries(stubs))
-      writeFileSync(join(bin, name), '#!/usr/bin/env bash\nset -eu\n' + common + body + '\n', { mode: 0o755 });
-    const script = join(root, 'deploy/deploy-ai.sh');
-    const result = spawnSync('bash', piped ? ['-s'] : [script, 'staging', ...(check ? ['--check'] : [])], {
-      ...(piped ? { input: `bash '${script}' staging --check\nprintf 'FOLLOWUP_REACHED\\n'\n` } : {}),
-      encoding: 'utf8', timeout: 10000,
-      env: { ...process.env, PATH: `${bin}:${process.env.PATH}`, DEPLOY_TEST_LOG: log,
-        DEPLOY_TEST_FAIL: fail, DEPLOY_TEST_BRANCH: branch, DEPLOY_TEST_MISMATCH: mismatch ? '1' : '0' },
+      writeFileSync(join(bin, name), '#!/usr/bin/env bash\nset -eu\n' + common + body + '\n', {
+        mode: 0o755,
       });
+    const script = join(root, 'deploy/deploy-ai.sh');
+    const result = spawnSync(
+      'bash',
+      piped ? ['-s'] : [script, target, ...(check ? ['--check'] : [])],
+      {
+        ...(piped
+          ? {
+              input: `bash '${script}' ${target} ${check ? '--check' : '--no-pull'}\nprintf 'FOLLOWUP_REACHED\\n'\n`,
+            }
+          : {}),
+        encoding: 'utf8',
+        timeout: 10000,
+        env: {
+          ...process.env,
+          PATH: `${bin}:${process.env.PATH}`,
+          DEPLOY_TEST_LOG: log,
+          COMPOSE_MANAGED: managed ? '1' : '0',
+          DEPLOY_TEST_FAIL: fail,
+          DEPLOY_TEST_BRANCH: branch,
+          DEPLOY_TEST_MISMATCH: mismatch ? '1' : '0',
+        },
+      },
+    );
     return { result, commands: readFileSync(log, 'utf8').trim().split('\n') };
-  } finally { rmSync(root, { recursive: true, force: true }); }
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
 }
 
 test('Docker 构建和预检成功后才停 AI、迁移、切换并校验镜像', () => {
@@ -52,9 +81,15 @@ test('Docker 构建和预检成功后才停 AI、迁移、切换并校验镜像'
   const at = (s) => commands.findIndex((c) => c.includes(s));
   assert.ok(at('build ai-web') < at('run --rm --no-deps -T --interactive=false ai-preflight'));
   assert.ok(at('ai-preflight') < at('stop ai-service ai-web'));
-  assert.ok(at('stop ai-service ai-web') < at('run --rm --no-deps -T --interactive=false ai-migrate'));
+  assert.ok(
+    at('stop ai-service ai-web') < at('run --rm --no-deps -T --interactive=false ai-migrate'),
+  );
   assert.ok(at('prisma migrate status') < at('up -d --no-deps --force-recreate --wait'));
-  assert.ok(commands.filter((c) => c.startsWith('docker compose')).every((c) => c.includes('-p pac-ai -f docker-compose.ai.yml')));
+  assert.ok(
+    commands
+      .filter((c) => c.startsWith('docker compose'))
+      .every((c) => c.includes('-p pac-ai -f docker-compose.ai.yml')),
+  );
   assert.ok(!commands.some((c) => /systemctl|pnpm install|stop pac-|up.*pac-service/.test(c)));
 });
 for (const failure of ['build ai-web', 'run --rm --no-deps -T --interactive=false ai-preflight']) {
@@ -65,7 +100,9 @@ for (const failure of ['build ai-web', 'run --rm --no-deps -T --interactive=fals
   });
 }
 test('迁移失败不启动新旧应用', () => {
-  const { result, commands } = runDeploy({ fail: 'run --rm --no-deps -T --interactive=false ai-migrate' });
+  const { result, commands } = runDeploy({
+    fail: 'run --rm --no-deps -T --interactive=false ai-migrate',
+  });
   assert.equal(result.status, 43);
   assert.ok(!commands.some((c) => c.includes('up -d')));
 });
@@ -85,7 +122,22 @@ test('运行容器镜像与构建镜像不一致时拒绝报告成功', () => {
 });
 
 test('SSH 管道执行预检后仍执行后续命令，不被 Compose 吞掉标准输入', () => {
+  const { result } = runDeploy({ piped: true, check: true });
+  assert.equal(result.status, 0, result.stderr);
+  assert.match(result.stdout, /FOLLOWUP_REACHED/);
+});
+
+test('完整发布也不吞后续 SSH 命令', () => {
   const { result } = runDeploy({ piped: true });
   assert.equal(result.status, 0, result.stderr);
   assert.match(result.stdout, /FOLLOWUP_REACHED/);
 });
+test('生产托管模式每个 Compose 操作均加载生产端口覆盖', () => {
+  const { result, commands } = runDeploy({ target: 'production', branch: 'main', managed: true });
+  assert.equal(result.status, 0, result.stderr);
+  assert.ok(
+    commands
+      .filter((c) => c.startsWith('docker compose'))
+      .every((c) => c.includes('-f docker-compose.ai.managed.yml')),
+  );
+});

deploy/tests/deploy-ingress.test.mjs

+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, copyFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { spawnSync } from 'node:child_process';
+function run({ check = false, fail = '' } = {}) {
+  const root = mkdtempSync(join(tmpdir(), 'ingress-deploy-'));
+  try {
+    for (const d of ['deploy', 'bin', '.git']) mkdirSync(join(root, d));
+    copyFileSync(resolve('deploy/deploy-ingress.sh'), join(root, 'deploy/deploy-ingress.sh'));
+    const log = join(root, 'log');
+    writeFileSync(log, '');
+    const scripts = {
+      git: `case "$*" in 'branch --show-current') echo main;; 'rev-parse --git-common-dir') echo .git;; 'rev-parse --short HEAD') echo abc123;; esac`,
+      flock: 'exit 0',
+      docker: `printf '%s\\n' "$*" >> "$LOG"\n[[ -z "$FAIL" || "$*" != *"$FAIL"* ]] || exit 43\ncase "$*" in *' ps -q channel-ingress') echo container;; 'image inspect '*|'inspect '*) echo sha256:expected;; esac`,
+    };
+    for (const [n, s] of Object.entries(scripts))
+      writeFileSync(join(root, 'bin', n), '#!/bin/bash\nset -eu\n' + s, { mode: 0o755 });
+    const r = spawnSync(
+      'bash',
+      [join(root, 'deploy/deploy-ingress.sh'), 'production', ...(check ? ['--check'] : [])],
+      {
+        encoding: 'utf8',
+        timeout: 10000,
+        env: {
+          ...process.env,
+          PATH: join(root, 'bin') + ':' + process.env.PATH,
+          COMPOSE_MANAGED: '1',
+          LOG: log,
+          FAIL: fail,
+        },
+      },
+    );
+    return { r, log: readFileSync(log, 'utf8') };
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+}
+test('生产中转预检加载生产端口，检查本地分流，不迁移不重启', () => {
+  const { r, log } = run({ check: true });
+  assert.equal(r.status, 0, r.stderr);
+  assert.ok(
+    log
+      .split('\n')
+      .filter((x) => x.startsWith('compose '))
+      .every((x) => x.includes('docker-compose.ingress.managed.yml')),
+  );
+  assert.match(log, /c.localAccounts.length\|\|c.localToken/);
+  assert.doesNotMatch(log, /ingress-migrate|up -d|stop /);
+});
+test('中转预检失败不能执行迁移或替换运行容器', () => {
+  const { r, log } = run({ fail: 'readConfig' });
+  assert.equal(r.status, 43);
+  assert.doesNotMatch(log, /ingress-migrate|up -d/);
+});
+test('中转路由校验失败保持原容器，成功后才切换镜像', () => {
+  const bad = run({ fail: 'verifyRouting' });
+  assert.equal(bad.r.status, 43);
+  assert.doesNotMatch(bad.log, /up -d/);
+  const good = run();
+  assert.equal(good.r.status, 0, good.r.stderr);
+  assert.ok(good.log.indexOf('verifyRouting') < good.log.indexOf('up -d'));
+  assert.doesNotMatch(good.log, /stop ai/);
+});

deploy/tests/health-watch.test.mjs

+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtempSync, mkdirSync, writeFileSync, readFileSync, rmSync } from 'node:fs';
+import { join, resolve } from 'node:path';
+import { tmpdir } from 'node:os';
+import { spawnSync } from 'node:child_process';
+function probe(body, ai) {
+  const root = mkdtempSync(join(tmpdir(), 'health-watch-'));
+  try {
+    const bin = join(root, 'bin');
+    mkdirSync(bin);
+    writeFileSync(
+      join(bin, 'curl'),
+      '#!/bin/bash\nprintf "%s\\n200" "$BODY"\nprintf "%s\\n" "$*" > "$CALL"\n',
+      { mode: 0o755 },
+    );
+    const r = spawnSync('bash', [resolve('deploy/health-watch.sh'), '--dry-run'], {
+      encoding: 'utf8',
+      env: {
+        ...process.env,
+        PATH: bin + ':' + process.env.PATH,
+        BODY: body,
+        CALL: join(root, 'call'),
+        WATCH_URL: 'https://test.invalid',
+        WATCH_STATE_DIR: root,
+        WATCH_NAME: 'check',
+        WATCH_HEARTBEAT_HOUR: '-1',
+        WATCH_CONTROL_URL: '',
+        ALERT_WEBHOOK_URL: '',
+        ...(ai
+          ? { WATCH_PATH: '/health/ready', WATCH_REQUIRE_DB: '1' }
+          : { WATCH_PATH: '/health', WATCH_REQUIRE_DB: '0' }),
+      },
+    });
+    assert.equal(r.status, 0, r.stderr);
+    return {
+      state: readFileSync(join(root, 'check.state'), 'utf8'),
+      call: readFileSync(join(root, 'call'), 'utf8'),
+    };
+  } finally {
+    rmSync(root, { recursive: true, force: true });
+  }
+}
+test('原 PAC 探针保持 /health 判据，拒绝 HTTP 200 错误信封', () => {
+  assert.match(probe('{"status":"ok"}', false).state, /fails=0/);
+  assert.match(probe('{"code":10004}', false).state, /fails=1/);
+});
+test('AI 外部探针要求数据库就绪且访问 ready 路由', () => {
+  const good = probe('{"status":"ok","db":"up"}', true);
+  assert.match(good.call, /health\/ready/);
+  assert.match(good.state, /fails=0/);
+  assert.match(probe('{"status":"ok","db":"down"}', true).state, /fails=1/);
+});

deploy/tests/prepare-pac-indexes.test.mjs

+import { createRequire } from 'node:module';
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+const { validIndex } = createRequire(import.meta.url)('../prepare-pac-indexes.cjs');
+const good = {
+  indisvalid: true,
+  indisready: true,
+  indisunique: false,
+  table_name: 'facts',
+  method: 'btree',
+  predicate: null,
+  expressions: null,
+  columns: ['host', 'tenant'],
+};
+test('同名索引必须有效、列序一致、无额外谓词，不能将失败残留当作完成', () => {
+  assert.equal(validIndex(good, 'facts', ['host', 'tenant']), true);
+  for (const patch of [
+    { indisvalid: false },
+    { indisready: false },
+    { indisunique: true },
+    { columns: ['tenant', 'host'] },
+    { predicate: 'false' },
+    { expressions: 'lower(host)' },
+    { table_name: 'other' },
+    { method: 'hash' },
+  ])
+    assert.equal(validIndex({ ...good, ...patch }, 'facts', ['host', 'tenant']), false);
+});

docker-compose.ai.managed.yml

+# 与 PAC 托管环境一致：远端网关经应用机私网访问，安全组仅放行网关。
+services:
+  ai-service:
+    ports: !override ['3104:3104']
+  ai-web:
+    ports: !override ['3103:3103']

docker-compose.ingress.managed.yml

+# 与 PAC 托管环境一致；3106 不得对公网开放，网关必须覆盖 X-Real-IP。
+services:
+  channel-ingress:
+    ports: !override ['3106:3106']

docs/friday-ai-production-handoff.md

+# FridayAI 首次生产上线：运维交接单
+
+应用机 `ssh pac-friday`，源码 `/data/pac`，分支 `main`。PAC 已运行；FridayAI 和中转尚未上线。详细命令见 [生产部署步骤](friday-ai-production-deployment.md)，执行证据见 [准备记录](qa/2026-10-10-production-readiness.md)。
+
+## 已准备
+
+- 生产发布脚本、远端网关端口覆盖、独立中转部署和配置预检。
+- 服务器健康/队列巡检、cron 与日志轮转模板；不依赖开发电脑。
+- 生产专用 JWT 私钥、读取密钥、中转密钥、数据库应用密码及 env 草稿，位于应用机 `/data/pac-ops/readiness-20261010/config/`，尚未激活。
+- PAC 索引后台准备任务；最终结果必须以准备记录、退出码和数据库有效性检查为准。
+- 未使用的治疗步骤汇总功能已移除，不需要回填或全量重算患者。
+
+## 请运维处理
+
+| 项目 | 运维执行 / 提供 | 验收 |
+| --- | --- | --- |
+| RDS | 备份 PAC 并登记恢复点；确认磁盘/IOPS；执行 `deploy/production/friday-ai-database.sql` 建独立库和账号，在新库安装两个扩展 | 新应用账号可迁移 `friday_ai`；UTF8、UTC；不改 PAC 时区 |
+| 数据库密码 | 安全交付专用应用账号密码，或使用已生成的 `database.key` 设置 | 两份 env 中 URL 一致；不需要交出 RDS 管理账号 |
+| 域名/证书 | 确认 AI 和回调的生产域名，配置 DNS、证书 | HTTPS 正常；不覆盖 PAC 域名 |
+| 网关/防火墙 | 按 `deploy/production/friday-ai-gateway.conf.example` 配置；仅网关访问应用机 3103/3104/3106 | API、SSE、上传可用；回调域名只开放 POST `/hook/ecom` |
+| OSS | 生产私有 bucket、region、应用 RAM 凭据 | 生产附件上传及签名下载成功 |
+| ECom | 生产 API 凭据、回调出口 IP；登记独立生产回调 | 测试回调保持原样；生产无本地分流 |
+| 模型 | 生产 API Key 和模型访问额度 | 测试调用成功 |
+| 告警 | 采集巡检日志；连续三次异常或五分钟缺日志告警；外部拨测；登记接收人 | 受控演练可收到告警和恢复通知 |
+
+业务还需提供：集团正式名称、首位管理员姓名/手机号、平台管理员资料、首批诊所/员工范围、批准的席位数量/期限、专用验收企微号。先接 5i5ya：`partner=jvs-dw`、`partnerTenant=ruier-grp`；无需提供其 PAC appSecret，不创建 Friday SaaS 生产租户。
+
+## 实施顺序
+
+1. 核对准备记录和索引状态；确认 RDS 备份、网关限制、所有生产配置已齐。
+2. 备份现有配置和镜像记录；`main` 快进到本次经过测试的 SHA，保护现有未跟踪文件。
+3. 补全 env 草稿、安装配置和私钥；PAC/AI 使用同一服务读取密钥。
+4. `COMPOSE_MANAGED=1 bash deploy/deploy-prod.sh --no-pull` 升级 PAC，核验七条迁移及真实读取；不首灌、不回填、不全量重算。
+5. `COMPOSE_MANAGED=1 bash deploy/deploy-ai.sh production --no-pull`。
+6. `COMPOSE_MANAGED=1 bash deploy/deploy-ingress.sh production`；检查健康，启用网关和巡检。
+7. CLI 开户，配置组织/员工范围及席位；验收 5i5ya 患者数据和诊所隔离。
+8. 先开 worker、后按业务确认开 scheduler；专用号验收收发/图片/文件及 AI 停服补投，再开放正式业务。
+
+**生产发布仍有短暂停服；本次没有承诺零停机。** 中转保护 AI 停服期间已持久接收的回调，不覆盖中转、主机或数据库自身故障。首次上线不能照搬测试账号、设备或密钥。

docs/friday-ai-test-deployment.md

@@ -196,3 +196,13 @@ docker compose -p pac-ai -f docker-compose.ai.yml exec ai-service \
 私密保存命令输出的临时密码，打开 `https://fai.jarvismedical.asia/login` 登录并改密。只读账号增加 `--role=platform_viewer`。
 
 已有账号使用 `grant --user='<用户UUID>' --role=platform_admin --reason='开通平台管理'`，继续使用原密码。完整命令见 [平台管理](friday-ai-platform-management.md)。
+
+## 9. 服务器巡检
+
+```sh
+bash deploy/check-ai-health.sh
+# 中转只构建和预检，不迁移或重启：
+bash deploy/deploy-ingress.sh staging --check
+```
+
+返回 JSON `status=ok`、退出码 0 为正常；`degraded` / 非 0 需检查对应项。测试保留现有本地调试路由；本地长时间离线也可能造成中转积压，不要因此清队列。生产部署和告警安装见 [生产部署](friday-ai-production-deployment.md)。

docs/qa/2026-10-10-production-readiness.md

+# 2026-10-10 FridayAI 生产准备
+
+## 范围
+
+- 功能分支 `codex/production-readiness`；功能分支 → test 验证 → 功能分支 → main。
+- 生产现有 PAC 保持运行；不发布生产 AI/中转、不启用生产发信、不改全局时区。
+- 移除未启用步骤汇总能力已经完成：main `3b81549`，测试运行 `3cd4acf`。
+
+## 生产核验与准备
+
+- 应用机 `/data/pac`：main `d9a9fda`；现有 PAC 健康。新代码尚未部署。
+- RDS PostgreSQL 18.3；PAC 库约 266 GB；现有应用账号无建库/建角色权限。未尝试提升权限。
+- RDS 云端剩余存储、备份和 IOPS 无控制台权限确认，列入运维上线门禁。
+- 独立生产密钥及未激活配置草稿：`/data/pac-ops/readiness-20261010/config/`；目录 700、文件 600。没有修改 PAC 运行 `.env`。
+- 现有配置/镜像/源码快照：`/data/pac-ops/readiness-20261010/`；不含数据库全量备份。
+
+## 索引任务
+
+后台任务、日志、退出码均在上述目录，断开本地不会中断。
+
+首次尝试在 00:20 开始；00:23 连续两次 PAC 健康检查超过五秒，任务按保护逻辑取消。核查同一时段有 00:15 启动的 `incremental_bundle` 正在摄入，不能仅据此归因于索引。随后健康恢复。已确认并删除本次创建的无效索引，未触碰既有索引，首次日志保留在 `attempt-1/`。
+
+调整后不仅看 PostgreSQL 长查询，还检查 `sync_logs`：摄入期间和完成后的 15 分钟不启动；之后需要连续六次健康/空闲采样。每次只建一个，`CONCURRENTLY`，维护内存 64 MB、并行维护 worker 0、锁超时 5 秒、语句超时 45 分钟。最多等待六小时；健康连续失败会取消并停止，不自动循环重试。任务仍需以最终日志和索引有效性查询确认。
+
+提前创建不修改 `_prisma_migrations`。正式发布仍按顺序运行七条迁移：三个索引已有效时跳过重复创建；未启用派生表按历史迁移创建后由新迁移删除，不回填事实。
+
+## 部署与监控验证
+
+- AI 生产端口覆盖和中转生产端口覆盖纳入仓库；两套发布脚本统一支持 `COMPOSE_MANAGED=1`。
+- 中转新增 `--check`，不执行迁移/重启；生产拒绝本地账号分流配置。
+- 修复 AI 发布最后一次 Compose exec 吞掉 SSH 管道后续命令的问题。
+- 新增服务器巡检命令和 cron/logrotate 模板；外部健康脚本支持 AI readiness。
+- 测试服务器真实巡检：AI、Web、PAC、ingress 均正常；中转 pending=0、retrying=0，outbox pending=0、expiredLeases=0。未发送任何测试消息。
+- 通知需运维接入云日志/云拨测与接收人；仅安装巡检日志不等于已收到告警。
+
+- 本地部署/预检/健康检查测试共 31 项通过；shell 语法检查通过。
+
+## 运维交付
+
+按 [生产部署](../friday-ai-production-deployment.md) 实施。需要运维执行的 SQL、网关、cron、日志轮转模板在 `deploy/production/`。待提供：生产域名/证书，专用数据库账号，ECom/OSS/模型生产配置，告警接收人，首个集团/管理员/诊所/席位信息。无需交付 RDS 管理账号。