OmniRoute/scripts/ops/deployCanary.ts

/**
 * Canary deploy policy (#10429) — pure planning + verdict logic.
 *
 * Deploying the internal gateway used to be a manual sequence (build → pack → scp →
 * `npm i -g` → `pm2 restart`) with nothing recording what landed and nothing proving the
 * new build served traffic. On 2026-08-14 that shipped a package built from a feature
 * branch predating #10373: the process came up, `/api/monitoring/health` answered
 * `healthy`, and every real request returned `502 … Executor result must contain a
 * Response` until a human hit it.
 *
 * Two lessons are encoded here:
 *  1. Refuse an artifact that cannot be traced to the release line (reuses #10427).
 *  2. A health check is NOT a smoke test. Only a real completion exercises the egress
 *     path where that outage lived, so the verdict requires at least one.
 *
 * Everything side-effecting (git, ssh, http) is injected or emitted as data, so the policy
 * is unit-testable without a host. The thin CLI that executes these steps lives in
 * `scripts/ops/deploy-canary.mjs`.
 */

import { resolveBuildProvenance } from "../build/buildProvenance.ts";

export type CanaryPlanInput = {
  buildSha: string;
  isAncestorOfRelease: (sha: string) => boolean;
  allowCanary: boolean;
};

export type CanaryPlan = {
  proceed: boolean;
  reason: string;
};

/**
 * Decide whether an artifact may be shipped at all. Delegates to the provenance policy so
 * the pack gate and the deploy path can never disagree about what "shippable" means.
 */
export function planCanaryDeploy(input: CanaryPlanInput): CanaryPlan {
  const provenance = resolveBuildProvenance({
    buildSha: input.buildSha,
    isAncestorOfRelease: input.isAncestorOfRelease,
    allowOverride: input.allowCanary,
  });
  return { proceed: provenance.ok, reason: provenance.message };
}

export type CompletionProbe = {
  model: string;
  ok: boolean;
  status: number;
};

export type SmokeInput = {
  healthOk: boolean;
  completions: CompletionProbe[];
};

export type SmokeVerdict = {
  ok: boolean;
  rollback: boolean;
  reason: string;
};

/**
 * Grade a deploy. Health first (cheap, and a dead process needs no further probing), then
 * every completion probe.
 *
 * An empty probe list FAILS: "no probe ran" must never read as "everything is fine" —
 * that is precisely how a broken egress path stays invisible behind a green health check.
 */
export function evaluateSmoke(input: SmokeInput): SmokeVerdict {
  if (!input.healthOk) {
    return {
      ok: false,
      rollback: true,
      reason: "health endpoint did not report healthy after restart",
    };
  }

  if (input.completions.length === 0) {
    return {
      ok: false,
      rollback: true,
      reason:
        "no completion probe ran — a health check alone cannot see a broken egress path (#10429)",
    };
  }

  const failed = input.completions.filter((probe) => !probe.ok);
  if (failed.length > 0) {
    const detail = failed.map((probe) => `${probe.model} → ${probe.status}`).join(", ");
    return {
      ok: false,
      rollback: true,
      reason: `completion probe failed: ${detail}`,
    };
  }

  return {
    ok: true,
    rollback: false,
    reason: `health + ${input.completions.length} completion probe(s) passed`,
  };
}

export type RemoteStep = {
  name: string;
  /** argv form only — never a shell string, so no value can be interpreted (Hard Rule #13). */
  argv: string[];
  description: string;
};

export type RemoteStepsInput = {
  host: string;
  tarballPath: string;
  pm2App: string;
};

/**
 * The remote sequence, as data. Ordered so the rollback anchor is captured BEFORE the
 * install overwrites it, and so the SHA is verified only after the restart has actually
 * loaded the new artifact.
 *
 * Emitted as argv arrays rather than shell strings: the paths and app names come from
 * config and CLI flags, and interpolating them into `sh -c` is exactly the pattern Hard
 * Rule #13 forbids.
 */
export function buildRemoteSteps(input: RemoteStepsInput): RemoteStep[] {
  const { host, tarballPath, pm2App } = input;
  const shaPath = "/usr/lib/node_modules/omniroute/dist/BUILD_SHA";

  return [
    {
      name: "capture-current-sha",
      argv: ["ssh", host, "cat", shaPath],
      description: "record the running BUILD_SHA so a failed smoke can be rolled back",
    },
    {
      name: "install",
      argv: ["ssh", host, "npm", "install", "-g", tarballPath, "--no-audit", "--no-fund"],
      description: "install the packaged artifact globally",
    },
    {
      name: "restart",
      argv: ["ssh", host, "pm2", "restart", pm2App, "--update-env"],
      description: "restart the service under its process manager",
    },
    {
      name: "verify-installed-sha",
      argv: ["ssh", host, "cat", shaPath],
      description: "confirm the running artifact is the one just shipped",
    },
  ];
}