mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-08-15 03:32:21 +03:00
Compare commits
1 Commits
release/v3
...
fix/10085-
| Author | SHA1 | Date | |
|---|---|---|---|
|
61e39bfe54 |
46
.github/workflows/quality.yml
vendored
46
.github/workflows/quality.yml
vendored
@@ -60,49 +60,13 @@ jobs:
|
||||
build:
|
||||
name: Build (advisory)
|
||||
needs: changes
|
||||
# FORK PRs ONLY. build.yml's `Fast Production Build` triggers on `push: branches: ["**"]`
|
||||
# and runs `build:release` — a superset of this job — so for an own-origin branch this job
|
||||
# was building the same tree twice. A fork contributor pushes to THEIR repo, so that push
|
||||
# never fires here, and this is the only pre-merge build signal they get. Measured
|
||||
# 2026-08-14: 72 of the last 100 PRs into release/** came from forks, so the fork case is
|
||||
# the majority of the traffic, not the exception — this job earns its place, it just should
|
||||
# not duplicate build.yml for the own-origin 28%.
|
||||
if: ${{ github.event_name != 'pull_request' || ((github.event.pull_request.draft == false || startsWith(github.head_ref, 'mergify/merge-queue/')) && needs.changes.outputs.code == 'true' && github.event.pull_request.head.repo.full_name != github.repository) }}
|
||||
# PINNED to hosted — this was the last job in THIS workflow still on the USE_VPS_RUNNER
|
||||
# switch (ci.yml's Build, nightly-release-green and npm-publish keep it, so the variable
|
||||
# stays meaningful), and with USE_VPS_RUNNER=true it produced NO signal at all here.
|
||||
# Measured 2026-08-14 over the last 25
|
||||
# quality.yml runs: not one Build (advisory) reached a conclusion. Every sample was either
|
||||
# queued on the self-hosted pool (2 runners, `omniroute-113-6/7`, both permanently busy — one
|
||||
# job sat queued 2h+ and was still unclaimed) or, when it did land, killed mid-build by this
|
||||
# workflow's own `cancel-in-progress` concurrency. 6/6 sampled "failures" are exit 143 /
|
||||
# "The runner has received a shutdown signal" at ~3.5 min into `npm run build` — zero OOM,
|
||||
# zero build errors. So the job burned a scarce runner that the gates actually need while
|
||||
# reporting a permanent red on every PR.
|
||||
#
|
||||
# Gap 19 left USE_VPS_RUNNER governing build-like jobs on the premise that "the build needs
|
||||
# the .113's RAM". That premise no longer holds: `Fast Production Build` (build.yml) runs
|
||||
# `build:release` — a SUPERSET of this job's `npm run build`, plus the CLI bundle — on plain
|
||||
# ubuntu-latest and passed 24/25 of its last runs in ~15 min. What it has and this job did
|
||||
# not is memory PROVISIONING: a 10 GB swapfile plus a 12 GB V8 heap. That matters because
|
||||
# --max-old-space-size only bounds V8's JS heap, never Turbopack's native (Rust) allocation
|
||||
# (#6409) — swap is what absorbs the native peak. Both are mirrored below.
|
||||
runs-on: ubuntu-latest
|
||||
if: ${{ github.event_name != 'pull_request' || ((github.event.pull_request.draft == false || startsWith(github.head_ref, 'mergify/merge-queue/')) && needs.changes.outputs.code == 'true') }}
|
||||
# Dynamic runner — same fork-safe rule as ci.yml / fast-gates.
|
||||
runs-on: ${{ (vars.USE_VPS_RUNNER == 'true' && (github.event_name != 'pull_request' || github.event.pull_request.head.repo.full_name == github.repository)) && fromJSON('["self-hosted","omni-release"]') || 'ubuntu-latest' }}
|
||||
# #7307: advisory for the first week of release-PR runs; remove
|
||||
# continue-on-error after the production-build signal is stable.
|
||||
continue-on-error: true
|
||||
steps:
|
||||
# Mirrors build.yml: Turbopack's native peak is not bounded by --max-old-space-size, so
|
||||
# the hosted runner needs swap headroom before the build starts.
|
||||
- name: Expand virtual memory (10 GB swap)
|
||||
run: |
|
||||
sudo swapoff -a || true
|
||||
sudo rm -f /mnt/swapfile /swapfile
|
||||
sudo fallocate -l 10G /mnt/swapfile || sudo dd if=/dev/zero of=/mnt/swapfile bs=1M count=10240
|
||||
sudo chmod 600 /mnt/swapfile
|
||||
sudo mkswap /mnt/swapfile
|
||||
sudo swapon /mnt/swapfile
|
||||
free -h
|
||||
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
|
||||
with:
|
||||
persist-credentials: false
|
||||
@@ -115,10 +79,6 @@ jobs:
|
||||
- run: npm run build
|
||||
env:
|
||||
OMNIROUTE_USE_TURBOPACK: "1"
|
||||
# Same heap build.yml proves sufficient. build-next-isolated.mjs defaults to 8192 and
|
||||
# honours OMNIROUTE_BUILD_MEMORY_MB; NODE_OPTIONS is set for parity with build.yml.
|
||||
NODE_OPTIONS: "--max-old-space-size=12288"
|
||||
OMNIROUTE_BUILD_MEMORY_MB: "12288"
|
||||
# No artifact upload here: the PR-to-release quality workflow has no
|
||||
# downstream package/e2e jobs that consume the Next.js build output.
|
||||
|
||||
|
||||
@@ -0,0 +1 @@
|
||||
- fix(sse): bridge generic openai-compatible/anthropic-compatible provider type ids to their concrete uuid node id in credential lookup (#10085)
|
||||
@@ -1 +0,0 @@
|
||||
- fix(ci): make `Build (advisory)` produce a signal again — pinned to a hosted runner with the swap/heap provisioning `Fast Production Build` proves sufficient, and scoped to fork PRs, which are the only ones `build.yml` cannot cover (72 of the last 100 PRs into `release/**`)
|
||||
@@ -330,75 +330,32 @@ excludeMarkers, defaultRetryAfterMs}`), matched via `applyStatusRestatement()`.
|
||||
|
||||
Permanent errors (agentrouter's `无权访问模型` — no access to this model) are
|
||||
NEVER restated: `excludeMarkers` vetoes the rule even when `textMarkers` hit,
|
||||
so the error keeps its original status and nothing retries it forever. The
|
||||
matching provider classification rule
|
||||
(`agentrouter-model-access-denied` in `open-sse/config/providerErrorRules.ts`:
|
||||
`reason: "auth_error"`, `scope: "model"`, a `6h` declared base cooldown) is
|
||||
consulted by `checkFallbackError` (`open-sse/services/accountFallback.ts`)
|
||||
*before* the generic apikey-category `FORBIDDEN` early-return, gated on
|
||||
`honorsRuleLockScope(provider)` (#10334 — currently agentrouter-exclusive via
|
||||
the `HONORS_RULE_LOCK_SCOPE_PROVIDERS` allowlist in
|
||||
`providerErrorRules.ts`). The rule's declared 6h cooldown flows through as
|
||||
`fallbackResult.baseCooldownMs`, but it still feeds the pre-existing
|
||||
per-model-quota lockout path (`lockModelIfPerModelQuota()` /
|
||||
`recordModelLockoutFailure()`, unchanged by #10334 except for the cooldown
|
||||
source): it is clamped down to the operator's `mlSettings.maxCooldownMs`
|
||||
(default `1_800_000ms` / 30min), like every other model lockout, and the
|
||||
*persisted lockout reason* stays the pre-existing hardcoded `"forbidden"`,
|
||||
not the rule's `"auth_error"` — only the cooldown duration is honored
|
||||
end-to-end, not the reason string. The connection itself stays active;
|
||||
sibling models on the same connection are unaffected.
|
||||
so the error keeps its original status and nothing retries it forever. A
|
||||
separate provider classification rule
|
||||
(`agentrouter-model-access-denied` in `open-sse/config/providerErrorRules.ts`)
|
||||
declares an `auth_error`/scope-`model` match for this text, but it does not
|
||||
fire on the live production path today: the rule only matches `status ===
|
||||
403`, and `checkFallbackError`'s apikey-category `FORBIDDEN` branch
|
||||
(`open-sse/services/accountFallback.ts`) returns early for a plain 403
|
||||
*before* the provider-rule lookup ever runs. In practice a `无权访问模型` 403
|
||||
is handled the same way as the base apikey-provider 403 path (see Connection
|
||||
Cooldown, §2), not as a 6h model lockout. The rule still exists as a
|
||||
declarative classification consumable by future callers of `classifyError`
|
||||
with context — wiring it into the production `checkFallbackError` path is
|
||||
tracked as a follow-up, not yet done.
|
||||
|
||||
Restated quota errors (`额度不足`) reach a provider rule in production
|
||||
(`agentrouter-user-quota-exhausted`: `reason: "quota_exhausted"`, `scope:
|
||||
"connection"`, no declared cooldown of its own — the persistence layer's
|
||||
scaled backoff default applies). Since #10334, `scope` on
|
||||
`ProviderErrorRuleMatch` IS consumed end-to-end, but **only** for providers in
|
||||
the `HONORS_RULE_LOCK_SCOPE_PROVIDERS` allowlist (`providerErrorRules.ts` —
|
||||
today only `"agentrouter"`, gated via `honorsRuleLockScope()`). For every
|
||||
other provider `scope` remains informational, exactly as before #10334.
|
||||
`checkFallbackError` surfaces the matched rule's scope as
|
||||
`fallbackResult.ruleScope`; `isAgentrouterConnectionQuotaScope()`
|
||||
(`src/sse/services/auth.ts`) is the shared guard that confirms a
|
||||
`ruleScope` is genuinely safe to honor as a connection-wide, self-recovering
|
||||
signal (scope `"connection"`, reason `quota_exhausted`, never `permanent`,
|
||||
never `creditsExhausted` — a defense against a future rule pairing scope
|
||||
`"connection"` with a permanent account state). Two consumers call it:
|
||||
|
||||
- **Persistence** (`markAccountUnavailable()`, `src/sse/services/auth.ts`):
|
||||
instead of falling into the passthrough-provider **per-model** lockout
|
||||
branch (agentrouter is `passthroughModels: true` → `hasPerModelQuota()`
|
||||
returns `true`), it applies a **temporary connection cooldown** —
|
||||
`testStatus: "unavailable"` + `rateLimitedUntil`, never a terminal status
|
||||
(`credits_exhausted`/`banned`/`expired`) — so the connection self-recovers
|
||||
once the cooldown lapses instead of requiring a manual credential reset.
|
||||
Skipped for connections with `disableCooling: true` (#2997): that opt-out
|
||||
falls through to the per-model lockout instead (a documented trade-off —
|
||||
see the code comment above the branch).
|
||||
- **Same-request combo routing** (`applyComboTargetExhaustion()`,
|
||||
`open-sse/services/combo/targetExhaustion.ts`): the same guard marks the
|
||||
connection into the in-memory `exhaustedConnections` set, keyed
|
||||
`${provider}:${connectionId}`. This only skips a remaining SAME-REQUEST
|
||||
target that *itself already carries that exact `connectionId`* on its own
|
||||
target object (`getExhaustedTargetSkipReason()`,
|
||||
`open-sse/services/combo/comboPredicates.ts`, `if (provider &&
|
||||
connectionId)` before the `exhaustedConnections` lookup) — a plain
|
||||
model-list combo, where sibling targets carry no pinned `connectionId` of
|
||||
their own and one is only resolved per-dispatch from the response's
|
||||
`X-OmniRoute-Selected-Connection-Id` header, never hits that key match. For
|
||||
that common case, the real protection against a remaining leg reusing the
|
||||
just-exhausted account is NOT this Set — it is the persistence layer above
|
||||
(the connection's `rateLimitedUntil` is now in the future) combined with
|
||||
this same guard suppressing `transientRateLimitedProviders` for the
|
||||
failure (see "Two-stage design" and the code comment on the
|
||||
`isAgentrouterConnectionQuotaScope` branch in `targetExhaustion.ts`): with
|
||||
that Set left unmarked, `combo.ts`'s `allowRateLimitedConnection` force-allow
|
||||
(`open-sse/services/combo.ts:1005-1013`, `:2734-2738`) does NOT kick in for
|
||||
the provider's remaining legs, so credential selection's `rateLimitedUntil`
|
||||
filter (`src/sse/services/auth.ts:1238`) is honored normally and a
|
||||
remaining leg either picks a different, still-eligible agentrouter
|
||||
connection or fails with no credentials available — it does not force its
|
||||
way back onto the connection this branch just cooled down.
|
||||
Restated quota errors (`额度不足`) do reach a provider rule in production
|
||||
(`agentrouter-user-quota-exhausted`, scope `"connection"`), but `scope` on
|
||||
`ProviderErrorRuleMatch` is currently informational — the persistence path
|
||||
(`checkFallbackError` → `combo.ts`) only consumes `reason` and `cooldownMs`,
|
||||
never `scope`. What actually happens for agentrouter (`passthroughModels:
|
||||
true` → `hasPerModelQuota()` returns `true`) is a **per-model** lockout via
|
||||
`recordModelLockoutFailure()`: the connection itself is never cooled down for
|
||||
this error (`combo.ts` skips `recordProviderCooldown` for 429 when
|
||||
`hasPerModelQuota` is true), so other models on the same account keep being
|
||||
tried — each one burns one call and its own lockout before combo routing
|
||||
moves on. Honoring `scope` end-to-end (so a `"connection"` match actually
|
||||
locks the connection) is tracked as a follow-up.
|
||||
|
||||
### Two-stage design: status restatement, then classification
|
||||
|
||||
@@ -423,15 +380,6 @@ allowlisted providers, the structured error otherwise. Adding a provider to
|
||||
that the default path for every provider not on the list stays
|
||||
byte-for-byte unchanged.
|
||||
|
||||
A rule's `scope` (`model` / `provider` / `connection`) is a separate opt-in
|
||||
from `FULL_TEXT_RULE_PROVIDERS`: `checkFallbackError` only surfaces it as
|
||||
`fallbackResult.ruleScope`, and downstream consumers only honor it as
|
||||
anything other than an informational label, for providers in the
|
||||
`HONORS_RULE_LOCK_SCOPE_PROVIDERS` allowlist in the same file (`gated via
|
||||
honorsRuleLockScope()` — today only `"agentrouter"`). See "Restated quota
|
||||
errors" above for what a `scope: "connection"` match actually does once a
|
||||
provider is on that allowlist.
|
||||
|
||||
### Adding a new quota-misstating gateway
|
||||
|
||||
1. Register one rule array in `statusRestatementRegistry`
|
||||
@@ -447,15 +395,7 @@ provider is on that allowlist.
|
||||
`checkFallbackError` only ever hands the rule the structured
|
||||
`{code, type}` error and a body-text rule will never match live traffic.
|
||||
Rules that match purely on `status`/`headers` (like Opencode's or
|
||||
Minimax's) do not need this opt-in. Separately, if the rule declares
|
||||
`scope: "connection"` and the intent is an actual connection-wide cooldown
|
||||
plus same-request combo skip (not just an informational label), add the
|
||||
provider id to `HONORS_RULE_LOCK_SCOPE_PROVIDERS` in the same file — this
|
||||
is what gates `isAgentrouterConnectionQuotaScope()`-style consumption in
|
||||
`markAccountUnavailable()` (`src/sse/services/auth.ts`) and
|
||||
`applyComboTargetExhaustion()`
|
||||
(`open-sse/services/combo/targetExhaustion.ts`); without it, `scope`
|
||||
still flows through `fallbackResult.ruleScope` but nothing acts on it.
|
||||
Minimax's) do not need this opt-in.
|
||||
3. Add unit tests mirroring `tests/unit/upstream-status-restatement.test.ts`
|
||||
and `tests/unit/agentrouter-error-rules.test.ts` (including the
|
||||
not-permanent / not-creditsExhausted guards, and — if the provider needs
|
||||
|
||||
@@ -32,7 +32,7 @@ different endpoint families, so all four products remain separate provider IDs.
|
||||
| Provider family | `global-sg` | `china-beijing` | Wire format |
|
||||
| ----------------------- | ------------------------------------------------------------------------ | -------------------------------------------------------------------- | ----------- |
|
||||
| `alibaba` | `https://dashscope-intl.aliyuncs.com/compatible-mode/v1` | `https://dashscope.aliyuncs.com/compatible-mode/v1` | OpenAI |
|
||||
| `bailian-coding-plan` | `https://token-plan.ap-southeast-1.maas.aliyuncs.com/apps/anthropic/v1` | `https://token-plan.cn-beijing.maas.aliyuncs.com/apps/anthropic/v1` | Anthropic |
|
||||
| `bailian-coding-plan` | `https://coding-intl.dashscope.aliyuncs.com/apps/anthropic/v1` | `https://coding.dashscope.aliyuncs.com/apps/anthropic/v1` | Anthropic |
|
||||
| `qwen-cloud` | `https://dashscope-intl.aliyuncs.com/compatible-mode/v1` | `https://dashscope.aliyuncs.com/compatible-mode/v1` | OpenAI |
|
||||
| `qwen-cloud-token-plan` | `https://token-plan.ap-southeast-1.maas.aliyuncs.com/compatible-mode/v1` | `https://token-plan.cn-beijing.maas.aliyuncs.com/compatible-mode/v1` | OpenAI |
|
||||
|
||||
|
||||
@@ -1195,8 +1195,8 @@ Provider quota endpoints, network tunnels (Tailscale, Ngrok, MITM debug proxy),
|
||||
| `TAILSCALED_BIN` | _(auto-detect)_ | `src/lib/tailscaleTunnel.ts` | Explicit path to the `tailscaled` daemon binary. |
|
||||
| `TAILSCALE_AUTHKEY` | _(unset)_ | `src/lib/tailscaleTunnel.ts` | Pre-shared Tailscale auth key for non-interactive / headless `tailscale up` (passed via `--auth-key=`). When unset, login falls back to the interactive browser auth URL. |
|
||||
| `NGROK_AUTHTOKEN` | _(unset)_ | `src/lib/ngrokTunnel.ts` | Authenticates outbound ngrok tunnels. |
|
||||
| `DB_BACKUP_MAX_FILES` | `20` | `src/lib/db/backup.ts`, `src/lib/db/migrationRunner.ts` | Maximum SQLite backup files retained on disk. Applies to manual/scheduled backups and to pre-migration snapshots. Overrides the value saved from Settings → Database backup retention. |
|
||||
| `DB_BACKUP_RETENTION_DAYS` | `0` | `src/lib/db/backup.ts`, `src/lib/db/migrationRunner.ts` | Maximum age (days) of retained backups. `0` disables age-based pruning. Applies to manual/scheduled backups and to pre-migration snapshots. Overrides the value saved from Settings → Database backup retention. |
|
||||
| `DB_BACKUP_MAX_FILES` | `20` | `src/lib/db/backup.ts` | Maximum SQLite backup files retained on disk. Overrides the value saved from Settings → Database backup retention. |
|
||||
| `DB_BACKUP_RETENTION_DAYS` | `0` | `src/lib/db/backup.ts` | Maximum age (days) of retained backups. `0` disables age-based pruning. Overrides the value saved from Settings → Database backup retention. |
|
||||
| `OMNIROUTE_BACKUP_SCHEDULE_JOB_INTERVAL_MS` | `30000` | `src/lib/jobs/backupScheduleJob.ts` | Tick interval (ms) of the server-side job that executes `backup-schedule.json`. Must stay well under the 1-minute cron granularity; values below `5000` or unparseable fall back to `30000`. |
|
||||
| `OMNIROUTE_TLS_PROXY_URL` | _(unset)_ | `open-sse/services/chatgptTlsClient.ts` | Override the TLS sidecar URL for tests. Production should leave unset. |
|
||||
| `CONTAINER_HOST` | `docker` | `scripts/check-permissions.sh` | Container runtime hint for the entrypoint permission check. Set to `podman` for any Podman topology. Because the container cannot determine whether the engine is local or reached through Podman Machine, the warning stays topology-neutral and points to `contrib/podman/README.md`. |
|
||||
|
||||
@@ -30,15 +30,13 @@ export type ProviderErrorRule = {
|
||||
export type ProviderErrorRuleMatch = {
|
||||
reason: ConfiguredErrorReason;
|
||||
/**
|
||||
* Intended lock scope. #10334: this field is CONSUMED end-to-end only for
|
||||
* providers in `HONORS_RULE_LOCK_SCOPE_PROVIDERS` (agentrouter-exclusive
|
||||
* today, gated by `honorsRuleLockScope()`) — for those, `checkFallbackError`
|
||||
* surfaces it as `ruleScope` on its return value for the persistence layer
|
||||
* to honor instead of re-deriving scope from `hasPerModelQuota()`. For
|
||||
* every other provider it remains INFORMATIONAL: `getProviderErrorRuleMatch`
|
||||
* callers still read only `reason`/`cooldownMs`, and the actual lock scope
|
||||
* is decided independently by each call site. Widening the allowlist is
|
||||
* tracked as a follow-up — see `docs/architecture/RESILIENCE_GUIDE.md` §7.
|
||||
* Intended lock scope. NOTE: this field is currently INFORMATIONAL — no
|
||||
* consumer of `getProviderErrorRuleMatch` (checkFallbackError, combo.ts)
|
||||
* reads `scope` today; only `reason` and `cooldownMs` are consulted. The
|
||||
* actual lock scope applied at runtime is decided independently by each
|
||||
* call site (e.g. `hasPerModelQuota()` deciding model- vs connection-level
|
||||
* lockout). Honoring this field end-to-end is tracked as a follow-up —
|
||||
* see `docs/architecture/RESILIENCE_GUIDE.md` §7.
|
||||
*/
|
||||
scope: "model" | "provider" | "connection";
|
||||
/** Optional explicit cooldown; falls back to the existing per-reason defaults. */
|
||||
@@ -190,29 +188,31 @@ function buildOpenrouterRules(): ProviderErrorRule[] {
|
||||
// agentrouter.org misstates temporary quota exhaustion as 403/400 with a
|
||||
// Chinese body. upstreamStatusRestatement.ts rewrites the status to 429
|
||||
// BEFORE classification, so rules here accept both the raw 403/400 and the
|
||||
// restated 429 (text is the real discriminator either way). Both the raw 403
|
||||
// path AND the restated 429 path reach these rules in production:
|
||||
// checkFallbackError's `honorsRuleLockScope("agentrouter")` pre-check
|
||||
// (#10334) consults these rules BEFORE the generic apikey-category FORBIDDEN
|
||||
// branch, and the restated 429 reaches them via the existing provider-rule
|
||||
// lookup in the configured-rule branch. Both paths use resolveRuleMatchBody,
|
||||
// the only mechanism in checkFallbackError that hands agentrouter's rules the
|
||||
// full error text instead of just {code, type}.
|
||||
// restated 429 (text is the real discriminator either way). In production,
|
||||
// the raw 403 path is what actually matters here: checkFallbackError's
|
||||
// apikey-category FORBIDDEN branch (~line 1699) returns EARLY for a plain
|
||||
// 403, before these rules are ever consulted — these rules fire on the
|
||||
// RESTATED 429 (chatCore's upstreamStatusRestatement hook runs first) via
|
||||
// resolveRuleMatchBody, which is the only path in checkFallbackError that
|
||||
// hands these rules the full error text instead of just {code, type}.
|
||||
// - "额度不足": account-wide temporary quota → quota_exhausted, scope
|
||||
// "connection" (mirror of the Opencode account-wide rationale above).
|
||||
// `scope` on ProviderErrorRuleMatch is CONSUMED for agentrouter (#10334,
|
||||
// exclusive allowlist via `honorsRuleLockScope`): checkFallbackError
|
||||
// surfaces it as `ruleScope` on its return value. Whether the persistence
|
||||
// layer (markAccountUnavailable / combo target exhaustion) actually
|
||||
// switches from `hasPerModelQuota()`-derived scope to honoring `ruleScope`
|
||||
// is Tasks 2/3 of #10334 — this task only surfaces the field.
|
||||
// NOTE: `scope` on ProviderErrorRuleMatch is currently informational —
|
||||
// checkFallbackError/combo.ts only consume `reason` and `cooldownMs`, not
|
||||
// `scope`. For agentrouter specifically (passthroughModels: true →
|
||||
// hasPerModelQuota() is true), this quota_exhausted match actually
|
||||
// resolves to a PER-MODEL lockout (recordModelLockoutFailure), not a
|
||||
// connection-wide lock — other models on the same account keep being
|
||||
// tried by combo routing (each burning one call) until they lock out
|
||||
// individually. Honoring `scope` end-to-end is tracked as a follow-up.
|
||||
// - "无权访问模型": declares auth_error/scope "model" (intent: lock only the
|
||||
// model so the connection keeps serving the rest — Model Lockout tier).
|
||||
// This rule now fires on the production 403 path (#10334): the
|
||||
// `honorsRuleLockScope` pre-check matches it and returns its declared
|
||||
// reason/cooldown/scope before the generic apikey-FORBIDDEN early-return
|
||||
// ever runs. A live `无权访问模型` 403 therefore no longer falls through to
|
||||
// the base apikey-provider 403 handling.
|
||||
// This rule does NOT fire on the production path today: it only matches
|
||||
// `status === 403`, but checkFallbackError's apikey FORBIDDEN branch
|
||||
// returns early for a plain 403 before this rule is ever consulted (see
|
||||
// the note above). A live `无权访问模型` 403 is handled like the base
|
||||
// apikey-provider 403 today. Wiring this rule into that path is tracked
|
||||
// as a follow-up.
|
||||
function buildAgentrouterRules(): ProviderErrorRule[] {
|
||||
const AGENTROUTER_ERROR_STATUSES = new Set([400, 403, 429]);
|
||||
return [
|
||||
@@ -231,15 +231,8 @@ function buildAgentrouterRules(): ProviderErrorRule[] {
|
||||
if (status !== 403) return null;
|
||||
const text = JSON.stringify(body ?? "").toLowerCase();
|
||||
if (!text.includes("无权访问模型")) return null;
|
||||
// Declares a 6h cooldown, but the effective cooldown is NOT 6h: the
|
||||
// model-lockout persistence layer (recordModelLockoutFailure, called from
|
||||
// markAccountUnavailable) clamps every base cooldown — this one included —
|
||||
// to the configured model-lockout maxCooldownMs, which defaults to
|
||||
// 1_800_000ms / 30min (src/lib/resilience/modelLockoutSettings.ts,
|
||||
// DEFAULT_MODEL_LOCKOUT_SETTINGS.maxCooldownMs). So in practice this is
|
||||
// "locked for ~30min by default (up to 6h if an operator raises the model-
|
||||
// lockout cap in settings)", not "until the operator fixes the key's model
|
||||
// grants" — it is a recoverable window, not a real fix-driven unlock.
|
||||
// 6h: effectively "until the operator fixes the key's model grants",
|
||||
// without being an unrecoverable terminal state.
|
||||
return { reason: "auth_error", scope: "model", cooldownMs: 6 * 60 * 60 * 1000 };
|
||||
},
|
||||
},
|
||||
@@ -262,21 +255,6 @@ export const providerRuleRegistry = new Map<string, ProviderErrorRule[]>([
|
||||
["agentrouter", buildAgentrouterRules()],
|
||||
]);
|
||||
|
||||
/**
|
||||
* Providers whose ProviderErrorRuleMatch.scope is actually CONSUMED at the
|
||||
* persistence layer (markAccountUnavailable / combo target exhaustion) to pick
|
||||
* connection-vs-model lock scope. EXCLUSIVE allowlist by owner decision
|
||||
* (2026-08-14, issue #10334) — deliberately SEPARATE from
|
||||
* FULL_TEXT_RULE_PROVIDERS: that set controls what body a rule matches against
|
||||
* (input), this one controls whether the matched scope changes caller behavior
|
||||
* (output). A provider could need one without the other.
|
||||
*/
|
||||
const HONORS_RULE_LOCK_SCOPE_PROVIDERS = new Set(["agentrouter"]);
|
||||
|
||||
export function honorsRuleLockScope(provider: string | null | undefined): boolean {
|
||||
return !!provider && HONORS_RULE_LOCK_SCOPE_PROVIDERS.has(provider.toLowerCase());
|
||||
}
|
||||
|
||||
/**
|
||||
* Providers whose rules match on the FULL upstream error text.
|
||||
* checkFallbackError's rule lookup normally passes only the structured
|
||||
|
||||
@@ -15,11 +15,7 @@ import {
|
||||
serviceSupervisorCooldown,
|
||||
isNimFunctionDegraded,
|
||||
} from "../config/errorConfig.ts";
|
||||
import {
|
||||
getProviderErrorRuleMatch,
|
||||
resolveRuleMatchBody,
|
||||
honorsRuleLockScope,
|
||||
} from "../config/providerErrorRules.ts";
|
||||
import { getProviderErrorRuleMatch, resolveRuleMatchBody } from "../config/providerErrorRules.ts";
|
||||
import * as rot from "./rotationConfig.ts";
|
||||
import { getPassthroughProviders, getProviderCategory } from "../config/providerRegistry.ts";
|
||||
import {
|
||||
@@ -1462,11 +1458,6 @@ export function checkFallbackError(
|
||||
/** #6061: the provider-configured cooldown (ms) before backoff scaling, surfaced so the
|
||||
* caller can persist an explicit reset window instead of the engine's scaled cooldown. */
|
||||
configuredCooldownMs?: number;
|
||||
/** #10334 — the matched ProviderErrorRule's declared lock scope, surfaced so the
|
||||
* persistence layer can honor it instead of re-deriving scope from
|
||||
* hasPerModelQuota(). Populated ONLY when honorsRuleLockScope(provider) is true;
|
||||
* always undefined for every other provider, so existing consumers are unaffected. */
|
||||
ruleScope?: "model" | "provider" | "connection";
|
||||
} {
|
||||
// #10360: an executor-result contract violation is OUR bug, not the provider's.
|
||||
// Retrying reproduces it verbatim, and cooling the connection down (or tripping
|
||||
@@ -1721,36 +1712,6 @@ export function checkFallbackError(
|
||||
return { shouldFallback: false, cooldownMs: 0, reason: RateLimitReason.UNKNOWN };
|
||||
}
|
||||
|
||||
// #10334 — agentrouter EXCLUSIVE: consult the provider rules BEFORE the
|
||||
// apikey-FORBIDDEN early-return below, so a recognized 403 body (e.g.
|
||||
// "无权访问模型") carries the rule's declared reason/cooldown/scope instead of
|
||||
// the generic short auth cooldown. Gated on honorsRuleLockScope — for any
|
||||
// other provider this block is a no-op and the early-return stays identical.
|
||||
if (status === HTTP_STATUS.FORBIDDEN && provider && honorsRuleLockScope(provider)) {
|
||||
const forbiddenMatch = getProviderErrorRuleMatch(
|
||||
provider,
|
||||
status,
|
||||
headers,
|
||||
resolveRuleMatchBody(provider, structuredError ?? null, errorStr)
|
||||
);
|
||||
if (forbiddenMatch) {
|
||||
const scaled = getScaledBaseCooldown(
|
||||
forbiddenMatch.reason as RateLimitReasonValue,
|
||||
backoffLevel
|
||||
);
|
||||
const ruleCooldownMs = forbiddenMatch.cooldownMs;
|
||||
return {
|
||||
shouldFallback: true,
|
||||
cooldownMs: ruleCooldownMs ?? scaled.cooldownMs,
|
||||
baseCooldownMs: ruleCooldownMs ?? scaled.baseCooldownMs,
|
||||
configuredCooldownMs: ruleCooldownMs,
|
||||
newBackoffLevel: ruleCooldownMs !== undefined ? 0 : scaled.newBackoffLevel,
|
||||
reason: forbiddenMatch.reason,
|
||||
ruleScope: forbiddenMatch.scope,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
status === HTTP_STATUS.FORBIDDEN &&
|
||||
provider &&
|
||||
@@ -1803,8 +1764,6 @@ export function checkFallbackError(
|
||||
providerMatch?.cooldownMs !== undefined && providerMatch.cooldownMs > 0
|
||||
? providerMatch.cooldownMs
|
||||
: undefined;
|
||||
const ruleScope =
|
||||
providerMatch && honorsRuleLockScope(provider) ? providerMatch.scope : undefined;
|
||||
const fallback = buildRetryableFallback(reason);
|
||||
if (providerCooldownMs !== undefined) {
|
||||
return {
|
||||
@@ -1812,10 +1771,9 @@ export function checkFallbackError(
|
||||
cooldownMs: providerCooldownMs,
|
||||
baseCooldownMs: providerCooldownMs,
|
||||
configuredCooldownMs: providerCooldownMs,
|
||||
ruleScope,
|
||||
};
|
||||
}
|
||||
return { ...fallback, ruleScope };
|
||||
return fallback;
|
||||
}
|
||||
// #6842: non-backoff configured rules (e.g. status_402) previously never
|
||||
// consulted providerRuleRegistry, so a provider-specific rule (like
|
||||
@@ -1831,15 +1789,12 @@ export function checkFallbackError(
|
||||
)
|
||||
: null;
|
||||
const cooldownMs = providerMatch?.cooldownMs ?? configuredRule.cooldownMs ?? 0;
|
||||
const ruleScope =
|
||||
providerMatch && honorsRuleLockScope(provider) ? providerMatch.scope : undefined;
|
||||
return {
|
||||
shouldFallback: true,
|
||||
cooldownMs,
|
||||
baseCooldownMs: cooldownMs,
|
||||
configuredCooldownMs: cooldownMs,
|
||||
reason: providerMatch?.reason ?? configuredRule.reason ?? RateLimitReason.UNKNOWN,
|
||||
ruleScope,
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -27,10 +27,6 @@ import {
|
||||
import { RateLimitReason } from "../../config/constants.ts";
|
||||
import { isProviderCircuitOpenResult, isRequestScopedUpstreamFailure } from "./comboPredicates.ts";
|
||||
import { isCloudflareFingerprintRejection } from "../errorClassifier.ts";
|
||||
// #10334 — agentrouter-exclusive predicate shared with the persistence layer
|
||||
// (markAccountUnavailable) so the same-request combo skip and the persisted
|
||||
// connection cooldown agree on exactly which fallbackResult shapes qualify.
|
||||
import { isAgentrouterConnectionQuotaScope } from "@/sse/services/auth";
|
||||
import type { ComboLogger, ResolvedComboTarget } from "./types.ts";
|
||||
|
||||
// Connection-level failure statuses: the provider connection itself is likely bad (upstream
|
||||
@@ -64,13 +60,7 @@ export type ComboExhaustionSets = {
|
||||
|
||||
export type ApplyComboTargetExhaustionOptions = {
|
||||
result: { status: number; headers?: Headers | null };
|
||||
fallbackResult: Parameters<typeof isProviderExhaustedReason>[0] & {
|
||||
/** #10334 — agentrouter-exclusive; see isAgentrouterConnectionQuotaScope
|
||||
* (src/sse/services/auth.ts). Populated only for providers in
|
||||
* HONORS_RULE_LOCK_SCOPE_PROVIDERS (today: agentrouter only). */
|
||||
ruleScope?: "model" | "provider" | "connection";
|
||||
permanent?: boolean;
|
||||
};
|
||||
fallbackResult: Parameters<typeof isProviderExhaustedReason>[0];
|
||||
errorText: string;
|
||||
rawModel: string;
|
||||
isTokenLimitBreach: boolean;
|
||||
@@ -96,56 +86,6 @@ export function applyComboTargetExhaustion(
|
||||
const { result, sets, log, tag, errorText, structuredError } = opts;
|
||||
const provider = target.provider;
|
||||
|
||||
// #10334: agentrouter-exclusive account-wide quota exhaustion ("额度不足")
|
||||
// must skip remaining SAME-CONNECTION targets within THIS request too, not
|
||||
// just via the persisted cooldown markAccountUnavailable applies for
|
||||
// whichever leg runs next. agentrouter is a passthroughModels provider
|
||||
// (hasPerModelQuota() === true), so without this branch the classification
|
||||
// below would fall straight through isProviderQuotaExhausted's
|
||||
// !hasPerModelQuota() guard, and — for the restated-429 case —
|
||||
// markConnectionLevelExhaustion's connection-level guard (429 is not in
|
||||
// CONNECTION_LEVEL_ERROR_STATUSES), marking nothing: combo would keep
|
||||
// burning one upstream call per remaining model of the same exhausted
|
||||
// account. isAgentrouterConnectionQuotaScope is the same guard
|
||||
// markAccountUnavailable uses, so both consumers agree on exactly which
|
||||
// fallbackResult shapes qualify (never a permanent/credits-exhausted
|
||||
// result, even one carrying ruleScope "connection").
|
||||
//
|
||||
// Runs BEFORE the auth-level (401/403) branch below. This is deliberate,
|
||||
// not incidental: the "额度不足" rule matches statuses {400, 403, 429}
|
||||
// (buildAgentrouterRules, providerErrorRules.ts), and Task 1's FORBIDDEN
|
||||
// pre-check (accountFallback.ts ~1729-1751) surfaces `ruleScope:
|
||||
// "connection"` for a RAW 403 carrying that body too — so this branch can
|
||||
// also fire on a 403, not just the restated 429. That is safe: for a 403
|
||||
// this branch and markAuthLevelExhaustion below write the SAME set with
|
||||
// the SAME `${provider}:${connId}` key and both return `true` — they are
|
||||
// set-equivalent for agentrouter on that status. The Cloudflare-1010 and
|
||||
// Alibaba free-tier EXEMPTIONS further down in the 401/403 branch cannot
|
||||
// apply here regardless of ordering: 1010 is a CDN fingerprint rejection
|
||||
// agentrouter's own text never carries, and the Alibaba exemption is
|
||||
// gated on isAlibabaModelStudioProvider(provider), which agentrouter is
|
||||
// not.
|
||||
//
|
||||
// Unlike the connection-level/auth-level branches, this path deliberately
|
||||
// does NOT fall through to markTransientOrConnectionLevel, so
|
||||
// sets.transientRateLimitedProviders is NEVER populated for this failure.
|
||||
// That is required, not just incidental: combo.ts (both dispatchers, see
|
||||
// the `allowRateLimitedConnection` reads keyed off
|
||||
// transientRateLimitedProviders) uses that set to force-allow reusing a
|
||||
// rate-limited CONNECTION for the provider's remaining legs — i.e. it
|
||||
// bypasses the very `rateLimitedUntil` filter this branch (and Task 2's
|
||||
// markAccountUnavailable) just set. Marking it here would silently
|
||||
// re-open the account this branch just cooled down. One secondary
|
||||
// consequence: a SIBLING agentrouter connection that is merely
|
||||
// rate-limited (not the one this branch exhausted) will also no longer be
|
||||
// force-allowed for a later leg on the same provider — a remaining leg
|
||||
// can now resolve to "no credentials available" instead of retrying a
|
||||
// rate-limited sibling account, which is the intended, safer outcome.
|
||||
if (isAgentrouterConnectionQuotaScope(provider, opts.fallbackResult)) {
|
||||
markAgentrouterConnectionQuotaExhaustion(target, { sets, log, tag });
|
||||
return true;
|
||||
}
|
||||
|
||||
// #8133/#8137: auth-level failures (401/403) mean that connection's credentials are bad.
|
||||
// Split out to keep applyComboTargetExhaustion under the complexity ceiling.
|
||||
// Cloudflare 1010 (a 403 carrying error_code 1010 / browser_signature_banned) is NOT an
|
||||
@@ -319,35 +259,6 @@ function markAuthLevelExhaustion(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* #10334: agentrouter-exclusive connection-scope account quota exhaustion. Mirrors
|
||||
* markAuthLevelExhaustion's connectionId-present/absent split — when the target carries a
|
||||
* connectionId, only that connection's account is exhausted (sibling agentrouter connections
|
||||
* for the same user may still have quota); fall back to whole-provider exhaustion only when no
|
||||
* connectionId is available.
|
||||
*/
|
||||
function markAgentrouterConnectionQuotaExhaustion(
|
||||
target: ResolvedComboTarget,
|
||||
opts: Pick<ApplyComboTargetExhaustionOptions, "sets" | "log" | "tag">
|
||||
): void {
|
||||
const { sets, log, tag } = opts;
|
||||
const provider = target.provider;
|
||||
const connId = target.connectionId ?? undefined;
|
||||
if (connId) {
|
||||
sets.exhaustedConnections.add(`${provider}:${connId}`);
|
||||
log.info(
|
||||
tag,
|
||||
`Provider ${provider} connection ${connId} account quota exhausted (rule scope=connection) — marking for skip on remaining targets (#10334)`
|
||||
);
|
||||
} else {
|
||||
sets.exhaustedProviders.add(provider as string);
|
||||
log.info(
|
||||
tag,
|
||||
`Provider ${provider} account quota exhausted (rule scope=connection, no connectionId) — marking for skip on remaining targets (#10334)`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* #1731v2: connection-level errors (408/5xx, excluding the OmniRoute circuit-open signal) suggest
|
||||
* the provider connection itself is bad → skip remaining same-connection (or same-provider, when
|
||||
|
||||
@@ -117,21 +117,6 @@ export const OUTPUT_STYLE_CATALOG: Record<string, OutputStyle> = {
|
||||
full: `# Eu tenho TDAH — saída action-first\n\nO leitor tem TDAH. Molde a saída para que um cérebro TDAH consiga agir sobre ela:\n1. Comece pela próxima ação — comando, path ou snippet primeiro; contexto depois, se necessário.\n2. Numere trabalho multi-passo; cada passo é uma ação delimitada; use o menor número de passos que funcione.\n3. Termine com UMA próxima ação concreta executável em menos de dois minutos.\n4. Suprima tangentes: termine a primeira questão, ofereça a segunda como pergunta separada.\n5. Em trabalho multi-turno, reafirme onde as coisas estão ("passo 3 de 5 feito") — o leitor não guarda estado entre mensagens.\n6. Quando houver esforço humano, estime em unidades concretas (minutos, uma tarde), nunca "um pouco de trabalho".\n7. Torne vitórias visíveis: diga o que funciona agora e como testar.\n8. Erros de forma direta: causa e fix; nunca "Opa!".\n9. Listas com no máximo 5 itens; acima disso, divida em "agora" vs "depois".\n10. Sem preâmbulo, sem recap, sem despedidas ("Espero ter ajudado").\nExceções: pedido explícito de "explique" recebe corpo completo (ainda sem preâmbulo/despedida); ações destrutivas recebem confirmação antes; ambiguidade real recebe uma pergunta curta de esclarecimento. ${SHARED_BOUNDARIES}`,
|
||||
ultra: `# Eu tenho TDAH (ultra)\nAção primeiro: comando/path/snippet, prosa depois se precisar. Passos numerados e delimitados, o mínimo que funcione. UMA próxima ação <2 min no fim. Sem tangentes — pergunta separada. Multi-turno: reafirme o estado. Esforço humano: unidades concretas de tempo. Vitórias visíveis. Erros: causa + fix. Listas ≤5. Zero preâmbulo/recap/despedidas. "Explique" recebe corpo completo; ação destrutiva recebe confirmação; ambiguidade real recebe uma pergunta. ${SHARED_BOUNDARIES}`,
|
||||
},
|
||||
vi: {
|
||||
lite: `# Tôi bị ADHD (rút gọn)\nBắt đầu bằng hành động: lệnh, đường dẫn hoặc đoạn mã trước, văn xuôi sau. Đánh số công việc nhiều bước; mỗi bước là một hành động giới hạn. Kết thúc bằng MỘT hành động cụ thể tiếp theo. Không mở đầu, không tóm tắt lại, không lời chào cuối. ${SHARED_BOUNDARIES}`,
|
||||
full: `# Tôi bị ADHD — đầu ra ưu tiên hành động\n\nNgười đọc bị ADHD. Hãy định hình đầu ra để một bộ não ADHD có thể hành động ngay:\n1. Mở đầu bằng hành động kế tiếp — lệnh, đường dẫn hoặc đoạn mã trước; ngữ cảnh sau, nếu cần.\n2. Đánh số công việc nhiều bước; mỗi bước là một hành động giới hạn; dùng ít bước nhất mà vẫn chạy được.\n3. Kết thúc bằng MỘT hành động cụ thể làm được dưới hai phút.\n4. Chặn lạc đề: xong việc thứ nhất, việc thứ hai đưa ra thành câu hỏi riêng.\n5. Trong công việc nhiều lượt, nhắc lại đang ở đâu ("xong bước 3 trên 5") — người đọc không giữ trạng thái giữa các tin nhắn.\n6. Khi có công sức của con người, ước lượng bằng đơn vị cụ thể (phút, một buổi chiều), không bao giờ nói "hơi tốn công".\n7. Cho thấy kết quả: nói rõ cái gì đã chạy được và thử thế nào.\n8. Báo lỗi thẳng thắn: nguyên nhân và cách sửa; không "Ôi không".\n9. Danh sách tối đa 5 mục; nhiều hơn thì tách "làm ngay" và "để sau".\n10. Không mở đầu, không tóm tắt lại, không lời chào cuối ("Hy vọng giúp ích").\nNgoại lệ: yêu cầu "giải thích" thì viết đầy đủ (vẫn không mở đầu/chào cuối); hành động phá huỷ phải xác nhận trước; mơ hồ thật sự thì hỏi một câu ngắn. ${SHARED_BOUNDARIES}`,
|
||||
ultra: `# Tôi bị ADHD (siêu gọn)\nHành động trước: lệnh/đường dẫn/đoạn mã, văn xuôi sau nếu cần. Bước đánh số, giới hạn, ít nhất có thể. MỘT hành động <2 phút ở cuối. Không lạc đề — hỏi riêng. Nhiều lượt: nhắc lại trạng thái. Công sức người: đơn vị thời gian cụ thể. Kết quả rõ ràng. Lỗi: nguyên nhân + cách sửa. Danh sách ≤5. Không mở đầu/tóm tắt/chào cuối. "Giải thích" thì viết đầy đủ; hành động phá huỷ phải xác nhận; mơ hồ thật thì hỏi một câu. ${SHARED_BOUNDARIES}`,
|
||||
},
|
||||
ja: {
|
||||
lite: `# ADHDです（軽量）\n行動から始める：コマンド、パス、スニペットを先に、散文は後。複数手順は番号付き；各手順は一つの区切られた行動。最後は具体的な次の行動を一つ。前置きなし、要約の繰り返しなし、締めの挨拶なし。${SHARED_BOUNDARIES}`,
|
||||
full: `# ADHDです — 行動優先の出力\n\n読み手はADHDです。ADHDの脳が動けるように出力を整えること：\n1. 次の行動から始める — コマンド、パス、スニペットを先に；文脈は必要なら後。\n2. 複数手順は番号付き；各手順は一つの区切られた行動；動く最小の手順数で。\n3. 最後は2分以内でできる具体的な次の行動を一つ。\n4. 脱線を抑える：最初の件を終えてから、二件目は別の質問として出す。\n5. 複数ターンの作業では現在地を言い直す（「5つ中3つ完了」）— 読み手はメッセージ間で状態を保持できない。\n6. 人手がかかる場合は具体的な単位で見積もる（分、半日）。「少し手間」は禁止。\n7. 成果を見せる：今何が動くか、どう試すかを述べる。\n8. エラーは淡々と：原因と対処；「おっと」は禁止。\n9. リストは5項目まで；超えるなら「今やる」と「後で」に分ける。\n10. 前置きなし、要約の繰り返しなし、締めの挨拶なし（「お役に立てば幸いです」）。\n例外：明示的な「説明して」には本文を十分に書く（前置き・締めはなし）；破壊的操作は先に確認；本当に曖昧なら短い確認質問を一つ。${SHARED_BOUNDARIES}`,
|
||||
ultra: `# ADHDです（超軽量）\n行動優先：コマンド/パス/スニペット、必要なら散文。番号付きの区切られた手順、動く最小限。最後に2分未満の次の行動を一つ。脱線なし — 別の質問へ。複数ターン：状態を言い直す。人手：具体的な時間単位。成果を明示。エラー：原因＋対処。リストは5まで。前置き/要約/締めの挨拶はゼロ。「説明して」には本文を十分に；破壊的操作は確認；本当の曖昧さには質問を一つ。${SHARED_BOUNDARIES}`,
|
||||
},
|
||||
id: {
|
||||
lite: `# Saya punya ADHD (ringkas)\nMulai dari aksi: perintah, path, atau cuplikan kode dulu, prosa belakangan. Beri nomor untuk pekerjaan banyak langkah; tiap langkah satu aksi yang terbatas. Akhiri dengan SATU langkah berikutnya yang konkret. Tanpa pembuka, tanpa rekap, tanpa basa-basi penutup. ${SHARED_BOUNDARIES}`,
|
||||
full: `# Saya punya ADHD — keluaran yang mengutamakan aksi\n\nPembaca punya ADHD. Bentuk keluaran supaya otak ADHD bisa langsung bertindak:\n1. Mulai dari aksi berikutnya — perintah, path, atau cuplikan kode dulu; konteks belakangan, kalau perlu.\n2. Beri nomor untuk pekerjaan banyak langkah; tiap langkah satu aksi terbatas; pakai langkah sesedikit mungkin yang tetap jalan.\n3. Akhiri dengan SATU langkah konkret yang bisa dikerjakan di bawah dua menit.\n4. Tahan bahasan sampingan: selesaikan yang pertama, tawarkan yang kedua sebagai pertanyaan terpisah.\n5. Pada pekerjaan banyak giliran, ulangi posisi saat ini ("langkah 3 dari 5 selesai") — pembaca tidak menyimpan status antar pesan.\n6. Kalau ada usaha manusia, perkirakan dalam satuan konkret (menit, satu sore), jangan "agak butuh kerja".\n7. Tunjukkan hasil: sebutkan apa yang sekarang jalan dan cara mencobanya.\n8. Error apa adanya: sebab dan perbaikannya; jangan "Waduh".\n9. Daftar maksimal 5 butir; lebih dari itu pisahkan "sekarang" dan "nanti".\n10. Tanpa pembuka, tanpa rekap, tanpa basa-basi penutup ("Semoga membantu").\nPengecualian: permintaan eksplisit "jelaskan" dapat isi penuh (tetap tanpa pembuka/penutup); aksi merusak dikonfirmasi dulu; ambiguitas nyata dapat satu pertanyaan singkat. ${SHARED_BOUNDARIES}`,
|
||||
ultra: `# Saya punya ADHD (ultra)\nAksi dulu: perintah/path/cuplikan, prosa kalau perlu. Langkah bernomor dan terbatas, sesedikit mungkin. SATU langkah <2 menit di akhir. Tanpa bahasan sampingan — jadikan pertanyaan terpisah. Banyak giliran: ulangi status. Usaha manusia: satuan waktu konkret. Hasil terlihat. Error: sebab + perbaikan. Daftar ≤5. Nol pembuka/rekap/penutup. "Jelaskan" dapat isi penuh; aksi merusak dikonfirmasi; ambiguitas nyata dapat satu pertanyaan. ${SHARED_BOUNDARIES}`,
|
||||
},
|
||||
},
|
||||
},
|
||||
"terse-cjk": {
|
||||
|
||||
@@ -42,7 +42,6 @@ export const INTENTIONALLY_INTERNAL = new Set([
|
||||
"accessTokens", // intentionally-internal: 4 rotas /api/cli/* (connect, whoami, tokens, tokens/[id]) + server/authz/accessTokenAuth.ts via import direto "@/lib/db/accessTokens" (Rule #2)
|
||||
"apiKeyColumnFallbacks", // db-internal: importado só por db/apiKeys.ts (API_KEY_COLUMN_FALLBACKS — fallbacks de coluna split do apiKeys.ts)
|
||||
"apiKeyUsageLimitFields", // db-internal: importado só por db/apiKeys.ts (helpers de campo de limite de uso split do apiKeys.ts; mig 101)
|
||||
"backupRetention", // db-internal: importado só por db/backup.ts e db/migrationRunner.ts (política de retenção compartilhada; mora fora de backup.ts porque core.ts importa migrationRunner.ts — importar backup.ts de lá fecharia um ciclo, #10421)
|
||||
"caseMapping", // db-internal: importado só por db/core.ts (toSnakeCase/toCamelCase/objToSnake — column-mapping snake↔camel split do core.ts, #4947)
|
||||
"cleanup", // intentionally-internal: 3 API routes (purge-quota-snapshots, purge-call-logs, purge-detailed-logs)
|
||||
"cliToolState", // intentionally-internal: 14+ API routes em /api/cli-tools/*-settings
|
||||
|
||||
@@ -14,13 +14,6 @@ import {
|
||||
DATA_DIR,
|
||||
} from "./core";
|
||||
import { resetAllDbModuleState } from "./stateReset";
|
||||
import {
|
||||
MAX_DB_BACKUPS,
|
||||
DEFAULT_DB_BACKUP_RETENTION_DAYS,
|
||||
parsePositiveInt,
|
||||
parseNonNegativeInt,
|
||||
pruneBackupDirectory,
|
||||
} from "./backupRetention";
|
||||
import { isAutomatedTestProcess } from "@/shared/utils/testProcess";
|
||||
|
||||
type CountRow = { cnt?: number };
|
||||
@@ -29,8 +22,22 @@ type CountRow = { cnt?: number };
|
||||
|
||||
let _lastBackupAt = 0;
|
||||
const BACKUP_THROTTLE_MS = 60 * 60 * 1000; // 60 minutes
|
||||
const MAX_DB_BACKUPS = 20;
|
||||
const DEFAULT_DB_BACKUP_RETENTION_DAYS = 0;
|
||||
const TRUE_ENV_VALUES = new Set(["1", "true", "yes", "on"]);
|
||||
|
||||
function parsePositiveInt(value: string | undefined, fallback: number) {
|
||||
if (!value) return fallback;
|
||||
const parsed = Number.parseInt(value, 10);
|
||||
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
function parseNonNegativeInt(value: string | undefined, fallback: number) {
|
||||
if (value === undefined) return fallback;
|
||||
const parsed = Number.parseInt(value, 10);
|
||||
return Number.isInteger(parsed) && parsed >= 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
// #3834: the "Keep latest backups" UI value is persisted here so it survives a page
|
||||
// refresh / the loadStorageHealth() refetch. A dedicated namespace avoids any
|
||||
// cross-talk with the databaseSettings key_value store (which rewrites all of its own
|
||||
@@ -101,16 +108,108 @@ function getBackupDir() {
|
||||
return DB_BACKUPS_DIR || path.join(DATA_DIR, "db_backups");
|
||||
}
|
||||
|
||||
export function cleanupDbBackups(options?: {
|
||||
maxFiles?: number;
|
||||
retentionDays?: number;
|
||||
backupDir?: string;
|
||||
}) {
|
||||
const backupDir = options?.backupDir ?? getBackupDir();
|
||||
const maxFiles = options?.maxFiles ?? getDbBackupMaxFiles();
|
||||
const retentionDays = options?.retentionDays ?? getDbBackupRetentionDays();
|
||||
function getBackupFamilyBase(filename: string) {
|
||||
if (filename.endsWith("-wal") || filename.endsWith("-shm")) return filename.slice(0, -4);
|
||||
if (filename.endsWith("-journal")) return filename.slice(0, -8);
|
||||
return filename;
|
||||
}
|
||||
|
||||
return pruneBackupDirectory({ backupDir, maxFiles, retentionDays });
|
||||
function collectBackupFamilies(backupDir: string) {
|
||||
if (!fs.existsSync(backupDir)) return [];
|
||||
|
||||
const families = new Map<
|
||||
string,
|
||||
{
|
||||
base: string;
|
||||
hasPrimary: boolean;
|
||||
primaryMtimeMs: number;
|
||||
latestMtimeMs: number;
|
||||
files: string[];
|
||||
}
|
||||
>();
|
||||
|
||||
for (const name of fs.readdirSync(backupDir)) {
|
||||
if (!name.startsWith("db_")) continue;
|
||||
const base = getBackupFamilyBase(name);
|
||||
const filePath = path.join(backupDir, name);
|
||||
|
||||
let stat;
|
||||
try {
|
||||
stat = fs.statSync(filePath);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
const family = families.get(base) || {
|
||||
base,
|
||||
hasPrimary: false,
|
||||
primaryMtimeMs: 0,
|
||||
latestMtimeMs: 0,
|
||||
files: [],
|
||||
};
|
||||
|
||||
family.files.push(name);
|
||||
family.latestMtimeMs = Math.max(family.latestMtimeMs, stat.mtimeMs);
|
||||
if (name === base && name.endsWith(".sqlite")) {
|
||||
family.hasPrimary = true;
|
||||
family.primaryMtimeMs = stat.mtimeMs;
|
||||
}
|
||||
|
||||
families.set(base, family);
|
||||
}
|
||||
|
||||
return [...families.values()];
|
||||
}
|
||||
|
||||
export function cleanupDbBackups(options?: { maxFiles?: number; retentionDays?: number }) {
|
||||
const backupDir = getBackupDir();
|
||||
if (!fs.existsSync(backupDir)) {
|
||||
return {
|
||||
deletedBackupFamilies: 0,
|
||||
deletedFiles: 0,
|
||||
keptBackupFamilies: 0,
|
||||
maxFiles: options?.maxFiles ?? getDbBackupMaxFiles(),
|
||||
retentionDays: options?.retentionDays ?? getDbBackupRetentionDays(),
|
||||
};
|
||||
}
|
||||
|
||||
const maxFiles = Math.max(1, options?.maxFiles ?? getDbBackupMaxFiles());
|
||||
const retentionDays = Math.max(0, options?.retentionDays ?? getDbBackupRetentionDays());
|
||||
const cutoffMs = retentionDays > 0 ? Date.now() - retentionDays * 24 * 60 * 60 * 1000 : 0;
|
||||
const families = collectBackupFamilies(backupDir);
|
||||
const primaryFamilies = families
|
||||
.filter((family) => family.hasPrimary)
|
||||
.sort((a, b) => b.primaryMtimeMs - a.primaryMtimeMs);
|
||||
const keepPrimaryBases = new Set(primaryFamilies.slice(0, maxFiles).map((family) => family.base));
|
||||
|
||||
let deletedBackupFamilies = 0;
|
||||
let deletedFiles = 0;
|
||||
|
||||
for (const family of families) {
|
||||
const isOverflowPrimary = family.hasPrimary && !keepPrimaryBases.has(family.base);
|
||||
const isExpired = retentionDays > 0 && family.latestMtimeMs < cutoffMs;
|
||||
const isOrphan = !family.hasPrimary;
|
||||
if (!isOverflowPrimary && !isExpired && !isOrphan) continue;
|
||||
|
||||
deletedBackupFamilies += 1;
|
||||
for (const name of family.files) {
|
||||
try {
|
||||
fs.unlinkSync(path.join(backupDir, name));
|
||||
deletedFiles += 1;
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
deletedBackupFamilies,
|
||||
deletedFiles,
|
||||
keptBackupFamilies: collectBackupFamilies(backupDir).filter((family) => family.hasPrimary)
|
||||
.length,
|
||||
maxFiles,
|
||||
retentionDays,
|
||||
};
|
||||
}
|
||||
|
||||
function coerceBoolean(value: unknown): boolean | null {
|
||||
|
||||
@@ -1,158 +0,0 @@
|
||||
/**
|
||||
* Backup retention primitives — pure filesystem work, no `core.ts` dependency.
|
||||
*
|
||||
* This module exists so BOTH backup call sites can share one retention policy:
|
||||
*
|
||||
* - `backup.ts` (manual/API/auto backups) — resolves the operator's settings from the
|
||||
* database and delegates here.
|
||||
* - `migrationRunner.ts` (pre-migration snapshots) — cannot import `backup.ts`, because
|
||||
* `core.ts` already imports `migrationRunner.ts` and `backup.ts` imports `core.ts`;
|
||||
* that edge would close a cycle. Keeping the policy here, free of `core`, lets the
|
||||
* migration path prune without one.
|
||||
*
|
||||
* Before #10421 the migration path had no retention at all and `db_backups/` grew
|
||||
* without bound (observed: 48.999 files / 204 GB against a 5,3 MB live database).
|
||||
*/
|
||||
|
||||
import fs from "fs";
|
||||
import path from "path";
|
||||
|
||||
export const MAX_DB_BACKUPS = 20;
|
||||
export const DEFAULT_DB_BACKUP_RETENTION_DAYS = 0;
|
||||
|
||||
export function parsePositiveInt(value: string | undefined, fallback: number) {
|
||||
if (!value) return fallback;
|
||||
const parsed = Number.parseInt(value, 10);
|
||||
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
export function parseNonNegativeInt(value: string | undefined, fallback: number) {
|
||||
if (value === undefined) return fallback;
|
||||
const parsed = Number.parseInt(value, 10);
|
||||
return Number.isInteger(parsed) && parsed >= 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
/**
|
||||
* A backup "family" is the primary `.sqlite` file plus its SQLite sidecars
|
||||
* (`-wal` / `-shm` / `-journal`). Retention operates on families so a sidecar is never
|
||||
* orphaned from — or outlives — the snapshot it belongs to.
|
||||
*/
|
||||
export function getBackupFamilyBase(filename: string) {
|
||||
if (filename.endsWith("-wal") || filename.endsWith("-shm")) return filename.slice(0, -4);
|
||||
if (filename.endsWith("-journal")) return filename.slice(0, -8);
|
||||
return filename;
|
||||
}
|
||||
|
||||
export type BackupFamily = {
|
||||
base: string;
|
||||
hasPrimary: boolean;
|
||||
primaryMtimeMs: number;
|
||||
latestMtimeMs: number;
|
||||
files: string[];
|
||||
};
|
||||
|
||||
export function collectBackupFamilies(backupDir: string): BackupFamily[] {
|
||||
if (!fs.existsSync(backupDir)) return [];
|
||||
|
||||
const families = new Map<string, BackupFamily>();
|
||||
|
||||
for (const name of fs.readdirSync(backupDir)) {
|
||||
if (!name.startsWith("db_")) continue;
|
||||
const base = getBackupFamilyBase(name);
|
||||
const filePath = path.join(backupDir, name);
|
||||
|
||||
let stat;
|
||||
try {
|
||||
stat = fs.statSync(filePath);
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
|
||||
const family = families.get(base) || {
|
||||
base,
|
||||
hasPrimary: false,
|
||||
primaryMtimeMs: 0,
|
||||
latestMtimeMs: 0,
|
||||
files: [],
|
||||
};
|
||||
|
||||
family.files.push(name);
|
||||
family.latestMtimeMs = Math.max(family.latestMtimeMs, stat.mtimeMs);
|
||||
if (name === base && name.endsWith(".sqlite")) {
|
||||
family.hasPrimary = true;
|
||||
family.primaryMtimeMs = stat.mtimeMs;
|
||||
}
|
||||
|
||||
families.set(base, family);
|
||||
}
|
||||
|
||||
return [...families.values()];
|
||||
}
|
||||
|
||||
export type PruneResult = {
|
||||
deletedBackupFamilies: number;
|
||||
deletedFiles: number;
|
||||
keptBackupFamilies: number;
|
||||
maxFiles: number;
|
||||
retentionDays: number;
|
||||
};
|
||||
|
||||
/**
|
||||
* Delete backup families beyond `maxFiles` (newest kept), older than `retentionDays`
|
||||
* (0 disables the age rule), or orphaned (sidecars whose primary is already gone).
|
||||
*/
|
||||
export function pruneBackupDirectory(options: {
|
||||
backupDir: string;
|
||||
maxFiles: number;
|
||||
retentionDays: number;
|
||||
}): PruneResult {
|
||||
const { backupDir } = options;
|
||||
const maxFiles = Math.max(1, options.maxFiles);
|
||||
const retentionDays = Math.max(0, options.retentionDays);
|
||||
|
||||
if (!fs.existsSync(backupDir)) {
|
||||
return {
|
||||
deletedBackupFamilies: 0,
|
||||
deletedFiles: 0,
|
||||
keptBackupFamilies: 0,
|
||||
maxFiles,
|
||||
retentionDays,
|
||||
};
|
||||
}
|
||||
|
||||
const cutoffMs = retentionDays > 0 ? Date.now() - retentionDays * 24 * 60 * 60 * 1000 : 0;
|
||||
const families = collectBackupFamilies(backupDir);
|
||||
const primaryFamilies = families
|
||||
.filter((family) => family.hasPrimary)
|
||||
.sort((a, b) => b.primaryMtimeMs - a.primaryMtimeMs);
|
||||
const keepPrimaryBases = new Set(primaryFamilies.slice(0, maxFiles).map((family) => family.base));
|
||||
|
||||
let deletedBackupFamilies = 0;
|
||||
let deletedFiles = 0;
|
||||
|
||||
for (const family of families) {
|
||||
const isOverflowPrimary = family.hasPrimary && !keepPrimaryBases.has(family.base);
|
||||
const isExpired = retentionDays > 0 && family.latestMtimeMs < cutoffMs;
|
||||
const isOrphan = !family.hasPrimary;
|
||||
if (!isOverflowPrimary && !isExpired && !isOrphan) continue;
|
||||
|
||||
deletedBackupFamilies += 1;
|
||||
for (const name of family.files) {
|
||||
try {
|
||||
fs.unlinkSync(path.join(backupDir, name));
|
||||
deletedFiles += 1;
|
||||
} catch {
|
||||
/* ignore */
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
deletedBackupFamilies,
|
||||
deletedFiles,
|
||||
keptBackupFamilies: collectBackupFamilies(backupDir).filter((family) => family.hasPrimary)
|
||||
.length,
|
||||
maxFiles,
|
||||
retentionDays,
|
||||
};
|
||||
}
|
||||
@@ -29,15 +29,6 @@ import {
|
||||
OPTIONAL_FTS5_MIGRATION_VERSIONS,
|
||||
} from "./migrationRunner/constants";
|
||||
import { getExtraMigrationFiles } from "./migrationRunner/extraDirs";
|
||||
// Retention primitives live in their own `core`-free module: `core.ts` imports this file,
|
||||
// so importing `backup.ts` (which imports `core.ts`) here would close a dependency cycle.
|
||||
import {
|
||||
MAX_DB_BACKUPS,
|
||||
DEFAULT_DB_BACKUP_RETENTION_DAYS,
|
||||
parsePositiveInt,
|
||||
parseNonNegativeInt,
|
||||
pruneBackupDirectory,
|
||||
} from "./backupRetention";
|
||||
|
||||
const isNodeTestRunnerChild = typeof process.env.NODE_TEST_CONTEXT === "string";
|
||||
|
||||
@@ -814,56 +805,6 @@ function rehomeLegacyVersionSlotMigrations(
|
||||
return repaired;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read a persisted `dbBackup` retention setting through the adapter that is ALREADY open
|
||||
* for this migration run.
|
||||
*
|
||||
* `backup.ts`'s equivalent goes through `getDbInstance()`, which is unsafe here: this
|
||||
* code runs from inside database initialization, so asking for the singleton would
|
||||
* re-enter it. Reading off `db` keeps the same stored values without that risk. A DB too
|
||||
* old to have `key_value` yet simply falls back to the default.
|
||||
*/
|
||||
function readStoredBackupSetting(db: SqliteAdapter, key: string, min: number): number | undefined {
|
||||
try {
|
||||
const row = db
|
||||
.prepare("SELECT value FROM key_value WHERE namespace = ? AND key = ?")
|
||||
.get("dbBackup", key) as { value?: string } | undefined;
|
||||
if (!row?.value) return undefined;
|
||||
const parsed = JSON.parse(row.value);
|
||||
return Number.isInteger(parsed) && parsed >= min ? parsed : undefined;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Enforce the backup retention budget after a pre-migration snapshot (#10421).
|
||||
*
|
||||
* Precedence matches `backup.ts`: env override → persisted operator setting → default.
|
||||
* Never throws: a migration must not fail because housekeeping did.
|
||||
*/
|
||||
function pruneMigrationBackups(db: SqliteAdapter, backupDir: string): void {
|
||||
try {
|
||||
const maxFiles = process.env.DB_BACKUP_MAX_FILES
|
||||
? parsePositiveInt(process.env.DB_BACKUP_MAX_FILES, MAX_DB_BACKUPS)
|
||||
: (readStoredBackupSetting(db, "maxFiles", 1) ?? MAX_DB_BACKUPS);
|
||||
const retentionDays = process.env.DB_BACKUP_RETENTION_DAYS
|
||||
? parseNonNegativeInt(process.env.DB_BACKUP_RETENTION_DAYS, DEFAULT_DB_BACKUP_RETENTION_DAYS)
|
||||
: (readStoredBackupSetting(db, "retentionDays", 0) ?? DEFAULT_DB_BACKUP_RETENTION_DAYS);
|
||||
|
||||
const result = pruneBackupDirectory({ backupDir, maxFiles, retentionDays });
|
||||
if (result.deletedFiles > 0) {
|
||||
console.log(
|
||||
`[Migration] Pruned ${result.deletedFiles} old backup file(s) ` +
|
||||
`(${result.keptBackupFamilies} kept, maxFiles=${maxFiles}, retentionDays=${retentionDays}).`
|
||||
);
|
||||
}
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
console.warn(`[Migration] Failed to prune old backups: ${message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Create a pre-migration backup of the SQLite database using VACUUM INTO.
|
||||
* Returns the backup path on success, null on failure.
|
||||
@@ -884,12 +825,6 @@ function createPreMigrationBackup(db: SqliteAdapter): string | null {
|
||||
|
||||
db.exec(`VACUUM INTO '${escapedBackupPath}'`);
|
||||
console.log(`[Migration] Pre-migration backup created: ${backupPath}`);
|
||||
|
||||
// #10421: apply the operator's retention budget right here. Without this the
|
||||
// migration path was the one backup producer that never pruned, so every process
|
||||
// start with a pending migration added ~5 MB forever (observed: 49k files / 204 GB).
|
||||
pruneMigrationBackups(db, backupDir);
|
||||
|
||||
return backupPath;
|
||||
} catch (err: unknown) {
|
||||
const message = err instanceof Error ? err.message : String(err);
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { randomUUID, createHash } from "crypto";
|
||||
import { nodeTypeFromId } from "@/lib/db/providerNodeSelect";
|
||||
import { extractGoogApiKeyHeader } from "./googApiKeyAuth.ts";
|
||||
import {
|
||||
getCachedRawProviderConnections,
|
||||
@@ -45,7 +46,6 @@ import {
|
||||
} from "@omniroute/open-sse/services/accountFallback.ts";
|
||||
import { isLocalProvider } from "@omniroute/open-sse/config/providerRegistry.ts";
|
||||
import { COOLDOWN_MS, RateLimitReason } from "@omniroute/open-sse/config/constants.ts";
|
||||
import { honorsRuleLockScope } from "@omniroute/open-sse/config/providerErrorRules.ts";
|
||||
import {
|
||||
preflightQuota,
|
||||
isQuotaPreflightEnabled,
|
||||
@@ -968,14 +968,33 @@ async function getProviderSearchPool(provider: string): Promise<string[]> {
|
||||
const nodeRecord = asRecord(node);
|
||||
const nodePrefix = typeof nodeRecord.prefix === "string" ? nodeRecord.prefix.trim() : "";
|
||||
const nodeId = typeof nodeRecord.id === "string" ? nodeRecord.id.trim() : "";
|
||||
if (!nodePrefix || !nodeId) continue;
|
||||
if (!nodeId) continue;
|
||||
if (
|
||||
nodePrefix === provider ||
|
||||
nodePrefix === canonicalProvider ||
|
||||
nodePrefix === canonicalAlias
|
||||
nodePrefix &&
|
||||
(nodePrefix === provider || nodePrefix === canonicalProvider || nodePrefix === canonicalAlias)
|
||||
) {
|
||||
searchPool.add(nodeId);
|
||||
}
|
||||
|
||||
// #10085: bridge the concrete uuid node id (what the chat path resolves,
|
||||
// "<generic-type>-<uuid>") to the GENERIC derived type id (what
|
||||
// resolveProviderNodeForConnection also accepts for connection creation,
|
||||
// #4421) -- and back. A connection created via the bare generic type
|
||||
// (e.g. "openai-compatible-chat") must still be found when the chat path
|
||||
// looks up the concrete node id, and vice versa.
|
||||
const derivedType = nodeTypeFromId(nodeId);
|
||||
if (derivedType && derivedType !== nodeId) {
|
||||
if (nodeId === provider || nodeId === canonicalProvider || nodeId === canonicalAlias) {
|
||||
searchPool.add(derivedType);
|
||||
}
|
||||
if (
|
||||
derivedType === provider ||
|
||||
derivedType === canonicalProvider ||
|
||||
derivedType === canonicalAlias
|
||||
) {
|
||||
searchPool.add(nodeId);
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Best-effort alias expansion only.
|
||||
@@ -1982,46 +2001,6 @@ export async function getProviderCredentialsWithQuotaPreflight(
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* #10334 — Guard for the agentrouter-exclusive "connection scope" quota
|
||||
* cooldown branch in markAccountUnavailable. The "never terminal" invariant of
|
||||
* that branch is NOT structurally guaranteed by `ruleScope === "connection"`
|
||||
* alone — it also depends on the provider rule table only ever pairing scope
|
||||
* "connection" with a genuinely transient reason. Today
|
||||
* (`buildAgentrouterRules()` in providerErrorRules.ts) that is true: the only
|
||||
* rule declaring scope "connection" is the quota-exhausted one. But a FUTURE
|
||||
* agentrouter rule for a permanent account state (e.g. "账号已封禁") — or a 402
|
||||
* added to `AGENTROUTER_ERROR_STATUSES` with scope "connection", a natural-
|
||||
* looking choice for an account ban — would otherwise be silently downgraded
|
||||
* to a transient cooldown here instead of going through
|
||||
* resolveTerminalConnectionStatus()/auto-disable below. Require the
|
||||
* reason/permanent/creditsExhausted signals checkFallbackError already
|
||||
* computes to explicitly confirm "this is quota, not a permanent state"
|
||||
* before taking the early return.
|
||||
*
|
||||
* Exported (not just inlined) so a synthetic permanent/credits-exhausted
|
||||
* `fallbackResult` can be tested directly — no rule in the table produces
|
||||
* that combination today, so this predicate is the only way to pin the guard
|
||||
* without editing the (production) rule table just for a test.
|
||||
*/
|
||||
export function isAgentrouterConnectionQuotaScope(
|
||||
provider: string | null | undefined,
|
||||
fallbackResult: {
|
||||
ruleScope?: "model" | "provider" | "connection";
|
||||
reason?: string;
|
||||
permanent?: boolean;
|
||||
creditsExhausted?: boolean;
|
||||
}
|
||||
): boolean {
|
||||
return (
|
||||
honorsRuleLockScope(provider) &&
|
||||
fallbackResult.ruleScope === "connection" &&
|
||||
fallbackResult.reason === RateLimitReason.QUOTA_EXHAUSTED &&
|
||||
!fallbackResult.permanent &&
|
||||
!fallbackResult.creditsExhausted
|
||||
);
|
||||
}
|
||||
|
||||
/** Persist exponential-backoff state for an unavailable provider connection. */
|
||||
export async function markAccountUnavailable(
|
||||
connectionId: string,
|
||||
@@ -2142,53 +2121,6 @@ export async function markAccountUnavailable(
|
||||
const disableCooling = connProviderSpecificData.disableCooling === true;
|
||||
|
||||
const isPerModelQuotaProvider = hasPerModelQuota(provider, model, connectionPassthroughModels);
|
||||
|
||||
// #10334 — agentrouter EXCLUSIVE: the matched provider rule declared scope
|
||||
// "connection" for account-wide quota exhaustion ("额度不足"). agentrouter is
|
||||
// a passthroughModels provider (isPerModelQuotaProvider === true), so without
|
||||
// this branch the next `if` would treat it like any other passthrough 429 and
|
||||
// lock a SINGLE model — leaving combo routing to burn one upstream call per
|
||||
// remaining model of the same exhausted account. Must run BEFORE that block.
|
||||
// Deliberately ignores persistUnavailableState/isCombo: for combo the caller
|
||||
// downgrades persistUnavailableState to false, and the generic path further
|
||||
// below would then lock per MODEL instead of cooling the connection — exactly
|
||||
// what this scope must override. NEVER sets a terminal status: this is a
|
||||
// renewing quota window, not "credits_exhausted"/"banned"/"expired".
|
||||
//
|
||||
// The "never terminal" invariant above is NOT structurally guaranteed by
|
||||
// ruleScope === "connection" alone — see isAgentrouterConnectionQuotaScope's
|
||||
// doc comment for why (a future permanent-state rule could pair scope
|
||||
// "connection" with a non-quota reason). That predicate is the actual guard.
|
||||
const ruleScopeIsConnection = isAgentrouterConnectionQuotaScope(provider, fallbackResult);
|
||||
// #2997's disableCooling opt-out is respected here (`!disableCooling` below):
|
||||
// a connection with disableCooling=true skips this branch entirely and falls
|
||||
// into the per-model-quota block further down, which locks the model for up
|
||||
// to ~30min (mlSettings.maxCooldownMs) instead of cooling the connection for
|
||||
// the rule's shorter transient window. That is a deliberate, if counter-
|
||||
// intuitive, consequence of #2997's scope (opt-out was designed only for the
|
||||
// CONNECTION-level cooldown, never extended to model lockout) — "opting out
|
||||
// of cooldown" ends up producing a LONGER effective block for this one rule.
|
||||
// Not addressed here; flagged for a future #2997 follow-up if it proves to be
|
||||
// a real operator complaint.
|
||||
if (ruleScopeIsConnection && provider && !disableCooling) {
|
||||
const connectionCooldownMs =
|
||||
fallbackResult.cooldownMs > 0 ? fallbackResult.cooldownMs : COOLDOWN_MS.rateLimit;
|
||||
await updateProviderConnection(connectionId, {
|
||||
lastErrorType: fallbackResult.reason || RateLimitReason.QUOTA_EXHAUSTED,
|
||||
lastError: `Account quota exhausted (${provider})`,
|
||||
lastErrorAt: new Date().toISOString(),
|
||||
errorCode: status,
|
||||
backoffLevel: fallbackResult.newBackoffLevel ?? backoffLevel,
|
||||
rateLimitedUntil: getUnavailableUntil(connectionCooldownMs),
|
||||
testStatus: "unavailable",
|
||||
});
|
||||
log.info(
|
||||
"AUTH",
|
||||
`Connection-scoped cooldown for ${provider}:${connectionId.slice(0, 8)} — ${status} ${fallbackResult.reason} ${Math.ceil(connectionCooldownMs / 1000)}s (rule scope=connection, overrides per-model lockout)`
|
||||
);
|
||||
return { shouldFallback: true, cooldownMs: connectionCooldownMs };
|
||||
}
|
||||
|
||||
const isNvidiaModelGone = provider === "nvidia" && status === 410;
|
||||
const modelLockoutOptions = { maxCooldownMs: effectiveProviderProfile?.maxCooldownMs };
|
||||
if (
|
||||
|
||||
@@ -64,7 +64,6 @@
|
||||
"tests/unit/adaptive-admission-runtime.test.ts",
|
||||
"tests/unit/adobe-firefly.test.ts",
|
||||
"tests/unit/agentrouter-error-rules.test.ts",
|
||||
"tests/unit/agentrouter-lock-scope-10334.test.ts",
|
||||
"tests/unit/alibaba-free-tier-exhaustion.test.ts",
|
||||
"tests/unit/anthropic-thinking-signature-recovery.test.ts",
|
||||
"tests/unit/antigravity-429-quota-tdd.test.ts",
|
||||
|
||||
@@ -565,8 +565,8 @@
|
||||
}
|
||||
},
|
||||
"url": {
|
||||
"nonStream": "https://token-plan.ap-southeast-1.maas.aliyuncs.com/apps/anthropic/v1",
|
||||
"stream": "https://token-plan.ap-southeast-1.maas.aliyuncs.com/apps/anthropic/v1"
|
||||
"nonStream": "https://coding-intl.dashscope.aliyuncs.com/apps/anthropic/v1",
|
||||
"stream": "https://coding-intl.dashscope.aliyuncs.com/apps/anthropic/v1"
|
||||
}
|
||||
},
|
||||
"baseten": {
|
||||
|
||||
152
tests/unit/10085-compatible-generic-vs-uuid-credential.test.ts
Normal file
152
tests/unit/10085-compatible-generic-vs-uuid-credential.test.ts
Normal file
@@ -0,0 +1,152 @@
|
||||
/**
|
||||
* #10085 -- a custom openai-compatible provider connection persisted under the
|
||||
* GENERIC derived type id ("openai-compatible-chat") must still be reachable
|
||||
* when the chat path looks up the concrete uuid node id
|
||||
* ("openai-compatible-chat-<uuid>"), and vice versa.
|
||||
*
|
||||
* `resolveProviderNodeForConnection` (src/lib/db/providers/nodes.ts, #4421)
|
||||
* already accepts the bare generic type id when a connection is created via
|
||||
* `/api/providers`. But `getProviderSearchPool` (src/sse/services/auth.ts)
|
||||
* only bridged the search pool via a node's `prefix`, never via the generic
|
||||
* type id <-> concrete node id relationship, so a connection created under
|
||||
* the generic type id went permanently unreachable from the chat path --
|
||||
* "No active credentials for provider: openai-compatible-chat-<uuid>", the
|
||||
* exact error reported in #10085.
|
||||
*/
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-10085-compat-"));
|
||||
process.env.DATA_DIR = TEST_DATA_DIR;
|
||||
|
||||
const core = await import("../../src/lib/db/core.ts");
|
||||
const nodesDb = await import("../../src/lib/db/providers/nodes.ts");
|
||||
const providersDb = await import("../../src/lib/db/providers.ts");
|
||||
const auth = await import("../../src/sse/services/auth.ts");
|
||||
|
||||
const NODE_PREFIX = "my-compat-10085";
|
||||
const NODE_ID = `openai-compatible-chat-458d982b-0000-4000-8000-000000000000`;
|
||||
|
||||
async function resetStorage() {
|
||||
core.resetDbInstance();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
||||
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
test.after(() => {
|
||||
core.resetDbInstance();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
async function seedNode() {
|
||||
await nodesDb.createProviderNode({
|
||||
id: NODE_ID,
|
||||
type: "openai-compatible",
|
||||
name: "My Compat",
|
||||
prefix: NODE_PREFIX,
|
||||
apiType: "chat",
|
||||
baseUrl: "https://example.test/v1",
|
||||
});
|
||||
}
|
||||
|
||||
test("a connection stored under the GENERIC type id is reachable when chat resolves the uuid node id (#10085)", async () => {
|
||||
await resetStorage();
|
||||
await seedNode();
|
||||
await providersDb.createProviderConnection({
|
||||
provider: "openai-compatible-chat", // generic type id, NOT the uuid node id
|
||||
authType: "apikey",
|
||||
apiKey: "sk-test-10085",
|
||||
name: "test-compat",
|
||||
isActive: true,
|
||||
testStatus: "active",
|
||||
priority: 1,
|
||||
providerSpecificData: { prefix: NODE_PREFIX, baseUrl: "https://example.test/v1" },
|
||||
});
|
||||
|
||||
const creds = await auth.getProviderCredentials(NODE_ID);
|
||||
|
||||
assert.ok(
|
||||
creds,
|
||||
`chat looked up "${NODE_ID}" but the connection is parked under the generic ` +
|
||||
`"openai-compatible-chat" provider id -- getProviderSearchPool never bridges the ` +
|
||||
`generic type id to the concrete node id. This matches #10085 exactly.`
|
||||
);
|
||||
});
|
||||
|
||||
test("the bridge works in the other direction too: a uuid-stored connection is reachable via the generic type id", async () => {
|
||||
await resetStorage();
|
||||
await seedNode();
|
||||
await providersDb.createProviderConnection({
|
||||
provider: NODE_ID, // concrete uuid node id
|
||||
authType: "apikey",
|
||||
apiKey: "sk-test-10085-b",
|
||||
name: "test-compat-b",
|
||||
isActive: true,
|
||||
testStatus: "active",
|
||||
priority: 1,
|
||||
providerSpecificData: { prefix: NODE_PREFIX, baseUrl: "https://example.test/v1" },
|
||||
});
|
||||
|
||||
const creds = await auth.getProviderCredentials("openai-compatible-chat");
|
||||
|
||||
assert.ok(
|
||||
creds,
|
||||
`a connection stored under the uuid node id "${NODE_ID}" must also be reachable via ` +
|
||||
`a lookup using the bare generic type id "openai-compatible-chat"`
|
||||
);
|
||||
});
|
||||
|
||||
test("control: a connection stored under the uuid node id is found by a uuid node id lookup", async () => {
|
||||
await resetStorage();
|
||||
await seedNode();
|
||||
await providersDb.createProviderConnection({
|
||||
provider: NODE_ID,
|
||||
authType: "apikey",
|
||||
apiKey: "sk-test-10085-c",
|
||||
name: "test-compat-c",
|
||||
isActive: true,
|
||||
testStatus: "active",
|
||||
priority: 1,
|
||||
providerSpecificData: { prefix: NODE_PREFIX, baseUrl: "https://example.test/v1" },
|
||||
});
|
||||
|
||||
assert.ok(await auth.getProviderCredentials(NODE_ID));
|
||||
});
|
||||
|
||||
test("control: a connection stored under the uuid node id is found via prefix lookup", async () => {
|
||||
await resetStorage();
|
||||
await seedNode();
|
||||
await providersDb.createProviderConnection({
|
||||
provider: NODE_ID,
|
||||
authType: "apikey",
|
||||
apiKey: "sk-test-10085-d",
|
||||
name: "test-compat-d",
|
||||
isActive: true,
|
||||
testStatus: "active",
|
||||
priority: 1,
|
||||
providerSpecificData: { prefix: NODE_PREFIX, baseUrl: "https://example.test/v1" },
|
||||
});
|
||||
|
||||
assert.ok(await auth.getProviderCredentials(NODE_PREFIX));
|
||||
});
|
||||
|
||||
test("the bridge does not make unrelated generic types findable", async () => {
|
||||
await resetStorage();
|
||||
await seedNode();
|
||||
await providersDb.createProviderConnection({
|
||||
provider: "openai-compatible-chat",
|
||||
authType: "apikey",
|
||||
apiKey: "sk-test-10085-e",
|
||||
name: "test-compat-e",
|
||||
isActive: true,
|
||||
testStatus: "active",
|
||||
priority: 1,
|
||||
providerSpecificData: { prefix: NODE_PREFIX, baseUrl: "https://example.test/v1" },
|
||||
});
|
||||
|
||||
// A different generic type (responses, not chat) must stay unrelated.
|
||||
assert.equal(await auth.getProviderCredentials("openai-compatible-responses"), null);
|
||||
});
|
||||
@@ -11,17 +11,18 @@ import assert from "node:assert/strict";
|
||||
* Status matching accepts both the raw upstream 403 AND the restated 429
|
||||
* (upstreamStatusRestatement.ts rewrites 403→429 before classification).
|
||||
*
|
||||
* #10334 — `ProviderErrorRuleMatch.scope` is now CONSUMED for agentrouter:
|
||||
* `checkFallbackError` surfaces it as `ruleScope` on its return value (see
|
||||
* A11/A12 below), and a raw 403 is no longer an early-return dead end for
|
||||
* this provider — `honorsRuleLockScope("agentrouter")` gates a dedicated
|
||||
* pre-check that consults the provider rules BEFORE the generic apikey
|
||||
* FORBIDDEN branch (see A7/A12). This is an EXCLUSIVE allowlist
|
||||
* (`honorsRuleLockScope`, A14): every other provider's `scope` stays
|
||||
* declared-but-unconsumed exactly as before (A13). See
|
||||
* `docs/architecture/RESILIENCE_GUIDE.md` §7 for the full writeup — Tasks 2/3
|
||||
* of #10334 wire the surfaced `ruleScope` into the persistence layer
|
||||
* (markAccountUnavailable / combo target exhaustion).
|
||||
* IMPORTANT — `scope` above is what the rule DECLARES, not what production
|
||||
* enforces: `ProviderErrorRuleMatch.scope` is not consumed by
|
||||
* checkFallbackError/combo.ts today (only `reason`/`cooldownMs` are). For
|
||||
* agentrouter (passthroughModels: true → hasPerModelQuota() true), the
|
||||
* quota_exhausted match actually resolves to a PER-MODEL lockout in
|
||||
* production, not a connection-wide lock — other models on the same account
|
||||
* keep being tried by combo routing until they lock out individually. And
|
||||
* the "无权访问模型" rule never reaches production traffic at all today: it
|
||||
* only matches raw `status === 403`, but checkFallbackError's apikey
|
||||
* FORBIDDEN branch returns early for a plain 403 before any provider rule is
|
||||
* consulted (see A7). See `docs/architecture/RESILIENCE_GUIDE.md` §7 for the
|
||||
* full writeup and the tracked follow-up to honor `scope`.
|
||||
*/
|
||||
|
||||
const { providerRuleRegistry, getProviderErrorRuleMatch } = await import(
|
||||
@@ -52,7 +53,7 @@ test("A3: quota body also matches the raw (pre-restatement) 403", () => {
|
||||
assert.equal(match.reason, "quota_exhausted");
|
||||
});
|
||||
|
||||
test("A4: 无权访问模型 → auth_error scope model, at the RULE layer (getProviderErrorRuleMatch directly) — since #10334 this rule DOES receive production traffic for agentrouter via the honorsRuleLockScope pre-check in checkFallbackError (see A12)", () => {
|
||||
test("A4: 无权访问模型 → auth_error scope model, at the RULE layer only (getProviderErrorRuleMatch directly) — this rule never receives production traffic (see A7): checkFallbackError's apikey FORBIDDEN branch returns early for a plain 403 before reaching this rule", () => {
|
||||
const match = getProviderErrorRuleMatch("agentrouter", 403, {}, {
|
||||
error: { message: "无权访问模型 claude-sonnet-4" },
|
||||
});
|
||||
@@ -85,15 +86,14 @@ test("A6: guard — restated quota error is retryable, never terminal, and now a
|
||||
});
|
||||
|
||||
test("A7: guard — raw 403 quota (hook bypassed) is still not account-deactivation", () => {
|
||||
// Since #10334, a raw (pre-restatement) 403 for agentrouter DOES reach the
|
||||
// provider rules: checkFallbackError's honorsRuleLockScope pre-check runs
|
||||
// BEFORE the generic apikey-category FORBIDDEN branch and matches the
|
||||
// "额度不足" rule here (reason quota_exhausted, scope connection — see A11).
|
||||
// In the real pipeline, chatCore's upstreamStatusRestatement hook (Task 2)
|
||||
// still converts 403→429 before checkFallbackError sees it, so this raw-403
|
||||
// path is what a hook-bypassed request hits — and it must still not be
|
||||
// misclassified as permanent account deactivation, regardless of which
|
||||
// branch (pre-check or the old apikey-FORBIDDEN fallback) ultimately fires.
|
||||
// A raw (pre-restatement) 403 never actually reaches the agentrouter provider
|
||||
// rules in production: checkFallbackError's apikey-category FORBIDDEN branch
|
||||
// (status === 403 && getProviderCategory(provider) === "apikey") returns
|
||||
// EARLY via resolveApiKeyForbiddenFallback before the provider-rule lookup
|
||||
// is ever consulted. In the real pipeline, chatCore's upstreamStatusRestatement
|
||||
// hook (Task 2) already converts 403→429 before checkFallbackError ever sees
|
||||
// it, so this early-return path is what a hook-bypassed raw 403 hits — and it
|
||||
// must still not be misclassified as permanent account deactivation.
|
||||
const result = checkFallbackError(403, "用户额度不足", 0, null, "agentrouter", null);
|
||||
assert.equal(result.shouldFallback, true);
|
||||
assert.ok(!result.permanent);
|
||||
@@ -139,39 +139,3 @@ test("A10: other providers' checkFallbackError behavior is unchanged (exclusivit
|
||||
assert.equal(result.reason, "rate_limit_exceeded");
|
||||
assert.equal(result.cooldownMs, 3000);
|
||||
});
|
||||
|
||||
test("A11: checkFallbackError surfaces ruleScope=connection for agentrouter quota", () => {
|
||||
const result = checkFallbackError(429, "用户额度不足", 0, null, "agentrouter", null);
|
||||
assert.equal(result.ruleScope, "connection");
|
||||
assert.equal(result.reason, "quota_exhausted");
|
||||
assert.ok(!result.permanent);
|
||||
});
|
||||
|
||||
test("A12: checkFallbackError 403 无权访问模型 carries the rule's scope + cooldown", () => {
|
||||
const result = checkFallbackError(403, "无权访问模型 claude-opus-5", 0, null, "agentrouter", null);
|
||||
assert.equal(result.ruleScope, "model");
|
||||
assert.equal(result.reason, "auth_error");
|
||||
assert.equal(result.baseCooldownMs, 6 * 60 * 60 * 1000);
|
||||
});
|
||||
|
||||
test("A13: exclusivity — ruleScope stays undefined for other providers", () => {
|
||||
const opencode = checkFallbackError(
|
||||
429,
|
||||
'{"error":{"message":"organization_quota_exceeded"}}',
|
||||
0,
|
||||
null,
|
||||
"opencode",
|
||||
null
|
||||
);
|
||||
assert.equal(opencode.ruleScope, undefined);
|
||||
const openrouter = checkFallbackError(402, "credits exhausted", 0, null, "openrouter", null);
|
||||
assert.equal(openrouter.ruleScope, undefined);
|
||||
});
|
||||
|
||||
test("A14: honorsRuleLockScope allowlist is agentrouter-only", async () => {
|
||||
const { honorsRuleLockScope } = await import("../../open-sse/config/providerErrorRules.ts");
|
||||
assert.equal(honorsRuleLockScope("agentrouter"), true);
|
||||
assert.equal(honorsRuleLockScope("AgentRouter"), true);
|
||||
assert.equal(honorsRuleLockScope("opencode"), false);
|
||||
assert.equal(honorsRuleLockScope(null), false);
|
||||
});
|
||||
|
||||
@@ -1,638 +0,0 @@
|
||||
// #10334 — agentrouter EXCLUSIVE: markAccountUnavailable must honor the
|
||||
// provider rule's declared lock scope instead of always deriving it from
|
||||
// hasPerModelQuota(). agentrouter is a passthroughModels provider, so a
|
||||
// naive account-wide quota exhaustion ("额度不足") would otherwise be treated
|
||||
// as a per-model 429 and lock only ONE model, leaving combo routing to burn
|
||||
// one upstream call per remaining model of the same exhausted account. This
|
||||
// suite pins the connection-scoped cooldown behavior AND its invariants:
|
||||
// never a terminal status, must also win when the caller is combo (isCombo),
|
||||
// must not lock the model, and must be EXCLUSIVE to agentrouter — every other
|
||||
// passthroughModels/compatible provider keeps today's per-model lockout.
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
|
||||
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-agentrouter-lock-"));
|
||||
process.env.DATA_DIR = TEST_DATA_DIR;
|
||||
|
||||
const core = await import("../../src/lib/db/core.ts");
|
||||
const providersDb = await import("../../src/lib/db/providers.ts");
|
||||
const auth = await import("../../src/sse/services/auth.ts");
|
||||
const accountFallback = await import("../../open-sse/services/accountFallback.ts");
|
||||
const { applyComboTargetExhaustion } = await import(
|
||||
"../../open-sse/services/combo/targetExhaustion.ts"
|
||||
);
|
||||
const { classifyProviderError } = await import("../../open-sse/services/errorClassifier.ts");
|
||||
|
||||
const QUOTA_EXHAUSTED_429 = '{"error":{"message":"账户额度不足，请充值后重试"}}';
|
||||
const MODEL_ACCESS_DENIED_403 = '{"error":{"message":"无权访问模型 claude-opus-5"}}';
|
||||
|
||||
async function resetStorage() {
|
||||
core.resetDbInstance();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
||||
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
|
||||
}
|
||||
|
||||
async function seedConnection(
|
||||
provider: string,
|
||||
overrides: Record<string, unknown> = {}
|
||||
): Promise<string> {
|
||||
const conn = await providersDb.createProviderConnection({
|
||||
provider,
|
||||
authType: "apikey",
|
||||
apiKey: `${provider}-key`,
|
||||
isActive: true,
|
||||
testStatus: "active",
|
||||
...overrides,
|
||||
});
|
||||
return (conn as Record<string, unknown>).id as string;
|
||||
}
|
||||
|
||||
test.after(() => {
|
||||
core.resetDbInstance();
|
||||
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test("agentrouter 429 account quota exhausted -> connection cooldown, never terminal", async () => {
|
||||
await resetStorage();
|
||||
const connId = await seedConnection("agentrouter");
|
||||
|
||||
const result = await auth.markAccountUnavailable(
|
||||
connId,
|
||||
429,
|
||||
QUOTA_EXHAUSTED_429,
|
||||
"agentrouter",
|
||||
"claude-opus-5"
|
||||
);
|
||||
|
||||
assert.equal(result.shouldFallback, true);
|
||||
assert.ok(result.cooldownMs > 0, "connection cooldown must be positive");
|
||||
|
||||
const after = await providersDb.getProviderConnectionById(connId);
|
||||
assert.equal(after.testStatus, "unavailable");
|
||||
assert.notEqual(after.testStatus, "credits_exhausted");
|
||||
assert.ok(after.rateLimitedUntil, "connection must carry a rateLimitedUntil");
|
||||
assert.ok(
|
||||
new Date(String(after.rateLimitedUntil)).getTime() > Date.now(),
|
||||
"rateLimitedUntil must be in the future"
|
||||
);
|
||||
});
|
||||
|
||||
test("agentrouter 429 quota exhausted with isCombo: true still cools the connection (not a model lock)", async () => {
|
||||
await resetStorage();
|
||||
const connId = await seedConnection("agentrouter");
|
||||
|
||||
const result = await auth.markAccountUnavailable(
|
||||
connId,
|
||||
429,
|
||||
QUOTA_EXHAUSTED_429,
|
||||
"agentrouter",
|
||||
"claude-opus-5",
|
||||
null,
|
||||
{ isCombo: true, persistUnavailableState: false }
|
||||
);
|
||||
|
||||
assert.equal(result.shouldFallback, true);
|
||||
assert.ok(result.cooldownMs > 0);
|
||||
|
||||
const after = await providersDb.getProviderConnectionById(connId);
|
||||
assert.equal(after.testStatus, "unavailable");
|
||||
assert.notEqual(after.testStatus, "credits_exhausted");
|
||||
assert.ok(after.rateLimitedUntil, "connection must be cooled down even for combo callers");
|
||||
});
|
||||
|
||||
test("agentrouter quota cooldown does NOT lock the model", async () => {
|
||||
await resetStorage();
|
||||
const connId = await seedConnection("agentrouter");
|
||||
|
||||
await auth.markAccountUnavailable(
|
||||
connId,
|
||||
429,
|
||||
QUOTA_EXHAUSTED_429,
|
||||
"agentrouter",
|
||||
"claude-opus-5"
|
||||
);
|
||||
|
||||
const lockout = accountFallback.getModelLockoutInfo("agentrouter", connId, "claude-opus-5");
|
||||
assert.equal(lockout, null, "connection-scoped quota must not also record a model lockout");
|
||||
});
|
||||
|
||||
test("agentrouter 403 model-access-denied -> model lockout, connection stays active", async () => {
|
||||
await resetStorage();
|
||||
const connId = await seedConnection("agentrouter");
|
||||
|
||||
const result = await auth.markAccountUnavailable(
|
||||
connId,
|
||||
403,
|
||||
MODEL_ACCESS_DENIED_403,
|
||||
"agentrouter",
|
||||
"claude-opus-5"
|
||||
);
|
||||
|
||||
assert.equal(result.shouldFallback, true);
|
||||
|
||||
const after = await providersDb.getProviderConnectionById(connId);
|
||||
assert.equal(after.testStatus, "active");
|
||||
assert.ok(!after.rateLimitedUntil, "connection must not be rate-limited by a model-scoped rule");
|
||||
|
||||
// #3027's existing per-model-quota-provider branch handles this 403 (it is
|
||||
// unmodified by #10334 except that it now reads the rule's declared
|
||||
// cooldown via fallbackResult.baseCooldownMs) — the recorded reason stays
|
||||
// the pre-existing hardcoded "forbidden", not the rule's "auth_error".
|
||||
const lockout = accountFallback.getModelLockoutInfo("agentrouter", connId, "claude-opus-5");
|
||||
assert.equal(lockout?.reason, "forbidden");
|
||||
// The 6h base cooldown declared by the "agentrouter-model-access-denied"
|
||||
// rule (open-sse/config/providerErrorRules.ts) must flow through as
|
||||
// fallbackResult.baseCooldownMs instead of the generic
|
||||
// COOLDOWN_MS.serviceUnavailable (2s) default — it then gets clamped down
|
||||
// to the model-lockout maxCooldownMs setting (default 1_800_000ms / 30min)
|
||||
// by recordModelLockoutFailure, same as every other model lockout. What
|
||||
// this pins is that the rule's cooldown was consulted at all: a plain 2s
|
||||
// default would be immediately visible as a tiny remainingMs, not ~max.
|
||||
assert.ok(
|
||||
lockout && lockout.remainingMs > 1_700_000,
|
||||
`expected the rule cooldown to be clamped to ~maxCooldownMs (1_800_000ms), got ${lockout?.remainingMs}ms`
|
||||
);
|
||||
});
|
||||
|
||||
test("exclusivity: ollama-cloud with an equivalent account-wide-looking 429 keeps today's per-model lockout, no connection cooldown", async () => {
|
||||
await resetStorage();
|
||||
const connId = await seedConnection("ollama-cloud");
|
||||
|
||||
const result = await auth.markAccountUnavailable(
|
||||
connId,
|
||||
429,
|
||||
QUOTA_EXHAUSTED_429,
|
||||
"ollama-cloud",
|
||||
"claude-opus-5"
|
||||
);
|
||||
|
||||
assert.equal(result.shouldFallback, true);
|
||||
|
||||
const after = await providersDb.getProviderConnectionById(connId);
|
||||
// ollama-cloud is NOT in the honorsRuleLockScope allowlist: today's
|
||||
// per-model-quota behavior for a 429 must be unchanged — connection stays
|
||||
// active, no rateLimitedUntil.
|
||||
assert.equal(after.testStatus, "active");
|
||||
assert.ok(!after.rateLimitedUntil, "non-agentrouter providers must not gain connection cooldown");
|
||||
|
||||
// Positive assertion, not just the negative: the model lockout must have
|
||||
// actually been recorded. Without this, a future refactor that stops
|
||||
// locking anything for these providers would pass this test silently.
|
||||
const lockout = accountFallback.getModelLockoutInfo("ollama-cloud", connId, "claude-opus-5");
|
||||
assert.ok(lockout, "expected the pre-existing per-model lockout to be recorded");
|
||||
});
|
||||
|
||||
test("exclusivity: vertex with an equivalent account-wide-looking 429 keeps today's per-model lockout, no connection cooldown", async () => {
|
||||
await resetStorage();
|
||||
const connId = await seedConnection("vertex");
|
||||
|
||||
const result = await auth.markAccountUnavailable(
|
||||
connId,
|
||||
429,
|
||||
QUOTA_EXHAUSTED_429,
|
||||
"vertex",
|
||||
"claude-opus-5"
|
||||
);
|
||||
|
||||
assert.equal(result.shouldFallback, true);
|
||||
|
||||
const after = await providersDb.getProviderConnectionById(connId);
|
||||
assert.equal(after.testStatus, "active");
|
||||
assert.ok(!after.rateLimitedUntil, "non-agentrouter providers must not gain connection cooldown");
|
||||
|
||||
// Positive assertion, not just the negative — see the ollama-cloud case above.
|
||||
const lockout = accountFallback.getModelLockoutInfo("vertex", connId, "claude-opus-5");
|
||||
assert.ok(lockout, "expected the pre-existing per-model lockout to be recorded");
|
||||
});
|
||||
|
||||
// ─── Fix round 1 (#10334 review) ───────────────────────────────────────────
|
||||
|
||||
// Important finding: the "never terminal" invariant is not structurally
|
||||
// guaranteed by `ruleScope === "connection"` alone — it depends on the
|
||||
// provider rule table only ever pairing scope "connection" with a genuinely
|
||||
// transient reason. isAgentrouterConnectionQuotaScope() is the actual guard;
|
||||
// pin its predicate directly with synthetic fallbackResult shapes, since no
|
||||
// rule in the current table produces a permanent/credits-exhausted result
|
||||
// with scope "connection" (exercising it end-to-end would require editing
|
||||
// the production rule table just for a test).
|
||||
test("isAgentrouterConnectionQuotaScope: rejects a permanent rule result even with scope connection", () => {
|
||||
const permanentConnectionScopeResult = {
|
||||
ruleScope: "connection" as const,
|
||||
reason: "auth_error",
|
||||
permanent: true,
|
||||
};
|
||||
assert.equal(
|
||||
auth.isAgentrouterConnectionQuotaScope("agentrouter", permanentConnectionScopeResult),
|
||||
false,
|
||||
"a future permanent-state rule with scope connection must NOT take the transient-cooldown branch"
|
||||
);
|
||||
});
|
||||
|
||||
test("isAgentrouterConnectionQuotaScope: rejects a credits-exhausted rule result even with scope connection", () => {
|
||||
const creditsExhaustedConnectionScopeResult = {
|
||||
ruleScope: "connection" as const,
|
||||
reason: "quota_exhausted",
|
||||
creditsExhausted: true,
|
||||
};
|
||||
assert.equal(
|
||||
auth.isAgentrouterConnectionQuotaScope("agentrouter", creditsExhaustedConnectionScopeResult),
|
||||
false,
|
||||
"a future credits-exhausted rule with scope connection must NOT take the transient-cooldown branch"
|
||||
);
|
||||
});
|
||||
|
||||
test("isAgentrouterConnectionQuotaScope: accepts the real quota-exhausted/connection shape", () => {
|
||||
const quotaConnectionScopeResult = {
|
||||
ruleScope: "connection" as const,
|
||||
reason: "quota_exhausted",
|
||||
};
|
||||
assert.equal(
|
||||
auth.isAgentrouterConnectionQuotaScope("agentrouter", quotaConnectionScopeResult),
|
||||
true,
|
||||
"today's only connection-scope rule result (quota_exhausted, no permanent/creditsExhausted) must pass"
|
||||
);
|
||||
});
|
||||
|
||||
test("isAgentrouterConnectionQuotaScope: rejects non-agentrouter providers regardless of shape", () => {
|
||||
const quotaConnectionScopeResult = {
|
||||
ruleScope: "connection" as const,
|
||||
reason: "quota_exhausted",
|
||||
};
|
||||
assert.equal(
|
||||
auth.isAgentrouterConnectionQuotaScope("ollama-cloud", quotaConnectionScopeResult),
|
||||
false,
|
||||
"honorsRuleLockScope must still gate every provider outside the agentrouter allowlist"
|
||||
);
|
||||
});
|
||||
|
||||
// Minor finding: guard the branch's POSITION in markAccountUnavailable. If a
|
||||
// future refactor moved the branch above the terminal-status guard (~line
|
||||
// 2023) or the anti-thundering-herd guard (~line 2038), a credits_exhausted
|
||||
// connection would be silently overwritten, or a live cooldown would be
|
||||
// shortened — and the 6 tests above would stay green because none of them
|
||||
// seed a connection with pre-existing terminal/cooldown state.
|
||||
test("position guard: a connection already credits_exhausted stays terminal through an agentrouter quota 429", async () => {
|
||||
await resetStorage();
|
||||
const connId = await seedConnection("agentrouter", { testStatus: "credits_exhausted" });
|
||||
|
||||
const result = await auth.markAccountUnavailable(
|
||||
connId,
|
||||
429,
|
||||
QUOTA_EXHAUSTED_429,
|
||||
"agentrouter",
|
||||
"claude-opus-5"
|
||||
);
|
||||
|
||||
assert.equal(result.shouldFallback, true);
|
||||
assert.equal(result.cooldownMs, 0, "terminal-status short-circuit returns cooldownMs 0");
|
||||
|
||||
const after = await providersDb.getProviderConnectionById(connId);
|
||||
assert.equal(
|
||||
after.testStatus,
|
||||
"credits_exhausted",
|
||||
"the connection-scope branch must never overwrite a pre-existing terminal status"
|
||||
);
|
||||
});
|
||||
|
||||
test("position guard: an existing live cooldown is not shortened by the connection-scope branch", async () => {
|
||||
await resetStorage();
|
||||
const futureCooldown = new Date(Date.now() + 10 * 60 * 1000).toISOString();
|
||||
const connId = await seedConnection("agentrouter", {
|
||||
testStatus: "unavailable",
|
||||
rateLimitedUntil: futureCooldown,
|
||||
});
|
||||
|
||||
const result = await auth.markAccountUnavailable(
|
||||
connId,
|
||||
429,
|
||||
QUOTA_EXHAUSTED_429,
|
||||
"agentrouter",
|
||||
"claude-opus-5"
|
||||
);
|
||||
|
||||
assert.equal(result.shouldFallback, true);
|
||||
|
||||
const after = await providersDb.getProviderConnectionById(connId);
|
||||
assert.equal(
|
||||
after.rateLimitedUntil,
|
||||
futureCooldown,
|
||||
"the anti-thundering-herd guard must win: an existing live cooldown must not be reset/shortened"
|
||||
);
|
||||
});
|
||||
|
||||
// Minor finding: disableCooling=true skips the connection-scope branch (the
|
||||
// `!disableCooling` condition), so the #10334 bug survives for connections
|
||||
// with that opt-out — they fall into the ~30min per-model lockout instead of
|
||||
// the shorter connection cooldown. Documented in the block comment above the
|
||||
// branch; pin the behavior so a future change to the guard is deliberate.
|
||||
test("disableCooling=true skips the connection-scope branch and falls back to per-model lockout", async () => {
|
||||
await resetStorage();
|
||||
const connId = await seedConnection("agentrouter", {
|
||||
providerSpecificData: { disableCooling: true },
|
||||
});
|
||||
|
||||
const result = await auth.markAccountUnavailable(
|
||||
connId,
|
||||
429,
|
||||
QUOTA_EXHAUSTED_429,
|
||||
"agentrouter",
|
||||
"claude-opus-5"
|
||||
);
|
||||
|
||||
assert.equal(result.shouldFallback, true);
|
||||
|
||||
const after = await providersDb.getProviderConnectionById(connId);
|
||||
// Connection is NOT cooled down — disableCooling's documented CONNECTION-
|
||||
// level opt-out (#2997) is honored.
|
||||
assert.equal(after.testStatus, "active");
|
||||
assert.ok(!after.rateLimitedUntil, "disableCooling must keep the connection selectable");
|
||||
|
||||
// But the model IS locked out instead (the #10334 bug's exact symptom for
|
||||
// disableCooling connections — a deliberate, documented trade-off).
|
||||
const lockout = accountFallback.getModelLockoutInfo("agentrouter", connId, "claude-opus-5");
|
||||
assert.ok(
|
||||
lockout,
|
||||
"expected a per-model lockout when disableCooling bypasses the connection branch"
|
||||
);
|
||||
});
|
||||
|
||||
// ─── Task 3 (#10334): combo skips the exhausted agentrouter connection
|
||||
// WITHIN THE SAME REQUEST ──────────────────────────────────────────────────
|
||||
// The tests above pin markAccountUnavailable's PERSISTED connection cooldown
|
||||
// — that only protects the NEXT request. applyComboTargetExhaustion (the
|
||||
// #1731/#1731v2 shared classifier both combo dispatchers call after every
|
||||
// target's upstream error — open-sse/services/combo/targetExhaustion.ts) is
|
||||
// what decides whether remaining targets of the CURRENT request are skipped.
|
||||
// Without a matching gate there, a combo with 5 legs on the same exhausted
|
||||
// agentrouter account would still burn all 5 upstream calls before the
|
||||
// persisted cooldown from the tests above ever kicks in.
|
||||
|
||||
function comboSets() {
|
||||
return {
|
||||
exhaustedProviders: new Set<string>(),
|
||||
exhaustedConnections: new Set<string>(),
|
||||
transientRateLimitedProviders: new Set<string>(),
|
||||
};
|
||||
}
|
||||
|
||||
function comboTarget(overrides: Record<string, unknown> = {}) {
|
||||
return {
|
||||
kind: "model",
|
||||
executionKey: "ek",
|
||||
modelStr: "agentrouter/claude-opus-5",
|
||||
provider: "agentrouter",
|
||||
providerId: null,
|
||||
connectionId: "conn-agentrouter-1",
|
||||
...overrides,
|
||||
} as Parameters<typeof applyComboTargetExhaustion>[0];
|
||||
}
|
||||
|
||||
const comboLog = { info() {}, warn() {}, error() {}, debug() {} };
|
||||
|
||||
const comboBaseOpts = {
|
||||
errorText: QUOTA_EXHAUSTED_429,
|
||||
rawModel: "claude-opus-5",
|
||||
isTokenLimitBreach: false,
|
||||
allAccountsRateLimited: false,
|
||||
requestScopedFailure: false,
|
||||
log: comboLog,
|
||||
tag: "COMBO",
|
||||
exhaustedLogLevel: "info" as const,
|
||||
};
|
||||
|
||||
// The real shape checkFallbackError surfaces for agentrouter's restated 429
|
||||
// (open-sse/config/providerErrorRules.ts's "agentrouter-user-quota-exhausted"
|
||||
// rule: reason "quota_exhausted", scope "connection") — same shape pinned by
|
||||
// isAgentrouterConnectionQuotaScope's own tests above.
|
||||
const CONNECTION_SCOPE_FALLBACK_RESULT = {
|
||||
ruleScope: "connection" as const,
|
||||
reason: "quota_exhausted",
|
||||
};
|
||||
|
||||
test("combo in-request skip: agentrouter connection-scope quota marks exhaustedConnections (#10334)", () => {
|
||||
const sets = comboSets();
|
||||
const exhausted = applyComboTargetExhaustion(comboTarget(), {
|
||||
...comboBaseOpts,
|
||||
result: { status: 429 },
|
||||
fallbackResult: CONNECTION_SCOPE_FALLBACK_RESULT,
|
||||
sets,
|
||||
});
|
||||
assert.equal(
|
||||
exhausted,
|
||||
true,
|
||||
"combo must treat this like an exhausted target — no same-target retry"
|
||||
);
|
||||
assert.ok(
|
||||
sets.exhaustedConnections.has("agentrouter:conn-agentrouter-1"),
|
||||
"the exhausted account's connection must be marked so remaining same-connection targets are skipped this request"
|
||||
);
|
||||
assert.equal(
|
||||
sets.exhaustedProviders.size,
|
||||
0,
|
||||
"must NOT exhaust the whole provider — sibling agentrouter connections keep their own quota"
|
||||
);
|
||||
// Important finding (review round 1): unlike markConnectionLevelExhaustion's
|
||||
// path, this branch must NEVER populate transientRateLimitedProviders. That
|
||||
// set drives combo.ts's `allowRateLimitedConnection` force-allow
|
||||
// (open-sse/services/combo.ts:1005-1013 and :2734-2738), which bypasses the
|
||||
// `rateLimitedUntil` filter in credential selection (src/sse/services/auth.ts:1238)
|
||||
// for the provider's remaining legs this request. Marking it here would
|
||||
// silently re-open the very connection Task 2's markAccountUnavailable (and
|
||||
// this branch) just cooled down.
|
||||
assert.equal(
|
||||
sets.transientRateLimitedProviders.size,
|
||||
0,
|
||||
"must NOT mark transientRateLimitedProviders — that would force-allow reusing the connection this branch just exhausted"
|
||||
);
|
||||
});
|
||||
|
||||
test("combo in-request skip: no connectionId falls back to whole-provider exhaustion", () => {
|
||||
const sets = comboSets();
|
||||
const exhausted = applyComboTargetExhaustion(comboTarget({ connectionId: null }), {
|
||||
...comboBaseOpts,
|
||||
result: { status: 429 },
|
||||
fallbackResult: CONNECTION_SCOPE_FALLBACK_RESULT,
|
||||
sets,
|
||||
});
|
||||
assert.equal(exhausted, true);
|
||||
assert.ok(
|
||||
sets.exhaustedProviders.has("agentrouter"),
|
||||
"no connectionId to scope to — must fall back to whole-provider, mirroring markAuthLevelExhaustion"
|
||||
);
|
||||
assert.equal(sets.exhaustedConnections.size, 0);
|
||||
});
|
||||
|
||||
test("exclusivity: an equivalent connection-scope-shaped result for ollama-cloud marks nothing (#10334 is agentrouter-only)", () => {
|
||||
const sets = comboSets();
|
||||
// Synthetic: production never actually produces ruleScope for a
|
||||
// non-allowlisted provider (honorsRuleLockScope gates it upstream inside
|
||||
// checkFallbackError) — feeding it here directly proves
|
||||
// applyComboTargetExhaustion ALSO re-checks the provider via
|
||||
// isAgentrouterConnectionQuotaScope rather than trusting whatever shape
|
||||
// it is handed.
|
||||
const exhausted = applyComboTargetExhaustion(
|
||||
comboTarget({ provider: "ollama-cloud", connectionId: "conn-ollama-1" }),
|
||||
{
|
||||
...comboBaseOpts,
|
||||
result: { status: 429 },
|
||||
fallbackResult: CONNECTION_SCOPE_FALLBACK_RESULT,
|
||||
sets,
|
||||
}
|
||||
);
|
||||
assert.equal(
|
||||
exhausted,
|
||||
false,
|
||||
"ollama-cloud must fall through to today's per-model-quota behavior unchanged"
|
||||
);
|
||||
assert.equal(sets.exhaustedConnections.size, 0);
|
||||
assert.equal(sets.exhaustedProviders.size, 0);
|
||||
});
|
||||
|
||||
test("exclusivity: vertex with the same synthetic connection-scope result marks nothing", () => {
|
||||
const sets = comboSets();
|
||||
const exhausted = applyComboTargetExhaustion(
|
||||
comboTarget({ provider: "vertex", connectionId: "conn-vertex-1" }),
|
||||
{
|
||||
...comboBaseOpts,
|
||||
result: { status: 429 },
|
||||
fallbackResult: CONNECTION_SCOPE_FALLBACK_RESULT,
|
||||
sets,
|
||||
}
|
||||
);
|
||||
assert.equal(exhausted, false);
|
||||
assert.equal(sets.exhaustedConnections.size, 0);
|
||||
assert.equal(sets.exhaustedProviders.size, 0);
|
||||
});
|
||||
|
||||
test("guard: a permanent agentrouter fallbackResult with scope connection does NOT mark the connection exhausted here either", () => {
|
||||
const sets = comboSets();
|
||||
const exhausted = applyComboTargetExhaustion(comboTarget(), {
|
||||
...comboBaseOpts,
|
||||
result: { status: 429 },
|
||||
fallbackResult: { ruleScope: "connection" as const, reason: "auth_error", permanent: true },
|
||||
sets,
|
||||
});
|
||||
assert.equal(exhausted, false);
|
||||
assert.equal(sets.exhaustedConnections.has("agentrouter:conn-agentrouter-1"), false);
|
||||
assert.equal(sets.exhaustedProviders.size, 0);
|
||||
});
|
||||
|
||||
test("guard: a credits-exhausted agentrouter fallbackResult with scope connection does NOT mark the connection exhausted here either", () => {
|
||||
const sets = comboSets();
|
||||
const exhausted = applyComboTargetExhaustion(comboTarget(), {
|
||||
...comboBaseOpts,
|
||||
result: { status: 429 },
|
||||
fallbackResult: {
|
||||
ruleScope: "connection" as const,
|
||||
reason: "quota_exhausted",
|
||||
creditsExhausted: true,
|
||||
},
|
||||
sets,
|
||||
});
|
||||
assert.equal(exhausted, false);
|
||||
assert.equal(sets.exhaustedConnections.has("agentrouter:conn-agentrouter-1"), false);
|
||||
assert.equal(sets.exhaustedProviders.size, 0);
|
||||
});
|
||||
|
||||
// Minor finding (review round 1): the connection-scope branch is NOT
|
||||
// 429-only. The "额度不足" rule (buildAgentrouterRules, providerErrorRules.ts)
|
||||
// matches statuses {400, 403, 429}, and Task 1's FORBIDDEN pre-check
|
||||
// (accountFallback.ts ~1729-1751, gated on honorsRuleLockScope) surfaces
|
||||
// `ruleScope: "connection"` for a RAW 403 carrying that body too — before the
|
||||
// generic apikey FORBIDDEN early-return, and before markAuthLevelExhaustion
|
||||
// below ever sees it. Pin that a raw 403 with this shape takes the SAME
|
||||
// connection-scope branch (not markAuthLevelExhaustion) and lands in the SAME
|
||||
// set with the SAME key — the two paths are set-equivalent for agentrouter on
|
||||
// this status, so this is not a behavior change, just documenting which
|
||||
// branch actually runs.
|
||||
//
|
||||
// Fix round 2 finding: the Set-content assertions alone (exhausted===true,
|
||||
// the connection key present, the other two sets empty) do NOT discriminate
|
||||
// which branch ran — markAuthLevelExhaustion (the 401/403 branch below)
|
||||
// produces the byte-identical Set effects for a 403 with a connectionId (same
|
||||
// key, same untouched sibling sets, same `true` return), so deleting the new
|
||||
// branch entirely would leave this test green. Use a log spy — the one real
|
||||
// observable difference between the two paths — to prove the NEW branch
|
||||
// actually fired: its message is tagged `#10334` / "account quota exhausted"
|
||||
// (markAgentrouterConnectionQuotaExhaustion), never `#8133` / "auth failure"
|
||||
// (markAuthLevelExhaustion).
|
||||
function makeLogSpy() {
|
||||
const calls: { level: string; tag: string; message: string }[] = [];
|
||||
const record = (level: string) => (tag: string, message: string) => {
|
||||
calls.push({ level, tag, message });
|
||||
};
|
||||
return {
|
||||
calls,
|
||||
log: {
|
||||
info: record("info"),
|
||||
warn: record("warn"),
|
||||
error: record("error"),
|
||||
debug: record("debug"),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
test("combo in-request skip: a RAW 403 with connection-scope quota also takes this branch (not markAuthLevelExhaustion)", () => {
|
||||
const sets = comboSets();
|
||||
const spy = makeLogSpy();
|
||||
const exhausted = applyComboTargetExhaustion(comboTarget(), {
|
||||
...comboBaseOpts,
|
||||
result: { status: 403 },
|
||||
fallbackResult: CONNECTION_SCOPE_FALLBACK_RESULT,
|
||||
sets,
|
||||
log: spy.log,
|
||||
});
|
||||
assert.equal(exhausted, true);
|
||||
assert.ok(
|
||||
sets.exhaustedConnections.has("agentrouter:conn-agentrouter-1"),
|
||||
"a raw 403 carrying ruleScope=connection must exhaust the connection just like the restated-429 case"
|
||||
);
|
||||
assert.equal(sets.exhaustedProviders.size, 0);
|
||||
assert.equal(
|
||||
sets.transientRateLimitedProviders.size,
|
||||
0,
|
||||
"same suppression as the 429 case — must not force-allow reusing this connection"
|
||||
);
|
||||
// The discriminant: prove the NEW (#10334) branch emitted the log, not
|
||||
// markAuthLevelExhaustion's (#8133) — the Set assertions above cannot tell
|
||||
// the two apart on their own.
|
||||
assert.equal(spy.calls.length, 1, "exactly one log call expected for this failure");
|
||||
assert.match(
|
||||
spy.calls[0].message,
|
||||
/#10334/,
|
||||
"must be markAgentrouterConnectionQuotaExhaustion's log line, not markAuthLevelExhaustion's"
|
||||
);
|
||||
assert.ok(
|
||||
/account quota exhausted/.test(spy.calls[0].message),
|
||||
"must carry the new branch's wording, not markAuthLevelExhaustion's 'auth failure'"
|
||||
);
|
||||
assert.doesNotMatch(
|
||||
spy.calls[0].message,
|
||||
/#8133/,
|
||||
"must NOT be markAuthLevelExhaustion's log line"
|
||||
);
|
||||
});
|
||||
|
||||
// ─── Invariant sentinel ─────────────────────────────────────────────────
|
||||
// classifyProviderError (open-sse/services/errorClassifier.ts) must NEVER
|
||||
// classify agentrouter's restated 429 body ("用户额度不足") as quota_exhausted.
|
||||
// If it ever does, open-sse/handlers/chatCore.ts's providerFailure handling
|
||||
// (~line 3835-3856) can reach the terminal `else` branch
|
||||
// (`testStatus: "credits_exhausted"`) for agentrouter whenever
|
||||
// lockModelIfPerModelQuota does not itself claim the failure — turning a
|
||||
// transient, self-recovering account-quota window into a connection that
|
||||
// requires a manual operator reset. agentrouter is an apikey-category
|
||||
// provider (not oauth), so shouldPreserveQuotaSignalsFor429 in
|
||||
// errorClassifier.ts returns false for it and the 429 branch falls through
|
||||
// to RATE_LIMITED instead — pin that this stays true.
|
||||
test("sentinel: classifyProviderError never returns quota_exhausted for agentrouter's restated 429 body", () => {
|
||||
const classification = classifyProviderError(429, "用户额度不足", "agentrouter");
|
||||
assert.notEqual(
|
||||
classification,
|
||||
"quota_exhausted",
|
||||
"a quota_exhausted classification here would route agentrouter's transient account quota into chatCore's terminal credits_exhausted branch (~chatCore.ts:3849)"
|
||||
);
|
||||
});
|
||||
@@ -329,35 +329,11 @@ test("#7307 quality.yml adds an advisory production build for release PR code ch
|
||||
assert.match(buildJob[0], /needs\.changes\.outputs\.code == 'true'/);
|
||||
assert.match(buildJob[0], /github\.event\.pull_request\.draft == false/);
|
||||
assert.match(buildJob[0], /startsWith\(github\.head_ref, 'mergify\/merge-queue\/'\)/);
|
||||
// FORK PRs ONLY (2026-08-14). build.yml's `Fast Production Build` fires on
|
||||
// `push: branches: ["**"]` and runs the superset `build:release`, so own-origin branches
|
||||
// were building twice; a fork's push never reaches this repo, making this their only
|
||||
// pre-merge build signal — and forks are 72 of the last 100 PRs into release/**.
|
||||
assert.match(
|
||||
buildJob[0],
|
||||
/github\.event\.pull_request\.head\.repo\.full_name != github\.repository/
|
||||
/github\.event\.pull_request\.head\.repo\.full_name == github\.repository/
|
||||
);
|
||||
// Runner PINNED to hosted. The self-hosted pool is 2 permanently-busy runners, where this
|
||||
// job either queued for hours or was killed by cancel-in-progress — ~10-15% of runs ever
|
||||
// reached a conclusion across 2026-08-13/14. It must NOT go back on the USE_VPS_RUNNER
|
||||
// switch (other workflows keep that variable).
|
||||
assert.match(buildJob[0], /\n {4}runs-on: ubuntu-latest\n/);
|
||||
// Check the DIRECTIVES, not the prose: the comment above legitimately explains why the
|
||||
// self-hosted pool was abandoned, so a naive /self-hosted/ scan over the whole block would
|
||||
// match its own rationale.
|
||||
const buildDirectives = buildJob[0]
|
||||
.split("\n")
|
||||
.filter((line) => !/^\s*#/.test(line))
|
||||
.join("\n");
|
||||
assert.doesNotMatch(buildDirectives, /self-hosted/);
|
||||
assert.doesNotMatch(buildDirectives, /USE_VPS_RUNNER/);
|
||||
// Memory provisioning mirrored from build.yml: --max-old-space-size bounds only V8's heap,
|
||||
// never Turbopack's native Rust allocation (#6409), so the swapfile is the load-bearing
|
||||
// half. Dropping either one puts the hosted build back at risk of an OOM.
|
||||
assert.match(buildJob[0], /fallocate -l 10G \/mnt\/swapfile/);
|
||||
assert.match(buildJob[0], /swapon \/mnt\/swapfile/);
|
||||
assert.match(buildJob[0], /NODE_OPTIONS: "--max-old-space-size=12288"/);
|
||||
assert.match(buildJob[0], /OMNIROUTE_BUILD_MEMORY_MB: "12288"/);
|
||||
assert.match(buildJob[0], /fromJSON\('\["self-hosted","omni-release"\]'\) \|\| 'ubuntu-latest'/);
|
||||
assert.match(buildJob[0], /continue-on-error: true/);
|
||||
assert.match(buildJob[0], /uses: actions\/checkout@[0-9a-f]{40} # v7/);
|
||||
assert.match(buildJob[0], /uses: actions\/setup-node@[0-9a-f]{40} # v7/);
|
||||
|
||||
@@ -121,13 +121,12 @@ test("INTENTIONALLY_INTERNAL is exported from check-db-rules.mjs", () => {
|
||||
assert.ok(INTENTIONALLY_INTERNAL.size > 0, "INTENTIONALLY_INTERNAL must not be empty");
|
||||
});
|
||||
|
||||
test("INTENTIONALLY_INTERNAL contains the expected 40 audited modules", () => {
|
||||
test("INTENTIONALLY_INTERNAL contains the expected 39 audited modules", () => {
|
||||
const expected = [
|
||||
"_rowTypes",
|
||||
"accessTokens",
|
||||
"apiKeyColumnFallbacks",
|
||||
"apiKeyUsageLimitFields",
|
||||
"backupRetention",
|
||||
"caseMapping",
|
||||
"cleanup",
|
||||
"cliToolState",
|
||||
|
||||
@@ -70,37 +70,6 @@ describe("i-have-adhd output style", () => {
|
||||
assert.ok(/preâmbulo/.test(pt.full), "pt-BR.full mentions preâmbulo");
|
||||
});
|
||||
|
||||
it("has i18n maps for pt-BR, vi, ja and id (parity with ponytail)", () => {
|
||||
assert.ok(ADHD.i18n, "i18n must be defined");
|
||||
for (const lang of ["pt-BR", "vi", "ja", "id"]) {
|
||||
const levels = ADHD.i18n[lang];
|
||||
assert.ok(levels, `${lang} must exist`);
|
||||
assertString(levels.lite, `${lang}.lite`);
|
||||
assertString(levels.full, `${lang}.full`);
|
||||
assertString(levels.ultra, `${lang}.ultra`);
|
||||
assert.ok(
|
||||
levels.full.includes("Code blocks"),
|
||||
`${lang}.full must keep the shared boundaries clause`
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
it("each translation is written in its own language, not copied English", () => {
|
||||
// Native-script / native-word anchors: a level that merely duplicated the
|
||||
// English text would pass the structural checks above but fail here.
|
||||
const anchors: Record<string, RegExp> = {
|
||||
vi: /hành động/, // "action"
|
||||
ja: /[-ヿ]/, // kana — Japanese-exclusive
|
||||
id: /aksi|langkah/, // "action" / "step"
|
||||
};
|
||||
for (const [lang, pattern] of Object.entries(anchors)) {
|
||||
const levels = ADHD.i18n?.[lang];
|
||||
assert.ok(levels, `${lang} i18n must exist`);
|
||||
assert.ok(pattern.test(levels.full), `${lang}.full must use ${lang} vocabulary`);
|
||||
assert.ok(pattern.test(levels.ultra), `${lang}.ultra must use ${lang} vocabulary`);
|
||||
}
|
||||
});
|
||||
|
||||
it("carries no locale gate", () => {
|
||||
assert.equal(outputStyleMeta("i-have-adhd").locale, undefined);
|
||||
});
|
||||
|
||||
@@ -1,141 +0,0 @@
|
||||
/**
|
||||
* Guard for the output-style × language matrix.
|
||||
*
|
||||
* Why this exists: the compression INPUT engines understand 10 languages (rule
|
||||
* packs under open-sse/services/compression/rules/ + the detector), but the
|
||||
* OUTPUT styles only instruct in a subset. `less-code` shipped English-only and
|
||||
* nobody noticed for months, because every other test is per-style. This test
|
||||
* is per-MATRIX: it pins the expected coverage so a new style cannot silently
|
||||
* be born English-only, and so an existing style cannot silently lose a locale.
|
||||
*
|
||||
* Adding a language to a style: extend BASELINE_LANGUAGES below (the assertion
|
||||
* is "at least these", so growth never fails the gate).
|
||||
* Adding a NEW style: it must cover REQUIRED_LANGUAGES, or be listed in
|
||||
* KNOWN_ENGLISH_ONLY with a tracking issue.
|
||||
*/
|
||||
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import {
|
||||
OUTPUT_STYLE_CATALOG,
|
||||
OUTPUT_STYLE_IDS,
|
||||
outputStyleMeta,
|
||||
} from "../../../open-sse/services/compression/outputStyles/catalog.ts";
|
||||
|
||||
/** Minimum i18n coverage every new non-locale-gated style must ship with. */
|
||||
const REQUIRED_LANGUAGES = ["pt-BR"];
|
||||
|
||||
/**
|
||||
* Styles that predate this guard and are still English-only.
|
||||
* Do NOT add entries here without an issue — fix the coverage instead.
|
||||
*/
|
||||
const KNOWN_ENGLISH_ONLY: Record<string, string> = {
|
||||
// 9router port that never got translated. Tracked in the compression i18n
|
||||
// backlog; the fix is mechanical (same shape as ponytail/i-have-adhd).
|
||||
"less-code": "pre-existing gap — English-only since the 9router port",
|
||||
};
|
||||
|
||||
/**
|
||||
* Frozen per-style coverage. The assertion is a SUPERSET check, so adding a
|
||||
* language is always allowed; removing one fails the gate.
|
||||
*/
|
||||
const BASELINE_LANGUAGES: Record<string, string[]> = {
|
||||
// terse-prose reuses CAVEMAN_INSTRUCTION_BY_LANGUAGE (outputMode.ts), which
|
||||
// localizes to pt-BR/ja/id — keep the two in sync when adding a language.
|
||||
"terse-prose": ["pt-BR", "ja", "id"],
|
||||
"less-code": [],
|
||||
ponytail: ["pt-BR", "vi", "ja", "id"],
|
||||
"i-have-adhd": ["pt-BR", "vi", "ja", "id"],
|
||||
// locale-gated to zh: the single-language instruction IS the feature.
|
||||
"terse-cjk": [],
|
||||
};
|
||||
|
||||
function languagesOf(id: string): string[] {
|
||||
return Object.keys(outputStyleMeta(id).i18n ?? {});
|
||||
}
|
||||
|
||||
test("every catalog style is covered by the matrix baseline", () => {
|
||||
for (const id of OUTPUT_STYLE_IDS) {
|
||||
assert.ok(
|
||||
id in BASELINE_LANGUAGES,
|
||||
`style "${id}" is missing from BASELINE_LANGUAGES — add its expected languages ` +
|
||||
`(and translate it: a new style must cover ${REQUIRED_LANGUAGES.join(", ")})`
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("no style loses a language it already had", () => {
|
||||
for (const [id, expected] of Object.entries(BASELINE_LANGUAGES)) {
|
||||
if (!OUTPUT_STYLE_CATALOG[id]) continue; // style removed — covered by the catalog tests
|
||||
const actual = languagesOf(id);
|
||||
for (const lang of expected) {
|
||||
assert.ok(
|
||||
actual.includes(lang),
|
||||
`style "${id}" lost its "${lang}" translation (has: ${actual.join(", ") || "none"})`
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("a non-locale-gated style ships the required languages, or is a known gap", () => {
|
||||
for (const id of OUTPUT_STYLE_IDS) {
|
||||
const meta = outputStyleMeta(id);
|
||||
// A locale-gated style is only ever offered under its own locale, so a
|
||||
// single-language instruction is correct by design (e.g. terse-cjk → zh).
|
||||
if (meta.locale) continue;
|
||||
if (id in KNOWN_ENGLISH_ONLY) continue;
|
||||
const actual = languagesOf(id);
|
||||
for (const lang of REQUIRED_LANGUAGES) {
|
||||
assert.ok(
|
||||
actual.includes(lang),
|
||||
`style "${id}" must ship a "${lang}" translation (has: ${actual.join(", ") || "none"}). ` +
|
||||
`English-only styles need an entry in KNOWN_ENGLISH_ONLY with a tracking issue.`
|
||||
);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("every declared translation has all three intensity levels", () => {
|
||||
for (const id of OUTPUT_STYLE_IDS) {
|
||||
const i18n = outputStyleMeta(id).i18n ?? {};
|
||||
for (const [lang, levels] of Object.entries(i18n)) {
|
||||
for (const level of ["lite", "full", "ultra"] as const) {
|
||||
assert.equal(
|
||||
typeof levels[level],
|
||||
"string",
|
||||
`${id}.i18n["${lang}"].${level} must be a string`
|
||||
);
|
||||
assert.ok(levels[level].length > 0, `${id}.i18n["${lang}"].${level} must be non-empty`);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("every translated level carries the shared boundaries clause", () => {
|
||||
// The boundary clause is what keeps code, paths, commands, errors and URLs
|
||||
// verbatim. A translation that drops it would let the model rewrite them.
|
||||
const anchor = "Code blocks";
|
||||
for (const id of OUTPUT_STYLE_IDS) {
|
||||
const i18n = outputStyleMeta(id).i18n ?? {};
|
||||
for (const [lang, levels] of Object.entries(i18n)) {
|
||||
for (const level of ["lite", "full", "ultra"] as const) {
|
||||
assert.ok(
|
||||
levels[level].includes(anchor),
|
||||
`${id}.i18n["${lang}"].${level} is missing the shared boundaries clause`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("KNOWN_ENGLISH_ONLY does not hide a style that is actually translated", () => {
|
||||
// Stale-allowlist guard: once a gap is fixed, its entry must be removed.
|
||||
for (const id of Object.keys(KNOWN_ENGLISH_ONLY)) {
|
||||
if (!OUTPUT_STYLE_CATALOG[id]) continue;
|
||||
assert.equal(
|
||||
languagesOf(id).length,
|
||||
0,
|
||||
`style "${id}" now has translations — remove it from KNOWN_ENGLISH_ONLY`
|
||||
);
|
||||
}
|
||||
});
|
||||
@@ -1,258 +0,0 @@
|
||||
// #10421 — pre-migration backups were created on every migration run and never pruned,
|
||||
// so `db_backups/` grew without bound (observed: 48.999 files / 204 GB against a 5,3 MB
|
||||
// live database). The pruning logic already existed in `cleanupDbBackups()` but nothing
|
||||
// on the migration path ever reached it. These tests pin the retention step to the
|
||||
// backup call site so the operator's maxFiles/retentionDays budget is honored there too.
|
||||
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { pathToFileURL } from "node:url";
|
||||
import Database from "better-sqlite3";
|
||||
|
||||
const serial = { concurrency: false };
|
||||
|
||||
async function importFresh(modulePath: string) {
|
||||
const url = pathToFileURL(path.resolve(modulePath)).href;
|
||||
return import(`${url}?test=${Date.now()}-${Math.random().toString(16).slice(2)}`);
|
||||
}
|
||||
|
||||
function withMockedMigrationFs(files: Record<string, string>, fn: () => void) {
|
||||
const originalExistsSync = fs.existsSync;
|
||||
const originalReaddirSync = fs.readdirSync;
|
||||
const originalReadFileSync = fs.readFileSync;
|
||||
|
||||
const isMigrationDir = (target: unknown) =>
|
||||
String(target).replaceAll("\\", "/").endsWith("/src/lib/db/migrations") ||
|
||||
String(target).replaceAll("\\", "/").endsWith("/migrations");
|
||||
|
||||
fs.existsSync = ((target: unknown) => {
|
||||
if (isMigrationDir(target)) return true;
|
||||
const fileName = path.basename(String(target));
|
||||
if (Object.hasOwn(files, fileName)) return true;
|
||||
return originalExistsSync(target as string);
|
||||
}) as typeof fs.existsSync;
|
||||
|
||||
fs.readdirSync = ((target: string, options?: unknown) => {
|
||||
if (isMigrationDir(target)) return Object.keys(files);
|
||||
return originalReaddirSync(target, options as never);
|
||||
}) as typeof fs.readdirSync;
|
||||
|
||||
fs.readFileSync = ((target: unknown, options?: unknown) => {
|
||||
const fileName = path.basename(String(target));
|
||||
if (Object.hasOwn(files, fileName)) return files[fileName];
|
||||
return originalReadFileSync(target as string, options as never);
|
||||
}) as typeof fs.readFileSync;
|
||||
|
||||
try {
|
||||
return fn();
|
||||
} finally {
|
||||
fs.existsSync = originalExistsSync;
|
||||
fs.readdirSync = originalReaddirSync;
|
||||
fs.readFileSync = originalReadFileSync;
|
||||
}
|
||||
}
|
||||
|
||||
/** Minimal SqliteAdapter over a real on-disk file (VACUUM INTO needs a file, not :memory:). */
|
||||
function createFileDb(sqlitePath: string) {
|
||||
const db = new Database(sqlitePath);
|
||||
|
||||
return {
|
||||
driver: "better-sqlite3",
|
||||
get open() {
|
||||
return db.open;
|
||||
},
|
||||
get name() {
|
||||
return db.name;
|
||||
},
|
||||
prepare: (sql: string) => db.prepare(sql),
|
||||
exec: (sql: string) => db.exec(sql),
|
||||
pragma: (str: string, options?: unknown) => db.pragma(str, options as never),
|
||||
transaction: (fn: (...args: unknown[]) => unknown) => {
|
||||
const tx = db.transaction((...args: unknown[]) => fn(...args));
|
||||
return (...args: unknown[]) => tx(...args);
|
||||
},
|
||||
immediate: (fn: () => void) => fn(),
|
||||
async backup() {},
|
||||
checkpoint() {},
|
||||
close: () => db.close(),
|
||||
get raw() {
|
||||
return db;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build a DB that already has migrations applied (so the pre-migration backup path is
|
||||
* reached: it requires `applied.size > 0`) plus one pending migration to trigger a run.
|
||||
*/
|
||||
function seedAppliedDb(db: ReturnType<typeof createFileDb>) {
|
||||
db.exec(`
|
||||
CREATE TABLE provider_connections (id TEXT PRIMARY KEY);
|
||||
CREATE TABLE combos (id TEXT PRIMARY KEY);
|
||||
CREATE TABLE call_logs (id TEXT PRIMARY KEY);
|
||||
`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Record 001 as applied in the runner's own ledger table. `runMigrations` only takes a
|
||||
* pre-migration backup when `applied.size > 0`, so this is what puts the test on the
|
||||
* code path under exercise.
|
||||
*/
|
||||
function seedAppliedMigration(db: ReturnType<typeof createFileDb>) {
|
||||
db.exec(`
|
||||
CREATE TABLE IF NOT EXISTS _omniroute_migrations (
|
||||
version TEXT PRIMARY KEY,
|
||||
name TEXT NOT NULL,
|
||||
applied_at TEXT NOT NULL DEFAULT (datetime('now'))
|
||||
);
|
||||
`);
|
||||
db.prepare(
|
||||
"INSERT OR REPLACE INTO _omniroute_migrations (version, name, applied_at) VALUES (?, ?, ?)"
|
||||
).run("001", "initial_schema", new Date().toISOString());
|
||||
}
|
||||
|
||||
function makeTempDataDir() {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-backup-retention-"));
|
||||
fs.mkdirSync(path.join(dir, "db_backups"), { recursive: true });
|
||||
return dir;
|
||||
}
|
||||
|
||||
/** Pre-existing backups, oldest first, with distinct mtimes so retention ordering is stable. */
|
||||
function seedBackups(backupDir: string, count: number) {
|
||||
const names: string[] = [];
|
||||
for (let i = 0; i < count; i++) {
|
||||
const name = `db_2026-08-${String(i + 1).padStart(2, "0")}T00-00-00-000Z_pre-migration.sqlite`;
|
||||
const filePath = path.join(backupDir, name);
|
||||
fs.writeFileSync(filePath, "x");
|
||||
const t = new Date(2026, 7, i + 1).getTime() / 1000;
|
||||
fs.utimesSync(filePath, t, t);
|
||||
names.push(name);
|
||||
}
|
||||
return names;
|
||||
}
|
||||
|
||||
function countBackups(backupDir: string) {
|
||||
return fs.readdirSync(backupDir).filter((n) => n.startsWith("db_")).length;
|
||||
}
|
||||
|
||||
function withEnv(vars: Record<string, string | undefined>, fn: () => void) {
|
||||
const saved: Record<string, string | undefined> = {};
|
||||
for (const [k, v] of Object.entries(vars)) {
|
||||
saved[k] = process.env[k];
|
||||
if (v === undefined) delete process.env[k];
|
||||
else process.env[k] = v;
|
||||
}
|
||||
try {
|
||||
return fn();
|
||||
} finally {
|
||||
for (const [k, v] of Object.entries(saved)) {
|
||||
if (v === undefined) delete process.env[k];
|
||||
else process.env[k] = v;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
test(
|
||||
"#10421 runMigrations prunes pre-migration backups to the configured maxFiles",
|
||||
serial,
|
||||
async () => {
|
||||
const dataDir = makeTempDataDir();
|
||||
const backupDir = path.join(dataDir, "db_backups");
|
||||
const sqlitePath = path.join(dataDir, "storage.sqlite");
|
||||
const db = createFileDb(sqlitePath);
|
||||
|
||||
try {
|
||||
seedAppliedDb(db);
|
||||
seedBackups(backupDir, 30);
|
||||
assert.equal(countBackups(backupDir), 30, "precondition: 30 stale backups on disk");
|
||||
|
||||
const { runMigrations } = await importFresh("src/lib/db/migrationRunner.ts");
|
||||
|
||||
withEnv(
|
||||
{
|
||||
DB_BACKUP_MAX_FILES: "5",
|
||||
DB_BACKUP_RETENTION_DAYS: "0",
|
||||
DISABLE_SQLITE_AUTO_BACKUP: undefined,
|
||||
},
|
||||
() => {
|
||||
withMockedMigrationFs(
|
||||
{
|
||||
"001_initial_schema.sql": "SELECT 1;",
|
||||
"002_retention_probe.sql": "CREATE TABLE retention_probe_10421 (id INTEGER);",
|
||||
},
|
||||
() => {
|
||||
// Mark 001 as applied so `applied.size > 0` and the backup path is reached.
|
||||
seedAppliedMigration(db);
|
||||
|
||||
runMigrations(db);
|
||||
}
|
||||
);
|
||||
}
|
||||
);
|
||||
|
||||
const remaining = countBackups(backupDir);
|
||||
assert.ok(
|
||||
remaining <= 5,
|
||||
`expected retention to cap db_backups at 5 files, found ${remaining} — ` +
|
||||
`pre-migration backups are accumulating unbounded (#10421)`
|
||||
);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(dataDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
test("#10421 the newest pre-migration backup survives pruning", serial, async () => {
|
||||
const dataDir = makeTempDataDir();
|
||||
const backupDir = path.join(dataDir, "db_backups");
|
||||
const sqlitePath = path.join(dataDir, "storage.sqlite");
|
||||
const db = createFileDb(sqlitePath);
|
||||
|
||||
try {
|
||||
seedAppliedDb(db);
|
||||
seedBackups(backupDir, 10);
|
||||
|
||||
const { runMigrations } = await importFresh("src/lib/db/migrationRunner.ts");
|
||||
|
||||
withEnv(
|
||||
{
|
||||
DB_BACKUP_MAX_FILES: "3",
|
||||
DB_BACKUP_RETENTION_DAYS: "0",
|
||||
DISABLE_SQLITE_AUTO_BACKUP: undefined,
|
||||
},
|
||||
() => {
|
||||
withMockedMigrationFs(
|
||||
{
|
||||
"001_initial_schema.sql": "SELECT 1;",
|
||||
"002_retention_probe.sql": "CREATE TABLE retention_probe_10421b (id INTEGER);",
|
||||
},
|
||||
() => {
|
||||
seedAppliedMigration(db);
|
||||
|
||||
runMigrations(db);
|
||||
}
|
||||
);
|
||||
}
|
||||
);
|
||||
|
||||
const remaining = fs.readdirSync(backupDir).filter((n) => n.startsWith("db_"));
|
||||
assert.ok(remaining.length <= 3, `expected <=3 backups, found ${remaining.length}`);
|
||||
|
||||
// The backup written by THIS run must be among the survivors — pruning must never
|
||||
// discard the snapshot that protects the migration it was taken for.
|
||||
const seededNames = new Set(
|
||||
Array.from({ length: 10 }, (_, i) => {
|
||||
return `db_2026-08-${String(i + 1).padStart(2, "0")}T00-00-00-000Z_pre-migration.sqlite`;
|
||||
})
|
||||
);
|
||||
const fresh = remaining.filter((n) => !seededNames.has(n));
|
||||
assert.equal(fresh.length, 1, `expected the run's own backup to survive, got ${fresh.length}`);
|
||||
} finally {
|
||||
db.close();
|
||||
fs.rmSync(dataDir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user