diff --git a/changelog.d/fixes/13440-daily-reset-tz.md b/changelog.d/fixes/13440-daily-reset-tz.md new file mode 100644 index 0000000000..7211394453 --- /dev/null +++ b/changelog.d/fixes/13440-daily-reset-tz.md @@ -0,0 +1 @@ +- **fix(resilience):** non-TPD daily-quota cooldowns honor the provider node's configured daily-reset clock (timezone + hour) instead of server midnight, on single-model and combo (priority and round-robin) paths; timezone edits apply without a restart ([#13440](https://github.com/diegosouzapw/OmniRoute/pull/13440)) — thanks @maxmad64bis diff --git a/config/quality/eslint-suppressions.json b/config/quality/eslint-suppressions.json index 27ebed935a..5ffac46924 100644 --- a/config/quality/eslint-suppressions.json +++ b/config/quality/eslint-suppressions.json @@ -469,7 +469,7 @@ }, "open-sse/services/combo.ts": { "@typescript-eslint/no-unused-vars": { - "count": 21 + "count": 1 } }, "open-sse/services/combo/providerWildcard.ts": { diff --git a/config/quality/file-size-baseline.json b/config/quality/file-size-baseline.json index 59f818b9f1..42f5bbe028 100644 --- a/config/quality/file-size-baseline.json +++ b/config/quality/file-size-baseline.json @@ -430,6 +430,7 @@ "_rebaseline_2026_08_29_11481_model_exposure_list": "Feature #11481 (explicit model exposure allow/deny list for /v1/models, mirrored into auto/* combo pools) own growth on top of #9133's +1: open-sse/services/autoCombo/virtualFactory.ts 1139->1145 (measured real line count after both #9133 and #11481 merged together = one import line for filterModelExposureCandidates plus the filter-and-reassign block at the existing buildPreparedPool chokepoint, immediately after the filterPaidOnlyCandidates call it mirrors — the exact pattern #6512 already established for hidePaidModels). The actual predicate (isModelExposureAllowed, glob support via the shared globToRegex matcher) lives in the new src/shared/utils/modelExposureList.ts leaf, and the pool-filter wrapper lives in the new open-sse/services/autoCombo/modelExposureFilter.ts leaf (both well under cap) — this file only carries the minimal call-site wiring plus import, not extractable further without hiding the buildPreparedPool filter chain. Covered by tests/unit/autoCombo/model-exposure-filter-11481.test.ts (pure filter, all branches) and tests/unit/model-exposure-list.test.ts (predicate).", "_rebaseline_2026_08_29_9133_candidates_inspector_skip_flag": "#9133 own growth: open-sse/services/autoCombo/virtualFactory.ts 1138->1139 (+1, net of extraction). Fix: prepareVirtualAutoComboInputs gained an opt-in `skip` parameter so the read-only #7819 candidate inspector (open-sse/handlers/autoComboCandidates.ts) can build the FULL, unfiltered pool and decorate a resilience-blocked candidate as reachable:false instead of filterResilienceBlockedCandidates silently dropping the row before the inspector ever sees it (routing is unaffected — it never passes `skip`). The connectionsById map-building loop was extracted to buildConnectionResilienceMap() in resilienceCandidateFilter.ts (net 0 there since Prettier still breaks the call over multiple lines) and the now-unused ConnectionResilienceView import was dropped; the sole remaining growth is the new `skip` default parameter itself, which Prettier always places on its own line once the preceding options object parameter already breaks across lines — not further reducible without splitting prepareVirtualAutoComboInputs's signature away from its own body. Covered by tests/unit/auto-combo-candidates-locked-model-visible.test.ts (TDD repro: red before the fix, green after) plus the existing tests/unit/noauth-autocombo-lockout-7623.test.ts and tests/unit/auto-combo-credentialed-model-pool.test.ts (unaffected routing-path behavior).", "_rebaseline_2026_08_30_11703_json_tree_viewer": "/merge-batch 2026-08-30 (v3.8.51): #11703 (hartmark) own growth: src/shared/components/RequestLoggerDetail.tsx 1018->1111 (+93). The 2026-07-22 annotation on this same file said 'no further growth without split rationale' — this PR does split: the collapsible-JSON-tree rendering logic itself lives in the sibling RequestLoggerDetail.sections.tsx (PayloadSection/StreamSection extraction, +82 lines there) plus two new leaves (JsonTreeExpandControls.tsx, useTimestampTitles.ts) and a new store (jsonTreeExpandStore.ts) — all well under cap. The +93 remaining here is the irreducible call-site wiring: import + mount JsonTreeExpandControls, wire the per-section expand-level state and timestamp-tooltip hook into the existing detail panel layout. Covered by the PR's own tests/unit/dashboard/payload-section-collapsible-json.test.tsx, timestamp-titles.test.tsx, tests/unit/shared/json-tree-expand-store.test.ts, short-call-id.test.ts (43/43 vitest + 11/11 native pass).", + "_rebaseline_2026_09_15_13440_daily_reset_tz": "#13440 rework: open-sse/services/accountFallback.ts 2469->2493 (+24): +6 for the operator-clock-first branch in checkFallbackError non-TPD daily quota (nextConfiguredResetMs leaf lives in dailyQuotaReset.ts, under cap) and +18 from the mandatory lint-staged Prettier pass over pre-existing unformatted lines of the touched file (no logic). executeTargetAttempt.ts 1212->1215 and roundRobinCombo.ts 1205->1208 (+3 each): one import plus the rotation/dailyReset arguments at the existing checkFallbackError call site; the lookup itself is the new comboDailyResetClock.ts leaf (under cap). Covered by tests/unit/daily-reset-tz-threading.test.ts.", "_rebaseline_pr1043_minimax_tts": "Upstream port decolua/9router#1043 (toanalien) own growth: audioSpeech.ts 965->1061 (+96). Adds MiniMax T2A v2 TTS dispatch (handleMinimaxSpeech + hexToBytes helper) — provider entry was already in audioRegistry (format: minimax-tts) but no handler existed, falling through to the OpenAI-compatible default that fails (T2A has custom shape + hex-encoded audio + base_resp envelope). New branch sits next to the other inline provider branches (xiaomi-mimo, coqui, tortoise, aws-polly) — extracting would just create indirection. Covered by tests/unit/minimax-tts-1043.test.ts (3 tests, GREEN: success, base_resp error, invalid-hex).", "_rebaseline_pr4592_exclude_exhausted_auto": "Reconcile #4592 already-merged growth: combo.ts 2991->3036 (+45, terminal-status quota-cutoff exclusion in buildAutoCandidates + opt-in gate). Fast-gate PR->release does not run check:file-size.", "open-sse/executors/antigravity.ts": 1665, @@ -443,10 +444,10 @@ "open-sse/handlers/search.ts": 1789, "open-sse/mcp-server/schemas/tools.ts": 1621, "open-sse/mcp-server/server.ts": 1572, - "open-sse/services/accountFallback.ts": 2469, + "open-sse/services/accountFallback.ts": 2493, "open-sse/services/adobeFireflyBrowserLogin.ts": 1401, "open-sse/services/combo.ts": 4080, - "open-sse/services/combo/executeTargetAttempt.ts": 1212, + "open-sse/services/combo/executeTargetAttempt.ts": 1215, "open-sse/translator/response/openai-responses.ts": 1466, "open-sse/utils/cursorAgentProtobuf.ts": 1547, "open-sse/utils/proxyFetch.ts": 1271, @@ -483,7 +484,7 @@ "tests/unit/account-fallback-service.test.ts": 2453, "tests/unit/provider-validation-specialty.test.ts": 4656, "open-sse/services/autoCombo/virtualFactory.ts": 1230, - "open-sse/services/combo/roundRobinCombo.ts": 1205 + "open-sse/services/combo/roundRobinCombo.ts": 1208 }, "_rebaseline_base_2026_08_10_proxyfetch": "Base-red fix (green-prs sweep, issue #9985): open-sse/utils/proxyFetch.ts 1207 > cap 1000 — new proxied-TLS fetch helper introduced by the Fal reference-image work. Owner-authorized quick rebaseline to green; structural slim tracked for v3.9.0.", "_rebaseline_2026_07_27_v3849_train2": "Merge-train 2 (7 PRs) — owner-approved 2026-07-27. Single entry: chatCore.ts 4955->5006 (#8595, Responses multi-turn image compaction before the context hard-reject). Genuine irreducible growth at the existing compaction chokepoint in handleChatCore — the PR adds a last-resort retry against the concrete budget plus the estimateFinalInputTokens helper, both wired at the pre-existing call site rather than a new branch. Covered by tests/unit/8560-responses-image-compaction.test.ts (4 tests).", diff --git a/open-sse/services/accountFallback.ts b/open-sse/services/accountFallback.ts index 5d947d429a..818183e6b7 100644 --- a/open-sse/services/accountFallback.ts +++ b/open-sse/services/accountFallback.ts @@ -50,7 +50,10 @@ import { } from "../../src/shared/constants/providers"; import { resolveUseUpstream429BreakerHints } from "../../src/shared/utils/providerHints"; import { getCodexModelScope } from "../config/codexQuotaScopes.ts"; -import { getQuotaScopedModelForProvider, isAntigravityQuotaProvider } from "./antigravityQuotaFamily.ts"; +import { + getQuotaScopedModelForProvider, + isAntigravityQuotaProvider, +} from "./antigravityQuotaFamily.ts"; import { persistAntigravityFamilyCooldownIfQuota } from "./antigravityFamilyCooldown.ts"; import { classifyGeminiQuotaMetricFromText, @@ -66,12 +69,13 @@ import { MAX_SHORT_RETRY_HINT_MS, } from "./retryAfterJson.ts"; import { isMoonshotAccountBalanceExhausted } from "./usage/moonshotOpenPlatform.ts"; -import { isTpdRateLimit, resolveTpdCooldownMs } from "./dailyQuotaReset.ts"; +import { isTpdRateLimit, resolveTpdCooldownMs, nextConfiguredResetMs } from "./dailyQuotaReset.ts"; // Pre-compiled regex constants for hot-path retry parsing (avoid per-call compilation) const RETRY_AFTER_RE = /retry\s+after\s+(\d+)\s*s/i; const PLEASE_RETRY_RE = /please retry in\s+([\d.]+\s*s)/i; -const ISO_RETRY_RE = /\b(?:try again at|wait until|reset(?:s)? at|available at|retry after)\s+(\d{4}-\d{2}-\d{2}[Tt ]\d{2}:\d{2}(?::\d{2})?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?)/i; +const ISO_RETRY_RE = + /\b(?:try again at|wait until|reset(?:s)? at|available at|retry after)\s+(\d{4}-\d{2}-\d{2}[Tt ]\d{2}:\d{2}(?::\d{2})?(?:\.\d+)?(?:Z|[+-]\d{2}:?\d{2})?)/i; const RESETS_AFTER_RE = /resets? after (\d+h)?(\d+m)?(\d+s)?/i; const WILL_RESET_AFTER_RE = /will reset after (\d+h)?(\d+m)?(\d+s)?/i; const RESETS_IN_RE = /resets? in (\d+h)?(\d+m)?(\d+s)?/i; @@ -376,7 +380,8 @@ export const MODEL_ACCESS_DENIED_PATTERNS = [ /\bunsupported\s+model\b/i, /\baccess.*denied.*model\b/i, /\bmodel.*access.*denied\b/i, - /\bplease select a different model\b/i, /\bunknown\s+provider\s+for\s+model\b/i, + /\bplease select a different model\b/i, + /\bunknown\s+provider\s+for\s+model\b/i, // "...access to the requested model" / "model ... access" — bounded lookahead // (no nested quantifiers) so it stays ReDoS-safe while requiring BOTH an // access/permission word and "model" so a pure auth error never matches. @@ -416,7 +421,8 @@ const PROVIDER_MODEL_UNSUPPORTED_PATTERNS = [ /\bmodel\b[\s\S]{0,80}?\b(?:does\s+not\s+support|doesn't\s+support|unsupported)\b/i, /\b(?:does\s+not\s+support|doesn't\s+support|unsupported)\b[\s\S]{0,80}?\bmodel\b/i, /\bunsupported\s+model\b/i, - /\bplease select a different model\b/i, /\bunknown\s+provider\s+for\s+model\b/i, + /\bplease select a different model\b/i, + /\bunknown\s+provider\s+for\s+model\b/i, ]; /** @@ -656,7 +662,13 @@ export async function recordCoreOwnedAntigravityQuotaState({ } ); if (lockout.cooldownMs > 0 && isProviderExhaustedReason(fallback)) { - persistAntigravityFamilyCooldownIfQuota({ provider, connectionId, model, cooldownMs: lockout.cooldownMs, reason: "quota_exhausted" }); + persistAntigravityFamilyCooldownIfQuota({ + provider, + connectionId, + model, + cooldownMs: lockout.cooldownMs, + reason: "quota_exhausted", + }); } return { cooldownMs: lockout.cooldownMs, failureCount: lockout.failureCount }; } @@ -1661,7 +1673,7 @@ export function checkFallbackError( timezone?: unknown; hour?: unknown; nowMs?: number; - } | null, + } | null ): { shouldFallback: boolean; cooldownMs: number; @@ -1987,7 +1999,7 @@ export function checkFallbackError( // no clock, no header — short 429, do not guess midnight console.warn( "[accountFallback] TPD 429 without node daily-reset clock or Reset header; using short cooldown", - { provider }, + { provider } ); } else { return { @@ -1998,7 +2010,13 @@ export function checkFallbackError( }; } } else { - const msUntilTomorrow = getMsUntilTomorrow(); + // Operator node clock first; host-midnight estimate when unconfigured. + const tzMs = nextConfiguredResetMs( + dailyReset?.timezone, + dailyReset?.hour, + dailyReset?.nowMs ?? Date.now() + ); + const msUntilTomorrow = tzMs ?? getMsUntilTomorrow(); // Cap at 24 hours to handle timezone edge cases const cooldownMs = Math.min(msUntilTomorrow, 24 * 60 * 60 * 1000); return { @@ -2434,7 +2452,13 @@ export function applyErrorState( // (`markConnectionQuotaExhausted`) so a DB failure can never crash the // chat path. See issue #1 (per-account 429 cascade not persisting). const connId = (account as AccountState | null | undefined)?.id; - if (typeof connId === "string" && connId.length > 0 && effectiveCooldownMs > 0 && nextState.rateLimitedUntil && !isAntigravityQuotaProvider(prov)) { + if ( + typeof connId === "string" && + connId.length > 0 && + effectiveCooldownMs > 0 && + nextState.rateLimitedUntil && + !isAntigravityQuotaProvider(prov) + ) { try { const untilMs = cooldownUntilMs(nextState.rateLimitedUntil); if (Number.isFinite(untilMs) && untilMs > Date.now()) { diff --git a/open-sse/services/combo.ts b/open-sse/services/combo.ts index acc72de4ae..89ab50a142 100644 --- a/open-sse/services/combo.ts +++ b/open-sse/services/combo.ts @@ -20,11 +20,7 @@ import { import { getHiddenModelsByProvider } from "@/models"; -import { - evaluateQuotaCutoff, - getQuotaFetcher, - type QuotaInfo, -} from "./quotaPreflight.ts"; +import { evaluateQuotaCutoff, getQuotaFetcher, type QuotaInfo } from "./quotaPreflight.ts"; import { resolveProviderId } from "../../src/shared/constants/providers.ts"; import { getQuotaFetchScope } from "./antigravityQuotaFamily.ts"; import { getCircuitBreaker } from "../../src/shared/utils/circuitBreaker"; @@ -37,10 +33,7 @@ import { projectAccountTier, type ProviderCandidate } from "./autoCombo/scoring. import { getSessionConnection } from "./sessionManager.ts"; import { getOAuthSessionAvailability } from "./oauthSessionOccupancy.ts"; -import { - clearStickyBinding, - peekStickyConnectionId, -} from "./combo/sessionStickiness.ts"; +import { clearStickyBinding, peekStickyConnectionId } from "./combo/sessionStickiness.ts"; import { lookupPositiveCap } from "./combo/concurrencyCaps.ts"; import { acquireQuotaShareConcurrencySlot } from "./combo/quotaShareConcurrency.ts"; @@ -107,20 +100,13 @@ import { tryPipelineDispatch, tryRuntimeUnitDispatch, } from "./combo/dispatchPrelude.ts"; -import { - resolveShadowTargets, - scheduleShadowRouting, -} from "./combo/shadowRouting.ts"; +import { resolveShadowTargets, scheduleShadowRouting } from "./combo/shadowRouting.ts"; import { filterTargetsByRequestCompatibility, resolveComboRuntimeUnits, resolveComboTargets, } from "./combo/comboStructure.ts"; -import { - createInvocationId, - getComboTrace, - startComboTrace, -} from "./combo/decisionTrace.ts"; +import { createInvocationId, getComboTrace, startComboTrace } from "./combo/decisionTrace.ts"; import { QUOTA_SOFT_DEPRIORITIZE_FACTOR, setCandidateQuotaSoftPenalty, @@ -135,20 +121,14 @@ import { calculateResetWindowAffinity, type ResetWindowConfig, } from "./combo/quotaScoring.ts"; -import { - fetchResetAwareQuotaWithCache, - preScreenTargets, -} from "./combo/quotaStrategies.ts"; +import { fetchResetAwareQuotaWithCache, preScreenTargets } from "./combo/quotaStrategies.ts"; import { buildAutoQuotaThresholds } from "./combo/quotaExhaustionCutoff.ts"; import { expandTargetsByFingerprints } from "./combo/fingerprintExpansion.ts"; import { resolveComboTargetPipeline } from "./combo/targetResolution.ts"; import { dispatchWithCooldownRetry } from "./combo/comboAttemptLoop.ts"; import { evaluateExecuteTargetGates } from "./combo/executeTargetGates.ts"; import { executeTargetAttempt } from "./combo/executeTargetAttempt.ts"; -import type { - AttemptLoopDeps, - AttemptLoopState, -} from "./combo/attemptLoopTypes.ts"; +import type { AttemptLoopDeps, AttemptLoopState } from "./combo/attemptLoopTypes.ts"; export { RESET_WINDOW_NAMES, QUOTA_SOFT_DEPRIORITIZE_FACTOR, setCandidateQuotaSoftPenalty }; export { scoreAutoTargets, expandAutoComboCandidatePool }; @@ -1081,4 +1061,3 @@ async function handleComboChatInner({ _unregisterExecutionCandidates(_registeredExecutionKeys); } } - diff --git a/open-sse/services/combo/comboDailyResetClock.ts b/open-sse/services/combo/comboDailyResetClock.ts new file mode 100644 index 0000000000..f514935b45 --- /dev/null +++ b/open-sse/services/combo/comboDailyResetClock.ts @@ -0,0 +1,34 @@ +/** + * Operator daily-reset clock lookup for the combo failure paths. + * + * Combo targets classify upstream failures with `checkFallbackError` directly, + * so they need the same per-provider `{ timezone, hour }` clock that the + * single-model path resolves in `src/sse/services/auth.ts` + * (`resolveDailyResetForProvider`): the provider node matched by id or prefix. + * + * Resolved on every failure through `getCachedProviderNodes`, which already + * owns caching (short TTL, invalidated on every provider_nodes write). There is + * deliberately no second cache here: a timezone/hour edit reaches combos + * without a restart, and a failed lookup returns null (host-midnight fallback in + * `checkFallbackError`) without being remembered. + * + * Dynamic import keeps the combo leaf free of a static edge into the DB layer. + */ + +export type ComboDailyResetClock = { timezone?: unknown; hour?: unknown }; + +export async function resolveComboDailyReset( + provider: string | null | undefined +): Promise { + if (!provider || provider === "unknown") return null; + try { + const { getCachedProviderNodes } = await import("@/lib/db/readCache"); + const nodes = await getCachedProviderNodes(); + const node = nodes.find((n) => n && (n.id === provider || n.prefix === provider)); + if (!node) return null; + return { timezone: node.dailyQuotaResetTimezone, hour: node.dailyQuotaResetHour }; + } catch { + // no-effect: an unreadable node table falls back to host midnight in checkFallbackError + return null; + } +} diff --git a/open-sse/services/combo/executeTargetAttempt.ts b/open-sse/services/combo/executeTargetAttempt.ts index bd948cf535..7c99d9902b 100644 --- a/open-sse/services/combo/executeTargetAttempt.ts +++ b/open-sse/services/combo/executeTargetAttempt.ts @@ -91,6 +91,7 @@ import type { AttemptLoopDeps, AttemptLoopState, ExecuteTargetResult } from "./a import type { ComboDiagnostics } from "../../utils/error.ts"; import type { ComboErrorBody, ComboRetryAfter, ResolvedComboTarget } from "./types.ts"; import type { ResponseValidationConfig } from "./responseValidation.ts"; +import { resolveComboDailyReset } from "./comboDailyResetClock.ts"; export async function executeTargetAttempt(opts: { index: number; @@ -835,7 +836,9 @@ export async function executeTargetAttempt(opts: { provider, result.headers, profile, - structuredError + structuredError, + null, + await resolveComboDailyReset(provider) ); const { cooldownMs } = fallbackResult; // #6863: a parsed upstream quota reset (e.g. Antigravity "Resets in 92h27m28s") diff --git a/open-sse/services/combo/roundRobinCombo.ts b/open-sse/services/combo/roundRobinCombo.ts index 2cc3c6895f..1e79a94663 100644 --- a/open-sse/services/combo/roundRobinCombo.ts +++ b/open-sse/services/combo/roundRobinCombo.ts @@ -112,6 +112,7 @@ import { resolveComboTargets, } from "./comboStructure.ts"; import { releaseStickyPinOnFailure, clearStaleLKGP } from "../combo.ts"; +import { resolveComboDailyReset } from "./comboDailyResetClock.ts"; /** Per-connection TPM budget for quota reservation. Undefined = store keeps prior limit. */ async function resolveTargetTokenLimit(target: { @@ -921,7 +922,9 @@ export async function handleRoundRobinCombo({ provider, result.headers, profile, - structuredError + structuredError, + null, + await resolveComboDailyReset(provider) ); const { cooldownMs } = fallbackResult; const selectedConnectionId = diff --git a/open-sse/services/dailyQuotaReset.ts b/open-sse/services/dailyQuotaReset.ts index a5108d132f..46db112236 100644 --- a/open-sse/services/dailyQuotaReset.ts +++ b/open-sse/services/dailyQuotaReset.ts @@ -57,7 +57,11 @@ function zonedParts(ms: number, timeZone: string): ZonedParts { }; } -function addCalendarDay(year: number, month: number, day: number): { +function addCalendarDay( + year: number, + month: number, + day: number +): { year: number; month: number; day: number; @@ -75,7 +79,7 @@ function zonedLocalToUtc( hour: number, minute: number, second: number, - timeZone: string, + timeZone: string ): number { const wanted = Date.UTC(year, month - 1, day, hour, minute, second); let guess = wanted; @@ -130,7 +134,7 @@ export type TpdCooldownOptions = { */ export function resolveTpdCooldownMs( errorText: string | null | undefined, - options: TpdCooldownOptions = {}, + options: TpdCooldownOptions = {} ): number | null { if (!isTpdRateLimit(errorText)) return null; const now = options.nowMs ?? Date.now(); @@ -143,3 +147,19 @@ export function resolveTpdCooldownMs( } return null; } + +/** + * Milliseconds until the next operator-configured daily reset, or null when + * the clock is absent, invalid, or already passed. Shared by the non-TPD + * daily-quota paths so configured and unconfigured behavior stay in one place. + */ +export function nextConfiguredResetMs( + timezone: unknown, + hour: unknown, + nowMs: number +): number | null { + if (typeof timezone !== "string" || !isValidResetHour(hour)) return null; + if (!nodeDailyResetConfigured(timezone, hour)) return null; + const ms = nextDailyResetAtMs(timezone, hour, nowMs) - nowMs; + return ms > 0 ? ms : null; +} diff --git a/tests/unit/daily-reset-tz-threading.test.ts b/tests/unit/daily-reset-tz-threading.test.ts new file mode 100644 index 0000000000..42979e3d9d --- /dev/null +++ b/tests/unit/daily-reset-tz-threading.test.ts @@ -0,0 +1,248 @@ +/** + * #13440 — non-TPD daily-quota cooldowns honor the provider node's configured + * daily-reset clock (dailyQuotaResetTimezone + dailyQuotaResetHour) instead of + * host midnight, on both the single-model classifier and the combo call sites. + */ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +process.env.TZ = "UTC"; +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-daily-reset-13440-")); +const ORIGINAL_DATA_DIR = process.env.DATA_DIR; +process.env.DATA_DIR = TEST_DATA_DIR; + +const { checkFallbackError, getMsUntilTomorrow } = + await import("../../open-sse/services/accountFallback.ts"); +const { nextDailyResetAtMs } = await import("../../open-sse/services/dailyQuotaReset.ts"); +const { resolveComboDailyReset } = + await import("../../open-sse/services/combo/comboDailyResetClock.ts"); +const { handleComboChat } = await import("../../open-sse/services/combo.ts"); +const rrState = await import("../../open-sse/services/combo/rrState.ts"); +const { createProviderNode, updateProviderNode } = + await import("../../src/lib/db/providers/nodes.ts"); +const dbCore = await import("../../src/lib/db/core.ts"); + +const DAILY_TEXT = "daily quota exceeded, try again tomorrow"; +const HOUR_MS = 3_600_000; +const realDateNow = Date.now; + +/** Shift the wall clock so "now" is `nowMs` (keeps advancing in real time). */ +function shiftClockTo(nowMs: number): void { + const offset = nowMs - realDateNow(); + Date.now = () => realDateNow() + offset; +} + +function dailyCooldownMs(timezone: unknown, hour: unknown, nowMs: number): number { + return checkFallbackError(403, DAILY_TEXT, 0, null, "tz-thread-prov", null, null, null, null, { + timezone, + hour, + nowMs, + }).cooldownMs; +} + +type LogCall = { level: string; msg: string }; +function captureLog(calls: LogCall[]) { + const push = (level: string) => (_tag: string, msg: unknown) => + calls.push({ level, msg: String(msg) }); + return { info: push("info"), warn: push("warn"), debug: push("debug"), error: push("error") }; +} + +function dailyQuotaResponse(status: number): Response { + return new Response(JSON.stringify({ error: { message: DAILY_TEXT } }), { + status, + headers: { "content-type": "application/json" }, + }); +} + +async function dispatch( + combo: Record, + failingProvider: string, + failStatus: number, + calls: LogCall[] +) { + const res = await handleComboChat({ + body: { model: combo.name, messages: [{ role: "user", content: "hi" }], stream: false }, + combo, + allCombos: [combo], + isModelAvailable: async () => true, + relayOptions: undefined, + signal: undefined, + settings: {}, + log: captureLog(calls), + handleSingleModel: async (_b: unknown, modelStr: string) => { + if (modelStr.startsWith(`${failingProvider}/`)) return dailyQuotaResponse(failStatus); + return Response.json({ choices: [{ message: { role: "assistant", content: "ok" } }] }); + }, + }); + await (res as Response | undefined)?.body?.cancel().catch(() => {}); +} + +function comboFor(name: string, strategy: string, nodeId: string, config = {}) { + return { + name, + strategy, + config: { maxRetries: 0, disableSessionStickiness: true, ...config }, + models: [ + { kind: "model", provider: nodeId, providerId: nodeId, model: "m-a", id: `${name}-0` }, + { kind: "model", provider: "glm-cn", providerId: "glm-cn", model: "m-b", id: `${name}-1` }, + ], + }; +} + +function rrCooldownFromLogs(calls: LogCall[]): number | null { + for (const c of calls) { + const m = /error 429, cooldown (\d+)ms/.exec(c.msg); + if (c.level === "warn" && m) return Number(m[1]); + } + return null; +} + +test.beforeEach(() => { + rrState.rrCounters.clear(); + rrState.rrStickyTargets.clear(); +}); + +test.afterEach(() => { + Date.now = realDateNow; +}); + +test.after(() => { + Date.now = realDateNow; + try { + dbCore.resetDbInstance(); + } catch { + /* ignore */ + } + if (ORIGINAL_DATA_DIR === undefined) delete process.env.DATA_DIR; + else process.env.DATA_DIR = ORIGINAL_DATA_DIR; + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 }); +}); + +test("checkFallbackError: Paris pre-spring-forward resolves to provider midnight", () => { + const nowMs = Date.parse("2026-03-28T21:00:00Z"); + assert.equal(dailyCooldownMs("Europe/Paris", 0, nowMs), 2 * HOUR_MS); +}); + +test("checkFallbackError: Paris pre-fall-back resolves to provider midnight", () => { + const nowMs = Date.parse("2026-10-24T10:00:00Z"); + assert.equal(dailyCooldownMs("Europe/Paris", 0, nowMs), 12 * HOUR_MS); +}); + +test("checkFallbackError: New York resolves to provider midnight, not host midnight", () => { + const nowMs = Date.parse("2026-01-16T04:00:00Z"); + assert.equal(dailyCooldownMs("America/New_York", 0, nowMs), HOUR_MS); +}); + +// The legacy value is recomputed from a live Date.now() inside getMsUntilTomorrow(), so the two +// reads are a few ms apart under load; compare within a second instead of strictly. +function assertWithinASecond(actual: number, expected: number, label: string): void { + assert.ok( + Math.abs(actual - expected) <= 1000, + `${label}: expected ${actual} within 1s of ${expected}` + ); +} + +test("checkFallbackError: unconfigured clock keeps the legacy host-midnight value", () => { + shiftClockTo(Date.parse("2026-01-15T12:00:00Z")); + assertWithinASecond( + dailyCooldownMs(undefined, undefined, Date.now()), + getMsUntilTomorrow(), + "unconfigured clock" + ); +}); + +test("checkFallbackError: invalid timezone falls back to legacy without throwing", () => { + shiftClockTo(Date.parse("2026-01-15T12:00:00Z")); + assertWithinASecond( + dailyCooldownMs("Mars/Olympus", 0, Date.now()), + getMsUntilTomorrow(), + "invalid tz" + ); +}); + +test("resolveComboDailyReset: matches id and prefix, null for unknown providers", async () => { + const node = await createProviderNode({ + type: "openai-compatible", + name: "Daily reset lookup", + prefix: "drlookup13440", + apiType: "chat", + baseUrl: "http://127.0.0.1:9/v1", + dailyQuotaResetTimezone: "Europe/Paris", + dailyQuotaResetHour: 7, + }); + const expected = { timezone: "Europe/Paris", hour: 7 }; + assert.deepEqual(await resolveComboDailyReset(String(node.id)), expected); + assert.deepEqual(await resolveComboDailyReset("drlookup13440"), expected); + assert.equal(await resolveComboDailyReset("no-such-provider-13440"), null); + assert.equal(await resolveComboDailyReset("unknown"), null); + assert.equal(await resolveComboDailyReset(null), null); +}); + +test("round-robin combo passes the node clock, and a timezone edit applies without restart", async () => { + const node = await createProviderNode({ + type: "openai-compatible", + name: "Daily reset RR", + prefix: "drrr13440", + apiType: "chat", + baseUrl: "http://127.0.0.1:9/v1", + dailyQuotaResetTimezone: "America/New_York", + dailyQuotaResetHour: 0, + }); + const nodeId = String(node.id); + + // 1h before New York midnight: provider clock says 1h, host (UTC) midnight is ~19-20h away. + shiftClockTo(nextDailyResetAtMs("America/New_York", 0, realDateNow()) - HOUR_MS); + assert.ok(getMsUntilTomorrow() > 3 * HOUR_MS, "fixture must separate host and provider clocks"); + const first: LogCall[] = []; + await dispatch(comboFor("rr13440-a", "round-robin", nodeId), nodeId, 429, first); + const firstCooldown = rrCooldownFromLogs(first); + assert.ok(firstCooldown !== null, "RR must log the semaphore cooldown for the 429"); + assert.ok( + Math.abs(firstCooldown - HOUR_MS) < 10_000, + `expected ~1h (New York midnight), got ${firstCooldown}ms` + ); + + // Operator edits the node: Tokyo midnight. No restart, no cache reset in the test. + await updateProviderNode(nodeId, { dailyQuotaResetTimezone: "Asia/Tokyo" }); + shiftClockTo(nextDailyResetAtMs("Asia/Tokyo", 0, realDateNow()) - 2 * HOUR_MS); + assert.ok(Math.abs(getMsUntilTomorrow() - 2 * HOUR_MS) > HOUR_MS); + const second: LogCall[] = []; + await dispatch(comboFor("rr13440-b", "round-robin", nodeId), nodeId, 429, second); + const secondCooldown = rrCooldownFromLogs(second); + assert.ok(secondCooldown !== null, "RR must log the semaphore cooldown for the 429"); + assert.ok( + Math.abs(secondCooldown - 2 * HOUR_MS) < 10_000, + `expected ~2h (Tokyo midnight after the edit), got ${secondCooldown}ms` + ); +}); + +test("priority combo attempt path passes the node clock to checkFallbackError", async () => { + const node = await createProviderNode({ + type: "openai-compatible", + name: "Daily reset priority", + prefix: "drprio13440", + apiType: "chat", + baseUrl: "http://127.0.0.1:9/v1", + dailyQuotaResetTimezone: "America/New_York", + dailyQuotaResetHour: 0, + }); + const nodeId = String(node.id); + + // 2s before New York midnight: the provider-clock cooldown (~2s) is short enough for + // the pre-fallback wait (<= MAX_FALLBACK_WAIT_MS); host midnight (hours) is not. + shiftClockTo(nextDailyResetAtMs("America/New_York", 0, realDateNow()) - 2_000); + const calls: LogCall[] = []; + await dispatch( + comboFor("prio13440", "priority", nodeId, { fallbackDelayMs: 25 }), + nodeId, + 503, + calls + ); + assert.ok( + calls.some((c) => c.level === "debug" && /Waiting 25ms before fallback/.test(c.msg)), + `expected the provider-clock fallback wait; logs: ${JSON.stringify(calls.map((c) => c.msg))}` + ); +});