diff --git a/src/app/api/v1/audio/speech/route.ts b/src/app/api/v1/audio/speech/route.ts index 4ade1612da..7cdbf75424 100644 --- a/src/app/api/v1/audio/speech/route.ts +++ b/src/app/api/v1/audio/speech/route.ts @@ -1,6 +1,11 @@ import { CORS_ORIGIN } from "@/shared/utils/cors"; import { handleAudioSpeech } from "@omniroute/open-sse/handlers/audioSpeech.ts"; -import { getProviderCredentials, extractApiKey, isValidApiKey } from "@/sse/services/auth"; +import { + getProviderCredentials, + clearRecoveredProviderState, + extractApiKey, + isValidApiKey, +} from "@/sse/services/auth"; import { parseSpeechModel, getSpeechProvider } from "@omniroute/open-sse/config/audioRegistry.ts"; import { errorResponse } from "@omniroute/open-sse/utils/error.ts"; import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts"; @@ -70,5 +75,9 @@ export async function POST(request) { } } - return handleAudioSpeech({ body, credentials }); + const response = await handleAudioSpeech({ body, credentials }); + if (response?.ok) { + await clearRecoveredProviderState(credentials); + } + return response; } diff --git a/src/app/api/v1/audio/transcriptions/route.ts b/src/app/api/v1/audio/transcriptions/route.ts index 6fe5ab4dbc..93e594b24c 100644 --- a/src/app/api/v1/audio/transcriptions/route.ts +++ b/src/app/api/v1/audio/transcriptions/route.ts @@ -1,6 +1,11 @@ import { CORS_ORIGIN } from "@/shared/utils/cors"; import { handleAudioTranscription } from "@omniroute/open-sse/handlers/audioTranscription.ts"; -import { getProviderCredentials, extractApiKey, isValidApiKey } from "@/sse/services/auth"; +import { + getProviderCredentials, + clearRecoveredProviderState, + extractApiKey, + isValidApiKey, +} from "@/sse/services/auth"; import { parseTranscriptionModel, getTranscriptionProvider } from "@omniroute/open-sse/config/audioRegistry.ts"; import { errorResponse } from "@omniroute/open-sse/utils/error.ts"; import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts"; @@ -68,5 +73,9 @@ export async function POST(request) { } } - return handleAudioTranscription({ formData, credentials }); + const response = await handleAudioTranscription({ formData, credentials }); + if (response?.ok) { + await clearRecoveredProviderState(credentials); + } + return response; } diff --git a/src/app/api/v1/embeddings/route.ts b/src/app/api/v1/embeddings/route.ts index 11272d4143..bd4c5a96ba 100644 --- a/src/app/api/v1/embeddings/route.ts +++ b/src/app/api/v1/embeddings/route.ts @@ -1,6 +1,11 @@ import { CORS_ORIGIN } from "@/shared/utils/cors"; import { handleEmbedding } from "@omniroute/open-sse/handlers/embeddings.ts"; -import { getProviderCredentials, extractApiKey, isValidApiKey } from "@/sse/services/auth"; +import { + getProviderCredentials, + clearRecoveredProviderState, + extractApiKey, + isValidApiKey, +} from "@/sse/services/auth"; import { parseEmbeddingModel, getAllEmbeddingModels, @@ -126,6 +131,7 @@ export async function POST(request) { const result = await handleEmbedding({ body, credentials, log }); if (result.success) { + await clearRecoveredProviderState(credentials); return new Response(JSON.stringify(result.data), { status: 200, headers: { "Content-Type": "application/json" }, diff --git a/src/app/api/v1/images/generations/route.ts b/src/app/api/v1/images/generations/route.ts index c64da344b3..046d63f8f7 100644 --- a/src/app/api/v1/images/generations/route.ts +++ b/src/app/api/v1/images/generations/route.ts @@ -1,6 +1,11 @@ import { CORS_ORIGIN } from "@/shared/utils/cors"; import { handleImageGeneration } from "@omniroute/open-sse/handlers/imageGeneration.ts"; -import { getProviderCredentials, extractApiKey, isValidApiKey } from "@/sse/services/auth"; +import { + getProviderCredentials, + clearRecoveredProviderState, + extractApiKey, + isValidApiKey, +} from "@/sse/services/auth"; import { parseImageModel, getAllImageModels, @@ -170,6 +175,7 @@ export async function POST(request) { }); if (result.success) { + await clearRecoveredProviderState(credentials); return new Response(JSON.stringify((result as any).data), { status: 200, headers: { "Content-Type": "application/json" }, diff --git a/src/app/api/v1/moderations/route.ts b/src/app/api/v1/moderations/route.ts index e3f5534592..8b407c0a5e 100644 --- a/src/app/api/v1/moderations/route.ts +++ b/src/app/api/v1/moderations/route.ts @@ -1,6 +1,11 @@ import { CORS_ORIGIN } from "@/shared/utils/cors"; import { handleModeration } from "@omniroute/open-sse/handlers/moderations.ts"; -import { getProviderCredentials, extractApiKey, isValidApiKey } from "@/sse/services/auth"; +import { + getProviderCredentials, + clearRecoveredProviderState, + extractApiKey, + isValidApiKey, +} from "@/sse/services/auth"; import { parseModerationModel } from "@omniroute/open-sse/config/moderationRegistry.ts"; import { errorResponse } from "@omniroute/open-sse/utils/error.ts"; import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts"; @@ -64,5 +69,9 @@ export async function POST(request) { ); } - return handleModeration({ body: { ...body, model }, credentials }); + const response = await handleModeration({ body: { ...body, model }, credentials }); + if (response?.ok) { + await clearRecoveredProviderState(credentials); + } + return response; } diff --git a/src/app/api/v1/music/generations/route.ts b/src/app/api/v1/music/generations/route.ts index 1f5eb2367a..7e88643dd0 100644 --- a/src/app/api/v1/music/generations/route.ts +++ b/src/app/api/v1/music/generations/route.ts @@ -1,6 +1,11 @@ import { CORS_ORIGIN } from "@/shared/utils/cors"; import { handleMusicGeneration } from "@omniroute/open-sse/handlers/musicGeneration.ts"; -import { getProviderCredentials, extractApiKey, isValidApiKey } from "@/sse/services/auth"; +import { + getProviderCredentials, + clearRecoveredProviderState, + extractApiKey, + isValidApiKey, +} from "@/sse/services/auth"; import { parseMusicModel, getAllMusicModels, @@ -110,6 +115,7 @@ export async function POST(request) { const result = await handleMusicGeneration({ body, credentials, log }); if (result.success) { + await clearRecoveredProviderState(credentials); return new Response(JSON.stringify((result as any).data), { status: 200, headers: { "Content-Type": "application/json" }, diff --git a/src/app/api/v1/providers/[provider]/embeddings/route.ts b/src/app/api/v1/providers/[provider]/embeddings/route.ts index 1928303adb..7f640b80cd 100644 --- a/src/app/api/v1/providers/[provider]/embeddings/route.ts +++ b/src/app/api/v1/providers/[provider]/embeddings/route.ts @@ -2,7 +2,12 @@ import { CORS_ORIGIN } from "@/shared/utils/cors"; import { errorResponse } from "@omniroute/open-sse/utils/error.ts"; import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts"; import { getRegistryEntry } from "@omniroute/open-sse/config/providerRegistry.ts"; -import { getProviderCredentials, extractApiKey, isValidApiKey } from "@/sse/services/auth"; +import { + getProviderCredentials, + clearRecoveredProviderState, + extractApiKey, + isValidApiKey, +} from "@/sse/services/auth"; import { handleEmbedding } from "@omniroute/open-sse/handlers/embeddings.ts"; import * as log from "@/sse/utils/logger"; import { enforceApiKeyPolicy } from "@/shared/utils/apiKeyPolicy"; @@ -84,6 +89,7 @@ export async function POST(request, { params }) { const result = await handleEmbedding({ body, credentials, log }); if (result.success) { + await clearRecoveredProviderState(credentials); return new Response(JSON.stringify(result.data), { status: 200, headers: { "Content-Type": "application/json" }, diff --git a/src/app/api/v1/providers/[provider]/images/generations/route.ts b/src/app/api/v1/providers/[provider]/images/generations/route.ts index ff6db542b1..f96b05dcce 100644 --- a/src/app/api/v1/providers/[provider]/images/generations/route.ts +++ b/src/app/api/v1/providers/[provider]/images/generations/route.ts @@ -2,7 +2,12 @@ import { CORS_ORIGIN } from "@/shared/utils/cors"; import { handleImageGeneration } from "@omniroute/open-sse/handlers/imageGeneration.ts"; import { errorResponse } from "@omniroute/open-sse/utils/error.ts"; import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts"; -import { getProviderCredentials, extractApiKey, isValidApiKey } from "@/sse/services/auth"; +import { + getProviderCredentials, + clearRecoveredProviderState, + extractApiKey, + isValidApiKey, +} from "@/sse/services/auth"; import { getImageProvider } from "@omniroute/open-sse/config/imageRegistry.ts"; import * as log from "@/sse/utils/logger"; import { toJsonErrorPayload } from "@/shared/utils/upstreamError"; @@ -84,6 +89,7 @@ export async function POST(request, { params }) { const result = await handleImageGeneration({ body, credentials, log }); if (result.success) { + await clearRecoveredProviderState(credentials); return new Response(JSON.stringify((result as any).data), { status: 200, headers: { "Content-Type": "application/json" }, diff --git a/src/app/api/v1/rerank/route.ts b/src/app/api/v1/rerank/route.ts index a8371f135d..99627cea1a 100644 --- a/src/app/api/v1/rerank/route.ts +++ b/src/app/api/v1/rerank/route.ts @@ -1,6 +1,11 @@ import { CORS_ORIGIN } from "@/shared/utils/cors"; import { handleRerank } from "@omniroute/open-sse/handlers/rerank.ts"; -import { getProviderCredentials, extractApiKey, isValidApiKey } from "@/sse/services/auth"; +import { + getProviderCredentials, + clearRecoveredProviderState, + extractApiKey, + isValidApiKey, +} from "@/sse/services/auth"; import { parseRerankModel } from "@omniroute/open-sse/config/rerankRegistry.ts"; import { errorResponse } from "@omniroute/open-sse/utils/error.ts"; import { HTTP_STATUS } from "@omniroute/open-sse/config/constants.ts"; @@ -66,7 +71,7 @@ export async function POST(request) { return errorResponse(HTTP_STATUS.BAD_REQUEST, `No credentials for provider: ${provider}`); } - return handleRerank({ + const response = await handleRerank({ model: body.model, query: body.query, documents: body.documents, @@ -74,4 +79,8 @@ export async function POST(request) { return_documents: body.return_documents, credentials, }); + if (response?.ok) { + await clearRecoveredProviderState(credentials); + } + return response; } diff --git a/src/app/api/v1/videos/generations/route.ts b/src/app/api/v1/videos/generations/route.ts index 8184d3e874..702012e793 100644 --- a/src/app/api/v1/videos/generations/route.ts +++ b/src/app/api/v1/videos/generations/route.ts @@ -1,6 +1,11 @@ import { CORS_ORIGIN } from "@/shared/utils/cors"; import { handleVideoGeneration } from "@omniroute/open-sse/handlers/videoGeneration.ts"; -import { getProviderCredentials, extractApiKey, isValidApiKey } from "@/sse/services/auth"; +import { + getProviderCredentials, + clearRecoveredProviderState, + extractApiKey, + isValidApiKey, +} from "@/sse/services/auth"; import { parseVideoModel, getAllVideoModels, @@ -110,6 +115,7 @@ export async function POST(request) { const result = await handleVideoGeneration({ body, credentials, log }); if (result.success) { + await clearRecoveredProviderState(credentials); return new Response(JSON.stringify((result as any).data), { status: 200, headers: { "Content-Type": "application/json" }, diff --git a/src/sse/services/auth.ts b/src/sse/services/auth.ts index a106fb6a55..1e3214af1f 100644 --- a/src/sse/services/auth.ts +++ b/src/sse/services/auth.ts @@ -35,10 +35,22 @@ interface ProviderConnectionView { consecutiveUseCount: number; priority: number; lastError: string | null; + lastErrorType: string | null; + lastErrorSource: string | null; errorCode: string | number | null; backoffLevel: number; } +interface RecoverableConnectionState { + connectionId: string; + testStatus?: string | null; + lastError?: string | null; + rateLimitedUntil?: string | null; + errorCode?: string | number | null; + lastErrorType?: string | null; + lastErrorSource?: string | null; +} + const CODEX_QUOTA_THRESHOLD_PERCENT = 90; function asRecord(value: unknown): JsonRecord { @@ -76,6 +88,8 @@ function toProviderConnection(value: unknown): ProviderConnectionView { consecutiveUseCount: toNumber(row.consecutiveUseCount, 0), priority: toNumber(row.priority, 999), lastError: toStringOrNull(row.lastError), + lastErrorType: toStringOrNull(row.lastErrorType), + lastErrorSource: toStringOrNull(row.lastErrorSource), errorCode: typeof row.errorCode === "string" || typeof row.errorCode === "number" ? row.errorCode : null, backoffLevel: toNumber(row.backoffLevel, 0), @@ -469,6 +483,9 @@ export async function getProviderCredentials( // Include current status for optimization check testStatus: connection.testStatus, lastError: connection.lastError, + lastErrorType: connection.lastErrorType, + lastErrorSource: connection.lastErrorSource, + errorCode: connection.errorCode, rateLimitedUntil: connection.rateLimitedUntil, }; } finally { @@ -569,12 +586,18 @@ export async function markAccountUnavailable( * Clear account error status (only if currently has error) * Optimized to avoid unnecessary DB updates */ -export async function clearAccountError(connectionId: string, currentConnection: any) { +export async function clearAccountError( + connectionId: string, + currentConnection: Partial +) { // Only update if currently has error status const hasError = - currentConnection.testStatus === "unavailable" || + (currentConnection.testStatus && currentConnection.testStatus !== "active") || currentConnection.lastError || - currentConnection.rateLimitedUntil; + currentConnection.rateLimitedUntil || + currentConnection.errorCode || + currentConnection.lastErrorType || + currentConnection.lastErrorSource; if (!hasError) return; // Skip if already clean @@ -582,12 +605,22 @@ export async function clearAccountError(connectionId: string, currentConnection: testStatus: "active", lastError: null, lastErrorAt: null, + lastErrorType: null, + lastErrorSource: null, + errorCode: null, rateLimitedUntil: null, backoffLevel: 0, }); log.info("AUTH", `Account ${connectionId.slice(0, 8)} error cleared`); } +export async function clearRecoveredProviderState( + credentials: Partial | null +) { + if (!credentials?.connectionId) return; + await clearAccountError(credentials.connectionId, credentials); +} + /** * Extract API key from request headers */ diff --git a/tests/unit/auth-clear-account-error.test.mjs b/tests/unit/auth-clear-account-error.test.mjs new file mode 100644 index 0000000000..cb7f283f7f --- /dev/null +++ b/tests/unit/auth-clear-account-error.test.mjs @@ -0,0 +1,59 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-auth-clear-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); +const auth = await import("../../src/sse/services/auth.ts"); + +async function resetStorage() { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); +} + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +test("clearAccountError clears stale provider error metadata after recovery", async () => { + await resetStorage(); + core.getDbInstance().exec('ALTER TABLE provider_connections ADD COLUMN "group" TEXT'); + + const created = await providersDb.createProviderConnection({ + provider: "codex", + authType: "oauth", + email: "recover@example.com", + accessToken: "access", + refreshToken: "refresh", + testStatus: "active", + lastError: null, + lastErrorType: "token_refresh_failed", + lastErrorSource: "oauth", + errorCode: "refresh_failed", + rateLimitedUntil: null, + backoffLevel: 2, + }); + + const credentials = await auth.getProviderCredentials("codex"); + assert.equal(credentials.connectionId, created.id); + assert.equal(credentials.errorCode, "refresh_failed"); + assert.equal(credentials.lastErrorType, "token_refresh_failed"); + assert.equal(credentials.lastErrorSource, "oauth"); + await auth.clearAccountError(created.id, credentials); + + const updated = await providersDb.getProviderConnectionById(created.id); + assert.equal(updated.testStatus, "active"); + assert.equal(updated.lastError, undefined); + assert.equal(updated.lastErrorType, undefined); + assert.equal(updated.lastErrorSource, undefined); + assert.equal(updated.errorCode, undefined); + assert.equal(updated.rateLimitedUntil, undefined); + assert.equal(updated.backoffLevel, 0); +}); diff --git a/tests/unit/auth-clear-provider-routes.test.mjs b/tests/unit/auth-clear-provider-routes.test.mjs new file mode 100644 index 0000000000..30c192ee2e --- /dev/null +++ b/tests/unit/auth-clear-provider-routes.test.mjs @@ -0,0 +1,109 @@ +import test from "node:test"; +import assert from "node:assert/strict"; +import fs from "node:fs"; +import os from "node:os"; +import path from "node:path"; + +const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-auth-routes-")); +process.env.DATA_DIR = TEST_DATA_DIR; + +const core = await import("../../src/lib/db/core.ts"); +const providersDb = await import("../../src/lib/db/providers.ts"); +const moderationRoute = await import("../../src/app/api/v1/moderations/route.ts"); +const embeddingsRoute = await import("../../src/app/api/v1/embeddings/route.ts"); + +async function resetStorage() { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); + fs.mkdirSync(TEST_DATA_DIR, { recursive: true }); + core.getDbInstance().exec('ALTER TABLE provider_connections ADD COLUMN "group" TEXT'); +} + +async function seedOpenAIConnection(email) { + return await providersDb.createProviderConnection({ + provider: "openai", + authType: "apikey", + email, + name: email, + apiKey: "sk-test", + testStatus: "active", + lastError: null, + lastErrorType: "token_refresh_failed", + lastErrorSource: "oauth", + errorCode: "refresh_failed", + rateLimitedUntil: null, + backoffLevel: 2, + }); +} + +async function readConnection(id) { + return await providersDb.getProviderConnectionById(id); +} + +test.after(() => { + core.resetDbInstance(); + fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true }); +}); + +test("moderations route clears stale provider error metadata on success", async () => { + await resetStorage(); + const created = await seedOpenAIConnection("moderation@example.com"); + + const originalFetch = globalThis.fetch; + globalThis.fetch = async () => + Response.json({ + id: "modr-1", + model: "omni-moderation-latest", + results: [{ flagged: false }], + }); + + try { + const request = new Request("http://localhost/v1/moderations", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ input: "hello" }), + }); + + const response = await moderationRoute.POST(request); + assert.equal(response.status, 200); + + const updated = await readConnection(created.id); + assert.equal(updated.testStatus, "active"); + assert.equal(updated.errorCode, undefined); + assert.equal(updated.lastErrorType, undefined); + assert.equal(updated.lastErrorSource, undefined); + } finally { + globalThis.fetch = originalFetch; + } +}); + +test("embeddings route clears stale provider error metadata on success", async () => { + await resetStorage(); + const created = await seedOpenAIConnection("embeddings@example.com"); + + const originalFetch = globalThis.fetch; + globalThis.fetch = async () => + Response.json({ + data: [{ object: "embedding", index: 0, embedding: [0.1, 0.2] }], + usage: { prompt_tokens: 3, total_tokens: 3 }, + }); + + try { + const request = new Request("http://localhost/v1/embeddings", { + method: "POST", + headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ model: "openai/text-embedding-3-small", input: "hello" }), + }); + + const response = await embeddingsRoute.POST(request); + assert.equal(response.status, 200); + + const updated = await readConnection(created.id); + assert.equal(updated.testStatus, "active"); + assert.equal(updated.errorCode, undefined); + assert.equal(updated.lastErrorType, undefined); + assert.equal(updated.lastErrorSource, undefined); + } finally { + globalThis.fetch = originalFetch; + } +});