mirror of
https://github.com/diegosouzapw/OmniRoute.git
synced 2026-09-14 02:42:24 +03:00
65e81158ab
Landed with the design call resolved per the owner's pick — **option 1**: the synced store is now endpoint-agnostic (persistDiscoveredModels and managedModelImport no longer drop non-chat models at write time), and chat selectability moved to read time (auto-pool expansion in autoStrategy applies filterChatSelectableModels; the models-route projection already had its chatOnly filter). Your discovery test now passes end-to-end (3/3): /api/show capabilities persist per connection and image/embedding requests route through the advertising host. Reconciliation notes: conflicted areas merged onto the current tip (adobe discovery import, requestedModel preflight signature, resolvedProvider fast-path coexists with the synced-route override — explicit resolution wins); carried base-red drains (#10055 memoization, #11071 test variants) dropped as already-landed; the managed-model-import exclusion test was propagated to the new contract (image/video models persist; the read filter still hides them from chat pickers — pinned by a new assertion). Full battery: 205/206 focused (the one red is a confirmed periodic-timer timing flake on the loaded devbox — 20/20 isolated), autoCombo vitest 30/30, combo suites 46/46, gates + typecheck clean. Thank you @yourspraveen — the capability probe + routing design was right; it just needed the store contract opened up. Fixes #11087.
84 lines
2.9 KiB
TypeScript
84 lines
2.9 KiB
TypeScript
/**
|
|
* Canonical streaming timing instrumentation (TTFT / ITL / interruption).
|
|
*
|
|
* One reusable seam for measuring the streaming path. It is created once per
|
|
* stream and marked from the SSE transform:
|
|
*
|
|
* markByte() — first upstream chunk received (bytes arrived from provider)
|
|
* markForward() — first chunk forwarded to the client (first SSE chunk enqueued)
|
|
*
|
|
* `ttft()` is therefore **first-forwarded-SSE-chunk latency**, NOT token-level
|
|
* TTFT. We document that distinction explicitly: a single SSE chunk can carry
|
|
* zero, one, or many tokens, and chunk boundaries do not map to token
|
|
* boundaries. If a future implementation can measure actual token timing it
|
|
* should extend this seam, not bypass it.
|
|
*
|
|
* ITL (inter-token latency) is approximated by the mean gap between forwarded
|
|
* SSE chunks (bounded sample window). It is a chunk-latency proxy, again not
|
|
* true token timing — callers must label it as such.
|
|
*
|
|
* The object is cheap to construct, plain mutable state, and safe under the
|
|
* event loop's single thread (each stream owns its own instance).
|
|
*/
|
|
export interface StreamTiming {
|
|
startedAt: number;
|
|
firstByteAt: number | null;
|
|
firstForwardAt: number | null;
|
|
lastForwardAt: number | null;
|
|
/** Mean gap between forwarded chunks (ms), bounded window. */
|
|
interChunkGaps: number[];
|
|
forwardedChunks: number;
|
|
interrupted: boolean;
|
|
markByte(): void;
|
|
markForward(): void;
|
|
markInterrupted(): void;
|
|
/** First-forwarded-SSE-chunk latency in ms, or null if nothing was forwarded. */
|
|
ttftMs(): number | null;
|
|
/** Mean inter-chunk gap in ms, or null when fewer than 2 chunks were forwarded. */
|
|
avgItlMs(): number | null;
|
|
/** Time from stream start to completion (ms). */
|
|
totalMs(): number;
|
|
}
|
|
|
|
/** Max number of inter-chunk samples kept (bounds memory). */
|
|
const MAX_INTER_CHUNK_GAPS = 32;
|
|
|
|
export function createStreamTiming(): StreamTiming {
|
|
const timing: StreamTiming = {
|
|
startedAt: Date.now(),
|
|
firstByteAt: null,
|
|
firstForwardAt: null,
|
|
lastForwardAt: null,
|
|
interChunkGaps: [],
|
|
forwardedChunks: 0,
|
|
interrupted: false,
|
|
markByte() {
|
|
if (this.firstByteAt === null) this.firstByteAt = Date.now();
|
|
},
|
|
markForward() {
|
|
const now = Date.now();
|
|
if (this.firstForwardAt === null) this.firstForwardAt = now;
|
|
if (this.lastForwardAt !== null && this.interChunkGaps.length < MAX_INTER_CHUNK_GAPS) {
|
|
this.interChunkGaps.push(now - this.lastForwardAt);
|
|
}
|
|
this.lastForwardAt = now;
|
|
this.forwardedChunks += 1;
|
|
},
|
|
markInterrupted() {
|
|
this.interrupted = true;
|
|
},
|
|
ttftMs() {
|
|
return this.firstForwardAt === null ? null : this.firstForwardAt - this.startedAt;
|
|
},
|
|
avgItlMs() {
|
|
if (this.interChunkGaps.length === 0) return null;
|
|
const sum = this.interChunkGaps.reduce((a, b) => a + b, 0);
|
|
return sum / this.interChunkGaps.length;
|
|
},
|
|
totalMs() {
|
|
return Date.now() - this.startedAt;
|
|
},
|
|
};
|
|
return timing;
|
|
}
|