Files
OmniRoute/tests/unit/lib/jobRegistry/registry.test.ts
Diego Rodrigues de Sa e Souza 3d4f3e4960 test(infra): retry recursive temp-dir removal instead of failing a shard on ENOTEMPTY (#11966) (#11968)
* test(infra): retry recursive temp-dir removal instead of failing a shard on ENOTEMPTY (#11966)

Two shards on release/v3.8.51 went red in one day with the same signature —
"ENOTEMPTY, Directory not empty: /tmp/omniroute-<test>-XXXXXX" — from
combo-same-provider-cascade (Unit Tests fast-path 4/4, on a PR that touches only
.github/) and auth-policy-embeddings-webfetch-7785 (the 20k-test TIA step). Both pass
alone and on re-run: the cleanup races something still writing into the directory
(SQLite WAL/-shm checkpoint, a worker, the backup) and under a loaded hosted runner
the window opens. 1154 test files do their own cleanup with
fs.rmSync(dir, { recursive: true, force: true }); 57 already asked for retries.

One-shot codemod (scripts/ad-hoc/codemod-rm-maxretries.mjs, kept for the record):
every rm / rmSync / rmdirSync option object with `recursive: true` and no
`maxRetries` gains `maxRetries: 5, retryDelay: 100` — Node itself then retries
ENOTEMPTY/EBUSY/EPERM for up to ~0.5 s before giving up. 2243 call sites in 1292
files under tests/, the shared tests/_setup/isolateDataDir.ts exit hook included.
Only the option object changes: no call site, assertion or import is touched.

Validation: prettier and ESLint (with the frozen suppressions) clean on all 1292
files; a random 20-file sample runs green (quota-redis-store hangs identically on
the untouched tree — it needs a Redis on localhost, an environment matter). The
four unit shards on this PR are the full run.

* fix(quality): let check-forgotten-sibling-tests read a 1,000-file diff

The gate shells out to `git diff` through execFileSync with Node's default 1 MB
maxBuffer; the 1,292-file codemod in this PR is the first diff large enough to
overflow it, and the gate died with `spawnSync git ENOBUFS` before comparing
anything. 64 MB is far above any real PR and costs nothing when unused.
2026-08-29 01:17:40 -03:00

595 lines
17 KiB
TypeScript

/** JobRegistry runtime tests . Uses real timers + isolated temp DB. */
// Access private internals (avoids `as any`).
type TestRegistry = JobRegistry & {
timers: Map<string, unknown>;
cronFailCount: Map<string, number>;
};
function regInternals(reg: JobRegistry): TestRegistry {
return reg as TestRegistry;
}
import test from "node:test";
import assert from "node:assert/strict";
import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import type { JobDefinition } from "@/lib/jobRegistry/core.ts";
const TEST_DATA_DIR = fs.mkdtempSync(path.join(os.tmpdir(), "omniroute-jr-rt-"));
process.env.DATA_DIR = TEST_DATA_DIR;
process.env.DISABLE_SQLITE_AUTO_BACKUP = "true";
const core = await import("@/lib/db/core.ts");
const index = await import("@/lib/jobRegistry/index.ts");
const { getJobRegistry, __resetJobRegistry } = index;
let nowIso: string;
function def(
id: string,
handler: () => Promise<{ success: boolean; recordsAffected?: number; error?: string }>,
over: Record<string, unknown> = {}
): JobDefinition {
return {
id,
type: "interval",
cron: null,
intervalMs: 100000,
enabled: true,
envFlag: null,
config: {},
createdAt: nowIso,
updatedAt: nowIso,
handler,
...over,
} as JobDefinition;
}
function resetAll() {
// Stop any live timers on the current singleton before wiping the DB, otherwise
// an in-flight safeRun from the previous test writes to a closed/wiped DB (FK error).
try {
getJobRegistry().stopAll();
} catch {
// no singleton yet on first run
}
__resetJobRegistry();
core.resetDbInstance();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
fs.mkdirSync(TEST_DATA_DIR, { recursive: true });
}
test.beforeEach(() => {
nowIso = new Date().toISOString();
resetAll();
});
test.after(() => {
core.resetDbInstance();
fs.rmSync(TEST_DATA_DIR, { recursive: true, force: true, maxRetries: 5, retryDelay: 100 });
});
test("register + start (interval) fires handler immediately", async () => {
const reg = getJobRegistry();
let calls = 0;
reg.register(
def("j", async () => {
calls++;
return { success: true, recordsAffected: 0 };
})
);
reg.start("j");
await new Promise((r) => setTimeout(r, 30));
assert.equal(calls, 1, "interval fires once on start");
reg.stop("j");
});
test("re-entrancy: second tick skipped while handler is running", async () => {
const reg = getJobRegistry();
let calls = 0;
let release: (v: void) => void = () => {};
const gate = new Promise<void>((res) => (release = res));
reg.register(
def(
"j",
async () => {
calls++;
await gate;
return { success: true };
},
{ intervalMs: 30 }
)
);
reg.start("j"); // fires immediately, blocks on gate
await new Promise((r) => setTimeout(r, 10));
assert.equal(calls, 1);
// ~100ms passes; setInterval would tick again but must be skipped (running guard).
await new Promise((r) => setTimeout(r, 120));
assert.equal(calls, 1, "must not re-enter while handler runs");
release();
await new Promise((r) => setTimeout(r, 30));
reg.stop("j");
});
test("cron nextTick: handler fires via recursive setTimeout", async () => {
const reg = getJobRegistry();
let calls = 0;
reg.register(
def(
"j",
async () => {
calls++;
return { success: true };
},
{
type: "cron",
cron: "* * * * * *", // every second (6-field) for fast deterministic test
intervalMs: null,
config: { timezone: "UTC" },
}
)
);
reg.start("j");
await new Promise((r) => setTimeout(r, 1400));
reg.stop("j");
assert.ok(calls >= 1, `cron handler should fire at least once, got ${calls}`);
});
test("runNow: manual trigger starts a disabled job's run", async () => {
const reg = getJobRegistry();
let calls = 0;
reg.register(
def("j", async () => {
calls++;
return { success: true };
})
);
const res = await reg.runNow("j");
assert.deepEqual(res, { started: true });
await new Promise((r) => setTimeout(r, 30));
assert.equal(calls, 1);
});
test("runNow: disabled job returns reason=disabled", async () => {
const reg = getJobRegistry();
reg.register(def("j", async () => ({ success: true }), { enabled: false }));
const res = await reg.runNow("j");
assert.deepEqual(res, { started: false, reason: "disabled" });
});
test("runNow: unknown job returns reason=not_found", async () => {
const reg = getJobRegistry();
const res = await reg.runNow("nope");
assert.deepEqual(res, { started: false, reason: "not_found" });
});
test("runNow: queue depth=1, coalesces concurrent triggers then re-runs", async () => {
const reg = getJobRegistry();
let calls = 0;
let release: (v: void) => void = () => {};
const gate = new Promise<void>((res) => (release = res));
reg.register(
def("j", async () => {
calls++;
await gate;
return { success: true };
})
);
const first = await reg.runNow("j"); // starts, blocks on gate
assert.deepEqual(first, { started: true });
// While running, a second runNow should queue (depth=1) and resolve after.
const queued = reg.runNow("j");
// The queued promise must not pile up: only one extra fire after release.
release();
const queuedRes = await queued;
await new Promise((r) => setTimeout(r, 30));
assert.equal(calls, 2, "initial + one queued re-run = 2 fires, no pile-up");
assert.deepEqual(queuedRes, { started: true });
});
test("runNow: env gate blocks when env explicitly disabled", async () => {
const reg = getJobRegistry();
let calls = 0;
reg.register(
def("j", async () => ({ success: true }), { envFlag: "OMNIROUTE_TEST_JOB_ENABLED" })
);
process.env.OMNIROUTE_TEST_JOB_ENABLED = "0";
const res = await reg.runNow("j");
delete process.env.OMNIROUTE_TEST_JOB_ENABLED;
assert.deepEqual(res, { started: false, reason: "env_disabled" });
assert.equal(calls, 0);
});
test("setEnabled(false) stops timer; handler no longer fires", async () => {
const reg = getJobRegistry();
let calls = 0;
reg.register(
def(
"j",
async () => {
calls++;
return { success: true };
},
{ intervalMs: 40 }
)
);
reg.start("j");
await new Promise((r) => setTimeout(r, 25));
reg.setEnabled("j", false);
await new Promise((r) => setTimeout(r, 120));
reg.stop("j");
assert.equal(calls, 1, "only the immediate fire before disable");
});
test("setEnabled(true) restarts a stopped job", async () => {
const reg = getJobRegistry();
let calls = 0;
reg.register(
def(
"j",
async () => {
calls++;
return { success: true };
},
{ intervalMs: 40 }
)
);
reg.setEnabled("j", true);
await new Promise((r) => setTimeout(r, 130));
reg.stop("j");
assert.ok(calls >= 2, `expected repeated fires, got ${calls}`);
});
test("envFlag gate generic: unset env fires (defaultWhenUnset=true)", async () => {
const reg = getJobRegistry();
let calls = 0;
reg.register(
def(
"j",
async () => {
calls++;
return { success: true };
},
{
type: "cron",
cron: "* * * * * *",
intervalMs: null,
envFlag: "OMNIROUTE_GENERIC_JOB_ENABLED", // unset -> default true -> fires
config: { timezone: "UTC" },
}
)
);
reg.start("j");
await new Promise((r) => setTimeout(r, 1400));
reg.stop("j");
assert.ok(calls >= 1, "unset env with default=true should fire");
});
test("envFlag gate warmup: unset env does NOT fire (envDefault=false)", async () => {
const reg = getJobRegistry();
let calls = 0;
reg.register(
def("j", async () => ({ success: true }), {
type: "cron",
cron: "* * * * * *",
intervalMs: null,
envFlag: "OMNIROUTE_WARMUP_ENABLED",
config: { timezone: "UTC", envDefault: false },
})
);
reg.start("j");
await new Promise((r) => setTimeout(r, 1400));
reg.stop("j");
assert.equal(calls, 0, "unset env with envDefault=false must not fire");
});
test("startAll: registers + starts all enabled jobs; throws if no handlers", async () => {
const reg = getJobRegistry();
await assert.rejects(() => reg.startAll(), /No handlers registered/);
let a = 0;
let b = 0;
reg.register(
def("a", async () => {
a++;
return { success: true, recordsAffected: 0 };
})
);
reg.register(
def("b", async () => {
b++;
return { success: true, recordsAffected: 0 };
})
);
await reg.startAll();
await new Promise((r) => setTimeout(r, 30));
assert.ok(a >= 1, "job a started");
assert.ok(b >= 1, "job b started");
reg.stopAll();
});
test("startAll isolation: a missing handler skips that job, starts the rest", async () => {
const reg = getJobRegistry();
let a = 0;
reg.register(
def("a", async () => {
a++;
return { success: true, recordsAffected: 0 };
})
);
// Seed a job in DB with no handler registered in the registry.
core.getDbInstance();
const { upsertJob } = await import("@/lib/db/jobRegistryDb.ts");
upsertJob({
id: "orphan",
type: "interval",
cron: null,
intervalMs: 1000,
enabled: true,
envFlag: null,
config: {},
createdAt: nowIso,
updatedAt: nowIso,
});
await reg.startAll();
await new Promise((r) => setTimeout(r, 30));
assert.ok(a >= 1, "registered job a still starts despite orphan in DB");
reg.stopAll();
});
test("startAll isolation: one job whose start() throws does not abort the rest", async () => {
const reg = getJobRegistry();
let a = 0;
let c = 0;
for (const [id, counter] of [
["a", () => a++],
["boom", () => 0],
["c", () => c++],
] as Array<[string, () => number]>) {
reg.register(
def(id, async () => {
counter();
return { success: true, recordsAffected: 0 };
})
);
}
// start() reads the DB (getJob), so it can throw for one job while the others
// are fine. Nothing in the registry catches per-job, so this asserts the loop
// in startAll() isolates it.
const realStart = reg.start.bind(reg);
reg.start = (jobId: string): void => {
if (jobId === "boom") throw new Error("start blew up for boom");
realStart(jobId);
};
try {
await reg.startAll();
} finally {
reg.start = realStart;
}
await new Promise((r) => setTimeout(r, 30));
assert.ok(a >= 1, "job registered before the throwing one still started");
assert.ok(c >= 1, "job registered after the throwing one still started");
reg.stopAll();
});
test("error isolation: handler throw records failure and survives", async () => {
const reg = getJobRegistry();
let calls = 0;
reg.register(
def("j", async () => {
calls++;
if (calls === 1) throw new Error("boom at /some/file.ts");
return { success: true };
})
);
await reg.runNow("j");
await new Promise((r) => setTimeout(r, 30));
const runs = reg.getRuns("j");
assert.equal(runs.length, 1);
assert.equal(runs[0].status, "failure");
// Error message must be sanitized - no stack path leaked.
assert.ok(!runs[0].errorMessage?.includes("at /"), "error must be sanitized");
});
test("getRuns returns recent records newest-first", async () => {
const reg = getJobRegistry();
reg.register(def("j", async () => ({ success: true, recordsAffected: 0 })));
await reg.runNow("j");
await reg.runNow("j");
await new Promise((r) => setTimeout(r, 30));
const runs = reg.getRuns("j");
assert.equal(runs.length, 2);
assert.ok(
new Date(runs[0].startedAt).getTime() >= new Date(runs[1].startedAt).getTime(),
"newest first"
);
});
test("listJobs returns registered jobs with handlers", () => {
const reg = getJobRegistry();
reg.register(def("j", async () => ({ success: true })));
const jobs = reg.listJobs();
const j = jobs.find((x) => x.id === "j");
assert.ok(j, "registered job j should appear in listJobs");
assert.equal(typeof j!.handler, "function");
});
test("cron: invalid expression stops re-scheduling after MAX_CRON_PARSE_FAILURES", async () => {
const reg = getJobRegistry();
let calls = 0;
reg.register(
def(
"badcron",
async () => {
calls++;
return { success: true };
},
{
type: "cron",
cron: "not-a-valid-cron-expression",
intervalMs: null,
config: { timezone: "UTC" },
}
)
);
reg.start("badcron");
const failCount = regInternals(reg).cronFailCount.get("badcron") ?? 0;
assert.ok(failCount >= 1, `parse should fail, got failCount=${failCount}`);
assert.equal(calls, 0, "invalid cron handler should never fire");
reg.stop("badcron");
});
test("runNow: queued trigger waits for current run then re-fires", async () => {
const reg = getJobRegistry();
let calls = 0;
let release: (v: void) => void = () => {};
const gate = new Promise<void>((res) => (release = res));
reg.register(
def("slow", async () => {
calls++;
await gate;
return { success: true };
})
);
const first = await reg.runNow("slow");
assert.deepEqual(first, { started: true });
const queued = reg.runNow("slow");
release();
const queuedRes = await queued;
await new Promise((r) => setTimeout(r, 30));
assert.equal(calls, 2, "initial + one queued re-run = 2 fires");
assert.deepEqual(queuedRes, { started: true });
});
test("dispose clears timers and cron failure counts", async () => {
const reg = getJobRegistry();
let calls = 0;
reg.register(
def(
"j",
async () => {
calls++;
return { success: true };
},
{ intervalMs: 30 }
)
);
reg.start("j");
await new Promise((r) => setTimeout(r, 10));
assert.ok(regInternals(reg).timers.has("j"));
reg.dispose();
assert.equal(regInternals(reg).timers.size, 0);
assert.equal(regInternals(reg).cronFailCount.size, 0);
const callsBefore = calls;
await new Promise((r) => setTimeout(r, 100));
assert.equal(calls, callsBefore, "disposed timers must not fire");
});
test("cronGetter: re-reads cron on each fire", async () => {
const reg = getJobRegistry();
let calls = 0;
let cronExpr = "*/2 * * * * *";
reg.register({
id: "dynamic",
type: "cron",
cron: cronExpr,
intervalMs: null,
enabled: true,
envFlag: null,
config: { timezone: "UTC" },
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
handler: async () => {
calls++;
return { success: true };
},
cronGetter: () => cronExpr,
});
reg.start("dynamic");
await new Promise((r) => setTimeout(r, 2500));
const callsAfterInitial = calls;
assert.ok(callsAfterInitial >= 1, `should fire with initial cron, got ${callsAfterInitial}`);
cronExpr = "0 0 1 1 * 2099";
await new Promise((r) => setTimeout(r, 3000));
reg.stop("dynamic");
const extraFires = calls - callsAfterInitial;
assert.ok(extraFires <= 1, `cronGetter change should stop fires, got ${extraFires} extra`);
});
test("cronFailCount: resets after successful parse", async () => {
const reg = getJobRegistry();
let calls = 0;
reg.register({
id: "recover",
type: "cron",
cron: "not-a-cron",
intervalMs: null,
enabled: true,
envFlag: null,
config: { timezone: "UTC" },
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
handler: async () => {
calls++;
return { success: true };
},
});
reg.start("recover");
const failCountAfterFail = regInternals(reg).cronFailCount.get("recover") ?? 0;
assert.ok(failCountAfterFail >= 1, `parse should fail, got failCount=${failCountAfterFail}`);
reg.register({
id: "recover",
type: "cron",
cron: "* * * * * *",
intervalMs: null,
enabled: true,
envFlag: null,
config: { timezone: "UTC" },
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
handler: async () => {
calls++;
return { success: true };
},
cronGetter: () => "* * * * * *",
});
reg.stop("recover");
reg.start("recover");
const failCountAfterRecovery = regInternals(reg).cronFailCount.get("recover") ?? 0;
assert.equal(failCountAfterRecovery, 0, "successful parse should reset failure count");
reg.stop("recover");
});
test("cronGetter: throws falls back to static cron", async () => {
const reg = getJobRegistry();
let calls = 0;
let shouldThrow = true;
reg.register({
id: "throwing",
type: "cron",
cron: "* * * * * *",
intervalMs: null,
enabled: true,
envFlag: null,
config: { timezone: "UTC" },
createdAt: new Date().toISOString(),
updatedAt: new Date().toISOString(),
handler: async () => {
calls++;
return { success: true };
},
cronGetter: () => {
if (shouldThrow) throw new Error("transient failure");
return "* * * * * *";
},
});
reg.start("throwing");
await new Promise((r) => setTimeout(r, 1100));
assert.ok(calls >= 1, "job should fire with static cron fallback when cronGetter throws");
reg.stop("throwing");
});