diff --git a/deno.json b/deno.json index e77c1e0..cc83a9a 100644 --- a/deno.json +++ b/deno.json @@ -20,6 +20,7 @@ "@oak/oak": "jsr:@oak/oak@^17.2.0", "@opensearch-project/opensearch": "npm:@opensearch-project/opensearch@^3.6.0", "@skyware/jetstream": "npm:@skyware/jetstream@^0.2.5", - "@std/assert": "jsr:@std/assert@1" + "@std/assert": "jsr:@std/assert@1", + "@std/testing": "jsr:@std/testing@1" } } diff --git a/deno.lock b/deno.lock index fb60f9b..92ffcff 100644 --- a/deno.lock +++ b/deno.lock @@ -7,8 +7,10 @@ "jsr:@oak/commons@1": "1.0.0", "jsr:@oak/oak@^17.2.0": "17.2.0", "jsr:@std/assert@1": "1.0.18", + "jsr:@std/async@^1.4.0": "1.4.0", "jsr:@std/bytes@1": "1.0.4", "jsr:@std/crypto@1": "1.0.3", + "jsr:@std/data-structures@^1.1.0": "1.1.0", "jsr:@std/encoding@1": "1.0.6", "jsr:@std/encoding@^1.0.5": "1.0.6", "jsr:@std/fmt@1": "1.0.9", @@ -20,6 +22,7 @@ "jsr:@std/path@1": "1.1.4", "jsr:@std/path@1.0": "1.0.9", "jsr:@std/path@^1.1.1": "1.1.4", + "jsr:@std/testing@1": "1.0.19", "npm:@opensearch-project/opensearch@3": "3.6.0", "npm:@opensearch-project/opensearch@^3.6.0": "3.6.0", "npm:@skyware/jetstream@~0.2.5": "0.2.5", @@ -71,12 +74,18 @@ "jsr:@std/internal@^1.0.12" ] }, + "@std/async@1.4.0": { + "integrity": "4d70b008634f571cff9b554090d628c76141c32613aae0ff283fd5fa23d0c379" + }, "@std/bytes@1.0.4": { "integrity": "11a0debe522707c95c7b7ef89b478c13fb1583a7cfb9a85674cd2cc2e3a28abc" }, "@std/crypto@1.0.3": { "integrity": "a2a32f51ddef632d299e3879cd027c630dcd4d1d9a5285d6e6788072f4e51e7f" }, + "@std/data-structures@1.1.0": { + "integrity": "c35ae4ad5d8e41a38573c2fe3e19b18ea2505f63cfea201edcb8720aca1f7f58" + }, "@std/encoding@1.0.6": { "integrity": "ca87122c196e8831737d9547acf001766618e78cd8c33920776c7f5885546069" }, @@ -110,6 +119,13 @@ "dependencies": [ "jsr:@std/internal@^1.0.12" ] + }, + "@std/testing@1.0.19": { + "integrity": "f4236172365b216728dc3cc8b5e80a9f4c33083d1e4ede7613d5b25b4014898e", + "dependencies": [ + "jsr:@std/async", + "jsr:@std/data-structures" + ] } }, "npm": { @@ -231,6 +247,7 @@ "jsr:@db/sqlite@0.13", "jsr:@oak/oak@^17.2.0", "jsr:@std/assert@1", + "jsr:@std/testing@1", "npm:@opensearch-project/opensearch@^3.6.0", "npm:@skyware/jetstream@~0.2.5" ] diff --git a/tests/jetstream_bulk_test.ts b/tests/jetstream_bulk_test.ts new file mode 100644 index 0000000..2b70dbd --- /dev/null +++ b/tests/jetstream_bulk_test.ts @@ -0,0 +1,147 @@ +import { assertEquals } from "@std/assert"; +import { FakeTime } from "@std/testing/time"; +import { + _resetBulkBuffersForTest, + type BulkDeps, + queueBulkCreate, +} from "../src/jetstream/bulk.ts"; +import type { WebhookTarget } from "../src/jetstream/webhooks.ts"; +import type { WebhookVerb } from "../src/data/db.ts"; + +type DispatchCall = { + webhooks: WebhookTarget[]; + payload: Record; + label: string; +}; + +const flushMicrotasks = async (): Promise => { + for (let i = 0; i < 5; i++) { + await Promise.resolve(); + } +}; + +const makeDeps = ( + webhooks: WebhookTarget[] = [], +): { + deps: BulkDeps; + getSubsCalls: { did: string; verb: WebhookVerb }[]; + dispatchCalls: DispatchCall[]; +} => { + const getSubsCalls: { did: string; verb: WebhookVerb }[] = []; + const dispatchCalls: DispatchCall[] = []; + const deps: BulkDeps = { + getSubs: (did, verb) => { + getSubsCalls.push({ did, verb }); + return webhooks; + }, + dispatch: (w, payload, label) => { + dispatchCalls.push({ webhooks: w, payload, label }); + return Promise.resolve(); + }, + debounceMs: 500, + }; + return { deps, getSubsCalls, dispatchCalls }; +}; + +Deno.test("queueBulkCreate accumulates records and flushes once after debounce", async () => { + const time = new FakeTime(); + try { + _resetBulkBuffersForTest(); + const { deps, dispatchCalls } = makeDeps(); + + queueBulkCreate("did:plc:a", { rkey: "r1" }, deps); + queueBulkCreate("did:plc:a", { rkey: "r2" }, deps); + queueBulkCreate("did:plc:a", { rkey: "r3" }, deps); + + assertEquals(dispatchCalls.length, 0); + + await time.tickAsync(500); + await flushMicrotasks(); + + assertEquals(dispatchCalls.length, 1); + const payload = dispatchCalls[0].payload as { + records: { rkey: string }[]; + }; + assertEquals(payload.records.map((r) => r.rkey), ["r1", "r2", "r3"]); + } finally { + _resetBulkBuffersForTest(); + time.restore(); + } +}); + +Deno.test("queueBulkCreate restarts the timer on each push (debounce)", async () => { + const time = new FakeTime(); + try { + _resetBulkBuffersForTest(); + const { deps, dispatchCalls } = makeDeps(); + + queueBulkCreate("did:plc:a", { rkey: "r1" }, deps); + await time.tickAsync(400); + queueBulkCreate("did:plc:a", { rkey: "r2" }, deps); + await time.tickAsync(400); + + assertEquals(dispatchCalls.length, 0); + + await time.tickAsync(100); + await flushMicrotasks(); + + assertEquals(dispatchCalls.length, 1); + } finally { + _resetBulkBuffersForTest(); + time.restore(); + } +}); + +Deno.test("flushBulkCreate uses bulk-create verb and the standard payload shape", async () => { + const time = new FakeTime(); + try { + _resetBulkBuffersForTest(); + const targets: WebhookTarget[] = [ + { method: "POST", url: "https://hook.example/bulk" }, + ]; + const { deps, getSubsCalls, dispatchCalls } = makeDeps(targets); + + queueBulkCreate("did:plc:a", { rkey: "r1", title: "T" }, deps); + await time.tickAsync(500); + await flushMicrotasks(); + + assertEquals(getSubsCalls, [{ did: "did:plc:a", verb: "bulk-create" }]); + assertEquals(dispatchCalls.length, 1); + assertEquals(dispatchCalls[0].webhooks, targets); + assertEquals(dispatchCalls[0].label, "bulk-create did:plc:a"); + assertEquals(dispatchCalls[0].payload, { + event: "bulk-create", + did: "did:plc:a", + records: [{ rkey: "r1", title: "T" }], + }); + } finally { + _resetBulkBuffersForTest(); + time.restore(); + } +}); + +Deno.test("bulkBuffers are cleared after flush and isolated per did", async () => { + const time = new FakeTime(); + try { + _resetBulkBuffersForTest(); + const { deps, dispatchCalls } = makeDeps(); + + queueBulkCreate("did:plc:a", { rkey: "a1" }, deps); + queueBulkCreate("did:plc:b", { rkey: "b1" }, deps); + await time.tickAsync(500); + await flushMicrotasks(); + + assertEquals(dispatchCalls.length, 2); + const dids = dispatchCalls.map((c) => (c.payload as { did: string }).did) + .sort(); + assertEquals(dids, ["did:plc:a", "did:plc:b"]); + + // Another flush window should not re-dispatch the same records. + await time.tickAsync(500); + await flushMicrotasks(); + assertEquals(dispatchCalls.length, 2); + } finally { + _resetBulkBuffersForTest(); + time.restore(); + } +});