perf(offline): parallelise note caching with batched writes

Fetch blobs with a concurrency of 8 instead of one at a time, batch
PouchDB writes via bulkUpdate, filter already-cached files up front,
and run the README fetch in parallel with the worker loop.
This commit is contained in:
Julien Calixte
2026-05-16 23:48:06 +02:00
parent 2002ac670a
commit f3ed5e063f
2 changed files with 78 additions and 30 deletions

View File

@@ -3,11 +3,31 @@ import { computed, ref } from "vue"
import { data, generateId } from "@/data/data"
import { DataType } from "@/data/DataType.enum"
import { prepareNoteCache } from "@/modules/note/cache/prepareNoteCache"
import { buildNoteDocs } from "@/modules/note/cache/prepareNoteCache"
import { Note } from "@/modules/note/models/Note"
import { getMainReadme, queryFileContent } from "@/modules/repo/services/repo"
import { useUserRepoStore } from "@/modules/repo/store/userRepo.store"
const CONCURRENCY = 8
const BULK_FLUSH_SIZE = 50
const runWithConcurrency = async <T>(
items: T[],
limit: number,
worker: (item: T) => Promise<void>
) => {
let cursor = 0
const next = async (): Promise<void> => {
const i = cursor++
if (i >= items.length) return
await worker(items[i])
return next()
}
await Promise.all(
Array.from({ length: Math.min(limit, items.length) }, next)
)
}
export const useOfflineNotes = () => {
const store = useUserRepoStore()
const totalOfNotes = computed(() => store.files.length)
@@ -30,40 +50,51 @@ export const useOfflineNotes = () => {
const cachedNotesSet = new Set(cachedNotesFromSha.map((note) => note._id))
noteCompleted.value = 0
const filesToFetch = store.files.filter(
(file) =>
file.sha && !cachedNotesSet.has(generateId(DataType.Note, file.sha))
)
noteCompleted.value = store.files.length - filesToFetch.length
failedNotes.value = 0
for (const file of store.files) {
noteCompleted.value++
if (
!file.sha ||
cachedNotesSet.has(generateId(DataType.Note, file.sha))
) {
continue
}
const { saveCacheNote } = prepareNoteCache(file.sha, file.path)
const contentFile = await queryFileContent(
store.user,
store.repo,
file.sha
)
if (!contentFile) {
failedNotes.value++
continue
}
saveCacheNote(contentFile)
const pendingDocs: Note[] = []
const flush = async () => {
if (!pendingDocs.length) return
const batch = pendingDocs.splice(0, pendingDocs.length)
await data.bulkUpdate(batch)
}
try {
await getMainReadme(store.user, store.repo)
} catch {
const fetchWork = runWithConcurrency(
filesToFetch,
CONCURRENCY,
async (file) => {
try {
const content = await queryFileContent(
store.user,
store.repo,
file.sha!
)
if (!content) {
failedNotes.value++
return
}
pendingDocs.push(...buildNoteDocs(file.sha!, file.path, content))
if (pendingDocs.length >= BULK_FLUSH_SIZE) {
await flush()
}
} finally {
noteCompleted.value++
}
}
)
const readmeWork = getMainReadme(store.user, store.repo).catch(() => {
failedNotes.value++
}
})
await Promise.all([fetchWork, readmeWork])
await flush()
}
const { execute, isLoading } = useAsyncState(cacheAllNotes, null, {
immediate: false