diff --git a/src/hooks/useOfflineNotes.hook.ts b/src/hooks/useOfflineNotes.hook.ts index 9c2d8c1..f3e4324 100644 --- a/src/hooks/useOfflineNotes.hook.ts +++ b/src/hooks/useOfflineNotes.hook.ts @@ -3,11 +3,31 @@ import { computed, ref } from "vue" import { data, generateId } from "@/data/data" import { DataType } from "@/data/DataType.enum" -import { prepareNoteCache } from "@/modules/note/cache/prepareNoteCache" +import { buildNoteDocs } from "@/modules/note/cache/prepareNoteCache" import { Note } from "@/modules/note/models/Note" import { getMainReadme, queryFileContent } from "@/modules/repo/services/repo" import { useUserRepoStore } from "@/modules/repo/store/userRepo.store" +const CONCURRENCY = 8 +const BULK_FLUSH_SIZE = 50 + +const runWithConcurrency = async ( + items: T[], + limit: number, + worker: (item: T) => Promise +) => { + let cursor = 0 + const next = async (): Promise => { + const i = cursor++ + if (i >= items.length) return + await worker(items[i]) + return next() + } + await Promise.all( + Array.from({ length: Math.min(limit, items.length) }, next) + ) +} + export const useOfflineNotes = () => { const store = useUserRepoStore() const totalOfNotes = computed(() => store.files.length) @@ -30,40 +50,51 @@ export const useOfflineNotes = () => { const cachedNotesSet = new Set(cachedNotesFromSha.map((note) => note._id)) - noteCompleted.value = 0 + const filesToFetch = store.files.filter( + (file) => + file.sha && !cachedNotesSet.has(generateId(DataType.Note, file.sha)) + ) + + noteCompleted.value = store.files.length - filesToFetch.length failedNotes.value = 0 - for (const file of store.files) { - noteCompleted.value++ - - if ( - !file.sha || - cachedNotesSet.has(generateId(DataType.Note, file.sha)) - ) { - continue - } - - const { saveCacheNote } = prepareNoteCache(file.sha, file.path) - - const contentFile = await queryFileContent( - store.user, - store.repo, - file.sha - ) - - if (!contentFile) { - failedNotes.value++ - continue - } - - saveCacheNote(contentFile) + const pendingDocs: Note[] = [] + const flush = async () => { + if (!pendingDocs.length) return + const batch = pendingDocs.splice(0, pendingDocs.length) + await data.bulkUpdate(batch) } - try { - await getMainReadme(store.user, store.repo) - } catch { + const fetchWork = runWithConcurrency( + filesToFetch, + CONCURRENCY, + async (file) => { + try { + const content = await queryFileContent( + store.user, + store.repo, + file.sha! + ) + if (!content) { + failedNotes.value++ + return + } + pendingDocs.push(...buildNoteDocs(file.sha!, file.path, content)) + if (pendingDocs.length >= BULK_FLUSH_SIZE) { + await flush() + } + } finally { + noteCompleted.value++ + } + } + ) + + const readmeWork = getMainReadme(store.user, store.repo).catch(() => { failedNotes.value++ - } + }) + + await Promise.all([fetchWork, readmeWork]) + await flush() } const { execute, isLoading } = useAsyncState(cacheAllNotes, null, { immediate: false diff --git a/src/modules/note/cache/prepareNoteCache.ts b/src/modules/note/cache/prepareNoteCache.ts index 54e73d3..65d5b32 100644 --- a/src/modules/note/cache/prepareNoteCache.ts +++ b/src/modules/note/cache/prepareNoteCache.ts @@ -11,6 +11,23 @@ type NoteCacheResult = | { note: Note; from: "path" } | { note: null; from: null } +export const buildNoteDocs = ( + sha: string, + path: string | undefined, + content: string, + editedSha?: string +): Note[] => { + const base: Note = { + _id: generateId(DataType.Note, sha), + $type: DataType.Note, + content, + editedSha + } + return path + ? [base, { ...base, _id: generateId(DataType.Note, path) }] + : [base] +} + export const prepareNoteCache = (sha: string, path?: string) => { const store = useUserRepoStore()