perf(offline): parallelise note caching with batched writes
Fetch blobs with a concurrency of 8 instead of one at a time, batch PouchDB writes via bulkUpdate, filter already-cached files up front, and run the README fetch in parallel with the worker loop.
This commit is contained in:
@@ -3,11 +3,31 @@ import { computed, ref } from "vue"
|
||||
|
||||
import { data, generateId } from "@/data/data"
|
||||
import { DataType } from "@/data/DataType.enum"
|
||||
import { prepareNoteCache } from "@/modules/note/cache/prepareNoteCache"
|
||||
import { buildNoteDocs } from "@/modules/note/cache/prepareNoteCache"
|
||||
import { Note } from "@/modules/note/models/Note"
|
||||
import { getMainReadme, queryFileContent } from "@/modules/repo/services/repo"
|
||||
import { useUserRepoStore } from "@/modules/repo/store/userRepo.store"
|
||||
|
||||
const CONCURRENCY = 8
|
||||
const BULK_FLUSH_SIZE = 50
|
||||
|
||||
const runWithConcurrency = async <T>(
|
||||
items: T[],
|
||||
limit: number,
|
||||
worker: (item: T) => Promise<void>
|
||||
) => {
|
||||
let cursor = 0
|
||||
const next = async (): Promise<void> => {
|
||||
const i = cursor++
|
||||
if (i >= items.length) return
|
||||
await worker(items[i])
|
||||
return next()
|
||||
}
|
||||
await Promise.all(
|
||||
Array.from({ length: Math.min(limit, items.length) }, next)
|
||||
)
|
||||
}
|
||||
|
||||
export const useOfflineNotes = () => {
|
||||
const store = useUserRepoStore()
|
||||
const totalOfNotes = computed(() => store.files.length)
|
||||
@@ -30,40 +50,51 @@ export const useOfflineNotes = () => {
|
||||
|
||||
const cachedNotesSet = new Set(cachedNotesFromSha.map((note) => note._id))
|
||||
|
||||
noteCompleted.value = 0
|
||||
const filesToFetch = store.files.filter(
|
||||
(file) =>
|
||||
file.sha && !cachedNotesSet.has(generateId(DataType.Note, file.sha))
|
||||
)
|
||||
|
||||
noteCompleted.value = store.files.length - filesToFetch.length
|
||||
failedNotes.value = 0
|
||||
|
||||
for (const file of store.files) {
|
||||
noteCompleted.value++
|
||||
|
||||
if (
|
||||
!file.sha ||
|
||||
cachedNotesSet.has(generateId(DataType.Note, file.sha))
|
||||
) {
|
||||
continue
|
||||
}
|
||||
|
||||
const { saveCacheNote } = prepareNoteCache(file.sha, file.path)
|
||||
|
||||
const contentFile = await queryFileContent(
|
||||
store.user,
|
||||
store.repo,
|
||||
file.sha
|
||||
)
|
||||
|
||||
if (!contentFile) {
|
||||
failedNotes.value++
|
||||
continue
|
||||
}
|
||||
|
||||
saveCacheNote(contentFile)
|
||||
const pendingDocs: Note[] = []
|
||||
const flush = async () => {
|
||||
if (!pendingDocs.length) return
|
||||
const batch = pendingDocs.splice(0, pendingDocs.length)
|
||||
await data.bulkUpdate(batch)
|
||||
}
|
||||
|
||||
try {
|
||||
await getMainReadme(store.user, store.repo)
|
||||
} catch {
|
||||
const fetchWork = runWithConcurrency(
|
||||
filesToFetch,
|
||||
CONCURRENCY,
|
||||
async (file) => {
|
||||
try {
|
||||
const content = await queryFileContent(
|
||||
store.user,
|
||||
store.repo,
|
||||
file.sha!
|
||||
)
|
||||
if (!content) {
|
||||
failedNotes.value++
|
||||
return
|
||||
}
|
||||
pendingDocs.push(...buildNoteDocs(file.sha!, file.path, content))
|
||||
if (pendingDocs.length >= BULK_FLUSH_SIZE) {
|
||||
await flush()
|
||||
}
|
||||
} finally {
|
||||
noteCompleted.value++
|
||||
}
|
||||
}
|
||||
)
|
||||
|
||||
const readmeWork = getMainReadme(store.user, store.repo).catch(() => {
|
||||
failedNotes.value++
|
||||
}
|
||||
})
|
||||
|
||||
await Promise.all([fetchWork, readmeWork])
|
||||
await flush()
|
||||
}
|
||||
const { execute, isLoading } = useAsyncState(cacheAllNotes, null, {
|
||||
immediate: false
|
||||
|
||||
17
src/modules/note/cache/prepareNoteCache.ts
vendored
17
src/modules/note/cache/prepareNoteCache.ts
vendored
@@ -11,6 +11,23 @@ type NoteCacheResult =
|
||||
| { note: Note; from: "path" }
|
||||
| { note: null; from: null }
|
||||
|
||||
export const buildNoteDocs = (
|
||||
sha: string,
|
||||
path: string | undefined,
|
||||
content: string,
|
||||
editedSha?: string
|
||||
): Note[] => {
|
||||
const base: Note = {
|
||||
_id: generateId(DataType.Note, sha),
|
||||
$type: DataType.Note,
|
||||
content,
|
||||
editedSha
|
||||
}
|
||||
return path
|
||||
? [base, { ...base, _id: generateId(DataType.Note, path) }]
|
||||
: [base]
|
||||
}
|
||||
|
||||
export const prepareNoteCache = (sha: string, path?: string) => {
|
||||
const store = useUserRepoStore()
|
||||
|
||||
|
||||
Reference in New Issue
Block a user