perf(offline): parallelise note caching with batched writes
Fetch blobs with a concurrency of 8 instead of one at a time, batch PouchDB writes via bulkUpdate, filter already-cached files up front, and run the README fetch in parallel with the worker loop.
This commit is contained in:
@@ -3,11 +3,31 @@ import { computed, ref } from "vue"
|
|||||||
|
|
||||||
import { data, generateId } from "@/data/data"
|
import { data, generateId } from "@/data/data"
|
||||||
import { DataType } from "@/data/DataType.enum"
|
import { DataType } from "@/data/DataType.enum"
|
||||||
import { prepareNoteCache } from "@/modules/note/cache/prepareNoteCache"
|
import { buildNoteDocs } from "@/modules/note/cache/prepareNoteCache"
|
||||||
import { Note } from "@/modules/note/models/Note"
|
import { Note } from "@/modules/note/models/Note"
|
||||||
import { getMainReadme, queryFileContent } from "@/modules/repo/services/repo"
|
import { getMainReadme, queryFileContent } from "@/modules/repo/services/repo"
|
||||||
import { useUserRepoStore } from "@/modules/repo/store/userRepo.store"
|
import { useUserRepoStore } from "@/modules/repo/store/userRepo.store"
|
||||||
|
|
||||||
|
const CONCURRENCY = 8
|
||||||
|
const BULK_FLUSH_SIZE = 50
|
||||||
|
|
||||||
|
const runWithConcurrency = async <T>(
|
||||||
|
items: T[],
|
||||||
|
limit: number,
|
||||||
|
worker: (item: T) => Promise<void>
|
||||||
|
) => {
|
||||||
|
let cursor = 0
|
||||||
|
const next = async (): Promise<void> => {
|
||||||
|
const i = cursor++
|
||||||
|
if (i >= items.length) return
|
||||||
|
await worker(items[i])
|
||||||
|
return next()
|
||||||
|
}
|
||||||
|
await Promise.all(
|
||||||
|
Array.from({ length: Math.min(limit, items.length) }, next)
|
||||||
|
)
|
||||||
|
}
|
||||||
|
|
||||||
export const useOfflineNotes = () => {
|
export const useOfflineNotes = () => {
|
||||||
const store = useUserRepoStore()
|
const store = useUserRepoStore()
|
||||||
const totalOfNotes = computed(() => store.files.length)
|
const totalOfNotes = computed(() => store.files.length)
|
||||||
@@ -30,40 +50,51 @@ export const useOfflineNotes = () => {
|
|||||||
|
|
||||||
const cachedNotesSet = new Set(cachedNotesFromSha.map((note) => note._id))
|
const cachedNotesSet = new Set(cachedNotesFromSha.map((note) => note._id))
|
||||||
|
|
||||||
noteCompleted.value = 0
|
const filesToFetch = store.files.filter(
|
||||||
failedNotes.value = 0
|
(file) =>
|
||||||
|
file.sha && !cachedNotesSet.has(generateId(DataType.Note, file.sha))
|
||||||
for (const file of store.files) {
|
|
||||||
noteCompleted.value++
|
|
||||||
|
|
||||||
if (
|
|
||||||
!file.sha ||
|
|
||||||
cachedNotesSet.has(generateId(DataType.Note, file.sha))
|
|
||||||
) {
|
|
||||||
continue
|
|
||||||
}
|
|
||||||
|
|
||||||
const { saveCacheNote } = prepareNoteCache(file.sha, file.path)
|
|
||||||
|
|
||||||
const contentFile = await queryFileContent(
|
|
||||||
store.user,
|
|
||||||
store.repo,
|
|
||||||
file.sha
|
|
||||||
)
|
)
|
||||||
|
|
||||||
if (!contentFile) {
|
noteCompleted.value = store.files.length - filesToFetch.length
|
||||||
failedNotes.value++
|
failedNotes.value = 0
|
||||||
continue
|
|
||||||
}
|
const pendingDocs: Note[] = []
|
||||||
|
const flush = async () => {
|
||||||
saveCacheNote(contentFile)
|
if (!pendingDocs.length) return
|
||||||
|
const batch = pendingDocs.splice(0, pendingDocs.length)
|
||||||
|
await data.bulkUpdate(batch)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const fetchWork = runWithConcurrency(
|
||||||
|
filesToFetch,
|
||||||
|
CONCURRENCY,
|
||||||
|
async (file) => {
|
||||||
try {
|
try {
|
||||||
await getMainReadme(store.user, store.repo)
|
const content = await queryFileContent(
|
||||||
} catch {
|
store.user,
|
||||||
|
store.repo,
|
||||||
|
file.sha!
|
||||||
|
)
|
||||||
|
if (!content) {
|
||||||
failedNotes.value++
|
failedNotes.value++
|
||||||
|
return
|
||||||
}
|
}
|
||||||
|
pendingDocs.push(...buildNoteDocs(file.sha!, file.path, content))
|
||||||
|
if (pendingDocs.length >= BULK_FLUSH_SIZE) {
|
||||||
|
await flush()
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
noteCompleted.value++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
const readmeWork = getMainReadme(store.user, store.repo).catch(() => {
|
||||||
|
failedNotes.value++
|
||||||
|
})
|
||||||
|
|
||||||
|
await Promise.all([fetchWork, readmeWork])
|
||||||
|
await flush()
|
||||||
}
|
}
|
||||||
const { execute, isLoading } = useAsyncState(cacheAllNotes, null, {
|
const { execute, isLoading } = useAsyncState(cacheAllNotes, null, {
|
||||||
immediate: false
|
immediate: false
|
||||||
|
|||||||
17
src/modules/note/cache/prepareNoteCache.ts
vendored
17
src/modules/note/cache/prepareNoteCache.ts
vendored
@@ -11,6 +11,23 @@ type NoteCacheResult =
|
|||||||
| { note: Note; from: "path" }
|
| { note: Note; from: "path" }
|
||||||
| { note: null; from: null }
|
| { note: null; from: null }
|
||||||
|
|
||||||
|
export const buildNoteDocs = (
|
||||||
|
sha: string,
|
||||||
|
path: string | undefined,
|
||||||
|
content: string,
|
||||||
|
editedSha?: string
|
||||||
|
): Note[] => {
|
||||||
|
const base: Note = {
|
||||||
|
_id: generateId(DataType.Note, sha),
|
||||||
|
$type: DataType.Note,
|
||||||
|
content,
|
||||||
|
editedSha
|
||||||
|
}
|
||||||
|
return path
|
||||||
|
? [base, { ...base, _id: generateId(DataType.Note, path) }]
|
||||||
|
: [base]
|
||||||
|
}
|
||||||
|
|
||||||
export const prepareNoteCache = (sha: string, path?: string) => {
|
export const prepareNoteCache = (sha: string, path?: string) => {
|
||||||
const store = useUserRepoStore()
|
const store = useUserRepoStore()
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user