Speed up boot, prune stores

src/app/core/storage.ts

@@ -1,6 +1,5 @@
 import {call} from "@welshman/lib"
 import {Preferences} from "@capacitor/preferences"
-import {Filesystem, Directory} from "@capacitor/filesystem"
 import {IDB} from "@lib/indexeddb"
 
 export const kv = call(() => {
@@ -31,22 +30,17 @@ export const kv = call(() => {
   return {get, set, clear}
 })
 
-export const db = new IDB({name: "flotilla-9gl", version: 1})
+export const db = new IDB({
-
+  name: "flotilla-9gl",
-// Migration - we used to use capacitor's filesystem for storage, clear it out since we're
+  version: 1,
-// going back to indexeddb
+  stores: [
-call(async () => {
+    {name: "events", keyPath: "id"},
-  const res = await Filesystem.readdir({
+    {name: "tracker", keyPath: "id"},
-    path: "",
+    {name: "relays", keyPath: "url"},
-    directory: Directory.Data,
+    {name: "relayStats", keyPath: "url"},
-  })
+    {name: "handles", keyPath: "nip05"},
-
+    {name: "zappers", keyPath: "lnurl"},
-  await Promise.all(
+    {name: "plaintext", keyPath: "key"},
-    res.files.map(file =>
+    {name: "wrapManager", keyPath: "id"},
-      Filesystem.deleteFile({
+  ],
-        path: file.name,
-        directory: Directory.Data,
-      }),
-    ),
-  )
 })

src/app/util/storage.ts

@@ -45,9 +45,8 @@ import {
   wrapManager,
   onRelay,
 } from "@welshman/app"
-import {isMobile} from "@lib/html"
+import type {Unsubscriber} from "svelte/store"
-import type {IDBTable} from "@lib/indexeddb"
+import {db} from "@app/core/storage"
-import {MESSAGE_KINDS, DM_KINDS} from "@app/core/state"
 
 const kinds = {
   meta: [PROFILE, FOLLOWS, MUTES, RELAYS, BLOSSOM_SERVERS, MESSAGING_RELAYS, APP_DATA, ROOMS],
@@ -62,30 +61,42 @@ const kinds = {
     ROOM_REMOVE_MEMBER,
     ROOM_CREATE_PERMISSION,
   ],
-  content: [...MESSAGE_KINDS, ...DM_KINDS],
 }
 
-const rankEvent = (event: TrustedEvent) => {
+const shouldPersistEvent = (event: TrustedEvent) =>
-  if (kinds.meta.includes(event.kind)) return 9
+  kinds.meta.includes(event.kind) ||
-  if (kinds.alert.includes(event.kind)) return 8
+  kinds.alert.includes(event.kind) ||
-  if (kinds.space.includes(event.kind)) return 7
+  kinds.space.includes(event.kind) ||
-  if (kinds.room.includes(event.kind)) return 6
+  kinds.room.includes(event.kind)
-  if (!isMobile && kinds.content.includes(event.kind)) return 5
-  return 0
-}
 
-const eventsAdapter = {
+type TrackerItem = {id: string; relays: string[]}
-  name: "events",
+
-  keyPath: "id",
+type PlaintextItem = {key: string; value: string}
-  init: async (table: IDBTable<TrustedEvent>) => {
+
+const loadCriticalEvents = async () => {
+  const table = db.table<TrustedEvent>("events")
   const initialEvents = await table.getAll()
+  const keep: TrustedEvent[] = []
+  const drop: string[] = []
 
-    // Mark events verified to avoid re-verification of signatures
   for (const event of initialEvents) {
+    if (shouldPersistEvent(event)) {
       event[verifiedSymbol] = true
+      keep.push(event)
+    } else {
+      drop.push(event.id)
+    }
   }
 
-    repository.load(initialEvents)
+  repository.load(keep)
+
+  if (drop.length > 0) {
+    void table.bulkDelete(drop)
+  }
+}
+
+const syncEvents = () => {
+  const table = db.table<TrustedEvent>("events")
 
   return on(
     repository,
@@ -96,7 +107,7 @@ const eventsAdapter = {
 
       for (const update of updates) {
         for (const event of update.added) {
-            if (rankEvent(event) > 0) {
+          if (shouldPersistEvent(event)) {
             add.push(event)
             remove.delete(event.id)
           }
@@ -116,30 +127,39 @@ const eventsAdapter = {
       }
     }),
   )
-  },
 }
 
-type TrackerItem = {id: string; relays: string[]}
+const loadCriticalTracker = async () => {
-
+  const table = db.table<TrackerItem>("tracker")
-const trackerAdapter = {
-  name: "tracker",
-  keyPath: "id",
-  init: async (table: IDBTable<TrackerItem>) => {
   const relaysById = new Map<string, Set<string>>()
+  const stale: string[] = []
 
   for (const {id, relays} of await table.getAll()) {
+    if (!repository.getEvent(id)) {
+      stale.push(id)
+      continue
+    }
+
     relaysById.set(id, new Set(relays))
   }
 
   tracker.load(relaysById)
 
+  if (stale.length > 0) {
+    void table.bulkDelete(stale)
+  }
+}
+
+const syncTracker = () => {
+  const table = db.table<TrackerItem>("tracker")
+
   const _onAdd = async (ids: Iterable<string>) => {
     const items: TrackerItem[] = []
 
     for (const id of ids) {
       const event = repository.getEvent(id)
 
-        if (!event || rankEvent(event) === 0) continue
+      if (!event || !shouldPersistEvent(event)) continue
 
       const relays = Array.from(tracker.getRelays(id))
 
@@ -156,11 +176,8 @@ const trackerAdapter = {
   }
 
   const onAdd = batch(3000, _onAdd)
-
   const onRemove = batch(3000, _onRemove)
-
   const onLoad = () => _onAdd(tracker.relaysById.keys())
-
   const onClear = () => _onRemove(tracker.relaysById.keys())
 
   tracker.on("add", onAdd)
@@ -174,55 +191,42 @@ const trackerAdapter = {
     tracker.off("load", onLoad)
     tracker.off("clear", onClear)
   }
-  },
 }
 
-const relaysAdapter = {
+const loadCriticalRelays = async () => {
-  name: "relays",
+  const table = db.table<RelayProfile>("relays")
-  keyPath: "url",
+
-  init: async (table: IDBTable<RelayProfile>) => {
   relaysByUrl.set(indexBy(r => r.url, await table.getAll()))
-
-    return onRelay(batch(1000, table.bulkPut))
-  },
 }
 
-const relayStatsAdapter = {
+const syncRelays = () => onRelay(batch(1000, db.table<RelayProfile>("relays").bulkPut))
-  name: "relayStats",
+
-  keyPath: "url",
+const initRelayStats = async () => {
-  init: async (table: IDBTable<RelayStats>) => {
+  const table = db.table<RelayStats>("relayStats")
+
   relayStatsByUrl.set(indexBy(r => r.url, await table.getAll()))
 
   return onRelayStats(batch(1000, table.bulkPut))
-  },
 }
 
-const handlesAdapter = {
+const initHandles = async () => {
-  name: "handles",
+  const table = db.table<Handle>("handles")
-  keyPath: "nip05",
+
-  init: async (table: IDBTable<Handle>) => {
   handlesByNip05.set(indexBy(r => r.nip05, await table.getAll()))
 
   return onHandle(batch(1000, table.bulkPut))
-  },
 }
 
-const zappersAdapter = {
+const initZappers = async () => {
-  name: "zappers",
+  const table = db.table<Zapper>("zappers")
-  keyPath: "lnurl",
+
-  init: async (table: IDBTable<Zapper>) => {
   zappersByLnurl.set(indexBy(z => z.lnurl, await table.getAll()))
 
   return onZapper(batch(3000, table.bulkPut))
-  },
 }
 
-type PlaintextItem = {key: string; value: string}
+const initPlaintext = async () => {
-
+  const table = db.table<PlaintextItem>("plaintext")
-const plaintextAdapter = {
-  name: "plaintext",
-  keyPath: "key",
-  init: async (table: IDBTable<PlaintextItem>) => {
   const initialRecords = await table.getAll()
 
   plaintext.set(fromPairs(initialRecords.map(({key, value}) => [key, value])))
@@ -230,17 +234,14 @@ const plaintextAdapter = {
   return throttled(3000, plaintext).subscribe($plaintext => {
     table.bulkPut(Object.entries($plaintext).map(([key, value]) => ({key, value})))
   })
-  },
 }
 
-const wrapManagerAdapter = {
+const initWrapManager = async () => {
-  name: "wrapManager",
+  const table = db.table<WrapItem>("wrapManager")
-  keyPath: "id",
+
-  init: async (table: IDBTable<WrapItem>) => {
   wrapManager.load(await table.getAll())
 
   const addOne = batch(3000, table.bulkPut)
-
   const removeOne = throttle(3000, table.bulkDelete)
 
   wrapManager.on("add", addOne)
@@ -250,16 +251,76 @@ const wrapManagerAdapter = {
     wrapManager.off("add", addOne)
     wrapManager.off("remove", removeOne)
   }
-  },
 }
 
-export const adapters = [
+type StorageSync = {
-  eventsAdapter,
+  unsubscribe: Unsubscriber
-  trackerAdapter,
+  ready: Promise<void>
-  relaysAdapter,
+}
-  relayStatsAdapter,
+
-  handlesAdapter,
+export const sync = (): StorageSync => {
-  zappersAdapter,
+  const unsubscribers: Unsubscriber[] = []
-  plaintextAdapter,
+  const deferredTimers: ReturnType<typeof setTimeout>[] = []
-  wrapManagerAdapter,
+  let stopped = false
-]
+
+  const addUnsubscriber = (unsubscriber: Unsubscriber) => {
+    if (stopped) {
+      unsubscriber()
+    } else {
+      unsubscribers.push(unsubscriber)
+    }
+  }
+
+  const scheduleDeferred = (task: () => Promise<void>) => {
+    const timeout = setTimeout(() => {
+      if (stopped) return
+
+      void task()
+    }, 0)
+
+    deferredTimers.push(timeout)
+  }
+
+  const ready = (async () => {
+    await db.connect()
+
+    await Promise.all([loadCriticalEvents(), loadCriticalRelays()])
+    await loadCriticalTracker()
+
+    addUnsubscriber(syncEvents())
+    addUnsubscriber(syncTracker())
+    addUnsubscriber(syncRelays())
+
+    scheduleDeferred(async () => {
+      addUnsubscriber(await initRelayStats())
+    })
+
+    scheduleDeferred(async () => {
+      addUnsubscriber(await initHandles())
+    })
+
+    scheduleDeferred(async () => {
+      addUnsubscriber(await initZappers())
+    })
+
+    scheduleDeferred(async () => {
+      addUnsubscriber(await initPlaintext())
+    })
+
+    scheduleDeferred(async () => {
+      addUnsubscriber(await initWrapManager())
+    })
+  })()
+
+  const unsubscribe = () => {
+    stopped = true
+
+    for (const timeout of deferredTimers) {
+      clearTimeout(timeout)
+    }
+
+    unsubscribers.forEach(unsubscriber => unsubscriber())
+  }
+
+  return {unsubscribe, ready}
+}

src/lib/indexeddb.ts

@@ -1,39 +1,32 @@
 import {openDB, deleteDB} from "idb"
 import type {IDBPDatabase} from "idb"
-import type {Unsubscriber} from "svelte/store"
-import {call} from "@welshman/lib"
 import type {Maybe} from "@welshman/lib"
 
-export type IDBAdapter = {
+export type IDBStore = {
   name: string
   keyPath: string
-  init: (table: IDBTable<any>) => Promise<Unsubscriber>
 }
 
-export type IDBAdapters = IDBAdapter[]
-
 export type IDBOptions = {
   name: string
   version: number
+  stores: IDBStore[]
 }
 
 export class IDB {
-  adapters: IDBAdapters = []
   connection: Maybe<Promise<IDBPDatabase>>
-  unsubscribers: Maybe<Unsubscriber[]>
   failedToConnect = false
 
   constructor(readonly options: IDBOptions) {}
 
   async connect() {
     if (!this.failedToConnect && !this.connection) {
-      const {name, version} = this.options
+      const {name, version, stores} = this.options
-      const adapters = this.adapters
 
       try {
         this.connection = openDB(name, version, {
           upgrade(idbDb: IDBPDatabase) {
-            const names = new Set(adapters.map(a => a.name))
+            const names = new Set(stores.map(store => store.name))
 
             for (const table of idbDb.objectStoreNames) {
               if (!names.has(table)) {
@@ -41,7 +34,7 @@ export class IDB {
               }
             }
 
-            for (const {name, keyPath} of adapters) {
+            for (const {name, keyPath} of stores) {
               try {
                 idbDb.createObjectStore(name, {keyPath})
               } catch (e) {
@@ -52,10 +45,6 @@ export class IDB {
           blocked() {},
           blocking() {},
         })
-
-        this.unsubscribers = await Promise.all(
-          adapters.map(({name, init}) => init(this.table(name))),
-        )
       } catch (e) {
         console.error("Failed to connect to indexeddb", e)
         this.failedToConnect = true
@@ -115,9 +104,6 @@ export class IDB {
   }
 
   close = () => {
-    this.unsubscribers?.forEach(call)
-    this.unsubscribers = undefined
-
     this.connection?.then(c => c.close())
     this.connection = undefined
   }

src/routes/+layout.svelte

@@ -126,11 +126,12 @@
       }),
     ])
 
-    // Set up our storage adapters
+    const storageSync = storage.sync()
-    db.adapters = storage.adapters
 
-    // Wait until data storage is initialized before syncing other stuff
+    unsubscribers.push(storageSync.unsubscribe)
-    await db.connect()
+
+    // Wait for critical storage data only
+    await storageSync.ready
 
     // Close the database connection on reload
     unsubscribers.push(() => db.close())