import {writable} from "svelte/store" import type {Unsubscriber} from "svelte/store" import {deleteDB} from "idb" import {SecureStorage} from "@aparajita/capacitor-secure-storage" import {Capacitor, registerPlugin} from "@capacitor/core" import {Preferences} from "@capacitor/preferences" import {WEEK, ago, noop, now, on, batch, call, makeQueue} from "@welshman/lib" import type {Maybe} from "@welshman/lib" import { ALERT_ANDROID, ALERT_EMAIL, ALERT_IOS, ALERT_STATUS, ALERT_WEB, APP_DATA, BLOSSOM_SERVERS, COMMAND, FOLLOWS, MESSAGING_RELAYS, MUTES, PROFILE, RELAY_ADD_MEMBER, RELAY_JOIN, RELAY_LEAVE, RELAY_MEMBERS, RELAY_REMOVE_MEMBER, RELAYS, ROOM_ADD_MEMBER, ROOM_CREATE_PERMISSION, ROOM_MEMBERS, ROOM_ADMINS, ROOM_META, ROOM_PINS, ROOM_DELETE, ROOM_REMOVE_MEMBER, ROOMS, DELETE, REACTION, hexTags, isSignedEvent, tagValues, verifiedSymbol, } from "@welshman/util" import type {Handle, TrustedEvent} from "@welshman/util" import {synced, withGetter} from "@welshman/store" import type {SyncedConfig} from "@welshman/store" import type {RepositoryUpdate, WrapItem} from "@welshman/net" import {Relay, Zapper} from "@welshman/domain" import type {RelayInfo, ZapperValues} from "@welshman/domain" import {Handles, Plaintext, Relays, RelayStats, User, Zappers} from "@welshman/app" import type {AppPolicy, IApp, MapPlugin, RelayStatsItem} from "@welshman/app" import {IDB} from "@lib/indexeddb" import {appPolicies} from "@app/core" import {DM_KINDS} from "@app/content" // withGetter's type drops the ready promise synced adds export const syncedWithGetter = (config: SyncedConfig) => { const store = synced(config) return {...withGetter(store), ready: store.ready} } type StringStorage = { get: (key: string) => Promise> set: (key: string, value: string) => Promise remove: (key: string) => Promise clear: () => Promise } const makeJsonStore = (storage: StringStorage) => { const enqueue = makeQueue() const get = async (key: string): Promise => { const value = await storage.get(key) if (value) { try { return JSON.parse(value) } catch (e) { return undefined } } } const set = async (key: string, value: T): Promise => { await enqueue(() => storage.set(key, JSON.stringify(value))) } const remove = async (key: string) => { await enqueue(() => storage.remove(key)) } const clear = async () => { await enqueue(() => storage.clear()) } return {get, set, remove, clear} } export const kv = makeJsonStore({ get: async key => (await Preferences.get({key})).value ?? undefined, set: (key, value) => Preferences.set({key, value}), remove: key => Preferences.remove({key}), clear: () => Preferences.clear(), }) const secretStorage: StringStorage = Capacitor.getPlatform() === "electron" ? call(() => { const plugin = registerPlugin<{ get(options: {key: string}): Promise<{value?: string}> set(options: {key: string; value: string}): Promise remove(options: {key: string}): Promise clear(): Promise }>("DesktopSecureStorage") return { get: async key => (await plugin.get({key})).value, set: (key, value) => value === undefined ? plugin.remove({key}) : plugin.set({key, value}), remove: key => plugin.remove({key}), clear: () => plugin.clear(), } }) : { get: async key => { let value = await SecureStorage.getItem(key) if (!value) { const legacy = await Preferences.get({key}) if (legacy.value) { value = legacy.value await SecureStorage.setItem(key, legacy.value) await Preferences.remove({key}) } } return value ?? undefined }, // Android's SecureStorage rejects undefined set: (key, value) => value === undefined ? SecureStorage.removeItem(key) : SecureStorage.setItem(key, value), remove: key => SecureStorage.removeItem(key), clear: () => SecureStorage.clear(), } export const ss = makeJsonStore(secretStorage) const TABLES = [ {name: "events", keyPath: "id"}, {name: "relays", keyPath: "url"}, {name: "relayStats", keyPath: "url"}, {name: "handles", keyPath: "nip05"}, {name: "zappers", keyPath: "lnurl"}, {name: "plaintext", keyPath: "key"}, {name: "wrapManager", keyPath: "id"}, ] const FLUSH_INTERVAL = 3000 const idleWrite = (f: (xs: T[]) => void): ((xs: T[]) => void) => { if (typeof requestIdleCallback !== "undefined") { return (xs: T[]) => requestIdleCallback(() => f(xs)) } return f } const kinds = { meta: [PROFILE, FOLLOWS, MUTES, RELAYS, BLOSSOM_SERVERS, MESSAGING_RELAYS, APP_DATA, ROOMS], alert: [ALERT_STATUS, ALERT_EMAIL, ALERT_WEB, ALERT_IOS, ALERT_ANDROID], space: [RELAY_ADD_MEMBER, RELAY_REMOVE_MEMBER, RELAY_MEMBERS, RELAY_JOIN, RELAY_LEAVE, COMMAND], room: [ ROOM_META, ROOM_DELETE, ROOM_ADMINS, ROOM_MEMBERS, ROOM_ADD_MEMBER, ROOM_REMOVE_MEMBER, ROOM_CREATE_PERMISSION, ROOM_PINS, ], } const isRelayScoped = (event: TrustedEvent) => kinds.space.includes(event.kind) || kinds.room.includes(event.kind) const isMembershipChange = (event: TrustedEvent) => event.kind === ROOM_ADD_MEMBER || event.kind === ROOM_REMOVE_MEMBER const isConversation = (event: TrustedEvent) => DM_KINDS.includes(event.kind) || ([DELETE, REACTION].includes(event.kind) && !isSignedEvent(event)) const shouldPersistEvent = (event: TrustedEvent, pubkey: string) => isMembershipChange(event) ? tagValues(hexTags("p"), event.tags).includes(pubkey) : kinds.meta.includes(event.kind) || kinds.alert.includes(event.kind) || isRelayScoped(event) || isConversation(event) type EventItem = {id: string; event: TrustedEvent; relays: string[]; cachedAt?: number} // A command definition is advisory, so one not republished in a week is dropped. const isExpired = (item: EventItem) => item.event.kind === COMMAND && (item.cachedAt ?? now()) < ago(1, WEEK) type PlaintextItem = {key: string; value: string} const DATABASE_PREFIX = "flotilla-9gl" // indexeddb goes with the app container on every platform; the iOS keychain the session lives in does not. export const hasCachedIdentity = async () => typeof indexedDB.databases === "function" ? (await indexedDB.databases()).some(db => db.name?.startsWith(DATABASE_PREFIX)) : true /** Caches an app's repository, tracker and local collections in indexeddb, one database per identity. */ class Storage { ready: Promise private db: IDB private cachedAt = new Map() private unsubscribers: Unsubscriber[] = [] private timeouts: ReturnType[] = [] private stopped = false private readonly pubkey: string constructor(private readonly app: IApp) { // Every identity used to share one database; drop it rather than leave it on disk. void deleteDB(DATABASE_PREFIX) this.pubkey = User.require(app).pubkey this.db = new IDB({name: `${DATABASE_PREFIX}-${this.pubkey}`, stores: TABLES}) this.ready = this.start() } close = () => this.db.close() clear = () => this.db.clear() cleanup = () => { this.stopped = true this.timeouts.forEach(clearTimeout) this.unsubscribers.forEach(call) this.close() } private start = async () => { await this.db.connect() const [, unsubscribeRelays] = await Promise.all([this.loadCriticalData(), this.initRelays()]) this.addUnsubscriber(this.syncEvents()) this.addUnsubscriber(this.syncTracker()) this.addUnsubscriber(unsubscribeRelays) const defer = (init: () => Promise) => { this.timeouts.push( setTimeout(async () => { if (!this.stopped) { this.addUnsubscriber(await init()) } }, 0), ) } defer(this.initRelayStats) defer(this.initHandles) defer(this.initZappers) defer(this.initPlaintext) defer(this.initWrapManager) } private addUnsubscriber = (unsubscriber: Unsubscriber) => { if (this.stopped) { unsubscriber() } else { this.unsubscribers.push(unsubscriber) } } /** A relay-scoped event that lost its provenance can never be keyed to a space again, so drop it. */ private loadCriticalData = async () => { const table = this.db.table("events") const items: EventItem[] = [] const stale: string[] = [] for (const item of await table.getAll()) { // Rows written before provenance was stored inline hold a bare event if ( item.event && shouldPersistEvent(item.event, this.pubkey) && (!isRelayScoped(item.event) || item.relays.length > 0) && !isExpired(item) ) { item.event[verifiedSymbol] = true this.cachedAt.set(item.id, item.cachedAt ?? now()) items.push(item) } else { stale.push(item.id) } } this.app.repository.load(items.map(item => item.event)) const relaysById = new Map>() for (const {id, relays} of items) { // Anything the repository rejected was superseded while we were loading. if (this.app.repository.getEvent(id)) { relaysById.set(id, new Set(relays)) } else { stale.push(id) } } this.app.tracker.load(relaysById) if (stale.length > 0) { void table.bulkDelete(stale) } } private syncEvents = () => { const table = this.db.table("events") return on( this.app.repository, "update", batch(3000, async (updates: RepositoryUpdate[]) => { const add: TrustedEvent[] = [] const remove = new Set() for (const update of updates) { for (const event of update.added) { if (shouldPersistEvent(event, this.pubkey)) { add.push(event) remove.delete(event.id) } } for (const id of update.removed) { remove.add(id) } } if (add.length > 0) { const cachedAt = now() // The ingest policy tracks an event before publishing it, so the tracker already has its provenance. await table.bulkPut( add.map(event => { this.cachedAt.set(event.id, cachedAt) return { id: event.id, event, relays: Array.from(this.app.tracker.getRelays(event.id)), cachedAt, } }), ) } if (remove.size > 0) { remove.forEach(id => this.cachedAt.delete(id)) await table.bulkDelete(remove) } }), ) } private syncTracker = () => { const table = this.db.table("events") const _onChange = async (ids: Iterable) => { const items: EventItem[] = [] for (const id of ids) { const event = this.app.repository.getEvent(id) // A brand-new event isn't queryable here, so syncEvents persists its provenance with the event. if (event && shouldPersistEvent(event, this.pubkey)) { items.push({ id, event, relays: Array.from(this.app.tracker.getRelays(id)), cachedAt: this.cachedAt.get(id) ?? now(), }) } } await table.bulkPut(items) } const onAdd = batch(3000, _onChange) const onRemove = batch(3000, _onChange) const onLoad = () => _onChange(this.app.tracker.relaysById.keys()) this.app.tracker.on("add", onAdd) this.app.tracker.on("remove", onRemove) this.app.tracker.on("load", onLoad) return () => { this.app.tracker.off("add", onAdd) this.app.tracker.off("remove", onRemove) this.app.tracker.off("load", onLoad) } } private persistMapPlugin = async ( name: string, plugin: MapPlugin, fromRow: (row: Row) => Maybe<[string, T]>, toRow: (key: string, item: T) => Row, ) => { const table = this.db.table(name) for (const row of await table.getAll()) { const entry = fromRow(row) if (entry) { plugin.set(...entry) } } const enqueue = batch(FLUSH_INTERVAL, idleWrite(table.bulkPut)) return plugin.onItem((key, item) => { if (item) { enqueue(toRow(key, item)) } }) } private initRelays = () => this.persistMapPlugin( "relays", this.app.use(Relays), (row: RelayInfo & {url: string}) => [row.url, new Relay(row.url, row)], (_url, relay) => ({...relay}), ) private initRelayStats = () => this.persistMapPlugin( "relayStats", this.app.use(RelayStats), (row: RelayStatsItem) => [row.url, row], (_url, stats) => stats, ) private initHandles = () => this.persistMapPlugin( "handles", this.app.use(Handles), (row: Handle) => [row.nip05, row], (_nip05, handle) => handle, ) // Validation is meaningless without pubkey and nostrPubkey, and rows cached before they were required lack them. private initZappers = () => this.persistMapPlugin( "zappers", this.app.use(Zappers), (row: ZapperValues) => row.pubkey && row.nostrPubkey ? [row.lnurl, new Zapper(row)] : undefined, (_lnurl, zapper) => ({...zapper}), ) private initPlaintext = () => this.persistMapPlugin( "plaintext", this.app.use(Plaintext), (row: PlaintextItem) => [row.key, row.value], (key, value) => ({key, value}), ) private initWrapManager = async () => { const table = this.db.table("wrapManager") this.app.wrapManager.load(await table.getAll()) const addOne = batch(3000, table.bulkPut) const removeOne = batch(3000, (items: WrapItem[]) => table.bulkDelete(items.map(item => item.id)), ) this.app.wrapManager.on("add", addOne) this.app.wrapManager.on("remove", removeOne) return () => { this.app.wrapManager.off("add", addOne) this.app.wrapManager.off("remove", removeOne) } } } // The current app's cache, which only exists once there's an identity to cache for. export const storage = withGetter(writable>(undefined)) // Storage is scoped to one app's repository, tracker and caches, so it is built and torn down with it. export const storagePolicy: AppPolicy = $app => { if ($app.user) { const $storage = new Storage($app) storage.set($storage) return () => { $storage.cleanup() storage.set(undefined) } } return noop } appPolicies.push(storagePolicy)