From a8d1c4bbbc7886959fa0031300f78a8ac6751395 Mon Sep 17 00:00:00 2001 From: Jon Staab Date: Tue, 30 Sep 2025 16:11:49 -0700 Subject: [PATCH] Refactor storage --- package.json | 22 +- pnpm-lock.yaml | Bin 340264 -> 340247 bytes src/app/components/SpaceQuickLinks.svelte | 6 +- .../WalletAsReceivingAddress.svelte | 2 +- src/app/components/WalletConnect.svelte | 3 +- .../WalletUpdateReceivingAddress.svelte | 1 - src/app/core/commands.ts | 9 +- src/app/util/storage.ts | 211 ++++++++++++++++++ src/lib/storage.ts | 160 +++++++------ src/lib/storage/events.ts | 102 --------- src/lib/storage/freshness.ts | 45 ---- src/lib/storage/handles.ts | 40 ---- src/lib/storage/plaintext.ts | 45 ---- src/lib/storage/relays.ts | 40 ---- src/lib/storage/tracker.ts | 80 ------- src/lib/storage/zappers.ts | 41 ---- src/routes/+layout.svelte | 79 ++----- 17 files changed, 332 insertions(+), 554 deletions(-) create mode 100644 src/app/util/storage.ts delete mode 100644 src/lib/storage/events.ts delete mode 100644 src/lib/storage/freshness.ts delete mode 100644 src/lib/storage/handles.ts delete mode 100644 src/lib/storage/plaintext.ts delete mode 100644 src/lib/storage/relays.ts delete mode 100644 src/lib/storage/tracker.ts delete mode 100644 src/lib/storage/zappers.ts diff --git a/package.json b/package.json index 7de750cb..59f89338 100644 --- a/package.json +++ b/package.json @@ -59,17 +59,17 @@ "@types/throttle-debounce": "^5.0.2", "@vite-pwa/assets-generator": "^0.2.6", "@vite-pwa/sveltekit": "^0.6.6", - "@welshman/app": "^0.4.7", - "@welshman/content": "^0.4.7", - "@welshman/editor": "^0.4.7", - "@welshman/feeds": "^0.4.7", - "@welshman/lib": "^0.4.7", - "@welshman/net": "^0.4.7", - "@welshman/relay": "^0.4.7", - "@welshman/router": "^0.4.7", - "@welshman/signer": "^0.4.7", - "@welshman/store": "^0.4.7", - "@welshman/util": "^0.4.7", + "@welshman/app": "^0.5.0", + "@welshman/content": "^0.5.0", + "@welshman/editor": "^0.5.0", + "@welshman/feeds": "^0.5.0", + "@welshman/lib": "^0.5.0", + "@welshman/net": "^0.5.0", + "@welshman/relay": "^0.5.0", + "@welshman/router": "^0.5.0", + "@welshman/signer": "^0.5.0", + "@welshman/store": "^0.5.0", + "@welshman/util": "^0.5.0", "compressorjs": "^1.2.1", "daisyui": "^4.12.10", "date-picker-svelte": "^2.13.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5b4d399d85649a10efe860ae091003723524d938..9c550054856959e3cb42691ddab1fe12dec11b9a 100644 GIT binary patch delta 4520 zcmbVQZLHhob;iA-?bwce?Zh{+eRJ)5lQwqB)f7chBm)GCM2V6}i6SL_N!!|yA|;Zd zNPSQesbWim4k)(nBdyjF+X5u*52GJJkRU+rHVj)b3@x@l>!1BdQ6x+M49k*v12Qbw zP;8WLWM4a5fbG|%-uImIyyraUyze>pyFa@S{p`Xk|8Q#Y?D6em$s6d6dDrcjc2FMQ5)7#~0&!tiSe?bMnna?!T^y zU5zQW>teHPXAx*llB_D{+m=wJ%1NE_hUb%pobtNa z(i9X+#U7HM7B$HN^PRqz3^mVH6l_|TN(B%Z3RtbJ3-ufY4#<>8x?_7joKfv!jze-< zYXRTF7w>(vzwmCIi+=i#iBl`6z-8#^+{q3wR5y)KWl+DuWP9UMhv59i2v@P%gtI%T zveG~qkgxjgEC?_rTPj(0-)MJayHZci!U>U`3J@hz9mYZ{-}uU7i^0F|N9SUc$bK(z zb_peolMshw@p&LuAQLfC$w00O0yJvR@<|ZI%9*}E&|Nmi`BRfKxDeMf1YXQQLy7?d zu&9?qn+~1Mc$mtK8U>;R@@6^LU2J^i(c55)XKtNaVR@*XF7_rP4U%&cc*doC%WS$W zTraTmz7tTgTn#|5n^CpGz$F5GqGM7}5>Ph8`ng7K;Fs}9b;bxNGBcWF(sL2qMb=o` zx5N0{1>of7)2YSzeNa?0c;CXMWZz7pxH!s{dnu%YCK;42N=CiqQW?pL?K(y_53wQ4 zWTc8+B4_3@($oL?%=?om<3VYt-gO|+w;&gl5WY(WRE4jT zj1ZJ+q%{WXPL8y5ofUZyNq#2g~y<_A5?q+G_%woP?Ezclh)h`I&GL4XxI>*acbj6 zU(n>g$9jUn=-AbHa)q#=|^6f#v!&)8$gLSj$)`M(Hppu5e%UD4T#kK@GG}6IJ zJnD=zloedH*KEU~ZnW^SnVE`1jY*NbI4(2{M3hUVZfTt+chd5={$+KYTo*LFVn7uO zELZ{0wxL0>SD*(K+nWov+~kO^qc%IP&G~g{mi9|+9qXA=AMG2Xq`=#rtK~#nBH?*s zqD+ad)lq8GO#U#AKU~Ew=Ud$}64Yp3(^)fBGZ+?~@%b`T;EHUzsOST)S~3f4%TSn+ zlHsN@s}AELkj8O(T9xiJ!y#zOZOQGfhsO7U(oE6akBNHiy^mhqU3~qmCz_ItPDWBA zbcTIr(4X~N(+p2cFlsJmWmxHGLg<@@;dbJR8CQ@}wnO)ct}7E}+2yQ&DAq{C)&)Xg z$X35ZfJCF?VnX(*r+zK+&HM&1|4l3>uq5>J4oxhol! zChJ(qVrSBw(q734Qfv+$P(s6Lq?+WkkMnYVTu!yfR@V`YAy;kJQq73HxUsWz(bbxG z-_C-BLwjMIM=7T=P{7odE-AH9DjbC;BH{bSy3QZwV_VOWp(!_x9!CaadLT6ZjU6qD~xlV!{Df>tZ>DDK(ZHyI)Qw&GeG9yzX3@u&n z5pX#rOj;F2r!@uTrfLn;AxHCFH&@LV*~r>Cv@aHTlkG?#=CTE)S#M*o!cKeL5KH?L zxa`RzKi|ssvW-wB17U7;MVv|1F}_1me8EBDCc^1YV6H_dBiN5$LJz4aP??LHoc-wL zK5#Y4J+iSXW@)DLU2z)7=G4lBT98VX+eEQd&6oO0evJ4;Gm{@+q&UHYTx)DK%EOLR zEn2><<7Hu5LMu5*s%qv?m$N}x^&!htIA}DB-hO0bRmqYCaox-$Bm$i2)C5zTHcJE5 zB5xH$7AbOctv654D5yErMO|YnggMT&tGv)-=9-ts z1ZarVnzhM(EeA$A!3%w zBco=iZWdP?ikl(hv(IBYd0wlH8k~W3hm=bW7!Cxhyn)p6-5TPVsUl6)>b-1MAC3&Y zre>1!3{uqM%jjEw7!&dNcQ&pb>^#2t+~=Yn{#EQvNf|13;FsAX*(R}ZMM8t7T^RFJ zQlpxcKFb6R3=P#(*edeS9L|W~gvn@Hs-$BnJ{&4eNeN{(*-xblXt6yT;}C1sBlzc= zdtasa%>#;0TnaX@NG&IJqpL3i$D?0;8906LyMLW{{%Smx?0=AWb=CbIeqWEa-b+|V zzJGb~6e(#0d+XO_d#>9u-1BAo#C?wa_P z^U=w*!s}bW!_nJY!1vdd-*^DnjsEl;@J%2-!2fL=d?1RI?X|mSF94SpZ@+))1>8(8wZmVERm zc696^;K3uC=Ogza;Qk{L=b~3WNZh}qa{o|FT!s1BEnsIIovqIR_nkg`eV^QeW1imJ z20j=4{dwSI^xNM9wqvzBbMhB9QU`;J0J6EVvLf{5tHAmHKXrHU^rc6Dug0f(<)?8$ z?Ck=rHC?~A3lJ-=-#mNi(|DG4fpbyz3NT+Q{`d-T;~syj(NnQ!r$*kyH&)%o^1lQZ zqP67WQy~tOqc81joLmRsH{%NCUyfkn#0hxiTmP9j7QOHo@Y&082Ctf>y*+#Uc}*P~ne>x;$PMfU9=*E{zy~qp&EW=q z@cK6a3W(|Z`5yqkvK9U1Ht@v%&T$lT{Nw225AII%u+*(Hz{83Hi%04B z9Q)W?v0?wdkKcQL-uL_7m+_C!e)Gp?zxyx8*D{OYxwD6%o0*%~V(rB5KfI4eYrw_$ zN2h>ei@!N>^}tEtG;k#zod<3#swY`d5V#==2;W>1A;uG5@&1$Wpjsa?uY724z~ED=zS z%nsmyjMu(*XlwD4KfCbGdpjH1#hH(;i=a>}Ho^wuKqFGrGbtRE@{B9{&3=^6nidA; z0z%6vxgM{Trj`6u$bgZ{(_V*~P5dc6kTW75fMqNLTF^X{vRD_wQ;WC9&n<@kac=R! z&#x`KJDc$j{&oHMQk_r|^M;AT1yC4gs5%>9Ffn!Xeu)nGZb^wyDCBa3OuL>UYpO0) zbd_(zrpD(qa>iFk*`4x{&joY7M{rSyFwl&v!i#&q^L(Q1TD+NPiTfX{pIWMt5EiMZ zLzv+UWD0=?d_dtnHr+yGx}&1DNFZsZN)O$frZ?IHCGBa-i0~oDFR@)RWAZsS#}yo; zQvuWciq+>K%ph`JxLEuAv-`>xFW=c*MX_oZIT2A6su2j;0i$TSMo%tNg@I(lQ};KI*8pYW`f6}`0`oc81NEu z6Iq=8=!Qr)dUaJ#vthNF?(&e^9AynisP>S)&8T)wB^_NrhXm<%`EtaTQG>4s`BFVw z?PK|z%8PSI$cvOtSKAJywu>`&z`Mf*d?&Y$+RL}scil74py?Y7-OJg1)+ra}jcmS- z(j~bdFk!bw6fwiKAg0v`ORk|KNXPF+6&pnRc?xxWZkC3+!8|+S#_a*xPgh$ZQy{$i z?(M>MqT8GJ-gv&}oisod9=2?|9aM^5n&lnHVpCW{&$!vfs2J&esf*U8GEWXvHs=m{ z`AoMnF%@mpXo(uiv*n5oRcQ)B%pTwC2$_1*K-@j_9wg>>&I2b_2`yP608(UW>R)9YJ{F*^K#@$tVI-N{lUE4?$3q<&9|(P zpxTXDznZ7}(aN{Ky0dfI2+-F12E2+gvP4FG8Kg=Sy z9^noO>!bX9Zds0>Zw>1(EKqgam^Uq>R>sFQpM!9$gFBDI^?M)g-XqcwadWa%H$Y^n z+eX##93<1$+d{FT7P=K|;}#7^kC<73@+M;+it1f`Szqsb@bkAXCofUq-p7?HX^qoNH!9Z5 zh9@-DOsYUO>8_lz!;wNbHLYf#x?Yx2avSURJ+?YWC`U<;i@|Uh=DDURFhr4TOh?3| zZ^}xPOFJT7_{1kZ8-HdKxUj3_WtKvyQIAY>;#_keA`IJP1!DyWReOyOAWwOtuq6wjTGZgrxkNR z7!%Z@`U=+86K}bI_>1eh_aQrRF? zY?$wpY>A-ivy9$D=BaMpL$YI{Xw0m%EOtl5rlDtzTy|0t8qlbO3>kOI)H_YE7=SM2 zGz#(GZ|!0D$+C;XqcL4#X0=?|C(vOAo0mnmT`Zf^UaCU-Bbk(l3RUMjFjh*c8Q0U{ zFr6~e!yr%Nc&p-5Y6Q2#0zOsg)R3V$-P%pm_%AL3XMjEABq>`p2AZQGHrvQ*g(*x# z(%3W9UPuoW#q$SEv`)1;@?buu>0v65XKAl4M=c+ncZ^I~Ry2|(`?an%fUQW=?O-V9 znC1Pk9lv%BxEkY^*6x?Mlw;kV3>O*=h4O2>QV-g56!F`^K*k~~^znfggkHT1XGcirbEllQ0F2vY`+Aj31l9YCHwMwxsz(IZhvx8P|j0;d|L@}kB+@7R| z7*gTtHOtzWUS9jz-B@$hUJ$V{;W}m)iV(tXb;~GL&C#&pN05dnsrgJ`YblU!RJ}F= zbvv%afoV`cWsvSTN+qN5Wxj1OX3G>2&1OoK^c41*ptJMdQ-?nHRQ!ck*IsB9`KE*C zi?k#|89mb!jnb5+vlNeNW+v+hV!n={RoAsftZ9~tBQ{iYC^AHH>`xe}?$6kkCCXB! zthF?k$V`|yKC_hs`S{N32^ufHw{~^s;tPjf{gwF3FDCA!gj%`;4KR7qGRZ>EpG90Z zaIx;Zz|Tf}f$5hjO(e<_xaRhV(o_KD2vus9=!)}+i(Ba=J*>KFWhuGaF0x@~YD~3e zQn#0Xm*5WV5ZwBuS^WMl*002;-U5!qjkkakJ74zu1{BO@oL10 zH$GVJ?XNzy_}rb1M`@SM-{EC@gvP`3KzcO`?Mhj;Y5TmNJo!*3^$6)J52zk=B?tBV zX8hLEiOEmiS>ITL7w_MF;USjgV}Fo(8ZRH@eG+cx)W5CYK6H?#v-7pP>!WAZcZWYE z{@E6AIezsQ>qi!!dE!j`^(TR6?;owEv?rFES;N(8$-?&-krvIEhURe+Re`Lr*6hzx6HP+)DBz^Z2VL&&G{s zfFxgbe&ZP+eP}5x8M;?U-&{%l|G@bOa?T~-iz}I@Pb7Z_e*JaeRJ?r|sH{Hy;}$@# z7H?k#F0B^(8S{vg(&dB?x&lNisuk#ecojIk+U>{V{Y^kPIrakIyx-Eq2MK?fmL+fe zcNc*pM|R)Ry?9(+-UvtIf8GX8t|9SDZzcEv*MQ3_#d}eT3)g_$p7|G+h{RvL2K?!N zW9l!nAaVUdeD7I6e!#aZy7)`a0Uz5+5L$I-+uQxgi&}ePqM2LU>6=&r+$(3!?EL6C z;4epO*v_B422el(>+RQpPjCF6G9gYf;UD6o-v;{oxwy5wa*hHAN@8n2Kaa#$-b_*m Zi*LRO {/if} @@ -81,7 +81,7 @@ Threads {#if $notifications.has(threadsPath)}
{/if} @@ -93,7 +93,7 @@ Calendar {#if $notifications.has(calendarPath)}
{/if} diff --git a/src/app/components/WalletAsReceivingAddress.svelte b/src/app/components/WalletAsReceivingAddress.svelte index a5f16fbf..2114e298 100644 --- a/src/app/components/WalletAsReceivingAddress.svelte +++ b/src/app/components/WalletAsReceivingAddress.svelte @@ -7,7 +7,7 @@ import {updateProfile} from "@app/core/commands" import {clearModals} from "@app/util/modal" import {userProfile, session} from "@welshman/app" - import {makeProfile, type NWCInfo} from "@welshman/util" + import {makeProfile} from "@welshman/util" const lud16 = getWalletAddress($session!.wallet!) diff --git a/src/app/components/WalletConnect.svelte b/src/app/components/WalletConnect.svelte index d6f97507..b8c6a38a 100644 --- a/src/app/components/WalletConnect.svelte +++ b/src/app/components/WalletConnect.svelte @@ -3,8 +3,7 @@ import {nwc} from "@getalby/sdk" import {sleep, assoc} from "@welshman/lib" import type {NWCInfo} from "@welshman/util" - import {pubkey, userProfile, updateSession, profilesByPubkey} from "@welshman/app" - import {makeProfile} from "@welshman/util" + import {pubkey, userProfile, updateSession} from "@welshman/app" import Link from "@lib/components/Link.svelte" import Cpu from "@assets/icons/cpu-bolt.svg?dataurl" import Lock from "@assets/icons/lock-keyhole.svg?dataurl" diff --git a/src/app/components/WalletUpdateReceivingAddress.svelte b/src/app/components/WalletUpdateReceivingAddress.svelte index f058eb12..0c941448 100644 --- a/src/app/components/WalletUpdateReceivingAddress.svelte +++ b/src/app/components/WalletUpdateReceivingAddress.svelte @@ -7,7 +7,6 @@ import ModalFooter from "@lib/components/ModalFooter.svelte" import Wallet from "@assets/icons/wallet.svg?dataurl" import CheckCircle from "@assets/icons/check-circle.svg?dataurl" - import CloseCircle from "@assets/icons/close-circle.svg?dataurl" import {updateProfile} from "@app/core/commands" import {pushToast} from "@app/util/toast" diff --git a/src/app/core/commands.ts b/src/app/core/commands.ts index 4ca68e4a..7dd5c9be 100644 --- a/src/app/core/commands.ts +++ b/src/app/core/commands.ts @@ -84,7 +84,6 @@ import { userInboxRelaySelections, nip44EncryptToSelf, loadRelay, - clearStorage, dropSession, tagEventForComment, tagEventForQuote, @@ -111,7 +110,7 @@ import { } from "@app/core/state" import {loadAlertStatuses} from "@app/core/requests" import {platform, platformName, getPushInfo} from "@app/util/push" -import {clearFileStorage, preferencesStorageProvider} from "@src/lib/storage" +import {preferencesStorageProvider, collectionStorageProvider} from "@src/lib/storage" // Utils @@ -154,12 +153,10 @@ export const logout = async () => { dropSession($pubkey) } - await clearStorage() - localStorage.clear() - await preferencesStorageProvider.clear() - await clearFileStorage() + await preferencesStorageProvider.clear() + await collectionStorageProvider.clear() } // Synchronization diff --git a/src/app/util/storage.ts b/src/app/util/storage.ts new file mode 100644 index 00000000..be74144d --- /dev/null +++ b/src/app/util/storage.ts @@ -0,0 +1,211 @@ +import {on, throttle, fromPairs, batch, sortBy, concat} from "@welshman/lib" +import {throttled, freshness} from "@welshman/store" +import { + PROFILE, + FOLLOWS, + MUTES, + RELAYS, + BLOSSOM_SERVERS, + INBOX_RELAYS, + ROOMS, + APP_DATA, + ALERT_STATUS, + ALERT_EMAIL, + ALERT_WEB, + ALERT_IOS, + ALERT_ANDROID, + EVENT_TIME, + THREAD, + MESSAGE, + DIRECT_MESSAGE, + DIRECT_MESSAGE_FILE, +} from "@welshman/util" +import type {Zapper, TrustedEvent} from "@welshman/util" +import type {RepositoryUpdate} from "@welshman/relay" +import type {Handle, Relay} from "@welshman/app" +import { + plaintext, + tracker, + relays, + repository, + handles, + zappers, + onZapper, + onHandle, +} from "@welshman/app" +import {collectionStorageProvider} from "@lib/storage" + +const syncEvents = async () => { + repository.load(await collectionStorageProvider.get("events")) + + const rankEvent = (event: TrustedEvent) => { + switch (event.kind) { + case PROFILE: + return 1 + case FOLLOWS: + return 1 + case MUTES: + return 1 + case RELAYS: + return 1 + case BLOSSOM_SERVERS: + return 1 + case INBOX_RELAYS: + return 1 + case ROOMS: + return 1 + case APP_DATA: + return 1 + case ALERT_STATUS: + return 1 + case ALERT_EMAIL: + return 1 + case ALERT_WEB: + return 1 + case ALERT_IOS: + return 1 + case ALERT_ANDROID: + return 1 + case EVENT_TIME: + return 0.9 + case THREAD: + return 0.9 + case MESSAGE: + return 0.9 + case DIRECT_MESSAGE: + return 0.9 + case DIRECT_MESSAGE_FILE: + return 0.9 + default: + return 0 + } + } + + return on( + repository, + "update", + batch(3000, async (updates: RepositoryUpdate[]) => { + let added: TrustedEvent[] = [] + const removed = new Set() + + for (const update of updates) { + for (const event of update.added) { + if (rankEvent(event) > 0) { + added.push(event) + removed.delete(event.id) + } + } + + for (const id of update.removed) { + added = added.filter(event => !update.removed.has(event.id)) + removed.add(id) + } + } + + if (added.length > 0) { + let events = concat(await collectionStorageProvider.get("events"), added) + + // If we're well above our retention limit, drop lowest-ranked events + if (events.length > 15_000) { + events = sortBy(e => -rankEvent(e), events).slice(10_000) + } + + await collectionStorageProvider.set("events", events) + } + }), + ) +} + +const syncTracker = async () => { + const relaysById = new Map>() + + for (const [id, relays] of await collectionStorageProvider.get<[string, string[]]>("tracker")) { + relaysById.set(id, new Set(relays)) + } + + tracker.load(relaysById) + + let p = Promise.resolve() + + const updateOne = batch(3000, (ids: string[]) => { + p = p.then(() => { + collectionStorageProvider.add( + "tracker", + ids.map(id => [id, Array.from(tracker.getRelays(id))]), + ) + }) + }) + + const updateAll = throttle(3000, () => { + p = p.then(() => { + collectionStorageProvider.set("tracker", Array.from(tracker.relaysById.entries())) + }) + }) + + tracker.on("add", updateOne) + tracker.on("remove", updateOne) + tracker.on("load", updateAll) + tracker.on("clear", updateAll) + + return () => { + tracker.off("add", updateOne) + tracker.off("remove", updateOne) + tracker.off("load", updateAll) + tracker.off("clear", updateAll) + } +} + +const syncRelays = async () => { + relays.set(await collectionStorageProvider.get("relays")) + + return throttled(3000, relays).subscribe($relays => { + collectionStorageProvider.set("relays", $relays) + }) +} + +const syncHandles = async () => { + handles.set(await collectionStorageProvider.get("handles")) + + return onHandle( + batch(3000, async $handles => { + await collectionStorageProvider.add("handles", $handles) + }), + ) +} + +const syncZappers = async () => { + zappers.set(await collectionStorageProvider.get("zappers")) + + return onZapper( + batch(3000, async $zappers => { + await collectionStorageProvider.add("zappers", $zappers) + }), + ) +} + +const syncFreshness = async () => { + freshness.set(fromPairs(await collectionStorageProvider.get<[string, number]>("freshness"))) + + return throttled(3000, freshness).subscribe($freshness => { + collectionStorageProvider.set("freshness", Object.entries($freshness)) + }) +} + +const syncPlaintext = async () => { + plaintext.set(fromPairs(await collectionStorageProvider.get<[string, string]>("plaintext"))) + + return throttled(3000, plaintext).subscribe($plaintext => { + collectionStorageProvider.set("plaintext", Object.entries($plaintext)) + }) +} + +export const syncDataStores = () => + Promise.all([ + syncEvents(), + syncTracker(), + syncRelays(), + syncHandles(), + syncZappers(), + syncFreshness(), + syncPlaintext(), + ]) diff --git a/src/lib/storage.ts b/src/lib/storage.ts index 02a879e2..b5abb458 100644 --- a/src/lib/storage.ts +++ b/src/lib/storage.ts @@ -1,17 +1,11 @@ +import {parseJson} from "@welshman/lib" import {type StorageProvider} from "@welshman/store" import {Preferences} from "@capacitor/preferences" -import type {Unsubscriber} from "svelte/store" -import {Encoding, Filesystem, type Directory} from "@capacitor/filesystem" -import {EventsStorageProvider} from "@lib/storage/events" -import {FreshnessStorageProvider} from "@lib/storage/freshness" -import {HandlesStorageProvider} from "@lib/storage/handles" -import {PlaintextStorageProvider} from "@lib/storage/plaintext" -import {RelaysStorageProvider} from "@lib/storage/relays" -import {TrackerStorageProvider} from "@lib/storage/tracker" -import {ZappersStorageProvider} from "@lib/storage/zappers" -import {repository, tracker, unsubscribers} from "@welshman/app" +import {Encoding, Filesystem, Directory} from "@capacitor/filesystem" export class PreferencesStorageProvider implements StorageProvider { + p = Promise.resolve() + get = async (key: string): Promise => { const result = await Preferences.get({key}) if (!result.value) return undefined @@ -22,74 +16,94 @@ export class PreferencesStorageProvider implements StorageProvider { } } - p = Promise.resolve() set = async (key: string, value: T): Promise => { - this.p = this.p.then(async () => await Preferences.set({key, value: JSON.stringify(value)})) + this.p = this.p.then(() => Preferences.set({key, value: JSON.stringify(value)})) + + await this.p + } + + clear = async () => { + this.p = this.p.then(() => Preferences.clear()) + + await this.p + } +} + +export const preferencesStorageProvider = new PreferencesStorageProvider() + +export class CollectionStorageProvider implements StorageProvider { + p = Promise.resolve() + + get = async (key: string): Promise => { + try { + const file = await Filesystem.readFile({ + path: key + ".json", + directory: Directory.Data, + encoding: Encoding.UTF8, + }) + + const items: T[] = [] + for (const line of file.data.toString().split("\n")) { + const item = parseJson(line) + + if (item) { + items.push(item) + } + } + + return items + } catch (err) { + // file doesn't exist, or isn't valid json + return [] + } + } + + set = async (key: string, value: T[]): Promise => { + this.p = this.p.then(async () => { + await Filesystem.writeFile({ + path: key + ".json", + directory: Directory.Data, + encoding: Encoding.UTF8, + data: value.map(v => JSON.stringify(v)).join("\n"), + }) + }) + + await this.p + } + + add = async (key: string, value: T[]): Promise => { + this.p = this.p.then(async () => { + await Filesystem.appendFile({ + path: key + ".json", + directory: Directory.Data, + encoding: Encoding.UTF8, + data: value.map(v => JSON.stringify(v)).join("\n"), + }) + }) + await this.p } clear = async (): Promise => { - await Preferences.clear() - this.p = Promise.resolve() + this.p = this.p.then(async () => { + try { + const res = await Filesystem.readdir({path: "./", directory: Directory.Data}) + + await Promise.all( + res.files.map(file => + Filesystem.deleteFile({ + path: file.name + ".json", + directory: Directory.Data, + }), + ), + ) + } catch (e) { + // Directory might not have been created + } + }) + + await this.p } } -// singleton instance of PreferencesStorageProvider -export const preferencesStorageProvider = new PreferencesStorageProvider() - -export interface FilesystemStorageProvider { - initializeState(): Promise - sync(): Unsubscriber -} - -export const getAllFromFile = async ( - filepath: string, - directory: Directory, - encoding: Encoding, -): Promise => { - try { - const contents = ( - await Filesystem.readFile({ - path: filepath, - directory, - encoding, - }) - ).data.toString() - - if (!contents || contents == "") { - return [] - } - - return JSON.parse(contents) - } catch (err) { - // file doesn't exist - return [] - } -} - -export const defaultStorageProviders = { - relays: new RelaysStorageProvider(), - handles: new RelaysStorageProvider(), - zappers: new ZappersStorageProvider(), - freshness: new FreshnessStorageProvider(), - plaintext: new PlaintextStorageProvider(), - tracker: new TrackerStorageProvider({tracker}), - events: new EventsStorageProvider({limit: 10_000, repository, rankEvent: () => 1}), -} - -export const initFileStorage = async (storageProviders: Record) => { - await Promise.all(Object.values(storageProviders).map(async provider => { - await provider.initializeState() - unsubscribers.push(provider.sync()) - })) -} - -export const clearFileStorage = async (): Promise => { - await EventsStorageProvider.clearStorage() - await FreshnessStorageProvider.clearStorage() - await HandlesStorageProvider.clearStorage() - await PlaintextStorageProvider.clearStorage() - await RelaysStorageProvider.clearStorage() - await TrackerStorageProvider.clearStorage() - await ZappersStorageProvider.clearStorage() -} \ No newline at end of file +export const collectionStorageProvider = new CollectionStorageProvider() diff --git a/src/lib/storage/events.ts b/src/lib/storage/events.ts deleted file mode 100644 index 9efbdfe1..00000000 --- a/src/lib/storage/events.ts +++ /dev/null @@ -1,102 +0,0 @@ -import {getAllFromFile, type FilesystemStorageProvider} from "@lib/storage" -import type {TrustedEvent} from "@welshman/util" -import type {Unsubscriber} from "svelte/store" -import {Filesystem, Directory, Encoding} from "@capacitor/filesystem" -import type {Repository, RepositoryUpdate} from "@welshman/relay" -import {on, sortBy} from "@welshman/lib" - -export class EventsStorageProvider implements FilesystemStorageProvider { - static filepath = "events.json" - static directory = Directory.Data - static encoding = Encoding.UTF8 - limit: number - repository: Repository - rankEvent: (event: TrustedEvent) => number - eventCount: number = 0 - isDeleting = false - - constructor({ - limit, - repository, - rankEvent, - }: { - limit: number - repository: Repository - rankEvent: (event: TrustedEvent) => number - }) { - this.limit = limit - this.repository = repository - this.rankEvent = rankEvent - } - - async initializeState(): Promise { - const events = await this.getAll() - this.eventCount = events.length - this.repository.load(events) - } - - sync(): Unsubscriber { - const onUpdate = async ({added, removed}: RepositoryUpdate) => { - // Only add events we want to keep - const keep = added.filter(e => this.rankEvent(e) > 0) - - // Add new events - if (keep.length > 0) { - await this.updateEvents(keep) - } - - // If we're well above our retention limit, drop lowest-ranked events - if (!this.isDeleting && this.eventCount > this.limit * 1.5) { - try { - this.isDeleting = true - - for (const event of sortBy(e => -this.rankEvent(e), await this.getAll()).slice( - this.limit, - )) { - removed.add(event.id) - } - - if (removed.size > 0) { - await this.deleteEvents(Array.from(removed)) - } - } finally { - this.isDeleting = false - } - } - - // Keep track of our total number of events. This isn't strictly accurate, but it's close enough - this.eventCount = this.eventCount + keep.length - removed.size - } - - return on(this.repository, "update", onUpdate) - } - - async getAll(): Promise { - return await getAllFromFile(EventsStorageProvider.filepath, EventsStorageProvider.directory, EventsStorageProvider.encoding) - } - - async writeAll(events: TrustedEvent[]) { - await Filesystem.writeFile({ - path: EventsStorageProvider.filepath, - directory: EventsStorageProvider.directory, - encoding: EventsStorageProvider.encoding, - data: JSON.stringify(events), - }) - } - - async updateEvents(events: TrustedEvent[]) { - const existing = await this.getAll() - const updated = existing.concat(events) - await this.writeAll(updated) - } - - async deleteEvents(ids: string[]) { - const existing = await this.getAll() - const updated = existing.filter(e => !ids.includes(e.id)) - await this.writeAll(updated) - } - - static async clearStorage(): Promise { - await Filesystem.deleteFile({path: EventsStorageProvider.filepath, directory: EventsStorageProvider.directory}) - } -} diff --git a/src/lib/storage/freshness.ts b/src/lib/storage/freshness.ts deleted file mode 100644 index ad8cb913..00000000 --- a/src/lib/storage/freshness.ts +++ /dev/null @@ -1,45 +0,0 @@ -import {getAllFromFile, type FilesystemStorageProvider} from "@lib/storage" -import type {Unsubscriber} from "svelte/store" -import {Filesystem, Directory, Encoding} from "@capacitor/filesystem" -import {fromPairs} from "@welshman/lib" -import {freshness} from "@welshman/store" - -type KV = {key: string; value: any} - -export class FreshnessStorageProvider implements FilesystemStorageProvider { - static filepath = "freshness.json" - static directory = Directory.Data - static encoding = Encoding.UTF8 - - async initializeState(): Promise { - const items = await this.getAll() - freshness.set(fromPairs(items.map(item => [item.key, item.value]))) - } - - sync(): Unsubscriber { - const interval = setInterval(() => { - this.writeAll(freshness.get()) - }, 10_000) - - return () => clearInterval(interval) - } - - async getAll(): Promise { - return await getAllFromFile(FreshnessStorageProvider.filepath, FreshnessStorageProvider.directory, FreshnessStorageProvider.encoding) - } - - async writeAll(items: Record) { - const kvs = Object.entries(items).map(([key, value]) => ({key, value})) - - await Filesystem.writeFile({ - path: FreshnessStorageProvider.filepath, - directory: FreshnessStorageProvider.directory, - encoding: FreshnessStorageProvider.encoding, - data: JSON.stringify(kvs), - }) - } - - static async clearStorage(): Promise { - await Filesystem.deleteFile({path: FreshnessStorageProvider.filepath, directory: FreshnessStorageProvider.directory}) - } -} diff --git a/src/lib/storage/handles.ts b/src/lib/storage/handles.ts deleted file mode 100644 index 41ad520a..00000000 --- a/src/lib/storage/handles.ts +++ /dev/null @@ -1,40 +0,0 @@ -import {getAllFromFile, type FilesystemStorageProvider} from "@lib/storage" -import {get, type Unsubscriber} from "svelte/store" -import {Filesystem, Directory, Encoding} from "@capacitor/filesystem" -import {batch} from "@welshman/lib" -import {handles, onHandle, type Handle} from "@welshman/app" - -export class HandlesStorageProvider implements FilesystemStorageProvider { - static filepath = "handles.json" - static directory = Directory.Data - static encoding = Encoding.UTF8 - - async initializeState(): Promise { - handles.set(await this.getAll()) - } - - sync(): Unsubscriber { - return onHandle(batch(300, () => this.saveState())) - } - - async getAll(): Promise { - return await getAllFromFile(HandlesStorageProvider.filepath, HandlesStorageProvider.directory, HandlesStorageProvider.encoding) - } - - async writeAll(handles: Handle[]) { - await Filesystem.writeFile({ - path: HandlesStorageProvider.filepath, - directory: HandlesStorageProvider.directory, - encoding: HandlesStorageProvider.encoding, - data: JSON.stringify(handles), - }) - } - - async saveState() { - await this.writeAll(get(handles)) - } - - static async clearStorage(): Promise { - await Filesystem.deleteFile({path: HandlesStorageProvider.filepath, directory: HandlesStorageProvider.directory}) - } -} diff --git a/src/lib/storage/plaintext.ts b/src/lib/storage/plaintext.ts deleted file mode 100644 index 01df9055..00000000 --- a/src/lib/storage/plaintext.ts +++ /dev/null @@ -1,45 +0,0 @@ -import {getAllFromFile, type FilesystemStorageProvider} from "@lib/storage" -import type {Unsubscriber} from "svelte/store" -import {Filesystem, Directory, Encoding} from "@capacitor/filesystem" -import {fromPairs} from "@welshman/lib" -import {plaintext} from "@welshman/app" - -type KV = {key: string; value: any} - -export class PlaintextStorageProvider implements FilesystemStorageProvider { - static filepath = "plaintext.json" - static directory = Directory.Data - static encoding = Encoding.UTF8 - - async initializeState(): Promise { - const items = await this.getAll() - plaintext.set(fromPairs(items.map(item => [item.key, item.value]))) - } - - sync(): Unsubscriber { - const interval = setInterval(() => { - this.writeAll(plaintext.get()) - }, 10_000) - - return () => clearInterval(interval) - } - - async getAll(): Promise { - return await getAllFromFile(PlaintextStorageProvider.filepath, PlaintextStorageProvider.directory, PlaintextStorageProvider.encoding) - } - - async writeAll(items: Record) { - const kvs = Object.entries(items).map(([key, value]) => ({key, value})) - - await Filesystem.writeFile({ - path: PlaintextStorageProvider.filepath, - directory: PlaintextStorageProvider.directory, - encoding: PlaintextStorageProvider.encoding, - data: JSON.stringify(kvs), - }) - } - - static async clearStorage(): Promise { - await Filesystem.deleteFile({path: PlaintextStorageProvider.filepath, directory: PlaintextStorageProvider.directory}) - } -} diff --git a/src/lib/storage/relays.ts b/src/lib/storage/relays.ts deleted file mode 100644 index c4feb885..00000000 --- a/src/lib/storage/relays.ts +++ /dev/null @@ -1,40 +0,0 @@ -import {getAllFromFile, type FilesystemStorageProvider} from "@lib/storage" -import type {Unsubscriber} from "svelte/store" -import {Filesystem, Directory, Encoding} from "@capacitor/filesystem" -import {relays, type Relay} from "@welshman/app" -import {throttled} from "@welshman/store" - -export class RelaysStorageProvider implements FilesystemStorageProvider { - static filepath = "relays.json" - static directory = Directory.Data - static encoding = Encoding.UTF8 - - async initializeState(): Promise { - relays.set(await this.getAll()) - } - - sync(): Unsubscriber { - return throttled(3000, relays).subscribe(() => this.saveState()) - } - - async getAll(): Promise { - return await getAllFromFile(RelaysStorageProvider.filepath, RelaysStorageProvider.directory, RelaysStorageProvider.encoding) - } - - async writeAll(relays: Relay[]) { - await Filesystem.writeFile({ - path: RelaysStorageProvider.filepath, - directory: RelaysStorageProvider.directory, - encoding: RelaysStorageProvider.encoding, - data: JSON.stringify(relays), - }) - } - - async saveState() { - await this.writeAll(relays.get()) - } - - static async clearStorage(): Promise { - await Filesystem.deleteFile({path: RelaysStorageProvider.filepath, directory: RelaysStorageProvider.directory}) - } -} diff --git a/src/lib/storage/tracker.ts b/src/lib/storage/tracker.ts deleted file mode 100644 index 59b46899..00000000 --- a/src/lib/storage/tracker.ts +++ /dev/null @@ -1,80 +0,0 @@ -import {getAllFromFile, type FilesystemStorageProvider} from "@lib/storage" -import type {Unsubscriber} from "svelte/store" -import {Filesystem, Directory, Encoding} from "@capacitor/filesystem" -import type {Tracker} from "@welshman/net" -import {call, on} from "@welshman/lib" - -type Entry = {id: string; relays: string[]} - -export class TrackerStorageProvider implements FilesystemStorageProvider { - static filepath = "tracker.json" - static directory = Directory.Data - static encoding = Encoding.UTF8 - tracker: Tracker - - constructor({tracker}: {tracker: Tracker}) { - this.tracker = tracker - } - - async initializeState(): Promise { - const relaysByid = new Map>() - - for (const {id, relays} of await this.getAll()) { - relaysByid.set(id, new Set(relays)) - } - - this.tracker.load(relaysByid) - } - - sync(): Unsubscriber { - const updateOne = async (id: string, relay: string) => { - const relays = new Set(await this.getAll()) - relays.add({id, relays: Array.from(this.tracker.getRelays(id))}) - await this.writeAll([...relays]) - } - - const updateAll = async () => { - await this.writeAll(Array.from(this.tracker.relaysById.entries()).map(([id, relays]) => ({ - id, - relays: Array.from(relays), - }))) - } - - const unsubscribers = [ - on(this.tracker, "add", updateOne), - on(this.tracker, "remove", updateOne), - on(this.tracker, "load", updateAll), - on(this.tracker, "clear", updateAll), - ] - - return () => { - unsubscribers.forEach(call) - } - } - - async getAll(): Promise { - return await getAllFromFile(TrackerStorageProvider.filepath, TrackerStorageProvider.directory, TrackerStorageProvider.encoding) - } - - async writeAll(relays: Entry[]) { - await Filesystem.writeFile({ - path: TrackerStorageProvider.filepath, - directory: TrackerStorageProvider.directory, - encoding: TrackerStorageProvider.encoding, - data: JSON.stringify(relays), - }) - } - - async saveState() { - await this.writeAll( - Array.from(this.tracker.relaysById.entries()).map(([id, relays]) => ({ - id, - relays: Array.from(relays), - })), - ) - } - - static async clearStorage(): Promise { - await Filesystem.deleteFile({path: TrackerStorageProvider.filepath, directory: TrackerStorageProvider.directory}) - } -} diff --git a/src/lib/storage/zappers.ts b/src/lib/storage/zappers.ts deleted file mode 100644 index 463b41ec..00000000 --- a/src/lib/storage/zappers.ts +++ /dev/null @@ -1,41 +0,0 @@ -import {getAllFromFile, type FilesystemStorageProvider} from "@lib/storage" -import {get, type Unsubscriber} from "svelte/store" -import {Filesystem, Directory, Encoding} from "@capacitor/filesystem" -import {onZapper, zappers} from "@welshman/app" -import {batch} from "@welshman/lib" -import type {Zapper} from "@welshman/util" - -export class ZappersStorageProvider implements FilesystemStorageProvider { - static filepath = "zappers.json" - static directory = Directory.Data - static encoding = Encoding.UTF8 - - async initializeState(): Promise { - zappers.set(await this.getAll()) - } - - sync(): Unsubscriber { - return onZapper(batch(300, () => this.saveState())) - } - - async getAll(): Promise { - return await getAllFromFile(ZappersStorageProvider.filepath, ZappersStorageProvider.directory, ZappersStorageProvider.encoding) - } - - async writeAll(zappers: Zapper[]) { - await Filesystem.writeFile({ - path: ZappersStorageProvider.filepath, - directory: ZappersStorageProvider.directory, - encoding: ZappersStorageProvider.encoding, - data: JSON.stringify(zappers), - }) - } - - async saveState() { - await this.writeAll(get(zappers)) - } - - static async clearStorage(): Promise { - await Filesystem.deleteFile({path: ZappersStorageProvider.filepath, directory: ZappersStorageProvider.directory}) - } -} diff --git a/src/routes/+layout.svelte b/src/routes/+layout.svelte index 77af3367..d48063ec 100644 --- a/src/routes/+layout.svelte +++ b/src/routes/+layout.svelte @@ -24,27 +24,7 @@ WEEK, } from "@welshman/lib" import type {TrustedEvent, StampedEvent} from "@welshman/util" - import { - WRAP, - ALERT_STATUS, - ALERT_EMAIL, - ALERT_WEB, - ALERT_IOS, - ALERT_ANDROID, - EVENT_TIME, - APP_DATA, - THREAD, - MESSAGE, - INBOX_RELAYS, - DIRECT_MESSAGE, - DIRECT_MESSAGE_FILE, - MUTES, - FOLLOWS, - PROFILE, - RELAYS, - BLOSSOM_SERVERS, - ROOMS, - } from "@welshman/util" + import {WRAP} from "@welshman/util" import {Nip46Broker, makeSecret} from "@welshman/signer" import type {Socket, RelayMessage, ClientMessage} from "@welshman/net" import { @@ -61,7 +41,6 @@ } from "@welshman/net" import { loadRelay, - db, repository, pubkey, session, @@ -71,7 +50,6 @@ dropSession, loginWithNip01, loginWithNip46, - EventsStorageAdapter, loadRelaySelections, SignerLogEntryStatus, } from "@welshman/app" @@ -83,7 +61,7 @@ import * as net from "@welshman/net" import * as app from "@welshman/app" import {nsecDecode} from "@lib/util" - import {defaultStorageProviders, initFileStorage, preferencesStorageProvider} from "@lib/storage" + import {preferencesStorageProvider} from "@lib/storage" import AppContainer from "@app/components/AppContainer.svelte" import ModalContainer from "@app/components/ModalContainer.svelte" import {setupTracking} from "@app/util/tracking" @@ -105,11 +83,13 @@ import {initializePushNotifications} from "@app/util/push" import * as commands from "@app/core/commands" import * as requests from "@app/core/requests" - import * as notifications from "@app/util/notifications" import * as appState from "@app/core/state" - import {badgeCount, handleBadgeCountChanges} from "@app/util/notifications" + import * as notifications from "@app/util/notifications" + import * as storage from "@app/util/storage" import NewNotificationSound from "@src/app/components/NewNotificationSound.svelte" - import {EventsStorageProvider} from "@lib/storage/events" + + // Migration: delete old indexeddb database + indexedDB?.deleteDatabase('flotilla') // Migration: old nostrtalk instance used different sessions if ($session && !$signer) { @@ -123,6 +103,8 @@ const ready = $state(defer()) + let initialized = false + onMount(async () => { Object.assign(window, { get, @@ -240,7 +222,8 @@ document.documentElement.style["font-size"] = `${$userSettingsValues.font_size}rem` }) - if (!db) { + if (!initialized) { + initialized = true setupTracking() setupAnalytics() @@ -279,42 +262,10 @@ storage: preferencesStorageProvider, }) - await initFileStorage({...defaultStorageProviders, - events: new EventsStorageProvider({ - limit: 10_000, - repository, - rankEvent: (e: TrustedEvent) => { - if ( - [ - PROFILE, - FOLLOWS, - MUTES, - RELAYS, - BLOSSOM_SERVERS, - INBOX_RELAYS, - ROOMS, - APP_DATA, - ALERT_STATUS, - ALERT_EMAIL, - ALERT_WEB, - ALERT_IOS, - ALERT_ANDROID, - ].includes(e.kind) - ) { - return 1 - } - - if ( - [EVENT_TIME, THREAD, MESSAGE, DIRECT_MESSAGE, DIRECT_MESSAGE_FILE].includes(e.kind) - ) { - return 0.9 - } - - return 0 - }, - }), - }) + // Sync application data (relay, events, etc) + await storage.syncDataStores() + // Wait 300 ms for any throttled stores to finish sleep(300).then(() => ready.resolve()) defaultSocketPolicies.push( @@ -465,7 +416,7 @@ ) // subscribe to badge count for changes - badgeCount.subscribe(handleBadgeCountChanges) + notifications.badgeCount.subscribe(notifications.handleBadgeCountChanges) // Listen for signer errors, report to user via toast signerLog.subscribe(