feat: WebSocket based implementation of server sent events for cache busting (#527)

* rough implementation of WS based event system for server side notifications of mutation

* fix test construction

* fix deadlock on event bus

* disable linter error

* add item mutation events

* remove old event bus code

* refactor event system to use composables

* refresh items table when new item is added

* fix create form errors

* cleanup unnecessary calls

* fix importer erorrs + limit fn calls on import

frontend/components/App/ImportDialog.vue

@@ -86,8 +86,6 @@
     importRef.value?.click();
   }
 
-  const eventBus = useEventBus();
-
   async function submitCsvFile() {
     if (!importCsv.value) {
       toast.error("Please select a file to import.");
@@ -111,8 +109,6 @@
       importRef.value.value = "";
     }
 
-    eventBus.emit(EventTypes.InvalidStores);
-
     toast.success("Import successful!");
   }
 </script>

frontend/components/Form/Password.vue

@@ -15,7 +15,7 @@
 <script setup lang="ts">
   type Props = {
     modelValue: string;
-    placeholder: string;
+    placeholder?: string;
     label: string;
   };
 

frontend/components/Item/CreateModal.vue

@@ -1,14 +1,14 @@
 <template>
   <BaseModal v-model="modal">
     <template #title> Create Item </template>
-    <div @keyup="keySubmit">
+    <form @submit.prevent="create()">
       <LocationSelector v-model="form.location" />
       <FormTextField ref="nameInput" v-model="form.name" :trigger-focus="focused" :autofocus="true" label="Item Name" />
       <FormTextArea v-model="form.description" label="Item Description" />
       <FormMultiselect v-model="form.labels" label="Labels" :items="labels ?? []" />
       <div class="modal-action">
         <div class="flex justify-center">
-          <BaseButton class="rounded-r-none" :loading="loading" @click="create(true)">
+          <BaseButton class="rounded-r-none" :loading="loading" type="submit">
             <template #icon>
               <Icon name="mdi-package-variant" class="swap-off h-5 w-5" />
               <Icon name="mdi-package-variant-closed" class="swap-on h-5 w-5" />
@@ -21,13 +21,13 @@
             </label>
             <ul tabindex="0" class="dropdown-content menu p-2 shadow bg-base-100 rounded-box w-64">
               <li>
-                <button type="button" @click.prevent="create(false)">Create and Add Another</button>
+                <button type="button" @click="create(false)">Create and Add Another</button>
               </li>
             </ul>
           </div>
         </div>
       </div>
-    </div>
+    </form>
     <p class="text-sm text-center mt-4">
       use <kbd class="kbd kbd-xs">Shift</kbd> + <kbd class="kbd kbd-xs"> Enter </kbd> to create and add another
     </p>
@@ -84,6 +84,8 @@
     labels: [] as LabelOut[],
   });
 
+  const { shift } = useMagicKeys();
+
   whenever(
     () => modal.value,
     () => {
@@ -102,11 +104,15 @@
     }
   );
 
-  async function create(close = false) {
+  async function create(close = true) {
     if (!form.location) {
       return;
     }
 
+    if (shift.value) {
+      close = false;
+    }
+
     const out: ItemCreate = {
       parentId: null,
       name: form.name,
@@ -135,18 +141,4 @@
       navigateTo(`/item/${data.id}`);
     }
   }
-
-  async function keySubmit(e: KeyboardEvent) {
-    // Shift + Enter
-    if (e.shiftKey && e.key === "Enter") {
-      console.log("Shift + Enter");
-      e.preventDefault();
-      await create(false);
-      focused.value = true;
-    } else if (e.key === "Enter") {
-      e.preventDefault();
-      console.log("Enter");
-      await create(true);
-    }
-  }
 </script>

frontend/components/Label/CreateModal.vue

@@ -1,7 +1,7 @@
 <template>
   <BaseModal v-model="modal">
     <template #title> Create Label </template>
-    <div @keyup="keySubmit">
+    <form @submit.prevent="create()">
       <FormTextField
         ref="locationNameRef"
         v-model="form.name"
@@ -12,24 +12,22 @@
       <FormTextArea v-model="form.description" label="Label Description" />
       <div class="modal-action">
         <div class="flex justify-center">
-          <BaseButton class="rounded-r-none" type="submit" :loading="loading" @click.prevent="create(true)">
-            Create
-          </BaseButton>
+          <BaseButton class="rounded-r-none" :loading="loading" type="submit"> Create </BaseButton>
           <div class="dropdown dropdown-top">
             <label tabindex="0" class="btn rounded-l-none rounded-r-xl">
               <Icon class="h-5 w-5" name="mdi-chevron-down" />
             </label>
             <ul tabindex="0" class="dropdown-content menu p-2 shadow bg-base-100 rounded-box w-64">
               <li>
-                <button @click.prevent="create(false)">Create and Add Another</button>
+                <button type="button" @click="create(false)">Create and Add Another</button>
               </li>
             </ul>
           </div>
         </div>
       </div>
-    </div>
+    </form>
     <p class="text-sm text-center mt-4">
-      use <kbd class="kbd kbd-xs">Shift</kbd> + <kdb class="kbd kbd-xs"> Enter </kdb> to create and add another
+      use <kbd class="kbd kbd-xs">Shift</kbd> + <kbd class="kbd kbd-xs"> Enter </kbd> to create and add another
     </p>
   </BaseModal>
 </template>
@@ -69,7 +67,13 @@
   const api = useUserApi();
   const toast = useNotifier();
 
-  async function create(close: boolean) {
+  const { shift } = useMagicKeys();
+
+  async function create(close = true) {
+    if (shift.value) {
+      close = false;
+    }
+
     const { error, data } = await api.labels.create(form);
     if (error) {
       toast.error("Couldn't create label");
@@ -84,16 +88,4 @@
       navigateTo(`/label/${data.id}`);
     }
   }
-
-  async function keySubmit(e: KeyboardEvent) {
-    // Shift + Enter
-    if (e.shiftKey && e.key === "Enter") {
-      e.preventDefault();
-      await create(false);
-      focused.value = true;
-    } else if (e.key === "Enter") {
-      e.preventDefault();
-      await create(true);
-    }
-  }
 </script>

frontend/components/Location/CreateModal.vue

@@ -1,7 +1,7 @@
 <template>
   <BaseModal v-model="modal">
     <template #title> Create Location </template>
-    <div @keyup="keySubmit">
+    <form @submit.prevent="create()">
       <FormTextField
         ref="locationNameRef"
         v-model="form.name"
@@ -13,22 +13,20 @@
       <LocationSelector v-model="form.parent" />
       <div class="modal-action">
         <div class="flex justify-center">
-          <BaseButton class="rounded-r-none" type="submit" :loading="loading" @click="create(true)">
-            Create
-          </BaseButton>
+          <BaseButton class="rounded-r-none" type="submit" :loading="loading"> Create </BaseButton>
           <div class="dropdown dropdown-top">
             <label tabindex="0" class="btn rounded-l-none rounded-r-xl">
               <Icon class="h-5 w-5" name="mdi-chevron-down" />
             </label>
             <ul tabindex="0" class="dropdown-content menu p-2 shadow bg-base-100 rounded-box w-64">
               <li>
-                <button @click.prevent="create(false)">Create and Add Another</button>
+                <button type="button" @click="create(false)">Create and Add Another</button>
               </li>
             </ul>
           </div>
         </div>
       </div>
-    </div>
+    </form>
     <p class="text-sm text-center mt-4">
       use <kbd class="kbd kbd-xs">Shift</kbd> + <kbd class="kbd kbd-xs"> Enter </kbd> to create and add another
     </p>
@@ -71,9 +69,15 @@
   const api = useUserApi();
   const toast = useNotifier();
 
-  async function create(close: boolean) {
+  const { shift } = useMagicKeys();
+
+  async function create(close = true) {
     loading.value = true;
 
+    if (shift.value) {
+      close = false;
+    }
+
     const { data, error } = await api.locations.create({
       name: form.name,
       description: form.description,
@@ -94,16 +98,4 @@
       navigateTo(`/location/${data.id}`);
     }
   }
-
-  async function keySubmit(e: KeyboardEvent) {
-    // Shift + Enter
-    if (e.shiftKey && e.key === "Enter") {
-      e.preventDefault();
-      await create(false);
-      focused.value = true;
-    } else if (e.key === "Enter") {
-      e.preventDefault();
-      await create(true);
-    }
-  }
 </script>

frontend/composables/use-events.ts

@@ -1,38 +0,0 @@
-export enum EventTypes {
-  // ClearStores event is used to inform the stores that _all_ the data they are using
-  // is now out of date and they should refresh - This is used when the user makes large
-  // changes to the data such as bulk actions or importing a CSV file
-  InvalidStores,
-}
-
-export type EventFn = () => void;
-
-export interface IEventBus {
-  on(event: EventTypes, fn: EventFn, key: string): void;
-  off(event: EventTypes, key: string): void;
-  emit(event: EventTypes): void;
-}
-
-class EventBus implements IEventBus {
-  private listeners: Record<EventTypes, Record<string, EventFn>> = {
-    [EventTypes.InvalidStores]: {},
-  };
-
-  on(event: EventTypes, fn: EventFn, key: string): void {
-    this.listeners[event][key] = fn;
-  }
-
-  off(event: EventTypes, key: string): void {
-    delete this.listeners[event][key];
-  }
-
-  emit(event: EventTypes): void {
-    Object.values(this.listeners[event]).forEach(fn => fn());
-  }
-}
-
-const bus = new EventBus();
-
-export function useEventBus(): IEventBus {
-  return bus;
-}

frontend/composables/use-server-events.ts

@@ -0,0 +1,78 @@
+export enum ServerEvent {
+  LocationMutation = "location.mutation",
+  ItemMutation = "item.mutation",
+  LabelMutation = "label.mutation",
+}
+
+export type EventMessage = {
+  event: ServerEvent;
+};
+
+let socket: WebSocket | null = null;
+
+const listeners = new Map<ServerEvent, (() => void)[]>();
+
+function connect(onmessage: (m: EventMessage) => void) {
+  const ws = new WebSocket(`ws://${window.location.host}/api/v1/ws/events`);
+
+  ws.onopen = () => {
+    console.debug("connected to server");
+  };
+
+  ws.onclose = () => {
+    console.debug("disconnected from server");
+    setTimeout(() => {
+      connect(onmessage);
+    }, 3000);
+  };
+
+  ws.onerror = err => {
+    console.error("websocket error", err);
+  };
+
+  const thorttled = new Map<ServerEvent, any>();
+
+  thorttled.set(ServerEvent.LocationMutation, useThrottleFn(onmessage, 1000));
+  thorttled.set(ServerEvent.ItemMutation, useThrottleFn(onmessage, 1000));
+  thorttled.set(ServerEvent.LabelMutation, useThrottleFn(onmessage, 1000));
+
+  ws.onmessage = msg => {
+    const pm = JSON.parse(msg.data);
+    const fn = thorttled.get(pm.event);
+    if (fn) {
+      fn(pm);
+    }
+  };
+
+  socket = ws;
+}
+
+export function onServerEvent(event: ServerEvent, callback: () => void) {
+  if (socket === null) {
+    connect(e => {
+      console.debug("received event", e);
+      listeners.get(e.event)?.forEach(c => c());
+    });
+  }
+
+  onMounted(() => {
+    if (!listeners.has(event)) {
+      listeners.set(event, []);
+    }
+    listeners.get(event)?.push(callback);
+  });
+
+  onUnmounted(() => {
+    const got = listeners.get(event);
+    if (got) {
+      listeners.set(
+        event,
+        got.filter(c => c !== callback)
+      );
+    }
+
+    if (listeners.get(event)?.length === 0) {
+      listeners.delete(event);
+    }
+  });
+}

frontend/layouts/default.vue

@@ -174,52 +174,24 @@
     },
   ];
 
-  function isMutation(method: string | undefined) {
-    return method === "POST" || method === "PUT" || method === "DELETE";
-  }
-  function isSuccess(status: number) {
-    return status >= 200 && status < 300;
-  }
-
   const labelStore = useLabelStore();
-  const reLabel = /\/api\/v1\/labels\/.*/gm;
-  const rmLabelStoreObserver = defineObserver("labelStore", {
-    handler: (resp, req) => {
-      if (isMutation(req?.method) && isSuccess(resp.status) && resp.url.match(reLabel)) {
-        labelStore.refresh();
-      }
-      console.debug("labelStore handler called by observer");
-    },
-  });
 
   const locationStore = useLocationStore();
-  const reLocation = /\/api\/v1\/locations\/.*/gm;
-  const rmLocationStoreObserver = defineObserver("locationStore", {
-    handler: (resp, req) => {
-      if (isMutation(req?.method) && isSuccess(resp.status) && resp.url.match(reLocation)) {
-        locationStore.refreshChildren();
-        locationStore.refreshParents();
-      }
 
-      console.debug("locationStore handler called by observer");
-    },
+  onServerEvent(ServerEvent.LabelMutation, () => {
+    labelStore.refresh();
   });
 
-  const eventBus = useEventBus();
-  eventBus.on(
-    EventTypes.InvalidStores,
-    () => {
-      labelStore.refresh();
-      locationStore.refreshChildren();
-      locationStore.refreshParents();
-    },
-    "stores"
-  );
+  onServerEvent(ServerEvent.LocationMutation, () => {
+    locationStore.refreshChildren();
+    locationStore.refreshParents();
+  });
 
-  onUnmounted(() => {
-    rmLabelStoreObserver();
-    rmLocationStoreObserver();
-    eventBus.off(EventTypes.InvalidStores, "stores");
+  onServerEvent(ServerEvent.ItemMutation, () => {
+    // item mutations can affect locations counts
+    // so we need to refresh those as well
+    locationStore.refreshChildren();
+    locationStore.refreshParents();
   });
 
   const authCtx = useAuthContext();

frontend/nuxt.config.ts

@@ -3,11 +3,12 @@ import { defineNuxtConfig } from "nuxt/config";
 // https://v3.nuxtjs.org/api/configuration/nuxt.config
 export default defineNuxtConfig({
   ssr: false,
-  modules: ["@nuxtjs/tailwindcss", "@pinia/nuxt", "@vueuse/nuxt", "@vite-pwa/nuxt"],
+  modules: ["@nuxtjs/tailwindcss", "@pinia/nuxt", "@vueuse/nuxt", "@vite-pwa/nuxt", "./nuxt.proxyoverride.ts"],
   nitro: {
     devProxy: {
       "/api": {
         target: "http://localhost:7745/api",
+        ws: true,
         changeOrigin: true,
       },
     },

frontend/nuxt.proxyoverride.ts

@@ -0,0 +1,46 @@
+// https://gist.github.com/ucw/67f7291c64777fb24341e8eae72bcd24
+import { IncomingMessage } from "http";
+import internal from "stream";
+import { defineNuxtModule, logger } from "@nuxt/kit";
+// eslint-disable-next-line
+import { createProxyServer } from "http-proxy";
+
+export default defineNuxtModule({
+  defaults: {
+    target: "ws://localhost:7745",
+    path: "/api/v1/ws",
+  },
+  meta: {
+    configKey: "websocketProxy",
+    name: "Websocket proxy",
+  },
+  setup(resolvedOptions, nuxt) {
+    if (!nuxt.options.dev || !resolvedOptions.target) {
+      return;
+    }
+
+    nuxt.hook("listen", server => {
+      const proxy = createProxyServer({
+        ws: true,
+        secure: false,
+        changeOrigin: true,
+        target: resolvedOptions.target,
+      });
+
+      const proxyFn = (req: IncomingMessage, socket: internal.Duplex, head: Buffer) => {
+        if (req.url && req.url.startsWith(resolvedOptions.path)) {
+          proxy.ws(req, socket, head);
+        }
+      };
+
+      server.on("upgrade", proxyFn);
+
+      nuxt.hook("close", () => {
+        server.off("upgrade", proxyFn);
+        proxy.close();
+      });
+
+      logger.info(`Websocket dev proxy started on ${resolvedOptions.path}`);
+    });
+  },
+});

frontend/package.json

@@ -47,6 +47,7 @@
     "chart.js": "^4.0.1",
     "daisyui": "^2.24.0",
     "dompurify": "^3.0.0",
+    "h3": "^1.7.1",
     "markdown-it": "^13.0.1",
     "pinia": "^2.0.21",
     "postcss": "^8.4.16",

frontend/pages/home/table.ts

@@ -1,7 +1,7 @@
 import { UserClient } from "~~/lib/api/user";
 
 export function itemsTable(api: UserClient) {
-  const { data: items } = useAsyncData(async () => {
+  const { data: items, refresh } = useAsyncData(async () => {
     const { data } = await api.items.getAll({
       page: 1,
       pageSize: 5,
@@ -10,6 +10,11 @@ export function itemsTable(api: UserClient) {
     return data.items;
   });
 
+  onServerEvent(ServerEvent.ItemMutation, () => {
+    console.log("item mutation");
+    refresh();
+  });
+
   return computed(() => {
     return {
       items: items.value || [],

frontend/pages/profile.vue

@@ -288,20 +288,22 @@
     <BaseModal v-model="passwordChange.dialog">
       <template #title> Change Password </template>
 
-      <FormPassword v-model="passwordChange.current" label="Current Password" />
-      <FormPassword v-model="passwordChange.new" label="New Password" />
-      <PasswordScore v-model:valid="passwordChange.isValid" :password="passwordChange.new" />
+      <form @submit.prevent="changePassword">
+        <FormPassword v-model="passwordChange.current" label="Current Password" placeholder="" />
+        <FormPassword v-model="passwordChange.new" label="New Password" placeholder="" />
+        <PasswordScore v-model:valid="passwordChange.isValid" :password="passwordChange.new" />
 
-      <div class="flex">
-        <BaseButton
-          class="ml-auto"
-          :loading="passwordChange.loading"
-          :disabled="!passwordChange.isValid"
-          @click="changePassword"
-        >
-          Submit
-        </BaseButton>
-      </div>
+        <div class="flex">
+          <BaseButton
+            class="ml-auto"
+            :loading="passwordChange.loading"
+            :disabled="!passwordChange.isValid"
+            type="submit"
+          >
+            Submit
+          </BaseButton>
+        </div>
+      </form>
     </BaseModal>
 
     <BaseModal v-model="notifierDialog">

frontend/pnpm-lock.yaml

@@ -44,6 +44,9 @@ dependencies:
   dompurify:
     specifier: ^3.0.0
     version: 3.0.0
+  h3:
+    specifier: ^1.7.1
+    version: 1.7.1
   markdown-it:
     specifier: ^13.0.1
     version: 13.0.1