Giant refactor

server/jest.config.js

@@ -0,0 +1,12 @@
+export default {
+  extensionsToTreatAsEsm: [".ts"],
+  preset: "ts-jest/presets/default-esm", // or other ESM presets
+  globals: {
+    "ts-jest": {
+      useESM: true,
+    },
+  },
+  moduleNameMapper: {
+    "^(\\.{1,2}/.*)\\.js$": "$1",
+  },
+};

server/package.json

@@ -4,25 +4,34 @@
   "license": "MIT",
   "source": "src/server.ts",
   "main": "dist/server.js",
+  "type": "module",
   "scripts": {
     "build": "parcel build",
-    "watch": "parcel watch",
+    "watch": "parcel watch --no-cache",
-    "start": "node dist/server.js",
+    "start": "node dist/server.js ../pages",
-    "nodemon": "nodemon dist/server.js"
+    "nodemon": "nodemon dist/server.js ../pages",
+    "test": "jest"
   },
   "dependencies": {
     "@codemirror/collab": "^0.19.0",
     "@codemirror/state": "^0.19.9",
+    "@vscode/sqlite3": "^5.0.7",
+    "better-sqlite3": "^7.5.0",
     "body-parser": "^1.19.2",
     "cors": "^2.8.5",
     "express": "^4.17.3",
+    "knex": "^1.0.4",
     "socket.io": "^4.4.1",
-    "typescript": "^4.6.2"
+    "socket.io-client": "^4.4.1",
+    "typescript": "^4.6.2",
+    "yargs": "^17.3.1"
   },
   "devDependencies": {
     "@types/cors": "^2.8.12",
     "@types/express": "^4.17.13",
+    "jest": "^27.5.1",
     "nodemon": "^2.0.15",
-    "parcel": "^2.3.2"
+    "parcel": "^2.3.2",
+    "ts-jest": "^27.1.3"
   }
 }

server/src/api.test.ts

@@ -0,0 +1,91 @@
+import { test, expect, beforeAll, afterAll, describe } from "@jest/globals";
+
+import { createServer } from "http";
+import { io as Client } from "socket.io-client";
+import { Server } from "socket.io";
+import { SocketServer } from "./api";
+import * as path from "path";
+import * as fs from "fs";
+
+describe("Server test", () => {
+  let io,
+    socketServer: SocketServer,
+    cleaner,
+    clientSocket,
+    reqId = 0;
+  const tmpDir = path.join(__dirname, "test");
+
+  function wsCall(eventName: string, ...args: any[]): Promise<any> {
+    return new Promise((resolve, reject) => {
+      reqId++;
+      clientSocket.once(`${eventName}Resp${reqId}`, (err, result) => {
+        if (err) {
+          reject(err);
+        } else {
+          resolve(result);
+        }
+      });
+      clientSocket.emit(eventName, reqId, ...args);
+    });
+  }
+
+  beforeAll((done) => {
+    const httpServer = createServer();
+    io = new Server(httpServer);
+    fs.mkdirSync(tmpDir, { recursive: true });
+    fs.writeFileSync(`${tmpDir}/test.md`, "This is a simple test");
+    httpServer.listen(async () => {
+      // @ts-ignore
+      const port = httpServer.address().port;
+      // @ts-ignore
+      clientSocket = new Client(`http://localhost:${port}`);
+      socketServer = new SocketServer(tmpDir, io);
+      clientSocket.on("connect", done);
+    });
+  });
+
+  afterAll(() => {
+    io.close();
+    clientSocket.close();
+    socketServer.close();
+    fs.rmSync(tmpDir, { recursive: true, force: true });
+  });
+
+  test("List pages", async () => {
+    let pages = await wsCall("listPages");
+    console.log(pages);
+    expect(pages.length).toBe(1);
+  });
+
+  test("Index operations", async () => {
+    await wsCall("index:clearPageIndexForPage", "test");
+    await wsCall("index:set", "test", "testkey", "value");
+    expect(await wsCall("index:get", "test", "testkey")).toBe("value");
+    await wsCall("index:delete", "test", "testkey");
+    expect(await wsCall("index:get", "test", "testkey")).toBe(null);
+    await wsCall("index:set", "test", "unrelated", 10);
+    await wsCall("index:set", "test", "unrelated", 12);
+    await wsCall("index:set", "test2", "complicated", {
+      name: "Bla",
+      age: 123123,
+    });
+    await wsCall("index:set", "test", "complicated", { name: "Bla", age: 100 });
+    await wsCall("index:set", "test", "complicated2", {
+      name: "Bla",
+      age: 101,
+    });
+    expect(await wsCall("index:get", "test", "complicated")).toStrictEqual({
+      name: "Bla",
+      age: 100,
+    });
+    let result = await wsCall("index:scanPrefixForPage", "test", "compli");
+    expect(result.length).toBe(2);
+    let result2 = await wsCall("index:scanPrefixGlobal", "compli");
+    expect(result2.length).toBe(3);
+    await wsCall("index:deletePrefixForPage", "test", "compli");
+    let result3 = await wsCall("index:scanPrefixForPage", "test", "compli");
+    expect(result3.length).toBe(0);
+    let result4 = await wsCall("index:scanPrefixGlobal", "compli");
+    expect(result4.length).toBe(1);
+  });
+});

server/src/api.ts

@@ -1,229 +1,376 @@
-import fs from "fs";
 import { stat } from "fs/promises";
-import path from "path";
 import { ChangeSet } from "@codemirror/state";
 import { Update } from "@codemirror/collab";
 import { Server } from "socket.io";
 import { Cursor, cursorEffect } from "../../webapp/src/cursorEffect";
 import { Socket } from "socket.io";
 import { DiskStorage } from "./disk_storage";
-import { PageMeta } from "./server";
+import { ClientPageState, Page, PageMeta } from "./types";
-import { ClientPageState, Page } from "./types";
 import { safeRun } from "./util";
+import * as fs from "fs";
+import * as path from "path";
+import knex, { Knex } from "knex";
 
-export function exposeSocketAPI(rootPath: string, io: Server) {
+type IndexItem = {
-  const openPages = new Map<string, Page>();
+  page: string;
-  const connectedSockets: Set<Socket> = new Set();
+  key: string;
-  const pageStore = new DiskStorage(rootPath);
+  value: any;
-  fileWatcher(rootPath);
+};
 
-  io.on("connection", (socket) => {
+class ClientConnection {
-    const socketOpenPages = new Set<string>();
+  openPages = new Set<string>();
+  constructor(readonly sock: Socket) {}
+}
 
-    console.log("Connected", socket.id);
+export class SocketServer {
-    connectedSockets.add(socket);
+  rootPath: string;
+  serverSock: Server;
+  openPages = new Map<string, Page>();
+  connectedSockets = new Set<Socket>();
+  pageStore: DiskStorage;
+  db: Knex;
+  serverSocket: Server;
 
-    socket.on("disconnect", () => {
+  api = {
-      console.log("Disconnected", socket.id);
+    openPage: async (clientConn: ClientConnection, pageName: string) => {
-      socketOpenPages.forEach(disconnectPageSocket);
+      let page = this.openPages.get(pageName);
-      connectedSockets.delete(socket);
-    });
-
-    socket.on("closePage", (pageName: string) => {
-      console.log("Closing page", pageName);
-      socketOpenPages.delete(pageName);
-      disconnectPageSocket(pageName);
-    });
-
-    const onCall = (
-      eventName: string,
-      cb: (...args: any[]) => Promise<any>
-    ) => {
-      socket.on(eventName, (reqId: number, ...args) => {
-        cb(...args).then((result) => {
-          socket.emit(`${eventName}Resp${reqId}`, result);
-        });
-      });
-    };
-
-    const disconnectPageSocket = (pageName: string) => {
-      let page = openPages.get(pageName);
-      if (page) {
-        for (let client of page.clientStates) {
-          if (client.socket === socket) {
-            disconnectClient(client, page);
-          }
-        }
-      }
-    };
-
-    onCall("openPage", async (pageName: string) => {
-      let page = openPages.get(pageName);
       if (!page) {
         try {
-          let { text, meta } = await pageStore.readPage(pageName);
+          let { text, meta } = await this.pageStore.readPage(pageName);
           page = new Page(pageName, text, meta);
         } catch (e) {
           console.log("Creating new page", pageName);
           page = new Page(pageName, "", { name: pageName, lastModified: 0 });
         }
-        openPages.set(pageName, page);
+        this.openPages.set(pageName, page);
       }
-      page.clientStates.add(new ClientPageState(socket, page.version));
+      page.clientStates.add(new ClientPageState(clientConn.sock, page.version));
-      socketOpenPages.add(pageName);
+      clientConn.openPages.add(pageName);
       console.log("Opened page", pageName);
-      broadcastCursors(page);
+      this.broadcastCursors(page);
       return page.toJSON();
-    });
+    },
+    pushUpdates: async (
+      clientConn: ClientConnection,
+      pageName: string,
+      version: number,
+      updates: any[]
+    ): Promise<boolean> => {
+      let page = this.openPages.get(pageName);
 
-    onCall(
+      if (!page) {
-      "pushUpdates",
+        console.error(
-      async (
+          "Received updates for not open page",
-        pageName: string,
+          pageName,
-        version: number,
+          this.openPages.keys()
-        updates: any[]
+        );
-      ): Promise<boolean> => {
+        return false;
-        let page = openPages.get(pageName);
-
-        if (!page) {
-          console.error(
-            "Received updates for not open page",
-            pageName,
-            openPages.keys()
-          );
-          return false;
-        }
-        if (version !== page.version) {
-          console.error("Invalid version", version, page.version);
-          return false;
-        } else {
-          console.log("Applying", updates.length, "updates to", pageName);
-          let transformedUpdates = [];
-          let textChanged = false;
-          for (let update of updates) {
-            let changes = ChangeSet.fromJSON(update.changes);
-            let transformedUpdate = {
-              changes,
-              clientID: update.clientID,
-              effects: update.cursors?.map((c: Cursor) => {
-                page.cursors.set(c.userId, c);
-                return cursorEffect.of(c);
-              }),
-            };
-            page.updates.push(transformedUpdate);
-            transformedUpdates.push(transformedUpdate);
-            let oldText = page.text;
-            page.text = changes.apply(page.text);
-            if (oldText !== page.text) {
-              textChanged = true;
-            }
-          }
-
-          if (textChanged) {
-            if (page.saveTimer) {
-              clearTimeout(page.saveTimer);
-            }
-
-            page.saveTimer = setTimeout(() => {
-              flushPageToDisk(pageName, page);
-            }, 1000);
-          }
-          while (page.pending.length) {
-            page.pending.pop()!(transformedUpdates);
-          }
-          return true;
-        }
       }
-    );
+      if (version !== page.version) {
-
+        console.error("Invalid version", version, page.version);
-    onCall(
+        return false;
-      "pullUpdates",
+      } else {
-      async (pageName: string, version: number): Promise<Update[]> => {
+        console.log("Applying", updates.length, "updates to", pageName);
-        let page = openPages.get(pageName);
+        let transformedUpdates = [];
-        // console.log("Pulling updates for", pageName);
+        let textChanged = false;
-        if (!page) {
+        for (let update of updates) {
-          console.error("Fetching updates for not open page");
+          let changes = ChangeSet.fromJSON(update.changes);
-          return [];
+          let transformedUpdate = {
-        }
+            changes,
-        // TODO: Optimize this
+            clientID: update.clientID,
-        let oldestVersion = Infinity;
+            effects: update.cursors?.map((c: Cursor) => {
-        page.clientStates.forEach((client) => {
+              page.cursors.set(c.userId, c);
-          oldestVersion = Math.min(client.version, oldestVersion);
+              return cursorEffect.of(c);
-          if (client.socket === socket) {
+            }),
-            client.version = version;
-          }
-        });
-        page.flushUpdates(oldestVersion);
-        if (version < page.version) {
-          return page.updatesSince(version);
-        } else {
-          return new Promise((resolve) => {
-            page.pending.push(resolve);
-          });
-        }
-      }
-    );
-
-    onCall(
-      "readPage",
-      async (pageName: string): Promise<{ text: string; meta: PageMeta }> => {
-        let page = openPages.get(pageName);
-        if (page) {
-          console.log("Serving page from memory", pageName);
-          return {
-            text: page.text.sliceString(0),
-            meta: page.meta,
           };
-        } else {
+          page.updates.push(transformedUpdate);
-          return pageStore.readPage(pageName);
+          transformedUpdates.push(transformedUpdate);
+          let oldText = page.text;
+          page.text = changes.apply(page.text);
+          if (oldText !== page.text) {
+            textChanged = true;
+          }
         }
-      }
-    );
 
-    onCall("writePage", async (pageName: string, text: string) => {
+        if (textChanged) {
-      let page = openPages.get(pageName);
+          if (page.saveTimer) {
+            clearTimeout(page.saveTimer);
+          }
+
+          page.saveTimer = setTimeout(() => {
+            this.flushPageToDisk(pageName, page);
+          }, 1000);
+        }
+        while (page.pending.length) {
+          page.pending.pop()!(transformedUpdates);
+        }
+        return true;
+      }
+    },
+
+    pullUpdates: async (
+      clientConn: ClientConnection,
+      pageName: string,
+      version: number
+    ): Promise<Update[]> => {
+      let page = this.openPages.get(pageName);
+      // console.log("Pulling updates for", pageName);
+      if (!page) {
+        console.error("Fetching updates for not open page");
+        return [];
+      }
+      // TODO: Optimize this
+      let oldestVersion = Infinity;
+      page.clientStates.forEach((client) => {
+        oldestVersion = Math.min(client.version, oldestVersion);
+        if (client.socket === clientConn.sock) {
+          client.version = version;
+        }
+      });
+      page.flushUpdates(oldestVersion);
+      if (version < page.version) {
+        return page.updatesSince(version);
+      } else {
+        return new Promise((resolve) => {
+          page.pending.push(resolve);
+        });
+      }
+    },
+
+    readPage: async (
+      clientConn: ClientConnection,
+      pageName: string
+    ): Promise<{ text: string; meta: PageMeta }> => {
+      let page = this.openPages.get(pageName);
+      if (page) {
+        console.log("Serving page from memory", pageName);
+        return {
+          text: page.text.sliceString(0),
+          meta: page.meta,
+        };
+      } else {
+        return this.pageStore.readPage(pageName);
+      }
+    },
+
+    writePage: async (
+      clientConn: ClientConnection,
+      pageName: string,
+      text: string
+    ) => {
+      let page = this.openPages.get(pageName);
       if (page) {
         for (let client of page.clientStates) {
           client.socket.emit("reloadPage", pageName);
         }
-        openPages.delete(pageName);
+        this.openPages.delete(pageName);
       }
-      return pageStore.writePage(pageName, text);
+      return this.pageStore.writePage(pageName, text);
-    });
+    },
 
-    onCall("deletePage", async (pageName: string) => {
+    deletePage: async (clientConn: ClientConnection, pageName: string) => {
-      openPages.delete(pageName);
+      this.openPages.delete(pageName);
-      socketOpenPages.delete(pageName);
+      clientConn.openPages.delete(pageName);
       // Cascading of this to all connected clients will be handled by file watcher
-      return pageStore.deletePage(pageName);
+      return this.pageStore.deletePage(pageName);
-    });
+    },
 
-    onCall("listPages", async (): Promise<PageMeta[]> => {
+    listPages: async (clientConn: ClientConnection): Promise<PageMeta[]> => {
-      return pageStore.listPages();
+      return this.pageStore.listPages();
-    });
+    },
 
-    onCall("getPageMeta", async (pageName: string): Promise<PageMeta> => {
+    getPageMeta: async (
-      let page = openPages.get(pageName);
+      clientConn: ClientConnection,
+      pageName: string
+    ): Promise<PageMeta> => {
+      let page = this.openPages.get(pageName);
       if (page) {
         return page.meta;
       }
-      return pageStore.getPageMeta(pageName);
+      return this.pageStore.getPageMeta(pageName);
-    });
+    },
-  });
 
-  function disconnectClient(client: ClientPageState, page: Page) {
+    "index:clearPageIndexForPage": async (
-    page.clientStates.delete(client);
+      clientConn: ClientConnection,
-    if (page.clientStates.size === 0) {
+      page: string
-      console.log("No more clients for", page.name, "flushing");
+    ) => {
-      flushPageToDisk(page.name, page);
+      await this.db<IndexItem>("page_index").where({ page }).del();
-      openPages.delete(page.name);
+    },
-    } else {
+    "index:set": async (
-      page.cursors.delete(client.socket.id);
+      clientConn: ClientConnection,
-      broadcastCursors(page);
+      page: string,
+      key: string,
+      value: any
+    ) => {
+      let changed = await this.db<IndexItem>("page_index")
+        .where({ page, key })
+        .update("value", JSON.stringify(value));
+      if (changed === 0) {
+        await this.db<IndexItem>("page_index").insert({
+          page,
+          key,
+          value: JSON.stringify(value),
+        });
+      }
+    },
+    "index:get": async (
+      clientConn: ClientConnection,
+      page: string,
+      key: string
+    ) => {
+      let result = await this.db<IndexItem>("page_index")
+        .where({ page, key })
+        .select("value");
+      if (result.length) {
+        return JSON.parse(result[0].value);
+      } else {
+        return null;
+      }
+    },
+    "index:delete": async (
+      clientConn: ClientConnection,
+      page: string,
+      key: string
+    ) => {
+      await this.db<IndexItem>("page_index").where({ page, key }).del();
+    },
+    "index:scanPrefixForPage": async (
+      clientConn: ClientConnection,
+      page: string,
+      prefix: string
+    ) => {
+      return (
+        await this.db<IndexItem>("page_index")
+          .where({ page })
+          .andWhereLike("key", `${prefix}%`)
+          .select("page", "key", "value")
+      ).map(({ page, key, value }) => ({
+        page,
+        key,
+        value: JSON.parse(value),
+      }));
+    },
+    "index:scanPrefixGlobal": async (
+      clientConn: ClientConnection,
+      prefix: string
+    ) => {
+      return (
+        await this.db<IndexItem>("page_index")
+          .andWhereLike("key", `${prefix}%`)
+          .select("page", "key", "value")
+      ).map(({ page, key, value }) => ({
+        page,
+        key,
+        value: JSON.parse(value),
+      }));
+    },
+    "index:deletePrefixForPage": async (
+      clientConn: ClientConnection,
+      page: string,
+      prefix: string
+    ) => {
+      return await this.db<IndexItem>("page_index")
+        .where({ page })
+        .andWhereLike("key", `${prefix}%`)
+        .del();
+    },
+
+    "index:clearPageIndex": async (clientConn: ClientConnection) => {
+      return await this.db<IndexItem>("page_index").del();
+    },
+  };
+
+  constructor(rootPath: string, serverSocket: Server) {
+    this.rootPath = path.resolve(rootPath);
+    this.serverSocket = serverSocket;
+    this.pageStore = new DiskStorage(this.rootPath);
+
+    this.db = knex({
+      client: "better-sqlite3",
+      connection: {
+        filename: path.join(rootPath, "data.db"),
+      },
+      useNullAsDefault: true,
+    });
+    this.initDb();
+
+    serverSocket.on("connection", (socket) => {
+      const clientConn = new ClientConnection(socket);
+      // const socketOpenPages = new Set<string>();
+
+      console.log("Connected", socket.id);
+      this.connectedSockets.add(socket);
+
+      socket.on("disconnect", () => {
+        console.log("Disconnected", socket.id);
+        clientConn.openPages.forEach(disconnectPageSocket);
+        this.connectedSockets.delete(socket);
+      });
+
+      socket.on("closePage", (pageName: string) => {
+        console.log("Closing page", pageName);
+        clientConn.openPages.delete(pageName);
+        disconnectPageSocket(pageName);
+      });
+
+      const onCall = (
+        eventName: string,
+        cb: (...args: any[]) => Promise<any>
+      ) => {
+        socket.on(eventName, (reqId: number, ...args) => {
+          cb(...args)
+            .then((result) => {
+              socket.emit(`${eventName}Resp${reqId}`, null, result);
+            })
+            .catch((err) => {
+              socket.emit(`${eventName}Resp${reqId}`, err.message);
+            });
+        });
+      };
+
+      const disconnectPageSocket = (pageName: string) => {
+        let page = this.openPages.get(pageName);
+        if (page) {
+          for (let client of page.clientStates) {
+            if (client.socket === socket) {
+              this.disconnectClient(client, page);
+            }
+          }
+        }
+      };
+      Object.entries(this.api).forEach(([eventName, cb]) => {
+        onCall(eventName, (...args: any[]): any => {
+          return cb.call(this, clientConn, ...args);
+        });
+      });
+    });
+  }
+
+  async initDb() {
+    if (!(await this.db.schema.hasTable("page_index"))) {
+      await this.db.schema.createTable("page_index", (table) => {
+        table.string("page");
+        table.string("key");
+        table.text("value");
+        table.primary(["page", "key"]);
+      });
+      console.log("Created table page_index");
     }
   }
 
-  function broadcastCursors(page: Page) {
+  disconnectClient(client: ClientPageState, page: Page) {
+    page.clientStates.delete(client);
+    if (page.clientStates.size === 0) {
+      console.log("No more clients for", page.name, "flushing");
+      this.flushPageToDisk(page.name, page);
+      this.openPages.delete(page.name);
+    } else {
+      page.cursors.delete(client.socket.id);
+      this.broadcastCursors(page);
+    }
+  }
+
+  broadcastCursors(page: Page) {
     page.clientStates.forEach((client) => {
       client.socket.emit(
         "cursorSnapshot",
@@ -233,27 +380,27 @@ export function exposeSocketAPI(rootPath: string, io: Server) {
     });
   }
 
-  function flushPageToDisk(name: string, page: Page) {
+  flushPageToDisk(name: string, page: Page) {
     safeRun(async () => {
-      let meta = await pageStore.writePage(name, page.text.sliceString(0));
+      let meta = await this.pageStore.writePage(name, page.text.sliceString(0));
       console.log(`Wrote page ${name} to disk`);
       page.meta = meta;
     });
   }
 
-  function fileWatcher(rootPath: string) {
+  fileWatcher() {
     fs.watch(
-      rootPath,
+      this.rootPath,
       {
         recursive: true,
         persistent: false,
       },
       (eventType, filename) => {
         safeRun(async () => {
-          if (path.extname(filename) !== ".md") {
+          if (!filename.endsWith(".md")) {
             return;
           }
-          let localPath = path.join(rootPath, filename);
+          let localPath = path.join(this.rootPath, filename);
           let pageName = filename.substring(0, filename.length - 3);
           // console.log("Edit in", pageName, eventType);
           let modifiedTime = 0;
@@ -263,16 +410,16 @@ export function exposeSocketAPI(rootPath: string, io: Server) {
           } catch (e) {
             // File was deleted
             console.log("Deleted", pageName);
-            for (let socket of connectedSockets) {
+            for (let socket of this.connectedSockets) {
               socket.emit("pageDeleted", pageName);
             }
             return;
           }
-          const openPage = openPages.get(pageName);
+          const openPage = this.openPages.get(pageName);
           if (openPage) {
             if (openPage.meta.lastModified < modifiedTime) {
               console.log("Page changed on disk outside of editor, reloading");
-              openPages.delete(pageName);
+              this.openPages.delete(pageName);
               const meta = {
                 name: pageName,
                 lastModified: modifiedTime,
@@ -285,9 +432,10 @@ export function exposeSocketAPI(rootPath: string, io: Server) {
           if (eventType === "rename") {
             // This most likely means a new file was created, let's push new file listings to all connected sockets
             console.log(
-              "New file created, broadcasting to all connected sockets"
+              "New file created, broadcasting to all connected sockets",
+              pageName
             );
-            for (let socket of connectedSockets) {
+            for (let socket of this.connectedSockets) {
               socket.emit("pageCreated", {
                 name: pageName,
                 lastModified: modifiedTime,
@@ -298,4 +446,8 @@ export function exposeSocketAPI(rootPath: string, io: Server) {
       }
     );
   }
+
+  close() {
+    this.db.destroy();
+  }
 }

server/src/disk_storage.ts

@@ -1,6 +1,6 @@
 import { readdir, readFile, stat, unlink, writeFile } from "fs/promises";
-import path from "path";
+import * as path from "path";
-import { PageMeta, pagesPath } from "./server";
+import { PageMeta } from "./types";
 
 export class DiskStorage {
   rootPath: string;
@@ -37,41 +37,55 @@ export class DiskStorage {
   }
 
   async readPage(pageName: string): Promise<{ text: string; meta: PageMeta }> {
-    const localPath = path.join(pagesPath, pageName + ".md");
+    const localPath = path.join(this.rootPath, pageName + ".md");
-    const s = await stat(localPath);
+    try {
-    return {
+      const s = await stat(localPath);
-      text: await readFile(localPath, "utf8"),
+      return {
-      meta: {
+        text: await readFile(localPath, "utf8"),
-        name: pageName,
+        meta: {
-        lastModified: s.mtime.getTime(),
+          name: pageName,
-      },
+          lastModified: s.mtime.getTime(),
-    };
+        },
+      };
+    } catch (e) {
+      // console.error("Error while writing page", pageName, e);
+      throw Error(`Could not read page ${pageName}`);
+    }
   }
 
   async writePage(pageName: string, text: string): Promise<PageMeta> {
-    let localPath = path.join(pagesPath, pageName + ".md");
+    let localPath = path.join(this.rootPath, pageName + ".md");
-    // await pipeline(body, fs.createWriteStream(localPath));
+    try {
-    await writeFile(localPath, text);
+      await writeFile(localPath, text);
 
-    // console.log(`Wrote to ${localPath}`);
+      // console.log(`Wrote to ${localPath}`);
-    const s = await stat(localPath);
+      const s = await stat(localPath);
-    return {
+      return {
-      name: pageName,
+        name: pageName,
-      lastModified: s.mtime.getTime(),
+        lastModified: s.mtime.getTime(),
-    };
+      };
+    } catch (e) {
+      console.error("Error while writing page", pageName, e);
+      throw Error(`Could not write ${pageName}`);
+    }
   }
 
   async getPageMeta(pageName: string): Promise<PageMeta> {
-    let localPath = path.join(pagesPath, pageName + ".md");
+    let localPath = path.join(this.rootPath, pageName + ".md");
-    const s = await stat(localPath);
+    try {
-    return {
+      const s = await stat(localPath);
-      name: pageName,
+      return {
-      lastModified: s.mtime.getTime(),
+        name: pageName,
-    };
+        lastModified: s.mtime.getTime(),
+      };
+    } catch (e) {
+      console.error("Error while getting page meta", pageName, e);
+      throw Error(`Could not get meta for ${pageName}`);
+    }
   }
 
   async deletePage(pageName: string) {
-    let localPath = path.join(pagesPath, pageName + ".md");
+    let localPath = path.join(this.rootPath, pageName + ".md");
     await unlink(localPath);
   }
 }

server/src/server.ts

@@ -2,7 +2,19 @@ import express from "express";
 import { readFile } from "fs/promises";
 import http from "http";
 import { Server } from "socket.io";
-import { exposeSocketAPI } from "./api";
+import { SocketServer } from "./api";
+import yargs from "yargs";
+import { hideBin } from "yargs/helpers";
+
+let args = yargs(hideBin(process.argv))
+  .option("debug", {
+    type: "boolean",
+  })
+  .option("port", {
+    type: "number",
+    default: 3000,
+  })
+  .parse();
 
 const app = express();
 const server = http.createServer(app);
@@ -13,18 +25,11 @@ const io = new Server(server, {
   },
 });
 
-const port = 3000;
+const port = args.port;
-export const pagesPath = "../pages";
 const distDir = `${__dirname}/../../webapp/dist`;
 
-export type PageMeta = {
-  name: string;
-  lastModified: number;
-  version?: number;
-};
-
 app.use("/", express.static(distDir));
-exposeSocketAPI(pagesPath, io);
+let socketServer = new SocketServer(args._[0] as string, io);
 
 // Fallback, serve index.html
 let cachedIndex: string | undefined = undefined;
@@ -36,5 +41,5 @@ app.get("/*", async (req, res) => {
 });
 
 server.listen(port, () => {
-  console.log(`Server istening on port ${port}`);
+  console.log(`Server listening on port ${port}`);
 });

server/src/types.ts

@@ -2,12 +2,16 @@ import { Update } from "@codemirror/collab";
 import { Text } from "@codemirror/state";
 import { Socket } from "socket.io";
 import { Cursor } from "../../webapp/src/cursorEffect";
-import { PageMeta } from "./server";
-
 export class ClientPageState {
   constructor(public socket: Socket, public version: number) {}
 }
 
+export type PageMeta = {
+  name: string;
+  lastModified: number;
+  version?: number;
+};
+
 export class Page {
   versionOffset = 0;
   updates: Update[] = [];

webapp/package.json

@@ -1,5 +1,5 @@
 {
-  "name": "noot",
+  "name": "silverbullet",
   "version": "1.0.0",
   "source": [
     "src/index.html"
@@ -7,10 +7,9 @@
   "license": "MIT",
   "browserslist": "> 0.5%, last 2 versions, not dead",
   "scripts": {
-    "start": "parcel",
+    "start": "parcel serve --no-cache",
     "build": "parcel build",
-    "clean": "rm -rf dist",
+    "clean": "rm -rf dist"
-    "check-watch": "tsc --noEmit --watch"
   },
   "devDependencies": {
     "@parcel/packager-raw-url": "2.3.2",

webapp/src/boot.ts

@@ -1,14 +1,11 @@
 import { Editor } from "./editor";
-import { RealtimeSpace } from "./space";
+import { Space } from "./space";
 import { safeRun } from "./util";
 import { io } from "socket.io-client";
 
 let socket = io(`http://${location.hostname}:3000`);
 
-let editor = new Editor(
+let editor = new Editor(new Space(socket), document.getElementById("root")!);
-  new RealtimeSpace(socket),
-  document.getElementById("root")!
-);
 
 safeRun(async () => {
   await editor.init();
@@ -16,3 +13,9 @@ safeRun(async () => {
 
 // @ts-ignore
 window.editor = editor;
+
+navigator.serviceWorker
+  .register(new URL("service_worker.ts", import.meta.url), { type: "module" })
+  .then((r) => {
+    console.log("Service worker registered", r);
+  });

webapp/src/editor.tsx

@@ -46,7 +46,7 @@ import { IPageNavigator, PathPageNavigator } from "./navigator";
 import customMarkDown from "./parser";
 import reducer from "./reducer";
 import { smartQuoteKeymap } from "./smart_quotes";
-import { RealtimeSpace } from "./space";
+import { Space } from "./space";
 import customMarkdownStyle from "./style";
 import dbSyscalls from "./syscalls/db.localstorage";
 import editorSyscalls from "./syscalls/editor.browser";
@@ -77,7 +77,7 @@ export class Editor implements AppEventDispatcher {
   viewState: AppViewState;
   viewDispatch: React.Dispatch<Action>;
   openPages: Map<string, PageState>;
-  space: RealtimeSpace;
+  space: Space;
   editorCommands: Map<string, AppCommand>;
   plugs: Plug<NuggetHook>[];
   indexer: Indexer;
@@ -85,7 +85,7 @@ export class Editor implements AppEventDispatcher {
   pageNavigator: IPageNavigator;
   indexCurrentPageDebounced: () => any;
 
-  constructor(space: RealtimeSpace, parent: Element) {
+  constructor(space: Space, parent: Element) {
     this.editorCommands = new Map();
     this.openPages = new Map();
     this.plugs = [];
@@ -101,7 +101,7 @@ export class Editor implements AppEventDispatcher {
       parent: document.getElementById("editor")!,
     });
     this.pageNavigator = new PathPageNavigator();
-    this.indexer = new Indexer("page-index", space);
+    this.indexer = new Indexer(space);
 
     this.indexCurrentPageDebounced = throttle(
       this.indexCurrentPage.bind(this),
@@ -175,7 +175,7 @@ export class Editor implements AppEventDispatcher {
       dbSyscalls,
       editorSyscalls(this),
       spaceSyscalls(this),
-      indexerSyscalls(this.indexer)
+      indexerSyscalls(this.space)
     );
 
     console.log("Now loading core plug");
@@ -319,8 +319,6 @@ export class Editor implements AppEventDispatcher {
             key: "Ctrl-.",
             mac: "Cmd-.",
             run: (target): boolean => {
-              console.log("YO");
-
               this.viewDispatch({
                 type: "show-palette",
               });

webapp/src/index.html

@@ -2,7 +2,7 @@
 <html lang="en">
   <head>
     <meta charset="utf-8" />
-    <title>Page</title>
+    <title>Silver Bullet</title>
     <link rel="stylesheet" href="styles/main.scss" />
     <script type="module" src="boot.ts"></script>
     <link rel="manifest" href="manifest.json" />

webapp/src/indexer.ts

@@ -1,61 +1,13 @@
-import { Dexie, Table } from "dexie";
 import { AppEventDispatcher, IndexEvent } from "./app_event";
 import { Space } from "./space";
-import { PageMeta } from "./types";
-
-function constructKey(pageName: string, key: string): string {
-  return `${pageName}:${key}`;
-}
-
-function cleanKey(pageName: string, fromKey: string): string {
-  return fromKey.substring(pageName.length + 1);
-}
-
-export type KV = {
-  key: string;
-  value: any;
-};
 
 export class Indexer {
-  db: Dexie;
-  pageIndex: Table;
   space: Space;
 
-  constructor(name: string, space: Space) {
+  constructor(space: Space) {
-    this.db = new Dexie(name);
     this.space = space;
-    this.db.version(1).stores({
-      pageIndex: "ck, page, key",
-    });
-    this.pageIndex = this.db.table("pageIndex");
   }
 
-  async clearPageIndexForPage(pageName: string) {
-    await this.pageIndex.where({ page: pageName }).delete();
-  }
-
-  async clearPageIndex() {
-    await this.pageIndex.clear();
-  }
-
-  // async setPageIndexPageMeta(pageName: string, meta: PageMeta) {
-  //   await this.set(pageName, "$meta", {
-  //     lastModified: meta.lastModified.getTime(),
-  //   });
-  // }
-
-  // async getPageIndexPageMeta(pageName: string): Promise<PageMeta | null> {
-  //   let meta = await this.get(pageName, "$meta");
-  //   if (meta) {
-  //     return {
-  //       name: pageName,
-  //       lastModified: new Date(meta.lastModified),
-  //     };
-  //   } else {
-  //     return null;
-  //   }
-  // }
-
   async indexPage(
     appEventDispatcher: AppEventDispatcher,
     pageName: string,
@@ -63,7 +15,7 @@ export class Indexer {
     withFlush: boolean
   ) {
     if (withFlush) {
-      await this.clearPageIndexForPage(pageName);
+      await this.space.indexDeletePrefixForPage(pageName, "");
     }
     let indexEvent: IndexEvent = {
       name: pageName,
@@ -74,77 +26,12 @@ export class Indexer {
   }
 
   async reindexSpace(space: Space, appEventDispatcher: AppEventDispatcher) {
-    await this.clearPageIndex();
     let allPages = await space.listPages();
     // TODO: Parallelize?
     for (let page of allPages) {
+      await space.indexDeletePrefixForPage(page.name, "");
       let pageData = await space.readPage(page.name);
       await this.indexPage(appEventDispatcher, page.name, pageData.text, false);
     }
   }
-
-  async set(pageName: string, key: string, value: any) {
-    await this.pageIndex.put({
-      ck: constructKey(pageName, key),
-      page: pageName,
-      key: key,
-      value: value,
-    });
-  }
-
-  async batchSet(pageName: string, kvs: KV[]) {
-    await this.pageIndex.bulkPut(
-      kvs.map(({ key, value }) => ({
-        ck: constructKey(pageName, key),
-        key: key,
-        page: pageName,
-        value: value,
-      }))
-    );
-  }
-
-  async get(pageName: string, key: string): Promise<any | null> {
-    let result = await this.pageIndex.get({
-      ck: constructKey(pageName, key),
-    });
-    return result ? result.value : null;
-  }
-
-  async scanPrefixForPage(
-    pageName: string,
-    keyPrefix: string
-  ): Promise<{ key: string; value: any }[]> {
-    let results = await this.pageIndex
-      .where("ck")
-      .startsWith(constructKey(pageName, keyPrefix))
-      .toArray();
-    return results.map((result) => ({
-      key: cleanKey(pageName, result.key),
-      value: result.value,
-    }));
-  }
-
-  async scanPrefixGlobal(
-    keyPrefix: string
-  ): Promise<{ key: string; value: any }[]> {
-    let results = await this.pageIndex
-      .where("key")
-      .startsWith(keyPrefix)
-      .toArray();
-    return results.map((result) => ({
-      key: result.key,
-      value: result.value,
-    }));
-  }
-
-  async deletePrefixForPage(pageName: string, keyPrefix: string) {
-    await this.pageIndex
-      .where("ck")
-      .startsWith(constructKey(pageName, keyPrefix))
-      .delete();
-  }
-
-  async delete(pageName: string, key: string) {
-    await this.pageIndex.delete(constructKey(pageName, key));
-  }
 }

webapp/src/manifest.json

@@ -1,6 +1,6 @@
 {
-  "short_name": "Nugget",
+  "short_name": "Silver Bullet",
-  "name": "Nugget",
+  "name": "Silver Bullet",
   "icons": [
     {
       "src": "./images/logo.png",
@@ -9,7 +9,7 @@
     }
   ],
   "capture_links": "new-client",
-  "start_url": "/start",
+  "start_url": "/",
   "display": "standalone",
   "scope": "/",
   "theme_color": "#000",

webapp/src/service_worker.ts

@@ -0,0 +1,17 @@
+import { manifest, version } from "@parcel/service-worker";
+
+async function install() {
+  const cache = await caches.open(version);
+  await cache.addAll(manifest);
+}
+//@ts-ignore
+self.addEventListener("install", (e) => e.waitUntil(install()));
+
+async function activate() {
+  const keys = await caches.keys();
+  await Promise.all(keys.map((key) => key !== version && caches.delete(key)));
+}
+//@ts-ignore
+self.addEventListener("activate", (e) => e.waitUntil(activate()));
+
+self.addEventListener("fetch", function (event) {});

webapp/src/space.ts

@@ -7,14 +7,6 @@ import { CollabEvents, CollabDocument } from "./collab";
 import { Cursor, cursorEffect } from "./cursorEffect";
 import { EventEmitter } from "./event";
 
-export interface Space {
-  listPages(): Promise<PageMeta[]>;
-  readPage(name: string): Promise<{ text: string; meta: PageMeta }>;
-  writePage(name: string, text: string): Promise<PageMeta>;
-  deletePage(name: string): Promise<void>;
-  getPageMeta(name: string): Promise<PageMeta>;
-}
-
 export type SpaceEvents = {
   connect: () => void;
   pageCreated: (meta: PageMeta) => void;
@@ -23,7 +15,12 @@ export type SpaceEvents = {
   pageListUpdated: (pages: Set<PageMeta>) => void;
 } & CollabEvents;
 
-export class RealtimeSpace extends EventEmitter<SpaceEvents> implements Space {
+export type KV = {
+  key: string;
+  value: any;
+};
+
+export class Space extends EventEmitter<SpaceEvents> {
   socket: Socket;
   reqId = 0;
   allPages = new Set<PageMeta>();
@@ -66,9 +63,15 @@ export class RealtimeSpace extends EventEmitter<SpaceEvents> implements Space {
   }
 
   private wsCall(eventName: string, ...args: any[]): Promise<any> {
-    return new Promise((resolve) => {
+    return new Promise((resolve, reject) => {
       this.reqId++;
-      this.socket!.once(`${eventName}Resp${this.reqId}`, resolve);
+      this.socket!.once(`${eventName}Resp${this.reqId}`, (err, result) => {
+        if (err) {
+          reject(err);
+        } else {
+          resolve(result);
+        }
+      });
       this.socket!.emit(eventName, this.reqId, ...args);
     });
   }
@@ -136,4 +139,40 @@ export class RealtimeSpace extends EventEmitter<SpaceEvents> implements Space {
   async getPageMeta(name: string): Promise<PageMeta> {
     return this.wsCall("deletePage", name);
   }
+
+  async indexSet(pageName: string, key: string, value: any) {
+    await this.wsCall("index:set", pageName, key, value);
+  }
+
+  async indexBatchSet(pageName: string, kvs: KV[]) {
+    // TODO: Optimize with batch call
+    for (let { key, value } of kvs) {
+      await this.indexSet(pageName, key, value);
+    }
+  }
+
+  async indexGet(pageName: string, key: string): Promise<any | null> {
+    return await this.wsCall("index:get", pageName, key);
+  }
+
+  async indexScanPrefixForPage(
+    pageName: string,
+    keyPrefix: string
+  ): Promise<{ key: string; value: any }[]> {
+    return await this.wsCall("index:scanPrefixForPage", pageName, keyPrefix);
+  }
+
+  async indexScanPrefixGlobal(
+    keyPrefix: string
+  ): Promise<{ key: string; value: any }[]> {
+    return await this.wsCall("index:scanPrefixGlobal", keyPrefix);
+  }
+
+  async indexDeletePrefixForPage(pageName: string, keyPrefix: string) {
+    await this.wsCall("index:deletePrefixForPage", keyPrefix);
+  }
+
+  async indexDelete(pageName: string, key: string) {
+    await this.wsCall("index:delete", pageName, key);
+  }
 }

webapp/src/styles/editor.scss

@@ -128,7 +128,8 @@
   }
 
   .mention {
-    color: gray;
+    color: #0330cb;
+    text-decoration: underline;
   }
 
   .tag {

webapp/src/syscalls/indexer.native.ts

@@ -1,22 +1,22 @@
-import { Indexer, KV } from "../indexer";
+import { Space, KV } from "../space";
 
-export default (indexer: Indexer) => ({
+export default (space: Space) => ({
   "indexer.scanPrefixForPage": async (pageName: string, keyPrefix: string) => {
-    return await indexer.scanPrefixForPage(pageName, keyPrefix);
+    return await space.indexScanPrefixForPage(pageName, keyPrefix);
   },
   "indexer.scanPrefixGlobal": async (keyPrefix: string) => {
-    return await indexer.scanPrefixGlobal(keyPrefix);
+    return await space.indexScanPrefixGlobal(keyPrefix);
   },
   "indexer.get": async (pageName: string, key: string): Promise<any> => {
-    return await indexer.get(pageName, key);
+    return await space.indexGet(pageName, key);
   },
   "indexer.set": async (pageName: string, key: string, value: any) => {
-    await indexer.set(pageName, key, value);
+    await space.indexSet(pageName, key, value);
   },
   "indexer.batchSet": async (pageName: string, kvs: KV[]) => {
-    await indexer.batchSet(pageName, kvs);
+    await space.indexBatchSet(pageName, kvs);
   },
   "indexer.delete": async (pageName: string, key: string) => {
-    await indexer.delete(pageName, key);
+    await space.indexDelete(pageName, key);
   },
 });

webapp/src/syscalls/space.native.ts

@@ -18,7 +18,7 @@ export default (editor: Editor) => ({
   },
   "space.deletePage": async (name: string) => {
     console.log("Clearing page index", name);
-    await editor.indexer.clearPageIndexForPage(name);
+    await editor.space.indexDeletePrefixForPage(name, "");
     // If we're deleting the current page, navigate to the start page
     if (editor.currentPage === name) {
       await editor.navigate("start");