More event refactoring work

common/spaces/evented_space_primitives.ts

@@ -5,64 +5,59 @@ import type { SpacePrimitives } from "./space_primitives.ts";
 
 /**
  * Events exposed:
- * - file:changed (FileMeta)
+ * - file:changed (string, localUpdate: boolean)
  * - file:deleted (string)
  * - file:listed (FileMeta[])
  * - page:saved (string, FileMeta)
  * - page:deleted (string)
  */
-
 export class EventedSpacePrimitives implements SpacePrimitives {
-  private fileMetaCache = new Map<string, FileMeta>();
+  alreadyFetching = false;
   initialFileListLoad = true;
 
+  spaceSnapshot: Record<string, number> = {};
   constructor(
     private wrapped: SpacePrimitives,
     private eventHook: EventHook,
-    private eventsToDispatch = [
-      "file:changed",
-      "file:deleted",
-      "file:listed",
-      "page:saved",
-      "page:deleted",
-    ],
   ) {}
 
   dispatchEvent(name: string, ...args: any[]): Promise<any[]> {
-    if (this.eventsToDispatch.includes(name)) {
     return this.eventHook.dispatchEvent(name, ...args);
-    } else {
-      return Promise.resolve([]);
-    }
   }
 
   async fetchFileList(): Promise<FileMeta[]> {
     const newFileList = await this.wrapped.fetchFileList();
-    const deletedFiles = new Set<string>(this.fileMetaCache.keys());
+    if (this.alreadyFetching) {
+      // Avoid race conditions
+      return newFileList;
+    }
+    // console.log("HEREEREEEREEREE");
+    this.alreadyFetching = true;
+    const deletedFiles = new Set<string>(Object.keys(this.spaceSnapshot));
     for (const meta of newFileList) {
-      const oldFileMeta = this.fileMetaCache.get(meta.name);
+      const oldHash = this.spaceSnapshot[meta.name];
-      const newFileMeta: FileMeta = { ...meta };
+      const newHash = meta.lastModified;
       if (
         (
           // New file scenario
-          !oldFileMeta && !this.initialFileListLoad
+          !oldHash && !this.initialFileListLoad
         ) || (
           // Changed file scenario
-          oldFileMeta &&
+          oldHash &&
-          oldFileMeta.lastModified !== newFileMeta.lastModified
+          oldHash !== newHash
         )
       ) {
-        this.dispatchEvent("file:changed", newFileMeta);
+        this.dispatchEvent("file:changed", meta.name);
       }
       // Page found, not deleted
       deletedFiles.delete(meta.name);
 
-      // Update in cache
+      // Update in snapshot
-      this.fileMetaCache.set(meta.name, newFileMeta);
+      this.spaceSnapshot[meta.name] = newHash;
     }
 
     for (const deletedFile of deletedFiles) {
-      this.fileMetaCache.delete(deletedFile);
+      delete this.spaceSnapshot[deletedFile];
       this.dispatchEvent("file:deleted", deletedFile);
 
       if (deletedFile.endsWith(".md")) {
@@ -71,28 +66,18 @@ export class EventedSpacePrimitives implements SpacePrimitives {
       }
     }
 
-    const fileList = [...new Set(this.fileMetaCache.values())];
+    this.dispatchEvent("file:listed", newFileList);
-    this.dispatchEvent("file:listed", fileList);
+    this.alreadyFetching = false;
     this.initialFileListLoad = false;
-    return fileList;
+    return newFileList;
   }
 
   async readFile(
     name: string,
   ): Promise<{ data: Uint8Array; meta: FileMeta }> {
     const data = await this.wrapped.readFile(name);
-    const previousMeta = this.fileMetaCache.get(name);
+    this.triggerEventsAndCache(name, data.meta.lastModified);
-    const newMeta = data.meta;
+    return data;
-    if (previousMeta) {
-      if (previousMeta.lastModified !== newMeta.lastModified) {
-        // Page changed since last cached metadata, trigger event
-        this.dispatchEvent("file:changed", newMeta);
-      }
-    }
-    return {
-      data: data.data,
-      meta: this.metaCacher(name, newMeta),
-    };
   }
 
   async writeFile(
@@ -108,9 +93,9 @@ export class EventedSpacePrimitives implements SpacePrimitives {
       meta,
     );
     if (!selfUpdate) {
-      this.dispatchEvent("file:changed", newMeta);
+      this.dispatchEvent("file:changed", name, true);
     }
-    this.metaCacher(name, newMeta);
+    this.spaceSnapshot[name] = newMeta.lastModified;
 
     // This can happen async
     if (name.endsWith(".md")) {
@@ -136,17 +121,21 @@ export class EventedSpacePrimitives implements SpacePrimitives {
     return newMeta;
   }
 
+  triggerEventsAndCache(name: string, newHash: number) {
+    const oldHash = this.spaceSnapshot[name];
+    if (oldHash && oldHash !== newHash) {
+      // Page changed since last cached metadata, trigger event
+      this.dispatchEvent("file:changed", name);
+    }
+    this.spaceSnapshot[name] = newHash;
+    return;
+  }
+
   async getFileMeta(name: string): Promise<FileMeta> {
     try {
-      const oldMeta = this.fileMetaCache.get(name);
       const newMeta = await this.wrapped.getFileMeta(name);
-      if (oldMeta) {
+      this.triggerEventsAndCache(name, newMeta.lastModified);
-        if (oldMeta.lastModified !== newMeta.lastModified) {
+      return newMeta;
-          // Changed on disk, trigger event
-          this.dispatchEvent("file:changed", newMeta);
-        }
-      }
-      return this.metaCacher(name, newMeta);
     } catch (e: any) {
       console.log("Checking error", e, name);
       if (e.message === "Not found") {
@@ -167,15 +156,7 @@ export class EventedSpacePrimitives implements SpacePrimitives {
     }
     // await this.getPageMeta(name); // Check if page exists, if not throws Error
     await this.wrapped.deleteFile(name);
-    this.fileMetaCache.delete(name);
+    delete this.spaceSnapshot[name];
     this.dispatchEvent("file:deleted", name);
   }
-
-  private metaCacher(name: string, meta: FileMeta): FileMeta {
-    if (meta.lastModified !== 0) {
-      // Don't cache metadata for pages with a 0 lastModified timestamp (usualy dynamically generated pages)
-      this.fileMetaCache.set(name, meta);
-    }
-    return meta;
-  }
 }

plug-api/silverbullet-syscall/system.ts

@@ -1,6 +1,14 @@
 import type { CommandDef } from "../../web/hooks/command.ts";
 import { syscall } from "./syscall.ts";
 
+export function invoke(
+  name: string,
+  ...args: any[]
+): Promise<any> {
+  return syscall("system.invoke", name, ...args);
+}
+
+// @deprecated use invoke instead
 export function invokeFunction(
   env: string,
   name: string,

plugos/types.ts

@@ -28,9 +28,7 @@ export interface Manifest<HookT> {
    *
    * see: common/manifest.ts#SilverBulletHooks
    */
-  functions: {
+  functions: Record<string, FunctionDef<HookT>>;
-    [key: string]: FunctionDef<HookT>;
-  };
 }
 
 /** Associates hooks with a function. This is the generic base structure, that identifies the function. Hooks are defined by the type parameter. */

plugs/core/core.plug.yaml

@@ -37,6 +37,7 @@ functions:
       name: "Editor: Move Cursor to Position"
   clearPageIndex:
     path: "./page.ts:clearPageIndex"
+    env: server
     events:
       - page:saved
       - page:deleted
@@ -46,6 +47,7 @@ functions:
       - query:page
   parseIndexTextRepublish:
     path: "./page.ts:parseIndexTextRepublish"
+    env: server
     events:
       - page:index_text
   reindexSpaceCommand:

plugs/core/page.ts

@@ -13,7 +13,7 @@ import {
 import { events, mq } from "$sb/plugos-syscall/mod.ts";
 
 import { applyQuery } from "$sb/lib/query.ts";
-import { invokeFunction } from "$sb/silverbullet-syscall/system.ts";
+import { invoke } from "$sb/silverbullet-syscall/system.ts";
 import type { Message } from "$sb/types.ts";
 import { sleep } from "../../common/async_util.ts";
 import { cacheFileListing } from "../federation/federation.ts";
@@ -135,7 +135,7 @@ export async function reindexSpace() {
   console.log("Clearing page index...");
   await index.clearPageIndex();
   // Executed this way to not have to embed the search plug code here
-  await invokeFunction("client", "search.clearIndex");
+  await invoke("search.clearIndex");
   const pages = await space.listPages();
 
   // Queue all page names to be indexed
@@ -172,7 +172,7 @@ export async function clearPageIndex(page: string) {
 }
 
 export async function parseIndexTextRepublish({ name, text }: IndexEvent) {
-  // console.log("Reindexing", name);
+  console.log("Reindexing", name);
   await events.dispatchEvent("page:index", {
     name,
     tree: await markdown.parseMarkdown(text),

plugs/directive/eval_directive.ts

@@ -44,7 +44,7 @@ export async function evalDirectiveRenderer(
     const result = await (0, eval)(
       `(async () => { 
         function invokeFunction(name, ...args) {
-          return syscall("system.invokeFunction", "server", name, ...args);
+          return syscall("system.invoke", name, ...args);
         }
         return ${replaceTemplateVars(translateJs(expression), pageMeta)};
       })()`,

plugs/search/search.plug.yaml

@@ -12,6 +12,7 @@ functions:
 
   searchUnindex:
     path: "./search.ts:pageUnindex"
+    env: client
     events:
       - page:deleted
   searchQueryProvider:

server/server_system.ts

@@ -30,10 +30,12 @@ import { shellSyscalls } from "../plugos/syscalls/shell.deno.ts";
 import { IDBKeyRange, indexedDB } from "https://esm.sh/fake-indexeddb@4.0.2";
 import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
 
+const fileListInterval = 30 * 1000; // 30s
+
 export class ServerSystem {
   system: System<SilverBulletHooks> = new System("server");
   spacePrimitives!: SpacePrimitives;
-  requeueInterval?: number;
+  private requeueInterval?: number;
   kvStore?: DenoKVStore;
 
   constructor(
@@ -114,19 +116,31 @@ export class ServerSystem {
 
     await this.loadPlugs();
 
-    // for (let plugPath of await space.listPlugs()) {
-    //   plugPath = path.resolve(this.spacePath, plugPath);
-    //   await this.system.load(
-    //     new URL(`file://${plugPath}`),
-    //     createSandbox,
-    //   );
-    // }
-
     // Load markdown syscalls based on all new syntax (if any)
     this.system.registerSyscalls(
       [],
       markdownSyscalls(buildMarkdown(loadMarkdownExtensions(this.system))),
     );
+
+    setInterval(() => {
+      space.updatePageList().catch(console.error);
+    }, fileListInterval);
+
+    eventHook.addLocalListener("file:changed", (path, localChange) => {
+      (async () => {
+        // console.log("!!!!! FILE CHANGED", path, localChange);
+        if (!localChange && path.endsWith(".md")) {
+          const pageName = path.slice(0, -3);
+          const data = await this.spacePrimitives.readFile(path);
+          console.log("Outside page change: reindexing", pageName);
+          // Change made outside of editor, trigger reindex
+          await eventHook.dispatchEvent("page:index_text", {
+            name: pageName,
+            text: new TextDecoder().decode(data.data),
+          });
+        }
+      })().catch(console.error);
+    });
   }
 
   async loadPlugs() {

web/client.ts

@@ -321,12 +321,6 @@ export class Client {
         }
       })().catch(console.error);
     }
-
-    // this.eventHook.addLocalListener("page:deleted", (pageName) => {
-    //   if (pageName === this.currentPage) {
-    //     this.flashNotification("Page does exist, creating as a new page");
-    //   }
-    // });
   }
 
   initSpace(): SpacePrimitives {
@@ -375,20 +369,15 @@ export class Client {
       localSpacePrimitives = new EventedSpacePrimitives(
         this.plugSpaceRemotePrimitives,
         this.eventHook,
-        [
-          "file:changed",
-          "file:listed",
-          "page:deleted",
-        ],
       );
     }
 
     this.space = new Space(localSpacePrimitives, this.kvStore, this.eventHook);
 
-    this.eventHook.addLocalListener("file:changed", (fileMeta: FileMeta) => {
+    this.eventHook.addLocalListener("file:changed", (path: string) => {
       // Only reload when watching the current page (to avoid reloading when switching pages)
       if (
-        this.space.watchInterval && `${this.currentPage}.md` === fileMeta.name
+        this.space.watchInterval && `${this.currentPage}.md` === path
       ) {
         console.log("Page changed elsewhere, reloading");
         this.flashNotification("Page changed elsewhere, reloading");

web/client_system.ts

@@ -37,7 +37,6 @@ import { indexProxySyscalls } from "./syscalls/index.proxy.ts";
 import { storeProxySyscalls } from "./syscalls/store.proxy.ts";
 
 export class ClientSystem {
-  system: System<SilverBulletHooks> = new System("client");
   commandHook: CommandHook;
   slashCommandHook: SlashCommandHook;
   namespaceHook: PlugNamespaceHook;
@@ -45,15 +44,19 @@ export class ClientSystem {
   codeWidgetHook: CodeWidgetHook;
   plugsUpdated = false;
   mdExtensions: MDExt[] = [];
+  system: System<SilverBulletHooks>;
 
   constructor(
     private client: Client,
     private kvStore: DexieKVStore,
     private mq: DexieMQ,
-    private dbPrefix: string,
+    dbPrefix: string,
     private eventHook: EventHook,
     private thinClientMode: boolean,
   ) {
+    // Only set environment to "client" when running in thin client mode, otherwise we run everything locally (hybrid)
+    this.system = new System(thinClientMode ? "client" : undefined);
+
     this.system.addHook(this.eventHook);
 
     // Plug page namespace hook

web/space.ts

@@ -65,36 +65,6 @@ export class Space {
   public async updatePageList() {
     // This will trigger appropriate events automatically
     await this.fetchPageList();
-    // const deletedPages = new Set<string>(this.pageMetaCache.keys());
-    // newPageList.forEach((meta) => {
-    //   const pageName = meta.name;
-    //   const oldPageMeta = this.pageMetaCache.get(pageName);
-    //   const newPageMeta: PageMeta = { ...meta };
-    //   if (
-    //     !oldPageMeta &&
-    //     (pageName.startsWith(plugPrefix) || !this.initialPageListLoad)
-    //   ) {
-    //     this.emit("pageCreated", newPageMeta);
-    //   } else if (
-    //     oldPageMeta &&
-    //     oldPageMeta.lastModified !== newPageMeta.lastModified
-    //   ) {
-    //     this.emit("pageChanged", newPageMeta);
-    //   }
-    //   // Page found, not deleted
-    //   deletedPages.delete(pageName);
-
-    //   // Update in cache
-    //   this.pageMetaCache.set(pageName, newPageMeta);
-    // });
-
-    // for (const deletedPage of deletedPages) {
-    //   this.pageMetaCache.delete(deletedPage);
-    //   this.emit("pageDeleted", deletedPage);
-    // }
-
-    // this.emit("pageListUpdated", this.listPages());
-    // this.initialPageListLoad = false;
   }
 
   async deletePage(name: string): Promise<void> {

web/syscalls/system.ts

@@ -2,17 +2,27 @@ import type { Plug } from "../../plugos/plug.ts";
 import { SysCallMapping, System } from "../../plugos/system.ts";
 import type { Client } from "../client.ts";
 import { CommandDef } from "../hooks/command.ts";
+import { proxySyscall } from "./util.ts";
 
 export function systemSyscalls(
   editor: Client,
   system: System<any>,
 ): SysCallMapping {
-  return {
+  const api: SysCallMapping = {
     "system.invokeFunction": (
       ctx,
       _env: string,
       name: string,
       ...args: any[]
+    ) => {
+      // For backwards compatibility
+      // TODO: Remove at some point
+      return api["system.invoke"](ctx, name, ...args);
+    },
+    "system.invoke": (
+      ctx,
+      name: string,
+      ...args: any[]
     ) => {
       if (!ctx.plug) {
         throw Error("No plug associated with context");
@@ -28,6 +38,14 @@ export function systemSyscalls(
         }
         name = functionName;
       }
+      const functionDef = plug.manifest!.functions[name];
+      if (!functionDef) {
+        throw Error(`Function ${name} not found`);
+      }
+      if (functionDef.env && system.env && functionDef.env !== system.env) {
+        // Proxy to another environment
+        return proxySyscall(editor.remoteSpacePrimitives, name, args);
+      }
       return plug.invoke(name, args);
     },
     "system.invokeCommand": (_ctx, name: string) => {
@@ -47,4 +65,5 @@ export function systemSyscalls(
       return system.env;
     },
   };
+  return api;
 }

web/syscalls/util.ts

@@ -1,3 +1,4 @@
+import { HttpSpacePrimitives } from "../../common/spaces/http_space_primitives.ts";
 import { SysCallMapping } from "../../plugos/system.ts";
 import { SyscallResponse } from "../../server/rpc.ts";
 import { Client } from "../client.ts";
@@ -5,12 +6,20 @@ import { Client } from "../client.ts";
 export function proxySyscalls(client: Client, names: string[]): SysCallMapping {
   const syscalls: SysCallMapping = {};
   for (const name of names) {
-    syscalls[name] = async (_ctx, ...args: any[]) => {
+    syscalls[name] = (_ctx, ...args: any[]) => {
-      if (!client.remoteSpacePrimitives) {
+      return proxySyscall(client.remoteSpacePrimitives, name, args);
-        throw new Error("Not supported");
+    };
   }
-      const resp = await client.remoteSpacePrimitives.authenticatedFetch(
+  return syscalls;
-        `${client.remoteSpacePrimitives.url}/.rpc`,
+}
+
+export async function proxySyscall(
+  httpSpacePrimitives: HttpSpacePrimitives,
+  name: string,
+  args: any[],
+): Promise<any> {
+  const resp = await httpSpacePrimitives.authenticatedFetch(
+    `${httpSpacePrimitives.url}/.rpc`,
     {
       method: "POST",
       body: JSON.stringify({
@@ -27,7 +36,4 @@ export function proxySyscalls(client: Client, names: string[]): SysCallMapping {
   } else {
     return result.result;
   }
-    };
-  }
-  return syscalls;
 }