Work on plug:run

Dockerfile

@@ -42,9 +42,12 @@ USER ${SILVERBULLET_USERNAME}
 # Port map this when running, e.g. with -p 3002:3000 (where 3002 is the host port)
 EXPOSE 3000
 
+ENV SB_HOSTNAME 0.0.0.0
+ENV SB_FOLDER /space
+
 # Copy the bundled version of silverbullet into the container
 ADD ./dist/silverbullet.js /silverbullet.js
 
 # Run the server, allowing to pass in additional argument at run time, e.g.
 #   docker run -p 3002:3000 -v myspace:/space -it zefhemel/silverbullet --user me:letmein
-ENTRYPOINT ["/tini", "--", "deno", "run", "-A", "/silverbullet.js", "-L0.0.0.0", "/space"]
+ENTRYPOINT ["/tini", "--", "deno", "run", "-A", "--unstable", "/silverbullet.js"]

cli/plug_run.ts

@@ -11,27 +11,37 @@ import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
 import { createSandbox } from "../plugos/environments/deno_sandbox.ts";
 import { CronHook } from "../plugos/hooks/cron.ts";
 import { EventHook } from "../plugos/hooks/event.ts";
+import { MQHook } from "../plugos/hooks/mq.ts";
 import { DenoKVStore } from "../plugos/lib/kv_store.deno_kv.ts";
+import { DexieMQ } from "../plugos/lib/mq.dexie.ts";
 import assetSyscalls from "../plugos/syscalls/asset.ts";
 import { eventSyscalls } from "../plugos/syscalls/event.ts";
 import { sandboxFetchSyscalls } from "../plugos/syscalls/fetch.ts";
+import { mqSyscalls } from "../plugos/syscalls/mq.dexie.ts";
 import { shellSyscalls } from "../plugos/syscalls/shell.deno.ts";
 import { storeSyscalls } from "../plugos/syscalls/store.ts";
 import { System } from "../plugos/system.ts";
 import { Space } from "../web/space.ts";
 import { debugSyscalls } from "../web/syscalls/debug.ts";
+import { pageIndexSyscalls } from "./syscalls/index.ts";
 import { markdownSyscalls } from "../web/syscalls/markdown.ts";
 import { systemSyscalls } from "../web/syscalls/system.ts";
 import { yamlSyscalls } from "../web/syscalls/yaml.ts";
-import { pageIndexSyscalls } from "./syscalls/index.ts";
 import { spaceSyscalls } from "./syscalls/space.ts";
 
+import { IDBKeyRange, indexedDB } from "https://esm.sh/fake-indexeddb@4.0.2";
+import { Application } from "../server/deps.ts";
+import { EndpointHook } from "../plugos/hooks/endpoint.ts";
+import { sleep } from "../common/async_util.ts";
+
 export async function runPlug(
   spacePath: string,
-  functionName: string,
+  functionName: string | undefined,
   args: string[] = [],
   builtinAssetBundle: AssetBundle,
   indexFirst = false,
+  httpServerPort = 3123,
+  httpHostname = "127.0.0.1",
 ) {
   spacePath = path.resolve(spacePath);
   const system = new System<SilverBulletHooks>("cli");
@@ -45,15 +55,29 @@ export async function runPlug(
   system.addHook(cronHook);
 
   const kvStore = new DenoKVStore();
-  await kvStore.init("run.db");
+  const tempFile = Deno.makeTempFileSync({ suffix: ".db" });
+  await kvStore.init(tempFile);
+
+  // Endpoint hook
+  const app = new Application();
+  system.addHook(new EndpointHook(app, "/_"));
+  const serverController = new AbortController();
+  app.listen({
+    hostname: httpHostname,
+    port: httpServerPort,
+    signal: serverController.signal,
+  });
+
+  // Use DexieMQ for this, in memory
+  const mq = new DexieMQ("mq", indexedDB, IDBKeyRange);
 
   const pageIndexCalls = pageIndexSyscalls(kvStore);
 
-  // TODO: Add endpoint
-
   const plugNamespaceHook = new PlugNamespaceHook();
   system.addHook(plugNamespaceHook);
 
+  system.addHook(new MQHook(system, mq));
+
   const spacePrimitives = new FileMetaSpacePrimitives(
     new EventedSpacePrimitives(
       new PlugSpacePrimitives(
@@ -75,6 +99,7 @@ export async function runPlug(
     yamlSyscalls(),
     storeSyscalls(kvStore),
     systemSyscalls(undefined as any, system),
+    mqSyscalls(mq),
     pageIndexCalls,
     debugSyscalls(),
     markdownSyscalls(buildMarkdown([])), // Will later be replaced with markdown extensions
@@ -111,6 +136,7 @@ export async function runPlug(
     await system.loadedPlugs.get("core")!.invoke("reindexSpace", []);
   }
 
+  if (functionName) {
     const [plugName, funcName] = functionName.split(".");
 
     const plug = system.loadedPlugs.get(plugName);
@@ -118,10 +144,16 @@ export async function runPlug(
       throw new Error(`Plug ${plugName} not found`);
     }
     const result = await plug.invoke(funcName, args);
-
     await system.unloadAll();
     await kvStore.delete();
+    serverController.abort();
     return result;
+  } else {
+    console.log("Running in server mode, use Ctrl-c to stop");
+    while (true) {
+      await sleep(1000);
+    }
+  }
 }
 
 async function loadPlugsFromAssetBundle(
@@ -131,7 +163,9 @@ async function loadPlugsFromAssetBundle(
   const tempDir = await Deno.makeTempDir();
   try {
     for (const filePath of assetBundle.listFiles()) {
-      if (filePath.endsWith(".plug.js")) {
+      if (
+        filePath.endsWith(".plug.js") // && !filePath.includes("search.plug.js")
+      ) {
         const plugPath = path.join(tempDir, filePath);
         await Deno.mkdir(path.dirname(plugPath), { recursive: true });
         await Deno.writeFile(plugPath, assetBundle.readFileSync(filePath));

cmd/plug_compile.ts

@@ -26,4 +26,5 @@ export async function plugCompileCommand(
     },
   );
   esbuild.stop();
+  Deno.exit(0);
 }

cmd/plug_run.ts

@@ -8,11 +8,15 @@ import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
 export async function plugRunCommand(
   {
     noIndex,
+    hostname,
+    port,
   }: {
     noIndex: boolean;
+    hostname?: string;
+    port?: number;
   },
   spacePath: string,
-  functionName: string,
+  functionName: string | undefined,
   ...args: string[]
 ) {
   spacePath = path.resolve(spacePath);
@@ -25,8 +29,11 @@ export async function plugRunCommand(
       args,
       new AssetBundle(assets),
       !noIndex,
+      port,
+      hostname,
     );
     console.log("Output", result);
+    Deno.exit(0);
   } catch (e: any) {
     console.error(e.message);
     Deno.exit(1);

cmd/server.ts

@@ -18,7 +18,8 @@ export async function serveCommand(
   options: any,
   folder?: string,
 ) {
-  const hostname = options.hostname || "127.0.0.1";
+  const hostname = options.hostname || Deno.env.get("SB_HOSTNAME") ||
+    "127.0.0.1";
   const port = options.port ||
     (Deno.env.get("SB_PORT") && +Deno.env.get("SB_PORT")!) || 3000;
   const maxFileSizeMB = options.maxFileSizeMB || 20;

common/manifest.ts

@@ -6,6 +6,7 @@ import { SlashCommandHookT } from "../web/hooks/slash_command.ts";
 import { PlugNamespaceHookT } from "./hooks/plug_namespace.ts";
 import { CodeWidgetT } from "../web/hooks/code_widget.ts";
 import { MQHookT } from "../plugos/hooks/mq.ts";
+import { EndpointHookT } from "../plugos/hooks/endpoint.ts";
 
 export type SilverBulletHooks =
   & CommandHookT
@@ -14,6 +15,7 @@ export type SilverBulletHooks =
   & MQHookT
   & EventHookT
   & CodeWidgetT
+  & EndpointHookT
   & PlugNamespaceHookT;
 
 export type SyntaxExtensions = {

plug-api/plugos-syscall/mq.ts

@@ -1,4 +1,5 @@
 import { syscall } from "$sb/plugos-syscall/syscall.ts";
+import { QueueStats } from "$sb/types.ts";
 
 export function send(queue: string, body: any) {
   return syscall("mq.send", queue, body);
@@ -15,3 +16,7 @@ export function ack(queue: string, id: string) {
 export function batchAck(queue: string, ids: string[]) {
   return syscall("mq.batchAck", queue, ids);
 }
+
+export function getQueueStats(queue: string): Promise<QueueStats> {
+  return syscall("mq.getQueueStats", queue);
+}

plug-api/types.ts

@@ -4,3 +4,9 @@ export type Message = {
   body: any;
   retries?: number;
 };
+
+export type QueueStats = {
+  queued: number;
+  processing: number;
+  dlq: number;
+};

plugos/hooks/endpoint.test.ts

@@ -1,48 +1,42 @@
 import { createSandbox } from "../environments/deno_sandbox.ts";
-import { Manifest } from "../types.ts";
 import { EndpointHook, EndpointHookT } from "./endpoint.ts";
 import { System } from "../system.ts";
 
 import { Application } from "../../server/deps.ts";
 import { assertEquals } from "../../test_deps.ts";
+import { compileManifest } from "../compile.ts";
+import { esbuild } from "../deps.ts";
 
-// Deno.test("Run a plugos endpoint server", async () => {
-//   const system = new System<EndpointHookT>("server");
-//   await system.load(
-//     {
-//       name: "test",
-//       functions: {
-//         testhandler: {
-//           http: {
-//             path: "/",
-//           },
-//           code: `(() => {
-//           return {
-//             default: (req) => {
-//               console.log("Req", req);
-//               return {status: 200, body: [1, 2, 3], headers: {"Content-type": "application/json"}};
-//             }
-//           };
-//         })()`,
-//         },
-//       },
-//     } as Manifest<EndpointHookT>,
-//     createSandbox,
-//   );
+Deno.test("Run a plugos endpoint server", async () => {
+  const tempDir = await Deno.makeTempDir();
+  const system = new System<EndpointHookT>("server");
 
-//   const app = new Application();
-//   const port = 3123;
+  const workerPath = await compileManifest(
+    new URL("../test.plug.yaml", import.meta.url).pathname,
+    tempDir,
+  );
 
-//   system.addHook(new EndpointHook(app, "/_"));
+  await system.load(
+    new URL(`file://${workerPath}`),
+    createSandbox,
+  );
 
-//   const controller = new AbortController();
-//   app.listen({ port: port, signal: controller.signal });
+  const app = new Application();
+  const port = 3123;
 
-//   const res = await fetch(`http://localhost:${port}/_/test/?name=Pete`);
-//   assertEquals(res.status, 200);
-//   assertEquals(res.headers.get("Content-type"), "application/json");
-//   assertEquals(await res.json(), [1, 2, 3]);
-//   console.log("Aborting");
-//   controller.abort();
-//   await system.unloadAll();
-// });
+  system.addHook(new EndpointHook(app, "/_"));
+
+  const controller = new AbortController();
+  app.listen({ port: port, signal: controller.signal });
+
+  const res = await fetch(`http://localhost:${port}/_/test/?name=Pete`);
+  assertEquals(res.status, 200);
+  assertEquals(res.headers.get("Content-type"), "application/json");
+  assertEquals(await res.json(), [1, 2, 3]);
+  console.log("Aborting");
+  controller.abort();
+  await system.unloadAll();
+
+  await Deno.remove(tempDir, { recursive: true });
+  esbuild.stop();
+});

plugos/hooks/endpoint.ts

@@ -58,6 +58,7 @@ export class EndpointHook implements Hook<EndpointHookT> {
           if (!functionDef.http) {
             continue;
           }
+          console.log("Got config", functionDef);
           const endpoints = Array.isArray(functionDef.http)
             ? functionDef.http
             : [functionDef.http];

plugos/hooks/mq.ts

@@ -2,7 +2,7 @@ import { Hook, Manifest } from "../types.ts";
 import { System } from "../system.ts";
 import { DexieMQ } from "../lib/mq.dexie.ts";
 import { fullQueueName } from "../lib/mq_util.ts";
-import { Message } from "$sb/mq.ts";
+import { Message } from "$sb/types.ts";
 
 type MQSubscription = {
   queue: string;

plugos/lib/kv_store.dexie.ts

@@ -8,9 +8,11 @@ export class DexieKVStore implements KVStore {
     dbName: string,
     tableName: string,
     indexedDB?: any,
+    IDBKeyRange?: any,
   ) {
     this.db = new Dexie(dbName, {
       indexedDB,
+      IDBKeyRange,
     });
     this.db.version(1).stores({
       [tableName]: "key",

plugos/lib/mq.dexie.ts

@@ -1,5 +1,5 @@
 import Dexie, { Table } from "dexie";
-import { Message } from "$sb/mq.ts";
+import { Message, QueueStats } from "$sb/types.ts";
 
 export type ProcessingMessage = Message & {
   ts: number;
@@ -10,12 +10,6 @@ export type SubscribeOptions = {
   pollInterval?: number;
 };
 
-export type QueueStats = {
-  queued: number;
-  processing: number;
-  dlq: number;
-};
-
 export class DexieMQ {
   db: Dexie;
   queued: Table<Message, [string, string]>;

plugos/syscalls/mq.dexie.ts

@@ -18,5 +18,8 @@ export function mqSyscalls(
     "mq.batchAck": (ctx, queue: string, ids: string[]) => {
       return mq.batchAck(fullQueueName(ctx.plug.name!, queue), ids);
     },
+    "mq.getQueueStats": (ctx, queue: string) => {
+      return mq.getQueueStats(fullQueueName(ctx.plug.name!, queue));
+    },
   };
 }

plugos/test.plug.yaml

@@ -2,3 +2,7 @@ name: test
 functions:
   boot:
     path: "./test_func.test.ts:hello"
+  endpoint:
+    path: "./test_func.test.ts:endpoint"
+    http:
+      path: "/"

plugos/test_func.test.ts

@@ -1,7 +1,17 @@
 import * as YAML from "https://deno.land/std@0.184.0/yaml/mod.ts";
+import { EndpointRequest, EndpointResponse } from "./hooks/endpoint.ts";
 
 export function hello() {
   console.log(YAML.stringify({ hello: "world" }));
 
   return "hello";
 }
+
+export function endpoint(req: EndpointRequest): EndpointResponse {
+  console.log("Req", req);
+  return {
+    status: 200,
+    body: [1, 2, 3],
+    headers: { "Content-type": "application/json" },
+  };
+}

plugs/core/page.ts

@@ -14,7 +14,8 @@ import { events, mq } from "$sb/plugos-syscall/mod.ts";
 
 import { applyQuery } from "$sb/lib/query.ts";
 import { invokeFunction } from "$sb/silverbullet-syscall/system.ts";
-import type { Message } from "$sb/mq.ts";
+import type { Message } from "$sb/types.ts";
+import { sleep } from "../../common/async_util.ts";
 
 // Key space:
 //   meta: => metaJson
@@ -83,14 +84,9 @@ export async function newPageCommand() {
 }
 
 export async function reindexCommand() {
-  await editor.flashNotification("Scheduling full reindex...");
-  console.log("Clearing page index...");
-  await index.clearPageIndex();
-  // Executed this way to not have to embed the search plug code here
-  await invokeFunction("client", "search.clearIndex");
-  const pages = await space.listPages();
-
-  await mq.batchSend("indexQueue", pages.map((page) => page.name));
+  await editor.flashNotification("Performing full page reindex...");
+  await reindexSpace();
+  await editor.flashNotification("Done with page index!");
 }
 
 // Completion
@@ -117,20 +113,18 @@ export async function reindexSpace() {
   await index.clearPageIndex();
   // Executed this way to not have to embed the search plug code here
   await invokeFunction("client", "search.clearIndex");
-  console.log("Listing all pages");
   const pages = await space.listPages();
-  let counter = 0;
-  for (const { name } of pages) {
-    counter++;
 
-    console.log(`Indexing page ${counter}/${pages.length}: ${name}`);
-    const text = await space.readPage(name);
-    const parsed = await markdown.parseMarkdown(text);
-    await events.dispatchEvent("page:index", {
-      name,
-      tree: parsed,
-    });
+  // Queue all page names to be indexed
+  await mq.batchSend("indexQueue", pages.map((page) => page.name));
+
+  // Now let's wait for the processing to finish
+  let queueStats = await mq.getQueueStats("indexQueue");
+  while (queueStats.queued > 0 || queueStats.processing > 0) {
+    sleep(1000);
+    queueStats = await mq.getQueueStats("indexQueue");
   }
+  // And notify the user
   console.log("Indexing completed!");
 }
 

plugs/directive/command.ts

@@ -10,7 +10,10 @@ import { extractFrontmatter } from "$sb/lib/frontmatter.ts";
 import { PageMeta } from "../../web/types.ts";
 import { isFederationPath } from "$sb/lib/resolve.ts";
 import { mq } from "$sb/plugos-syscall/mod.ts";
-import { Message } from "$sb/mq.ts";
+import { Message } from "$sb/types.ts";
+import { sleep } from "../../common/async_util.ts";
+
+const directiveUpdateQueueName = "directiveUpdateQueue";
 
 export async function updateDirectivesOnPageCommand() {
   // If `arg` is a string, it's triggered automatically via an event, not explicitly via a command
@@ -83,10 +86,10 @@ export async function updateDirectivesInSpaceCommand() {
   await editor.flashNotification(
     "Updating directives in entire space, this can take a while...",
   );
-  // await updateDirectivesInSpace();
-  const pages = await space.listPages();
+  await updateDirectivesInSpace();
 
-  await mq.batchSend("directiveUpdateQueue", pages.map((page) => page.name));
+  // And notify the user
+  await editor.flashNotification("Updating of all directives completed!");
 }
 
 export async function processUpdateQueue(messages: Message[]) {
@@ -94,7 +97,7 @@ export async function processUpdateQueue(messages: Message[]) {
     const pageName: string = message.body;
     console.log("Updating directives in page", pageName);
     await updateDirectivesForPage(pageName);
-    await mq.ack("directiveUpdateQueue", message.id);
+    await mq.ack(directiveUpdateQueueName, message.id);
   }
 }
 
@@ -144,20 +147,17 @@ async function findReplacements(
 }
 
 export async function updateDirectivesInSpace() {
-  const allPages = await space.listPages();
-  let counter = 0;
-  for (const page of allPages) {
-    counter++;
-    console.log(
-      `Updating directives in page [${counter}/${allPages.length}]`,
-      page.name,
-    );
-    try {
-      await updateDirectivesForPage(page.name);
-    } catch (e: any) {
-      console.error("Error while updating directives on page", page.name, e);
-    }
+  const pages = await space.listPages();
+  await mq.batchSend(directiveUpdateQueueName, pages.map((page) => page.name));
+
+  // Now let's wait for the processing to finish
+  let queueStats = await mq.getQueueStats(directiveUpdateQueueName);
+  while (queueStats.queued > 0 || queueStats.processing > 0) {
+    sleep(1000);
+    queueStats = await mq.getQueueStats(directiveUpdateQueueName);
   }
+
+  console.log("Done updating directives in space!");
 }
 
 async function updateDirectivesForPage(
@@ -180,7 +180,7 @@ async function updateDirectivesForPage(
 
   const newText = await updateDirectives(pageMeta, tree, currentText);
   if (newText !== currentText) {
-    console.info("Content of page changed, saving.");
+    console.info("Content of page changed, saving", pageName);
     await space.writePage(pageName, newText);
   }
 }

plugs/search/engine.ts

@@ -1,4 +1,3 @@
-import { c } from "https://esm.sh/@codemirror/legacy-modes@6.3.1/mode/clike?external=@codemirror/language";
 import { stemmer } from "https://esm.sh/porter-stemmer@0.9.1";
 
 export type Document = {

silverbullet.ts

@@ -69,10 +69,15 @@ await new Command()
   .action(plugCompileCommand)
   // plug:run
   .command("plug:run", "Run a PlugOS function from the CLI")
-  .arguments("<spacePath> <function> [...args:string]")
+  .arguments("<spacePath> [function] [...args:string]")
   .option("--noIndex [type:boolean]", "Do not run a full space index first", {
     default: false,
   })
+  .option(
+    "--hostname, -L <hostname:string>",
+    "Hostname or address to listen on",
+  )
+  .option("-p, --port <port:number>", "Port to listen on")
   .action(plugRunCommand)
   .command("user:add", "Add a new user to an authentication file")
   .arguments("[username:string]")

web/client.ts

@@ -94,6 +94,7 @@ export class Client {
       `${this.dbPrefix}_store`,
       "data",
       globalThis.indexedDB,
+      globalThis.IDBKeyRange,
     );
 
     this.mq = new DexieMQ(`${this.dbPrefix}_mq`, indexedDB, IDBKeyRange);

web/client_system.ts

@@ -64,6 +64,7 @@ export class ClientSystem {
     this.indexSyscalls = pageIndexSyscalls(
       `${dbPrefix}_page_index`,
       globalThis.indexedDB,
+      globalThis.IDBKeyRange,
     );
 
     // Code widget hook

web/syscalls/index.ts

@@ -15,9 +15,11 @@ export type KV = {
 export function pageIndexSyscalls(
   dbName: string,
   indexedDB?: any,
+  IDBKeyRange?: any,
 ): SysCallMapping {
   const db = new Dexie(dbName, {
     indexedDB,
+    IDBKeyRange,
   });
   db.version(1).stores({
     "index": "[page+key], page, key",

website/Install.md

@@ -126,6 +126,7 @@ $env
 You can configure SB with environment variables instead of flags as well. The following environment variables are supported:
 
 * `SB_USER`: Sets single-user credentials (like `--user`), e.g. `SB_USER=pete:1234`
+* `SB_HOSTNAME`: Set to the hostname to bind to (defaults to `127.0.0.0`, set to `0.0.0.0` to accept outside connections)
 * `SB_PORT`: Sets the port to listen to, e.g. `SB_PORT=1234`
 * `SB_FOLDER`: Sets the folder to expose, e.g. `SB_FOLDER=/space`
 * `SB_AUTH`: Loads an [[Authentication]] database from a (JSON encoded) string, e.g. `SB_AUTH=$(cat /path/to/.auth.json)`