Sync engine (#298)

Fixes #261

.github/workflows/desktop.yml

@@ -3,7 +3,7 @@ name: Build & Release
 on:
   push:
     tags:
-    - '*'
+      - "*"
 jobs:
   build:
     name: Build (${{ matrix.os }} - ${{ matrix.arch }})
@@ -72,6 +72,7 @@ jobs:
           files: |
             desktop/out/**/*.deb
             desktop/out/**/*Setup.exe
+            desktop/out/**/RELEASES
             desktop/out/**/*.rpm
             desktop/out/**/*.zip
             dist/silverbullet.js

build_mobile.ts

@@ -1,4 +1,4 @@
-import { bundle, esbuild } from "./build.ts";
+import { bundle, esbuild } from "./build_web.ts";
 import * as flags from "https://deno.land/std@0.165.0/flags/mod.ts";
 import { copy } from "https://deno.land/std@0.165.0/fs/copy.ts";
 

cmd/server.ts

@@ -7,6 +7,7 @@ export function serveCommand(options: any, folder: string) {
   const pagesPath = path.resolve(Deno.cwd(), folder);
   const hostname = options.hostname || "127.0.0.1";
   const port = options.port || 3000;
+  const bareMode = options.bare;
 
   console.log(
     "Going to start Silver Bullet binding to",
@@ -27,6 +28,7 @@ export function serveCommand(options: any, folder: string) {
     dbPath: path.join(pagesPath, options.db),
     assetBundle: new AssetBundle(assetBundle as AssetJson),
     user: options.user,
+    bareMode,
   });
   httpServer.start().catch((e) => {
     console.error("HTTP Server error", e);

common/manifest.ts

@@ -4,7 +4,7 @@ import { CronHookT } from "../plugos/hooks/cron.deno.ts";
 import { EventHookT } from "../plugos/hooks/event.ts";
 import { CommandHookT } from "../web/hooks/command.ts";
 import { SlashCommandHookT } from "../web/hooks/slash_command.ts";
-import { PageNamespaceHookT } from "../server/hooks/page_namespace.ts";
+import { PageNamespaceHookT } from "./hooks/page_namespace.ts";
 import { CodeWidgetT } from "../web/hooks/code_widget.ts";
 
 export type SilverBulletHooks =

common/spaces/asset_bundle_space_primitives.ts

@@ -12,7 +12,7 @@ export class AssetBundlePlugSpacePrimitives implements SpacePrimitives {
   }
 
   async fetchFileList(): Promise<FileMeta[]> {
-    const l = await this.wrapped.fetchFileList();
+    const files = await this.wrapped.fetchFileList();
     return this.assetBundle.listFiles().filter((p) => p.startsWith("_plug/"))
       .map((p) => ({
         name: p,
@@ -20,7 +20,7 @@ export class AssetBundlePlugSpacePrimitives implements SpacePrimitives {
         lastModified: bootTime,
         perm: "ro",
         size: -1,
-      } as FileMeta)).concat(l);
+      } as FileMeta)).concat(files);
   }
 
   readFile(
@@ -31,7 +31,7 @@ export class AssetBundlePlugSpacePrimitives implements SpacePrimitives {
       const data = this.assetBundle.readFileSync(name);
       // console.log("Requested encoding", encoding);
       return Promise.resolve({
-        data: encoding === "string" ? new TextDecoder().decode(data) : data,
+        data: encoding === "utf8" ? new TextDecoder().decode(data) : data,
         meta: {
           lastModified: bootTime,
           size: data.byteLength,
@@ -60,7 +60,7 @@ export class AssetBundlePlugSpacePrimitives implements SpacePrimitives {
     name: string,
     encoding: FileEncoding,
     data: FileData,
-    selfUpdate?: boolean | undefined,
+    selfUpdate?: boolean,
   ): Promise<FileMeta> {
     return this.wrapped.writeFile(name, encoding, data, selfUpdate);
   }

common/spaces/disk_space_primitives.ts

@@ -15,6 +15,8 @@ function lookupContentType(path: string): string {
   return mime.getType(path) || "application/octet-stream";
 }
 
+const excludedFiles = ["data.db", "data.db-journal", "sync.json"];
+
 export class DiskSpacePrimitives implements SpacePrimitives {
   rootPath: string;
 
@@ -48,7 +50,7 @@ export class DiskSpacePrimitives implements SpacePrimitives {
       let data: FileData | null = null;
       const contentType = lookupContentType(name);
       switch (encoding) {
-        case "string":
+        case "utf8":
           data = await Deno.readTextFile(localPath);
           break;
         case "dataurl":
@@ -98,7 +100,7 @@ export class DiskSpacePrimitives implements SpacePrimitives {
 
       // Actually write the file
       switch (encoding) {
-        case "string":
+        case "utf8":
           await Deno.writeTextFile(`${localPath}`, data as string);
           break;
         case "dataurl":
@@ -165,8 +167,12 @@ export class DiskSpacePrimitives implements SpacePrimitives {
       const fullPath = file.path;
       try {
         const s = await Deno.stat(fullPath);
+        const name = fullPath.substring(this.rootPath.length + 1);
+        if (excludedFiles.includes(name)) {
+          continue;
+        }
         allFiles.push({
-          name: fullPath.substring(this.rootPath.length + 1),
+          name: name,
           lastModified: s.mtime!.getTime(),
           contentType: mime.getType(fullPath) || "application/octet-stream",
           size: s.size,

common/spaces/evented_space_primitives.ts

@@ -35,7 +35,7 @@ export class EventedSpacePrimitives implements SpacePrimitives {
     name: string,
     encoding: FileEncoding,
     data: FileData,
-    selfUpdate: boolean,
+    selfUpdate?: boolean,
   ): Promise<FileMeta> {
     const newMeta = await this.wrapped.writeFile(
       name,
@@ -48,7 +48,7 @@ export class EventedSpacePrimitives implements SpacePrimitives {
       const pageName = name.substring(0, name.length - 3);
       let text = "";
       switch (encoding) {
-        case "string":
+        case "utf8":
           text = data as string;
           break;
         case "arraybuffer":

common/spaces/file_meta_space_primitives.ts

@@ -12,10 +12,10 @@ export class FileMetaSpacePrimitives implements SpacePrimitives {
   }
 
   async fetchFileList(): Promise<FileMeta[]> {
-    const list = await this.wrapped.fetchFileList();
+    const files = await this.wrapped.fetchFileList();
     // Enrich the file list with custom meta data (for pages)
     const allFilesMap: Map<string, any> = new Map(
-      list.map((fm) => [fm.name, fm]),
+      files.map((fm) => [fm.name, fm]),
     );
     for (
       const { page, value } of await this.indexSyscalls["index.queryPrefix"](
@@ -53,7 +53,7 @@ export class FileMetaSpacePrimitives implements SpacePrimitives {
     name: string,
     encoding: FileEncoding,
     data: FileData,
-    selfUpdate?: boolean | undefined,
+    selfUpdate?: boolean,
   ): Promise<FileMeta> {
     return this.wrapped.writeFile(name, encoding, data, selfUpdate);
   }

common/spaces/http_space_primitives.ts

@@ -3,33 +3,51 @@ import { Plug } from "../../plugos/plug.ts";
 import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives.ts";
 import {
   base64DecodeDataUrl,
+  base64Encode,
   base64EncodedDataUrl,
 } from "../../plugos/asset_bundle/base64.ts";
 import { mime } from "../../plugos/deps.ts";
 
 export class HttpSpacePrimitives implements SpacePrimitives {
-  fsUrl: string;
+  private fsUrl: string;
   private plugUrl: string;
 
-  constructor(url: string) {
+  constructor(
+    url: string,
+    readonly user?: string,
+    readonly password?: string,
+    readonly base64Put?: boolean,
+  ) {
     this.fsUrl = url + "/fs";
     this.plugUrl = url + "/plug";
   }
 
   private async authenticatedFetch(
     url: string,
-    options: any,
+    options: Record<string, any>,
   ): Promise<Response> {
+    if (this.user && this.password) {
+      // Explicitly set an auth cookie
+      if (!options.headers) {
+        options.headers = {};
+      }
+      options.headers["cookie"] = `auth=${
+        btoa(`${this.user}:${this.password}`)
+      }`;
+    }
     const result = await fetch(url, options);
-    if (result.status === 401) {
+    if (result.status === 401 || result.redirected) {
       // Invalid credentials, reloading the browser should trigger authentication
-      location.reload();
+      if (typeof location !== "undefined") {
+        location.reload();
+      }
+
       throw Error("Unauthorized");
     }
     return result;
   }
 
-  public async fetchFileList(): Promise<FileMeta[]> {
+  async fetchFileList(): Promise<FileMeta[]> {
     const req = await this.authenticatedFetch(this.fsUrl, {
       method: "GET",
     });
@@ -41,9 +59,12 @@ export class HttpSpacePrimitives implements SpacePrimitives {
     name: string,
     encoding: FileEncoding,
   ): Promise<{ data: FileData; meta: FileMeta }> {
-    const res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
+    const res = await this.authenticatedFetch(
-      method: "GET",
+      `${this.fsUrl}/${encodeURI(name)}`,
-    });
+      {
+        method: "GET",
+      },
+    );
     if (res.status === 404) {
       throw new Error(`Page not found`);
     }
@@ -52,7 +73,6 @@ export class HttpSpacePrimitives implements SpacePrimitives {
       case "arraybuffer":
         {
           data = await res.arrayBuffer();
-          // data = await abBlob.arrayBuffer();
         }
         break;
       case "dataurl":
@@ -63,7 +83,7 @@ export class HttpSpacePrimitives implements SpacePrimitives {
           );
         }
         break;
-      case "string":
+      case "utf8":
         data = await res.text();
         break;
     }
@@ -82,37 +102,56 @@ export class HttpSpacePrimitives implements SpacePrimitives {
 
     switch (encoding) {
       case "arraybuffer":
-      case "string":
+        // actually we want an Uint8Array
+        body = data instanceof ArrayBuffer ? new Uint8Array(data) : data;
+        break;
+      case "utf8":
         body = data;
         break;
       case "dataurl":
         data = base64DecodeDataUrl(data as string);
         break;
     }
-    const res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
+    const headers: Record<string, string> = {
-      method: "PUT",
+      "Content-Type": "application/octet-stream",
-      headers: {
+    };
-        "Content-type": "application/octet-stream",
+    if (this.base64Put) {
+      headers["X-Content-Base64"] = "true";
+      headers["Content-Type"] = "text/plain";
+      body = base64Encode(body);
+    }
+
+    const res = await this.authenticatedFetch(
+      `${this.fsUrl}/${encodeURI(name)}`,
+      {
+        method: "PUT",
+        headers,
+        body,
       },
-      body,
+    );
-    });
     const newMeta = this.responseToMeta(name, res);
     return newMeta;
   }
 
   async deleteFile(name: string): Promise<void> {
-    const req = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
+    const req = await this.authenticatedFetch(
-      method: "DELETE",
+      `${this.fsUrl}/${encodeURI(name)}`,
-    });
+      {
+        method: "DELETE",
+      },
+    );
     if (req.status !== 200) {
       throw Error(`Failed to delete file: ${req.statusText}`);
     }
   }
 
   async getFileMeta(name: string): Promise<FileMeta> {
-    const res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
+    const res = await this.authenticatedFetch(
-      method: "OPTIONS",
+      `${this.fsUrl}/${encodeURI(name)}`,
-    });
+      {
+        method: "OPTIONS",
+      },
+    );
     if (res.status === 404) {
       throw new Error(`File not found`);
     }

common/spaces/plug_space_primitives.ts

@@ -5,7 +5,10 @@ import {
   SpacePrimitives,
 } from "../../common/spaces/space_primitives.ts";
 import { FileMeta } from "../../common/types.ts";
-import { NamespaceOperation, PageNamespaceHook } from "./page_namespace.ts";
+import {
+  NamespaceOperation,
+  PageNamespaceHook,
+} from "../hooks/page_namespace.ts";
 import { base64DecodeDataUrl } from "../../plugos/asset_bundle/base64.ts";
 
 export class PlugSpacePrimitives implements SpacePrimitives {
@@ -46,8 +49,8 @@ export class PlugSpacePrimitives implements SpacePrimitives {
         }
       }
     }
-    const result = await this.wrapped.fetchFileList();
+    const files = await this.wrapped.fetchFileList();
-    for (const pm of result) {
+    for (const pm of files) {
       allFiles.push(pm);
     }
     return allFiles;

common/spaces/space.ts

@@ -1,9 +1,13 @@
 import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives.ts";
-import { AttachmentMeta, FileMeta, PageMeta } from "../types.ts";
+import { AttachmentMeta, PageMeta } from "../types.ts";
 import { EventEmitter } from "../../plugos/event.ts";
 import { Plug } from "../../plugos/plug.ts";
 import { plugPrefix } from "./constants.ts";
 import { safeRun } from "../util.ts";
+import {
+  FileMeta,
+  ProxyFileSystem,
+} from "../../plug-api/plugos-syscall/types.ts";
 
 const pageWatchInterval = 2000;
 
@@ -14,16 +18,42 @@ export type SpaceEvents = {
   pageListUpdated: (pages: Set<PageMeta>) => void;
 };
 
-export class Space extends EventEmitter<SpaceEvents> {
+export class Space extends EventEmitter<SpaceEvents>
+  implements ProxyFileSystem {
   pageMetaCache = new Map<string, PageMeta>();
   watchedPages = new Set<string>();
   private initialPageListLoad = true;
   private saving = false;
 
-  constructor(private space: SpacePrimitives) {
+  constructor(readonly spacePrimitives: SpacePrimitives) {
     super();
   }
 
+  // Filesystem interface implementation
+  async readFile(path: string, encoding: "dataurl" | "utf8"): Promise<string> {
+    return (await this.spacePrimitives.readFile(path, encoding)).data as string;
+  }
+  getFileMeta(path: string): Promise<FileMeta> {
+    return this.spacePrimitives.getFileMeta(path);
+  }
+  writeFile(
+    path: string,
+    text: string,
+    encoding: "dataurl" | "utf8",
+  ): Promise<FileMeta> {
+    return this.spacePrimitives.writeFile(path, encoding, text);
+  }
+  deleteFile(path: string): Promise<void> {
+    return this.spacePrimitives.deleteFile(path);
+  }
+  async listFiles(path: string): Promise<FileMeta[]> {
+    return (await this.spacePrimitives.fetchFileList()).filter((f) =>
+      f.name.startsWith(path)
+    );
+  }
+
+  // The more domain-specific methods
+
   public async updatePageList() {
     const newPageList = await this.fetchPageList();
     const deletedPages = new Set<string>(this.pageMetaCache.keys());
@@ -81,7 +111,7 @@ export class Space extends EventEmitter<SpaceEvents> {
 
   async deletePage(name: string): Promise<void> {
     await this.getPageMeta(name); // Check if page exists, if not throws Error
-    await this.space.deleteFile(`${name}.md`);
+    await this.spacePrimitives.deleteFile(`${name}.md`);
 
     this.pageMetaCache.delete(name);
     this.emit("pageDeleted", name);
@@ -91,7 +121,7 @@ export class Space extends EventEmitter<SpaceEvents> {
   async getPageMeta(name: string): Promise<PageMeta> {
     const oldMeta = this.pageMetaCache.get(name);
     const newMeta = fileMetaToPageMeta(
-      await this.space.getFileMeta(`${name}.md`),
+      await this.spacePrimitives.getFileMeta(`${name}.md`),
     );
     if (oldMeta) {
       if (oldMeta.lastModified !== newMeta.lastModified) {
@@ -108,7 +138,7 @@ export class Space extends EventEmitter<SpaceEvents> {
     name: string,
     args: any[],
   ): Promise<any> {
-    return this.space.invokeFunction(plug, env, name, args);
+    return this.spacePrimitives.invokeFunction(plug, env, name, args);
   }
 
   listPages(): Set<PageMeta> {
@@ -116,18 +146,21 @@ export class Space extends EventEmitter<SpaceEvents> {
   }
 
   async listPlugs(): Promise<string[]> {
-    const allFiles = await this.space.fetchFileList();
+    const files = await this.spacePrimitives.fetchFileList();
-    return allFiles
+    return files
       .filter((fileMeta) => fileMeta.name.endsWith(".plug.json"))
       .map((fileMeta) => fileMeta.name);
   }
 
   proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
-    return this.space.proxySyscall(plug, name, args);
+    return this.spacePrimitives.proxySyscall(plug, name, args);
   }
 
   async readPage(name: string): Promise<{ text: string; meta: PageMeta }> {
-    const pageData = await this.space.readFile(`${name}.md`, "string");
+    const pageData = await this.spacePrimitives.readFile(
+      `${name}.md`,
+      "utf8",
+    );
     const previousMeta = this.pageMetaCache.get(name);
     const newMeta = fileMetaToPageMeta(pageData.meta);
     if (previousMeta) {
@@ -159,7 +192,12 @@ export class Space extends EventEmitter<SpaceEvents> {
     try {
       this.saving = true;
       const pageMeta = fileMetaToPageMeta(
-        await this.space.writeFile(`${name}.md`, "string", text, selfUpdate),
+        await this.spacePrimitives.writeFile(
+          `${name}.md`,
+          "utf8",
+          text,
+          selfUpdate,
+        ),
       );
       if (!selfUpdate) {
         this.emit("pageChanged", pageMeta);
@@ -171,13 +209,13 @@ export class Space extends EventEmitter<SpaceEvents> {
   }
 
   async fetchPageList(): Promise<PageMeta[]> {
-    return (await this.space.fetchFileList())
+    return (await this.spacePrimitives.fetchFileList())
       .filter((fileMeta) => fileMeta.name.endsWith(".md"))
       .map(fileMetaToPageMeta);
   }
 
   async fetchAttachmentList(): Promise<AttachmentMeta[]> {
-    return (await this.space.fetchFileList()).filter(
+    return (await this.spacePrimitives.fetchFileList()).filter(
       (fileMeta) =>
         !fileMeta.name.endsWith(".md") &&
         !fileMeta.name.endsWith(".plug.json") &&
@@ -195,11 +233,11 @@ export class Space extends EventEmitter<SpaceEvents> {
     name: string,
     encoding: FileEncoding,
   ): Promise<{ data: FileData; meta: AttachmentMeta }> {
-    return this.space.readFile(name, encoding);
+    return this.spacePrimitives.readFile(name, encoding);
   }
 
   getAttachmentMeta(name: string): Promise<AttachmentMeta> {
-    return this.space.getFileMeta(name);
+    return this.spacePrimitives.getFileMeta(name);
   }
 
   writeAttachment(
@@ -208,11 +246,11 @@ export class Space extends EventEmitter<SpaceEvents> {
     data: FileData,
     selfUpdate?: boolean | undefined,
   ): Promise<AttachmentMeta> {
-    return this.space.writeFile(name, encoding, data, selfUpdate);
+    return this.spacePrimitives.writeFile(name, encoding, data, selfUpdate);
   }
 
   deleteAttachment(name: string): Promise<void> {
-    return this.space.deleteFile(name);
+    return this.spacePrimitives.deleteFile(name);
   }
 
   private metaCacher(name: string, meta: PageMeta): PageMeta {

common/spaces/space_primitives.ts

@@ -1,10 +1,10 @@
 import { Plug } from "../../plugos/plug.ts";
 import { FileMeta } from "../types.ts";
 
-export type FileEncoding = "string" | "arraybuffer" | "dataurl";
+export type FileEncoding = "utf8" | "arraybuffer" | "dataurl";
 export type FileData = ArrayBuffer | string;
 export interface SpacePrimitives {
-  // Pages
+  // Returns a list of file meta data as well as the timestamp of this snapshot
   fetchFileList(): Promise<FileMeta[]>;
   readFile(
     name: string,
@@ -15,6 +15,7 @@ export interface SpacePrimitives {
     name: string,
     encoding: FileEncoding,
     data: FileData,
+    // Used to decide whether or not to emit change events
     selfUpdate?: boolean,
   ): Promise<FileMeta>;
   deleteFile(name: string): Promise<void>;

common/spaces/sync.test.ts

@@ -0,0 +1,143 @@
+import { SpaceSync, SyncStatusItem } from "./sync.ts";
+import { DiskSpacePrimitives } from "./disk_space_primitives.ts";
+import { assertEquals } from "../../test_deps.ts";
+
+Deno.test("Test store", async () => {
+  const primaryPath = await Deno.makeTempDir();
+  const secondaryPath = await Deno.makeTempDir();
+  console.log("Primary", primaryPath);
+  console.log("Secondary", secondaryPath);
+  const primary = new DiskSpacePrimitives(primaryPath);
+  const secondary = new DiskSpacePrimitives(secondaryPath);
+  const statusMap = new Map<string, SyncStatusItem>();
+  const sync = new SpaceSync(primary, secondary, statusMap);
+
+  // Write one page to primary
+  await primary.writeFile("index", "utf8", "Hello");
+  assertEquals((await secondary.fetchFileList()).length, 0);
+  console.log("Initial sync ops", await doSync());
+
+  assertEquals((await secondary.fetchFileList()).length, 1);
+  assertEquals((await secondary.readFile("index", "utf8")).data, "Hello");
+
+  // Should be a no-op
+  assertEquals(await doSync(), 0);
+
+  // Now let's make a change on the secondary
+  await secondary.writeFile("index", "utf8", "Hello!!");
+  await secondary.writeFile("test", "utf8", "Test page");
+
+  // And sync it
+  await doSync();
+
+  assertEquals((await primary.fetchFileList()).length, 2);
+  assertEquals((await secondary.fetchFileList()).length, 2);
+
+  assertEquals((await primary.readFile("index", "utf8")).data, "Hello!!");
+
+  // Let's make some random edits on both ends
+  await primary.writeFile("index", "utf8", "1");
+  await primary.writeFile("index2", "utf8", "2");
+  await secondary.writeFile("index3", "utf8", "3");
+  await secondary.writeFile("index4", "utf8", "4");
+  await doSync();
+
+  assertEquals((await primary.fetchFileList()).length, 5);
+  assertEquals((await secondary.fetchFileList()).length, 5);
+
+  assertEquals(await doSync(), 0);
+
+  console.log("Deleting pages");
+  // Delete some pages
+  await primary.deleteFile("index");
+  await primary.deleteFile("index3");
+
+  await doSync();
+
+  assertEquals((await primary.fetchFileList()).length, 3);
+  assertEquals((await secondary.fetchFileList()).length, 3);
+
+  // No-op
+  assertEquals(await doSync(), 0);
+
+  await secondary.deleteFile("index4");
+  await primary.deleteFile("index2");
+
+  await doSync();
+
+  // Just "test" left
+  assertEquals((await primary.fetchFileList()).length, 1);
+  assertEquals((await secondary.fetchFileList()).length, 1);
+
+  // No-op
+  assertEquals(await doSync(), 0);
+
+  await secondary.writeFile("index", "utf8", "I'm back");
+
+  await doSync();
+
+  assertEquals((await primary.readFile("index", "utf8")).data, "I'm back");
+
+  // Cause a conflict
+  console.log("Introducing a conflict now");
+  await primary.writeFile("index", "utf8", "Hello 1");
+  await secondary.writeFile("index", "utf8", "Hello 2");
+
+  await doSync();
+
+  // Sync conflicting copy back
+  await doSync();
+
+  // Verify that primary won
+  assertEquals((await primary.readFile("index", "utf8")).data, "Hello 1");
+  assertEquals((await secondary.readFile("index", "utf8")).data, "Hello 1");
+
+  // test + index + index.conflicting copy
+  assertEquals((await primary.fetchFileList()).length, 3);
+  assertEquals((await secondary.fetchFileList()).length, 3);
+
+  // Introducing a fake conflict (same content, so not really conflicting)
+  await primary.writeFile("index", "utf8", "Hello 1");
+  await secondary.writeFile("index", "utf8", "Hello 1");
+
+  await doSync();
+  await doSync();
+
+  // test + index + previous index.conflicting copy but nothing more
+  assertEquals((await primary.fetchFileList()).length, 3);
+
+  console.log("Bringing a third device in the mix");
+
+  const ternaryPath = await Deno.makeTempDir();
+
+  console.log("Ternary", ternaryPath);
+
+  const ternary = new DiskSpacePrimitives(ternaryPath);
+  const sync2 = new SpaceSync(
+    secondary,
+    ternary,
+    new Map<string, SyncStatusItem>(),
+  );
+  console.log("N ops", await sync2.syncFiles());
+  await sleep(2);
+  assertEquals(await sync2.syncFiles(), 0);
+
+  await Deno.remove(primaryPath, { recursive: true });
+  await Deno.remove(secondaryPath, { recursive: true });
+  await Deno.remove(ternaryPath, { recursive: true });
+
+  async function doSync() {
+    await sleep();
+    const r = await sync.syncFiles(
+      SpaceSync.primaryConflictResolver,
+    );
+    await sleep();
+    return r;
+  }
+});
+
+function sleep(ms = 10): Promise<void> {
+  return new Promise((resolve) => {
+    setTimeout(resolve, ms);
+  });
+}

common/spaces/sync.ts

@@ -0,0 +1,271 @@
+import type { FileMeta } from "../types.ts";
+import { SpacePrimitives } from "./space_primitives.ts";
+
+type SyncHash = number;
+
+// Tuple where the first value represents a lastModified timestamp for the primary space
+// and the second item the lastModified value of the secondary space
+export type SyncStatusItem = [SyncHash, SyncHash];
+
+// Implementation of this algorithm https://unterwaditzer.net/2016/sync-algorithm.html
+export class SpaceSync {
+  constructor(
+    private primary: SpacePrimitives,
+    private secondary: SpacePrimitives,
+    readonly snapshot: Map<string, SyncStatusItem>,
+  ) {}
+
+  async syncFiles(
+    conflictResolver?: (
+      name: string,
+      snapshot: Map<string, SyncStatusItem>,
+      primarySpace: SpacePrimitives,
+      secondarySpace: SpacePrimitives,
+    ) => Promise<void>,
+  ): Promise<number> {
+    let operations = 0;
+    console.log("Fetching snapshot from primary");
+    const primaryAllPages = this.syncCandidates(
+      await this.primary.fetchFileList(),
+    );
+
+    console.log("Fetching snapshot from secondary");
+    try {
+      const secondaryAllPages = this.syncCandidates(
+        await this.secondary.fetchFileList(),
+      );
+
+      const primaryFileMap = new Map<string, SyncHash>(
+        primaryAllPages.map((m) => [m.name, m.lastModified]),
+      );
+      const secondaryFileMap = new Map<string, SyncHash>(
+        secondaryAllPages.map((m) => [m.name, m.lastModified]),
+      );
+
+      const allFilesToProcess = new Set([
+        ...this.snapshot.keys(),
+        ...primaryFileMap.keys(),
+        ...secondaryFileMap.keys(),
+      ]);
+
+      console.log("Iterating over all files");
+      for (const name of allFilesToProcess) {
+        if (
+          primaryFileMap.has(name) && !secondaryFileMap.has(name) &&
+          !this.snapshot.has(name)
+        ) {
+          // New file, created on primary, copy from primary to secondary
+          console.log(
+            "New file created on primary, copying to secondary",
+            name,
+          );
+          const { data } = await this.primary.readFile(name, "arraybuffer");
+          const writtenMeta = await this.secondary.writeFile(
+            name,
+            "arraybuffer",
+            data,
+          );
+          this.snapshot.set(name, [
+            primaryFileMap.get(name)!,
+            writtenMeta.lastModified,
+          ]);
+          operations++;
+        } else if (
+          secondaryFileMap.has(name) && !primaryFileMap.has(name) &&
+          !this.snapshot.has(name)
+        ) {
+          // New file, created on secondary, copy from secondary to primary
+          console.log(
+            "New file created on secondary, copying from secondary to primary",
+            name,
+          );
+          const { data } = await this.secondary.readFile(name, "arraybuffer");
+          const writtenMeta = await this.primary.writeFile(
+            name,
+            "arraybuffer",
+            data,
+          );
+          this.snapshot.set(name, [
+            writtenMeta.lastModified,
+            secondaryFileMap.get(name)!,
+          ]);
+          operations++;
+        } else if (
+          primaryFileMap.has(name) && this.snapshot.has(name) &&
+          !secondaryFileMap.has(name)
+        ) {
+          // File deleted on B
+          console.log("File deleted on secondary, deleting from primary", name);
+          await this.primary.deleteFile(name);
+          this.snapshot.delete(name);
+          operations++;
+        } else if (
+          secondaryFileMap.has(name) && this.snapshot.has(name) &&
+          !primaryFileMap.has(name)
+        ) {
+          // File deleted on A
+          console.log("File deleted on primary, deleting from secondary", name);
+          await this.secondary.deleteFile(name);
+          this.snapshot.delete(name);
+          operations++;
+        } else if (
+          this.snapshot.has(name) && !primaryFileMap.has(name) &&
+          !secondaryFileMap.has(name)
+        ) {
+          // File deleted on both sides, :shrug:
+          console.log("File deleted on both ends, deleting from status", name);
+          this.snapshot.delete(name);
+          operations++;
+        } else if (
+          primaryFileMap.has(name) && secondaryFileMap.has(name) &&
+          this.snapshot.get(name) &&
+          primaryFileMap.get(name) !== this.snapshot.get(name)![0] &&
+          secondaryFileMap.get(name) === this.snapshot.get(name)![1]
+        ) {
+          // File has changed on primary, but not secondary: copy from primary to secondary
+          console.log("File changed on primary, copying to secondary", name);
+          const { data } = await this.primary.readFile(name, "arraybuffer");
+          const writtenMeta = await this.secondary.writeFile(
+            name,
+            "arraybuffer",
+            data,
+          );
+          this.snapshot.set(name, [
+            primaryFileMap.get(name)!,
+            writtenMeta.lastModified,
+          ]);
+          operations++;
+        } else if (
+          primaryFileMap.has(name) && secondaryFileMap.has(name) &&
+          this.snapshot.get(name) &&
+          secondaryFileMap.get(name) !== this.snapshot.get(name)![1] &&
+          primaryFileMap.get(name) === this.snapshot.get(name)![0]
+        ) {
+          // File has changed on secondary, but not primary: copy from secondary to primary
+          const { data } = await this.secondary.readFile(name, "arraybuffer");
+          const writtenMeta = await this.primary.writeFile(
+            name,
+            "arraybuffer",
+            data,
+          );
+          this.snapshot.set(name, [
+            writtenMeta.lastModified,
+            secondaryFileMap.get(name)!,
+          ]);
+          operations++;
+        } else if (
+          ( // File changed on both ends, but we don't have any info in the snapshot (resync scenario?): have to run through conflict handling
+            primaryFileMap.has(name) && secondaryFileMap.has(name) &&
+            !this.snapshot.has(name)
+          ) ||
+          ( // File changed on both ends, CONFLICT!
+            primaryFileMap.has(name) && secondaryFileMap.has(name) &&
+            this.snapshot.get(name) &&
+            secondaryFileMap.get(name) !== this.snapshot.get(name)![1] &&
+            primaryFileMap.get(name) !== this.snapshot.get(name)![0]
+          )
+        ) {
+          console.log("File changed on both ends, conflict!", name);
+          if (conflictResolver) {
+            await conflictResolver(
+              name,
+              this.snapshot,
+              this.primary,
+              this.secondary,
+            );
+          } else {
+            throw Error(
+              `Sync conflict for ${name} with no conflict resolver specified`,
+            );
+          }
+          operations++;
+        } else {
+          // Nothing needs to happen
+        }
+      }
+    } catch (e: any) {
+      console.error("Boom", e.message);
+      throw e;
+    }
+
+    return operations;
+  }
+
+  // Strategy: Primary wins
+  public static async primaryConflictResolver(
+    name: string,
+    snapshot: Map<string, SyncStatusItem>,
+    primary: SpacePrimitives,
+    secondary: SpacePrimitives,
+  ): Promise<void> {
+    console.log("Hit a conflict for", name);
+    const filePieces = name.split(".");
+    const fileNameBase = filePieces.slice(0, -1).join(".");
+    const fileNameExt = filePieces[filePieces.length - 1];
+    const pageData1 = await primary.readFile(name, "arraybuffer");
+    const pageData2 = await secondary.readFile(name, "arraybuffer");
+
+    let byteWiseMatch = true;
+    const arrayBuffer1 = new Uint8Array(pageData1.data as ArrayBuffer);
+    const arrayBuffer2 = new Uint8Array(pageData2.data as ArrayBuffer);
+    if (arrayBuffer1.byteLength !== arrayBuffer2.byteLength) {
+      byteWiseMatch = false;
+    }
+    if (byteWiseMatch) {
+      // Byte-wise comparison
+      for (let i = 0; i < arrayBuffer1.byteLength; i++) {
+        if (arrayBuffer1[i] !== arrayBuffer2[i]) {
+          byteWiseMatch = false;
+          break;
+        }
+      }
+      // Byte wise they're still the same, so no confict
+      if (byteWiseMatch) {
+        snapshot.set(name, [
+          pageData1.meta.lastModified,
+          pageData2.meta.lastModified,
+        ]);
+        return;
+      }
+    }
+    const revisionFileName = filePieces.length === 1
+      ? `${name}.conflicted.${pageData2.meta.lastModified}`
+      : `${fileNameBase}.conflicted.${pageData2.meta.lastModified}.${fileNameExt}`;
+    console.log(
+      "Going to create conflicting copy",
+      revisionFileName,
+    );
+
+    // Copy secondary to conflict copy
+    const localConflictMeta = await primary.writeFile(
+      revisionFileName,
+      "arraybuffer",
+      pageData2.data,
+    );
+    const remoteConflictMeta = await secondary.writeFile(
+      revisionFileName,
+      "arraybuffer",
+      pageData2.data,
+    );
+
+    // Updating snapshot
+    snapshot.set(revisionFileName, [
+      localConflictMeta.lastModified,
+      remoteConflictMeta.lastModified,
+    ]);
+
+    // Write replacement on top
+    const writeMeta = await secondary.writeFile(
+      name,
+      "arraybuffer",
+      pageData1.data,
+      true,
+    );
+
+    snapshot.set(name, [pageData1.meta.lastModified, writeMeta.lastModified]);
+  }
+
+  syncCandidates(files: FileMeta[]): FileMeta[] {
+    return files.filter((f) => !f.name.startsWith("_plug/"));
+  }
+}

common/syscalls/space.ts

@@ -6,6 +6,8 @@ import {
   FileEncoding,
 } from "../../common/spaces/space_primitives.ts";
 
+import { FileMeta as PlugFileMeta } from "../../plug-api/plugos-syscall/types.ts";
+
 export default (space: Space): SysCallMapping => {
   return {
     "space.listPages": (): PageMeta[] => {
@@ -59,5 +61,9 @@ export default (space: Space): SysCallMapping => {
     "space.deleteAttachment": async (_ctx, name: string) => {
       await space.deleteAttachment(name);
     },
+
+    "space.listFiles": (_ctx, path: string): Promise<PlugFileMeta[]> => {
+      return space.listFiles(path);
+    },
   };
 };

common/syscalls/sync.ts

@@ -0,0 +1,65 @@
+import { SysCallMapping } from "../../plugos/system.ts";
+import type { SyncEndpoint } from "../../plug-api/silverbullet-syscall/sync.ts";
+import { SpaceSync, SyncStatusItem } from "../spaces/sync.ts";
+import { HttpSpacePrimitives } from "../spaces/http_space_primitives.ts";
+import { SpacePrimitives } from "../spaces/space_primitives.ts";
+
+export function syncSyscalls(localSpace: SpacePrimitives): SysCallMapping {
+  return {
+    "sync.sync": async (
+      _ctx,
+      endpoint: SyncEndpoint,
+      snapshot: Record<string, SyncStatusItem>,
+    ): Promise<
+      {
+        snapshot: Record<string, SyncStatusItem>;
+        operations: number;
+        // The reason to not just throw an Error is so that the partially updated snapshot can still be saved
+        error?: string;
+      }
+    > => {
+      const syncSpace = new HttpSpacePrimitives(
+        endpoint.url,
+        endpoint.user,
+        endpoint.password,
+        // Base64 PUTs to support mobile
+        true,
+      );
+      // Convert from JSON to a Map
+      const syncStatusMap = new Map<string, SyncStatusItem>(
+        Object.entries(snapshot),
+      );
+      const spaceSync = new SpaceSync(
+        localSpace,
+        syncSpace,
+        syncStatusMap,
+      );
+
+      try {
+        const operations = await spaceSync.syncFiles(
+          SpaceSync.primaryConflictResolver,
+        );
+        return {
+          // And convert back to JSON
+          snapshot: Object.fromEntries(spaceSync.snapshot),
+          operations,
+        };
+      } catch (e: any) {
+        return {
+          snapshot: Object.fromEntries(spaceSync.snapshot),
+          operations: -1,
+          error: e.message,
+        };
+      }
+    },
+    "sync.check": async (_ctx, endpoint: SyncEndpoint): Promise<void> => {
+      const syncSpace = new HttpSpacePrimitives(
+        endpoint.url,
+        endpoint.user,
+        endpoint.password,
+      );
+      // Let's just fetch the file list to see if it works
+      await syncSpace.fetchFileList();
+    },
+  };
+}

common/util.ts

@@ -1,6 +1,7 @@
 import { SETTINGS_TEMPLATE } from "./settings_template.ts";
 import { YAML } from "./deps.ts";
 import { Space } from "./spaces/space.ts";
+import { BuiltinSettings } from "../web/types.ts";
 
 export function safeRun(fn: () => Promise<void>) {
   fn().catch((e) => {
@@ -45,7 +46,15 @@ export function parseYamlSettings(settingsMarkdown: string): {
   }
 }
 
-export async function ensureAndLoadSettings(space: Space) {
+export async function ensureAndLoadSettings(
+  space: Space,
+  dontCreate: boolean,
+): Promise<any> {
+  if (dontCreate) {
+    return {
+      indexPage: "index",
+    };
+  }
   try {
     await space.getPageMeta("SETTINGS");
   } catch {

deno.jsonc

@@ -3,9 +3,9 @@
     "clean": "rm -rf dist dist_bundle",
     "install": "deno install -f -A --unstable silverbullet.ts",
     "test": "deno test -A --unstable",
-    "build": "deno run -A --unstable --check build_plugs.ts && deno run -A --unstable --check build.ts",
+    "build": "deno run -A --unstable --check build_plugs.ts && deno run -A --unstable --check build_web.ts",
     "plugs": "deno run -A --unstable --check build_plugs.ts",
-    "watch-web": "deno run -A --unstable --check build.ts --watch",
+    "watch-web": "deno run -A --unstable --check build_web.ts --watch",
     "watch-mobile": "deno run -A --unstable --check build_mobile.ts --watch",
     "watch-server": "deno run -A --unstable --check --watch silverbullet.ts",
     // The only reason to run a shell script is that deno task doesn't support globs yet (e.g. *.plug.yaml)
@@ -22,7 +22,7 @@
     "desktop:build": "deno task build && deno task bundle && cd desktop && npm run make",
     // Mobile
     "mobile:deps": "cd mobile && npm install && npx cap sync",
-    "mobile:build": "deno run -A --unstable --check build_mobile.ts && cd mobile && npx cap copy && npx cap open ios"
+    "mobile:build": "deno task clean && deno task plugs && deno run -A --unstable --check build_mobile.ts && cd mobile && npx cap copy && npx cap open ios"
   },
 
   "compilerOptions": {

desktop/src/menu.ts

@@ -155,7 +155,18 @@ const template: MenuItemConstructorOptions[] = [
 
 if (process.platform === "darwin") {
   const name = app.getName();
-  template.unshift({ label: name, submenu: [] });
+  template.unshift({
+    label: name,
+    submenu: [
+      { role: "services" },
+      { type: "separator" },
+      { role: "hide" },
+      { role: "hideOthers" },
+      { role: "unhide" },
+      { type: "separator" },
+      { role: "quit" },
+    ],
+  });
 }
 
 export const menu = Menu.buildFromTemplate(template);

mobile/boot.ts

@@ -1,8 +1,8 @@
 import { Editor } from "../web/editor.tsx";
 import { ensureAndLoadSettings, safeRun } from "../common/util.ts";
 import { Space } from "../common/spaces/space.ts";
-import { PlugSpacePrimitives } from "../server/hooks/plug_space_primitives.ts";
+import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts";
-import { PageNamespaceHook } from "../server/hooks/page_namespace.ts";
+import { PageNamespaceHook } from "../common/hooks/page_namespace.ts";
 import { SilverBulletHooks } from "../common/manifest.ts";
 import { System } from "../plugos/system.ts";
 import { BuiltinSettings } from "../web/types.ts";
@@ -75,7 +75,10 @@ safeRun(async () => {
   const serverSpace = new Space(spacePrimitives);
   serverSpace.watch();
 
-  const settings = await ensureAndLoadSettings(serverSpace) as BuiltinSettings;
+  const settings = await ensureAndLoadSettings(
+    serverSpace,
+    false,
+  ) as BuiltinSettings;
 
   // Register some mobile-specific syscall implementations
   system.registerSyscalls(

mobile/spaces/capacitor_space_primitives.ts

@@ -13,7 +13,10 @@ import { Directory, Encoding, Filesystem } from "../deps.ts";
 import { mime } from "../../plugos/deps.ts";
 
 export class CapacitorSpacePrimitives implements SpacePrimitives {
-  constructor(readonly source: Directory, readonly root: string) {
+  constructor(
+    readonly source: Directory,
+    readonly root: string,
+  ) {
   }
 
   async fetchFileList(): Promise<FileMeta[]> {
@@ -28,8 +31,9 @@ export class CapacitorSpacePrimitives implements SpacePrimitives {
       });
       for (const file of files.files) {
         if (file.type === "file") {
+          const name = `${dir}/${file.name}`.substring(1);
           allFiles.push({
-            name: `${dir}/${file.name}`.substring(1),
+            name: name,
             lastModified: file.mtime,
             perm: "rw",
             contentType: mime.getType(file.name) || "application/octet-stream",
@@ -41,7 +45,6 @@ export class CapacitorSpacePrimitives implements SpacePrimitives {
       }
     }
     await readAllFiles("");
-    console.log("allFiles", allFiles);
     return allFiles;
   }
   async readFile(
@@ -51,7 +54,7 @@ export class CapacitorSpacePrimitives implements SpacePrimitives {
     let data: FileData | undefined;
     try {
       switch (encoding) {
-        case "string":
+        case "utf8":
           data = (await Filesystem.readFile({
             path: this.root + name,
             directory: this.source,
@@ -109,7 +112,7 @@ export class CapacitorSpacePrimitives implements SpacePrimitives {
     data: FileData,
   ): Promise<FileMeta> {
     switch (encoding) {
-      case "string":
+      case "utf8":
         await Filesystem.writeFile({
           path: this.root + name,
           directory: this.source,

plug-api/plugos-syscall/fs.ts

@@ -1,36 +1,48 @@
 import { syscall } from "./syscall.ts";
+import type { FileMeta, ProxyFileSystem } from "./types.ts";
 
-export type FileMeta = {
+export class LocalFileSystem implements ProxyFileSystem {
-  name: string;
+  constructor(readonly root: string) {
-  lastModified: number;
+  }
-};
 
-export function readFile(
+  readFile(
-  path: string,
+    path: string,
-  encoding: "utf8" | "dataurl" = "utf8",
+    encoding: "utf8" | "dataurl" = "utf8",
-): Promise<string> {
+  ): Promise<string> {
-  return syscall("fs.readFile", path, encoding);
+    return syscall("fs.readFile", `${this.root}/${path}`, encoding);
-}
+  }
-
+
-export function getFileMeta(path: string): Promise<FileMeta> {
+  async getFileMeta(path: string): Promise<FileMeta> {
-  return syscall("fs.getFileMeta", path);
+    return this.removeRootDir(
-}
+      await syscall("fs.getFileMeta", `${this.root}/${path}`),
-
+    );
-export function writeFile(
+  }
-  path: string,
+
-  text: string,
+  writeFile(
-  encoding: "utf8" | "dataurl" = "utf8",
+    path: string,
-): Promise<FileMeta> {
+    text: string,
-  return syscall("fs.writeFile", path, text, encoding);
+    encoding: "utf8" | "dataurl" = "utf8",
-}
+  ): Promise<FileMeta> {
-
+    return syscall("fs.writeFile", `${this.root}/${path}`, text, encoding);
-export function deleteFile(path: string): Promise<void> {
+  }
-  return syscall("fs.deleteFile", path);
+
-}
+  deleteFile(path: string): Promise<void> {
-
+    return syscall("fs.deleteFile", `${this.root}/${path}`);
-export function listFiles(
+  }
-  dirName: string,
+
-  recursive = false,
+  async listFiles(
-): Promise<FileMeta[]> {
+    dirName: string,
-  return syscall("fs.listFiles", dirName, recursive);
+    recursive = false,
+  ): Promise<FileMeta[]> {
+    return (await syscall(
+      "fs.listFiles",
+      `${this.root}/${dirName}`,
+      recursive,
+    )).map(this.removeRootDir.bind(this));
+  }
+
+  private removeRootDir(fileMeta: FileMeta): FileMeta {
+    fileMeta.name = fileMeta.name.substring(this.root.length + 1);
+    return fileMeta;
+  }
 }

plug-api/plugos-syscall/mod.ts

@@ -1,6 +1,7 @@
 export * as asset from "./asset.ts";
 export * as events from "./event.ts";
-export * as fs from "./fs.ts";
+// export * as fs from "./fs.ts";
+export { LocalFileSystem } from "./fs.ts";
 export * as sandbox from "./sandbox.ts";
 export * as fulltext from "./fulltext.ts";
 export * as shell from "./shell.ts";

plug-api/plugos-syscall/types.ts

@@ -0,0 +1,25 @@
+export type FileMeta = {
+  name: string;
+  lastModified: number;
+};
+
+export interface ProxyFileSystem {
+  readFile(
+    path: string,
+    encoding: "utf8" | "dataurl",
+  ): Promise<string>;
+
+  getFileMeta(path: string): Promise<FileMeta>;
+
+  writeFile(
+    path: string,
+    text: string,
+    encoding: "utf8" | "dataurl",
+  ): Promise<FileMeta>;
+
+  deleteFile(path: string): Promise<void>;
+
+  listFiles(
+    path: string,
+  ): Promise<FileMeta[]>;
+}

plug-api/silverbullet-syscall/mod.ts

@@ -3,6 +3,7 @@ export * as editor from "./editor.ts";
 export * as index from "./index.ts";
 export * as markdown from "./markdown.ts";
 export * as sandbox from "./sandbox.ts";
-export * as space from "./space.ts";
+export { default as space } from "./space.ts";
 export * as system from "./system.ts";
 export * as collab from "./collab.ts";
+export * as sync from "./sync.ts";

plug-api/silverbullet-syscall/space.ts

@@ -1,70 +1,98 @@
 import { syscall } from "./syscall.ts";
 import { AttachmentMeta, PageMeta } from "../../common/types.ts";
+import { FileMeta, ProxyFileSystem } from "../plugos-syscall/types.ts";
 
-export function listPages(unfiltered = false): Promise<PageMeta[]> {
+export class SpaceFileSystem implements ProxyFileSystem {
-  return syscall("space.listPages", unfiltered);
+  // More space-specific methods
+
+  listPages(unfiltered = false): Promise<PageMeta[]> {
+    return syscall("space.listPages", unfiltered);
+  }
+
+  getPageMeta(name: string): Promise<PageMeta> {
+    return syscall("space.getPageMeta", name);
+  }
+
+  readPage(
+    name: string,
+  ): Promise<string> {
+    return syscall("space.readPage", name);
+  }
+
+  writePage(name: string, text: string): Promise<PageMeta> {
+    return syscall("space.writePage", name, text);
+  }
+
+  deletePage(name: string): Promise<void> {
+    return syscall("space.deletePage", name);
+  }
+
+  listPlugs(): Promise<string[]> {
+    return syscall("space.listPlugs");
+  }
+
+  listAttachments(): Promise<PageMeta[]> {
+    return syscall("space.listAttachments");
+  }
+
+  getAttachmentMeta(name: string): Promise<AttachmentMeta> {
+    return syscall("space.getAttachmentMeta", name);
+  }
+
+  /**
+   * Read an attachment from the space
+   * @param name path of the attachment to read
+   * @returns the attachment data encoded as a data URL
+   */
+  readAttachment(
+    name: string,
+  ): Promise<string> {
+    return syscall("space.readAttachment", name);
+  }
+
+  /**
+   * Writes an attachment to the space
+   * @param name path of the attachment to write
+   * @param encoding encoding of the data ("utf8" or "dataurl)
+   * @param data data itself
+   * @returns
+   */
+  writeAttachment(
+    name: string,
+    encoding: "utf8" | "dataurl",
+    data: string,
+  ): Promise<AttachmentMeta> {
+    return syscall("space.writeAttachment", name, encoding, data);
+  }
+
+  /**
+   * Deletes an attachment from the space
+   * @param name path of the attachment to delete
+   */
+  deleteAttachment(name: string): Promise<void> {
+    return syscall("space.deleteAttachment", name);
+  }
+
+  // Filesystem implementation
+  readFile(path: string, encoding: "dataurl" | "utf8"): Promise<string> {
+    return syscall("space.readFile", path, encoding);
+  }
+  getFileMeta(path: string): Promise<FileMeta> {
+    return syscall("space.getFileMeta", path);
+  }
+  writeFile(
+    path: string,
+    text: string,
+    encoding: "dataurl" | "utf8",
+  ): Promise<FileMeta> {
+    return syscall("space.writeFile", path, text, encoding);
+  }
+  deleteFile(path: string): Promise<void> {
+    return syscall("space.deleteFile", path);
+  }
+  listFiles(path: string): Promise<FileMeta[]> {
+    return syscall("space.listFiles", path);
+  }
 }
 
-export function getPageMeta(name: string): Promise<PageMeta> {
+export default new SpaceFileSystem();
-  return syscall("space.getPageMeta", name);
-}
-
-export function readPage(
-  name: string,
-): Promise<string> {
-  return syscall("space.readPage", name);
-}
-
-export function writePage(name: string, text: string): Promise<PageMeta> {
-  return syscall("space.writePage", name, text);
-}
-
-export function deletePage(name: string): Promise<void> {
-  return syscall("space.deletePage", name);
-}
-
-export function listPlugs(): Promise<string[]> {
-  return syscall("space.listPlugs");
-}
-
-export function listAttachments(): Promise<PageMeta[]> {
-  return syscall("space.listAttachments");
-}
-
-export function getAttachmentMeta(name: string): Promise<AttachmentMeta> {
-  return syscall("space.getAttachmentMeta", name);
-}
-
-/**
- * Read an attachment from the space
- * @param name path of the attachment to read
- * @returns the attachment data encoded as a data URL
- */
-export function readAttachment(
-  name: string,
-): Promise<string> {
-  return syscall("space.readAttachment", name);
-}
-
-/**
- * Writes an attachment to the space
- * @param name path of the attachment to write
- * @param encoding encoding of the data ("string" or "dataurl)
- * @param data data itself
- * @returns
- */
-export function writeAttachment(
-  name: string,
-  encoding: "string" | "dataurl",
-  data: string,
-): Promise<AttachmentMeta> {
-  return syscall("space.writeAttachment", name, encoding, data);
-}
-
-/**
- * Deletes an attachment from the space
- * @param name path of the attachment to delete
- */
-export function deleteAttachment(name: string): Promise<void> {
-  return syscall("space.deleteAttachment", name);
-}

plug-api/silverbullet-syscall/sync.ts

@@ -0,0 +1,28 @@
+import type { SyncStatusItem } from "../../common/spaces/sync.ts";
+import { syscall } from "./syscall.ts";
+
+export type SyncEndpoint = {
+  url: string;
+  user?: string;
+  password?: string;
+};
+
+// Perform a sync with the server, based on the given status (to be persisted)
+// returns a new sync status to persist
+export function sync(
+  endpoint: SyncEndpoint,
+  snapshot: Record<string, SyncStatusItem>,
+): Promise<
+  {
+    snapshot: Record<string, SyncStatusItem>;
+    operations: number;
+    error?: string;
+  }
+> {
+  return syscall("sync.sync", endpoint, snapshot);
+}
+
+// Checks the sync endpoint for connectivity and authentication, throws and Error on failure
+export function check(endpoint: SyncEndpoint): Promise<void> {
+  return syscall("sync.check", endpoint);
+}

plugos/sqlite/deno-sqlite/src/db.test.ts

@@ -1,462 +0,0 @@
-import {
-  assertAlmostEquals,
-  assertEquals,
-  assertThrows,
-} from "https://deno.land/std@0.154.0/testing/asserts.ts";
-
-import { DB } from "../mod.ts";
-
-const TEST_DB = "test.db";
-const LARGE_TEST_DB = "build/2GB_test.db";
-
-async function dbPermissions(path: string): Promise<boolean> {
-  const query = async (name: "read" | "write") =>
-    (await Deno.permissions.query({ name, path })).state ===
-      "granted";
-  return await query("read") && await query("write");
-}
-
-const TEST_DB_PERMISSIONS = await dbPermissions(TEST_DB);
-const LARGE_TEST_DB_PERMISSIONS = await dbPermissions(LARGE_TEST_DB);
-
-async function deleteDatabase(file: string) {
-  try {
-    await Deno.remove(file);
-  } catch { /* no op */ }
-  try {
-    await Deno.remove(`${file}-journal`);
-  } catch { /* no op */ }
-}
-
-Deno.test("execute multiple statements", function () {
-  const db = new DB();
-
-  db.execute(`
-    CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT);
-
-    INSERT INTO test (id) VALUES (1);
-    INSERT INTO test (id) VALUES (2);
-    INSERT INTO test (id) VALUES (3);
-  `);
-  assertEquals(db.query("SELECT id FROM test"), [[1], [2], [3]]);
-
-  // table `test` already exists ...
-  assertThrows(function () {
-    db.execute(`
-      CREATE TABLE test2 (id INTEGER);
-      CREATE TABLE test (id INTEGER);
-    `);
-  });
-
-  // ... but table `test2` was created before the error
-  assertEquals(db.query("SELECT id FROM test2"), []);
-
-  // syntax error after first valid statement
-  assertThrows(() => db.execute("SELECT id FROM test; NOT SQL ANYMORE"));
-});
-
-Deno.test("foreign key constraints enabled", function () {
-  const db = new DB();
-  db.execute(`
-    CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT);
-    CREATE TABLE orders (id INTEGER PRIMARY KEY AUTOINCREMENT, user INTEGER, FOREIGN KEY(user) REFERENCES users(id));
-  `);
-
-  db.query("INSERT INTO users (id) VALUES (1)");
-  const [{ id }] = db.queryEntries<{ id: number }>("SELECT id FROM users");
-
-  // user must exist
-  assertThrows(() =>
-    db.query("INSERT INTO orders (user) VALUES (?)", [id + 1])
-  );
-  db.query("INSERT INTO orders (user) VALUES (?)", [id]);
-
-  // can't delete if that violates the constraint ...
-  assertThrows(() => {
-    db.query("DELETE FROM users WHERE id = ?", [id]);
-  });
-
-  // ... after deleting the order, deleting is OK
-  db.query("DELETE FROM orders WHERE user = ?", [id]);
-  db.query("DELETE FROM users WHERE id = ?", [id]);
-});
-
-Deno.test("json functions exist", function () {
-  const db = new DB();
-
-  // The JSON1 functions should exist and we should be able to call them without unexpected errors
-  db.query(`SELECT json('{"this is": ["json"]}')`);
-
-  // We should expect an error if we pass invalid JSON where valid JSON is expected
-  assertThrows(() => {
-    db.query(`SELECT json('this is not json')`);
-  });
-
-  // We should be able to use bound values as arguments to the JSON1 functions,
-  // and they should produce the expected results for these simple expressions.
-  const [[objectType]] = db.query(`SELECT json_type('{}')`);
-  assertEquals(objectType, "object");
-
-  const [[integerType]] = db.query(`SELECT json_type(?)`, ["2"]);
-  assertEquals(integerType, "integer");
-
-  const [[realType]] = db.query(`SELECT json_type(?)`, ["2.5"]);
-  assertEquals(realType, "real");
-
-  const [[stringType]] = db.query(`SELECT json_type(?)`, [`"hello"`]);
-  assertEquals(stringType, "text");
-
-  const [[integerTypeAtPath]] = db.query(
-    `SELECT json_type(?, ?)`,
-    [`["hello", 2, {"world": 4}]`, `$[2].world`],
-  );
-  assertEquals(integerTypeAtPath, "integer");
-});
-
-Deno.test("date time is correct", function () {
-  const db = new DB();
-  // the date/ time is passed from JS and should be current (note that it is GMT)
-  const [[now]] = [...db.query("SELECT STRFTIME('%Y-%m-%d %H:%M:%f', 'NOW')")];
-  const jsTime = new Date().getTime();
-  const dbTime = new Date(`${now}Z`).getTime();
-  // to account for runtime latency, a small difference is ok
-  const tolerance = 10;
-  assertAlmostEquals(jsTime, dbTime, tolerance);
-  db.close();
-});
-
-Deno.test("SQL localtime reflects system locale", function () {
-  const db = new DB();
-  const [[timeDb]] = db.query("SELECT datetime('now', 'localtime')");
-  const now = new Date();
-
-  const jsMonth = `${now.getMonth() + 1}`.padStart(2, "0");
-  const jsDate = `${now.getDate()}`.padStart(2, "0");
-  const jsHour = `${now.getHours()}`.padStart(2, "0");
-  const jsMinute = `${now.getMinutes()}`.padStart(2, "0");
-  const jsSecond = `${now.getSeconds()}`.padStart(2, "0");
-  const timeJs =
-    `${now.getFullYear()}-${jsMonth}-${jsDate} ${jsHour}:${jsMinute}:${jsSecond}`;
-
-  assertEquals(timeDb, timeJs);
-});
-
-Deno.test("database has correct changes and totalChanges", function () {
-  const db = new DB();
-
-  db.execute(
-    "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT)",
-  );
-
-  for (const name of ["a", "b", "c"]) {
-    db.query("INSERT INTO test (name) VALUES (?)", [name]);
-    assertEquals(1, db.changes);
-  }
-
-  assertEquals(3, db.totalChanges);
-
-  db.query("UPDATE test SET name = ?", ["new name"]);
-  assertEquals(3, db.changes);
-  assertEquals(6, db.totalChanges);
-});
-
-Deno.test("last inserted id", function () {
-  const db = new DB();
-
-  // By default, lastInsertRowId must be 0
-  assertEquals(db.lastInsertRowId, 0);
-
-  // Create table and insert value
-  db.query("CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT)");
-
-  const insertRowIds = [];
-
-  // Insert data to table and collect their ids
-  for (let i = 0; i < 10; i++) {
-    db.query("INSERT INTO users (name) VALUES ('John Doe')");
-    insertRowIds.push(db.lastInsertRowId);
-  }
-
-  // Now, the last inserted row id must be 10
-  assertEquals(db.lastInsertRowId, 10);
-
-  // All collected row ids must be the same as in the database
-  assertEquals(
-    insertRowIds,
-    [...db.query("SELECT id FROM users")].map(([i]) => i),
-  );
-
-  db.close();
-
-  // When the database is closed, the value
-  // will be reset to 0 again
-  assertEquals(db.lastInsertRowId, 0);
-});
-
-Deno.test("close database", function () {
-  const db = new DB();
-  db.close();
-  assertThrows(() => db.query("CREATE TABLE test (name TEXT PRIMARY KEY)"));
-  db.close(); // check close is idempotent and won't throw
-});
-
-Deno.test("open queries block close", function () {
-  const db = new DB();
-  db.query("CREATE TABLE test (name TEXT PRIMARY KEY)");
-
-  const query = db.prepareQuery("SELECT name FROM test");
-  assertThrows(() => db.close());
-  query.finalize();
-
-  db.close();
-});
-
-Deno.test("open queries cleaned up by forced close", function () {
-  const db = new DB();
-  db.query("CREATE TABLE test (name TEXT PRIMARY KEY)");
-  db.query("INSERT INTO test (name) VALUES (?)", ["Deno"]);
-
-  db.prepareQuery("SELECT name FROM test WHERE name like '%test%'");
-
-  assertThrows(() => db.close());
-  db.close(true);
-});
-
-Deno.test("invalid bind does not leak statements", function () {
-  const db = new DB();
-  db.query("CREATE TABLE test (id INTEGER)");
-
-  for (let n = 0; n < 100; n++) {
-    assertThrows(() => {
-      // deno-lint-ignore no-explicit-any
-      const badBinding: any = [{}];
-      db.query("INSERT INTO test (id) VALUES (?)", badBinding);
-    });
-    assertThrows(() => {
-      const badBinding = { missingKey: null };
-      db.query("INSERT INTO test (id) VALUES (?)", badBinding);
-    });
-  }
-
-  db.query("INSERT INTO test (id) VALUES (1)");
-
-  db.close();
-});
-
-Deno.test("transactions can be nested", function () {
-  const db = new DB();
-  db.query("CREATE TABLE test (id INTEGER PRIMARY KEY)");
-
-  db.transaction(() => {
-    db.query("INSERT INTO test (id) VALUES (1)");
-    try {
-      db.transaction(() => {
-        db.query("INSERT INTO test (id) VALUES (2)");
-        throw new Error("boom!");
-      });
-    } catch (_) { /* ignore */ }
-  });
-
-  assertEquals([{ id: 1 }], db.queryEntries("SELECT * FROM test"));
-});
-
-Deno.test("transactions commit when closure exists", function () {
-  const db = new DB();
-  db.query("CREATE TABLE test (id INTEGER PRIMARY KEY)");
-
-  db.transaction(() => {
-    db.query("INSERT INTO test (id) VALUES (1)");
-  });
-  assertThrows(() => db.query("ROLLBACK"));
-
-  assertEquals([{ id: 1 }], db.queryEntries("SELECT * FROM test"));
-});
-
-Deno.test("transaction rolls back on throw", function () {
-  const db = new DB();
-  db.query("CREATE TABLE test (id INTEGER PRIMARY KEY)");
-
-  assertThrows(() => {
-    db.transaction(() => {
-      db.query("INSERT INTO test (id) VALUES (1)");
-      throw new Error("boom!");
-    });
-  });
-
-  assertEquals([], db.query("SELECT * FROM test"));
-});
-
-Deno.test(
-  "persist database to file",
-  {
-    ignore: !TEST_DB_PERMISSIONS,
-    permissions: { read: true, write: true },
-    sanitizeResources: true,
-  },
-  async function () {
-    const data = [
-      "Hello World!",
-      "Hello Deno!",
-      "JavaScript <3",
-      "This costs 0€ / $0 / £0",
-      "Wéll, hällö thėrè¿",
-    ];
-
-    // ensure the test database file does not exist
-    await deleteDatabase(TEST_DB);
-
-    const db = new DB(TEST_DB);
-    db.execute(
-      "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, val TEXT)",
-    );
-    for (const val of data) {
-      db.query("INSERT INTO test (val) VALUES (?)", [val]);
-    }
-
-    // open the same database with a separate connection
-    const readOnlyDb = await new DB(TEST_DB, { mode: "read" });
-    for (
-      const [id, val] of readOnlyDb.query<[number, string]>(
-        "SELECT * FROM test",
-      )
-    ) {
-      assertEquals(data[id - 1], val);
-    }
-
-    await Deno.remove(TEST_DB);
-    db.close();
-    readOnlyDb.close();
-  },
-);
-
-Deno.test(
-  "temporary file database read / write",
-  {
-    ignore: !TEST_DB_PERMISSIONS,
-    permissions: { read: true, write: true },
-    sanitizeResources: true,
-  },
-  function () {
-    const data = [
-      "Hello World!",
-      "Hello Deno!",
-      "JavaScript <3",
-      "This costs 0€ / $0 / £0",
-      "Wéll, hällö thėrè¿",
-    ];
-
-    const tempDb = new DB("");
-    tempDb.execute(
-      "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, val TEXT)",
-    );
-    for (const val of data) {
-      tempDb.query("INSERT INTO test (val) VALUES (?)", [val]);
-    }
-
-    for (
-      const [id, val] of tempDb.query<[number, string]>("SELECT * FROM test")
-    ) {
-      assertEquals(data[id - 1], val);
-    }
-
-    tempDb.close();
-  },
-);
-
-Deno.test(
-  "database open options",
-  {
-    ignore: !TEST_DB_PERMISSIONS,
-    permissions: { read: true, write: true },
-    sanitizeResources: true,
-  },
-  async function () {
-    await deleteDatabase(TEST_DB);
-
-    // when no file exists, these should error
-    assertThrows(() => new DB(TEST_DB, { mode: "write" }));
-    assertThrows(() => new DB(TEST_DB, { mode: "read" }));
-
-    // create the database
-    const dbCreate = new DB(TEST_DB, { mode: "create" });
-    dbCreate.execute(
-      "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL)",
-    );
-    dbCreate.close();
-
-    // the default mode is create
-    await deleteDatabase(TEST_DB);
-    const dbCreateDefault = new DB(TEST_DB, { mode: "create" });
-    dbCreateDefault.execute(
-      "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL)",
-    );
-    dbCreateDefault.close();
-
-    // in write mode, we can run INSERT queries ...
-    const dbWrite = new DB(TEST_DB, { mode: "write" });
-    dbWrite.query("INSERT INTO test (name) VALUES (?)", ["open-options-test"]);
-    dbWrite.close();
-
-    // ... which we can read in read-only mode ...
-    const dbRead = new DB(TEST_DB, { mode: "read" });
-    const rows = [...dbRead.query("SELECT id, name FROM test")];
-    assertEquals(rows, [[1, "open-options-test"]]);
-
-    // ... but we can't write with a read-only connection
-    assertThrows(() =>
-      dbRead.query("INTERT INTO test (name) VALUES (?)", ["this-fails"])
-    );
-    dbRead.close();
-  },
-);
-
-Deno.test(
-  "create / write mode require write permissions",
-  {
-    ignore: !TEST_DB_PERMISSIONS,
-    permissions: { read: true, write: false },
-    sanitizeResources: true,
-  },
-  function () {
-    // opening with these modes requires write permissions ...
-    assertThrows(() => new DB(TEST_DB, { mode: "create" }));
-    assertThrows(() => new DB(TEST_DB, { mode: "write" }));
-
-    // ... and the default mode is create
-    assertThrows(() => new DB(TEST_DB));
-
-    // however, opening in read-only mode should work (the file was created
-    // in the previous test)
-    (new DB(TEST_DB, { mode: "read" })).close();
-
-    // with memory flag set, the database will be in memory and
-    // not require any permissions
-    (new DB(TEST_DB, { mode: "create", memory: true })).close();
-
-    // the mode can also be specified via a URI flag
-    (new DB(`file:${TEST_DB}?mode=memory`, { uri: true })).close();
-  },
-);
-
-Deno.test(
-  "database larger than 2GB read / write",
-  {
-    ignore: !LARGE_TEST_DB_PERMISSIONS,
-    permissions: { read: true, write: true },
-    sanitizeResources: true,
-  },
-  function () {
-    // generated with `cd build && make testdb`
-    const db = new DB(LARGE_TEST_DB, { mode: "write" });
-
-    db.query("INSERT INTO test (value) VALUES (?)", ["This is a test..."]);
-
-    const rows = [
-      ...db.query("SELECT value FROM test ORDER BY id DESC LIMIT 10"),
-    ];
-    assertEquals(rows.length, 10);
-    assertEquals(rows[0][0], "This is a test...");
-
-    db.close();
-  },
-);

plugos/sqlite/deno-sqlite/src/error.test.ts

@@ -1,55 +0,0 @@
-import {
-  assertEquals,
-  assertInstanceOf,
-  assertThrows,
-} from "https://deno.land/std@0.154.0/testing/asserts.ts";
-
-import { DB, SqliteError, Status } from "../mod.ts";
-
-Deno.test("invalid SQL", function () {
-  const db = new DB();
-  const queries = [
-    "INSERT INTO does_not_exist (balance) VALUES (5)",
-    "this is not sql",
-    ";;;",
-  ];
-  for (const query of queries) assertThrows(() => db.query(query));
-
-  db.close();
-});
-
-Deno.test("constraint error code is correct", function () {
-  const db = new DB();
-  db.query("CREATE TABLE test (name TEXT PRIMARY KEY)");
-  db.query("INSERT INTO test (name) VALUES (?)", ["A"]);
-
-  assertThrows(
-    () => db.query("INSERT INTO test (name) VALUES (?)", ["A"]),
-    (e: Error) => {
-      assertInstanceOf(e, SqliteError);
-      assertEquals(e.code, Status.SqliteConstraint, "Got wrong error code");
-      assertEquals(
-        Status[e.codeName],
-        Status.SqliteConstraint,
-        "Got wrong error code name",
-      );
-    },
-  );
-});
-
-Deno.test("syntax error code is correct", function () {
-  const db = new DB();
-
-  assertThrows(
-    () => db.query("CREATE TABLEX test (name TEXT PRIMARY KEY)"),
-    (e: Error) => {
-      assertInstanceOf(e, SqliteError);
-      assertEquals(e.code, Status.SqliteError, "Got wrong error code");
-      assertEquals(
-        Status[e.codeName],
-        Status.SqliteError,
-        "Got wrong error code name",
-      );
-    },
-  );
-});

plugos/sqlite/deno-sqlite/src/query.test.ts

@@ -1,521 +0,0 @@
-import {
-  assertEquals,
-  assertThrows,
-} from "https://deno.land/std@0.154.0/testing/asserts.ts";
-
-import { DB, QueryParameter } from "../mod.ts";
-
-function roundTripValues<T extends QueryParameter>(values: T[]): unknown[] {
-  const db = new DB();
-  db.execute(
-    "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, datum ANY)",
-  );
-
-  for (const value of values) {
-    db.query("INSERT INTO test (datum) VALUES (?)", [value]);
-  }
-
-  return db
-    .queryEntries<{ datum: unknown }>("SELECT datum FROM test")
-    .map(({ datum }) => datum);
-}
-
-Deno.test("bind string values", function () {
-  const values = ["Hello World!", "I love Deno.", "Täst strüng..."];
-  assertEquals(values, roundTripValues(values));
-});
-
-Deno.test("bind integer values", function () {
-  const values = [42, 1, 2, 3, 4, 3453246, 4536787093, 45536787093];
-  assertEquals(values, roundTripValues(values));
-});
-
-Deno.test("bind float values", function () {
-  const values = [42.1, 1.235, 2.999, 1 / 3, 4.2345, 345.3246, 4536787.953e-8];
-  assertEquals(values, roundTripValues(values));
-});
-
-Deno.test("bind boolean values", function () {
-  assertEquals([1, 0], roundTripValues([true, false]));
-});
-
-Deno.test("bind date values", function () {
-  const values = [new Date(), new Date("2018-11-20"), new Date(123456789)];
-  assertEquals(
-    values.map((date) => date.toISOString()),
-    roundTripValues(values),
-  );
-});
-
-Deno.test("bind blob values", function () {
-  const values = [
-    new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8, 9, 0]),
-    new Uint8Array([3, 57, 45]),
-  ];
-  assertEquals(values, roundTripValues(values));
-});
-
-Deno.test("blobs are copies", function () {
-  const db = new DB();
-
-  db.query(
-    "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, val BLOB)",
-  );
-  const data = new Uint8Array([1, 2, 3, 4, 5]);
-  db.query("INSERT INTO test (val) VALUES (?)", [data]);
-
-  const [[a]] = db.query<[Uint8Array]>("SELECT val FROM test");
-  const [[b]] = db.query<[Uint8Array]>("SELECT val FROM test");
-
-  assertEquals(data, a);
-  assertEquals(data, b);
-  assertEquals(a, b);
-
-  a[0] = 100;
-  assertEquals(a[0], 100);
-  assertEquals(b[0], 1);
-  assertEquals(data[0], 1);
-
-  data[0] = 5;
-  const [[c]] = db.query<[Uint8Array]>("SELECT val FROM test");
-  assertEquals(c[0], 1);
-});
-
-Deno.test("bind bigint values", function () {
-  assertEquals(
-    [9007199254741991n, 100],
-    roundTripValues([9007199254741991n, 100n]),
-  );
-});
-
-Deno.test("bind null / undefined", function () {
-  assertEquals([null, null], roundTripValues([null, undefined]));
-});
-
-Deno.test("bind mixed values", function () {
-  const values = [42, "Hello World!", 0.33333, null];
-  assertEquals(values, roundTripValues(values));
-});
-
-Deno.test("omitting a value binds NULL", function () {
-  const db = new DB();
-  db.execute("CREATE TABLE test (id INTEGER PRIMARY KEY, datum ANY)");
-
-  const insert = db.prepareQuery(
-    "INSERT INTO test (datum) VALUES (?) RETURNING datum",
-  );
-
-  assertEquals([null], insert.first());
-  assertEquals([null], insert.first([]));
-  assertEquals([null], insert.first({}));
-
-  // previously bound values are cleared
-  insert.execute(["this is not null"]);
-  assertEquals([null], insert.first());
-});
-
-Deno.test("prepared query clears bindings before reused", function () {
-  const db = new DB();
-  db.query("CREATE TABLE test (id INTEGER PRIMARY KEY, value INTEGER)");
-
-  const query = db.prepareQuery("INSERT INTO test (value) VALUES (?)");
-  query.execute([1]);
-  query.execute();
-
-  assertEquals([[1], [null]], db.query("SELECT value FROM test"));
-
-  query.finalize();
-  db.close();
-});
-
-Deno.test("bind very large floating point numbers", function () {
-  const db = new DB();
-
-  db.query("CREATE TABLE numbers (id INTEGER PRIMARY KEY, number REAL)");
-
-  db.query("INSERT INTO numbers (number) VALUES (?)", [+Infinity]);
-  db.query("INSERT INTO numbers (number) VALUES (?)", [-Infinity]);
-  db.query("INSERT INTO numbers (number) VALUES (?)", [+20e20]);
-  db.query("INSERT INTO numbers (number) VALUES (?)", [-20e20]);
-
-  const [
-    [positiveInfinity],
-    [negativeInfinity],
-    [positiveTwentyTwenty],
-    [negativeTwentyTwenty],
-  ] = db.query("SELECT number FROM numbers");
-
-  assertEquals(negativeInfinity, -Infinity);
-  assertEquals(positiveInfinity, +Infinity);
-  assertEquals(positiveTwentyTwenty, +20e20);
-  assertEquals(negativeTwentyTwenty, -20e20);
-});
-
-Deno.test("big very large integers", function () {
-  const db = new DB();
-  db.query(
-    "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, val INTEGER)",
-  );
-
-  const goodValues = [
-    0n,
-    42n,
-    -42n,
-    9223372036854775807n,
-    -9223372036854775808n,
-  ];
-  const overflowValues = [
-    9223372036854775807n + 1n,
-    -9223372036854775808n - 1n,
-    2352359223372036854775807n,
-    -32453249223372036854775807n,
-  ];
-
-  const query = db.prepareQuery("INSERT INTO test (val) VALUES (?)");
-  for (const val of goodValues) {
-    query.execute([val]);
-  }
-
-  const dbValues = db.query<[number | bigint]>(
-    "SELECT val FROM test ORDER BY id",
-  ).map((
-    [id],
-  ) => BigInt(id));
-  assertEquals(goodValues, dbValues);
-
-  for (const bigVal of overflowValues) {
-    assertThrows(() => {
-      query.execute([bigVal]);
-    });
-  }
-
-  query.finalize();
-  db.close();
-});
-
-Deno.test("bind named parameters", function () {
-  const db = new DB();
-
-  db.query(
-    "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, val TEXT)",
-  );
-
-  // :name
-  db.query("INSERT INTO test (val) VALUES (:val)", { val: "value" });
-  db.query(
-    "INSERT INTO test (val) VALUES (:otherVal)",
-    { otherVal: "value other" },
-  );
-  db.query(
-    "INSERT INTO test (val) VALUES (:explicitColon)",
-    { ":explicitColon": "value explicit" },
-  );
-
-  // @name
-  db.query(
-    "INSERT INTO test (val) VALUES (@someName)",
-    { "@someName": "@value" },
-  );
-
-  // $name
-  db.query(
-    "INSERT INTO test (val) VALUES ($var::Name)",
-    { "$var::Name": "$value" },
-  );
-
-  // explicit positional syntax
-  db.query("INSERT INTO test (id, val) VALUES (?2, ?1)", ["this-is-it", 1000]);
-
-  // names must exist
-  assertThrows(() => {
-    db.query(
-      "INSERT INTO test (val) VALUES (:val)",
-      { Val: "miss-spelled name" },
-    );
-  });
-
-  // make sure the data came through correctly
-  const vals = [...db.query("SELECT val FROM test ORDER BY id ASC")]
-    .map(([datum]) => datum);
-  assertEquals(
-    vals,
-    [
-      "value",
-      "value other",
-      "value explicit",
-      "@value",
-      "$value",
-      "this-is-it",
-    ],
-  );
-});
-
-Deno.test("iterate from prepared query", function () {
-  const db = new DB();
-  db.execute("CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT)");
-  db.execute("INSERT INTO test (id) VALUES (1), (2), (3)");
-
-  const res = [];
-  const query = db.prepareQuery<[number]>("SELECT id FROM test");
-  for (const [id] of query.iter()) {
-    res.push(id);
-  }
-  assertEquals(res, [1, 2, 3]);
-
-  query.finalize();
-  db.close();
-});
-
-Deno.test("query all from prepared query", function () {
-  const db = new DB();
-  db.query("CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT)");
-  const query = db.prepareQuery("SELECT id FROM test");
-
-  assertEquals(query.all(), []);
-  db.query("INSERT INTO test (id) VALUES (1), (2), (3)");
-  assertEquals(query.all(), [[1], [2], [3]]);
-
-  query.finalize();
-  db.close();
-});
-
-Deno.test("query first from prepared query", function () {
-  const db = new DB();
-  db.query("CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT)");
-  db.query("INSERT INTO test (id) VALUES (1), (2), (3)");
-
-  const querySingle = db.prepareQuery("SELECT id FROM test WHERE id = ?");
-  assertEquals(querySingle.first([42]), undefined);
-  assertEquals(querySingle.first([2]), [2]);
-
-  const queryAll = db.prepareQuery("SELECT id FROM test ORDER BY id ASC");
-  assertEquals(queryAll.first(), [1]);
-
-  querySingle.finalize();
-  queryAll.finalize();
-  db.close();
-});
-
-Deno.test("query one from prepared query", function () {
-  const db = new DB();
-  db.query("CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT)");
-  db.query("INSERT INTO test (id) VALUES (1), (2), (3)");
-
-  const queryOne = db.prepareQuery<[number]>(
-    "SELECT id FROM test WHERE id = ?",
-  );
-  assertThrows(() => queryOne.one([42]));
-  assertEquals(queryOne.one([2]), [2]);
-
-  const queryAll = db.prepareQuery("SELECT id FROM test");
-  assertThrows(() => queryAll.one());
-
-  queryOne.finalize();
-  queryAll.finalize();
-  db.close();
-});
-
-Deno.test("execute from prepared query", function () {
-  const db = new DB();
-  db.query("CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT)");
-
-  const insert = db.prepareQuery("INSERT INTO test (id) VALUES (:id)");
-  for (const id of [1, 2, 3]) {
-    insert.execute({ id });
-  }
-  insert.finalize();
-  assertEquals(db.query("SELECT id FROM test"), [[1], [2], [3]]);
-
-  db.close();
-});
-
-Deno.test("empty query returns empty array", function () {
-  const db = new DB();
-  db.query("CREATE TABLE test (id INTEGER PRIMARY KEY)");
-  assertEquals([], db.query("SELECT * FROM test"));
-  db.close();
-});
-
-Deno.test("query entries returns correct object shapes", function () {
-  const db = new DB();
-  db.query(
-    "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT NOT NULL, height REAL)",
-  );
-
-  const rowsOrig = [
-    { id: 1, name: "Peter Parker", height: 1.5 },
-    { id: 2, name: "Clark Kent", height: 1.9 },
-    { id: 3, name: "Robert Paar", height: 2.1 },
-  ];
-
-  const insertQuery = db.prepareQuery(
-    "INSERT INTO test (id, name, height) VALUES (:id, :name, :height)",
-  );
-  for (const row of rowsOrig) {
-    insertQuery.execute(row);
-  }
-  insertQuery.finalize();
-
-  const query = db.prepareQuery("SELECT * FROM test");
-  assertEquals(rowsOrig, [...query.iterEntries()]);
-  assertEquals(rowsOrig, query.allEntries());
-  assertEquals(rowsOrig[0], query.firstEntry());
-  assertEquals(rowsOrig, db.queryEntries("SELECT * FROM test"));
-
-  query.finalize();
-  db.close();
-});
-
-Deno.test("prepared query can be reused", function () {
-  const db = new DB();
-  db.query("CREATE TABLE test (id INTEGER PRIMARY KEY)");
-
-  const query = db.prepareQuery("INSERT INTO test (id) VALUES (?)");
-  query.execute([1]);
-  query.execute([2]);
-  query.execute([3]);
-
-  assertEquals([[1], [2], [3]], db.query("SELECT id FROM test"));
-
-  query.finalize();
-  db.close();
-});
-
-Deno.test("get columns from select query", function () {
-  const db = new DB();
-
-  db.query(
-    "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT)",
-  );
-
-  const query = db.prepareQuery("SELECT id, name from test");
-
-  assertEquals(query.columns(), [
-    { name: "id", originName: "id", tableName: "test" },
-    { name: "name", originName: "name", tableName: "test" },
-  ]);
-});
-
-Deno.test("get columns from returning query", function () {
-  const db = new DB();
-
-  db.query(
-    "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT)",
-  );
-  const query = db.prepareQuery(
-    "INSERT INTO test (name) VALUES (?) RETURNING *",
-  );
-
-  assertEquals(query.columns(), [
-    { name: "id", originName: "id", tableName: "test" },
-    { name: "name", originName: "name", tableName: "test" },
-  ]);
-
-  assertEquals(query.all(["name"]), [[1, "name"]]);
-});
-
-Deno.test("get columns with renamed column", function () {
-  const db = new DB();
-
-  db.query(
-    "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT)",
-  );
-  db.query("INSERT INTO test (name) VALUES (?)", ["name"]);
-
-  const query = db.prepareQuery(
-    "SELECT id AS test_id, name AS test_name from test",
-  );
-  const columns = query.columns();
-
-  assertEquals(columns, [
-    { name: "test_id", originName: "id", tableName: "test" },
-    { name: "test_name", originName: "name", tableName: "test" },
-  ]);
-});
-
-Deno.test("columns can be obtained from empty prepared query", function () {
-  const db = new DB();
-  db.query(
-    "CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEST, age INTEGER)",
-  );
-  db.query("INSERT INTO test (name, age) VALUES (?, ?)", ["Peter Parker", 21]);
-
-  const query = db.prepareQuery("SELECT * FROM test");
-  const columnsFromPreparedQuery = query.columns();
-  query.finalize();
-
-  const queryEmpty = db.prepareQuery("SELECT * FROM test WHERE 1 = 0");
-  const columnsFromPreparedQueryWithEmptyQuery = queryEmpty.columns();
-  assertEquals(queryEmpty.all(), []);
-  query.finalize();
-
-  assertEquals(
-    [{ name: "id", originName: "id", tableName: "test" }, {
-      name: "name",
-      originName: "name",
-      tableName: "test",
-    }, { name: "age", originName: "age", tableName: "test" }],
-    columnsFromPreparedQuery,
-  );
-  assertEquals(
-    columnsFromPreparedQueryWithEmptyQuery,
-    columnsFromPreparedQuery,
-  );
-});
-
-Deno.test("invalid number of bound parameters throws", function () {
-  const db = new DB();
-  db.execute("CREATE TABLE test (id INTEGER PRIMARY KEY)");
-
-  // too many
-  assertThrows(() => {
-    db.query("SELECT * FROM test", [null]);
-  });
-  assertThrows(() => {
-    db.query("SELECT * FROM test LIMIT ?", [5, "extra"]);
-  });
-
-  // too few
-  assertThrows(() => db.query("SELECT * FROM test LIMIT ?", []));
-  assertThrows(() => {
-    db.query(
-      "SELECT * FROM test WHERE id >= ? AND id <= ? LIMIT ?",
-      [42],
-    );
-  });
-});
-
-Deno.test("using finalized prepared query throws", function () {
-  const db = new DB();
-  db.query("CREATE TABLE test (name TEXT)");
-  const query = db.prepareQuery("INSERT INTO test (name) VALUES (?)");
-  query.finalize();
-
-  assertThrows(() => query.execute(["test"]));
-  db.close();
-});
-
-Deno.test("invalid binding throws", function () {
-  const db = new DB();
-  db.query("CREATE TABLE test (id INTEGER)");
-  assertThrows(() => {
-    // deno-lint-ignore no-explicit-any
-    const badBinding: any = [{}];
-    db.query("SELECT * FORM test WHERE id = ?", badBinding);
-  });
-  db.close();
-});
-
-Deno.test("get columns from finalized query throws", function () {
-  const db = new DB();
-
-  db.query("CREATE TABLE test (id INTEGER PRIMARY KEY AUTOINCREMENT)");
-
-  const query = db.prepareQuery("SELECT id from test");
-  query.finalize();
-
-  // after iteration is done
-  assertThrows(() => {
-    query.columns();
-  });
-});

plugos/sqlite/deno-sqlite/src/readme.test.ts

@@ -1,80 +0,0 @@
-import {
-  assertEquals,
-  assertMatch,
-} from "https://deno.land/std@0.154.0/testing/asserts.ts";
-
-import { DB } from "../mod.ts";
-
-Deno.test("README example", function () {
-  const db = new DB(/* in memory */);
-  db.execute(`
-    CREATE TABLE IF NOT EXISTS people (
-      id INTEGER PRIMARY KEY AUTOINCREMENT,
-      name TEXT
-    )
-  `);
-
-  const name =
-    ["Peter Parker", "Clark Kent", "Bruce Wane"][Math.floor(Math.random() * 3)];
-
-  // Run a simple query
-  db.query("INSERT INTO people (name) VALUES (?)", [name]);
-
-  // Print out data in table
-  for (const [_name] of db.query("SELECT name FROM people")) continue; // no console.log ;)
-
-  db.close();
-});
-
-Deno.test("old README example", function () {
-  const db = new DB();
-  const first = ["Bruce", "Clark", "Peter"];
-  const last = ["Wane", "Kent", "Parker"];
-  db.query(
-    "CREATE TABLE users (id INTEGER PRIMARY KEY AUTOINCREMENT, name TEXT, email TEXT, subscribed INTEGER)",
-  );
-
-  for (let i = 0; i < 100; i++) {
-    const name = `${first[Math.floor(Math.random() * first.length)]} ${
-      last[
-        Math.floor(
-          Math.random() * last.length,
-        )
-      ]
-    }`;
-    const email = `${name.replace(" ", "-")}@deno.land`;
-    const subscribed = Math.random() > 0.5 ? true : false;
-    db.query("INSERT INTO users (name, email, subscribed) VALUES (?, ?, ?)", [
-      name,
-      email,
-      subscribed,
-    ]);
-  }
-
-  for (
-    const [
-      name,
-      email,
-    ] of db.query<[string, string]>(
-      "SELECT name, email FROM users WHERE subscribed = ? LIMIT 100",
-      [true],
-    )
-  ) {
-    assertMatch(name, /(Bruce|Clark|Peter) (Wane|Kent|Parker)/);
-    assertEquals(email, `${name.replace(" ", "-")}@deno.land`);
-  }
-
-  const res = db.query("SELECT email FROM users WHERE name LIKE ?", [
-    "Robert Parr",
-  ]);
-  assertEquals(res, []);
-
-  const subscribers = db.query(
-    "SELECT name, email FROM users WHERE subscribed = ?",
-    [true],
-  );
-  for (const [_name, _email] of subscribers) {
-    if (Math.random() > 0.5) continue;
-    break;
-  }
-});

plugos/sqlite/deno-sqlite/src/wasm.test.ts

@@ -1,40 +0,0 @@
-import {
-  assertEquals,
-  assertThrows,
-} from "https://deno.land/std@0.154.0/testing/asserts.ts";
-
-import { Wasm } from "../build/sqlite.js";
-import * as wasm from "./wasm.ts";
-
-function mock(
-  malloc: () => number = () => 1,
-  free: (pts: number) => void = () => {},
-): Wasm {
-  const memory = new Uint8Array(2048);
-  return {
-    malloc,
-    free,
-    str_len: (ptr: number) => {
-      let len = 0;
-      for (let idx = ptr; memory.at(idx) != 0; idx++) len++;
-      return len;
-    },
-    memory,
-  } as unknown as Wasm;
-}
-
-Deno.test("round trip string", function () {
-  const mockWasm = mock();
-  const testCases = ["Hello world!", "Söme, fünky lëttêrß", "你好👋"];
-  for (const input of testCases) {
-    const output = wasm.setStr(mockWasm, input, (ptr) => {
-      return wasm.getStr(mockWasm, ptr);
-    });
-    assertEquals(input, output);
-  }
-});
-
-Deno.test("throws on allocation error", function () {
-  const mockWasm = mock(() => 0);
-  assertThrows(() => wasm.setStr(mockWasm, "Hello world!", (_) => null));
-});

plugs/collab/collab.ts

@@ -136,8 +136,8 @@ export async function readFileCollab(
   const text = `---\n$share: ${collabUri}\n---\n`;
 
   return {
-    // encoding === "arraybuffer" is not an option, so either it's "string" or "dataurl"
+    // encoding === "arraybuffer" is not an option, so either it's "utf8" or "dataurl"
-    data: encoding === "string" ? text : base64EncodedDataUrl(
+    data: encoding === "utf8" ? text : base64EncodedDataUrl(
       "text/markdown",
       new TextEncoder().encode(text),
     ),

plugs/core/cloud.ts

@@ -45,7 +45,7 @@ export async function readFileCloud(
     `${pagePrefix}${originalUrl.split("/")[0]}/`,
   );
   return {
-    data: encoding === "string" ? text : base64EncodedDataUrl(
+    data: encoding === "utf8" ? text : base64EncodedDataUrl(
       "text/markdown",
       new TextEncoder().encode(text),
     ),

plugs/core/plugmanager.ts

@@ -1,9 +1,7 @@
 import { events } from "$sb/plugos-syscall/mod.ts";
 import type { Manifest } from "../../common/manifest.ts";
 import { editor, space, system } from "$sb/silverbullet-syscall/mod.ts";
-
 import { readYamlPage } from "$sb/lib/yaml_page.ts";
-import { writePage } from "$sb/silverbullet-syscall/space.ts";
 
 const plugsPrelude =
   "This file lists all plugs that SilverBullet will load. Run the {[Plugs: Update]} command to update and reload this list of plugs.\n\n";
@@ -82,7 +80,7 @@ export async function updatePlugs() {
     // console.log("Writing", `_plug/${manifest.name}`);
     await space.writeAttachment(
       `_plug/${manifest.name}.plug.json`,
-      "string",
+      "utf8",
       JSON.stringify(manifest),
     );
   }

plugs/core/search.ts

@@ -85,8 +85,8 @@ export async function readFileSearch(
   `;
 
   return {
-    // encoding === "arraybuffer" is not an option, so either it's "string" or "dataurl"
+    // encoding === "arraybuffer" is not an option, so either it's "utf8" or "dataurl"
-    data: encoding === "string" ? text : base64EncodedDataUrl(
+    data: encoding === "utf8" ? text : base64EncodedDataUrl(
       "text/markdown",
       new TextEncoder().encode(text),
     ),

plugs/markdown/markdown.plug.yaml

@@ -24,14 +24,14 @@ functions:
     path: "./preview.ts:previewClickHandler"
     env: client
     events:
-        - preview:click
+      - preview:click
 
   # $share: file:* publisher for markdown files
   sharePublisher:
     path: ./share.ts:sharePublisher
     events:
-    - share:file
+      - share:file
 
   markdownWidget:
     path: ./widget.ts:markdownWidget
-    codeWidget: markdown
+    codeWidget: markdown

plugs/markdown/share.ts

@@ -1,5 +1,5 @@
 import { markdown, space } from "$sb/silverbullet-syscall/mod.ts";
-import { fs } from "$sb/plugos-syscall/mod.ts";
+import { LocalFileSystem } from "$sb/plugos-syscall/mod.ts";
 import { asset } from "$sb/plugos-syscall/mod.ts";
 import { renderMarkdownToHtml } from "./markdown_render.ts";
 import { PublishEvent } from "$sb/app_event.ts";
@@ -10,12 +10,14 @@ export async function sharePublisher(event: PublishEvent) {
   const text = await space.readPage(pageName);
   const tree = await markdown.parseMarkdown(text);
 
+  const rootFS = new LocalFileSystem("");
+
   const css = await asset.readAsset("assets/styles.css");
   const markdownHtml = renderMarkdownToHtml(tree, {
     smartHardBreak: true,
   });
   const html =
     `<html><head><style>${css}</style></head><body><div id="root">${markdownHtml}</div></body></html>`;
-  await fs.writeFile(path, html, "utf8");
+  await rootFS.writeFile(path, html, "utf8");
   return true;
 }

plugs/sync/sync.plug.yaml

@@ -0,0 +1,19 @@
+name: sync
+functions:
+  configureCommand:
+    path: sync.ts:configureCommand
+    command:
+      name: "Sync: Configure"
+
+  syncCommand:
+    path: sync.ts:syncCommand
+    command:
+      name: "Sync: Sync"
+
+  check:
+    env: server
+    path: sync.ts:check
+
+  performSync:
+    env: server
+    path: sync.ts:performSync

plugs/sync/sync.ts

@@ -0,0 +1,88 @@
+import { store } from "$sb/plugos-syscall/mod.ts";
+import { editor, sync, system } from "$sb/silverbullet-syscall/mod.ts";
+import type { SyncEndpoint } from "$sb/silverbullet-syscall/sync.ts";
+
+export async function configureCommand() {
+  const url = await editor.prompt(
+    "Enter the URL of the remote space to sync with",
+    "https://",
+  );
+  if (!url) {
+    return;
+  }
+
+  const user = await editor.prompt("Username (if any):");
+  let password = undefined;
+  if (user) {
+    password = await editor.prompt("Password:");
+  }
+
+  const syncConfig: SyncEndpoint = {
+    url,
+    user,
+    password,
+  };
+
+  try {
+    await system.invokeFunction("server", "check", syncConfig);
+  } catch (e: any) {
+    await editor.flashNotification(
+      `Sync configuration failed: ${e.message}`,
+      "error",
+    );
+    return;
+  }
+
+  await store.batchSet([
+    { key: "sync.config", value: syncConfig },
+    // Empty initial snapshot
+    { key: "sync.snapshot", value: {} },
+  ]);
+
+  await editor.flashNotification("Sync configuration saved.");
+
+  return syncConfig;
+}
+
+export async function syncCommand() {
+  let config: SyncEndpoint | undefined = await store.get("sync.config");
+  if (!config) {
+    config = await configureCommand();
+    if (!config) {
+      return;
+    }
+  }
+  await editor.flashNotification("Starting sync...");
+  try {
+    const operations = await system.invokeFunction("server", "performSync");
+    await editor.flashNotification(
+      `Sync complete. Performed ${operations} operations.`,
+    );
+  } catch (e: any) {
+    await editor.flashNotification(
+      `Sync failed: ${e.message}`,
+      "error",
+    );
+  }
+}
+
+// Run on server
+export function check(config: SyncEndpoint) {
+  return sync.check(config);
+}
+
+// Run on server
+export async function performSync() {
+  const config: SyncEndpoint = await store.get("sync.config");
+  const snapshot = await store.get("sync.snapshot");
+  const { snapshot: newSnapshot, operations, error } = await sync.sync(
+    config,
+    snapshot,
+  );
+  await store.set("sync.snapshot", newSnapshot);
+  if (error) {
+    console.error("Sync error", error);
+    throw new Error(error);
+  }
+  return operations;
+}

server/http_server.ts

@@ -5,6 +5,7 @@ import { EndpointHook } from "../plugos/hooks/endpoint.ts";
 import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
 import { SpaceSystem } from "./space_system.ts";
 import { ensureAndLoadSettings } from "../common/util.ts";
+import { base64Decode } from "../plugos/asset_bundle/base64.ts";
 
 export type ServerOptions = {
   hostname: string;
@@ -14,6 +15,7 @@ export type ServerOptions = {
   assetBundle: AssetBundle;
   user?: string;
   pass?: string;
+  bareMode?: boolean;
 };
 
 const staticLastModified = new Date().toUTCString();
@@ -26,6 +28,7 @@ export class HttpServer {
   user?: string;
   settings: { [key: string]: any } = {};
   abortController?: AbortController;
+  bareMode: boolean;
 
   constructor(options: ServerOptions) {
     this.hostname = options.hostname;
@@ -37,6 +40,7 @@ export class HttpServer {
       options.pagesPath,
       options.dbPath,
     );
+    this.bareMode = options.bareMode || false;
 
     // Second, for loading plug JSON files with absolute or relative (from CWD) paths
     this.systemBoot.eventHook.addLocalListener(
@@ -66,7 +70,7 @@ export class HttpServer {
   async start() {
     await this.systemBoot.start();
     await this.systemBoot.ensureSpaceIndex();
-    await ensureAndLoadSettings(this.systemBoot.space);
+    await ensureAndLoadSettings(this.systemBoot.space, this.bareMode);
 
     this.addPasswordAuth(this.app);
 
@@ -207,7 +211,8 @@ export class HttpServer {
     // File list
     fsRouter.get("/", async ({ response }) => {
       response.headers.set("Content-type", "application/json");
-      response.body = JSON.stringify(await spacePrimitives.fetchFileList());
+      const files = await spacePrimitives.fetchFileList();
+      response.body = JSON.stringify(files);
     });
 
     fsRouter
@@ -248,12 +253,21 @@ export class HttpServer {
         const name = params[0];
         console.log("Saving file", name);
 
+        let body: Uint8Array;
+        if (
+          request.headers.get("X-Content-Base64")
+        ) {
+          const content = await request.body({ type: "text" }).value;
+          body = base64Decode(content);
+        } else {
+          body = await request.body({ type: "bytes" }).value;
+        }
+
         try {
           const meta = await spacePrimitives.writeFile(
             name,
             "arraybuffer",
-            await request.body().value,
+            body,
-            false,
           );
           response.status = 200;
           response.headers.set("Content-Type", meta.contentType);
@@ -299,7 +313,6 @@ export class HttpServer {
 
   private buildPlugRouter(): Router {
     const plugRouter = new Router();
-    // this.addPasswordAuth(plugRouter);
     const system = this.systemBoot.system;
 
     plugRouter.post(

server/space_system.ts

@@ -23,13 +23,13 @@ import {
   storeSyscalls,
 } from "../plugos/syscalls/store.sqlite.ts";
 import { System } from "../plugos/system.ts";
-import { PageNamespaceHook } from "./hooks/page_namespace.ts";
+import { PageNamespaceHook } from "../common/hooks/page_namespace.ts";
-import { PlugSpacePrimitives } from "./hooks/plug_space_primitives.ts";
+import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts";
 import {
   ensureTable as ensureIndexTable,
   pageIndexSyscalls,
 } from "./syscalls/index.ts";
-import spaceSyscalls from "./syscalls/space.ts";
+import spaceSyscalls from "../common/syscalls/space.ts";
 import { systemSyscalls } from "./syscalls/system.ts";
 import { AssetBundlePlugSpacePrimitives } from "../common/spaces/asset_bundle_space_primitives.ts";
 import assetSyscalls from "../plugos/syscalls/asset.ts";
@@ -37,6 +37,7 @@ import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
 import { AsyncSQLite } from "../plugos/sqlite/async_sqlite.ts";
 import { FileMetaSpacePrimitives } from "../common/spaces/file_meta_space_primitives.ts";
 import { sandboxFetchSyscalls } from "../plugos/syscalls/fetch.ts";
+import { syncSyscalls } from "../common/syscalls/sync.ts";
 export const indexRequiredKey = "$spaceIndexed";
 
 // A composition of a PlugOS system attached to a Space for server-side use
@@ -111,6 +112,7 @@ export class SpaceSystem {
       storeSyscalls(this.db, "store"),
       fullTextSearchSyscalls(this.db, "fts"),
       spaceSyscalls(this.space),
+      syncSyscalls(this.spacePrimitives),
       eventSyscalls(this.eventHook),
       markdownSyscalls(buildMarkdown([])),
       esbuildSyscalls([globalModules]),
@@ -145,7 +147,7 @@ export class SpaceSystem {
 
     console.log("Going to load", allPlugs.length, "plugs...");
     await Promise.all(allPlugs.map(async (plugName) => {
-      const { data } = await this.space.readAttachment(plugName, "string");
+      const { data } = await this.space.readAttachment(plugName, "utf8");
       await this.system.load(JSON.parse(data as string), createSandbox);
     }));
 

silverbullet.ts

@@ -21,6 +21,9 @@ await new Command()
   .arguments("<folder:string>")
   .option("--hostname <hostname:string>", "Hostname or address to listen on")
   .option("-p, --port <port:number>", "Port to listen on")
+  .option("--bare [type:boolean]", "Don't auto generate pages", {
+    default: false,
+  })
   .option("--db <dbfile:string>", "Filename for the database", {
     default: "data.db",
   })

web/boot.ts

@@ -2,8 +2,8 @@ import { Editor } from "./editor.tsx";
 import { parseYamlSettings, safeRun } from "../common/util.ts";
 import { Space } from "../common/spaces/space.ts";
 import { HttpSpacePrimitives } from "../common/spaces/http_space_primitives.ts";
-import { PlugSpacePrimitives } from "../server/hooks/plug_space_primitives.ts";
+import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts";
-import { PageNamespaceHook } from "../server/hooks/page_namespace.ts";
+import { PageNamespaceHook } from "../common/hooks/page_namespace.ts";
 import { SilverBulletHooks } from "../common/manifest.ts";
 import { System } from "../plugos/system.ts";
 import { BuiltinSettings } from "./types.ts";
@@ -19,7 +19,7 @@ safeRun(async () => {
   let settingsPageText = "";
   try {
     settingsPageText = (
-      await httpPrimitives.readFile("SETTINGS.md", "string")
+      await httpPrimitives.readFile("SETTINGS.md", "utf8")
     ).data as string;
   } catch (e: any) {
     console.error("No settings page found", e.message);

web/editor.tsx

@@ -97,6 +97,7 @@ import type {
 } from "../plug-api/app_event.ts";
 import { CodeWidgetHook } from "./hooks/code_widget.ts";
 import { sandboxFetchSyscalls } from "../plugos/syscalls/fetch.ts";
+import { syncSyscalls } from "../common/syscalls/sync.ts";
 
 const frontMatterRegex = /^---\n(.*?)---\n/ms;
 
@@ -195,6 +196,7 @@ export class Editor {
       markdownSyscalls(buildMarkdown(this.mdExtensions)),
       sandboxSyscalls(this.system),
       assetSyscalls(this.system),
+      syncSyscalls(this.space.spacePrimitives),
       collabSyscalls(this),
     );
 
@@ -659,7 +661,7 @@ export class Editor {
     await this.system.unloadAll();
     console.log("(Re)loading plugs");
     await Promise.all((await this.space.listPlugs()).map(async (plugName) => {
-      const { data } = await this.space.readAttachment(plugName, "string");
+      const { data } = await this.space.readAttachment(plugName, "utf8");
       await this.system.load(JSON.parse(data as string), createSandbox);
     }));
     this.rebuildEditorState();

web/syscalls/space.ts

@@ -1,70 +1,19 @@
 import { Editor } from "../editor.tsx";
 import { SysCallMapping } from "../../plugos/system.ts";
-import { AttachmentMeta, PageMeta } from "../../common/types.ts";
+
-import {
+import commonSpaceSyscalls from "../../common/syscalls/space.ts";
-  FileData,
-  FileEncoding,
-} from "../../common/spaces/space_primitives.ts";
 
 export function spaceSyscalls(editor: Editor): SysCallMapping {
-  return {
+  const syscalls = commonSpaceSyscalls(editor.space);
-    "space.listPages": (): PageMeta[] => {
+  syscalls["space.deletePage"] = async (_ctx, name: string) => {
-      return [...editor.space.listPages()];
+    // If we're deleting the current page, navigate to the index page
-    },
+    if (editor.currentPage === name) {
-    "space.readPage": async (
+      await editor.navigate("");
-      _ctx,
+    }
-      name: string,
+    // Remove page from open pages in editor
-    ): Promise<string> => {
+    editor.openPages.delete(name);
-      return (await editor.space.readPage(name)).text;
+    console.log("Deleting page");
-    },
+    await editor.space.deletePage(name);
-    "space.getPageMeta": async (_ctx, name: string): Promise<PageMeta> => {
-      return await editor.space.getPageMeta(name);
-    },
-    "space.writePage": async (
-      _ctx,
-      name: string,
-      text: string,
-    ): Promise<PageMeta> => {
-      return await editor.space.writePage(name, text);
-    },
-    "space.deletePage": async (_ctx, name: string) => {
-      // If we're deleting the current page, navigate to the index page
-      if (editor.currentPage === name) {
-        await editor.navigate("");
-      }
-      // Remove page from open pages in editor
-      editor.openPages.delete(name);
-      console.log("Deleting page");
-      await editor.space.deletePage(name);
-    },
-    "space.listPlugs": (): Promise<string[]> => {
-      return editor.space.listPlugs();
-    },
-    "space.listAttachments": (): Promise<AttachmentMeta[]> => {
-      return editor.space.fetchAttachmentList();
-    },
-    "space.readAttachment": async (
-      _ctx,
-      name: string,
-    ): Promise<FileData> => {
-      return (await editor.space.readAttachment(name, "dataurl")).data;
-    },
-    "space.getAttachmentMeta": async (
-      _ctx,
-      name: string,
-    ): Promise<AttachmentMeta> => {
-      return await editor.space.getAttachmentMeta(name);
-    },
-    "space.writeAttachment": async (
-      _ctx,
-      name: string,
-      encoding: FileEncoding,
-      data: FileData,
-    ): Promise<AttachmentMeta> => {
-      return await editor.space.writeAttachment(name, encoding, data);
-    },
-    "space.deleteAttachment": async (_ctx, name: string) => {
-      await editor.space.deleteAttachment(name);
-    },
   };
+  return syscalls;
 }

web/types.ts

@@ -14,6 +14,7 @@ export type PanelMode = number;
 
 export type BuiltinSettings = {
   indexPage: string;
+  syncUrl?: string;
 };
 
 export type PanelConfig = {

website/.gitignore

@@ -1,3 +1,2 @@
 data.db
 _plug
-_trash

website/CHANGELOG.md

@@ -14,6 +14,7 @@ release.
   select * from my_table;
   ```
 * Merged code for experimental mobile app (iOS only for now)
+* Experimental sync engine, to be documented once it matures
 
 ---