Step 1 in major backend refactor

packages/common/spaces/constants.ts

@@ -1,2 +1 @@
-export const trashPrefix = "_trash/";
 export const plugPrefix = "_plug/";

packages/common/spaces/disk_space_primitives.ts

@@ -1,30 +1,20 @@
-import {
+import { mkdir, readdir, readFile, stat, unlink, writeFile } from "fs/promises";
-  mkdir,
-  readdir,
-  readFile,
-  stat,
-  unlink,
-  utimes,
-  writeFile,
-} from "fs/promises";
 import * as path from "path";
-import { AttachmentMeta, PageMeta } from "../types";
+import { FileMeta } from "../types";
-import {
+import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives";
-  AttachmentData,
-  AttachmentEncoding,
-  SpacePrimitives,
-} from "./space_primitives";
 import { Plug } from "@plugos/plugos/plug";
 import { realpathSync } from "fs";
 import mime from "mime-types";
 
+function lookupContentType(path: string): string {
+  return mime.lookup(path) || "application/octet-stream";
+}
+
 export class DiskSpacePrimitives implements SpacePrimitives {
   rootPath: string;
-  plugPrefix: string;
 
-  constructor(rootPath: string, plugPrefix: string = "_plug/") {
+  constructor(rootPath: string) {
     this.rootPath = realpathSync(rootPath);
-    this.plugPrefix = plugPrefix;
   }
 
   safePath(p: string): string {
@@ -35,265 +25,147 @@ export class DiskSpacePrimitives implements SpacePrimitives {
     return realPath;
   }
 
-  pageNameToPath(pageName: string) {
+  filenameToPath(pageName: string) {
-    if (pageName.startsWith(this.plugPrefix)) {
+    return this.safePath(path.join(this.rootPath, pageName));
-      return this.safePath(path.join(this.rootPath, pageName + ".plug.json"));
-    }
-    return this.safePath(path.join(this.rootPath, pageName + ".md"));
   }
 
-  pathToPageName(fullPath: string): string {
+  pathToFilename(fullPath: string): string {
-    let extLength = fullPath.endsWith(".plug.json")
+    return fullPath.substring(this.rootPath.length + 1);
-      ? ".plug.json".length
-      : ".md".length;
-    return fullPath.substring(
-      this.rootPath.length + 1,
-      fullPath.length - extLength
-    );
   }
 
-  // Pages
+  async readFile(
-  async readPage(pageName: string): Promise<{ text: string; meta: PageMeta }> {
+    name: string,
-    const localPath = this.pageNameToPath(pageName);
+    encoding: FileEncoding
+  ): Promise<{ data: FileData; meta: FileMeta }> {
+    const localPath = this.filenameToPath(name);
     try {
       const s = await stat(localPath);
+      let data: FileData | null = null;
+      let contentType = lookupContentType(name);
+      switch (encoding) {
+        case "string":
+          data = await readFile(localPath, "utf8");
+          break;
+        case "dataurl":
+          let fileBuffer = await readFile(localPath, {
+            encoding: "base64",
+          });
+          data = `data:${contentType};base64,${fileBuffer}`;
+          break;
+        case "arraybuffer":
+          let arrayBuffer = await readFile(localPath);
+          data = arrayBuffer.buffer;
+          break;
+      }
       return {
-        text: await readFile(localPath, "utf8"),
+        data,
         meta: {
-          name: pageName,
+          name: name,
           lastModified: s.mtime.getTime(),
           perm: "rw",
+          size: s.size,
+          contentType: contentType,
         },
       };
     } catch (e) {
-      // console.error("Error while reading page", pageName, e);
+      console.error("Error while reading file", name, e);
-      throw Error(`Could not read page ${pageName}`);
+      throw Error(`Could not read file ${name}`);
     }
   }
 
-  async writePage(
+  async writeFile(
-    pageName: string,
+    name: string,
-    text: string,
+    encoding: FileEncoding,
-    selfUpdate: boolean,
+    data: FileData,
-    lastModified?: number
+    selfUpdate?: boolean
-  ): Promise<PageMeta> {
+  ): Promise<FileMeta> {
-    let localPath = this.pageNameToPath(pageName);
+    let localPath = this.filenameToPath(name);
     try {
       // Ensure parent folder exists
       await mkdir(path.dirname(localPath), { recursive: true });
 
       // Actually write the file
-      await writeFile(localPath, text);
+      switch (encoding) {
-
+        case "string":
-      if (lastModified) {
+          await writeFile(localPath, data as string, "utf8");
-        let d = new Date(lastModified);
+          break;
-        console.log("Going to set the modified time", d);
+        case "dataurl":
-        await utimes(localPath, d, d);
+          await writeFile(localPath, (data as string).split(",")[1], {
+            encoding: "base64",
+          });
+          break;
+        case "arraybuffer":
+          await writeFile(localPath, Buffer.from(data as ArrayBuffer));
+          break;
       }
+
       // Fetch new metadata
       const s = await stat(localPath);
       return {
-        name: pageName,
+        name: name,
+        size: s.size,
+        contentType: lookupContentType(name),
         lastModified: s.mtime.getTime(),
         perm: "rw",
       };
     } catch (e) {
-      console.error("Error while writing page", pageName, e);
+      console.error("Error while writing file", name, e);
-      throw Error(`Could not write ${pageName}`);
+      throw Error(`Could not write ${name}`);
     }
   }
 
-  async getPageMeta(pageName: string): Promise<PageMeta> {
+  async getFileMeta(name: string): Promise<FileMeta> {
-    let localPath = this.pageNameToPath(pageName);
+    let localPath = this.filenameToPath(name);
     try {
       const s = await stat(localPath);
       return {
-        name: pageName,
+        name: name,
+        size: s.size,
+        contentType: lookupContentType(name),
         lastModified: s.mtime.getTime(),
         perm: "rw",
       };
     } catch (e) {
       // console.error("Error while getting page meta", pageName, e);
-      throw Error(`Could not get meta for ${pageName}`);
-    }
-  }
-
-  async deletePage(pageName: string): Promise<void> {
-    let localPath = this.pageNameToPath(pageName);
-    await unlink(localPath);
-  }
-
-  async fetchPageList(): Promise<{
-    pages: Set<PageMeta>;
-    nowTimestamp: number;
-  }> {
-    let pages = new Set<PageMeta>();
-
-    const walkPath = async (dir: string) => {
-      let files = await readdir(dir);
-      for (let file of files) {
-        const fullPath = path.join(dir, file);
-        let s = await stat(fullPath);
-        if (s.isDirectory()) {
-          await walkPath(fullPath);
-        } else {
-          if (file.endsWith(".md") || file.endsWith(".json")) {
-            pages.add({
-              name: this.pathToPageName(fullPath),
-              lastModified: s.mtime.getTime(),
-              perm: "rw",
-            });
-          }
-        }
-      }
-    };
-    await walkPath(this.rootPath);
-    return {
-      pages: pages,
-      nowTimestamp: Date.now(),
-    };
-  }
-
-  // Attachments
-  attachmentNameToPath(name: string) {
-    return this.safePath(path.join(this.rootPath, name));
-  }
-
-  pathToAttachmentName(fullPath: string): string {
-    return fullPath.substring(this.rootPath.length + 1);
-  }
-
-  async fetchAttachmentList(): Promise<{
-    attachments: Set<AttachmentMeta>;
-    nowTimestamp: number;
-  }> {
-    let attachments = new Set<AttachmentMeta>();
-
-    const walkPath = async (dir: string) => {
-      let files = await readdir(dir);
-      for (let file of files) {
-        const fullPath = path.join(dir, file);
-        let s = await stat(fullPath);
-        if (s.isDirectory()) {
-          if (!file.startsWith(".")) {
-            await walkPath(fullPath);
-          }
-        } else {
-          if (
-            !file.startsWith(".") &&
-            !file.endsWith(".md") &&
-            !file.endsWith(".json")
-          ) {
-            attachments.add({
-              name: this.pathToAttachmentName(fullPath),
-              lastModified: s.mtime.getTime(),
-              size: s.size,
-              contentType: mime.lookup(file) || "application/octet-stream",
-              perm: "rw",
-            } as AttachmentMeta);
-          }
-        }
-      }
-    };
-    await walkPath(this.rootPath);
-    return {
-      attachments,
-      nowTimestamp: Date.now(),
-    };
-  }
-
-  async readAttachment(
-    name: string,
-    encoding: AttachmentEncoding
-  ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> {
-    const localPath = this.attachmentNameToPath(name);
-    let fileBuffer = await readFile(localPath, {
-      encoding: encoding === "dataurl" ? "base64" : null,
-    });
-
-    try {
-      const s = await stat(localPath);
-      let contentType = mime.lookup(name) || "application/octet-stream";
-      return {
-        data:
-          encoding === "dataurl"
-            ? `data:${contentType};base64,${fileBuffer}`
-            : (fileBuffer as Buffer).buffer,
-        meta: {
-          name: name,
-          lastModified: s.mtime.getTime(),
-          size: s.size,
-          contentType: contentType,
-          perm: "rw",
-        },
-      };
-    } catch (e) {
-      // console.error("Error while reading attachment", name, e);
-      throw Error(`Could not read attachment ${name}`);
-    }
-  }
-
-  async getAttachmentMeta(name: string): Promise<AttachmentMeta> {
-    const localPath = this.attachmentNameToPath(name);
-    try {
-      const s = await stat(localPath);
-      return {
-        name: name,
-        lastModified: s.mtime.getTime(),
-        size: s.size,
-        contentType: mime.lookup(name) || "application/octet-stream",
-        perm: "rw",
-      };
-    } catch (e) {
-      // console.error("Error while getting attachment meta", name, e);
       throw Error(`Could not get meta for ${name}`);
     }
   }
 
-  async writeAttachment(
+  async deleteFile(name: string): Promise<void> {
-    name: string,
+    let localPath = this.filenameToPath(name);
-    data: AttachmentData,
-    selfUpdate?: boolean,
-    lastModified?: number
-  ): Promise<AttachmentMeta> {
-    let localPath = this.attachmentNameToPath(name);
-    try {
-      // Ensure parent folder exists
-      await mkdir(path.dirname(localPath), { recursive: true });
-
-      // Actually write the file
-      if (typeof data === "string") {
-        await writeFile(localPath, data.split(",")[1], { encoding: "base64" });
-      } else {
-        await writeFile(localPath, Buffer.from(data));
-      }
-
-      if (lastModified) {
-        let d = new Date(lastModified);
-        console.log("Going to set the modified time", d);
-        await utimes(localPath, d, d);
-      }
-
-      // Fetch new metadata
-      const s = await stat(localPath);
-      return {
-        name: name,
-        lastModified: s.mtime.getTime(),
-        size: s.size,
-        contentType: mime.lookup(name) || "application/octet-stream",
-        perm: "rw",
-      };
-    } catch (e) {
-      console.error("Error while writing attachment", name, e);
-      throw Error(`Could not write ${name}`);
-    }
-  }
-
-  async deleteAttachment(name: string): Promise<void> {
-    let localPath = this.attachmentNameToPath(name);
     await unlink(localPath);
   }
 
+  async fetchFileList(): Promise<FileMeta[]> {
+    let fileList: FileMeta[] = [];
+
+    const walkPath = async (dir: string) => {
+      let files = await readdir(dir);
+      for (let file of files) {
+        if (file.startsWith(".")) {
+          continue;
+        }
+        const fullPath = path.join(dir, file);
+        let s = await stat(fullPath);
+        if (s.isDirectory()) {
+          await walkPath(fullPath);
+        } else {
+          if (!file.startsWith(".")) {
+            fileList.push({
+              name: this.pathToFilename(fullPath),
+              size: s.size,
+              contentType: lookupContentType(fullPath),
+              lastModified: s.mtime.getTime(),
+              perm: "rw",
+            });
+          }
+        }
+      }
+    };
+    await walkPath(this.rootPath);
+    return fileList;
+  }
+
   // Plugs
   invokeFunction(
     plug: Plug<any>,

packages/common/spaces/evented_space_primitives.ts

@@ -1,19 +1,14 @@
 import { EventHook } from "@plugos/plugos/hooks/event";
 import { Plug } from "@plugos/plugos/plug";
 
-import { AttachmentMeta, PageMeta } from "../types";
+import { FileMeta } from "../types";
-import { plugPrefix, trashPrefix } from "./constants";
+import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives";
-import {
-  AttachmentData,
-  AttachmentEncoding,
-  SpacePrimitives,
-} from "./space_primitives";
 
 export class EventedSpacePrimitives implements SpacePrimitives {
   constructor(private wrapped: SpacePrimitives, private eventHook: EventHook) {}
 
-  fetchPageList(): Promise<{ pages: Set<PageMeta>; nowTimestamp: number }> {
+  fetchFileList(): Promise<FileMeta[]> {
-    return this.wrapped.fetchPageList();
+    return this.wrapped.fetchFileList();
   }
 
   proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
@@ -29,26 +24,43 @@ export class EventedSpacePrimitives implements SpacePrimitives {
     return this.wrapped.invokeFunction(plug, env, name, args);
   }
 
-  readPage(pageName: string): Promise<{ text: string; meta: PageMeta }> {
+  readFile(
-    return this.wrapped.readPage(pageName);
+    name: string,
+    encoding: FileEncoding
+  ): Promise<{ data: FileData; meta: FileMeta }> {
+    return this.wrapped.readFile(name, encoding);
   }
 
-  async writePage(
+  async writeFile(
-    pageName: string,
+    name: string,
-    text: string,
+    encoding: FileEncoding,
-    selfUpdate: boolean,
+    data: FileData,
-    lastModified?: number
+    selfUpdate: boolean
-  ): Promise<PageMeta> {
+  ): Promise<FileMeta> {
-    const newPageMeta = await this.wrapped.writePage(
+    const newMeta = await this.wrapped.writeFile(
-      pageName,
+      name,
-      text,
+      encoding,
-      selfUpdate,
+      data,
-      lastModified
+      selfUpdate
     );
     // This can happen async
-    if (!pageName.startsWith(trashPrefix) && !pageName.startsWith(plugPrefix)) {
+    if (name.endsWith(".md")) {
+      const pageName = name.substring(0, name.length - 3);
+      let text = "";
+      switch (encoding) {
+        case "string":
+          text = data as string;
+          break;
+        case "arraybuffer":
+          const decoder = new TextDecoder("utf-8");
+          text = decoder.decode(data as ArrayBuffer);
+          break;
+        case "dataurl":
+          throw Error("Data urls not supported in this context");
+      }
+
       this.eventHook
-        .dispatchEvent("page:saved", pageName)
+        .dispatchEvent("page:saved")
         .then(() => {
           return this.eventHook.dispatchEvent("page:index_text", {
             name: pageName,
@@ -59,54 +71,18 @@ export class EventedSpacePrimitives implements SpacePrimitives {
           console.error("Error dispatching page:saved event", e);
         });
     }
-    return newPageMeta;
+    return newMeta;
   }
 
-  getPageMeta(pageName: string): Promise<PageMeta> {
+  getFileMeta(name: string): Promise<FileMeta> {
-    return this.wrapped.getPageMeta(pageName);
+    return this.wrapped.getFileMeta(name);
   }
 
-  async deletePage(pageName: string): Promise<void> {
+  async deleteFile(name: string): Promise<void> {
+    if (name.endsWith(".md")) {
+      const pageName = name.substring(0, name.length - 3);
       await this.eventHook.dispatchEvent("page:deleted", pageName);
-    return this.wrapped.deletePage(pageName);
     }
-
+    return this.wrapped.deleteFile(name);
-  fetchAttachmentList(): Promise<{
-    attachments: Set<AttachmentMeta>;
-    nowTimestamp: number;
-  }> {
-    return this.wrapped.fetchAttachmentList();
-  }
-
-  readAttachment(
-    name: string,
-    encoding: AttachmentEncoding
-  ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> {
-    return this.wrapped.readAttachment(name, encoding);
-  }
-
-  getAttachmentMeta(name: string): Promise<AttachmentMeta> {
-    return this.wrapped.getAttachmentMeta(name);
-  }
-
-  async writeAttachment(
-    name: string,
-    blob: ArrayBuffer,
-    selfUpdate?: boolean | undefined,
-    lastModified?: number | undefined
-  ): Promise<AttachmentMeta> {
-    let meta = await this.wrapped.writeAttachment(
-      name,
-      blob,
-      selfUpdate,
-      lastModified
-    );
-    await this.eventHook.dispatchEvent("attachment:saved", name);
-    return meta;
-  }
-
-  async deleteAttachment(name: string): Promise<void> {
-    await this.eventHook.dispatchEvent("attachment:deleted", name);
-    return this.wrapped.deleteAttachment(name);
   }
 }

packages/common/spaces/http_space_primitives.ts

@@ -1,20 +1,14 @@
-import { AttachmentMeta, PageMeta } from "../types";
+import { AttachmentMeta, FileMeta, PageMeta } from "../types";
 import { Plug } from "@plugos/plugos/plug";
-import {
+import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives";
-  AttachmentData,
-  AttachmentEncoding,
-  SpacePrimitives,
-} from "./space_primitives";
 
 export class HttpSpacePrimitives implements SpacePrimitives {
   fsUrl: string;
-  fsaUrl: string;
   private plugUrl: string;
   token?: string;
 
   constructor(url: string, token?: string) {
-    this.fsUrl = url + "/page";
+    this.fsUrl = url + "/fs";
-    this.fsaUrl = url + "/attachment";
     this.plugUrl = url + "/plug";
     this.token = token;
   }
@@ -34,72 +28,105 @@ export class HttpSpacePrimitives implements SpacePrimitives {
     return result;
   }
 
-  public async fetchPageList(): Promise<{
+  public async fetchFileList(): Promise<FileMeta[]> {
-    pages: Set<PageMeta>;
-    nowTimestamp: number;
-  }> {
     let req = await this.authenticatedFetch(this.fsUrl, {
       method: "GET",
     });
 
-    let result = new Set<PageMeta>();
+    let result: FileMeta[] = await req.json();
-    ((await req.json()) as any[]).forEach((meta: any) => {
-      const pageName = meta.name;
-      result.add({
-        name: pageName,
-        lastModified: meta.lastModified,
-        perm: "rw",
-      });
-    });
 
-    return {
+    return result;
-      pages: result,
-      nowTimestamp: +req.headers.get("Now-Timestamp")!,
-    };
   }
 
-  async readPage(name: string): Promise<{ text: string; meta: PageMeta }> {
+  async readFile(
+    name: string,
+    encoding: FileEncoding
+  ): Promise<{ data: FileData; meta: FileMeta }> {
     let res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
       method: "GET",
     });
-    if (res.headers.get("X-Status") === "404") {
+    if (res.status === 404) {
       throw new Error(`Page not found`);
     }
+    let data: FileData | null = null;
+    switch (encoding) {
+      case "arraybuffer":
+        let abBlob = await res.blob();
+        data = await abBlob.arrayBuffer();
+        break;
+      case "dataurl":
+        let dUBlob = await res.blob();
+        data = arrayBufferToDataUrl(await dUBlob.arrayBuffer());
+        break;
+      case "string":
+        data = await res.text();
+        break;
+    }
     return {
-      text: await res.text(),
+      data: data,
-      meta: this.responseToPageMeta(name, res),
+      meta: this.responseToMeta(name, res),
     };
   }
 
-  async writePage(
+  async writeFile(
     name: string,
-    text: string,
+    encoding: FileEncoding,
-    selfUpdate?: boolean,
+    data: FileData,
-    lastModified?: number
+    selfUpdate?: boolean
-  ): Promise<PageMeta> {
+  ): Promise<FileMeta> {
-    // TODO: lastModified ignored for now
+    let body: any = null;
+
+    switch (encoding) {
+      case "arraybuffer":
+      case "string":
+        body = data;
+        break;
+      case "dataurl":
+        data = dataUrlToArrayBuffer(data as string);
+        break;
+    }
     let res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
       method: "PUT",
-      body: text,
+      headers: {
-      headers: lastModified
+        "Content-type": "application/octet-stream",
-        ? {
+      },
-            "Last-Modified": "" + lastModified,
+      body,
-          }
-        : undefined,
     });
-    const newMeta = this.responseToPageMeta(name, res);
+    const newMeta = this.responseToMeta(name, res);
     return newMeta;
   }
 
-  async deletePage(name: string): Promise<void> {
+  async deleteFile(name: string): Promise<void> {
     let req = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
       method: "DELETE",
     });
     if (req.status !== 200) {
-      throw Error(`Failed to delete page: ${req.statusText}`);
+      throw Error(`Failed to delete file: ${req.statusText}`);
     }
   }
 
+  async getFileMeta(name: string): Promise<FileMeta> {
+    let res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
+      method: "OPTIONS",
+    });
+    if (res.status === 404) {
+      throw new Error(`File not found`);
+    }
+    return this.responseToMeta(name, res);
+  }
+
+  private responseToMeta(name: string, res: Response): FileMeta {
+    return {
+      name,
+      size: +res.headers.get("Content-length")!,
+      contentType: res.headers.get("Content-type")!,
+      lastModified: +(res.headers.get("Last-Modified") || "0"),
+      perm: (res.headers.get("X-Permission") as "rw" | "ro") || "rw",
+    };
+  }
+
+  // Plugs
+
   async proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
     let req = await this.authenticatedFetch(
       `${this.plugUrl}/${plug.name}/syscall/${name}`,
@@ -121,95 +148,6 @@ export class HttpSpacePrimitives implements SpacePrimitives {
     return await req.json();
   }
 
-  // Attachments
-  public async fetchAttachmentList(): Promise<{
-    attachments: Set<AttachmentMeta>;
-    nowTimestamp: number;
-  }> {
-    let req = await this.authenticatedFetch(this.fsaUrl, {
-      method: "GET",
-    });
-
-    let result = new Set<AttachmentMeta>();
-    ((await req.json()) as any[]).forEach((meta: any) => {
-      const pageName = meta.name;
-      result.add({
-        name: pageName,
-        size: meta.size,
-        lastModified: meta.lastModified,
-        contentType: meta.contentType,
-        perm: "rw",
-      });
-    });
-
-    return {
-      attachments: result,
-      nowTimestamp: +req.headers.get("Now-Timestamp")!,
-    };
-  }
-
-  async readAttachment(
-    name: string,
-    encoding: AttachmentEncoding
-  ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> {
-    let res = await this.authenticatedFetch(`${this.fsaUrl}/${name}`, {
-      method: "GET",
-    });
-    if (res.headers.get("X-Status") === "404") {
-      throw new Error(`Page not found`);
-    }
-    let blob = await res.blob();
-    return {
-      data:
-        encoding === "arraybuffer"
-          ? await blob.arrayBuffer()
-          : arrayBufferToDataUrl(await blob.arrayBuffer()),
-      meta: this.responseToAttachmentMeta(name, res),
-    };
-  }
-
-  async writeAttachment(
-    name: string,
-    data: AttachmentData,
-    selfUpdate?: boolean,
-    lastModified?: number
-  ): Promise<AttachmentMeta> {
-    if (typeof data === "string") {
-      data = dataUrlToArrayBuffer(data);
-    }
-    let res = await this.authenticatedFetch(`${this.fsaUrl}/${name}`, {
-      method: "PUT",
-      body: data,
-      headers: {
-        "Last-Modified": lastModified ? "" + lastModified : undefined,
-        "Content-type": "application/octet-stream",
-      },
-    });
-    const newMeta = this.responseToAttachmentMeta(name, res);
-    return newMeta;
-  }
-
-  async getAttachmentMeta(name: string): Promise<AttachmentMeta> {
-    let res = await this.authenticatedFetch(`${this.fsaUrl}/${name}`, {
-      method: "OPTIONS",
-    });
-    if (res.headers.get("X-Status") === "404") {
-      throw new Error(`Page not found`);
-    }
-    return this.responseToAttachmentMeta(name, res);
-  }
-
-  async deleteAttachment(name: string): Promise<void> {
-    let req = await this.authenticatedFetch(`${this.fsaUrl}/${name}`, {
-      method: "DELETE",
-    });
-    if (req.status !== 200) {
-      throw Error(`Failed to delete attachment: ${req.statusText}`);
-    }
-  }
-
-  // Plugs
-
   async invokeFunction(
     plug: Plug<any>,
     env: string,
@@ -244,38 +182,6 @@ export class HttpSpacePrimitives implements SpacePrimitives {
       return await req.text();
     }
   }
-
-  async getPageMeta(name: string): Promise<PageMeta> {
-    let res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
-      method: "OPTIONS",
-    });
-    if (res.headers.get("X-Status") === "404") {
-      throw new Error(`Page not found`);
-    }
-    return this.responseToPageMeta(name, res);
-  }
-
-  private responseToPageMeta(name: string, res: Response): PageMeta {
-    return {
-      name,
-      lastModified: +(res.headers.get("Last-Modified") || "0"),
-      perm: (res.headers.get("X-Permission") as "rw" | "ro") || "rw",
-    };
-  }
-
-  private responseToAttachmentMeta(
-    name: string,
-    res: Response
-  ): AttachmentMeta {
-    return {
-      name,
-      lastModified: +(res.headers.get("Last-Modified") || "0"),
-      size: +(res.headers.get("Content-Length") || "0"),
-      contentType:
-        res.headers.get("Content-Type") || "application/octet-stream",
-      perm: (res.headers.get("X-Permission") as "rw" | "ro") || "rw",
-    };
-  }
 }
 
 function dataUrlToArrayBuffer(dataUrl: string): ArrayBuffer {

packages/common/spaces/indexeddb_space_primitives.ts

@@ -1,117 +0,0 @@
-import {
-  AttachmentData,
-  AttachmentEncoding,
-  SpacePrimitives,
-} from "./space_primitives";
-import { AttachmentMeta, PageMeta } from "../types";
-import Dexie, { Table } from "dexie";
-import { Plug } from "@plugos/plugos/plug";
-
-type Page = {
-  name: string;
-  text: string;
-  meta: PageMeta;
-};
-
-export class IndexedDBSpacePrimitives implements SpacePrimitives {
-  private pageTable: Table<Page, string>;
-
-  constructor(dbName: string, readonly timeSkew: number = 0) {
-    const db = new Dexie(dbName);
-    db.version(1).stores({
-      page: "name",
-    });
-    this.pageTable = db.table("page");
-  }
-  fetchAttachmentList(): Promise<{
-    attachments: Set<AttachmentMeta>;
-    nowTimestamp: number;
-  }> {
-    throw new Error("Method not implemented.");
-  }
-  readAttachment(
-    name: string,
-    encoding: AttachmentEncoding
-  ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> {
-    throw new Error("Method not implemented.");
-  }
-  getAttachmentMeta(name: string): Promise<AttachmentMeta> {
-    throw new Error("Method not implemented.");
-  }
-  writeAttachment(
-    name: string,
-    blob: ArrayBuffer,
-    selfUpdate?: boolean | undefined,
-    lastModified?: number | undefined
-  ): Promise<AttachmentMeta> {
-    throw new Error("Method not implemented.");
-  }
-  deleteAttachment(name: string): Promise<void> {
-    throw new Error("Method not implemented.");
-  }
-
-  async deletePage(name: string): Promise<void> {
-    return this.pageTable.delete(name);
-  }
-
-  async getPageMeta(name: string): Promise<PageMeta> {
-    let entry = await this.pageTable.get(name);
-    if (entry) {
-      return entry.meta;
-    } else {
-      throw Error(`Page not found`);
-    }
-  }
-
-  invokeFunction(
-    plug: Plug<any>,
-    env: string,
-    name: string,
-    args: any[]
-  ): Promise<any> {
-    return plug.invoke(name, args);
-  }
-
-  async fetchPageList(): Promise<{
-    pages: Set<PageMeta>;
-    nowTimestamp: number;
-  }> {
-    let allPages = await this.pageTable.toArray();
-    return {
-      pages: new Set(allPages.map((p) => p.meta)),
-      nowTimestamp: Date.now() + this.timeSkew,
-    };
-  }
-
-  proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
-    return plug.syscall(name, args);
-  }
-
-  async readPage(name: string): Promise<{ text: string; meta: PageMeta }> {
-    let page = await this.pageTable.get(name);
-    if (page) {
-      return page;
-    } else {
-      throw new Error("Page not found");
-    }
-  }
-
-  async writePage(
-    name: string,
-    text: string,
-    selfUpdate?: boolean,
-    lastModified?: number
-  ): Promise<PageMeta> {
-    const meta: PageMeta = {
-      name,
-      lastModified: lastModified ? lastModified : Date.now() + this.timeSkew,
-      perm: "rw",
-    };
-    await this.pageTable.put({
-      name,
-      text,
-      meta,
-    });
-    return meta;
-  }
-}

packages/common/spaces/space.ts

@@ -1,13 +1,8 @@
-import {
+import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives";
-  AttachmentData,
+import { AttachmentMeta, FileMeta, PageMeta } from "../types";
-  AttachmentEncoding,
-  SpacePrimitives,
-} from "./space_primitives";
-import { AttachmentMeta, PageMeta } from "../types";
 import { EventEmitter } from "@plugos/plugos/event";
 import { Plug } from "@plugos/plugos/plug";
-import { Manifest } from "../manifest";
+import { plugPrefix } from "./constants";
-import { plugPrefix, trashPrefix } from "./constants";
 import { safeRun } from "../util";
 
 const pageWatchInterval = 2000;
@@ -19,23 +14,21 @@ export type SpaceEvents = {
   pageListUpdated: (pages: Set<PageMeta>) => void;
 };
 
-export class Space
+export class Space extends EventEmitter<SpaceEvents> {
-  extends EventEmitter<SpaceEvents>
-  implements SpacePrimitives
-{
   pageMetaCache = new Map<string, PageMeta>();
   watchedPages = new Set<string>();
   private initialPageListLoad = true;
   private saving = false;
 
-  constructor(private space: SpacePrimitives, private trashEnabled = true) {
+  constructor(private space: SpacePrimitives) {
     super();
   }
 
   public async updatePageList() {
-    let newPageList = await this.space.fetchPageList();
+    let newPageList = await this.fetchPageList();
+    // console.log("Updating page list", newPageList);
     let deletedPages = new Set<string>(this.pageMetaCache.keys());
-    newPageList.pages.forEach((meta) => {
+    newPageList.forEach((meta) => {
       const pageName = meta.name;
       const oldPageMeta = this.pageMetaCache.get(pageName);
       const newPageMeta: PageMeta = {
@@ -50,9 +43,7 @@ export class Space
         this.emit("pageCreated", newPageMeta);
       } else if (
         oldPageMeta &&
-        oldPageMeta.lastModified !== newPageMeta.lastModified &&
+        oldPageMeta.lastModified !== newPageMeta.lastModified
-        (!this.trashEnabled ||
-          (this.trashEnabled && !pageName.startsWith(trashPrefix)))
       ) {
         this.emit("pageChanged", newPageMeta);
       }
@@ -95,17 +86,7 @@ export class Space
 
   async deletePage(name: string, deleteDate?: number): Promise<void> {
     await this.getPageMeta(name); // Check if page exists, if not throws Error
-    if (this.trashEnabled) {
+    await this.space.deleteFile(`${name}.md`);
-      let pageData = await this.readPage(name);
-      // Move to trash
-      await this.writePage(
-        `${trashPrefix}${name}`,
-        pageData.text,
-        true,
-        deleteDate
-      );
-    }
-    await this.space.deletePage(name);
 
     this.pageMetaCache.delete(name);
     this.emit("pageDeleted", name);
@@ -114,7 +95,9 @@ export class Space
 
   async getPageMeta(name: string): Promise<PageMeta> {
     let oldMeta = this.pageMetaCache.get(name);
-    let newMeta = await this.space.getPageMeta(name);
+    let newMeta = fileMetaToPageMeta(
+      await this.space.getFileMeta(`${name}.md`)
+    );
     if (oldMeta) {
       if (oldMeta.lastModified !== newMeta.lastModified) {
         // Changed on disk, trigger event
@@ -133,41 +116,15 @@ export class Space
     return this.space.invokeFunction(plug, env, name, args);
   }
 
-  listPages(unfiltered = false): Set<PageMeta> {
+  listPages(): Set<PageMeta> {
-    if (unfiltered) {
     return new Set(this.pageMetaCache.values());
-    } else {
-      return new Set(
-        [...this.pageMetaCache.values()].filter(
-          (pageMeta) =>
-            !pageMeta.name.startsWith(trashPrefix) &&
-            !pageMeta.name.startsWith(plugPrefix)
-        )
-      );
-    }
   }
 
-  listTrash(): Set<PageMeta> {
+  async listPlugs(): Promise<string[]> {
-    return new Set(
+    let allFiles = await this.space.fetchFileList();
-      [...this.pageMetaCache.values()]
+    return allFiles
-        .filter(
+      .filter((fileMeta) => fileMeta.name.endsWith(".plug.json"))
-          (pageMeta) =>
+      .map((fileMeta) => fileMeta.name);
-            pageMeta.name.startsWith(trashPrefix) &&
-            !pageMeta.name.startsWith(plugPrefix)
-        )
-        .map((pageMeta) => ({
-          ...pageMeta,
-          name: pageMeta.name.substring(trashPrefix.length),
-        }))
-    );
-  }
-
-  listPlugs(): Set<PageMeta> {
-    return new Set(
-      [...this.pageMetaCache.values()].filter((pageMeta) =>
-        pageMeta.name.startsWith(plugPrefix)
-      )
-    );
   }
 
   proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {
@@ -175,16 +132,20 @@ export class Space
   }
 
   async readPage(name: string): Promise<{ text: string; meta: PageMeta }> {
-    let pageData = await this.space.readPage(name);
+    let pageData = await this.space.readFile(`${name}.md`, "string");
     let previousMeta = this.pageMetaCache.get(name);
+    let newMeta = fileMetaToPageMeta(pageData.meta);
     if (previousMeta) {
-      if (previousMeta.lastModified !== pageData.meta.lastModified) {
+      if (previousMeta.lastModified !== newMeta.lastModified) {
         // Page changed since last cached metadata, trigger event
-        this.emit("pageChanged", pageData.meta);
+        this.emit("pageChanged", newMeta);
       }
     }
-    this.pageMetaCache.set(name, pageData.meta);
+    let meta = this.metaCacher(name, newMeta);
-    return pageData;
+    return {
+      text: pageData.data as string,
+      meta: meta,
+    };
   }
 
   watchPage(pageName: string) {
@@ -198,16 +159,12 @@ export class Space
   async writePage(
     name: string,
     text: string,
-    selfUpdate?: boolean,
+    selfUpdate?: boolean
-    lastModified?: number
   ): Promise<PageMeta> {
     try {
       this.saving = true;
-      let pageMeta = await this.space.writePage(
+      let pageMeta = fileMetaToPageMeta(
-        name,
+        await this.space.writeFile(`${name}.md`, "string", text, selfUpdate)
-        text,
-        selfUpdate,
-        lastModified
       );
       if (!selfUpdate) {
         this.emit("pageChanged", pageMeta);
@@ -218,39 +175,52 @@ export class Space
     }
   }
 
-  fetchPageList(): Promise<{ pages: Set<PageMeta>; nowTimestamp: number }> {
+  async fetchPageList(): Promise<PageMeta[]> {
-    return this.space.fetchPageList();
+    return (await this.space.fetchFileList())
+      .filter((fileMeta) => fileMeta.name.endsWith(".md"))
+      .map(fileMetaToPageMeta);
   }
 
-  fetchAttachmentList(): Promise<{
+  async fetchAttachmentList(): Promise<AttachmentMeta[]> {
-    attachments: Set<AttachmentMeta>;
+    return (await this.space.fetchFileList()).filter(
-    nowTimestamp: number;
+      (fileMeta) =>
-  }> {
+        !fileMeta.name.endsWith(".md") && !fileMeta.name.endsWith(".plug.json")
-    return this.space.fetchAttachmentList();
+    );
   }
+
   readAttachment(
     name: string,
-    encoding: AttachmentEncoding
+    encoding: FileEncoding
-  ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> {
+  ): Promise<{ data: FileData; meta: AttachmentMeta }> {
-    return this.space.readAttachment(name, encoding);
+    return this.space.readFile(name, encoding);
-  }
-  getAttachmentMeta(name: string): Promise<AttachmentMeta> {
-    return this.space.getAttachmentMeta(name);
-  }
-  writeAttachment(
-    name: string,
-    data: AttachmentData,
-    selfUpdate?: boolean | undefined,
-    lastModified?: number | undefined
-  ): Promise<AttachmentMeta> {
-    return this.space.writeAttachment(name, data, selfUpdate, lastModified);
-  }
-  deleteAttachment(name: string): Promise<void> {
-    return this.space.deleteAttachment(name);
   }
 
-  private metaCacher(name: string, pageMeta: PageMeta): PageMeta {
+  getAttachmentMeta(name: string): Promise<AttachmentMeta> {
-    this.pageMetaCache.set(name, pageMeta);
+    return this.space.getFileMeta(name);
-    return pageMeta;
+  }
+
+  writeAttachment(
+    name: string,
+    encoding: FileEncoding,
+    data: FileData,
+    selfUpdate?: boolean | undefined
+  ): Promise<AttachmentMeta> {
+    return this.space.writeFile(name, encoding, data, selfUpdate);
+  }
+
+  deleteAttachment(name: string): Promise<void> {
+    return this.space.deleteFile(name);
+  }
+
+  private metaCacher(name: string, meta: PageMeta): PageMeta {
+    this.pageMetaCache.set(name, meta);
+    return meta;
   }
 }
+
+function fileMetaToPageMeta(fileMeta: FileMeta): PageMeta {
+  return {
+    ...fileMeta,
+    name: fileMeta.name.substring(0, fileMeta.name.length - 3),
+  } as PageMeta;
+}

packages/common/spaces/space_primitives.ts

@@ -1,38 +1,23 @@
 import { Plug } from "@plugos/plugos/plug";
-import { AttachmentMeta, PageMeta } from "../types";
+import { FileMeta } from "../types";
 
-export type AttachmentEncoding = "arraybuffer" | "dataurl";
+export type FileEncoding = "string" | "arraybuffer" | "dataurl";
-export type AttachmentData = ArrayBuffer | string;
+export type FileData = ArrayBuffer | string;
 export interface SpacePrimitives {
   // Pages
-  fetchPageList(): Promise<{ pages: Set<PageMeta>; nowTimestamp: number }>;
+  fetchFileList(): Promise<FileMeta[]>;
-  readPage(name: string): Promise<{ text: string; meta: PageMeta }>;
+  readFile(
-  getPageMeta(name: string): Promise<PageMeta>;
-  writePage(
     name: string,
-    text: string,
+    encoding: FileEncoding
-    selfUpdate?: boolean,
+  ): Promise<{ data: FileData; meta: FileMeta }>;
-    lastModified?: number
+  getFileMeta(name: string): Promise<FileMeta>;
-  ): Promise<PageMeta>;
+  writeFile(
-  deletePage(name: string): Promise<void>;
-
-  // Attachments
-  fetchAttachmentList(): Promise<{
-    attachments: Set<AttachmentMeta>;
-    nowTimestamp: number;
-  }>;
-  readAttachment(
     name: string,
-    encoding: AttachmentEncoding
+    encoding: FileEncoding,
-  ): Promise<{ data: AttachmentData; meta: AttachmentMeta }>;
+    data: FileData,
-  getAttachmentMeta(name: string): Promise<AttachmentMeta>;
+    selfUpdate?: boolean
-  writeAttachment(
+  ): Promise<FileMeta>;
-    name: string,
+  deleteFile(name: string): Promise<void>;
-    data: AttachmentData,
-    selfUpdate?: boolean,
-    lastModified?: number
-  ): Promise<AttachmentMeta>;
-  deleteAttachment(name: string): Promise<void>;
 
   // Plugs
   proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any>;

packages/common/spaces/sync.test.ts

@@ -1,123 +0,0 @@
-import { expect, test } from "@jest/globals";
-import { IndexedDBSpacePrimitives } from "./indexeddb_space_primitives";
-import { SpaceSync } from "./sync";
-import { PageMeta } from "../types";
-import { Space } from "./space";
-
-// For testing in node.js
-require("fake-indexeddb/auto");
-
-test("Test store", async () => {
-  let primary = new Space(new IndexedDBSpacePrimitives("primary"), true);
-  let secondary = new Space(
-    new IndexedDBSpacePrimitives("secondary", -5000),
-    true
-  );
-  let sync = new SpaceSync(primary, secondary, 0, 0, "_trash/");
-
-  async function conflictResolver(pageMeta1: PageMeta, pageMeta2: PageMeta) {}
-
-  // Write one page to primary
-  await primary.writePage("index", "Hello");
-  expect((await secondary.listPages()).size).toBe(0);
-  await syncPages(conflictResolver);
-  expect((await secondary.listPages()).size).toBe(1);
-  expect((await secondary.readPage("index")).text).toBe("Hello");
-
-  // Should be a no-op
-  expect(await syncPages()).toBe(0);
-
-  // Now let's make a change on the secondary
-  await secondary.writePage("index", "Hello!!");
-  await secondary.writePage("test", "Test page");
-
-  // And sync it
-  await syncPages();
-
-  expect(primary.listPages().size).toBe(2);
-  expect(secondary.listPages().size).toBe(2);
-
-  expect((await primary.readPage("index")).text).toBe("Hello!!");
-
-  // Let's make some random edits on both ends
-  await primary.writePage("index", "1");
-  await primary.writePage("index2", "2");
-  await secondary.writePage("index3", "3");
-  await secondary.writePage("index4", "4");
-  await syncPages();
-
-  expect((await primary.listPages()).size).toBe(5);
-  expect((await secondary.listPages()).size).toBe(5);
-
-  expect(await syncPages()).toBe(0);
-
-  console.log("Deleting pages");
-  // Delete some pages
-  await primary.deletePage("index");
-  await primary.deletePage("index3");
-
-  console.log("Pages", await primary.listPages());
-  console.log("Trash", await primary.listTrash());
-
-  await syncPages();
-
-  expect((await primary.listPages()).size).toBe(3);
-  expect((await secondary.listPages()).size).toBe(3);
-
-  // No-op
-  expect(await syncPages()).toBe(0);
-
-  await secondary.deletePage("index4");
-  await primary.deletePage("index2");
-
-  await syncPages();
-
-  // Just "test" left
-  expect((await primary.listPages()).size).toBe(1);
-  expect((await secondary.listPages()).size).toBe(1);
-
-  // No-op
-  expect(await syncPages()).toBe(0);
-
-  await secondary.writePage("index", "I'm back");
-
-  await syncPages();
-
-  expect((await primary.readPage("index")).text).toBe("I'm back");
-
-  // Cause a conflict
-  await primary.writePage("index", "Hello 1");
-  await secondary.writePage("index", "Hello 2");
-
-  await syncPages(SpaceSync.primaryConflictResolver(primary, secondary));
-
-  // Sync conflicting copy back
-  await syncPages();
-
-  // Verify that primary won
-  expect((await primary.readPage("index")).text).toBe("Hello 1");
-  expect((await secondary.readPage("index")).text).toBe("Hello 1");
-
-  // test + index + index.conflicting copy
-  expect((await primary.listPages()).size).toBe(3);
-  expect((await secondary.listPages()).size).toBe(3);
-
-  async function syncPages(
-    conflictResolver?: (
-      pageMeta1: PageMeta,
-      pageMeta2: PageMeta
-    ) => Promise<void>
-  ): Promise<number> {
-    // Awesome practice: adding sleeps to fix issues!
-    await sleep(2);
-    let n = await sync.syncPages(conflictResolver);
-    await sleep(2);
-    return n;
-  }
-});
-
-function sleep(ms: number = 5): Promise<void> {
-  return new Promise((resolve) => {
-    setTimeout(resolve, ms);
-  });
-}

packages/common/spaces/sync.ts

@@ -1,208 +0,0 @@
-import { Space } from "./space";
-import { PageMeta } from "../types";
-import { SpacePrimitives } from "./space_primitives";
-
-export class SpaceSync {
-  constructor(
-    private primary: Space,
-    private secondary: Space,
-    public primaryLastSync: number,
-    public secondaryLastSync: number,
-    private trashPrefix: string
-  ) {}
-
-  // Strategy: Primary wins
-  public static primaryConflictResolver(
-    primary: Space,
-    secondary: Space
-  ): (pageMeta1: PageMeta, pageMeta2: PageMeta) => Promise<void> {
-    return async (pageMeta1, pageMeta2) => {
-      const pageName = pageMeta1.name;
-      const revisionPageName = `${pageName}.conflicted.${pageMeta2.lastModified}`;
-      // Copy secondary to conflict copy
-      let oldPageData = await secondary.readPage(pageName);
-      await secondary.writePage(revisionPageName, oldPageData.text);
-
-      // Write replacement on top
-      let newPageData = await primary.readPage(pageName);
-      await secondary.writePage(
-        pageName,
-        newPageData.text,
-        true,
-        newPageData.meta.lastModified
-      );
-    };
-  }
-
-  async syncablePages(
-    space: Space
-  ): Promise<{ pages: PageMeta[]; nowTimestamp: number }> {
-    let fetchResult = await space.fetchPageList();
-    return {
-      pages: [...fetchResult.pages].filter(
-        (pageMeta) => !pageMeta.name.startsWith(this.trashPrefix)
-      ),
-      nowTimestamp: fetchResult.nowTimestamp,
-    };
-  }
-
-  async trashPages(space: SpacePrimitives): Promise<PageMeta[]> {
-    return [...(await space.fetchPageList()).pages]
-      .filter((pageMeta) => pageMeta.name.startsWith(this.trashPrefix))
-      .map((pageMeta) => ({
-        ...pageMeta,
-        name: pageMeta.name.substring(this.trashPrefix.length),
-      }));
-  }
-
-  async syncPages(
-    conflictResolver?: (
-      pageMeta1: PageMeta,
-      pageMeta2: PageMeta
-    ) => Promise<void>
-  ): Promise<number> {
-    let syncOps = 0;
-
-    let { pages: primaryAllPagesSet, nowTimestamp: primarySyncTimestamp } =
-      await this.syncablePages(this.primary);
-    let allPagesPrimary = new Map(primaryAllPagesSet.map((p) => [p.name, p]));
-    let { pages: secondaryAllPagesSet, nowTimestamp: secondarySyncTimestamp } =
-      await this.syncablePages(this.secondary);
-    let allPagesSecondary = new Map(
-      secondaryAllPagesSet.map((p) => [p.name, p])
-    );
-
-    let allTrashPrimary = new Map(
-      (await this.trashPages(this.primary))
-        // Filter out old trash
-        .filter((p) => p.lastModified > this.primaryLastSync)
-        .map((p) => [p.name, p])
-    );
-    let allTrashSecondary = new Map(
-      (await this.trashPages(this.secondary))
-        // Filter out old trash
-        .filter((p) => p.lastModified > this.secondaryLastSync)
-        .map((p) => [p.name, p])
-    );
-
-    // Iterate over all pages on the primary first
-    for (let [name, pageMetaPrimary] of allPagesPrimary.entries()) {
-      let pageMetaSecondary = allPagesSecondary.get(pageMetaPrimary.name);
-      if (!pageMetaSecondary) {
-        // New page on primary
-        // Let's check it's not on the deleted list
-        if (allTrashSecondary.has(name)) {
-          // Explicitly deleted, let's skip
-          continue;
-        }
-
-        // Push from primary to secondary
-        console.log("New page on primary", name, "syncing to secondary");
-        let pageData = await this.primary.readPage(name);
-        await this.secondary.writePage(
-          name,
-          pageData.text,
-          true,
-          secondarySyncTimestamp // The reason for this is to not include it in the next sync cycle, we cannot blindly use the lastModified date due to time skew
-        );
-        syncOps++;
-      } else {
-        // Existing page
-        if (pageMetaPrimary.lastModified > this.primaryLastSync) {
-          // Primary updated since last sync
-          if (pageMetaSecondary.lastModified > this.secondaryLastSync) {
-            // Secondary also updated! CONFLICT
-            if (conflictResolver) {
-              await conflictResolver(pageMetaPrimary, pageMetaSecondary);
-            } else {
-              throw Error(
-                `Sync conflict for ${name} with no conflict resolver specified`
-              );
-            }
-          } else {
-            // Ok, not changed on secondary, push it secondary
-            console.log(
-              "Changed page on primary",
-              name,
-              "syncing to secondary"
-            );
-            let pageData = await this.primary.readPage(name);
-            await this.secondary.writePage(
-              name,
-              pageData.text,
-              false,
-              secondarySyncTimestamp
-            );
-            syncOps++;
-          }
-        } else if (pageMetaSecondary.lastModified > this.secondaryLastSync) {
-          // Secondary updated, but not primary (checked above)
-          // Push from secondary to primary
-          console.log("Changed page on secondary", name, "syncing to primary");
-          let pageData = await this.secondary.readPage(name);
-          await this.primary.writePage(
-            name,
-            pageData.text,
-            false,
-            primarySyncTimestamp
-          );
-          syncOps++;
-        } else {
-          // Neither updated, no-op
-        }
-      }
-    }
-
-    // Now do a simplified version in reverse, only detecting new pages
-    for (let [name, pageMetaSecondary] of allPagesSecondary.entries()) {
-      if (!allPagesPrimary.has(pageMetaSecondary.name)) {
-        // New page on secondary
-        // Let's check it's not on the deleted list
-        if (allTrashPrimary.has(name)) {
-          // Explicitly deleted, let's skip
-          continue;
-        }
-        // Push from secondary to primary
-        console.log("New page on secondary", name, "pushing to primary");
-        let pageData = await this.secondary.readPage(name);
-        await this.primary.writePage(
-          name,
-          pageData.text,
-          false,
-          primarySyncTimestamp
-        );
-        syncOps++;
-      }
-    }
-
-    // And finally, let's trash some pages
-    for (let pageToDelete of allTrashPrimary.values()) {
-      console.log("Deleting", pageToDelete.name, "on secondary");
-      try {
-        await this.secondary.deletePage(
-          pageToDelete.name,
-          secondarySyncTimestamp
-        );
-        syncOps++;
-      } catch (e: any) {
-        console.log("Page already gone", e.message);
-      }
-    }
-
-    for (let pageToDelete of allTrashSecondary.values()) {
-      console.log("Deleting", pageToDelete.name, "on primary");
-      try {
-        await this.primary.deletePage(pageToDelete.name, primarySyncTimestamp);
-        syncOps++;
-      } catch (e: any) {
-        console.log("Page already gone", e.message);
-      }
-    }
-
-    // Setting last sync time to the timestamps we got back when fetching the page lists on each end
-    this.primaryLastSync = primarySyncTimestamp;
-    this.secondaryLastSync = secondarySyncTimestamp;
-
-    return syncOps;
-  }
-}

packages/common/types.ts

@@ -1,6 +1,13 @@
-export const reservedPageNames = ["page", "attachment", "plug"];
 export const maximumAttachmentSize = 100 * 1024 * 1024; // 100 MB
 
+export type FileMeta = {
+  name: string;
+  lastModified: number;
+  contentType: string;
+  size: number;
+  perm: "ro" | "rw";
+};
+
 export type PageMeta = {
   name: string;
   lastModified: number;

packages/plugos-silverbullet-syscall/space.ts

@@ -23,6 +23,10 @@ export async function deletePage(name: string): Promise<void> {
   return syscall("space.deletePage", name);
 }
 
+export async function listPlugs(): Promise<string[]> {
+  return syscall("space.listPlugs");
+}
+
 export async function listAttachments(): Promise<PageMeta[]> {
   return syscall("space.listAttachments");
 }
@@ -39,9 +43,10 @@ export async function readAttachment(
 
 export async function writeAttachment(
   name: string,
-  buffer: ArrayBuffer
+  encoding: "string" | "dataurl",
+  data: string
 ): Promise<AttachmentMeta> {
-  return syscall("space.writeAttachment", name, buffer);
+  return syscall("space.writeAttachment", name, encoding, data);
 }
 
 export async function deleteAttachment(name: string): Promise<void> {

packages/plugos-syscall/fulltext.ts

@@ -5,7 +5,7 @@ export async function fullTextIndex(key: string, value: string) {
 }
 
 export async function fullTextDelete(key: string) {
-  return syscall("fulltext.index", key);
+  return syscall("fulltext.delete", key);
 }
 
 export async function fullTextSearch(phrase: string, limit: number = 100) {

packages/plugos/environments/node_worker.ts

@@ -40,6 +40,8 @@ let vm = new VM({
     setInterval,
     URL,
     clearInterval,
+    TextEncoder,
+    TextDecoder,
     fetch: require(`${nodeModulesPath}/node-fetch`),
     WebSocket: require(`${nodeModulesPath}/ws`),
     // This is only going to be called for pre-bundled modules, we won't allow

packages/plugs/core/cloud.ts

@@ -1,16 +1,24 @@
+import type {
+  FileData,
+  FileEncoding,
+} from "@silverbulletmd/common/spaces/space_primitives";
 import {
   renderToText,
   replaceNodesMatching,
 } from "@silverbulletmd/common/tree";
-import { PageMeta } from "@silverbulletmd/common/types";
+import type { FileMeta, PageMeta } from "@silverbulletmd/common/types";
 import { parseMarkdown } from "@silverbulletmd/plugos-silverbullet-syscall/markdown";
 
 const pagePrefix = "💭 ";
 
-export async function readPageCloud(
+export async function readFileCloud(
-  name: string
+  name: string,
-): Promise<{ text: string; meta: PageMeta } | undefined> {
+  encoding: FileEncoding
-  let originalUrl = name.substring(pagePrefix.length);
+): Promise<{ data: FileData; meta: FileMeta } | undefined> {
+  let originalUrl = name.substring(
+    pagePrefix.length,
+    name.length - ".md".length
+  );
   let url = originalUrl;
   if (!url.includes("/")) {
     url += "/index";
@@ -32,13 +40,15 @@ export async function readPageCloud(
     text = e.message;
   }
   return {
-    text: await translateLinksWithPrefix(
+    data: await translateLinksWithPrefix(
       text,
       `${pagePrefix}${originalUrl.split("/")[0]}/`
     ),
     meta: {
       name,
+      contentType: "text/markdown",
       lastModified: 0,
+      size: text.length,
       perm: "ro",
     },
   };
@@ -60,9 +70,11 @@ async function translateLinksWithPrefix(
   return text;
 }
 
-export async function getPageMetaCloud(name: string): Promise<PageMeta> {
+export async function getFileMetaCloud(name: string): Promise<FileMeta> {
   return {
     name,
+    size: 0,
+    contentType: "text/markdown",
     lastModified: 0,
     perm: "ro",
   };

packages/plugs/core/core.plug.yaml

@@ -156,12 +156,12 @@ functions:
     path: ./search.ts:readPageSearch
     pageNamespace:
       pattern: "🔍 .+"
-      operation: readPage
+      operation: readFile
   getPageMetaSearch:
     path: ./search.ts:getPageMetaSearch
     pageNamespace:
       pattern: "🔍 .+"
-      operation: getPageMeta
+      operation: getFileMeta
 
   # Template commands
   insertPageMeta:
@@ -374,12 +374,12 @@ functions:
 
   # Cloud pages
   readPageCloud:
-    path: ./cloud.ts:readPageCloud
+    path: ./cloud.ts:readFileCloud
     pageNamespace:
       pattern: "💭 .+"
-      operation: readPage
+      operation: readFile
   getPageMetaCloud:
-    path: ./cloud.ts:getPageMetaCloud
+    path: ./cloud.ts:getFileMetaCloud
     pageNamespace:
       pattern: "💭 .+"
-      operation: getPageMeta
+      operation: getFileMeta

packages/plugs/core/navigate.ts

@@ -14,7 +14,7 @@ import { invokeCommand } from "@silverbulletmd/plugos-silverbullet-syscall/syste
 // Checks if the URL contains a protocol, if so keeps it, otherwise assumes an attachment
 function patchUrl(url: string): string {
   if (url.indexOf("://") === -1) {
-    return `attachment/${url}`;
+    return `fs/${url}`;
   }
   return url;
 }

packages/plugs/core/plugmanager.ts

@@ -5,9 +5,9 @@ import {
   save,
 } from "@silverbulletmd/plugos-silverbullet-syscall/editor";
 import {
-  deletePage,
+  deleteAttachment,
-  listPages,
+  listPlugs,
-  writePage,
+  writeAttachment,
 } from "@silverbulletmd/plugos-silverbullet-syscall/space";
 import {
   invokeFunction,
@@ -16,13 +16,6 @@ import {
 
 import { readYamlPage } from "../lib/yaml_page";
 
-async function listPlugs(): Promise<string[]> {
-  let unfilteredPages = await listPages(true);
-  return unfilteredPages
-    .filter((p) => p.name.startsWith("_plug/"))
-    .map((p) => p.name.substring("_plug/".length));
-}
-
 export async function updatePlugsCommand() {
   await save();
   flashNotification("Updating plugs...");
@@ -39,9 +32,11 @@ export async function updatePlugs() {
   let plugList: string[] = [];
   try {
     const plugListRead: any[] = await readYamlPage("PLUGS");
-    plugList = plugListRead.filter((plug) => typeof plug === 'string');
+    plugList = plugListRead.filter((plug) => typeof plug === "string");
     if (plugList.length !== plugListRead.length) {
-      throw new Error(`Some of the plugs were not in a yaml list format, they were ignored`);
+      throw new Error(
+        `Some of the plugs were not in a yaml list format, they were ignored`
+      );
     }
   } catch (e: any) {
     throw new Error(`Error processing PLUGS: ${e.message}`);
@@ -58,17 +53,23 @@ export async function updatePlugs() {
     let manifest = manifests[0];
     allPlugNames.push(manifest.name);
     // console.log("Writing", `_plug/${manifest.name}`);
-    await writePage(
+    await writeAttachment(
-      `_plug/${manifest.name}`,
+      `_plug/${manifest.name}.plug.json`,
-      JSON.stringify(manifest, null, 2)
+      "string",
+      JSON.stringify(manifest)
     );
   }
 
   // And delete extra ones
   for (let existingPlug of await listPlugs()) {
-    if (!allPlugNames.includes(existingPlug)) {
+    let plugName = existingPlug.substring(
-      console.log("Removing plug", existingPlug);
+      "_plug/".length,
-      await deletePage(`_plug/${existingPlug}`);
+      existingPlug.length - ".plug.json".length
+    );
+    console.log("Considering", plugName);
+    if (!allPlugNames.includes(plugName)) {
+      console.log("Removing plug", plugName);
+      await deleteAttachment(existingPlug);
     }
   }
   await reloadPlugs();
@@ -97,13 +98,19 @@ export async function getPlugGithub(identifier: string): Promise<Manifest> {
   );
 }
 
-export async function getPlugGithubRelease(identifier: string): Promise<Manifest> {
+export async function getPlugGithubRelease(
+  identifier: string
+): Promise<Manifest> {
   let [owner, repo, version] = identifier.split("/");
   if (!version || version === "latest") {
-    console.log('fetching the latest version');
+    console.log("fetching the latest version");
-    const req = await fetch(`https://api.github.com/repos/${owner}/${repo}/releases/latest`);
+    const req = await fetch(
+      `https://api.github.com/repos/${owner}/${repo}/releases/latest`
+    );
     if (req.status !== 200) {
-      throw new Error(`Could not fetch latest relase manifest from ${identifier}}`);
+      throw new Error(
+        `Could not fetch latest relase manifest from ${identifier}}`
+      );
     }
     const result = await req.json();
     version = result.name;

packages/plugs/markdown/util.ts

@@ -44,7 +44,7 @@ export async function cleanMarkdown(
     if (n.type === "URL") {
       const url = n.children![0].text!;
       if (url.indexOf("://") === -1) {
-        n.children![0].text = `attachment/${url}`;
+        n.children![0].text = `fs/${url}`;
       }
       console.log("Link", url);
     }

packages/server/express_server.ts

@@ -3,7 +3,6 @@ import { Manifest, SilverBulletHooks } from "@silverbulletmd/common/manifest";
 import { EndpointHook } from "@plugos/plugos/hooks/endpoint";
 import { readdir, readFile } from "fs/promises";
 import { System } from "@plugos/plugos/system";
-import cors from "cors";
 import { DiskSpacePrimitives } from "@silverbulletmd/common/spaces/disk_space_primitives";
 import path from "path";
 import bodyParser from "body-parser";
@@ -32,14 +31,6 @@ import { plugPrefix } from "@silverbulletmd/common/spaces/constants";
 import sandboxSyscalls from "@plugos/plugos/syscalls/sandbox";
 // @ts-ignore
 import settingsTemplate from "bundle-text:./SETTINGS_template.md";
-
-const globalModules: any = JSON.parse(
-  readFileSync(
-    nodeModulesDir + "/node_modules/@silverbulletmd/web/dist/global.plug.json",
-    "utf-8"
-  )
-);
-
 import { safeRun } from "./util";
 import {
   ensureFTSTable,
@@ -50,10 +41,18 @@ import { PageNamespaceHook } from "./hooks/page_namespace";
 import { readFileSync } from "fs";
 import fileSystemSyscalls from "@plugos/plugos/syscalls/fs.node";
 import {
-  storeSyscalls,
   ensureTable as ensureStoreTable,
+  storeSyscalls,
 } from "@plugos/plugos/syscalls/store.knex_node";
 import { parseYamlSettings } from "@silverbulletmd/common/util";
+import { SpacePrimitives } from "@silverbulletmd/common/spaces/space_primitives";
+
+const globalModules: any = JSON.parse(
+  readFileSync(
+    nodeModulesDir + "/node_modules/@silverbulletmd/web/dist/global.plug.json",
+    "utf-8"
+  )
+);
 
 const safeFilename = /^[a-zA-Z0-9_\-\.]+$/;
 
@@ -76,6 +75,7 @@ export class ExpressServer {
   builtinPlugDir: string;
   password?: string;
   settings: { [key: string]: any } = {};
+  spacePrimitives: SpacePrimitives;
 
   constructor(options: ServerOptions) {
     this.port = options.port;
@@ -96,16 +96,14 @@ export class ExpressServer {
     this.system.addHook(namespaceHook);
 
     // The space
-    this.space = new Space(
+    this.spacePrimitives = new EventedSpacePrimitives(
-      new EventedSpacePrimitives(
       new PlugSpacePrimitives(
         new DiskSpacePrimitives(options.pagesPath),
         namespaceHook
       ),
       this.eventHook
-      ),
-      true
     );
+    this.space = new Space(this.spacePrimitives);
 
     // The database used for persistence (SQLite)
     this.db = knex({
@@ -222,8 +220,9 @@ export class ExpressServer {
         );
         let manifest: Manifest = JSON.parse(manifestJson);
         pluginNames.push(manifest.name);
-        await this.space.writePage(
+        await this.spacePrimitives.writeFile(
-          `${plugPrefix}${manifest.name}`,
+          `${plugPrefix}${file}`,
+          "string",
           manifestJson
         );
       }
@@ -245,16 +244,17 @@ export class ExpressServer {
 
   async reloadPlugs() {
     await this.space.updatePageList();
-    let allPlugs = this.space.listPlugs();
+    let allPlugs = await this.space.listPlugs();
-    if (allPlugs.size === 0) {
+    if (allPlugs.length === 0) {
       await this.bootstrapBuiltinPlugs();
-      allPlugs = this.space.listPlugs();
+      allPlugs = await this.space.listPlugs();
     }
     await this.system.unloadAll();
     console.log("Loading plugs");
-    for (let pageInfo of allPlugs) {
+    console.log(allPlugs);
-      let { text } = await this.space.readPage(pageInfo.name);
+    for (let plugName of allPlugs) {
-      await this.system.load(JSON.parse(text), createSandbox);
+      let { data } = await this.space.readAttachment(plugName, "string");
+      await this.system.load(JSON.parse(data as string), createSandbox);
     }
     this.rebuildMdExtensions();
   }
@@ -283,39 +283,16 @@ export class ExpressServer {
 
     // Pages API
     this.app.use(
-      "/page",
+      "/fs",
       passwordMiddleware,
-      cors({
+      buildFsRouter(this.spacePrimitives)
-        methods: "GET,HEAD,PUT,OPTIONS,POST,DELETE",
-        preflightContinue: true,
-      }),
-      this.buildFsRouter()
-    );
-
-    // Attachment API
-    this.app.use(
-      "/attachment",
-      passwordMiddleware,
-      cors({
-        methods: "GET,HEAD,PUT,OPTIONS,POST,DELETE",
-        preflightContinue: true,
-      }),
-      this.buildAttachmentRouter()
     );
 
     // Plug API
-    this.app.use(
+    this.app.use("/plug", passwordMiddleware, this.buildPlugRouter());
-      "/plug",
-      passwordMiddleware,
-      cors({
-        methods: "GET,HEAD,PUT,OPTIONS,POST,DELETE",
-        preflightContinue: true,
-      }),
-      this.buildPlugRouter()
-    );
 
     // Fallback, serve index.html
-    this.app.get("/*", async (req, res) => {
+    this.app.get(/^(\/((?!fs\/).)+)$/, async (req, res) => {
       res.sendFile(`${this.distDir}/index.html`, {});
     });
 
@@ -387,205 +364,6 @@ export class ExpressServer {
     return plugRouter;
   }
 
-  private buildFsRouter() {
-    let fsRouter = express.Router();
-
-    // Page list
-    fsRouter.route("/").get(async (req, res) => {
-      let { nowTimestamp, pages } = await this.space.fetchPageList();
-      res.header("Now-Timestamp", "" + nowTimestamp);
-      res.json([...pages]);
-    });
-
-    fsRouter
-      .route(/\/(.+)/)
-      .get(async (req, res) => {
-        let pageName = req.params[0];
-        // console.log("Getting", pageName);
-        try {
-          let pageData = await this.space.readPage(pageName);
-          res.status(200);
-          res.header("Last-Modified", "" + pageData.meta.lastModified);
-          res.header("X-Permission", pageData.meta.perm);
-          res.header("Content-Type", "text/markdown");
-          res.send(pageData.text);
-        } catch (e) {
-          // CORS
-          res.status(200);
-          res.header("X-Status", "404");
-          res.send("");
-        }
-      })
-      .put(bodyParser.text({ type: "*/*" }), async (req, res) => {
-        let pageName = req.params[0];
-        console.log("Saving", pageName);
-
-        try {
-          let meta = await this.space.writePage(
-            pageName,
-            req.body,
-            false,
-            req.header("Last-Modified")
-              ? +req.header("Last-Modified")!
-              : undefined
-          );
-          res.status(200);
-          res.header("Last-Modified", "" + meta.lastModified);
-          res.header("X-Permission", meta.perm);
-          res.send("OK");
-        } catch (err) {
-          res.status(500);
-          res.send("Write failed");
-          console.error("Pipeline failed", err);
-        }
-      })
-      .options(async (req, res) => {
-        let pageName = req.params[0];
-        try {
-          const meta = await this.space.getPageMeta(pageName);
-          res.status(200);
-          res.header("Last-Modified", "" + meta.lastModified);
-          res.header("X-Permission", meta.perm);
-          res.header("Content-Type", "text/markdown");
-          res.send("");
-        } catch (e) {
-          // CORS
-          res.status(200);
-          res.header("X-Status", "404");
-          res.send("Not found");
-        }
-      })
-      .delete(async (req, res) => {
-        let pageName = req.params[0];
-        try {
-          await this.space.deletePage(pageName);
-          res.status(200);
-          res.send("OK");
-        } catch (e) {
-          console.error("Error deleting file", e);
-          res.status(500);
-          res.send("OK");
-        }
-      });
-    return fsRouter;
-  }
-
-  // Build attachment router
-  private buildAttachmentRouter() {
-    let fsaRouter = express.Router();
-
-    // Page list
-    fsaRouter.route("/").get(async (req, res) => {
-      let { nowTimestamp, attachments } =
-        await this.space.fetchAttachmentList();
-      res.header("Now-Timestamp", "" + nowTimestamp);
-      res.json([...attachments]);
-    });
-
-    fsaRouter
-      .route(/\/(.+)/)
-      .get(async (req, res) => {
-        let attachmentName = req.params[0];
-        if (!this.attachmentCheck(attachmentName, res)) {
-          return;
-        }
-        console.log("Getting", attachmentName);
-        try {
-          let attachmentData = await this.space.readAttachment(
-            attachmentName,
-            "arraybuffer"
-          );
-          res.status(200);
-          res.header("Last-Modified", "" + attachmentData.meta.lastModified);
-          res.header("X-Permission", attachmentData.meta.perm);
-          res.header("Content-Type", attachmentData.meta.contentType);
-          // res.header("X-Content-Length", "" + attachmentData.meta.size);
-          res.send(Buffer.from(attachmentData.data as ArrayBuffer));
-        } catch (e) {
-          // CORS
-          res.status(200);
-          res.header("X-Status", "404");
-          res.send("");
-        }
-      })
-      .put(
-        bodyParser.raw({ type: "*/*", limit: "100mb" }),
-        async (req, res) => {
-          let attachmentName = req.params[0];
-          if (!this.attachmentCheck(attachmentName, res)) {
-            return;
-          }
-          console.log("Saving attachment", attachmentName);
-
-          try {
-            let meta = await this.space.writeAttachment(
-              attachmentName,
-              req.body,
-              false,
-              req.header("Last-Modified")
-                ? +req.header("Last-Modified")!
-                : undefined
-            );
-            res.status(200);
-            res.header("Last-Modified", "" + meta.lastModified);
-            res.header("Content-Type", meta.contentType);
-            res.header("Content-Length", "" + meta.size);
-            res.header("X-Permission", meta.perm);
-            res.send("OK");
-          } catch (err) {
-            res.status(500);
-            res.send("Write failed");
-            console.error("Pipeline failed", err);
-          }
-        }
-      )
-      .options(async (req, res) => {
-        let attachmentName = req.params[0];
-        if (!this.attachmentCheck(attachmentName, res)) {
-          return;
-        }
-        try {
-          const meta = await this.space.getAttachmentMeta(attachmentName);
-          res.status(200);
-          res.header("Last-Modified", "" + meta.lastModified);
-          res.header("X-Permission", meta.perm);
-          res.header("Content-Length", "" + meta.size);
-          res.header("Content-Type", meta.contentType);
-          res.send("");
-        } catch (e) {
-          // CORS
-          res.status(200);
-          res.header("X-Status", "404");
-          res.send("Not found");
-        }
-      })
-      .delete(async (req, res) => {
-        let attachmentName = req.params[0];
-        if (!this.attachmentCheck(attachmentName, res)) {
-          return;
-        }
-        try {
-          await this.space.deleteAttachment(attachmentName);
-          res.status(200);
-          res.send("OK");
-        } catch (e) {
-          console.error("Error deleting attachment", e);
-          res.status(500);
-          res.send("OK");
-        }
-      });
-    return fsaRouter;
-  }
-
-  attachmentCheck(attachmentName: string, res: express.Response): boolean {
-    if (attachmentName.endsWith(".md")) {
-      res.status(405);
-      res.send("No markdown files allowed through the attachment API");
-      return false;
-    }
-    return true;
-  }
-
   async ensureAndLoadSettings() {
     try {
       await this.space.getPageMeta("SETTINGS");
@@ -628,3 +406,82 @@ export class ExpressServer {
     }
   }
 }
+
+function buildFsRouter(spacePrimitives: SpacePrimitives) {
+  let fsRouter = express.Router();
+
+  // File list
+  fsRouter.route("/").get(async (req, res, next) => {
+    res.json(await spacePrimitives.fetchFileList());
+  });
+
+  fsRouter
+    .route(/\/(.+)/)
+    .get(async (req, res, next) => {
+      let name = req.params[0];
+      console.log("Getting", name);
+      try {
+        let attachmentData = await spacePrimitives.readFile(
+          name,
+          "arraybuffer"
+        );
+        res.status(200);
+        res.header("Last-Modified", "" + attachmentData.meta.lastModified);
+        res.header("X-Permission", attachmentData.meta.perm);
+        res.header("Content-Type", attachmentData.meta.contentType);
+        res.send(Buffer.from(attachmentData.data as ArrayBuffer));
+      } catch (e) {
+        next();
+      }
+    })
+    .put(bodyParser.raw({ type: "*/*", limit: "100mb" }), async (req, res) => {
+      let name = req.params[0];
+      console.log("Saving file", name);
+
+      try {
+        let meta = await spacePrimitives.writeFile(
+          name,
+          "arraybuffer",
+          req.body,
+          false
+        );
+        res.status(200);
+        res.header("Last-Modified", "" + meta.lastModified);
+        res.header("Content-Type", meta.contentType);
+        res.header("Content-Length", "" + meta.size);
+        res.header("X-Permission", meta.perm);
+        res.send("OK");
+      } catch (err) {
+        res.status(500);
+        res.send("Write failed");
+        console.error("Pipeline failed", err);
+      }
+    })
+    .options(async (req, res, next) => {
+      let name = req.params[0];
+      try {
+        const meta = await spacePrimitives.getFileMeta(name);
+        res.status(200);
+        res.header("Last-Modified", "" + meta.lastModified);
+        res.header("X-Permission", meta.perm);
+        res.header("Content-Length", "" + meta.size);
+        res.header("Content-Type", meta.contentType);
+        res.send("");
+      } catch (e) {
+        next();
+      }
+    })
+    .delete(async (req, res) => {
+      let name = req.params[0];
+      try {
+        await spacePrimitives.deleteFile(name);
+        res.status(200);
+        res.send("OK");
+      } catch (e) {
+        console.error("Error deleting attachment", e);
+        res.status(500);
+        res.send("OK");
+      }
+    });
+  return fsRouter;
+}

packages/server/hooks/page_namespace.ts

@@ -3,16 +3,16 @@ import { System } from "@plugos/plugos/system";
 import { Hook, Manifest } from "@plugos/plugos/types";
 import { Express, NextFunction, Request, Response, Router } from "express";
 
-export type PageNamespaceOperation =
+export type NamespaceOperation =
-  | "readPage"
+  | "readFile"
-  | "writePage"
+  | "writeFile"
-  | "listPages"
+  | "listFiles"
-  | "getPageMeta"
+  | "getFileMeta"
-  | "deletePage";
+  | "deleteFile";
 
 export type PageNamespaceDef = {
   pattern: string;
-  operation: PageNamespaceOperation;
+  operation: NamespaceOperation;
 };
 
 export type PageNamespaceHookT = {
@@ -20,7 +20,7 @@ export type PageNamespaceHookT = {
 };
 
 type SpaceFunction = {
-  operation: PageNamespaceOperation;
+  operation: NamespaceOperation;
   pattern: RegExp;
   plug: Plug<PageNamespaceHookT>;
   name: string;
@@ -76,11 +76,11 @@ export class PageNamespaceHook implements Hook<PageNamespaceHookT> {
         }
         if (
           ![
-            "readPage",
+            "readFile",
-            "writePage",
+            "writeFile",
-            "getPageMeta",
+            "getFileMeta",
-            "listPages",
+            "listFiles",
-            "deletePage",
+            "deleteFile",
           ].includes(funcDef.pageNamespace.operation)
         ) {
           errors.push(

packages/server/hooks/plug_space_primitives.ts

@@ -1,11 +1,15 @@
 import { Plug } from "@plugos/plugos/plug";
 import {
-  AttachmentData,
+  FileData,
-  AttachmentEncoding,
+  FileEncoding,
   SpacePrimitives,
 } from "@silverbulletmd/common/spaces/space_primitives";
-import { AttachmentMeta, PageMeta } from "@silverbulletmd/common/types";
+import {
-import { PageNamespaceHook, PageNamespaceOperation } from "./page_namespace";
+  AttachmentMeta,
+  FileMeta,
+  PageMeta,
+} from "@silverbulletmd/common/types";
+import { PageNamespaceHook, NamespaceOperation } from "./page_namespace";
 
 export class PlugSpacePrimitives implements SpacePrimitives {
   constructor(
@@ -14,7 +18,7 @@ export class PlugSpacePrimitives implements SpacePrimitives {
   ) {}
 
   performOperation(
-    type: PageNamespaceOperation,
+    type: NamespaceOperation,
     pageName: string,
     ...args: any[]
   ): Promise<any> | false {
@@ -26,101 +30,71 @@ export class PlugSpacePrimitives implements SpacePrimitives {
     return false;
   }
 
-  async fetchPageList(): Promise<{
+  async fetchFileList(): Promise<FileMeta[]> {
-    pages: Set<PageMeta>;
+    let allFiles: FileMeta[] = [];
-    nowTimestamp: number;
-  }> {
-    let allPages = new Set<PageMeta>();
     for (let { plug, name, operation } of this.hook.spaceFunctions) {
-      if (operation === "listPages") {
+      if (operation === "listFiles") {
         try {
           for (let pm of await plug.invoke(name, [])) {
-            allPages.add(pm);
+            allFiles.push(pm);
           }
         } catch (e) {
-          console.error("Error listing pages", e);
+          console.error("Error listing files", e);
         }
       }
     }
-    let result = await this.wrapped.fetchPageList();
+    let result = await this.wrapped.fetchFileList();
-    for (let pm of result.pages) {
+    for (let pm of result) {
-      allPages.add(pm);
+      allFiles.push(pm);
     }
-    return {
+    return allFiles;
-      nowTimestamp: result.nowTimestamp,
-      pages: allPages,
-    };
   }
 
-  readPage(name: string): Promise<{ text: string; meta: PageMeta }> {
+  readFile(
-    let result = this.performOperation("readPage", name);
-    if (result) {
-      return result;
-    }
-    return this.wrapped.readPage(name);
-  }
-
-  getPageMeta(name: string): Promise<PageMeta> {
-    let result = this.performOperation("getPageMeta", name);
-    if (result) {
-      return result;
-    }
-    return this.wrapped.getPageMeta(name);
-  }
-
-  writePage(
     name: string,
-    text: string,
+    encoding: FileEncoding
-    selfUpdate?: boolean,
+  ): Promise<{ data: FileData; meta: FileMeta }> {
-    lastModified?: number
+    let result = this.performOperation("readFile", name);
-  ): Promise<PageMeta> {
+    if (result) {
+      return result;
+    }
+    return this.wrapped.readFile(name, encoding);
+  }
+
+  getFileMeta(name: string): Promise<FileMeta> {
+    let result = this.performOperation("getFileMeta", name);
+    if (result) {
+      return result;
+    }
+    return this.wrapped.getFileMeta(name);
+  }
+
+  writeFile(
+    name: string,
+    encoding: FileEncoding,
+    data: FileData,
+    selfUpdate?: boolean
+  ): Promise<FileMeta> {
     let result = this.performOperation(
-      "writePage",
+      "writeFile",
       name,
-      text,
+      encoding,
-      selfUpdate,
+      data,
-      lastModified
+      selfUpdate
     );
     if (result) {
       return result;
     }
 
-    return this.wrapped.writePage(name, text, selfUpdate, lastModified);
+    return this.wrapped.writeFile(name, encoding, data, selfUpdate);
   }
 
-  deletePage(name: string): Promise<void> {
+  deleteFile(name: string): Promise<void> {
-    let result = this.performOperation("deletePage", name);
+    let result = this.performOperation("deleteFile", name);
     if (result) {
       return result;
     }
-    return this.wrapped.deletePage(name);
+    return this.wrapped.deleteFile(name);
-  }
-
-  fetchAttachmentList(): Promise<{
-    attachments: Set<AttachmentMeta>;
-    nowTimestamp: number;
-  }> {
-    return this.wrapped.fetchAttachmentList();
-  }
-  readAttachment(
-    name: string,
-    encoding: AttachmentEncoding
-  ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> {
-    return this.wrapped.readAttachment(name, encoding);
-  }
-  getAttachmentMeta(name: string): Promise<AttachmentMeta> {
-    return this.wrapped.getAttachmentMeta(name);
-  }
-  writeAttachment(
-    name: string,
-    blob: ArrayBuffer,
-    selfUpdate?: boolean | undefined,
-    lastModified?: number | undefined
-  ): Promise<AttachmentMeta> {
-    return this.wrapped.writeAttachment(name, blob, selfUpdate, lastModified);
-  }
-  deleteAttachment(name: string): Promise<void> {
-    return this.wrapped.deleteAttachment(name);
   }
 
   proxySyscall(plug: Plug<any>, name: string, args: any[]): Promise<any> {

packages/server/syscalls/space.ts

@@ -1,12 +1,15 @@
 import { AttachmentMeta, PageMeta } from "@silverbulletmd/common/types";
 import { SysCallMapping } from "@plugos/plugos/system";
 import { Space } from "@silverbulletmd/common/spaces/space";
-import { AttachmentData } from "@silverbulletmd/common/spaces/space_primitives";
+import {
+  FileData,
+  FileEncoding,
+} from "@silverbulletmd/common/spaces/space_primitives";
 
 export default (space: Space): SysCallMapping => {
   return {
-    "space.listPages": async (ctx, unfiltered = false): Promise<PageMeta[]> => {
+    "space.listPages": async (): Promise<PageMeta[]> => {
-      return [...space.listPages(unfiltered)];
+      return [...space.listPages()];
     },
     "space.readPage": async (
       ctx,
@@ -27,13 +30,16 @@ export default (space: Space): SysCallMapping => {
     "space.deletePage": async (ctx, name: string) => {
       return space.deletePage(name);
     },
+    "space.listPlugs": async (): Promise<string[]> => {
+      return await space.listPlugs();
+    },
     "space.listAttachments": async (ctx): Promise<AttachmentMeta[]> => {
-      return [...(await space.fetchAttachmentList()).attachments];
+      return await space.fetchAttachmentList();
     },
     "space.readAttachment": async (
       ctx,
       name: string
-    ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> => {
+    ): Promise<{ data: FileData; meta: AttachmentMeta }> => {
       return await space.readAttachment(name, "dataurl");
     },
     "space.getAttachmentMeta": async (
@@ -45,9 +51,10 @@ export default (space: Space): SysCallMapping => {
     "space.writeAttachment": async (
       ctx,
       name: string,
+      encoding: FileEncoding,
       data: string
     ): Promise<AttachmentMeta> => {
-      return await space.writeAttachment(name, data);
+      return await space.writeAttachment(name, encoding, data);
     },
     "space.deleteAttachment": async (ctx, name: string) => {
       await space.deleteAttachment(name);

packages/web/boot.ts

@@ -11,7 +11,9 @@ safeRun(async () => {
   let settingsPageText = "";
   while (true) {
     try {
-      settingsPageText = (await httpPrimitives.readPage("SETTINGS")).text;
+      settingsPageText = (await (
+        await httpPrimitives.readFile("SETTINGS.md", "string")
+      ).data) as string;
       break;
     } catch (e: any) {
       if (e.message === "Unauthorized") {
@@ -25,7 +27,7 @@ safeRun(async () => {
       }
     }
   }
-  let serverSpace = new Space(httpPrimitives, true);
+  let serverSpace = new Space(httpPrimitives);
   serverSpace.watch();
 
   console.log("Booting...");

packages/web/editor.tsx

@@ -55,11 +55,7 @@ import {
   MDExt,
 } from "@silverbulletmd/common/markdown_ext";
 import { FilterList } from "./components/filter";
-import {
+import { FilterOption, PageMeta } from "@silverbulletmd/common/types";
-  FilterOption,
-  PageMeta,
-  reservedPageNames,
-} from "@silverbulletmd/common/types";
 import { syntaxTree } from "@codemirror/language";
 import sandboxSyscalls from "@plugos/plugos/syscalls/sandbox";
 import { eventSyscalls } from "@plugos/plugos/syscalls/event";
@@ -208,13 +204,6 @@ export class Editor {
     this.focus();
 
     this.pageNavigator.subscribe(async (pageName, pos: number | string) => {
-      if (reservedPageNames.includes(pageName)) {
-        this.flashNotification(
-          `"${pageName}" is a reserved page name. It cannot be used.`,
-          "error"
-        );
-        return;
-      }
       console.log("Now navigating to", pageName, pos);
 
       if (!this.editorView) {
@@ -534,10 +523,10 @@ export class Editor {
     await this.space.updatePageList();
     await this.system.unloadAll();
     console.log("(Re)loading plugs");
-    for (let pageInfo of this.space.listPlugs()) {
+    for (let plugName of await this.space.listPlugs()) {
       // console.log("Loading plug", pageInfo.name);
-      let { text } = await this.space.readPage(pageInfo.name);
+      let { data } = await this.space.readAttachment(plugName, "string");
-      await this.system.load(JSON.parse(text), createIFrameSandbox);
+      await this.system.load(JSON.parse(data as string), createIFrameSandbox);
     }
     this.rebuildEditorState();
     await this.dispatchAppEvent("plugs:loaded");
@@ -617,11 +606,6 @@ export class Editor {
 
     const previousPage = this.currentPage;
 
-    this.viewDispatch({
-      type: "page-loading",
-      name: pageName,
-    });
-
     // Persist current page state and nicely close page
     if (previousPage) {
       this.saveState(previousPage);
@@ -629,6 +613,11 @@ export class Editor {
       await this.save(true);
     }
 
+    this.viewDispatch({
+      type: "page-loading",
+      name: pageName,
+    });
+
     // Fetch next page to open
     let doc;
     try {

packages/web/editor_paste.ts

@@ -118,7 +118,7 @@ export function attachmentExtension(editor: Editor) {
     if (!finalFileName) {
       return;
     }
-    await editor.space.writeAttachment(finalFileName, data!);
+    await editor.space.writeAttachment(finalFileName, "arraybuffer", data!);
     let attachmentMarkdown = `[${finalFileName}](${finalFileName})`;
     if (mimeType.startsWith("image/")) {
       attachmentMarkdown = `![](${finalFileName})`;

packages/web/inline_image.ts

@@ -23,7 +23,7 @@ class InlineImageWidget extends WidgetType {
     if (this.url.startsWith("http")) {
       img.src = this.url;
     } else {
-      img.src = `attachment/${this.url}`;
+      img.src = `fs/${this.url}`;
     }
     img.alt = this.title;
     img.title = this.title;

packages/web/service_worker.ts

@@ -35,10 +35,8 @@ self.addEventListener("fetch", (event: any) => {
         return response;
       } else {
         if (
-          parsedUrl.pathname !== "/page" &&
+          parsedUrl.pathname !== "/fs" &&
           !parsedUrl.pathname.startsWith("/page/") &&
-          parsedUrl.pathname !== "/attachment" &&
-          !parsedUrl.pathname.startsWith("/attachment/") &&
           !parsedUrl.pathname.startsWith("/plug/")
         ) {
           return cache.match("/index.html");

packages/web/syscalls/space.ts

@@ -1,12 +1,15 @@
 import { Editor } from "../editor";
 import { SysCallMapping } from "@plugos/plugos/system";
 import { AttachmentMeta, PageMeta } from "@silverbulletmd/common/types";
-import { AttachmentData } from "@silverbulletmd/common/spaces/space_primitives";
+import {
+  FileData,
+  FileEncoding,
+} from "@silverbulletmd/common/spaces/space_primitives";
 
 export function spaceSyscalls(editor: Editor): SysCallMapping {
   return {
-    "space.listPages": async (ctx, unfiltered = false): Promise<PageMeta[]> => {
+    "space.listPages": async (): Promise<PageMeta[]> => {
-      return [...(await editor.space.listPages(unfiltered))];
+      return [...editor.space.listPages()];
     },
     "space.readPage": async (
       ctx,
@@ -34,13 +37,16 @@ export function spaceSyscalls(editor: Editor): SysCallMapping {
       console.log("Deleting page");
       await editor.space.deletePage(name);
     },
+    "space.listPlugs": async (): Promise<string[]> => {
+      return await editor.space.listPlugs();
+    },
     "space.listAttachments": async (ctx): Promise<AttachmentMeta[]> => {
-      return [...(await editor.space.fetchAttachmentList()).attachments];
+      return await editor.space.fetchAttachmentList();
     },
     "space.readAttachment": async (
       ctx,
       name: string
-    ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> => {
+    ): Promise<{ data: FileData; meta: AttachmentMeta }> => {
       return await editor.space.readAttachment(name, "dataurl");
     },
     "space.getAttachmentMeta": async (
@@ -52,9 +58,10 @@ export function spaceSyscalls(editor: Editor): SysCallMapping {
     "space.writeAttachment": async (
       ctx,
       name: string,
-      buffer: ArrayBuffer
+      encoding: FileEncoding,
+      data: FileData
     ): Promise<AttachmentMeta> => {
-      return await editor.space.writeAttachment(name, buffer);
+      return await editor.space.writeAttachment(name, encoding, data);
     },
     "space.deleteAttachment": async (ctx, name: string) => {
       await editor.space.deleteAttachment(name);