diff --git a/packages/common/spaces/constants.ts b/packages/common/spaces/constants.ts index 3eb8b4fa..0922ecc5 100644 --- a/packages/common/spaces/constants.ts +++ b/packages/common/spaces/constants.ts @@ -1,2 +1 @@ -export const trashPrefix = "_trash/"; export const plugPrefix = "_plug/"; diff --git a/packages/common/spaces/disk_space_primitives.ts b/packages/common/spaces/disk_space_primitives.ts index 99a78a72..ff141f9d 100644 --- a/packages/common/spaces/disk_space_primitives.ts +++ b/packages/common/spaces/disk_space_primitives.ts @@ -1,30 +1,20 @@ -import { - mkdir, - readdir, - readFile, - stat, - unlink, - utimes, - writeFile, -} from "fs/promises"; +import { mkdir, readdir, readFile, stat, unlink, writeFile } from "fs/promises"; import * as path from "path"; -import { AttachmentMeta, PageMeta } from "../types"; -import { - AttachmentData, - AttachmentEncoding, - SpacePrimitives, -} from "./space_primitives"; +import { FileMeta } from "../types"; +import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives"; import { Plug } from "@plugos/plugos/plug"; import { realpathSync } from "fs"; import mime from "mime-types"; +function lookupContentType(path: string): string { + return mime.lookup(path) || "application/octet-stream"; +} + export class DiskSpacePrimitives implements SpacePrimitives { rootPath: string; - plugPrefix: string; - constructor(rootPath: string, plugPrefix: string = "_plug/") { + constructor(rootPath: string) { this.rootPath = realpathSync(rootPath); - this.plugPrefix = plugPrefix; } safePath(p: string): string { @@ -35,111 +25,136 @@ export class DiskSpacePrimitives implements SpacePrimitives { return realPath; } - pageNameToPath(pageName: string) { - if (pageName.startsWith(this.plugPrefix)) { - return this.safePath(path.join(this.rootPath, pageName + ".plug.json")); - } - return this.safePath(path.join(this.rootPath, pageName + ".md")); + filenameToPath(pageName: string) { + return this.safePath(path.join(this.rootPath, pageName)); } - pathToPageName(fullPath: string): string { - let extLength = fullPath.endsWith(".plug.json") - ? ".plug.json".length - : ".md".length; - return fullPath.substring( - this.rootPath.length + 1, - fullPath.length - extLength - ); + pathToFilename(fullPath: string): string { + return fullPath.substring(this.rootPath.length + 1); } - // Pages - async readPage(pageName: string): Promise<{ text: string; meta: PageMeta }> { - const localPath = this.pageNameToPath(pageName); + async readFile( + name: string, + encoding: FileEncoding + ): Promise<{ data: FileData; meta: FileMeta }> { + const localPath = this.filenameToPath(name); try { const s = await stat(localPath); + let data: FileData | null = null; + let contentType = lookupContentType(name); + switch (encoding) { + case "string": + data = await readFile(localPath, "utf8"); + break; + case "dataurl": + let fileBuffer = await readFile(localPath, { + encoding: "base64", + }); + data = `data:${contentType};base64,${fileBuffer}`; + break; + case "arraybuffer": + let arrayBuffer = await readFile(localPath); + data = arrayBuffer.buffer; + break; + } return { - text: await readFile(localPath, "utf8"), + data, meta: { - name: pageName, + name: name, lastModified: s.mtime.getTime(), perm: "rw", + size: s.size, + contentType: contentType, }, }; } catch (e) { - // console.error("Error while reading page", pageName, e); - throw Error(`Could not read page ${pageName}`); + console.error("Error while reading file", name, e); + throw Error(`Could not read file ${name}`); } } - async writePage( - pageName: string, - text: string, - selfUpdate: boolean, - lastModified?: number - ): Promise { - let localPath = this.pageNameToPath(pageName); + async writeFile( + name: string, + encoding: FileEncoding, + data: FileData, + selfUpdate?: boolean + ): Promise { + let localPath = this.filenameToPath(name); try { // Ensure parent folder exists await mkdir(path.dirname(localPath), { recursive: true }); // Actually write the file - await writeFile(localPath, text); - - if (lastModified) { - let d = new Date(lastModified); - console.log("Going to set the modified time", d); - await utimes(localPath, d, d); + switch (encoding) { + case "string": + await writeFile(localPath, data as string, "utf8"); + break; + case "dataurl": + await writeFile(localPath, (data as string).split(",")[1], { + encoding: "base64", + }); + break; + case "arraybuffer": + await writeFile(localPath, Buffer.from(data as ArrayBuffer)); + break; } + // Fetch new metadata const s = await stat(localPath); return { - name: pageName, + name: name, + size: s.size, + contentType: lookupContentType(name), lastModified: s.mtime.getTime(), perm: "rw", }; } catch (e) { - console.error("Error while writing page", pageName, e); - throw Error(`Could not write ${pageName}`); + console.error("Error while writing file", name, e); + throw Error(`Could not write ${name}`); } } - async getPageMeta(pageName: string): Promise { - let localPath = this.pageNameToPath(pageName); + async getFileMeta(name: string): Promise { + let localPath = this.filenameToPath(name); try { const s = await stat(localPath); return { - name: pageName, + name: name, + size: s.size, + contentType: lookupContentType(name), lastModified: s.mtime.getTime(), perm: "rw", }; } catch (e) { // console.error("Error while getting page meta", pageName, e); - throw Error(`Could not get meta for ${pageName}`); + throw Error(`Could not get meta for ${name}`); } } - async deletePage(pageName: string): Promise { - let localPath = this.pageNameToPath(pageName); + async deleteFile(name: string): Promise { + let localPath = this.filenameToPath(name); await unlink(localPath); } - async fetchPageList(): Promise<{ - pages: Set; - nowTimestamp: number; - }> { - let pages = new Set(); + async fetchFileList(): Promise { + let fileList: FileMeta[] = []; const walkPath = async (dir: string) => { let files = await readdir(dir); for (let file of files) { + if (file.startsWith(".")) { + continue; + } const fullPath = path.join(dir, file); let s = await stat(fullPath); if (s.isDirectory()) { await walkPath(fullPath); } else { - if (file.endsWith(".md") || file.endsWith(".json")) { - pages.add({ - name: this.pathToPageName(fullPath), + if (!file.startsWith(".")) { + fileList.push({ + name: this.pathToFilename(fullPath), + size: s.size, + contentType: lookupContentType(fullPath), lastModified: s.mtime.getTime(), perm: "rw", }); @@ -148,150 +163,7 @@ export class DiskSpacePrimitives implements SpacePrimitives { } }; await walkPath(this.rootPath); - return { - pages: pages, - nowTimestamp: Date.now(), - }; - } - - // Attachments - attachmentNameToPath(name: string) { - return this.safePath(path.join(this.rootPath, name)); - } - - pathToAttachmentName(fullPath: string): string { - return fullPath.substring(this.rootPath.length + 1); - } - - async fetchAttachmentList(): Promise<{ - attachments: Set; - nowTimestamp: number; - }> { - let attachments = new Set(); - - const walkPath = async (dir: string) => { - let files = await readdir(dir); - for (let file of files) { - const fullPath = path.join(dir, file); - let s = await stat(fullPath); - if (s.isDirectory()) { - if (!file.startsWith(".")) { - await walkPath(fullPath); - } - } else { - if ( - !file.startsWith(".") && - !file.endsWith(".md") && - !file.endsWith(".json") - ) { - attachments.add({ - name: this.pathToAttachmentName(fullPath), - lastModified: s.mtime.getTime(), - size: s.size, - contentType: mime.lookup(file) || "application/octet-stream", - perm: "rw", - } as AttachmentMeta); - } - } - } - }; - await walkPath(this.rootPath); - return { - attachments, - nowTimestamp: Date.now(), - }; - } - - async readAttachment( - name: string, - encoding: AttachmentEncoding - ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> { - const localPath = this.attachmentNameToPath(name); - let fileBuffer = await readFile(localPath, { - encoding: encoding === "dataurl" ? "base64" : null, - }); - - try { - const s = await stat(localPath); - let contentType = mime.lookup(name) || "application/octet-stream"; - return { - data: - encoding === "dataurl" - ? `data:${contentType};base64,${fileBuffer}` - : (fileBuffer as Buffer).buffer, - meta: { - name: name, - lastModified: s.mtime.getTime(), - size: s.size, - contentType: contentType, - perm: "rw", - }, - }; - } catch (e) { - // console.error("Error while reading attachment", name, e); - throw Error(`Could not read attachment ${name}`); - } - } - - async getAttachmentMeta(name: string): Promise { - const localPath = this.attachmentNameToPath(name); - try { - const s = await stat(localPath); - return { - name: name, - lastModified: s.mtime.getTime(), - size: s.size, - contentType: mime.lookup(name) || "application/octet-stream", - perm: "rw", - }; - } catch (e) { - // console.error("Error while getting attachment meta", name, e); - throw Error(`Could not get meta for ${name}`); - } - } - - async writeAttachment( - name: string, - data: AttachmentData, - selfUpdate?: boolean, - lastModified?: number - ): Promise { - let localPath = this.attachmentNameToPath(name); - try { - // Ensure parent folder exists - await mkdir(path.dirname(localPath), { recursive: true }); - - // Actually write the file - if (typeof data === "string") { - await writeFile(localPath, data.split(",")[1], { encoding: "base64" }); - } else { - await writeFile(localPath, Buffer.from(data)); - } - - if (lastModified) { - let d = new Date(lastModified); - console.log("Going to set the modified time", d); - await utimes(localPath, d, d); - } - - // Fetch new metadata - const s = await stat(localPath); - return { - name: name, - lastModified: s.mtime.getTime(), - size: s.size, - contentType: mime.lookup(name) || "application/octet-stream", - perm: "rw", - }; - } catch (e) { - console.error("Error while writing attachment", name, e); - throw Error(`Could not write ${name}`); - } - } - - async deleteAttachment(name: string): Promise { - let localPath = this.attachmentNameToPath(name); - await unlink(localPath); + return fileList; } // Plugs diff --git a/packages/common/spaces/evented_space_primitives.ts b/packages/common/spaces/evented_space_primitives.ts index dde97d8c..95b441c9 100644 --- a/packages/common/spaces/evented_space_primitives.ts +++ b/packages/common/spaces/evented_space_primitives.ts @@ -1,19 +1,14 @@ import { EventHook } from "@plugos/plugos/hooks/event"; import { Plug } from "@plugos/plugos/plug"; -import { AttachmentMeta, PageMeta } from "../types"; -import { plugPrefix, trashPrefix } from "./constants"; -import { - AttachmentData, - AttachmentEncoding, - SpacePrimitives, -} from "./space_primitives"; +import { FileMeta } from "../types"; +import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives"; export class EventedSpacePrimitives implements SpacePrimitives { constructor(private wrapped: SpacePrimitives, private eventHook: EventHook) {} - fetchPageList(): Promise<{ pages: Set; nowTimestamp: number }> { - return this.wrapped.fetchPageList(); + fetchFileList(): Promise { + return this.wrapped.fetchFileList(); } proxySyscall(plug: Plug, name: string, args: any[]): Promise { @@ -29,26 +24,43 @@ export class EventedSpacePrimitives implements SpacePrimitives { return this.wrapped.invokeFunction(plug, env, name, args); } - readPage(pageName: string): Promise<{ text: string; meta: PageMeta }> { - return this.wrapped.readPage(pageName); + readFile( + name: string, + encoding: FileEncoding + ): Promise<{ data: FileData; meta: FileMeta }> { + return this.wrapped.readFile(name, encoding); } - async writePage( - pageName: string, - text: string, - selfUpdate: boolean, - lastModified?: number - ): Promise { - const newPageMeta = await this.wrapped.writePage( - pageName, - text, - selfUpdate, - lastModified + async writeFile( + name: string, + encoding: FileEncoding, + data: FileData, + selfUpdate: boolean + ): Promise { + const newMeta = await this.wrapped.writeFile( + name, + encoding, + data, + selfUpdate ); // This can happen async - if (!pageName.startsWith(trashPrefix) && !pageName.startsWith(plugPrefix)) { + if (name.endsWith(".md")) { + const pageName = name.substring(0, name.length - 3); + let text = ""; + switch (encoding) { + case "string": + text = data as string; + break; + case "arraybuffer": + const decoder = new TextDecoder("utf-8"); + text = decoder.decode(data as ArrayBuffer); + break; + case "dataurl": + throw Error("Data urls not supported in this context"); + } + this.eventHook - .dispatchEvent("page:saved", pageName) + .dispatchEvent("page:saved") .then(() => { return this.eventHook.dispatchEvent("page:index_text", { name: pageName, @@ -59,54 +71,18 @@ export class EventedSpacePrimitives implements SpacePrimitives { console.error("Error dispatching page:saved event", e); }); } - return newPageMeta; + return newMeta; } - getPageMeta(pageName: string): Promise { - return this.wrapped.getPageMeta(pageName); + getFileMeta(name: string): Promise { + return this.wrapped.getFileMeta(name); } - async deletePage(pageName: string): Promise { - await this.eventHook.dispatchEvent("page:deleted", pageName); - return this.wrapped.deletePage(pageName); - } - - fetchAttachmentList(): Promise<{ - attachments: Set; - nowTimestamp: number; - }> { - return this.wrapped.fetchAttachmentList(); - } - - readAttachment( - name: string, - encoding: AttachmentEncoding - ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> { - return this.wrapped.readAttachment(name, encoding); - } - - getAttachmentMeta(name: string): Promise { - return this.wrapped.getAttachmentMeta(name); - } - - async writeAttachment( - name: string, - blob: ArrayBuffer, - selfUpdate?: boolean | undefined, - lastModified?: number | undefined - ): Promise { - let meta = await this.wrapped.writeAttachment( - name, - blob, - selfUpdate, - lastModified - ); - await this.eventHook.dispatchEvent("attachment:saved", name); - return meta; - } - - async deleteAttachment(name: string): Promise { - await this.eventHook.dispatchEvent("attachment:deleted", name); - return this.wrapped.deleteAttachment(name); + async deleteFile(name: string): Promise { + if (name.endsWith(".md")) { + const pageName = name.substring(0, name.length - 3); + await this.eventHook.dispatchEvent("page:deleted", pageName); + } + return this.wrapped.deleteFile(name); } } diff --git a/packages/common/spaces/http_space_primitives.ts b/packages/common/spaces/http_space_primitives.ts index 52992c25..8e18f09f 100644 --- a/packages/common/spaces/http_space_primitives.ts +++ b/packages/common/spaces/http_space_primitives.ts @@ -1,20 +1,14 @@ -import { AttachmentMeta, PageMeta } from "../types"; +import { AttachmentMeta, FileMeta, PageMeta } from "../types"; import { Plug } from "@plugos/plugos/plug"; -import { - AttachmentData, - AttachmentEncoding, - SpacePrimitives, -} from "./space_primitives"; +import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives"; export class HttpSpacePrimitives implements SpacePrimitives { fsUrl: string; - fsaUrl: string; private plugUrl: string; token?: string; constructor(url: string, token?: string) { - this.fsUrl = url + "/page"; - this.fsaUrl = url + "/attachment"; + this.fsUrl = url + "/fs"; this.plugUrl = url + "/plug"; this.token = token; } @@ -34,72 +28,105 @@ export class HttpSpacePrimitives implements SpacePrimitives { return result; } - public async fetchPageList(): Promise<{ - pages: Set; - nowTimestamp: number; - }> { + public async fetchFileList(): Promise { let req = await this.authenticatedFetch(this.fsUrl, { method: "GET", }); - let result = new Set(); - ((await req.json()) as any[]).forEach((meta: any) => { - const pageName = meta.name; - result.add({ - name: pageName, - lastModified: meta.lastModified, - perm: "rw", - }); - }); + let result: FileMeta[] = await req.json(); - return { - pages: result, - nowTimestamp: +req.headers.get("Now-Timestamp")!, - }; + return result; } - async readPage(name: string): Promise<{ text: string; meta: PageMeta }> { + async readFile( + name: string, + encoding: FileEncoding + ): Promise<{ data: FileData; meta: FileMeta }> { let res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, { method: "GET", }); - if (res.headers.get("X-Status") === "404") { + if (res.status === 404) { throw new Error(`Page not found`); } + let data: FileData | null = null; + switch (encoding) { + case "arraybuffer": + let abBlob = await res.blob(); + data = await abBlob.arrayBuffer(); + break; + case "dataurl": + let dUBlob = await res.blob(); + data = arrayBufferToDataUrl(await dUBlob.arrayBuffer()); + break; + case "string": + data = await res.text(); + break; + } return { - text: await res.text(), - meta: this.responseToPageMeta(name, res), + data: data, + meta: this.responseToMeta(name, res), }; } - async writePage( + async writeFile( name: string, - text: string, - selfUpdate?: boolean, - lastModified?: number - ): Promise { - // TODO: lastModified ignored for now + encoding: FileEncoding, + data: FileData, + selfUpdate?: boolean + ): Promise { + let body: any = null; + + switch (encoding) { + case "arraybuffer": + case "string": + body = data; + break; + case "dataurl": + data = dataUrlToArrayBuffer(data as string); + break; + } let res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, { method: "PUT", - body: text, - headers: lastModified - ? { - "Last-Modified": "" + lastModified, - } - : undefined, + headers: { + "Content-type": "application/octet-stream", + }, + body, }); - const newMeta = this.responseToPageMeta(name, res); + const newMeta = this.responseToMeta(name, res); return newMeta; } - async deletePage(name: string): Promise { + async deleteFile(name: string): Promise { let req = await this.authenticatedFetch(`${this.fsUrl}/${name}`, { method: "DELETE", }); if (req.status !== 200) { - throw Error(`Failed to delete page: ${req.statusText}`); + throw Error(`Failed to delete file: ${req.statusText}`); } } + async getFileMeta(name: string): Promise { + let res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, { + method: "OPTIONS", + }); + if (res.status === 404) { + throw new Error(`File not found`); + } + return this.responseToMeta(name, res); + } + + private responseToMeta(name: string, res: Response): FileMeta { + return { + name, + size: +res.headers.get("Content-length")!, + contentType: res.headers.get("Content-type")!, + lastModified: +(res.headers.get("Last-Modified") || "0"), + perm: (res.headers.get("X-Permission") as "rw" | "ro") || "rw", + }; + } + + // Plugs + async proxySyscall(plug: Plug, name: string, args: any[]): Promise { let req = await this.authenticatedFetch( `${this.plugUrl}/${plug.name}/syscall/${name}`, @@ -121,95 +148,6 @@ export class HttpSpacePrimitives implements SpacePrimitives { return await req.json(); } - // Attachments - public async fetchAttachmentList(): Promise<{ - attachments: Set; - nowTimestamp: number; - }> { - let req = await this.authenticatedFetch(this.fsaUrl, { - method: "GET", - }); - - let result = new Set(); - ((await req.json()) as any[]).forEach((meta: any) => { - const pageName = meta.name; - result.add({ - name: pageName, - size: meta.size, - lastModified: meta.lastModified, - contentType: meta.contentType, - perm: "rw", - }); - }); - - return { - attachments: result, - nowTimestamp: +req.headers.get("Now-Timestamp")!, - }; - } - - async readAttachment( - name: string, - encoding: AttachmentEncoding - ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> { - let res = await this.authenticatedFetch(`${this.fsaUrl}/${name}`, { - method: "GET", - }); - if (res.headers.get("X-Status") === "404") { - throw new Error(`Page not found`); - } - let blob = await res.blob(); - return { - data: - encoding === "arraybuffer" - ? await blob.arrayBuffer() - : arrayBufferToDataUrl(await blob.arrayBuffer()), - meta: this.responseToAttachmentMeta(name, res), - }; - } - - async writeAttachment( - name: string, - data: AttachmentData, - selfUpdate?: boolean, - lastModified?: number - ): Promise { - if (typeof data === "string") { - data = dataUrlToArrayBuffer(data); - } - let res = await this.authenticatedFetch(`${this.fsaUrl}/${name}`, { - method: "PUT", - body: data, - headers: { - "Last-Modified": lastModified ? "" + lastModified : undefined, - "Content-type": "application/octet-stream", - }, - }); - const newMeta = this.responseToAttachmentMeta(name, res); - return newMeta; - } - - async getAttachmentMeta(name: string): Promise { - let res = await this.authenticatedFetch(`${this.fsaUrl}/${name}`, { - method: "OPTIONS", - }); - if (res.headers.get("X-Status") === "404") { - throw new Error(`Page not found`); - } - return this.responseToAttachmentMeta(name, res); - } - - async deleteAttachment(name: string): Promise { - let req = await this.authenticatedFetch(`${this.fsaUrl}/${name}`, { - method: "DELETE", - }); - if (req.status !== 200) { - throw Error(`Failed to delete attachment: ${req.statusText}`); - } - } - - // Plugs - async invokeFunction( plug: Plug, env: string, @@ -244,38 +182,6 @@ export class HttpSpacePrimitives implements SpacePrimitives { return await req.text(); } } - - async getPageMeta(name: string): Promise { - let res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, { - method: "OPTIONS", - }); - if (res.headers.get("X-Status") === "404") { - throw new Error(`Page not found`); - } - return this.responseToPageMeta(name, res); - } - - private responseToPageMeta(name: string, res: Response): PageMeta { - return { - name, - lastModified: +(res.headers.get("Last-Modified") || "0"), - perm: (res.headers.get("X-Permission") as "rw" | "ro") || "rw", - }; - } - - private responseToAttachmentMeta( - name: string, - res: Response - ): AttachmentMeta { - return { - name, - lastModified: +(res.headers.get("Last-Modified") || "0"), - size: +(res.headers.get("Content-Length") || "0"), - contentType: - res.headers.get("Content-Type") || "application/octet-stream", - perm: (res.headers.get("X-Permission") as "rw" | "ro") || "rw", - }; - } } function dataUrlToArrayBuffer(dataUrl: string): ArrayBuffer { diff --git a/packages/common/spaces/indexeddb_space_primitives.ts b/packages/common/spaces/indexeddb_space_primitives.ts deleted file mode 100644 index 249bbc80..00000000 --- a/packages/common/spaces/indexeddb_space_primitives.ts +++ /dev/null @@ -1,117 +0,0 @@ -import { - AttachmentData, - AttachmentEncoding, - SpacePrimitives, -} from "./space_primitives"; -import { AttachmentMeta, PageMeta } from "../types"; -import Dexie, { Table } from "dexie"; -import { Plug } from "@plugos/plugos/plug"; - -type Page = { - name: string; - text: string; - meta: PageMeta; -}; - -export class IndexedDBSpacePrimitives implements SpacePrimitives { - private pageTable: Table; - - constructor(dbName: string, readonly timeSkew: number = 0) { - const db = new Dexie(dbName); - db.version(1).stores({ - page: "name", - }); - this.pageTable = db.table("page"); - } - fetchAttachmentList(): Promise<{ - attachments: Set; - nowTimestamp: number; - }> { - throw new Error("Method not implemented."); - } - readAttachment( - name: string, - encoding: AttachmentEncoding - ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> { - throw new Error("Method not implemented."); - } - getAttachmentMeta(name: string): Promise { - throw new Error("Method not implemented."); - } - writeAttachment( - name: string, - blob: ArrayBuffer, - selfUpdate?: boolean | undefined, - lastModified?: number | undefined - ): Promise { - throw new Error("Method not implemented."); - } - deleteAttachment(name: string): Promise { - throw new Error("Method not implemented."); - } - - async deletePage(name: string): Promise { - return this.pageTable.delete(name); - } - - async getPageMeta(name: string): Promise { - let entry = await this.pageTable.get(name); - if (entry) { - return entry.meta; - } else { - throw Error(`Page not found`); - } - } - - invokeFunction( - plug: Plug, - env: string, - name: string, - args: any[] - ): Promise { - return plug.invoke(name, args); - } - - async fetchPageList(): Promise<{ - pages: Set; - nowTimestamp: number; - }> { - let allPages = await this.pageTable.toArray(); - return { - pages: new Set(allPages.map((p) => p.meta)), - nowTimestamp: Date.now() + this.timeSkew, - }; - } - - proxySyscall(plug: Plug, name: string, args: any[]): Promise { - return plug.syscall(name, args); - } - - async readPage(name: string): Promise<{ text: string; meta: PageMeta }> { - let page = await this.pageTable.get(name); - if (page) { - return page; - } else { - throw new Error("Page not found"); - } - } - - async writePage( - name: string, - text: string, - selfUpdate?: boolean, - lastModified?: number - ): Promise { - const meta: PageMeta = { - name, - lastModified: lastModified ? lastModified : Date.now() + this.timeSkew, - perm: "rw", - }; - await this.pageTable.put({ - name, - text, - meta, - }); - return meta; - } -} diff --git a/packages/common/spaces/space.ts b/packages/common/spaces/space.ts index 38f11c7b..7ee73785 100644 --- a/packages/common/spaces/space.ts +++ b/packages/common/spaces/space.ts @@ -1,13 +1,8 @@ -import { - AttachmentData, - AttachmentEncoding, - SpacePrimitives, -} from "./space_primitives"; -import { AttachmentMeta, PageMeta } from "../types"; +import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives"; +import { AttachmentMeta, FileMeta, PageMeta } from "../types"; import { EventEmitter } from "@plugos/plugos/event"; import { Plug } from "@plugos/plugos/plug"; -import { Manifest } from "../manifest"; -import { plugPrefix, trashPrefix } from "./constants"; +import { plugPrefix } from "./constants"; import { safeRun } from "../util"; const pageWatchInterval = 2000; @@ -19,23 +14,21 @@ export type SpaceEvents = { pageListUpdated: (pages: Set) => void; }; -export class Space - extends EventEmitter - implements SpacePrimitives -{ +export class Space extends EventEmitter { pageMetaCache = new Map(); watchedPages = new Set(); private initialPageListLoad = true; private saving = false; - constructor(private space: SpacePrimitives, private trashEnabled = true) { + constructor(private space: SpacePrimitives) { super(); } public async updatePageList() { - let newPageList = await this.space.fetchPageList(); + let newPageList = await this.fetchPageList(); + // console.log("Updating page list", newPageList); let deletedPages = new Set(this.pageMetaCache.keys()); - newPageList.pages.forEach((meta) => { + newPageList.forEach((meta) => { const pageName = meta.name; const oldPageMeta = this.pageMetaCache.get(pageName); const newPageMeta: PageMeta = { @@ -50,9 +43,7 @@ export class Space this.emit("pageCreated", newPageMeta); } else if ( oldPageMeta && - oldPageMeta.lastModified !== newPageMeta.lastModified && - (!this.trashEnabled || - (this.trashEnabled && !pageName.startsWith(trashPrefix))) + oldPageMeta.lastModified !== newPageMeta.lastModified ) { this.emit("pageChanged", newPageMeta); } @@ -95,17 +86,7 @@ export class Space async deletePage(name: string, deleteDate?: number): Promise { await this.getPageMeta(name); // Check if page exists, if not throws Error - if (this.trashEnabled) { - let pageData = await this.readPage(name); - // Move to trash - await this.writePage( - `${trashPrefix}${name}`, - pageData.text, - true, - deleteDate - ); - } - await this.space.deletePage(name); + await this.space.deleteFile(`${name}.md`); this.pageMetaCache.delete(name); this.emit("pageDeleted", name); @@ -114,7 +95,9 @@ export class Space async getPageMeta(name: string): Promise { let oldMeta = this.pageMetaCache.get(name); - let newMeta = await this.space.getPageMeta(name); + let newMeta = fileMetaToPageMeta( + await this.space.getFileMeta(`${name}.md`) + ); if (oldMeta) { if (oldMeta.lastModified !== newMeta.lastModified) { // Changed on disk, trigger event @@ -133,41 +116,15 @@ export class Space return this.space.invokeFunction(plug, env, name, args); } - listPages(unfiltered = false): Set { - if (unfiltered) { - return new Set(this.pageMetaCache.values()); - } else { - return new Set( - [...this.pageMetaCache.values()].filter( - (pageMeta) => - !pageMeta.name.startsWith(trashPrefix) && - !pageMeta.name.startsWith(plugPrefix) - ) - ); - } + listPages(): Set { + return new Set(this.pageMetaCache.values()); } - listTrash(): Set { - return new Set( - [...this.pageMetaCache.values()] - .filter( - (pageMeta) => - pageMeta.name.startsWith(trashPrefix) && - !pageMeta.name.startsWith(plugPrefix) - ) - .map((pageMeta) => ({ - ...pageMeta, - name: pageMeta.name.substring(trashPrefix.length), - })) - ); - } - - listPlugs(): Set { - return new Set( - [...this.pageMetaCache.values()].filter((pageMeta) => - pageMeta.name.startsWith(plugPrefix) - ) - ); + async listPlugs(): Promise { + let allFiles = await this.space.fetchFileList(); + return allFiles + .filter((fileMeta) => fileMeta.name.endsWith(".plug.json")) + .map((fileMeta) => fileMeta.name); } proxySyscall(plug: Plug, name: string, args: any[]): Promise { @@ -175,16 +132,20 @@ export class Space } async readPage(name: string): Promise<{ text: string; meta: PageMeta }> { - let pageData = await this.space.readPage(name); + let pageData = await this.space.readFile(`${name}.md`, "string"); let previousMeta = this.pageMetaCache.get(name); + let newMeta = fileMetaToPageMeta(pageData.meta); if (previousMeta) { - if (previousMeta.lastModified !== pageData.meta.lastModified) { + if (previousMeta.lastModified !== newMeta.lastModified) { // Page changed since last cached metadata, trigger event - this.emit("pageChanged", pageData.meta); + this.emit("pageChanged", newMeta); } } - this.pageMetaCache.set(name, pageData.meta); - return pageData; + let meta = this.metaCacher(name, newMeta); + return { + text: pageData.data as string, + meta: meta, + }; } watchPage(pageName: string) { @@ -198,16 +159,12 @@ export class Space async writePage( name: string, text: string, - selfUpdate?: boolean, - lastModified?: number + selfUpdate?: boolean ): Promise { try { this.saving = true; - let pageMeta = await this.space.writePage( - name, - text, - selfUpdate, - lastModified + let pageMeta = fileMetaToPageMeta( + await this.space.writeFile(`${name}.md`, "string", text, selfUpdate) ); if (!selfUpdate) { this.emit("pageChanged", pageMeta); @@ -218,39 +175,52 @@ export class Space } } - fetchPageList(): Promise<{ pages: Set; nowTimestamp: number }> { - return this.space.fetchPageList(); + async fetchPageList(): Promise { + return (await this.space.fetchFileList()) + .filter((fileMeta) => fileMeta.name.endsWith(".md")) + .map(fileMetaToPageMeta); } - fetchAttachmentList(): Promise<{ - attachments: Set; - nowTimestamp: number; - }> { - return this.space.fetchAttachmentList(); + async fetchAttachmentList(): Promise { + return (await this.space.fetchFileList()).filter( + (fileMeta) => + !fileMeta.name.endsWith(".md") && !fileMeta.name.endsWith(".plug.json") + ); } + readAttachment( name: string, - encoding: AttachmentEncoding - ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> { - return this.space.readAttachment(name, encoding); - } - getAttachmentMeta(name: string): Promise { - return this.space.getAttachmentMeta(name); - } - writeAttachment( - name: string, - data: AttachmentData, - selfUpdate?: boolean | undefined, - lastModified?: number | undefined - ): Promise { - return this.space.writeAttachment(name, data, selfUpdate, lastModified); - } - deleteAttachment(name: string): Promise { - return this.space.deleteAttachment(name); + encoding: FileEncoding + ): Promise<{ data: FileData; meta: AttachmentMeta }> { + return this.space.readFile(name, encoding); } - private metaCacher(name: string, pageMeta: PageMeta): PageMeta { - this.pageMetaCache.set(name, pageMeta); - return pageMeta; + getAttachmentMeta(name: string): Promise { + return this.space.getFileMeta(name); + } + + writeAttachment( + name: string, + encoding: FileEncoding, + data: FileData, + selfUpdate?: boolean | undefined + ): Promise { + return this.space.writeFile(name, encoding, data, selfUpdate); + } + + deleteAttachment(name: string): Promise { + return this.space.deleteFile(name); + } + + private metaCacher(name: string, meta: PageMeta): PageMeta { + this.pageMetaCache.set(name, meta); + return meta; } } + +function fileMetaToPageMeta(fileMeta: FileMeta): PageMeta { + return { + ...fileMeta, + name: fileMeta.name.substring(0, fileMeta.name.length - 3), + } as PageMeta; +} diff --git a/packages/common/spaces/space_primitives.ts b/packages/common/spaces/space_primitives.ts index fb0431b2..4156a3a7 100644 --- a/packages/common/spaces/space_primitives.ts +++ b/packages/common/spaces/space_primitives.ts @@ -1,38 +1,23 @@ import { Plug } from "@plugos/plugos/plug"; -import { AttachmentMeta, PageMeta } from "../types"; +import { FileMeta } from "../types"; -export type AttachmentEncoding = "arraybuffer" | "dataurl"; -export type AttachmentData = ArrayBuffer | string; +export type FileEncoding = "string" | "arraybuffer" | "dataurl"; +export type FileData = ArrayBuffer | string; export interface SpacePrimitives { // Pages - fetchPageList(): Promise<{ pages: Set; nowTimestamp: number }>; - readPage(name: string): Promise<{ text: string; meta: PageMeta }>; - getPageMeta(name: string): Promise; - writePage( + fetchFileList(): Promise; + readFile( name: string, - text: string, - selfUpdate?: boolean, - lastModified?: number - ): Promise; - deletePage(name: string): Promise; - - // Attachments - fetchAttachmentList(): Promise<{ - attachments: Set; - nowTimestamp: number; - }>; - readAttachment( + encoding: FileEncoding + ): Promise<{ data: FileData; meta: FileMeta }>; + getFileMeta(name: string): Promise; + writeFile( name: string, - encoding: AttachmentEncoding - ): Promise<{ data: AttachmentData; meta: AttachmentMeta }>; - getAttachmentMeta(name: string): Promise; - writeAttachment( - name: string, - data: AttachmentData, - selfUpdate?: boolean, - lastModified?: number - ): Promise; - deleteAttachment(name: string): Promise; + encoding: FileEncoding, + data: FileData, + selfUpdate?: boolean + ): Promise; + deleteFile(name: string): Promise; // Plugs proxySyscall(plug: Plug, name: string, args: any[]): Promise; diff --git a/packages/common/spaces/sync.test.ts b/packages/common/spaces/sync.test.ts deleted file mode 100644 index e4cb8a5b..00000000 --- a/packages/common/spaces/sync.test.ts +++ /dev/null @@ -1,123 +0,0 @@ -import { expect, test } from "@jest/globals"; -import { IndexedDBSpacePrimitives } from "./indexeddb_space_primitives"; -import { SpaceSync } from "./sync"; -import { PageMeta } from "../types"; -import { Space } from "./space"; - -// For testing in node.js -require("fake-indexeddb/auto"); - -test("Test store", async () => { - let primary = new Space(new IndexedDBSpacePrimitives("primary"), true); - let secondary = new Space( - new IndexedDBSpacePrimitives("secondary", -5000), - true - ); - let sync = new SpaceSync(primary, secondary, 0, 0, "_trash/"); - - async function conflictResolver(pageMeta1: PageMeta, pageMeta2: PageMeta) {} - - // Write one page to primary - await primary.writePage("index", "Hello"); - expect((await secondary.listPages()).size).toBe(0); - await syncPages(conflictResolver); - expect((await secondary.listPages()).size).toBe(1); - expect((await secondary.readPage("index")).text).toBe("Hello"); - - // Should be a no-op - expect(await syncPages()).toBe(0); - - // Now let's make a change on the secondary - await secondary.writePage("index", "Hello!!"); - await secondary.writePage("test", "Test page"); - - // And sync it - await syncPages(); - - expect(primary.listPages().size).toBe(2); - expect(secondary.listPages().size).toBe(2); - - expect((await primary.readPage("index")).text).toBe("Hello!!"); - - // Let's make some random edits on both ends - await primary.writePage("index", "1"); - await primary.writePage("index2", "2"); - await secondary.writePage("index3", "3"); - await secondary.writePage("index4", "4"); - await syncPages(); - - expect((await primary.listPages()).size).toBe(5); - expect((await secondary.listPages()).size).toBe(5); - - expect(await syncPages()).toBe(0); - - console.log("Deleting pages"); - // Delete some pages - await primary.deletePage("index"); - await primary.deletePage("index3"); - - console.log("Pages", await primary.listPages()); - console.log("Trash", await primary.listTrash()); - - await syncPages(); - - expect((await primary.listPages()).size).toBe(3); - expect((await secondary.listPages()).size).toBe(3); - - // No-op - expect(await syncPages()).toBe(0); - - await secondary.deletePage("index4"); - await primary.deletePage("index2"); - - await syncPages(); - - // Just "test" left - expect((await primary.listPages()).size).toBe(1); - expect((await secondary.listPages()).size).toBe(1); - - // No-op - expect(await syncPages()).toBe(0); - - await secondary.writePage("index", "I'm back"); - - await syncPages(); - - expect((await primary.readPage("index")).text).toBe("I'm back"); - - // Cause a conflict - await primary.writePage("index", "Hello 1"); - await secondary.writePage("index", "Hello 2"); - - await syncPages(SpaceSync.primaryConflictResolver(primary, secondary)); - - // Sync conflicting copy back - await syncPages(); - - // Verify that primary won - expect((await primary.readPage("index")).text).toBe("Hello 1"); - expect((await secondary.readPage("index")).text).toBe("Hello 1"); - - // test + index + index.conflicting copy - expect((await primary.listPages()).size).toBe(3); - expect((await secondary.listPages()).size).toBe(3); - - async function syncPages( - conflictResolver?: ( - pageMeta1: PageMeta, - pageMeta2: PageMeta - ) => Promise - ): Promise { - // Awesome practice: adding sleeps to fix issues! - await sleep(2); - let n = await sync.syncPages(conflictResolver); - await sleep(2); - return n; - } -}); - -function sleep(ms: number = 5): Promise { - return new Promise((resolve) => { - setTimeout(resolve, ms); - }); -} diff --git a/packages/common/spaces/sync.ts b/packages/common/spaces/sync.ts deleted file mode 100644 index e2631c75..00000000 --- a/packages/common/spaces/sync.ts +++ /dev/null @@ -1,208 +0,0 @@ -import { Space } from "./space"; -import { PageMeta } from "../types"; -import { SpacePrimitives } from "./space_primitives"; - -export class SpaceSync { - constructor( - private primary: Space, - private secondary: Space, - public primaryLastSync: number, - public secondaryLastSync: number, - private trashPrefix: string - ) {} - - // Strategy: Primary wins - public static primaryConflictResolver( - primary: Space, - secondary: Space - ): (pageMeta1: PageMeta, pageMeta2: PageMeta) => Promise { - return async (pageMeta1, pageMeta2) => { - const pageName = pageMeta1.name; - const revisionPageName = `${pageName}.conflicted.${pageMeta2.lastModified}`; - // Copy secondary to conflict copy - let oldPageData = await secondary.readPage(pageName); - await secondary.writePage(revisionPageName, oldPageData.text); - - // Write replacement on top - let newPageData = await primary.readPage(pageName); - await secondary.writePage( - pageName, - newPageData.text, - true, - newPageData.meta.lastModified - ); - }; - } - - async syncablePages( - space: Space - ): Promise<{ pages: PageMeta[]; nowTimestamp: number }> { - let fetchResult = await space.fetchPageList(); - return { - pages: [...fetchResult.pages].filter( - (pageMeta) => !pageMeta.name.startsWith(this.trashPrefix) - ), - nowTimestamp: fetchResult.nowTimestamp, - }; - } - - async trashPages(space: SpacePrimitives): Promise { - return [...(await space.fetchPageList()).pages] - .filter((pageMeta) => pageMeta.name.startsWith(this.trashPrefix)) - .map((pageMeta) => ({ - ...pageMeta, - name: pageMeta.name.substring(this.trashPrefix.length), - })); - } - - async syncPages( - conflictResolver?: ( - pageMeta1: PageMeta, - pageMeta2: PageMeta - ) => Promise - ): Promise { - let syncOps = 0; - - let { pages: primaryAllPagesSet, nowTimestamp: primarySyncTimestamp } = - await this.syncablePages(this.primary); - let allPagesPrimary = new Map(primaryAllPagesSet.map((p) => [p.name, p])); - let { pages: secondaryAllPagesSet, nowTimestamp: secondarySyncTimestamp } = - await this.syncablePages(this.secondary); - let allPagesSecondary = new Map( - secondaryAllPagesSet.map((p) => [p.name, p]) - ); - - let allTrashPrimary = new Map( - (await this.trashPages(this.primary)) - // Filter out old trash - .filter((p) => p.lastModified > this.primaryLastSync) - .map((p) => [p.name, p]) - ); - let allTrashSecondary = new Map( - (await this.trashPages(this.secondary)) - // Filter out old trash - .filter((p) => p.lastModified > this.secondaryLastSync) - .map((p) => [p.name, p]) - ); - - // Iterate over all pages on the primary first - for (let [name, pageMetaPrimary] of allPagesPrimary.entries()) { - let pageMetaSecondary = allPagesSecondary.get(pageMetaPrimary.name); - if (!pageMetaSecondary) { - // New page on primary - // Let's check it's not on the deleted list - if (allTrashSecondary.has(name)) { - // Explicitly deleted, let's skip - continue; - } - - // Push from primary to secondary - console.log("New page on primary", name, "syncing to secondary"); - let pageData = await this.primary.readPage(name); - await this.secondary.writePage( - name, - pageData.text, - true, - secondarySyncTimestamp // The reason for this is to not include it in the next sync cycle, we cannot blindly use the lastModified date due to time skew - ); - syncOps++; - } else { - // Existing page - if (pageMetaPrimary.lastModified > this.primaryLastSync) { - // Primary updated since last sync - if (pageMetaSecondary.lastModified > this.secondaryLastSync) { - // Secondary also updated! CONFLICT - if (conflictResolver) { - await conflictResolver(pageMetaPrimary, pageMetaSecondary); - } else { - throw Error( - `Sync conflict for ${name} with no conflict resolver specified` - ); - } - } else { - // Ok, not changed on secondary, push it secondary - console.log( - "Changed page on primary", - name, - "syncing to secondary" - ); - let pageData = await this.primary.readPage(name); - await this.secondary.writePage( - name, - pageData.text, - false, - secondarySyncTimestamp - ); - syncOps++; - } - } else if (pageMetaSecondary.lastModified > this.secondaryLastSync) { - // Secondary updated, but not primary (checked above) - // Push from secondary to primary - console.log("Changed page on secondary", name, "syncing to primary"); - let pageData = await this.secondary.readPage(name); - await this.primary.writePage( - name, - pageData.text, - false, - primarySyncTimestamp - ); - syncOps++; - } else { - // Neither updated, no-op - } - } - } - - // Now do a simplified version in reverse, only detecting new pages - for (let [name, pageMetaSecondary] of allPagesSecondary.entries()) { - if (!allPagesPrimary.has(pageMetaSecondary.name)) { - // New page on secondary - // Let's check it's not on the deleted list - if (allTrashPrimary.has(name)) { - // Explicitly deleted, let's skip - continue; - } - // Push from secondary to primary - console.log("New page on secondary", name, "pushing to primary"); - let pageData = await this.secondary.readPage(name); - await this.primary.writePage( - name, - pageData.text, - false, - primarySyncTimestamp - ); - syncOps++; - } - } - - // And finally, let's trash some pages - for (let pageToDelete of allTrashPrimary.values()) { - console.log("Deleting", pageToDelete.name, "on secondary"); - try { - await this.secondary.deletePage( - pageToDelete.name, - secondarySyncTimestamp - ); - syncOps++; - } catch (e: any) { - console.log("Page already gone", e.message); - } - } - - for (let pageToDelete of allTrashSecondary.values()) { - console.log("Deleting", pageToDelete.name, "on primary"); - try { - await this.primary.deletePage(pageToDelete.name, primarySyncTimestamp); - syncOps++; - } catch (e: any) { - console.log("Page already gone", e.message); - } - } - - // Setting last sync time to the timestamps we got back when fetching the page lists on each end - this.primaryLastSync = primarySyncTimestamp; - this.secondaryLastSync = secondarySyncTimestamp; - - return syncOps; - } -} diff --git a/packages/common/types.ts b/packages/common/types.ts index 50d13dd4..79e3c396 100644 --- a/packages/common/types.ts +++ b/packages/common/types.ts @@ -1,6 +1,13 @@ -export const reservedPageNames = ["page", "attachment", "plug"]; export const maximumAttachmentSize = 100 * 1024 * 1024; // 100 MB +export type FileMeta = { + name: string; + lastModified: number; + contentType: string; + size: number; + perm: "ro" | "rw"; +}; + export type PageMeta = { name: string; lastModified: number; diff --git a/packages/plugos-silverbullet-syscall/space.ts b/packages/plugos-silverbullet-syscall/space.ts index eb23db68..4fc28ce1 100644 --- a/packages/plugos-silverbullet-syscall/space.ts +++ b/packages/plugos-silverbullet-syscall/space.ts @@ -23,6 +23,10 @@ export async function deletePage(name: string): Promise { return syscall("space.deletePage", name); } +export async function listPlugs(): Promise { + return syscall("space.listPlugs"); +} + export async function listAttachments(): Promise { return syscall("space.listAttachments"); } @@ -39,9 +43,10 @@ export async function readAttachment( export async function writeAttachment( name: string, - buffer: ArrayBuffer + encoding: "string" | "dataurl", + data: string ): Promise { - return syscall("space.writeAttachment", name, buffer); + return syscall("space.writeAttachment", name, encoding, data); } export async function deleteAttachment(name: string): Promise { diff --git a/packages/plugos-syscall/fulltext.ts b/packages/plugos-syscall/fulltext.ts index ebdc1dda..f0bf9f17 100644 --- a/packages/plugos-syscall/fulltext.ts +++ b/packages/plugos-syscall/fulltext.ts @@ -5,7 +5,7 @@ export async function fullTextIndex(key: string, value: string) { } export async function fullTextDelete(key: string) { - return syscall("fulltext.index", key); + return syscall("fulltext.delete", key); } export async function fullTextSearch(phrase: string, limit: number = 100) { diff --git a/packages/plugos/environments/node_worker.ts b/packages/plugos/environments/node_worker.ts index 1de96665..47f25dc0 100644 --- a/packages/plugos/environments/node_worker.ts +++ b/packages/plugos/environments/node_worker.ts @@ -40,6 +40,8 @@ let vm = new VM({ setInterval, URL, clearInterval, + TextEncoder, + TextDecoder, fetch: require(`${nodeModulesPath}/node-fetch`), WebSocket: require(`${nodeModulesPath}/ws`), // This is only going to be called for pre-bundled modules, we won't allow diff --git a/packages/plugs/core/cloud.ts b/packages/plugs/core/cloud.ts index 37358716..78d441e8 100644 --- a/packages/plugs/core/cloud.ts +++ b/packages/plugs/core/cloud.ts @@ -1,16 +1,24 @@ +import type { + FileData, + FileEncoding, +} from "@silverbulletmd/common/spaces/space_primitives"; import { renderToText, replaceNodesMatching, } from "@silverbulletmd/common/tree"; -import { PageMeta } from "@silverbulletmd/common/types"; +import type { FileMeta, PageMeta } from "@silverbulletmd/common/types"; import { parseMarkdown } from "@silverbulletmd/plugos-silverbullet-syscall/markdown"; const pagePrefix = "💭 "; -export async function readPageCloud( - name: string -): Promise<{ text: string; meta: PageMeta } | undefined> { - let originalUrl = name.substring(pagePrefix.length); +export async function readFileCloud( + name: string, + encoding: FileEncoding +): Promise<{ data: FileData; meta: FileMeta } | undefined> { + let originalUrl = name.substring( + pagePrefix.length, + name.length - ".md".length + ); let url = originalUrl; if (!url.includes("/")) { url += "/index"; @@ -32,13 +40,15 @@ export async function readPageCloud( text = e.message; } return { - text: await translateLinksWithPrefix( + data: await translateLinksWithPrefix( text, `${pagePrefix}${originalUrl.split("/")[0]}/` ), meta: { name, + contentType: "text/markdown", lastModified: 0, + size: text.length, perm: "ro", }, }; @@ -60,9 +70,11 @@ async function translateLinksWithPrefix( return text; } -export async function getPageMetaCloud(name: string): Promise { +export async function getFileMetaCloud(name: string): Promise { return { name, + size: 0, + contentType: "text/markdown", lastModified: 0, perm: "ro", }; diff --git a/packages/plugs/core/core.plug.yaml b/packages/plugs/core/core.plug.yaml index 39909aaf..e8b408bd 100644 --- a/packages/plugs/core/core.plug.yaml +++ b/packages/plugs/core/core.plug.yaml @@ -156,12 +156,12 @@ functions: path: ./search.ts:readPageSearch pageNamespace: pattern: "🔍 .+" - operation: readPage + operation: readFile getPageMetaSearch: path: ./search.ts:getPageMetaSearch pageNamespace: pattern: "🔍 .+" - operation: getPageMeta + operation: getFileMeta # Template commands insertPageMeta: @@ -374,12 +374,12 @@ functions: # Cloud pages readPageCloud: - path: ./cloud.ts:readPageCloud + path: ./cloud.ts:readFileCloud pageNamespace: pattern: "💭 .+" - operation: readPage + operation: readFile getPageMetaCloud: - path: ./cloud.ts:getPageMetaCloud + path: ./cloud.ts:getFileMetaCloud pageNamespace: pattern: "💭 .+" - operation: getPageMeta + operation: getFileMeta diff --git a/packages/plugs/core/navigate.ts b/packages/plugs/core/navigate.ts index c92286d5..28c7ef79 100644 --- a/packages/plugs/core/navigate.ts +++ b/packages/plugs/core/navigate.ts @@ -14,7 +14,7 @@ import { invokeCommand } from "@silverbulletmd/plugos-silverbullet-syscall/syste // Checks if the URL contains a protocol, if so keeps it, otherwise assumes an attachment function patchUrl(url: string): string { if (url.indexOf("://") === -1) { - return `attachment/${url}`; + return `fs/${url}`; } return url; } diff --git a/packages/plugs/core/plugmanager.ts b/packages/plugs/core/plugmanager.ts index 8a7ce0ff..8e3da1d7 100644 --- a/packages/plugs/core/plugmanager.ts +++ b/packages/plugs/core/plugmanager.ts @@ -5,9 +5,9 @@ import { save, } from "@silverbulletmd/plugos-silverbullet-syscall/editor"; import { - deletePage, - listPages, - writePage, + deleteAttachment, + listPlugs, + writeAttachment, } from "@silverbulletmd/plugos-silverbullet-syscall/space"; import { invokeFunction, @@ -16,13 +16,6 @@ import { import { readYamlPage } from "../lib/yaml_page"; -async function listPlugs(): Promise { - let unfilteredPages = await listPages(true); - return unfilteredPages - .filter((p) => p.name.startsWith("_plug/")) - .map((p) => p.name.substring("_plug/".length)); -} - export async function updatePlugsCommand() { await save(); flashNotification("Updating plugs..."); @@ -39,9 +32,11 @@ export async function updatePlugs() { let plugList: string[] = []; try { const plugListRead: any[] = await readYamlPage("PLUGS"); - plugList = plugListRead.filter((plug) => typeof plug === 'string'); + plugList = plugListRead.filter((plug) => typeof plug === "string"); if (plugList.length !== plugListRead.length) { - throw new Error(`Some of the plugs were not in a yaml list format, they were ignored`); + throw new Error( + `Some of the plugs were not in a yaml list format, they were ignored` + ); } } catch (e: any) { throw new Error(`Error processing PLUGS: ${e.message}`); @@ -58,17 +53,23 @@ export async function updatePlugs() { let manifest = manifests[0]; allPlugNames.push(manifest.name); // console.log("Writing", `_plug/${manifest.name}`); - await writePage( - `_plug/${manifest.name}`, - JSON.stringify(manifest, null, 2) + await writeAttachment( + `_plug/${manifest.name}.plug.json`, + "string", + JSON.stringify(manifest) ); } // And delete extra ones for (let existingPlug of await listPlugs()) { - if (!allPlugNames.includes(existingPlug)) { - console.log("Removing plug", existingPlug); - await deletePage(`_plug/${existingPlug}`); + let plugName = existingPlug.substring( + "_plug/".length, + existingPlug.length - ".plug.json".length + ); + console.log("Considering", plugName); + if (!allPlugNames.includes(plugName)) { + console.log("Removing plug", plugName); + await deleteAttachment(existingPlug); } } await reloadPlugs(); @@ -97,17 +98,23 @@ export async function getPlugGithub(identifier: string): Promise { ); } -export async function getPlugGithubRelease(identifier: string): Promise { +export async function getPlugGithubRelease( + identifier: string +): Promise { let [owner, repo, version] = identifier.split("/"); if (!version || version === "latest") { - console.log('fetching the latest version'); - const req = await fetch(`https://api.github.com/repos/${owner}/${repo}/releases/latest`); + console.log("fetching the latest version"); + const req = await fetch( + `https://api.github.com/repos/${owner}/${repo}/releases/latest` + ); if (req.status !== 200) { - throw new Error(`Could not fetch latest relase manifest from ${identifier}}`); + throw new Error( + `Could not fetch latest relase manifest from ${identifier}}` + ); } const result = await req.json(); version = result.name; - } + } const finalUrl = `//github.com/${owner}/${repo}/releases/download/${version}/${repo}.plug.json`; return getPlugHTTPS(finalUrl); -} \ No newline at end of file +} diff --git a/packages/plugs/markdown/util.ts b/packages/plugs/markdown/util.ts index 400ad28e..69428721 100644 --- a/packages/plugs/markdown/util.ts +++ b/packages/plugs/markdown/util.ts @@ -44,7 +44,7 @@ export async function cleanMarkdown( if (n.type === "URL") { const url = n.children![0].text!; if (url.indexOf("://") === -1) { - n.children![0].text = `attachment/${url}`; + n.children![0].text = `fs/${url}`; } console.log("Link", url); } diff --git a/packages/server/express_server.ts b/packages/server/express_server.ts index 9459e17a..7840dfc7 100644 --- a/packages/server/express_server.ts +++ b/packages/server/express_server.ts @@ -3,7 +3,6 @@ import { Manifest, SilverBulletHooks } from "@silverbulletmd/common/manifest"; import { EndpointHook } from "@plugos/plugos/hooks/endpoint"; import { readdir, readFile } from "fs/promises"; import { System } from "@plugos/plugos/system"; -import cors from "cors"; import { DiskSpacePrimitives } from "@silverbulletmd/common/spaces/disk_space_primitives"; import path from "path"; import bodyParser from "body-parser"; @@ -32,14 +31,6 @@ import { plugPrefix } from "@silverbulletmd/common/spaces/constants"; import sandboxSyscalls from "@plugos/plugos/syscalls/sandbox"; // @ts-ignore import settingsTemplate from "bundle-text:./SETTINGS_template.md"; - -const globalModules: any = JSON.parse( - readFileSync( - nodeModulesDir + "/node_modules/@silverbulletmd/web/dist/global.plug.json", - "utf-8" - ) -); - import { safeRun } from "./util"; import { ensureFTSTable, @@ -50,10 +41,18 @@ import { PageNamespaceHook } from "./hooks/page_namespace"; import { readFileSync } from "fs"; import fileSystemSyscalls from "@plugos/plugos/syscalls/fs.node"; import { - storeSyscalls, ensureTable as ensureStoreTable, + storeSyscalls, } from "@plugos/plugos/syscalls/store.knex_node"; import { parseYamlSettings } from "@silverbulletmd/common/util"; +import { SpacePrimitives } from "@silverbulletmd/common/spaces/space_primitives"; + +const globalModules: any = JSON.parse( + readFileSync( + nodeModulesDir + "/node_modules/@silverbulletmd/web/dist/global.plug.json", + "utf-8" + ) +); const safeFilename = /^[a-zA-Z0-9_\-\.]+$/; @@ -76,6 +75,7 @@ export class ExpressServer { builtinPlugDir: string; password?: string; settings: { [key: string]: any } = {}; + spacePrimitives: SpacePrimitives; constructor(options: ServerOptions) { this.port = options.port; @@ -96,16 +96,14 @@ export class ExpressServer { this.system.addHook(namespaceHook); // The space - this.space = new Space( - new EventedSpacePrimitives( - new PlugSpacePrimitives( - new DiskSpacePrimitives(options.pagesPath), - namespaceHook - ), - this.eventHook + this.spacePrimitives = new EventedSpacePrimitives( + new PlugSpacePrimitives( + new DiskSpacePrimitives(options.pagesPath), + namespaceHook ), - true + this.eventHook ); + this.space = new Space(this.spacePrimitives); // The database used for persistence (SQLite) this.db = knex({ @@ -222,8 +220,9 @@ export class ExpressServer { ); let manifest: Manifest = JSON.parse(manifestJson); pluginNames.push(manifest.name); - await this.space.writePage( - `${plugPrefix}${manifest.name}`, + await this.spacePrimitives.writeFile( + `${plugPrefix}${file}`, + "string", manifestJson ); } @@ -245,16 +244,17 @@ export class ExpressServer { async reloadPlugs() { await this.space.updatePageList(); - let allPlugs = this.space.listPlugs(); - if (allPlugs.size === 0) { + let allPlugs = await this.space.listPlugs(); + if (allPlugs.length === 0) { await this.bootstrapBuiltinPlugs(); - allPlugs = this.space.listPlugs(); + allPlugs = await this.space.listPlugs(); } await this.system.unloadAll(); console.log("Loading plugs"); - for (let pageInfo of allPlugs) { - let { text } = await this.space.readPage(pageInfo.name); - await this.system.load(JSON.parse(text), createSandbox); + console.log(allPlugs); + for (let plugName of allPlugs) { + let { data } = await this.space.readAttachment(plugName, "string"); + await this.system.load(JSON.parse(data as string), createSandbox); } this.rebuildMdExtensions(); } @@ -283,39 +283,16 @@ export class ExpressServer { // Pages API this.app.use( - "/page", + "/fs", passwordMiddleware, - cors({ - methods: "GET,HEAD,PUT,OPTIONS,POST,DELETE", - preflightContinue: true, - }), - this.buildFsRouter() - ); - - // Attachment API - this.app.use( - "/attachment", - passwordMiddleware, - cors({ - methods: "GET,HEAD,PUT,OPTIONS,POST,DELETE", - preflightContinue: true, - }), - this.buildAttachmentRouter() + buildFsRouter(this.spacePrimitives) ); // Plug API - this.app.use( - "/plug", - passwordMiddleware, - cors({ - methods: "GET,HEAD,PUT,OPTIONS,POST,DELETE", - preflightContinue: true, - }), - this.buildPlugRouter() - ); + this.app.use("/plug", passwordMiddleware, this.buildPlugRouter()); // Fallback, serve index.html - this.app.get("/*", async (req, res) => { + this.app.get(/^(\/((?!fs\/).)+)$/, async (req, res) => { res.sendFile(`${this.distDir}/index.html`, {}); }); @@ -387,205 +364,6 @@ export class ExpressServer { return plugRouter; } - private buildFsRouter() { - let fsRouter = express.Router(); - - // Page list - fsRouter.route("/").get(async (req, res) => { - let { nowTimestamp, pages } = await this.space.fetchPageList(); - res.header("Now-Timestamp", "" + nowTimestamp); - res.json([...pages]); - }); - - fsRouter - .route(/\/(.+)/) - .get(async (req, res) => { - let pageName = req.params[0]; - // console.log("Getting", pageName); - try { - let pageData = await this.space.readPage(pageName); - res.status(200); - res.header("Last-Modified", "" + pageData.meta.lastModified); - res.header("X-Permission", pageData.meta.perm); - res.header("Content-Type", "text/markdown"); - res.send(pageData.text); - } catch (e) { - // CORS - res.status(200); - res.header("X-Status", "404"); - res.send(""); - } - }) - .put(bodyParser.text({ type: "*/*" }), async (req, res) => { - let pageName = req.params[0]; - console.log("Saving", pageName); - - try { - let meta = await this.space.writePage( - pageName, - req.body, - false, - req.header("Last-Modified") - ? +req.header("Last-Modified")! - : undefined - ); - res.status(200); - res.header("Last-Modified", "" + meta.lastModified); - res.header("X-Permission", meta.perm); - res.send("OK"); - } catch (err) { - res.status(500); - res.send("Write failed"); - console.error("Pipeline failed", err); - } - }) - .options(async (req, res) => { - let pageName = req.params[0]; - try { - const meta = await this.space.getPageMeta(pageName); - res.status(200); - res.header("Last-Modified", "" + meta.lastModified); - res.header("X-Permission", meta.perm); - res.header("Content-Type", "text/markdown"); - res.send(""); - } catch (e) { - // CORS - res.status(200); - res.header("X-Status", "404"); - res.send("Not found"); - } - }) - .delete(async (req, res) => { - let pageName = req.params[0]; - try { - await this.space.deletePage(pageName); - res.status(200); - res.send("OK"); - } catch (e) { - console.error("Error deleting file", e); - res.status(500); - res.send("OK"); - } - }); - return fsRouter; - } - - // Build attachment router - private buildAttachmentRouter() { - let fsaRouter = express.Router(); - - // Page list - fsaRouter.route("/").get(async (req, res) => { - let { nowTimestamp, attachments } = - await this.space.fetchAttachmentList(); - res.header("Now-Timestamp", "" + nowTimestamp); - res.json([...attachments]); - }); - - fsaRouter - .route(/\/(.+)/) - .get(async (req, res) => { - let attachmentName = req.params[0]; - if (!this.attachmentCheck(attachmentName, res)) { - return; - } - console.log("Getting", attachmentName); - try { - let attachmentData = await this.space.readAttachment( - attachmentName, - "arraybuffer" - ); - res.status(200); - res.header("Last-Modified", "" + attachmentData.meta.lastModified); - res.header("X-Permission", attachmentData.meta.perm); - res.header("Content-Type", attachmentData.meta.contentType); - // res.header("X-Content-Length", "" + attachmentData.meta.size); - res.send(Buffer.from(attachmentData.data as ArrayBuffer)); - } catch (e) { - // CORS - res.status(200); - res.header("X-Status", "404"); - res.send(""); - } - }) - .put( - bodyParser.raw({ type: "*/*", limit: "100mb" }), - async (req, res) => { - let attachmentName = req.params[0]; - if (!this.attachmentCheck(attachmentName, res)) { - return; - } - console.log("Saving attachment", attachmentName); - - try { - let meta = await this.space.writeAttachment( - attachmentName, - req.body, - false, - req.header("Last-Modified") - ? +req.header("Last-Modified")! - : undefined - ); - res.status(200); - res.header("Last-Modified", "" + meta.lastModified); - res.header("Content-Type", meta.contentType); - res.header("Content-Length", "" + meta.size); - res.header("X-Permission", meta.perm); - res.send("OK"); - } catch (err) { - res.status(500); - res.send("Write failed"); - console.error("Pipeline failed", err); - } - } - ) - .options(async (req, res) => { - let attachmentName = req.params[0]; - if (!this.attachmentCheck(attachmentName, res)) { - return; - } - try { - const meta = await this.space.getAttachmentMeta(attachmentName); - res.status(200); - res.header("Last-Modified", "" + meta.lastModified); - res.header("X-Permission", meta.perm); - res.header("Content-Length", "" + meta.size); - res.header("Content-Type", meta.contentType); - res.send(""); - } catch (e) { - // CORS - res.status(200); - res.header("X-Status", "404"); - res.send("Not found"); - } - }) - .delete(async (req, res) => { - let attachmentName = req.params[0]; - if (!this.attachmentCheck(attachmentName, res)) { - return; - } - try { - await this.space.deleteAttachment(attachmentName); - res.status(200); - res.send("OK"); - } catch (e) { - console.error("Error deleting attachment", e); - res.status(500); - res.send("OK"); - } - }); - return fsaRouter; - } - - attachmentCheck(attachmentName: string, res: express.Response): boolean { - if (attachmentName.endsWith(".md")) { - res.status(405); - res.send("No markdown files allowed through the attachment API"); - return false; - } - return true; - } - async ensureAndLoadSettings() { try { await this.space.getPageMeta("SETTINGS"); @@ -628,3 +406,82 @@ export class ExpressServer { } } } + +function buildFsRouter(spacePrimitives: SpacePrimitives) { + let fsRouter = express.Router(); + + // File list + fsRouter.route("/").get(async (req, res, next) => { + res.json(await spacePrimitives.fetchFileList()); + }); + + fsRouter + .route(/\/(.+)/) + .get(async (req, res, next) => { + let name = req.params[0]; + console.log("Getting", name); + try { + let attachmentData = await spacePrimitives.readFile( + name, + "arraybuffer" + ); + res.status(200); + res.header("Last-Modified", "" + attachmentData.meta.lastModified); + res.header("X-Permission", attachmentData.meta.perm); + res.header("Content-Type", attachmentData.meta.contentType); + res.send(Buffer.from(attachmentData.data as ArrayBuffer)); + } catch (e) { + next(); + } + }) + .put(bodyParser.raw({ type: "*/*", limit: "100mb" }), async (req, res) => { + let name = req.params[0]; + console.log("Saving file", name); + + try { + let meta = await spacePrimitives.writeFile( + name, + "arraybuffer", + req.body, + false + ); + res.status(200); + res.header("Last-Modified", "" + meta.lastModified); + res.header("Content-Type", meta.contentType); + res.header("Content-Length", "" + meta.size); + res.header("X-Permission", meta.perm); + res.send("OK"); + } catch (err) { + res.status(500); + res.send("Write failed"); + console.error("Pipeline failed", err); + } + }) + .options(async (req, res, next) => { + let name = req.params[0]; + try { + const meta = await spacePrimitives.getFileMeta(name); + res.status(200); + res.header("Last-Modified", "" + meta.lastModified); + res.header("X-Permission", meta.perm); + res.header("Content-Length", "" + meta.size); + res.header("Content-Type", meta.contentType); + res.send(""); + } catch (e) { + next(); + } + }) + .delete(async (req, res) => { + let name = req.params[0]; + try { + await spacePrimitives.deleteFile(name); + res.status(200); + res.send("OK"); + } catch (e) { + console.error("Error deleting attachment", e); + res.status(500); + res.send("OK"); + } + }); + return fsRouter; +} diff --git a/packages/server/hooks/page_namespace.ts b/packages/server/hooks/page_namespace.ts index 4e99e0ac..9e29193a 100644 --- a/packages/server/hooks/page_namespace.ts +++ b/packages/server/hooks/page_namespace.ts @@ -3,16 +3,16 @@ import { System } from "@plugos/plugos/system"; import { Hook, Manifest } from "@plugos/plugos/types"; import { Express, NextFunction, Request, Response, Router } from "express"; -export type PageNamespaceOperation = - | "readPage" - | "writePage" - | "listPages" - | "getPageMeta" - | "deletePage"; +export type NamespaceOperation = + | "readFile" + | "writeFile" + | "listFiles" + | "getFileMeta" + | "deleteFile"; export type PageNamespaceDef = { pattern: string; - operation: PageNamespaceOperation; + operation: NamespaceOperation; }; export type PageNamespaceHookT = { @@ -20,7 +20,7 @@ export type PageNamespaceHookT = { }; type SpaceFunction = { - operation: PageNamespaceOperation; + operation: NamespaceOperation; pattern: RegExp; plug: Plug; name: string; @@ -76,11 +76,11 @@ export class PageNamespaceHook implements Hook { } if ( ![ - "readPage", - "writePage", - "getPageMeta", - "listPages", - "deletePage", + "readFile", + "writeFile", + "getFileMeta", + "listFiles", + "deleteFile", ].includes(funcDef.pageNamespace.operation) ) { errors.push( diff --git a/packages/server/hooks/plug_space_primitives.ts b/packages/server/hooks/plug_space_primitives.ts index 0b7d0715..6dc375b1 100644 --- a/packages/server/hooks/plug_space_primitives.ts +++ b/packages/server/hooks/plug_space_primitives.ts @@ -1,11 +1,15 @@ import { Plug } from "@plugos/plugos/plug"; import { - AttachmentData, - AttachmentEncoding, + FileData, + FileEncoding, SpacePrimitives, } from "@silverbulletmd/common/spaces/space_primitives"; -import { AttachmentMeta, PageMeta } from "@silverbulletmd/common/types"; -import { PageNamespaceHook, PageNamespaceOperation } from "./page_namespace"; +import { + AttachmentMeta, + FileMeta, + PageMeta, +} from "@silverbulletmd/common/types"; +import { PageNamespaceHook, NamespaceOperation } from "./page_namespace"; export class PlugSpacePrimitives implements SpacePrimitives { constructor( @@ -14,7 +18,7 @@ export class PlugSpacePrimitives implements SpacePrimitives { ) {} performOperation( - type: PageNamespaceOperation, + type: NamespaceOperation, pageName: string, ...args: any[] ): Promise | false { @@ -26,101 +30,71 @@ export class PlugSpacePrimitives implements SpacePrimitives { return false; } - async fetchPageList(): Promise<{ - pages: Set; - nowTimestamp: number; - }> { - let allPages = new Set(); + async fetchFileList(): Promise { + let allFiles: FileMeta[] = []; for (let { plug, name, operation } of this.hook.spaceFunctions) { - if (operation === "listPages") { + if (operation === "listFiles") { try { for (let pm of await plug.invoke(name, [])) { - allPages.add(pm); + allFiles.push(pm); } } catch (e) { - console.error("Error listing pages", e); + console.error("Error listing files", e); } } } - let result = await this.wrapped.fetchPageList(); - for (let pm of result.pages) { - allPages.add(pm); + let result = await this.wrapped.fetchFileList(); + for (let pm of result) { + allFiles.push(pm); } - return { - nowTimestamp: result.nowTimestamp, - pages: allPages, - }; + return allFiles; } - readPage(name: string): Promise<{ text: string; meta: PageMeta }> { - let result = this.performOperation("readPage", name); - if (result) { - return result; - } - return this.wrapped.readPage(name); - } - - getPageMeta(name: string): Promise { - let result = this.performOperation("getPageMeta", name); - if (result) { - return result; - } - return this.wrapped.getPageMeta(name); - } - - writePage( + readFile( name: string, - text: string, - selfUpdate?: boolean, - lastModified?: number - ): Promise { + encoding: FileEncoding + ): Promise<{ data: FileData; meta: FileMeta }> { + let result = this.performOperation("readFile", name); + if (result) { + return result; + } + return this.wrapped.readFile(name, encoding); + } + + getFileMeta(name: string): Promise { + let result = this.performOperation("getFileMeta", name); + if (result) { + return result; + } + return this.wrapped.getFileMeta(name); + } + + writeFile( + name: string, + encoding: FileEncoding, + data: FileData, + selfUpdate?: boolean + ): Promise { let result = this.performOperation( - "writePage", + "writeFile", name, - text, - selfUpdate, - lastModified + encoding, + data, + selfUpdate ); if (result) { return result; } - return this.wrapped.writePage(name, text, selfUpdate, lastModified); + return this.wrapped.writeFile(name, encoding, data, selfUpdate); } - deletePage(name: string): Promise { - let result = this.performOperation("deletePage", name); + deleteFile(name: string): Promise { + let result = this.performOperation("deleteFile", name); if (result) { return result; } - return this.wrapped.deletePage(name); - } - - fetchAttachmentList(): Promise<{ - attachments: Set; - nowTimestamp: number; - }> { - return this.wrapped.fetchAttachmentList(); - } - readAttachment( - name: string, - encoding: AttachmentEncoding - ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> { - return this.wrapped.readAttachment(name, encoding); - } - getAttachmentMeta(name: string): Promise { - return this.wrapped.getAttachmentMeta(name); - } - writeAttachment( - name: string, - blob: ArrayBuffer, - selfUpdate?: boolean | undefined, - lastModified?: number | undefined - ): Promise { - return this.wrapped.writeAttachment(name, blob, selfUpdate, lastModified); - } - deleteAttachment(name: string): Promise { - return this.wrapped.deleteAttachment(name); + return this.wrapped.deleteFile(name); } proxySyscall(plug: Plug, name: string, args: any[]): Promise { diff --git a/packages/server/syscalls/space.ts b/packages/server/syscalls/space.ts index 589d8c38..04338522 100644 --- a/packages/server/syscalls/space.ts +++ b/packages/server/syscalls/space.ts @@ -1,12 +1,15 @@ import { AttachmentMeta, PageMeta } from "@silverbulletmd/common/types"; import { SysCallMapping } from "@plugos/plugos/system"; import { Space } from "@silverbulletmd/common/spaces/space"; -import { AttachmentData } from "@silverbulletmd/common/spaces/space_primitives"; +import { + FileData, + FileEncoding, +} from "@silverbulletmd/common/spaces/space_primitives"; export default (space: Space): SysCallMapping => { return { - "space.listPages": async (ctx, unfiltered = false): Promise => { - return [...space.listPages(unfiltered)]; + "space.listPages": async (): Promise => { + return [...space.listPages()]; }, "space.readPage": async ( ctx, @@ -27,13 +30,16 @@ export default (space: Space): SysCallMapping => { "space.deletePage": async (ctx, name: string) => { return space.deletePage(name); }, + "space.listPlugs": async (): Promise => { + return await space.listPlugs(); + }, "space.listAttachments": async (ctx): Promise => { - return [...(await space.fetchAttachmentList()).attachments]; + return await space.fetchAttachmentList(); }, "space.readAttachment": async ( ctx, name: string - ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> => { + ): Promise<{ data: FileData; meta: AttachmentMeta }> => { return await space.readAttachment(name, "dataurl"); }, "space.getAttachmentMeta": async ( @@ -45,9 +51,10 @@ export default (space: Space): SysCallMapping => { "space.writeAttachment": async ( ctx, name: string, + encoding: FileEncoding, data: string ): Promise => { - return await space.writeAttachment(name, data); + return await space.writeAttachment(name, encoding, data); }, "space.deleteAttachment": async (ctx, name: string) => { await space.deleteAttachment(name); diff --git a/packages/web/boot.ts b/packages/web/boot.ts index 03104936..ce55a500 100644 --- a/packages/web/boot.ts +++ b/packages/web/boot.ts @@ -11,7 +11,9 @@ safeRun(async () => { let settingsPageText = ""; while (true) { try { - settingsPageText = (await httpPrimitives.readPage("SETTINGS")).text; + settingsPageText = (await ( + await httpPrimitives.readFile("SETTINGS.md", "string") + ).data) as string; break; } catch (e: any) { if (e.message === "Unauthorized") { @@ -25,7 +27,7 @@ safeRun(async () => { } } } - let serverSpace = new Space(httpPrimitives, true); + let serverSpace = new Space(httpPrimitives); serverSpace.watch(); console.log("Booting..."); diff --git a/packages/web/editor.tsx b/packages/web/editor.tsx index ef85f807..fee767a7 100644 --- a/packages/web/editor.tsx +++ b/packages/web/editor.tsx @@ -55,11 +55,7 @@ import { MDExt, } from "@silverbulletmd/common/markdown_ext"; import { FilterList } from "./components/filter"; -import { - FilterOption, - PageMeta, - reservedPageNames, -} from "@silverbulletmd/common/types"; +import { FilterOption, PageMeta } from "@silverbulletmd/common/types"; import { syntaxTree } from "@codemirror/language"; import sandboxSyscalls from "@plugos/plugos/syscalls/sandbox"; import { eventSyscalls } from "@plugos/plugos/syscalls/event"; @@ -208,13 +204,6 @@ export class Editor { this.focus(); this.pageNavigator.subscribe(async (pageName, pos: number | string) => { - if (reservedPageNames.includes(pageName)) { - this.flashNotification( - `"${pageName}" is a reserved page name. It cannot be used.`, - "error" - ); - return; - } console.log("Now navigating to", pageName, pos); if (!this.editorView) { @@ -534,10 +523,10 @@ export class Editor { await this.space.updatePageList(); await this.system.unloadAll(); console.log("(Re)loading plugs"); - for (let pageInfo of this.space.listPlugs()) { + for (let plugName of await this.space.listPlugs()) { // console.log("Loading plug", pageInfo.name); - let { text } = await this.space.readPage(pageInfo.name); - await this.system.load(JSON.parse(text), createIFrameSandbox); + let { data } = await this.space.readAttachment(plugName, "string"); + await this.system.load(JSON.parse(data as string), createIFrameSandbox); } this.rebuildEditorState(); await this.dispatchAppEvent("plugs:loaded"); @@ -617,11 +606,6 @@ export class Editor { const previousPage = this.currentPage; - this.viewDispatch({ - type: "page-loading", - name: pageName, - }); - // Persist current page state and nicely close page if (previousPage) { this.saveState(previousPage); @@ -629,6 +613,11 @@ export class Editor { await this.save(true); } + this.viewDispatch({ + type: "page-loading", + name: pageName, + }); + // Fetch next page to open let doc; try { diff --git a/packages/web/editor_paste.ts b/packages/web/editor_paste.ts index 550ee566..298e79ca 100644 --- a/packages/web/editor_paste.ts +++ b/packages/web/editor_paste.ts @@ -118,7 +118,7 @@ export function attachmentExtension(editor: Editor) { if (!finalFileName) { return; } - await editor.space.writeAttachment(finalFileName, data!); + await editor.space.writeAttachment(finalFileName, "arraybuffer", data!); let attachmentMarkdown = `[${finalFileName}](${finalFileName})`; if (mimeType.startsWith("image/")) { attachmentMarkdown = `![](${finalFileName})`; diff --git a/packages/web/inline_image.ts b/packages/web/inline_image.ts index 7114a687..d41074a5 100644 --- a/packages/web/inline_image.ts +++ b/packages/web/inline_image.ts @@ -23,7 +23,7 @@ class InlineImageWidget extends WidgetType { if (this.url.startsWith("http")) { img.src = this.url; } else { - img.src = `attachment/${this.url}`; + img.src = `fs/${this.url}`; } img.alt = this.title; img.title = this.title; diff --git a/packages/web/service_worker.ts b/packages/web/service_worker.ts index bd35bf79..c305cdc0 100644 --- a/packages/web/service_worker.ts +++ b/packages/web/service_worker.ts @@ -35,10 +35,8 @@ self.addEventListener("fetch", (event: any) => { return response; } else { if ( - parsedUrl.pathname !== "/page" && + parsedUrl.pathname !== "/fs" && !parsedUrl.pathname.startsWith("/page/") && - parsedUrl.pathname !== "/attachment" && - !parsedUrl.pathname.startsWith("/attachment/") && !parsedUrl.pathname.startsWith("/plug/") ) { return cache.match("/index.html"); diff --git a/packages/web/syscalls/space.ts b/packages/web/syscalls/space.ts index 51f27804..ccee3763 100644 --- a/packages/web/syscalls/space.ts +++ b/packages/web/syscalls/space.ts @@ -1,12 +1,15 @@ import { Editor } from "../editor"; import { SysCallMapping } from "@plugos/plugos/system"; import { AttachmentMeta, PageMeta } from "@silverbulletmd/common/types"; -import { AttachmentData } from "@silverbulletmd/common/spaces/space_primitives"; +import { + FileData, + FileEncoding, +} from "@silverbulletmd/common/spaces/space_primitives"; export function spaceSyscalls(editor: Editor): SysCallMapping { return { - "space.listPages": async (ctx, unfiltered = false): Promise => { - return [...(await editor.space.listPages(unfiltered))]; + "space.listPages": async (): Promise => { + return [...editor.space.listPages()]; }, "space.readPage": async ( ctx, @@ -34,13 +37,16 @@ export function spaceSyscalls(editor: Editor): SysCallMapping { console.log("Deleting page"); await editor.space.deletePage(name); }, + "space.listPlugs": async (): Promise => { + return await editor.space.listPlugs(); + }, "space.listAttachments": async (ctx): Promise => { - return [...(await editor.space.fetchAttachmentList()).attachments]; + return await editor.space.fetchAttachmentList(); }, "space.readAttachment": async ( ctx, name: string - ): Promise<{ data: AttachmentData; meta: AttachmentMeta }> => { + ): Promise<{ data: FileData; meta: AttachmentMeta }> => { return await editor.space.readAttachment(name, "dataurl"); }, "space.getAttachmentMeta": async ( @@ -52,9 +58,10 @@ export function spaceSyscalls(editor: Editor): SysCallMapping { "space.writeAttachment": async ( ctx, name: string, - buffer: ArrayBuffer + encoding: FileEncoding, + data: FileData ): Promise => { - return await editor.space.writeAttachment(name, buffer); + return await editor.space.writeAttachment(name, encoding, data); }, "space.deleteAttachment": async (ctx, name: string) => { await editor.space.deleteAttachment(name);