diff --git a/.gitignore b/.gitignore index a40e7c0a..114e103c 100644 --- a/.gitignore +++ b/.gitignore @@ -12,4 +12,6 @@ env.sh node_modules *.db test_space -silverbullet \ No newline at end of file +silverbullet +# Local Netlify folder +.netlify diff --git a/common/hooks/page_namespace.ts b/common/hooks/page_namespace.ts index 77db1ba0..c7c3e765 100644 --- a/common/hooks/page_namespace.ts +++ b/common/hooks/page_namespace.ts @@ -69,7 +69,7 @@ export class PageNamespaceHook implements Hook { if (!manifest.functions) { return []; } - for (let [funcName, funcDef] of Object.entries(manifest.functions)) { + for (const [funcName, funcDef] of Object.entries(manifest.functions)) { if (funcDef.pageNamespace) { if (!funcDef.pageNamespace.pattern) { errors.push(`Function ${funcName} has a namespace but no pattern`); diff --git a/common/spaces/http_space_primitives.ts b/common/spaces/http_space_primitives.ts index 2b3bd62e..753680a7 100644 --- a/common/spaces/http_space_primitives.ts +++ b/common/spaces/http_space_primitives.ts @@ -26,30 +26,31 @@ export class HttpSpacePrimitives implements SpacePrimitives { }); if (result.redirected) { // Got a redirect, we'll assume this is due to invalid credentials and redirecting to an auth page - console.log("Got a redirect via the API so will redirect to URL", url); + console.log( + "Got a redirect via the API so will redirect to URL", + result.url, + ); location.href = result.url; throw new Error("Invalid credentials"); } return result; } - getRealStatus(r: Response) { - if (r.headers.get("X-Status")) { - return +r.headers.get("X-Status")!; - } - return r.status; - } - async fetchFileList(): Promise { - const resp = await this.authenticatedFetch(this.url, { + const resp = await this.authenticatedFetch(`${this.url}/index.json`, { method: "GET", + headers: { + Accept: "application/json", + }, }); if ( - this.getRealStatus(resp) === 200 && + resp.status === 200 && this.expectedSpacePath && resp.headers.get("X-Space-Path") !== this.expectedSpacePath ) { + console.log("Expected space path", this.expectedSpacePath); + console.log("Got space path", resp.headers.get("X-Space-Path")); await flushCachesAndUnregisterServiceWorker(); alert("Space folder path different on server, reloading the page"); location.reload(); @@ -67,7 +68,7 @@ export class HttpSpacePrimitives implements SpacePrimitives { method: "GET", }, ); - if (this.getRealStatus(res) === 404) { + if (res.status === 404) { throw new Error(`Not found`); } return { @@ -109,7 +110,7 @@ export class HttpSpacePrimitives implements SpacePrimitives { method: "DELETE", }, ); - if (this.getRealStatus(req) !== 200) { + if (req.status !== 200) { throw Error(`Failed to delete file: ${req.statusText}`); } } @@ -118,10 +119,10 @@ export class HttpSpacePrimitives implements SpacePrimitives { const res = await this.authenticatedFetch( `${this.url}/${encodeURI(name)}`, { - method: "OPTIONS", + method: "HEAD", }, ); - if (this.getRealStatus(res) === 404) { + if (res.status === 404) { throw new Error(`Not found`); } return this.responseToMeta(name, res); @@ -130,7 +131,7 @@ export class HttpSpacePrimitives implements SpacePrimitives { private responseToMeta(name: string, res: Response): FileMeta { return { name, - size: +res.headers.get("X-Content-Length")!, + size: +res.headers.get("Content-Length")!, contentType: res.headers.get("Content-type")!, lastModified: +(res.headers.get("X-Last-Modified") || "0"), perm: (res.headers.get("X-Permission") as "rw" | "ro") || "rw", diff --git a/import_map.json b/import_map.json index 0b83e09e..696c2d1e 100644 --- a/import_map.json +++ b/import_map.json @@ -18,7 +18,6 @@ "@codemirror/search": "https://esm.sh/@codemirror/search@6.4.0?external=@codemirror/state,@codemirror/view&target=es2022", "preact": "https://esm.sh/preact@10.11.1", - "yjs": "https://esm.sh/yjs@13.5.42?deps=lib0@0.2.70&target=es2022", "$sb/": "./plug-api/", "handlebars": "https://esm.sh/handlebars@4.7.7?target=es2022", "dexie": "https://esm.sh/dexie@3.2.2" diff --git a/plug-api/lib/page.ts b/plug-api/lib/page.ts new file mode 100644 index 00000000..e90fe65b --- /dev/null +++ b/plug-api/lib/page.ts @@ -0,0 +1,4 @@ +export function isValidPageName(name: string): boolean { + // Page can not be empty and not end with a file extension (e.g. "bla.md") + return name !== "" && !/\.[a-zA-Z]+$/.test(name); +} diff --git a/plug-api/lib/settings_page.ts b/plug-api/lib/settings_page.ts index 9b09650f..1df934b8 100644 --- a/plug-api/lib/settings_page.ts +++ b/plug-api/lib/settings_page.ts @@ -1,8 +1,7 @@ -import { readYamlPage } from "./yaml_page.ts"; -import { notifyUser } from "./util.ts"; +import { readYamlPage } from "$sb/lib/yaml_page.ts"; import { YAML } from "$sb/plugos-syscall/mod.ts"; -import { space } from "$sb/silverbullet-syscall/mod.ts"; +import { editor, space } from "$sb/silverbullet-syscall/mod.ts"; /** * Convenience function to read a specific set of settings from the `SETTINGS` page as well as default values @@ -65,7 +64,7 @@ export async function writeSettings(settings: T) { try { readSettings = (await readYamlPage(SETTINGS_PAGE, ["yaml"])) || {}; } catch { - await notifyUser("Creating a new SETTINGS page...", "info"); + await editor.flashNotification("Creating a new SETTINGS page...", "info"); } const writeSettings: any = { ...readSettings, ...settings }; // const doc = new YAML.Document(); diff --git a/plug-api/lib/util.ts b/plug-api/lib/util.ts deleted file mode 100644 index 32d4974f..00000000 --- a/plug-api/lib/util.ts +++ /dev/null @@ -1,36 +0,0 @@ -import { editor } from "$sb/silverbullet-syscall/mod.ts"; - -export async function replaceAsync( - str: string, - regex: RegExp, - asyncFn: (match: string, ...args: any[]) => Promise, -) { - const promises: Promise[] = []; - str.replace(regex, (match: string, ...args: any[]): string => { - const promise = asyncFn(match, ...args); - promises.push(promise); - return ""; - }); - const data = await Promise.all(promises); - return str.replace(regex, () => data.shift()!); -} - -export function isServer() { - return ( - typeof window === "undefined" || typeof window.document === "undefined" - ); // if something defines window the same way as the browser, this will fail. -} - -// this helps keep if's condition as positive -export function isBrowser() { - return !isServer(); -} - -export function notifyUser(message: string, type?: "info" | "error") { - if (isBrowser()) { - return editor.flashNotification(message, type); - } - const log = type === "error" ? console.error : console.log; - log(message); // we should end up sending the message to the user, users dont read logs. - return; -} diff --git a/plug-api/silverbullet-syscall/collab.ts b/plug-api/silverbullet-syscall/collab.ts deleted file mode 100644 index 8cfe85de..00000000 --- a/plug-api/silverbullet-syscall/collab.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { syscall } from "./syscall.ts"; - -export function start(serverUrl: string, token: string, username: string) { - return syscall("collab.start", serverUrl, token, username); -} - -export function stop() { - return syscall("collab.stop"); -} diff --git a/plug-api/silverbullet-syscall/mod.ts b/plug-api/silverbullet-syscall/mod.ts index 8c217bcd..7500d2ce 100644 --- a/plug-api/silverbullet-syscall/mod.ts +++ b/plug-api/silverbullet-syscall/mod.ts @@ -3,7 +3,6 @@ export * as index from "./index.ts"; export * as markdown from "./markdown.ts"; export * as space from "./space.ts"; export * as system from "./system.ts"; -export * as collab from "./collab.ts"; // Legacy redirect, use "store" in $sb/plugos-syscall/mod.ts instead export * as clientStore from "./store.ts"; export * as sync from "./sync.ts"; diff --git a/plugos/asset_bundle/builder.ts b/plugos/asset_bundle/builder.ts index dabfeb3f..3981a3be 100644 --- a/plugos/asset_bundle/builder.ts +++ b/plugos/asset_bundle/builder.ts @@ -38,6 +38,7 @@ export async function bundleFolder( bundlePath: string, ) { const bundle = new AssetBundle(); + await Deno.mkdir(path.dirname(bundlePath), { recursive: true }); for await ( const { path: filePath } of walk(rootPath, { includeDirs: false }) diff --git a/plugs/builtin_plugs.ts b/plugs/builtin_plugs.ts index 7f3b10ac..2ac354f3 100644 --- a/plugs/builtin_plugs.ts +++ b/plugs/builtin_plugs.ts @@ -1,6 +1,5 @@ // TODO: Figure out how to keep this up-to-date automatically export const builtinPlugNames = [ - "collab", "core", "directive", "emoji", diff --git a/plugs/collab/collab.plug.yaml b/plugs/collab/collab.plug.yaml deleted file mode 100644 index 6e94c69a..00000000 --- a/plugs/collab/collab.plug.yaml +++ /dev/null @@ -1,36 +0,0 @@ -name: collab -functions: - detectCollabPage: - path: "./collab.ts:detectPage" - events: - - editor:pageLoaded - - plugs:loaded - joinCommand: - path: "./collab.ts:joinCommand" - command: - name: "Share: Join Collab" - shareCommand: - path: "./collab.ts:shareCommand" - command: - name: "Share: Collab" - shareNoop: - path: "./collab.ts:shareNoop" - events: - - share:collab - - # Space extension - readPageCollab: - path: ./collab.ts:readFileCollab - pageNamespace: - pattern: "collab:.+" - operation: readFile - writePageCollab: - path: ./collab.ts:writeFileCollab - pageNamespace: - pattern: "collab:.+" - operation: writeFile - getPageMetaCollab: - path: ./collab.ts:getFileMetaCollab - pageNamespace: - pattern: "collab:.+" - operation: getFileMeta diff --git a/plugs/collab/collab.ts b/plugs/collab/collab.ts deleted file mode 100644 index 56698736..00000000 --- a/plugs/collab/collab.ts +++ /dev/null @@ -1,158 +0,0 @@ -import { - findNodeOfType, - removeParentPointers, - renderToText, -} from "$sb/lib/tree.ts"; -import { getText } from "$sb/silverbullet-syscall/editor.ts"; -import { parseMarkdown } from "$sb/silverbullet-syscall/markdown.ts"; -import { - extractFrontmatter, - prepareFrontmatterDispatch, -} from "$sb/lib/frontmatter.ts"; -import { store, YAML } from "$sb/plugos-syscall/mod.ts"; -import { collab, editor, markdown } from "$sb/silverbullet-syscall/mod.ts"; - -import { nanoid } from "https://esm.sh/nanoid@4.0.0"; -import { FileMeta } from "../../common/types.ts"; - -const defaultServer = "wss://collab.silverbullet.md"; - -async function ensureUsername(): Promise { - let username = await store.get("collabUsername"); - if (!username) { - username = await editor.prompt( - "Please enter a publicly visible user name (or cancel for 'anonymous'):", - ); - if (!username) { - return "anonymous"; - } else { - await store.set("collabUsername", username); - } - } - return username; -} - -export async function joinCommand() { - let collabUri = await editor.prompt( - "Collab share URI:", - ); - if (!collabUri) { - return; - } - if (!collabUri.startsWith("collab:")) { - collabUri = "collab:" + collabUri; - } - await editor.navigate(collabUri); -} - -export async function shareCommand() { - const serverUrl = await editor.prompt( - "Please enter the URL of the collab server to use:", - defaultServer, - ); - if (!serverUrl) { - return; - } - const roomId = nanoid().replaceAll("_", "-"); - await editor.save(); - const text = await editor.getText(); - const tree = await markdown.parseMarkdown(text); - let { $share } = await extractFrontmatter(tree); - if (!$share) { - $share = []; - } - if (!Array.isArray($share)) { - $share = [$share]; - } - - removeParentPointers(tree); - const dispatchData = await prepareFrontmatterDispatch(tree, { - $share: [...$share, `collab:${serverUrl}/${roomId}`], - }); - - await editor.dispatch(dispatchData); - - collab.start( - serverUrl, - roomId, - await ensureUsername(), - ); -} - -export async function detectPage() { - const tree = await parseMarkdown(await getText()); - const frontMatter = findNodeOfType(tree, "FrontMatter"); - if (frontMatter) { - const yamlText = renderToText(frontMatter.children![1].children![0]); - try { - let { $share } = await YAML.parse(yamlText) as any; - if (!$share) { - return; - } - if (!Array.isArray($share)) { - $share = [$share]; - } - for (const uri of $share) { - if (uri.startsWith("collab:")) { - console.log("Going to enable collab"); - const uriPieces = uri.substring("collab:".length).split("/"); - await collab.start( - // All parts except the last one - uriPieces.slice(0, uriPieces.length - 1).join("/"), - // because the last one is the room ID - uriPieces[uriPieces.length - 1], - await ensureUsername(), - ); - } - } - } catch (e) { - console.error("Error parsing YAML", e); - } - } -} - -export function shareNoop() { - return true; -} - -export function readFileCollab( - name: string, -): { data: Uint8Array; meta: FileMeta } { - if (!name.endsWith(".md")) { - throw new Error("Not found"); - } - const collabUri = name.substring(0, name.length - ".md".length); - const text = `---\n$share: ${collabUri}\n---\n`; - - return { - // encoding === "arraybuffer" is not an option, so either it's "utf8" or "dataurl" - data: new TextEncoder().encode(text), - meta: { - name, - contentType: "text/markdown", - size: text.length, - lastModified: 0, - perm: "rw", - }, - }; -} - -export function getFileMetaCollab(name: string): FileMeta { - return { - name, - contentType: "text/markdown", - size: -1, - lastModified: 0, - perm: "rw", - }; -} - -export function writeFileCollab(name: string): FileMeta { - return { - name, - contentType: "text/markdown", - size: -1, - lastModified: 0, - perm: "rw", - }; -} diff --git a/plugs/collab/constants.ts b/plugs/collab/constants.ts deleted file mode 100644 index 53ec7f9d..00000000 --- a/plugs/collab/constants.ts +++ /dev/null @@ -1 +0,0 @@ -export const collabPingInterval = 2500; diff --git a/plugs/core/navigate.ts b/plugs/core/navigate.ts index ec56d867..50844e83 100644 --- a/plugs/core/navigate.ts +++ b/plugs/core/navigate.ts @@ -77,7 +77,7 @@ async function actionClickOrActionEnter( return editor.flashNotification("Empty link, ignoring", "error"); } if (url.indexOf("://") === -1 && !url.startsWith("mailto:")) { - return editor.openUrl(`/.fs/${decodeURI(url)}`); + return editor.openUrl(decodeURI(url)); } else { await editor.openUrl(url); } diff --git a/plugs/core/page.ts b/plugs/core/page.ts index 7cf410a9..b6268f1a 100644 --- a/plugs/core/page.ts +++ b/plugs/core/page.ts @@ -23,6 +23,7 @@ import { import { applyQuery, removeQueries } from "$sb/lib/query.ts"; import { extractFrontmatter } from "$sb/lib/frontmatter.ts"; import { invokeFunction } from "$sb/silverbullet-syscall/system.ts"; +import { isValidPageName } from "$sb/lib/page.ts"; // Key space: // pl:toPage:pos => pageName @@ -136,6 +137,13 @@ export async function renamePage(cmdDef: any) { return; } + if (!isValidPageName(newName)) { + return editor.flashNotification( + "Invalid page name: page names cannot end with a file extension", + "error", + ); + } + console.log("New name", newName); if (newName.trim() === oldName.trim()) { diff --git a/plugs/directive/command.ts b/plugs/directive/command.ts index e82e4177..ee06f71c 100644 --- a/plugs/directive/command.ts +++ b/plugs/directive/command.ts @@ -20,27 +20,6 @@ export async function updateDirectivesOnPageCommand(arg: any) { return; } - // if (!(await sync.hasInitialSyncCompleted())) { - // console.info("Initial sync hasn't completed yet, not updating directives."); - // return; - // } - - // If this page is shared ($share) via collab: disable directives as well - // due to security concerns - if (metaData.$share) { - for (const uri of metaData.$share) { - if (uri.startsWith("collab:")) { - if (explicitCall) { - await editor.flashNotification( - "Directives are disabled for 'collab' pages (safety reasons).", - "error", - ); - } - return; - } - } - } - // Collect all directives and their body replacements const replacements: { fullMatch: string; textPromise: Promise }[] = []; diff --git a/plugs/directive/template_directive.ts b/plugs/directive/template_directive.ts index 7190fb35..ef3feccb 100644 --- a/plugs/directive/template_directive.ts +++ b/plugs/directive/template_directive.ts @@ -1,6 +1,5 @@ import { queryRegex } from "$sb/lib/query.ts"; import { ParseTree, renderToText } from "$sb/lib/tree.ts"; -import { replaceAsync } from "$sb/lib/util.ts"; import { markdown, space } from "$sb/silverbullet-syscall/mod.ts"; import Handlebars from "handlebars"; @@ -68,32 +67,28 @@ export async function templateDirectiveRenderer( return newBody.trim(); } -export function cleanTemplateInstantiations(text: string): Promise { - return replaceAsync( - text, - directiveRegex, - ( - _fullMatch, - startInst, - type, - _args, - body, - endInst, - ): Promise => { - if (type === "use") { - body = body.replaceAll( - queryRegex, - ( - _fullMatch: string, - _startQuery: string, - _query: string, - body: string, - ) => { - return body.trim(); - }, - ); - } - return Promise.resolve(`${startInst}${body}${endInst}`); - }, - ); +export function cleanTemplateInstantiations(text: string) { + return text.replaceAll(directiveRegex, ( + _fullMatch, + startInst, + type, + _args, + body, + endInst, + ): string => { + if (type === "use") { + body = body.replaceAll( + queryRegex, + ( + _fullMatch: string, + _startQuery: string, + _query: string, + body: string, + ) => { + return body.trim(); + }, + ); + } + return `${startInst}${body}${endInst}`; + }); } diff --git a/plugs/federation/federation.ts b/plugs/federation/federation.ts index 7cfad9c3..05308e87 100644 --- a/plugs/federation/federation.ts +++ b/plugs/federation/federation.ts @@ -5,9 +5,6 @@ import { readSetting } from "$sb/lib/settings_page.ts"; function resolveFederated(pageName: string): string { // URL without the prefix "!"" let url = pageName.substring(1); - const pieces = url.split("/"); - pieces.splice(1, 0, ".fs"); - url = pieces.join("/"); if (!url.startsWith("127.0.0.1") && !url.startsWith("localhost")) { url = `https://${url}`; } else { @@ -153,7 +150,7 @@ export async function deleteFile( export async function getFileMeta(name: string): Promise { const url = resolveFederated(name); console.log("Fetching federation file meta", url); - const r = await nativeFetch(url, { method: "OPTIONS" }); + const r = await nativeFetch(url, { method: "HEAD" }); const fileMeta = await responseToFileMeta(r, name); if (!r.ok) { throw new Error("Not found"); diff --git a/plugs/markdown/preview.ts b/plugs/markdown/preview.ts index 621ccfdd..22633152 100644 --- a/plugs/markdown/preview.ts +++ b/plugs/markdown/preview.ts @@ -7,7 +7,6 @@ export async function updateMarkdownPreview() { if (!(await store.get("enableMarkdownPreview"))) { return; } - const pageName = await editor.getCurrentPage(); const text = await editor.getText(); const mdTree = await parseMarkdown(text); // const cleanMd = await cleanMarkdown(text); @@ -18,7 +17,7 @@ export async function updateMarkdownPreview() { annotationPositions: true, translateUrls: (url) => { if (!url.includes("://")) { - return `/.fs/${url}`; + return decodeURI(url); } return url; }, diff --git a/scripts/build_website.sh b/scripts/build_website.sh index 28c96fdb..ced612af 100755 --- a/scripts/build_website.sh +++ b/scripts/build_website.sh @@ -10,33 +10,29 @@ if [ "$1" != "local" ]; then fi +deno task clean +mkdir -p website_build/_plug website_build/_client + +echo "Copying website content" +cp -r website/* website_build/ +#rm website_build/{_redirects,_headers} echo "Building silver bullet" -rm -rf website_build -deno task clean deno task build -echo "Cleaning website build dir" -rm -rf website_build -mkdir -p website_build/_fs/_plug website_build/_client + echo "Copying silverbullet runtime files" cp dist_client_bundle/* website_build/ cp -r dist_client_bundle/.client/* website_build/_client/ echo "And all plugs" -cp -r dist_plug_bundle/_plug/* website_build/_fs/_plug/ +cp -r dist_plug_bundle/_plug/* website_build/_plug/ #echo "And additional ones" -curl https://raw.githubusercontent.com/silverbulletmd/silverbullet-mermaid/main/mermaid.plug.js > website_build/_fs/_plug/mermaid.plug.js +curl https://raw.githubusercontent.com/silverbulletmd/silverbullet-mermaid/main/mermaid.plug.js > website_build/_plug/mermaid.plug.js echo "But remove some plugs" -rm -rf website_build/_fs/_plug/{plugmd}.plug.js +rm -rf website_build/_plug/{plugmd}.plug.js -echo "Copying website content into fs/" -cp -r website/* website_build/_fs/ -rm website_build/_fs/{_redirects,_headers} -echo "Copy website files another time into the root" -cp -r website/* website_build/ - -# Genereate random modified date, and replace in _headers too +# Generate random modified date, and replace in _headers too export LAST_MODIFIED_TIMESTAMP=$RANDOM cat website/_headers | sed "s/12345/$LAST_MODIFIED_TIMESTAMP/g" > website_build/_headers diff --git a/scripts/generate_fs_list.ts b/scripts/generate_fs_list.ts index b4520089..86fd7b18 100644 --- a/scripts/generate_fs_list.ts +++ b/scripts/generate_fs_list.ts @@ -4,7 +4,7 @@ import { walk } from "https://deno.land/std@0.165.0/fs/mod.ts"; import { resolve } from "https://deno.land/std@0.165.0/path/mod.ts"; import { mime } from "https://deno.land/x/mimetypes@v1.0.0/mod.ts"; -const rootDir = resolve("website_build/_fs"); +const rootDir = resolve("website_build"); const lastModifiedTimestamp = +Deno.env.get("LAST_MODIFIED_TIMESTAMP")! || Date.now(); @@ -14,7 +14,7 @@ for await ( const file of walk(rootDir, { includeDirs: false, // Exclude hidden files - skip: [/^.*\/\..+$/], + skip: [/^.*\/(\..+|_redirects|_headers|_client\/.*)$/], }) ) { const fullPath = file.path; diff --git a/server/collab.test.ts b/server/collab.test.ts deleted file mode 100644 index 952cbab9..00000000 --- a/server/collab.test.ts +++ /dev/null @@ -1,42 +0,0 @@ -import { assert, assertEquals } from "../test_deps.ts"; -import { CollabServer } from "./collab.ts"; - -Deno.test("Collab server", async () => { - const collabServer = new CollabServer(null as any); - console.log("Client 1 joins page 1"); - assertEquals(collabServer.updatePresence("client1", "page1"), {}); - assertEquals(collabServer.pages.size, 1); - assertEquals(collabServer.updatePresence("client1", "page2"), {}); - assertEquals(collabServer.pages.size, 1); - console.log("Client 2 joins to page 2, collab id created, but not exposed"); - assertEquals( - collabServer.updatePresence("client2", "page2").collabId, - undefined, - ); - assert( - collabServer.updatePresence("client1", "page2").collabId !== undefined, - ); - console.log("Client 2 moves to page 1, collab id destroyed"); - assertEquals(collabServer.updatePresence("client2", "page1"), {}); - assertEquals(collabServer.updatePresence("client1", "page2"), {}); - assertEquals(collabServer.pages.get("page2")!.collabId, undefined); - assertEquals(collabServer.pages.get("page1")!.collabId, undefined); - console.log("Going to cleanup, which should have no effect"); - collabServer.cleanup(50); - assertEquals(collabServer.pages.size, 2); - collabServer.updatePresence("client2", "page2"); - console.log("Going to sleep 20ms"); - await sleep(20); - console.log("Then client 1 pings, but client 2 does not"); - collabServer.updatePresence("client1", "page2"); - await sleep(20); - console.log("Going to cleanup, which should clean client 2"); - collabServer.cleanup(35); - assertEquals(collabServer.pages.get("page2")!.collabId, undefined); - assertEquals(collabServer.clients.size, 1); - console.log(collabServer); -}); - -function sleep(ms: number) { - return new Promise((resolve) => setTimeout(resolve, ms)); -} diff --git a/server/collab.ts b/server/collab.ts deleted file mode 100644 index 8f439184..00000000 --- a/server/collab.ts +++ /dev/null @@ -1,241 +0,0 @@ -import { getAvailablePortSync } from "https://deno.land/x/port@1.0.0/mod.ts"; -import { nanoid } from "https://esm.sh/nanoid@4.0.0"; -import { race, timeout } from "../common/async_util.ts"; -import { Application } from "./deps.ts"; -import { SpacePrimitives } from "../common/spaces/space_primitives.ts"; -import { collabPingInterval } from "../plugs/collab/constants.ts"; -import { Hocuspocus } from "./deps.ts"; - -type CollabPage = { - clients: Map; // clientId -> lastPing - collabId?: string; - // The currently elected provider of the initial document - masterClientId: string; -}; - -export class CollabServer { - clients: Map = new Map(); // clientId -> openPage - pages: Map = new Map(); - yCollabServer?: Hocuspocus; - - constructor(private spacePrimitives: SpacePrimitives) { - } - - start() { - setInterval(() => { - this.cleanup(3 * collabPingInterval); - }, collabPingInterval); - } - - updatePresence( - clientId: string, - currentPage: string, - ): { collabId?: string } { - let client = this.clients.get(clientId); - if (!client) { - client = { openPage: "", lastUpdate: 0 }; - this.clients.set(clientId, client); - } - client.lastUpdate = Date.now(); - if (currentPage !== client.openPage) { - // Client switched pages - // Update last page record - const lastCollabPage = this.pages.get(client.openPage); - if (lastCollabPage) { - lastCollabPage.clients.delete(clientId); - if (lastCollabPage.clients.size === 1) { - delete lastCollabPage.collabId; - } - - if (lastCollabPage.clients.size === 0) { - this.pages.delete(client.openPage); - } else { - // Elect a new master client - if (lastCollabPage.masterClientId === clientId) { - // Any is fine, really - lastCollabPage.masterClientId = - [...lastCollabPage.clients.keys()][0]; - } - } - } - // Ok, let's update our records now - client.openPage = currentPage; - } - - // Update new page - let nextCollabPage = this.pages.get(currentPage); - if (!nextCollabPage) { - // Newly opened page (no other clients on this page right now) - nextCollabPage = { - clients: new Map(), - masterClientId: clientId, - }; - this.pages.set(currentPage, nextCollabPage); - } - // Register last ping from us - nextCollabPage.clients.set(clientId, Date.now()); - - if (nextCollabPage.clients.size > 1 && !nextCollabPage.collabId) { - // Create a new collabId - nextCollabPage.collabId = nanoid(); - } - // console.log("State", this.pages); - if (nextCollabPage.collabId) { - // We will now expose this collabId, except when we're just starting this session - // in which case we'll wait for the original client to publish the document - const existingyCollabSession = this.yCollabServer?.documents.get( - buildCollabId(nextCollabPage.collabId, `${currentPage}.md`), - ); - if (existingyCollabSession) { - // console.log("Found an existing collab session already, let's join!"); - return { collabId: nextCollabPage.collabId }; - } else if (clientId === nextCollabPage.masterClientId) { - // console.log("We're the master, so we should connect"); - return { collabId: nextCollabPage.collabId }; - } else { - // We're not the first client, so we need to wait for the first client to connect - // console.log("We're not the master, so we should wait"); - return {}; - } - } else { - return {}; - } - } - - cleanup(timeout: number) { - // Clean up pages and their clients that haven't pinged for some time - for (const [pageName, page] of this.pages) { - for (const [clientId, lastPing] of page.clients) { - if (Date.now() - lastPing > timeout) { - // Eject client - page.clients.delete(clientId); - // Elect a new master client - if (page.masterClientId === clientId && page.clients.size > 0) { - page.masterClientId = [...page.clients.keys()][0]; - } - } - } - if (page.clients.size === 1) { - // If there's only one client left, we don't need to keep the collabId around anymore - delete page.collabId; - } - if (page.clients.size === 0) { - // And if we have no clients left, well... - this.pages.delete(pageName); - } - } - - for (const [clientId, { lastUpdate }] of this.clients) { - if (Date.now() - lastUpdate > timeout) { - // Eject client - this.clients.delete(clientId); - } - } - } - - route(app: Application) { - // The way this works is that we spin up a separate WS server locally and then proxy requests to it - // This is the only way I could get Hocuspocus to work with Deno - const internalPort = getAvailablePortSync(); - this.yCollabServer = new Hocuspocus({ - port: internalPort, - address: "127.0.0.1", - quiet: true, - onStoreDocument: async (data) => { - const [_, path] = splitCollabId(data.documentName); - const text = data.document.getText("codemirror").toString(); - console.log( - "[Hocuspocus]", - "Persisting", - path, - "to space on server", - ); - const meta = await this.spacePrimitives.writeFile( - path, - new TextEncoder().encode(text), - ); - // Broadcast new persisted lastModified date - data.document.broadcastStateless( - JSON.stringify({ - type: "persisted", - path, - lastModified: meta.lastModified, - }), - ); - return; - }, - onDisconnect: (client) => { - console.log("[Hocuspocus]", "Client disconnected", client.clientsCount); - if (client.clientsCount === 0) { - console.log( - "[Hocuspocus]", - "Last client disconnected from", - client.documentName, - "purging from memory", - ); - this.yCollabServer!.documents.delete(client.documentName); - } - return Promise.resolve(); - }, - }); - - this.yCollabServer.listen(); - - app.use((ctx) => { - // if (ctx.request.url.pathname === "/.ws") { - // const sock = ctx.upgrade(); - // sock.onmessage = (e) => { - // console.log("WS: Got message", e.data); - // }; - // } - // Websocket proxy to hocuspocus - if (ctx.request.url.pathname === "/.ws-collab") { - const sock = ctx.upgrade(); - - const ws = new WebSocket(`ws://localhost:${internalPort}`); - const wsReady = race([ - new Promise((resolve) => { - ws.onopen = () => { - resolve(); - }; - }), - timeout(1000), - ]).catch(() => { - console.error("Timeout waiting for collab to open websocket"); - sock.close(); - }); - sock.onmessage = (e) => { - // console.log("Got message", e); - wsReady.then(() => ws.send(e.data)).catch(console.error); - }; - sock.onclose = () => { - if (ws.OPEN) { - ws.close(); - } - }; - ws.onmessage = (e) => { - if (sock.OPEN) { - sock.send(e.data); - } else { - console.error("Got message from websocket but socket is not open"); - } - }; - ws.onclose = () => { - if (sock.OPEN) { - sock.close(); - } - }; - } - }); - } -} - -function splitCollabId(documentName: string): [string, string] { - const [collabId, ...pathPieces] = documentName.split("/"); - const path = pathPieces.join("/"); - return [collabId, path]; -} - -function buildCollabId(collabId: string, path: string): string { - return `${collabId}/${path}`; -} diff --git a/server/deps.ts b/server/deps.ts index 15932b1a..2db7e91c 100644 --- a/server/deps.ts +++ b/server/deps.ts @@ -1,5 +1,9 @@ export * from "../common/deps.ts"; -export { Application, Router } from "https://deno.land/x/oak@v12.4.0/mod.ts"; +export type { Next } from "https://deno.land/x/oak@v12.4.0/mod.ts"; +export { + Application, + Context, + Router, +} from "https://deno.land/x/oak@v12.4.0/mod.ts"; export * as etag from "https://deno.land/x/oak@v12.4.0/etag.ts"; - -export { Hocuspocus } from "npm:@hocuspocus/server@2.1.0"; +export { oakCors } from "https://deno.land/x/cors@v1.2.2/mod.ts"; diff --git a/server/http_server.ts b/server/http_server.ts index 194a99c1..8d3622b9 100644 --- a/server/http_server.ts +++ b/server/http_server.ts @@ -1,15 +1,13 @@ -import { Application, Router } from "./deps.ts"; +import { Application, Context, Next, oakCors, Router } from "./deps.ts"; import { SpacePrimitives } from "../common/spaces/space_primitives.ts"; import { AssetBundle } from "../plugos/asset_bundle/bundle.ts"; -import { base64Decode } from "../plugos/asset_bundle/base64.ts"; import { ensureSettingsAndIndex } from "../common/util.ts"; import { performLocalFetch } from "../common/proxy_fetch.ts"; import { BuiltinSettings } from "../web/types.ts"; import { gitIgnoreCompiler } from "./deps.ts"; import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts"; -import { CollabServer } from "./collab.ts"; import { Authenticator } from "./auth.ts"; -import { oakCors } from "https://deno.land/x/cors@v1.2.2/mod.ts"; +import { FileMeta } from "../common/types.ts"; export type ServerOptions = { hostname: string; @@ -31,7 +29,6 @@ export class HttpServer { clientAssetBundle: AssetBundle; settings?: BuiltinSettings; spacePrimitives: SpacePrimitives; - collab: CollabServer; authenticator: Authenticator; constructor( @@ -66,8 +63,6 @@ export class HttpServer { } }, ); - this.collab = new CollabServer(this.spacePrimitives); - this.collab.start(); } // Replaces some template variables in index.html in a rather ad-hoc manner, but YOLO @@ -76,76 +71,25 @@ export class HttpServer { .replaceAll( "{{SPACE_PATH}}", this.options.pagesPath.replaceAll("\\", "\\\\"), - ).replaceAll( - "{{SYNC_ENDPOINT}}", - "/.fs", ); } async start() { await this.reloadSettings(); + // Serve static files (javascript, css, html) - this.app.use(async ({ request, response }, next) => { - if (request.url.pathname === "/") { - // Note: we're explicitly not setting Last-Modified and If-Modified-Since header here because this page is dynamic - response.headers.set("Content-type", "text/html"); - response.body = this.renderIndexHtml(); - return; - } - try { - const assetName = request.url.pathname.slice(1); - if ( - this.clientAssetBundle.has(assetName) && - request.headers.get("If-Modified-Since") === - utcDateString(this.clientAssetBundle.getMtime(assetName)) - ) { - response.status = 304; - return; - } - response.status = 200; - response.headers.set( - "Content-type", - this.clientAssetBundle.getMimeType(assetName), - ); - const data = this.clientAssetBundle.readFileSync( - assetName, - ); - response.headers.set("Cache-Control", "no-cache"); - response.headers.set("Content-length", "" + data.length); - response.headers.set( - "Last-Modified", - utcDateString(this.clientAssetBundle.getMtime(assetName)), - ); + this.app.use(this.serveStatic.bind(this)); - if (request.method === "GET") { - response.body = data; - } - } catch { - await next(); - } - }); - - // Fallback, serve index.html - this.app.use(({ request, response }, next) => { - if ( - !request.url.pathname.startsWith("/.fs") && - request.url.pathname !== "/.auth" && - !request.url.pathname.startsWith("/.ws") - ) { - response.headers.set("Content-type", "text/html"); - response.body = this.renderIndexHtml(); - } else { - return next(); - } - }); - - // Pages API - const fsRouter = this.buildFsRouter(this.spacePrimitives); await this.addPasswordAuth(this.app); + const fsRouter = this.addFsRoutes(this.spacePrimitives); this.app.use(fsRouter.routes()); this.app.use(fsRouter.allowedMethods()); - this.collab.route(this.app); + // Fallback, serve the UI index.html + this.app.use(({ response }) => { + response.headers.set("Content-type", "text/html"); + response.body = this.renderIndexHtml(); + }); this.abortController = new AbortController(); const listenOptions: any = { @@ -172,6 +116,52 @@ export class HttpServer { ); } + serveStatic( + { request, response }: Context, Record>, + next: Next, + ) { + if ( + request.url.pathname === "/" + ) { + // Serve the UI (index.html) + // Note: we're explicitly not setting Last-Modified and If-Modified-Since header here because this page is dynamic + response.headers.set("Content-type", "text/html"); + response.body = this.renderIndexHtml(); + return; + } + try { + const assetName = request.url.pathname.slice(1); + if ( + this.clientAssetBundle.has(assetName) && + request.headers.get("If-Modified-Since") === + utcDateString(this.clientAssetBundle.getMtime(assetName)) + ) { + response.status = 304; + return; + } + response.status = 200; + response.headers.set( + "Content-type", + this.clientAssetBundle.getMimeType(assetName), + ); + const data = this.clientAssetBundle.readFileSync( + assetName, + ); + response.headers.set("Cache-Control", "no-cache"); + response.headers.set("Content-length", "" + data.length); + response.headers.set( + "Last-Modified", + utcDateString(this.clientAssetBundle.getMtime(assetName)), + ); + + if (request.method === "GET") { + response.body = data; + } + } catch { + return next(); + } + } + async reloadSettings() { // TODO: Throttle this? this.settings = await ensureSettingsAndIndex(this.spacePrimitives); @@ -185,6 +175,7 @@ export class HttpServer { "/.auth", ]; + // Middleware handling the /.auth page and flow app.use(async ({ request, response, cookies }, next) => { if (request.url.pathname === "/.auth") { if (request.url.search === "?logout") { @@ -227,6 +218,7 @@ export class HttpServer { }); if ((await this.authenticator.getAllUsers()).length > 0) { + // Users defined, so enabling auth app.use(async ({ request, response, cookies }, next) => { if (!excludedPaths.includes(request.url.pathname)) { const authCookie = await cookies.get("auth"); @@ -250,23 +242,37 @@ export class HttpServer { } } - private buildFsRouter(spacePrimitives: SpacePrimitives): Router { + private addFsRoutes(spacePrimitives: SpacePrimitives): Router { const fsRouter = new Router(); const corsMiddleware = oakCors({ allowedHeaders: "*", exposedHeaders: "*", - methods: ["GET", "POST", "PUT", "DELETE", "OPTIONS"], - }); - // File list - fsRouter.get("/", corsMiddleware, async ({ response }) => { - response.headers.set("Content-type", "application/json"); - response.headers.set("X-Space-Path", this.options.pagesPath); - const files = await spacePrimitives.fetchFileList(); - response.body = JSON.stringify(files); + methods: ["GET", "POST", "PUT", "DELETE", "HEAD", "OPTIONS"], }); + fsRouter.use(corsMiddleware); + + // File list + fsRouter.get( + "/index.json", + // corsMiddleware, + async ({ request, response }) => { + if (request.headers.get("Accept") === "application/json") { + // Only handle direct requests for a JSON representation of the file list + response.headers.set("Content-type", "application/json"); + response.headers.set("X-Space-Path", this.options.pagesPath); + const files = await spacePrimitives.fetchFileList(); + response.body = JSON.stringify(files); + } else { + // Otherwise, redirect to the UI + // The reason to do this is to handle authentication systems like Authelia nicely + response.redirect("/"); + } + }, + ); + // RPC - fsRouter.post("/", corsMiddleware, async ({ request, response }) => { + fsRouter.post("/.rpc", async ({ request, response }) => { const body = await request.body({ type: "json" }).value; try { switch (body.operation) { @@ -288,6 +294,7 @@ export class HttpServer { }); return; } + console.log("Running shell command:", body.cmd, body.args); const p = new Deno.Command(body.cmd, { args: body.args, cwd: this.options.pagesPath, @@ -304,6 +311,9 @@ export class HttpServer { stderr, code: output.code, }); + if (output.code !== 0) { + console.error("Error running shell command", stdout, stderr); + } return; } default: @@ -319,129 +329,104 @@ export class HttpServer { } }); + const filePathRegex = "\/(.+\\.[a-zA-Z]+)"; + fsRouter - .get("\/(.+)", corsMiddleware, async ({ params, response, request }) => { - const name = params[0]; - console.log("Loading file", name); - if (name.startsWith(".")) { - // Don't expose hidden files - response.status = 404; - return; - } - try { - const attachmentData = await spacePrimitives.readFile( - name, - ); - const lastModifiedHeader = new Date(attachmentData.meta.lastModified) - .toUTCString(); - if (request.headers.get("If-Modified-Since") === lastModifiedHeader) { - response.status = 304; + .get( + filePathRegex, + // corsMiddleware, + async ({ params, response, request }) => { + const name = params[0]; + console.log("Requested file", name); + if (name.startsWith(".")) { + // Don't expose hidden files + response.status = 404; + response.body = "Not exposed"; return; } - response.status = 200; - response.headers.set( - "X-Last-Modified", - "" + attachmentData.meta.lastModified, - ); - response.headers.set("Cache-Control", "no-cache"); - response.headers.set("X-Permission", attachmentData.meta.perm); - response.headers.set( - "Last-Modified", - lastModifiedHeader, - ); - response.headers.set("Content-Type", attachmentData.meta.contentType); - response.body = attachmentData.data; - } catch { - // console.error("Error in main router", e); - response.status = 404; - response.body = ""; - } - }) - .put("\/(.+)", corsMiddleware, async ({ request, response, params }) => { + try { + const fileData = await spacePrimitives.readFile( + name, + ); + const lastModifiedHeader = new Date(fileData.meta.lastModified) + .toUTCString(); + if ( + request.headers.get("If-Modified-Since") === lastModifiedHeader + ) { + response.status = 304; + return; + } + response.status = 200; + this.fileMetaToHeaders(response.headers, fileData.meta); + response.headers.set("Last-Modified", lastModifiedHeader); + + response.body = fileData.data; + } catch { + // console.error("Error GETting of file", name, e); + response.status = 404; + response.body = "Not found"; + } + }, + ) + .put( + filePathRegex, + // corsMiddleware, + async ({ request, response, params }) => { + const name = params[0]; + console.log("Saving file", name); + if (name.startsWith(".")) { + // Don't expose hidden files + response.status = 403; + return; + } + + const body = await request.body({ type: "bytes" }).value; + + try { + const meta = await spacePrimitives.writeFile( + name, + body, + ); + response.status = 200; + this.fileMetaToHeaders(response.headers, meta); + response.body = "OK"; + } catch (err) { + console.error("Write failed", err); + response.status = 500; + response.body = "Write failed"; + } + }, + ) + .delete(filePathRegex, async ({ response, params }) => { const name = params[0]; - console.log("Saving file", name); + console.log("Deleting file", name); if (name.startsWith(".")) { // Don't expose hidden files response.status = 403; return; } - - let body: Uint8Array; - if ( - request.headers.get("X-Content-Base64") - ) { - const content = await request.body({ type: "text" }).value; - body = base64Decode(content); - } else { - body = await request.body({ type: "bytes" }).value; - } - - try { - const meta = await spacePrimitives.writeFile( - name, - body, - ); - response.status = 200; - response.headers.set("Content-Type", meta.contentType); - response.headers.set("X-Last-Modified", "" + meta.lastModified); - response.headers.set("X-Content-Length", "" + meta.size); - response.headers.set("X-Permission", meta.perm); - response.body = "OK"; - } catch (err) { - response.status = 500; - response.body = "Write failed"; - console.error("Pipeline failed", err); - } - }) - .options("\/(.+)", async ({ request, response, params }) => { - const name = params[0]; - // Manually set CORS headers - response.headers.set("access-control-allow-headers", "*"); - response.headers.set( - "access-control-allow-methods", - "GET,POST,PUT,DELETE,OPTIONS", - ); - response.headers.set("access-control-allow-origin", "*"); - response.headers.set("access-control-expose-headers", "*"); - try { - const meta = await spacePrimitives.getFileMeta(name); - response.status = 200; - response.headers.set("Content-Type", meta.contentType); - response.headers.set("X-Last-Modified", "" + meta.lastModified); - response.headers.set("X-Content-Length", "" + meta.size); - response.headers.set("X-Permission", meta.perm); - - const clientId = request.headers.get("X-Client-Id"); - if (name.endsWith(".md") && clientId) { - const pageName = name.substring(0, name.length - ".md".length); - console.log(`Got presence update from ${clientId}: ${pageName}`); - const { collabId } = this.collab.updatePresence(clientId, pageName); - if (collabId) { - response.headers.set("X-Collab-Id", collabId); - } - } - } catch { - // Have to do this because of CORS - response.status = 200; - response.headers.set("X-Status", "404"); - response.body = "Not found"; - // console.error("Options failed", err); - } - }) - .delete("\/(.+)", corsMiddleware, async ({ response, params }) => { - const name = params[0]; - console.log("Deleting file", name); try { await spacePrimitives.deleteFile(name); response.status = 200; response.body = "OK"; } catch (e: any) { console.error("Error deleting attachment", e); - response.status = 200; + response.status = 500; response.body = e.message; } - }); - return new Router().use("/.fs", fsRouter.routes()); + }) + .options(filePathRegex, corsMiddleware); + return fsRouter; + } + + private fileMetaToHeaders(headers: Headers, fileMeta: FileMeta) { + headers.set("Content-Type", fileMeta.contentType); + headers.set( + "X-Last-Modified", + "" + fileMeta.lastModified, + ); + headers.set("Cache-Control", "no-cache"); + headers.set("X-Permission", fileMeta.perm); } stop() { diff --git a/web/cm_plugins/collab.ts b/web/cm_plugins/collab.ts deleted file mode 100644 index d2352ed0..00000000 --- a/web/cm_plugins/collab.ts +++ /dev/null @@ -1,99 +0,0 @@ -import { safeRun } from "../../common/util.ts"; -import { Extension, HocuspocusProvider, Y, yCollab } from "../deps.ts"; -import { SyncService } from "../sync_service.ts"; - -const userColors = [ - { color: "#30bced", light: "#30bced33" }, - { color: "#6eeb83", light: "#6eeb8333" }, - { color: "#ffbc42", light: "#ffbc4233" }, - { color: "#ecd444", light: "#ecd44433" }, - { color: "#ee6352", light: "#ee635233" }, - { color: "#9ac2c9", light: "#9ac2c933" }, - { color: "#8acb88", light: "#8acb8833" }, - { color: "#1be7ff", light: "#1be7ff33" }, -]; - -export class CollabState { - public ytext: Y.Text; - collabProvider: HocuspocusProvider; - private yundoManager: Y.UndoManager; - interval?: number; - - constructor( - serverUrl: string, - readonly path: string, - readonly token: string, - username: string, - private syncService: SyncService, - public isLocalCollab: boolean, - ) { - this.collabProvider = new HocuspocusProvider({ - url: serverUrl, - name: token, - - // Receive broadcasted messages from the server (right now only "page has been persisted" notifications) - onStateless: ( - { payload }, - ) => { - const message = JSON.parse(payload); - switch (message.type) { - case "persisted": { - // Received remote persist notification, updating snapshot - syncService.updateRemoteLastModified( - message.path, - message.lastModified, - ).catch(console.error); - } - } - }, - }); - - this.collabProvider.on("status", (e: any) => { - console.log("Collab status change", e); - }); - - this.ytext = this.collabProvider.document.getText("codemirror"); - this.yundoManager = new Y.UndoManager(this.ytext); - - const randomColor = - userColors[Math.floor(Math.random() * userColors.length)]; - - this.collabProvider.awareness.setLocalStateField("user", { - name: username, - color: randomColor.color, - colorLight: randomColor.light, - }); - if (isLocalCollab) { - syncService.excludeFromSync(path).catch(console.error); - - this.interval = setInterval(() => { - // Ping the store to make sure the file remains in exclusion - syncService.excludeFromSync(path).catch(console.error); - }, 1000); - } - } - - stop() { - console.log("[COLLAB] Destroying collab provider"); - if (this.interval) { - clearInterval(this.interval); - } - this.collabProvider.destroy(); - // For whatever reason, destroy() doesn't properly clean up everything so we need to help a bit - this.collabProvider.configuration.websocketProvider.webSocket = null; - this.collabProvider.configuration.websocketProvider.destroy(); - - // When stopping collaboration, we're going back to sync mode. Make sure we got the latest and greatest remote timestamp to avoid - // conflicts - safeRun(async () => { - await this.syncService.unExcludeFromSync(this.path); - await this.syncService.fetchAndPersistRemoteLastModified(this.path); - }); - } - - collabExtension(): Extension { - return yCollab(this.ytext, this.collabProvider.awareness, { - undoManager: this.yundoManager, - }); - } -} diff --git a/web/cm_plugins/fenced_code.ts b/web/cm_plugins/fenced_code.ts index 04faba95..f33ff3e8 100644 --- a/web/cm_plugins/fenced_code.ts +++ b/web/cm_plugins/fenced_code.ts @@ -21,7 +21,6 @@ class IFrameWidget extends WidgetType { } toDOM(): HTMLElement { - console.log("toDOM"); const iframe = document.createElement("iframe"); iframe.srcdoc = panelHtml; // iframe.style.height = "0"; diff --git a/web/cm_plugins/inline_image.ts b/web/cm_plugins/inline_image.ts index 8a26a0ca..37b08e61 100644 --- a/web/cm_plugins/inline_image.ts +++ b/web/cm_plugins/inline_image.ts @@ -74,7 +74,7 @@ export function inlineImagesPlugin(editor: Editor) { let url = imageRexexResult.groups.url; const title = imageRexexResult.groups.title; if (url.indexOf("://") === -1) { - url = `/.fs/${url}`; + url = decodeURI(url); } widgets.push( Decoration.widget({ diff --git a/web/cm_plugins/table.ts b/web/cm_plugins/table.ts index 004990be..1778cadd 100644 --- a/web/cm_plugins/table.ts +++ b/web/cm_plugins/table.ts @@ -39,7 +39,7 @@ class TableViewWidget extends WidgetType { annotationPositions: true, translateUrls: (url) => { if (!url.includes("://")) { - return `/.fs/${url}`; + return `/${url}`; } return url; }, diff --git a/web/collab_manager.ts b/web/collab_manager.ts deleted file mode 100644 index cbc6b26d..00000000 --- a/web/collab_manager.ts +++ /dev/null @@ -1,83 +0,0 @@ -import { nanoid } from "https://esm.sh/nanoid@4.0.0"; -import type { Editor } from "./editor.tsx"; - -const collabPingInterval = 2500; - -export class CollabManager { - clientId = nanoid(); - localCollabServer: string; - - constructor(private editor: Editor) { - this.localCollabServer = location.protocol === "http:" - ? `ws://${location.host}/.ws-collab` - : `wss://${location.host}/.ws-collab`; - editor.eventHook.addLocalListener( - "editor:pageLoaded", - (pageName, previousPage) => { - console.log("Page loaded", pageName, previousPage); - this.updatePresence(pageName).catch(console.error); - }, - ); - } - - start() { - setInterval(() => { - this.updatePresence(this.editor.currentPage!).catch(console.error); - }, collabPingInterval); - } - - async updatePresence(currentPage: string) { - try { - // This is signaled through an OPTIONS call on the file we have open - const resp = await this.editor.remoteSpacePrimitives.authenticatedFetch( - `${this.editor.remoteSpacePrimitives.url}/${currentPage}.md`, - { - method: "OPTIONS", - headers: { - "X-Client-Id": this.clientId, - }, - }, - ); - const collabId = resp.headers.get("X-Collab-Id"); - // Not reading body at all, is that a problem? - - if (this.editor.collabState && !this.editor.collabState.isLocalCollab) { - // We're in a remote collab mode, don't do anything - return; - } - - // console.log("Collab ID", collabId); - const previousCollabId = this.editor.collabState?.token.split("/")[0]; - if (!collabId && this.editor.collabState) { - // Stop collab - console.log("Stopping collab"); - if (this.editor.collabState.path === `${currentPage}.md`) { - this.editor.flashNotification( - "Other users have left this page, switched back to single-user mode.", - ); - } - this.editor.stopCollab(); - } else if (collabId && collabId !== previousCollabId) { - // Start collab - console.log("Starting collab"); - this.editor.flashNotification( - "Opening page in multi-user mode.", - ); - this.editor.startCollab( - this.localCollabServer, - `${collabId}/${currentPage}.md`, - this.editor.getUsername(), - true, - ); - } - } catch (e: any) { - // console.error("Ping error", e); - if ( - e.message.toLowerCase().includes("failed") && this.editor.collabState - ) { - console.log("Offline, stopping collab"); - this.editor.stopCollab(); - } - } - } -} diff --git a/web/deps.ts b/web/deps.ts index 5eb1c335..179ff312 100644 --- a/web/deps.ts +++ b/web/deps.ts @@ -15,14 +15,6 @@ export { Terminal as TerminalIcon, } from "https://esm.sh/preact-feather@4.2.1?external=preact"; -// Y collab -export * as Y from "yjs"; -export { - yCollab, - yUndoManagerKeymap, -} from "https://esm.sh/y-codemirror.next@0.3.2?external=yjs,@codemirror/state,@codemirror/commands,@codemirror/history,@codemirror/view"; -export { HocuspocusProvider } from "https://esm.sh/@hocuspocus/provider@2.2.0?deps=lib0@0.2.70&external=yjs,ws&target=es2022"; - // Vim mode export { getCM as vimGetCm, diff --git a/web/editor.tsx b/web/editor.tsx index 0d97e507..dd4de055 100644 --- a/web/editor.tsx +++ b/web/editor.tsx @@ -68,7 +68,6 @@ import assetSyscalls from "../plugos/syscalls/asset.ts"; import { eventSyscalls } from "../plugos/syscalls/event.ts"; import { System } from "../plugos/system.ts"; import { cleanModePlugins } from "./cm_plugins/clean.ts"; -import { CollabState } from "./cm_plugins/collab.ts"; import { attachmentExtension, pasteLinkExtension, @@ -91,14 +90,12 @@ import { useEffect, useReducer, vim, - yUndoManagerKeymap, } from "./deps.ts"; import { AppCommand, CommandHook } from "./hooks/command.ts"; import { SlashCommandHook } from "./hooks/slash_command.ts"; import { PathPageNavigator } from "./navigator.ts"; import reducer from "./reducer.ts"; import customMarkdownStyle from "./style.ts"; -import { collabSyscalls } from "./syscalls/collab.ts"; import { editorSyscalls } from "./syscalls/editor.ts"; import { spaceSyscalls } from "./syscalls/space.ts"; import { systemSyscalls } from "./syscalls/system.ts"; @@ -137,7 +134,8 @@ import { HttpSpacePrimitives } from "../common/spaces/http_space_primitives.ts"; import { FallbackSpacePrimitives } from "../common/spaces/fallback_space_primitives.ts"; import { syncSyscalls } from "./syscalls/sync.ts"; import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts"; -import { CollabManager } from "./collab_manager.ts"; +import { run } from "../plug-api/plugos-syscall/shell.ts"; +import { isValidPageName } from "$sb/lib/page.ts"; const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/; @@ -155,7 +153,6 @@ declare global { // Injected via index.html silverBulletConfig: { spaceFolderPath: string; - syncEndpoint: string; }; editor: Editor; } @@ -190,11 +187,9 @@ export class Editor { fullSyncCompleted = false; // Runtime state (that doesn't make sense in viewState) - collabState?: CollabState; syncService: SyncService; settings?: BuiltinSettings; kvStore: DexieKVStore; - collabManager: CollabManager; constructor( parent: Element, @@ -216,8 +211,6 @@ export class Editor { this.eventHook = new EventHook(); system.addHook(this.eventHook); - this.collabManager = new CollabManager(this); - // Cron hook const cronHook = new CronHook(system); system.addHook(cronHook); @@ -237,7 +230,7 @@ export class Editor { // Setup space this.remoteSpacePrimitives = new HttpSpacePrimitives( - runtimeConfig.syncEndpoint, + location.origin, runtimeConfig.spaceFolderPath, true, ); @@ -328,7 +321,6 @@ export class Editor { systemSyscalls(this, this.system), markdownSyscalls(buildMarkdown(this.mdExtensions)), assetSyscalls(this.system), - collabSyscalls(this), yamlSyscalls(), storeCalls, indexSyscalls, @@ -376,17 +368,12 @@ export class Editor { } }); - // globalThis.addEventListener("beforeunload", (e) => { - // console.log("Pinging with with undefined page name"); - // this.collabManager.updatePresence(undefined, this.currentPage); - // }); - this.eventHook.addLocalListener("plug:changed", async (fileName) => { console.log("Plug updated, reloading:", fileName); system.unload(fileName); await system.load( // await this.space.readFile(fileName, "utf8"), - new URL(`/.fs/${fileName}`, location.href), + new URL(`/${fileName}`, location.href), createSandbox, ); this.plugsUpdated = true; @@ -402,7 +389,7 @@ export class Editor { this.space.on({ pageChanged: (meta) => { - // Only reload when watching the current page (to avoid reloading when switching pages and in collab mode) + // Only reload when watching the current page (to avoid reloading when switching pages) if (this.space.watchInterval && this.currentPage === meta.name) { console.log("Page changed elsewhere, reloading"); this.flashNotification("Page changed elsewhere, reloading"); @@ -485,7 +472,6 @@ export class Editor { // Kick off background sync this.syncService.start(); - this.collabManager.start(); this.eventHook.addLocalListener("sync:success", async (operations) => { // console.log("Operations", operations); @@ -738,7 +724,7 @@ export class Editor { let touchCount = 0; return EditorState.create({ - doc: this.collabState ? this.collabState.ytext.toString() : text, + doc: text, extensions: [ // Not using CM theming right now, but some extensions depend on the "dark" thing EditorView.theme({}, { dark: this.viewState.uiOptions.darkMode }), @@ -934,7 +920,6 @@ export class Editor { ...searchKeymap, ...historyKeymap, ...completionKeymap, - ...(this.collabState ? yUndoManagerKeymap : []), indentWithTab, ...commandKeyBindings, { @@ -1057,7 +1042,6 @@ export class Editor { pasteLinkExtension, attachmentExtension(this), closeBrackets(), - ...[this.collabState ? this.collabState.collabExtension() : []], ], }); } @@ -1070,7 +1054,7 @@ export class Editor { await Promise.all((await this.space.listPlugs()).map(async (plugName) => { try { await this.system.load( - new URL(`/.fs/${plugName}`, location.href), + new URL(plugName, location.origin), createSandbox, ); } catch (e: any) { @@ -1181,6 +1165,13 @@ export class Editor { name = this.settings!.indexPage; } + if (!isValidPageName(name)) { + return this.flashNotification( + "Invalid page name: page names cannot end with a file extension", + "error", + ); + } + if (newWindow) { const win = window.open(`${location.origin}/${name}`, "_blank"); if (win) { @@ -1206,10 +1197,6 @@ export class Editor { this.space.unwatchPage(previousPage); if (previousPage !== pageName) { await this.save(true); - // And stop the collab session - if (this.collabState) { - this.stopCollab(); - } } } @@ -1222,6 +1209,9 @@ export class Editor { let doc; try { doc = await this.space.readPage(pageName); + if (doc.meta.contentType.startsWith("text/html")) { + throw new Error("Got HTML page, not markdown"); + } } catch (e: any) { // Not found, new page console.log("Creating new page", pageName); @@ -1578,50 +1568,4 @@ export class Editor { } return; } - - startCollab( - serverUrl: string, - token: string, - username: string, - isLocalCollab = false, - ) { - if (this.collabState) { - // Clean up old collab state - this.collabState.stop(); - } - const initialText = this.editorView!.state.sliceDoc(); - this.collabState = new CollabState( - serverUrl, - `${this.currentPage!}.md`, - token, - username, - this.syncService, - isLocalCollab, - ); - - this.collabState.collabProvider.on("synced", () => { - if (this.collabState!.ytext.toString() === "") { - console.log( - "[Collab]", - "Synced value is empty (new collab session), inserting local copy", - ); - this.collabState!.ytext.insert(0, initialText); - } - }); - - this.rebuildEditorState(); - - // Don't watch for local changes in this mode - this.space.unwatch(); - } - - stopCollab() { - if (this.collabState) { - this.collabState.stop(); - this.collabState = undefined; - this.rebuildEditorState(); - } - // Start file watching again - this.space.watch(); - } } diff --git a/web/index.html b/web/index.html index 3d0f04ad..84b6fc4e 100644 --- a/web/index.html +++ b/web/index.html @@ -35,13 +35,11 @@ window.silverBulletConfig = { // These {{VARIABLES}} are replaced by http_server.ts spaceFolderPath: "{{SPACE_PATH}}", - syncEndpoint: "{{SYNC_ENDPOINT}}", }; // But in case these variables aren't replaced by the server, fall back fully static mode (no sync) if (window.silverBulletConfig.spaceFolderPath.includes("{{")) { window.silverBulletConfig = { spaceFolderPath: "", - syncEndpoint: "/.fs" }; } diff --git a/web/service_worker.ts b/web/service_worker.ts index 7fd36312..368da214 100644 --- a/web/service_worker.ts +++ b/web/service_worker.ts @@ -24,42 +24,40 @@ const precacheFiles = Object.fromEntries([ self.addEventListener("install", (event: any) => { console.log("[Service worker]", "Installing service worker..."); event.waitUntil( - caches.open(CACHE_NAME) - .then((cache) => { - console.log( - "[Service worker]", - "Now pre-caching client files", - ); - return cache.addAll(Object.values(precacheFiles)).then(() => { - console.log( - "[Service worker]", - Object.keys(precacheFiles).length, - "client files cached", - ); - // @ts-ignore: No need to wait - self.skipWaiting(); - }); - }), + (async () => { + const cache = await caches.open(CACHE_NAME); + console.log( + "[Service worker]", + "Now pre-caching client files", + ); + await cache.addAll(Object.values(precacheFiles)); + console.log( + "[Service worker]", + Object.keys(precacheFiles).length, + "client files cached", + ); + // @ts-ignore: No need to wait + self.skipWaiting(); + })(), ); }); self.addEventListener("activate", (event: any) => { console.log("[Service worker]", "Activating new service worker!!!"); event.waitUntil( - caches.keys().then((cacheNames) => { - return Promise.all( + (async () => { + const cacheNames = await caches.keys(); + await Promise.all( cacheNames.map((cacheName) => { if (cacheName !== CACHE_NAME) { console.log("[Service worker]", "Removing old cache", cacheName); return caches.delete(cacheName); } }), - ).then(() => { - // Let's activate ourselves for all existing clients - // @ts-ignore: No need to wait, clients is a serviceworker thing - return clients.claim(); - }); - }), + ); + // @ts-ignore: No need to wait + return clients.claim(); + })(), ); }); @@ -75,6 +73,15 @@ self.addEventListener("fetch", (event: any) => { event.respondWith( (async () => { + const request = event.request; + + // console.log("Getting request", request, [...request.headers.entries()]); + + // Any request with the X-Sync-Mode header originates from the sync engine: pass it on to the server + if (request.headers.has("x-sync-mode")) { + return fetch(request); + } + // Try the static (client) file cache first const cachedResponse = await caches.match(cacheKey); // Return the cached response if found @@ -82,67 +89,65 @@ self.addEventListener("fetch", (event: any) => { return cachedResponse; } - const requestUrl = new URL(event.request.url); - + const requestUrl = new URL(request.url); const pathname = requestUrl.pathname; - // console.log("In service worker, pathname is", pathname); - // Are we fetching a URL from the same origin as the app? If not, we don't handle it here - const fetchingLocal = location.host === requestUrl.host; - if (!fetchingLocal) { - return fetch(event.request); + // Are we fetching a URL from the same origin as the app? If not, we don't handle it and pass it on + if (location.host !== requestUrl.host) { + return fetch(request); } - // If this is a /.fs request, this can either be a plug worker load or an attachment load - if (pathname.startsWith("/.fs")) { - if (!fileContentTable || event.request.headers.has("x-sync-mode")) { - // Not initialzed yet, or explicitly in sync mode (so direct server communication requested) - return fetch(event.request); - } - // console.log( - // "Attempting to serve file from locally synced space:", - // pathname, - // ); - const path = decodeURIComponent( - requestUrl.pathname.slice("/.fs/".length), - ); - const data = await fileContentTable.get(path); - if (data) { - // console.log("Serving from space", path); - if (!data.meta) { - // Legacy database not fully synced yet - data.meta = (await fileMetatable!.get(path))!; - } - return new Response( - data.data, - { - headers: { - "Content-type": data.meta.contentType, - "Content-Length": "" + data.meta.size, - "X-Permission": data.meta.perm, - "X-Last-Modified": "" + data.meta.lastModified, - }, - }, - ); - } else { - console.error( - "Did not find file in locally synced space", - path, - ); - return new Response("Not found", { - status: 404, - }); - } + // If this is a /*.* request, this can either be a plug worker load or an attachment load + if (/\/.+\.[a-zA-Z]+$/.test(pathname)) { + return handleLocalFileRequest(request, pathname); } else if (pathname === "/.auth") { - return fetch(event.request); + return fetch(request); } else { // Must be a page URL, let's serve index.html which will handle it - return (await caches.match(precacheFiles["/"])) || fetch(event.request); + return (await caches.match(precacheFiles["/"])) || fetch(request); } })(), ); }); +async function handleLocalFileRequest( + request: Request, + pathname: string, +): Promise { + if (!fileContentTable) { + // Not initialzed yet, or explicitly in sync mode (so direct server communication requested) + return fetch(request); + } + const path = decodeURIComponent(pathname.slice(1)); + const data = await fileContentTable.get(path); + if (data) { + // console.log("Serving from space", path); + if (!data.meta) { + // Legacy database not fully synced yet + data.meta = (await fileMetatable!.get(path))!; + } + return new Response( + data.data, + { + headers: { + "Content-type": data.meta.contentType, + "Content-Length": "" + data.meta.size, + "X-Permission": data.meta.perm, + "X-Last-Modified": "" + data.meta.lastModified, + }, + }, + ); + } else { + console.error( + "Did not find file in locally synced space", + path, + ); + return new Response("Not found", { + status: 404, + }); + } +} + self.addEventListener("message", (event: any) => { if (event.data.type === "flushCache") { caches.delete(CACHE_NAME) diff --git a/web/sync_service.ts b/web/sync_service.ts index 2b82fcb9..3820e424 100644 --- a/web/sync_service.ts +++ b/web/sync_service.ts @@ -17,7 +17,7 @@ const syncStartTimeKey = "syncStartTime"; // Keeps the last time an activity was registered, used to detect if a sync is still alive and whether a new one should be started already const syncLastActivityKey = "syncLastActivity"; -const syncExcludePrefix = "syncExclude:"; +const syncInitialFullSyncCompletedKey = "syncInitialFullSyncCompleted"; // maximum time between two activities before we consider a sync crashed const syncMaxIdleTimeout = 1000 * 20; // 20s @@ -56,17 +56,9 @@ export class SyncService { await this.syncFile(`${name}.md`); }); - eventHook.addLocalListener("editor:pageSaved", async (name, meta) => { + eventHook.addLocalListener("editor:pageSaved", async (name) => { const path = `${name}.md`; await this.syncFile(path); - if (await this.isExcludedFromSync(path)) { - // So we're editing a page and just saved it, but it's excluded from sync - // Assumption: we're in collab mode for this file, so we're going to constantly update our local hash - // console.log( - // "Locally updating last modified in snapshot because we're in collab mode", - // ); - await this.updateLocalLastModified(path, meta.lastModified); - } }); } @@ -86,10 +78,9 @@ export class SyncService { return true; } - async hasInitialSyncCompleted(): Promise { + hasInitialSyncCompleted(): Promise { // Initial sync has happened when sync progress has been reported at least once, but the syncStartTime has been reset (which happens after sync finishes) - return !(await this.kvStore.has(syncStartTimeKey)) && - (await this.kvStore.has(syncLastActivityKey)); + return this.kvStore.has(syncInitialFullSyncCompletedKey); } async registerSyncStart(): Promise { @@ -116,40 +107,7 @@ export class SyncService { async registerSyncStop(): Promise { await this.registerSyncProgress(); await this.kvStore.del(syncStartTimeKey); - } - - // Temporarily exclude a specific file from sync (e.g. when in collab mode) - excludeFromSync(path: string): Promise { - return this.kvStore.set(syncExcludePrefix + path, Date.now()); - } - - unExcludeFromSync(path: string): Promise { - return this.kvStore.del(syncExcludePrefix + path); - } - - async isExcludedFromSync(path: string): Promise { - const lastExcluded = await this.kvStore.get(syncExcludePrefix + path); - return lastExcluded && Date.now() - lastExcluded < syncMaxIdleTimeout; - } - - async fetchAllExcludedFromSync(): Promise { - const entries = await this.kvStore.queryPrefix(syncExcludePrefix); - const expiredPaths: string[] = []; - const now = Date.now(); - const result = entries.filter(({ key, value }) => { - if (now - value > syncMaxIdleTimeout) { - expiredPaths.push(key); - return false; - } - return true; - }).map(({ key }) => key.slice(syncExcludePrefix.length)); - - if (expiredPaths.length > 0) { - console.log("Purging expired sync exclusions: ", expiredPaths); - await this.kvStore.batchDelete(expiredPaths); - } - - return result; + await this.kvStore.set(syncInitialFullSyncCompletedKey, true); } async getSnapshot(): Promise> { @@ -167,83 +125,6 @@ export class SyncService { } } - // When in collab mode, we delegate the sync to the CDRT engine, to avoid conflicts, we try to keep the lastModified time in sync with the remote - async updateRemoteLastModified(path: string, lastModified: number) { - await this.noOngoingSync(); - await this.registerSyncStart(); - const snapshot = await this.getSnapshot(); - const entry = snapshot.get(path); - if (entry) { - snapshot.set(path, [entry[0], lastModified]); - } else { - // In the unlikely scenario that a space first openen on a collab page before every being synced - try { - console.log( - "Received lastModified time for file not in snapshot", - path, - lastModified, - ); - snapshot.set(path, [ - (await this.localSpacePrimitives.getFileMeta(path)).lastModified, - lastModified, - ]); - } catch (e) { - console.warn( - "Received lastModified time for non-existing file not in snapshot", - path, - lastModified, - e, - ); - } - } - await this.saveSnapshot(snapshot); - await this.registerSyncStop(); - } - - // Reach out out to remote space, fetch the latest lastModified time and update the local snapshot - // This is used when exiting collab mode - async fetchAndPersistRemoteLastModified(path: string) { - const meta = await this.remoteSpace.getFileMeta(path); - await this.updateRemoteLastModified( - path, - meta.lastModified, - ); - } - - // When in collab mode, we delegate the sync to the CDRT engine, to avoid conflicts, we try to keep the lastModified time in sync when local changes happen - async updateLocalLastModified(path: string, lastModified: number) { - await this.noOngoingSync(); - await this.registerSyncStart(); - const snapshot = await this.getSnapshot(); - const entry = snapshot.get(path); - if (entry) { - snapshot.set(path, [lastModified, entry[1]]); - } else { - // In the unlikely scenario that a space first openen on a collab page before every being synced - try { - console.log( - "Setting lastModified time for file not in snapshot", - path, - lastModified, - ); - snapshot.set(path, [ - lastModified, - (await this.localSpacePrimitives.getFileMeta(path)).lastModified, - ]); - } catch (e) { - console.warn( - "Received lastModified time for non-existing file not in snapshot", - path, - lastModified, - e, - ); - } - } - await this.saveSnapshot(snapshot); - await this.registerSyncStop(); - // console.log("All done!"); - } - start() { this.syncSpace().catch( console.error, @@ -271,14 +152,11 @@ export class SyncService { await this.registerSyncStart(); let operations = 0; const snapshot = await this.getSnapshot(); - // Fetch the list of files that are excluded from sync (e.g. because they're in collab mode) - const excludedFromSync = await this.fetchAllExcludedFromSync(); // console.log("Excluded from sync", excludedFromSync); try { operations = await this.spaceSync!.syncFiles( snapshot, - (path) => - this.isSyncCandidate(path) && !excludedFromSync.includes(path), + (path) => this.isSyncCandidate(path), ); this.eventHook.dispatchEvent("sync:success", operations); } catch (e: any) { @@ -295,7 +173,7 @@ export class SyncService { // console.log("Already syncing"); return; } - if (!this.isSyncCandidate(name) || (await this.isExcludedFromSync(name))) { + if (!this.isSyncCandidate(name)) { return; } await this.registerSyncStart(); diff --git a/web/syscalls/collab.ts b/web/syscalls/collab.ts deleted file mode 100644 index b2592ecd..00000000 --- a/web/syscalls/collab.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { SysCallMapping } from "../../plugos/system.ts"; -import type { Editor } from "../editor.tsx"; - -export function collabSyscalls(editor: Editor): SysCallMapping { - return { - "collab.start": ( - _ctx, - serverUrl: string, - token: string, - username: string, - ) => { - editor.startCollab(serverUrl, token, username); - }, - "collab.stop": ( - _ctx, - ) => { - editor.stopCollab(); - }, - }; -} diff --git a/web/syscalls/fetch.ts b/web/syscalls/fetch.ts index ce0321be..2e690c34 100644 --- a/web/syscalls/fetch.ts +++ b/web/syscalls/fetch.ts @@ -21,7 +21,7 @@ export function sandboxFetchSyscalls( return performLocalFetch(url, options); } const resp = httpSpacePrimitives.authenticatedFetch( - httpSpacePrimitives.url, + `${httpSpacePrimitives.url}/.rpc`, { method: "POST", body: JSON.stringify({ diff --git a/web/syscalls/shell.ts b/web/syscalls/shell.ts index 9ffb8094..a7270c49 100644 --- a/web/syscalls/shell.ts +++ b/web/syscalls/shell.ts @@ -14,7 +14,7 @@ export function shellSyscalls( throw new Error("Not supported in fully local mode"); } const resp = httpSpacePrimitives.authenticatedFetch( - httpSpacePrimitives.url, + `${httpSpacePrimitives.url}/.rpc`, { method: "POST", body: JSON.stringify({ diff --git a/website/API.md b/website/API.md new file mode 100644 index 00000000..e49217a9 --- /dev/null +++ b/website/API.md @@ -0,0 +1,12 @@ +The server API is relatively small. The client primarily communicates with the server for file “CRUD” (Create, Read, Update, Delete) style operations. + +Here’s an attempt to document this API: + +* `GET /index.json` (when sent with an `Accept: application/json` request header): will return a full listing of all files in your space including meta data like when the file was last modified, as well as permissions. This is primarily for sync purposes with the client. A request sent without the mentioned `Accept` header will redirect to `/` (to better support authentication layers like [Authelia](https://www.authelia.com/)). +* `GET /*.*`: _Reads_ and returns the content of the file at the given path. This means that if you `GET /index.md` you will receive the content of your `index` page. The `GET` response will have a few additional SB-specific headers: + * `X-Last-Modified` as a UNIX timestamp in ms (as coming from `Data.now()`) + * `X-Permission`: either `rw` or `ro` which will change whether the editor opens in read-only or regular mode. +* `PUT /*.*`: The same as `GET` except that it takes the body of the request and _writes_ it a file. +* `DELETE /*.*`: Again the same, except this will _delete_ the given file. +* `GET /.client/*`: Retrieve files implementing the client +* `GET /*` and `GET /`: Anything else (any path without a file extension) will serve the SilverBullet UI HTML. diff --git a/website/CHANGELOG.md b/website/CHANGELOG.md index ee49cc21..f6661d4f 100644 --- a/website/CHANGELOG.md +++ b/website/CHANGELOG.md @@ -3,6 +3,12 @@ release. --- +## Next +* **Removal of all real-time collaboration features**: this was causing too many edge cases, and complicated the code too much. To simplify the product as well as the code, we completely removed all real-time collaboration features for now. We may introduce this at some point in the future when the demand and focus is there. +* **Change of APIs**: This is mostly internal, but will likely have effects on the first load after the upgrade: you may see errors or a message around “the path has changed”, or your page may not properly load. Don’t freak out, just reload once or twice and all should resync and be fine. There’s a beginning of documenting the server [[API]] now. + +--- + ## 0.3.4 * **Breaking change (for some templates):** Template in various places allowed you to use `{{variables}}` and various handlebars functions. There also used to be a magic `{{page}}` variable that you could use in various places, but not everywhere. This has now been unified. And the magical `{{page}}` now has been replaced with the global `@page` which does not just expose the page’s name, but any page meta data. More information here: [[🔌 Core/Templates@vars]]. You will now get completion for built-in handlebars helpers after typing `{{`. @@ -18,8 +24,7 @@ release. ## 0.3.2 -* **Real-time collaboration support** between clients: Open the same page in multiple windows (browser tabs, mobile devices) and within a few seconds you should get kicked into real-time collaboration mode, showing other participants cursors, selections and edits in real time (Google doc style). This only works when a connection with the server can be established. - * This **breaks** existing [[🔌 Collab]] links, since we switched real-time collaboration libraries. We’re still looking at the best way to keep supporting this feature. +* REMOVED: **Real-time collaboration support** between clients: Open the same page in multiple windows (browser tabs, mobile devices) and within a few seconds you should get kicked into real-time collaboration mode, showing other participants cursors, selections and edits in real time (Google doc style). This only works when a connection with the server can be established. * [[Authentication|Multi-user authentication]]: you can now allow multiple user accounts authenticate, which makes the real-time collaboration support actually useful. This feature is still experimental and will likely evolve over time. * Added `spaceIgnore` setting to not sync specific folders or file patterns to the client, see [[SETTINGS]] for documentation * Much improved image loading behavior on page (previously scroll bars would jump up and down like a mad person) @@ -209,7 +214,7 @@ Besides these architectural changes, a few other breaking changes were made to s * New `Plugs: Add` command to quickly add a new plug (will create a `PLUGS` page if you don't have one yet). * **Paste without formatting**: holding `Shift` while pasting will disable "rich text paste." * **New core plug:** [[🔌 Share]] for sharing your pages with the outside work (such as collab, see below). -* **New plug:** [[🔌 Collab]] for real-time collaboration on your pages. +* **New plug:** 🔌 Collab for real-time collaboration on your pages. --- diff --git a/website/SilverBullet.md b/website/SilverBullet.md index 09cc69d0..bcde4b1c 100644 --- a/website/SilverBullet.md +++ b/website/SilverBullet.md @@ -15,7 +15,6 @@ Now that we got that out of the way, let’s have a look at some of SilverBullet * Run commands via their keyboard shortcuts, or the **command palette** (triggered with `Cmd-/` or `Ctrl-/` on Linux and Windows). * Use [[🔌 Core/Slash Commands|slash commands]] to perform common text editing operations. * Provides a platform for [end-user programming](https://www.inkandswitch.com/end-user-programming/) through its support for annotating pages with [[Frontmatter]] and [[🔌 Directive|directives]] (such as [[🔌 Directive/Query|#query]]), making parts of pages _dynamic_. -* Experimental [[🔌 Collab|real-time collaboration support]]. * Robust extension mechanism using [[🔌 Plugs]]. * **Self-hosted**: you own your data. All content is stored as plain files in a folder on disk. Back up, sync, edit, publish, script with any additional tools you like. * SilverBullet is [open source, MIT licensed](https://github.com/silverbulletmd/silverbullet) software. diff --git a/website/_headers b/website/_headers index 2a052f6c..882a7200 100644 --- a/website/_headers +++ b/website/_headers @@ -1,14 +1,7 @@ -/.fs/_plug/* - X-Last-Modified: 12345 - X-Permission: ro - access-control-allow-headers: * - access-control-allow-methods: GET,POST,PUT,DELETE,OPTIONS - access-control-allow-origin: * - access-control-expose-headers: * -/.fs/* +/* X-Last-Modified: 12345 X-Permission: rw access-control-allow-headers: * - access-control-allow-methods: GET,POST,PUT,DELETE,OPTIONS + access-control-allow-methods: GET,POST,PUT,DELETE,OPTIONS,HEAD access-control-allow-origin: * access-control-expose-headers: * \ No newline at end of file diff --git a/website/_redirects b/website/_redirects index 572ea646..f39e12c4 100644 --- a/website/_redirects +++ b/website/_redirects @@ -1,5 +1,3 @@ -/.fs /index.json 200 -/.fs/* /_fs/:splat 200! -/.fs/* /empty.md 200 +# /.fs /index.json 200 /.client/* /_client/:splat 200! -/* /_client/index.html 200 \ No newline at end of file +/* /_client/index.html 200 \ No newline at end of file diff --git a/website/🔌 Collab.md b/website/🔌 Collab.md deleted file mode 100644 index 15d76331..00000000 --- a/website/🔌 Collab.md +++ /dev/null @@ -1,41 +0,0 @@ ---- -type: plug -repo: https://github.com/silverbulletmd/silverbullet -share-support: true ---- - -The Collab plug implements real-time “Google Doc” style collaboration with other SilverBullet users using the [Hocuspocus](https://hocuspocus.dev/) library. It supports: - -* Real-time editing -* Showing other participant’s cursors and selections - -The philosophy behind this plug is that by default your data is private and not shared with others. However, there are cases where you would like to collaborate on individual pages. - -Some example use cases: - -* Sharing a meeting agenda or meeting notes -* Writing or editing an article with others - -The collab plug allows you to share individual pages. All collaborators will keep their own local copy on disk (which they can back up, and you probably should), but the “source of truth” moves to a central collaboration server. There is one deployed at `wss://collab.silverbullet.md`, but you can also run your own, see [[@deploy|the instructions below]]. The collab plugin leverages SB’s [[🔌 Share]] infrastructure. - -To use it: - -1. Open a page you would like to collaborate on -2. Run the {[Share: Collab]} command and select the collab server to use (an open one runs at `wss://collab.silverbullet.md`) -3. Copy & paste the `collab:...` URI that is injected into the `$share` [[Frontmatter]] and send to a collaborator **or** if your collaborator is not (yet) a SilverBullet user, you can use the silverbullet.md website (which is an SB instance) directly via the `https://silverbullet.md/collab:...` URL scheme. -4. If your collaborator is an SB user, have them use the {[Share: Join Collab]} command, or directly open the `collab:...` URI as a page in SilverBullet (both do the same). -5. If the collaborator wants to keep a persistent copy of the page collaborated page, they can simply _rename_ the page to something not prefixed with `collab:`. Everything will keep working for as long as the `collab:` will appear in the `$share` attribute of [[Frontmatter]] - -## How it works -The Collab plug uses Hocuspocus for real-time collaboration via a WebSocket. A random ID is assigned to every shared page, and a copy of this page (as well as its history) will be stored on the collaboration server. Therefore, be cautious about what you share, especially when using a public collab server like `collab.silverbullet.md`. For “production use” we recommend deploying your own collab server. - -## Deploying your own collab server -$deploy - -Collaboration uses the excellent Hocuspocus library. You can easily deploy your own collaboration server as follows (requires node.js and npm): - -```shell -npx @hocuspocus/cli@2.0.6 --sqlite documents.db --port 1337 -``` - -This will run the hocuspocus server on port 1337, and store page data persistently in a SQLite database `documents.db`. You can connect to this server via `ws://ip:1337`. To use SSL, put a TLS terminator in front of it, in which case you can use `wss://` instead. \ No newline at end of file