Robustness and federation sync

common/spaces/http_space_primitives.ts

@@ -67,6 +67,7 @@ export class HttpSpacePrimitives implements SpacePrimitives {
     if (
       resp.status === 200 &&
       this.expectedSpacePath &&
+      resp.headers.get("X-Space-Path") &&
       resp.headers.get("X-Space-Path") !== this.expectedSpacePath
     ) {
       console.log("Expected space path", this.expectedSpacePath);

plug-api/lib/query.test.ts

@@ -1,8 +1,7 @@
 import { renderToText } from "./tree.ts";
-import wikiMarkdownLang from "../../common/markdown_parser/parser.ts";
 import { assert, assertEquals } from "../../test_deps.ts";
-import { parse } from "../../common/markdown_parser/parse_tree.ts";
 import { removeQueries } from "./query.ts";
+import { parseMarkdown } from "$sb/lib/test_utils.ts";
 
 const queryRemovalTest = `
 # Heading
@@ -14,8 +13,7 @@ End
 `;
 
 Deno.test("White out queries", () => {
-  const lang = wikiMarkdownLang([]);
+  const mdTree = parseMarkdown(queryRemovalTest);
-  const mdTree = parse(lang, queryRemovalTest);
   removeQueries(mdTree);
   const text = renderToText(mdTree);
   // Same length? We should be good

plug-api/lib/resolve.test.ts

@@ -1,5 +1,11 @@
-import { resolvePath } from "$sb/lib/resolve.ts";
+import {
+  federatedPathToUrl,
+  resolvePath,
+  rewritePageRefs,
+} from "$sb/lib/resolve.ts";
 import { assertEquals } from "../../test_deps.ts";
+import { parseMarkdown } from "$sb/lib/test_utils.ts";
+import { renderToText } from "$sb/lib/tree.ts";
 
 Deno.test("Test URL resolver", () => {
   assertEquals(resolvePath("test", "some page"), "some page");
@@ -17,4 +23,67 @@ Deno.test("Test URL resolver", () => {
     resolvePath("!silverbullet.md", "test/image.png", true),
     "https://silverbullet.md/test/image.png",
   );
+
+  assertEquals(
+    resolvePath("!silverbullet.md", "bla@123"),
+    "!silverbullet.md/bla@123",
+  );
+  assertEquals(resolvePath("somewhere", "bla@123"), "bla@123");
+
+  assertEquals(
+    federatedPathToUrl("!silverbullet.md"),
+    "https://silverbullet.md",
+  );
+  assertEquals(
+    federatedPathToUrl("!silverbullet.md/index"),
+    "https://silverbullet.md/index",
+  );
+});
+
+Deno.test("Test rewritePageRefs", () => {
+  let tree = parseMarkdown(`
+This is a [[local link]] and [[local link|with alias]].
+
+<!-- #query page render [[template/page]] -->
+<!-- /query -->
+
+<!-- #use [[template/use-template]] {} -->
+
+<!-- /use -->
+
+<!-- #include [[template/include-template]] {} -->
+
+<!-- /include -->
+`);
+  rewritePageRefs(tree, "!silverbullet.md");
+  let rewrittenText = renderToText(tree);
+
+  assertEquals(
+    rewrittenText,
+    `
+This is a [[!silverbullet.md/local link]] and [[!silverbullet.md/local link|with alias]].
+
+<!-- #query page render [[!silverbullet.md/template/page]] -->
+<!-- /query -->
+
+<!-- #use [[!silverbullet.md/template/use-template]] {} -->
+
+<!-- /use -->
+
+<!-- #include [[!silverbullet.md/template/include-template]] {} -->
+
+<!-- /include -->
+`,
+  );
+
+  tree = parseMarkdown(
+    `This is a [[local link]] and [[local link|with alias]].`,
+  );
+  // Now test the default case without federated links
+  rewritePageRefs(tree, "index");
+  rewrittenText = renderToText(tree);
+  assertEquals(
+    rewrittenText,
+    `This is a [[local link]] and [[local link|with alias]].`,
+  );
 });

plug-api/lib/resolve.ts

@@ -1,3 +1,5 @@
+import { findNodeOfType, ParseTree, traverseTree } from "$sb/lib/tree.ts";
+
 export function resolvePath(
   currentPage: string,
   pathToResolve: string,
@@ -6,12 +8,7 @@ export function resolvePath(
   if (isFederationPath(currentPage) && !isFederationPath(pathToResolve)) {
     let domainPart = currentPage.split("/")[0];
     if (fullUrl) {
-      domainPart = domainPart.substring(1);
+      domainPart = federatedPathToUrl(domainPart);
-      if (domainPart.startsWith("localhost")) {
-        domainPart = "http://" + domainPart;
-      } else {
-        domainPart = "https://" + domainPart;
-      }
     }
     return `${domainPart}/${pathToResolve}`;
   } else {
@@ -19,6 +16,50 @@ export function resolvePath(
   }
 }
 
+export function federatedPathToUrl(path: string): string {
+  path = path.substring(1);
+  if (path.startsWith("localhost")) {
+    path = "http://" + path;
+  } else {
+    path = "https://" + path;
+  }
+  return path;
+}
+
 export function isFederationPath(path: string) {
   return path.startsWith("!");
 }
+
+export function rewritePageRefs(tree: ParseTree, templatePath: string) {
+  traverseTree(tree, (n): boolean => {
+    if (n.type === "DirectiveStart") {
+      const pageRef = findNodeOfType(n, "PageRef")!;
+      if (pageRef) {
+        const pageRefName = pageRef.children![0].text!.slice(2, -2);
+        pageRef.children![0].text = `[[${
+          resolvePath(templatePath, pageRefName)
+        }]]`;
+      }
+      const directiveText = n.children![0].text;
+      // #use or #import
+      if (directiveText) {
+        const match = /\[\[(.+)\]\]/.exec(directiveText);
+        if (match) {
+          const pageRefName = match[1];
+          n.children![0].text = directiveText.replace(
+            match[0],
+            `[[${resolvePath(templatePath, pageRefName)}]]`,
+          );
+        }
+      }
+
+      return true;
+    }
+    if (n.type === "WikiLinkPage") {
+      n.children![0].text = resolvePath(templatePath, n.children![0].text!);
+      return true;
+    }
+
+    return false;
+  });
+}

plug-api/lib/test_utils.ts

@@ -0,0 +1,8 @@
+import wikiMarkdownLang from "../../common/markdown_parser/parser.ts";
+import type { ParseTree } from "$sb/lib/tree.ts";
+import { parse } from "../../common/markdown_parser/parse_tree.ts";
+
+export function parseMarkdown(text: string): ParseTree {
+  const lang = wikiMarkdownLang([]);
+  return parse(lang, text);
+}

plugs/directive/command.ts

@@ -19,6 +19,7 @@ export async function updateDirectivesOnPageCommand() {
 
   if (isFederationPath(currentPage)) {
     console.info("Current page is a federation page, not updating directives.");
+    return;
   }
 
   if (metaData.$disableDirectives) {

plugs/directive/template_directive.ts

@@ -14,7 +14,7 @@ import { directiveRegex } from "./directives.ts";
 import { updateDirectives } from "./command.ts";
 import { buildHandebarOptions } from "./util.ts";
 import { PageMeta } from "../../web/types.ts";
-import { resolvePath } from "$sb/lib/resolve.ts";
+import { resolvePath, rewritePageRefs } from "$sb/lib/resolve.ts";
 
 const templateRegex = /\[\[([^\]]+)\]\]\s*(.*)\s*/;
 
@@ -82,40 +82,6 @@ export async function templateDirectiveRenderer(
   return newBody.trim();
 }
 
-function rewritePageRefs(tree: ParseTree, templatePath: string) {
-  traverseTree(tree, (n): boolean => {
-    if (n.type === "DirectiveStart") {
-      const pageRef = findNodeOfType(n, "PageRef")!;
-      if (pageRef) {
-        const pageRefName = pageRef.children![0].text!.slice(2, -2);
-        pageRef.children![0].text = `[[${
-          resolvePath(templatePath, pageRefName)
-        }]]`;
-      }
-      const directiveText = n.children![0].text;
-      // #use or #import
-      if (directiveText) {
-        const match = /\[\[(.+)\]\]/.exec(directiveText);
-        if (match) {
-          const pageRefName = match[1];
-          n.children![0].text = directiveText.replace(
-            match[0],
-            `[[${resolvePath(templatePath, pageRefName)}]]`,
-          );
-        }
-      }
-
-      return true;
-    }
-    if (n.type === "WikiLinkPage") {
-      n.children![0].text = resolvePath(templatePath, n.children![0].text!);
-      return true;
-    }
-
-    return false;
-  });
-}
-
 export function cleanTemplateInstantiations(text: string) {
   return text.replaceAll(directiveRegex, (
     _fullMatch,

plugs/federation/config.ts

@@ -0,0 +1,36 @@
+import { readSetting } from "$sb/lib/settings_page.ts";
+
+type FederationConfig = {
+  uri: string;
+  perm?: "ro" | "rw";
+  // TODO: alias?: string;
+};
+
+let federationConfigs: FederationConfig[] = [];
+let lastFederationUrlFetch = 0;
+
+export async function readFederationConfigs(): Promise<FederationConfig[]> {
+  // Update at most every 5 seconds
+  if (Date.now() > lastFederationUrlFetch + 5000) {
+    federationConfigs = await readSetting("federate", []);
+    if (!Array.isArray(federationConfigs)) {
+      console.error("'federate' setting should be an array of objects");
+      return [];
+    }
+    // Normalize URIs
+    for (const config of federationConfigs) {
+      if (!config.uri) {
+        console.error(
+          "'federate' setting should be an array of objects with at least an 'uri' property",
+          config,
+        );
+        continue;
+      }
+      if (!config.uri.startsWith("!")) {
+        config.uri = `!${config.uri}`;
+      }
+    }
+    lastFederationUrlFetch = Date.now();
+  }
+  return federationConfigs;
+}

plugs/federation/federation.plug.yaml

@@ -2,11 +2,11 @@ name: federation
 requiredPermissions:
   - fetch
 functions:
-  #listFiles:
+  listFiles:
-  #  path: ./federation.ts:listFiles
+   path: ./federation.ts:listFiles
-  #  pageNamespace:
+   pageNamespace:
-  #    pattern: "!.+"
+     pattern: "!.+"
-  #    operation: listFiles
+     operation: listFiles
   readFile:
     path: ./federation.ts:readFile
     pageNamespace:

plugs/federation/federation.ts

@@ -1,90 +1,115 @@
 import "$sb/lib/fetch.ts";
 import type { FileMeta } from "../../common/types.ts";
-import { readSetting } from "$sb/lib/settings_page.ts";
+import { federatedPathToUrl } from "$sb/lib/resolve.ts";
-
+import { readFederationConfigs } from "./config.ts";
-function resolveFederated(pageName: string): string {
+import { store } from "$sb/plugos-syscall/mod.ts";
-  // URL without the prefix "!""
-  let url = pageName.substring(1);
-  if (!url.startsWith("127.0.0.1") && !url.startsWith("localhost")) {
-    url = `https://${url}`;
-  } else {
-    url = `http://${url}`;
-  }
-  return url;
-}
 
 async function responseToFileMeta(
   r: Response,
   name: string,
 ): Promise<FileMeta> {
-  // const perm = r.headers.get("X-Permission") as any || "ro";
+  const federationConfigs = await readFederationConfigs();
-  // const federationConfigs = await readFederationConfigs();
+
-  // const federationConfig = federationConfigs.find((config) =>
+  // Default permission is "ro" unless explicitly set otherwise
-  //   name.startsWith(config.uri)
+  let perm: "ro" | "rw" = "ro";
-  // );
+  const federationConfig = federationConfigs.find((config) =>
-  // if (federationConfig?.perm) {
+    name.startsWith(config.uri)
-  //   perm = federationConfig.perm;
+  );
-  // }
+  if (federationConfig?.perm) {
+    perm = federationConfig.perm;
+  }
   return {
     name: name,
     size: r.headers.get("Content-length")
       ? +r.headers.get("Content-length")!
       : 0,
     contentType: r.headers.get("Content-type")!,
-    perm: "ro",
+    perm,
     lastModified: +(r.headers.get("X-Last-Modified") || "0"),
   };
 }
 
-type FederationConfig = {
+const fileListingPrefixCacheKey = `federationListCache:`;
-  uri: string;
+const listingCacheTimeout = 1000 * 30;
-  // perm?: "ro" | "rw";
-};
-let federationConfigs: FederationConfig[] = [];
-let lastFederationUrlFetch = 0;
 
-async function readFederationConfigs() {
+type FileListingCacheEntry = {
-  // Update at most every 5 seconds
+  items: FileMeta[];
-  if (Date.now() > lastFederationUrlFetch + 5000) {
+  lastUpdated: number;
-    federationConfigs = await readSetting("federate", []);
+};
-    // Normalize URIs
-    for (const config of federationConfigs) {
-      if (!config.uri.startsWith("!")) {
-        config.uri = `!${config.uri}`;
-      }
-    }
-    lastFederationUrlFetch = Date.now();
-  }
-  return federationConfigs;
-}
 
 export async function listFiles(): Promise<FileMeta[]> {
   let fileMetas: FileMeta[] = [];
   // Fetch them all in parallel
-  await Promise.all((await readFederationConfigs()).map(async (config) => {
+  try {
-    // console.log("Fetching from federated", config);
+    await Promise.all((await readFederationConfigs()).map(async (config) => {
-    const uriParts = config.uri.split("/");
+      const cachedListing = await store.get(
-    const rootUri = uriParts[0];
+        `${fileListingPrefixCacheKey}${config.uri}`,
-    const prefix = uriParts.slice(1).join("/");
+      ) as FileListingCacheEntry;
-    const r = await nativeFetch(resolveFederated(rootUri));
+      if (
-    fileMetas = fileMetas.concat(
+        cachedListing &&
-      (await r.json()).filter((meta: FileMeta) => meta.name.startsWith(prefix))
+        cachedListing.lastUpdated > Date.now() - listingCacheTimeout
-        .map((meta: FileMeta) => ({
+      ) {
+        fileMetas = fileMetas.concat(cachedListing.items);
+        return;
+      }
+      console.log("Fetching from federated", config);
+      const uriParts = config.uri.split("/");
+      const rootUri = uriParts[0];
+      const prefix = uriParts.slice(1).join("/");
+      const indexUrl = `${federatedPathToUrl(rootUri)}/index.json`;
+      try {
+        const r = await nativeFetch(indexUrl, {
+          method: "GET",
+          headers: {
+            Accept: "application/json",
+          },
+        });
+        if (r.status !== 200) {
+          console.error(
+            `Failed to fetch ${indexUrl}. Skipping.`,
+            r.status,
+            r.statusText,
+          );
+          if (cachedListing) {
+            console.info("Using cached listing");
+            fileMetas = fileMetas.concat(cachedListing.items);
+          }
+          return;
+        }
+        const jsonResult = await r.json();
+        const items: FileMeta[] = jsonResult.filter((meta: FileMeta) =>
+          meta.name.startsWith(prefix)
+        ).map((meta: FileMeta) => ({
           ...meta,
-          perm: "ro", //config.perm || meta.perm,
+          perm: config.perm || "ro",
           name: `${rootUri}/${meta.name}`,
-        })),
+        }));
-    );
+        await store.set(`${fileListingPrefixCacheKey}${config.uri}`, {
-  }));
+          items,
-  // console.log("All of em: ", fileMetas);
+          lastUpdated: Date.now(),
-  return fileMetas;
+        } as FileListingCacheEntry);
+        fileMetas = fileMetas.concat(items);
+      } catch (e: any) {
+        console.error("Failed to process", indexUrl, e);
+      }
+    }));
+
+    // console.log("All of em: ", fileMetas);
+    return fileMetas;
+  } catch (e: any) {
+    console.error("Error listing federation files", e);
+    return [];
+  }
 }
 
 export async function readFile(
   name: string,
 ): Promise<{ data: Uint8Array; meta: FileMeta } | undefined> {
-  const url = resolveFederated(name);
+  const url = federatedPathToUrl(name);
   const r = await nativeFetch(url);
+  if (r.status === 503) {
+    throw new Error("Offline");
+  }
   const fileMeta = await responseToFileMeta(r, name);
   console.log("Fetching", url);
   if (r.status === 404) {
@@ -151,9 +176,12 @@ export async function deleteFile(
 }
 
 export async function getFileMeta(name: string): Promise<FileMeta> {
-  const url = resolveFederated(name);
+  const url = federatedPathToUrl(name);
   console.log("Fetching federation file meta", url);
   const r = await nativeFetch(url, { method: "HEAD" });
+  if (r.status === 503) {
+    throw new Error("Offline");
+  }
   const fileMeta = await responseToFileMeta(r, name);
   if (!r.ok) {
     throw new Error("Not found");

server/http_server.ts

@@ -355,7 +355,13 @@ export class HttpServer {
             try {
               const req = await fetch(url);
               response.status = req.status;
-              response.headers = req.headers;
+              // Override X-Permssion header to always be "ro"
+              const newHeaders = new Headers();
+              for (const [key, value] of req.headers.entries()) {
+                newHeaders.set(key, value);
+              }
+              newHeaders.set("X-Permission", "ro");
+              response.headers = newHeaders;
               response.body = req.body;
             } catch (e: any) {
               console.error("Error fetching federated link", e);