diff --git a/common/spaces/http_space_primitives.ts b/common/spaces/http_space_primitives.ts index c983d740..cdaa4458 100644 --- a/common/spaces/http_space_primitives.ts +++ b/common/spaces/http_space_primitives.ts @@ -67,6 +67,7 @@ export class HttpSpacePrimitives implements SpacePrimitives { if ( resp.status === 200 && this.expectedSpacePath && + resp.headers.get("X-Space-Path") && resp.headers.get("X-Space-Path") !== this.expectedSpacePath ) { console.log("Expected space path", this.expectedSpacePath); diff --git a/plug-api/lib/query.test.ts b/plug-api/lib/query.test.ts index fc2c6881..194195fa 100644 --- a/plug-api/lib/query.test.ts +++ b/plug-api/lib/query.test.ts @@ -1,8 +1,7 @@ import { renderToText } from "./tree.ts"; -import wikiMarkdownLang from "../../common/markdown_parser/parser.ts"; import { assert, assertEquals } from "../../test_deps.ts"; -import { parse } from "../../common/markdown_parser/parse_tree.ts"; import { removeQueries } from "./query.ts"; +import { parseMarkdown } from "$sb/lib/test_utils.ts"; const queryRemovalTest = ` # Heading @@ -14,8 +13,7 @@ End `; Deno.test("White out queries", () => { - const lang = wikiMarkdownLang([]); - const mdTree = parse(lang, queryRemovalTest); + const mdTree = parseMarkdown(queryRemovalTest); removeQueries(mdTree); const text = renderToText(mdTree); // Same length? We should be good diff --git a/plug-api/lib/resolve.test.ts b/plug-api/lib/resolve.test.ts index d124c82b..7200286a 100644 --- a/plug-api/lib/resolve.test.ts +++ b/plug-api/lib/resolve.test.ts @@ -1,5 +1,11 @@ -import { resolvePath } from "$sb/lib/resolve.ts"; +import { + federatedPathToUrl, + resolvePath, + rewritePageRefs, +} from "$sb/lib/resolve.ts"; import { assertEquals } from "../../test_deps.ts"; +import { parseMarkdown } from "$sb/lib/test_utils.ts"; +import { renderToText } from "$sb/lib/tree.ts"; Deno.test("Test URL resolver", () => { assertEquals(resolvePath("test", "some page"), "some page"); @@ -17,4 +23,67 @@ Deno.test("Test URL resolver", () => { resolvePath("!silverbullet.md", "test/image.png", true), "https://silverbullet.md/test/image.png", ); + + assertEquals( + resolvePath("!silverbullet.md", "bla@123"), + "!silverbullet.md/bla@123", + ); + assertEquals(resolvePath("somewhere", "bla@123"), "bla@123"); + + assertEquals( + federatedPathToUrl("!silverbullet.md"), + "https://silverbullet.md", + ); + assertEquals( + federatedPathToUrl("!silverbullet.md/index"), + "https://silverbullet.md/index", + ); +}); + +Deno.test("Test rewritePageRefs", () => { + let tree = parseMarkdown(` +This is a [[local link]] and [[local link|with alias]]. + + + + + + + + + + + +`); + rewritePageRefs(tree, "!silverbullet.md"); + let rewrittenText = renderToText(tree); + + assertEquals( + rewrittenText, + ` +This is a [[!silverbullet.md/local link]] and [[!silverbullet.md/local link|with alias]]. + + + + + + + + + + + +`, + ); + + tree = parseMarkdown( + `This is a [[local link]] and [[local link|with alias]].`, + ); + // Now test the default case without federated links + rewritePageRefs(tree, "index"); + rewrittenText = renderToText(tree); + assertEquals( + rewrittenText, + `This is a [[local link]] and [[local link|with alias]].`, + ); }); diff --git a/plug-api/lib/resolve.ts b/plug-api/lib/resolve.ts index a4dfd6a3..d89096a2 100644 --- a/plug-api/lib/resolve.ts +++ b/plug-api/lib/resolve.ts @@ -1,3 +1,5 @@ +import { findNodeOfType, ParseTree, traverseTree } from "$sb/lib/tree.ts"; + export function resolvePath( currentPage: string, pathToResolve: string, @@ -6,12 +8,7 @@ export function resolvePath( if (isFederationPath(currentPage) && !isFederationPath(pathToResolve)) { let domainPart = currentPage.split("/")[0]; if (fullUrl) { - domainPart = domainPart.substring(1); - if (domainPart.startsWith("localhost")) { - domainPart = "http://" + domainPart; - } else { - domainPart = "https://" + domainPart; - } + domainPart = federatedPathToUrl(domainPart); } return `${domainPart}/${pathToResolve}`; } else { @@ -19,6 +16,50 @@ export function resolvePath( } } +export function federatedPathToUrl(path: string): string { + path = path.substring(1); + if (path.startsWith("localhost")) { + path = "http://" + path; + } else { + path = "https://" + path; + } + return path; +} + export function isFederationPath(path: string) { return path.startsWith("!"); } + +export function rewritePageRefs(tree: ParseTree, templatePath: string) { + traverseTree(tree, (n): boolean => { + if (n.type === "DirectiveStart") { + const pageRef = findNodeOfType(n, "PageRef")!; + if (pageRef) { + const pageRefName = pageRef.children![0].text!.slice(2, -2); + pageRef.children![0].text = `[[${ + resolvePath(templatePath, pageRefName) + }]]`; + } + const directiveText = n.children![0].text; + // #use or #import + if (directiveText) { + const match = /\[\[(.+)\]\]/.exec(directiveText); + if (match) { + const pageRefName = match[1]; + n.children![0].text = directiveText.replace( + match[0], + `[[${resolvePath(templatePath, pageRefName)}]]`, + ); + } + } + + return true; + } + if (n.type === "WikiLinkPage") { + n.children![0].text = resolvePath(templatePath, n.children![0].text!); + return true; + } + + return false; + }); +} diff --git a/plug-api/lib/test_utils.ts b/plug-api/lib/test_utils.ts new file mode 100644 index 00000000..08d54696 --- /dev/null +++ b/plug-api/lib/test_utils.ts @@ -0,0 +1,8 @@ +import wikiMarkdownLang from "../../common/markdown_parser/parser.ts"; +import type { ParseTree } from "$sb/lib/tree.ts"; +import { parse } from "../../common/markdown_parser/parse_tree.ts"; + +export function parseMarkdown(text: string): ParseTree { + const lang = wikiMarkdownLang([]); + return parse(lang, text); +} diff --git a/plugs/directive/command.ts b/plugs/directive/command.ts index c76c0099..65e15fc6 100644 --- a/plugs/directive/command.ts +++ b/plugs/directive/command.ts @@ -19,6 +19,7 @@ export async function updateDirectivesOnPageCommand() { if (isFederationPath(currentPage)) { console.info("Current page is a federation page, not updating directives."); + return; } if (metaData.$disableDirectives) { diff --git a/plugs/directive/template_directive.ts b/plugs/directive/template_directive.ts index 58c9284c..ebdb18ea 100644 --- a/plugs/directive/template_directive.ts +++ b/plugs/directive/template_directive.ts @@ -14,7 +14,7 @@ import { directiveRegex } from "./directives.ts"; import { updateDirectives } from "./command.ts"; import { buildHandebarOptions } from "./util.ts"; import { PageMeta } from "../../web/types.ts"; -import { resolvePath } from "$sb/lib/resolve.ts"; +import { resolvePath, rewritePageRefs } from "$sb/lib/resolve.ts"; const templateRegex = /\[\[([^\]]+)\]\]\s*(.*)\s*/; @@ -82,40 +82,6 @@ export async function templateDirectiveRenderer( return newBody.trim(); } -function rewritePageRefs(tree: ParseTree, templatePath: string) { - traverseTree(tree, (n): boolean => { - if (n.type === "DirectiveStart") { - const pageRef = findNodeOfType(n, "PageRef")!; - if (pageRef) { - const pageRefName = pageRef.children![0].text!.slice(2, -2); - pageRef.children![0].text = `[[${ - resolvePath(templatePath, pageRefName) - }]]`; - } - const directiveText = n.children![0].text; - // #use or #import - if (directiveText) { - const match = /\[\[(.+)\]\]/.exec(directiveText); - if (match) { - const pageRefName = match[1]; - n.children![0].text = directiveText.replace( - match[0], - `[[${resolvePath(templatePath, pageRefName)}]]`, - ); - } - } - - return true; - } - if (n.type === "WikiLinkPage") { - n.children![0].text = resolvePath(templatePath, n.children![0].text!); - return true; - } - - return false; - }); -} - export function cleanTemplateInstantiations(text: string) { return text.replaceAll(directiveRegex, ( _fullMatch, diff --git a/plugs/federation/config.ts b/plugs/federation/config.ts new file mode 100644 index 00000000..02bd3c5a --- /dev/null +++ b/plugs/federation/config.ts @@ -0,0 +1,36 @@ +import { readSetting } from "$sb/lib/settings_page.ts"; + +type FederationConfig = { + uri: string; + perm?: "ro" | "rw"; + // TODO: alias?: string; +}; + +let federationConfigs: FederationConfig[] = []; +let lastFederationUrlFetch = 0; + +export async function readFederationConfigs(): Promise { + // Update at most every 5 seconds + if (Date.now() > lastFederationUrlFetch + 5000) { + federationConfigs = await readSetting("federate", []); + if (!Array.isArray(federationConfigs)) { + console.error("'federate' setting should be an array of objects"); + return []; + } + // Normalize URIs + for (const config of federationConfigs) { + if (!config.uri) { + console.error( + "'federate' setting should be an array of objects with at least an 'uri' property", + config, + ); + continue; + } + if (!config.uri.startsWith("!")) { + config.uri = `!${config.uri}`; + } + } + lastFederationUrlFetch = Date.now(); + } + return federationConfigs; +} diff --git a/plugs/federation/federation.plug.yaml b/plugs/federation/federation.plug.yaml index a7e6e3f0..480266cb 100644 --- a/plugs/federation/federation.plug.yaml +++ b/plugs/federation/federation.plug.yaml @@ -2,11 +2,11 @@ name: federation requiredPermissions: - fetch functions: - #listFiles: - # path: ./federation.ts:listFiles - # pageNamespace: - # pattern: "!.+" - # operation: listFiles + listFiles: + path: ./federation.ts:listFiles + pageNamespace: + pattern: "!.+" + operation: listFiles readFile: path: ./federation.ts:readFile pageNamespace: diff --git a/plugs/federation/federation.ts b/plugs/federation/federation.ts index 632eb4b9..c978d15c 100644 --- a/plugs/federation/federation.ts +++ b/plugs/federation/federation.ts @@ -1,90 +1,115 @@ import "$sb/lib/fetch.ts"; import type { FileMeta } from "../../common/types.ts"; -import { readSetting } from "$sb/lib/settings_page.ts"; - -function resolveFederated(pageName: string): string { - // URL without the prefix "!"" - let url = pageName.substring(1); - if (!url.startsWith("127.0.0.1") && !url.startsWith("localhost")) { - url = `https://${url}`; - } else { - url = `http://${url}`; - } - return url; -} +import { federatedPathToUrl } from "$sb/lib/resolve.ts"; +import { readFederationConfigs } from "./config.ts"; +import { store } from "$sb/plugos-syscall/mod.ts"; async function responseToFileMeta( r: Response, name: string, ): Promise { - // const perm = r.headers.get("X-Permission") as any || "ro"; - // const federationConfigs = await readFederationConfigs(); - // const federationConfig = federationConfigs.find((config) => - // name.startsWith(config.uri) - // ); - // if (federationConfig?.perm) { - // perm = federationConfig.perm; - // } + const federationConfigs = await readFederationConfigs(); + + // Default permission is "ro" unless explicitly set otherwise + let perm: "ro" | "rw" = "ro"; + const federationConfig = federationConfigs.find((config) => + name.startsWith(config.uri) + ); + if (federationConfig?.perm) { + perm = federationConfig.perm; + } return { name: name, size: r.headers.get("Content-length") ? +r.headers.get("Content-length")! : 0, contentType: r.headers.get("Content-type")!, - perm: "ro", + perm, lastModified: +(r.headers.get("X-Last-Modified") || "0"), }; } -type FederationConfig = { - uri: string; - // perm?: "ro" | "rw"; -}; -let federationConfigs: FederationConfig[] = []; -let lastFederationUrlFetch = 0; +const fileListingPrefixCacheKey = `federationListCache:`; +const listingCacheTimeout = 1000 * 30; -async function readFederationConfigs() { - // Update at most every 5 seconds - if (Date.now() > lastFederationUrlFetch + 5000) { - federationConfigs = await readSetting("federate", []); - // Normalize URIs - for (const config of federationConfigs) { - if (!config.uri.startsWith("!")) { - config.uri = `!${config.uri}`; - } - } - lastFederationUrlFetch = Date.now(); - } - return federationConfigs; -} +type FileListingCacheEntry = { + items: FileMeta[]; + lastUpdated: number; +}; export async function listFiles(): Promise { let fileMetas: FileMeta[] = []; // Fetch them all in parallel - await Promise.all((await readFederationConfigs()).map(async (config) => { - // console.log("Fetching from federated", config); - const uriParts = config.uri.split("/"); - const rootUri = uriParts[0]; - const prefix = uriParts.slice(1).join("/"); - const r = await nativeFetch(resolveFederated(rootUri)); - fileMetas = fileMetas.concat( - (await r.json()).filter((meta: FileMeta) => meta.name.startsWith(prefix)) - .map((meta: FileMeta) => ({ + try { + await Promise.all((await readFederationConfigs()).map(async (config) => { + const cachedListing = await store.get( + `${fileListingPrefixCacheKey}${config.uri}`, + ) as FileListingCacheEntry; + if ( + cachedListing && + cachedListing.lastUpdated > Date.now() - listingCacheTimeout + ) { + fileMetas = fileMetas.concat(cachedListing.items); + return; + } + console.log("Fetching from federated", config); + const uriParts = config.uri.split("/"); + const rootUri = uriParts[0]; + const prefix = uriParts.slice(1).join("/"); + const indexUrl = `${federatedPathToUrl(rootUri)}/index.json`; + try { + const r = await nativeFetch(indexUrl, { + method: "GET", + headers: { + Accept: "application/json", + }, + }); + if (r.status !== 200) { + console.error( + `Failed to fetch ${indexUrl}. Skipping.`, + r.status, + r.statusText, + ); + if (cachedListing) { + console.info("Using cached listing"); + fileMetas = fileMetas.concat(cachedListing.items); + } + return; + } + const jsonResult = await r.json(); + const items: FileMeta[] = jsonResult.filter((meta: FileMeta) => + meta.name.startsWith(prefix) + ).map((meta: FileMeta) => ({ ...meta, - perm: "ro", //config.perm || meta.perm, + perm: config.perm || "ro", name: `${rootUri}/${meta.name}`, - })), - ); - })); - // console.log("All of em: ", fileMetas); - return fileMetas; + })); + await store.set(`${fileListingPrefixCacheKey}${config.uri}`, { + items, + lastUpdated: Date.now(), + } as FileListingCacheEntry); + fileMetas = fileMetas.concat(items); + } catch (e: any) { + console.error("Failed to process", indexUrl, e); + } + })); + + // console.log("All of em: ", fileMetas); + return fileMetas; + } catch (e: any) { + console.error("Error listing federation files", e); + return []; + } } export async function readFile( name: string, ): Promise<{ data: Uint8Array; meta: FileMeta } | undefined> { - const url = resolveFederated(name); + const url = federatedPathToUrl(name); const r = await nativeFetch(url); + if (r.status === 503) { + throw new Error("Offline"); + } const fileMeta = await responseToFileMeta(r, name); console.log("Fetching", url); if (r.status === 404) { @@ -151,9 +176,12 @@ export async function deleteFile( } export async function getFileMeta(name: string): Promise { - const url = resolveFederated(name); + const url = federatedPathToUrl(name); console.log("Fetching federation file meta", url); const r = await nativeFetch(url, { method: "HEAD" }); + if (r.status === 503) { + throw new Error("Offline"); + } const fileMeta = await responseToFileMeta(r, name); if (!r.ok) { throw new Error("Not found"); diff --git a/server/http_server.ts b/server/http_server.ts index 0250efba..f786e4c3 100644 --- a/server/http_server.ts +++ b/server/http_server.ts @@ -355,7 +355,13 @@ export class HttpServer { try { const req = await fetch(url); response.status = req.status; - response.headers = req.headers; + // Override X-Permssion header to always be "ro" + const newHeaders = new Headers(); + for (const [key, value] of req.headers.entries()) { + newHeaders.set(key, value); + } + newHeaders.set("X-Permission", "ro"); + response.headers = newHeaders; response.body = req.body; } catch (e: any) { console.error("Error fetching federated link", e);