Transclosure support for pages (#925)

Transclusion implementation

common/query_functions.ts

@@ -1,8 +1,8 @@
-import { FunctionMap } from "../plug-api/types.ts";
+import { FunctionMap, Query } from "$sb/types.ts";
-import { builtinFunctions } from "../lib/builtin_query_functions.ts";
+import { builtinFunctions } from "$lib/builtin_query_functions.ts";
-import { System } from "../lib/plugos/system.ts";
+import { System } from "$lib/plugos/system.ts";
-import { Query } from "../plug-api/types.ts";
 import { LimitedMap } from "$lib/limited_map.ts";
+import { parsePageRef } from "$sb/lib/page_ref.ts";
 
 const pageCacheTtl = 10 * 1000; // 10s
 
@@ -49,20 +49,58 @@ export function buildQueryFunctions(
         variables,
       ]);
     },
-    // INTERNAL: Used to implement resolving [[links]] in expressions
+    // INTERNAL: Used to implement resolving [[links]] in expressions, also supports [[link#header]] and [[link$pos]] as well as [[link$anchor]]
-    readPage(name: string): Promise<string> | string {
+    async readPage(name: string): Promise<string> {
       const cachedPage = pageCache.get(name);
       if (cachedPage) {
         return cachedPage;
       } else {
-        return system.localSyscall("space.readPage", [name]).then((page) => {
+        const pageRef = parsePageRef(name);
+        try {
+          let page: string = await system.localSyscall("space.readPage", [
+            pageRef.page,
+          ]);
+
+          // Extract page section if pos, anchor, or header are included
+          if (pageRef.pos) {
+            // If the page link includes a position, slice the page from that position
+            page = page.slice(pageRef.pos);
+          } else if (pageRef.anchor) {
+            // If the page link includes an anchor, slice the page from that anchor
+            const pos = page.indexOf(`$${pageRef.anchor}`);
+            page = page.slice(pos);
+          } else if (pageRef.header) {
+            // If the page link includes a header, select that header (up to the next header at the same level)
+            // Note: this an approximation, should ideally use the AST
+            let pos = page.indexOf(`# ${pageRef.header}\n`);
+            let headingLevel = 1;
+            while (page.charAt(pos - 1) === "#") {
+              pos--;
+              headingLevel++;
+            }
+            page = page.slice(pos);
+
+            // Slice up to the next equal or higher level heading
+            const headRegex = new RegExp(
+              `[^#]#{1,${headingLevel}} [^\n]*\n`,
+              "g",
+            );
+            const endPos = page.slice(headingLevel).search(headRegex) +
+              headingLevel;
+            if (endPos) {
+              page = page.slice(0, endPos);
+            }
+          }
+
           pageCache.set(name, page, pageCacheTtl);
+
           return page;
-        }).catch((e: any) => {
+        } catch (e: any) {
           if (e.message === "Not found") {
             throw new Error(`Page not found: ${name}`);
           }
-        });
+          throw e;
+        }
       }
     },
   };

plugs/markdown/api.ts

@@ -10,6 +10,8 @@ import {
   MarkdownRenderOptions,
   renderMarkdownToHtml,
 } from "./markdown_render.ts";
+import { validatePageName } from "$sb/lib/page_ref.ts";
+import { parsePageRef } from "$sb/lib/page_ref.ts";
 
 /**
  * Finds code widgets, runs their plug code to render and inlines their content in the parse tree
@@ -58,6 +60,48 @@ export async function expandCodeWidgets(
           console.error("Error rendering code", e.message);
         }
       }
+    } else if (n.type === "Image") {
+      // Let's scan for ![[embeds]] that are codified as Images, confusingly
+      const wikiLinkMark = findNodeOfType(n, "WikiLinkMark");
+      if (!wikiLinkMark) {
+        return;
+      }
+      const wikiLinkPage = findNodeOfType(n, "WikiLinkPage");
+      if (!wikiLinkPage) {
+        return;
+      }
+
+      const page = wikiLinkPage.children![0].text!;
+
+      // Check if this is likely a page link (based on the path format, e.g. if it contains an extension, it's probably not a page link)
+      try {
+        const ref = parsePageRef(page);
+        validatePageName(ref.page);
+      } catch {
+        // Not a valid page name, so not a page reference
+        return;
+      }
+
+      // Internally translate this to a template that inlines a page, then render that
+      const result = await codeWidget.render(
+        "template",
+        `{{[[${page}]]}}`,
+        page,
+      );
+      if (!result) {
+        return {
+          text: "",
+        };
+      }
+      // Only do this for "markdown" widgets, that is: that can render to markdown
+      if (result.markdown !== undefined) {
+        const parsedBody = await parseMarkdown(result.markdown);
+        // Recursively process
+        return expandCodeWidgets(
+          parsedBody,
+          page,
+        );
+      }
     }
   });
   return mdTree;

web/cm_plugins/inline_content.ts

@@ -1,16 +1,18 @@
 import { EditorState, Range } from "@codemirror/state";
 import { syntaxTree } from "@codemirror/language";
 import { Decoration, WidgetType } from "@codemirror/view";
+import { MarkdownWidget } from "./markdown_widget.ts";
 import { decoratorStateField } from "./util.ts";
 import type { Client } from "../client.ts";
-import { isLocalPath, resolvePath } from "$sb/lib/resolve.ts";
+import { isFederationPath, isLocalPath, resolvePath } from "$sb/lib/resolve.ts";
+import { parsePageRef } from "$sb/lib/page_ref.ts";
 
 type ImageDimensions = {
   width?: number;
   height?: number;
 };
 
-class InlineImageWidget extends WidgetType {
+class InlineContentWidget extends WidgetType {
   constructor(
     readonly url: string,
     readonly title: string,
@@ -20,14 +22,13 @@ class InlineImageWidget extends WidgetType {
     super();
   }
 
-  eq(other: InlineImageWidget) {
+  eq(other: InlineContentWidget) {
     return other.url === this.url && other.title === this.title &&
       JSON.stringify(other.dim) === JSON.stringify(this.dim);
   }
 
   get estimatedHeight(): number {
     const cachedHeight = this.client.getCachedWidgetHeight(`image:${this.url}`);
-    // console.log("Estimated height requested", this.url, cachedHeight);
     return cachedHeight;
   }
 
@@ -113,7 +114,9 @@ export function inlineImagesPlugin(client: Client) {
           (match = /(!?\[\[)([^\]\|]+)(?:\|([^\]]+))?(\]\])/g.exec(text))
         ) {
           [/* fullMatch */, /* firstMark */ , url, alias] = match;
+          if (!isFederationPath(url)) {
             url = "/" + url;
+          }
         } else {
           return;
         }
@@ -130,11 +133,45 @@ export function inlineImagesPlugin(client: Client) {
 
         if (isLocalPath(url)) {
           url = resolvePath(client.currentPage, decodeURI(url), true);
+          const pageRef = parsePageRef(url);
+          if (
+            isFederationPath(pageRef.page) ||
+            client.clientSystem.allKnownFiles.has(pageRef.page + ".md")
+          ) {
+            // This is a page reference, let's inline the content
+            const codeWidgetCallback = client.clientSystem.codeWidgetHook
+              .codeWidgetCallbacks.get("template");
+
+            if (!codeWidgetCallback) {
+              return;
+            }
+
+            widgets.push(
+              Decoration.line({
+                class: "sb-fenced-code-iframe",
+              }).range(node.to),
+            );
+
+            widgets.push(
+              Decoration.widget({
+                widget: new MarkdownWidget(
+                  node.from,
+                  client,
+                  `widget:${client.currentPage}:${text}`,
+                  `{{[[${url}]]}}`,
+                  codeWidgetCallback,
+                  "sb-markdown-widget sb-markdown-widget-inline",
+                ),
+                block: true,
+              }).range(node.to),
+            );
+            return;
+          }
         }
 
         widgets.push(
           Decoration.widget({
-            widget: new InlineImageWidget(url, alias, dim, client),
+            widget: new InlineContentWidget(url, alias, dim, client),
             block: true,
           }).range(node.to),
         );

web/editor_state.ts

@@ -27,7 +27,7 @@ import {
 import { vim } from "@replit/codemirror-vim";
 import { markdown } from "@codemirror/lang-markdown";
 import { Client } from "./client.ts";
-import { inlineImagesPlugin } from "./cm_plugins/inline_image.ts";
+import { inlineImagesPlugin } from "./cm_plugins/inline_content.ts";
 import { cleanModePlugins } from "./cm_plugins/clean.ts";
 import { lineWrapper } from "./cm_plugins/line_wrapper.ts";
 import { smartQuoteKeymap } from "./cm_plugins/smart_quotes.ts";

web/styles/editor.scss

@@ -581,6 +581,14 @@
     }
   }
 
+  .sb-markdown-widget-inline {
+    margin: 0 0 0 0;
+
+    &:hover .button-bar {
+      display: none !important;
+    }
+  }
+
   .sb-fenced-code-iframe {
     background-color: transparent;
 

website/Attachments.md

@@ -22,6 +22,8 @@ Images can also be embedded using the [[#Linking]] syntax, but prefixed with an
 
 These follow the same relative/absolute path rules as links described before.
 
+## Image resizing
+
 In addition, images can be _sized_ using the following syntax:
 * Specifying only a width: `![Alt text|300](image.png)` or `![[image.png|300]]`
 * Specifying both width and height: `![Hello|300x300](image.png)` or `![[image.png|300x300]]`

website/CHANGELOG.md

@@ -7,6 +7,7 @@ release.
 _These features are not yet properly released, you need to use [the edge builds](https://community.silverbullet.md/t/living-on-the-edge-builds/27) to try them._
 
 * The old **Template Picker** has now been rebranded to [[Meta Picker]] and surfaces pages in your space tagged as `#template` or `#meta`. Read more about this in [[Meta Pages]].
+* [[Transclusion]] has now been implementing, allowing inline embeddings of other pages as well as images (by onespaceman) using the convenient `![[link]]` syntax.
 * For new spaces, the default [[SETTINGS]] page is now tagged with `#meta`, which means it will only appear in the [[Meta Picker]]. There is also a new {[Navigate: Open SETTINGS]} command (bound to `Ctrl-,` and `Cmd-,`).
 * Attachments are now indexed, and smartly moved when pages are renamed (by onespaceman)
 * Images can now be resized: [[Attachments#Embedding]] (initial work done by [Florent](https://github.com/silverbulletmd/silverbullet/pull/833), later adapted by onespaceman)

website/Expression Language.md

@@ -63,9 +63,9 @@ A rendered query:
 Page references use the `[[page name]]` syntax and evaluate to the content of the referenced page (as a string), this makes them a good candidate to be used in conjunction with [[Functions#template(text, value)]] or to simply inline another page:
 
 ```template
-Including another page directly, without template rendering: {{[[internal/test page]]}}
+Including another page directly, without template rendering: {{[[internal/test page with template]]}}
 
-And rendered as a template: {{template([[internal/test page]], "actual value")}}
+And rendered as a template: {{template([[internal/test page with template]], "actual value")}}
 ```
 
 # Logical expressions 

website/Markdown/Extensions.md

@@ -7,6 +7,7 @@ In addition to supporting [[Markdown/Basics|markdown basics]] as standardized by
     * [[Live Queries]]
     * [[Live Templates]]
     * [[Table of Contents]]
+* [[Transclusion]] syntax
 * [[Markdown/Anchors]]
 * [[Markdown/Admonitions]]
 * Hashtags, e.g. `#mytag`.

website/Transclusion.md

@@ -0,0 +1,20 @@
+Transclusions are an extension of the [[Markdown]] syntax enabling inline embedding of content.
+
+The general syntax is `![[path]]`. Two types of transclusions are currently supported:
+
+# Images
+Syntax: `![[path/to/image.jpg]]` see [[Attachments#Embedding]] for more details.
+
+Image resizing is also supported:
+![[Attachments#Image resizing]]
+# Pages
+Syntax:
+* `![[page name]]` embed an entire page
+* `![[page name#header]]` embed only a section (guarded by the given header)
+* `![[page$anchor]]` embed a page _from_ the given anchor until the end of the page
+* `![[page@pos]]` embed a page _from_ the given character position until the end of the page
+
+For example, to embed a full page:
+![[internal/test page]]
+And just a header:
+![[internal/test page#This is a header]]

website/internal/test page with template.md

@@ -0,0 +1 @@
+This is a simple test page with a placeholder: {{.}}.

website/internal/test page.md

@@ -1 +1,4 @@
-This is a simple test page with a placeholder: {{.}}.
+This is a simple **test page**. Cool, no?
+
+# This is a header
+And some content underneath