Remove "syntax" support from plugs

common/manifest.ts

@@ -27,40 +27,8 @@ export type SilverBulletHooks =
   & EndpointHookT
   & PlugNamespaceHookT;
 
-/** Syntax extension allow plugs to declaratively add new *inline* parse tree nodes to the markdown parser. */
-export type SyntaxExtensions = {
-  /** A map of node **name** (also called "type"), to parsing and highlighting instructions. Each entry defines a new node. By convention node names (types) are UpperCamelCase (PascalCase).
-   *
-   * see: plug-api/lib/tree.ts#ParseTree
-   */
-  syntax?: { [key: string]: NodeDef };
-};
-
-/** Parsing and highlighting instructions for SyntaxExtension */
-export type NodeDef = {
-  /** An array of possible first characters to begin matching on.
-   *
-   * **Example**: If this node has the regex '[abc][123]', NodeDef.firstCharacters should be ["a", "b", "c"].
-   */
-  firstCharacters: string[];
-
-  /** A regular expression that matches the *entire* syntax, including the first character. */
-  regex: string;
-
-  /** CSS styles to apply to the matched text.
-   *
-   * Key-value pair of CSS key to value:
-   *
-   * **Example**: `backgroundColor: "rgba(22,22,22,0.07)"`
-   */
-  styles?: { [key: string]: string };
-
-  /** CSS class name to apply to the matched text */
-  className?: string;
-};
-
 /** A plug manifest configures hooks, declares syntax extensions, and describes plug metadata.
  *
  * Typically the manifest file is in a plug's root directory, named `${plugName}.plug.yaml`.
  */
-export type Manifest = plugos.Manifest<SilverBulletHooks> & SyntaxExtensions;
+export type Manifest = plugos.Manifest<SilverBulletHooks>;

common/markdown_parser/customtags.ts

@@ -17,6 +17,12 @@ export const AttributeTag = Tag.define();
 export const AttributeNameTag = Tag.define();
 export const AttributeValueTag = Tag.define();
 
+export const NamedAnchorTag = Tag.define();
+
 export const TaskTag = Tag.define();
 export const TaskMarkTag = Tag.define();
 export const TaskStateTag = Tag.define();
+export const TaskDeadlineTag = Tag.define();
+
+export const HashtagTag = Tag.define();
+export const NakedURLTag = Tag.define();

common/markdown_parser/markdown_ext.ts

@@ -1,72 +0,0 @@
-import { Tag } from "../deps.ts";
-import type { MarkdownConfig } from "../deps.ts";
-import { System } from "../../plugos/system.ts";
-import { Manifest, NodeDef } from "../manifest.ts";
-
-export type MDExt = {
-  // unicode char code for efficiency .charCodeAt(0)
-  firstCharCodes: number[];
-  regex: RegExp;
-  nodeType: string;
-  tag: Tag;
-  styles?: { [key: string]: string };
-  className?: string;
-};
-
-export function mdExtensionSyntaxConfig({
-  regex,
-  firstCharCodes,
-  nodeType,
-}: MDExt): MarkdownConfig {
-  return {
-    defineNodes: [nodeType],
-    parseInline: [
-      {
-        name: nodeType,
-        parse(cx, next, pos) {
-          if (!firstCharCodes.includes(next)) {
-            return -1;
-          }
-          const match = regex.exec(cx.slice(pos, cx.end));
-          if (!match) {
-            return -1;
-          }
-          return cx.addElement(cx.elt(nodeType, pos, pos + match[0].length));
-        },
-        // after: "Emphasis",
-      },
-    ],
-  };
-}
-
-export function mdExtensionStyleTags({ nodeType, tag }: MDExt): {
-  [selector: string]: Tag | readonly Tag[];
-} {
-  return {
-    [nodeType]: tag,
-  };
-}
-
-export function loadMarkdownExtensions(system: System<any>): MDExt[] {
-  const mdExtensions: MDExt[] = [];
-  for (const plug of system.loadedPlugs.values()) {
-    const manifest = plug.manifest as Manifest;
-    if (manifest.syntax) {
-      for (const [nodeType, def] of Object.entries(manifest.syntax)) {
-        mdExtensions.push(nodeDefToMDExt(nodeType, def));
-      }
-    }
-  }
-  return mdExtensions;
-}
-
-export function nodeDefToMDExt(nodeType: string, def: NodeDef): MDExt {
-  return {
-    nodeType,
-    tag: Tag.define(),
-    firstCharCodes: def.firstCharacters.map((ch) => ch.charCodeAt(0)),
-    regex: new RegExp("^" + def.regex),
-    styles: def.styles,
-    className: def.className,
-  };
-}

common/markdown_parser/parser.test.ts

@@ -1,11 +1,11 @@
 import { parse } from "./parse_tree.ts";
-import buildMarkdown from "./parser.ts";
 import {
   collectNodesOfType,
   findNodeOfType,
   renderToText,
 } from "../../plug-api/lib/tree.ts";
 import { assertEquals, assertNotEquals } from "../../test_deps.ts";
+import { extendedMarkdownLanguage } from "./parser.ts";
 
 const sample1 = `---
 type: page
@@ -26,8 +26,7 @@ name: Zef
 Supper`;
 
 Deno.test("Test parser", () => {
-  const lang = buildMarkdown([]);
-  let tree = parse(lang, sample1);
+  let tree = parse(extendedMarkdownLanguage, sample1);
   // console.log("tree", JSON.stringify(tree, null, 2));
   // Check if rendering back to text works
   assertEquals(renderToText(tree), sample1);
@@ -45,7 +44,7 @@ Deno.test("Test parser", () => {
   // Find frontmatter
   let node = findNodeOfType(tree, "FrontMatter");
   assertNotEquals(node, undefined);
-  tree = parse(lang, sampleInvalid1);
+  tree = parse(extendedMarkdownLanguage, sampleInvalid1);
   node = findNodeOfType(tree, "FrontMatter");
   // console.log("Invalid node", node);
   assertEquals(node, undefined);
@@ -62,8 +61,7 @@ And one with nested brackets: [array: [1, 2, 3]]
 `;
 
 Deno.test("Test inline attribute syntax", () => {
-  const lang = buildMarkdown([]);
-  const tree = parse(lang, inlineAttributeSample);
+  const tree = parse(extendedMarkdownLanguage, inlineAttributeSample);
   // console.log("Attribute parsed", JSON.stringify(tree, null, 2));
   const attributes = collectNodesOfType(tree, "Attribute");
   let nameNode = findNodeOfType(attributes[0], "AttributeName");
@@ -89,8 +87,7 @@ const multiStatusTaskExample = `
 `;
 
 Deno.test("Test multi-status tasks", () => {
-  const lang = buildMarkdown([]);
-  const tree = parse(lang, multiStatusTaskExample);
+  const tree = parse(extendedMarkdownLanguage, multiStatusTaskExample);
   // console.log("Tasks parsed", JSON.stringify(tree, null, 2));
   const tasks = collectNodesOfType(tree, "Task");
   assertEquals(tasks.length, 3);
@@ -107,8 +104,7 @@ const commandLinkSample = `
 `;
 
 Deno.test("Test command links", () => {
-  const lang = buildMarkdown([]);
-  const tree = parse(lang, commandLinkSample);
+  const tree = parse(extendedMarkdownLanguage, commandLinkSample);
   const commands = collectNodesOfType(tree, "CommandLink");
   console.log("Command links parsed", JSON.stringify(commands, null, 2));
   assertEquals(commands.length, 3);
@@ -125,8 +121,7 @@ const commandLinkArgsSample = `
 `;
 
 Deno.test("Test command link arguments", () => {
-  const lang = buildMarkdown([]);
-  const tree = parse(lang, commandLinkArgsSample);
+  const tree = parse(extendedMarkdownLanguage, commandLinkArgsSample);
   const commands = collectNodesOfType(tree, "CommandLink");
   assertEquals(commands.length, 2);
 
@@ -138,7 +133,6 @@ Deno.test("Test command link arguments", () => {
 });
 
 Deno.test("Test template directives", () => {
-  const lang = buildMarkdown([]);
-  const tree = parse(lang, `Hello there {{name}}!`);
+  const tree = parse(extendedMarkdownLanguage, `Hello there {{name}}!`);
   console.log("Template directive", JSON.stringify(tree, null, 2));
 });

common/markdown_parser/parser.ts

@@ -1,6 +1,5 @@
 import {
   BlockContext,
-  Language,
   LeafBlock,
   LeafBlockParser,
   Line,
@@ -9,16 +8,14 @@ import {
   StreamLanguage,
   Strikethrough,
   styleTags,
+  Tag,
   tags as t,
   yamlLanguage,
 } from "../deps.ts";
 import * as ct from "./customtags.ts";
+import { HashtagTag, TaskDeadlineTag } from "./customtags.ts";
+import { NakedURLTag } from "./customtags.ts";
 import { TaskList } from "./extended_task.ts";
-import {
-  MDExt,
-  mdExtensionStyleTags,
-  mdExtensionSyntaxConfig,
-} from "./markdown_ext.ts";
 
 export const pageLinkRegex = /^\[\[([^\]\|]+)(\|([^\]]+))?\]\]/;
 
@@ -313,6 +310,77 @@ export const Comment: MarkdownConfig = {
   ],
 };
 
+type RegexParserExtension = {
+  // unicode char code for efficiency .charCodeAt(0)
+  firstCharCode: number;
+  regex: RegExp;
+  nodeType: string;
+  tag: Tag;
+  className?: string;
+};
+
+function regexParser({
+  regex,
+  firstCharCode,
+  nodeType,
+}: RegexParserExtension): MarkdownConfig {
+  return {
+    defineNodes: [nodeType],
+    parseInline: [
+      {
+        name: nodeType,
+        parse(cx, next, pos) {
+          if (firstCharCode !== next) {
+            return -1;
+          }
+          const match = regex.exec(cx.slice(pos, cx.end));
+          if (!match) {
+            return -1;
+          }
+          return cx.addElement(cx.elt(nodeType, pos, pos + match[0].length));
+        },
+      },
+    ],
+  };
+}
+
+const NakedURL = regexParser(
+  {
+    firstCharCode: 104, // h
+    regex:
+      /^https?:\/\/[-a-zA-Z0-9@:%._\+~#=]{1,256}([-a-zA-Z0-9()@:%_\+.~#?&=\/]*)/,
+    nodeType: "NakedURL",
+    className: "sb-naked-url",
+    tag: NakedURLTag,
+  },
+);
+
+const Hashtag = regexParser(
+  {
+    firstCharCode: 35, // #
+    regex: /^#[^#\d\s\[\]]+\w+/,
+    nodeType: "Hashtag",
+    className: "sb-hashtag",
+    tag: ct.HashtagTag,
+  },
+);
+
+const TaskDeadline = regexParser({
+  firstCharCode: 55357, // 📅
+  regex: /^📅\s*\d{4}\-\d{2}\-\d{2}/,
+  className: "sb-task-deadline",
+  nodeType: "DeadlineDate",
+  tag: ct.TaskDeadlineTag,
+});
+
+const NamedAnchor = regexParser({
+  firstCharCode: 36, // $
+  regex: /^\$[a-zA-Z\.\-\/]+[\w\.\-\/]*/,
+  className: "sb-named-anchor",
+  nodeType: "NamedAnchor",
+  tag: ct.NamedAnchorTag,
+});
+
 import { Table } from "./table_parser.ts";
 import { foldNodeProp } from "@codemirror/language";
 
@@ -379,54 +447,56 @@ export const FrontMatter: MarkdownConfig = {
   }],
 };
 
-export default function buildMarkdown(mdExtensions: MDExt[]): Language {
-  return markdown({
-    extensions: [
-      WikiLink,
-      CommandLink,
-      Attribute,
-      FrontMatter,
-      TaskList,
-      Comment,
-      Highlight,
-      TemplateDirective,
-      Strikethrough,
-      Table,
-      ...mdExtensions.map(mdExtensionSyntaxConfig),
-      {
-        props: [
-          foldNodeProp.add({
-            // Don't fold at the list level
-            BulletList: () => null,
-            OrderedList: () => null,
-            // Fold list items
-            ListItem: (tree, state) => ({
-              from: state.doc.lineAt(tree.from).to,
-              to: tree.to,
-            }),
-            // Fold frontmatter
-            FrontMatter: (tree) => ({
-              from: tree.from,
-              to: tree.to,
-            }),
+export const extendedMarkdownLanguage = markdown({
+  extensions: [
+    WikiLink,
+    CommandLink,
+    Attribute,
+    FrontMatter,
+    TaskList,
+    Comment,
+    Highlight,
+    TemplateDirective,
+    Strikethrough,
+    Table,
+    NakedURL,
+    Hashtag,
+    TaskDeadline,
+    NamedAnchor,
+    {
+      props: [
+        foldNodeProp.add({
+          // Don't fold at the list level
+          BulletList: () => null,
+          OrderedList: () => null,
+          // Fold list items
+          ListItem: (tree, state) => ({
+            from: state.doc.lineAt(tree.from).to,
+            to: tree.to,
           }),
+          // Fold frontmatter
+          FrontMatter: (tree) => ({
+            from: tree.from,
+            to: tree.to,
+          }),
+        }),
 
-          styleTags({
-            Task: ct.TaskTag,
-            TaskMark: ct.TaskMarkTag,
-            Comment: ct.CommentTag,
-            "TableDelimiter SubscriptMark SuperscriptMark StrikethroughMark":
-              t.processingInstruction,
-            "TableHeader/...": t.heading,
-            TableCell: t.content,
-            CodeInfo: ct.CodeInfoTag,
-            HorizontalRule: ct.HorizontalRuleTag,
-          }),
-          ...mdExtensions.map((mdExt) =>
-            styleTags(mdExtensionStyleTags(mdExt))
-          ),
-        ],
-      },
-    ],
-  }).language;
-}
+        styleTags({
+          Task: ct.TaskTag,
+          TaskMark: ct.TaskMarkTag,
+          Comment: ct.CommentTag,
+          "TableDelimiter SubscriptMark SuperscriptMark StrikethroughMark":
+            t.processingInstruction,
+          "TableHeader/...": t.heading,
+          TableCell: t.content,
+          CodeInfo: ct.CodeInfoTag,
+          HorizontalRule: ct.HorizontalRuleTag,
+          Hashtag: ct.HashtagTag,
+          NakedURL: ct.NakedURLTag,
+          DeadlineDate: ct.TaskDeadlineTag,
+          NamedAnchor: ct.NamedAnchorTag,
+        }),
+      ],
+    },
+  ],
+}).language;

common/syscalls/markdown.ts

@@ -1,12 +1,12 @@
 import { SysCallMapping } from "../../plugos/system.ts";
 import { parse } from "../markdown_parser/parse_tree.ts";
-import { Language } from "../../web/deps.ts";
 import type { ParseTree } from "$sb/lib/tree.ts";
+import { extendedMarkdownLanguage } from "../markdown_parser/parser.ts";
 
-export function markdownSyscalls(lang: Language): SysCallMapping {
+export function markdownSyscalls(): SysCallMapping {
   return {
     "markdown.parseMarkdown": (_ctx, text: string): ParseTree => {
-      return parse(lang, text);
+      return parse(extendedMarkdownLanguage, text);
     },
   };
 }

plug-api/lib/attribute.test.ts

@@ -1,9 +1,9 @@
 import "$sb/lib/syscall_mock.ts";
 import { parse } from "../../common/markdown_parser/parse_tree.ts";
-import buildMarkdown from "../../common/markdown_parser/parser.ts";
 import { extractAttributes } from "$sb/lib/attribute.ts";
 import { assertEquals } from "../../test_deps.ts";
 import { renderToText } from "$sb/lib/tree.ts";
+import { extendedMarkdownLanguage } from "../../common/markdown_parser/parser.ts";
 
 const inlineAttributeSample = `
 # My document
@@ -26,8 +26,7 @@ Top level attributes:
 `;
 
 Deno.test("Test attribute extraction", async () => {
-  const lang = buildMarkdown([]);
-  const tree = parse(lang, inlineAttributeSample);
+  const tree = parse(extendedMarkdownLanguage, inlineAttributeSample);
   const toplevelAttributes = await extractAttributes(tree, false);
   // console.log("All attributes", toplevelAttributes);
   assertEquals(toplevelAttributes.name, "sup");

plug-api/lib/feed.test.ts

@@ -1,9 +1,8 @@
 import "$sb/lib/syscall_mock.ts";
 import { parse } from "../../common/markdown_parser/parse_tree.ts";
-import buildMarkdown from "../../common/markdown_parser/parser.ts";
 import { assertEquals } from "../../test_deps.ts";
 import { extractFeedItems } from "$sb/lib/feed.ts";
-import { nodeDefToMDExt } from "../../common/markdown_parser/markdown_ext.ts";
+import { extendedMarkdownLanguage } from "../../common/markdown_parser/parser.ts";
 
 const feedSample1 = `---
 test: ignore me
@@ -25,11 +24,7 @@ Completely free form
 
 Deno.test("Test feed parsing", async () => {
   // Ad hoc added the NamedAnchor extension from the core plug-in inline here
-  const lang = buildMarkdown([nodeDefToMDExt("NamedAnchor", {
-    firstCharacters: ["$"],
-    regex: "\\$[a-zA-Z\\.\\-\\/]+[\\w\\.\\-\\/]*",
-  })]);
-  const tree = parse(lang, feedSample1);
+  const tree = parse(extendedMarkdownLanguage, feedSample1);
   const items = await extractFeedItems(tree);
   assertEquals(items.length, 3);
   assertEquals(items[0], {

plug-api/lib/test_utils.ts

@@ -1,8 +1,7 @@
-import wikiMarkdownLang from "../../common/markdown_parser/parser.ts";
 import type { ParseTree } from "$sb/lib/tree.ts";
 import { parse } from "../../common/markdown_parser/parse_tree.ts";
+import { extendedMarkdownLanguage } from "../../common/markdown_parser/parser.ts";
 
 export function parseMarkdown(text: string): ParseTree {
-  const lang = wikiMarkdownLang([]);
-  return parse(lang, text);
+  return parse(extendedMarkdownLanguage, text);
 }

plug-api/lib/tree.test.ts

@@ -9,9 +9,9 @@ import {
   renderToText,
   replaceNodesMatching,
 } from "./tree.ts";
-import wikiMarkdownLang from "../../common/markdown_parser/parser.ts";
 import { assertEquals, assertNotEquals } from "../../test_deps.ts";
 import { parse } from "../../common/markdown_parser/parse_tree.ts";
+import { extendedMarkdownLanguage } from "../../common/markdown_parser/parser.ts";
 
 const mdTest1 = `
 # Heading
@@ -49,8 +49,7 @@ name: something
 `;
 
 Deno.test("Test parsing", () => {
-  const lang = wikiMarkdownLang([]);
-  const mdTree = parse(lang, mdTest1);
+  const mdTree = parse(extendedMarkdownLanguage, mdTest1);
   addParentPointers(mdTree);
   // console.log(JSON.stringify(mdTree, null, 2));
   const wikiLink = nodeAtPos(mdTree, mdTest1.indexOf("Wiki Page"))!;
@@ -75,12 +74,11 @@ Deno.test("Test parsing", () => {
     }
   });
   // console.log(JSON.stringify(mdTree, null, 2));
-  let mdTree3 = parse(lang, mdTest3);
+  let mdTree3 = parse(extendedMarkdownLanguage, mdTest3);
   // console.log(JSON.stringify(mdTree3, null, 2));
 });
 
 Deno.test("AST functions", () => {
-  const lang = wikiMarkdownLang([]);
-  const mdTree = parse(lang, mdTest1);
+  const mdTree = parse(extendedMarkdownLanguage, mdTest1);
   console.log(JSON.stringify(parseTreeToAST(mdTree), null, 2));
 });

plugs/editor/editor.plug.yaml

@@ -1,10 +1,4 @@
 name: editor
-syntax:
-  NakedURL:
-    firstCharacters:
-      - "h"
-    regex: "https?:\\/\\/[-a-zA-Z0-9@:%._\\+~#=]{1,256}([-a-zA-Z0-9()@:%_\\+.~#?&=\\/]*)"
-    className: sb-naked-url
 functions:
   setEditorMode:
     path: "./editor.ts:setEditorMode"

plugs/index/index.plug.yaml

@@ -1,15 +1,4 @@
 name: index
-syntax:
-  Hashtag:
-    firstCharacters:
-      - "#"
-    regex: "#[^#\\d\\s\\[\\]]+\\w+"
-    className: sb-hashtag
-  NamedAnchor:
-    firstCharacters:
-      - "$"
-    regex: "\\$[a-zA-Z\\.\\-\\/]+[\\w\\.\\-\\/]*"
-    className: sb-named-anchor
 functions:
   loadBuiltinsIntoIndex:
     path: builtins.ts:loadBuiltinsIntoIndex

plugs/markdown/markdown_render.test.ts

@@ -1,10 +1,9 @@
-import buildMarkdown from "../../common/markdown_parser/parser.ts";
 import { parse } from "../../common/markdown_parser/parse_tree.ts";
 import { System } from "../../plugos/system.ts";
 
 import { createSandbox } from "../../plugos/sandboxes/deno_worker_sandbox.ts";
-import { loadMarkdownExtensions } from "../../common/markdown_parser/markdown_ext.ts";
 import { renderMarkdownToHtml } from "./markdown_render.ts";
+import { extendedMarkdownLanguage } from "../../common/markdown_parser/parser.ts";
 
 Deno.test("Markdown render", async () => {
   const system = new System<any>("server");
@@ -20,11 +19,10 @@ Deno.test("Markdown render", async () => {
       new URL("../../dist_plug_bundle/_plug/tasks.plug.js", import.meta.url),
     ),
   );
-  const lang = buildMarkdown(loadMarkdownExtensions(system));
   const testFile = Deno.readTextFileSync(
     new URL("test/example.md", import.meta.url).pathname,
   );
-  const tree = parse(lang, testFile);
+  const tree = parse(extendedMarkdownLanguage, testFile);
   renderMarkdownToHtml(tree, {
     failOnUnknown: true,
   });
@@ -35,8 +33,7 @@ Deno.test("Markdown render", async () => {
 Deno.test("Smart hard break test", () => {
   const example = `**Hello**
 *world!*`;
-  const lang = buildMarkdown([]);
-  const tree = parse(lang, example);
+  const tree = parse(extendedMarkdownLanguage, example);
   const html = renderMarkdownToHtml(tree, {
     failOnUnknown: true,
     smartHardBreak: true,
@@ -58,7 +55,7 @@ And another
   Server: something else
   📅 last_updated - [Release notes](release_notes_url)`;
 
-  const tree2 = parse(lang, example2);
+  const tree2 = parse(extendedMarkdownLanguage, example2);
   const html2 = renderMarkdownToHtml(tree2, {
     failOnUnknown: true,
     smartHardBreak: true,

plugs/tasks/tasks.plug.yaml

@@ -1,23 +1,4 @@
 name: tasks
-syntax:
-  DeadlineDate:
-    firstCharacters:
-      - "📅"
-    regex: "📅\\s*\\d{4}\\-\\d{2}\\-\\d{2}"
-    styles:
-      backgroundColor: "rgba(22,22,22,0.07)"
-  CompletedDate:
-    firstCharacters:
-      - "✅"
-    regex: "✅\\s*\\d{4}\\-\\d{2}\\-\\d{2}"
-    styles:
-      backgroundColor: "rgba(22,22,22,0.07)"
-  RepeatInterval:
-    firstCharacters:
-      - "🔁"
-    regex: "🔁\\s*every\\s+\\w+"
-    styles:
-      backgroundColor: "rgba(22,22,22,0.07)"
 functions:
   # API
   updateTaskState:

server/server_system.ts

@@ -1,7 +1,5 @@
 import { PlugNamespaceHook } from "../common/hooks/plug_namespace.ts";
 import { SilverBulletHooks } from "../common/manifest.ts";
-import { loadMarkdownExtensions } from "../common/markdown_parser/markdown_ext.ts";
-import buildMarkdown from "../common/markdown_parser/parser.ts";
 import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitives.ts";
 import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts";
 import { createSandbox } from "../plugos/sandboxes/web_worker_sandbox.ts";
@@ -135,7 +133,7 @@ export class ServerSystem {
       dataStoreSyscalls(this.ds),
       debugSyscalls(),
       codeWidgetSyscalls(codeWidgetHook),
-      markdownSyscalls(buildMarkdown([])), // Will later be replaced with markdown extensions
+      markdownSyscalls(),
     );
 
     // Syscalls that require some additional permissions
@@ -151,12 +149,6 @@ export class ServerSystem {
 
     await this.loadPlugs();
 
-    // Load markdown syscalls based on all new syntax (if any)
-    this.system.registerSyscalls(
-      [],
-      markdownSyscalls(buildMarkdown(loadMarkdownExtensions(this.system))),
-    );
-
     this.listInterval = setInterval(() => {
       // runWithSystemLock(this.system, async () => {
       //   await space.updatePageList();

web/client.ts

@@ -94,17 +94,6 @@ export class Client {
       .catch((e) => console.error("Error dispatching editor:updated event", e));
   }, 1000);
 
-  debouncedPlugsUpdatedEvent = throttle(async () => {
-    // To register new commands, update editor state based on new plugs
-    this.rebuildEditorState();
-    await this.dispatchAppEvent(
-      "editor:pageLoaded",
-      this.currentPage,
-      undefined,
-      true,
-    );
-  }, 1000);
-
   // Track if plugs have been updated since sync cycle
   fullSyncCompleted = false;
 
@@ -195,7 +184,7 @@ export class Client {
 
     this.focus();
 
-    await this.system.init();
+    this.system.init();
 
     await this.loadSettings();
 
@@ -773,7 +762,6 @@ export class Client {
 
   async loadPlugs() {
     await this.system.reloadPlugsFromSpace(this.space);
-    this.rebuildEditorState();
     await this.eventHook.dispatchEvent("system:ready");
     await this.dispatchAppEvent("plugs:loaded");
   }
@@ -782,12 +770,7 @@ export class Client {
     const editorView = this.editorView;
     console.log("Rebuilding editor state");
 
-    this.system.updateMarkdownParser();
-
     if (this.currentPage) {
-      // And update the editor if a page is loaded
-      // this.openPages.saveState(this.currentPage);
-
       editorView.setState(
         createEditorState(
           this,
@@ -801,8 +784,6 @@ export class Client {
           editorView.contentDOM,
         );
       }
-
-      // this.openPages.restoreState(this.currentPage);
     }
   }
 
@@ -930,8 +911,6 @@ export class Client {
     const editorView = this.editorView;
     const previousPage = this.currentPage;
 
-    // console.log("Navigating to", pageName, restoreState);
-
     // Persist current page state and nicely close page
     if (previousPage) {
       // this.openPages.saveState(previousPage);

web/client_system.ts

@@ -1,6 +1,5 @@
 import { PlugNamespaceHook } from "../common/hooks/plug_namespace.ts";
-import { Manifest, SilverBulletHooks } from "../common/manifest.ts";
-import buildMarkdown from "../common/markdown_parser/parser.ts";
+import { SilverBulletHooks } from "../common/manifest.ts";
 import { CronHook } from "../plugos/hooks/cron.ts";
 import { EventHook } from "../plugos/hooks/event.ts";
 import { createSandbox } from "../plugos/sandboxes/web_worker_sandbox.ts";
@@ -23,10 +22,6 @@ import { syncSyscalls } from "./syscalls/sync.ts";
 import { systemSyscalls } from "./syscalls/system.ts";
 import { yamlSyscalls } from "../common/syscalls/yaml.ts";
 import { Space } from "./space.ts";
-import {
-  loadMarkdownExtensions,
-  MDExt,
-} from "../common/markdown_parser/markdown_ext.ts";
 import { MQHook } from "../plugos/hooks/mq.ts";
 import { mqSyscalls } from "../plugos/syscalls/mq.ts";
 import { mqProxySyscalls } from "./syscalls/mq.proxy.ts";
@@ -51,7 +46,6 @@ export class ClientSystem {
   slashCommandHook: SlashCommandHook;
   namespaceHook: PlugNamespaceHook;
   codeWidgetHook: CodeWidgetHook;
-  mdExtensions: MDExt[] = [];
   system: System<SilverBulletHooks>;
   panelWidgetHook: PanelWidgetHook;
 
@@ -139,22 +133,17 @@ export class ClientSystem {
           const plugName = plugNameExtractRegex.exec(path)![1];
           console.log("Plug updated, reloading", plugName, "from", path);
           this.system.unload(path);
-          const plug = await this.system.load(
+          await this.system.load(
             plugName,
             createSandbox(new URL(`/${path}`, location.href)),
             newHash,
           );
-          if ((plug.manifest! as Manifest).syntax) {
-            // If there are syntax extensions, rebuild the markdown parser immediately
-            this.updateMarkdownParser();
-          }
-          this.client.debouncedPlugsUpdatedEvent();
         }
       },
     );
   }
 
-  async init() {
+  init() {
     // Slash command hook
     this.slashCommandHook = new SlashCommandHook(this.client);
     this.system.addHook(this.slashCommandHook);
@@ -166,7 +155,7 @@ export class ClientSystem {
       editorSyscalls(this.client),
       spaceSyscalls(this.client),
       systemSyscalls(this.system, this.client),
-      markdownSyscalls(buildMarkdown(this.mdExtensions)),
+      markdownSyscalls(),
       assetSyscalls(this.system),
       yamlSyscalls(),
       handlebarsSyscalls(),
@@ -222,16 +211,6 @@ export class ClientSystem {
     }));
   }
 
-  updateMarkdownParser() {
-    // Load all syntax extensions
-    this.mdExtensions = loadMarkdownExtensions(this.system);
-    // And reload the syscalls to use the new syntax extensions
-    this.system.registerSyscalls(
-      [],
-      markdownSyscalls(buildMarkdown(this.mdExtensions)),
-    );
-  }
-
   localSyscall(name: string, args: any[]) {
     return this.system.localSyscall(name, args);
   }

web/cm_plugins/lint.ts

@@ -1,13 +1,13 @@
 import { Diagnostic, linter } from "@codemirror/lint";
 import type { Client } from "../client.ts";
 import { parse } from "../../common/markdown_parser/parse_tree.ts";
-import buildMarkdown from "../../common/markdown_parser/parser.ts";
 import { LintEvent } from "$sb/app_event.ts";
+import { extendedMarkdownLanguage } from "../../common/markdown_parser/parser.ts";
 
 export function plugLinter(client: Client) {
   return linter(async (view): Promise<Diagnostic[]> => {
     const tree = parse(
-      buildMarkdown(client.system.mdExtensions),
+      extendedMarkdownLanguage,
       view.state.sliceDoc(),
     );
     const results = (await client.dispatchAppEvent("editor:lint", {

web/cm_plugins/markdown_widget.ts

@@ -4,8 +4,8 @@ import type { CodeWidgetButton, CodeWidgetCallback } from "$sb/types.ts";
 import { renderMarkdownToHtml } from "../../plugs/markdown/markdown_render.ts";
 import { resolveAttachmentPath } from "$sb/lib/resolve.ts";
 import { parse } from "../../common/markdown_parser/parse_tree.ts";
-import buildMarkdown from "../../common/markdown_parser/parser.ts";
 import { parsePageRef } from "$sb/lib/page.ts";
+import { extendedMarkdownLanguage } from "../../common/markdown_parser/parser.ts";
 
 const activeWidgets = new Set<MarkdownWidget>();
 
@@ -59,9 +59,8 @@ export class MarkdownWidget extends WidgetType {
       );
       return;
     }
-    const lang = buildMarkdown(this.client.system.mdExtensions);
     let mdTree = parse(
-      lang,
+      extendedMarkdownLanguage,
       widgetContent.markdown!,
     );
     mdTree = await this.client.system.localSyscall(

web/components/page_navigator.tsx

@@ -4,7 +4,7 @@ import { CompletionContext, CompletionResult } from "../deps.ts";
 import { PageMeta } from "$sb/types.ts";
 import { isFederationPath } from "$sb/lib/resolve.ts";
 
-const tagRegex = /#[^#\d\s\[\]]+\w+/g;
+export const tagRegex = /#[^#\d\s\[\]]+\w+/g;
 
 export function PageNavigator({
   allPages,

web/editor_state.ts

@@ -1,6 +1,4 @@
-import buildMarkdown, {
-  commandLinkRegex,
-} from "../common/markdown_parser/parser.ts";
+import { commandLinkRegex } from "../common/markdown_parser/parser.ts";
 import { readonlyMode } from "./cm_plugins/readonly.ts";
 import customMarkdownStyle from "./style.ts";
 import {
@@ -46,6 +44,7 @@ import { postScriptPrefacePlugin } from "./cm_plugins/top_bottom_panels.ts";
 import { languageFor } from "../common/languages.ts";
 import { plugLinter } from "./cm_plugins/lint.ts";
 import { Compartment, Extension } from "@codemirror/state";
+import { extendedMarkdownLanguage } from "../common/markdown_parser/parser.ts";
 
 export function createEditorState(
   client: Client,
@@ -55,8 +54,6 @@ export function createEditorState(
 ): EditorState {
   let touchCount = 0;
 
-  const markdownLanguage = buildMarkdown(client.system.mdExtensions);
-
   // Ugly: keep the keyhandler compartment in the client, to be replaced later once more commands are loaded
   client.keyHandlerCompartment = new Compartment();
   const keyBindings = client.keyHandlerCompartment.of(
@@ -82,7 +79,7 @@ export function createEditorState(
 
       // The uber markdown mode
       markdown({
-        base: markdownLanguage,
+        base: extendedMarkdownLanguage,
         codeLanguages: (info) => {
           const lang = languageFor(info);
           if (lang) {
@@ -96,10 +93,10 @@ export function createEditorState(
         },
         addKeymap: true,
       }),
-      markdownLanguage.data.of({
+      extendedMarkdownLanguage.data.of({
         closeBrackets: { brackets: ["(", "{", "[", "`"] },
       }),
-      syntaxHighlighting(customMarkdownStyle(client.system.mdExtensions)),
+      syntaxHighlighting(customMarkdownStyle()),
       autocompletion({
         override: [
           client.editorComplete.bind(client),

web/style.ts

@@ -1,9 +1,8 @@
 import { HighlightStyle } from "../common/deps.ts";
 import { tagHighlighter, tags as t } from "./deps.ts";
 import * as ct from "../common/markdown_parser/customtags.ts";
-import { MDExt } from "../common/markdown_parser/markdown_ext.ts";
 
-export default function highlightStyles(mdExtension: MDExt[]) {
+export default function highlightStyles() {
   tagHighlighter;
   return HighlightStyle.define([
     { tag: t.heading1, class: "sb-h1" },
@@ -49,8 +48,9 @@ export default function highlightStyles(mdExtension: MDExt[]) {
     { tag: t.processingInstruction, class: "sb-meta" },
     { tag: t.punctuation, class: "sb-punctuation" },
     { tag: ct.HorizontalRuleTag, class: "sb-hr" },
-    ...mdExtension.map((mdExt) => {
-      return { tag: mdExt.tag, ...mdExt.styles, class: mdExt.className };
-    }),
+    { tag: ct.HashtagTag, class: "sb-hashtag" },
+    { tag: ct.NakedURLTag, class: "sb-naked-url" },
+    { tag: ct.TaskDeadlineTag, class: "sb-task-deadline" },
+    { tag: ct.NamedAnchorTag, class: "sb-named-anchor" },
   ]);
 }

web/styles/editor.scss

@@ -317,6 +317,10 @@
     font-size: 91%;
   }
 
+  .sb-task-deadline {
+    background-color: rgba(22, 22, 22, 0.07)
+  }
+
   .sb-line-frontmatter-outside,
   .sb-line-code-outside {
     .sb-meta {

website/CHANGELOG.md

@@ -6,7 +6,9 @@ release.
 ## Edge
 _The changes below are not yet released “properly”. To them out early, check out [the docs on edge](https://community.silverbullet.md/t/living-on-the-edge-builds/27)._
 
-* Nothing new yet, be patient, but check out 0.6.0 below.
+* Internal changes:
+  * Big refactor: of navigation and browser history, fixed some {[Page: Rename]} bugs along the way
+  * Plugs now can no longer define their own markdown syntax, migrated all plug-specific syntax into the main parser. This should remove a bunch of editor “flashing” especially during sync.
 
 ---