Reapplied all the things

.idea/.gitignore

@@ -0,0 +1,8 @@
+# Default ignored files
+/shelf/
+/workspace.xml
+# Editor-based HTTP Client requests
+/httpRequests/
+# Datasource local storage ignored files
+/dataSources/
+/dataSources.local.xml

.idea/modules.xml

@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/.idea/silverbullet.iml" filepath="$PROJECT_DIR$/.idea/silverbullet.iml" />
+    </modules>
+  </component>
+</project>

.idea/silverbullet.iml

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module type="WEB_MODULE" version="4">
+  <component name="Go" enabled="true" />
+  <component name="NewModuleRootManager">
+    <content url="file://$MODULE_DIR$" />
+    <orderEntry type="inheritedJdk" />
+    <orderEntry type="sourceFolder" forTests="false" />
+  </component>
+</module>

.idea/vcs.xml

@@ -0,0 +1,6 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="VcsDirectoryMappings">
+    <mapping directory="$PROJECT_DIR$" vcs="Git" />
+  </component>
+</project>

common/syscalls/markdown.ts

@@ -0,0 +1,16 @@
+import {SysCallMapping} from "../../plugos/system";
+import {MarkdownTree, nodeAtPos, parse, render} from "../tree";
+
+export function markdownSyscalls(): SysCallMapping {
+    return {
+        parse(ctx, text: string): MarkdownTree {
+            return parse(text);
+        },
+        nodeAtPos(ctx, mdTree: MarkdownTree, pos: number): MarkdownTree | null {
+            return nodeAtPos(mdTree, pos);
+        },
+        render(ctx, mdTree: MarkdownTree): string {
+            return render(mdTree);
+        },
+    };
+}

common/tree.test.ts

@@ -0,0 +1,26 @@
+import {expect, test} from "@jest/globals";
+import {nodeAtPos, parse, render} from "./tree";
+
+const mdTest1 = `
+# Heading
+## Sub _heading_ cool
+
+Hello, this is some **bold** text and *italic*. And [a link](http://zef.me).
+
+- This is a list
+- With another item
+- TODOs:
+  - [ ] A task that's not yet done
+  - [x] Hello
+- And a _third_ one [[Wiki Page]] yo
+`;
+
+test("Run a Node sandbox", async () => {
+  let mdTree = parse(mdTest1);
+  console.log(JSON.stringify(mdTree, null, 2));
+  expect(nodeAtPos(mdTree, 4)!.type).toBe("ATXHeading1");
+  expect(nodeAtPos(mdTree, mdTest1.indexOf("Wiki Page"))!.type).toBe(
+    "WikiLink"
+  );
+  expect(render(mdTree)).toBe(mdTest1);
+});

common/tree.ts

@@ -0,0 +1,115 @@
+import {SyntaxNode} from "@lezer/common";
+import wikiMarkdownLang from "../webapp/parser";
+
+export type MarkdownTree = {
+    type?: string; // undefined === text node
+    from: number;
+    to: number;
+    text?: string;
+    children?: MarkdownTree[];
+    parent?: MarkdownTree;
+};
+
+function treeToAST(text: string, n: SyntaxNode): MarkdownTree {
+    let children: MarkdownTree[] = [];
+    let nodeText: string | undefined;
+    let child = n.firstChild;
+    while (child) {
+        children.push(treeToAST(text, child));
+        child = child.nextSibling;
+    }
+
+    if (children.length === 0) {
+        children = [
+            {
+                from: n.from,
+                to: n.to,
+                text: text.substring(n.from, n.to),
+            },
+        ];
+    } else {
+        let newChildren: MarkdownTree[] | string = [];
+        let index = n.from;
+        for (let child of children) {
+            let s = text.substring(index, child.from);
+            if (s) {
+                newChildren.push({
+                    from: index,
+                    to: child.from,
+                    text: s,
+                });
+            }
+            newChildren.push(child);
+            index = child.to;
+        }
+        let s = text.substring(index, n.to);
+        if (s) {
+            newChildren.push({ from: index, to: n.to, text: s });
+        }
+        children = newChildren;
+    }
+
+    let result: MarkdownTree = {
+        type: n.name,
+        from: n.from,
+        to: n.to,
+    };
+    if (children.length > 0) {
+        result.children = children;
+    }
+    if (nodeText) {
+        result.text = nodeText;
+    }
+    return result;
+}
+
+// Currently unused
+function addParentPointers(mdTree: MarkdownTree) {
+    if (!mdTree.children) {
+        return;
+    }
+    for (let child of mdTree.children) {
+        child.parent = mdTree;
+        addParentPointers(child);
+    }
+}
+
+// Finds non-text node at position
+export function nodeAtPos(
+    mdTree: MarkdownTree,
+    pos: number
+): MarkdownTree | null {
+    if (pos < mdTree.from || pos > mdTree.to) {
+        return null;
+    }
+    if (!mdTree.children) {
+        return mdTree;
+    }
+    for (let child of mdTree.children) {
+        let n = nodeAtPos(child, pos);
+        if (n && n.text) {
+            // Got a text node, let's return its parent
+            return mdTree;
+        } else if (n) {
+            // Got it
+            return n;
+        }
+    }
+    return null;
+}
+
+// Turn MarkdownTree back into regular markdown text
+export function render(mdTree: MarkdownTree): string {
+    let pieces: string[] = [];
+    if (mdTree.text) {
+        return mdTree.text;
+    }
+    for (let child of mdTree.children!) {
+        pieces.push(render(child));
+    }
+    return pieces.join("");
+}
+
+export function parse(text: string): MarkdownTree {
+    return treeToAST(text, wikiMarkdownLang.parser.parse(text).topNode);
+}

package.json

@@ -35,7 +35,7 @@
       "context": "node"
     },
     "test": {
-      "source": [],
+      "source": ["common/tree.test.ts"],
       "outputFormat": "commonjs",
       "isLibrary": true,
       "context": "node"
@@ -54,6 +54,9 @@
     "@fortawesome/fontawesome-svg-core": "1.3.0",
     "@fortawesome/free-solid-svg-icons": "6.0.0",
     "@fortawesome/react-fontawesome": "0.1.17",
+    "@codemirror/highlight": "^0.19.0",
+    "@codemirror/language": "^0.19.0",
+    "@lezer/markdown": "^0.15.0",
     "@jest/globals": "^27.5.1",
     "better-sqlite3": "^7.5.0",
     "body-parser": "^1.19.2",

plugos-silverbullet-syscall/markdown.ts

@@ -0,0 +1,17 @@
+import {syscall} from "./syscall";
+import type {MarkdownTree} from "../common/tree";
+
+export async function parse(text: string): Promise<MarkdownTree> {
+  return syscall("markdown.parse", text);
+}
+
+export async function nodeAtPos(
+  mdTree: MarkdownTree,
+  pos: number
+): Promise<any | null> {
+  return syscall("markdown.nodeAtPos", mdTree, pos);
+}
+
+export async function render(mdTree: MarkdownTree): Promise<string> {
+  return syscall("markdown.render", mdTree);
+}

plugs/core/materialized_queries.ts

@@ -1,16 +1,11 @@
-import {
-  flashNotification,
-  getCurrentPage,
-  reloadPage,
-  save,
-} from "plugos-silverbullet-syscall/editor";
+import {flashNotification, getCurrentPage, reloadPage, save,} from "plugos-silverbullet-syscall/editor";
 
-import { readPage, writePage } from "plugos-silverbullet-syscall/space";
-import { invokeFunctionOnServer } from "plugos-silverbullet-syscall/system";
-import { scanPrefixGlobal } from "plugos-silverbullet-syscall";
+import {readPage, writePage} from "plugos-silverbullet-syscall/space";
+import {invokeFunctionOnServer} from "plugos-silverbullet-syscall/system";
+import {scanPrefixGlobal} from "plugos-silverbullet-syscall";
 
 export const queryRegex =
-  /(<!--\s*#query\s+(?<table>\w+)\s*(filter\s+["'“”‘’](?<filter>[^"'“”‘’]+)["'“”‘’])?\s*-->)(.+?)(<!--\s*#end\s*-->)/gs;
+  /(<!--\s*#query\s+(?<table>\w+)\s*(filter\s+["'“”‘’](?<filter>[^"'“”‘’]+)["'“”‘’])?\s*(group by\s+(?<groupBy>\w+))?\s*-->)(.+?)(<!--\s*#end\s*-->)/gs;
 
 export function whiteOutQueries(text: string): string {
   return text.replaceAll(queryRegex, (match) =>

plugs/core/navigate.ts

@@ -1,12 +1,14 @@
-import { ClickEvent } from "../../webapp/app_event";
-import { updateMaterializedQueriesCommand } from "./materialized_queries";
+import {ClickEvent} from "../../webapp/app_event";
+import {updateMaterializedQueriesCommand} from "./materialized_queries";
 import {
     getSyntaxNodeAtPos,
     getSyntaxNodeUnderCursor,
+    getText,
     navigate as navigateTo,
     openUrl,
 } from "plugos-silverbullet-syscall/editor";
-import { taskToggleAtPos } from "../tasks/task";
+import {taskToggleAtPos} from "../tasks/task";
+import {nodeAtPos, parse} from "plugos-silverbullet-syscall/markdown";
 
 const materializedQueryPrefix = /<!--\s*#query\s+/;
 
@@ -52,6 +54,9 @@ export async function linkNavigate() {
 export async function clickNavigate(event: ClickEvent) {
   if (event.ctrlKey || event.metaKey) {
     let syntaxNode = await getSyntaxNodeAtPos(event.pos);
+    let mdTree = await parse(await getText());
+    let newNode = await nodeAtPos(mdTree, event.pos);
+    console.log("New node", newNode);
     await actionClickOrActionEnter(syntaxNode);
   }
 }

server/api_server.ts

@@ -1,19 +1,20 @@
-import express, { Express } from "express";
-import { SilverBulletHooks } from "../common/manifest";
-import { EndpointHook } from "../plugos/hooks/endpoint";
-import { readFile } from "fs/promises";
-import { System } from "../plugos/system";
+import express, {Express} from "express";
+import {SilverBulletHooks} from "../common/manifest";
+import {EndpointHook} from "../plugos/hooks/endpoint";
+import {readFile} from "fs/promises";
+import {System} from "../plugos/system";
 import cors from "cors";
-import { DiskStorage, EventedStorage, Storage } from "./disk_storage";
+import {DiskStorage, EventedStorage, Storage} from "./disk_storage";
 import path from "path";
 import bodyParser from "body-parser";
-import { EventHook } from "../plugos/hooks/event";
+import {EventHook} from "../plugos/hooks/event";
 import spaceSyscalls from "./syscalls/space";
-import { eventSyscalls } from "../plugos/syscalls/event";
-import { pageIndexSyscalls } from "./syscalls";
-import knex, { Knex } from "knex";
+import {eventSyscalls} from "../plugos/syscalls/event";
+import {pageIndexSyscalls} from "./syscalls";
+import knex, {Knex} from "knex";
 import shellSyscalls from "../plugos/syscalls/shell.node";
-import { NodeCronHook } from "../plugos/hooks/node_cron";
+import {NodeCronHook} from "../plugos/hooks/node_cron";
+import {markdownSyscalls} from "../common/syscalls/markdown";
 
 export class ExpressServer {
   app: Express;
@@ -56,6 +57,7 @@ export class ExpressServer {
     system.registerSyscalls("index", [], pageIndexSyscalls(this.db));
     system.registerSyscalls("space", [], spaceSyscalls(this.storage));
     system.registerSyscalls("event", [], eventSyscalls(this.eventHook));
+    system.registerSyscalls("markdown", [], markdownSyscalls());
     system.addHook(new EndpointHook(app, "/_"));
   }
 

webapp/editor.tsx

@@ -1,10 +1,10 @@
-import { autocompletion, completionKeymap } from "@codemirror/autocomplete";
-import { closeBrackets, closeBracketsKeymap } from "@codemirror/closebrackets";
-import { indentWithTab, standardKeymap } from "@codemirror/commands";
-import { history, historyKeymap } from "@codemirror/history";
-import { bracketMatching } from "@codemirror/matchbrackets";
-import { searchKeymap } from "@codemirror/search";
-import { EditorSelection, EditorState } from "@codemirror/state";
+import {autocompletion, completionKeymap} from "@codemirror/autocomplete";
+import {closeBrackets, closeBracketsKeymap} from "@codemirror/closebrackets";
+import {indentWithTab, standardKeymap} from "@codemirror/commands";
+import {history, historyKeymap} from "@codemirror/history";
+import {bracketMatching} from "@codemirror/matchbrackets";
+import {searchKeymap} from "@codemirror/search";
+import {EditorSelection, EditorState} from "@codemirror/state";
 import {
   drawSelection,
   dropCursor,
@@ -15,36 +15,38 @@ import {
   ViewPlugin,
   ViewUpdate,
 } from "@codemirror/view";
-import React, { useEffect, useReducer } from "react";
+import React, {useEffect, useReducer} from "react";
 import ReactDOM from "react-dom";
-import { createSandbox as createIFrameSandbox } from "../plugos/environments/iframe_sandbox";
-import { AppEvent, AppEventDispatcher, ClickEvent } from "./app_event";
+import {createSandbox as createIFrameSandbox} from "../plugos/environments/iframe_sandbox";
+import {AppEvent, AppEventDispatcher, ClickEvent} from "./app_event";
 import * as commands from "./commands";
-import { CommandPalette } from "./components/command_palette";
-import { PageNavigator } from "./components/page_navigator";
-import { TopBar } from "./components/top_bar";
-import { lineWrapper } from "./line_wrapper";
-import { markdown } from "./markdown";
-import { PathPageNavigator } from "./navigator";
+import {CommandPalette} from "./components/command_palette";
+import {PageNavigator} from "./components/page_navigator";
+import {TopBar} from "./components/top_bar";
+import {lineWrapper} from "./line_wrapper";
+import {markdown} from "./markdown";
+import {PathPageNavigator} from "./navigator";
 import customMarkDown from "./parser";
 import reducer from "./reducer";
-import { smartQuoteKeymap } from "./smart_quotes";
-import { Space } from "./space";
+import {smartQuoteKeymap} from "./smart_quotes";
+import {Space} from "./space";
 import customMarkdownStyle from "./style";
-import editorSyscalls from "./syscalls/editor";
-import indexerSyscalls from "./syscalls/indexer";
-import spaceSyscalls from "./syscalls/space";
-import { Action, AppViewState, initialViewState } from "./types";
-import { SilverBulletHooks } from "../common/manifest";
-import { safeRun, throttle } from "./util";
-import { System } from "../plugos/system";
-import { EventHook } from "../plugos/hooks/event";
-import { systemSyscalls } from "./syscalls/system";
-import { Panel } from "./components/panel";
-import { CommandHook } from "./hooks/command";
-import { SlashCommandHook } from "./hooks/slash_command";
-import { CompleterHook } from "./hooks/completer";
-import { pasteLinkExtension } from "./editor_paste";
+import {editorSyscalls} from "./syscalls/editor";
+import {indexerSyscalls} from "./syscalls/indexer";
+import {spaceSyscalls} from "./syscalls/space";
+import {Action, AppViewState, initialViewState} from "./types";
+import {SilverBulletHooks} from "../common/manifest";
+import {safeRun, throttle} from "./util";
+import {System} from "../plugos/system";
+import {EventHook} from "../plugos/hooks/event";
+import {systemSyscalls} from "./syscalls/system";
+import {Panel} from "./components/panel";
+import {CommandHook} from "./hooks/command";
+import {SlashCommandHook} from "./hooks/slash_command";
+import {CompleterHook} from "./hooks/completer";
+import {pasteLinkExtension} from "./editor_paste";
+import {markdownSyscalls} from "../common/syscalls/markdown";
+
 
 class PageState {
   scrollTop: number;
@@ -112,6 +114,7 @@ export class Editor implements AppEventDispatcher {
     this.system.registerSyscalls("space", [], spaceSyscalls(this));
     this.system.registerSyscalls("index", [], indexerSyscalls(this.space));
     this.system.registerSyscalls("system", [], systemSyscalls(this.space));
+    this.system.registerSyscalls("markdown", [], markdownSyscalls());
   }
 
   async init() {

webapp/styles/main.scss

@@ -37,7 +37,7 @@ body {
 
 #top {
   height: 55px;
-  position: fixed;
+  position: absolute;
   top: 0;
   left: 0;
   right: 0;
@@ -83,7 +83,7 @@ body {
 #editor {
   position: absolute;
   top: 55px;
-  bottom: 0;
+  bottom: 50px;
   left: 0;
   right: 0;
   overflow-y: hidden;

webapp/syscalls/editor.ts

@@ -1,7 +1,7 @@
-import { Editor } from "../editor";
-import { syntaxTree } from "@codemirror/language";
-import { Transaction } from "@codemirror/state";
-import { SysCallMapping } from "../../plugos/system";
+import {Editor} from "../editor";
+import {syntaxTree} from "@codemirror/language";
+import {Transaction} from "@codemirror/state";
+import {SysCallMapping} from "../../plugos/system";
 
 type SyntaxNode = {
   name: string;
@@ -26,7 +26,8 @@ function ensureAnchor(expr: any, start: boolean) {
   );
 }
 
-export default (editor: Editor): SysCallMapping => ({
+export function editorSyscalls(editor: Editor): SysCallMapping {
+  return {
     getCurrentPage: (): string => {
       return editor.currentPage!;
     },
@@ -152,4 +153,5 @@ export default (editor: Editor): SysCallMapping => ({
     prompt: (ctx, message: string, defaultValue = ""): string | null => {
       return prompt(message, defaultValue);
     },
-});
+  };
+}

webapp/syscalls/indexer.ts

@@ -1,8 +1,8 @@
-import { Space } from "../space";
-import { SysCallMapping } from "../../plugos/system";
-import { transportSyscalls } from "../../plugos/syscalls/transport";
+import {Space} from "../space";
+import {SysCallMapping} from "../../plugos/system";
+import {transportSyscalls} from "../../plugos/syscalls/transport";
 
-export default function indexerSyscalls(space: Space): SysCallMapping {
+export function indexerSyscalls(space: Space): SysCallMapping {
   return transportSyscalls(
     [
       "scanPrefixForPage",

webapp/syscalls/space.ts

@@ -1,8 +1,9 @@
-import { Editor } from "../editor";
-import { SysCallMapping } from "../../plugos/system";
-import { PageMeta } from "../../common/types";
+import {Editor} from "../editor";
+import {SysCallMapping} from "../../plugos/system";
+import {PageMeta} from "../../common/types";
 
-export default (editor: Editor): SysCallMapping => ({
+export function spaceSyscalls(editor: Editor): SysCallMapping {
+  return {
     listPages: async (): Promise<PageMeta[]> => {
       return [...(await editor.space.listPages())];
     },
@@ -25,4 +26,5 @@ export default (editor: Editor): SysCallMapping => ({
       console.log("Deleting page");
       await editor.space.deletePage(name);
     },
-});
+  };
+}