Rewrote all plugs using MarkdownTree

pull/3/head
Zef Hemel 2022-04-04 11:51:41 +02:00
parent 07453d638b
commit 16577c8ea2
17 changed files with 332 additions and 255 deletions

View File

@ -1,16 +1,10 @@
import {SysCallMapping} from "../../plugos/system";
import {MarkdownTree, nodeAtPos, parse, render} from "../tree";
import {MarkdownTree, parse} from "../tree";
export function markdownSyscalls(): SysCallMapping {
return {
"markdown.parse": (ctx, text: string): MarkdownTree => {
return parse(text);
},
"markdown.nodeAtPos": (ctx, mdTree: MarkdownTree, pos: number): MarkdownTree | null => {
return nodeAtPos(mdTree, pos);
},
"markdown.render": (ctx, mdTree: MarkdownTree): string => {
return render(mdTree);
},
};
return {
"markdown.parseMarkdown": (ctx, text: string): MarkdownTree => {
return parse(text);
},
};
}

View File

@ -1,26 +0,0 @@
import {expect, test} from "@jest/globals";
import {nodeAtPos, parse, render} from "./tree";
const mdTest1 = `
# Heading
## Sub _heading_ cool
Hello, this is some **bold** text and *italic*. And [a link](http://zef.me).
- This is a list
- With another item
- TODOs:
- [ ] A task that's not yet done
- [x] Hello
- And a _third_ one [[Wiki Page]] yo
`;
test("Run a Node sandbox", async () => {
let mdTree = parse(mdTest1);
console.log(JSON.stringify(mdTree, null, 2));
expect(nodeAtPos(mdTree, 4)!.type).toBe("ATXHeading1");
expect(nodeAtPos(mdTree, mdTest1.indexOf("Wiki Page"))!.type).toBe(
"WikiLink"
);
expect(render(mdTree)).toBe(mdTest1);
});

View File

@ -2,114 +2,66 @@ import {SyntaxNode} from "@lezer/common";
import wikiMarkdownLang from "../webapp/parser";
export type MarkdownTree = {
type?: string; // undefined === text node
from: number;
to: number;
text?: string;
children?: MarkdownTree[];
parent?: MarkdownTree;
type?: string; // undefined === text node
from: number;
to: number;
text?: string;
children?: MarkdownTree[];
};
function treeToAST(text: string, n: SyntaxNode): MarkdownTree {
let children: MarkdownTree[] = [];
let nodeText: string | undefined;
let child = n.firstChild;
while (child) {
children.push(treeToAST(text, child));
child = child.nextSibling;
}
let children: MarkdownTree[] = [];
let nodeText: string | undefined;
let child = n.firstChild;
while (child) {
children.push(treeToAST(text, child));
child = child.nextSibling;
}
if (children.length === 0) {
children = [
{
from: n.from,
to: n.to,
text: text.substring(n.from, n.to),
},
];
} else {
let newChildren: MarkdownTree[] | string = [];
let index = n.from;
for (let child of children) {
let s = text.substring(index, child.from);
if (s) {
newChildren.push({
from: index,
to: child.from,
text: s,
});
}
newChildren.push(child);
index = child.to;
}
let s = text.substring(index, n.to);
if (s) {
newChildren.push({ from: index, to: n.to, text: s });
}
children = newChildren;
}
let result: MarkdownTree = {
type: n.name,
if (children.length === 0) {
children = [
{
from: n.from,
to: n.to,
};
if (children.length > 0) {
result.children = children;
text: text.substring(n.from, n.to),
},
];
} else {
let newChildren: MarkdownTree[] | string = [];
let index = n.from;
for (let child of children) {
let s = text.substring(index, child.from);
if (s) {
newChildren.push({
from: index,
to: child.from,
text: s,
});
}
newChildren.push(child);
index = child.to;
}
if (nodeText) {
result.text = nodeText;
let s = text.substring(index, n.to);
if (s) {
newChildren.push({ from: index, to: n.to, text: s });
}
return result;
}
children = newChildren;
}
// Currently unused
function addParentPointers(mdTree: MarkdownTree) {
if (!mdTree.children) {
return;
}
for (let child of mdTree.children) {
child.parent = mdTree;
addParentPointers(child);
}
}
// Finds non-text node at position
export function nodeAtPos(
mdTree: MarkdownTree,
pos: number
): MarkdownTree | null {
if (pos < mdTree.from || pos > mdTree.to) {
return null;
}
if (!mdTree.children) {
return mdTree;
}
for (let child of mdTree.children) {
let n = nodeAtPos(child, pos);
if (n && n.text) {
// Got a text node, let's return its parent
return mdTree;
} else if (n) {
// Got it
return n;
}
}
return null;
}
// Turn MarkdownTree back into regular markdown text
export function render(mdTree: MarkdownTree): string {
let pieces: string[] = [];
if (mdTree.text) {
return mdTree.text;
}
for (let child of mdTree.children!) {
pieces.push(render(child));
}
return pieces.join("");
let result: MarkdownTree = {
type: n.name,
from: n.from,
to: n.to,
};
if (children.length > 0) {
result.children = children;
}
if (nodeText) {
result.text = nodeText;
}
return result;
}
export function parse(text: string): MarkdownTree {
return treeToAST(text, wikiMarkdownLang.parser.parse(text).topNode);
return treeToAST(text, wikiMarkdownLang.parser.parse(text).topNode);
}

View File

@ -35,7 +35,7 @@
"context": "node"
},
"test": {
"source": ["common/tree.test.ts"],
"source": ["plugs/lib/tree.test.ts"],
"outputFormat": "commonjs",
"isLibrary": true,
"context": "node"

View File

@ -1,17 +1,6 @@
import {syscall} from "./syscall";
import type {MarkdownTree} from "../common/tree";
export async function parse(text: string): Promise<MarkdownTree> {
return syscall("markdown.parse", text);
}
export async function nodeAtPos(
mdTree: MarkdownTree,
pos: number
): Promise<MarkdownTree | null> {
return syscall("markdown.nodeAtPos", mdTree, pos);
}
export async function render(mdTree: MarkdownTree): Promise<string> {
return syscall("markdown.render", mdTree);
export async function parseMarkdown(text: string): Promise<MarkdownTree> {
return syscall("markdown.parseMarkdown", text);
}

View File

@ -1,6 +1,6 @@
import { Hook, Manifest } from "../types";
import { System } from "../system";
import { safeRun } from "../util";
import {Hook, Manifest} from "../types";
import {System} from "../system";
import {safeRun} from "../util";
// System events:
// - plug:load (plugName: string)
@ -16,7 +16,6 @@ export class EventHook implements Hook<EventHookT> {
if (!this.system) {
throw new Error("Event hook is not initialized");
}
let promises: Promise<void>[] = [];
for (const plug of this.system.loadedPlugs.values()) {
for (const [name, functionDef] of Object.entries(
plug.manifest!.functions
@ -24,12 +23,11 @@ export class EventHook implements Hook<EventHookT> {
if (functionDef.events && functionDef.events.includes(eventName)) {
// Only dispatch functions that can run in this environment
if (plug.canInvoke(name)) {
promises.push(plug.invoke(name, [data]));
await plug.invoke(name, [data]);
}
}
}
}
await Promise.all(promises);
}
apply(system: System<EventHookT>): void {

View File

@ -3,7 +3,7 @@ import {EventHook} from "../hooks/event";
export function eventSyscalls(eventHook: EventHook): SysCallMapping {
return {
"event.dispatch": async(ctx, eventName: string, data: any) => {
"event.dispatch": async (ctx, eventName: string, data: any) => {
return eventHook.dispatchEvent(eventName, data);
},
};

View File

@ -7,7 +7,7 @@ export function fetchSyscalls(): SysCallMapping {
let resp = await fetch(url, init);
return resp.json();
},
"fetch.text": async(ctx, url: RequestInfo, init: RequestInit) => {
"fetch.text": async (ctx, url: RequestInfo, init: RequestInit) => {
let resp = await fetch(url, init);
return resp.text();
},

View File

@ -1,40 +1,49 @@
import { IndexEvent } from "../../webapp/app_event";
import { whiteOutQueries } from "./materialized_queries";
import {IndexEvent} from "../../webapp/app_event";
import {whiteOutQueries} from "./materialized_queries";
import { batchSet } from "plugos-silverbullet-syscall/index";
import {batchSet} from "plugos-silverbullet-syscall/index";
import {parseMarkdown} from "plugos-silverbullet-syscall/markdown";
import {collectNodesMatching, MarkdownTree, render} from "../lib/tree";
type Item = {
item: string;
children?: string[];
nested?: string;
};
const pageRefRe = /\[\[[^\]]+@\d+\]\]/;
const itemFullRe =
/(?<prefix>[\t ]*)[\-\*]\s*([^\n]+)(\n\k<prefix>\s+[\-\*][^\n]+)*/g;
export async function indexItems({ name, text }: IndexEvent) {
let items: { key: string; value: Item }[] = [];
text = whiteOutQueries(text);
for (let match of text.matchAll(itemFullRe)) {
let entire = match[0];
let item = match[2];
if (item.match(pageRefRe)) {
continue;
}
let pos = match.index!;
let lines = entire.split("\n");
console.log("Indexing items", name);
let mdTree = await parseMarkdown(text);
let coll = collectNodesMatching(mdTree, (n) => n.type === "ListItem");
coll.forEach((n) => {
if (!n.children) {
return;
}
let textNodes: MarkdownTree[] = [];
let nested: string | undefined;
for (let child of n.children!.slice(1)) {
if (child.type === "OrderedList" || child.type === "BulletList") {
nested = render(child);
break;
}
textNodes.push(child);
}
let item = textNodes.map(render).join("").trim();
let value: Item = {
item,
};
if (lines.length > 1) {
value.children = lines.slice(1);
if (nested) {
value.nested = nested;
}
items.push({
key: `it:${pos}`,
key: `it:${n.from}`,
value,
});
}
});
console.log("Found", items.length, "item(s)");
await batchSet(name, items);
}

View File

@ -49,13 +49,13 @@ export async function updateMaterializedQueriesOnPage(pageName: string) {
for (let {
key,
page,
value: { task, complete, children },
value: { task, complete, nested },
} of await scanPrefixGlobal("task:")) {
let [, pos] = key.split(":");
if (!filter || (filter && task.includes(filter))) {
results.push(
`* [${complete ? "x" : " "}] [[${page}@${pos}]] ${task}` +
(children ? "\n" + children.join("\n") : "")
(nested ? "\n " + nested : "")
);
}
}
@ -78,13 +78,12 @@ export async function updateMaterializedQueriesOnPage(pageName: string) {
for (let {
key,
page,
value: { item, children },
value: { item, nested },
} of await scanPrefixGlobal("it:")) {
let [, pos] = key.split(":");
if (!filter || (filter && item.includes(filter))) {
results.push(
`* [[${page}@${pos}]] ${item}` +
(children ? "\n" + children.join("\n") : "")
`* [[${page}@${pos}]] ${item}` + (nested ? "\n " + nested : "")
);
}
}

View File

@ -2,7 +2,8 @@ import {ClickEvent} from "../../webapp/app_event";
import {updateMaterializedQueriesCommand} from "./materialized_queries";
import {getCursor, getText, navigate as navigateTo, openUrl,} from "plugos-silverbullet-syscall/editor";
import {taskToggleAtPos} from "../tasks/task";
import {nodeAtPos, parse} from "plugos-silverbullet-syscall/markdown";
import {parseMarkdown} from "plugos-silverbullet-syscall/markdown";
import {nodeAtPos} from "../lib/tree";
import type {MarkdownTree} from "../../common/tree";
const materializedQueryPrefix = /<!--\s*#query\s+/;
@ -39,16 +40,15 @@ async function actionClickOrActionEnter(mdTree: MarkdownTree | null) {
}
export async function linkNavigate() {
let mdTree = await parse(await getText());
let mdTree = await parseMarkdown(await getText());
let newNode = await nodeAtPos(mdTree, await getCursor());
await actionClickOrActionEnter(newNode);
}
export async function clickNavigate(event: ClickEvent) {
if (event.ctrlKey || event.metaKey) {
let mdTree = await parse(await getText());
let newNode = await nodeAtPos(mdTree, event.pos);
console.log("New node", newNode);
let mdTree = await parseMarkdown(await getText());
let newNode = nodeAtPos(mdTree, event.pos);
await actionClickOrActionEnter(newNode);
}
}

52
plugs/lib/tree.test.ts Normal file
View File

@ -0,0 +1,52 @@
import {expect, test} from "@jest/globals";
import {parse} from "../../common/tree";
import {addParentPointers, collectNodesMatching, findParentMatching, nodeAtPos, render,} from "./tree";
const mdTest1 = `
# Heading
## Sub _heading_ cool
Hello, this is some **bold** text and *italic*. And [a link](http://zef.me).
%% My comment here
%% And second line
And an @mention
http://zef.plus
- This is a list [[PageLink]]
- With another item
- TODOs:
- [ ] A task that's not yet done
- [x] Hello
- And a _third_ one [[Wiki Page]] yo
`;
const mdTest2 = `
Hello
* Item 1
*
Sup`;
test("Run a Node sandbox", async () => {
let mdTree = parse(mdTest1);
addParentPointers(mdTree);
// console.log(JSON.stringify(mdTree, null, 2));
let wikiLink = nodeAtPos(mdTree, mdTest1.indexOf("Wiki Page"))!;
expect(wikiLink.type).toBe("WikiLink");
expect(
findParentMatching(wikiLink, (n) => n.type === "BulletList")
).toBeDefined();
let allTodos = collectNodesMatching(mdTree, (n) => n.type === "Task");
expect(allTodos.length).toBe(2);
// Render back into markdown should be equivalent
expect(render(mdTree)).toBe(mdTest1);
let mdTree2 = parse(mdTest2);
console.log(JSON.stringify(mdTree2, null, 2));
});

101
plugs/lib/tree.ts Normal file
View File

@ -0,0 +1,101 @@
export type MarkdownTree = {
type?: string; // undefined === text node
from: number;
to: number;
text?: string;
children?: MarkdownTree[];
parent?: MarkdownTree;
};
export function addParentPointers(mdTree: MarkdownTree) {
if (!mdTree.children) {
return;
}
for (let child of mdTree.children) {
child.parent = mdTree;
addParentPointers(child);
}
}
export function removeParentPointers(mdTree: MarkdownTree) {
delete mdTree.parent;
if (!mdTree.children) {
return;
}
for (let child of mdTree.children) {
removeParentPointers(child);
}
}
export function findParentMatching(
mdTree: MarkdownTree,
matchFn: (mdTree: MarkdownTree) => boolean
): MarkdownTree | null {
let node = mdTree.parent;
while (node) {
if (matchFn(node)) {
return node;
}
node = node.parent;
}
return null;
}
export function collectNodesMatching(
mdTree: MarkdownTree,
matchFn: (mdTree: MarkdownTree) => boolean
): MarkdownTree[] {
if (matchFn(mdTree)) {
return [mdTree];
}
let results: MarkdownTree[] = [];
if (mdTree.children) {
for (let child of mdTree.children) {
results = [...results, ...collectNodesMatching(child, matchFn)];
}
}
return results;
}
export function findNodeMatching(
mdTree: MarkdownTree,
matchFn: (mdTree: MarkdownTree) => boolean
): MarkdownTree | null {
return collectNodesMatching(mdTree, matchFn)[0];
}
// Finds non-text node at position
export function nodeAtPos(
mdTree: MarkdownTree,
pos: number
): MarkdownTree | null {
if (pos < mdTree.from || pos > mdTree.to) {
return null;
}
if (!mdTree.children) {
return mdTree;
}
for (let child of mdTree.children) {
let n = nodeAtPos(child, pos);
if (n && n.text !== undefined) {
// Got a text node, let's return its parent
return mdTree;
} else if (n) {
// Got it
return n;
}
}
return null;
}
// Turn MarkdownTree back into regular markdown text
export function render(mdTree: MarkdownTree): string {
let pieces: string[] = [];
if (mdTree.text !== undefined) {
return mdTree.text;
}
for (let child of mdTree.children!) {
pieces.push(render(child));
}
return pieces.join("");
}

View File

@ -2,6 +2,7 @@
"name": "plugs",
"version": "1.0.0",
"dependencies": {
"@jest/globals": "^27.5.1",
"plugos-silverbullet-syscall": "file:../plugos-silverbullet-syscall",
"plugos-syscall": "file:../plugos-syscall"
}

View File

@ -1,50 +1,45 @@
import type { ClickEvent } from "../../webapp/app_event";
import { IndexEvent } from "../../webapp/app_event";
import type {ClickEvent} from "../../webapp/app_event";
import {IndexEvent} from "../../webapp/app_event";
import { whiteOutQueries } from "../core/materialized_queries";
import { batchSet } from "plugos-silverbullet-syscall/index";
import { readPage, writePage } from "plugos-silverbullet-syscall/space";
import {
dispatch,
getLineUnderCursor,
getSyntaxNodeAtPos,
} from "plugos-silverbullet-syscall/editor";
const taskFullRe =
/(?<prefix>[\t ]*)[\-\*]\s*\[([ Xx])\]\s*([^\n]+)(\n\k<prefix>\s+[\-\*][^\n]+)*/g;
const extractPageLink = /[\-\*]\s*\[[ Xx]\]\s\[\[([^\]]+)@(\d+)\]\]\s*(.*)/;
import {whiteOutQueries} from "../core/materialized_queries";
import {batchSet} from "plugos-silverbullet-syscall/index";
import {readPage, writePage} from "plugos-silverbullet-syscall/space";
import {parseMarkdown} from "plugos-silverbullet-syscall/markdown";
import {dispatch, getText,} from "plugos-silverbullet-syscall/editor";
import {addParentPointers, collectNodesMatching, nodeAtPos, render,} from "../lib/tree";
type Task = {
task: string;
complete: boolean;
pos?: number;
children?: string[];
nested?: string;
};
export async function indexTasks({ name, text }: IndexEvent) {
console.log("Indexing tasks");
let tasks: { key: string; value: Task }[] = [];
text = whiteOutQueries(text);
for (let match of text.matchAll(taskFullRe)) {
let entire = match[0];
let complete = match[2] !== " ";
let task = match[3];
let pos = match.index!;
let lines = entire.split("\n");
let mdTree = await parseMarkdown(text);
addParentPointers(mdTree);
collectNodesMatching(mdTree, (n) => n.type === "Task").forEach((n) => {
let task = n.children!.slice(1).map(render).join("").trim();
let complete = n.children![0].children![0].text! !== "[ ]";
let value: Task = {
task,
complete,
};
if (lines.length > 1) {
value.children = lines.slice(1);
let taskIndex = n.parent!.children!.indexOf(n);
let nestedItems = n.parent!.children!.slice(taskIndex + 1);
if (nestedItems.length > 0) {
value.nested = nestedItems.map(render).join("").trim();
}
tasks.push({
key: `task:${pos}`,
key: `task:${n.from}`,
value,
});
}
});
console.log("Found", tasks.length, "task(s)");
await batchSet(name, tasks);
}
@ -54,41 +49,55 @@ export async function taskToggle(event: ClickEvent) {
}
export async function taskToggleAtPos(pos: number) {
let syntaxNode = await getSyntaxNodeAtPos(pos);
if (syntaxNode && syntaxNode.name === "TaskMarker") {
let text = await getText();
let mdTree = await parseMarkdown(text);
addParentPointers(mdTree);
let node = nodeAtPos(mdTree, pos);
if (node && node.type === "TaskMarker") {
let changeTo = "[x]";
if (syntaxNode.text === "[x]" || syntaxNode.text === "[X]") {
if (node.children![0].text === "[x]" || node.children![0].text === "[X]") {
changeTo = "[ ]";
}
await dispatch({
changes: {
from: syntaxNode.from,
to: syntaxNode.to,
from: node.from,
to: node.to,
insert: changeTo,
},
selection: {
anchor: pos,
},
});
// In case there's a page reference with @ position in the task, let's propagate this change back to that page
// Example: * [ ] [[MyPage@123]] My task
let line = await getLineUnderCursor();
let match = line.match(extractPageLink);
if (match) {
console.log("Found a remote task reference, updating other page");
let [, page, posS] = match;
let pos = +posS;
let pageData = await readPage(page);
let text = pageData.text;
// Apply the toggle
text =
text.substring(0, pos) +
text
.substring(pos)
.replace(/^(\s*[\-\*]\s*)\[[ xX]\]/, "$1" + changeTo);
let parentWikiLinks = collectNodesMatching(
node.parent!,
(n) => n.type === "WikiLinkPage"
);
for (let wikiLink of parentWikiLinks) {
let ref = wikiLink.children![0].text!;
if (ref.includes("@")) {
let [page, pos] = ref.split("@");
let pageData = await readPage(page);
let text = pageData.text;
await writePage(page, text);
let referenceMdTree = await parseMarkdown(text);
// Adding +1 to immediately hit the task marker
let taskMarkerNode = nodeAtPos(referenceMdTree, +pos + 1);
if (!taskMarkerNode || taskMarkerNode.type !== "TaskMarker") {
console.error(
"Reference not a task marker, out of date?",
taskMarkerNode
);
return;
}
taskMarkerNode.children![0].text = changeTo;
console.log("This will be the new marker", render(taskMarkerNode));
text = render(referenceMdTree);
console.log("Updated reference paged text", text);
await writePage(page, text);
}
}
}
}

View File

@ -1,7 +1,7 @@
import { mkdir, readdir, readFile, stat, unlink, writeFile } from "fs/promises";
import {mkdir, readdir, readFile, stat, unlink, writeFile} from "fs/promises";
import * as path from "path";
import { PageMeta } from "../common/types";
import { EventHook } from "../plugos/hooks/event";
import {PageMeta} from "../common/types";
import {EventHook} from "../plugos/hooks/event";
export interface Storage {
listPages(): Promise<PageMeta[]>;
@ -29,12 +29,17 @@ export class EventedStorage implements Storage {
async writePage(pageName: string, text: string): Promise<PageMeta> {
const newPageMeta = this.wrapped.writePage(pageName, text);
// This can happen async
this.eventHook.dispatchEvent("page:saved", pageName).then(() => {
return this.eventHook.dispatchEvent("page:index", {
name: pageName,
text: text,
this.eventHook
.dispatchEvent("page:saved", pageName)
.then(() => {
return this.eventHook.dispatchEvent("page:index", {
name: pageName,
text: text,
});
})
.catch((e) => {
console.error("Error dispatching page:saved event", e);
});
});
return newPageMeta;
}

View File

@ -1,14 +1,8 @@
import { styleTags, tags as t } from "@codemirror/highlight";
import {
BlockContext,
LeafBlock,
LeafBlockParser,
MarkdownConfig,
TaskList,
} from "@lezer/markdown";
import { commonmark, mkLang } from "./markdown/markdown";
import {styleTags, tags as t} from "@codemirror/highlight";
import {BlockContext, LeafBlock, LeafBlockParser, MarkdownConfig, TaskList,} from "@lezer/markdown";
import {commonmark, mkLang} from "./markdown/markdown";
import * as ct from "./customtags";
import { pageLinkRegex } from "./constant";
import {pageLinkRegex} from "./constant";
const pageLinkRegexPrefix = new RegExp(
"^" + pageLinkRegex.toString().slice(1, -1)
@ -28,7 +22,7 @@ const WikiLink: MarkdownConfig = {
return -1;
}
return cx.addElement(
cx.elt("WikiLink", pos, pos + match[0].length + 1, [
cx.elt("WikiLink", pos, pos + match[0].length, [
cx.elt("WikiLinkPage", pos + 2, pos + match[0].length - 2),
])
);