diff --git a/common/deps.ts b/common/deps.ts index e98c681b..07f78935 100644 --- a/common/deps.ts +++ b/common/deps.ts @@ -42,11 +42,9 @@ export { TaskList, } from "@lezer/markdown"; -export { parseMixed } from "@lezer/common"; - export type { NodeType, SyntaxNode, SyntaxNodeRef, Tree } from "@lezer/common"; -export { searchKeymap } from "https://esm.sh/@codemirror/search@6.2.2?external=@codemirror/state,@codemirror/view"; +export { searchKeymap } from "https://esm.sh/@codemirror/search@6.2.3?external=@codemirror/state,@codemirror/view"; export { Decoration, drawSelection, @@ -61,7 +59,7 @@ export { } from "@codemirror/view"; export type { DecorationSet, KeyBinding } from "@codemirror/view"; -export { markdown } from "https://esm.sh/@codemirror/lang-markdown@6.0.4?external=@codemirror/state,@lezer/common,@codemirror/language,@lezer/markdown,@codemirror/view,@lezer/highlight"; +export { markdown } from "https://esm.sh/@codemirror/lang-markdown@6.0.5?external=@codemirror/state,@lezer/common,@codemirror/language,@lezer/markdown,@codemirror/view,@lezer/highlight,@@codemirror/lang-html"; export { EditorSelection, @@ -96,4 +94,4 @@ export { yaml as yamlLanguage } from "https://esm.sh/@codemirror/legacy-modes@6. export { javascriptLanguage, typescriptLanguage, -} from "https://esm.sh/@codemirror/lang-javascript@6.1.1?external=@codemirror/language,@codemirror/autocomplete,@codemirror/view,@codemirror/state,@codemirror/lint,@lezer/common,@lezer/lr,@lezer/javascript,@codemirror/commands"; +} from "https://esm.sh/@codemirror/lang-javascript@6.1.2?external=@codemirror/language,@codemirror/autocomplete,@codemirror/view,@codemirror/state,@codemirror/lint,@lezer/common,@lezer/lr,@lezer/javascript,@codemirror/commands"; diff --git a/common/customtags.ts b/common/markdown_parser/customtags.ts similarity index 71% rename from common/customtags.ts rename to common/markdown_parser/customtags.ts index f0faf573..ef9104e8 100644 --- a/common/customtags.ts +++ b/common/markdown_parser/customtags.ts @@ -1,4 +1,4 @@ -import { Tag } from "./deps.ts"; +import { Tag } from "../deps.ts"; export const CommandLinkTag = Tag.define(); export const CommandLinkNameTag = Tag.define(); @@ -13,3 +13,8 @@ export const BulletList = Tag.define(); export const OrderedList = Tag.define(); export const Highlight = Tag.define(); export const HorizontalRuleTag = Tag.define(); + +export const DirectiveTag = Tag.define(); +export const DirectiveStartTag = Tag.define(); +export const DirectiveEndTag = Tag.define(); +export const DirectiveProgramTag = Tag.define(); diff --git a/common/markdown_ext.ts b/common/markdown_parser/markdown_ext.ts similarity index 89% rename from common/markdown_ext.ts rename to common/markdown_parser/markdown_ext.ts index 54fc8fae..74682256 100644 --- a/common/markdown_ext.ts +++ b/common/markdown_parser/markdown_ext.ts @@ -1,7 +1,7 @@ -import { Tag } from "./deps.ts"; -import type { MarkdownConfig } from "./deps.ts"; -import { System } from "../plugos/system.ts"; -import { Manifest } from "./manifest.ts"; +import { Tag } from "../deps.ts"; +import type { MarkdownConfig } from "../deps.ts"; +import { System } from "../../plugos/system.ts"; +import { Manifest } from "../manifest.ts"; export type MDExt = { // unicode char code for efficiency .charCodeAt(0) diff --git a/common/markdown_parser/parse-query.js b/common/markdown_parser/parse-query.js new file mode 100644 index 00000000..24b23657 --- /dev/null +++ b/common/markdown_parser/parse-query.js @@ -0,0 +1,16 @@ +// This file was generated by lezer-generator. You probably shouldn't edit it. +import {LRParser} from "@lezer/lr" +export const parser = LRParser.deserialize({ + version: 14, + states: "&`OVQPOOOmQQO'#C^QOQPOOOtQPO'#C`OyQPO'#ClO!OQPO'#CnO!TQPO'#CqO!YQPO'#CsOOQO'#Cv'#CvO!bQQO,58xO!iQQO'#CcO#WQQO'#CbOOQO,58z,58zOOQO,59W,59WO#oQQO,59YO$ZQQO'#D`OOQO,59],59]OOQO,59_,59_OOQO-E6t-E6tO$rQQO,58}OtQPO'#CxO%ZQQO,58|OOQO'#Cp'#CpOOQO1G.t1G.tO%rQPO'#CyO%wQQO,59zOOQO'#Cg'#CgO$rQQO'#CjOOQO'#Cd'#CdOOQO1G.i1G.iOOQO,59d,59dOOQO-E6v-E6vOOQO,59e,59eOOQO-E6w-E6wO&`QPO'#DRO&hQPO,59UO$rQQO'#CwO&mQPO,59mOOQO1G.p1G.pOOQO,59c,59cOOQO-E6u-E6u", + stateData: "&u~OpOS~ORPO~OTROaSOcTOfUOhVO~OnQX~P[ORYO~OX]O~OR^O~OR_O~OYaOiaO~OnQa~P[OqcOxcOycOzcO{cO|cO}cO!OcO!PcO~O_dOTUXaUXcUXfUXhUXnUX~O!QfO!RfOTbaabacbafbahbanba~OvhOT!SXa!SXc!SXf!SXh!SXn!SX~OXlOYlO[lO]lOrjOsjOtkO~O_dOTUaaUacUafUahUanUa~ORpO~OvhOT!Saa!Sac!Saf!Sah!San!Sa~OvtOwuX~OwvO~OvtOwua~O", + goto: "#g!TPP!UP!XP!]!`!fPP!oPP!oP!XP!XP!t!XP!XPP!w!}#T#ZPPPPPPP#aPPPPPPPPPPPP#dRQOTWPXR[RQZRRndQmcQrkRwtVlcktRg^QXPRbXQurRxuQeZRoeQi_RqiRskR`U", + nodeNames: "⚠ Program Query Name WhereClause Where LogicalExpr FilterExpr Value Number String Bool Regex Null List And LimitClause Limit OrderClause Order OrderDirection SelectClause Select RenderClause Render PageRef", + maxTerm: 50, + skippedNodes: [0], + repeatNodeCount: 4, + tokenData: "Ao~R|X^#{pq#{qr$prs%T|}%o}!O%t!P!Q&V!Q![&|!^!_'U!_!`'c!`!a'p!c!}%t!}#O'}#P#Q(n#R#S%t#T#U(s#U#W%t#W#X+Y#X#Y%t#Y#Z-U#Z#]%t#]#^/f#^#`%t#`#a0b#a#b%t#b#c2u#c#d4q#d#f%t#f#g7h#g#h:d#h#i=`#i#k%t#k#l?[#l#o%t#y#z#{$f$g#{#BY#BZ#{$IS$I_#{$Ip$Iq%T$Iq$Ir%T$I|$JO#{$JT$JU#{$KV$KW#{&FU&FV#{~$QYp~X^#{pq#{#y#z#{$f$g#{#BY#BZ#{$IS$I_#{$I|$JO#{$JT$JU#{$KV$KW#{&FU&FV#{~$sP!_!`$v~${Pz~#r#s%O~%TO!O~~%WUOr%Trs%js$Ip%T$Ip$Iq%j$Iq$Ir%j$Ir~%T~%oOY~~%tOv~P%ySRP}!O%t!c!}%t#R#S%t#T#o%t~&[V[~OY&VZ]&V^!P&V!P!Q&q!Q#O&V#O#P&v#P~&V~&vO[~~&yPO~&V~'RPX~!Q![&|~'ZPq~!_!`'^~'cOx~~'hPy~#r#s'k~'pO}~~'uP|~!_!`'x~'}O{~R(SPtQ!}#O(VP(YRO#P(V#P#Q(c#Q~(VP(fP#P#Q(iP(nOiP~(sOw~R(xWRP}!O%t!c!}%t#R#S%t#T#b%t#b#c)b#c#g%t#g#h*^#h#o%tR)gURP}!O%t!c!}%t#R#S%t#T#W%t#W#X)y#X#o%tR*QS_QRP}!O%t!c!}%t#R#S%t#T#o%tR*cURP}!O%t!c!}%t#R#S%t#T#V%t#V#W*u#W#o%tR*|S!RQRP}!O%t!c!}%t#R#S%t#T#o%tR+_URP}!O%t!c!}%t#R#S%t#T#X%t#X#Y+q#Y#o%tR+vURP}!O%t!c!}%t#R#S%t#T#g%t#g#h,Y#h#o%tR,_URP}!O%t!c!}%t#R#S%t#T#V%t#V#W,q#W#o%tR,xS!QQRP}!O%t!c!}%t#R#S%t#T#o%tR-ZTRP}!O%t!c!}%t#R#S%t#T#U-j#U#o%tR-oURP}!O%t!c!}%t#R#S%t#T#`%t#`#a.R#a#o%tR.WURP}!O%t!c!}%t#R#S%t#T#g%t#g#h.j#h#o%tR.oURP}!O%t!c!}%t#R#S%t#T#X%t#X#Y/R#Y#o%tR/YSsQRP}!O%t!c!}%t#R#S%t#T#o%tR/kURP}!O%t!c!}%t#R#S%t#T#b%t#b#c/}#c#o%tR0US!PQRP}!O%t!c!}%t#R#S%t#T#o%tR0gURP}!O%t!c!}%t#R#S%t#T#]%t#]#^0y#^#o%tR1OURP}!O%t!c!}%t#R#S%t#T#a%t#a#b1b#b#o%tR1gURP}!O%t!c!}%t#R#S%t#T#]%t#]#^1y#^#o%tR2OURP}!O%t!c!}%t#R#S%t#T#h%t#h#i2b#i#o%tR2iSaQRP}!O%t!c!}%t#R#S%t#T#o%tR2zURP}!O%t!c!}%t#R#S%t#T#i%t#i#j3^#j#o%tR3cURP}!O%t!c!}%t#R#S%t#T#`%t#`#a3u#a#o%tR3zURP}!O%t!c!}%t#R#S%t#T#`%t#`#a4^#a#o%tR4eSRP]Q}!O%t!c!}%t#R#S%t#T#o%tR4vURP}!O%t!c!}%t#R#S%t#T#f%t#f#g5Y#g#o%tR5_URP}!O%t!c!}%t#R#S%t#T#W%t#W#X5q#X#o%tR5vURP}!O%t!c!}%t#R#S%t#T#X%t#X#Y6Y#Y#o%tR6_URP}!O%t!c!}%t#R#S%t#T#f%t#f#g6q#g#o%tR6vTRPpq7V}!O%t!c!}%t#R#S%t#T#o%tQ7YP#U#V7]Q7`P#m#n7cQ7hOcQR7mURP}!O%t!c!}%t#R#S%t#T#X%t#X#Y8P#Y#o%tR8UURP}!O%t!c!}%t#R#S%t#T#b%t#b#c8h#c#o%tR8mURP}!O%t!c!}%t#R#S%t#T#W%t#W#X9P#X#o%tR9UURP}!O%t!c!}%t#R#S%t#T#X%t#X#Y9h#Y#o%tR9mURP}!O%t!c!}%t#R#S%t#T#f%t#f#g:P#g#o%tR:WSRPhQ}!O%t!c!}%t#R#S%t#T#o%tR:iURP}!O%t!c!}%t#R#S%t#T#X%t#X#Y:{#Y#o%tR;QURP}!O%t!c!}%t#R#S%t#T#`%t#`#a;d#a#o%tR;iURP}!O%t!c!}%t#R#S%t#T#X%t#X#Y;{#Y#o%tR`#j#o%tR>eURP}!O%t!c!}%t#R#S%t#T#X%t#X#Y>w#Y#o%tR?OSrQRP}!O%t!c!}%t#R#S%t#T#o%tR?aURP}!O%t!c!}%t#R#S%t#T#[%t#[#]?s#]#o%tR?xURP}!O%t!c!}%t#R#S%t#T#X%t#X#Y@[#Y#o%tR@aURP}!O%t!c!}%t#R#S%t#T#f%t#f#g@s#g#o%tR@xURP}!O%t!c!}%t#R#S%t#T#X%t#X#YA[#Y#o%tRAcSRPTQ}!O%t!c!}%t#R#S%t#T#o%t", + tokenizers: [0, 1], + topRules: {"Program":[0,1]}, + tokenPrec: 0 +}) diff --git a/plugs/directive/parse-query.terms.js b/common/markdown_parser/parse-query.terms.js similarity index 54% rename from plugs/directive/parse-query.terms.js rename to common/markdown_parser/parse-query.terms.js index 9d41b67a..4b169e8e 100644 --- a/plugs/directive/parse-query.terms.js +++ b/common/markdown_parser/parse-query.terms.js @@ -4,8 +4,8 @@ export const Query = 2, Name = 3, WhereClause = 4, - LogicalExpr = 5, - AndExpr = 6, + Where = 5, + LogicalExpr = 6, FilterExpr = 7, Value = 8, Number = 9, @@ -14,9 +14,14 @@ export const Regex = 12, Null = 13, List = 14, - OrderClause = 15, - Order = 16, - LimitClause = 17, - SelectClause = 18, - RenderClause = 19, - PageRef = 20 + And = 15, + LimitClause = 16, + Limit = 17, + OrderClause = 18, + Order = 19, + OrderDirection = 20, + SelectClause = 21, + Select = 22, + RenderClause = 23, + Render = 24, + PageRef = 25 diff --git a/common/parse_tree.ts b/common/markdown_parser/parse_tree.ts similarity index 64% rename from common/parse_tree.ts rename to common/markdown_parser/parse_tree.ts index 3cf783ba..23eef1d6 100644 --- a/common/parse_tree.ts +++ b/common/markdown_parser/parse_tree.ts @@ -1,5 +1,5 @@ import type { ParseTree } from "$sb/lib/tree.ts"; -import type { Language, SyntaxNode } from "./deps.ts"; +import type { Language, SyntaxNode } from "../deps.ts"; export function lezerToParseTree( text: string, @@ -60,24 +60,5 @@ export function lezerToParseTree( export function parse(language: Language, text: string): ParseTree { const tree = lezerToParseTree(text, language.parser.parse(text).topNode); - // replaceNodesMatching(tree, (n): MarkdownTree | undefined | null => { - // if (n.type === "FencedCode") { - // let infoN = findNodeMatching(n, (n) => n.type === "CodeInfo"); - // let language = infoN!.children![0].text; - // let textN = findNodeMatching(n, (n) => n.type === "CodeText"); - // let text = textN!.children![0].text!; - // - // console.log(language, text); - // switch (language) { - // case "yaml": - // let parsed = StreamLanguage.define(yaml).parser.parse(text); - // let subTree = treeToAST(text, parsed.topNode, n.from); - // // console.log(JSON.stringify(subTree, null, 2)); - // subTree.type = "yaml"; - // return subTree; - // } - // } - // return; - // }); return tree; } diff --git a/common/parser.test.ts b/common/markdown_parser/parser.test.ts similarity index 56% rename from common/parser.test.ts rename to common/markdown_parser/parser.test.ts index 71ddec3a..0d6e3d38 100644 --- a/common/parser.test.ts +++ b/common/markdown_parser/parser.test.ts @@ -4,8 +4,8 @@ import { collectNodesOfType, findNodeOfType, renderToText, -} from "../plug-api/lib/tree.ts"; -import { assertEquals, assertNotEquals } from "../test_deps.ts"; +} from "../../plug-api/lib/tree.ts"; +import { assertEquals, assertNotEquals } from "../../test_deps.ts"; const sample1 = `--- type: page @@ -27,10 +27,7 @@ Supper`; Deno.test("Test parser", () => { const lang = buildMarkdown([]); - let tree = parse( - lang, - sample1, - ); + let tree = parse(lang, sample1); // console.log("tree", JSON.stringify(tree, null, 2)); // Check if rendering back to text works assertEquals(renderToText(tree), sample1); @@ -53,3 +50,42 @@ Deno.test("Test parser", () => { // console.log("Invalid node", node); assertEquals(node, undefined); }); + +const directiveSample = ` +Before + +Body line 1 + +Body line 2 + +End +`; + +const nestedDirectiveExample = ` +Before + +1 + +100 + +3 + +End +`; + +Deno.test("Test directive parser", () => { + const lang = buildMarkdown([]); + let tree = parse(lang, directiveSample); + // console.log("tree", JSON.stringify(tree, null, 2)); + assertEquals(renderToText(tree), directiveSample); + + tree = parse(lang, nestedDirectiveExample); + // console.log("tree", JSON.stringify(tree, null, 2)); + assertEquals(renderToText(tree), nestedDirectiveExample); + + const orderByExample = ` + + `; + tree = parse(lang, orderByExample); + console.log("Tree", JSON.stringify(tree, null, 2)); +}); diff --git a/common/parser.ts b/common/markdown_parser/parser.ts similarity index 70% rename from common/parser.ts rename to common/markdown_parser/parser.ts index 6de72a3e..be1d3479 100644 --- a/common/parser.ts +++ b/common/markdown_parser/parser.ts @@ -13,7 +13,7 @@ import { tags as t, TaskList, yamlLanguage, -} from "./deps.ts"; +} from "../deps.ts"; import * as ct from "./customtags.ts"; import { MDExt, @@ -112,14 +112,6 @@ const CommandLink: MarkdownConfig = { cx.elt("CommandLinkMark", endPos - 2, endPos), ]), ); - - // return cx.addElement( - // cx.elt("CommandLink", pos, endPos, [ - // cx.elt("CommandLinkMark", pos, pos + 2), - // cx.elt("CommandLinkName", pos + 2, endPos - 2), - // cx.elt("CommandLinkMark", endPos - 2, endPos), - // ]), - // ); }, after: "Emphasis", }, @@ -180,6 +172,108 @@ export const Comment: MarkdownConfig = { ], }; +// Directive parser + +const directiveStart = /^\s*\s*/; +const directiveEnd = /^\s*\s*/; + +import { parser as directiveParser } from "./parse-query.js"; + +const highlightingDirectiveParser = directiveParser.configure({ + props: [ + styleTags({ + "Name": t.variableName, + "String PageRef": t.string, + "Number": t.number, + "Where Limit Select Render Order OrderDirection And": t.keyword, + }), + ], +}); + +export const Directive: MarkdownConfig = { + defineNodes: [ + { name: "Directive", block: true, style: ct.DirectiveTag }, + { name: "DirectiveStart", style: ct.DirectiveStartTag, block: true }, + { name: "DirectiveEnd", style: ct.DirectiveEndTag }, + { name: "DirectiveBody", block: true }, + ], + parseBlock: [{ + name: "Directive", + parse: (cx, line: Line) => { + const match = directiveStart.exec(line.text); + if (!match) { + return false; + } + + // console.log("Parsing directive", line.text); + + const frontStart = cx.parsedPos; + const [fullMatch, directive, arg] = match; + const elts = []; + if (directive === "query") { + const queryParseTree = highlightingDirectiveParser.parse(arg); + elts.push(cx.elt( + "DirectiveStart", + cx.parsedPos, + cx.parsedPos + line.text.length + 1, + [cx.elt(queryParseTree, frontStart + fullMatch.indexOf(arg))], + )); + } else { + elts.push(cx.elt( + "DirectiveStart", + cx.parsedPos, + cx.parsedPos + line.text.length + 1, + )); + } + + // console.log("Query parse tree", queryParseTree.topNode); + + cx.nextLine(); + const startPos = cx.parsedPos; + let endPos = startPos; + let text = ""; + let lastPos = cx.parsedPos; + let nesting = 0; + while (true) { + if (directiveEnd.exec(line.text) && nesting === 0) { + break; + } + text += line.text + "\n"; + endPos += line.text.length + 1; + if (directiveStart.exec(line.text)) { + nesting++; + } + if (directiveEnd.exec(line.text)) { + nesting--; + } + cx.nextLine(); + if (cx.parsedPos === lastPos) { + // End of file, no progress made, there may be a better way to do this but :shrug: + return false; + } + lastPos = cx.parsedPos; + } + const directiveBodyTree = cx.parser.parse(text); + + elts.push( + cx.elt("DirectiveBody", startPos, endPos, [ + cx.elt(directiveBodyTree, startPos), + ]), + ); + endPos = cx.parsedPos + line.text.length; + elts.push(cx.elt( + "DirectiveEnd", + cx.parsedPos, + cx.parsedPos + line.text.length, + )); + cx.nextLine(); + cx.addElement(cx.elt("Directive", frontStart, endPos, elts)); + return true; + }, + before: "HTMLBlock", + }], +}; + // FrontMatter parser const yamlLang = StreamLanguage.define(yamlLanguage); @@ -249,6 +343,7 @@ export default function buildMarkdown(mdExtensions: MDExt[]): Language { WikiLink, CommandLink, FrontMatter, + Directive, TaskList, Comment, Highlight, diff --git a/plugs/directive/query.grammar b/common/markdown_parser/query.grammar similarity index 56% rename from plugs/directive/query.grammar rename to common/markdown_parser/query.grammar index 489bff8b..b5e85cdb 100644 --- a/plugs/directive/query.grammar +++ b/common/markdown_parser/query.grammar @@ -1,28 +1,26 @@ @precedence { logic @left } - @top Program { Query } +@skip { space } Query { - Name ( WhereClause | OrderClause | LimitClause | SelectClause | RenderClause )* + Name ( WhereClause | LimitClause | OrderClause | SelectClause | RenderClause )* } commaSep { content ("," content)* } -WhereClause { "where" LogicalExpr } -OrderClause { "order" "by" Name Order? } -LimitClause { "limit" Number } -SelectClause { "select" commaSep } -RenderClause { "render" (PageRef | String) } +WhereClause { Where LogicalExpr } +LimitClause { Limit Number } +OrderClause { Order Name OrderDirection? } +SelectClause { Select commaSep } +RenderClause { Render (PageRef | String) } -Order { +OrderDirection { "desc" | "asc" } Value { Number | String | Bool | Regex | Null | List } -LogicalExpr { AndExpr | FilterExpr } - -AndExpr { FilterExpr !logic "and" FilterExpr } +LogicalExpr { FilterExpr (And FilterExpr)* } FilterExpr { Name "<" Value @@ -38,21 +36,24 @@ FilterExpr { List { "[" commaSep "]" } -@skip { space } - - Bool { "true" | "false" } -Null { - "null" -} @tokens { space { std.whitespace+ } Name { (std.asciiLetter | "-" | "_")+ } + + Where { "where" } + Order { "order by" } + Select { "select" } + Render { "render" } + Limit { "limit" } + And { "and" } + Null { "null" } + String { ("\"" | "“" | "”") ![\"”“]* ("\"" | "“" | "”") } @@ -62,4 +63,6 @@ Null { Regex { "/" ( ![/\\\n\r] | "\\" _ )* "/"? } Number { std.digit+ } + + // @precedence { Where, Sort, Select, Render, Limit, And, Null, Name } } diff --git a/common/syscalls/markdown.ts b/common/syscalls/markdown.ts index 27a1da69..69efe45e 100644 --- a/common/syscalls/markdown.ts +++ b/common/syscalls/markdown.ts @@ -1,5 +1,5 @@ import { SysCallMapping } from "../../plugos/system.ts"; -import { parse } from "../parse_tree.ts"; +import { parse } from "../markdown_parser/parse_tree.ts"; import { Language } from "../deps.ts"; import type { ParseTree } from "$sb/lib/tree.ts"; diff --git a/deno.jsonc b/deno.jsonc index 3b7ab94f..63ff6183 100644 --- a/deno.jsonc +++ b/deno.jsonc @@ -11,7 +11,7 @@ "bundle": "deno bundle silverbullet.ts dist/silverbullet.js", // Regenerates some bundle files (checked into the repo) // Install lezer-generator with "npm install -g @lezer/generator" - "generate": "deno run -A plugos/gen.ts && lezer-generator plugs/directive/query.grammar -o plugs/directive/parse-query.js" + "generate": "deno run -A plugos/gen.ts && lezer-generator common/markdown_parser/query.grammar -o common/markdown_parser/parse-query.js" }, "compilerOptions": { diff --git a/import_map.json b/import_map.json index 3a079731..230624b5 100644 --- a/import_map.json +++ b/import_map.json @@ -1,20 +1,21 @@ { "imports": { - "@codemirror/state": "https://esm.sh/@codemirror/state@6.1.2", - "@lezer/common": "https://esm.sh/@lezer/common@1.0.1", + "@codemirror/state": "https://esm.sh/@codemirror/state@6.1.4", + "@lezer/common": "https://esm.sh/@lezer/common@1.0.2", "@lezer/markdown": "https://esm.sh/@lezer/markdown@1.0.2?external=@lezer/common,@codemirror/language,@lezer/highlight", - "@lezer/javascript": "https://esm.sh/@lezer/javascript@1.0.2?external=@lezer/common,@codemirror/language,@lezer/highlight", - "@codemirror/language": "https://esm.sh/@codemirror/language@6.3.0?external=@codemirror/state,@lezer/common,@lezer/lr,@codemirror/view,@lezer/highlight", + "@lezer/javascript": "https://esm.sh/@lezer/javascript@1.3.1?external=@lezer/common,@codemirror/language,@lezer/highlight", + "@codemirror/language": "https://esm.sh/@codemirror/language@6.3.1?external=@codemirror/state,@lezer/common,@lezer/lr,@codemirror/view,@lezer/highlight", "@codemirror/commands": "https://esm.sh/@codemirror/commands@6.1.2?external=@codemirror/state,@codemirror/view", - "@codemirror/view": "https://esm.sh/@codemirror/view@6.4.1?external=@codemirror/state,@lezer/common", - "@lezer/highlight": "https://esm.sh/@lezer/highlight@1.1.1?external=@lezer/common", - "@codemirror/autocomplete": "https://esm.sh/@codemirror/autocomplete@6.3.0?external=@codemirror/state,@codemirror/commands,@lezer/common,@codemirror/view", - "@codemirror/lint": "https://esm.sh/@codemirror/lint@6.0.0?external=@codemirror/state,@lezer/common", + "@codemirror/view": "https://esm.sh/@codemirror/view@6.7.1?external=@codemirror/state,@lezer/common", + "@lezer/highlight": "https://esm.sh/@lezer/highlight@1.1.3?external=@lezer/common", + "@codemirror/autocomplete": "https://esm.sh/@codemirror/autocomplete@6.3.4?external=@codemirror/state,@codemirror/commands,@lezer/common,@codemirror/view", + "@codemirror/lint": "https://esm.sh/@codemirror/lint@6.1.0?external=@codemirror/state,@lezer/common", + "@codemirror/lang-html": "https://esm.sh/@codemirror/lang-html@6.4.0", "preact": "https://esm.sh/preact@10.11.1", "yjs": "https://esm.sh/yjs@13.5.42", "$sb/": "./plug-api/", "handlebars": "https://esm.sh/handlebars", - "@lezer/lr": "https://esm.sh/@lezer/lr", + "@lezer/lr": "https://esm.sh/@lezer/lr@1.2.5?external=@lezer/common", "yaml": "https://deno.land/std/encoding/yaml.ts" } } diff --git a/plug-api/lib/query.test.ts b/plug-api/lib/query.test.ts new file mode 100644 index 00000000..fc2c6881 --- /dev/null +++ b/plug-api/lib/query.test.ts @@ -0,0 +1,25 @@ +import { renderToText } from "./tree.ts"; +import wikiMarkdownLang from "../../common/markdown_parser/parser.ts"; +import { assert, assertEquals } from "../../test_deps.ts"; +import { parse } from "../../common/markdown_parser/parse_tree.ts"; +import { removeQueries } from "./query.ts"; + +const queryRemovalTest = ` +# Heading +Before + +Bla bla remove me + +End +`; + +Deno.test("White out queries", () => { + const lang = wikiMarkdownLang([]); + const mdTree = parse(lang, queryRemovalTest); + removeQueries(mdTree); + const text = renderToText(mdTree); + // Same length? We should be good + assertEquals(text.length, queryRemovalTest.length); + assert(text.indexOf("remove me") === -1); + console.log("Whited out text", text); +}); diff --git a/plug-api/lib/query.ts b/plug-api/lib/query.ts index 3d8b550a..afe33c96 100644 --- a/plug-api/lib/query.ts +++ b/plug-api/lib/query.ts @@ -3,6 +3,7 @@ import { collectNodesMatching, ParseTree, renderToText, + replaceNodesMatching, } from "$sb/lib/tree.ts"; export const queryRegex = @@ -134,35 +135,15 @@ export function applyQuery(parsedQuery: ParsedQuery, records: T[]): T[] { } export function removeQueries(pt: ParseTree) { - addParentPointers(pt); - collectNodesMatching(pt, (t) => { - if (t.type !== "CommentBlock") { - return false; + replaceNodesMatching(pt, (t) => { + if (t.type !== "Directive") { + return; } - const text = t.children![0].text!; - const match = directiveStartRegex.exec(text); - if (!match) { - return false; - } - const directiveType = match[1]; - const parentChildren = t.parent!.children!; - const index = parentChildren.indexOf(t); - const nodesToReplace: ParseTree[] = []; - for (let i = index + 1; i < parentChildren.length; i++) { - const n = parentChildren[i]; - if (n.type === "CommentBlock") { - const text = n.children![0].text!; - const match = directiveEndRegex.exec(text); - if (match && match[1] === directiveType) { - break; - } - } - nodesToReplace.push(n); - } - const renderedText = nodesToReplace.map(renderToText).join(""); - parentChildren.splice(index + 1, nodesToReplace.length, { + const renderedText = renderToText(t); + return { + from: t.from, + to: t.to, text: new Array(renderedText.length + 1).join(" "), - }); - return true; + }; }); } diff --git a/plug-api/lib/tree.test.ts b/plug-api/lib/tree.test.ts index 66696820..d23cb3a4 100644 --- a/plug-api/lib/tree.test.ts +++ b/plug-api/lib/tree.test.ts @@ -8,9 +8,9 @@ import { renderToText, replaceNodesMatching, } from "./tree.ts"; -import wikiMarkdownLang from "../../common/parser.ts"; +import wikiMarkdownLang from "../../common/markdown_parser/parser.ts"; import { assertEquals, assertNotEquals } from "../../test_deps.ts"; -import { parse } from "../../common/parse_tree.ts"; +import { parse } from "../../common/markdown_parser/parse_tree.ts"; const mdTest1 = ` # Heading @@ -47,7 +47,7 @@ name: something \`\`\` `; -Deno.test("Run a Node sandbox", () => { +Deno.test("Test parsing", () => { const lang = wikiMarkdownLang([]); const mdTree = parse(lang, mdTest1); addParentPointers(mdTree); diff --git a/plug-api/lib/tree.ts b/plug-api/lib/tree.ts index 25984534..022a02ae 100644 --- a/plug-api/lib/tree.ts +++ b/plug-api/lib/tree.ts @@ -148,3 +148,12 @@ export function renderToText(tree: ParseTree): string { } return pieces.join(""); } + +export function cloneTree(tree: ParseTree): ParseTree { + const newTree = { ...tree }; + if (tree.children) { + newTree.children = tree.children.map(cloneTree); + } + delete newTree.parent; + return newTree; +} diff --git a/plugs/collab/collab.ts b/plugs/collab/collab.ts index 079c38f1..6be051d9 100644 --- a/plugs/collab/collab.ts +++ b/plugs/collab/collab.ts @@ -63,7 +63,7 @@ export async function shareCommand() { if (!serverUrl) { return; } - const roomId = nanoid(); + const roomId = nanoid().replaceAll("_", "-"); await editor.save(); const text = await editor.getText(); const tree = await markdown.parseMarkdown(text); diff --git a/plugs/directive/command.ts b/plugs/directive/command.ts index 8d8f9cb3..958a78b8 100644 --- a/plugs/directive/command.ts +++ b/plugs/directive/command.ts @@ -1,7 +1,11 @@ import { editor, markdown, system } from "$sb/silverbullet-syscall/mod.ts"; -import { nodeAtPos } from "$sb/lib/tree.ts"; -import { replaceAsync } from "$sb/lib/util.ts"; -import { directiveRegex, renderDirectives } from "./directives.ts"; +import { + ParseTree, + removeParentPointers, + renderToText, + traverseTree, +} from "$sb/lib/tree.ts"; +import { renderDirectives } from "./directives.ts"; import { extractFrontmatter } from "$sb/lib/frontmatter.ts"; export async function updateDirectivesOnPageCommand(arg: any) { @@ -33,38 +37,47 @@ export async function updateDirectivesOnPageCommand(arg: any) { } // Collect all directives and their body replacements - const replacements: { fullMatch: string; text?: string }[] = []; + const replacements: { fullMatch: string; textPromise: Promise }[] = + []; + + // Convenience array to wait for all promises to resolve + const allPromises: Promise[] = []; + + removeParentPointers(tree); + + traverseTree(tree, (tree) => { + if (tree.type !== "Directive") { + return false; + } + const fullMatch = text.substring(tree.from!, tree.to!); + try { + const promise = system.invokeFunction( + "server", + "serverRenderDirective", + pageName, + tree, + ); + replacements.push({ + textPromise: promise, + fullMatch, + }); + allPromises.push(promise); + } catch (e: any) { + replacements.push({ + fullMatch, + textPromise: Promise.resolve( + `${renderToText(tree.children![0])}\n**ERROR:** ${e.message}\n${ + renderToText(tree.children![tree.children!.length - 1]) + }`, + ), + }); + } + return true; + }); + + // Wait for all to have processed + await Promise.all(allPromises); - await replaceAsync( - text, - directiveRegex, - async (fullMatch, startInst, _type, _arg, _body, endInst, index) => { - const replacement: { fullMatch: string; text?: string } = { fullMatch }; - // Pushing to the replacement array - const currentNode = nodeAtPos(tree, index + 1); - if (currentNode?.type !== "CommentBlock") { - // If not a comment block, it's likely a code block, ignore - // console.log("Not comment block, ignoring", fullMatch); - return fullMatch; - } - replacements.push(replacement); - try { - const replacementText = await system.invokeFunction( - "server", - "serverRenderDirective", - pageName, - fullMatch, - ); - replacement.text = replacementText; - // Return value is ignored, we're using the replacements array - return fullMatch; - } catch (e: any) { - replacement.text = `${startInst}\n**ERROR:** ${e.message}\n${endInst}`; - // Return value is ignored, we're using the replacements array - return fullMatch; - } - }, - ); // Iterate again and replace the bodies. Iterating again (not using previous positions) // because text may have changed in the mean time (directive processing may take some time) // Hypothetically in the mean time directives in text may have been changed/swapped, in which @@ -77,6 +90,10 @@ export async function updateDirectivesOnPageCommand(arg: any) { // This may happen if the query itself, or the user is editing inside the directive block (WHY!?) if (index === -1) { + console.warn( + "Text I got", + text, + ); console.warn( "Could not find directive in text, skipping", replacement.fullMatch, @@ -84,7 +101,8 @@ export async function updateDirectivesOnPageCommand(arg: any) { continue; } const from = index, to = index + replacement.fullMatch.length; - if (text.substring(from, to) === replacement.text) { + const newText = await replacement.textPromise; + if (text.substring(from, to) === newText) { // No change, skip continue; } @@ -92,21 +110,67 @@ export async function updateDirectivesOnPageCommand(arg: any) { changes: { from, to, - insert: replacement.text, + insert: newText, }, }); } } -export function serverPing() { - return "pong"; -} - // Called from client, running on server // The text passed here is going to be a single directive block (not a full page) export function serverRenderDirective( pageName: string, - text: string, + tree: ParseTree, ): Promise { - return renderDirectives(pageName, text); + return renderDirectives(pageName, tree); +} + +// Pure server driven implementation of directive updating +export async function serverUpdateDirectives( + pageName: string, + text: string, +) { + const tree = await markdown.parseMarkdown(text); + // Collect all directives and their body replacements + const replacements: { fullMatch: string; textPromise: Promise }[] = + []; + + const allPromises: Promise[] = []; + + traverseTree(tree, (tree) => { + if (tree.type !== "Directive") { + return false; + } + const fullMatch = text.substring(tree.from!, tree.to!); + try { + const promise = renderDirectives( + pageName, + tree, + ); + replacements.push({ + textPromise: promise, + fullMatch, + }); + allPromises.push(promise); + } catch (e: any) { + replacements.push({ + fullMatch, + textPromise: Promise.resolve( + `${renderToText(tree.children![0])}\n**ERROR:** ${e.message}\n${ + renderToText(tree.children![tree.children!.length - 1]) + }`, + ), + }); + } + return true; + }); + + // Wait for all to have processed + await Promise.all(allPromises); + + // Iterate again and replace the bodies. + for (const replacement of replacements) { + text = text.replace(replacement.fullMatch, await replacement.textPromise); + } + return text; } diff --git a/plugs/directive/directives.ts b/plugs/directive/directives.ts index a8f5e1e5..d7d12ccc 100644 --- a/plugs/directive/directives.ts +++ b/plugs/directive/directives.ts @@ -1,4 +1,4 @@ -import { nodeAtPos, ParseTree } from "$sb/lib/tree.ts"; +import { nodeAtPos, ParseTree, renderToText } from "$sb/lib/tree.ts"; import { replaceAsync } from "$sb/lib/util.ts"; import { markdown } from "$sb/silverbullet-syscall/mod.ts"; @@ -9,56 +9,72 @@ import { templateDirectiveRenderer, } from "./template_directive.ts"; +export const directiveStartRegex = + //i; + export const directiveRegex = /()(.+?)()/gs; - /** * Looks for directives in the text dispatches them based on name */ -export function directiveDispatcher( +export async function directiveDispatcher( pageName: string, - text: string, - tree: ParseTree, + directiveTree: ParseTree, directiveRenderers: Record< string, - (directive: string, pageName: string, arg: string) => Promise + ( + directive: string, + pageName: string, + arg: string | ParseTree, + ) => Promise >, ): Promise { - return replaceAsync( - text, - directiveRegex, - async (fullMatch, startInst, type, arg, _body, endInst, index) => { - const currentNode = nodeAtPos(tree, index + 1); - // console.log("Node type", currentNode?.type); - if (currentNode?.type !== "CommentBlock") { - // If not a comment block, it's likely a code block, ignore - // console.log("Not comment block, ingoring", fullMatch); - return fullMatch; - } + const directiveStart = directiveTree.children![0]; // + const directiveEnd = directiveTree.children![2]; // + + const directiveStartText = renderToText(directiveStart).trim(); + const directiveEndText = renderToText(directiveEnd).trim(); + + if (directiveStart.children!.length === 1) { + // Everything not #query + const match = directiveStartRegex.exec(directiveStart.children![0].text!); + if (!match) { + throw Error("No match"); + } + + let [_fullMatch, type, arg] = match; + try { arg = arg.trim(); - try { - const newBody = await directiveRenderers[type](type, pageName, arg); - return `${startInst}\n${newBody.trim()}\n${endInst}`; - } catch (e: any) { - return `${startInst}\n**ERROR:** ${e.message}\n${endInst}`; - } - }, - ); + const newBody = await directiveRenderers[type](type, pageName, arg); + const result = + `${directiveStartText}\n${newBody.trim()}\n${directiveEndText}`; + return result; + } catch (e: any) { + return `${directiveStartText}\n**ERROR:** ${e.message}\n${directiveEndText}`; + } + } else { + // #query + const newBody = await directiveRenderers["query"]( + "query", + pageName, + directiveStart.children![1], // The query ParseTree + ); + const result = + `${directiveStartText}\n${newBody.trim()}\n${directiveEndText}`; + return result; + } } export async function renderDirectives( pageName: string, - text: string, + directiveTree: ParseTree, ): Promise { - const tree = await markdown.parseMarkdown(text); - - text = await directiveDispatcher(pageName, text, tree, { + const replacementText = await directiveDispatcher(pageName, directiveTree, { use: templateDirectiveRenderer, - "use-verbose": templateDirectiveRenderer, - "include": templateDirectiveRenderer, + include: templateDirectiveRenderer, query: queryDirectiveRenderer, eval: evalDirectiveRenderer, }); - return await cleanTemplateInstantiations(text); + return cleanTemplateInstantiations(replacementText); } diff --git a/plugs/directive/eval_directive.ts b/plugs/directive/eval_directive.ts index 2aa67554..9c57844f 100644 --- a/plugs/directive/eval_directive.ts +++ b/plugs/directive/eval_directive.ts @@ -1,6 +1,7 @@ // This is some shocking stuff. My profession would kill me for this. import * as YAML from "yaml"; +import { ParseTree } from "../../plug-api/lib/tree.ts"; import { jsonToMDTable, renderTemplate } from "./util.ts"; // Enables plugName.functionName(arg1, arg2) syntax in JS expressions @@ -20,8 +21,11 @@ const expressionRegex = /(.+?)(\s+render\s+\[\[([^\]]+)\]\])?$/; export async function evalDirectiveRenderer( _directive: string, _pageName: string, - expression: string, + expression: string | ParseTree, ): Promise { + if (typeof expression !== "string") { + throw new Error("Expected a string"); + } console.log("Got JS expression", expression); const match = expressionRegex.exec(expression); if (!match) { diff --git a/plugs/directive/parse-query.js b/plugs/directive/parse-query.js deleted file mode 100644 index e77c6597..00000000 --- a/plugs/directive/parse-query.js +++ /dev/null @@ -1,16 +0,0 @@ -// This file was generated by lezer-generator. You probably shouldn't edit it. -import {LRParser} from "@lezer/lr" -export const parser = LRParser.deserialize({ - version: 14, - states: "&fOVQPOOOmQQO'#C^QOQPOOOtQPO'#C`OyQQO'#CkO!OQPO'#CmO!TQPO'#CnO!YQPO'#CoOOQO'#Cq'#CqO!bQQO,58xO!iQQO'#CcO#WQQO'#CaOOQO'#Ca'#CaOOQO,58z,58zO#oQPO,59VOOQO,59X,59XO#tQQO'#DaOOQO,59Y,59YOOQO,59Z,59ZOOQO-E6o-E6oO$]QQO,58}OtQPO,58|O$tQQO1G.qO%`QPO'#CsO%eQQO,59{OOQO'#Cg'#CgOOQO'#Ci'#CiO$]QQO'#CjOOQO'#Cd'#CdOOQO1G.i1G.iOOQO1G.h1G.hOOQO'#Cl'#ClOOQO7+$]7+$]OOQO,59_,59_OOQO-E6q-E6qO%|QPO'#C}O&UQPO,59UO$]QQO'#CrO&ZQPO,59iOOQO1G.p1G.pOOQO,59^,59^OOQO-E6p-E6p", - stateData: "&c~OjOS~ORPO~OkRO}SO!RTO!SUO!UVO~OhQX~P[ORYO~O!O^O~OX_O~OR`O~OYbOdbO~OhQa~P[OldOtdOudOvdOwdOxdOydOzdO{dO~O|eOhTXkTX}TX!RTX!STX!UTX~ORfO~OrgOh!TXk!TX}!TX!R!TX!S!TX!U!TX~OXlOYlO[lOmiOniOojOpkO~O!PoO!QoOh_ik_i}_i!R_i!S_i!U_i~ORqO~OrgOh!Tak!Ta}!Ta!R!Ta!S!Ta!U!Ta~OruOsqX~OswO~OruOsqa~O", - goto: "#e!UPP!VP!Y!^!a!d!jPP!sP!s!s!Y!x!Y!Y!YP!{#R#XPPPPPPPPP#_PPPPPPPPPPPPPPPPP#bRQOTWPXR]RR[RQZRRneQmdQskRxuVldkuRpfQXPRcXQvsRyvQh`RrhRtkRaU", - nodeNames: "⚠ Program Query Name WhereClause LogicalExpr AndExpr FilterExpr Value Number String Bool Regex Null List OrderClause Order LimitClause SelectClause RenderClause PageRef", - maxTerm: 52, - skippedNodes: [0], - repeatNodeCount: 3, - tokenData: "B[~R}X^$Opq$Oqr$srs%W|}%r}!O%w!P!Q&Y!Q!['P!^!_'X!_!`'f!`!a's!c!}%w!}#O(Q#P#Q(q#R#S%w#T#U(v#U#V+]#V#W%w#W#X,X#X#Y%w#Y#Z.T#Z#]%w#]#^0e#^#`%w#`#a1a#a#b%w#b#c3t#c#d5p#d#f%w#f#g8T#g#h;P#h#i={#i#k%w#k#l?w#l#o%w#y#z$O$f$g$O#BY#BZ$O$IS$I_$O$Ip$Iq%W$Iq$Ir%W$I|$JO$O$JT$JU$O$KV$KW$O&FU&FV$O~$TYj~X^$Opq$O#y#z$O$f$g$O#BY#BZ$O$IS$I_$O$I|$JO$O$JT$JU$O$KV$KW$O&FU&FV$O~$vP!_!`$y~%OPv~#r#s%R~%WOz~~%ZUOr%Wrs%ms$Ip%W$Ip$Iq%m$Iq$Ir%m$Ir~%W~%rOY~~%wOr~P%|SRP}!O%w!c!}%w#R#S%w#T#o%w~&_V[~OY&YZ]&Y^!P&Y!P!Q&t!Q#O&Y#O#P&y#P~&Y~&yO[~~&|PO~&Y~'UPX~!Q!['P~'^Pl~!_!`'a~'fOt~~'kPu~#r#s'n~'sOy~~'xPx~!_!`'{~(QOw~R(VPpQ!}#O(YP(]RO#P(Y#P#Q(f#Q~(YP(iP#P#Q(lP(qOdP~(vOs~R({WRP}!O%w!c!}%w#R#S%w#T#b%w#b#c)e#c#g%w#g#h*a#h#o%wR)jURP}!O%w!c!}%w#R#S%w#T#W%w#W#X)|#X#o%wR*TS|QRP}!O%w!c!}%w#R#S%w#T#o%wR*fURP}!O%w!c!}%w#R#S%w#T#V%w#V#W*x#W#o%wR+PS!QQRP}!O%w!c!}%w#R#S%w#T#o%wR+bURP}!O%w!c!}%w#R#S%w#T#m%w#m#n+t#n#o%wR+{S!OQRP}!O%w!c!}%w#R#S%w#T#o%wR,^URP}!O%w!c!}%w#R#S%w#T#X%w#X#Y,p#Y#o%wR,uURP}!O%w!c!}%w#R#S%w#T#g%w#g#h-X#h#o%wR-^URP}!O%w!c!}%w#R#S%w#T#V%w#V#W-p#W#o%wR-wS!PQRP}!O%w!c!}%w#R#S%w#T#o%wR.YTRP}!O%w!c!}%w#R#S%w#T#U.i#U#o%wR.nURP}!O%w!c!}%w#R#S%w#T#`%w#`#a/Q#a#o%wR/VURP}!O%w!c!}%w#R#S%w#T#g%w#g#h/i#h#o%wR/nURP}!O%w!c!}%w#R#S%w#T#X%w#X#Y0Q#Y#o%wR0XSnQRP}!O%w!c!}%w#R#S%w#T#o%wR0jURP}!O%w!c!}%w#R#S%w#T#b%w#b#c0|#c#o%wR1TS{QRP}!O%w!c!}%w#R#S%w#T#o%wR1fURP}!O%w!c!}%w#R#S%w#T#]%w#]#^1x#^#o%wR1}URP}!O%w!c!}%w#R#S%w#T#a%w#a#b2a#b#o%wR2fURP}!O%w!c!}%w#R#S%w#T#]%w#]#^2x#^#o%wR2}URP}!O%w!c!}%w#R#S%w#T#h%w#h#i3a#i#o%wR3hS!RQRP}!O%w!c!}%w#R#S%w#T#o%wR3yURP}!O%w!c!}%w#R#S%w#T#i%w#i#j4]#j#o%wR4bURP}!O%w!c!}%w#R#S%w#T#`%w#`#a4t#a#o%wR4yURP}!O%w!c!}%w#R#S%w#T#`%w#`#a5]#a#o%wR5dSoQRP}!O%w!c!}%w#R#S%w#T#o%wR5uURP}!O%w!c!}%w#R#S%w#T#f%w#f#g6X#g#o%wR6^URP}!O%w!c!}%w#R#S%w#T#W%w#W#X6p#X#o%wR6uURP}!O%w!c!}%w#R#S%w#T#X%w#X#Y7X#Y#o%wR7^URP}!O%w!c!}%w#R#S%w#T#f%w#f#g7p#g#o%wR7wS}QRP}!O%w!c!}%w#R#S%w#T#o%wR8YURP}!O%w!c!}%w#R#S%w#T#X%w#X#Y8l#Y#o%wR8qURP}!O%w!c!}%w#R#S%w#T#b%w#b#c9T#c#o%wR9YURP}!O%w!c!}%w#R#S%w#T#W%w#W#X9l#X#o%wR9qURP}!O%w!c!}%w#R#S%w#T#X%w#X#Y:T#Y#o%wR:YURP}!O%w!c!}%w#R#S%w#T#f%w#f#g:l#g#o%wR:sS!UQRP}!O%w!c!}%w#R#S%w#T#o%wR;UURP}!O%w!c!}%w#R#S%w#T#X%w#X#Y;h#Y#o%wR;mURP}!O%w!c!}%w#R#S%w#T#`%w#`#aQURP}!O%w!c!}%w#R#S%w#T#f%w#f#g>d#g#o%wR>iURP}!O%w!c!}%w#R#S%w#T#i%w#i#j>{#j#o%wR?QURP}!O%w!c!}%w#R#S%w#T#X%w#X#Y?d#Y#o%wR?kSmQRP}!O%w!c!}%w#R#S%w#T#o%wR?|URP}!O%w!c!}%w#R#S%w#T#[%w#[#]@`#]#o%wR@eURP}!O%w!c!}%w#R#S%w#T#X%w#X#Y@w#Y#o%wR@|URP}!O%w!c!}%w#R#S%w#T#f%w#f#gA`#g#o%wRAeURP}!O%w!c!}%w#R#S%w#T#X%w#X#YAw#Y#o%wRBOSkQRP}!O%w!c!}%w#R#S%w#T#o%w", - tokenizers: [0, 1], - topRules: {"Program":[0,1]}, - tokenPrec: 0 -}) diff --git a/plugs/directive/parser.ts b/plugs/directive/parser.ts index c88f4fce..f656f78d 100644 --- a/plugs/directive/parser.ts +++ b/plugs/directive/parser.ts @@ -4,16 +4,14 @@ import { ParseTree, replaceNodesMatching, } from "$sb/lib/tree.ts"; -import { lezerToParseTree } from "../../common/parse_tree.ts"; // @ts-ignore auto generated -import { parser } from "./parse-query.js"; import { ParsedQuery, QueryFilter } from "$sb/lib/query.ts"; -export function parseQuery(query: string): ParsedQuery { - const n = lezerToParseTree(query, parser.parse(query).topNode); +export function parseQuery(queryTree: ParseTree): ParsedQuery { + // const n = lezerToParseTree(query, parser.parse(query).topNode); // Clean the tree a bit - replaceNodesMatching(n, (n) => { + replaceNodesMatching(queryTree, (n) => { if (!n.type) { const trimmed = n.text!.trim(); if (!trimmed) { @@ -24,7 +22,7 @@ export function parseQuery(query: string): ParsedQuery { }); // console.log("Parsed", JSON.stringify(n, null, 2)); - const queryNode = n.children![0]; + const queryNode = queryTree.children![0]; const parsedQuery: ParsedQuery = { table: queryNode.children![0].children![0].text!, filter: [], @@ -33,7 +31,7 @@ export function parseQuery(query: string): ParsedQuery { if (orderByNode) { const nameNode = findNodeOfType(orderByNode, "Name"); parsedQuery.orderBy = nameNode!.children![0].text!; - const orderNode = findNodeOfType(orderByNode, "Order"); + const orderNode = findNodeOfType(orderByNode, "OrderDirection"); parsedQuery.orderDesc = orderNode ? orderNode.children![0].text! === "desc" : false; diff --git a/plugs/directive/query.test.ts b/plugs/directive/query.test.ts index 2257b5a0..83a3b48c 100644 --- a/plugs/directive/query.test.ts +++ b/plugs/directive/query.test.ts @@ -1,6 +1,22 @@ import { assertEquals } from "../../test_deps.ts"; import { applyQuery } from "$sb/lib/query.ts"; -import { parseQuery } from "./parser.ts"; + +import wikiMarkdownLang from "../../common/markdown_parser/parser.ts"; +import { parse } from "../../common/markdown_parser/parse_tree.ts"; +import { parseQuery as parseQueryQuery } from "./parser.ts"; +import { findNodeOfType, renderToText } from "../../plug-api/lib/tree.ts"; + +function parseQuery(query: string) { + const lang = wikiMarkdownLang([]); + const mdTree = parse( + lang, + ` + + `, + ); + const programNode = findNodeOfType(mdTree, "Program")!; + return parseQueryQuery(programNode); +} Deno.test("Test parser", () => { const parsedBasicQuery = parseQuery(`page`); @@ -154,3 +170,14 @@ Deno.test("Test applyQuery with multi value", () => { ], ); }); + +const testQuery = ` + +`; + +Deno.test("Query parsing and serialization", () => { + const lang = wikiMarkdownLang([]); + const mdTree = parse(lang, testQuery); + // console.log(JSON.stringify(mdTree, null, 2)); + assertEquals(renderToText(mdTree), testQuery); +}); diff --git a/plugs/directive/query_directive.ts b/plugs/directive/query_directive.ts index 899f28cd..a3eb4d76 100644 --- a/plugs/directive/query_directive.ts +++ b/plugs/directive/query_directive.ts @@ -4,15 +4,21 @@ import { replaceTemplateVars } from "../core/template.ts"; import { renderTemplate } from "./util.ts"; import { parseQuery } from "./parser.ts"; import { jsonToMDTable } from "./util.ts"; +import { ParseTree } from "../../plug-api/lib/tree.ts"; export async function queryDirectiveRenderer( _directive: string, pageName: string, - query: string, + query: string | ParseTree, ): Promise { - const parsedQuery = parseQuery(replaceTemplateVars(query, pageName)); + if (typeof query === "string") { + throw new Error("Argument must be a ParseTree"); + } + const parsedQuery = parseQuery( + JSON.parse(replaceTemplateVars(JSON.stringify(query), pageName)), + ); - console.log("Parsed query", parsedQuery); + // console.log("Parsed query", parsedQuery); // Let's dispatch an event and see what happens const results = await events.dispatchEvent( `query:${parsedQuery.table}`, diff --git a/plugs/directive/template_directive.ts b/plugs/directive/template_directive.ts index d26378a0..9dc66d3e 100644 --- a/plugs/directive/template_directive.ts +++ b/plugs/directive/template_directive.ts @@ -1,20 +1,24 @@ import { queryRegex } from "$sb/lib/query.ts"; -import { renderToText } from "$sb/lib/tree.ts"; +import { ParseTree, renderToText } from "$sb/lib/tree.ts"; import { replaceAsync } from "$sb/lib/util.ts"; import { markdown, space } from "$sb/silverbullet-syscall/mod.ts"; import Handlebars from "handlebars"; import { replaceTemplateVars } from "../core/template.ts"; import { extractFrontmatter } from "$sb/lib/frontmatter.ts"; -import { directiveRegex, renderDirectives } from "./directives.ts"; +import { directiveRegex } from "./directives.ts"; +import { serverUpdateDirectives } from "./command.ts"; const templateRegex = /\[\[([^\]]+)\]\]\s*(.*)\s*/; export async function templateDirectiveRenderer( directive: string, pageName: string, - arg: string, + arg: string | ParseTree, ): Promise { + if (typeof arg !== "string") { + throw new Error("Template directives must be a string"); + } const match = arg.match(templateRegex); if (!match) { throw new Error(`Invalid template directive: ${arg}`); @@ -42,7 +46,7 @@ export async function templateDirectiveRenderer( } let newBody = templateText; // if it's a template injection (not a literal "include") - if (directive === "use" || directive === "use-verbose") { + if (directive === "use") { const tree = await markdown.parseMarkdown(templateText); extractFrontmatter(tree, ["$disableDirectives"]); templateText = renderToText(tree); @@ -53,7 +57,7 @@ export async function templateDirectiveRenderer( newBody = templateFn(parsedArgs); // Recursively render directives - newBody = await renderDirectives(pageName, newBody); + newBody = await serverUpdateDirectives(pageName, newBody); } return newBody.trim(); } diff --git a/plugs/global.plug.yaml b/plugs/global.plug.yaml index 3b17bb8c..72173591 100644 --- a/plugs/global.plug.yaml +++ b/plugs/global.plug.yaml @@ -2,4 +2,3 @@ name: global dependencies: "https://esm.sh/handlebars": "https://esm.sh/handlebars@4.7.7" "https://deno.land/std/encoding/yaml.ts": "https://deno.land/std@0.165.0/encoding/yaml.ts" - "https://esm.sh/@lezer/lr": "https://esm.sh/@lezer/lr@1.2.3" diff --git a/plugs/markdown/markdown_render.test.ts b/plugs/markdown/markdown_render.test.ts index 2596b92b..a7638bee 100644 --- a/plugs/markdown/markdown_render.test.ts +++ b/plugs/markdown/markdown_render.test.ts @@ -1,6 +1,5 @@ -import buildMarkdown from "../../common/parser.ts"; -import { parse } from "../../common/parse_tree.ts"; -import { renderHtml } from "./html_render.ts"; +import buildMarkdown from "../../common/markdown_parser/parser.ts"; +import { parse } from "../../common/markdown_parser/parse_tree.ts"; import { System } from "../../plugos/system.ts"; import corePlug from "../../dist_bundle/_plug/core.plug.json" assert { @@ -10,9 +9,9 @@ import tasksPlug from "../../dist_bundle/_plug/tasks.plug.json" assert { type: "json", }; import { createSandbox } from "../../plugos/environments/deno_sandbox.ts"; -import { loadMarkdownExtensions } from "../../common/markdown_ext.ts"; +import { loadMarkdownExtensions } from "../../common/markdown_parser/markdown_ext.ts"; import { renderMarkdownToHtml } from "./markdown_render.ts"; -import { assertEquals } from "https://deno.land/std@0.165.0/testing/asserts.ts"; +import { assertEquals } from "../../test_deps.ts"; Deno.test("Markdown render", async () => { const system = new System("server"); diff --git a/plugs/markdown/markdown_render.ts b/plugs/markdown/markdown_render.ts index 6a522021..4e82b4b4 100644 --- a/plugs/markdown/markdown_render.ts +++ b/plugs/markdown/markdown_render.ts @@ -362,6 +362,10 @@ function render( body: cleanTags(mapRender(newChildren)), }; } + case "Directive": { + const body = findNodeOfType(t, "DirectiveBody")!; + return posPreservingRender(body.children![0], options); + } // Text case undefined: return t.text!; diff --git a/plugs/tasks/task.ts b/plugs/tasks/task.ts index 3cc283ee..8d3308dd 100644 --- a/plugs/tasks/task.ts +++ b/plugs/tasks/task.ts @@ -40,9 +40,9 @@ function getDeadline(deadlineNode: ParseTree): string { } export async function indexTasks({ name, tree }: IndexTreeEvent) { - // console.log("Indexing tasks"); const tasks: { key: string; value: Task }[] = []; removeQueries(tree); + addParentPointers(tree); collectNodesOfType(tree, "Task").forEach((n) => { const complete = n.children![0].children![0].text! !== "[ ]"; const task: Task = { @@ -78,7 +78,6 @@ export async function indexTasks({ name, tree }: IndexTreeEvent) { key: `task:${n.from}`, value: task, }); - // console.log("Task", task); }); // console.log("Found", tasks.length, "task(s)"); @@ -132,7 +131,6 @@ async function toggleTaskMarker(node: ParseTree, moveToPos: number) { } taskMarkerNode.children![0].text = changeTo; text = renderToText(referenceMdTree); - console.log("Updated reference paged text", text); await space.writePage(page, text); } } diff --git a/server/space_system.ts b/server/space_system.ts index 1cd946cf..eb138756 100644 --- a/server/space_system.ts +++ b/server/space_system.ts @@ -1,6 +1,6 @@ import { SilverBulletHooks } from "../common/manifest.ts"; -import { loadMarkdownExtensions } from "../common/markdown_ext.ts"; -import buildMarkdown from "../common/parser.ts"; +import { loadMarkdownExtensions } from "../common/markdown_parser/markdown_ext.ts"; +import buildMarkdown from "../common/markdown_parser/parser.ts"; import { DiskSpacePrimitives } from "../common/spaces/disk_space_primitives.ts"; import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitives.ts"; import { Space } from "../common/spaces/space.ts"; diff --git a/web/cm_plugins/command_link.ts b/web/cm_plugins/command_link.ts index 51f67213..e63671cb 100644 --- a/web/cm_plugins/command_link.ts +++ b/web/cm_plugins/command_link.ts @@ -1,4 +1,4 @@ -import { commandLinkRegex } from "../../common/parser.ts"; +import { commandLinkRegex } from "../../common/markdown_parser/parser.ts"; import { ClickEvent } from "$sb/app_event.ts"; import { Decoration, syntaxTree } from "../deps.ts"; import { Editor } from "../editor.tsx"; diff --git a/web/cm_plugins/directive.ts b/web/cm_plugins/directive.ts index dd4faf3e..08181c44 100644 --- a/web/cm_plugins/directive.ts +++ b/web/cm_plugins/directive.ts @@ -10,49 +10,64 @@ import { export function directivePlugin() { return decoratorStateField((state: EditorState) => { const widgets: any[] = []; - const cursorPos = state.selection.main.head; - - // TODO: This doesn't handle nested directives properly - let posOfLastOpen = { from: 0, to: 0 }; syntaxTree(state).iterate({ - enter: ({ type, from, to }) => { - if (type.name !== "CommentBlock") { + enter: ({ type, from, to, node }) => { + const parent = node.parent; + + if (!parent) { return; } - const text = state.sliceDoc(from, to); - if (/` and `` comments), they just don’t update their content dynamically. +* [[🔌 Directive|Directives]] are disabled, although you will see them being used across this site (look for sections with subtle curved lines around them, if you move your cursor inside you’ll see where their content is generated from), they just don’t update their content dynamically. * **Full-text search**. * **Extending** and updating SB’s functionality by installing additional [[🔌 Plugs]] (SB parlance for plug-ins) and writing your own. diff --git a/website/🔌 Directive.md b/website/🔌 Directive.md index 614d2de4..e8d8f74f 100644 --- a/website/🔌 Directive.md +++ b/website/🔌 Directive.md @@ -33,11 +33,10 @@ which renders as follows: * [[🔌 Directive]] by **Pete** ([repo](https://google.com)) +* [ ] #test This is a test task Note that a string is also a valid JSON value: -* [ ] #test This is a test task - So, a template can take, for instance a tag name as an argument: @@ -48,6 +47,8 @@ So, a template can take, for instance a tag name as an argument: $eval The `#eval` directive can be used to evaluate arbitrary JavaScript expressions. It’s also possible to invoke arbitrary plug functions this way. +**Note:** This feature is experimental and will likely evolve. + A simple example is multiplying numbers: diff --git a/website/🔌 Directive/Query.md b/website/🔌 Directive/Query.md index 426cf20f..49ce9236 100644 --- a/website/🔌 Directive/Query.md +++ b/website/🔌 Directive/Query.md @@ -2,7 +2,6 @@ The `#query` is the most widely used directive. It can be used to query various data sources and render results in various ways. ### Syntax - 1. _start with_: `` 2. _end with_: `` 3. _write your query_: replace `[QUERY GOES HERE]` with any query you want using the options below. @@ -166,13 +165,13 @@ For the sake of simplicity, we will use the `page` data source and limit the res **Result:** Okay, this is what we wanted but there is also information such as `perm`, `type` and `lastModified` that we don't need. -|name |lastModified |contentType |size|perm|type|repo |share-support| -|--|--|--|--|--|--|--|--| -|🔌 Collab |1669545776517|text/markdown|2926|rw|plug|https://github.com/silverbulletmd/silverbullet|true| -|🔌 Tasks |1669536555227|text/markdown|1231|rw|plug|https://github.com/silverbulletmd/silverbullet| | -|🔌 Share |1669536545411|text/markdown|672 |rw|plug|https://github.com/silverbulletmd/silverbullet| | -|🔌 Markdown|1669536539800|text/markdown|268 |rw|plug|https://github.com/silverbulletmd/silverbullet|true| -|🔌 Emoji |1669536531680|text/markdown|155 |rw|plug|https://github.com/silverbulletmd/silverbullet| | +|name |lastModified |contentType |size|perm|type|repo |uri |author |share-support| +|--|--|--|--|--|--|--|--|--|--| +|🔌 Directive|1671044429696|text/markdown|2605|rw|plug|https://github.com/silverbulletmd/silverbullet | | | | +|🔌 Backlinks|1670833065065|text/markdown|960 |rw|plug|https://github.com/Willyfrog/silverbullet-backlinks|ghr:Willyfrog/silverbullet-backlinks|Guillermo Vayá| | +|🔌 Collab |1670435068917|text/markdown|2923|rw|plug|https://github.com/silverbulletmd/silverbullet | | |true| +|🔌 Tasks |1669536555227|text/markdown|1231|rw|plug|https://github.com/silverbulletmd/silverbullet | | | | +|🔌 Share |1669536545411|text/markdown|672 |rw|plug|https://github.com/silverbulletmd/silverbullet | | | | #### 6.3 Query to select only certain fields @@ -183,14 +182,14 @@ and `repo` columns and then sort by last modified time. **Result:** Okay, this is much better. However, I believe this needs a touch from a visual perspective. - -|name |author|repo |ri| -|-----------||----------------------------------------------|| -|🔌 Collab ||https://github.com/silverbulletmd/silverbullet|| -|🔌 Tasks ||https://github.com/silverbulletmd/silverbullet|| -|🔌 Share ||https://github.com/silverbulletmd/silverbullet|| -|🔌 Markdown||https://github.com/silverbulletmd/silverbullet|| -|🔌 Emoji ||https://github.com/silverbulletmd/silverbullet|| + +|name |author |repo |ri| +|--|--|--|--| +|🔌 Directive| |https://github.com/silverbulletmd/silverbullet || +|🔌 Backlinks|Guillermo Vayá|https://github.com/Willyfrog/silverbullet-backlinks|| +|🔌 Collab | |https://github.com/silverbulletmd/silverbullet || +|🔌 Tasks | |https://github.com/silverbulletmd/silverbullet || +|🔌 Share | |https://github.com/silverbulletmd/silverbullet || #### 6.4 Display the data in a format defined by a template @@ -199,12 +198,12 @@ from a visual perspective. **Result:** Here you go. This is the result we would like to achieve 🎉. Did you see how I used `render` and `template/plug` in a query? 🚀 - + +* [[🔌 Directive]] +* [[🔌 Backlinks]] by **Guillermo Vayá** ([repo](https://github.com/Willyfrog/silverbullet-backlinks)) * [[🔌 Collab]] * [[🔌 Tasks]] -* [[🔌 Share]] -* [[🔌 Markdown]] -* [[🔌 Emoji]] +* [[🔌 Share]] PS: You don't need to select only certain fields to use templates. Templates are diff --git a/website/🔌 Plugs.md b/website/🔌 Plugs.md index df4b6a8f..211547cd 100644 --- a/website/🔌 Plugs.md +++ b/website/🔌 Plugs.md @@ -25,10 +25,8 @@ These plugs are distributed with Silver Bullet and are automatically enabled: * [[🔌 Share]] * [[🔌 Tasks]] - ## Third-party plugs These plugs are written either by third parties or distributed separately from the main SB distribution: - * [[🔌 Backlinks]] by **Guillermo Vayá** ([repo](https://github.com/Willyfrog/silverbullet-backlinks)) * [[🔌 Ghost]] by **Zef Hemel** ([repo](https://github.com/silverbulletmd/silverbullet-ghost)) @@ -39,7 +37,6 @@ These plugs are written either by third parties or distributed separately from t * [[🔌 Serendipity]] by **Pantelis Vratsalis** ([repo](https://github.com/m1lt0n/silverbullet-serendipity)) * [[🔌 Twitter]] by **Silver Bullet Authors** ([repo](https://github.com/silverbulletmd/silverbullet-twitter)) - ## How to develop your own plug The easiest way to get started is to click the “Use this template” on the [silverbullet-plug-template](https://github.com/silverbulletmd/silverbullet-plug-template) repo.