diff --git a/common/spaces/evented_space_primitives.ts b/common/spaces/evented_space_primitives.ts index ccf5347b..714dd142 100644 --- a/common/spaces/evented_space_primitives.ts +++ b/common/spaces/evented_space_primitives.ts @@ -45,9 +45,9 @@ export class EventedSpacePrimitives implements SpacePrimitives { this.eventHook .dispatchEvent("page:saved", pageName) .then(() => { - return this.eventHook.dispatchEvent("page:index", { + return this.eventHook.dispatchEvent("page:index_text", { name: pageName, - text: text, + text, }); }) .catch((e) => { diff --git a/common/tree.ts b/common/tree.ts index ea4b1e85..f75c69b4 100644 --- a/common/tree.ts +++ b/common/tree.ts @@ -13,6 +13,10 @@ export function addParentPointers(tree: ParseTree) { return; } for (let child of tree.children) { + if (child.parent) { + // Already added parent pointers before + return; + } child.parent = tree; addParentPointers(child); } diff --git a/plugos-syscall/event.ts b/plugos-syscall/event.ts index 891386de..513bec53 100644 --- a/plugos-syscall/event.ts +++ b/plugos-syscall/event.ts @@ -6,13 +6,18 @@ export async function dispatch( timeout?: number ): Promise { return new Promise((resolve, reject) => { - let timeOut = setTimeout(() => { - console.log("Timeout!"); - reject("timeout"); - }, timeout); + let timeouter: any = -1; + if (timeout) { + timeouter = setTimeout(() => { + console.log("Timeout!"); + reject("timeout"); + }, timeout); + } syscall("event.dispatch", eventName, data) .then((r) => { - clearTimeout(timeOut); + if (timeouter !== -1) { + clearTimeout(timeouter); + } resolve(r); }) .catch(reject); diff --git a/plugs/core/core.plug.yaml b/plugs/core/core.plug.yaml index 03fe5858..4dccdc2a 100644 --- a/plugs/core/core.plug.yaml +++ b/plugs/core/core.plug.yaml @@ -29,6 +29,10 @@ functions: path: ./page.ts:pageQueryProvider events: - query:page + parseIndexTextRepublish: + path: "./page.ts:parseIndexTextRepublish" + events: + - page:index_text indexLinks: path: "./page.ts:indexLinks" events: diff --git a/plugs/core/dates.ts b/plugs/core/dates.ts index 0cad52e6..ce0e2248 100644 --- a/plugs/core/dates.ts +++ b/plugs/core/dates.ts @@ -1,30 +1,9 @@ import { insertAtCursor } from "plugos-silverbullet-syscall/editor"; -import { IndexEvent } from "../../webapp/app_event"; -import { batchSet } from "plugos-silverbullet-syscall"; -import { whiteOutQueries } from "../query/util"; const dateMatchRegex = /(\d{4}\-\d{2}\-\d{2})/g; -// Index key space: -// d:[date]:page@pos - -export async function indexDates({ name, text }: IndexEvent) { - let dates: { key: string; value: boolean }[] = []; - text = whiteOutQueries(text); - console.log("Now date indexing", name); - for (let match of text.matchAll(dateMatchRegex)) { - // console.log("Date match", match[0]); - dates.push({ - key: `d:${match[0]}:${name}@${match.index}`, - value: true, - }); - } - console.log("Found", dates.length, "dates"); - await batchSet(name, dates); -} - export function niceDate(d: Date): string { - return new Date().toISOString().split("T")[0]; + return d.toISOString().split("T")[0]; } export async function insertToday() { diff --git a/plugs/core/item.ts b/plugs/core/item.ts index e16074a7..0c4a8d2f 100644 --- a/plugs/core/item.ts +++ b/plugs/core/item.ts @@ -1,9 +1,8 @@ -import { IndexEvent } from "../../webapp/app_event"; +import { IndexTreeEvent } from "../../webapp/app_event"; import { batchSet, scanPrefixGlobal } from "plugos-silverbullet-syscall/index"; -import { parseMarkdown } from "plugos-silverbullet-syscall/markdown"; import { collectNodesOfType, ParseTree, renderToText } from "../../common/tree"; -import { whiteOutQueries } from "../query/util"; +import { removeQueries } from "../query/util"; import { applyQuery, QueryProviderEvent } from "../query/engine"; export type Item = { @@ -14,14 +13,13 @@ export type Item = { pos?: number; }; -export async function indexItems({ name, text }: IndexEvent) { +export async function indexItems({ name, tree }: IndexTreeEvent) { let items: { key: string; value: Item }[] = []; - text = whiteOutQueries(text); + removeQueries(tree); console.log("Indexing items", name); - let mdTree = await parseMarkdown(text); - let coll = collectNodesOfType(mdTree, "ListItem"); + let coll = collectNodesOfType(tree, "ListItem"); coll.forEach((n) => { if (!n.children) { diff --git a/plugs/core/page.ts b/plugs/core/page.ts index d4572f7f..e00b475c 100644 --- a/plugs/core/page.ts +++ b/plugs/core/page.ts @@ -1,4 +1,4 @@ -import { IndexEvent } from "../../webapp/app_event"; +import { IndexEvent, IndexTreeEvent } from "../../webapp/app_event"; import { batchSet, clearPageIndex as clearPageIndexSyscall, @@ -28,23 +28,20 @@ import { import { applyQuery, QueryProviderEvent } from "../query/engine"; import { PageMeta } from "../../common/types"; -export async function indexLinks({ name, text }: IndexEvent) { +export async function indexLinks({ name, tree }: IndexTreeEvent) { let backLinks: { key: string; value: string }[] = []; // [[Style Links]] console.log("Now indexing", name); - let mdTree = await parseMarkdown(text); - collectNodesMatching(mdTree, (n) => n.type === "WikiLinkPage").forEach( - (n) => { - let toPage = n.children![0].text!; - if (toPage.includes("@")) { - toPage = toPage.split("@")[0]; - } - backLinks.push({ - key: `pl:${toPage}:${n.from}`, - value: name, - }); + collectNodesMatching(tree, (n) => n.type === "WikiLinkPage").forEach((n) => { + let toPage = n.children![0].text!; + if (toPage.includes("@")) { + toPage = toPage.split("@")[0]; } - ); + backLinks.push({ + key: `pl:${toPage}:${n.from}`, + value: name, + }); + }); console.log("Found", backLinks.length, "wiki link(s)"); await batchSet(name, backLinks); } @@ -193,10 +190,11 @@ export async function reindexSpace() { let pages = await listPages(); for (let { name } of pages) { console.log("Indexing", name); - const pageObj = await readPage(name); + const { text } = await readPage(name); + let parsed = await parseMarkdown(text); await dispatch("page:index", { name, - text: pageObj.text, + tree: parsed, }); } } @@ -206,6 +204,13 @@ export async function clearPageIndex(page: string) { await clearPageIndexForPage(page); } +export async function parseIndexTextRepublish({ name, text }: IndexEvent) { + await dispatch("page:index", { + name, + tree: await parseMarkdown(text), + }); +} + export async function parseServerPageCommand() { console.log(await invokeFunction("server", "parsePage", await getText())); } diff --git a/plugs/core/template.ts b/plugs/core/template.ts index 505d57bd..53bef7d1 100644 --- a/plugs/core/template.ts +++ b/plugs/core/template.ts @@ -4,7 +4,6 @@ import { parseMarkdown } from "plugos-silverbullet-syscall/markdown"; import { extractMeta } from "../query/data"; import { renderToText } from "../../common/tree"; import { niceDate } from "./dates"; -import { dispatch } from "plugos-syscall/event"; import { invokeFunction } from "plugos-silverbullet-syscall/system"; const pageTemplatePrefix = `template/page/`; @@ -60,17 +59,17 @@ export async function replaceTemplateVarsCommand() { export function replaceTemplateVars(s: string, pageName: string): string { return s.replaceAll(/\{\{([^\}]+)\}\}/g, (match, v) => { - if (v === "today") { - return niceDate(new Date()); - } - if (v.startsWith("placeholder:")) { - // Dispatch event, to be replaced in the file async later - dispatch(v, { - pageName: pageName, - placeholder: v, - }).catch((e) => { - console.error("Failed to dispatch placeholder event", e); - }); + switch (v) { + case "today": + return niceDate(new Date()); + case "yesterday": + let yesterday = new Date(); + yesterday.setDate(yesterday.getDate() - 1); + return niceDate(yesterday); + case "lastWeek": + let lastWeek = new Date(); + lastWeek.setDate(lastWeek.getDate() - 7); + return niceDate(lastWeek); } return match; }); diff --git a/plugs/markdown/preview.ts b/plugs/markdown/preview.ts index 35510f62..4aa5dd50 100644 --- a/plugs/markdown/preview.ts +++ b/plugs/markdown/preview.ts @@ -15,6 +15,24 @@ body { padding-right: 20px; } +table { + width: 100%; + border-spacing: 0; +} + +thead tr { + background-color: #333; + color: #eee; +} + +th, td { + padding: 8px; +} + +tbody tr:nth-of-type(even) { + background-color: #f3f3f3; +} + a[href] { text-decoration: none; } diff --git a/plugs/mattermost/client.ts b/plugs/mattermost/client.ts index fe3b370a..378f815b 100644 --- a/plugs/mattermost/client.ts +++ b/plugs/mattermost/client.ts @@ -1,11 +1,12 @@ import { readPage } from "plugos-silverbullet-syscall/space"; import { parseMarkdown } from "plugos-silverbullet-syscall/markdown"; import { extractMeta } from "../query/data"; -import { UserProfile } from "@hmhealey/types/lib/users"; import { json } from "plugos-syscall/fetch"; -import { Post } from "@hmhealey/types/lib/posts"; -import { Channel } from "@hmhealey/types/lib/channels"; -import { Team } from "@hmhealey/types/lib/teams"; +import type { UserProfile } from "@mattermost/types/lib/users"; +import type { Post } from "@mattermost/types/lib/posts"; +import type { Channel } from "@mattermost/types/lib/channels"; +import type { Team } from "@mattermost/types/lib/teams"; +import { niceDate } from "../core/dates"; type MattermostConfig = { url: string; @@ -19,6 +20,13 @@ async function getConfig(): Promise { return pageMeta as MattermostConfig; } +type AugmentedPost = Post & { + // Dates we can use to filter + createdAt: string; + updatedAt: string; + editedAt: string; +}; + export class MattermostClient { userCache = new Map(); channelCache = new Map(); @@ -77,7 +85,10 @@ export class MattermostClient { return team!; } - async getFlaggedPosts(userId: string, perPage: number = 10): Promise { + async getFlaggedPosts( + userId: string, + perPage: number = 10 + ): Promise { let postCollection = await json( `${this.url}/api/v4/users/${userId}/posts/flagged?per_page=${perPage}`, { @@ -86,10 +97,24 @@ export class MattermostClient { }, } ); - let posts: Post[] = []; + let posts: AugmentedPost[] = []; for (let order of postCollection.order) { - posts.push(postCollection.posts[order]); + let post = postCollection.posts[order]; + augmentPost(post); + posts.push(post); } return posts; } } + +function augmentPost(post: AugmentedPost) { + if (post.create_at) { + post.createdAt = niceDate(new Date(post.create_at)); + } + if (post.update_at) { + post.updatedAt = niceDate(new Date(post.update_at)); + } + if (post.edit_at) { + post.editedAt = niceDate(new Date(post.edit_at)); + } +} diff --git a/plugs/mattermost/mattermost.ts b/plugs/mattermost/mattermost.ts index cf7fcffa..c6034208 100644 --- a/plugs/mattermost/mattermost.ts +++ b/plugs/mattermost/mattermost.ts @@ -20,17 +20,16 @@ export async function savedPostsQueryProvider({ let savedPostsMd = []; savedPosts = applyQuery(query, savedPosts); for (let savedPost of savedPosts) { - // savedPost. let channel = await client.getChannel(savedPost.channel_id); let team = await client.getTeam(channel.team_id); savedPostsMd.push( - `@${ - (await client.getUser(savedPost.user_id)).username - } [link](${mattermostDesktopUrlForPost( + `@${(await client.getUser(savedPost.user_id)).username} [${ + savedPost.createdAt + }](${mattermostDesktopUrlForPost( client.url, team.name, savedPost.id - )}):\n> ${savedPost.message.replaceAll(/\n/g, "\n> ")}` + )}):\n> ${savedPost.message.substring(0, 1000).replaceAll(/\n/g, "\n> ")}` ); } return savedPostsMd.join("\n\n"); diff --git a/plugs/package-lock.json b/plugs/package-lock.json index 0bff9536..bbb9f9fe 100644 --- a/plugs/package-lock.json +++ b/plugs/package-lock.json @@ -8,10 +8,10 @@ "name": "plugs", "version": "1.0.0", "dependencies": { - "@hmhealey/types": "^6.6.0-4", "@jest/globals": "^27.5.1", "@lezer/generator": "^0.15.4", "@lezer/lr": "^0.15.8", + "@mattermost/types": "^6.7.0-0", "@types/yaml": "^1.9.7", "plugos-silverbullet-syscall": "file:../plugos-silverbullet-syscall", "plugos-syscall": "file:../plugos-syscall", @@ -120,19 +120,6 @@ "node": ">=4" } }, - "node_modules/@hmhealey/types": { - "version": "6.6.0-4", - "resolved": "https://registry.npmjs.org/@hmhealey/types/-/types-6.6.0-4.tgz", - "integrity": "sha512-71IxVaXhrUesmLnvQQh4RtUqqhmVL+ejci4qo4R6rTWTdY77BniRtBx269uAz34wzTlAgITysN8x7MBTdt/XBg==", - "peerDependencies": { - "typescript": "^4.3" - }, - "peerDependenciesMeta": { - "typescript": { - "optional": true - } - } - }, "node_modules/@jest/environment": { "version": "27.5.1", "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-27.5.1.tgz", @@ -216,6 +203,19 @@ "@lezer/common": "^0.15.0" } }, + "node_modules/@mattermost/types": { + "version": "6.7.0-0", + "resolved": "https://registry.npmjs.org/@mattermost/types/-/types-6.7.0-0.tgz", + "integrity": "sha512-mT8wJwWEp20KPo9D12y7bW7EdUHO7VhUHxr3gH8nPGapWooGcl0Ra0H3u1iCjPpqPWvp7LiodcneU0IysunYKQ==", + "peerDependencies": { + "typescript": "^4.3" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, "node_modules/@sinonjs/commons": { "version": "1.8.3", "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.3.tgz", @@ -697,12 +697,6 @@ } } }, - "@hmhealey/types": { - "version": "6.6.0-4", - "resolved": "https://registry.npmjs.org/@hmhealey/types/-/types-6.6.0-4.tgz", - "integrity": "sha512-71IxVaXhrUesmLnvQQh4RtUqqhmVL+ejci4qo4R6rTWTdY77BniRtBx269uAz34wzTlAgITysN8x7MBTdt/XBg==", - "requires": {} - }, "@jest/environment": { "version": "27.5.1", "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-27.5.1.tgz", @@ -771,6 +765,12 @@ "@lezer/common": "^0.15.0" } }, + "@mattermost/types": { + "version": "6.7.0-0", + "resolved": "https://registry.npmjs.org/@mattermost/types/-/types-6.7.0-0.tgz", + "integrity": "sha512-mT8wJwWEp20KPo9D12y7bW7EdUHO7VhUHxr3gH8nPGapWooGcl0Ra0H3u1iCjPpqPWvp7LiodcneU0IysunYKQ==", + "requires": {} + }, "@sinonjs/commons": { "version": "1.8.3", "resolved": "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.3.tgz", diff --git a/plugs/package.json b/plugs/package.json index a4cc2920..4f737180 100644 --- a/plugs/package.json +++ b/plugs/package.json @@ -5,7 +5,7 @@ "generate": "lezer-generator query/query.grammar -o query/parse-query.js" }, "dependencies": { - "@hmhealey/types": "^6.6.0-4", + "@mattermost/types": "^6.7.0-0", "@jest/globals": "^27.5.1", "@lezer/generator": "^0.15.4", "@lezer/lr": "^0.15.8", diff --git a/plugs/query/data.ts b/plugs/query/data.ts index 12ed32d5..c07c0338 100644 --- a/plugs/query/data.ts +++ b/plugs/query/data.ts @@ -1,23 +1,17 @@ // Index key space: // data:page@pos -import { IndexEvent } from "../../webapp/app_event"; +import { IndexTreeEvent } from "../../webapp/app_event"; import { batchSet, scanPrefixGlobal } from "plugos-silverbullet-syscall"; -import { parseMarkdown } from "plugos-silverbullet-syscall/markdown"; import { collectNodesOfType, findNodeOfType, ParseTree, replaceNodesMatching } from "../../common/tree"; import YAML, { parse as parseYaml, parseAllDocuments } from "yaml"; -import { whiteOutQueries } from "./util"; import type { QueryProviderEvent } from "./engine"; import { applyQuery } from "./engine"; -export async function indexData({ name, text }: IndexEvent) { - text = whiteOutQueries(text); - // console.log("Now data indexing", name); - let mdTree = await parseMarkdown(text); - +export async function indexData({ name, tree }: IndexTreeEvent) { let dataObjects: { key: string; value: Object }[] = []; - collectNodesOfType(mdTree, "FencedCode").forEach((t) => { + collectNodesOfType(tree, "FencedCode").forEach((t) => { let codeInfoNode = findNodeOfType(t, "CodeInfo"); if (!codeInfoNode) { return; diff --git a/plugs/query/materialized_queries.ts b/plugs/query/materialized_queries.ts index 637eb7d4..d112b8e7 100644 --- a/plugs/query/materialized_queries.ts +++ b/plugs/query/materialized_queries.ts @@ -1,30 +1,18 @@ -import { flashNotification, getCurrentPage, reloadPage, save } from "plugos-silverbullet-syscall/editor"; +import { flashNotification, getCurrentPage, getText, reloadPage, save } from "plugos-silverbullet-syscall/editor"; import { readPage, writePage } from "plugos-silverbullet-syscall/space"; import { invokeFunction } from "plugos-silverbullet-syscall/system"; import { parseQuery } from "./engine"; import { replaceTemplateVars } from "../core/template"; -import { queryRegex } from "./util"; +import { queryRegex, removeQueries } from "./util"; import { dispatch } from "plugos-syscall/event"; - -async function replaceAsync( - str: string, - regex: RegExp, - asyncFn: (match: string, ...args: any[]) => Promise -) { - const promises: Promise[] = []; - str.replace(regex, (match: string, ...args: any[]): string => { - const promise = asyncFn(match, ...args); - promises.push(promise); - return ""; - }); - const data = await Promise.all(promises); - return str.replace(regex, () => data.shift()!); -} +import { replaceAsync } from "../lib/util"; +import { parseMarkdown } from "plugos-silverbullet-syscall/markdown"; export async function updateMaterializedQueriesCommand() { const currentPage = await getCurrentPage(); await save(); + await flashNotification("Updating materialized queries..."); await invokeFunction( "server", "updateMaterializedQueriesOnPage", @@ -34,6 +22,12 @@ export async function updateMaterializedQueriesCommand() { await flashNotification("Updated materialized queries"); } +export async function whiteOutQueriesCommand() { + const text = await getText(); + const parsed = await parseMarkdown(text); + console.log(removeQueries(parsed)); +} + // Called from client, running on server export async function updateMaterializedQueriesOnPage(pageName: string) { let { text } = await readPage(pageName); @@ -49,7 +43,7 @@ export async function updateMaterializedQueriesOnPage(pageName: string) { let results = await dispatch( `query:${parsedQuery.table}`, { query: parsedQuery, pageName: pageName }, - 5000 + 10 * 1000 ); if (results.length === 0) { return `${startQuery}\n${endQuery}`; diff --git a/plugs/query/parse-query.js b/plugs/query/parse-query.js index 0bca3b60..6fe25cb3 100644 --- a/plugs/query/parse-query.js +++ b/plugs/query/parse-query.js @@ -3,15 +3,19 @@ import { LRParser } from "@lezer/lr"; export const parser = LRParser.deserialize({ version: 13, - states: "$[OVQPOOO[QQO'#C^QOQPOOOjQPO'#C`OoQQO'#CjOtQPO'#ClOOQO'#Cm'#CmOyQQO,58xO!XQPO'#CcO!sQQO'#CaOOQO'#Ca'#CaOOQO,58z,58zO#UQPO,59UOOQO,59W,59WOOQO-E6k-E6kO#ZQQO,58}OjQPO,58|O#oQQO1G.pOOQO'#Cg'#CgOOQO'#Ci'#CiOOQO'#Cd'#CdOOQO1G.i1G.iOOQO1G.h1G.hOOQO'#Ck'#CkOOQO7+$[7+$[", - stateData: "$T~OdOS~ORPO~OeROrSOvTObQX~ORWO~Os[O~OX]O~OeROrSOvTObQa~Of_Oj_Ok_Ol_Om_On_Oo_Op_O~Oq`ObTXeTXrTXvTX~ORaO~OXdOYdO[dOgbOhbOicO~OtgOugOb^ie^ir^iv^i~O", + states: + "$[OVQPOOO[QQO'#C^QOQPOOOjQPO'#C`OoQQO'#CjOtQPO'#ClOOQO'#Cm'#CmOyQQO,58xO!XQPO'#CcO!sQQO'#CaOOQO'#Ca'#CaOOQO,58z,58zO#UQPO,59UOOQO,59W,59WOOQO-E6k-E6kO#ZQQO,58}OjQPO,58|O#oQQO1G.pOOQO'#Cg'#CgOOQO'#Ci'#CiOOQO'#Cd'#CdOOQO1G.i1G.iOOQO1G.h1G.hOOQO'#Ck'#CkOOQO7+$[7+$[", + stateData: + "$T~OdOS~ORPO~OeROrSOvTObQX~ORWO~Os[O~OX]O~OeROrSOvTObQa~Of_Oj_Ok_Ol_Om_On_Oo_Op_O~Oq`ObTXeTXrTXvTX~ORaO~OXdOYdO[dOgbOhbOicO~OtgOugOb^ie^ir^iv^i~O", goto: "!VbPPcPfjmpvPPyPyf|f!PRQOTUPVRZRRYRQXRRf`Re_Rd_RhaQVPR^V", - nodeNames: "⚠ Program Query Name WhereClause LogicalExpr AndExpr FilterExpr Value Number String Bool Regex Null OrderClause Order LimitClause", + nodeNames: + "⚠ Program Query Name WhereClause LogicalExpr AndExpr FilterExpr Value Number String Bool Regex Null OrderClause Order LimitClause", maxTerm: 38, skippedNodes: [0], repeatNodeCount: 1, - tokenData: ":W~RvX^#ipq#iqr$^rs$q}!O%]!P!Q%n!Q![&e!^!_&m!_!`&z!`!a'X!c!}%]#R#S%]#T#U'f#U#V){#V#W%]#W#X*w#X#Y%]#Y#Z,s#Z#`%]#`#a/T#a#b%]#b#c1h#c#d3d#d#h%]#h#i5w#i#k%]#k#l7s#l#o%]#y#z#i$f$g#i#BY#BZ#i$IS$I_#i$Ip$Iq$q$Iq$Ir$q$I|$JO#i$JT$JU#i$KV$KW#i&FU&FV#i~#nYd~X^#ipq#i#y#z#i$f$g#i#BY#BZ#i$IS$I_#i$I|$JO#i$JT$JU#i$KV$KW#i&FU&FV#i~$aP!_!`$d~$iPl~#r#s$l~$qOp~~$tUOr$qrs%Ws$Ip$q$Ip$Iq%W$Iq$Ir%W$Ir~$q~%]OY~P%bSRP}!O%]!c!}%]#R#S%]#T#o%]~%sV[~OY%nZ]%n^!P%n!P!Q&Y!Q#O%n#O#P&_#P~%n~&_O[~~&bPO~%n~&jPX~!Q![&e~&rPf~!_!`&u~&zOj~~'PPk~#r#s'S~'XOo~~'^Pn~!_!`'a~'fOm~R'kWRP}!O%]!c!}%]#R#S%]#T#b%]#b#c(T#c#g%]#g#h)P#h#o%]R(YURP}!O%]!c!}%]#R#S%]#T#W%]#W#X(l#X#o%]R(sSqQRP}!O%]!c!}%]#R#S%]#T#o%]R)UURP}!O%]!c!}%]#R#S%]#T#V%]#V#W)h#W#o%]R)oSuQRP}!O%]!c!}%]#R#S%]#T#o%]R*QURP}!O%]!c!}%]#R#S%]#T#m%]#m#n*d#n#o%]R*kSsQRP}!O%]!c!}%]#R#S%]#T#o%]R*|URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y+`#Y#o%]R+eURP}!O%]!c!}%]#R#S%]#T#g%]#g#h+w#h#o%]R+|URP}!O%]!c!}%]#R#S%]#T#V%]#V#W,`#W#o%]R,gStQRP}!O%]!c!}%]#R#S%]#T#o%]R,xTRP}!O%]!c!}%]#R#S%]#T#U-X#U#o%]R-^URP}!O%]!c!}%]#R#S%]#T#`%]#`#a-p#a#o%]R-uURP}!O%]!c!}%]#R#S%]#T#g%]#g#h.X#h#o%]R.^URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y.p#Y#o%]R.wShQRP}!O%]!c!}%]#R#S%]#T#o%]R/YURP}!O%]!c!}%]#R#S%]#T#]%]#]#^/l#^#o%]R/qURP}!O%]!c!}%]#R#S%]#T#a%]#a#b0T#b#o%]R0YURP}!O%]!c!}%]#R#S%]#T#]%]#]#^0l#^#o%]R0qURP}!O%]!c!}%]#R#S%]#T#h%]#h#i1T#i#o%]R1[SvQRP}!O%]!c!}%]#R#S%]#T#o%]R1mURP}!O%]!c!}%]#R#S%]#T#i%]#i#j2P#j#o%]R2UURP}!O%]!c!}%]#R#S%]#T#`%]#`#a2h#a#o%]R2mURP}!O%]!c!}%]#R#S%]#T#`%]#`#a3P#a#o%]R3WSiQRP}!O%]!c!}%]#R#S%]#T#o%]R3iURP}!O%]!c!}%]#R#S%]#T#f%]#f#g3{#g#o%]R4QURP}!O%]!c!}%]#R#S%]#T#W%]#W#X4d#X#o%]R4iURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y4{#Y#o%]R5QURP}!O%]!c!}%]#R#S%]#T#f%]#f#g5d#g#o%]R5kSrQRP}!O%]!c!}%]#R#S%]#T#o%]R5|URP}!O%]!c!}%]#R#S%]#T#f%]#f#g6`#g#o%]R6eURP}!O%]!c!}%]#R#S%]#T#i%]#i#j6w#j#o%]R6|URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y7`#Y#o%]R7gSgQRP}!O%]!c!}%]#R#S%]#T#o%]R7xURP}!O%]!c!}%]#R#S%]#T#[%]#[#]8[#]#o%]R8aURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y8s#Y#o%]R8xURP}!O%]!c!}%]#R#S%]#T#f%]#f#g9[#g#o%]R9aURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y9s#Y#o%]R9zSeQRP}!O%]!c!}%]#R#S%]#T#o%]", + tokenData: + ":W~RvX^#ipq#iqr$^rs$q}!O%]!P!Q%n!Q![&e!^!_&m!_!`&z!`!a'X!c!}%]#R#S%]#T#U'f#U#V){#V#W%]#W#X*w#X#Y%]#Y#Z,s#Z#`%]#`#a/T#a#b%]#b#c1h#c#d3d#d#h%]#h#i5w#i#k%]#k#l7s#l#o%]#y#z#i$f$g#i#BY#BZ#i$IS$I_#i$Ip$Iq$q$Iq$Ir$q$I|$JO#i$JT$JU#i$KV$KW#i&FU&FV#i~#nYd~X^#ipq#i#y#z#i$f$g#i#BY#BZ#i$IS$I_#i$I|$JO#i$JT$JU#i$KV$KW#i&FU&FV#i~$aP!_!`$d~$iPl~#r#s$l~$qOp~~$tUOr$qrs%Ws$Ip$q$Ip$Iq%W$Iq$Ir%W$Ir~$q~%]OY~P%bSRP}!O%]!c!}%]#R#S%]#T#o%]~%sV[~OY%nZ]%n^!P%n!P!Q&Y!Q#O%n#O#P&_#P~%n~&_O[~~&bPO~%n~&jPX~!Q![&e~&rPf~!_!`&u~&zOj~~'PPk~#r#s'S~'XOo~~'^Pn~!_!`'a~'fOm~R'kWRP}!O%]!c!}%]#R#S%]#T#b%]#b#c(T#c#g%]#g#h)P#h#o%]R(YURP}!O%]!c!}%]#R#S%]#T#W%]#W#X(l#X#o%]R(sSqQRP}!O%]!c!}%]#R#S%]#T#o%]R)UURP}!O%]!c!}%]#R#S%]#T#V%]#V#W)h#W#o%]R)oSuQRP}!O%]!c!}%]#R#S%]#T#o%]R*QURP}!O%]!c!}%]#R#S%]#T#m%]#m#n*d#n#o%]R*kSsQRP}!O%]!c!}%]#R#S%]#T#o%]R*|URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y+`#Y#o%]R+eURP}!O%]!c!}%]#R#S%]#T#g%]#g#h+w#h#o%]R+|URP}!O%]!c!}%]#R#S%]#T#V%]#V#W,`#W#o%]R,gStQRP}!O%]!c!}%]#R#S%]#T#o%]R,xTRP}!O%]!c!}%]#R#S%]#T#U-X#U#o%]R-^URP}!O%]!c!}%]#R#S%]#T#`%]#`#a-p#a#o%]R-uURP}!O%]!c!}%]#R#S%]#T#g%]#g#h.X#h#o%]R.^URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y.p#Y#o%]R.wShQRP}!O%]!c!}%]#R#S%]#T#o%]R/YURP}!O%]!c!}%]#R#S%]#T#]%]#]#^/l#^#o%]R/qURP}!O%]!c!}%]#R#S%]#T#a%]#a#b0T#b#o%]R0YURP}!O%]!c!}%]#R#S%]#T#]%]#]#^0l#^#o%]R0qURP}!O%]!c!}%]#R#S%]#T#h%]#h#i1T#i#o%]R1[SvQRP}!O%]!c!}%]#R#S%]#T#o%]R1mURP}!O%]!c!}%]#R#S%]#T#i%]#i#j2P#j#o%]R2UURP}!O%]!c!}%]#R#S%]#T#`%]#`#a2h#a#o%]R2mURP}!O%]!c!}%]#R#S%]#T#`%]#`#a3P#a#o%]R3WSiQRP}!O%]!c!}%]#R#S%]#T#o%]R3iURP}!O%]!c!}%]#R#S%]#T#f%]#f#g3{#g#o%]R4QURP}!O%]!c!}%]#R#S%]#T#W%]#W#X4d#X#o%]R4iURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y4{#Y#o%]R5QURP}!O%]!c!}%]#R#S%]#T#f%]#f#g5d#g#o%]R5kSrQRP}!O%]!c!}%]#R#S%]#T#o%]R5|URP}!O%]!c!}%]#R#S%]#T#f%]#f#g6`#g#o%]R6eURP}!O%]!c!}%]#R#S%]#T#i%]#i#j6w#j#o%]R6|URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y7`#Y#o%]R7gSgQRP}!O%]!c!}%]#R#S%]#T#o%]R7xURP}!O%]!c!}%]#R#S%]#T#[%]#[#]8[#]#o%]R8aURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y8s#Y#o%]R8xURP}!O%]!c!}%]#R#S%]#T#f%]#f#g9[#g#o%]R9aURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y9s#Y#o%]R9zSeQRP}!O%]!c!}%]#R#S%]#T#o%]", tokenizers: [0, 1], - topRules: {"Program":[0,1]}, - tokenPrec: 0 -}) + topRules: { Program: [0, 1] }, + tokenPrec: 0, +}); diff --git a/plugs/query/query.plug.yaml b/plugs/query/query.plug.yaml index dabb8ba2..0271ceb1 100644 --- a/plugs/query/query.plug.yaml +++ b/plugs/query/query.plug.yaml @@ -6,6 +6,10 @@ functions: command: name: "Materialized Queries: Update" key: "Alt-q" + whiteOutQueriesCommand: + path: ./materialized_queries.ts:whiteOutQueriesCommand + command: + name: "Debug: Whiteout Queries" indexData: path: ./data.ts:indexData events: diff --git a/plugs/query/util.ts b/plugs/query/util.ts index 2826807f..8142a521 100644 --- a/plugs/query/util.ts +++ b/plugs/query/util.ts @@ -1,8 +1,45 @@ +import { addParentPointers, collectNodesMatching, ParseTree, renderToText } from "../../common/tree"; + export const queryRegex = /()(.+?)()/gs; -export function whiteOutQueries(text: string): string { - return text.replaceAll(queryRegex, (match) => - new Array(match.length + 1).join(" ") - ); +export const queryStartRegex = //s; + +export const queryEndRegex = //s; + +// export function whiteOutQueries(text: string): string { +// return text.replaceAll(queryRegex, (match) => +// new Array(match.length + 1).join(" ") +// ); +// } + +export function removeQueries(pt: ParseTree) { + addParentPointers(pt); + collectNodesMatching(pt, (t) => { + if (t.type !== "CommentBlock") { + return false; + } + let text = t.children![0].text!; + if (!queryStartRegex.exec(text)) { + return false; + } + let parentChildren = t.parent!.children!; + let index = parentChildren.indexOf(t); + let nodesToReplace: ParseTree[] = []; + for (let i = index + 1; i < parentChildren.length; i++) { + let n = parentChildren[i]; + if (n.type === "CommentBlock") { + let text = n.children![0].text!; + if (queryEndRegex.exec(text)) { + break; + } + } + nodesToReplace.push(n); + } + let renderedText = nodesToReplace.map(renderToText).join(""); + parentChildren.splice(index + 1, nodesToReplace.length, { + text: new Array(renderedText.length + 1).join(" "), + }); + return true; + }); } diff --git a/plugs/tasks/task.ts b/plugs/tasks/task.ts index 7b015304..9aa8aef8 100644 --- a/plugs/tasks/task.ts +++ b/plugs/tasks/task.ts @@ -1,4 +1,4 @@ -import type { ClickEvent, IndexEvent } from "../../webapp/app_event"; +import type { ClickEvent, IndexTreeEvent } from "../../webapp/app_event"; import { batchSet, scanPrefixGlobal } from "plugos-silverbullet-syscall/index"; import { readPage, writePage } from "plugos-silverbullet-syscall/space"; @@ -11,7 +11,7 @@ import { nodeAtPos, renderToText } from "../../common/tree"; -import { whiteOutQueries } from "../query/util"; +import { removeQueries } from "../query/util"; import { applyQuery, QueryProviderEvent } from "../query/engine"; export type Task = { @@ -24,13 +24,11 @@ export type Task = { page?: string; }; -export async function indexTasks({ name, text }: IndexEvent) { +export async function indexTasks({ name, tree }: IndexTreeEvent) { // console.log("Indexing tasks"); let tasks: { key: string; value: Task }[] = []; - text = whiteOutQueries(text); - let mdTree = await parseMarkdown(text); - addParentPointers(mdTree); - collectNodesOfType(mdTree, "Task").forEach((n) => { + removeQueries(tree); + collectNodesOfType(tree, "Task").forEach((n) => { let task = n.children!.slice(1).map(renderToText).join("").trim(); let complete = n.children![0].children![0].text! !== "[ ]"; let value: Task = { diff --git a/plugs/yarn.lock b/plugs/yarn.lock index 9a6bbf1d..5ffe0286 100644 --- a/plugs/yarn.lock +++ b/plugs/yarn.lock @@ -23,11 +23,6 @@ "chalk" "^2.0.0" "js-tokens" "^4.0.0" -"@hmhealey/types@^6.6.0-4": - "integrity" "sha512-71IxVaXhrUesmLnvQQh4RtUqqhmVL+ejci4qo4R6rTWTdY77BniRtBx269uAz34wzTlAgITysN8x7MBTdt/XBg==" - "resolved" "https://registry.npmjs.org/@hmhealey/types/-/types-6.6.0-4.tgz" - "version" "6.6.0-4" - "@jest/environment@^27.5.1": "integrity" "sha512-/WQjhPJe3/ghaol/4Bq480JKXV/Rfw8nQdN7f41fM8VDHLcxKXou6QyXAh3EFr9/bVG3x74z1NWDkP87EiY8gA==" "resolved" "https://registry.npmjs.org/@jest/environment/-/environment-27.5.1.tgz" @@ -90,6 +85,11 @@ dependencies: "@lezer/common" "^0.15.0" +"@mattermost/types@^6.7.0-0": + "integrity" "sha512-mT8wJwWEp20KPo9D12y7bW7EdUHO7VhUHxr3gH8nPGapWooGcl0Ra0H3u1iCjPpqPWvp7LiodcneU0IysunYKQ==" + "resolved" "https://registry.npmjs.org/@mattermost/types/-/types-6.7.0-0.tgz" + "version" "6.7.0-0" + "@sinonjs/commons@^1.7.0": "integrity" "sha512-xkNcLAn/wZaX14RPlwizcKicDk9G3F8m2nU3L7Ukm5zBgTwiT0wsoFAHx9Jq56fJA1z/7uKGtCRu16sOUCLIHQ==" "resolved" "https://registry.npmjs.org/@sinonjs/commons/-/commons-1.8.3.tgz" diff --git a/webapp/app_event.ts b/webapp/app_event.ts index b1d8bbf8..b6c9a0fc 100644 --- a/webapp/app_event.ts +++ b/webapp/app_event.ts @@ -1,3 +1,5 @@ +import type { ParseTree } from "../common/tree"; + export type AppEvent = "page:click" | "page:complete"; export type ClickEvent = { @@ -12,3 +14,8 @@ export type IndexEvent = { name: string; text: string; }; + +export type IndexTreeEvent = { + name: string; + tree: ParseTree; +}; diff --git a/webapp/editor.tsx b/webapp/editor.tsx index 731e6b37..be7dac07 100644 --- a/webapp/editor.tsx +++ b/webapp/editor.tsx @@ -300,7 +300,6 @@ export class Editor { closeBrackets(), autocompletion({ override: [ - // this.completerHook.plugCompleter.bind(this.completerHook), this.completer.bind(this), this.slashCommandHook.slashCommandCompleter.bind( this.slashCommandHook @@ -320,6 +319,7 @@ export class Editor { { selector: "Comment", class: "line-comment" }, { selector: "BulletList", class: "line-ul" }, { selector: "OrderedList", class: "line-ol" }, + { selector: "TableHeader", class: "line-tbl-header" }, ]), keymap.of([ ...smartQuoteKeymap, @@ -433,6 +433,9 @@ export class Editor { editorView.setState( this.createEditorState(this.currentPage, editorView.state.sliceDoc()) ); + if (editorView.contentDOM) { + editorView.contentDOM.spellcheck = true; + } } } @@ -503,6 +506,9 @@ export class Editor { let editorState = this.createEditorState(pageName, doc.text); let pageState = this.openPages.get(pageName); editorView.setState(editorState); + if (editorView.contentDOM) { + editorView.contentDOM.spellcheck = true; + } if (!pageState) { pageState = new PageState(0, editorState.selection); this.openPages.set(pageName, pageState!); diff --git a/webapp/markdown/markdown.ts b/webapp/markdown/markdown.ts index 8d50bea5..e63e2548 100644 --- a/webapp/markdown/markdown.ts +++ b/webapp/markdown/markdown.ts @@ -5,17 +5,10 @@ import { Language, languageDataProp, LanguageDescription, - ParseContext, + ParseContext } from "@codemirror/language"; import { styleTags, tags as t } from "@codemirror/highlight"; -import { - Emoji, - GFM, - MarkdownParser, - parser as baseParser, - Subscript, - Superscript, -} from "@lezer/markdown"; +import { Emoji, GFM, MarkdownParser, parser as baseParser, Subscript, Superscript } from "@lezer/markdown"; const data = defineLanguageFacet({ block: { open: "" } }); @@ -37,8 +30,6 @@ export const commonmark = baseParser.configure({ "StrongEmphasis/...": t.strong, "Link/... Image/...": t.link, "OrderedList/... BulletList/...": t.list, - - // "CodeBlock/... FencedCode/...": t.blockComment, "InlineCode CodeText": t.monospace, URL: t.url, "HeaderMark HardBreak QuoteMark ListMark LinkMark EmphasisMark CodeMark": diff --git a/webapp/parser.ts b/webapp/parser.ts index 9a37bb37..baebccf6 100644 --- a/webapp/parser.ts +++ b/webapp/parser.ts @@ -1,5 +1,5 @@ -import { styleTags } from "@codemirror/highlight"; -import { BlockContext, LeafBlock, LeafBlockParser, MarkdownConfig, parseCode, TaskList } from "@lezer/markdown"; +import { styleTags, tags as t } from "@codemirror/highlight"; +import { BlockContext, LeafBlock, LeafBlockParser, MarkdownConfig, parseCode, Table, TaskList } from "@lezer/markdown"; import { commonmark, getCodeParser, mkLang } from "./markdown/markdown"; import * as ct from "./customtags"; import { Language, LanguageDescription, LanguageSupport } from "@codemirror/language"; @@ -73,6 +73,7 @@ export default function buildMarkdown(mdExtensions: MDExt[]): Language { WikiLink, TaskList, Comment, + Table, ...mdExtensions.map(mdExtensionSyntaxConfig), parseCode({ codeParser: getCodeParser([ @@ -96,6 +97,10 @@ export default function buildMarkdown(mdExtensions: MDExt[]): Language { Task: ct.TaskTag, TaskMarker: ct.TaskMarkerTag, Comment: ct.CommentTag, + "TableDelimiter SubscriptMark SuperscriptMark StrikethroughMark": + t.processingInstruction, + "TableHeader/...": t.heading, + TableCell: t.content, }), ...mdExtensions.map((mdExt) => styleTags(mdExtensionStyleTags(mdExt)) diff --git a/webapp/style.ts b/webapp/style.ts index 56f003c0..a59ba124 100644 --- a/webapp/style.ts +++ b/webapp/style.ts @@ -36,6 +36,8 @@ export default function highlightStyles(mdExtension: MDExt[]) { { tag: t.variableName, class: "variableName" }, { tag: t.comment, class: "comment" }, { tag: t.invalid, class: "invalid" }, + { tag: t.processingInstruction, class: "meta" }, + // { tag: t.content, class: "tbl-content" }, { tag: t.punctuation, class: "punctuation" }, ...mdExtension.map((mdExt) => { return { tag: mdExt.tag, ...mdExt.styles }; diff --git a/webapp/styles/editor.scss b/webapp/styles/editor.scss index ae9c7176..b90e4a68 100644 --- a/webapp/styles/editor.scss +++ b/webapp/styles/editor.scss @@ -58,12 +58,35 @@ } .line-code { - background-color: #efefef; - margin-left: 30px; + background-color: rgba(72, 72, 72, 0.1); + + .code { + background-color: transparent; + } + } + + .line-tbl-header { + font-weight: bold; + + .meta { + font-weight: normal; + } + } + + .struct { + color: darkred; + } + + .code { + background-color: rgba(72, 72, 72, 0.1); } .line-fenced-code { background-color: rgba(72, 72, 72, 0.1); + + .code { + background-color: transparent; + } } .meta { @@ -117,10 +140,6 @@ - .code { - background-color: #efefef; - } - // Indentation of follow-up lines @mixin lineOverflow($baseIndent) { text-indent: -1 * ($baseIndent + 2ch);