From c7176b00fac6e430f24c2e38e393375dd04b98c2 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Tue, 19 Apr 2022 16:54:47 +0200 Subject: [PATCH] Factored out materialized query providers --- common/manifest.ts | 2 - plugos-syscall/event.ts | 19 +++++- plugos-syscall/fetch.ts | 2 +- plugos/bin/plugos-bundle.ts | 2 +- plugos/environments/node_worker.ts | 6 +- plugos/hooks/event.ts | 9 ++- plugs/core/core.plug.yaml | 26 +++++++- plugs/core/item.ts | 23 ++++++- plugs/core/page.ts | 27 ++++++++ plugs/core/template.ts | 44 ++++++++++--- plugs/emoji/emoji.plug.yaml | 3 +- plugs/ghost/ghost.ts | 15 ++--- plugs/lib/util.ts | 14 ++++ plugs/mattermost/client.ts | 95 +++++++++++++++++++++++++++ plugs/mattermost/mattermost.plug.yaml | 5 ++ plugs/mattermost/mattermost.ts | 37 +++++++++++ plugs/package-lock.json | 20 ++++++ plugs/package.json | 1 + plugs/query/data.ts | 25 ++++++- plugs/query/engine.test.ts | 2 +- plugs/query/engine.ts | 16 +++-- plugs/query/materialized_queries.ts | 95 +++++---------------------- plugs/query/parse-query.js | 2 +- plugs/query/query.grammar | 2 +- plugs/query/query.plug.yaml | 4 ++ plugs/tasks/task.ts | 23 ++++++- plugs/tasks/tasks.plug.yaml | 4 ++ plugs/yarn.lock | 5 ++ webapp/app_event.ts | 6 +- webapp/editor.tsx | 41 ++++++++---- webapp/hooks/completer.ts | 49 -------------- 31 files changed, 437 insertions(+), 187 deletions(-) create mode 100644 plugs/lib/util.ts create mode 100644 plugs/mattermost/client.ts create mode 100644 plugs/mattermost/mattermost.plug.yaml create mode 100644 plugs/mattermost/mattermost.ts delete mode 100644 webapp/hooks/completer.ts diff --git a/common/manifest.ts b/common/manifest.ts index acecdcde..927889db 100644 --- a/common/manifest.ts +++ b/common/manifest.ts @@ -4,10 +4,8 @@ import { CronHookT } from "../plugos/hooks/node_cron"; import { EventHookT } from "../plugos/hooks/event"; import { CommandHookT } from "../webapp/hooks/command"; import { SlashCommandHookT } from "../webapp/hooks/slash_command"; -import { CompleterHookT } from "../webapp/hooks/completer"; export type SilverBulletHooks = CommandHookT & - CompleterHookT & SlashCommandHookT & EndpointHookT & CronHookT & diff --git a/plugos-syscall/event.ts b/plugos-syscall/event.ts index fa8114c0..891386de 100644 --- a/plugos-syscall/event.ts +++ b/plugos-syscall/event.ts @@ -1,5 +1,20 @@ import { syscall } from "./syscall"; -export async function dispatch(eventName: string, data: any): Promise { - return syscall("event.dispatch", eventName, data); +export async function dispatch( + eventName: string, + data: any, + timeout?: number +): Promise { + return new Promise((resolve, reject) => { + let timeOut = setTimeout(() => { + console.log("Timeout!"); + reject("timeout"); + }, timeout); + syscall("event.dispatch", eventName, data) + .then((r) => { + clearTimeout(timeOut); + resolve(r); + }) + .catch(reject); + }); } diff --git a/plugos-syscall/fetch.ts b/plugos-syscall/fetch.ts index 766a1a65..ff0e14e8 100644 --- a/plugos-syscall/fetch.ts +++ b/plugos-syscall/fetch.ts @@ -6,7 +6,7 @@ export async function json(url: RequestInfo, init: RequestInit): Promise { export async function text( url: RequestInfo, - init: RequestInit + init: RequestInit = {} ): Promise { return syscall("fetch.text", url, init); } diff --git a/plugos/bin/plugos-bundle.ts b/plugos/bin/plugos-bundle.ts index 19d756b9..b2a0060f 100755 --- a/plugos/bin/plugos-bundle.ts +++ b/plugos/bin/plugos-bundle.ts @@ -14,7 +14,7 @@ async function compile( filePath: string, functionName: string, debug: boolean, - meta = true + meta = false ) { let outFile = "_out.tmp"; let inFile = filePath; diff --git a/plugos/environments/node_worker.ts b/plugos/environments/node_worker.ts index 0ce0ac27..a4e5822e 100644 --- a/plugos/environments/node_worker.ts +++ b/plugos/environments/node_worker.ts @@ -21,8 +21,12 @@ let syscallReqId = 0; let vm = new VM({ sandbox: { console, + setTimeout, + clearTimeout, + setInterval, + clearInterval, require: (moduleName: string): any => { - console.log("Loading", moduleName); + // console.log("Loading", moduleName); if (preloadModules.includes(moduleName)) { return require(`${workerData}/${moduleName}`); } else { diff --git a/plugos/hooks/event.ts b/plugos/hooks/event.ts index df0c2310..d90be3bf 100644 --- a/plugos/hooks/event.ts +++ b/plugos/hooks/event.ts @@ -12,10 +12,11 @@ export type EventHookT = { export class EventHook implements Hook { private system?: System; - async dispatchEvent(eventName: string, data?: any): Promise { + async dispatchEvent(eventName: string, data?: any): Promise { if (!this.system) { throw new Error("Event hook is not initialized"); } + let responses: any[] = []; for (const plug of this.system.loadedPlugs.values()) { for (const [name, functionDef] of Object.entries( plug.manifest!.functions @@ -23,11 +24,15 @@ export class EventHook implements Hook { if (functionDef.events && functionDef.events.includes(eventName)) { // Only dispatch functions that can run in this environment if (plug.canInvoke(name)) { - await plug.invoke(name, [data]); + let result = await plug.invoke(name, [data]); + if (result !== undefined) { + responses.push(result); + } } } } } + return responses; } apply(system: System): void { diff --git a/plugs/core/core.plug.yaml b/plugs/core/core.plug.yaml index 8b077ecb..03fe5858 100644 --- a/plugs/core/core.plug.yaml +++ b/plugs/core/core.plug.yaml @@ -25,14 +25,26 @@ functions: events: - page:saved - page:deleted + pageQueryProvider: + path: ./page.ts:pageQueryProvider + events: + - query:page indexLinks: path: "./page.ts:indexLinks" events: - page:index + linkQueryProvider: + path: ./page.ts:linkQueryProvider + events: + - query:link indexItems: path: "./item.ts:indexItems" events: - page:index + itemQueryProvider: + path: ./item.ts:queryProvider + events: + - query:item deletePage: path: "./page.ts:deletePage" command: @@ -52,7 +64,8 @@ functions: key: Ctrl-Alt-r pageComplete: path: "./page.ts:pageComplete" - isCompleter: true + events: + - page:complete linkNavigate: path: "./navigate.ts:linkNavigate" command: @@ -86,3 +99,14 @@ functions: path: ./template.ts:instantiateTemplateCommand command: name: "Template: Instantiate for Page" + + instantiateTemplate: + path: ./template.ts:instantiateTemplate + env: server + + + + replaceTemplateVarsCommand: + path: ./template.ts:replaceTemplateVarsCommand + command: + name: "Template: Replace Variables" diff --git a/plugs/core/item.ts b/plugs/core/item.ts index 20d1ca63..e16074a7 100644 --- a/plugs/core/item.ts +++ b/plugs/core/item.ts @@ -1,9 +1,10 @@ import { IndexEvent } from "../../webapp/app_event"; -import { batchSet } from "plugos-silverbullet-syscall/index"; +import { batchSet, scanPrefixGlobal } from "plugos-silverbullet-syscall/index"; import { parseMarkdown } from "plugos-silverbullet-syscall/markdown"; import { collectNodesOfType, ParseTree, renderToText } from "../../common/tree"; import { whiteOutQueries } from "../query/util"; +import { applyQuery, QueryProviderEvent } from "../query/engine"; export type Item = { name: string; @@ -50,3 +51,23 @@ export async function indexItems({ name, text }: IndexEvent) { console.log("Found", items.length, "item(s)"); await batchSet(name, items); } + +export async function queryProvider({ + query, +}: QueryProviderEvent): Promise { + let allItems: Item[] = []; + for (let { key, page, value } of await scanPrefixGlobal("it:")) { + let [, pos] = key.split(":"); + allItems.push({ + ...value, + page: page, + pos: +pos, + }); + } + let markdownItems = applyQuery(query, allItems).map( + (item) => + `* [[${item.page}@${item.pos}]] ${item.name}` + + (item.nested ? "\n " + item.nested : "") + ); + return markdownItems.join("\n"); +} diff --git a/plugs/core/page.ts b/plugs/core/page.ts index b16201f6..d4572f7f 100644 --- a/plugs/core/page.ts +++ b/plugs/core/page.ts @@ -25,6 +25,8 @@ import { renderToText, replaceNodesMatching } from "../../common/tree"; +import { applyQuery, QueryProviderEvent } from "../query/engine"; +import { PageMeta } from "../../common/types"; export async function indexLinks({ name, text }: IndexEvent) { let backLinks: { key: string; value: string }[] = []; @@ -47,6 +49,31 @@ export async function indexLinks({ name, text }: IndexEvent) { await batchSet(name, backLinks); } +export async function pageQueryProvider({ + query, +}: QueryProviderEvent): Promise { + let allPages = await listPages(); + let markdownPages = applyQuery(query, allPages).map( + (pageMeta: PageMeta) => `* [[${pageMeta.name}]]` + ); + return markdownPages.join("\n"); +} + +export async function linkQueryProvider({ + query, + pageName, +}: QueryProviderEvent): Promise { + let uniqueLinks = new Set(); + for (let { value: name } of await scanPrefixGlobal(`pl:${pageName}:`)) { + uniqueLinks.add(name); + } + let markdownLinks = applyQuery( + query, + [...uniqueLinks].map((l) => ({ name: l })) + ).map((pageMeta) => `* [[${pageMeta.name}]]`); + return markdownLinks.join("\n"); +} + export async function deletePage() { let pageName = await getCurrentPage(); console.log("Navigating to start page"); diff --git a/plugs/core/template.ts b/plugs/core/template.ts index bbda674b..505d57bd 100644 --- a/plugs/core/template.ts +++ b/plugs/core/template.ts @@ -1,9 +1,11 @@ import { listPages, readPage, writePage } from "plugos-silverbullet-syscall/space"; -import { filterBox, navigate, prompt } from "plugos-silverbullet-syscall/editor"; +import { filterBox, getCurrentPage, getText, navigate, prompt } from "plugos-silverbullet-syscall/editor"; import { parseMarkdown } from "plugos-silverbullet-syscall/markdown"; import { extractMeta } from "../query/data"; import { renderToText } from "../../common/tree"; import { niceDate } from "./dates"; +import { dispatch } from "plugos-syscall/event"; +import { invokeFunction } from "plugos-silverbullet-syscall/system"; const pageTemplatePrefix = `template/page/`; @@ -34,17 +36,41 @@ export async function instantiateTemplateCommand() { if (!pageName) { return; } - let pageText = replaceTemplateVars(renderToText(parseTree)); - await writePage(pageName, pageText); + await invokeFunction( + "server", + "instantiateTemplate", + pageName, + renderToText(parseTree) + ); + // let pageText = replaceTemplateVars(, pageName); + // await writePage(pageName, pageText); await navigate(pageName); } -export function replaceTemplateVars(s: string): string { - return s.replaceAll(/\{\{(\w+)\}\}/g, (match, v) => { - switch (v) { - case "today": - return niceDate(new Date()); - break; +export async function instantiateTemplate(pageName: string, text: string) { + let pageText = replaceTemplateVars(text, pageName); + await writePage(pageName, pageText); +} + +export async function replaceTemplateVarsCommand() { + let currentPage = await getCurrentPage(); + let text = await getText(); + await invokeFunction("server", "instantiateTemplate", currentPage, text); +} + +export function replaceTemplateVars(s: string, pageName: string): string { + return s.replaceAll(/\{\{([^\}]+)\}\}/g, (match, v) => { + if (v === "today") { + return niceDate(new Date()); + } + if (v.startsWith("placeholder:")) { + // Dispatch event, to be replaced in the file async later + dispatch(v, { + pageName: pageName, + placeholder: v, + }).catch((e) => { + console.error("Failed to dispatch placeholder event", e); + }); } return match; }); diff --git a/plugs/emoji/emoji.plug.yaml b/plugs/emoji/emoji.plug.yaml index 2f03abe8..dc53541e 100644 --- a/plugs/emoji/emoji.plug.yaml +++ b/plugs/emoji/emoji.plug.yaml @@ -1,4 +1,5 @@ functions: emojiCompleter: path: "./emoji.ts:emojiCompleter" - isCompleter: true + events: + - page:complete diff --git a/plugs/ghost/ghost.ts b/plugs/ghost/ghost.ts index 9d6ee1ee..d40a286a 100644 --- a/plugs/ghost/ghost.ts +++ b/plugs/ghost/ghost.ts @@ -1,9 +1,10 @@ import { readPage, writePage } from "plugos-silverbullet-syscall/space"; import { json } from "plugos-syscall/fetch"; -import { parse as parseYaml } from "yaml"; import { invokeFunction } from "plugos-silverbullet-syscall/system"; import { getCurrentPage, getText } from "plugos-silverbullet-syscall/editor"; import { cleanMarkdown } from "../markdown/util"; +import { parseMarkdown } from "plugos-silverbullet-syscall/markdown"; +import { extractMeta } from "../query/data"; type GhostConfig = { url: string; @@ -182,14 +183,10 @@ async function markdownToPost(text: string): Promise> { } async function getConfig(): Promise { - let configPage = await readPage("ghost-config"); - return parseYaml(configPage.text) as GhostConfig; - // return { - // adminKey: "", - // pagePrefix: "", - // postPrefix: "", - // url: "", - // }; + let { text } = await readPage("ghost-config"); + let parsedContent = await parseMarkdown(text); + let pageMeta = await extractMeta(parsedContent); + return pageMeta as GhostConfig; } export async function downloadAllPostsCommand() { diff --git a/plugs/lib/util.ts b/plugs/lib/util.ts new file mode 100644 index 00000000..6bc6993a --- /dev/null +++ b/plugs/lib/util.ts @@ -0,0 +1,14 @@ +export async function replaceAsync( + str: string, + regex: RegExp, + asyncFn: (match: string, ...args: any[]) => Promise +) { + const promises: Promise[] = []; + str.replace(regex, (match: string, ...args: any[]): string => { + const promise = asyncFn(match, ...args); + promises.push(promise); + return ""; + }); + const data = await Promise.all(promises); + return str.replace(regex, () => data.shift()!); +} diff --git a/plugs/mattermost/client.ts b/plugs/mattermost/client.ts new file mode 100644 index 00000000..fe3b370a --- /dev/null +++ b/plugs/mattermost/client.ts @@ -0,0 +1,95 @@ +import { readPage } from "plugos-silverbullet-syscall/space"; +import { parseMarkdown } from "plugos-silverbullet-syscall/markdown"; +import { extractMeta } from "../query/data"; +import { UserProfile } from "@hmhealey/types/lib/users"; +import { json } from "plugos-syscall/fetch"; +import { Post } from "@hmhealey/types/lib/posts"; +import { Channel } from "@hmhealey/types/lib/channels"; +import { Team } from "@hmhealey/types/lib/teams"; + +type MattermostConfig = { + url: string; + token: string; +}; + +async function getConfig(): Promise { + let { text } = await readPage("mattermost-config"); + let parsedContent = await parseMarkdown(text); + let pageMeta = await extractMeta(parsedContent); + return pageMeta as MattermostConfig; +} + +export class MattermostClient { + userCache = new Map(); + channelCache = new Map(); + teamCache = new Map(); + + constructor(readonly url: string, readonly token: string) {} + + static async fromConfig(): Promise { + let config = await getConfig(); + return new MattermostClient(config.url, config.token); + } + + getMe(): Promise { + return this.getUser("me"); + } + + async getUser(userId: string): Promise { + let user = this.userCache.get(userId); + if (user) { + return user; + } + user = await json(`${this.url}/api/v4/users/${userId}`, { + headers: { + Authorization: `Bearer ${this.token}`, + }, + }); + this.userCache.set(userId, user!); + return user!; + } + + async getChannel(channelId: string): Promise { + let channel = this.channelCache.get(channelId); + if (channel) { + return channel; + } + channel = await json(`${this.url}/api/v4/channels/${channelId}`, { + headers: { + Authorization: `Bearer ${this.token}`, + }, + }); + this.channelCache.set(channelId, channel!); + return channel!; + } + + async getTeam(teamId: string): Promise { + let team = this.teamCache.get(teamId); + if (team) { + return team; + } + team = await json(`${this.url}/api/v4/teams/${teamId}`, { + headers: { + Authorization: `Bearer ${this.token}`, + }, + }); + this.teamCache.set(teamId, team!); + return team!; + } + + async getFlaggedPosts(userId: string, perPage: number = 10): Promise { + let postCollection = await json( + `${this.url}/api/v4/users/${userId}/posts/flagged?per_page=${perPage}`, + { + headers: { + Authorization: `Bearer ${this.token}`, + }, + } + ); + let posts: Post[] = []; + for (let order of postCollection.order) { + posts.push(postCollection.posts[order]); + } + return posts; + } +} diff --git a/plugs/mattermost/mattermost.plug.yaml b/plugs/mattermost/mattermost.plug.yaml new file mode 100644 index 00000000..6f333a3d --- /dev/null +++ b/plugs/mattermost/mattermost.plug.yaml @@ -0,0 +1,5 @@ +functions: + test: + path: mattermost.ts:savedPostsQueryProvider + events: + - query:mm-saved diff --git a/plugs/mattermost/mattermost.ts b/plugs/mattermost/mattermost.ts new file mode 100644 index 00000000..cf7fcffa --- /dev/null +++ b/plugs/mattermost/mattermost.ts @@ -0,0 +1,37 @@ +import { MattermostClient } from "./client"; +import { applyQuery, QueryProviderEvent } from "../query/engine"; + +// https://community.mattermost.com/private-core/pl/rbp7a7jtr3f89nzsefo6ftqt3o + +function mattermostDesktopUrlForPost( + url: string, + teamName: string, + postId: string +) { + return `${url.replace("https://", "mattermost://")}/${teamName}/pl/${postId}`; +} + +export async function savedPostsQueryProvider({ + query, +}: QueryProviderEvent): Promise { + let client = await MattermostClient.fromConfig(); + let me = await client.getMe(); + let savedPosts = await client.getFlaggedPosts(me.id); + let savedPostsMd = []; + savedPosts = applyQuery(query, savedPosts); + for (let savedPost of savedPosts) { + // savedPost. + let channel = await client.getChannel(savedPost.channel_id); + let team = await client.getTeam(channel.team_id); + savedPostsMd.push( + `@${ + (await client.getUser(savedPost.user_id)).username + } [link](${mattermostDesktopUrlForPost( + client.url, + team.name, + savedPost.id + )}):\n> ${savedPost.message.replaceAll(/\n/g, "\n> ")}` + ); + } + return savedPostsMd.join("\n\n"); +} diff --git a/plugs/package-lock.json b/plugs/package-lock.json index 87c4bc37..0bff9536 100644 --- a/plugs/package-lock.json +++ b/plugs/package-lock.json @@ -8,6 +8,7 @@ "name": "plugs", "version": "1.0.0", "dependencies": { + "@hmhealey/types": "^6.6.0-4", "@jest/globals": "^27.5.1", "@lezer/generator": "^0.15.4", "@lezer/lr": "^0.15.8", @@ -119,6 +120,19 @@ "node": ">=4" } }, + "node_modules/@hmhealey/types": { + "version": "6.6.0-4", + "resolved": "https://registry.npmjs.org/@hmhealey/types/-/types-6.6.0-4.tgz", + "integrity": "sha512-71IxVaXhrUesmLnvQQh4RtUqqhmVL+ejci4qo4R6rTWTdY77BniRtBx269uAz34wzTlAgITysN8x7MBTdt/XBg==", + "peerDependencies": { + "typescript": "^4.3" + }, + "peerDependenciesMeta": { + "typescript": { + "optional": true + } + } + }, "node_modules/@jest/environment": { "version": "27.5.1", "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-27.5.1.tgz", @@ -683,6 +697,12 @@ } } }, + "@hmhealey/types": { + "version": "6.6.0-4", + "resolved": "https://registry.npmjs.org/@hmhealey/types/-/types-6.6.0-4.tgz", + "integrity": "sha512-71IxVaXhrUesmLnvQQh4RtUqqhmVL+ejci4qo4R6rTWTdY77BniRtBx269uAz34wzTlAgITysN8x7MBTdt/XBg==", + "requires": {} + }, "@jest/environment": { "version": "27.5.1", "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-27.5.1.tgz", diff --git a/plugs/package.json b/plugs/package.json index 17f6256a..a4cc2920 100644 --- a/plugs/package.json +++ b/plugs/package.json @@ -5,6 +5,7 @@ "generate": "lezer-generator query/query.grammar -o query/parse-query.js" }, "dependencies": { + "@hmhealey/types": "^6.6.0-4", "@jest/globals": "^27.5.1", "@lezer/generator": "^0.15.4", "@lezer/lr": "^0.15.8", diff --git a/plugs/query/data.ts b/plugs/query/data.ts index bf8f1a27..12ed32d5 100644 --- a/plugs/query/data.ts +++ b/plugs/query/data.ts @@ -2,14 +2,15 @@ // data:page@pos import { IndexEvent } from "../../webapp/app_event"; -import { batchSet } from "plugos-silverbullet-syscall"; +import { batchSet, scanPrefixGlobal } from "plugos-silverbullet-syscall"; import { parseMarkdown } from "plugos-silverbullet-syscall/markdown"; import { collectNodesOfType, findNodeOfType, ParseTree, replaceNodesMatching } from "../../common/tree"; -import { parse as parseYaml, parseAllDocuments } from "yaml"; +import YAML, { parse as parseYaml, parseAllDocuments } from "yaml"; import { whiteOutQueries } from "./util"; +import type { QueryProviderEvent } from "./engine"; +import { applyQuery } from "./engine"; export async function indexData({ name, text }: IndexEvent) { - let e; text = whiteOutQueries(text); // console.log("Now data indexing", name); let mdTree = await parseMarkdown(text); @@ -77,3 +78,21 @@ export function extractMeta(parseTree: ParseTree, remove = false): any { return data; } + +export async function queryProvider({ + query, +}: QueryProviderEvent): Promise { + let allData: any[] = []; + for (let { key, page, value } of await scanPrefixGlobal("data:")) { + let [, pos] = key.split("@"); + allData.push({ + ...value, + page: page, + pos: +pos, + }); + } + let markdownData = applyQuery(query, allData).map((item) => + YAML.stringify(item) + ); + return `\`\`\`data\n${markdownData.join("---\n")}\`\`\``; +} diff --git a/plugs/query/engine.test.ts b/plugs/query/engine.test.ts index 39bd1686..cef74c39 100644 --- a/plugs/query/engine.test.ts +++ b/plugs/query/engine.test.ts @@ -30,7 +30,7 @@ test("Test parser", () => { expect(parsedQuery2.filter[0]).toStrictEqual({ op: "=~", prop: "name", - value: /interview\/.*/, + value: "interview\\/.*", }); let parsedQuery3 = parseQuery(`page where something != null`); diff --git a/plugs/query/engine.ts b/plugs/query/engine.ts index 6e7943a0..df524fd0 100644 --- a/plugs/query/engine.ts +++ b/plugs/query/engine.ts @@ -4,13 +4,18 @@ import { lezerToParseTree } from "../../common/parse_tree"; // @ts-ignore import { parser } from "./parse-query"; -type Filter = { +export type QueryProviderEvent = { + query: ParsedQuery; + pageName: string; +}; + +export type Filter = { op: string; prop: string; value: any; }; -type ParsedQuery = { +export type ParsedQuery = { table: string; orderBy?: string; orderDesc?: boolean; @@ -71,7 +76,7 @@ export function parseQuery(query: string): ParsedQuery { break; case "Regex": val = valNode.children![0].text!; - val = new RegExp(val.substring(1, val.length - 1)); + val = val.substring(1, val.length - 1); break; case "String": val = valNode.children![0].text!; @@ -129,12 +134,13 @@ export function applyQuery(parsedQuery: ParsedQuery, records: T[]): T[] { } break; case "=~": - if (!value.exec(recordAny[prop])) { + // TODO: Cache regexps somehow + if (!new RegExp(value).exec(recordAny[prop])) { continue recordLoop; } break; case "!=~": - if (value.exec(recordAny[prop])) { + if (new RegExp(value).exec(recordAny[prop])) { continue recordLoop; } break; diff --git a/plugs/query/materialized_queries.ts b/plugs/query/materialized_queries.ts index c9e30fcb..637eb7d4 100644 --- a/plugs/query/materialized_queries.ts +++ b/plugs/query/materialized_queries.ts @@ -1,15 +1,11 @@ import { flashNotification, getCurrentPage, reloadPage, save } from "plugos-silverbullet-syscall/editor"; -import { listPages, readPage, writePage } from "plugos-silverbullet-syscall/space"; +import { readPage, writePage } from "plugos-silverbullet-syscall/space"; import { invokeFunction } from "plugos-silverbullet-syscall/system"; -import { scanPrefixGlobal } from "plugos-silverbullet-syscall"; -import { applyQuery, parseQuery } from "./engine"; -import { PageMeta } from "../../common/types"; -import type { Task } from "../tasks/task"; -import { Item } from "../core/item"; -import YAML from "yaml"; +import { parseQuery } from "./engine"; import { replaceTemplateVars } from "../core/template"; import { queryRegex } from "./util"; +import { dispatch } from "plugos-syscall/event"; async function replaceAsync( str: string, @@ -46,79 +42,22 @@ export async function updateMaterializedQueriesOnPage(pageName: string) { text, queryRegex, async (fullMatch, startQuery, query, body, endQuery) => { - let parsedQuery = parseQuery(replaceTemplateVars(query)); + let parsedQuery = parseQuery(replaceTemplateVars(query, pageName)); console.log("Parsed query", parsedQuery); - - switch (parsedQuery.table) { - case "page": - let allPages = await listPages(); - let markdownPages = applyQuery(parsedQuery, allPages).map( - (pageMeta: PageMeta) => `* [[${pageMeta.name}]]` - ); - return `${startQuery}\n${markdownPages.join("\n")}\n${endQuery}`; - case "task": - let allTasks: Task[] = []; - for (let { key, page, value } of await scanPrefixGlobal("task:")) { - let [, pos] = key.split(":"); - allTasks.push({ - ...value, - page: page, - pos: pos, - }); - } - let markdownTasks = applyQuery(parsedQuery, allTasks).map( - (t) => - `* [${t.done ? "x" : " "}] [[${t.page}@${t.pos}]] ${t.name}` + - (t.nested ? "\n " + t.nested : "") - ); - return `${startQuery}\n${markdownTasks.join("\n")}\n${endQuery}`; - case "link": - let uniqueLinks = new Set(); - for (let { value: name } of await scanPrefixGlobal( - `pl:${pageName}:` - )) { - uniqueLinks.add(name); - } - let markdownLinks = applyQuery( - parsedQuery, - [...uniqueLinks].map((l) => ({ name: l })) - ).map((pageMeta) => `* [[${pageMeta.name}]]`); - return `${startQuery}\n${markdownLinks.join("\n")}\n${endQuery}`; - case "item": - let allItems: Item[] = []; - for (let { key, page, value } of await scanPrefixGlobal("it:")) { - let [, pos] = key.split(":"); - allItems.push({ - ...value, - page: page, - pos: +pos, - }); - } - let markdownItems = applyQuery(parsedQuery, allItems).map( - (item) => - `* [[${item.page}@${item.pos}]] ${item.name}` + - (item.nested ? "\n " + item.nested : "") - ); - return `${startQuery}\n${markdownItems.join("\n")}\n${endQuery}`; - case "data": - let allData: Object[] = []; - for (let { key, page, value } of await scanPrefixGlobal("data:")) { - let [, pos] = key.split("@"); - allData.push({ - ...value, - page: page, - pos: +pos, - }); - } - let markdownData = applyQuery(parsedQuery, allData).map((item) => - YAML.stringify(item) - ); - return `${startQuery}\n\`\`\`data\n${markdownData.join( - "---\n" - )}\`\`\`\n${endQuery}`; - default: - return fullMatch; + // Let's dispatch an event and see what happens + let results = await dispatch( + `query:${parsedQuery.table}`, + { query: parsedQuery, pageName: pageName }, + 5000 + ); + if (results.length === 0) { + return `${startQuery}\n${endQuery}`; + } else if (results.length === 1) { + return `${startQuery}\n${results[0]}\n${endQuery}`; + } else { + console.error("Too many query results", results); + return fullMatch; } } ); diff --git a/plugs/query/parse-query.js b/plugs/query/parse-query.js index a8e68a3a..0bca3b60 100644 --- a/plugs/query/parse-query.js +++ b/plugs/query/parse-query.js @@ -10,7 +10,7 @@ export const parser = LRParser.deserialize({ maxTerm: 38, skippedNodes: [0], repeatNodeCount: 1, - tokenData: "4v~RtX^#cpq#cqr$Wrs$k!P!Q%V!Q![%|!^!_&U!_!`&c!`!a&p!c!}&}#T#U'Y#U#V)P#V#W&}#W#X)o#X#Y&}#Y#Z+R#Z#`&}#`#a,s#a#b&}#b#c.h#c#d/z#d#h&}#h#i1o#i#k&}#k#l3R#l#o&}#y#z#c$f$g#c#BY#BZ#c$IS$I_#c$Ip$Iq$k$Iq$Ir$k$I|$JO#c$JT$JU#c$KV$KW#c&FU&FV#c~#hYd~X^#cpq#c#y#z#c$f$g#c#BY#BZ#c$IS$I_#c$I|$JO#c$JT$JU#c$KV$KW#c&FU&FV#c~$ZP!_!`$^~$cPl~#r#s$f~$kOp~~$nUOr$krs%Qs$Ip$k$Ip$Iq%Q$Iq$Ir%Q$Ir~$k~%VOY~~%[V[~OY%VZ]%V^!P%V!P!Q%q!Q#O%V#O#P%v#P~%V~%vO[~~%yPO~%V~&RPX~!Q![%|~&ZPf~!_!`&^~&cOj~~&hPk~#r#s&k~&pOo~~&uPn~!_!`&x~&}Om~P'SQRP!c!}&}#T#o&}R'_URP!c!}&}#T#b&}#b#c'q#c#g&}#g#h(a#h#o&}R'vSRP!c!}&}#T#W&}#W#X(S#X#o&}R(ZQqQRP!c!}&}#T#o&}R(fSRP!c!}&}#T#V&}#V#W(r#W#o&}R(yQuQRP!c!}&}#T#o&}R)USRP!c!}&}#T#m&}#m#n)b#n#o&}R)iQsQRP!c!}&}#T#o&}R)tSRP!c!}&}#T#X&}#X#Y*Q#Y#o&}R*VSRP!c!}&}#T#g&}#g#h*c#h#o&}R*hSRP!c!}&}#T#V&}#V#W*t#W#o&}R*{QtQRP!c!}&}#T#o&}R+WRRP!c!}&}#T#U+a#U#o&}R+fSRP!c!}&}#T#`&}#`#a+r#a#o&}R+wSRP!c!}&}#T#g&}#g#h,T#h#o&}R,YSRP!c!}&}#T#X&}#X#Y,f#Y#o&}R,mQhQRP!c!}&}#T#o&}R,xSRP!c!}&}#T#]&}#]#^-U#^#o&}R-ZSRP!c!}&}#T#a&}#a#b-g#b#o&}R-lSRP!c!}&}#T#]&}#]#^-x#^#o&}R-}SRP!c!}&}#T#h&}#h#i.Z#i#o&}R.bQvQRP!c!}&}#T#o&}R.mSRP!c!}&}#T#i&}#i#j.y#j#o&}R/OSRP!c!}&}#T#`&}#`#a/[#a#o&}R/aSRP!c!}&}#T#`&}#`#a/m#a#o&}R/tQiQRP!c!}&}#T#o&}R0PSRP!c!}&}#T#f&}#f#g0]#g#o&}R0bSRP!c!}&}#T#W&}#W#X0n#X#o&}R0sSRP!c!}&}#T#X&}#X#Y1P#Y#o&}R1USRP!c!}&}#T#f&}#f#g1b#g#o&}R1iQrQRP!c!}&}#T#o&}R1tSRP!c!}&}#T#f&}#f#g2Q#g#o&}R2VSRP!c!}&}#T#i&}#i#j2c#j#o&}R2hSRP!c!}&}#T#X&}#X#Y2t#Y#o&}R2{QgQRP!c!}&}#T#o&}R3WSRP!c!}&}#T#[&}#[#]3d#]#o&}R3iSRP!c!}&}#T#X&}#X#Y3u#Y#o&}R3zSRP!c!}&}#T#f&}#f#g4W#g#o&}R4]SRP!c!}&}#T#X&}#X#Y4i#Y#o&}R4pQeQRP!c!}&}#T#o&}", + tokenData: ":W~RvX^#ipq#iqr$^rs$q}!O%]!P!Q%n!Q![&e!^!_&m!_!`&z!`!a'X!c!}%]#R#S%]#T#U'f#U#V){#V#W%]#W#X*w#X#Y%]#Y#Z,s#Z#`%]#`#a/T#a#b%]#b#c1h#c#d3d#d#h%]#h#i5w#i#k%]#k#l7s#l#o%]#y#z#i$f$g#i#BY#BZ#i$IS$I_#i$Ip$Iq$q$Iq$Ir$q$I|$JO#i$JT$JU#i$KV$KW#i&FU&FV#i~#nYd~X^#ipq#i#y#z#i$f$g#i#BY#BZ#i$IS$I_#i$I|$JO#i$JT$JU#i$KV$KW#i&FU&FV#i~$aP!_!`$d~$iPl~#r#s$l~$qOp~~$tUOr$qrs%Ws$Ip$q$Ip$Iq%W$Iq$Ir%W$Ir~$q~%]OY~P%bSRP}!O%]!c!}%]#R#S%]#T#o%]~%sV[~OY%nZ]%n^!P%n!P!Q&Y!Q#O%n#O#P&_#P~%n~&_O[~~&bPO~%n~&jPX~!Q![&e~&rPf~!_!`&u~&zOj~~'PPk~#r#s'S~'XOo~~'^Pn~!_!`'a~'fOm~R'kWRP}!O%]!c!}%]#R#S%]#T#b%]#b#c(T#c#g%]#g#h)P#h#o%]R(YURP}!O%]!c!}%]#R#S%]#T#W%]#W#X(l#X#o%]R(sSqQRP}!O%]!c!}%]#R#S%]#T#o%]R)UURP}!O%]!c!}%]#R#S%]#T#V%]#V#W)h#W#o%]R)oSuQRP}!O%]!c!}%]#R#S%]#T#o%]R*QURP}!O%]!c!}%]#R#S%]#T#m%]#m#n*d#n#o%]R*kSsQRP}!O%]!c!}%]#R#S%]#T#o%]R*|URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y+`#Y#o%]R+eURP}!O%]!c!}%]#R#S%]#T#g%]#g#h+w#h#o%]R+|URP}!O%]!c!}%]#R#S%]#T#V%]#V#W,`#W#o%]R,gStQRP}!O%]!c!}%]#R#S%]#T#o%]R,xTRP}!O%]!c!}%]#R#S%]#T#U-X#U#o%]R-^URP}!O%]!c!}%]#R#S%]#T#`%]#`#a-p#a#o%]R-uURP}!O%]!c!}%]#R#S%]#T#g%]#g#h.X#h#o%]R.^URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y.p#Y#o%]R.wShQRP}!O%]!c!}%]#R#S%]#T#o%]R/YURP}!O%]!c!}%]#R#S%]#T#]%]#]#^/l#^#o%]R/qURP}!O%]!c!}%]#R#S%]#T#a%]#a#b0T#b#o%]R0YURP}!O%]!c!}%]#R#S%]#T#]%]#]#^0l#^#o%]R0qURP}!O%]!c!}%]#R#S%]#T#h%]#h#i1T#i#o%]R1[SvQRP}!O%]!c!}%]#R#S%]#T#o%]R1mURP}!O%]!c!}%]#R#S%]#T#i%]#i#j2P#j#o%]R2UURP}!O%]!c!}%]#R#S%]#T#`%]#`#a2h#a#o%]R2mURP}!O%]!c!}%]#R#S%]#T#`%]#`#a3P#a#o%]R3WSiQRP}!O%]!c!}%]#R#S%]#T#o%]R3iURP}!O%]!c!}%]#R#S%]#T#f%]#f#g3{#g#o%]R4QURP}!O%]!c!}%]#R#S%]#T#W%]#W#X4d#X#o%]R4iURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y4{#Y#o%]R5QURP}!O%]!c!}%]#R#S%]#T#f%]#f#g5d#g#o%]R5kSrQRP}!O%]!c!}%]#R#S%]#T#o%]R5|URP}!O%]!c!}%]#R#S%]#T#f%]#f#g6`#g#o%]R6eURP}!O%]!c!}%]#R#S%]#T#i%]#i#j6w#j#o%]R6|URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y7`#Y#o%]R7gSgQRP}!O%]!c!}%]#R#S%]#T#o%]R7xURP}!O%]!c!}%]#R#S%]#T#[%]#[#]8[#]#o%]R8aURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y8s#Y#o%]R8xURP}!O%]!c!}%]#R#S%]#T#f%]#f#g9[#g#o%]R9aURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y9s#Y#o%]R9zSeQRP}!O%]!c!}%]#R#S%]#T#o%]", tokenizers: [0, 1], topRules: {"Program":[0,1]}, tokenPrec: 0 diff --git a/plugs/query/query.grammar b/plugs/query/query.grammar index 34b517e1..d596b3b8 100644 --- a/plugs/query/query.grammar +++ b/plugs/query/query.grammar @@ -43,7 +43,7 @@ Null { @tokens { space { std.whitespace+ } - Name { std.asciiLetter+ } + Name { (std.asciiLetter | "-" | "_")+ } String { ("\"" | "“" | "”") ![\"”“]* ("\"" | "“" | "”") } diff --git a/plugs/query/query.plug.yaml b/plugs/query/query.plug.yaml index da0df0e8..dabb8ba2 100644 --- a/plugs/query/query.plug.yaml +++ b/plugs/query/query.plug.yaml @@ -10,3 +10,7 @@ functions: path: ./data.ts:indexData events: - page:index + dataQueryProvider: + path: ./data.ts:queryProvider + events: + - query:data diff --git a/plugs/tasks/task.ts b/plugs/tasks/task.ts index 095ff91d..7b015304 100644 --- a/plugs/tasks/task.ts +++ b/plugs/tasks/task.ts @@ -1,6 +1,6 @@ import type { ClickEvent, IndexEvent } from "../../webapp/app_event"; -import { batchSet } from "plugos-silverbullet-syscall/index"; +import { batchSet, scanPrefixGlobal } from "plugos-silverbullet-syscall/index"; import { readPage, writePage } from "plugos-silverbullet-syscall/space"; import { parseMarkdown } from "plugos-silverbullet-syscall/markdown"; import { dispatch, getCurrentPage, getText } from "plugos-silverbullet-syscall/editor"; @@ -12,6 +12,7 @@ import { renderToText } from "../../common/tree"; import { whiteOutQueries } from "../query/util"; +import { applyQuery, QueryProviderEvent } from "../query/engine"; export type Task = { name: string; @@ -120,3 +121,23 @@ export async function taskToggleAtPos(pos: number) { } } } + +export async function queryProvider({ + query, +}: QueryProviderEvent): Promise { + let allTasks: Task[] = []; + for (let { key, page, value } of await scanPrefixGlobal("task:")) { + let [, pos] = key.split(":"); + allTasks.push({ + ...value, + page: page, + pos: pos, + }); + } + let markdownTasks = applyQuery(query, allTasks).map( + (t) => + `* [${t.done ? "x" : " "}] [[${t.page}@${t.pos}]] ${t.name}` + + (t.nested ? "\n " + t.nested : "") + ); + return markdownTasks.join("\n"); +} diff --git a/plugs/tasks/tasks.plug.yaml b/plugs/tasks/tasks.plug.yaml index 06c58610..943787d0 100644 --- a/plugs/tasks/tasks.plug.yaml +++ b/plugs/tasks/tasks.plug.yaml @@ -26,4 +26,8 @@ functions: path: "./task.ts:taskToggle" events: - page:click + itemQueryProvider: + path: ./task.ts:queryProvider + events: + - query:task diff --git a/plugs/yarn.lock b/plugs/yarn.lock index 3868630c..9a6bbf1d 100644 --- a/plugs/yarn.lock +++ b/plugs/yarn.lock @@ -23,6 +23,11 @@ "chalk" "^2.0.0" "js-tokens" "^4.0.0" +"@hmhealey/types@^6.6.0-4": + "integrity" "sha512-71IxVaXhrUesmLnvQQh4RtUqqhmVL+ejci4qo4R6rTWTdY77BniRtBx269uAz34wzTlAgITysN8x7MBTdt/XBg==" + "resolved" "https://registry.npmjs.org/@hmhealey/types/-/types-6.6.0-4.tgz" + "version" "6.6.0-4" + "@jest/environment@^27.5.1": "integrity" "sha512-/WQjhPJe3/ghaol/4Bq480JKXV/Rfw8nQdN7f41fM8VDHLcxKXou6QyXAh3EFr9/bVG3x74z1NWDkP87EiY8gA==" "resolved" "https://registry.npmjs.org/@jest/environment/-/environment-27.5.1.tgz" diff --git a/webapp/app_event.ts b/webapp/app_event.ts index baaa240a..b1d8bbf8 100644 --- a/webapp/app_event.ts +++ b/webapp/app_event.ts @@ -1,4 +1,4 @@ -export type AppEvent = "page:click" | "editor:complete"; +export type AppEvent = "page:click" | "page:complete"; export type ClickEvent = { page: string; @@ -12,7 +12,3 @@ export type IndexEvent = { name: string; text: string; }; - -export interface AppEventDispatcher { - dispatchAppEvent(name: AppEvent, data?: any): Promise; -} diff --git a/webapp/editor.tsx b/webapp/editor.tsx index 9f98fe3c..731e6b37 100644 --- a/webapp/editor.tsx +++ b/webapp/editor.tsx @@ -1,4 +1,4 @@ -import { autocompletion, completionKeymap } from "@codemirror/autocomplete"; +import { autocompletion, completionKeymap, CompletionResult } from "@codemirror/autocomplete"; import { closeBrackets, closeBracketsKeymap } from "@codemirror/closebrackets"; import { indentWithTab, standardKeymap } from "@codemirror/commands"; import { history, historyKeymap } from "@codemirror/history"; @@ -18,7 +18,7 @@ import { import React, { useEffect, useReducer } from "react"; import ReactDOM from "react-dom"; import { createSandbox as createIFrameSandbox } from "../plugos/environments/webworker_sandbox"; -import { AppEvent, AppEventDispatcher, ClickEvent } from "./app_event"; +import { AppEvent, ClickEvent } from "./app_event"; import * as commands from "./commands"; import { CommandPalette } from "./components/command_palette"; import { PageNavigator } from "./components/page_navigator"; @@ -44,7 +44,6 @@ import { systemSyscalls } from "./syscalls/system"; import { Panel } from "./components/panel"; import { CommandHook } from "./hooks/command"; import { SlashCommandHook } from "./hooks/slash_command"; -import { CompleterHook } from "./hooks/completer"; import { pasteLinkExtension } from "./editor_paste"; import { markdownSyscalls } from "../common/syscalls/markdown"; import { clientStoreSyscalls } from "./syscalls/clientStore"; @@ -65,10 +64,9 @@ class PageState { const saveInterval = 1000; -export class Editor implements AppEventDispatcher { +export class Editor { readonly commandHook: CommandHook; readonly slashCommandHook: SlashCommandHook; - readonly completerHook: CompleterHook; openPages = new Map(); editorView?: EditorView; viewState: AppViewState; @@ -78,7 +76,9 @@ export class Editor implements AppEventDispatcher { eventHook: EventHook; saveTimeout: any; debouncedUpdateEvent = throttle(() => { - this.eventHook.dispatchEvent("editor:updated"); + this.eventHook + .dispatchEvent("editor:updated") + .catch((e) => console.error("Error dispatching editor:updated event", e)); }, 1000); private system = new System("client"); private mdExtensions: MDExt[] = []; @@ -108,10 +108,6 @@ export class Editor implements AppEventDispatcher { this.slashCommandHook = new SlashCommandHook(this); this.system.addHook(this.slashCommandHook); - // Completer hook - this.completerHook = new CompleterHook(); - this.system.addHook(this.completerHook); - this.render(parent); this.editorView = new EditorView({ state: this.createEditorState("", ""), @@ -261,13 +257,14 @@ export class Editor implements AppEventDispatcher { helpText, onSelect: (option) => { this.viewDispatch({ type: "hide-filterbox" }); + this.focus(); resolve(option); }, }); }); } - async dispatchAppEvent(name: AppEvent, data?: any): Promise { + async dispatchAppEvent(name: AppEvent, data?: any): Promise { return this.eventHook.dispatchEvent(name, data); } @@ -303,7 +300,8 @@ export class Editor implements AppEventDispatcher { closeBrackets(), autocompletion({ override: [ - this.completerHook.plugCompleter.bind(this.completerHook), + // this.completerHook.plugCompleter.bind(this.completerHook), + this.completer.bind(this), this.slashCommandHook.slashCommandCompleter.bind( this.slashCommandHook ), @@ -408,7 +406,7 @@ export class Editor implements AppEventDispatcher { if (update.docChanged) { editor.viewDispatch({ type: "page-changed" }); editor.debouncedUpdateEvent(); - editor.save(); + editor.save().catch((e) => console.error("Error saving", e)); } } } @@ -438,6 +436,23 @@ export class Editor implements AppEventDispatcher { } } + async completer(): Promise { + let results = await this.dispatchAppEvent("page:complete"); + let actualResult = null; + for (const result of results) { + if (result) { + if (actualResult) { + console.error( + "Got completion results from multiple sources, cannot deal with that" + ); + return null; + } + actualResult = result; + } + } + return actualResult; + } + reloadPage() { console.log("Reloading page"); safeRun(async () => { diff --git a/webapp/hooks/completer.ts b/webapp/hooks/completer.ts deleted file mode 100644 index ecf1d4d2..00000000 --- a/webapp/hooks/completer.ts +++ /dev/null @@ -1,49 +0,0 @@ -import { Hook, Manifest } from "../../plugos/types"; -import { System } from "../../plugos/system"; -import { CompletionResult } from "@codemirror/autocomplete"; - -export type CompleterHookT = { - isCompleter?: boolean; -}; - -export class CompleterHook implements Hook { - private system?: System; - - public async plugCompleter(): Promise { - let completerPromises = []; - // TODO: Can be optimized (cache all functions) - for (const plug of this.system!.loadedPlugs.values()) { - if (!plug.manifest) { - continue; - } - for (const [functionName, functionDef] of Object.entries( - plug.manifest.functions - )) { - if (functionDef.isCompleter) { - completerPromises.push(plug.invoke(functionName, [])); - } - } - } - let actualResult = null; - for (const result of await Promise.all(completerPromises)) { - if (result) { - if (actualResult) { - console.error( - "Got completion results from multiple sources, cannot deal with that" - ); - return null; - } - actualResult = result; - } - } - return actualResult; - } - - apply(system: System): void { - this.system = system; - } - - validateManifest(manifest: Manifest): string[] { - return []; - } -}