Factored out materialized query providers

common/manifest.ts

@@ -4,10 +4,8 @@ import { CronHookT } from "../plugos/hooks/node_cron";
 import { EventHookT } from "../plugos/hooks/event";
 import { CommandHookT } from "../webapp/hooks/command";
 import { SlashCommandHookT } from "../webapp/hooks/slash_command";
-import { CompleterHookT } from "../webapp/hooks/completer";
 
 export type SilverBulletHooks = CommandHookT &
-  CompleterHookT &
   SlashCommandHookT &
   EndpointHookT &
   CronHookT &

plugos-syscall/event.ts

@@ -1,5 +1,20 @@
 import { syscall } from "./syscall";
 
-export async function dispatch(eventName: string, data: any): Promise<void> {
+export async function dispatch(
-  return syscall("event.dispatch", eventName, data);
+  eventName: string,
+  data: any,
+  timeout?: number
+): Promise<any[]> {
+  return new Promise((resolve, reject) => {
+    let timeOut = setTimeout(() => {
+      console.log("Timeout!");
+      reject("timeout");
+    }, timeout);
+    syscall("event.dispatch", eventName, data)
+      .then((r) => {
+        clearTimeout(timeOut);
+        resolve(r);
+      })
+      .catch(reject);
+  });
 }

plugos-syscall/fetch.ts

@@ -6,7 +6,7 @@ export async function json(url: RequestInfo, init: RequestInit): Promise<any> {
 
 export async function text(
   url: RequestInfo,
-  init: RequestInit
+  init: RequestInit = {}
 ): Promise<string> {
   return syscall("fetch.text", url, init);
 }

plugos/bin/plugos-bundle.ts

@@ -14,7 +14,7 @@ async function compile(
   filePath: string,
   functionName: string,
   debug: boolean,
-  meta = true
+  meta = false
 ) {
   let outFile = "_out.tmp";
   let inFile = filePath;

plugos/environments/node_worker.ts

@@ -21,8 +21,12 @@ let syscallReqId = 0;
 let vm = new VM({
   sandbox: {
     console,
+    setTimeout,
+    clearTimeout,
+    setInterval,
+    clearInterval,
     require: (moduleName: string): any => {
-      console.log("Loading", moduleName);
+      // console.log("Loading", moduleName);
       if (preloadModules.includes(moduleName)) {
         return require(`${workerData}/${moduleName}`);
       } else {

plugos/hooks/event.ts

@@ -12,10 +12,11 @@ export type EventHookT = {
 export class EventHook implements Hook<EventHookT> {
   private system?: System<EventHookT>;
 
-  async dispatchEvent(eventName: string, data?: any): Promise<void> {
+  async dispatchEvent(eventName: string, data?: any): Promise<any[]> {
     if (!this.system) {
       throw new Error("Event hook is not initialized");
     }
+    let responses: any[] = [];
     for (const plug of this.system.loadedPlugs.values()) {
       for (const [name, functionDef] of Object.entries(
         plug.manifest!.functions
@@ -23,11 +24,15 @@ export class EventHook implements Hook<EventHookT> {
         if (functionDef.events && functionDef.events.includes(eventName)) {
           // Only dispatch functions that can run in this environment
           if (plug.canInvoke(name)) {
-            await plug.invoke(name, [data]);
+            let result = await plug.invoke(name, [data]);
+            if (result !== undefined) {
+              responses.push(result);
+            }
           }
         }
       }
     }
+    return responses;
   }
 
   apply(system: System<EventHookT>): void {

plugs/core/core.plug.yaml

@@ -25,14 +25,26 @@ functions:
     events:
       - page:saved
       - page:deleted
+  pageQueryProvider:
+    path: ./page.ts:pageQueryProvider
+    events:
+      - query:page
   indexLinks:
     path: "./page.ts:indexLinks"
     events:
       - page:index
+  linkQueryProvider:
+    path: ./page.ts:linkQueryProvider
+    events:
+      - query:link
   indexItems:
     path: "./item.ts:indexItems"
     events:
       - page:index
+  itemQueryProvider:
+    path: ./item.ts:queryProvider
+    events:
+      - query:item
   deletePage:
     path: "./page.ts:deletePage"
     command:
@@ -52,7 +64,8 @@ functions:
       key: Ctrl-Alt-r
   pageComplete:
     path: "./page.ts:pageComplete"
-    isCompleter: true
+    events:
+      - page:complete
   linkNavigate:
     path: "./navigate.ts:linkNavigate"
     command:
@@ -86,3 +99,14 @@ functions:
     path: ./template.ts:instantiateTemplateCommand
     command:
       name: "Template: Instantiate for Page"
+
+  instantiateTemplate:
+    path: ./template.ts:instantiateTemplate
+    env: server
+
+
+
+  replaceTemplateVarsCommand:
+    path: ./template.ts:replaceTemplateVarsCommand
+    command:
+      name: "Template: Replace Variables"

plugs/core/item.ts

@@ -1,9 +1,10 @@
 import { IndexEvent } from "../../webapp/app_event";
 
-import { batchSet } from "plugos-silverbullet-syscall/index";
+import { batchSet, scanPrefixGlobal } from "plugos-silverbullet-syscall/index";
 import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
 import { collectNodesOfType, ParseTree, renderToText } from "../../common/tree";
 import { whiteOutQueries } from "../query/util";
+import { applyQuery, QueryProviderEvent } from "../query/engine";
 
 export type Item = {
   name: string;
@@ -50,3 +51,23 @@ export async function indexItems({ name, text }: IndexEvent) {
   console.log("Found", items.length, "item(s)");
   await batchSet(name, items);
 }
+
+export async function queryProvider({
+  query,
+}: QueryProviderEvent): Promise<string> {
+  let allItems: Item[] = [];
+  for (let { key, page, value } of await scanPrefixGlobal("it:")) {
+    let [, pos] = key.split(":");
+    allItems.push({
+      ...value,
+      page: page,
+      pos: +pos,
+    });
+  }
+  let markdownItems = applyQuery(query, allItems).map(
+    (item) =>
+      `* [[${item.page}@${item.pos}]] ${item.name}` +
+      (item.nested ? "\n  " + item.nested : "")
+  );
+  return markdownItems.join("\n");
+}

plugs/core/page.ts

@@ -25,6 +25,8 @@ import {
   renderToText,
   replaceNodesMatching
 } from "../../common/tree";
+import { applyQuery, QueryProviderEvent } from "../query/engine";
+import { PageMeta } from "../../common/types";
 
 export async function indexLinks({ name, text }: IndexEvent) {
   let backLinks: { key: string; value: string }[] = [];
@@ -47,6 +49,31 @@ export async function indexLinks({ name, text }: IndexEvent) {
   await batchSet(name, backLinks);
 }
 
+export async function pageQueryProvider({
+  query,
+}: QueryProviderEvent): Promise<string> {
+  let allPages = await listPages();
+  let markdownPages = applyQuery(query, allPages).map(
+    (pageMeta: PageMeta) => `* [[${pageMeta.name}]]`
+  );
+  return markdownPages.join("\n");
+}
+
+export async function linkQueryProvider({
+  query,
+  pageName,
+}: QueryProviderEvent): Promise<string> {
+  let uniqueLinks = new Set<string>();
+  for (let { value: name } of await scanPrefixGlobal(`pl:${pageName}:`)) {
+    uniqueLinks.add(name);
+  }
+  let markdownLinks = applyQuery(
+    query,
+    [...uniqueLinks].map((l) => ({ name: l }))
+  ).map((pageMeta) => `* [[${pageMeta.name}]]`);
+  return markdownLinks.join("\n");
+}
+
 export async function deletePage() {
   let pageName = await getCurrentPage();
   console.log("Navigating to start page");

plugs/core/template.ts

@@ -1,9 +1,11 @@
 import { listPages, readPage, writePage } from "plugos-silverbullet-syscall/space";
-import { filterBox, navigate, prompt } from "plugos-silverbullet-syscall/editor";
+import { filterBox, getCurrentPage, getText, navigate, prompt } from "plugos-silverbullet-syscall/editor";
 import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
 import { extractMeta } from "../query/data";
 import { renderToText } from "../../common/tree";
 import { niceDate } from "./dates";
+import { dispatch } from "plugos-syscall/event";
+import { invokeFunction } from "plugos-silverbullet-syscall/system";
 
 const pageTemplatePrefix = `template/page/`;
 
@@ -34,17 +36,41 @@ export async function instantiateTemplateCommand() {
   if (!pageName) {
     return;
   }
-  let pageText = replaceTemplateVars(renderToText(parseTree));
+  await invokeFunction(
-  await writePage(pageName, pageText);
+    "server",
+    "instantiateTemplate",
+    pageName,
+    renderToText(parseTree)
+  );
+  // let pageText = replaceTemplateVars(, pageName);
+  // await writePage(pageName, pageText);
   await navigate(pageName);
 }
 
-export function replaceTemplateVars(s: string): string {
+export async function instantiateTemplate(pageName: string, text: string) {
-  return s.replaceAll(/\{\{(\w+)\}\}/g, (match, v) => {
+  let pageText = replaceTemplateVars(text, pageName);
-    switch (v) {
+  await writePage(pageName, pageText);
-      case "today":
+}
-        return niceDate(new Date());
+
-        break;
+export async function replaceTemplateVarsCommand() {
+  let currentPage = await getCurrentPage();
+  let text = await getText();
+  await invokeFunction("server", "instantiateTemplate", currentPage, text);
+}
+
+export function replaceTemplateVars(s: string, pageName: string): string {
+  return s.replaceAll(/\{\{([^\}]+)\}\}/g, (match, v) => {
+    if (v === "today") {
+      return niceDate(new Date());
+    }
+    if (v.startsWith("placeholder:")) {
+      // Dispatch event, to be replaced in the file async later
+      dispatch(v, {
+        pageName: pageName,
+        placeholder: v,
+      }).catch((e) => {
+        console.error("Failed to dispatch placeholder event", e);
+      });
     }
     return match;
   });

plugs/emoji/emoji.plug.yaml

@@ -1,4 +1,5 @@
 functions:
   emojiCompleter:
     path: "./emoji.ts:emojiCompleter"
-    isCompleter: true
+    events:
+      - page:complete

plugs/ghost/ghost.ts

@@ -1,9 +1,10 @@
 import { readPage, writePage } from "plugos-silverbullet-syscall/space";
 import { json } from "plugos-syscall/fetch";
-import { parse as parseYaml } from "yaml";
 import { invokeFunction } from "plugos-silverbullet-syscall/system";
 import { getCurrentPage, getText } from "plugos-silverbullet-syscall/editor";
 import { cleanMarkdown } from "../markdown/util";
+import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
+import { extractMeta } from "../query/data";
 
 type GhostConfig = {
   url: string;
@@ -182,14 +183,10 @@ async function markdownToPost(text: string): Promise<Partial<Post>> {
 }
 
 async function getConfig(): Promise<GhostConfig> {
-  let configPage = await readPage("ghost-config");
+  let { text } = await readPage("ghost-config");
-  return parseYaml(configPage.text) as GhostConfig;
+  let parsedContent = await parseMarkdown(text);
-  // return {
+  let pageMeta = await extractMeta(parsedContent);
-  //   adminKey: "",
+  return pageMeta as GhostConfig;
-  //   pagePrefix: "",
-  //   postPrefix: "",
-  //   url: "",
-  // };
 }
 
 export async function downloadAllPostsCommand() {

plugs/lib/util.ts

@@ -0,0 +1,14 @@
+export async function replaceAsync(
+  str: string,
+  regex: RegExp,
+  asyncFn: (match: string, ...args: any[]) => Promise<string>
+) {
+  const promises: Promise<string>[] = [];
+  str.replace(regex, (match: string, ...args: any[]): string => {
+    const promise = asyncFn(match, ...args);
+    promises.push(promise);
+    return "";
+  });
+  const data = await Promise.all(promises);
+  return str.replace(regex, () => data.shift()!);
+}

plugs/mattermost/client.ts

@@ -0,0 +1,95 @@
+import { readPage } from "plugos-silverbullet-syscall/space";
+import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
+import { extractMeta } from "../query/data";
+import { UserProfile } from "@hmhealey/types/lib/users";
+import { json } from "plugos-syscall/fetch";
+import { Post } from "@hmhealey/types/lib/posts";
+import { Channel } from "@hmhealey/types/lib/channels";
+import { Team } from "@hmhealey/types/lib/teams";
+
+type MattermostConfig = {
+  url: string;
+  token: string;
+};
+
+async function getConfig(): Promise<MattermostConfig> {
+  let { text } = await readPage("mattermost-config");
+  let parsedContent = await parseMarkdown(text);
+  let pageMeta = await extractMeta(parsedContent);
+  return pageMeta as MattermostConfig;
+}
+
+export class MattermostClient {
+  userCache = new Map<string, UserProfile>();
+  channelCache = new Map<string, Channel>();
+  teamCache = new Map<string, Team>();
+
+  constructor(readonly url: string, readonly token: string) {}
+
+  static async fromConfig(): Promise<MattermostClient> {
+    let config = await getConfig();
+    return new MattermostClient(config.url, config.token);
+  }
+
+  getMe(): Promise<UserProfile> {
+    return this.getUser("me");
+  }
+
+  async getUser(userId: string): Promise<UserProfile> {
+    let user = this.userCache.get(userId);
+    if (user) {
+      return user;
+    }
+    user = await json(`${this.url}/api/v4/users/${userId}`, {
+      headers: {
+        Authorization: `Bearer ${this.token}`,
+      },
+    });
+    this.userCache.set(userId, user!);
+    return user!;
+  }
+
+  async getChannel(channelId: string): Promise<Channel> {
+    let channel = this.channelCache.get(channelId);
+    if (channel) {
+      return channel;
+    }
+    channel = await json(`${this.url}/api/v4/channels/${channelId}`, {
+      headers: {
+        Authorization: `Bearer ${this.token}`,
+      },
+    });
+    this.channelCache.set(channelId, channel!);
+    return channel!;
+  }
+
+  async getTeam(teamId: string): Promise<Team> {
+    let team = this.teamCache.get(teamId);
+    if (team) {
+      return team;
+    }
+    team = await json(`${this.url}/api/v4/teams/${teamId}`, {
+      headers: {
+        Authorization: `Bearer ${this.token}`,
+      },
+    });
+    this.teamCache.set(teamId, team!);
+    return team!;
+  }
+
+  async getFlaggedPosts(userId: string, perPage: number = 10): Promise<Post[]> {
+    let postCollection = await json(
+      `${this.url}/api/v4/users/${userId}/posts/flagged?per_page=${perPage}`,
+      {
+        headers: {
+          Authorization: `Bearer ${this.token}`,
+        },
+      }
+    );
+    let posts: Post[] = [];
+    for (let order of postCollection.order) {
+      posts.push(postCollection.posts[order]);
+    }
+    return posts;
+  }
+}

plugs/mattermost/mattermost.plug.yaml

@@ -0,0 +1,5 @@
+functions:
+  test:
+    path: mattermost.ts:savedPostsQueryProvider
+    events:
+      - query:mm-saved

plugs/mattermost/mattermost.ts

@@ -0,0 +1,37 @@
+import { MattermostClient } from "./client";
+import { applyQuery, QueryProviderEvent } from "../query/engine";
+
+// https://community.mattermost.com/private-core/pl/rbp7a7jtr3f89nzsefo6ftqt3o
+
+function mattermostDesktopUrlForPost(
+  url: string,
+  teamName: string,
+  postId: string
+) {
+  return `${url.replace("https://", "mattermost://")}/${teamName}/pl/${postId}`;
+}
+
+export async function savedPostsQueryProvider({
+  query,
+}: QueryProviderEvent): Promise<string> {
+  let client = await MattermostClient.fromConfig();
+  let me = await client.getMe();
+  let savedPosts = await client.getFlaggedPosts(me.id);
+  let savedPostsMd = [];
+  savedPosts = applyQuery(query, savedPosts);
+  for (let savedPost of savedPosts) {
+    // savedPost.
+    let channel = await client.getChannel(savedPost.channel_id);
+    let team = await client.getTeam(channel.team_id);
+    savedPostsMd.push(
+      `@${
+        (await client.getUser(savedPost.user_id)).username
+      } [link](${mattermostDesktopUrlForPost(
+        client.url,
+        team.name,
+        savedPost.id
+      )}):\n> ${savedPost.message.replaceAll(/\n/g, "\n> ")}`
+    );
+  }
+  return savedPostsMd.join("\n\n");
+}

plugs/package-lock.json

@@ -8,6 +8,7 @@
       "name": "plugs",
       "version": "1.0.0",
       "dependencies": {
+        "@hmhealey/types": "^6.6.0-4",
         "@jest/globals": "^27.5.1",
         "@lezer/generator": "^0.15.4",
         "@lezer/lr": "^0.15.8",
@@ -119,6 +120,19 @@
         "node": ">=4"
       }
     },
+    "node_modules/@hmhealey/types": {
+      "version": "6.6.0-4",
+      "resolved": "https://registry.npmjs.org/@hmhealey/types/-/types-6.6.0-4.tgz",
+      "integrity": "sha512-71IxVaXhrUesmLnvQQh4RtUqqhmVL+ejci4qo4R6rTWTdY77BniRtBx269uAz34wzTlAgITysN8x7MBTdt/XBg==",
+      "peerDependencies": {
+        "typescript": "^4.3"
+      },
+      "peerDependenciesMeta": {
+        "typescript": {
+          "optional": true
+        }
+      }
+    },
     "node_modules/@jest/environment": {
       "version": "27.5.1",
       "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-27.5.1.tgz",
@@ -683,6 +697,12 @@
         }
       }
     },
+    "@hmhealey/types": {
+      "version": "6.6.0-4",
+      "resolved": "https://registry.npmjs.org/@hmhealey/types/-/types-6.6.0-4.tgz",
+      "integrity": "sha512-71IxVaXhrUesmLnvQQh4RtUqqhmVL+ejci4qo4R6rTWTdY77BniRtBx269uAz34wzTlAgITysN8x7MBTdt/XBg==",
+      "requires": {}
+    },
     "@jest/environment": {
       "version": "27.5.1",
       "resolved": "https://registry.npmjs.org/@jest/environment/-/environment-27.5.1.tgz",

plugs/package.json

@@ -5,6 +5,7 @@
     "generate": "lezer-generator query/query.grammar -o query/parse-query.js"
   },
   "dependencies": {
+    "@hmhealey/types": "^6.6.0-4",
     "@jest/globals": "^27.5.1",
     "@lezer/generator": "^0.15.4",
     "@lezer/lr": "^0.15.8",

plugs/query/data.ts

@@ -2,14 +2,15 @@
 // data:page@pos
 
 import { IndexEvent } from "../../webapp/app_event";
-import { batchSet } from "plugos-silverbullet-syscall";
+import { batchSet, scanPrefixGlobal } from "plugos-silverbullet-syscall";
 import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
 import { collectNodesOfType, findNodeOfType, ParseTree, replaceNodesMatching } from "../../common/tree";
-import { parse as parseYaml, parseAllDocuments } from "yaml";
+import YAML, { parse as parseYaml, parseAllDocuments } from "yaml";
 import { whiteOutQueries } from "./util";
+import type { QueryProviderEvent } from "./engine";
+import { applyQuery } from "./engine";
 
 export async function indexData({ name, text }: IndexEvent) {
-  let e;
   text = whiteOutQueries(text);
   // console.log("Now data indexing", name);
   let mdTree = await parseMarkdown(text);
@@ -77,3 +78,21 @@ export function extractMeta(parseTree: ParseTree, remove = false): any {
 
   return data;
 }
+
+export async function queryProvider({
+  query,
+}: QueryProviderEvent): Promise<string> {
+  let allData: any[] = [];
+  for (let { key, page, value } of await scanPrefixGlobal("data:")) {
+    let [, pos] = key.split("@");
+    allData.push({
+      ...value,
+      page: page,
+      pos: +pos,
+    });
+  }
+  let markdownData = applyQuery(query, allData).map((item) =>
+    YAML.stringify(item)
+  );
+  return `\`\`\`data\n${markdownData.join("---\n")}\`\`\``;
+}

plugs/query/engine.test.ts

@@ -30,7 +30,7 @@ test("Test parser", () => {
   expect(parsedQuery2.filter[0]).toStrictEqual({
     op: "=~",
     prop: "name",
-    value: /interview\/.*/,
+    value: "interview\\/.*",
   });
 
   let parsedQuery3 = parseQuery(`page where something != null`);

plugs/query/engine.ts

@@ -4,13 +4,18 @@ import { lezerToParseTree } from "../../common/parse_tree";
 // @ts-ignore
 import { parser } from "./parse-query";
 
-type Filter = {
+export type QueryProviderEvent = {
+  query: ParsedQuery;
+  pageName: string;
+};
+
+export type Filter = {
   op: string;
   prop: string;
   value: any;
 };
 
-type ParsedQuery = {
+export type ParsedQuery = {
   table: string;
   orderBy?: string;
   orderDesc?: boolean;
@@ -71,7 +76,7 @@ export function parseQuery(query: string): ParsedQuery {
         break;
       case "Regex":
         val = valNode.children![0].text!;
-        val = new RegExp(val.substring(1, val.length - 1));
+        val = val.substring(1, val.length - 1);
         break;
       case "String":
         val = valNode.children![0].text!;
@@ -129,12 +134,13 @@ export function applyQuery<T>(parsedQuery: ParsedQuery, records: T[]): T[] {
             }
             break;
           case "=~":
-            if (!value.exec(recordAny[prop])) {
+            // TODO: Cache regexps somehow
+            if (!new RegExp(value).exec(recordAny[prop])) {
               continue recordLoop;
             }
             break;
           case "!=~":
-            if (value.exec(recordAny[prop])) {
+            if (new RegExp(value).exec(recordAny[prop])) {
               continue recordLoop;
             }
             break;

plugs/query/materialized_queries.ts

@@ -1,15 +1,11 @@
 import { flashNotification, getCurrentPage, reloadPage, save } from "plugos-silverbullet-syscall/editor";
 
-import { listPages, readPage, writePage } from "plugos-silverbullet-syscall/space";
+import { readPage, writePage } from "plugos-silverbullet-syscall/space";
 import { invokeFunction } from "plugos-silverbullet-syscall/system";
-import { scanPrefixGlobal } from "plugos-silverbullet-syscall";
+import { parseQuery } from "./engine";
-import { applyQuery, parseQuery } from "./engine";
-import { PageMeta } from "../../common/types";
-import type { Task } from "../tasks/task";
-import { Item } from "../core/item";
-import YAML from "yaml";
 import { replaceTemplateVars } from "../core/template";
 import { queryRegex } from "./util";
+import { dispatch } from "plugos-syscall/event";
 
 async function replaceAsync(
   str: string,
@@ -46,79 +42,22 @@ export async function updateMaterializedQueriesOnPage(pageName: string) {
     text,
     queryRegex,
     async (fullMatch, startQuery, query, body, endQuery) => {
-      let parsedQuery = parseQuery(replaceTemplateVars(query));
+      let parsedQuery = parseQuery(replaceTemplateVars(query, pageName));
 
       console.log("Parsed query", parsedQuery);
-
+      // Let's dispatch an event and see what happens
-      switch (parsedQuery.table) {
+      let results = await dispatch(
-        case "page":
+        `query:${parsedQuery.table}`,
-          let allPages = await listPages();
+        { query: parsedQuery, pageName: pageName },
-          let markdownPages = applyQuery(parsedQuery, allPages).map(
+        5000
-            (pageMeta: PageMeta) => `* [[${pageMeta.name}]]`
+      );
-          );
+      if (results.length === 0) {
-          return `${startQuery}\n${markdownPages.join("\n")}\n${endQuery}`;
+        return `${startQuery}\n${endQuery}`;
-        case "task":
+      } else if (results.length === 1) {
-          let allTasks: Task[] = [];
+        return `${startQuery}\n${results[0]}\n${endQuery}`;
-          for (let { key, page, value } of await scanPrefixGlobal("task:")) {
+      } else {
-            let [, pos] = key.split(":");
+        console.error("Too many query results", results);
-            allTasks.push({
+        return fullMatch;
-              ...value,
-              page: page,
-              pos: pos,
-            });
-          }
-          let markdownTasks = applyQuery(parsedQuery, allTasks).map(
-            (t) =>
-              `* [${t.done ? "x" : " "}] [[${t.page}@${t.pos}]] ${t.name}` +
-              (t.nested ? "\n  " + t.nested : "")
-          );
-          return `${startQuery}\n${markdownTasks.join("\n")}\n${endQuery}`;
-        case "link":
-          let uniqueLinks = new Set<string>();
-          for (let { value: name } of await scanPrefixGlobal(
-            `pl:${pageName}:`
-          )) {
-            uniqueLinks.add(name);
-          }
-          let markdownLinks = applyQuery(
-            parsedQuery,
-            [...uniqueLinks].map((l) => ({ name: l }))
-          ).map((pageMeta) => `* [[${pageMeta.name}]]`);
-          return `${startQuery}\n${markdownLinks.join("\n")}\n${endQuery}`;
-        case "item":
-          let allItems: Item[] = [];
-          for (let { key, page, value } of await scanPrefixGlobal("it:")) {
-            let [, pos] = key.split(":");
-            allItems.push({
-              ...value,
-              page: page,
-              pos: +pos,
-            });
-          }
-          let markdownItems = applyQuery(parsedQuery, allItems).map(
-            (item) =>
-              `* [[${item.page}@${item.pos}]] ${item.name}` +
-              (item.nested ? "\n  " + item.nested : "")
-          );
-          return `${startQuery}\n${markdownItems.join("\n")}\n${endQuery}`;
-        case "data":
-          let allData: Object[] = [];
-          for (let { key, page, value } of await scanPrefixGlobal("data:")) {
-            let [, pos] = key.split("@");
-            allData.push({
-              ...value,
-              page: page,
-              pos: +pos,
-            });
-          }
-          let markdownData = applyQuery(parsedQuery, allData).map((item) =>
-            YAML.stringify(item)
-          );
-          return `${startQuery}\n\`\`\`data\n${markdownData.join(
-            "---\n"
-          )}\`\`\`\n${endQuery}`;
-        default:
-          return fullMatch;
       }
     }
   );

plugs/query/parse-query.js

@@ -10,7 +10,7 @@ export const parser = LRParser.deserialize({
   maxTerm: 38,
   skippedNodes: [0],
   repeatNodeCount: 1,
-  tokenData: "4v~RtX^#cpq#cqr$Wrs$k!P!Q%V!Q![%|!^!_&U!_!`&c!`!a&p!c!}&}#T#U'Y#U#V)P#V#W&}#W#X)o#X#Y&}#Y#Z+R#Z#`&}#`#a,s#a#b&}#b#c.h#c#d/z#d#h&}#h#i1o#i#k&}#k#l3R#l#o&}#y#z#c$f$g#c#BY#BZ#c$IS$I_#c$Ip$Iq$k$Iq$Ir$k$I|$JO#c$JT$JU#c$KV$KW#c&FU&FV#c~#hYd~X^#cpq#c#y#z#c$f$g#c#BY#BZ#c$IS$I_#c$I|$JO#c$JT$JU#c$KV$KW#c&FU&FV#c~$ZP!_!`$^~$cPl~#r#s$f~$kOp~~$nUOr$krs%Qs$Ip$k$Ip$Iq%Q$Iq$Ir%Q$Ir~$k~%VOY~~%[V[~OY%VZ]%V^!P%V!P!Q%q!Q#O%V#O#P%v#P~%V~%vO[~~%yPO~%V~&RPX~!Q![%|~&ZPf~!_!`&^~&cOj~~&hPk~#r#s&k~&pOo~~&uPn~!_!`&x~&}Om~P'SQRP!c!}&}#T#o&}R'_URP!c!}&}#T#b&}#b#c'q#c#g&}#g#h(a#h#o&}R'vSRP!c!}&}#T#W&}#W#X(S#X#o&}R(ZQqQRP!c!}&}#T#o&}R(fSRP!c!}&}#T#V&}#V#W(r#W#o&}R(yQuQRP!c!}&}#T#o&}R)USRP!c!}&}#T#m&}#m#n)b#n#o&}R)iQsQRP!c!}&}#T#o&}R)tSRP!c!}&}#T#X&}#X#Y*Q#Y#o&}R*VSRP!c!}&}#T#g&}#g#h*c#h#o&}R*hSRP!c!}&}#T#V&}#V#W*t#W#o&}R*{QtQRP!c!}&}#T#o&}R+WRRP!c!}&}#T#U+a#U#o&}R+fSRP!c!}&}#T#`&}#`#a+r#a#o&}R+wSRP!c!}&}#T#g&}#g#h,T#h#o&}R,YSRP!c!}&}#T#X&}#X#Y,f#Y#o&}R,mQhQRP!c!}&}#T#o&}R,xSRP!c!}&}#T#]&}#]#^-U#^#o&}R-ZSRP!c!}&}#T#a&}#a#b-g#b#o&}R-lSRP!c!}&}#T#]&}#]#^-x#^#o&}R-}SRP!c!}&}#T#h&}#h#i.Z#i#o&}R.bQvQRP!c!}&}#T#o&}R.mSRP!c!}&}#T#i&}#i#j.y#j#o&}R/OSRP!c!}&}#T#`&}#`#a/[#a#o&}R/aSRP!c!}&}#T#`&}#`#a/m#a#o&}R/tQiQRP!c!}&}#T#o&}R0PSRP!c!}&}#T#f&}#f#g0]#g#o&}R0bSRP!c!}&}#T#W&}#W#X0n#X#o&}R0sSRP!c!}&}#T#X&}#X#Y1P#Y#o&}R1USRP!c!}&}#T#f&}#f#g1b#g#o&}R1iQrQRP!c!}&}#T#o&}R1tSRP!c!}&}#T#f&}#f#g2Q#g#o&}R2VSRP!c!}&}#T#i&}#i#j2c#j#o&}R2hSRP!c!}&}#T#X&}#X#Y2t#Y#o&}R2{QgQRP!c!}&}#T#o&}R3WSRP!c!}&}#T#[&}#[#]3d#]#o&}R3iSRP!c!}&}#T#X&}#X#Y3u#Y#o&}R3zSRP!c!}&}#T#f&}#f#g4W#g#o&}R4]SRP!c!}&}#T#X&}#X#Y4i#Y#o&}R4pQeQRP!c!}&}#T#o&}",
+  tokenData: ":W~RvX^#ipq#iqr$^rs$q}!O%]!P!Q%n!Q![&e!^!_&m!_!`&z!`!a'X!c!}%]#R#S%]#T#U'f#U#V){#V#W%]#W#X*w#X#Y%]#Y#Z,s#Z#`%]#`#a/T#a#b%]#b#c1h#c#d3d#d#h%]#h#i5w#i#k%]#k#l7s#l#o%]#y#z#i$f$g#i#BY#BZ#i$IS$I_#i$Ip$Iq$q$Iq$Ir$q$I|$JO#i$JT$JU#i$KV$KW#i&FU&FV#i~#nYd~X^#ipq#i#y#z#i$f$g#i#BY#BZ#i$IS$I_#i$I|$JO#i$JT$JU#i$KV$KW#i&FU&FV#i~$aP!_!`$d~$iPl~#r#s$l~$qOp~~$tUOr$qrs%Ws$Ip$q$Ip$Iq%W$Iq$Ir%W$Ir~$q~%]OY~P%bSRP}!O%]!c!}%]#R#S%]#T#o%]~%sV[~OY%nZ]%n^!P%n!P!Q&Y!Q#O%n#O#P&_#P~%n~&_O[~~&bPO~%n~&jPX~!Q![&e~&rPf~!_!`&u~&zOj~~'PPk~#r#s'S~'XOo~~'^Pn~!_!`'a~'fOm~R'kWRP}!O%]!c!}%]#R#S%]#T#b%]#b#c(T#c#g%]#g#h)P#h#o%]R(YURP}!O%]!c!}%]#R#S%]#T#W%]#W#X(l#X#o%]R(sSqQRP}!O%]!c!}%]#R#S%]#T#o%]R)UURP}!O%]!c!}%]#R#S%]#T#V%]#V#W)h#W#o%]R)oSuQRP}!O%]!c!}%]#R#S%]#T#o%]R*QURP}!O%]!c!}%]#R#S%]#T#m%]#m#n*d#n#o%]R*kSsQRP}!O%]!c!}%]#R#S%]#T#o%]R*|URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y+`#Y#o%]R+eURP}!O%]!c!}%]#R#S%]#T#g%]#g#h+w#h#o%]R+|URP}!O%]!c!}%]#R#S%]#T#V%]#V#W,`#W#o%]R,gStQRP}!O%]!c!}%]#R#S%]#T#o%]R,xTRP}!O%]!c!}%]#R#S%]#T#U-X#U#o%]R-^URP}!O%]!c!}%]#R#S%]#T#`%]#`#a-p#a#o%]R-uURP}!O%]!c!}%]#R#S%]#T#g%]#g#h.X#h#o%]R.^URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y.p#Y#o%]R.wShQRP}!O%]!c!}%]#R#S%]#T#o%]R/YURP}!O%]!c!}%]#R#S%]#T#]%]#]#^/l#^#o%]R/qURP}!O%]!c!}%]#R#S%]#T#a%]#a#b0T#b#o%]R0YURP}!O%]!c!}%]#R#S%]#T#]%]#]#^0l#^#o%]R0qURP}!O%]!c!}%]#R#S%]#T#h%]#h#i1T#i#o%]R1[SvQRP}!O%]!c!}%]#R#S%]#T#o%]R1mURP}!O%]!c!}%]#R#S%]#T#i%]#i#j2P#j#o%]R2UURP}!O%]!c!}%]#R#S%]#T#`%]#`#a2h#a#o%]R2mURP}!O%]!c!}%]#R#S%]#T#`%]#`#a3P#a#o%]R3WSiQRP}!O%]!c!}%]#R#S%]#T#o%]R3iURP}!O%]!c!}%]#R#S%]#T#f%]#f#g3{#g#o%]R4QURP}!O%]!c!}%]#R#S%]#T#W%]#W#X4d#X#o%]R4iURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y4{#Y#o%]R5QURP}!O%]!c!}%]#R#S%]#T#f%]#f#g5d#g#o%]R5kSrQRP}!O%]!c!}%]#R#S%]#T#o%]R5|URP}!O%]!c!}%]#R#S%]#T#f%]#f#g6`#g#o%]R6eURP}!O%]!c!}%]#R#S%]#T#i%]#i#j6w#j#o%]R6|URP}!O%]!c!}%]#R#S%]#T#X%]#X#Y7`#Y#o%]R7gSgQRP}!O%]!c!}%]#R#S%]#T#o%]R7xURP}!O%]!c!}%]#R#S%]#T#[%]#[#]8[#]#o%]R8aURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y8s#Y#o%]R8xURP}!O%]!c!}%]#R#S%]#T#f%]#f#g9[#g#o%]R9aURP}!O%]!c!}%]#R#S%]#T#X%]#X#Y9s#Y#o%]R9zSeQRP}!O%]!c!}%]#R#S%]#T#o%]",
   tokenizers: [0, 1],
   topRules: {"Program":[0,1]},
   tokenPrec: 0

plugs/query/query.grammar

@@ -43,7 +43,7 @@ Null {
 
 @tokens {
   space { std.whitespace+ }
-  Name { std.asciiLetter+ }
+  Name { (std.asciiLetter | "-" | "_")+ }
   String {
     ("\"" | "“" | "”") ![\"”“]* ("\"" | "“" | "”")
   }

plugs/query/query.plug.yaml

@@ -10,3 +10,7 @@ functions:
     path: ./data.ts:indexData
     events:
       - page:index
+  dataQueryProvider:
+    path: ./data.ts:queryProvider
+    events:
+      - query:data

plugs/tasks/task.ts

@@ -1,6 +1,6 @@
 import type { ClickEvent, IndexEvent } from "../../webapp/app_event";
 
-import { batchSet } from "plugos-silverbullet-syscall/index";
+import { batchSet, scanPrefixGlobal } from "plugos-silverbullet-syscall/index";
 import { readPage, writePage } from "plugos-silverbullet-syscall/space";
 import { parseMarkdown } from "plugos-silverbullet-syscall/markdown";
 import { dispatch, getCurrentPage, getText } from "plugos-silverbullet-syscall/editor";
@@ -12,6 +12,7 @@ import {
   renderToText
 } from "../../common/tree";
 import { whiteOutQueries } from "../query/util";
+import { applyQuery, QueryProviderEvent } from "../query/engine";
 
 export type Task = {
   name: string;
@@ -120,3 +121,23 @@ export async function taskToggleAtPos(pos: number) {
     }
   }
 }
+
+export async function queryProvider({
+  query,
+}: QueryProviderEvent): Promise<string> {
+  let allTasks: Task[] = [];
+  for (let { key, page, value } of await scanPrefixGlobal("task:")) {
+    let [, pos] = key.split(":");
+    allTasks.push({
+      ...value,
+      page: page,
+      pos: pos,
+    });
+  }
+  let markdownTasks = applyQuery(query, allTasks).map(
+    (t) =>
+      `* [${t.done ? "x" : " "}] [[${t.page}@${t.pos}]] ${t.name}` +
+      (t.nested ? "\n  " + t.nested : "")
+  );
+  return markdownTasks.join("\n");
+}

plugs/tasks/tasks.plug.yaml

@@ -26,4 +26,8 @@ functions:
     path: "./task.ts:taskToggle"
     events:
       - page:click
+  itemQueryProvider:
+    path: ./task.ts:queryProvider
+    events:
+      - query:task
 

plugs/yarn.lock

@@ -23,6 +23,11 @@
     "chalk" "^2.0.0"
     "js-tokens" "^4.0.0"
 
+"@hmhealey/types@^6.6.0-4":
+  "integrity" "sha512-71IxVaXhrUesmLnvQQh4RtUqqhmVL+ejci4qo4R6rTWTdY77BniRtBx269uAz34wzTlAgITysN8x7MBTdt/XBg=="
+  "resolved" "https://registry.npmjs.org/@hmhealey/types/-/types-6.6.0-4.tgz"
+  "version" "6.6.0-4"
+
 "@jest/environment@^27.5.1":
   "integrity" "sha512-/WQjhPJe3/ghaol/4Bq480JKXV/Rfw8nQdN7f41fM8VDHLcxKXou6QyXAh3EFr9/bVG3x74z1NWDkP87EiY8gA=="
   "resolved" "https://registry.npmjs.org/@jest/environment/-/environment-27.5.1.tgz"

webapp/app_event.ts

@@ -1,4 +1,4 @@
-export type AppEvent = "page:click" | "editor:complete";
+export type AppEvent = "page:click" | "page:complete";
 
 export type ClickEvent = {
   page: string;
@@ -12,7 +12,3 @@ export type IndexEvent = {
   name: string;
   text: string;
 };
-
-export interface AppEventDispatcher {
-  dispatchAppEvent(name: AppEvent, data?: any): Promise<void>;
-}

webapp/editor.tsx

@@ -1,4 +1,4 @@
-import { autocompletion, completionKeymap } from "@codemirror/autocomplete";
+import { autocompletion, completionKeymap, CompletionResult } from "@codemirror/autocomplete";
 import { closeBrackets, closeBracketsKeymap } from "@codemirror/closebrackets";
 import { indentWithTab, standardKeymap } from "@codemirror/commands";
 import { history, historyKeymap } from "@codemirror/history";
@@ -18,7 +18,7 @@ import {
 import React, { useEffect, useReducer } from "react";
 import ReactDOM from "react-dom";
 import { createSandbox as createIFrameSandbox } from "../plugos/environments/webworker_sandbox";
-import { AppEvent, AppEventDispatcher, ClickEvent } from "./app_event";
+import { AppEvent, ClickEvent } from "./app_event";
 import * as commands from "./commands";
 import { CommandPalette } from "./components/command_palette";
 import { PageNavigator } from "./components/page_navigator";
@@ -44,7 +44,6 @@ import { systemSyscalls } from "./syscalls/system";
 import { Panel } from "./components/panel";
 import { CommandHook } from "./hooks/command";
 import { SlashCommandHook } from "./hooks/slash_command";
-import { CompleterHook } from "./hooks/completer";
 import { pasteLinkExtension } from "./editor_paste";
 import { markdownSyscalls } from "../common/syscalls/markdown";
 import { clientStoreSyscalls } from "./syscalls/clientStore";
@@ -65,10 +64,9 @@ class PageState {
 
 const saveInterval = 1000;
 
-export class Editor implements AppEventDispatcher {
+export class Editor {
   readonly commandHook: CommandHook;
   readonly slashCommandHook: SlashCommandHook;
-  readonly completerHook: CompleterHook;
   openPages = new Map<string, PageState>();
   editorView?: EditorView;
   viewState: AppViewState;
@@ -78,7 +76,9 @@ export class Editor implements AppEventDispatcher {
   eventHook: EventHook;
   saveTimeout: any;
   debouncedUpdateEvent = throttle(() => {
-    this.eventHook.dispatchEvent("editor:updated");
+    this.eventHook
+      .dispatchEvent("editor:updated")
+      .catch((e) => console.error("Error dispatching editor:updated event", e));
   }, 1000);
   private system = new System<SilverBulletHooks>("client");
   private mdExtensions: MDExt[] = [];
@@ -108,10 +108,6 @@ export class Editor implements AppEventDispatcher {
     this.slashCommandHook = new SlashCommandHook(this);
     this.system.addHook(this.slashCommandHook);
 
-    // Completer hook
-    this.completerHook = new CompleterHook();
-    this.system.addHook(this.completerHook);
-
     this.render(parent);
     this.editorView = new EditorView({
       state: this.createEditorState("", ""),
@@ -261,13 +257,14 @@ export class Editor implements AppEventDispatcher {
         helpText,
         onSelect: (option) => {
           this.viewDispatch({ type: "hide-filterbox" });
+          this.focus();
           resolve(option);
         },
       });
     });
   }
 
-  async dispatchAppEvent(name: AppEvent, data?: any): Promise<void> {
+  async dispatchAppEvent(name: AppEvent, data?: any): Promise<any[]> {
     return this.eventHook.dispatchEvent(name, data);
   }
 
@@ -303,7 +300,8 @@ export class Editor implements AppEventDispatcher {
         closeBrackets(),
         autocompletion({
           override: [
-            this.completerHook.plugCompleter.bind(this.completerHook),
+            // this.completerHook.plugCompleter.bind(this.completerHook),
+            this.completer.bind(this),
             this.slashCommandHook.slashCommandCompleter.bind(
               this.slashCommandHook
             ),
@@ -408,7 +406,7 @@ export class Editor implements AppEventDispatcher {
               if (update.docChanged) {
                 editor.viewDispatch({ type: "page-changed" });
                 editor.debouncedUpdateEvent();
-                editor.save();
+                editor.save().catch((e) => console.error("Error saving", e));
               }
             }
           }
@@ -438,6 +436,23 @@ export class Editor implements AppEventDispatcher {
     }
   }
 
+  async completer(): Promise<CompletionResult | null> {
+    let results = await this.dispatchAppEvent("page:complete");
+    let actualResult = null;
+    for (const result of results) {
+      if (result) {
+        if (actualResult) {
+          console.error(
+            "Got completion results from multiple sources, cannot deal with that"
+          );
+          return null;
+        }
+        actualResult = result;
+      }
+    }
+    return actualResult;
+  }
+
   reloadPage() {
     console.log("Reloading page");
     safeRun(async () => {

webapp/hooks/completer.ts

@@ -1,49 +0,0 @@
-import { Hook, Manifest } from "../../plugos/types";
-import { System } from "../../plugos/system";
-import { CompletionResult } from "@codemirror/autocomplete";
-
-export type CompleterHookT = {
-  isCompleter?: boolean;
-};
-
-export class CompleterHook implements Hook<CompleterHookT> {
-  private system?: System<CompleterHookT>;
-
-  public async plugCompleter(): Promise<CompletionResult | null> {
-    let completerPromises = [];
-    // TODO: Can be optimized (cache all functions)
-    for (const plug of this.system!.loadedPlugs.values()) {
-      if (!plug.manifest) {
-        continue;
-      }
-      for (const [functionName, functionDef] of Object.entries(
-        plug.manifest.functions
-      )) {
-        if (functionDef.isCompleter) {
-          completerPromises.push(plug.invoke(functionName, []));
-        }
-      }
-    }
-    let actualResult = null;
-    for (const result of await Promise.all(completerPromises)) {
-      if (result) {
-        if (actualResult) {
-          console.error(
-            "Got completion results from multiple sources, cannot deal with that"
-          );
-          return null;
-        }
-        actualResult = result;
-      }
-    }
-    return actualResult;
-  }
-
-  apply(system: System<CompleterHookT>): void {
-    this.system = system;
-  }
-
-  validateManifest(manifest: Manifest<CompleterHookT>): string[] {
-    return [];
-  }
-}