Changed how data is stored quite a bit with nice query capabilities

packages/plugos-silverbullet-syscall/index.ts

@@ -1,3 +1,4 @@
+import type { Query } from "@plugos/plugos-syscall/store";
 import { syscall } from "./syscall";
 
 export type KV = {
@@ -25,17 +26,16 @@ export async function del(page: string, key: string): Promise<void> {
   return syscall("index.delete", page, key);
 }
 
-export async function scanPrefixForPage(
-  page: string,
+export async function queryPrefix(
   prefix: string
 ): Promise<{ key: string; page: string; value: any }[]> {
-  return syscall("index.scanPrefixForPage", page, prefix);
+  return syscall("index.queryPrefix", prefix);
 }
 
-export async function scanPrefixGlobal(
-  prefix: string
+export async function query(
+  query: Query
 ): Promise<{ key: string; page: string; value: any }[]> {
-  return syscall("index.scanPrefixGlobal", prefix);
+  return syscall("index.query", query);
 }
 
 export async function clearPageIndexForPage(page: string): Promise<void> {

packages/plugos-syscall/store.ts

@@ -5,6 +5,20 @@ export type KV = {
   value: any;
 };
 
+export type Query = {
+  filter?: Filter[];
+  orderBy?: string;
+  orderDesc?: boolean;
+  limit?: number;
+  select?: string[];
+};
+
+export type Filter = {
+  op: string;
+  prop: string;
+  value: any;
+};
+
 export async function set(key: string, value: any): Promise<void> {
   return syscall("store.set", key, value);
 }
@@ -28,7 +42,13 @@ export async function batchDel(keys: string[]): Promise<void> {
 export async function queryPrefix(
   prefix: string
 ): Promise<{ key: string; value: any }[]> {
-  return syscall("store.scanPrefix", prefix);
+  return syscall("store.queryPrefix", prefix);
+}
+
+export async function query(
+  query: Query
+): Promise<{ key: string; value: any }[]> {
+  return syscall("store.query", query);
 }
 
 export async function deletePrefix(prefix: string): Promise<void> {

packages/plugos/runtime.test.ts

@@ -93,19 +93,19 @@ test("Run a Node sandbox", async () => {
     await plug.invoke("errorOut", []);
     expect(true).toBe(false);
   } catch (e: any) {
-    expect(e.message).toBe("BOOM");
+    expect(e.message).toContain("BOOM");
   }
   try {
     await plug.invoke("errorOutSys", []);
     expect(true).toBe(false);
   } catch (e: any) {
-    expect(e.message).toBe("#fail");
+    expect(e.message).toContain("#fail");
   }
   try {
     await plug.invoke("restrictedTest", []);
     expect(true).toBe(false);
   } catch (e: any) {
-    expect(e.message).toBe(
+    expect(e.message).toContain(
       "Missing permission 'restricted' for syscall restrictedSyscall"
     );
   }

packages/plugos/syscalls/store.knex_node.test.ts

@@ -15,7 +15,8 @@ test("Test store", async () => {
   });
   await ensureTable(db, "test_table");
   let system = new System("server");
-  system.registerSyscalls([], storeSyscalls(db, "test_table"));
+  let syscalls = storeSyscalls(db, "test_table");
+  system.registerSyscalls([], syscalls);
   let plug = await system.load(
     {
       name: "test",
@@ -36,5 +37,79 @@ test("Test store", async () => {
   );
   expect(await plug.invoke("test1", [])).toBe("Pete");
   await system.unloadAll();
+
+  let dummyCtx: any = {};
+
+  await syscalls["store.deleteAll"](dummyCtx);
+  await syscalls["store.batchSet"](dummyCtx, [
+    {
+      key: "pete",
+      value: {
+        age: 20,
+        firstName: "Pete",
+        lastName: "Roberts",
+      },
+    },
+    {
+      key: "petejr",
+      value: {
+        age: 8,
+        firstName: "Pete Jr",
+        lastName: "Roberts",
+      },
+    },
+    {
+      key: "petesr",
+      value: {
+        age: 78,
+        firstName: "Pete Sr",
+        lastName: "Roberts",
+      },
+    },
+  ]);
+
+  let allRoberts = await syscalls["store.query"](dummyCtx, {
+    filter: [{ op: "=", prop: "lastName", value: "Roberts" }],
+    orderBy: "age",
+    orderDesc: true,
+  });
+
+  expect(allRoberts.length).toBe(3);
+  expect(allRoberts[0].key).toBe("petesr");
+
+  allRoberts = await syscalls["store.query"](dummyCtx, {
+    filter: [{ op: "=", prop: "lastName", value: "Roberts" }],
+    orderBy: "age",
+    limit: 1,
+  });
+
+  expect(allRoberts.length).toBe(1);
+  expect(allRoberts[0].key).toBe("petejr");
+
+  allRoberts = await syscalls["store.query"](dummyCtx, {
+    filter: [
+      { op: ">", prop: "age", value: 10 },
+      { op: "<", prop: "age", value: 30 },
+    ],
+    orderBy: "age",
+  });
+
+  expect(allRoberts.length).toBe(1);
+  expect(allRoberts[0].key).toBe("pete");
+
+  // Delete the middle one
+
+  await syscalls["store.deleteQuery"](dummyCtx, {
+    filter: [
+      { op: ">", prop: "age", value: 10 },
+      { op: "<", prop: "age", value: 30 },
+    ],
+  });
+
+  allRoberts = await syscalls["store.query"](dummyCtx, {});
+  expect(allRoberts.length).toBe(2);
+
+  await db.destroy();
+
   await fs.unlink("test.db");
 });

packages/plugos/syscalls/store.knex_node.ts

@@ -24,6 +24,45 @@ export async function ensureTable(db: Knex<any, unknown>, tableName: string) {
   }
 }
 
+export type Query = {
+  filter?: Filter[];
+  orderBy?: string;
+  orderDesc?: boolean;
+  limit?: number;
+  select?: string[];
+};
+
+export type Filter = {
+  op: string;
+  prop: string;
+  value: any;
+};
+
+export function queryToKnex(
+  queryBuilder: Knex.QueryBuilder<Item, any>,
+  query: Query
+): Knex.QueryBuilder<Item, any> {
+  if (query.filter) {
+    for (let filter of query.filter) {
+      queryBuilder = queryBuilder.andWhereRaw(
+        `json_extract(value, '$.${filter.prop}') ${filter.op} ?`,
+        [filter.value]
+      );
+    }
+  }
+  if (query.limit) {
+    queryBuilder = queryBuilder.limit(query.limit);
+  }
+  if (query.orderBy) {
+    queryBuilder = queryBuilder.orderByRaw(
+      `json_extract(value, '$.${query.orderBy}') ${
+        query.orderDesc ? "desc" : "asc"
+      }`
+    );
+  }
+  return queryBuilder;
+}
+
 export function storeSyscalls(
   db: Knex<any, unknown>,
   tableName: string
@@ -35,6 +74,9 @@ export function storeSyscalls(
     "store.deletePrefix": async (ctx, prefix: string) => {
       return db<Item>(tableName).andWhereLike("key", `${prefix}%`).del();
     },
+    "store.deleteQuery": async (ctx, query: Query) => {
+      await queryToKnex(db<Item>(tableName), query).del();
+    },
     "store.deleteAll": async (ctx) => {
       await db<Item>(tableName).del();
     },
@@ -78,6 +120,14 @@ export function storeSyscalls(
         value: JSON.parse(value),
       }));
     },
+    "store.query": async (ctx, query: Query) => {
+      return (
+        await queryToKnex(db<Item>(tableName), query).select("key", "value")
+      ).map(({ key, value }: { key: string; value: string }) => ({
+        key,
+        value: JSON.parse(value),
+      }));
+    },
   };
   return apiObj;
 }

packages/plugs/build/test/engine.test.js

@@ -422,7 +422,6 @@ function $9072202279b76d33$export$5884dae03c64f759(parsedQuery, records) {
 }
 async function $9072202279b76d33$export$b3c659c1456e61b0(parsedQuery, data) {
     if (parsedQuery.render) {
-        console.log("Handlebars", ($parcel$interopDefault($hVExJ$handlebars)));
         ($parcel$interopDefault($hVExJ$handlebars)).registerHelper("json", (v)=>JSON.stringify(v)
         );
         ($parcel$interopDefault($hVExJ$handlebars)).registerHelper("niceDate", (ts)=>$c3893eec0c49ec96$export$5dc1410f87262ed6(new Date(ts))

packages/plugs/core/item.ts

@@ -2,7 +2,7 @@ import type { IndexTreeEvent } from "@silverbulletmd/web/app_event";
 
 import {
   batchSet,
-  scanPrefixGlobal,
+  queryPrefix,
 } from "@silverbulletmd/plugos-silverbullet-syscall/index";
 import {
   collectNodesOfType,
@@ -61,7 +61,7 @@ export async function queryProvider({
   query,
 }: QueryProviderEvent): Promise<any[]> {
   let allItems: Item[] = [];
-  for (let { key, page, value } of await scanPrefixGlobal("it:")) {
+  for (let { key, page, value } of await queryPrefix("it:")) {
     let [, pos] = key.split(":");
     allItems.push({
       ...value,

packages/plugs/core/page.ts

@@ -3,7 +3,7 @@ import {
   batchSet,
   clearPageIndex as clearPageIndexSyscall,
   clearPageIndexForPage,
-  scanPrefixGlobal,
+  queryPrefix,
   set,
 } from "@silverbulletmd/plugos-silverbullet-syscall/index";
 import {
@@ -73,7 +73,7 @@ export async function pageQueryProvider({
   let allPageMap: Map<string, any> = new Map(
     allPages.map((pm) => [pm.name, pm])
   );
-  for (let { page, value } of await scanPrefixGlobal("meta:")) {
+  for (let { page, value } of await queryPrefix("meta:")) {
     let p = allPageMap.get(page);
     if (p) {
       for (let [k, v] of Object.entries(value)) {
@@ -90,7 +90,7 @@ export async function linkQueryProvider({
   pageName,
 }: QueryProviderEvent): Promise<any[]> {
   let uniqueLinks = new Set<string>();
-  for (let { value: name } of await scanPrefixGlobal(`pl:${pageName}:`)) {
+  for (let { value: name } of await queryPrefix(`pl:${pageName}:`)) {
     uniqueLinks.add(name);
   }
   return applyQuery(
@@ -176,7 +176,7 @@ type BackLink = {
 };
 
 async function getBackLinks(pageName: string): Promise<BackLink[]> {
-  let allBackLinks = await scanPrefixGlobal(`pl:${pageName}:`);
+  let allBackLinks = await queryPrefix(`pl:${pageName}:`);
   let pagesToUpdate: BackLink[] = [];
   for (let { key, value } of allBackLinks) {
     let keyParts = key.split(":");

packages/plugs/query/data.ts

@@ -4,7 +4,7 @@
 import type { IndexTreeEvent } from "@silverbulletmd/web/app_event";
 import {
   batchSet,
-  scanPrefixGlobal,
+  queryPrefix,
 } from "@silverbulletmd/plugos-silverbullet-syscall";
 import {
   collectNodesOfType,
@@ -102,7 +102,7 @@ export async function queryProvider({
   query,
 }: QueryProviderEvent): Promise<any[]> {
   let allData: any[] = [];
-  for (let { key, page, value } of await scanPrefixGlobal("data:")) {
+  for (let { key, page, value } of await queryPrefix("data:")) {
     let [, pos] = key.split("@");
     allData.push({
       ...value,

packages/plugs/query/engine.ts

@@ -221,7 +221,6 @@ export async function renderQuery(
   data: any[]
 ): Promise<string> {
   if (parsedQuery.render) {
-    console.log("Handlebars", Handlebars);
     Handlebars.registerHelper("json", (v) => JSON.stringify(v));
     Handlebars.registerHelper("niceDate", (ts) => niceDate(new Date(ts)));
     Handlebars.registerHelper("yaml", (v, prefix) => {

packages/plugs/query/materialized_queries.ts

@@ -22,25 +22,24 @@ export async function updateMaterializedQueriesCommand() {
   const currentPage = await getCurrentPage();
   await save();
   await flashNotification("Updating materialized queries...");
-  await invokeFunction(
-    "server",
-    "updateMaterializedQueriesOnPage",
-    currentPage
-  );
-  await reloadPage();
-}
-
-export async function whiteOutQueriesCommand() {
-  const text = await getText();
-  const parsed = await parseMarkdown(text);
-  console.log(removeQueries(parsed));
+  if (
+    await invokeFunction(
+      "server",
+      "updateMaterializedQueriesOnPage",
+      currentPage
+    )
+  ) {
+    await reloadPage();
+  }
 }
 
 // Called from client, running on server
-export async function updateMaterializedQueriesOnPage(pageName: string) {
+export async function updateMaterializedQueriesOnPage(
+  pageName: string
+): Promise<boolean> {
   let { text } = await readPage(pageName);
 
-  text = await replaceAsync(
+  let newText = await replaceAsync(
     text,
     queryRegex,
     async (fullMatch, startQuery, query, body, endQuery) => {
@@ -68,6 +67,9 @@ export async function updateMaterializedQueriesOnPage(pageName: string) {
       }
     }
   );
-  // console.log("New text", text);
-  await writePage(pageName, text);
+  if (text !== newText) {
+    await writePage(pageName, newText);
+    return true;
+  }
+  return false;
 }

packages/plugs/query/query.plug.yaml

@@ -12,10 +12,8 @@ functions:
       key: "Alt-q"
       button:
         label: "🔄"
-  whiteOutQueriesCommand:
-    path: ./materialized_queries.ts:whiteOutQueriesCommand
-    command:
-      name: "Debug: Whiteout Queries"
+    events:
+      - editor:pageLoaded
   indexData:
     path: ./data.ts:indexData
     events:

packages/plugs/search/search.ts

@@ -1,7 +1,7 @@
 import { fullTextIndex, fullTextSearch } from "@plugos/plugos-syscall/fulltext";
 import { renderToText } from "@silverbulletmd/common/tree";
 import { PageMeta } from "@silverbulletmd/common/types";
-import { scanPrefixGlobal } from "@silverbulletmd/plugos-silverbullet-syscall";
+import { queryPrefix } from "@silverbulletmd/plugos-silverbullet-syscall";
 import {
   navigate,
   prompt,
@@ -28,7 +28,7 @@ export async function queryProvider({
   let allPageMap: Map<string, any> = new Map(
     results.map((r: any) => [r.name, r])
   );
-  for (let { page, value } of await scanPrefixGlobal("meta:")) {
+  for (let { page, value } of await queryPrefix("meta:")) {
     let p = allPageMap.get(page);
     if (p) {
       for (let [k, v] of Object.entries(value)) {

packages/plugs/tasks/task.ts

@@ -2,7 +2,7 @@ import type { ClickEvent, IndexTreeEvent } from "@silverbulletmd/web/app_event";
 
 import {
   batchSet,
-  scanPrefixGlobal,
+  queryPrefix,
 } from "@silverbulletmd/plugos-silverbullet-syscall/index";
 import {
   readPage,
@@ -197,7 +197,7 @@ export async function queryProvider({
   query,
 }: QueryProviderEvent): Promise<Task[]> {
   let allTasks: Task[] = [];
-  for (let { key, page, value } of await scanPrefixGlobal("task:")) {
+  for (let { key, page, value } of await queryPrefix("task:")) {
     let [, pos] = key.split(":");
     allTasks.push({
       ...value,

packages/server/express_server.ts

@@ -10,7 +10,7 @@ import bodyParser from "body-parser";
 import { EventHook } from "@plugos/plugos/hooks/event";
 import spaceSyscalls from "./syscalls/space";
 import { eventSyscalls } from "@plugos/plugos/syscalls/event";
-import { ensurePageIndexTable, pageIndexSyscalls } from "./syscalls";
+import { ensureTable, pageIndexSyscalls } from "./syscalls";
 import knex, { Knex } from "knex";
 import shellSyscalls from "@plugos/plugos/syscalls/shell.node";
 import { NodeCronHook } from "@plugos/plugos/hooks/node_cron";
@@ -207,7 +207,7 @@ export class ExpressServer {
           next();
         };
 
-    await ensurePageIndexTable(this.db);
+    await ensureTable(this.db);
     await ensureFTSTable(this.db, "fts");
     console.log("Setting up router");
 

packages/server/syscalls/index.ts

@@ -1,12 +1,8 @@
 import { Knex } from "knex";
 import { SysCallMapping } from "@plugos/plugos/system";
+import { Query, queryToKnex } from "@plugos/plugos/syscalls/store.knex_node";
 
-import {
-  ensureTable,
-  storeSyscalls,
-} from "@plugos/plugos/syscalls/store.knex_node";
-
-type IndexItem = {
+type Item = {
   page: string;
   key: string;
   value: any;
@@ -17,45 +13,35 @@ export type KV = {
   value: any;
 };
 
-/*
- Keyspace design:
+const tableName = "page_index";
 
- for page lookups:
- p~page~key
+export async function ensureTable(db: Knex<any, unknown>) {
+  if (!(await db.schema.hasTable(tableName))) {
+    await db.schema.createTable(tableName, (table) => {
+      table.string("page");
+      table.string("key");
+      table.text("value");
+      table.primary(["page", "key"]);
+      table.index(["key"]);
+    });
 
- for global lookups:
- k~key~page
-
- */
-
-function pageKey(page: string, key: string) {
-  return `p~${page}~${key}`;
-}
-
-function unpackPageKey(dbKey: string): { page: string; key: string } {
-  const [, page, key] = dbKey.split("~");
-  return { page, key };
-}
-
-function globalKey(page: string, key: string) {
-  return `k~${key}~${page}`;
-}
-
-function unpackGlobalKey(dbKey: string): { page: string; key: string } {
-  const [, key, page] = dbKey.split("~");
-  return { page, key };
-}
-
-export async function ensurePageIndexTable(db: Knex<any, unknown>) {
-  await ensureTable(db, "page_index");
+    console.log(`Created table ${tableName}`);
+  }
 }
 
 export function pageIndexSyscalls(db: Knex<any, unknown>): SysCallMapping {
-  const storeCalls = storeSyscalls(db, "page_index");
   const apiObj: SysCallMapping = {
     "index.set": async (ctx, page: string, key: string, value: any) => {
-      await storeCalls["store.set"](ctx, pageKey(page, key), value);
-      await storeCalls["store.set"](ctx, globalKey(page, key), value);
+      let changed = await db<Item>(tableName)
+        .where({ key, page })
+        .update("value", JSON.stringify(value));
+      if (changed === 0) {
+        await db<Item>(tableName).insert({
+          key,
+          page,
+          value: JSON.stringify(value),
+        });
+      }
     },
     "index.batchSet": async (ctx, page: string, kvs: KV[]) => {
       for (let { key, value } of kvs) {
@@ -63,53 +49,53 @@ export function pageIndexSyscalls(db: Knex<any, unknown>): SysCallMapping {
       }
     },
     "index.delete": async (ctx, page: string, key: string) => {
-      await storeCalls["store.delete"](ctx, pageKey(page, key));
-      await storeCalls["store.delete"](ctx, globalKey(page, key));
+      await db<Item>(tableName).where({ key, page }).del();
     },
     "index.get": async (ctx, page: string, key: string) => {
-      return storeCalls["store.get"](ctx, pageKey(page, key));
+      let result = await db<Item>(tableName)
+        .where({ key, page })
+        .select("value");
+      if (result.length) {
+        return JSON.parse(result[0].value);
+      } else {
+        return null;
+      }
     },
-    "index.scanPrefixForPage": async (ctx, page: string, prefix: string) => {
+    "index.queryPrefix": async (ctx, prefix: string) => {
       return (
-        await storeCalls["store.queryPrefix"](ctx, pageKey(page, prefix))
-      ).map(({ key, value }: { key: string; value: any }) => {
-        const { key: pageKey } = unpackPageKey(key);
-        return {
-          page,
-          key: pageKey,
-          value,
-        };
-      });
+        await db<Item>(tableName)
+          .andWhereLike("key", `${prefix}%`)
+          .select("key", "value", "page")
+      ).map(({ key, value, page }) => ({
+        key,
+        page,
+        value: JSON.parse(value),
+      }));
     },
-    "index.scanPrefixGlobal": async (ctx, prefix: string) => {
-      return (await storeCalls["store.queryPrefix"](ctx, `k~${prefix}`)).map(
-        ({ key, value }: { key: string; value: any }) => {
-          const { page, key: pageKey } = unpackGlobalKey(key);
-          return {
-            page,
-            key: pageKey,
-            value,
-          };
-        }
-      );
+    "index.query": async (ctx, query: Query) => {
+      return (
+        await queryToKnex(db<Item>(tableName), query).select(
+          "key",
+          "value",
+          "page"
+        )
+      ).map(({ key, value, page }: any) => ({
+        key,
+        page,
+        value: JSON.parse(value),
+      }));
     },
     "index.clearPageIndexForPage": async (ctx, page: string) => {
       await apiObj["index.deletePrefixForPage"](ctx, page, "");
     },
     "index.deletePrefixForPage": async (ctx, page: string, prefix: string) => {
-      // Collect all global keys for this page to delete
-      let keysToDelete = (
-        await storeCalls["store.queryPrefix"](ctx, pageKey(page, prefix))
-      ).map(({ key }: { key: string; value: string }) =>
-        globalKey(page, unpackPageKey(key).key)
-      );
-      // Delete all page keys
-      await storeCalls["store.deletePrefix"](ctx, pageKey(page, prefix));
-      // console.log("Deleting keys", keysToDelete);
-      await storeCalls["store.batchDelete"](ctx, keysToDelete);
+      return db<Item>(tableName)
+        .where({ page })
+        .andWhereLike("key", `${prefix}%`)
+        .del();
     },
     "index.clearPageIndex": async (ctx) => {
-      await storeCalls["store.deleteAll"](ctx);
+      await db<Item>(tableName).del();
     },
   };
   return apiObj;

packages/web/app_event.ts

@@ -1,6 +1,6 @@
 import type { ParseTree } from "@silverbulletmd/common/tree";
 
-export type AppEvent = "page:click" | "page:complete";
+export type AppEvent = "page:click" | "page:complete" | "page:load";
 
 export type ClickEvent = {
   page: string;

packages/web/editor.tsx

@@ -509,6 +509,7 @@ export class Editor {
   }
 
   async loadPage(pageName: string) {
+    const loadingDifferentPage = pageName !== this.currentPage;
     const editorView = this.editorView;
     if (!editorView) {
       return;
@@ -547,7 +548,9 @@ export class Editor {
       meta: doc.meta,
     });
 
-    await this.eventHook.dispatchEvent("editor:pageSwitched");
+    if (loadingDifferentPage) {
+      await this.eventHook.dispatchEvent("editor:pageLoaded", pageName);
+    }
   }
 
   tweakEditorDOM(contentDOM: HTMLElement, readOnly: boolean) {

packages/web/syscalls/editor.ts

@@ -31,13 +31,6 @@ export function editorSyscalls(editor: Editor): SysCallMapping {
     "editor.getCurrentPage": (): string => {
       return editor.currentPage!;
     },
-    // sets the current page name, without changing the content
-    "editor.setPage": (ctx, newName: string) => {
-      return editor.viewDispatch({
-        type: "page-loaded",
-        name: newName,
-      });
-    },
     "editor.getText": () => {
       return editor.editorView?.state.sliceDoc();
     },

packages/web/syscalls/index.ts

@@ -5,8 +5,7 @@ import { Space } from "@silverbulletmd/common/spaces/space";
 export function indexerSyscalls(space: Space): SysCallMapping {
   return proxySyscalls(
     [
-      "index.scanPrefixForPage",
-      "index.scanPrefixGlobal",
+      "index.queryPrefix",
       "index.get",
       "index.set",
       "index.batchSet",