Fixes #90: Re-enables full text search

common/spaces/disk_space_primitives.ts

@@ -6,8 +6,8 @@ import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives.ts";
 import { Plug } from "../../plugos/plug.ts";
 import { mime } from "https://deno.land/x/mimetypes@v1.0.0/mod.ts";
 import {
-  base64Decode,
+  base64DecodeDataUrl,
-  base64Encode,
+  base64EncodedDataUrl,
 } from "../../plugos/asset_bundle/base64.ts";
 
 function lookupContentType(path: string): string {
@@ -53,10 +53,10 @@ export class DiskSpacePrimitives implements SpacePrimitives {
         case "dataurl":
           {
             const f = await Deno.open(localPath, { read: true });
-            const buf = base64Encode(await readAll(f));
+            const buf = await readAll(f);
             Deno.close(f.rid);
 
-            data = `data:${contentType};base64,${buf}`;
+            data = base64EncodedDataUrl(contentType, buf);
           }
           break;
         case "arraybuffer":
@@ -103,7 +103,7 @@ export class DiskSpacePrimitives implements SpacePrimitives {
         case "dataurl":
           await Deno.writeFile(
             localPath,
-            base64Decode((data as string).split(",")[1]),
+            base64DecodeDataUrl(data as string),
           );
           break;
         case "arraybuffer":

common/spaces/http_space_primitives.ts

@@ -1,6 +1,11 @@
 import { FileMeta } from "../types.ts";
 import { Plug } from "../../plugos/plug.ts";
 import { FileData, FileEncoding, SpacePrimitives } from "./space_primitives.ts";
+import {
+  base64DecodeDataUrl,
+  base64EncodedDataUrl,
+} from "../../plugos/asset_bundle/base64.ts";
+import { mime } from "../../plugos/deps.ts";
 
 export class HttpSpacePrimitives implements SpacePrimitives {
   fsUrl: string;
@@ -50,14 +55,16 @@ export class HttpSpacePrimitives implements SpacePrimitives {
     switch (encoding) {
       case "arraybuffer":
         {
-          const abBlob = await res.blob();
+          data = await res.arrayBuffer();
-          data = await abBlob.arrayBuffer();
+          // data = await abBlob.arrayBuffer();
         }
         break;
       case "dataurl":
         {
-          const dUBlob = await res.blob();
+          data = base64EncodedDataUrl(
-          data = arrayBufferToDataUrl(await dUBlob.arrayBuffer());
+            mime.getType(name) || "application/octet-stream",
+            new Uint8Array(await res.arrayBuffer()),
+          );
         }
         break;
       case "string":
@@ -83,7 +90,7 @@ export class HttpSpacePrimitives implements SpacePrimitives {
         body = data;
         break;
       case "dataurl":
-        data = dataUrlToArrayBuffer(data as string);
+        data = base64DecodeDataUrl(data as string);
         break;
     }
     const res = await this.authenticatedFetch(`${this.fsUrl}/${name}`, {
@@ -184,23 +191,3 @@ export class HttpSpacePrimitives implements SpacePrimitives {
     }
   }
 }
-
-function dataUrlToArrayBuffer(dataUrl: string): ArrayBuffer {
-  const binary_string = atob(dataUrl.split(",")[1]);
-  const len = binary_string.length;
-  const bytes = new Uint8Array(len);
-  for (let i = 0; i < len; i++) {
-    bytes[i] = binary_string.charCodeAt(i);
-  }
-  return bytes.buffer;
-}
-
-function arrayBufferToDataUrl(buffer: ArrayBuffer): string {
-  let binary = "";
-  const bytes = new Uint8Array(buffer);
-  const len = bytes.byteLength;
-  for (let i = 0; i < len; i++) {
-    binary += String.fromCharCode(bytes[i]);
-  }
-  return `data:application/octet-stream,${btoa(binary)}`;
-}

plug-api/plugos-syscall/asset.ts

@@ -1,4 +1,4 @@
-import { base64Decode } from "../../plugos/asset_bundle/base64.ts";
+import { base64DecodeDataUrl } from "../../plugos/asset_bundle/base64.ts";
 import { syscall } from "./syscall.ts";
 
 export async function readAsset(
@@ -8,7 +8,7 @@ export async function readAsset(
   const dataUrl = await syscall("asset.readAsset", name) as string;
   switch (encoding) {
     case "utf8":
-      return new TextDecoder().decode(base64Decode(dataUrl.split(",")[1]));
+      return new TextDecoder().decode(base64DecodeDataUrl(dataUrl));
     case "dataurl":
       return dataUrl;
   }

plugos/asset_bundle/base64.test.ts

@@ -1,5 +1,9 @@
 import { assertEquals } from "../../test_deps.ts";
-import { base64Decode } from "./base64.ts";
+import {
+  base64Decode,
+  base64DecodeDataUrl,
+  base64EncodedDataUrl,
+} from "./base64.ts";
 import { base64Encode } from "./base64.ts";
 
 Deno.test("Base 64 encoding", () => {
@@ -9,4 +13,9 @@ Deno.test("Base 64 encoding", () => {
   buf[2] = 3;
 
   assertEquals(buf, base64Decode(base64Encode(buf)));
+
+  assertEquals(
+    buf,
+    base64DecodeDataUrl(base64EncodedDataUrl("application/octet-stream", buf)),
+  );
 });

plugos/asset_bundle/base64.ts

@@ -16,3 +16,15 @@ export function base64Encode(buffer: Uint8Array): string {
   }
   return btoa(binary);
 }
+
+export function base64EncodedDataUrl(
+  mimeType: string,
+  buffer: Uint8Array,
+): string {
+  return `data:${mimeType};base64,${base64Encode(buffer)}`;
+}
+
+export function base64DecodeDataUrl(dataUrl: string): Uint8Array {
+  const b64Encoded = dataUrl.split(",", 2)[1];
+  return base64Decode(b64Encoded);
+}

plugos/asset_bundle/bundle.ts

@@ -1,4 +1,4 @@
-import { base64Decode, base64Encode } from "./base64.ts";
+import { base64Decode, base64EncodedDataUrl } from "./base64.ts";
 import { mime } from "../deps.ts";
 
 type DataUrl = string;
@@ -57,9 +57,8 @@ export class AssetBundle {
   }
 
   writeFileSync(path: string, data: Uint8Array) {
-    const encoded = base64Encode(data);
+    const mimeType = mime.getType(path) || "application/octet-stream";
-    const mimeType = mime.getType(path);
+    this.bundle[path] = base64EncodedDataUrl(mimeType, data);
-    this.bundle[path] = `data:${mimeType};base64,${encoded}`;
   }
 
   writeTextFileSync(path: string, s: string) {

plugos/syscalls/fs.deno.ts

@@ -1,6 +1,6 @@
 import type { SysCallMapping } from "../system.ts";
 import { mime, path } from "../deps.ts";
-import { base64Decode, base64Encode } from "../asset_bundle/base64.ts";
+import { base64DecodeDataUrl, base64Encode } from "../asset_bundle/base64.ts";
 import { FileMeta } from "../../common/types.ts";
 
 export default function fileSystemSyscalls(root = "/"): SysCallMapping {
@@ -51,7 +51,7 @@ export default function fileSystemSyscalls(root = "/"): SysCallMapping {
       if (encoding === "utf8") {
         await Deno.writeTextFile(p, text);
       } else {
-        await Deno.writeFile(p, base64Decode(text.split(",")[1]));
+        await Deno.writeFile(p, base64DecodeDataUrl(text));
       }
       const s = await Deno.stat(p);
       return {

plugos/syscalls/fulltext.knex_sqlite.ts

@@ -2,27 +2,22 @@ import { SQLite } from "../../server/deps.ts";
 import { SysCallMapping } from "../system.ts";
 import { asyncExecute, asyncQuery } from "./store.deno.ts";
 
-type Item = {
-  key: string;
-  value: string;
-};
-
 export function ensureFTSTable(
   db: SQLite,
   tableName: string,
 ) {
-  // const stmt = db.prepare(
+  const result = db.query(
-  //   `SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
+    `SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
-  // );
+    [tableName],
-  // const result = stmt.all(tableName);
+  );
-  // if (result.length === 0) {
+  if (result.length === 0) {
-  //   asyncExecute(
+    asyncExecute(
-  //     db,
+      db,
-  //     `CREATE VIRTUAL TABLE ${tableName} USING fts5(key, value);`,
+      `CREATE VIRTUAL TABLE ${tableName} USING fts5(key, value);`,
-  //   );
+    );
 
-  //   console.log(`Created fts5 table ${tableName}`);
+    console.log(`Created fts5 table ${tableName}`);
-  // }
+  }
   return Promise.resolve();
 }
 
@@ -44,6 +39,7 @@ export function fullTextSearchSyscalls(
       await asyncExecute(db, `DELETE FROM ${tableName} WHERE key = ?`, key);
     },
     "fulltext.search": async (_ctx, phrase: string, limit: number) => {
+      console.log("Got search query", phrase);
       return (
         await asyncQuery<any>(
           db,

plugs/core/core.plug.yaml

@@ -138,15 +138,15 @@ functions:
       - page:complete
 
   # Full text search
-  # searchIndex:
+  searchIndex:
-  #   path: ./search.ts:pageIndex
+    path: ./search.ts:pageIndex
-  #   events:
+    events:
-  #     - page:index
+      - page:index
-  # searchUnindex:
+  searchUnindex:
-  #   path: "./search.ts:pageUnindex"
+    path: "./search.ts:pageUnindex"
-  #   env: server
+    env: server
-  #   events:
+    events:
-  #     - page:deleted
+      - page:deleted
   searchQueryProvider:
     path: ./search.ts:queryProvider
     events:
@@ -158,12 +158,12 @@ functions:
       key: Ctrl-Shift-f
       mac: Cmd-Shift-f
   readPageSearch:
-    path: ./search.ts:readPageSearch
+    path: ./search.ts:readFileSearch
     pageNamespace:
       pattern: "🔍 .+"
       operation: readFile
   getPageMetaSearch:
-    path: ./search.ts:getPageMetaSearch
+    path: ./search.ts:getFileMetaSearch
     pageNamespace:
       pattern: "🔍 .+"
       operation: getFileMeta

plugs/core/search.ts

@@ -1,9 +1,18 @@
 import { fulltext } from "$sb/plugos-syscall/mod.ts";
 import { renderToText } from "$sb/lib/tree.ts";
-import type { PageMeta } from "../../common/types.ts";
+import type { FileMeta, PageMeta } from "../../common/types.ts";
 import { editor, index } from "$sb/silverbullet-syscall/mod.ts";
 import { IndexTreeEvent, QueryProviderEvent } from "$sb/app_event.ts";
 import { applyQuery, removeQueries } from "$sb/lib/query.ts";
+import {
+  FileData,
+  FileEncoding,
+} from "../../common/spaces/space_primitives.ts";
+import {
+  base64DecodeDataUrl,
+  base64Encode,
+  base64EncodedDataUrl,
+} from "../../plugos/asset_bundle/base64.ts";
 
 const searchPrefix = "🔍 ";
 
@@ -46,16 +55,20 @@ export async function queryProvider({
 }
 
 export async function searchCommand() {
-  const phrase = await prompt("Search for: ");
+  const phrase = await editor.prompt("Search for: ");
   if (phrase) {
     await editor.navigate(`${searchPrefix}${phrase}`);
   }
 }
 
-export async function readPageSearch(
+export async function readFileSearch(
   name: string,
-): Promise<{ text: string; meta: PageMeta }> {
+  encoding: FileEncoding,
-  const phrase = name.substring(searchPrefix.length);
+): Promise<{ data: FileData; meta: FileMeta }> {
+  const phrase = name.substring(
+    searchPrefix.length,
+    name.length - ".md".length,
+  );
   const results = await fulltext.fullTextSearch(phrase, 100);
   const text = `# Search results for "${phrase}"\n${
     results
@@ -63,19 +76,28 @@ export async function readPageSearch(
       .join("\n")
   }
   `;
+
   return {
-    text: text,
+    // encoding === "arraybuffer" is not an option, so either it's "string" or "dataurl"
+    data: encoding === "string" ? text : base64EncodedDataUrl(
+      "text/markdown",
+      new TextEncoder().encode(text),
+    ),
     meta: {
       name,
+      contentType: "text/markdown",
+      size: text.length,
       lastModified: 0,
       perm: "ro",
     },
   };
 }
 
-export function getPageMetaSearch(name: string): PageMeta {
+export function getFileMetaSearch(name: string): FileMeta {
   return {
     name,
+    contentType: "text/markdown",
+    size: -1,
     lastModified: 0,
     perm: "ro",
   };

server/deps.ts

@@ -1,3 +1,4 @@
 export * from "../common/deps.ts";
-export { DB as SQLite } from "https://deno.land/x/sqlite@v3.5.0/mod.ts";
+export { DB as SQLite } from "../plugos/forked/deno-sqlite/mod.ts";
 export { Application, Router } from "https://deno.land/x/oak@v11.1.0/mod.ts";
+export * as etag from "https://deno.land/x/oak@v11.1.0/etag.ts";

server/hooks/plug_space_primitives.ts

@@ -6,6 +6,7 @@ import {
 } from "../../common/spaces/space_primitives.ts";
 import { FileMeta } from "../../common/types.ts";
 import { NamespaceOperation, PageNamespaceHook } from "./page_namespace.ts";
+import { base64DecodeDataUrl } from "../../plugos/asset_bundle/base64.ts";
 
 export class PlugSpacePrimitives implements SpacePrimitives {
   constructor(
@@ -46,14 +47,27 @@ export class PlugSpacePrimitives implements SpacePrimitives {
     return allFiles;
   }
 
-  readFile(
+  async readFile(
     name: string,
     encoding: FileEncoding,
   ): Promise<{ data: FileData; meta: FileMeta }> {
-    const result = this.performOperation("readFile", name);
+    const wantArrayBuffer = encoding === "arraybuffer";
+    const result: { data: FileData; meta: FileMeta } | false = await this
+      .performOperation(
+        "readFile",
+        name,
+        wantArrayBuffer ? "dataurl" : encoding,
+      );
     if (result) {
+      if (wantArrayBuffer) {
+        return {
+          data: base64DecodeDataUrl(result.data as string),
+          meta: result.meta,
+        };
+      } else {
         return result;
       }
+    }
     return this.wrapped.readFile(name, encoding);
   }
 

server/http_server.ts

@@ -1,4 +1,4 @@
-import { Application, path, Router, SQLite } from "./deps.ts";
+import { Application, etag, path, Router, SQLite } from "./deps.ts";
 import { Manifest, SilverBulletHooks } from "../common/manifest.ts";
 import { loadMarkdownExtensions } from "../common/markdown_ext.ts";
 import buildMarkdown from "../common/parser.ts";
@@ -15,7 +15,10 @@ import { DenoCronHook } from "../plugos/hooks/cron.deno.ts";
 import { esbuildSyscalls } from "../plugos/syscalls/esbuild.ts";
 import { eventSyscalls } from "../plugos/syscalls/event.ts";
 import fileSystemSyscalls from "../plugos/syscalls/fs.deno.ts";
-import { fullTextSearchSyscalls } from "../plugos/syscalls/fulltext.knex_sqlite.ts";
+import {
+  ensureFTSTable,
+  fullTextSearchSyscalls,
+} from "../plugos/syscalls/fulltext.knex_sqlite.ts";
 import sandboxSyscalls from "../plugos/syscalls/sandbox.ts";
 import shellSyscalls from "../plugos/syscalls/shell.node.ts";
 import {
@@ -43,6 +46,7 @@ export type ServerOptions = {
 };
 
 const indexRequiredKey = "$spaceIndexed";
+const staticLastModified = new Date().toUTCString();
 
 export class HttpServer {
   app: Application;
@@ -198,7 +202,7 @@ export class HttpServer {
   async start() {
     await ensureIndexTable(this.db);
     await ensureStoreTable(this.db, "store");
-    // await ensureFTSTable(this.db, "fts");
+    await ensureFTSTable(this.db, "fts");
     await this.ensureAndLoadSettings();
 
     // Load plugs
@@ -211,6 +215,8 @@ export class HttpServer {
         response.body = this.assetBundle.readTextFileSync(
           "web/index.html",
         );
+        response.headers.set("Last-Modified", staticLastModified);
+        response.headers.set("ETag", await etag.calculate(response.body));
         return;
       }
       try {
@@ -224,6 +230,8 @@ export class HttpServer {
           assetName,
         );
         response.headers.set("Content-length", "" + data.length);
+        response.headers.set("Last-Modified", staticLastModified);
+        response.headers.set("ETag", await etag.calculate(data));
 
         if (request.method === "GET") {
           response.body = data;