Exposing Objects indexing as syscalls

common/syscalls/index.ts

@@ -0,0 +1,49 @@
+import type {
+  KvQuery,
+  ObjectQuery,
+  ObjectValue,
+} from "@silverbulletmd/silverbullet/types";
+import type { SysCallMapping, System } from "$lib/plugos/system.ts";
+import type { LuaCollectionQuery } from "$common/space_lua/query_collection.ts";
+
+// These are just wrappers around the system.invokeFunction calls, but they make it easier to use the index
+
+export function indexSyscalls(system: System<any>): SysCallMapping {
+  return {
+    "index.indexObjects": (_ctx, page: string, objects: ObjectValue<any>[]) => {
+      return system.invokeFunction("index.indexObjects", [page, objects]);
+    },
+    "index.queryObjects": (
+      _ctx,
+      tag: string,
+      query: ObjectQuery,
+      ttlSecs?: number,
+    ) => {
+      return system.invokeFunction("index.queryObjects", [
+        tag,
+        query,
+        ttlSecs,
+      ]);
+    },
+    "index.queryLuaObjects": (
+      _ctx,
+      tag: string,
+      query: LuaCollectionQuery,
+      scopedVariables?: Record<string, any>,
+    ) => {
+      return system.invokeFunction(
+        "index.queryLuaObjects",
+        [tag, query, scopedVariables],
+      );
+    },
+    "index.queryDeleteObjects": (_ctx, tag: string, query: ObjectQuery) => {
+      return system.invokeFunction("index.queryDeleteObjects", [tag, query]);
+    },
+    "index.query": (_ctx, query: KvQuery, variables?: Record<string, any>) => {
+      return system.invokeFunction("index.query", [query, variables]);
+    },
+    "index.getObjectByRef": (_ctx, page: string, tag: string, ref: string) => {
+      return system.invokeFunction("index.getObjectByRef", [page, tag, ref]);
+    },
+  };
+}

lib/plugos/syscalls/datastore.ts

@@ -47,7 +47,7 @@ export function dataStoreReadSyscalls(
       prefix: string[],
       query: LuaCollectionQuery,
       scopeVariables: Record<string, any> = {},
-    ): Promise<KV[]> => {
+    ): Promise<any[]> => {
       const dsQueryCollection = new DataStoreQueryCollection(ds, prefix);
       const env = new LuaEnv(commonSystem.spaceLuaEnv.env);
       for (const [key, value] of Object.entries(scopeVariables)) {

plug-api/syscalls.ts

@@ -17,4 +17,7 @@ export * as datastore from "./syscalls/datastore.ts";
 export * as jsonschema from "./syscalls/jsonschema.ts";
 export * as lua from "./syscalls/lua.ts";
 
+// Not technically syscalls, but we want to export them for convenience
+export * as index from "./syscalls/index.ts";
+
 export * from "./syscall.ts";

plug-api/syscalls/datastore.ts

@@ -75,7 +75,7 @@ export function queryLua(
   prefix: string[],
   query: LuaCollectionQuery,
   scopeVariables: Record<string, any>,
-): Promise<KV[]> {
+): Promise<any[]> {
   return syscall("datastore.queryLua", prefix, query, scopeVariables);
 }
 

plug-api/syscalls/index.ts

@@ -0,0 +1,82 @@
+import type {
+  ObjectQuery,
+  ObjectValue,
+} from "@silverbulletmd/silverbullet/types";
+import type { LuaCollectionQuery } from "$common/space_lua/query_collection.ts";
+import { syscall } from "@silverbulletmd/silverbullet/syscall";
+
+/**
+ * Exposes the SilverBullet object indexing system
+ * @module
+ */
+
+/**
+ * Indexes objects for a specific page
+ * @param page - The page identifier where objects will be indexed
+ * @param objects - Array of objects to be indexed
+ * @returns Promise that resolves when indexing is complete
+ */
+export function indexObjects<T>(
+  page: string,
+  objects: ObjectValue<T>[],
+): Promise<void> {
+  return syscall("index.indexObjects", page, objects);
+}
+
+/**
+ * Queries objects based on specified criteria
+ * @param tag - The tag to filter objects by
+ * @param query - Query parameters to filter objects
+ * @param ttlSecs - Optional time-to-live in seconds for the query cache
+ * @returns Promise that resolves with an array of matching objects
+ */
+export function queryObjects<T>(
+  tag: string,
+  query: ObjectQuery,
+  ttlSecs?: number,
+): Promise<ObjectValue<T>[]> {
+  return syscall("index.queryObjects", tag, query, ttlSecs);
+}
+
+/**
+ * Queries objects using a Lua-based collection query
+ * @param tag - The tag to filter objects by
+ * @param query - Lua query parameters to filter objects
+ * @param scopedVariables - Optional variables to be used in the Lua query
+ * @returns Promise that resolves with an array of matching objects
+ */
+export function queryLuaObjects<T>(
+  tag: string,
+  query: LuaCollectionQuery,
+  scopedVariables?: Record<string, any>,
+): Promise<ObjectValue<T>[]> {
+  return syscall("index.queryLuaObjects", tag, query, scopedVariables);
+}
+
+/**
+ * Deletes objects that match the specified query criteria
+ * @param tag - The tag of objects to be deleted
+ * @param query - Query parameters to identify objects for deletion
+ * @returns Promise that resolves when deletion is complete
+ */
+export function queryDeleteObjects(
+  tag: string,
+  query: ObjectQuery,
+): Promise<void> {
+  return syscall("index.queryDeleteObjects", tag, query);
+}
+
+/**
+ * Retrieves a specific object by its reference
+ * @param page - The page identifier where the object is located
+ * @param tag - The tag of the object
+ * @param ref - The reference identifier of the object
+ * @returns Promise that resolves with the matching object or undefined if not found
+ */
+export function getObjectByRef<T>(
+  page: string,
+  tag: string,
+  ref: string,
+): Promise<ObjectValue<T> | undefined> {
+  return syscall("index.getObjectByRef", page, tag, ref);
+}

plugs/index/api.ts

@@ -9,6 +9,7 @@ import type {
 import type { QueryProviderEvent } from "../../plug-api/types.ts";
 import { determineType, type SimpleJSONType } from "./attributes.ts";
 import { ttlCache } from "$lib/memory_cache.ts";
+import type { LuaCollectionQuery } from "$common/space_lua/query_collection.ts";
 
 const indexKey = "idx";
 const pageKey = "ridx";
@@ -179,6 +180,17 @@ export function queryObjects<T>(
   }, ttlSecs);
 }
 
+export function queryLuaObjects<T>(
+  tag: string,
+  query: LuaCollectionQuery,
+  scopedVariables: Record<string, any> = {},
+  ttlSecs?: number,
+): Promise<ObjectValue<T>[]> {
+  return ttlCache(query, () => {
+    return datastore.queryLua([indexKey, tag], query, scopedVariables);
+  }, ttlSecs);
+}
+
 export function queryDeleteObjects<T>(
   tag: string,
   query: ObjectQuery,

plugs/index/index.plug.yaml

@@ -12,6 +12,9 @@ functions:
   queryObjects:
     path: api.ts:queryObjects
     # Note: not setting env: server to allow for client-side datastore query caching
+  queryLuaObjects:
+    path: api.ts:queryLuaObjects
+    # Note: not setting env: server to allow for client-side datastore query caching
   getObjectByRef:
     path: api.ts:getObjectByRef
     env: server

server/server_system.ts

@@ -41,6 +41,7 @@ import type { DataStoreMQ } from "$lib/data/mq.datastore.ts";
 import { plugPrefix } from "$common/spaces/constants.ts";
 import { base64EncodedDataUrl } from "$lib/crypto.ts";
 import type { ConfigContainer } from "../type/config.ts";
+import { indexSyscalls } from "$common/syscalls/index.ts";
 
 const fileListInterval = 30 * 1000; // 30s
 
@@ -133,6 +134,7 @@ export class ServerSystem extends CommonSystem {
       mqSyscalls(this.mq),
       languageSyscalls(),
       jsonschemaSyscalls(),
+      indexSyscalls(this.system),
       luaSyscalls(),
       templateSyscalls(this.ds),
       dataStoreReadSyscalls(this.ds, this),

web/client_system.ts

@@ -44,6 +44,7 @@ import type { DataStoreMQ } from "$lib/data/mq.datastore.ts";
 import { plugPrefix } from "$common/spaces/constants.ts";
 import { jsonschemaSyscalls } from "$common/syscalls/jsonschema.ts";
 import { luaSyscalls } from "$common/syscalls/lua.ts";
+import { indexSyscalls } from "$common/syscalls/index.ts";
 
 const plugNameExtractRegex = /\/(.+)\.plug\.js$/;
 
@@ -162,6 +163,7 @@ export class ClientSystem extends CommonSystem {
       clientCodeWidgetSyscalls(),
       languageSyscalls(),
       jsonschemaSyscalls(),
+      indexSyscalls(this.system),
       luaSyscalls(),
       this.client.syncMode
         // In sync mode handle locally

website/API/clientStore.md

@@ -1,6 +1,5 @@
 The Client Store API provides a simple key-value store for client-specific states and preferences.
 
-# Client Store API
 
 ## clientStore.set(key, value)
 Sets a value in the client store.

website/API/datastore.md

@@ -1,10 +1,9 @@
-# Datastore API
 
 The Datastore API provides functions for interacting with a key-value store that has query capabilities.
 
-## Key-Value Operations
+# Key-Value Operations
 
-### datastore.set(key, value)
+## datastore.set(key, value)
 Sets a value in the key-value store.
 
 Example:
@@ -12,7 +11,7 @@ Example:
 datastore.set("user:123", {name = "John", age = 30})
 ```
 
-### datastore.get(key)
+## datastore.get(key)
 Gets a value from the key-value store.
 
 Example:
@@ -21,7 +20,7 @@ local user = datastore.get("user:123")
 print(user.name)  -- prints "John"
 ```
 
-### datastore.del(key)
+## datastore.del(key)
 Deletes a value from the key-value store.
 
 Example:
@@ -29,9 +28,9 @@ Example:
 datastore.del("user:123")
 ```
 
-## Batch Operations
+# Batch Operations
 
-### datastore.batch_set(kvs)
+## datastore.batch_set(kvs)
 Sets multiple key-value pairs in a single operation.
 
 Example:
@@ -43,7 +42,7 @@ local kvs = {
 datastore.batch_set(kvs)
 ```
 
-### datastore.batch_get(keys)
+## datastore.batch_get(keys)
 Gets multiple values in a single operation.
 
 Example:
@@ -55,7 +54,7 @@ for _, value in ipairs(values) do
 end
 ```
 
-### datastore.batch_del(keys)
+## datastore.batch_del(keys)
 Deletes multiple values in a single operation.
 
 Example:

website/API/index.md

@@ -0,0 +1,34 @@
+The `index` API provides functions for interacting with SilverBullet's [[Objects]], allowing you to store and query page-associated data.
+
+## Object Operations
+
+### index.index_objects(page, objects)
+Indexes an array of objects for a specific page.
+
+Example:
+```lua
+local objects = {
+    {tag = "mytask", ref="task1", content = "Buy groceries"},
+    {tag = "mytask", ref="task2", content = "Write docs"}
+}
+index.index_objects("my page", objects)
+```
+
+### index.query_lua_objects(tag, query, scoped_variables?)
+Queries objects using a Lua-based collection query.
+
+Example:
+```lua
+local tasks = index.query_lua_objects("mytask", {limit=3})
+```
+
+### index.get_object_by_ref(page, tag, ref)
+Retrieves a specific object by its reference.
+
+Example:
+```lua
+local task = index.get_object_by_ref("my page", "mytask", "task1")
+if task then
+    print("Found task: " .. task.content)
+end
+```

website/API/space.md

@@ -1,10 +1,8 @@
-# Space API
-
 The Space API provides functions for interacting with pages, attachments, and files in the space.
 
-## Page Operations
+# Page Operations
 
-### space.list_pages()
+## space.list_pages()
 Returns a list of all pages in the space.
 
 Example:
@@ -15,7 +13,7 @@ for page in each(pages) do
 end
 ```
 
-### space.read_page(name)
+## space.read_page(name)
 Reads the content of a page.
 
 Example:
@@ -24,7 +22,7 @@ local content = space.read_page("welcome")
 print(content)  -- prints the content of the "welcome" page
 ```
 
-### space.get_page_meta(name)
+## space.get_page_meta(name)
 Gets metadata for a specific page.
 
 Example:
@@ -33,7 +31,7 @@ local meta = space.get_page_meta("welcome")
 print(meta.name, meta.lastModified)  -- prints page name and last modified date
 ```
 
-### space.write_page(name, text)
+## space.write_page(name, text)
 Writes content to a page.
 
 Example:
@@ -42,7 +40,7 @@ local meta = space.write_page("notes", "My new note content")
 print("Page updated at: " .. meta.lastModified)
 ```
 
-### space.delete_page(name)
+## space.delete_page(name)
 Deletes a page from the space.
 
 Example:
@@ -50,9 +48,9 @@ Example:
 space.delete_page("old-notes")
 ```
 
-## Attachment Operations
+# Attachment Operations
 
-### space.list_attachments()
+## space.list_attachments()
 Returns a list of all attachments in the space.
 
 Example:
@@ -63,7 +61,7 @@ for att in each(attachments) do
 end
 ```
 
-### space.read_attachment(name)
+## space.read_attachment(name)
 Reads the content of an attachment.
 
 Example:
@@ -72,7 +70,7 @@ local data = space.read_attachment("image.png")
 print("Attachment size: " .. #data .. " bytes")
 ```
 
-### space.write_attachment(name, data)
+## space.write_attachment(name, data)
 Writes binary data to an attachment.
 
 Example:
@@ -82,7 +80,7 @@ local meta = space.write_attachment("test.bin", binary_data)
 print("Attachment saved with size: " .. meta.size)
 ```
 
-### space.delete_attachment(name)
+## space.delete_attachment(name)
 Deletes an attachment from the space.
 
 Example:
@@ -90,9 +88,9 @@ Example:
 space.delete_attachment("old-image.png")
 ```
 
-## File Operations
+# File Operations
 
-### space.list_files()
+## space.list_files()
 Returns a list of all files in the space.
 
 Example:
@@ -103,7 +101,7 @@ for _, file in ipairs(files) do
 end
 ```
 
-### space.get_file_meta(name)
+## space.get_file_meta(name)
 Gets metadata for a specific file.
 
 Example:
@@ -112,7 +110,7 @@ local meta = space.get_file_meta("document.txt")
 print(meta.name, meta.modified, meta.size)
 ```
 
-### space.read_file(name)
+## space.read_file(name)
 Reads the content of a file.
 
 Example:
@@ -121,7 +119,7 @@ local content = space.read_file("document.txt")
 print("File size: " .. #content .. " bytes")
 ```
 
-### space.write_file(name, data)
+## space.write_file(name, data)
 Writes binary data to a file.
 
 Example:
@@ -131,7 +129,7 @@ local meta = space.write_file("greeting.txt", text)
 print("File written with size: " .. meta.size)
 ```
 
-### space.delete_file(name)
+## space.delete_file(name)
 Deletes a file from the space.
 
 Example:
@@ -139,7 +137,7 @@ Example:
 space.delete_file("old-document.txt")
 ```
 
-### space.file_exists(name)
+## space.file_exists(name)
 Checks if a file exists in the space.
 
 Example:

website/Library/Lua API/OpenAI.md

@@ -9,65 +9,62 @@ OPENAI_API_KEY: yourapikeyhere
 
 # Implementation
 ```space-lua
-openai = {}
+openai = {
+  Client = {}
+}
+openai.Client.__index = openai.Client
 
--- Initialize OpenAI, optionally OPENAI_API_KEY from your SECRETS page if not supplied directly
+-- Create a new OpenAI client instance
-function openai.init(openaiApiKey)
+function openai.Client.new(apiKey)
-  if openai.client then
+    -- Read SECRETS if no API key provided
-    -- Already initialized
+    if not apiKey then
-    return
+        local secretsPage = space.readPage("SECRETS")
-  end
+        apiKey = string.match(secretsPage, "OPENAI_API_KEY: (%S+)")
-  if not openaiApiKey then
+    end
-    -- Read SECRETS
+    if not apiKey then
-    local secretsPage = space.readPage("SECRETS")
+        error("No OpenAI API key supplied")
-    -- Find the line with the pattern OPENAI_API_KEY: <key> and extract the key
+    end
-    openaiApiKey = string.match(secretsPage, "OPENAI_API_KEY: (%S+)")
+
-  end
+    local openai_lib = js.import("https://esm.sh/openai")
-  if not openaiApiKey then
+    local client = js.new(openai_lib.OpenAI, {
-    error("No OpenAI API key supplied")
+        apiKey = apiKey,
-  end
+        dangerouslyAllowBrowser = true
-  
+    })
-  local openai_lib = js.import("https://esm.sh/openai")
+
-  openai.client = js.new(openai_lib.OpenAI, {
+    local self = setmetatable({
-      apiKey = openaiApiKey,
+        client = client
-      dangerouslyAllowBrowser = true
+    }, OpenAIClient)
-  })
+    
-end
+    return self
-
+end
-function openai.ensure_inited()
+
-  if not openai.client then
+-- Chat completion method
-    error("OpenAI not yet initialized")
+function openai.Client:chat(message)
-  end
+    local r = self.client.chat.completions.create({
-end
+        model = "gpt-4o-mini",
-
+        messages = {
-function openai.chat(message)
+            { role = "user", content = message },
-  openai.ensure_inited()
+        },
-  local r = openai.client.chat.completions.create({
+    })
-    model = "gpt-4o-mini",
+    return r.choices[1].message.content
-    messages = {
+end
-        { role = "user", content = message },
+
-    },
+-- Streaming chat completion method
-  })
+function openai.Client:stream_chat(message)
-  return r.choices[1].message.content
+    local r = self.client.chat.completions.create({
-end
+        model = "gpt-4o-mini",
-
+        messages = {
-
+            { role = "user", content = message },
-function openai.stream_chat(message)
+        },
-  openai.ensure_inited()
+        stream = true,
-  local r = openai.client.chat.completions.create({
+    })
-    model = "gpt-4o-mini",
+    local iterator = js.each_iterable(r)
-    messages = {
+    return function()
-        { role = "user", content = message },
+        local el = iterator()
-    },
+        if el then
-    stream = true,
+            return el.choices[1].delta.content
-  })
+        end
-  local iterator = js.each_iterable(r)
-  return function()
-    local el = iterator()
-    if el then
-      return el.choices[1].delta.content
     end
-  end
 end
 
 ```