From 44a1ce698e6140d562e018e40793b84995eabc3e Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Fri, 7 Feb 2025 15:16:57 +0100 Subject: [PATCH 1/2] Support plugs to register their own syscalls, using `syscall` (see the index plug for an example) --- common/syscalls/index.ts | 60 -------------------------------- lib/manifest.ts | 7 +++- plugs/index/index.plug.yaml | 5 +++ server/server_system.ts | 4 +++ web/client_system.ts | 4 +++ web/hooks/syscall.ts | 68 +++++++++++++++++++++++++++++++++++++ 6 files changed, 87 insertions(+), 61 deletions(-) create mode 100644 web/hooks/syscall.ts diff --git a/common/syscalls/index.ts b/common/syscalls/index.ts index 4fa012db..87458c33 100644 --- a/common/syscalls/index.ts +++ b/common/syscalls/index.ts @@ -1,8 +1,3 @@ -import type { - KvQuery, - ObjectQuery, - ObjectValue, -} from "@silverbulletmd/silverbullet/types"; import type { SysCallMapping } from "$lib/plugos/system.ts"; import { findAllQueryVariables, @@ -21,61 +16,6 @@ import type { CommonSystem } from "$common/common_system.ts"; export function indexSyscalls(commonSystem: CommonSystem): SysCallMapping { return { - "index.indexObjects": (ctx, page: string, objects: ObjectValue[]) => { - return commonSystem.system.syscall(ctx, "system.invokeFunction", [ - "index.indexObjects", - page, - objects, - ]); - }, - "index.queryObjects": ( - ctx, - tag: string, - query: ObjectQuery, - ttlSecs?: number, - ) => { - return commonSystem.system.syscall(ctx, "system.invokeFunction", [ - "index.queryObjects", - tag, - query, - ttlSecs, - ]); - }, - "index.queryLuaObjects": ( - ctx, - tag: string, - query: LuaCollectionQuery, - scopedVariables?: Record, - ) => { - return commonSystem.system.syscall(ctx, "system.invokeFunction", [ - "index.queryLuaObjects", - tag, - query, - scopedVariables, - ]); - }, - "index.queryDeleteObjects": (ctx, tag: string, query: ObjectQuery) => { - return commonSystem.system.syscall(ctx, "system.invokeFunction", [ - "index.queryDeleteObjects", - tag, - query, - ]); - }, - "index.query": (ctx, query: KvQuery, variables?: Record) => { - return commonSystem.system.syscall(ctx, "system.invokeFunction", [ - "index.query", - query, - variables, - ]); - }, - "index.getObjectByRef": (ctx, page: string, tag: string, ref: string) => { - return commonSystem.system.syscall(ctx, "system.invokeFunction", [ - "index.getObjectByRef", - page, - tag, - ref, - ]); - }, "index.tag": (_ctx, tagName: string): LuaQueryCollection => { return { query: async ( diff --git a/lib/manifest.ts b/lib/manifest.ts index d57036fd..5d34c3f2 100644 --- a/lib/manifest.ts +++ b/lib/manifest.ts @@ -62,6 +62,10 @@ export type SlashCommandHookT = { slashCommand?: SlashCommandDef; }; +export type SyscallHookT = { + syscall?: string; +}; + /** Silverbullet hooks give plugs access to silverbullet core systems. * * Hooks are associated with typescript functions through a manifest file. @@ -78,7 +82,8 @@ export type SilverBulletHooks = & CodeWidgetT & PanelWidgetT & EndpointHookT - & PlugNamespaceHookT; + & PlugNamespaceHookT + & SyscallHookT; /** A plug manifest configures hooks, declares syntax extensions, and describes plug metadata. * diff --git a/plugs/index/index.plug.yaml b/plugs/index/index.plug.yaml index 3acf3cd6..39e0ee8c 100644 --- a/plugs/index/index.plug.yaml +++ b/plugs/index/index.plug.yaml @@ -6,17 +6,22 @@ functions: env: server query: path: api.ts:query + syscall: index.query indexObjects: path: api.ts:indexObjects + syscall: index.indexObjects env: server queryObjects: path: api.ts:queryObjects + syscall: index.queryObjects # Note: not setting env: server to allow for client-side datastore query caching queryLuaObjects: path: api.ts:queryLuaObjects + syscall: index.queryLuaObjects # Note: not setting env: server to allow for client-side datastore query caching getObjectByRef: path: api.ts:getObjectByRef + syscall: index.getObjectByRef env: server objectSourceProvider: path: api.ts:objectSourceProvider diff --git a/server/server_system.ts b/server/server_system.ts index 0d6c32c5..de99102c 100644 --- a/server/server_system.ts +++ b/server/server_system.ts @@ -37,6 +37,7 @@ import { ensureSpaceIndex } from "$common/space_index.ts"; import type { FileMeta } from "../plug-api/types.ts"; import { CommandHook } from "$common/hooks/command.ts"; import { CommonSystem } from "$common/common_system.ts"; +import { SyscallHook } from "../web/hooks/syscall.ts"; import type { DataStoreMQ } from "$lib/data/mq.datastore.ts"; import { plugPrefix } from "$common/spaces/constants.ts"; import { base64EncodedDataUrl } from "$lib/crypto.ts"; @@ -100,6 +101,9 @@ export class ServerSystem extends CommonSystem { this.system.addHook(new MQHook(this.system, this.mq)); + // Syscall hook + this.system.addHook(new SyscallHook()); + const codeWidgetHook = new CodeWidgetHook(); this.system.addHook(codeWidgetHook); diff --git a/web/client_system.ts b/web/client_system.ts index d71b07fd..53c0f1a0 100644 --- a/web/client_system.ts +++ b/web/client_system.ts @@ -11,6 +11,7 @@ import type { Client } from "./client.ts"; import { CodeWidgetHook } from "./hooks/code_widget.ts"; import { CommandHook } from "$common/hooks/command.ts"; import { SlashCommandHook } from "./hooks/slash_command.ts"; +import { SyscallHook } from "./hooks/syscall.ts"; import { clientStoreSyscalls } from "./syscalls/clientStore.ts"; import { debugSyscalls } from "./syscalls/debug.ts"; import { editorSyscalls } from "./syscalls/editor.ts"; @@ -128,6 +129,9 @@ export class ClientSystem extends CommonSystem { this.slashCommandHook = new SlashCommandHook(this.client, this); this.system.addHook(this.slashCommandHook); + // Syscall hook + this.system.addHook(new SyscallHook()); + this.eventHook.addLocalListener( "file:changed", async (path: string, _selfUpdate, _oldHash, newHash) => { diff --git a/web/hooks/syscall.ts b/web/hooks/syscall.ts new file mode 100644 index 00000000..bf22d58a --- /dev/null +++ b/web/hooks/syscall.ts @@ -0,0 +1,68 @@ +import type { Hook, Manifest } from "$lib/plugos/types.ts"; +import type { System } from "$lib/plugos/system.ts"; +import type { SyscallHookT } from "$lib/manifest.ts"; +import type { SysCallMapping } from "$lib/plugos/system.ts"; + +export class SyscallHook implements Hook { + apply(system: System): void { + this.registerSyscalls(system); + system.on({ + plugLoaded: () => { + this.registerSyscalls(system); + }, + }); + } + + registerSyscalls(system: System) { + // Register syscalls from all loaded plugs + for (const plug of system.loadedPlugs.values()) { + const syscalls: SysCallMapping = {}; + + for ( + const [name, functionDef] of Object.entries(plug.manifest!.functions) + ) { + if (!functionDef.syscall) { + continue; + } + + const syscallName = functionDef.syscall; + + console.log("Registering plug syscall", syscallName, "for", name); + // Add the syscall to our mapping + syscalls[syscallName] = (ctx, ...args) => { + // Delegate to the system to invoke the function + return system.syscall(ctx, "system.invokeFunction", [ + name, + ...args, + ]); + }; + + // Register the syscalls with no required permissions + system.registerSyscalls([], syscalls); + } + } + } + + validateManifest(manifest: Manifest): string[] { + const errors: string[] = []; + for (const [name, functionDef] of Object.entries(manifest.functions)) { + if (!functionDef.syscall) { + continue; + } + + // Validate syscall name is provided + if (!functionDef.syscall) { + errors.push(`Function ${name} has a syscall but no name`); + continue; + } + + // Validate syscall name format (should be namespaced) + if (!functionDef.syscall.includes(".")) { + errors.push( + `Function ${name} has invalid syscall name "${functionDef.syscall}" - must be in format "namespace.name"`, + ); + } + } + return errors; + } +} From 8eebb8dd67ac1f7e74652ddd8c98404120fc72ac Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Fri, 7 Feb 2025 15:39:51 +0100 Subject: [PATCH 2/2] Expose index.extractFrontmatter as syscall --- plugs/index/api.ts | 21 ++++++++++++++++++++- plugs/index/index.plug.yaml | 6 ++++++ web/hooks/syscall.ts | 2 +- website/API/index.md | 22 ++++++++++++++++++++++ 4 files changed, 49 insertions(+), 2 deletions(-) diff --git a/plugs/index/api.ts b/plugs/index/api.ts index b45984a4..227f6562 100644 --- a/plugs/index/api.ts +++ b/plugs/index/api.ts @@ -1,4 +1,8 @@ -import { datastore, system } from "@silverbulletmd/silverbullet/syscalls"; +import { + datastore, + markdown, + system, +} from "@silverbulletmd/silverbullet/syscalls"; import type { KV, KvKey, @@ -10,6 +14,12 @@ import type { QueryProviderEvent } from "../../plug-api/types.ts"; import { determineType, type SimpleJSONType } from "./attributes.ts"; import { ttlCache } from "$lib/memory_cache.ts"; import type { LuaCollectionQuery } from "$common/space_lua/query_collection.ts"; +import { + extractFrontmatter as extractFrontmatterFromTree, + type FrontMatter, + type FrontmatterExtractOptions, +} from "../../plug-api/lib/frontmatter.ts"; +import { renderToText } from "@silverbulletmd/silverbullet/lib/tree"; const indexKey = "idx"; const pageKey = "ridx"; @@ -245,3 +255,12 @@ export async function discoverSources() { // And concatenate all the tags from the schema .concat(Object.keys(schema.tag)); } + +export async function extractFrontmatter( + text: string, + extractOptions: FrontmatterExtractOptions = {}, +): Promise<{ frontmatter: FrontMatter; text: string }> { + const tree = await markdown.parseMarkdown(text); + const frontmatter = await extractFrontmatterFromTree(tree, extractOptions); + return { frontmatter, text: renderToText(tree) }; +} diff --git a/plugs/index/index.plug.yaml b/plugs/index/index.plug.yaml index 39e0ee8c..befc123f 100644 --- a/plugs/index/index.plug.yaml +++ b/plugs/index/index.plug.yaml @@ -23,6 +23,12 @@ functions: path: api.ts:getObjectByRef syscall: index.getObjectByRef env: server + + extractFrontmatter: + path: api.ts:extractFrontmatter + syscall: index.extractFrontmatter + + # Event handlers objectSourceProvider: path: api.ts:objectSourceProvider events: diff --git a/web/hooks/syscall.ts b/web/hooks/syscall.ts index bf22d58a..763ccdb0 100644 --- a/web/hooks/syscall.ts +++ b/web/hooks/syscall.ts @@ -32,7 +32,7 @@ export class SyscallHook implements Hook { syscalls[syscallName] = (ctx, ...args) => { // Delegate to the system to invoke the function return system.syscall(ctx, "system.invokeFunction", [ - name, + `${plug.manifest!.name}.${name}`, ...args, ]); }; diff --git a/website/API/index.md b/website/API/index.md index daae7b32..bb92a4f5 100644 --- a/website/API/index.md +++ b/website/API/index.md @@ -1,3 +1,9 @@ +--- +testattribute: 10 +--- + +#apidoc + The `index` API provides functions for interacting with SilverBullet's [[Objects]], allowing you to store and query page-associated data. ## Object Operations @@ -38,3 +44,19 @@ local task = index.getObjectByRef("my page", "mytask", "task1") if task then print("Found task: " .. task.content) end +``` + +## index.extractFrontmatter(text, extractOptions) +Extracts frontmatter from a markdown document (whose text is provided as argument), possibly cleaning it up. It also parses top-level tags consistent with SilverBullet's tag indexing system. + +It returns a table with two keys: +- `frontmatter`: A table containing the parsed frontmatter. +- `text`: The text of the document, with any changes applied requested with the `extractOptions`. + +The `extractOptions` is an optional table that can contain the following keys (which will affect the returned `text`): +- `removeKeys`: An array of keys to remove from the frontmatter. +- `removeTags`: A boolean or array of tags to remove from the frontmatter. +- `removeFrontmatterSection`: A boolean to remove the frontmatter section from the document. + +Example applied to this page: +${(index.extractFrontmatter(editor.getText())).frontmatter} \ No newline at end of file