From fce78a22f8ca65063ff3c8035c5410c377480ed3 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Wed, 22 Jan 2025 20:26:37 +0100 Subject: [PATCH] More Lua template work --- common/common_system.ts | 6 +- common/space_lua/parse.test.ts | 27 ++++-- common/space_lua/parse.ts | 147 +++++++++++++++++------------ common/space_lua/runtime.ts | 11 +++ common/space_lua_api.ts | 23 +++-- common/space_script.ts | 14 ++- common/syscalls/command.ts | 22 +++++ common/syscalls/index.ts | 9 +- lib/command.ts | 7 ++ plugs/core/Library/Std/Config.md | 2 +- plugs/core/Library/Std/Template.md | 10 +- plugs/index/script.ts | 1 - plugs/template/snippet.ts | 32 +++++-- plugs/template/template.plug.yaml | 4 + web/client.ts | 4 +- web/client_system.ts | 4 +- web/cm_plugins/inline_content.ts | 2 +- web/cm_plugins/lua_directive.ts | 6 +- web/components/top_bar.tsx | 5 +- web/editor_state.ts | 2 +- web/hooks/slash_command.ts | 26 ++++- website/Space Lua.md | 3 +- 22 files changed, 254 insertions(+), 113 deletions(-) diff --git a/common/common_system.ts b/common/common_system.ts index 4aa08a46..59f9c340 100644 --- a/common/common_system.ts +++ b/common/common_system.ts @@ -22,7 +22,7 @@ export abstract class CommonSystem { // Hooks commandHook!: CommandHook; - slashCommandHook!: SlashCommandHook; + slashCommandHook?: SlashCommandHook; namespaceHook!: PlugNamespaceHook; codeWidgetHook!: CodeWidgetHook; panelWidgetHook!: PanelWidgetHook; @@ -78,6 +78,10 @@ export abstract class CommonSystem { this.eventHook.scriptEnvironment = this.scriptEnv; this.commandHook.throttledBuildAllCommands(); + if (this.slashCommandHook) { + // Only on client + this.slashCommandHook.throttledBuildAllCommands(); + } } // Swap in the expanded function map diff --git a/common/space_lua/parse.test.ts b/common/space_lua/parse.test.ts index c2c4fae6..78cff97b 100644 --- a/common/space_lua/parse.test.ts +++ b/common/space_lua/parse.test.ts @@ -1,4 +1,5 @@ -import { parse } from "$common/space_lua/parse.ts"; +import { parse, stripLuaComments } from "$common/space_lua/parse.ts"; +import { assertEquals } from "@std/assert/equals"; Deno.test("Test Lua parser", () => { // Basic block test @@ -108,15 +109,21 @@ Deno.test("Test Lua parser", () => { }); Deno.test("Test comment handling", () => { - parse(` - -- Single line comment - --[[ Multi - line - comment ]] - f([[ - hello - -- yo - ]])`); + const code = ` +-- Single line comment +--[[ Multi +line +comment ]] +f([[ +hello +-- yo +]])`; + const code2 = stripLuaComments(code); + assertEquals(code2.length, code.length); + console.log(code2); + console.log(stripLuaComments(`e([==[ + --- Hello + ]==])`)); }); Deno.test("Test query parsing", () => { diff --git a/common/space_lua/parse.ts b/common/space_lua/parse.ts index ac9a5614..82b162b3 100644 --- a/common/space_lua/parse.ts +++ b/common/space_lua/parse.ts @@ -635,76 +635,103 @@ function parseTableField(t: ParseTree, ctx: ASTCtx): LuaTableField { throw new Error(`Unknown table field type: ${t.type}`); } } -function stripLuaComments(s: string): string { - // Strips Lua comments (single-line and multi-line) and replaces them with equivalent length whitespace + +export function stripLuaComments(s: string): string { let result = ""; - let inString = false; - let inMultilineString = false; - let inComment = false; - let inMultilineComment = false; + let i = 0; - for (let i = 0; i < s.length; i++) { - // Handle string detection for single-line strings (to avoid stripping comments inside strings) - if ( - s[i] === '"' && !inComment && !inMultilineComment && !inMultilineString - ) { - inString = !inString; - } + while (i < s.length) { + // Check for long string + if (s[i] === "[") { + let j = i + 1; + let equalsCount = 0; + while (s[j] === "=") { + equalsCount++; + j++; + } + if (s[j] === "[") { + // Found long string start + const openBracket = s.substring(i, j + 1); + const closeBracket = "]" + "=".repeat(equalsCount) + "]"; + result += openBracket; + i = j + 1; - // Handle multi-line string literals (starting with "[[") - if ( - !inString && !inComment && !inMultilineComment && s[i] === "[" && - s[i + 1] === "[" - ) { - inMultilineString = true; - result += "[["; // Copy "[[" into result - i += 1; // Skip over "[[" - continue; - } - - // Handle end of multi-line string literals (ending with "]]") - if (inMultilineString && s[i] === "]" && s[i + 1] === "]") { - inMultilineString = false; - result += "]]"; // Copy "]]" into result - i += 1; // Skip over "]]" - continue; - } - - // Handle single-line comments (starting with "--") - if ( - !inString && !inMultilineString && !inMultilineComment && s[i] === "-" && - s[i + 1] === "-" - ) { - if (s[i + 2] === "[" && s[i + 3] === "[") { - // Detect multi-line comment start "--[[" - inMultilineComment = true; - i += 3; // Skip over "--[[" - result += " "; // Add equivalent length spaces for "--[[" - continue; - } else { - inComment = true; + // Find matching closing bracket + const content = s.substring(i); + const closeIndex = content.indexOf(closeBracket); + if (closeIndex !== -1) { + // Copy string content verbatim, including any comment-like sequences + result += content.substring(0, closeIndex) + closeBracket; + i += closeIndex + closeBracket.length; + continue; + } } } - // Handle end of single-line comment - if (inComment && s[i] === "\n") { - inComment = false; - } - - // Handle multi-line comment ending "]]" - if (inMultilineComment && s[i] === "]" && s[i + 1] === "]") { - inMultilineComment = false; - i += 1; // Skip over "]]" - result += " "; // Add equivalent length spaces for "]]" + // Check for single quoted string + if (s[i] === '"' || s[i] === "'") { + const quote = s[i]; + result += quote; + i++; + while (i < s.length && s[i] !== quote) { + if (s[i] === "\\") { + result += s[i] + s[i + 1]; + i += 2; + } else { + result += s[i]; + i++; + } + } + if (i < s.length) { + result += s[i]; // closing quote + i++; + } continue; } - // Replace comment content with spaces, or copy original content if not in comment or multi-line string - if (inComment || inMultilineComment) { - result += " "; // Replace comment characters with spaces - } else { - result += s[i]; + // Check for comments + if (s[i] === "-" && s[i + 1] === "-") { + // Replace the -- with spaces + result += " "; + i += 2; + + // Check for long comment + if (s[i] === "[") { + let j = i + 1; + let equalsCount = 0; + while (s[j] === "=") { + equalsCount++; + j++; + } + if (s[j] === "[") { + // Found long comment start + const closeBracket = "]" + "=".repeat(equalsCount) + "]"; + // Replace opening bracket with spaces + result += " ".repeat(j - i + 1); + i = j + 1; + + // Find matching closing bracket + const content = s.substring(i); + const closeIndex = content.indexOf(closeBracket); + if (closeIndex !== -1) { + // Replace comment content and closing bracket with spaces + result += " ".repeat(closeIndex) + " ".repeat(closeBracket.length); + i += closeIndex + closeBracket.length; + continue; + } + } + } + + // Single line comment - replace rest of line with spaces + while (i < s.length && s[i] !== "\n") { + result += " "; + i++; + } + continue; } + + result += s[i]; + i++; } return result; diff --git a/common/space_lua/runtime.ts b/common/space_lua/runtime.ts index f3739030..61228a7a 100644 --- a/common/space_lua/runtime.ts +++ b/common/space_lua/runtime.ts @@ -80,6 +80,17 @@ export class LuaEnv implements ILuaSettable, ILuaGettable { } return keys; } + + toJSON(omitKeys: string[] = []): Record { + const result: Record = {}; + for (const key of this.keys()) { + if (omitKeys.includes(key)) { + continue; + } + result[key] = luaValueToJS(this.get(key)); + } + return result; + } } export class LuaStackFrame { diff --git a/common/space_lua_api.ts b/common/space_lua_api.ts index 8ed8d6ff..3dea6a01 100644 --- a/common/space_lua_api.ts +++ b/common/space_lua_api.ts @@ -39,13 +39,24 @@ function exposeSyscalls(env: LuaEnv, system: System) { } } -export function buildThreadLocalEnv(_system: System, globalEnv: LuaEnv) { +export async function buildThreadLocalEnv( + system: System, + globalEnv: LuaEnv, +) { const tl = new LuaEnv(); - // const currentPageMeta = await system.localSyscall( - // "editor.getCurrentPageMeta", - // [], - // ); - // tl.setLocal("pageMeta", currentPageMeta); + if (system.registeredSyscalls.has("editor.getCurrentPageMeta")) { + const currentPageMeta = await system.localSyscall( + "editor.getCurrentPageMeta", + [], + ); + if (currentPageMeta) { + tl.setLocal("currentPage", currentPageMeta); + } else { + tl.setLocal("currentPage", { + name: await system.localSyscall("editor.getCurrentPage", []), + }); + } + } tl.setLocal("_GLOBAL", globalEnv); return Promise.resolve(tl); } diff --git a/common/space_script.ts b/common/space_script.ts index db44c8a3..aa13ce45 100644 --- a/common/space_script.ts +++ b/common/space_script.ts @@ -1,9 +1,10 @@ import type { System } from "../lib/plugos/system.ts"; import type { ParseTree } from "../plug-api/lib/tree.ts"; import type { ScriptObject } from "../plugs/index/script.ts"; -import type { AppCommand, CommandDef } from "$lib/command.ts"; +import type { AppCommand, CommandDef, SlashCommand } from "$lib/command.ts"; import { Intl, Temporal, toTemporalInstant } from "@js-temporal/polyfill"; import * as syscalls from "@silverbulletmd/silverbullet/syscalls"; +import type { SlashCommandDef } from "$lib/manifest.ts"; // @ts-ignore: Temporal polyfill Date.prototype.toTemporalInstant = toTemporalInstant; @@ -32,6 +33,7 @@ type AttributeExtractorCallback = ( export class ScriptEnvironment { functions: Record any> = {}; commands: Record = {}; + slashCommands: Record = {}; attributeExtractors: Record = {}; eventHandlers: Record any)[]> = {}; @@ -71,6 +73,16 @@ export class ScriptEnvironment { }; } + registerSlashCommand( + def: SlashCommandDef, + fn: (...args: any[]) => any, + ) { + this.slashCommands[def.name] = { + slashCommand: def, + run: fn, + }; + } + registerAttributeExtractor( def: AttributeExtractorDef, callback: AttributeExtractorCallback, diff --git a/common/syscalls/command.ts b/common/syscalls/command.ts index 565009da..491dfeb8 100644 --- a/common/syscalls/command.ts +++ b/common/syscalls/command.ts @@ -46,5 +46,27 @@ export function commandSyscalls( }, ); }, + "slash_command.define": ( + _ctx, + def: CallbackCommandDef, + ) => { + commonSystem.scriptEnv.registerSlashCommand( + def, + async (...args: any[]) => { + const tl = await buildThreadLocalEnv( + commonSystem.system, + commonSystem.spaceLuaEnv.env, + ); + const sf = new LuaStackFrame(tl, null); + try { + return luaValueToJS( + await luaCall(def.run, args.map(jsToLuaValue), sf), + ); + } catch (e: any) { + await handleLuaError(e, commonSystem.system); + } + }, + ); + }, }; } diff --git a/common/syscalls/index.ts b/common/syscalls/index.ts index 387f2423..c9dbb55f 100644 --- a/common/syscalls/index.ts +++ b/common/syscalls/index.ts @@ -10,7 +10,7 @@ import { type LuaQueryCollection, } from "$common/space_lua/query_collection.ts"; import { - type LuaEnv, + LuaEnv, LuaRuntimeError, type LuaStackFrame, luaValueToJS, @@ -90,7 +90,12 @@ export function indexSyscalls(commonSystem: CommonSystem): SysCallMapping { const scopedVariables: Record = {}; for (const v of localVars) { try { - const jsonValue = await luaValueToJS(env.get(v)); + let value = env.get(v); + if (value instanceof LuaEnv) { + // We don't want to include the global environment in the serialized value + value = value.toJSON(["_GLOBAL"]); + } + const jsonValue = await luaValueToJS(value); // Ensure this is JSON serializable JSON.stringify(jsonValue); scopedVariables[v] = jsonValue; diff --git a/lib/command.ts b/lib/command.ts index 8187e97d..f483a0f1 100644 --- a/lib/command.ts +++ b/lib/command.ts @@ -1,3 +1,5 @@ +import type { SlashCommandDef } from "$lib/manifest.ts"; + export type CommandDef = { name: string; @@ -19,6 +21,11 @@ export type AppCommand = { run: (args?: any[]) => Promise; }; +export type SlashCommand = { + slashCommand: SlashCommandDef; + run: (args?: any[]) => Promise; +}; + export type CommandHookEvents = { commandsUpdated(commandMap: Map): void; }; diff --git a/plugs/core/Library/Std/Config.md b/plugs/core/Library/Std/Config.md index a9aa284d..e8af163b 100644 --- a/plugs/core/Library/Std/Config.md +++ b/plugs/core/Library/Std/Config.md @@ -3,7 +3,7 @@ Config library for defining and getting config values ```space-lua --- priority: 100 +-- priority: 10 config = {} local config_values = {} diff --git a/plugs/core/Library/Std/Template.md b/plugs/core/Library/Std/Template.md index 07805dcf..f8e836c6 100644 --- a/plugs/core/Library/Std/Template.md +++ b/plugs/core/Library/Std/Template.md @@ -3,9 +3,11 @@ Implements useful template functions ```space-lua --- priority: 100 --- Template library for working with templates and iterables +-- priority: 10 +-- Template API root table template = {} +-- Template storage table +templates = {} -- Iterates over a table/array and applies a function to each element, -- concatenating the results @@ -31,8 +33,4 @@ function template.new(template_str) return space_lua.interpolate(template_str, obj) end end - -print("template loaded!!") - - ``` diff --git a/plugs/index/script.ts b/plugs/index/script.ts index 50cc0453..0ac51c4d 100644 --- a/plugs/index/script.ts +++ b/plugs/index/script.ts @@ -2,7 +2,6 @@ import type { IndexTreeEvent } from "../../plug-api/types.ts"; import { collectNodesOfType, findNodeOfType } from "../../plug-api/lib/tree.ts"; import type { ObjectValue } from "../../plug-api/types.ts"; import { indexObjects } from "./api.ts"; -import { space } from "@silverbulletmd/silverbullet/syscalls"; export type ScriptObject = ObjectValue<{ script: string; priority?: number; diff --git a/plugs/template/snippet.ts b/plugs/template/snippet.ts index dda1178f..e14e9e38 100644 --- a/plugs/template/snippet.ts +++ b/plugs/template/snippet.ts @@ -55,14 +55,13 @@ export async function insertSnippetTemplate( const config = await system.getSpaceConfig(); const templateText = await space.readPage(slashCompletion.templatePage); - let { renderedFrontmatter, text: replacementText, frontmatter } = + const { renderedFrontmatter, text: replacementText, frontmatter } = await renderTemplate( templateText, pageObject, { page: pageObject, config }, ); const snippetTemplate: SnippetConfig = frontmatter.hooks.snippet; - let cursorPos = await editor.getCursor(); if (renderedFrontmatter) { @@ -108,8 +107,21 @@ export async function insertSnippetTemplate( cursorPos = await editor.getCursor(); } - if (snippetTemplate.insertAt) { - switch (snippetTemplate.insertAt) { + await applySnippetTemplate(replacementText, snippetTemplate); +} + +export async function applySnippetTemplate( + templateText: string, + config: { + insertAt?: string; + match?: string; + matchRegex?: string; + }, +) { + let cursorPos = await editor.getCursor(); + + if (config.insertAt) { + switch (config.insertAt) { case "page-start": await editor.moveCursor(0); break; @@ -141,12 +153,12 @@ export async function insertSnippetTemplate( cursorPos = await editor.getCursor(); - if (snippetTemplate.match || snippetTemplate.matchRegex) { + if (config.match || config.matchRegex) { const pageText = await editor.getText(); // Regex matching mode const matchRegex = new RegExp( - (snippetTemplate.match || snippetTemplate.matchRegex)!, + (config.match || config.matchRegex)!, ); let startOfLine = cursorPos; @@ -158,7 +170,7 @@ export async function insertSnippetTemplate( endOfLine++; } let currentLine = pageText.slice(startOfLine, endOfLine); - const caretParts = replacementText.split("|^|"); + const caretParts = templateText.split("|^|"); const emptyLine = !currentLine; currentLine = currentLine.replace(matchRegex, caretParts[0]); @@ -190,9 +202,9 @@ export async function insertSnippetTemplate( selection: newSelection, }); } else { - const carretPos = replacementText.indexOf("|^|"); - replacementText = replacementText.replace("|^|", ""); - await editor.insertAtCursor(replacementText); + const carretPos = templateText.indexOf("|^|"); + templateText = templateText.replace("|^|", ""); + await editor.insertAtCursor(templateText); if (carretPos !== -1) { await editor.moveCursor(cursorPos + carretPos); } diff --git a/plugs/template/template.plug.yaml b/plugs/template/template.plug.yaml index 1094ed74..a7cbed92 100644 --- a/plugs/template/template.plug.yaml +++ b/plugs/template/template.plug.yaml @@ -11,6 +11,10 @@ functions: instantiatePageTemplate: path: page.ts:instantiatePageTemplate + applySnippetTemplate: + path: snippet.ts:applySnippetTemplate + env: client + # Indexing indexTemplate: path: index.ts:indexTemplate diff --git a/web/client.ts b/web/client.ts index 912c9958..90cb455c 100644 --- a/web/client.ts +++ b/web/client.ts @@ -273,9 +273,7 @@ export class Client implements ConfigContainer { type: "config-loaded", config: this.config, }); - this.clientSystem.slashCommandHook.buildAllCommands( - this.clientSystem.system, - ); + this.clientSystem.slashCommandHook!.buildAllCommands(); this.eventHook.dispatchEvent("config:loaded", this.config); } diff --git a/web/client_system.ts b/web/client_system.ts index ba6a99ef..c1647da1 100644 --- a/web/client_system.ts +++ b/web/client_system.ts @@ -125,7 +125,7 @@ export class ClientSystem extends CommonSystem { this.system.addHook(this.commandHook); // Slash command hook - this.slashCommandHook = new SlashCommandHook(this.client); + this.slashCommandHook = new SlashCommandHook(this.client, this); this.system.addHook(this.slashCommandHook); this.eventHook.addLocalListener( @@ -147,7 +147,7 @@ export class ClientSystem extends CommonSystem { init() { // Slash command hook - this.slashCommandHook = new SlashCommandHook(this.client); + this.slashCommandHook = new SlashCommandHook(this.client, this); this.system.addHook(this.slashCommandHook); // Syscalls available to all plugs diff --git a/web/cm_plugins/inline_content.ts b/web/cm_plugins/inline_content.ts index 52b4856f..7bfb10f6 100644 --- a/web/cm_plugins/inline_content.ts +++ b/web/cm_plugins/inline_content.ts @@ -56,7 +56,7 @@ class InlineContentWidget extends WidgetType { return div; } - const url = encodeURIComponent(this.url) + const url = encodeURIComponent(this.url); if (mimeType.startsWith("image/")) { const img = document.createElement("img"); diff --git a/web/cm_plugins/lua_directive.ts b/web/cm_plugins/lua_directive.ts index da1a7b4a..f21b9a8f 100644 --- a/web/cm_plugins/lua_directive.ts +++ b/web/cm_plugins/lua_directive.ts @@ -59,7 +59,11 @@ export function luaDirectivePlugin(client: Client) { .args[0]; const tl = new LuaEnv(); - tl.setLocal("pageMeta", currentPageMeta); + tl.setLocal( + "currentPage", + currentPageMeta || + { name: client.ui.viewState.currentPage }, + ); tl.setLocal("_GLOBAL", client.clientSystem.spaceLuaEnv.env); const sf = new LuaStackFrame(tl, expr.ctx); const threadLocalizedEnv = new LuaEnv( diff --git a/web/components/top_bar.tsx b/web/components/top_bar.tsx index 206ce596..09d6741f 100644 --- a/web/components/top_bar.tsx +++ b/web/components/top_bar.tsx @@ -104,7 +104,10 @@ export function TopBar({
{progressPerc !== undefined && ( -
+
{ slashCommands = new Map(); private editor: Client; - constructor(editor: Client) { + constructor(editor: Client, private commonSystem: CommonSystem) { this.editor = editor; } - buildAllCommands(system: System) { + throttledBuildAllCommands = throttle(() => { + this.buildAllCommands(); + }, 200); + + buildAllCommands() { + const system = this.commonSystem.system; + this.slashCommands.clear(); for (const plug of system.loadedPlugs.values()) { for ( @@ -50,6 +57,15 @@ export class SlashCommandHook implements Hook { }); } } + // Iterate over script defined slash commands + for ( + const [name, command] of Object.entries( + this.commonSystem.scriptEnv.slashCommands, + ) + ) { + this.slashCommands.set(name, command); + } + // Iterate over all shortcuts if (this.editor.config?.shortcuts) { // Add slash commands for shortcuts that configure them for (const shortcut of this.editor.config.shortcuts) { @@ -161,10 +177,10 @@ export class SlashCommandHook implements Hook { } apply(system: System): void { - this.buildAllCommands(system); + this.buildAllCommands(); system.on({ plugLoaded: () => { - this.buildAllCommands(system); + this.buildAllCommands(); }, }); } diff --git a/website/Space Lua.md b/website/Space Lua.md index 7f26ae17..7abf8c04 100644 --- a/website/Space Lua.md +++ b/website/Space Lua.md @@ -137,7 +137,8 @@ Space Lua currently introduces a few new features on top core Lua: ## Thread locals There’s a magic `_CTX` global variable available from which you can access useful context-specific values. Currently the following keys are available: -* `_CTX.GLOBAL` providing access to the global scope +* `_CTX.currentPage` providing access (in the client only) to the currently open page (PageMeta object) +* `_CTX._GLOBAL` providing access to the global scope # API Lua APIs, which should be (roughly) implemented according to the Lua standard.