Lua Integrated Query

pull/1210/head
Zef Hemel 2025-01-13 20:25:39 +01:00
parent 2283d16d09
commit bf6a34f82c
19 changed files with 324 additions and 145 deletions

View File

@ -2,19 +2,16 @@ import type { System } from "../lib/plugos/system.ts";
import type { ScriptObject } from "../plugs/index/script.ts";
import {
LuaEnv,
LuaFunction,
LuaRuntimeError,
LuaStackFrame,
} from "$common/space_lua/runtime.ts";
import { parse as parseLua } from "$common/space_lua/parse.ts";
import { evalStatement } from "$common/space_lua/eval.ts";
import { jsToLuaValue } from "$common/space_lua/runtime.ts";
import {
type PageRef,
parsePageRef,
} from "@silverbulletmd/silverbullet/lib/page_ref";
import type { ScriptEnvironment } from "$common/space_script.ts";
import { luaValueToJS } from "$common/space_lua/runtime.ts";
import type { ASTCtx } from "$common/space_lua/ast.ts";
import { buildLuaEnv } from "$common/space_lua_api.ts";
@ -59,19 +56,6 @@ export class SpaceLuaEnvironment {
}
}
// Find all functions and register them
for (const globalName of this.env.keys()) {
const value = this.env.get(globalName);
if (value instanceof LuaFunction) {
console.log(
`[Lua] Registering global function '${globalName}' (source: ${value.body.ctx.ref})`,
);
scriptEnv.registerFunction({ name: globalName }, (...args: any[]) => {
const sf = new LuaStackFrame(tl, value.body.ctx);
return luaValueToJS(value.call(sf, ...args.map(jsToLuaValue)));
});
}
}
console.log("[Lua] Loaded", allScripts.length, "scripts");
}
}

View File

@ -271,7 +271,7 @@ export type LuaQueryClause =
export type LuaFromClause = {
type: "From";
name: string;
name?: string;
expression: LuaExpression;
} & ASTContext;
@ -299,5 +299,5 @@ export type LuaOrderBy = {
export type LuaSelectClause = {
type: "Select";
tableConstructor: LuaTableConstructor;
expression: LuaExpression;
} & ASTContext;

View File

@ -7,6 +7,7 @@ import type {
import { evalPromiseValues } from "$common/space_lua/util.ts";
import {
luaCall,
luaEquals,
luaSet,
type LuaStackFrame,
} from "$common/space_lua/runtime.ts";
@ -256,7 +257,7 @@ export function evalExpression(
if (!findFromClause) {
throw new LuaRuntimeError("No from clause found", sf.withCtx(e.ctx));
}
const objectVariable = findFromClause.name;
const objectVariable = findFromClause.name || "_";
const objectExpression = findFromClause.expression;
return Promise.resolve(evalExpression(objectExpression, env, sf)).then(
async (collection: LuaValue) => {
@ -298,20 +299,7 @@ export function evalExpression(
break;
}
case "Select": {
query.select = clause.tableConstructor.fields.map((f) => {
if (f.type === "PropField") {
return {
name: f.key,
expr: f.value,
};
} else {
throw new LuaRuntimeError(
"Select fields must be named",
sf.withCtx(f.ctx),
);
}
});
query.select = clause.expression;
break;
}
case "Limit": {
@ -524,15 +512,15 @@ const operatorsMetaMethods: Record<string, {
},
"==": {
metaMethod: "__eq",
nativeImplementation: (a, b) => a === b,
nativeImplementation: (a, b) => luaEquals(a, b),
},
"~=": {
metaMethod: "__ne",
nativeImplementation: (a, b) => a !== b,
nativeImplementation: (a, b) => !luaEquals(a, b),
},
"!=": {
metaMethod: "__ne",
nativeImplementation: (a, b) => a !== b,
nativeImplementation: (a, b) => !luaEquals(a, b),
},
"<": { metaMethod: "__lt", nativeImplementation: (a, b) => a < b },
"<=": { metaMethod: "__le", nativeImplementation: (a, b) => a <= b },

View File

@ -730,3 +730,10 @@ assert_equal(r[2].name, "Jane")
assert_equal(r[2].age, 21)
assert_equal(r[1].lastModified, nil)
assert_equal(r[2].lastModified, nil)
-- Random select test
local r = query [[from {1, 2, 3} select _ + 1]]
assert_equal(#r, 3)
assert_equal(r[1], 2)
assert_equal(r[2], 3)
assert_equal(r[3], 4)

View File

@ -97,12 +97,12 @@ QueryClause {
LimitClause
}
FromClause { ckw<"from"> Name "=" exp }
FromClause { ckw<"from"> (Name "=")? exp }
WhereClause { ckw<"where"> exp }
LimitClause { ckw<"limit"> exp ("," exp)? }
OrderByClause { ckw<"order"> ckw<"by"> list<OrderBy> }
OrderBy { exp ckw<"desc">? }
SelectClause { ckw<"select"> TableConstructor }
SelectClause { ckw<"select"> exp }
field[@isGroup=Field] {

File diff suppressed because one or more lines are too long

View File

@ -105,7 +105,7 @@ Deno.test("Test comment handling", () => {
Deno.test("Test query parsing", () => {
parse(`_(query[[from p = tag("page") where p.name == "John" limit 10, 3]])`);
parse(`_(query[[from p = tag("page") select {name="hello", age=10}]])`);
parse(`_(query[[from tag("page") select {name="hello", age=10}]])`);
parse(
`_(query[[from p = tag("page") order by p.lastModified desc, p.name]])`,
);

View File

@ -20,7 +20,6 @@ import type {
LuaPrefixExpression,
LuaQueryClause,
LuaStatement,
LuaTableConstructor,
LuaTableField,
} from "./ast.ts";
import { tags as t } from "@lezer/highlight";
@ -486,12 +485,21 @@ function parseQueryClause(t: ParseTree, ctx: ASTCtx): LuaQueryClause {
t = t.children![0];
switch (t.type) {
case "FromClause": {
return {
type: "From",
name: t.children![1].children![0].text!,
expression: parseExpression(t.children![3], ctx),
ctx: context(t, ctx),
};
if (t.children!.length === 4) {
// From clause with a name
return {
type: "From",
name: t.children![1].children![0].text!,
expression: parseExpression(t.children![3], ctx),
ctx: context(t, ctx),
};
} else {
return {
type: "From",
expression: parseExpression(t.children![1], ctx),
ctx: context(t, ctx),
};
}
}
case "WhereClause":
return {
@ -532,10 +540,7 @@ function parseQueryClause(t: ParseTree, ctx: ASTCtx): LuaQueryClause {
case "SelectClause": {
return {
type: "Select",
tableConstructor: parseExpression(
t.children![1],
ctx,
) as LuaTableConstructor,
expression: parseExpression(t.children![1], ctx),
ctx: context(t, ctx),
};
}

View File

@ -114,33 +114,27 @@ Deno.test("ArrayQueryCollection", async () => {
const result8 = await collection2.query(
{
objectVariable: "p",
select: [{
name: "fullName",
expr: parseExpressionString("p.firstName .. ' ' .. p.lastName"),
}],
select: parseExpressionString("p.firstName .. ' ' .. p.lastName"),
},
rootEnv,
LuaStackFrame.lostFrame,
);
assertEquals(result8[0].fullName, "John Doe");
assertEquals(result8[1].fullName, "Alice Johnson");
assertEquals(result8[2].fullName, "Jane Doe");
assertEquals(result8[3].fullName, "Bob Johnson");
assertEquals(result8[0], "John Doe");
assertEquals(result8[1], "Alice Johnson");
assertEquals(result8[2], "Jane Doe");
assertEquals(result8[3], "Bob Johnson");
// Test select with native function
const result9 = await collection2.query(
{
objectVariable: "p",
select: [{
name: "fullName",
expr: parseExpressionString("build_name(p.firstName, p.lastName)"),
}],
select: parseExpressionString("build_name(p.firstName, p.lastName)"),
},
rootEnv,
LuaStackFrame.lostFrame,
);
assertEquals(result9[0].fullName, "John Doe");
assertEquals(result9[1].fullName, "Alice Johnson");
assertEquals(result9[2].fullName, "Jane Doe");
assertEquals(result9[3].fullName, "Bob Johnson");
assertEquals(result9[0], "John Doe");
assertEquals(result9[1], "Alice Johnson");
assertEquals(result9[2], "Jane Doe");
assertEquals(result9[3], "Bob Johnson");
});

View File

@ -15,11 +15,6 @@ export type LuaOrderBy = {
desc: boolean;
};
export type LuaSelect = {
name: string;
expr?: LuaExpression;
};
/**
* Represents a query for a collection
*/
@ -30,7 +25,7 @@ export type LuaCollectionQuery = {
// The order by expression evaluated with Lua
orderBy?: LuaOrderBy[];
// The select expression evaluated with Lua
select?: LuaSelect[];
select?: LuaExpression;
// The limit of the query
limit?: number;
// The offset of the query
@ -77,28 +72,6 @@ async function applyTransforms(
env: LuaEnv,
sf: LuaStackFrame,
): Promise<any[]> {
// Apply the select
if (query.select) {
const newResult = [];
for (const item of result) {
const itemEnv = buildItemEnv(query.objectVariable, item, env);
const newItem: Record<string, any> = {};
for (const select of query.select) {
if (select.expr) {
newItem[select.name] = await evalExpression(
select.expr,
itemEnv,
sf,
);
} else {
newItem[select.name] = item[select.name];
}
}
newResult.push(newItem);
}
result = newResult;
}
// Apply the order by
if (query.orderBy) {
result = await asyncQuickSort(result, async (a, b) => {
@ -122,6 +95,16 @@ async function applyTransforms(
});
}
// Apply the select
if (query.select) {
const newResult = [];
for (const item of result) {
const itemEnv = buildItemEnv(query.objectVariable, item, env);
newResult.push(await evalExpression(query.select, itemEnv, sf));
}
result = newResult;
}
// Apply the limit and offset
if (query.limit !== undefined && query.offset !== undefined) {
result = result.slice(query.offset, query.offset + query.limit);

View File

@ -437,6 +437,14 @@ export class LuaTable implements ILuaSettable, ILuaGettable {
return this.arrayPart.map(luaValueToJS);
}
asJS(): Record<string, any> | any[] {
if (this.length > 0) {
return this.asJSArray();
} else {
return this.asJSObject();
}
}
async toStringAsync(): Promise<string> {
if (this.metatable?.has("__tostring")) {
const metaValue = await this.metatable.get("__tostring");
@ -557,6 +565,10 @@ export function luaCall(
return fn.call((sf || LuaStackFrame.lostFrame).withCtx(ctx), ...args);
}
export function luaEquals(a: any, b: any): boolean {
return a === b;
}
export function luaKeys(val: any): any[] {
if (val instanceof LuaTable) {
return val.keys();

View File

@ -1,7 +1,10 @@
import {
type ILuaFunction,
LuaBuiltinFunction,
luaEquals,
LuaRuntimeError,
LuaTable,
type LuaValue,
} from "$common/space_lua/runtime.ts";
export const tableApi = new LuaTable({
@ -40,4 +43,24 @@ export const tableApi = new LuaTable({
keys: new LuaBuiltinFunction((_sf, tbl: LuaTable) => {
return tbl.keys();
}),
includes: new LuaBuiltinFunction(
(sf, tbl: LuaTable | Record<string, any>, value: LuaValue) => {
if (tbl instanceof LuaTable) {
// Iterate over the table
for (const key of tbl.keys()) {
if (luaEquals(tbl.get(key), value)) {
return true;
}
}
return false;
} else if (Array.isArray(tbl)) {
return !!tbl.find((item) => luaEquals(item, value));
} else {
throw new LuaRuntimeError(
`Cannot use includes on a non-table or non-array value`,
sf,
);
}
},
),
});

View File

@ -6,7 +6,7 @@ import type { CommonSystem } from "$common/common_system.ts";
import type { KV, KvKey, KvQuery } from "../../../plug-api/types.ts";
import type { DataStore } from "../../data/datastore.ts";
import type { SysCallMapping } from "../system.ts";
import { LuaStackFrame } from "$common/space_lua/runtime.ts";
import { LuaStackFrame, luaValueToJS } from "$common/space_lua/runtime.ts";
/**
* Exposes the datastore API to plugs, but scoping everything to a prefix based on the plug's name
@ -37,17 +37,17 @@ export function dataStoreReadSyscalls(
return ds.query(query, variables);
},
"datastore.queryLua": (
"datastore.queryLua": async (
_ctx,
prefix: string[],
query: LuaCollectionQuery,
): Promise<KV[]> => {
const dsQueryCollection = new DataStoreQueryCollection(ds, prefix);
return dsQueryCollection.query(
return (await dsQueryCollection.query(
query,
commonSystem.spaceLuaEnv.env,
LuaStackFrame.lostFrame,
);
)).map((item) => luaValueToJS(item));
},
"datastore.listFunctions": (): string[] => {

View File

@ -45,7 +45,7 @@ export class LuaWidget extends WidgetType {
toDOM(): HTMLElement {
const div = document.createElement("div");
div.className = "sb-lua-directive";
// div.className = "sb-lua-directive-inline";
const cacheItem = this.client.getWidgetCache(this.cacheKey);
if (cacheItem) {
div.innerHTML = cacheItem.html;
@ -89,9 +89,9 @@ export class LuaWidget extends WidgetType {
html = widgetContent.html;
div.innerHTML = html;
if ((widgetContent as any)?.display === "block") {
div.style.display = "block";
div.className = "sb-lua-directive-block";
} else {
div.style.display = "inline";
div.className = "sb-lua-directive-inline";
}
attachWidgetEventHandlers(div, this.client, this.from);
this.client.setWidgetCache(
@ -128,10 +128,13 @@ export class LuaWidget extends WidgetType {
return;
}
if ((widgetContent as any)?.display === "block") {
div.style.display = "block";
if (
(widgetContent as any)?.display === "block" ||
trimmedMarkdown.includes("\n")
) {
div.className = "sb-lua-directive-block";
} else {
div.style.display = "inline";
div.className = "sb-lua-directive-inline";
}
// Parse the markdown again after trimming

View File

@ -6,6 +6,7 @@ import type { Client } from "../client.ts";
const straightQuoteContexts = [
"CommentBlock",
"CodeBlock",
"CodeText",
"FencedCode",
"InlineCode",
"FrontMatterCode",

View File

@ -334,14 +334,22 @@
cursor: pointer;
}
.sb-lua-directive {
background-color: rgb(233, 232, 232, 35%);
border: 1px #d3d3d373 solid;
/* box-shadow: #d1d1d1 0 0 4px; */
.sb-lua-directive-inline {
display: inline;
border: 1px var(--editor-widget-background-color) solid;
border-radius: 8px;
padding: 2px;
}
.sb-lua-directive-block {
display: block;
border: 1px var(--editor-widget-background-color) solid;
border-radius: 8px;
padding: 5px;
margin: -1em 0;
overflow: auto;
}
a.sb-wiki-link-page-missing,
.sb-wiki-link-page-missing > .sb-wiki-link-page {
border-radius: 5px;

View File

@ -1,18 +1,32 @@
> **warning** Experimental
> This is a **highly experimental** feature still under active development. It is documented here primarily for the real early adopters as this feature develops.
>
> If you want to experiment, be sure to use the [edge builds](https://community.silverbullet.md/t/living-on-the-edge-builds/27/5).
Space Lua is a custom implementation of the [Lua programming language](https://lua.org/) embedded in SilverBullet.
Space Lua is a custom implementation of the [Lua programming language](https://lua.org/), embedded in SilverBullet. It aims to be a largely complete Lua implementation, and adds a few non-standard features while remaining syntactically compatible with “real” Lua.
# Goals
These are current, long term goals that are subject to change.
The introduction of Lua aims to unify and simplify a few SilverBullet features, specifically:
* Provide a safe, integrated, productive way to extend SilverBullets feature set with a low barrier to entry
* Ultimately succeed [[Space Script]] (for most, if not all) use cases
* Ultimately replace [[Expression Language]] with Luas expression language, also in [[Query Language]].
* Ultimately replace [[Template Language]] with a variant using Luas control flows (`for`, `if` etc.)
* Scripting: replace [[Space Script]] (JavaScript) with a more controlled, simple and extensible language.
* Replace [[Expression Language]], [[Template Language]] and [[Query Language]] with Lua-based equivalents.
* (Potentially) provide an alternative way to specify [[Space Config]]
# Use
Space Lua functions analogously to [[Space Script]], [[Space Style]] and [[Space Config]] in that it is defined in fenced code blocks, in this case with the `space-lua` language. As follows:
# Introduction approach
This is a big effort. During its development, Space Lua will be offered as a kind of “alternative universe” to the things mentioned above. Existing [[Live Templates]], [[Live Queries]] and [[Space Script]] will continue to work as before, unaltered.
Once these features stabilize and best practices are ironed out, old mechanisms will likely be deprecated and possibly removed at some point.
Were not there yet, though.
# Basics
In its essence, Space Lua adds two features to its [[Markdown]] language:
* **Definitions**: Code written in `space-lua` code blocks are enabled across your entire space.
* **Expressions**: The `${expression}` syntax will [[Live Preview]] to its evaluated value.
## Definitions
Space Lua definitions are defined in fenced code blocks, in this case with the `space-lua` language. As follows:
```space-lua
-- adds two numbers
@ -21,14 +35,41 @@ function adder(a, b)
end
```
Each `space-lua` block has its own local scope, however when functions and variables are not explicitly defined as `local` they will be available from anywhere (following regular Lua scoping rule).
Each `space-lua` block has its own local scope. However, following Lua semantics, when functions and variables are not explicitly defined as `local` they will be available globally across your space. This means that the `adder` function above can be used in any other page.
A new syntax introduced with Space Lua is the `${lua expression}` syntax that you can use in your pages, this syntax will [[Live Preview]] to the evaluation of that expression.
Since there is a single global namespace, it is good practice to manually namespace things using the following pattern:
Example: 10 + 2 = ${adder(10, 2)} (Alt-click on this value to see the expression using the just defined `adder` function to calculate this).
```space-lua
-- This initializes the stuff variable with an empty table if it's not already defined
stuff = stuff or {}
function stuff.adder(a, b)
return a + b
end
```
> **note** Tip
> All your space-lua scripts are loaded on boot, to reload them without reloading the page, simply run the {[System: Reload]} command.
## Expressions
A new syntax introduced with Space Lua is the `${lua expression}` syntax that you can use in your pages. This syntax will [[Live Preview]] to the evaluation of that expression.
For example: 10 + 2 = ${adder(10, 2)} (Alt-click, or select to see the expression) is using the just defined `adder` function to this rather impressive calculation. Yes, this may as well be written as `${10 + 2}` (${10 + 2}), but... you know.
## Queries
Space Lua has a feature called [[Space Lua/Lua Integrated Query]], which integrate SQL-like queries into Lua. By using this feature, you can easily replicate [[Live Queries]]. More detail in [[Space Lua/Lua Integrated Query]], but heres a small example querying the last 3 modifies pages:
${query[[
from tag "page"
order by _.lastModified desc
select _.name
limit 3
]]}
## Widgets
The `${lua expression}` syntax can be used to implement simple widgets. If the lua expression evaluates to a simple string, it will live preview as that string rendered as simple markdown. However, if the expression returns a Lua table with specific keys, you can do some cooler stuff. The following keys are supported:
The `${lua expression}` syntax can be used to implement simple widgets. If the Lua expression evaluates to a simple string, it will live preview as that string rendered as markdown. However, if the expression returns a Lua table with specific keys, you can do some cooler stuff.
The following keys are supported:
* `markdown`: Renders the value as markdown
* `html`: Renders the value as HTML
@ -55,7 +96,7 @@ And some [[Space Style]] to style it:
}
```
Now, lets use it (put your cursor in there to see the code):
Now, lets use it:
${marquee "Finally, marqeeeeeeee!"}
Oh boy, the times we live in!
@ -88,30 +129,17 @@ define_event_listener {
}
```
## Custom functions
Any global function (so not marked with `local`) is automatically exposed to be used in [[Live Queries]] and [[Live Templates]]:
# Space Lua Extensions
Space Lua currently introduces a few new features on top core Lua:
```space-lua
-- This is a global function, therefore automatically exposed
function greet_me(name)
return "Hello, " .. name
end
1. [[Space Lua/Lua Integrated Query]], embedding a [[Query Language]]-like language into Lua itself
2. Thread locals
-- Whereas this one is not
local function greet_you(name)
error("This is not exposed")
end
```
Template:
```template
Here's a greeting: {{greet_me("Pete")}}
```
# Thread locals
Theres a magic `_CTX` global variable available from which you can access useful context-specific value. Currently the following keys are available:
## Thread locals
Theres a magic `_CTX` global variable available from which you can access useful context-specific values. Currently the following keys are available:
* `_CTX.pageMeta` contains a reference to the loaded page metadata (can be `nil` when not yet loaded)
* `_CTX.GLOBAL` providing access to the global scope
# API
Lua APIs, which should be (roughly) implemented according to the Lua standard.

View File

@ -0,0 +1,110 @@
Lua Integrated Query (LIQ) is a SilverBullet specific Lua extension. It adds a convenient query syntax to the language in a backwards compatible way. It does so by overloading Luas default function call + single argument syntax when using `query` as the function call. As a result, Lua programs using LIQ are still syntactically valid Lua.
The syntax for LIQ is `query[[my query]]`. In regular Lua `[[my query]]` is just another way of writing `"my query"` (it is an alternative string syntax). Function calls that only take a string argument can omit parentheses, therefore `query[[my query]]` is equivalent to `query("my query")`.
However, in [[Space Lua]] it interpreted as an SQL (and [LINQ](https://learn.microsoft.com/en-us/dotnet/csharp/linq/))-inspired integrated query language.
General syntax:
query[[
from <var> = <expression>
where <expression>
order by <expression>
limit <expression>, <expression>
select <expression>
]]
Unlike [[Query Language]] which operates on [[Objects]] only, LIQ can operate on any Lua collection.
For instance, to sort a list of numbers in descending order:
${query[[from n = {1, 2, 3} order by n desc]]}
However, in most cases youll use it in conjunction with [[Space Lua/stdlib#tag(name)]]. Heres an example querying the 3 pages that were last modified:
${query[[
from p = tag "page"
order by p.lastModified desc
select p.name
limit 3
]]}
# Clauses
Here are the clauses that are currently supported:
## `from <expression>`
The `from` clause specifies the source of your data. There are two syntactic variants:
With explicit variable binding:
from v = <<expression>>
binding each item to the variable `v`.
And the shorter:
from <<expression>>
implicitly binding each item to the variable `_`.
Example without variable binding:
${query[[from {1, 2, 3} select _]]}
With variable binding:
${query[[from n = {1, 2, 3} select n]]}
A more realist example using `tag`:
${query[[from t = tag "page" limit 3 select t.name]]}
## `where <expression>`
The `where` clause allows you to filter data. When the expression evaluated to a truthy value, the item is included in the result.
Example:
${query[[from {1, 2, 3, 4, 5} where _ > 2]]}
Or to select all pages tagged with `#meta`:
${query[[from tag "page" where table.includes(_.tags, "meta")]]}
## `order by <expression> [desc]`
The `order by` clause allows you to sort data, when `desc` is specified it reverts the sort order.
As an example, the last 3 modified pages:
${query[[
from tag "page"
order by _.lastModified desc
select _.name
limit 3
]]}
## `limit <expression>[, <expression>]`
The `limit` clause allows you to limit the number of results, optionally with an offset.
Example:
${query[[from {1, 2, 3, 4, 5} limit 3]]}
You can also specify an offset to skip some results:
${query[[from {1, 2, 3, 4, 5} limit 3, 2]]}
## `select <expression>`
The `select` clause allows you to transform each item in the result set. If omitted, it defaults to returning the item itself.
Some examples:
Double each number:
${query[[from {1, 2, 3} select _ * 2]]}
Extract just the name from pages:
${query[[from tag "page" select _.name limit 3]]}
You can also return tables or other complex values:
${query[[
from tag "page"
select {
name = _.name,
modified = _.lastModified
}
limit 3
]]}

View File

@ -0,0 +1,33 @@
These are Lua functions defined in the global namespace:
# Standard Lua
## print(...)
Prints to your log (browser or server log).
## assert(expr)
Asserts `expr` to be true otherwise raises an [[#error]]
## ipairs
## pairs
## unpack
## type
## tostring
## tonumber
## error(message)
Throw an error.
Example: `error("FAIL")`
## pcall
## xpcall
## setmetatable
## getmetatable
## rawset
# Space Lua specific
## tag(name)
Returns a given [[Objects#Tags]] as a query collection, to be queried using [[Space Lua/Lua Integrated Query]].
Example:
${query[[from tag("page") limit 1]]}