From 077d58d4398b0c2553234b9623103bc51c8b2e3a Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Wed, 5 Oct 2022 10:35:01 +0200 Subject: [PATCH] WIP --- mod.ts | 29 ++-- packages/plugos/syscalls/esbuild.ts | 10 +- ...e.knex_node.test.ts => store.deno.test.ts} | 43 +++--- .../{store.dex_deno.ts => store.deno.ts} | 114 +++++++-------- .../plugos/syscalls/store.dex_deno.test.ts | 9 -- packages/plugos/syscalls/store.knex_node.ts | 133 ------------------ packages/plugos/tsconfig.json | 14 -- test.db | Bin 0 -> 12288 bytes test_dep.ts | 12 ++ 9 files changed, 106 insertions(+), 258 deletions(-) rename packages/plugos/syscalls/{store.knex_node.test.ts => store.deno.test.ts} (69%) rename packages/plugos/syscalls/{store.dex_deno.ts => store.deno.ts} (53%) delete mode 100644 packages/plugos/syscalls/store.dex_deno.test.ts delete mode 100644 packages/plugos/syscalls/store.knex_node.ts delete mode 100644 packages/plugos/tsconfig.json create mode 100644 test.db create mode 100644 test_dep.ts diff --git a/mod.ts b/mod.ts index 53592a39..7a1b4f8b 100644 --- a/mod.ts +++ b/mod.ts @@ -9,22 +9,22 @@ export * as path from "https://deno.land/std@0.158.0/path/mod.ts"; export { readAll } from "https://deno.land/std@0.158.0/streams/conversion.ts"; export { - encode as b64encode, decode as b64decode, -} from "https://deno.land/std/encoding/base64.ts"; + encode as b64encode, +} from "https://deno.land/std@0.158.0/encoding/base64.ts"; export { defaultHighlightStyle, - Language, - LanguageSupport, - LanguageDescription, - syntaxHighlighting, - syntaxTree, defineLanguageFacet, - languageDataProp, foldNodeProp, indentNodeProp, + Language, + languageDataProp, + LanguageDescription, + LanguageSupport, ParseContext, + syntaxHighlighting, + syntaxTree, } from "@codemirror/language"; export { markdown } from "https://esm.sh/@codemirror/lang-markdown@6.0.1?external=@codemirror/state"; export { @@ -45,19 +45,19 @@ export type { LeafBlock, LeafBlockParser, MarkdownConfig, + MarkdownExtension, Table, TaskList, - MarkdownExtension, } from "https://esm.sh/@lezer/markdown"; export { + Emoji, + GFM, MarkdownParser, parseCode, parser as baseParser, - GFM, Subscript, Superscript, - Emoji, } from "https://esm.sh/@lezer/markdown"; export type { SyntaxNode, Tree } from "https://esm.sh/@lezer/common"; @@ -77,9 +77,6 @@ export { export type { KeyBinding } from "https://esm.sh/@codemirror/view@6.3.0?external=@codemirror/state"; export { EditorSelection, EditorState, Text } from "@codemirror/state"; -export type { StateCommand, ChangeSpec } from "@codemirror/state"; +export type { ChangeSpec, StateCommand } from "@codemirror/state"; -export { DB as SQLite3 } from "https://deno.land/x/sqlite/mod.ts"; - -// @deno-types="https://deno.land/x/dex@1.0.2/types/index.d.ts" -export { default as Dex } from "https://deno.land/x/dex@1.0.2/mod.ts"; +export { Database as SQLite } from "https://deno.land/x/sqlite3@0.6.1/mod.ts"; diff --git a/packages/plugos/syscalls/esbuild.ts b/packages/plugos/syscalls/esbuild.ts index 35105b78..8289b22d 100644 --- a/packages/plugos/syscalls/esbuild.ts +++ b/packages/plugos/syscalls/esbuild.ts @@ -1,5 +1,5 @@ import { sandboxCompile, sandboxCompileModule } from "../compile"; -import { SysCallMapping } from "../system"; +import { SysCallMapping } from "../system.ts"; // TODO: FIgure out a better way to do this const builtinModules = ["yaml", "handlebars"]; @@ -9,14 +9,14 @@ export function esbuildSyscalls(): SysCallMapping { "tsc.analyze": async ( ctx, filename: string, - code: string + code: string, ): Promise => {}, "esbuild.compile": async ( ctx, filename: string, code: string, functionName?: string, - excludeModules: string[] = [] + excludeModules: string[] = [], ): Promise => { return await sandboxCompile( filename, @@ -24,12 +24,12 @@ export function esbuildSyscalls(): SysCallMapping { functionName, true, [], - [...builtinModules, ...excludeModules] + [...builtinModules, ...excludeModules], ); }, "esbuild.compileModule": async ( ctx, - moduleName: string + moduleName: string, ): Promise => { return await sandboxCompileModule(moduleName, builtinModules); }, diff --git a/packages/plugos/syscalls/store.knex_node.test.ts b/packages/plugos/syscalls/store.deno.test.ts similarity index 69% rename from packages/plugos/syscalls/store.knex_node.test.ts rename to packages/plugos/syscalls/store.deno.test.ts index b0bf66c5..4ed76596 100644 --- a/packages/plugos/syscalls/store.knex_node.test.ts +++ b/packages/plugos/syscalls/store.deno.test.ts @@ -1,18 +1,11 @@ -import { createSandbox } from "../environments/node_sandbox"; -import { expect, test } from "@jest/globals"; -import { System } from "../system"; -import { ensureTable, storeSyscalls } from "./store.knex_node"; -import knex from "knex"; -import fs from "fs/promises"; +import { assertEquals } from "../../../test_dep.ts"; +import { SQLite } from "../../../mod.ts"; +import { createSandbox } from "../environments/deno_sandbox.ts"; +import { System } from "../system.ts"; +import { ensureTable, storeSyscalls } from "./store.deno.ts"; -test("Test store", async () => { - const db = knex({ - client: "better-sqlite3", - connection: { - filename: "test.db", - }, - useNullAsDefault: true, - }); +Deno.test("Test store", async () => { + const db = new SQLite(":memory:"); await ensureTable(db, "test_table"); let system = new System("server"); let syscalls = storeSyscalls(db, "test_table"); @@ -33,9 +26,9 @@ test("Test store", async () => { }, }, }, - createSandbox + createSandbox, ); - expect(await plug.invoke("test1", [])).toBe("Pete"); + assertEquals(await plug.invoke("test1", []), "Pete"); await system.unloadAll(); let dummyCtx: any = {}; @@ -74,8 +67,8 @@ test("Test store", async () => { orderDesc: true, }); - expect(allRoberts.length).toBe(3); - expect(allRoberts[0].key).toBe("petesr"); + assertEquals(allRoberts.length, 3); + assertEquals(allRoberts[0].key, "petesr"); allRoberts = await syscalls["store.query"](dummyCtx, { filter: [{ op: "=", prop: "lastName", value: "Roberts" }], @@ -83,8 +76,8 @@ test("Test store", async () => { limit: 1, }); - expect(allRoberts.length).toBe(1); - expect(allRoberts[0].key).toBe("petejr"); + assertEquals(allRoberts.length, 1); + assertEquals(allRoberts[0].key, "petejr"); allRoberts = await syscalls["store.query"](dummyCtx, { filter: [ @@ -94,8 +87,8 @@ test("Test store", async () => { orderBy: "age", }); - expect(allRoberts.length).toBe(1); - expect(allRoberts[0].key).toBe("pete"); + assertEquals(allRoberts.length, 1); + assertEquals(allRoberts[0].key, "pete"); // Delete the middle one @@ -107,9 +100,7 @@ test("Test store", async () => { }); allRoberts = await syscalls["store.query"](dummyCtx, {}); - expect(allRoberts.length).toBe(2); + assertEquals(allRoberts.length, 2); - await db.destroy(); - - await fs.unlink("test.db"); + db.close(); }); diff --git a/packages/plugos/syscalls/store.dex_deno.ts b/packages/plugos/syscalls/store.deno.ts similarity index 53% rename from packages/plugos/syscalls/store.dex_deno.ts rename to packages/plugos/syscalls/store.deno.ts index f366e8fc..5ca0894e 100644 --- a/packages/plugos/syscalls/store.dex_deno.ts +++ b/packages/plugos/syscalls/store.deno.ts @@ -1,8 +1,4 @@ -import type { QueryBuilder } from "https://deno.land/x/dex@1.0.2/types/index.d.ts"; -import { RowObject } from "https://deno.land/x/sqlite/mod.ts"; -import type { SQLite3 } from "../../../mod.ts"; - -import { Dex } from "../../../mod.ts"; +import { SQLite } from "../../../mod.ts"; import { SysCallMapping } from "../system.ts"; export type Item = { @@ -16,24 +12,16 @@ export type KV = { value: any; }; -const dex = Dex({ client: "sqlite3" }); - -export function ensureTable(db: SQLite3, tableName: string) { - const result = db.query<[string]>( +export function ensureTable(db: SQLite, tableName: string) { + const stmt = db.prepare( `SELECT name FROM sqlite_master WHERE type='table' AND name=?`, - [tableName], ); + const result = stmt.all(tableName); if (result.length === 0) { - const createQuery = dex.schema.createTable(tableName, (table) => { - table.string("key"); - table.text("value"); - table.primary(["key"]); - }).toString(); - - db.query(createQuery); - + db.exec(`CREATE TABLE ${tableName} (key STRING PRIMARY KEY, value TEXT);`); console.log(`Created table ${tableName}`); } + return Promise.resolve(); } export type Query = { @@ -50,95 +38,110 @@ export type Filter = { value: any; }; -export function queryToKnex( - queryBuilder: QueryBuilder, +export function queryToSql( query: Query, -): QueryBuilder { +): { sql: string; params: any[] } { + const whereClauses: string[] = []; + const clauses: string[] = []; + const params: any[] = []; if (query.filter) { for (const filter of query.filter) { - queryBuilder = queryBuilder.andWhereRaw( + whereClauses.push( `json_extract(value, '$.${filter.prop}') ${filter.op} ?`, - [filter.value], ); + params.push(filter.value); } } - if (query.limit) { - queryBuilder = queryBuilder.limit(query.limit); - } if (query.orderBy) { - queryBuilder = queryBuilder.orderByRaw( - `json_extract(value, '$.${query.orderBy}') ${ + clauses.push( + `ORDER BY json_extract(value, '$.${query.orderBy}') ${ query.orderDesc ? "desc" : "asc" }`, ); } - return queryBuilder; + if (query.limit) { + clauses.push(`LIMIT ${query.limit}`); + } + return { + sql: whereClauses.length > 0 + ? `WHERE ${whereClauses.join(" AND ")} ${clauses.join(" ")}` + : clauses.join(" "), + params, + }; } -function asyncQuery( - db: SQLite3, - query: QueryBuilder, +function asyncQuery>( + db: SQLite, + query: string, + ...params: any[] ): Promise { - return Promise.resolve(db.queryEntries(query.toString())); + // console.log("Querying", query, params); + return Promise.resolve(db.prepare(query).all(params)); } function asyncExecute( - db: SQLite3, - query: QueryBuilder, -): Promise { - return Promise.resolve(db.execute(query.toString())); + db: SQLite, + query: string, + ...params: any[] +): Promise { + // console.log("Exdecting", query, params); + return Promise.resolve(db.exec(query, params)); } export function storeSyscalls( - db: SQLite3, + db: SQLite, tableName: string, ): SysCallMapping { const apiObj: SysCallMapping = { "store.delete": async (_ctx, key: string) => { - await asyncExecute(db, dex(tableName).where({ key }).del()); + await asyncExecute(db, `DELETE FROM ${tableName} WHERE key = ?`, key); }, "store.deletePrefix": async (_ctx, prefix: string) => { await asyncExecute( db, - dex(tableName).whereRaw(`"key" LIKE "${prefix}%"`).del(), + `DELETE FROM ${tableName} WHERE key LIKE "?%"`, + prefix, ); }, "store.deleteQuery": async (_ctx, query: Query) => { - await asyncExecute(db, queryToKnex(dex(tableName), query).del()); + const { sql, params } = queryToSql(query); + await asyncExecute(db, `DELETE FROM ${tableName} ${sql}`, ...params); }, "store.deleteAll": async () => { - await asyncExecute(db, dex(tableName).del()); + await asyncExecute(db, `DELETE FROM ${tableName}`); }, "store.set": async (_ctx, key: string, value: any) => { await asyncExecute( db, - dex(tableName).where({ key }).update("value", JSON.stringify(value)), + `UPDATE ${tableName} SET value = ? WHERE key = ?`, + JSON.stringify(value), + key, ); if (db.changes === 0) { await asyncExecute( db, - dex(tableName).insert({ - key, - value: JSON.stringify(value), - }), + `INSERT INTO ${tableName} (key, value) VALUES (?, ?)`, + key, + JSON.stringify(value), ); } }, // TODO: Optimize "store.batchSet": async (ctx, kvs: KV[]) => { - for (let { key, value } of kvs) { + for (const { key, value } of kvs) { await apiObj["store.set"](ctx, key, value); } }, "store.batchDelete": async (ctx, keys: string[]) => { - for (let key of keys) { + for (const key of keys) { await apiObj["store.delete"](ctx, key); } }, "store.get": async (_ctx, key: string): Promise => { const result = await asyncQuery( db, - dex(tableName).where({ key }).select("value"), + `SELECT value FROM ${tableName} WHERE key = ?`, + key, ); if (result.length) { return JSON.parse(result[0].value); @@ -146,13 +149,12 @@ export function storeSyscalls( return null; } }, - "store.queryPrefix": async (ctx, prefix: string) => { + "store.queryPrefix": async (_ctx, prefix: string) => { return ( await asyncQuery( db, - dex(tableName) - .andWhereRaw(`"key" LIKE "${prefix}%"`) - .select("key", "value"), + `SELECT key, value FROM ${tableName} WHERE key LIKE "?%"`, + prefix, ) ).map(({ key, value }) => ({ key, @@ -160,10 +162,12 @@ export function storeSyscalls( })); }, "store.query": async (_ctx, query: Query) => { + const { sql, params } = queryToSql(query); return ( await asyncQuery( db, - queryToKnex(dex(tableName), query).select("key", "value"), + `SELECT key, value FROM ${tableName} ${sql}`, + ...params, ) ).map(({ key, value }: { key: string; value: string }) => ({ key, diff --git a/packages/plugos/syscalls/store.dex_deno.test.ts b/packages/plugos/syscalls/store.dex_deno.test.ts deleted file mode 100644 index 108f1e71..00000000 --- a/packages/plugos/syscalls/store.dex_deno.test.ts +++ /dev/null @@ -1,9 +0,0 @@ -import { SQLite3 } from "../../../mod.ts"; -import { storeSyscalls } from "./store.dex_deno.ts"; - -Deno.test("store.dex", async () => { - const db = new SQLite3(":memory:"); - const syscalls = storeSyscalls(db, "test"); - const fakeCtx = {} as any; - await syscalls["store.put"](fakeCtx, "key", { value: "value" }); -}); diff --git a/packages/plugos/syscalls/store.knex_node.ts b/packages/plugos/syscalls/store.knex_node.ts deleted file mode 100644 index f05b7f68..00000000 --- a/packages/plugos/syscalls/store.knex_node.ts +++ /dev/null @@ -1,133 +0,0 @@ -import { Knex } from "knex"; -import { SysCallMapping } from "../system"; - -export type Item = { - page: string; - key: string; - value: any; -}; - -export type KV = { - key: string; - value: any; -}; - -export async function ensureTable(db: Knex, tableName: string) { - if (!(await db.schema.hasTable(tableName))) { - await db.schema.createTable(tableName, (table) => { - table.string("key"); - table.text("value"); - table.primary(["key"]); - }); - - console.log(`Created table ${tableName}`); - } -} - -export type Query = { - filter?: Filter[]; - orderBy?: string; - orderDesc?: boolean; - limit?: number; - select?: string[]; -}; - -export type Filter = { - op: string; - prop: string; - value: any; -}; - -export function queryToKnex( - queryBuilder: Knex.QueryBuilder, - query: Query -): Knex.QueryBuilder { - if (query.filter) { - for (let filter of query.filter) { - queryBuilder = queryBuilder.andWhereRaw( - `json_extract(value, '$.${filter.prop}') ${filter.op} ?`, - [filter.value] - ); - } - } - if (query.limit) { - queryBuilder = queryBuilder.limit(query.limit); - } - if (query.orderBy) { - queryBuilder = queryBuilder.orderByRaw( - `json_extract(value, '$.${query.orderBy}') ${ - query.orderDesc ? "desc" : "asc" - }` - ); - } - return queryBuilder; -} - -export function storeSyscalls( - db: Knex, - tableName: string -): SysCallMapping { - const apiObj: SysCallMapping = { - "store.delete": async (ctx, key: string) => { - await db(tableName).where({ key }).del(); - }, - "store.deletePrefix": async (ctx, prefix: string) => { - return db(tableName).andWhereLike("key", `${prefix}%`).del(); - }, - "store.deleteQuery": async (ctx, query: Query) => { - await queryToKnex(db(tableName), query).del(); - }, - "store.deleteAll": async (ctx) => { - await db(tableName).del(); - }, - "store.set": async (ctx, key: string, value: any) => { - let changed = await db(tableName) - .where({ key }) - .update("value", JSON.stringify(value)); - if (changed === 0) { - await db(tableName).insert({ - key, - value: JSON.stringify(value), - }); - } - }, - // TODO: Optimize - "store.batchSet": async (ctx, kvs: KV[]) => { - for (let { key, value } of kvs) { - await apiObj["store.set"](ctx, key, value); - } - }, - "store.batchDelete": async (ctx, keys: string[]) => { - for (let key of keys) { - await apiObj["store.delete"](ctx, key); - } - }, - "store.get": async (ctx, key: string): Promise => { - let result = await db(tableName).where({ key }).select("value"); - if (result.length) { - return JSON.parse(result[0].value); - } else { - return null; - } - }, - "store.queryPrefix": async (ctx, prefix: string) => { - return ( - await db(tableName) - .andWhereLike("key", `${prefix}%`) - .select("key", "value") - ).map(({ key, value }) => ({ - key, - value: JSON.parse(value), - })); - }, - "store.query": async (ctx, query: Query) => { - return ( - await queryToKnex(db(tableName), query).select("key", "value") - ).map(({ key, value }: { key: string; value: string }) => ({ - key, - value: JSON.parse(value), - })); - }, - }; - return apiObj; -} diff --git a/packages/plugos/tsconfig.json b/packages/plugos/tsconfig.json deleted file mode 100644 index 386241d6..00000000 --- a/packages/plugos/tsconfig.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "include": ["bin/*", "environments/*", "hooks/*", "syscalls/*", "*"], - "compilerOptions": { - "target": "esnext", - "strict": true, - "moduleResolution": "node", - "module": "esnext", - "esModuleInterop": true, - "allowSyntheticDefaultImports": true, - "resolveJsonModule": true, - "jsx": "react-jsx", - "downlevelIteration": true - } -} diff --git a/test.db b/test.db new file mode 100644 index 0000000000000000000000000000000000000000..bc854e203f5984b95d617f5af7d7d622373e9683 GIT binary patch literal 12288 zcmeI$%}T>S5C`ztRO$zm2qok?)PoHaMWq+LSS7b!)ZVjeT``zgO}C{*Jotn@hOeME zXG?qTx$+-&vYE{$kl$_R?xvqR%a5a~G>*?`pOm6YW+DnCYm%!_aBqwUG-Deeo1p%Kij$Hsi}rfrfQ#dyGe)j)D#n|+1Z)v@N1Qqrkb;CTy9U2!=N5t zM2hm42Rj?T6>_%E%*2frb>=^vo%pLi*Y9}GhNKF=Nd^J}5P$##AOHafKmY;|fB*y_ z0D;XGh*hn5ERBNq)}n>bI$zEod-7i@{3RI(2tWV=5P$##AOHafKmY;|fB*#kn?SwR I+?9sFAG_s4wg3PC literal 0 HcmV?d00001 diff --git a/test_dep.ts b/test_dep.ts new file mode 100644 index 00000000..42f1d284 --- /dev/null +++ b/test_dep.ts @@ -0,0 +1,12 @@ +export { + assert, + assertEquals, + AssertionError, + assertMatch, + assertNotEquals, + assertThrows, + equal, + fail, + unimplemented, + unreachable, +} from "https://deno.land/std@0.158.0/testing/asserts.ts";