From b2cd68f82c6702aae4801f5fb091db8bd9908e01 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Fri, 12 Jan 2024 17:44:25 +0100 Subject: [PATCH] Doing some fancy caching now --- cmd/server.ts | 4 - common/limited_map.test.ts | 9 +- common/limited_map.ts | 45 +- common/manifest.ts | 2 - common/space_index.ts | 6 +- .../spaces/datastore_space_primitives.test.ts | 4 +- plug-api/types.ts | 5 + plugos/hooks/endpoint.test.ts | 48 -- plugos/hooks/endpoint.ts | 140 ---- plugos/lib/datastore.test.ts | 4 +- plugos/lib/datastore.ts | 90 +-- plugos/lib/kv_datastore.ts | 88 +++ plugos/lib/mq.datastore.test.ts | 4 +- plugos/lib/mq.datastore.ts | 4 +- plugos/manifest_cache.ts | 18 +- plugos/syscalls/datastore.ts | 4 +- plugos/test.plug.yaml | 5 +- plugos/test_func.test.ts | 10 - plugs/editor/complete.ts | 4 +- server/deps.ts | 19 +- server/http_server.ts | 655 ++++++++---------- server/instance.ts | 9 +- web/client.ts | 11 +- web/client_system.ts | 8 +- web/remote_datastore.ts | 39 +- web/service_worker.ts | 3 +- web/space.ts | 3 - 27 files changed, 518 insertions(+), 723 deletions(-) delete mode 100644 plugos/hooks/endpoint.test.ts delete mode 100644 plugos/hooks/endpoint.ts create mode 100644 plugos/lib/kv_datastore.ts diff --git a/cmd/server.ts b/cmd/server.ts index 4356acb7..d5fb0aa2 100644 --- a/cmd/server.ts +++ b/cmd/server.ts @@ -1,4 +1,3 @@ -import { Application } from "../server/deps.ts"; import { HttpServer } from "../server/http_server.ts"; import clientAssetBundle from "../dist/client_asset_bundle.json" assert { type: "json", @@ -46,8 +45,6 @@ export async function serveCommand( console.log("Running in sync-only mode (no backend processing)"); } - const app = new Application(); - if (!folder) { // Didn't get a folder as an argument, check if we got it as an environment variable folder = Deno.env.get("SB_FOLDER"); @@ -90,7 +87,6 @@ export async function serveCommand( }); const httpServer = new HttpServer({ - app, hostname, port, clientAssetBundle: new AssetBundle(clientAssetBundle as AssetJson), diff --git a/common/limited_map.test.ts b/common/limited_map.test.ts index 2014d359..50901230 100644 --- a/common/limited_map.test.ts +++ b/common/limited_map.test.ts @@ -5,7 +5,7 @@ import { LimitedMap } from "./limited_map.ts"; Deno.test("limited map", async () => { const mp = new LimitedMap(3); mp.set("a", "a"); - mp.set("b", "b"); + mp.set("b", "b", 10); mp.set("c", "c"); await sleep(2); assertEquals(mp.get("a"), "a"); @@ -15,6 +15,11 @@ Deno.test("limited map", async () => { assertEquals(mp.get("c"), "c"); // Drops the first key mp.set("d", "d"); - await sleep(2); assertEquals(mp.get("a"), undefined); + await sleep(20); + // "b" should have been dropped + assertEquals(mp.get("b"), undefined); + assertEquals(mp.get("c"), "c"); + + console.log(mp.toJSON()); }); diff --git a/common/limited_map.ts b/common/limited_map.ts index 4561cef6..3ba22594 100644 --- a/common/limited_map.ts +++ b/common/limited_map.ts @@ -1,20 +1,36 @@ -type LimitedMapRecord = Record; +type LimitedMapRecord = { value: V; la: number }; export class LimitedMap { - constructor(private maxSize: number, private map: LimitedMapRecord = {}) { + private map: Map>; + + constructor( + private maxSize: number, + initialJson: Record> = {}, + ) { + this.map = new Map(Object.entries(initialJson)); } - set(key: string, value: V) { - if (Object.keys(this.map).length >= this.maxSize) { + /** + * @param key + * @param value + * @param ttl time to live (in ms) + */ + set(key: string, value: V, ttl?: number) { + if (ttl) { + setTimeout(() => { + this.map.delete(key); + }, ttl); + } + if (this.map.size >= this.maxSize) { // Remove the oldest key before adding a new one const oldestKey = this.getOldestKey(); - delete this.map[oldestKey!]; + this.map.delete(oldestKey!); } - this.map[key] = { value, la: Date.now() }; + this.map.set(key, { value, la: Date.now() }); } get(key: string): V | undefined { - const entry = this.map[key]; + const entry = this.map.get(key); if (entry) { // Update the last accessed timestamp entry.la = Date.now(); @@ -24,24 +40,21 @@ export class LimitedMap { } remove(key: string) { - delete this.map[key]; + this.map.delete(key); } toJSON() { - return this.map; + return Object.fromEntries(this.map.entries()); } private getOldestKey(): string | undefined { let oldestKey: string | undefined; let oldestTimestamp: number | undefined; - for (const key in this.map) { - if (Object.prototype.hasOwnProperty.call(this.map, key)) { - const entry = this.map[key]; - if (!oldestTimestamp || entry.la < oldestTimestamp) { - oldestKey = key; - oldestTimestamp = entry.la; - } + for (const [key, entry] of this.map.entries()) { + if (!oldestTimestamp || entry.la < oldestTimestamp) { + oldestKey = key; + oldestTimestamp = entry.la; } } diff --git a/common/manifest.ts b/common/manifest.ts index 278de77b..7ef2ff14 100644 --- a/common/manifest.ts +++ b/common/manifest.ts @@ -6,7 +6,6 @@ import { SlashCommandHookT } from "../web/hooks/slash_command.ts"; import { PlugNamespaceHookT } from "./hooks/plug_namespace.ts"; import { CodeWidgetT } from "../web/hooks/code_widget.ts"; import { MQHookT } from "../plugos/hooks/mq.ts"; -import { EndpointHookT } from "../plugos/hooks/endpoint.ts"; import { PanelWidgetT } from "../web/hooks/panel_widget.ts"; /** Silverbullet hooks give plugs access to silverbullet core systems. @@ -24,7 +23,6 @@ export type SilverBulletHooks = & EventHookT & CodeWidgetT & PanelWidgetT - & EndpointHookT & PlugNamespaceHookT; /** Syntax extension allow plugs to declaratively add new *inline* parse tree nodes to the markdown parser. */ diff --git a/common/space_index.ts b/common/space_index.ts index 4df2beef..c0fc9b6d 100644 --- a/common/space_index.ts +++ b/common/space_index.ts @@ -1,4 +1,4 @@ -import { IDataStore } from "../plugos/lib/datastore.ts"; +import { DataStore } from "../plugos/lib/datastore.ts"; import { System } from "../plugos/system.ts"; const indexVersionKey = ["$indexVersion"]; @@ -8,7 +8,7 @@ const desiredIndexVersion = 1; let indexOngoing = false; -export async function ensureSpaceIndex(ds: IDataStore, system: System) { +export async function ensureSpaceIndex(ds: DataStore, system: System) { const currentIndexVersion = await ds.get(indexVersionKey); console.info("Current space index version", currentIndexVersion); @@ -25,6 +25,6 @@ export async function ensureSpaceIndex(ds: IDataStore, system: System) { } } -export async function markFullSpaceIndexComplete(ds: IDataStore) { +export async function markFullSpaceIndexComplete(ds: DataStore) { await ds.set(indexVersionKey, desiredIndexVersion); } diff --git a/common/spaces/datastore_space_primitives.test.ts b/common/spaces/datastore_space_primitives.test.ts index 0222d4fc..0da093e3 100644 --- a/common/spaces/datastore_space_primitives.test.ts +++ b/common/spaces/datastore_space_primitives.test.ts @@ -1,8 +1,8 @@ import "https://esm.sh/fake-indexeddb@4.0.2/auto"; -import { DataStore } from "../../plugos/lib/datastore.ts"; import { IndexedDBKvPrimitives } from "../../plugos/lib/indexeddb_kv_primitives.ts"; import { DataStoreSpacePrimitives } from "./datastore_space_primitives.ts"; import { testSpacePrimitives } from "./space_primitives.test.ts"; +import { KvDataStore } from "../../plugos/lib/kv_datastore.ts"; Deno.test("DataStoreSpacePrimitives", { sanitizeResources: false, @@ -11,7 +11,7 @@ Deno.test("DataStoreSpacePrimitives", { const db = new IndexedDBKvPrimitives("test"); await db.init(); - const space = new DataStoreSpacePrimitives(new DataStore(db)); + const space = new DataStoreSpacePrimitives(new KvDataStore(db)); await testSpacePrimitives(space); db.close(); }); diff --git a/plug-api/types.ts b/plug-api/types.ts index 0786cde9..3d6e8152 100644 --- a/plug-api/types.ts +++ b/plug-api/types.ts @@ -73,6 +73,11 @@ export type Query = { render?: string; renderAll?: boolean; distinct?: boolean; + + /** + * When set, the DS implementation _may_ cache the result for the given number of seconds. + */ + cacheSecs?: number; }; export type KvQuery = Omit & { diff --git a/plugos/hooks/endpoint.test.ts b/plugos/hooks/endpoint.test.ts deleted file mode 100644 index 4bdaa471..00000000 --- a/plugos/hooks/endpoint.test.ts +++ /dev/null @@ -1,48 +0,0 @@ -import { createSandbox } from "../environments/deno_sandbox.ts"; -import { EndpointHook, EndpointHookT } from "./endpoint.ts"; -import { System } from "../system.ts"; - -import { Application } from "../../server/deps.ts"; -import { assertEquals } from "../../test_deps.ts"; -import { compileManifest } from "../compile.ts"; -import { esbuild } from "../deps.ts"; - -Deno.test("Run a plugos endpoint server", async () => { - const tempDir = await Deno.makeTempDir(); - const system = new System("server"); - - const workerPath = await compileManifest( - new URL("../test.plug.yaml", import.meta.url).pathname, - tempDir, - ); - - await system.load( - new URL(`file://${workerPath}`), - "test", - 0, - createSandbox, - ); - - const app = new Application(); - const port = 3123; - - const endpointHook = new EndpointHook("/_/"); - - app.use((context, next) => { - return endpointHook.handleRequest(system, context, next); - }); - - const controller = new AbortController(); - app.listen({ port: port, signal: controller.signal }); - - const res = await fetch(`http://localhost:${port}/_/test/?name=Pete`); - assertEquals(res.status, 200); - assertEquals(res.headers.get("Content-type"), "application/json"); - assertEquals(await res.json(), [1, 2, 3]); - console.log("Aborting"); - controller.abort(); - await system.unloadAll(); - - await Deno.remove(tempDir, { recursive: true }); - esbuild.stop(); -}); diff --git a/plugos/hooks/endpoint.ts b/plugos/hooks/endpoint.ts deleted file mode 100644 index 1523f6c9..00000000 --- a/plugos/hooks/endpoint.ts +++ /dev/null @@ -1,140 +0,0 @@ -import { Hook, Manifest } from "../types.ts"; -import { System } from "../system.ts"; -import { Application, Context, Next } from "../../server/deps.ts"; - -export type EndpointRequest = { - method: string; - path: string; - query: { [key: string]: string }; - headers: { [key: string]: string }; - body: any; -}; - -export type EndpointResponse = { - status: number; - headers?: { [key: string]: string }; - body: any; -}; - -export type EndpointHookT = { - http?: EndPointDef | EndPointDef[]; -}; - -export type EndPointDef = { - method?: "GET" | "POST" | "PUT" | "DELETE" | "HEAD" | "OPTIONS" | "ANY"; - path: string; -}; - -export class EndpointHook implements Hook { - readonly prefix: string; - - constructor(prefix: string) { - this.prefix = prefix; - } - - public async handleRequest( - system: System, - ctx: Context, - next: Next, - ) { - const req = ctx.request; - const requestPath = ctx.request.url.pathname; - if (!requestPath.startsWith(this.prefix)) { - return next(); - } - console.log("Endpoint request", requestPath); - // Iterate over all loaded plugins - for (const [plugName, plug] of system.loadedPlugs.entries()) { - const manifest = plug.manifest; - if (!manifest) { - continue; - } - const functions = manifest.functions; - // console.log("Checking plug", plugName); - const prefix = `${this.prefix}${plugName}`; - if (!requestPath.startsWith(prefix)) { - continue; - } - for (const [name, functionDef] of Object.entries(functions)) { - if (!functionDef.http) { - continue; - } - // console.log("Got config", functionDef); - const endpoints = Array.isArray(functionDef.http) - ? functionDef.http - : [functionDef.http]; - // console.log(endpoints); - for (const { path, method } of endpoints) { - const prefixedPath = `${prefix}${path}`; - if ( - prefixedPath === requestPath && - ((method || "GET") === req.method || method === "ANY") - ) { - try { - const response: EndpointResponse = await plug.invoke(name, [ - { - path: req.url.pathname, - method: req.method, - body: req.body(), - query: Object.fromEntries( - req.url.searchParams.entries(), - ), - headers: Object.fromEntries(req.headers.entries()), - } as EndpointRequest, - ]); - if (response.headers) { - for ( - const [key, value] of Object.entries( - response.headers, - ) - ) { - ctx.response.headers.set(key, value); - } - } - ctx.response.status = response.status; - ctx.response.body = response.body; - // console.log("Sent result"); - return; - } catch (e: any) { - console.error("Error executing function", e); - ctx.response.status = 500; - ctx.response.body = e.message; - return; - } - } - } - } - } - // console.log("Shouldn't get here"); - await next(); - } - - apply(): void { - } - - validateManifest(manifest: Manifest): string[] { - const errors = []; - for (const functionDef of Object.values(manifest.functions)) { - if (!functionDef.http) { - continue; - } - const endpoints = Array.isArray(functionDef.http) - ? functionDef.http - : [functionDef.http]; - for (const { path, method } of endpoints) { - if (!path) { - errors.push("Path not defined for endpoint"); - } - if ( - method && - ["GET", "POST", "PUT", "DELETE", "ANY"].indexOf(method) === -1 - ) { - errors.push( - `Invalid method ${method} for end point with with ${path}`, - ); - } - } - } - return errors; - } -} diff --git a/plugos/lib/datastore.test.ts b/plugos/lib/datastore.test.ts index 15ca0e75..216638e1 100644 --- a/plugos/lib/datastore.test.ts +++ b/plugos/lib/datastore.test.ts @@ -1,13 +1,13 @@ import "https://esm.sh/fake-indexeddb@4.0.2/auto"; import { IndexedDBKvPrimitives } from "./indexeddb_kv_primitives.ts"; -import { DataStore } from "./datastore.ts"; import { DenoKvPrimitives } from "./deno_kv_primitives.ts"; import { KvPrimitives } from "./kv_primitives.ts"; import { assertEquals } from "https://deno.land/std@0.165.0/testing/asserts.ts"; import { PrefixedKvPrimitives } from "./prefixed_kv_primitives.ts"; +import { KvDataStore } from "./kv_datastore.ts"; async function test(db: KvPrimitives) { - const datastore = new DataStore(new PrefixedKvPrimitives(db, ["ds"]), { + const datastore = new KvDataStore(new PrefixedKvPrimitives(db, ["ds"]), { count: (arr: any[]) => arr.length, }); await datastore.set(["user", "peter"], { name: "Peter" }); diff --git a/plugos/lib/datastore.ts b/plugos/lib/datastore.ts index 5491b63c..17e659c4 100644 --- a/plugos/lib/datastore.ts +++ b/plugos/lib/datastore.ts @@ -1,13 +1,10 @@ -import { applyQueryNoFilterKV, evalQueryExpression } from "$sb/lib/query.ts"; -import { FunctionMap, KV, KvKey, KvQuery } from "$sb/types.ts"; -import { builtinFunctions } from "$sb/lib/builtin_query_functions.ts"; -import { KvPrimitives } from "./kv_primitives.ts"; +import { KV, KvKey, KvQuery } from "$sb/types.ts"; /** * This is the data store class you'll actually want to use, wrapping the primitives * in a more user-friendly way */ -export interface IDataStore { +export interface DataStore { get(key: KvKey): Promise; batchGet(keys: KvKey[]): Promise<(T | null)[]>; set(key: KvKey, value: any): Promise; @@ -17,86 +14,3 @@ export interface IDataStore { query(query: KvQuery): Promise[]>; queryDelete(query: KvQuery): Promise; } - -export class DataStore implements IDataStore { - constructor( - readonly kv: KvPrimitives, - private functionMap: FunctionMap = builtinFunctions, - ) { - } - - async get(key: KvKey): Promise { - return (await this.batchGet([key]))[0]; - } - - batchGet(keys: KvKey[]): Promise<(T | null)[]> { - return this.kv.batchGet(keys); - } - - set(key: KvKey, value: any): Promise { - return this.batchSet([{ key, value }]); - } - - batchSet(entries: KV[]): Promise { - const allKeyStrings = new Set(); - const uniqueEntries: KV[] = []; - for (const { key, value } of entries) { - const keyString = JSON.stringify(key); - if (allKeyStrings.has(keyString)) { - console.warn(`Duplicate key ${keyString} in batchSet, skipping`); - } else { - allKeyStrings.add(keyString); - uniqueEntries.push({ key, value }); - } - } - return this.kv.batchSet(uniqueEntries); - } - - delete(key: KvKey): Promise { - return this.batchDelete([key]); - } - - batchDelete(keys: KvKey[]): Promise { - return this.kv.batchDelete(keys); - } - - async query(query: KvQuery): Promise[]> { - const results: KV[] = []; - let itemCount = 0; - // Accumulate results - let limit = Infinity; - if (query.limit) { - limit = evalQueryExpression(query.limit, {}, this.functionMap); - } - for await ( - const entry of this.kv.query(query) - ) { - // Filter - if ( - query.filter && - !evalQueryExpression(query.filter, entry.value, this.functionMap) - ) { - continue; - } - results.push(entry); - itemCount++; - // Stop when the limit has been reached - if (itemCount === limit && !query.orderBy) { - // Only break when not also ordering in which case we need all results - break; - } - } - // Apply order by, limit, and select - return applyQueryNoFilterKV(query, results, this.functionMap); - } - - async queryDelete(query: KvQuery): Promise { - const keys: KvKey[] = []; - for ( - const { key } of await this.query(query) - ) { - keys.push(key); - } - return this.batchDelete(keys); - } -} diff --git a/plugos/lib/kv_datastore.ts b/plugos/lib/kv_datastore.ts new file mode 100644 index 00000000..c02053fe --- /dev/null +++ b/plugos/lib/kv_datastore.ts @@ -0,0 +1,88 @@ +import { FunctionMap, KV, KvKey, KvQuery } from "$sb/types.ts"; +import { builtinFunctions } from "$sb/lib/builtin_query_functions.ts"; +import { DataStore } from "./datastore.ts"; +import { KvPrimitives } from "./kv_primitives.ts"; +import { applyQueryNoFilterKV, evalQueryExpression } from "$sb/lib/query.ts"; + +export class KvDataStore implements DataStore { + constructor( + readonly kv: KvPrimitives, + private functionMap: FunctionMap = builtinFunctions, + ) { + } + + async get(key: KvKey): Promise { + return (await this.batchGet([key]))[0]; + } + + batchGet(keys: KvKey[]): Promise<(T | null)[]> { + return this.kv.batchGet(keys); + } + + set(key: KvKey, value: any): Promise { + return this.batchSet([{ key, value }]); + } + + batchSet(entries: KV[]): Promise { + const allKeyStrings = new Set(); + const uniqueEntries: KV[] = []; + for (const { key, value } of entries) { + const keyString = JSON.stringify(key); + if (allKeyStrings.has(keyString)) { + console.warn(`Duplicate key ${keyString} in batchSet, skipping`); + } else { + allKeyStrings.add(keyString); + uniqueEntries.push({ key, value }); + } + } + return this.kv.batchSet(uniqueEntries); + } + + delete(key: KvKey): Promise { + return this.batchDelete([key]); + } + + batchDelete(keys: KvKey[]): Promise { + return this.kv.batchDelete(keys); + } + + async query(query: KvQuery): Promise[]> { + const results: KV[] = []; + let itemCount = 0; + // Accumulate results + let limit = Infinity; + if (query.limit) { + limit = evalQueryExpression(query.limit, {}, this.functionMap); + } + for await ( + const entry of this.kv.query(query) + ) { + // Filter + if ( + query.filter && + !evalQueryExpression(query.filter, entry.value, this.functionMap) + ) { + continue; + } + results.push(entry); + itemCount++; + // Stop when the limit has been reached + if (itemCount === limit && !query.orderBy) { + // Only break when not also ordering in which case we need all results + break; + } + } + // Apply order by, limit, and select + return applyQueryNoFilterKV(query, results, this.functionMap); + } + + async queryDelete(query: KvQuery): Promise { + const keys: KvKey[] = []; + for ( + const { key } of await this.query(query) + ) { + keys.push(key); + } + return this.batchDelete(keys); + } +} diff --git a/plugos/lib/mq.datastore.test.ts b/plugos/lib/mq.datastore.test.ts index dca3171e..1a17fcdc 100644 --- a/plugos/lib/mq.datastore.test.ts +++ b/plugos/lib/mq.datastore.test.ts @@ -2,15 +2,15 @@ import { DataStoreMQ } from "./mq.datastore.ts"; import { assertEquals } from "../../test_deps.ts"; import { sleep } from "$sb/lib/async.ts"; import { DenoKvPrimitives } from "./deno_kv_primitives.ts"; -import { DataStore } from "./datastore.ts"; import { PrefixedKvPrimitives } from "./prefixed_kv_primitives.ts"; +import { KvDataStore } from "./kv_datastore.ts"; Deno.test("DataStore MQ", async () => { const tmpFile = await Deno.makeTempFile(); const db = new DenoKvPrimitives(await Deno.openKv(tmpFile)); const mq = new DataStoreMQ( - new DataStore(new PrefixedKvPrimitives(db, ["mq"])), + new KvDataStore(new PrefixedKvPrimitives(db, ["mq"])), ); await mq.send("test", "Hello World"); let messages = await mq.poll("test", 10); diff --git a/plugos/lib/mq.datastore.ts b/plugos/lib/mq.datastore.ts index bc2da069..3de3d300 100644 --- a/plugos/lib/mq.datastore.ts +++ b/plugos/lib/mq.datastore.ts @@ -1,6 +1,6 @@ import { KV, MQMessage, MQStats, MQSubscribeOptions } from "$sb/types.ts"; import { MessageQueue } from "./mq.ts"; -import { IDataStore } from "./datastore.ts"; +import { DataStore } from "./datastore.ts"; export type ProcessingMessage = MQMessage & { ts: number; @@ -15,7 +15,7 @@ export class DataStoreMQ implements MessageQueue { localSubscriptions = new Map void>>(); constructor( - private ds: IDataStore, + private ds: DataStore, ) { } diff --git a/plugos/manifest_cache.ts b/plugos/manifest_cache.ts index fa810b95..b71c1338 100644 --- a/plugos/manifest_cache.ts +++ b/plugos/manifest_cache.ts @@ -1,4 +1,4 @@ -import { KvPrimitives } from "./lib/kv_primitives.ts"; +import { DataStore } from "./lib/datastore.ts"; import { Plug } from "./plug.ts"; import { Manifest } from "./types.ts"; @@ -7,25 +7,21 @@ export interface ManifestCache { } export class KVPrimitivesManifestCache implements ManifestCache { - constructor(private kv: KvPrimitives, private manifestPrefix: string) { + constructor(private ds: DataStore, private manifestPrefix: string) { } async getManifest(plug: Plug, hash: number): Promise> { - const [cached] = await this.kv.batchGet([[ - this.manifestPrefix, - plug.name, - ]]); + const cached = await this.ds.get([this.manifestPrefix, plug.name]); if (cached && cached.hash === hash) { // console.log("Using KV cached manifest for", plug.name); return cached.manifest; } await plug.sandbox.init(); const manifest = plug.sandbox.manifest!; - await this.kv.batchSet([{ - key: [this.manifestPrefix, plug.name], - // Deliverately removing the assets from the manifest to preserve space, will be re-added upon load of actual worker - value: { manifest: { ...manifest, assets: undefined }, hash }, - }]); + await this.ds.set([this.manifestPrefix, plug.name], { + manifest: { ...manifest, assets: undefined }, + hash, + }); return manifest; } } diff --git a/plugos/syscalls/datastore.ts b/plugos/syscalls/datastore.ts index bd60be05..83f7efa9 100644 --- a/plugos/syscalls/datastore.ts +++ b/plugos/syscalls/datastore.ts @@ -1,5 +1,5 @@ import { KV, KvKey, KvQuery } from "$sb/types.ts"; -import type { IDataStore } from "../lib/datastore.ts"; +import type { DataStore } from "../lib/datastore.ts"; import type { SyscallContext, SysCallMapping } from "../system.ts"; /** @@ -8,7 +8,7 @@ import type { SyscallContext, SysCallMapping } from "../system.ts"; * @param prefix prefix to scope all keys to to which the plug name will be appended */ export function dataStoreSyscalls( - ds: IDataStore, + ds: DataStore, ): SysCallMapping { return { "datastore.delete": (ctx, key: KvKey) => { diff --git a/plugos/test.plug.yaml b/plugos/test.plug.yaml index 581a3294..4ae2092d 100644 --- a/plugos/test.plug.yaml +++ b/plugos/test.plug.yaml @@ -2,7 +2,4 @@ name: test functions: boot: path: "./test_func.test.ts:hello" - endpoint: - path: "./test_func.test.ts:endpoint" - http: - path: "/" + \ No newline at end of file diff --git a/plugos/test_func.test.ts b/plugos/test_func.test.ts index 6471a19e..3f668a08 100644 --- a/plugos/test_func.test.ts +++ b/plugos/test_func.test.ts @@ -1,17 +1,7 @@ import * as YAML from "https://deno.land/std@0.184.0/yaml/mod.ts"; -import { EndpointRequest, EndpointResponse } from "./hooks/endpoint.ts"; export function hello() { console.log(YAML.stringify({ hello: "world" })); return "hello"; } - -export function endpoint(req: EndpointRequest): EndpointResponse { - console.log("Req", req); - return { - status: 200, - body: [1, 2, 3], - headers: { "Content-type": "application/json" }, - }; -} diff --git a/plugs/editor/complete.ts b/plugs/editor/complete.ts index 41f8c94e..5fd0e03c 100644 --- a/plugs/editor/complete.ts +++ b/plugs/editor/complete.ts @@ -18,7 +18,9 @@ export async function pageComplete(completeEvent: CompleteEvent) { completeEvent.linePrefix, ); const tagToQuery = isInTemplateContext ? "template" : "page"; - let allPages: PageMeta[] = await queryObjects(tagToQuery, {}); + let allPages: PageMeta[] = await queryObjects(tagToQuery, { + cacheSecs: 5, + }); const prefix = match[1]; if (prefix.startsWith("!")) { // Federation prefix, let's first see if we're matching anything from federation that is locally synced diff --git a/server/deps.ts b/server/deps.ts index b6feb828..752099d0 100644 --- a/server/deps.ts +++ b/server/deps.ts @@ -1,11 +1,14 @@ export * from "../common/deps.ts"; -export type { Next } from "https://deno.land/x/oak@v12.4.0/mod.ts"; + +export { Hono } from "https://deno.land/x/hono@v3.12.2/mod.ts"; export { - Application, + deleteCookie, + getCookie, + setCookie, +} from "https://deno.land/x/hono@v3.12.2/helper.ts"; +export { cors } from "https://deno.land/x/hono@v3.12.2/middleware.ts"; +export type { Context, - Request, - Response, - Router, -} from "https://deno.land/x/oak@v12.4.0/mod.ts"; -export * as etag from "https://deno.land/x/oak@v12.4.0/etag.ts"; -export { oakCors } from "https://deno.land/x/cors@v1.2.2/mod.ts"; + HonoRequest, + Next, +} from "https://deno.land/x/hono@v3.12.2/mod.ts"; diff --git a/server/http_server.ts b/server/http_server.ts index fc26d7e2..c1ab0db5 100644 --- a/server/http_server.ts +++ b/server/http_server.ts @@ -1,30 +1,22 @@ import { - Application, - Context, - Next, - oakCors, - Request, - Router, + deleteCookie, + getCookie, + Hono, + HonoRequest, + setCookie, } from "./deps.ts"; import { AssetBundle } from "../plugos/asset_bundle/bundle.ts"; import { FileMeta } from "$sb/types.ts"; -import { - handleRpc, - ShellRequest, - SyscallRequest, - SyscallResponse, -} from "./rpc.ts"; +import { handleRpc } from "./rpc.ts"; import { determineShellBackend } from "./shell_backend.ts"; import { SpaceServer, SpaceServerConfig } from "./instance.ts"; import { KvPrimitives } from "../plugos/lib/kv_primitives.ts"; -import { EndpointHook } from "../plugos/hooks/endpoint.ts"; import { PrefixedKvPrimitives } from "../plugos/lib/prefixed_kv_primitives.ts"; import { base64Encode } from "../plugos/asset_bundle/base64.ts"; const authenticationExpirySeconds = 60 * 60 * 24 * 7; // 1 week export type ServerOptions = { - app: Application; hostname: string; port: number; clientAssetBundle: AssetBundle; @@ -42,7 +34,7 @@ export class HttpServer { plugAssetBundle: AssetBundle; hostname: string; port: number; - app: Application>; + app: Hono; keyFile: string | undefined; certFile: string | undefined; @@ -51,11 +43,11 @@ export class HttpServer { configs: Map; constructor(options: ServerOptions) { + this.app = new Hono(); this.clientAssetBundle = options.clientAssetBundle; this.plugAssetBundle = options.plugAssetBundle; this.hostname = options.hostname; this.port = options.port; - this.app = options.app; this.keyFile = options.keyFile; this.certFile = options.certFile; this.baseKvPrimitives = options.baseKvPrimitives; @@ -76,8 +68,9 @@ export class HttpServer { return spaceServer; } - determineConfig(req: Request): [string, SpaceServerConfig] { - let hostname = req.url.host; // hostname:port + determineConfig(req: HonoRequest): [string, SpaceServerConfig] { + const url = new URL(req.url); + let hostname = url.host; // hostname:port // First try a full match let config = this.configs.get(hostname); @@ -102,7 +95,7 @@ export class HttpServer { throw new Error(`No space server config found for hostname ${hostname}`); } - ensureSpaceServer(req: Request): Promise { + ensureSpaceServer(req: HonoRequest): Promise { const [matchedHostname, config] = this.determineConfig(req); const spaceServer = this.spaceServers.get(matchedHostname); if (spaceServer) { @@ -132,22 +125,17 @@ export class HttpServer { } start() { - // Initialize JWT issuer - // First check if auth string (username:password) has changed // Serve static files (javascript, css, html) - this.app.use(this.serveStatic.bind(this)); - - this.addAuth(this.app); - const fsRouter = this.addFsRoutes(); - this.app.use(fsRouter.routes()); - this.app.use(fsRouter.allowedMethods()); + this.serveStatic(); + this.addAuth(); + this.addFsRoutes(); // Fallback, serve the UI index.html - this.app.use(async ({ request, response }) => { - response.headers.set("Content-type", "text/html"); - response.headers.set("Cache-Control", "no-cache"); - const spaceServer = await this.ensureSpaceServer(request); - response.body = this.renderIndexHtml(spaceServer); + this.app.use("*", async (c) => { + const spaceServer = await this.ensureSpaceServer(c.req); + return c.html(this.renderIndexHtml(spaceServer), 200, { + "Cache-Control": "no-cache", + }); }); this.abortController = new AbortController(); @@ -162,11 +150,12 @@ export class HttpServer { if (this.certFile) { listenOptions.cert = Deno.readTextFileSync(this.certFile); } - this.app.listen(listenOptions) - .catch((e: any) => { - console.log("Server listen error:", e.message); - Deno.exit(1); - }); + Deno.serve(listenOptions, this.app.fetch); + // this.app.listen(listenOptions) + // .catch((e: any) => { + // console.log("Server listen error:", e.message); + // Deno.exit(1); + // }); const visibleHostname = this.hostname === "0.0.0.0" ? "localhost" : this.hostname; @@ -175,73 +164,75 @@ export class HttpServer { ); } - async serveStatic( - { request, response }: Context, Record>, - next: Next, - ) { - const spaceServer = await this.ensureSpaceServer(request); - if ( - request.url.pathname === "/" - ) { - // Serve the UI (index.html) - // Note: we're explicitly not setting Last-Modified and If-Modified-Since header here because this page is dynamic - response.headers.set("Content-type", "text/html"); - response.headers.set("Cache-Control", "no-cache"); - response.body = this.renderIndexHtml(spaceServer); - return; - } - try { - const assetName = request.url.pathname.slice(1); + serveStatic() { + this.app.use("*", async (c, next) => { + const req = c.req; + const spaceServer = await this.ensureSpaceServer(req); + const url = new URL(req.url); + // console.log("URL", url); if ( - this.clientAssetBundle.has(assetName) && - request.headers.get("If-Modified-Since") === - utcDateString(this.clientAssetBundle.getMtime(assetName)) && - assetName !== "service_worker.js" + url.pathname === "/" ) { - response.status = 304; - return; + // Serve the UI (index.html) + // Note: we're explicitly not setting Last-Modified and If-Modified-Since header here because this page is dynamic + return c.html(this.renderIndexHtml(spaceServer), 200, { + "Cache-Control": "no-cache", + }); } - response.status = 200; - response.headers.set( - "Content-type", - this.clientAssetBundle.getMimeType(assetName), - ); - let data: Uint8Array | string = this.clientAssetBundle.readFileSync( - assetName, - ); - response.headers.set("Cache-Control", "no-cache"); - response.headers.set("Content-length", "" + data.length); - if (assetName !== "service_worker.js") { - response.headers.set( - "Last-Modified", - utcDateString(this.clientAssetBundle.getMtime(assetName)), + try { + const assetName = url.pathname.slice(1); + if (!this.clientAssetBundle.has(assetName)) { + return next(); + } + console.log("Asset name", assetName); + if ( + this.clientAssetBundle.has(assetName) && + req.header("If-Modified-Since") === + utcDateString(this.clientAssetBundle.getMtime(assetName)) && + assetName !== "service_worker.js" + ) { + return c.text("", 304); + } + c.status(200); + c.header("Content-type", this.clientAssetBundle.getMimeType(assetName)); + let data: Uint8Array | string = this.clientAssetBundle.readFileSync( + assetName, ); - } - - if (request.method === "GET") { - if (assetName === "service_worker.js") { - const textData = new TextDecoder().decode(data); - // console.log( - // "Swapping out config hash in service worker", - // ); - data = textData.replaceAll( - "{{CONFIG_HASH}}", - base64Encode( - JSON.stringify([ - spaceServer.clientEncryption, - spaceServer.syncOnly, - ]), - ), + c.header("Cache-Control", "no-cache"); + c.header("Content-length", "" + data.length); + if (assetName !== "service_worker.js") { + c.header( + "Last-Modified", + utcDateString(this.clientAssetBundle.getMtime(assetName)), ); } - response.body = data; + // console.log("Serving it now", assetName); + + if (req.method === "GET") { + if (assetName === "service_worker.js") { + const textData = new TextDecoder().decode(data); + // console.log( + // "Swapping out config hash in service worker", + // ); + data = textData.replaceAll( + "{{CONFIG_HASH}}", + base64Encode( + JSON.stringify([ + spaceServer.clientEncryption, + spaceServer.syncOnly, + ]), + ), + ); + } + return c.body(data); + } // else e.g. HEAD, OPTIONS, don't send body + } catch { + return next(); } - } catch { - return next(); - } + }); } - private addAuth(app: Application) { + private addAuth() { const excludedPaths = [ "/manifest.json", "/favicon.png", @@ -250,70 +241,63 @@ export class HttpServer { ]; // Middleware handling the /.auth page and flow - app.use(async ({ request, response, cookies }, next) => { - const host = request.url.host; // e.g. localhost:3000 - if (request.url.pathname === "/.auth") { - if (request.url.search === "?logout") { - await cookies.delete(authCookieName(host)); - // Implicit fallthrough to login page - } - if (request.method === "GET") { - response.headers.set("Content-type", "text/html"); - response.body = this.clientAssetBundle.readTextFileSync( - ".client/auth.html", + this.app.all("/.auth", async (c) => { + const url = new URL(c.req.url); + const req = c.req; + const host = url.host; // e.g. localhost:3000 + if (url.search === "?logout") { + deleteCookie(c, authCookieName(host)); + } + if (req.method === "GET") { + return c.html( + this.clientAssetBundle.readTextFileSync(".client/auth.html"), + ); + } else if (req.method === "POST") { + const values = await c.req.parseBody(); + const username = values["username"]; + const password = values["password"]; + const spaceServer = await this.ensureSpaceServer(req); + const { user: expectedUser, pass: expectedPassword } = spaceServer + .auth!; + if (username === expectedUser && password === expectedPassword) { + // Generate a JWT and set it as a cookie + const jwt = await spaceServer.jwtIssuer.createJWT( + { username }, + authenticationExpirySeconds, ); - return; - } else if (request.method === "POST") { - const values = await request.body({ type: "form" }).value; - const username = values.get("username")!; - const password = values.get("password")!; - const spaceServer = await this.ensureSpaceServer(request); - const { user: expectedUser, pass: expectedPassword } = spaceServer - .auth!; - if (username === expectedUser && password === expectedPassword) { - // Generate a JWT and set it as a cookie - const jwt = await spaceServer.jwtIssuer.createJWT( - { username }, - authenticationExpirySeconds, - ); - await cookies.set( - authCookieName(host), - jwt, - { - expires: new Date( - Date.now() + authenticationExpirySeconds * 1000, - ), // in a week - sameSite: "strict", - }, - ); - response.redirect("/"); - } else { - response.redirect("/.auth?error=1"); - } - return; + console.log("Successful auth"); + setCookie(c, authCookieName(host), jwt, { + expires: new Date( + Date.now() + authenticationExpirySeconds * 1000, + ), // in a week + // sameSite: "Strict", + // httpOnly: true, + }); + return c.redirect("/"); } else { - response.redirect("/.auth"); - return; + return c.redirect("/.auth?error=1"); } } else { - await next(); + return c.redirect("/.auth"); } }); // Check auth - app.use(async ({ request, response, cookies }, next) => { - const spaceServer = await this.ensureSpaceServer(request); + this.app.use("*", async (c, next) => { + const req = c.req; + const spaceServer = await this.ensureSpaceServer(req); if (!spaceServer.auth) { // Auth disabled in this config, skip return next(); } - const host = request.url.host; - if (!excludedPaths.includes(request.url.pathname)) { - const authToken = await cookies.get(authCookieName(host)); + const url = new URL(req.url); + const host = url.host; + if (!excludedPaths.includes(url.pathname)) { + const authCookie = getCookie(c, authCookieName(host)); - if (!authToken && spaceServer.authToken) { + if (!authCookie && spaceServer.authToken) { // Attempt Bearer Authorization based authentication - const authHeader = request.headers.get("Authorization"); + const authHeader = req.header("Authorization"); if (authHeader && authHeader.startsWith("Bearer ")) { const authToken = authHeader.slice("Bearer ".length); if (authToken === spaceServer.authToken) { @@ -323,21 +307,19 @@ export class HttpServer { console.log( "Unauthorized token access, redirecting to auth page", ); - response.status = 401; - response.body = "Unauthorized"; - return; + return c.text("Unauthorized", 401); } } } - if (!authToken) { + if (!authCookie) { console.log("Unauthorized access, redirecting to auth page"); - return response.redirect("/.auth"); + return c.redirect("/.auth"); } const { user: expectedUser } = spaceServer.auth!; try { const verifiedJwt = await spaceServer.jwtIssuer.verifyAndDecodeJWT( - authToken, + authCookie, ); if (verifiedJwt.username !== expectedUser) { throw new Error("Username mismatch"); @@ -347,214 +329,191 @@ export class HttpServer { "Error verifying JWT, redirecting to auth page", e.message, ); - return response.redirect("/.auth"); + return c.redirect("/.auth"); } } return next(); }); } - private addFsRoutes(): Router { - const fsRouter = new Router(); - const corsMiddleware = oakCors({ - allowedHeaders: "*", - exposedHeaders: "*", - methods: ["GET", "POST", "PUT", "DELETE", "HEAD", "OPTIONS"], - }); - - fsRouter.use(corsMiddleware); + private addFsRoutes() { + // this.app.use( + // "*", + // cors({ + // origin: "*", + // allowHeaders: ["*"], + // exposeHeaders: ["*"], + // allowMethods: ["GET", "POST", "PUT", "DELETE", "HEAD", "OPTIONS"], + // }), + // ); // File list - fsRouter.get( + this.app.get( "/index.json", - // corsMiddleware, - async ({ request, response }) => { - const spaceServer = await this.ensureSpaceServer(request); - if (request.headers.has("X-Sync-Mode")) { + async (c) => { + const req = c.req; + const spaceServer = await this.ensureSpaceServer(req); + if (req.header("X-Sync-Mode")) { // Only handle direct requests for a JSON representation of the file list - response.headers.set("Content-type", "application/json"); - response.headers.set("X-Space-Path", spaceServer.pagesPath); const files = await spaceServer.spacePrimitives.fetchFileList(); - response.body = JSON.stringify(files); + return c.json(files, 200, { + "X-Space-Path": spaceServer.pagesPath, + }); } else { // Otherwise, redirect to the UI // The reason to do this is to handle authentication systems like Authelia nicely - response.redirect("/"); + return c.redirect("/"); } }, ); // RPC - fsRouter.post("/.rpc/(.+)", async ({ request, response, params }) => { - const operation = params[0]; - const spaceServer = await this.ensureSpaceServer(request); - const body = await request.body({ - type: "json", - limit: 100 * 1024 * 1024, - }).value; + this.app.post("/.rpc/:operation", async (c) => { + const req = c.req; + const operation = req.param("operation")!; + const spaceServer = await this.ensureSpaceServer(req); + req.parseBody; + const body = await req.json(); try { const resp = await handleRpc(spaceServer, operation, body); - response.headers.set("Content-Type", "application/json"); - response.body = JSON.stringify({ r: resp }); + return c.json({ r: resp }, 200); } catch (e: any) { - console.log("Error", e); - response.status = 500; - response.body = e.message; - return; + return c.text(e.message, 500); } }); - const filePathRegex = "\/([^!].+\\.[a-zA-Z]+)"; + const filePathRegex = "/:path{[^!].+\\.[a-zA-Z]+}"; - fsRouter - .get( - filePathRegex, - async ({ params, response, request }) => { - const name = params[0]; - const spaceServer = await this.ensureSpaceServer(request); - console.log( - "Requested file", - name, + this.app.get( + filePathRegex, + async (c) => { + const req = c.req; + const name = req.param("path")!; + const spaceServer = await this.ensureSpaceServer(req); + console.log( + "Requested file", + name, + ); + if ( + name.endsWith(".md") && + // This header signififies the requests comes directly from the http_space_primitives client (not the browser) + !req.header("X-Sync-Mode") && + // This Accept header is used by federation to still work with CORS + req.header("Accept") !== + "application/octet-stream" && + req.header("sec-fetch-mode") !== "cors" + ) { + // It can happen that during a sync, authentication expires, this may result in a redirect to the login page and then back to this particular file. This particular file may be an .md file, which isn't great to show so we're redirecting to the associated SB UI page. + console.warn( + "Request was without X-Sync-Mode nor a CORS request, redirecting to page", ); - if ( - name.endsWith(".md") && - // This header signififies the requests comes directly from the http_space_primitives client (not the browser) - !request.headers.has("X-Sync-Mode") && - // This Accept header is used by federation to still work with CORS - request.headers.get("Accept") !== - "application/octet-stream" && - request.headers.get("sec-fetch-mode") !== "cors" - ) { - // It can happen that during a sync, authentication expires, this may result in a redirect to the login page and then back to this particular file. This particular file may be an .md file, which isn't great to show so we're redirecting to the associated SB UI page. - console.warn( - "Request was without X-Sync-Mode nor a CORS request, redirecting to page", - ); - response.redirect(`/${name.slice(0, -3)}`); - return; - } - if (name.startsWith(".")) { - // Don't expose hidden files - response.status = 404; - response.body = "Not exposed"; - return; - } - // Handle federated links through a simple redirect, only used for attachments loads with service workers disabled - if (name.startsWith("!")) { - let url = name.slice(1); - console.log("Handling this as a federated link", url); - if (url.startsWith("localhost")) { - url = `http://${url}`; - } else { - url = `https://${url}`; - } - try { - const req = await fetch(url); - response.status = req.status; - // Override X-Permssion header to always be "ro" - const newHeaders = new Headers(); - for (const [key, value] of req.headers.entries()) { - newHeaders.set(key, value); - } - newHeaders.set("X-Permission", "ro"); - response.headers = newHeaders; - response.body = req.body; - } catch (e: any) { - console.error("Error fetching federated link", e); - response.status = 500; - response.body = e.message; - } - return; - } - try { - if (request.headers.has("X-Get-Meta")) { - // Getting meta via GET request - const fileData = await spaceServer.spacePrimitives.getFileMeta( - name, - ); - response.status = 200; - this.fileMetaToHeaders(response.headers, fileData); - response.body = ""; - return; - } - const fileData = await spaceServer.spacePrimitives.readFile(name); - const lastModifiedHeader = new Date(fileData.meta.lastModified) - .toUTCString(); - if ( - request.headers.get("If-Modified-Since") === lastModifiedHeader - ) { - response.status = 304; - return; - } - response.status = 200; - this.fileMetaToHeaders(response.headers, fileData.meta); - response.headers.set("Last-Modified", lastModifiedHeader); - - response.body = fileData.data; - } catch (e: any) { - console.error("Error GETting file", name, e.message); - response.status = 404; - response.headers.set("Cache-Control", "no-cache"); - response.body = "Not found"; - } - }, - ) - .put( - filePathRegex, - async ({ request, response, params }) => { - const name = params[0]; - const spaceServer = await this.ensureSpaceServer(request); - console.log("Saving file", name); - if (name.startsWith(".")) { - // Don't expose hidden files - response.status = 403; - return; - } - - const body = await request.body({ type: "bytes" }).value; - - try { - const meta = await spaceServer.spacePrimitives.writeFile( - name, - body, - ); - response.status = 200; - this.fileMetaToHeaders(response.headers, meta); - response.body = "OK"; - } catch (err) { - console.error("Write failed", err); - response.status = 500; - response.body = "Write failed"; - } - }, - ) - .delete(filePathRegex, async ({ request, response, params }) => { - const name = params[0]; - const spaceServer = await this.ensureSpaceServer(request); - console.log("Deleting file", name); + return c.redirect(`/${name.slice(0, -3)}`); + } if (name.startsWith(".")) { // Don't expose hidden files - response.status = 403; - return; + return c.notFound(); + } + // Handle federated links through a simple redirect, only used for attachments loads with service workers disabled + if (name.startsWith("!")) { + let url = name.slice(1); + console.log("Handling this as a federated link", url); + if (url.startsWith("localhost")) { + url = `http://${url}`; + } else { + url = `https://${url}`; + } + try { + const req = await fetch(url); + // Override X-Permssion header to always be "ro" + const newHeaders = new Headers(); + for (const [key, value] of req.headers.entries()) { + newHeaders.set(key, value); + } + newHeaders.set("X-Permission", "ro"); + return new Response(req.body, { + status: req.status, + headers: newHeaders, + }); + } catch (e: any) { + console.error("Error fetching federated link", e); + return c.text(e.message, 500); + } } try { - await spaceServer.spacePrimitives.deleteFile(name); - response.status = 200; - response.body = "OK"; + if (req.header("X-Get-Meta")) { + // Getting meta via GET request + const fileData = await spaceServer.spacePrimitives.getFileMeta( + name, + ); + return c.text("", 200, this.fileMetaToHeaders(fileData)); + } + const fileData = await spaceServer.spacePrimitives.readFile(name); + const lastModifiedHeader = new Date(fileData.meta.lastModified) + .toUTCString(); + if ( + req.header("If-Modified-Since") === lastModifiedHeader + ) { + return c.text("", 304); + } + return c.body(fileData.data, 200, { + ...this.fileMetaToHeaders(fileData.meta), + "Last-Modified": lastModifiedHeader, + }); } catch (e: any) { - console.error("Error deleting attachment", e); - response.status = 500; - response.body = e.message; + console.error("Error GETting file", name, e.message); + return c.notFound(); } - }) - .options(filePathRegex, corsMiddleware); + }, + ).put( + async (c) => { + const req = c.req; + const name = req.param("path")!; + const spaceServer = await this.ensureSpaceServer(req); + console.log("Saving file", name); + if (name.startsWith(".")) { + // Don't expose hidden files + return c.text("Forbidden", 403); + } + + const body = await req.arrayBuffer(); + + try { + const meta = await spaceServer.spacePrimitives.writeFile( + name, + new Uint8Array(body), + ); + return c.text("OK", 200, this.fileMetaToHeaders(meta)); + } catch (err) { + console.error("Write failed", err); + return c.text("Write failed", 500); + } + }, + ).delete(async (c) => { + const req = c.req; + const name = req.param("path")!; + const spaceServer = await this.ensureSpaceServer(req); + console.log("Deleting file", name); + if (name.startsWith(".")) { + // Don't expose hidden files + return c.text("Forbidden", 403); + } + try { + await spaceServer.spacePrimitives.deleteFile(name); + return c.text("OK"); + } catch (e: any) { + console.error("Error deleting attachment", e); + return c.text(e.message, 500); + } + }).options(); // Federation proxy - const proxyPathRegex = "\/!(.+)"; - fsRouter.all( + const proxyPathRegex = "/:uri{!.+}"; + this.app.all( proxyPathRegex, - async ({ params, response, request }, next) => { - let url = params[0]; - if (!request.headers.has("X-Proxy-Request")) { + async (c, next) => { + const req = c.req; + let url = req.param("uri")!.slice(1); + if (!req.header("X-Proxy-Request")) { // Direct browser request, not explicity fetch proxy request if (!/\.[a-zA-Z0-9]+$/.test(url)) { console.log("Directly loading federation page via URL:", url); @@ -572,47 +531,41 @@ export class HttpServer { for ( const headerName of ["Authorization", "Accept", "Content-Type"] ) { - if (request.headers.has(headerName)) { + if (req.header(headerName)) { safeRequestHeaders.set( headerName, - request.headers.get(headerName)!, + req.header(headerName)!, ); } } - const req = await fetch(url, { - method: request.method, + const body = await req.arrayBuffer(); + const fetchReq = await fetch(url, { + method: req.method, headers: safeRequestHeaders, - body: request.hasBody - ? request.body({ type: "stream" }).value - : undefined, + body: body.byteLength > 0 ? body : undefined, }); - response.status = req.status; - response.headers = req.headers; - response.body = req.body; + const responseHeaders: Record = {}; + for (const [key, value] of fetchReq.headers.entries()) { + responseHeaders[key] = value; + } + return c.body(fetchReq.body, fetchReq.status, responseHeaders); } catch (e: any) { console.error("Error fetching federated link", e); - response.status = 500; - response.body = e.message; + return c.text(e.message, 500); } - return; }, ); - return fsRouter; } - private fileMetaToHeaders(headers: Headers, fileMeta: FileMeta) { - headers.set("Content-Type", fileMeta.contentType); - headers.set( - "X-Last-Modified", - "" + fileMeta.lastModified, - ); - headers.set( - "X-Created", - "" + fileMeta.created, - ); - headers.set("Cache-Control", "no-cache"); - headers.set("X-Permission", fileMeta.perm); - headers.set("X-Content-Length", "" + fileMeta.size); + private fileMetaToHeaders(fileMeta: FileMeta) { + return { + "Content-Type": fileMeta.contentType, + "X-Last-Modified": "" + fileMeta.lastModified, + "X-Created": "" + fileMeta.created, + "Cache-Control": "no-cache", + "X-Permission": fileMeta.perm, + "X-Content-Length": "" + fileMeta.size, + }; } stop() { @@ -628,5 +581,5 @@ function utcDateString(mtime: number): string { } function authCookieName(host: string) { - return `auth:${host}`; + return `auth_${host.replaceAll(/\W/g, "_")}`; } diff --git a/server/instance.ts b/server/instance.ts index 054d595e..d72276ea 100644 --- a/server/instance.ts +++ b/server/instance.ts @@ -1,14 +1,11 @@ import { AssetBundlePlugSpacePrimitives } from "../common/spaces/asset_bundle_space_primitives.ts"; -import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts"; import { SpacePrimitives } from "../common/spaces/space_primitives.ts"; -import { ensureSettingsAndIndex } from "../common/util.ts"; import { AssetBundle } from "../plugos/asset_bundle/bundle.ts"; -import { DataStore } from "../plugos/lib/datastore.ts"; +import type { DataStore } from "../plugos/lib/datastore.ts"; +import { KvDataStore } from "../plugos/lib/kv_datastore.ts"; import { KvPrimitives } from "../plugos/lib/kv_primitives.ts"; import { PrefixedKvPrimitives } from "../plugos/lib/prefixed_kv_primitives.ts"; -import { BuiltinSettings } from "../web/types.ts"; import { JWTIssuer } from "./crypto.ts"; -import { gitIgnoreCompiler } from "./deps.ts"; import { ShellBackend } from "./shell_backend.ts"; import { determineStorageBackend } from "./storage_backend.ts"; @@ -57,7 +54,7 @@ export class SpaceServer { } this.jwtIssuer = new JWTIssuer(kvPrimitives); - this.ds = new DataStore(new PrefixedKvPrimitives(kvPrimitives, ["ds"])); + this.ds = new KvDataStore(new PrefixedKvPrimitives(kvPrimitives, ["ds"])); } async init() { diff --git a/web/client.ts b/web/client.ts index 774cc1ca..d9c533a0 100644 --- a/web/client.ts +++ b/web/client.ts @@ -39,7 +39,7 @@ import { MainUI } from "./editor_ui.tsx"; import { cleanPageRef } from "$sb/lib/resolve.ts"; import { SpacePrimitives } from "../common/spaces/space_primitives.ts"; import { CodeWidgetButton, FileMeta, PageMeta } from "$sb/types.ts"; -import { DataStore, IDataStore } from "../plugos/lib/datastore.ts"; +import type { DataStore } from "../plugos/lib/datastore.ts"; import { IndexedDBKvPrimitives } from "../plugos/lib/indexeddb_kv_primitives.ts"; import { DataStoreMQ } from "../plugos/lib/mq.datastore.ts"; import { DataStoreSpacePrimitives } from "../common/spaces/datastore_space_primitives.ts"; @@ -52,6 +52,7 @@ import { markFullSpaceIndexComplete, } from "../common/space_index.ts"; import { RemoteDataStore } from "./remote_datastore.ts"; +import { KvDataStore } from "../plugos/lib/kv_datastore.ts"; const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/; const autoSaveInterval = 1000; @@ -114,7 +115,7 @@ export class Client { public allKnownPages = new Set(); clientDS!: DataStore; mq!: DataStoreMQ; - ds!: IDataStore; + ds!: DataStore; constructor( private parent: Element, @@ -140,7 +141,7 @@ export class Client { `${this.dbPrefix}_state`, ); await stateKvPrimitives.init(); - this.clientDS = new DataStore(stateKvPrimitives); + this.clientDS = new KvDataStore(stateKvPrimitives); // In sync mode, reuse the clientDS, otherwise talk to a remote data store (over HTTP) this.ds = this.syncMode @@ -198,7 +199,7 @@ export class Client { this.focus(); - await this.system.init(); + this.system.init(); // Load settings this.settings = await ensureSettingsAndIndex(localSpacePrimitives); @@ -485,7 +486,7 @@ export class Client { new EventedSpacePrimitives( // Using fallback space primitives here to allow (by default) local reads to "fall through" to HTTP when files aren't synced yet new FallbackSpacePrimitives( - new DataStoreSpacePrimitives(new DataStore(spaceKvPrimitives)), + new DataStoreSpacePrimitives(new KvDataStore(spaceKvPrimitives)), this.plugSpaceRemotePrimitives, ), this.eventHook, diff --git a/web/client_system.ts b/web/client_system.ts index 426c1e09..d4c50e74 100644 --- a/web/client_system.ts +++ b/web/client_system.ts @@ -30,7 +30,7 @@ import { import { MQHook } from "../plugos/hooks/mq.ts"; import { mqSyscalls } from "../plugos/syscalls/mq.ts"; import { dataStoreSyscalls } from "../plugos/syscalls/datastore.ts"; -import { DataStore, IDataStore } from "../plugos/lib/datastore.ts"; +import { DataStore } from "../plugos/lib/datastore.ts"; import { MessageQueue } from "../plugos/lib/mq.ts"; import { languageSyscalls } from "../common/syscalls/language.ts"; import { handlebarsSyscalls } from "../common/syscalls/handlebars.ts"; @@ -56,7 +56,7 @@ export class ClientSystem { private client: Client, private mq: MessageQueue, private clientDs: DataStore, - private dataStore: IDataStore, + private dataStore: DataStore, private eventHook: EventHook, ) { // Only set environment to "client" when running in thin client mode, otherwise we run everything locally (hybrid) @@ -65,7 +65,7 @@ export class ClientSystem { undefined, { manifestCache: new KVPrimitivesManifestCache( - clientDs.kv, + clientDs, "manifest", ), }, @@ -145,7 +145,7 @@ export class ClientSystem { ); } - async init() { + init() { // Slash command hook this.slashCommandHook = new SlashCommandHook(this.client); this.system.addHook(this.slashCommandHook); diff --git a/web/remote_datastore.ts b/web/remote_datastore.ts index 2e0f1680..de0ed563 100644 --- a/web/remote_datastore.ts +++ b/web/remote_datastore.ts @@ -1,11 +1,15 @@ import { HttpSpacePrimitives } from "../common/spaces/http_space_primitives.ts"; import { KV, KvKey, KvQuery } from "$sb/types.ts"; -import { IDataStore } from "../plugos/lib/datastore.ts"; +import { DataStore } from "../plugos/lib/datastore.ts"; import { rpcCall } from "./syscalls/datastore.proxy.ts"; +import { LimitedMap } from "../common/limited_map.ts"; -// implements DataStore "interface" -export class RemoteDataStore implements IDataStore { - constructor(private httpPrimitives: HttpSpacePrimitives) { +export class RemoteDataStore implements DataStore { + private cache = new LimitedMap(20); + + constructor( + private httpPrimitives: HttpSpacePrimitives, + ) { } private proxy( @@ -45,8 +49,31 @@ export class RemoteDataStore implements IDataStore { return this.proxy("datastore.batchDelete", keys); } - query(query: KvQuery): Promise[]> { - return this.proxy("datastore.query", query); + /** + * Proxies the query to the server, and caches the result if cacheSecs is set + * @param query query to execute + * @returns + */ + async query(query: KvQuery): Promise[]> { + let cacheKey: string | undefined; + const cacheSecs = query.cacheSecs; + // Should we do caching? + if (cacheSecs) { + // Remove the cacheSecs from the query + query = { ...query, cacheSecs: undefined }; + cacheKey = JSON.stringify(query); + const cachedResult = this.cache.get(cacheKey); + if (cachedResult) { + // Let's use the cached result + return cachedResult; + } + } + const result = await this.proxy("datastore.query", query); + if (cacheKey) { + // Store in the cache + this.cache.set(cacheKey, result, cacheSecs! * 1000); + } + return result; } queryDelete(query: KvQuery): Promise { diff --git a/web/service_worker.ts b/web/service_worker.ts index 05e39c6b..072c119a 100644 --- a/web/service_worker.ts +++ b/web/service_worker.ts @@ -2,6 +2,7 @@ import type { FileContent } from "../common/spaces/datastore_space_primitives.ts import { simpleHash } from "../common/crypto.ts"; import { DataStore } from "../plugos/lib/datastore.ts"; import { IndexedDBKvPrimitives } from "../plugos/lib/indexeddb_kv_primitives.ts"; +import { KvDataStore } from "../plugos/lib/kv_datastore.ts"; const CACHE_NAME = "{{CACHE_NAME}}_{{CONFIG_HASH}}"; @@ -182,7 +183,7 @@ self.addEventListener("message", (event: any) => { // Setup space const kv = new IndexedDBKvPrimitives(`${dbPrefix}_synced_space`); kv.init().then(() => { - ds = new DataStore(kv); + ds = new KvDataStore(kv); console.log("Datastore in service worker initialized..."); }); } diff --git a/web/space.ts b/web/space.ts index 92757681..ac577f12 100644 --- a/web/space.ts +++ b/web/space.ts @@ -4,9 +4,6 @@ import { safeRun } from "../common/util.ts"; import { AttachmentMeta, FileMeta, PageMeta } from "$sb/types.ts"; import { EventHook } from "../plugos/hooks/event.ts"; -import { throttle } from "$sb/lib/async.ts"; -import { DataStore } from "../plugos/lib/datastore.ts"; -import { LimitedMap } from "../common/limited_map.ts"; const pageWatchInterval = 5000;