mingzailao
/
silverbullet
mirror of https://github.com/silverbulletmd/silverbullet.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Doing some fancy caching now

pull/628/head
Zef Hemel 2024-01-12 17:44:25 +01:00
parent f366a2fd63
commit b2cd68f82c
27 changed files with 518 additions and 723 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cmd/server.ts
View File

@ -1,4 +1,3 @@
import { Application } from "../server/deps.ts";
import { HttpServer } from "../server/http_server.ts"; import { HttpServer } from "../server/http_server.ts";
import clientAssetBundle from "../dist/client_asset_bundle.json" assert { import clientAssetBundle from "../dist/client_asset_bundle.json" assert {
type: "json", type: "json",
@ -46,8 +45,6 @@ export async function serveCommand(
console.log("Running in sync-only mode (no backend processing)"); console.log("Running in sync-only mode (no backend processing)");
} }
const app = new Application();
if (!folder) { if (!folder) {
// Didn't get a folder as an argument, check if we got it as an environment variable // Didn't get a folder as an argument, check if we got it as an environment variable
folder = Deno.env.get("SB_FOLDER"); folder = Deno.env.get("SB_FOLDER");
@ -90,7 +87,6 @@ export async function serveCommand(
}); });
const httpServer = new HttpServer({ const httpServer = new HttpServer({
app,
hostname, hostname,
port, port,
clientAssetBundle: new AssetBundle(clientAssetBundle as AssetJson), clientAssetBundle: new AssetBundle(clientAssetBundle as AssetJson),

9
common/limited_map.test.ts
View File

@ -5,7 +5,7 @@ import { LimitedMap } from "./limited_map.ts";
Deno.test("limited map", async () => { Deno.test("limited map", async () => {
const mp = new LimitedMap<string>(3); const mp = new LimitedMap<string>(3);
mp.set("a", "a"); mp.set("a", "a");
mp.set("b", "b"); mp.set("b", "b", 10);
mp.set("c", "c"); mp.set("c", "c");
await sleep(2); await sleep(2);
assertEquals(mp.get("a"), "a"); assertEquals(mp.get("a"), "a");
@ -15,6 +15,11 @@ Deno.test("limited map", async () => {
assertEquals(mp.get("c"), "c"); assertEquals(mp.get("c"), "c");
// Drops the first key // Drops the first key
mp.set("d", "d"); mp.set("d", "d");
await sleep(2);
assertEquals(mp.get("a"), undefined); assertEquals(mp.get("a"), undefined);
await sleep(20);
// "b" should have been dropped
assertEquals(mp.get("b"), undefined);
assertEquals(mp.get("c"), "c");
console.log(mp.toJSON());
}); });

45
common/limited_map.ts
View File

@ -1,20 +1,36 @@
type LimitedMapRecord<V> = Record<string, { value: V; la: number }>; type LimitedMapRecord<V> = { value: V; la: number };
export class LimitedMap<V> { export class LimitedMap<V> {
constructor(private maxSize: number, private map: LimitedMapRecord<V> = {}) { private map: Map<string, LimitedMapRecord<V>>;
constructor(
private maxSize: number,
initialJson: Record<string, LimitedMapRecord<V>> = {},
) {
this.map = new Map(Object.entries(initialJson));
} }
set(key: string, value: V) { /**
if (Object.keys(this.map).length >= this.maxSize) { * @param key
* @param value
* @param ttl time to live (in ms)
*/
set(key: string, value: V, ttl?: number) {
if (ttl) {
setTimeout(() => {
this.map.delete(key);
}, ttl);
}
if (this.map.size >= this.maxSize) {
// Remove the oldest key before adding a new one // Remove the oldest key before adding a new one
const oldestKey = this.getOldestKey(); const oldestKey = this.getOldestKey();
delete this.map[oldestKey!]; this.map.delete(oldestKey!);
} }
this.map[key] = { value, la: Date.now() }; this.map.set(key, { value, la: Date.now() });
} }
get(key: string): V | undefined { get(key: string): V | undefined {
const entry = this.map[key]; const entry = this.map.get(key);
if (entry) { if (entry) {
// Update the last accessed timestamp // Update the last accessed timestamp
entry.la = Date.now(); entry.la = Date.now();
@ -24,24 +40,21 @@ export class LimitedMap<V> {
} }
remove(key: string) { remove(key: string) {
delete this.map[key]; this.map.delete(key);
} }
toJSON() { toJSON() {
return this.map; return Object.fromEntries(this.map.entries());
} }
private getOldestKey(): string | undefined { private getOldestKey(): string | undefined {
let oldestKey: string | undefined; let oldestKey: string | undefined;
let oldestTimestamp: number | undefined; let oldestTimestamp: number | undefined;
for (const key in this.map) { for (const [key, entry] of this.map.entries()) {
if (Object.prototype.hasOwnProperty.call(this.map, key)) { if (!oldestTimestamp || entry.la < oldestTimestamp) {
const entry = this.map[key]; oldestKey = key;
if (!oldestTimestamp || entry.la < oldestTimestamp) { oldestTimestamp = entry.la;
oldestKey = key;
oldestTimestamp = entry.la;
}
} }
} }

2
common/manifest.ts
View File

@ -6,7 +6,6 @@ import { SlashCommandHookT } from "../web/hooks/slash_command.ts";
import { PlugNamespaceHookT } from "./hooks/plug_namespace.ts"; import { PlugNamespaceHookT } from "./hooks/plug_namespace.ts";
import { CodeWidgetT } from "../web/hooks/code_widget.ts"; import { CodeWidgetT } from "../web/hooks/code_widget.ts";
import { MQHookT } from "../plugos/hooks/mq.ts"; import { MQHookT } from "../plugos/hooks/mq.ts";
import { EndpointHookT } from "../plugos/hooks/endpoint.ts";
import { PanelWidgetT } from "../web/hooks/panel_widget.ts"; import { PanelWidgetT } from "../web/hooks/panel_widget.ts";
/** Silverbullet hooks give plugs access to silverbullet core systems. /** Silverbullet hooks give plugs access to silverbullet core systems.
@ -24,7 +23,6 @@ export type SilverBulletHooks =
& EventHookT & EventHookT
& CodeWidgetT & CodeWidgetT
& PanelWidgetT & PanelWidgetT
& EndpointHookT
& PlugNamespaceHookT; & PlugNamespaceHookT;
/** Syntax extension allow plugs to declaratively add new *inline* parse tree nodes to the markdown parser. */ /** Syntax extension allow plugs to declaratively add new *inline* parse tree nodes to the markdown parser. */

6
common/space_index.ts
View File

@ -1,4 +1,4 @@
import { IDataStore } from "../plugos/lib/datastore.ts"; import { DataStore } from "../plugos/lib/datastore.ts";
import { System } from "../plugos/system.ts"; import { System } from "../plugos/system.ts";
const indexVersionKey = ["$indexVersion"]; const indexVersionKey = ["$indexVersion"];
@ -8,7 +8,7 @@ const desiredIndexVersion = 1;
let indexOngoing = false; let indexOngoing = false;
export async function ensureSpaceIndex(ds: IDataStore, system: System<any>) { export async function ensureSpaceIndex(ds: DataStore, system: System<any>) {
const currentIndexVersion = await ds.get(indexVersionKey); const currentIndexVersion = await ds.get(indexVersionKey);
console.info("Current space index version", currentIndexVersion); console.info("Current space index version", currentIndexVersion);
@ -25,6 +25,6 @@ export async function ensureSpaceIndex(ds: IDataStore, system: System<any>) {
} }
} }
export async function markFullSpaceIndexComplete(ds: IDataStore) { export async function markFullSpaceIndexComplete(ds: DataStore) {
await ds.set(indexVersionKey, desiredIndexVersion); await ds.set(indexVersionKey, desiredIndexVersion);
} }

4
common/spaces/datastore_space_primitives.test.ts
View File

@ -1,8 +1,8 @@
import "https://esm.sh/fake-indexeddb@4.0.2/auto"; import "https://esm.sh/fake-indexeddb@4.0.2/auto";
import { DataStore } from "../../plugos/lib/datastore.ts";
import { IndexedDBKvPrimitives } from "../../plugos/lib/indexeddb_kv_primitives.ts"; import { IndexedDBKvPrimitives } from "../../plugos/lib/indexeddb_kv_primitives.ts";
import { DataStoreSpacePrimitives } from "./datastore_space_primitives.ts"; import { DataStoreSpacePrimitives } from "./datastore_space_primitives.ts";
import { testSpacePrimitives } from "./space_primitives.test.ts"; import { testSpacePrimitives } from "./space_primitives.test.ts";
import { KvDataStore } from "../../plugos/lib/kv_datastore.ts";
Deno.test("DataStoreSpacePrimitives", { Deno.test("DataStoreSpacePrimitives", {
sanitizeResources: false, sanitizeResources: false,
@ -11,7 +11,7 @@ Deno.test("DataStoreSpacePrimitives", {
const db = new IndexedDBKvPrimitives("test"); const db = new IndexedDBKvPrimitives("test");
await db.init(); await db.init();
const space = new DataStoreSpacePrimitives(new DataStore(db)); const space = new DataStoreSpacePrimitives(new KvDataStore(db));
await testSpacePrimitives(space); await testSpacePrimitives(space);
db.close(); db.close();
}); });

5
plug-api/types.ts
View File

@ -73,6 +73,11 @@ export type Query = {
render?: string; render?: string;
renderAll?: boolean; renderAll?: boolean;
distinct?: boolean; distinct?: boolean;
/**
* When set, the DS implementation _may_ cache the result for the given number of seconds.
*/
cacheSecs?: number;
}; };
export type KvQuery = Omit<Query, "querySource"> & { export type KvQuery = Omit<Query, "querySource"> & {

48
plugos/hooks/endpoint.test.ts
View File

@ -1,48 +0,0 @@
import { createSandbox } from "../environments/deno_sandbox.ts";
import { EndpointHook, EndpointHookT } from "./endpoint.ts";
import { System } from "../system.ts";
import { Application } from "../../server/deps.ts";
import { assertEquals } from "../../test_deps.ts";
import { compileManifest } from "../compile.ts";
import { esbuild } from "../deps.ts";
Deno.test("Run a plugos endpoint server", async () => {
const tempDir = await Deno.makeTempDir();
const system = new System<EndpointHookT>("server");
const workerPath = await compileManifest(
new URL("../test.plug.yaml", import.meta.url).pathname,
tempDir,
);
await system.load(
new URL(`file://${workerPath}`),
"test",
0,
createSandbox,
);
const app = new Application();
const port = 3123;
const endpointHook = new EndpointHook("/_/");
app.use((context, next) => {
return endpointHook.handleRequest(system, context, next);
});
const controller = new AbortController();
app.listen({ port: port, signal: controller.signal });
const res = await fetch(`http://localhost:${port}/_/test/?name=Pete`);
assertEquals(res.status, 200);
assertEquals(res.headers.get("Content-type"), "application/json");
assertEquals(await res.json(), [1, 2, 3]);
console.log("Aborting");
controller.abort();
await system.unloadAll();
await Deno.remove(tempDir, { recursive: true });
esbuild.stop();
});

140
plugos/hooks/endpoint.ts
View File

@ -1,140 +0,0 @@
import { Hook, Manifest } from "../types.ts";
import { System } from "../system.ts";
import { Application, Context, Next } from "../../server/deps.ts";
export type EndpointRequest = {
method: string;
path: string;
query: { [key: string]: string };
headers: { [key: string]: string };
body: any;
};
export type EndpointResponse = {
status: number;
headers?: { [key: string]: string };
body: any;
};
export type EndpointHookT = {
http?: EndPointDef | EndPointDef[];
};
export type EndPointDef = {
method?: "GET" | "POST" | "PUT" | "DELETE" | "HEAD" | "OPTIONS" | "ANY";
path: string;
};
export class EndpointHook implements Hook<EndpointHookT> {
readonly prefix: string;
constructor(prefix: string) {
this.prefix = prefix;
}
public async handleRequest(
system: System<EndpointHookT>,
ctx: Context,
next: Next,
) {
const req = ctx.request;
const requestPath = ctx.request.url.pathname;
if (!requestPath.startsWith(this.prefix)) {
return next();
}
console.log("Endpoint request", requestPath);
// Iterate over all loaded plugins
for (const [plugName, plug] of system.loadedPlugs.entries()) {
const manifest = plug.manifest;
if (!manifest) {
continue;
}
const functions = manifest.functions;
// console.log("Checking plug", plugName);
const prefix = `${this.prefix}${plugName}`;
if (!requestPath.startsWith(prefix)) {
continue;
}
for (const [name, functionDef] of Object.entries(functions)) {
if (!functionDef.http) {
continue;
}
// console.log("Got config", functionDef);
const endpoints = Array.isArray(functionDef.http)
? functionDef.http
: [functionDef.http];
// console.log(endpoints);
for (const { path, method } of endpoints) {
const prefixedPath = `${prefix}${path}`;
if (
prefixedPath === requestPath &&
((method || "GET") === req.method || method === "ANY")
) {
try {
const response: EndpointResponse = await plug.invoke(name, [
{
path: req.url.pathname,
method: req.method,
body: req.body(),
query: Object.fromEntries(
req.url.searchParams.entries(),
),
headers: Object.fromEntries(req.headers.entries()),
} as EndpointRequest,
]);
if (response.headers) {
for (
const [key, value] of Object.entries(
response.headers,
)
) {
ctx.response.headers.set(key, value);
}
}
ctx.response.status = response.status;
ctx.response.body = response.body;
// console.log("Sent result");
return;
} catch (e: any) {
console.error("Error executing function", e);
ctx.response.status = 500;
ctx.response.body = e.message;
return;
}
}
}
}
}
// console.log("Shouldn't get here");
await next();
}
apply(): void {
}
validateManifest(manifest: Manifest<EndpointHookT>): string[] {
const errors = [];
for (const functionDef of Object.values(manifest.functions)) {
if (!functionDef.http) {
continue;
}
const endpoints = Array.isArray(functionDef.http)
? functionDef.http
: [functionDef.http];
for (const { path, method } of endpoints) {
if (!path) {
errors.push("Path not defined for endpoint");
}
if (
method &&
["GET", "POST", "PUT", "DELETE", "ANY"].indexOf(method) === -1
) {
errors.push(
`Invalid method ${method} for end point with with ${path}`,
);
}
}
}
return errors;
}
}

4
plugos/lib/datastore.test.ts
View File

@ -1,13 +1,13 @@
import "https://esm.sh/fake-indexeddb@4.0.2/auto"; import "https://esm.sh/fake-indexeddb@4.0.2/auto";
import { IndexedDBKvPrimitives } from "./indexeddb_kv_primitives.ts"; import { IndexedDBKvPrimitives } from "./indexeddb_kv_primitives.ts";
import { DataStore } from "./datastore.ts";
import { DenoKvPrimitives } from "./deno_kv_primitives.ts"; import { DenoKvPrimitives } from "./deno_kv_primitives.ts";
import { KvPrimitives } from "./kv_primitives.ts"; import { KvPrimitives } from "./kv_primitives.ts";
import { assertEquals } from "https://deno.land/std@0.165.0/testing/asserts.ts"; import { assertEquals } from "https://deno.land/std@0.165.0/testing/asserts.ts";
import { PrefixedKvPrimitives } from "./prefixed_kv_primitives.ts"; import { PrefixedKvPrimitives } from "./prefixed_kv_primitives.ts";
import { KvDataStore } from "./kv_datastore.ts";
async function test(db: KvPrimitives) { async function test(db: KvPrimitives) {
const datastore = new DataStore(new PrefixedKvPrimitives(db, ["ds"]), { const datastore = new KvDataStore(new PrefixedKvPrimitives(db, ["ds"]), {
count: (arr: any[]) => arr.length, count: (arr: any[]) => arr.length,
}); });
await datastore.set(["user", "peter"], { name: "Peter" }); await datastore.set(["user", "peter"], { name: "Peter" });

90
plugos/lib/datastore.ts
View File

@ -1,13 +1,10 @@
import { applyQueryNoFilterKV, evalQueryExpression } from "$sb/lib/query.ts"; import { KV, KvKey, KvQuery } from "$sb/types.ts";
import { FunctionMap, KV, KvKey, KvQuery } from "$sb/types.ts";
import { builtinFunctions } from "$sb/lib/builtin_query_functions.ts";
import { KvPrimitives } from "./kv_primitives.ts";
/** /**
* This is the data store class you'll actually want to use, wrapping the primitives * This is the data store class you'll actually want to use, wrapping the primitives
* in a more user-friendly way * in a more user-friendly way
*/ */
export interface IDataStore { export interface DataStore {
get<T = any>(key: KvKey): Promise<T | null>; get<T = any>(key: KvKey): Promise<T | null>;
batchGet<T = any>(keys: KvKey[]): Promise<(T | null)[]>; batchGet<T = any>(keys: KvKey[]): Promise<(T | null)[]>;
set(key: KvKey, value: any): Promise<void>; set(key: KvKey, value: any): Promise<void>;
@ -17,86 +14,3 @@ export interface IDataStore {
query<T = any>(query: KvQuery): Promise<KV<T>[]>; query<T = any>(query: KvQuery): Promise<KV<T>[]>;
queryDelete(query: KvQuery): Promise<void>; queryDelete(query: KvQuery): Promise<void>;
} }
export class DataStore implements IDataStore {
constructor(
readonly kv: KvPrimitives,
private functionMap: FunctionMap = builtinFunctions,
) {
}
async get<T = any>(key: KvKey): Promise<T | null> {
return (await this.batchGet([key]))[0];
}
batchGet<T = any>(keys: KvKey[]): Promise<(T | null)[]> {
return this.kv.batchGet(keys);
}
set(key: KvKey, value: any): Promise<void> {
return this.batchSet([{ key, value }]);
}
batchSet<T = any>(entries: KV<T>[]): Promise<void> {
const allKeyStrings = new Set<string>();
const uniqueEntries: KV[] = [];
for (const { key, value } of entries) {
const keyString = JSON.stringify(key);
if (allKeyStrings.has(keyString)) {
console.warn(`Duplicate key ${keyString} in batchSet, skipping`);
} else {
allKeyStrings.add(keyString);
uniqueEntries.push({ key, value });
}
}
return this.kv.batchSet(uniqueEntries);
}
delete(key: KvKey): Promise<void> {
return this.batchDelete([key]);
}
batchDelete(keys: KvKey[]): Promise<void> {
return this.kv.batchDelete(keys);
}
async query<T = any>(query: KvQuery): Promise<KV<T>[]> {
const results: KV<T>[] = [];
let itemCount = 0;
// Accumulate results
let limit = Infinity;
if (query.limit) {
limit = evalQueryExpression(query.limit, {}, this.functionMap);
}
for await (
const entry of this.kv.query(query)
) {
// Filter
if (
query.filter &&
!evalQueryExpression(query.filter, entry.value, this.functionMap)
) {
continue;
}
results.push(entry);
itemCount++;
// Stop when the limit has been reached
if (itemCount === limit && !query.orderBy) {
// Only break when not also ordering in which case we need all results
break;
}
}
// Apply order by, limit, and select
return applyQueryNoFilterKV(query, results, this.functionMap);
}
async queryDelete(query: KvQuery): Promise<void> {
const keys: KvKey[] = [];
for (
const { key } of await this.query(query)
) {
keys.push(key);
}
return this.batchDelete(keys);
}
}

88
plugos/lib/kv_datastore.ts Normal file
View File

@ -0,0 +1,88 @@
import { FunctionMap, KV, KvKey, KvQuery } from "$sb/types.ts";
import { builtinFunctions } from "$sb/lib/builtin_query_functions.ts";
import { DataStore } from "./datastore.ts";
import { KvPrimitives } from "./kv_primitives.ts";
import { applyQueryNoFilterKV, evalQueryExpression } from "$sb/lib/query.ts";
export class KvDataStore implements DataStore {
constructor(
readonly kv: KvPrimitives,
private functionMap: FunctionMap = builtinFunctions,
) {
}
async get<T = any>(key: KvKey): Promise<T | null> {
return (await this.batchGet([key]))[0];
}
batchGet<T = any>(keys: KvKey[]): Promise<(T | null)[]> {
return this.kv.batchGet(keys);
}
set(key: KvKey, value: any): Promise<void> {
return this.batchSet([{ key, value }]);
}
batchSet<T = any>(entries: KV<T>[]): Promise<void> {
const allKeyStrings = new Set<string>();
const uniqueEntries: KV[] = [];
for (const { key, value } of entries) {
const keyString = JSON.stringify(key);
if (allKeyStrings.has(keyString)) {
console.warn(`Duplicate key ${keyString} in batchSet, skipping`);
} else {
allKeyStrings.add(keyString);
uniqueEntries.push({ key, value });
}
}
return this.kv.batchSet(uniqueEntries);
}
delete(key: KvKey): Promise<void> {
return this.batchDelete([key]);
}
batchDelete(keys: KvKey[]): Promise<void> {
return this.kv.batchDelete(keys);
}
async query<T = any>(query: KvQuery): Promise<KV<T>[]> {
const results: KV<T>[] = [];
let itemCount = 0;
// Accumulate results
let limit = Infinity;
if (query.limit) {
limit = evalQueryExpression(query.limit, {}, this.functionMap);
}
for await (
const entry of this.kv.query(query)
) {
// Filter
if (
query.filter &&
!evalQueryExpression(query.filter, entry.value, this.functionMap)
) {
continue;
}
results.push(entry);
itemCount++;
// Stop when the limit has been reached
if (itemCount === limit && !query.orderBy) {
// Only break when not also ordering in which case we need all results
break;
}
}
// Apply order by, limit, and select
return applyQueryNoFilterKV(query, results, this.functionMap);
}
async queryDelete(query: KvQuery): Promise<void> {
const keys: KvKey[] = [];
for (
const { key } of await this.query(query)
) {
keys.push(key);
}
return this.batchDelete(keys);
}
}

4
plugos/lib/mq.datastore.test.ts
View File

@ -2,15 +2,15 @@ import { DataStoreMQ } from "./mq.datastore.ts";
import { assertEquals } from "../../test_deps.ts"; import { assertEquals } from "../../test_deps.ts";
import { sleep } from "$sb/lib/async.ts"; import { sleep } from "$sb/lib/async.ts";
import { DenoKvPrimitives } from "./deno_kv_primitives.ts"; import { DenoKvPrimitives } from "./deno_kv_primitives.ts";
import { DataStore } from "./datastore.ts";
import { PrefixedKvPrimitives } from "./prefixed_kv_primitives.ts"; import { PrefixedKvPrimitives } from "./prefixed_kv_primitives.ts";
import { KvDataStore } from "./kv_datastore.ts";
Deno.test("DataStore MQ", async () => { Deno.test("DataStore MQ", async () => {
const tmpFile = await Deno.makeTempFile(); const tmpFile = await Deno.makeTempFile();
const db = new DenoKvPrimitives(await Deno.openKv(tmpFile)); const db = new DenoKvPrimitives(await Deno.openKv(tmpFile));
const mq = new DataStoreMQ( const mq = new DataStoreMQ(
new DataStore(new PrefixedKvPrimitives(db, ["mq"])), new KvDataStore(new PrefixedKvPrimitives(db, ["mq"])),
); );
await mq.send("test", "Hello World"); await mq.send("test", "Hello World");
let messages = await mq.poll("test", 10); let messages = await mq.poll("test", 10);

4
plugos/lib/mq.datastore.ts
View File

@ -1,6 +1,6 @@
import { KV, MQMessage, MQStats, MQSubscribeOptions } from "$sb/types.ts"; import { KV, MQMessage, MQStats, MQSubscribeOptions } from "$sb/types.ts";
import { MessageQueue } from "./mq.ts"; import { MessageQueue } from "./mq.ts";
import { IDataStore } from "./datastore.ts"; import { DataStore } from "./datastore.ts";
export type ProcessingMessage = MQMessage & { export type ProcessingMessage = MQMessage & {
ts: number; ts: number;
@ -15,7 +15,7 @@ export class DataStoreMQ implements MessageQueue {
localSubscriptions = new Map<string, Set<() => void>>(); localSubscriptions = new Map<string, Set<() => void>>();
constructor( constructor(
private ds: IDataStore, private ds: DataStore,
) { ) {
} }

18
plugos/manifest_cache.ts
View File

@ -1,4 +1,4 @@
import { KvPrimitives } from "./lib/kv_primitives.ts"; import { DataStore } from "./lib/datastore.ts";
import { Plug } from "./plug.ts"; import { Plug } from "./plug.ts";
import { Manifest } from "./types.ts"; import { Manifest } from "./types.ts";
@ -7,25 +7,21 @@ export interface ManifestCache<T> {
} }
export class KVPrimitivesManifestCache<T> implements ManifestCache<T> { export class KVPrimitivesManifestCache<T> implements ManifestCache<T> {
constructor(private kv: KvPrimitives, private manifestPrefix: string) { constructor(private ds: DataStore, private manifestPrefix: string) {
} }
async getManifest(plug: Plug<T>, hash: number): Promise<Manifest<T>> { async getManifest(plug: Plug<T>, hash: number): Promise<Manifest<T>> {
const [cached] = await this.kv.batchGet([[ const cached = await this.ds.get([this.manifestPrefix, plug.name]);
this.manifestPrefix,
plug.name,
]]);
if (cached && cached.hash === hash) { if (cached && cached.hash === hash) {
// console.log("Using KV cached manifest for", plug.name); // console.log("Using KV cached manifest for", plug.name);
return cached.manifest; return cached.manifest;
} }
await plug.sandbox.init(); await plug.sandbox.init();
const manifest = plug.sandbox.manifest!; const manifest = plug.sandbox.manifest!;
await this.kv.batchSet([{ await this.ds.set([this.manifestPrefix, plug.name], {
key: [this.manifestPrefix, plug.name], manifest: { ...manifest, assets: undefined },
// Deliverately removing the assets from the manifest to preserve space, will be re-added upon load of actual worker hash,
value: { manifest: { ...manifest, assets: undefined }, hash }, });
}]);
return manifest; return manifest;
} }
} }

4
plugos/syscalls/datastore.ts
View File

@ -1,5 +1,5 @@
import { KV, KvKey, KvQuery } from "$sb/types.ts"; import { KV, KvKey, KvQuery } from "$sb/types.ts";
import type { IDataStore } from "../lib/datastore.ts"; import type { DataStore } from "../lib/datastore.ts";
import type { SyscallContext, SysCallMapping } from "../system.ts"; import type { SyscallContext, SysCallMapping } from "../system.ts";
/** /**
@ -8,7 +8,7 @@ import type { SyscallContext, SysCallMapping } from "../system.ts";
* @param prefix prefix to scope all keys to to which the plug name will be appended * @param prefix prefix to scope all keys to to which the plug name will be appended
*/ */
export function dataStoreSyscalls( export function dataStoreSyscalls(
ds: IDataStore, ds: DataStore,
): SysCallMapping { ): SysCallMapping {
return { return {
"datastore.delete": (ctx, key: KvKey) => { "datastore.delete": (ctx, key: KvKey) => {

5
plugos/test.plug.yaml
View File

@ -2,7 +2,4 @@ name: test
functions: functions:
boot: boot:
path: "./test_func.test.ts:hello" path: "./test_func.test.ts:hello"
endpoint:
path: "./test_func.test.ts:endpoint"
http:
path: "/"

10
plugos/test_func.test.ts
View File

@ -1,17 +1,7 @@
import * as YAML from "https://deno.land/std@0.184.0/yaml/mod.ts"; import * as YAML from "https://deno.land/std@0.184.0/yaml/mod.ts";
import { EndpointRequest, EndpointResponse } from "./hooks/endpoint.ts";
export function hello() { export function hello() {
console.log(YAML.stringify({ hello: "world" })); console.log(YAML.stringify({ hello: "world" }));
return "hello"; return "hello";
} }
export function endpoint(req: EndpointRequest): EndpointResponse {
console.log("Req", req);
return {
status: 200,
body: [1, 2, 3],
headers: { "Content-type": "application/json" },
};
}

4
plugs/editor/complete.ts
View File

@ -18,7 +18,9 @@ export async function pageComplete(completeEvent: CompleteEvent) {
completeEvent.linePrefix, completeEvent.linePrefix,
); );
const tagToQuery = isInTemplateContext ? "template" : "page"; const tagToQuery = isInTemplateContext ? "template" : "page";
let allPages: PageMeta[] = await queryObjects<PageMeta>(tagToQuery, {}); let allPages: PageMeta[] = await queryObjects<PageMeta>(tagToQuery, {
cacheSecs: 5,
});
const prefix = match[1]; const prefix = match[1];
if (prefix.startsWith("!")) { if (prefix.startsWith("!")) {
// Federation prefix, let's first see if we're matching anything from federation that is locally synced // Federation prefix, let's first see if we're matching anything from federation that is locally synced

19
server/deps.ts
View File

@ -1,11 +1,14 @@
export * from "../common/deps.ts"; export * from "../common/deps.ts";
export type { Next } from "https://deno.land/x/oak@v12.4.0/mod.ts";
export { Hono } from "https://deno.land/x/hono@v3.12.2/mod.ts";
export { export {
Application, deleteCookie,
getCookie,
setCookie,
} from "https://deno.land/x/hono@v3.12.2/helper.ts";
export { cors } from "https://deno.land/x/hono@v3.12.2/middleware.ts";
export type {
Context, Context,
Request, HonoRequest,
Response, Next,
Router, } from "https://deno.land/x/hono@v3.12.2/mod.ts";
} from "https://deno.land/x/oak@v12.4.0/mod.ts";
export * as etag from "https://deno.land/x/oak@v12.4.0/etag.ts";
export { oakCors } from "https://deno.land/x/cors@v1.2.2/mod.ts";

655
server/http_server.ts
View File

@ -1,30 +1,22 @@
import { import {
Application, deleteCookie,
Context, getCookie,
Next, Hono,
oakCors, HonoRequest,
Request, setCookie,
Router,
} from "./deps.ts"; } from "./deps.ts";
import { AssetBundle } from "../plugos/asset_bundle/bundle.ts"; import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
import { FileMeta } from "$sb/types.ts"; import { FileMeta } from "$sb/types.ts";
import { import { handleRpc } from "./rpc.ts";
handleRpc,
ShellRequest,
SyscallRequest,
SyscallResponse,
} from "./rpc.ts";
import { determineShellBackend } from "./shell_backend.ts"; import { determineShellBackend } from "./shell_backend.ts";
import { SpaceServer, SpaceServerConfig } from "./instance.ts"; import { SpaceServer, SpaceServerConfig } from "./instance.ts";
import { KvPrimitives } from "../plugos/lib/kv_primitives.ts"; import { KvPrimitives } from "../plugos/lib/kv_primitives.ts";
import { EndpointHook } from "../plugos/hooks/endpoint.ts";
import { PrefixedKvPrimitives } from "../plugos/lib/prefixed_kv_primitives.ts"; import { PrefixedKvPrimitives } from "../plugos/lib/prefixed_kv_primitives.ts";
import { base64Encode } from "../plugos/asset_bundle/base64.ts"; import { base64Encode } from "../plugos/asset_bundle/base64.ts";
const authenticationExpirySeconds = 60 * 60 * 24 * 7; // 1 week const authenticationExpirySeconds = 60 * 60 * 24 * 7; // 1 week
export type ServerOptions = { export type ServerOptions = {
app: Application;
hostname: string; hostname: string;
port: number; port: number;
clientAssetBundle: AssetBundle; clientAssetBundle: AssetBundle;
@ -42,7 +34,7 @@ export class HttpServer {
plugAssetBundle: AssetBundle; plugAssetBundle: AssetBundle;
hostname: string; hostname: string;
port: number; port: number;
app: Application<Record<string, any>>; app: Hono;
keyFile: string | undefined; keyFile: string | undefined;
certFile: string | undefined; certFile: string | undefined;
@ -51,11 +43,11 @@ export class HttpServer {
configs: Map<string, SpaceServerConfig>; configs: Map<string, SpaceServerConfig>;
constructor(options: ServerOptions) { constructor(options: ServerOptions) {
this.app = new Hono();
this.clientAssetBundle = options.clientAssetBundle; this.clientAssetBundle = options.clientAssetBundle;
this.plugAssetBundle = options.plugAssetBundle; this.plugAssetBundle = options.plugAssetBundle;
this.hostname = options.hostname; this.hostname = options.hostname;
this.port = options.port; this.port = options.port;
this.app = options.app;
this.keyFile = options.keyFile; this.keyFile = options.keyFile;
this.certFile = options.certFile; this.certFile = options.certFile;
this.baseKvPrimitives = options.baseKvPrimitives; this.baseKvPrimitives = options.baseKvPrimitives;
@ -76,8 +68,9 @@ export class HttpServer {
return spaceServer; return spaceServer;
} }
determineConfig(req: Request): [string, SpaceServerConfig] { determineConfig(req: HonoRequest): [string, SpaceServerConfig] {
let hostname = req.url.host; // hostname:port const url = new URL(req.url);
let hostname = url.host; // hostname:port
// First try a full match // First try a full match
let config = this.configs.get(hostname); let config = this.configs.get(hostname);
@ -102,7 +95,7 @@ export class HttpServer {
throw new Error(`No space server config found for hostname ${hostname}`); throw new Error(`No space server config found for hostname ${hostname}`);
} }
ensureSpaceServer(req: Request): Promise<SpaceServer> { ensureSpaceServer(req: HonoRequest): Promise<SpaceServer> {
const [matchedHostname, config] = this.determineConfig(req); const [matchedHostname, config] = this.determineConfig(req);
const spaceServer = this.spaceServers.get(matchedHostname); const spaceServer = this.spaceServers.get(matchedHostname);
if (spaceServer) { if (spaceServer) {
@ -132,22 +125,17 @@ export class HttpServer {
} }
start() { start() {
// Initialize JWT issuer
// First check if auth string (username:password) has changed
// Serve static files (javascript, css, html) // Serve static files (javascript, css, html)
this.app.use(this.serveStatic.bind(this)); this.serveStatic();
this.addAuth();
this.addAuth(this.app); this.addFsRoutes();
const fsRouter = this.addFsRoutes();
this.app.use(fsRouter.routes());
this.app.use(fsRouter.allowedMethods());
// Fallback, serve the UI index.html // Fallback, serve the UI index.html
this.app.use(async ({ request, response }) => { this.app.use("*", async (c) => {
response.headers.set("Content-type", "text/html"); const spaceServer = await this.ensureSpaceServer(c.req);
response.headers.set("Cache-Control", "no-cache"); return c.html(this.renderIndexHtml(spaceServer), 200, {
const spaceServer = await this.ensureSpaceServer(request); "Cache-Control": "no-cache",
response.body = this.renderIndexHtml(spaceServer); });
}); });
this.abortController = new AbortController(); this.abortController = new AbortController();
@ -162,11 +150,12 @@ export class HttpServer {
if (this.certFile) { if (this.certFile) {
listenOptions.cert = Deno.readTextFileSync(this.certFile); listenOptions.cert = Deno.readTextFileSync(this.certFile);
} }
this.app.listen(listenOptions) Deno.serve(listenOptions, this.app.fetch);
.catch((e: any) => { // this.app.listen(listenOptions)
console.log("Server listen error:", e.message); // .catch((e: any) => {
Deno.exit(1); // console.log("Server listen error:", e.message);
}); // Deno.exit(1);
// });
const visibleHostname = this.hostname === "0.0.0.0" const visibleHostname = this.hostname === "0.0.0.0"
? "localhost" ? "localhost"
: this.hostname; : this.hostname;
@ -175,73 +164,75 @@ export class HttpServer {
); );
} }
async serveStatic( serveStatic() {
{ request, response }: Context<Record<string, any>, Record<string, any>>, this.app.use("*", async (c, next) => {
next: Next, const req = c.req;
) { const spaceServer = await this.ensureSpaceServer(req);
const spaceServer = await this.ensureSpaceServer(request); const url = new URL(req.url);
if ( // console.log("URL", url);
request.url.pathname === "/"
) {
// Serve the UI (index.html)
// Note: we're explicitly not setting Last-Modified and If-Modified-Since header here because this page is dynamic
response.headers.set("Content-type", "text/html");
response.headers.set("Cache-Control", "no-cache");
response.body = this.renderIndexHtml(spaceServer);
return;
}
try {
const assetName = request.url.pathname.slice(1);
if ( if (
this.clientAssetBundle.has(assetName) && url.pathname === "/"
request.headers.get("If-Modified-Since") ===
utcDateString(this.clientAssetBundle.getMtime(assetName)) &&
assetName !== "service_worker.js"
) { ) {
response.status = 304; // Serve the UI (index.html)
return; // Note: we're explicitly not setting Last-Modified and If-Modified-Since header here because this page is dynamic
return c.html(this.renderIndexHtml(spaceServer), 200, {
"Cache-Control": "no-cache",
});
} }
response.status = 200; try {
response.headers.set( const assetName = url.pathname.slice(1);
"Content-type", if (!this.clientAssetBundle.has(assetName)) {
this.clientAssetBundle.getMimeType(assetName), return next();
); }
let data: Uint8Array | string = this.clientAssetBundle.readFileSync( console.log("Asset name", assetName);
assetName, if (
); this.clientAssetBundle.has(assetName) &&
response.headers.set("Cache-Control", "no-cache"); req.header("If-Modified-Since") ===
response.headers.set("Content-length", "" + data.length); utcDateString(this.clientAssetBundle.getMtime(assetName)) &&
if (assetName !== "service_worker.js") { assetName !== "service_worker.js"
response.headers.set( ) {
"Last-Modified", return c.text("", 304);
utcDateString(this.clientAssetBundle.getMtime(assetName)), }
c.status(200);
c.header("Content-type", this.clientAssetBundle.getMimeType(assetName));
let data: Uint8Array | string = this.clientAssetBundle.readFileSync(
assetName,
); );
} c.header("Cache-Control", "no-cache");
c.header("Content-length", "" + data.length);
if (request.method === "GET") { if (assetName !== "service_worker.js") {
if (assetName === "service_worker.js") { c.header(
const textData = new TextDecoder().decode(data); "Last-Modified",
// console.log( utcDateString(this.clientAssetBundle.getMtime(assetName)),
// "Swapping out config hash in service worker",
// );
data = textData.replaceAll(
"{{CONFIG_HASH}}",
base64Encode(
JSON.stringify([
spaceServer.clientEncryption,
spaceServer.syncOnly,
]),
),
); );
} }
response.body = data; // console.log("Serving it now", assetName);
if (req.method === "GET") {
if (assetName === "service_worker.js") {
const textData = new TextDecoder().decode(data);
// console.log(
// "Swapping out config hash in service worker",
// );
data = textData.replaceAll(
"{{CONFIG_HASH}}",
base64Encode(
JSON.stringify([
spaceServer.clientEncryption,
spaceServer.syncOnly,
]),
),
);
}
return c.body(data);
} // else e.g. HEAD, OPTIONS, don't send body
} catch {
return next();
} }
} catch { });
return next();
}
} }
private addAuth(app: Application) { private addAuth() {
const excludedPaths = [ const excludedPaths = [
"/manifest.json", "/manifest.json",
"/favicon.png", "/favicon.png",
@ -250,70 +241,63 @@ export class HttpServer {
]; ];
// Middleware handling the /.auth page and flow // Middleware handling the /.auth page and flow
app.use(async ({ request, response, cookies }, next) => { this.app.all("/.auth", async (c) => {
const host = request.url.host; // e.g. localhost:3000 const url = new URL(c.req.url);
if (request.url.pathname === "/.auth") { const req = c.req;
if (request.url.search === "?logout") { const host = url.host; // e.g. localhost:3000
await cookies.delete(authCookieName(host)); if (url.search === "?logout") {
// Implicit fallthrough to login page deleteCookie(c, authCookieName(host));
} }
if (request.method === "GET") { if (req.method === "GET") {
response.headers.set("Content-type", "text/html"); return c.html(
response.body = this.clientAssetBundle.readTextFileSync( this.clientAssetBundle.readTextFileSync(".client/auth.html"),
".client/auth.html", );
} else if (req.method === "POST") {
const values = await c.req.parseBody();
const username = values["username"];
const password = values["password"];
const spaceServer = await this.ensureSpaceServer(req);
const { user: expectedUser, pass: expectedPassword } = spaceServer
.auth!;
if (username === expectedUser && password === expectedPassword) {
// Generate a JWT and set it as a cookie
const jwt = await spaceServer.jwtIssuer.createJWT(
{ username },
authenticationExpirySeconds,
); );
return; console.log("Successful auth");
} else if (request.method === "POST") { setCookie(c, authCookieName(host), jwt, {
const values = await request.body({ type: "form" }).value; expires: new Date(
const username = values.get("username")!; Date.now() + authenticationExpirySeconds * 1000,
const password = values.get("password")!; ), // in a week
const spaceServer = await this.ensureSpaceServer(request); // sameSite: "Strict",
const { user: expectedUser, pass: expectedPassword } = spaceServer // httpOnly: true,
.auth!; });
if (username === expectedUser && password === expectedPassword) { return c.redirect("/");
// Generate a JWT and set it as a cookie
const jwt = await spaceServer.jwtIssuer.createJWT(
{ username },
authenticationExpirySeconds,
);
await cookies.set(
authCookieName(host),
jwt,
{
expires: new Date(
Date.now() + authenticationExpirySeconds * 1000,
), // in a week
sameSite: "strict",
},
);
response.redirect("/");
} else {
response.redirect("/.auth?error=1");
}
return;
} else { } else {
response.redirect("/.auth"); return c.redirect("/.auth?error=1");
return;
} }
} else { } else {
await next(); return c.redirect("/.auth");
} }
}); });
// Check auth // Check auth
app.use(async ({ request, response, cookies }, next) => { this.app.use("*", async (c, next) => {
const spaceServer = await this.ensureSpaceServer(request); const req = c.req;
const spaceServer = await this.ensureSpaceServer(req);
if (!spaceServer.auth) { if (!spaceServer.auth) {
// Auth disabled in this config, skip // Auth disabled in this config, skip
return next(); return next();
} }
const host = request.url.host; const url = new URL(req.url);
if (!excludedPaths.includes(request.url.pathname)) { const host = url.host;
const authToken = await cookies.get(authCookieName(host)); if (!excludedPaths.includes(url.pathname)) {
const authCookie = getCookie(c, authCookieName(host));
if (!authToken && spaceServer.authToken) { if (!authCookie && spaceServer.authToken) {
// Attempt Bearer Authorization based authentication // Attempt Bearer Authorization based authentication
const authHeader = request.headers.get("Authorization"); const authHeader = req.header("Authorization");
if (authHeader && authHeader.startsWith("Bearer ")) { if (authHeader && authHeader.startsWith("Bearer ")) {
const authToken = authHeader.slice("Bearer ".length); const authToken = authHeader.slice("Bearer ".length);
if (authToken === spaceServer.authToken) { if (authToken === spaceServer.authToken) {
@ -323,21 +307,19 @@ export class HttpServer {
console.log( console.log(
"Unauthorized token access, redirecting to auth page", "Unauthorized token access, redirecting to auth page",
); );
response.status = 401; return c.text("Unauthorized", 401);
response.body = "Unauthorized";
return;
} }
} }
} }
if (!authToken) { if (!authCookie) {
console.log("Unauthorized access, redirecting to auth page"); console.log("Unauthorized access, redirecting to auth page");
return response.redirect("/.auth"); return c.redirect("/.auth");
} }
const { user: expectedUser } = spaceServer.auth!; const { user: expectedUser } = spaceServer.auth!;
try { try {
const verifiedJwt = await spaceServer.jwtIssuer.verifyAndDecodeJWT( const verifiedJwt = await spaceServer.jwtIssuer.verifyAndDecodeJWT(
authToken, authCookie,
); );
if (verifiedJwt.username !== expectedUser) { if (verifiedJwt.username !== expectedUser) {
throw new Error("Username mismatch"); throw new Error("Username mismatch");
@ -347,214 +329,191 @@ export class HttpServer {
"Error verifying JWT, redirecting to auth page", "Error verifying JWT, redirecting to auth page",
e.message, e.message,
); );
return response.redirect("/.auth"); return c.redirect("/.auth");
} }
} }
return next(); return next();
}); });
} }
private addFsRoutes(): Router { private addFsRoutes() {
const fsRouter = new Router(); // this.app.use(
const corsMiddleware = oakCors({ // "*",
allowedHeaders: "*", // cors({
exposedHeaders: "*", // origin: "*",
methods: ["GET", "POST", "PUT", "DELETE", "HEAD", "OPTIONS"], // allowHeaders: ["*"],
}); // exposeHeaders: ["*"],
// allowMethods: ["GET", "POST", "PUT", "DELETE", "HEAD", "OPTIONS"],
fsRouter.use(corsMiddleware); // }),
// );
// File list // File list
fsRouter.get( this.app.get(
"/index.json", "/index.json",
// corsMiddleware, async (c) => {
async ({ request, response }) => { const req = c.req;
const spaceServer = await this.ensureSpaceServer(request); const spaceServer = await this.ensureSpaceServer(req);
if (request.headers.has("X-Sync-Mode")) { if (req.header("X-Sync-Mode")) {
// Only handle direct requests for a JSON representation of the file list // Only handle direct requests for a JSON representation of the file list
response.headers.set("Content-type", "application/json");
response.headers.set("X-Space-Path", spaceServer.pagesPath);
const files = await spaceServer.spacePrimitives.fetchFileList(); const files = await spaceServer.spacePrimitives.fetchFileList();
response.body = JSON.stringify(files); return c.json(files, 200, {
"X-Space-Path": spaceServer.pagesPath,
});
} else { } else {
// Otherwise, redirect to the UI // Otherwise, redirect to the UI
// The reason to do this is to handle authentication systems like Authelia nicely // The reason to do this is to handle authentication systems like Authelia nicely
response.redirect("/"); return c.redirect("/");
} }
}, },
); );
// RPC // RPC
fsRouter.post("/.rpc/(.+)", async ({ request, response, params }) => { this.app.post("/.rpc/:operation", async (c) => {
const operation = params[0]; const req = c.req;
const spaceServer = await this.ensureSpaceServer(request); const operation = req.param("operation")!;
const body = await request.body({ const spaceServer = await this.ensureSpaceServer(req);
type: "json", req.parseBody;
limit: 100 * 1024 * 1024, const body = await req.json();
}).value;
try { try {
const resp = await handleRpc(spaceServer, operation, body); const resp = await handleRpc(spaceServer, operation, body);
response.headers.set("Content-Type", "application/json"); return c.json({ r: resp }, 200);
response.body = JSON.stringify({ r: resp });
} catch (e: any) { } catch (e: any) {
console.log("Error", e); return c.text(e.message, 500);
response.status = 500;
response.body = e.message;
return;
} }
}); });
const filePathRegex = "\/([^!].+\\.[a-zA-Z]+)"; const filePathRegex = "/:path{[^!].+\\.[a-zA-Z]+}";
fsRouter this.app.get(
.get( filePathRegex,
filePathRegex, async (c) => {
async ({ params, response, request }) => { const req = c.req;
const name = params[0]; const name = req.param("path")!;
const spaceServer = await this.ensureSpaceServer(request); const spaceServer = await this.ensureSpaceServer(req);
console.log( console.log(
"Requested file", "Requested file",
name, name,
);
if (
name.endsWith(".md") &&
// This header signififies the requests comes directly from the http_space_primitives client (not the browser)
!req.header("X-Sync-Mode") &&
// This Accept header is used by federation to still work with CORS
req.header("Accept") !==
"application/octet-stream" &&
req.header("sec-fetch-mode") !== "cors"
) {
// It can happen that during a sync, authentication expires, this may result in a redirect to the login page and then back to this particular file. This particular file may be an .md file, which isn't great to show so we're redirecting to the associated SB UI page.
console.warn(
"Request was without X-Sync-Mode nor a CORS request, redirecting to page",
); );
if ( return c.redirect(`/${name.slice(0, -3)}`);
name.endsWith(".md") && }
// This header signififies the requests comes directly from the http_space_primitives client (not the browser)
!request.headers.has("X-Sync-Mode") &&
// This Accept header is used by federation to still work with CORS
request.headers.get("Accept") !==
"application/octet-stream" &&
request.headers.get("sec-fetch-mode") !== "cors"
) {
// It can happen that during a sync, authentication expires, this may result in a redirect to the login page and then back to this particular file. This particular file may be an .md file, which isn't great to show so we're redirecting to the associated SB UI page.
console.warn(
"Request was without X-Sync-Mode nor a CORS request, redirecting to page",
);
response.redirect(`/${name.slice(0, -3)}`);
return;
}
if (name.startsWith(".")) {
// Don't expose hidden files
response.status = 404;
response.body = "Not exposed";
return;
}
// Handle federated links through a simple redirect, only used for attachments loads with service workers disabled
if (name.startsWith("!")) {
let url = name.slice(1);
console.log("Handling this as a federated link", url);
if (url.startsWith("localhost")) {
url = `http://${url}`;
} else {
url = `https://${url}`;
}
try {
const req = await fetch(url);
response.status = req.status;
// Override X-Permssion header to always be "ro"
const newHeaders = new Headers();
for (const [key, value] of req.headers.entries()) {
newHeaders.set(key, value);
}
newHeaders.set("X-Permission", "ro");
response.headers = newHeaders;
response.body = req.body;
} catch (e: any) {
console.error("Error fetching federated link", e);
response.status = 500;
response.body = e.message;
}
return;
}
try {
if (request.headers.has("X-Get-Meta")) {
// Getting meta via GET request
const fileData = await spaceServer.spacePrimitives.getFileMeta(
name,
);
response.status = 200;
this.fileMetaToHeaders(response.headers, fileData);
response.body = "";
return;
}
const fileData = await spaceServer.spacePrimitives.readFile(name);
const lastModifiedHeader = new Date(fileData.meta.lastModified)
.toUTCString();
if (
request.headers.get("If-Modified-Since") === lastModifiedHeader
) {
response.status = 304;
return;
}
response.status = 200;
this.fileMetaToHeaders(response.headers, fileData.meta);
response.headers.set("Last-Modified", lastModifiedHeader);
response.body = fileData.data;
} catch (e: any) {
console.error("Error GETting file", name, e.message);
response.status = 404;
response.headers.set("Cache-Control", "no-cache");
response.body = "Not found";
}
},
)
.put(
filePathRegex,
async ({ request, response, params }) => {
const name = params[0];
const spaceServer = await this.ensureSpaceServer(request);
console.log("Saving file", name);
if (name.startsWith(".")) {
// Don't expose hidden files
response.status = 403;
return;
}
const body = await request.body({ type: "bytes" }).value;
try {
const meta = await spaceServer.spacePrimitives.writeFile(
name,
body,
);
response.status = 200;
this.fileMetaToHeaders(response.headers, meta);
response.body = "OK";
} catch (err) {
console.error("Write failed", err);
response.status = 500;
response.body = "Write failed";
}
},
)
.delete(filePathRegex, async ({ request, response, params }) => {
const name = params[0];
const spaceServer = await this.ensureSpaceServer(request);
console.log("Deleting file", name);
if (name.startsWith(".")) { if (name.startsWith(".")) {
// Don't expose hidden files // Don't expose hidden files
response.status = 403; return c.notFound();
return; }
// Handle federated links through a simple redirect, only used for attachments loads with service workers disabled
if (name.startsWith("!")) {
let url = name.slice(1);
console.log("Handling this as a federated link", url);
if (url.startsWith("localhost")) {
url = `http://${url}`;
} else {
url = `https://${url}`;
}
try {
const req = await fetch(url);
// Override X-Permssion header to always be "ro"
const newHeaders = new Headers();
for (const [key, value] of req.headers.entries()) {
newHeaders.set(key, value);
}
newHeaders.set("X-Permission", "ro");
return new Response(req.body, {
status: req.status,
headers: newHeaders,
});
} catch (e: any) {
console.error("Error fetching federated link", e);
return c.text(e.message, 500);
}
} }
try { try {
await spaceServer.spacePrimitives.deleteFile(name); if (req.header("X-Get-Meta")) {
response.status = 200; // Getting meta via GET request
response.body = "OK"; const fileData = await spaceServer.spacePrimitives.getFileMeta(
name,
);
return c.text("", 200, this.fileMetaToHeaders(fileData));
}
const fileData = await spaceServer.spacePrimitives.readFile(name);
const lastModifiedHeader = new Date(fileData.meta.lastModified)
.toUTCString();
if (
req.header("If-Modified-Since") === lastModifiedHeader
) {
return c.text("", 304);
}
return c.body(fileData.data, 200, {
...this.fileMetaToHeaders(fileData.meta),
"Last-Modified": lastModifiedHeader,
});
} catch (e: any) { } catch (e: any) {
console.error("Error deleting attachment", e); console.error("Error GETting file", name, e.message);
response.status = 500; return c.notFound();
response.body = e.message;
} }
}) },
.options(filePathRegex, corsMiddleware); ).put(
async (c) => {
const req = c.req;
const name = req.param("path")!;
const spaceServer = await this.ensureSpaceServer(req);
console.log("Saving file", name);
if (name.startsWith(".")) {
// Don't expose hidden files
return c.text("Forbidden", 403);
}
const body = await req.arrayBuffer();
try {
const meta = await spaceServer.spacePrimitives.writeFile(
name,
new Uint8Array(body),
);
return c.text("OK", 200, this.fileMetaToHeaders(meta));
} catch (err) {
console.error("Write failed", err);
return c.text("Write failed", 500);
}
},
).delete(async (c) => {
const req = c.req;
const name = req.param("path")!;
const spaceServer = await this.ensureSpaceServer(req);
console.log("Deleting file", name);
if (name.startsWith(".")) {
// Don't expose hidden files
return c.text("Forbidden", 403);
}
try {
await spaceServer.spacePrimitives.deleteFile(name);
return c.text("OK");
} catch (e: any) {
console.error("Error deleting attachment", e);
return c.text(e.message, 500);
}
}).options();
// Federation proxy // Federation proxy
const proxyPathRegex = "\/!(.+)"; const proxyPathRegex = "/:uri{!.+}";
fsRouter.all( this.app.all(
proxyPathRegex, proxyPathRegex,
async ({ params, response, request }, next) => { async (c, next) => {
let url = params[0]; const req = c.req;
if (!request.headers.has("X-Proxy-Request")) { let url = req.param("uri")!.slice(1);
if (!req.header("X-Proxy-Request")) {
// Direct browser request, not explicity fetch proxy request // Direct browser request, not explicity fetch proxy request
if (!/\.[a-zA-Z0-9]+$/.test(url)) { if (!/\.[a-zA-Z0-9]+$/.test(url)) {
console.log("Directly loading federation page via URL:", url); console.log("Directly loading federation page via URL:", url);
@ -572,47 +531,41 @@ export class HttpServer {
for ( for (
const headerName of ["Authorization", "Accept", "Content-Type"] const headerName of ["Authorization", "Accept", "Content-Type"]
) { ) {
if (request.headers.has(headerName)) { if (req.header(headerName)) {
safeRequestHeaders.set( safeRequestHeaders.set(
headerName, headerName,
request.headers.get(headerName)!, req.header(headerName)!,
); );
} }
} }
const req = await fetch(url, { const body = await req.arrayBuffer();
method: request.method, const fetchReq = await fetch(url, {
method: req.method,
headers: safeRequestHeaders, headers: safeRequestHeaders,
body: request.hasBody body: body.byteLength > 0 ? body : undefined,
? request.body({ type: "stream" }).value
: undefined,
}); });
response.status = req.status; const responseHeaders: Record<string, any> = {};
response.headers = req.headers; for (const [key, value] of fetchReq.headers.entries()) {
response.body = req.body; responseHeaders[key] = value;
}
return c.body(fetchReq.body, fetchReq.status, responseHeaders);
} catch (e: any) { } catch (e: any) {
console.error("Error fetching federated link", e); console.error("Error fetching federated link", e);
response.status = 500; return c.text(e.message, 500);
response.body = e.message;
} }
return;
}, },
); );
return fsRouter;
} }
private fileMetaToHeaders(headers: Headers, fileMeta: FileMeta) { private fileMetaToHeaders(fileMeta: FileMeta) {
headers.set("Content-Type", fileMeta.contentType); return {
headers.set( "Content-Type": fileMeta.contentType,
"X-Last-Modified", "X-Last-Modified": "" + fileMeta.lastModified,
"" + fileMeta.lastModified, "X-Created": "" + fileMeta.created,
); "Cache-Control": "no-cache",
headers.set( "X-Permission": fileMeta.perm,
"X-Created", "X-Content-Length": "" + fileMeta.size,
"" + fileMeta.created, };
);
headers.set("Cache-Control", "no-cache");
headers.set("X-Permission", fileMeta.perm);
headers.set("X-Content-Length", "" + fileMeta.size);
} }
stop() { stop() {
@ -628,5 +581,5 @@ function utcDateString(mtime: number): string {
} }
function authCookieName(host: string) { function authCookieName(host: string) {
return `auth:${host}`; return `auth_${host.replaceAll(/\W/g, "_")}`;
} }

9
server/instance.ts
View File

@ -1,14 +1,11 @@
import { AssetBundlePlugSpacePrimitives } from "../common/spaces/asset_bundle_space_primitives.ts"; import { AssetBundlePlugSpacePrimitives } from "../common/spaces/asset_bundle_space_primitives.ts";
import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts";
import { SpacePrimitives } from "../common/spaces/space_primitives.ts"; import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
import { ensureSettingsAndIndex } from "../common/util.ts";
import { AssetBundle } from "../plugos/asset_bundle/bundle.ts"; import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
import { DataStore } from "../plugos/lib/datastore.ts"; import type { DataStore } from "../plugos/lib/datastore.ts";
import { KvDataStore } from "../plugos/lib/kv_datastore.ts";
import { KvPrimitives } from "../plugos/lib/kv_primitives.ts"; import { KvPrimitives } from "../plugos/lib/kv_primitives.ts";
import { PrefixedKvPrimitives } from "../plugos/lib/prefixed_kv_primitives.ts"; import { PrefixedKvPrimitives } from "../plugos/lib/prefixed_kv_primitives.ts";
import { BuiltinSettings } from "../web/types.ts";
import { JWTIssuer } from "./crypto.ts"; import { JWTIssuer } from "./crypto.ts";
import { gitIgnoreCompiler } from "./deps.ts";
import { ShellBackend } from "./shell_backend.ts"; import { ShellBackend } from "./shell_backend.ts";
import { determineStorageBackend } from "./storage_backend.ts"; import { determineStorageBackend } from "./storage_backend.ts";
@ -57,7 +54,7 @@ export class SpaceServer {
} }
this.jwtIssuer = new JWTIssuer(kvPrimitives); this.jwtIssuer = new JWTIssuer(kvPrimitives);
this.ds = new DataStore(new PrefixedKvPrimitives(kvPrimitives, ["ds"])); this.ds = new KvDataStore(new PrefixedKvPrimitives(kvPrimitives, ["ds"]));
} }
async init() { async init() {

11
web/client.ts
View File

@ -39,7 +39,7 @@ import { MainUI } from "./editor_ui.tsx";
import { cleanPageRef } from "$sb/lib/resolve.ts"; import { cleanPageRef } from "$sb/lib/resolve.ts";
import { SpacePrimitives } from "../common/spaces/space_primitives.ts"; import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
import { CodeWidgetButton, FileMeta, PageMeta } from "$sb/types.ts"; import { CodeWidgetButton, FileMeta, PageMeta } from "$sb/types.ts";
import { DataStore, IDataStore } from "../plugos/lib/datastore.ts"; import type { DataStore } from "../plugos/lib/datastore.ts";
import { IndexedDBKvPrimitives } from "../plugos/lib/indexeddb_kv_primitives.ts"; import { IndexedDBKvPrimitives } from "../plugos/lib/indexeddb_kv_primitives.ts";
import { DataStoreMQ } from "../plugos/lib/mq.datastore.ts"; import { DataStoreMQ } from "../plugos/lib/mq.datastore.ts";
import { DataStoreSpacePrimitives } from "../common/spaces/datastore_space_primitives.ts"; import { DataStoreSpacePrimitives } from "../common/spaces/datastore_space_primitives.ts";
@ -52,6 +52,7 @@ import {
markFullSpaceIndexComplete, markFullSpaceIndexComplete,
} from "../common/space_index.ts"; } from "../common/space_index.ts";
import { RemoteDataStore } from "./remote_datastore.ts"; import { RemoteDataStore } from "./remote_datastore.ts";
import { KvDataStore } from "../plugos/lib/kv_datastore.ts";
const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/; const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/;
const autoSaveInterval = 1000; const autoSaveInterval = 1000;
@ -114,7 +115,7 @@ export class Client {
public allKnownPages = new Set<string>(); public allKnownPages = new Set<string>();
clientDS!: DataStore; clientDS!: DataStore;
mq!: DataStoreMQ; mq!: DataStoreMQ;
ds!: IDataStore; ds!: DataStore;
constructor( constructor(
private parent: Element, private parent: Element,
@ -140,7 +141,7 @@ export class Client {
`${this.dbPrefix}_state`, `${this.dbPrefix}_state`,
); );
await stateKvPrimitives.init(); await stateKvPrimitives.init();
this.clientDS = new DataStore(stateKvPrimitives); this.clientDS = new KvDataStore(stateKvPrimitives);
// In sync mode, reuse the clientDS, otherwise talk to a remote data store (over HTTP) // In sync mode, reuse the clientDS, otherwise talk to a remote data store (over HTTP)
this.ds = this.syncMode this.ds = this.syncMode
@ -198,7 +199,7 @@ export class Client {
this.focus(); this.focus();
await this.system.init(); this.system.init();
// Load settings // Load settings
this.settings = await ensureSettingsAndIndex(localSpacePrimitives); this.settings = await ensureSettingsAndIndex(localSpacePrimitives);
@ -485,7 +486,7 @@ export class Client {
new EventedSpacePrimitives( new EventedSpacePrimitives(
// Using fallback space primitives here to allow (by default) local reads to "fall through" to HTTP when files aren't synced yet // Using fallback space primitives here to allow (by default) local reads to "fall through" to HTTP when files aren't synced yet
new FallbackSpacePrimitives( new FallbackSpacePrimitives(
new DataStoreSpacePrimitives(new DataStore(spaceKvPrimitives)), new DataStoreSpacePrimitives(new KvDataStore(spaceKvPrimitives)),
this.plugSpaceRemotePrimitives, this.plugSpaceRemotePrimitives,
), ),
this.eventHook, this.eventHook,

8
web/client_system.ts
View File

@ -30,7 +30,7 @@ import {
import { MQHook } from "../plugos/hooks/mq.ts"; import { MQHook } from "../plugos/hooks/mq.ts";
import { mqSyscalls } from "../plugos/syscalls/mq.ts"; import { mqSyscalls } from "../plugos/syscalls/mq.ts";
import { dataStoreSyscalls } from "../plugos/syscalls/datastore.ts"; import { dataStoreSyscalls } from "../plugos/syscalls/datastore.ts";
import { DataStore, IDataStore } from "../plugos/lib/datastore.ts"; import { DataStore } from "../plugos/lib/datastore.ts";
import { MessageQueue } from "../plugos/lib/mq.ts"; import { MessageQueue } from "../plugos/lib/mq.ts";
import { languageSyscalls } from "../common/syscalls/language.ts"; import { languageSyscalls } from "../common/syscalls/language.ts";
import { handlebarsSyscalls } from "../common/syscalls/handlebars.ts"; import { handlebarsSyscalls } from "../common/syscalls/handlebars.ts";
@ -56,7 +56,7 @@ export class ClientSystem {
private client: Client, private client: Client,
private mq: MessageQueue, private mq: MessageQueue,
private clientDs: DataStore, private clientDs: DataStore,
private dataStore: IDataStore, private dataStore: DataStore,
private eventHook: EventHook, private eventHook: EventHook,
) { ) {
// Only set environment to "client" when running in thin client mode, otherwise we run everything locally (hybrid) // Only set environment to "client" when running in thin client mode, otherwise we run everything locally (hybrid)
@ -65,7 +65,7 @@ export class ClientSystem {
undefined, undefined,
{ {
manifestCache: new KVPrimitivesManifestCache<SilverBulletHooks>( manifestCache: new KVPrimitivesManifestCache<SilverBulletHooks>(
clientDs.kv, clientDs,
"manifest", "manifest",
), ),
}, },
@ -145,7 +145,7 @@ export class ClientSystem {
); );
} }
async init() { init() {
// Slash command hook // Slash command hook
this.slashCommandHook = new SlashCommandHook(this.client); this.slashCommandHook = new SlashCommandHook(this.client);
this.system.addHook(this.slashCommandHook); this.system.addHook(this.slashCommandHook);

39
web/remote_datastore.ts
View File

@ -1,11 +1,15 @@
import { HttpSpacePrimitives } from "../common/spaces/http_space_primitives.ts"; import { HttpSpacePrimitives } from "../common/spaces/http_space_primitives.ts";
import { KV, KvKey, KvQuery } from "$sb/types.ts"; import { KV, KvKey, KvQuery } from "$sb/types.ts";
import { IDataStore } from "../plugos/lib/datastore.ts"; import { DataStore } from "../plugos/lib/datastore.ts";
import { rpcCall } from "./syscalls/datastore.proxy.ts"; import { rpcCall } from "./syscalls/datastore.proxy.ts";
import { LimitedMap } from "../common/limited_map.ts";
// implements DataStore "interface" export class RemoteDataStore implements DataStore {
export class RemoteDataStore implements IDataStore { private cache = new LimitedMap<any>(20);
constructor(private httpPrimitives: HttpSpacePrimitives) {
constructor(
private httpPrimitives: HttpSpacePrimitives,
) {
} }
private proxy( private proxy(
@ -45,8 +49,31 @@ export class RemoteDataStore implements IDataStore {
return this.proxy("datastore.batchDelete", keys); return this.proxy("datastore.batchDelete", keys);
} }
query<T = any>(query: KvQuery): Promise<KV<T>[]> { /**
return this.proxy("datastore.query", query); * Proxies the query to the server, and caches the result if cacheSecs is set
* @param query query to execute
* @returns
*/
async query<T = any>(query: KvQuery): Promise<KV<T>[]> {
let cacheKey: string | undefined;
const cacheSecs = query.cacheSecs;
// Should we do caching?
if (cacheSecs) {
// Remove the cacheSecs from the query
query = { ...query, cacheSecs: undefined };
cacheKey = JSON.stringify(query);
const cachedResult = this.cache.get(cacheKey);
if (cachedResult) {
// Let's use the cached result
return cachedResult;
}
}
const result = await this.proxy("datastore.query", query);
if (cacheKey) {
// Store in the cache
this.cache.set(cacheKey, result, cacheSecs! * 1000);
}
return result;
} }
queryDelete(query: KvQuery): Promise<void> { queryDelete(query: KvQuery): Promise<void> {

3
web/service_worker.ts
View File

@ -2,6 +2,7 @@ import type { FileContent } from "../common/spaces/datastore_space_primitives.ts
import { simpleHash } from "../common/crypto.ts"; import { simpleHash } from "../common/crypto.ts";
import { DataStore } from "../plugos/lib/datastore.ts"; import { DataStore } from "../plugos/lib/datastore.ts";
import { IndexedDBKvPrimitives } from "../plugos/lib/indexeddb_kv_primitives.ts"; import { IndexedDBKvPrimitives } from "../plugos/lib/indexeddb_kv_primitives.ts";
import { KvDataStore } from "../plugos/lib/kv_datastore.ts";
const CACHE_NAME = "{{CACHE_NAME}}_{{CONFIG_HASH}}"; const CACHE_NAME = "{{CACHE_NAME}}_{{CONFIG_HASH}}";
@ -182,7 +183,7 @@ self.addEventListener("message", (event: any) => {
// Setup space // Setup space
const kv = new IndexedDBKvPrimitives(`${dbPrefix}_synced_space`); const kv = new IndexedDBKvPrimitives(`${dbPrefix}_synced_space`);
kv.init().then(() => { kv.init().then(() => {
ds = new DataStore(kv); ds = new KvDataStore(kv);
console.log("Datastore in service worker initialized..."); console.log("Datastore in service worker initialized...");
}); });
} }

3
web/space.ts
View File

@ -4,9 +4,6 @@ import { safeRun } from "../common/util.ts";
import { AttachmentMeta, FileMeta, PageMeta } from "$sb/types.ts"; import { AttachmentMeta, FileMeta, PageMeta } from "$sb/types.ts";
import { EventHook } from "../plugos/hooks/event.ts"; import { EventHook } from "../plugos/hooks/event.ts";
import { throttle } from "$sb/lib/async.ts";
import { DataStore } from "../plugos/lib/datastore.ts";
import { LimitedMap } from "../common/limited_map.ts";
const pageWatchInterval = 5000; const pageWatchInterval = 5000;