mingzailao
/
silverbullet
mirror of https://github.com/silverbulletmd/silverbullet.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactoring work to support multi-tenancy and multiple storage, database backends (#598) 

* Backend infrastructure
* New backend configuration work
* Factor out KV prefixing
* Don't put assets in the manifest cache
* Removed fancy authentication stuff
* Documentation updates
pull/599/head
Zef Hemel 2023-12-10 13:23:42 +01:00 committed by GitHub
parent 573eca3676
commit 30ba3fcca7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 647 additions and 781 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cli/plug_run.test.ts
View File

@ -30,7 +30,6 @@ Deno.test("Test plug run", {
assertEquals( assertEquals(
await runPlug( await runPlug(
testSpaceFolder, testSpaceFolder,
tempDbFile,
"test.run", "test.run",
[], [],
assetBundle, assetBundle,

21
cli/plug_run.ts
View File

@ -5,10 +5,11 @@ import { Application } from "../server/deps.ts";
import { sleep } from "$sb/lib/async.ts"; import { sleep } from "$sb/lib/async.ts";
import { ServerSystem } from "../server/server_system.ts"; import { ServerSystem } from "../server/server_system.ts";
import { AssetBundlePlugSpacePrimitives } from "../common/spaces/asset_bundle_space_primitives.ts"; import { AssetBundlePlugSpacePrimitives } from "../common/spaces/asset_bundle_space_primitives.ts";
import { determineDatabaseBackend } from "../server/db_backend.ts";
import { EndpointHook } from "../plugos/hooks/endpoint.ts";
export async function runPlug( export async function runPlug(
spacePath: string, spacePath: string,
dbPath: string,
functionName: string | undefined, functionName: string | undefined,
args: string[] = [], args: string[] = [],
builtinAssetBundle: AssetBundle, builtinAssetBundle: AssetBundle,
@ -18,15 +19,27 @@ export async function runPlug(
const serverController = new AbortController(); const serverController = new AbortController();
const app = new Application(); const app = new Application();
const dbBackend = await determineDatabaseBackend();
if (!dbBackend) {
console.error("Cannot run plugs in databaseless mode.");
return;
}
const endpointHook = new EndpointHook("/_/");
const serverSystem = new ServerSystem( const serverSystem = new ServerSystem(
new AssetBundlePlugSpacePrimitives( new AssetBundlePlugSpacePrimitives(
new DiskSpacePrimitives(spacePath), new DiskSpacePrimitives(spacePath),
builtinAssetBundle, builtinAssetBundle,
), ),
dbPath, dbBackend,
app,
); );
await serverSystem.init(true); await serverSystem.init(true);
app.use((context, next) => {
return endpointHook.handleRequest(serverSystem.system!, context, next);
});
app.listen({ app.listen({
hostname: httpHostname, hostname: httpHostname,
port: httpServerPort, port: httpServerPort,
@ -42,7 +55,7 @@ export async function runPlug(
} }
const result = await plug.invoke(funcName, args); const result = await plug.invoke(funcName, args);
await serverSystem.close(); await serverSystem.close();
serverSystem.denoKv.close(); serverSystem.kvPrimitives.close();
serverController.abort(); serverController.abort();
return result; return result;
} else { } else {

8
cmd/plug_run.ts
View File

@ -4,15 +4,12 @@ import assets from "../dist/plug_asset_bundle.json" assert {
type: "json", type: "json",
}; };
import { AssetBundle } from "../plugos/asset_bundle/bundle.ts"; import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
import { silverBulletDbFile } from "./constants.ts";
export async function plugRunCommand( export async function plugRunCommand(
{ {
db,
hostname, hostname,
port, port,
}: { }: {
db?: string;
hostname?: string; hostname?: string;
port?: number; port?: number;
}, },
@ -22,15 +19,10 @@ export async function plugRunCommand(
) { ) {
spacePath = path.resolve(spacePath); spacePath = path.resolve(spacePath);
console.log("Space path", spacePath); console.log("Space path", spacePath);
let dbPath = path.resolve(spacePath, silverBulletDbFile);
if (db) {
dbPath = path.resolve(db);
}
console.log("Function to run:", functionName, "with arguments", args); console.log("Function to run:", functionName, "with arguments", args);
try { try {
const result = await runPlug( const result = await runPlug(
spacePath, spacePath,
dbPath,
functionName, functionName,
args, args,
new AssetBundle(assets), new AssetBundle(assets),

128
cmd/server.ts
View File

@ -1,4 +1,4 @@
import { Application, path } from "../server/deps.ts"; import { Application } from "../server/deps.ts";
import { HttpServer } from "../server/http_server.ts"; import { HttpServer } from "../server/http_server.ts";
import clientAssetBundle from "../dist/client_asset_bundle.json" assert { import clientAssetBundle from "../dist/client_asset_bundle.json" assert {
type: "json", type: "json",
@ -7,17 +7,11 @@ import plugAssetBundle from "../dist/plug_asset_bundle.json" assert {
type: "json", type: "json",
}; };
import { AssetBundle, AssetJson } from "../plugos/asset_bundle/bundle.ts"; import { AssetBundle, AssetJson } from "../plugos/asset_bundle/bundle.ts";
import { AssetBundlePlugSpacePrimitives } from "../common/spaces/asset_bundle_space_primitives.ts";
import { DiskSpacePrimitives } from "../common/spaces/disk_space_primitives.ts";
import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
import { S3SpacePrimitives } from "../server/spaces/s3_space_primitives.ts";
import { Authenticator } from "../server/auth.ts";
import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts";
import { ServerSystem } from "../server/server_system.ts";
import { sleep } from "$sb/lib/async.ts"; import { sleep } from "$sb/lib/async.ts";
import { SilverBulletHooks } from "../common/manifest.ts";
import { System } from "../plugos/system.ts"; import { determineDatabaseBackend } from "../server/db_backend.ts";
import { silverBulletDbFile } from "./constants.ts"; import { SpaceServerConfig } from "../server/instance.ts";
import { path } from "../common/deps.ts";
export async function serveCommand( export async function serveCommand(
options: { options: {
@ -28,8 +22,6 @@ export async function serveCommand(
cert?: string; cert?: string;
key?: string; key?: string;
reindex?: boolean; reindex?: boolean;
db?: string;
syncOnly?: boolean;
}, },
folder?: string, folder?: string,
) { ) {
@ -37,12 +29,11 @@ export async function serveCommand(
"127.0.0.1"; "127.0.0.1";
const port = options.port || const port = options.port ||
(Deno.env.get("SB_PORT") && +Deno.env.get("SB_PORT")!) || 3000; (Deno.env.get("SB_PORT") && +Deno.env.get("SB_PORT")!) || 3000;
const syncOnly = options.syncOnly || Deno.env.get("SB_SYNC_ONLY");
let dbFile = options.db || Deno.env.get("SB_DB_FILE") || silverBulletDbFile;
const app = new Application(); const app = new Application();
if (!folder) { if (!folder) {
// Didn't get a folder as an argument, check if we got it as an environment variable
folder = Deno.env.get("SB_FOLDER"); folder = Deno.env.get("SB_FOLDER");
if (!folder) { if (!folder) {
console.error( console.error(
@ -51,105 +42,44 @@ export async function serveCommand(
Deno.exit(1); Deno.exit(1);
} }
} }
folder = path.resolve(Deno.cwd(), folder);
const baseKvPrimitives = await determineDatabaseBackend(folder);
console.log( console.log(
"Going to start SilverBullet binding to", "Going to start SilverBullet binding to",
`${hostname}:${port}`, `${hostname}:${port}`,
); );
if (hostname === "127.0.0.1") { if (hostname === "127.0.0.1") {
console.log( console.info(
`NOTE: SilverBullet will only be available locally (via http://localhost:${port}). `SilverBullet will only be available locally (via http://localhost:${port}).
To allow outside connections, pass -L 0.0.0.0 as a flag, and put a TLS terminator on top.`, To allow outside connections, pass -L 0.0.0.0 as a flag, and put a TLS terminator on top.`,
); );
} }
let spacePrimitives: SpacePrimitives | undefined;
if (folder === "s3://") {
spacePrimitives = new S3SpacePrimitives({
accessKey: Deno.env.get("AWS_ACCESS_KEY_ID")!,
secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!,
endPoint: Deno.env.get("AWS_ENDPOINT")!,
region: Deno.env.get("AWS_REGION")!,
bucket: Deno.env.get("AWS_BUCKET")!,
});
console.log("Running in S3 mode");
folder = Deno.cwd();
} else {
// Regular disk mode
folder = path.resolve(Deno.cwd(), folder);
spacePrimitives = new DiskSpacePrimitives(folder);
}
spacePrimitives = new AssetBundlePlugSpacePrimitives( const userAuth = options.user ?? Deno.env.get("SB_USER");
spacePrimitives,
new AssetBundle(plugAssetBundle as AssetJson),
);
let system: System<SilverBulletHooks> | undefined; const configs = new Map<string, SpaceServerConfig>();
// system = undefined in syncOnly mode (no PlugOS instance on the server) configs.set("*", {
if (!syncOnly) {
// Enable server-side processing
dbFile = path.resolve(folder, dbFile);
console.log(
`Running in server-processing mode, keeping state in ${dbFile}`,
);
const serverSystem = new ServerSystem(spacePrimitives, dbFile, app);
await serverSystem.init();
spacePrimitives = serverSystem.spacePrimitives;
system = serverSystem.system;
if (options.reindex) {
console.log("Reindexing space (requested via --reindex flag)");
serverSystem.system.loadedPlugs.get("index")!.invoke(
"reindexSpace",
[],
).catch(console.error);
}
}
const authStore = new JSONKVStore();
const authenticator = new Authenticator(authStore);
const flagUser = options.user ?? Deno.env.get("SB_USER");
if (flagUser) {
// If explicitly added via env/parameter, add on the fly
const [username, password] = flagUser.split(":");
await authenticator.register(username, password, ["admin"], "");
}
if (options.auth) {
// Load auth file
const authFile: string = options.auth;
console.log("Loading authentication credentials from", authFile);
await authStore.load(authFile);
(async () => {
// Asynchronously kick off file watcher
for await (const _event of Deno.watchFs(options.auth!)) {
console.log("Authentication file changed, reloading...");
await authStore.load(authFile);
}
})().catch(console.error);
}
const envAuth = Deno.env.get("SB_AUTH");
if (envAuth) {
console.log("Loading authentication from SB_AUTH");
authStore.loadString(envAuth);
}
const httpServer = new HttpServer(
spacePrimitives!,
app,
system,
{
hostname, hostname,
port: port, namespace: "*",
pagesPath: folder!, auth: userAuth,
pagesPath: folder,
});
const httpServer = new HttpServer({
app,
hostname,
port,
clientAssetBundle: new AssetBundle(clientAssetBundle as AssetJson), clientAssetBundle: new AssetBundle(clientAssetBundle as AssetJson),
authenticator, plugAssetBundle: new AssetBundle(plugAssetBundle as AssetJson),
baseKvPrimitives,
syncOnly: baseKvPrimitives === undefined,
keyFile: options.key, keyFile: options.key,
certFile: options.cert, certFile: options.cert,
}, configs,
); });
await httpServer.start(); httpServer.start();
// Wait in an infinite loop (to keep the HTTP server running, only cancelable via Ctrl+C or other signal) // Wait in an infinite loop (to keep the HTTP server running, only cancelable via Ctrl+C or other signal)
while (true) { while (true) {

35
cmd/user_add.ts
View File

@ -1,35 +0,0 @@
import getpass from "https://deno.land/x/getpass@0.3.1/mod.ts";
import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts";
import { Authenticator } from "../server/auth.ts";
export async function userAdd(
options: any,
username?: string,
) {
const authFile = options.auth || ".auth.json";
console.log("Using auth file", authFile);
if (!username) {
username = prompt("Username:")!;
}
if (!username) {
return;
}
const pw = getpass("Password: ");
if (!pw) {
return;
}
console.log("Adding user to groups", options.group);
const store = new JSONKVStore();
try {
await store.load(authFile);
} catch (e: any) {
if (e instanceof Deno.errors.NotFound) {
console.log("Creating new auth database because it didn't exist.");
}
}
const auth = new Authenticator(store);
await auth.register(username!, pw!, options.group);
await store.save(authFile);
}

30
cmd/user_chgrp.ts
View File

@ -1,30 +0,0 @@
import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts";
import { Authenticator } from "../server/auth.ts";
export async function userChgrp(
options: any,
username?: string,
) {
const authFile = options.auth || ".auth.json";
console.log("Using auth file", authFile);
if (!username) {
username = prompt("Username:")!;
}
if (!username) {
return;
}
console.log("Setting groups for user:", options.group);
const store = new JSONKVStore();
try {
await store.load(authFile);
} catch (e: any) {
if (e instanceof Deno.errors.NotFound) {
console.log("Creating new auth database because it didn't exist.");
}
}
const auth = new Authenticator(store);
await auth.setGroups(username!, options.group);
await store.save(authFile);
}

37
cmd/user_delete.ts
View File

@ -1,37 +0,0 @@
import getpass from "https://deno.land/x/getpass@0.3.1/mod.ts";
import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts";
import { Authenticator } from "../server/auth.ts";
export async function userDelete(
options: any,
username?: string,
) {
const authFile = options.auth || ".auth.json";
console.log("Using auth file", authFile);
if (!username) {
username = prompt("Username:")!;
}
if (!username) {
return;
}
const store = new JSONKVStore();
try {
await store.load(authFile);
} catch (e: any) {
if (e instanceof Deno.errors.NotFound) {
console.log("Creating new auth database because it didn't exist.");
}
}
const auth = new Authenticator(store);
const user = await auth.getUser(username);
if (!user) {
console.error("User", username, "not found.");
Deno.exit(1);
}
await auth.deleteUser(username!);
await store.save(authFile);
}

42
cmd/user_passwd.ts
View File

@ -1,42 +0,0 @@
import getpass from "https://deno.land/x/getpass@0.3.1/mod.ts";
import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts";
import { Authenticator } from "../server/auth.ts";
export async function userPasswd(
options: any,
username?: string,
) {
const authFile = options.auth || ".auth.json";
console.log("Using auth file", authFile);
if (!username) {
username = prompt("Username:")!;
}
if (!username) {
return;
}
const store = new JSONKVStore();
try {
await store.load(authFile);
} catch (e: any) {
if (e instanceof Deno.errors.NotFound) {
console.log("Creating new auth database because it didn't exist.");
}
}
const auth = new Authenticator(store);
const user = await auth.getUser(username);
if (!user) {
console.error("User", username, "not found.");
Deno.exit(1);
}
const pw = getpass("New password: ");
if (!pw) {
return;
}
await auth.setPassword(username!, pw!);
await store.save(authFile);
}

6
plugos/hooks/endpoint.test.ts
View File

@ -26,7 +26,11 @@ Deno.test("Run a plugos endpoint server", async () => {
const app = new Application(); const app = new Application();
const port = 3123; const port = 3123;
system.addHook(new EndpointHook(app, "/_/")); const endpointHook = new EndpointHook("/_/");
app.use((context, next) => {
return endpointHook.handleRequest(system, context, next);
});
const controller = new AbortController(); const controller = new AbortController();
app.listen({ port: port, signal: controller.signal }); app.listen({ port: port, signal: controller.signal });

17
plugos/hooks/endpoint.ts
View File

@ -1,6 +1,6 @@
import { Hook, Manifest } from "../types.ts"; import { Hook, Manifest } from "../types.ts";
import { System } from "../system.ts"; import { System } from "../system.ts";
import { Application } from "../../server/deps.ts"; import { Application, Context, Next } from "../../server/deps.ts";
export type EndpointRequest = { export type EndpointRequest = {
method: string; method: string;
@ -26,16 +26,17 @@ export type EndPointDef = {
}; };
export class EndpointHook implements Hook<EndpointHookT> { export class EndpointHook implements Hook<EndpointHookT> {
private app: Application;
readonly prefix: string; readonly prefix: string;
constructor(app: Application, prefix: string) { constructor(prefix: string) {
this.app = app;
this.prefix = prefix; this.prefix = prefix;
} }
apply(system: System<EndpointHookT>): void { public async handleRequest(
this.app.use(async (ctx, next) => { system: System<EndpointHookT>,
ctx: Context,
next: Next,
) {
const req = ctx.request; const req = ctx.request;
const requestPath = ctx.request.url.pathname; const requestPath = ctx.request.url.pathname;
if (!requestPath.startsWith(this.prefix)) { if (!requestPath.startsWith(this.prefix)) {
@ -106,7 +107,9 @@ export class EndpointHook implements Hook<EndpointHookT> {
} }
// console.log("Shouldn't get here"); // console.log("Shouldn't get here");
await next(); await next();
}); }
apply(): void {
} }
validateManifest(manifest: Manifest<EndpointHookT>): string[] { validateManifest(manifest: Manifest<EndpointHookT>): string[] {

4
plugos/lib/datastore.test.ts
View File

@ -2,11 +2,11 @@ import "https://esm.sh/fake-indexeddb@4.0.2/auto";
import { IndexedDBKvPrimitives } from "./indexeddb_kv_primitives.ts"; import { IndexedDBKvPrimitives } from "./indexeddb_kv_primitives.ts";
import { DataStore } from "./datastore.ts"; import { DataStore } from "./datastore.ts";
import { DenoKvPrimitives } from "./deno_kv_primitives.ts"; import { DenoKvPrimitives } from "./deno_kv_primitives.ts";
import { KvPrimitives } from "./kv_primitives.ts"; import { KvPrimitives, PrefixedKvPrimitives } from "./kv_primitives.ts";
import { assertEquals } from "https://deno.land/std@0.165.0/testing/asserts.ts"; import { assertEquals } from "https://deno.land/std@0.165.0/testing/asserts.ts";
async function test(db: KvPrimitives) { async function test(db: KvPrimitives) {
const datastore = new DataStore(db, ["ds"], { const datastore = new DataStore(new PrefixedKvPrimitives(db, ["ds"]), {
count: (arr: any[]) => arr.length, count: (arr: any[]) => arr.length,
}); });
await datastore.set(["user", "peter"], { name: "Peter" }); await datastore.set(["user", "peter"], { name: "Peter" });

38
plugos/lib/datastore.ts
View File

@ -10,25 +10,16 @@ import { KvPrimitives } from "./kv_primitives.ts";
export class DataStore { export class DataStore {
constructor( constructor(
readonly kv: KvPrimitives, readonly kv: KvPrimitives,
private prefix: KvKey = [],
private functionMap: FunctionMap = builtinFunctions, private functionMap: FunctionMap = builtinFunctions,
) { ) {
} }
prefixed(prefix: KvKey): DataStore {
return new DataStore(
this.kv,
[...this.prefix, ...prefix],
this.functionMap,
);
}
async get<T = any>(key: KvKey): Promise<T | null> { async get<T = any>(key: KvKey): Promise<T | null> {
return (await this.batchGet([key]))[0]; return (await this.batchGet([key]))[0];
} }
batchGet<T = any>(keys: KvKey[]): Promise<(T | null)[]> { batchGet<T = any>(keys: KvKey[]): Promise<(T | null)[]> {
return this.kv.batchGet(keys.map((key) => this.applyPrefix(key))); return this.kv.batchGet(keys);
} }
set(key: KvKey, value: any): Promise<void> { set(key: KvKey, value: any): Promise<void> {
@ -44,7 +35,7 @@ export class DataStore {
console.warn(`Duplicate key ${keyString} in batchSet, skipping`); console.warn(`Duplicate key ${keyString} in batchSet, skipping`);
} else { } else {
allKeyStrings.add(keyString); allKeyStrings.add(keyString);
uniqueEntries.push({ key: this.applyPrefix(key), value }); uniqueEntries.push({ key, value });
} }
} }
return this.kv.batchSet(uniqueEntries); return this.kv.batchSet(uniqueEntries);
@ -55,7 +46,7 @@ export class DataStore {
} }
batchDelete(keys: KvKey[]): Promise<void> { batchDelete(keys: KvKey[]): Promise<void> {
return this.kv.batchDelete(keys.map((key) => this.applyPrefix(key))); return this.kv.batchDelete(keys);
} }
async query<T = any>(query: KvQuery): Promise<KV<T>[]> { async query<T = any>(query: KvQuery): Promise<KV<T>[]> {
@ -63,15 +54,11 @@ export class DataStore {
let itemCount = 0; let itemCount = 0;
// Accumulate results // Accumulate results
let limit = Infinity; let limit = Infinity;
const prefixedQuery: KvQuery = {
...query,
prefix: query.prefix ? this.applyPrefix(query.prefix) : undefined,
};
if (query.limit) { if (query.limit) {
limit = evalQueryExpression(query.limit, {}, this.functionMap); limit = evalQueryExpression(query.limit, {}, this.functionMap);
} }
for await ( for await (
const entry of this.kv.query(prefixedQuery) const entry of this.kv.query(query)
) { ) {
// Filter // Filter
if ( if (
@ -89,29 +76,16 @@ export class DataStore {
} }
} }
// Apply order by, limit, and select // Apply order by, limit, and select
return applyQueryNoFilterKV(prefixedQuery, results, this.functionMap).map(( return applyQueryNoFilterKV(query, results, this.functionMap);
{ key, value },
) => ({ key: this.stripPrefix(key), value }));
} }
async queryDelete(query: KvQuery): Promise<void> { async queryDelete(query: KvQuery): Promise<void> {
const keys: KvKey[] = []; const keys: KvKey[] = [];
for ( for (
const { key } of await this.query({ const { key } of await this.query(query)
...query,
prefix: query.prefix ? this.applyPrefix(query.prefix) : undefined,
})
) { ) {
keys.push(key); keys.push(key);
} }
return this.batchDelete(keys); return this.batchDelete(keys);
} }
private applyPrefix(key: KvKey): KvKey {
return [...this.prefix, ...(key ? key : [])];
}
private stripPrefix(key: KvKey): KvKey {
return key.slice(this.prefix.length);
}
} }

44
plugos/lib/kv_primitives.ts
View File

@ -9,4 +9,48 @@ export interface KvPrimitives {
batchSet(entries: KV[]): Promise<void>; batchSet(entries: KV[]): Promise<void>;
batchDelete(keys: KvKey[]): Promise<void>; batchDelete(keys: KvKey[]): Promise<void>;
query(options: KvQueryOptions): AsyncIterableIterator<KV>; query(options: KvQueryOptions): AsyncIterableIterator<KV>;
close(): void;
}
/**
* Turns any KvPrimitives into a KvPrimitives that automatically prefixes all keys (and removes them again when reading)
*/
export class PrefixedKvPrimitives implements KvPrimitives {
constructor(private wrapped: KvPrimitives, private prefix: KvKey) {
}
batchGet(keys: KvKey[]): Promise<any[]> {
return this.wrapped.batchGet(keys.map((key) => this.applyPrefix(key)));
}
batchSet(entries: KV[]): Promise<void> {
return this.wrapped.batchSet(
entries.map(({ key, value }) => ({ key: this.applyPrefix(key), value })),
);
}
batchDelete(keys: KvKey[]): Promise<void> {
return this.wrapped.batchDelete(keys.map((key) => this.applyPrefix(key)));
}
async *query(options: KvQueryOptions): AsyncIterableIterator<KV> {
for await (
const result of this.wrapped.query({
prefix: this.applyPrefix(options.prefix),
})
) {
yield { key: this.stripPrefix(result.key), value: result.value };
}
}
close(): void {
this.wrapped.close();
}
private applyPrefix(key?: KvKey): KvKey {
return [...this.prefix, ...(key ? key : [])];
}
private stripPrefix(key: KvKey): KvKey {
return key.slice(this.prefix.length);
}
} }

56
plugos/lib/kv_store.json_file.ts
View File

@ -1,56 +0,0 @@
import { KV } from "$sb/types.ts";
export class JSONKVStore {
private data: { [key: string]: any } = {};
async load(path: string) {
this.loadString(await Deno.readTextFile(path));
}
loadString(jsonString: string) {
this.data = JSON.parse(jsonString);
}
async save(path: string) {
await Deno.writeTextFile(path, JSON.stringify(this.data));
}
del(key: string): Promise<void> {
delete this.data[key];
return Promise.resolve();
}
deletePrefix(prefix: string): Promise<void> {
for (const key in this.data) {
if (key.startsWith(prefix)) {
delete this.data[key];
}
}
return Promise.resolve();
}
deleteAll(): Promise<void> {
this.data = {};
return Promise.resolve();
}
set(key: string, value: any): Promise<void> {
this.data[key] = value;
return Promise.resolve();
}
get(key: string): Promise<any> {
return Promise.resolve(this.data[key]);
}
has(key: string): Promise<boolean> {
return Promise.resolve(key in this.data);
}
queryPrefix(keyPrefix: string): Promise<{ key: string; value: any }[]> {
const results: { key: string; value: any }[] = [];
for (const key in this.data) {
if (key.startsWith(keyPrefix)) {
results.push({ key, value: this.data[key] });
}
}
return Promise.resolve(results);
}
}

5
plugos/lib/mq.datastore.test.ts
View File

@ -3,12 +3,15 @@ import { assertEquals } from "../../test_deps.ts";
import { sleep } from "$sb/lib/async.ts"; import { sleep } from "$sb/lib/async.ts";
import { DenoKvPrimitives } from "./deno_kv_primitives.ts"; import { DenoKvPrimitives } from "./deno_kv_primitives.ts";
import { DataStore } from "./datastore.ts"; import { DataStore } from "./datastore.ts";
import { PrefixedKvPrimitives } from "./kv_primitives.ts";
Deno.test("DataStore MQ", async () => { Deno.test("DataStore MQ", async () => {
const tmpFile = await Deno.makeTempFile(); const tmpFile = await Deno.makeTempFile();
const db = new DenoKvPrimitives(await Deno.openKv(tmpFile)); const db = new DenoKvPrimitives(await Deno.openKv(tmpFile));
const mq = new DataStoreMQ(new DataStore(db, ["mq"])); const mq = new DataStoreMQ(
new DataStore(new PrefixedKvPrimitives(db, ["mq"])),
);
await mq.send("test", "Hello World"); await mq.send("test", "Hello World");
let messages = await mq.poll("test", 10); let messages = await mq.poll("test", 10);
assertEquals(messages.length, 1); assertEquals(messages.length, 1);

2
plugos/system.ts
View File

@ -141,7 +141,7 @@ export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
if (this.plugs.has(manifest.name)) { if (this.plugs.has(manifest.name)) {
this.unload(manifest.name); this.unload(manifest.name);
} }
console.log("Loaded plug", manifest.name); console.log("Activated plug", manifest.name);
this.plugs.set(manifest.name, plug); this.plugs.set(manifest.name, plug);
await this.emit("plugLoaded", plug); await this.emit("plugLoaded", plug);

120
server/auth.ts
View File

@ -1,120 +0,0 @@
import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts";
export type User = {
username: string;
passwordHash: string; // hashed password
salt: string;
groups: string[]; // special "admin"
};
async function createUser(
username: string,
password: string,
groups: string[],
salt = generateSalt(16),
): Promise<User> {
return {
username,
passwordHash: await hashSHA256(`${salt}${password}`),
salt,
groups,
};
}
const userPrefix = `u:`;
export class Authenticator {
constructor(private store: JSONKVStore) {
}
async register(
username: string,
password: string,
groups: string[],
salt?: string,
): Promise<void> {
await this.store.set(
`${userPrefix}${username}`,
await createUser(username, password, groups, salt),
);
}
async authenticateHashed(
username: string,
hashedPassword: string,
): Promise<boolean> {
const user = await this.store.get(`${userPrefix}${username}`) as User;
if (!user) {
return false;
}
return user.passwordHash === hashedPassword;
}
async authenticate(
username: string,
password: string,
): Promise<string | undefined> {
const user = await this.store.get(`${userPrefix}${username}`) as User;
if (!user) {
return undefined;
}
const hashedPassword = await hashSHA256(`${user.salt}${password}`);
return user.passwordHash === hashedPassword ? hashedPassword : undefined;
}
async getAllUsers(): Promise<User[]> {
return (await this.store.queryPrefix(userPrefix)).map((item) => item.value);
}
getUser(username: string): Promise<User | undefined> {
return this.store.get(`${userPrefix}${username}`);
}
async setPassword(username: string, password: string): Promise<void> {
const user = await this.getUser(username);
if (!user) {
throw new Error(`User does not exist`);
}
user.passwordHash = await hashSHA256(`${user.salt}${password}`);
await this.store.set(`${userPrefix}${username}`, user);
}
async deleteUser(username: string): Promise<void> {
const user = await this.getUser(username);
if (!user) {
throw new Error(`User does not exist`);
}
await this.store.del(`${userPrefix}${username}`);
}
async setGroups(username: string, groups: string[]): Promise<void> {
const user = await this.getUser(username);
if (!user) {
throw new Error(`User does not exist`);
}
user.groups = groups;
await this.store.set(`${userPrefix}${username}`, user);
}
}
async function hashSHA256(message: string): Promise<string> {
// Transform the string into an ArrayBuffer
const encoder = new TextEncoder();
const data = encoder.encode(message);
// Generate the hash
const hashBuffer = await window.crypto.subtle.digest("SHA-256", data);
// Transform the hash into a hex string
return Array.from(new Uint8Array(hashBuffer)).map((b) =>
b.toString(16).padStart(2, "0")
).join("");
}
function generateSalt(length: number): string {
const array = new Uint8Array(length / 2); // because two characters represent one byte in hex
crypto.getRandomValues(array);
return Array.from(array, (byte) => ("00" + byte.toString(16)).slice(-2)).join(
"",
);
}

13
server/crypto.ts Normal file
View File

@ -0,0 +1,13 @@
export async function hashSHA256(message: string): Promise<string> {
// Transform the string into an ArrayBuffer
const encoder = new TextEncoder();
const data = encoder.encode(message);
// Generate the hash
const hashBuffer = await window.crypto.subtle.digest("SHA-256", data);
// Transform the hash into a hex string
return Array.from(new Uint8Array(hashBuffer)).map((b) =>
b.toString(16).padStart(2, "0")
).join("");
}

44
server/db_backend.ts Normal file
View File

@ -0,0 +1,44 @@
import { DenoKvPrimitives } from "../plugos/lib/deno_kv_primitives.ts";
import { KvPrimitives } from "../plugos/lib/kv_primitives.ts";
import { path } from "./deps.ts";
/**
* Environment variables:
* - SB_DB_BACKEND: "denokv" or "off" (default: denokv)
* - SB_KV_DB (denokv only): path to the database file (default .silverbullet.db) or ":cloud:" for cloud storage
*/
export async function determineDatabaseBackend(
singleTenantFolder?: string,
): Promise<
KvPrimitives | undefined
> {
const backendConfig = Deno.env.get("SB_DB_BACKEND") || "denokv";
switch (backendConfig) {
case "denokv": {
let dbFile: string | undefined = Deno.env.get("SB_KV_DB") ||
".silverbullet.db";
if (singleTenantFolder) {
// If we're running in single tenant mode, we may as well use the tenant's space folder to keep the database
dbFile = path.resolve(singleTenantFolder, dbFile);
}
if (dbFile === ":cloud:") {
dbFile = undefined; // Deno Deploy will use the default KV store
}
const denoDb = await Deno.openKv(dbFile);
console.info(
`Using DenoKV as a database backend (${
dbFile || "cloud"
}), running in server-processing mode.`,
);
return new DenoKvPrimitives(denoDb);
}
default:
console.info(
"Running in databaseless mode: no server-side indexing and state keeping (beyond space files) will happen.",
);
return;
}
}

2
server/deps.ts
View File

@ -3,6 +3,8 @@ export type { Next } from "https://deno.land/x/oak@v12.4.0/mod.ts";
export { export {
Application, Application,
Context, Context,
Request,
Response,
Router, Router,
} from "https://deno.land/x/oak@v12.4.0/mod.ts"; } from "https://deno.land/x/oak@v12.4.0/mod.ts";
export * as etag from "https://deno.land/x/oak@v12.4.0/etag.ts"; export * as etag from "https://deno.land/x/oak@v12.4.0/etag.ts";

297
server/http_server.ts
View File

@ -1,95 +1,155 @@
import { Application, Context, Next, oakCors, Router } from "./deps.ts";
import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
import { ensureSettingsAndIndex } from "../common/util.ts";
import { BuiltinSettings } from "../web/types.ts";
import { gitIgnoreCompiler } from "./deps.ts";
import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts";
import { Authenticator } from "./auth.ts";
import { FileMeta } from "$sb/types.ts";
import { import {
ShellRequest, Application,
ShellResponse, Context,
SyscallRequest, Next,
SyscallResponse, oakCors,
} from "./rpc.ts"; Request,
import { SilverBulletHooks } from "../common/manifest.ts"; Router,
import { System } from "../plugos/system.ts"; } from "./deps.ts";
import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
import { FileMeta } from "$sb/types.ts";
import { ShellRequest, SyscallRequest, SyscallResponse } from "./rpc.ts";
import { determineShellBackend } from "./shell_backend.ts";
import { SpaceServer, SpaceServerConfig } from "./instance.ts";
import {
KvPrimitives,
PrefixedKvPrimitives,
} from "../plugos/lib/kv_primitives.ts";
import { EndpointHook } from "../plugos/hooks/endpoint.ts";
import { hashSHA256 } from "./crypto.ts";
export type ServerOptions = { export type ServerOptions = {
app: Application;
hostname: string; hostname: string;
port: number; port: number;
pagesPath: string;
clientAssetBundle: AssetBundle; clientAssetBundle: AssetBundle;
authenticator: Authenticator; plugAssetBundle: AssetBundle;
pass?: string; baseKvPrimitives?: KvPrimitives;
syncOnly: boolean;
certFile?: string; certFile?: string;
keyFile?: string; keyFile?: string;
configs: Map<string, SpaceServerConfig>;
}; };
export class HttpServer { export class HttpServer {
private hostname: string;
private port: number;
abortController?: AbortController; abortController?: AbortController;
clientAssetBundle: AssetBundle; clientAssetBundle: AssetBundle;
settings?: BuiltinSettings; plugAssetBundle: AssetBundle;
spacePrimitives: SpacePrimitives; hostname: string;
authenticator: Authenticator; port: number;
app: Application<Record<string, any>>;
keyFile: string | undefined;
certFile: string | undefined;
constructor( spaceServers = new Map<string, Promise<SpaceServer>>();
spacePrimitives: SpacePrimitives, syncOnly: boolean;
private app: Application, baseKvPrimitives?: KvPrimitives;
private system: System<SilverBulletHooks> | undefined, configs: Map<string, SpaceServerConfig>;
private options: ServerOptions,
) { constructor(options: ServerOptions) {
this.clientAssetBundle = options.clientAssetBundle;
this.plugAssetBundle = options.plugAssetBundle;
this.hostname = options.hostname; this.hostname = options.hostname;
this.port = options.port; this.port = options.port;
this.authenticator = options.authenticator; this.app = options.app;
this.clientAssetBundle = options.clientAssetBundle; this.keyFile = options.keyFile;
this.certFile = options.certFile;
let fileFilterFn: (s: string) => boolean = () => true; this.syncOnly = options.syncOnly;
this.spacePrimitives = new FilteredSpacePrimitives( this.baseKvPrimitives = options.baseKvPrimitives;
spacePrimitives, this.configs = options.configs;
(meta) => fileFilterFn(meta.name),
async () => {
await this.reloadSettings();
if (typeof this.settings?.spaceIgnore === "string") {
fileFilterFn = gitIgnoreCompiler(this.settings.spaceIgnore).accepts;
} else {
fileFilterFn = () => true;
} }
},
async bootSpaceServer(config: SpaceServerConfig): Promise<SpaceServer> {
const spaceServer = new SpaceServer(
config,
determineShellBackend(config.pagesPath),
this.plugAssetBundle,
this.baseKvPrimitives
? new PrefixedKvPrimitives(this.baseKvPrimitives, [
config.namespace,
])
: undefined,
); );
await spaceServer.init();
return spaceServer;
}
determineConfig(req: Request): [string, SpaceServerConfig] {
let hostname = req.url.host; // hostname:port
// First try a full match
let config = this.configs.get(hostname);
if (config) {
return [hostname, config];
}
// Then rip off the port and try again
hostname = hostname.split(":")[0];
config = this.configs.get(hostname);
if (config) {
return [hostname, config];
}
// If all else fails, try the wildcard
config = this.configs.get("*");
if (config) {
return ["*", config];
}
throw new Error(`No space server config found for hostname ${hostname}`);
}
ensureSpaceServer(req: Request): Promise<SpaceServer> {
const [matchedHostname, config] = this.determineConfig(req);
const spaceServer = this.spaceServers.get(matchedHostname);
if (spaceServer) {
return spaceServer;
}
// And then boot the thing, async
const spaceServerPromise = this.bootSpaceServer(config);
// But immediately write the promise to the map so that we don't boot it twice
this.spaceServers.set(matchedHostname, spaceServerPromise);
return spaceServerPromise;
} }
// Replaces some template variables in index.html in a rather ad-hoc manner, but YOLO // Replaces some template variables in index.html in a rather ad-hoc manner, but YOLO
renderIndexHtml() { renderIndexHtml(pagesPath: string) {
return this.clientAssetBundle.readTextFileSync(".client/index.html") return this.clientAssetBundle.readTextFileSync(".client/index.html")
.replaceAll( .replaceAll(
"{{SPACE_PATH}}", "{{SPACE_PATH}}",
this.options.pagesPath.replaceAll("\\", "\\\\"), pagesPath.replaceAll("\\", "\\\\"),
// ); // );
).replaceAll( ).replaceAll(
"{{SYNC_ONLY}}", "{{SYNC_ONLY}}",
this.system ? "false" : "true", this.syncOnly ? "true" : "false",
); );
} }
async start() { start() {
await this.reloadSettings();
// Serve static files (javascript, css, html) // Serve static files (javascript, css, html)
this.app.use(this.serveStatic.bind(this)); this.app.use(this.serveStatic.bind(this));
await this.addPasswordAuth(this.app); const endpointHook = new EndpointHook("/_/");
const fsRouter = this.addFsRoutes(this.spacePrimitives);
this.app.use(async (context, next) => {
const spaceServer = await this.ensureSpaceServer(context.request);
return endpointHook.handleRequest(spaceServer.system!, context, next);
});
this.addPasswordAuth(this.app);
const fsRouter = this.addFsRoutes();
this.app.use(fsRouter.routes()); this.app.use(fsRouter.routes());
this.app.use(fsRouter.allowedMethods()); this.app.use(fsRouter.allowedMethods());
// Fallback, serve the UI index.html // Fallback, serve the UI index.html
this.app.use(({ response }) => { this.app.use(async ({ request, response }) => {
response.headers.set("Content-type", "text/html"); response.headers.set("Content-type", "text/html");
response.body = this.renderIndexHtml(); response.headers.set("Cache-Control", "no-cache");
const spaceServer = await this.ensureSpaceServer(request);
response.body = this.renderIndexHtml(spaceServer.pagesPath);
}); });
this.abortController = new AbortController(); this.abortController = new AbortController();
@ -98,11 +158,11 @@ export class HttpServer {
port: this.port, port: this.port,
signal: this.abortController.signal, signal: this.abortController.signal,
}; };
if (this.options.keyFile) { if (this.keyFile) {
listenOptions.key = Deno.readTextFileSync(this.options.keyFile); listenOptions.key = Deno.readTextFileSync(this.keyFile);
} }
if (this.options.certFile) { if (this.certFile) {
listenOptions.cert = Deno.readTextFileSync(this.options.certFile); listenOptions.cert = Deno.readTextFileSync(this.certFile);
} }
this.app.listen(listenOptions) this.app.listen(listenOptions)
.catch((e: any) => { .catch((e: any) => {
@ -117,7 +177,7 @@ export class HttpServer {
); );
} }
serveStatic( async serveStatic(
{ request, response }: Context<Record<string, any>, Record<string, any>>, { request, response }: Context<Record<string, any>, Record<string, any>>,
next: Next, next: Next,
) { ) {
@ -127,7 +187,9 @@ export class HttpServer {
// Serve the UI (index.html) // Serve the UI (index.html)
// Note: we're explicitly not setting Last-Modified and If-Modified-Since header here because this page is dynamic // Note: we're explicitly not setting Last-Modified and If-Modified-Since header here because this page is dynamic
response.headers.set("Content-type", "text/html"); response.headers.set("Content-type", "text/html");
response.body = this.renderIndexHtml(); response.headers.set("Cache-Control", "no-cache");
const spaceServer = await this.ensureSpaceServer(request);
response.body = this.renderIndexHtml(spaceServer.pagesPath);
return; return;
} }
try { try {
@ -163,12 +225,7 @@ export class HttpServer {
} }
} }
async reloadSettings() { private addPasswordAuth(app: Application) {
// TODO: Throttle this?
this.settings = await ensureSettingsAndIndex(this.spacePrimitives);
}
private async addPasswordAuth(app: Application) {
const excludedPaths = [ const excludedPaths = [
"/manifest.json", "/manifest.json",
"/favicon.png", "/favicon.png",
@ -192,14 +249,17 @@ export class HttpServer {
return; return;
} else if (request.method === "POST") { } else if (request.method === "POST") {
const values = await request.body({ type: "form" }).value; const values = await request.body({ type: "form" }).value;
const username = values.get("username")!, const username = values.get("username")!;
password = values.get("password")!, const password = values.get("password")!;
refer = values.get("refer"); const refer = values.get("refer");
const hashedPassword = await this.authenticator.authenticate(
username, const spaceServer = await this.ensureSpaceServer(request);
password, const hashedPassword = await hashSHA256(password);
); const [expectedUser, expectedPassword] = spaceServer.auth!.split(":");
if (hashedPassword) { if (
username === expectedUser &&
hashedPassword === await hashSHA256(expectedPassword)
) {
await cookies.set( await cookies.set(
authCookieName(host), authCookieName(host),
`${username}:${hashedPassword}`, `${username}:${hashedPassword}`,
@ -223,9 +283,13 @@ export class HttpServer {
} }
}); });
if ((await this.authenticator.getAllUsers()).length > 0) { // Check auth
// Users defined, so enabling auth
app.use(async ({ request, response, cookies }, next) => { app.use(async ({ request, response, cookies }, next) => {
const spaceServer = await this.ensureSpaceServer(request);
if (!spaceServer.auth) {
// Auth disabled in this config, skip
return next();
}
const host = request.url.host; const host = request.url.host;
if (!excludedPaths.includes(request.url.pathname)) { if (!excludedPaths.includes(request.url.pathname)) {
const authCookie = await cookies.get(authCookieName(host)); const authCookie = await cookies.get(authCookieName(host));
@ -233,12 +297,14 @@ export class HttpServer {
response.redirect("/.auth"); response.redirect("/.auth");
return; return;
} }
const spaceServer = await this.ensureSpaceServer(request);
const [username, hashedPassword] = authCookie.split(":"); const [username, hashedPassword] = authCookie.split(":");
const [expectedUser, expectedPassword] = spaceServer.auth!.split(
":",
);
if ( if (
!await this.authenticator.authenticateHashed( username !== expectedUser ||
username, hashedPassword !== await hashSHA256(expectedPassword)
hashedPassword,
)
) { ) {
response.redirect("/.auth"); response.redirect("/.auth");
return; return;
@ -247,9 +313,8 @@ export class HttpServer {
await next(); await next();
}); });
} }
}
private addFsRoutes(spacePrimitives: SpacePrimitives): Router { private addFsRoutes(): Router {
const fsRouter = new Router(); const fsRouter = new Router();
const corsMiddleware = oakCors({ const corsMiddleware = oakCors({
allowedHeaders: "*", allowedHeaders: "*",
@ -264,11 +329,12 @@ export class HttpServer {
"/index.json", "/index.json",
// corsMiddleware, // corsMiddleware,
async ({ request, response }) => { async ({ request, response }) => {
const spaceServer = await this.ensureSpaceServer(request);
if (request.headers.has("X-Sync-Mode")) { if (request.headers.has("X-Sync-Mode")) {
// Only handle direct requests for a JSON representation of the file list // Only handle direct requests for a JSON representation of the file list
response.headers.set("Content-type", "application/json"); response.headers.set("Content-type", "application/json");
response.headers.set("X-Space-Path", this.options.pagesPath); response.headers.set("X-Space-Path", spaceServer.pagesPath);
const files = await spacePrimitives.fetchFileList(); const files = await spaceServer.spacePrimitives.fetchFileList();
response.body = JSON.stringify(files); response.body = JSON.stringify(files);
} else { } else {
// Otherwise, redirect to the UI // Otherwise, redirect to the UI
@ -280,49 +346,29 @@ export class HttpServer {
// RPC // RPC
fsRouter.post("/.rpc", async ({ request, response }) => { fsRouter.post("/.rpc", async ({ request, response }) => {
const spaceServer = await this.ensureSpaceServer(request);
const body = await request.body({ type: "json" }).value; const body = await request.body({ type: "json" }).value;
try { try {
switch (body.operation) { switch (body.operation) {
case "shell": { case "shell": {
// TODO: Have a nicer way to do this
if (this.options.pagesPath.startsWith("s3://")) {
response.status = 500;
response.body = JSON.stringify({
stdout: "",
stderr: "Cannot run shell commands with S3 backend",
code: 500,
});
return;
}
const shellCommand: ShellRequest = body; const shellCommand: ShellRequest = body;
console.log( console.log(
"Running shell command:", "Running shell command:",
shellCommand.cmd, shellCommand.cmd,
shellCommand.args, shellCommand.args,
); );
const p = new Deno.Command(shellCommand.cmd, { const shellResponse = await spaceServer.shellBackend.handle(
args: shellCommand.args, shellCommand,
cwd: this.options.pagesPath, );
stdout: "piped",
stderr: "piped",
});
const output = await p.output();
const stdout = new TextDecoder().decode(output.stdout);
const stderr = new TextDecoder().decode(output.stderr);
response.headers.set("Content-Type", "application/json"); response.headers.set("Content-Type", "application/json");
response.body = JSON.stringify({ response.body = JSON.stringify(shellResponse);
stdout, if (shellResponse.code !== 0) {
stderr, console.error("Error running shell command", shellResponse);
code: output.code,
} as ShellResponse);
if (output.code !== 0) {
console.error("Error running shell command", stdout, stderr);
} }
return; return;
} }
case "syscall": { case "syscall": {
if (!this.system) { if (this.syncOnly) {
response.headers.set("Content-Type", "text/plain"); response.headers.set("Content-Type", "text/plain");
response.status = 400; response.status = 400;
response.body = "Unknown operation"; response.body = "Unknown operation";
@ -330,7 +376,9 @@ export class HttpServer {
} }
const syscallCommand: SyscallRequest = body; const syscallCommand: SyscallRequest = body;
try { try {
const plug = this.system.loadedPlugs.get(syscallCommand.ctx); const plug = spaceServer.system!.loadedPlugs.get(
syscallCommand.ctx,
);
if (!plug) { if (!plug) {
throw new Error(`Plug ${syscallCommand.ctx} not found`); throw new Error(`Plug ${syscallCommand.ctx} not found`);
} }
@ -372,6 +420,7 @@ export class HttpServer {
filePathRegex, filePathRegex,
async ({ params, response, request }) => { async ({ params, response, request }) => {
const name = params[0]; const name = params[0];
const spaceServer = await this.ensureSpaceServer(request);
console.log("Requested file", name); console.log("Requested file", name);
if (!request.headers.has("X-Sync-Mode") && name.endsWith(".md")) { if (!request.headers.has("X-Sync-Mode") && name.endsWith(".md")) {
// It can happen that during a sync, authentication expires, this may result in a redirect to the login page and then back to this particular file. This particular file may be an .md file, which isn't great to show so we're redirecting to the associated SB UI page. // It can happen that during a sync, authentication expires, this may result in a redirect to the login page and then back to this particular file. This particular file may be an .md file, which isn't great to show so we're redirecting to the associated SB UI page.
@ -415,13 +464,15 @@ export class HttpServer {
try { try {
if (request.headers.has("X-Get-Meta")) { if (request.headers.has("X-Get-Meta")) {
// Getting meta via GET request // Getting meta via GET request
const fileData = await spacePrimitives.getFileMeta(name); const fileData = await spaceServer.spacePrimitives.getFileMeta(
name,
);
response.status = 200; response.status = 200;
this.fileMetaToHeaders(response.headers, fileData); this.fileMetaToHeaders(response.headers, fileData);
response.body = ""; response.body = "";
return; return;
} }
const fileData = await spacePrimitives.readFile(name); const fileData = await spaceServer.spacePrimitives.readFile(name);
const lastModifiedHeader = new Date(fileData.meta.lastModified) const lastModifiedHeader = new Date(fileData.meta.lastModified)
.toUTCString(); .toUTCString();
if ( if (
@ -447,6 +498,7 @@ export class HttpServer {
filePathRegex, filePathRegex,
async ({ request, response, params }) => { async ({ request, response, params }) => {
const name = params[0]; const name = params[0];
const spaceServer = await this.ensureSpaceServer(request);
console.log("Saving file", name); console.log("Saving file", name);
if (name.startsWith(".")) { if (name.startsWith(".")) {
// Don't expose hidden files // Don't expose hidden files
@ -457,7 +509,7 @@ export class HttpServer {
const body = await request.body({ type: "bytes" }).value; const body = await request.body({ type: "bytes" }).value;
try { try {
const meta = await spacePrimitives.writeFile( const meta = await spaceServer.spacePrimitives.writeFile(
name, name,
body, body,
); );
@ -471,8 +523,9 @@ export class HttpServer {
} }
}, },
) )
.delete(filePathRegex, async ({ response, params }) => { .delete(filePathRegex, async ({ request, response, params }) => {
const name = params[0]; const name = params[0];
const spaceServer = await this.ensureSpaceServer(request);
console.log("Deleting file", name); console.log("Deleting file", name);
if (name.startsWith(".")) { if (name.startsWith(".")) {
// Don't expose hidden files // Don't expose hidden files
@ -480,7 +533,7 @@ export class HttpServer {
return; return;
} }
try { try {
await spacePrimitives.deleteFile(name); await spaceServer.spacePrimitives.deleteFile(name);
response.status = 200; response.status = 200;
response.body = "OK"; response.body = "OK";
} catch (e: any) { } catch (e: any) {
@ -573,11 +626,3 @@ function utcDateString(mtime: number): string {
function authCookieName(host: string) { function authCookieName(host: string) {
return `auth:${host}`; return `auth:${host}`;
} }
function headersToJson(headers: Headers) {
let headersObj: any = {};
for (const [key, value] of headers.entries()) {
headersObj[key] = value;
}
return JSON.stringify(headersObj);
}

87
server/instance.ts Normal file
View File

@ -0,0 +1,87 @@
import { SilverBulletHooks } from "../common/manifest.ts";
import { AssetBundlePlugSpacePrimitives } from "../common/spaces/asset_bundle_space_primitives.ts";
import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts";
import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
import { ensureSettingsAndIndex } from "../common/util.ts";
import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
import { KvPrimitives } from "../plugos/lib/kv_primitives.ts";
import { System } from "../plugos/system.ts";
import { BuiltinSettings } from "../web/types.ts";
import { gitIgnoreCompiler } from "./deps.ts";
import { ServerSystem } from "./server_system.ts";
import { ShellBackend } from "./shell_backend.ts";
import { determineStorageBackend } from "./storage_backend.ts";
export type SpaceServerConfig = {
hostname: string;
namespace: string;
auth?: string; // username:password
pagesPath: string;
};
export class SpaceServer {
public pagesPath: string;
auth?: string;
hostname: string;
private settings?: BuiltinSettings;
spacePrimitives: SpacePrimitives;
// Only set when syncOnly == false
private serverSystem?: ServerSystem;
system?: System<SilverBulletHooks>;
constructor(
config: SpaceServerConfig,
public shellBackend: ShellBackend,
plugAssetBundle: AssetBundle,
kvPrimitives?: KvPrimitives,
) {
this.pagesPath = config.pagesPath;
this.hostname = config.hostname;
this.auth = config.auth;
let fileFilterFn: (s: string) => boolean = () => true;
this.spacePrimitives = new FilteredSpacePrimitives(
new AssetBundlePlugSpacePrimitives(
determineStorageBackend(this.pagesPath),
plugAssetBundle,
),
(meta) => fileFilterFn(meta.name),
async () => {
await this.reloadSettings();
if (typeof this.settings?.spaceIgnore === "string") {
fileFilterFn = gitIgnoreCompiler(this.settings.spaceIgnore).accepts;
} else {
fileFilterFn = () => true;
}
},
);
// system = undefined in databaseless mode (no PlugOS instance on the server and no DB)
if (kvPrimitives) {
// Enable server-side processing
const serverSystem = new ServerSystem(
this.spacePrimitives,
kvPrimitives,
);
this.serverSystem = serverSystem;
}
}
async init() {
if (this.serverSystem) {
await this.serverSystem.init();
this.system = this.serverSystem.system;
}
await this.reloadSettings();
console.log("Booted server with hostname", this.hostname);
}
async reloadSettings() {
// TODO: Throttle this?
this.settings = await ensureSettingsAndIndex(this.spacePrimitives);
}
}

19
server/server_system.ts
View File

@ -25,7 +25,6 @@ import { shellSyscalls } from "../plugos/syscalls/shell.deno.ts";
import { SpacePrimitives } from "../common/spaces/space_primitives.ts"; import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
import { base64EncodedDataUrl } from "../plugos/asset_bundle/base64.ts"; import { base64EncodedDataUrl } from "../plugos/asset_bundle/base64.ts";
import { Plug } from "../plugos/plug.ts"; import { Plug } from "../plugos/plug.ts";
import { DenoKvPrimitives } from "../plugos/lib/deno_kv_primitives.ts";
import { DataStore } from "../plugos/lib/datastore.ts"; import { DataStore } from "../plugos/lib/datastore.ts";
import { dataStoreSyscalls } from "../plugos/syscalls/datastore.ts"; import { dataStoreSyscalls } from "../plugos/syscalls/datastore.ts";
import { DataStoreMQ } from "../plugos/lib/mq.datastore.ts"; import { DataStoreMQ } from "../plugos/lib/mq.datastore.ts";
@ -34,6 +33,7 @@ import { handlebarsSyscalls } from "../common/syscalls/handlebars.ts";
import { codeWidgetSyscalls } from "../web/syscalls/code_widget.ts"; import { codeWidgetSyscalls } from "../web/syscalls/code_widget.ts";
import { CodeWidgetHook } from "../web/hooks/code_widget.ts"; import { CodeWidgetHook } from "../web/hooks/code_widget.ts";
import { KVPrimitivesManifestCache } from "../plugos/manifest_cache.ts"; import { KVPrimitivesManifestCache } from "../plugos/manifest_cache.ts";
import { KvPrimitives } from "../plugos/lib/kv_primitives.ts";
const fileListInterval = 30 * 1000; // 30s const fileListInterval = 30 * 1000; // 30s
@ -42,27 +42,27 @@ const plugNameExtractRegex = /\/(.+)\.plug\.js$/;
export class ServerSystem { export class ServerSystem {
system!: System<SilverBulletHooks>; system!: System<SilverBulletHooks>;
spacePrimitives!: SpacePrimitives; spacePrimitives!: SpacePrimitives;
denoKv!: Deno.Kv; // denoKv!: Deno.Kv;
listInterval?: number; listInterval?: number;
ds!: DataStore; ds!: DataStore;
constructor( constructor(
private baseSpacePrimitives: SpacePrimitives, private baseSpacePrimitives: SpacePrimitives,
private dbPath: string, readonly kvPrimitives: KvPrimitives,
private app: Application,
) { ) {
} }
// Always needs to be invoked right after construction // Always needs to be invoked right after construction
async init(awaitIndex = false) { async init(awaitIndex = false) {
this.denoKv = await Deno.openKv(this.dbPath); this.ds = new DataStore(this.kvPrimitives);
const kvPrimitives = new DenoKvPrimitives(this.denoKv);
this.ds = new DataStore(kvPrimitives);
this.system = new System( this.system = new System(
"server", "server",
{ {
manifestCache: new KVPrimitivesManifestCache(kvPrimitives, "manifest"), manifestCache: new KVPrimitivesManifestCache(
this.kvPrimitives,
"manifest",
),
plugFlushTimeout: 5 * 60 * 1000, // 5 minutes plugFlushTimeout: 5 * 60 * 1000, // 5 minutes
}, },
); );
@ -75,9 +75,6 @@ export class ServerSystem {
const cronHook = new CronHook(this.system); const cronHook = new CronHook(this.system);
this.system.addHook(cronHook); this.system.addHook(cronHook);
// Endpoint hook
this.system.addHook(new EndpointHook(this.app, "/_/"));
const mq = new DataStoreMQ(this.ds); const mq = new DataStoreMQ(this.ds);
setInterval(() => { setInterval(() => {

64
server/shell_backend.ts Normal file
View File

@ -0,0 +1,64 @@
import { ShellRequest, ShellResponse } from "./rpc.ts";
/**
* Configuration via environment variables:
* - SB_SHELL_BACKEND: "local" or "off"
*/
export function determineShellBackend(path: string): ShellBackend {
const backendConfig = Deno.env.get("SB_SHELL_BACKEND") || "local";
switch (backendConfig) {
case "local":
return new LocalShell(path);
default:
console.info(
"Running in shellless mode, meaning shell commands are disabled",
);
return new NotSupportedShell();
}
}
export interface ShellBackend {
handle(shellRequest: ShellRequest): Promise<ShellResponse>;
}
class NotSupportedShell implements ShellBackend {
handle(): Promise<ShellResponse> {
return Promise.resolve({
stdout: "",
stderr: "Not supported",
code: 1,
});
}
}
class LocalShell implements ShellBackend {
constructor(private cwd: string) {
}
async handle(shellRequest: ShellRequest): Promise<ShellResponse> {
console.log(
"Running shell command:",
shellRequest.cmd,
shellRequest.args,
);
const p = new Deno.Command(shellRequest.cmd, {
cwd: this.cwd,
args: shellRequest.args,
stdout: "piped",
stderr: "piped",
});
const output = await p.output();
const stdout = new TextDecoder().decode(output.stdout);
const stderr = new TextDecoder().decode(output.stderr);
if (output.code !== 0) {
console.error("Error running shell command", stdout, stderr);
}
return {
stderr,
stdout,
code: output.code,
};
}
}

1
server/spaces/s3_space_primitives.test.ts
View File

@ -9,6 +9,7 @@ Deno.test("s3_space_primitives", async () => {
endPoint: "s3.eu-central-1.amazonaws.com", endPoint: "s3.eu-central-1.amazonaws.com",
region: "eu-central-1", region: "eu-central-1",
bucket: "zef-sb-space", bucket: "zef-sb-space",
prefix: "test",
}; };
const primitives = new S3SpacePrimitives(options); const primitives = new S3SpacePrimitives(options);

9
server/spaces/s3_space_primitives.ts
View File

@ -7,10 +7,15 @@ import { FileMeta } from "$sb/types.ts";
// TODO: IMPORTANT: This needs a different way to keep meta data (last modified and created dates) // TODO: IMPORTANT: This needs a different way to keep meta data (last modified and created dates)
export type S3SpacePrimitivesOptions = ClientOptions & { prefix: string };
export class S3SpacePrimitives implements SpacePrimitives { export class S3SpacePrimitives implements SpacePrimitives {
client: S3Client; client: S3Client;
constructor(options: ClientOptions) { prefix: string;
constructor(options: S3SpacePrimitivesOptions) {
this.client = new S3Client(options); this.client = new S3Client(options);
// TODO: Use this
this.prefix = options.prefix;
} }
private encodePath(name: string): string { private encodePath(name: string): string {
@ -18,7 +23,7 @@ export class S3SpacePrimitives implements SpacePrimitives {
} }
private decodePath(encoded: string): string { private decodePath(encoded: string): string {
// AWS only returns ' replace dwith &apos; // AWS only returns ' replace with &apos;
return encoded.replaceAll("&apos;", "'"); return encoded.replaceAll("&apos;", "'");
} }

22
server/storage_backend.ts Normal file
View File

@ -0,0 +1,22 @@
import { DiskSpacePrimitives } from "../common/spaces/disk_space_primitives.ts";
import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
import { path } from "./deps.ts";
import { S3SpacePrimitives } from "./spaces/s3_space_primitives.ts";
export function determineStorageBackend(folder: string): SpacePrimitives {
if (folder === "s3://") {
console.info("Using S3 as a storage backend");
return new S3SpacePrimitives({
accessKey: Deno.env.get("AWS_ACCESS_KEY_ID")!,
secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!,
endPoint: Deno.env.get("AWS_ENDPOINT")!,
region: Deno.env.get("AWS_REGION")!,
bucket: Deno.env.get("AWS_BUCKET")!,
prefix: folder.slice(5),
});
} else {
folder = path.resolve(Deno.cwd(), folder);
console.info(`Using local disk as a storage backend: ${folder}`);
return new DiskSpacePrimitives(folder);
}
}

44
silverbullet.ts
View File

@ -7,10 +7,6 @@ import { upgradeCommand } from "./cmd/upgrade.ts";
import { versionCommand } from "./cmd/version.ts"; import { versionCommand } from "./cmd/version.ts";
import { serveCommand } from "./cmd/server.ts"; import { serveCommand } from "./cmd/server.ts";
import { plugCompileCommand } from "./cmd/plug_compile.ts"; import { plugCompileCommand } from "./cmd/plug_compile.ts";
import { userAdd } from "./cmd/user_add.ts";
import { userPasswd } from "./cmd/user_passwd.ts";
import { userDelete } from "./cmd/user_delete.ts";
import { userChgrp } from "./cmd/user_chgrp.ts";
import { plugRunCommand } from "./cmd/plug_run.ts"; import { plugRunCommand } from "./cmd/plug_run.ts";
await new Command() await new Command()
@ -83,47 +79,7 @@ await new Command()
"Hostname or address to listen on", "Hostname or address to listen on",
) )
.option("-p, --port <port:number>", "Port to listen on") .option("-p, --port <port:number>", "Port to listen on")
.option(
"--db <db:string>",
"Path to database file",
)
.action(plugRunCommand) .action(plugRunCommand)
.command("user:add", "Add a new user to an authentication file")
.arguments("[username:string]")
.option(
"--auth <auth.json:string>",
"User authentication file to use",
)
.option("-G, --group <name:string>", "Add user to group", {
collect: true,
default: [] as string[],
})
.action(userAdd)
.command("user:delete", "Delete an existing user")
.arguments("[username:string]")
.option(
"--auth <auth.json:string>",
"User authentication file to use",
)
.action(userDelete)
.command("user:chgrp", "Update user groups")
.arguments("[username:string]")
.option(
"--auth <auth.json:string>",
"User authentication file to use",
)
.option("-G, --group <name:string>", "Groups to put user into", {
collect: true,
default: [] as string[],
})
.action(userChgrp)
.command("user:passwd", "Set the password for an existing user")
.arguments("[username:string]")
.option(
"--auth <auth.json:string>",
"User authentication file to use",
)
.action(userPasswd)
// upgrade // upgrade
.command("upgrade", "Upgrade SilverBullet") .command("upgrade", "Upgrade SilverBullet")
.action(upgradeCommand) .action(upgradeCommand)

5
web/boot.ts
View File

@ -5,7 +5,10 @@ const syncMode = window.silverBulletConfig.syncOnly ||
!!localStorage.getItem("syncMode"); !!localStorage.getItem("syncMode");
safeRun(async () => { safeRun(async () => {
console.log("Booting SilverBullet..."); console.log(
"Booting SilverBullet client",
syncMode ? "in Sync Mode" : "in Online Mode",
);
const client = new Client( const client = new Client(
document.getElementById("sb-root")!, document.getElementById("sb-root")!,

6
web/space.ts
View File

@ -71,6 +71,7 @@ export class Space {
} }
}); });
eventHook.addLocalListener("file:listed", (files: FileMeta[]) => { eventHook.addLocalListener("file:listed", (files: FileMeta[]) => {
// console.log("Files listed", files);
this.cachedPageList = files.filter(this.isListedPage).map( this.cachedPageList = files.filter(this.isListedPage).map(
fileMetaToPageMeta, fileMetaToPageMeta,
); );
@ -227,6 +228,7 @@ export class Space {
export function fileMetaToPageMeta(fileMeta: FileMeta): PageMeta { export function fileMetaToPageMeta(fileMeta: FileMeta): PageMeta {
const name = fileMeta.name.substring(0, fileMeta.name.length - 3); const name = fileMeta.name.substring(0, fileMeta.name.length - 3);
try {
return { return {
...fileMeta, ...fileMeta,
ref: name, ref: name,
@ -235,4 +237,8 @@ export function fileMetaToPageMeta(fileMeta: FileMeta): PageMeta {
created: new Date(fileMeta.created).toISOString(), created: new Date(fileMeta.created).toISOString(),
lastModified: new Date(fileMeta.lastModified).toISOString(), lastModified: new Date(fileMeta.lastModified).toISOString(),
} as PageMeta; } as PageMeta;
} catch (e) {
console.error("Failed to convert fileMeta to pageMeta", fileMeta, e);
throw e;
}
} }

37
website/Authentication.md
View File

@ -1,8 +1,5 @@
SilverBullet supports simple authentication for one or many users. SilverBullet supports simple authentication for a single user.
**Note**: This feature is experimental and will likely change significantly over time.
## Single User
By simply passing the `--user` flag with a username:password combination, you enable authentication for a single user. For instance: By simply passing the `--user` flag with a username:password combination, you enable authentication for a single user. For instance:
```shell ```shell
@ -11,36 +8,10 @@ silverbullet --user pete:1234 .
Will let `pete` authenticate with password `1234`. Will let `pete` authenticate with password `1234`.
## Multiple users Alternative, the same information can be passed in via the `SB_USER` environment variable, e.g.
Although multi-user support is still rudimentary, it is possible to have multiple users authenticate. These users can be configured using a JSON authentication file that SB can generate for you. It is usually named `.auth.json`.
You can enable authentication as follows:
```shell ```shell
silverbullet --auth /path/to/.auth.json SB_USER=pete:1234 silverbullet .
``` ```
To create and manage an `.auth.json` file, you can use the following commands: This is especially convenient when deploying using Docker
* `silverbullet user:add --auth /path/to/.auth.json [username]` to add a user
* `silverbullet user:delete --auth /path/to/.auth.json [username]` to delete a user
* `silverbullet user:passwd --auth /path/to/.auth.json [username]` to update a password
If the `.auth.json` file does not yet exist, it will be created.
When SB is run with a `--auth` flag, this fill will automatically be reloaded upon change.
### Group management
While this functionality is not yet used, users can also be added to groups which can be arbitrarily named. The `admin` group will likely have a special meaning down the line.
When adding a user, you can add one more `-G` or `--group` flags:
```shell
silverbullet user:add --auth /path/to/.auth.json -G admin pete
```
And you can update these groups later with `silverbullet user:chgrp`:
```shell
silverbullet user:chgrp --auth /path/to/.auth.json -G admin pete
```

9
website/CHANGELOG.md
View File

@ -1,6 +1,15 @@
An attempt at documenting the changes/new features introduced in each An attempt at documenting the changes/new features introduced in each
release. release.
---
## Next
* Removed built-in multi-user [[Authentication]], `SB_AUTH` is no longer supported, use `--user` or `SB_USER` instead, or an authentication layer such as [[Authelia]]
* Technical refactoring in preparation of multi-tenant deployment support (allowing you to run a single SB instance and serve multiple spaces and users at the same time)
* Lazy everything: plugs are now lazily loaded (after a first load, manifests are cached). On the server side, a whole lot of infrastructure is now only booted once the first HTTP request comes in
---
## 0.5.8 ## 0.5.8
* Various bugfixes, primarily related to the new way of running docker containers, which broke things for some people. Be sure to have a look at the new [[Install/Local$env|environment variable]] configuration options * Various bugfixes, primarily related to the new way of running docker containers, which broke things for some people. Be sure to have a look at the new [[Install/Local$env|environment variable]] configuration options

1
website/Install/Local.md
View File

@ -140,5 +140,4 @@ You can configure SB with environment variables instead of flags, which is proba
* `SB_HOSTNAME`: Set to the hostname to bind to (defaults to `127.0.0.0`, set to `0.0.0.0` to accept outside connections) * `SB_HOSTNAME`: Set to the hostname to bind to (defaults to `127.0.0.0`, set to `0.0.0.0` to accept outside connections)
* `SB_PORT`: Sets the port to listen to, e.g. `SB_PORT=1234` * `SB_PORT`: Sets the port to listen to, e.g. `SB_PORT=1234`
* `SB_FOLDER`: Sets the folder to expose, e.g. `SB_FOLDER=/space` * `SB_FOLDER`: Sets the folder to expose, e.g. `SB_FOLDER=/space`
* `SB_AUTH`: Loads an [[Authentication]] database from a (JSON encoded) string, e.g. `SB_AUTH=$(cat /path/to/.auth.json)`
* `SB_SYNC_ONLY`: Runs the server in a "dumb" space store-only mode (not indexing content or keeping other state), e.g. `SB_SYNC_ONLY=1`. This will disable the Online [[Client Modes]] altogether (and not even show the sync icon in the top bar). Conceptually, [silverbullet.md](https://silverbullet.md) runs in this mode. * `SB_SYNC_ONLY`: Runs the server in a "dumb" space store-only mode (not indexing content or keeping other state), e.g. `SB_SYNC_ONLY=1`. This will disable the Online [[Client Modes]] altogether (and not even show the sync icon in the top bar). Conceptually, [silverbullet.md](https://silverbullet.md) runs in this mode.