mingzailao
/
silverbullet
mirror of https://github.com/silverbulletmd/silverbullet.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Refactoring work to support multi-tenancy and multiple storage, database backends (#598) 

* Backend infrastructure
* New backend configuration work
* Factor out KV prefixing
* Don't put assets in the manifest cache
* Removed fancy authentication stuff
* Documentation updates
pull/599/head
Zef Hemel 2023-12-10 13:23:42 +01:00 committed by GitHub
parent 573eca3676
commit 30ba3fcca7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
33 changed files with 647 additions and 781 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cli/plug_run.test.ts
View File

@ -30,7 +30,6 @@ Deno.test("Test plug run", {
assertEquals(
await runPlug(
testSpaceFolder,
tempDbFile,
"test.run",
[],
assetBundle,

21
cli/plug_run.ts
View File

@ -5,10 +5,11 @@ import { Application } from "../server/deps.ts";
import { sleep } from "$sb/lib/async.ts";
import { ServerSystem } from "../server/server_system.ts";
import { AssetBundlePlugSpacePrimitives } from "../common/spaces/asset_bundle_space_primitives.ts";
import { determineDatabaseBackend } from "../server/db_backend.ts";
import { EndpointHook } from "../plugos/hooks/endpoint.ts";
export async function runPlug(
spacePath: string,
dbPath: string,
functionName: string | undefined,
args: string[] = [],
builtinAssetBundle: AssetBundle,
@ -18,15 +19,27 @@ export async function runPlug(
const serverController = new AbortController();
const app = new Application();
const dbBackend = await determineDatabaseBackend();
if (!dbBackend) {
console.error("Cannot run plugs in databaseless mode.");
return;
}
const endpointHook = new EndpointHook("/_/");
const serverSystem = new ServerSystem(
new AssetBundlePlugSpacePrimitives(
new DiskSpacePrimitives(spacePath),
builtinAssetBundle,
),
dbPath,
app,
dbBackend,
);
await serverSystem.init(true);
app.use((context, next) => {
return endpointHook.handleRequest(serverSystem.system!, context, next);
});
app.listen({
hostname: httpHostname,
port: httpServerPort,
@ -42,7 +55,7 @@ export async function runPlug(
}
const result = await plug.invoke(funcName, args);
await serverSystem.close();
serverSystem.denoKv.close();
serverSystem.kvPrimitives.close();
serverController.abort();
return result;
} else {

8
cmd/plug_run.ts
View File

@ -4,15 +4,12 @@ import assets from "../dist/plug_asset_bundle.json" assert {
type: "json",
};
import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
import { silverBulletDbFile } from "./constants.ts";
export async function plugRunCommand(
{
db,
hostname,
port,
}: {
db?: string;
hostname?: string;
port?: number;
},
@ -22,15 +19,10 @@ export async function plugRunCommand(
) {
spacePath = path.resolve(spacePath);
console.log("Space path", spacePath);
let dbPath = path.resolve(spacePath, silverBulletDbFile);
if (db) {
dbPath = path.resolve(db);
}
console.log("Function to run:", functionName, "with arguments", args);
try {
const result = await runPlug(
spacePath,
dbPath,
functionName,
args,
new AssetBundle(assets),

132
cmd/server.ts
View File

@ -1,4 +1,4 @@
import { Application, path } from "../server/deps.ts";
import { Application } from "../server/deps.ts";
import { HttpServer } from "../server/http_server.ts";
import clientAssetBundle from "../dist/client_asset_bundle.json" assert {
type: "json",
@ -7,17 +7,11 @@ import plugAssetBundle from "../dist/plug_asset_bundle.json" assert {
type: "json",
};
import { AssetBundle, AssetJson } from "../plugos/asset_bundle/bundle.ts";
import { AssetBundlePlugSpacePrimitives } from "../common/spaces/asset_bundle_space_primitives.ts";
import { DiskSpacePrimitives } from "../common/spaces/disk_space_primitives.ts";
import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
import { S3SpacePrimitives } from "../server/spaces/s3_space_primitives.ts";
import { Authenticator } from "../server/auth.ts";
import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts";
import { ServerSystem } from "../server/server_system.ts";
import { sleep } from "$sb/lib/async.ts";
import { SilverBulletHooks } from "../common/manifest.ts";
import { System } from "../plugos/system.ts";
import { silverBulletDbFile } from "./constants.ts";
import { determineDatabaseBackend } from "../server/db_backend.ts";
import { SpaceServerConfig } from "../server/instance.ts";
import { path } from "../common/deps.ts";
export async function serveCommand(
options: {
@ -28,8 +22,6 @@ export async function serveCommand(
cert?: string;
key?: string;
reindex?: boolean;
db?: string;
syncOnly?: boolean;
},
folder?: string,
) {
@ -37,12 +29,11 @@ export async function serveCommand(
"127.0.0.1";
const port = options.port ||
(Deno.env.get("SB_PORT") && +Deno.env.get("SB_PORT")!) || 3000;
const syncOnly = options.syncOnly || Deno.env.get("SB_SYNC_ONLY");
let dbFile = options.db || Deno.env.get("SB_DB_FILE") || silverBulletDbFile;
const app = new Application();
if (!folder) {
// Didn't get a folder as an argument, check if we got it as an environment variable
folder = Deno.env.get("SB_FOLDER");
if (!folder) {
console.error(
@ -51,105 +42,44 @@ export async function serveCommand(
Deno.exit(1);
}
}
folder = path.resolve(Deno.cwd(), folder);
const baseKvPrimitives = await determineDatabaseBackend(folder);
console.log(
"Going to start SilverBullet binding to",
`${hostname}:${port}`,
);
if (hostname === "127.0.0.1") {
console.log(
`NOTE: SilverBullet will only be available locally (via http://localhost:${port}).
console.info(
`SilverBullet will only be available locally (via http://localhost:${port}).
To allow outside connections, pass -L 0.0.0.0 as a flag, and put a TLS terminator on top.`,
);
}
let spacePrimitives: SpacePrimitives | undefined;
if (folder === "s3://") {
spacePrimitives = new S3SpacePrimitives({
accessKey: Deno.env.get("AWS_ACCESS_KEY_ID")!,
secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!,
endPoint: Deno.env.get("AWS_ENDPOINT")!,
region: Deno.env.get("AWS_REGION")!,
bucket: Deno.env.get("AWS_BUCKET")!,
});
console.log("Running in S3 mode");
folder = Deno.cwd();
} else {
// Regular disk mode
folder = path.resolve(Deno.cwd(), folder);
spacePrimitives = new DiskSpacePrimitives(folder);
}
spacePrimitives = new AssetBundlePlugSpacePrimitives(
spacePrimitives,
new AssetBundle(plugAssetBundle as AssetJson),
);
const userAuth = options.user ?? Deno.env.get("SB_USER");
let system: System<SilverBulletHooks> | undefined;
// system = undefined in syncOnly mode (no PlugOS instance on the server)
if (!syncOnly) {
// Enable server-side processing
dbFile = path.resolve(folder, dbFile);
console.log(
`Running in server-processing mode, keeping state in ${dbFile}`,
);
const serverSystem = new ServerSystem(spacePrimitives, dbFile, app);
await serverSystem.init();
spacePrimitives = serverSystem.spacePrimitives;
system = serverSystem.system;
if (options.reindex) {
console.log("Reindexing space (requested via --reindex flag)");
serverSystem.system.loadedPlugs.get("index")!.invoke(
"reindexSpace",
[],
).catch(console.error);
}
}
const configs = new Map<string, SpaceServerConfig>();
configs.set("*", {
hostname,
namespace: "*",
auth: userAuth,
pagesPath: folder,
});
const authStore = new JSONKVStore();
const authenticator = new Authenticator(authStore);
const flagUser = options.user ?? Deno.env.get("SB_USER");
if (flagUser) {
// If explicitly added via env/parameter, add on the fly
const [username, password] = flagUser.split(":");
await authenticator.register(username, password, ["admin"], "");
}
if (options.auth) {
// Load auth file
const authFile: string = options.auth;
console.log("Loading authentication credentials from", authFile);
await authStore.load(authFile);
(async () => {
// Asynchronously kick off file watcher
for await (const _event of Deno.watchFs(options.auth!)) {
console.log("Authentication file changed, reloading...");
await authStore.load(authFile);
}
})().catch(console.error);
}
const envAuth = Deno.env.get("SB_AUTH");
if (envAuth) {
console.log("Loading authentication from SB_AUTH");
authStore.loadString(envAuth);
}
const httpServer = new HttpServer(
spacePrimitives!,
const httpServer = new HttpServer({
app,
system,
{
hostname,
port: port,
pagesPath: folder!,
clientAssetBundle: new AssetBundle(clientAssetBundle as AssetJson),
authenticator,
keyFile: options.key,
certFile: options.cert,
},
);
await httpServer.start();
hostname,
port,
clientAssetBundle: new AssetBundle(clientAssetBundle as AssetJson),
plugAssetBundle: new AssetBundle(plugAssetBundle as AssetJson),
baseKvPrimitives,
syncOnly: baseKvPrimitives === undefined,
keyFile: options.key,
certFile: options.cert,
configs,
});
httpServer.start();
// Wait in an infinite loop (to keep the HTTP server running, only cancelable via Ctrl+C or other signal)
while (true) {

35
cmd/user_add.ts
View File

@ -1,35 +0,0 @@
import getpass from "https://deno.land/x/getpass@0.3.1/mod.ts";
import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts";
import { Authenticator } from "../server/auth.ts";
export async function userAdd(
options: any,
username?: string,
) {
const authFile = options.auth || ".auth.json";
console.log("Using auth file", authFile);
if (!username) {
username = prompt("Username:")!;
}
if (!username) {
return;
}
const pw = getpass("Password: ");
if (!pw) {
return;
}
console.log("Adding user to groups", options.group);
const store = new JSONKVStore();
try {
await store.load(authFile);
} catch (e: any) {
if (e instanceof Deno.errors.NotFound) {
console.log("Creating new auth database because it didn't exist.");
}
}
const auth = new Authenticator(store);
await auth.register(username!, pw!, options.group);
await store.save(authFile);
}

30
cmd/user_chgrp.ts
View File

@ -1,30 +0,0 @@
import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts";
import { Authenticator } from "../server/auth.ts";
export async function userChgrp(
options: any,
username?: string,
) {
const authFile = options.auth || ".auth.json";
console.log("Using auth file", authFile);
if (!username) {
username = prompt("Username:")!;
}
if (!username) {
return;
}
console.log("Setting groups for user:", options.group);
const store = new JSONKVStore();
try {
await store.load(authFile);
} catch (e: any) {
if (e instanceof Deno.errors.NotFound) {
console.log("Creating new auth database because it didn't exist.");
}
}
const auth = new Authenticator(store);
await auth.setGroups(username!, options.group);
await store.save(authFile);
}

37
cmd/user_delete.ts
View File

@ -1,37 +0,0 @@
import getpass from "https://deno.land/x/getpass@0.3.1/mod.ts";
import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts";
import { Authenticator } from "../server/auth.ts";
export async function userDelete(
options: any,
username?: string,
) {
const authFile = options.auth || ".auth.json";
console.log("Using auth file", authFile);
if (!username) {
username = prompt("Username:")!;
}
if (!username) {
return;
}
const store = new JSONKVStore();
try {
await store.load(authFile);
} catch (e: any) {
if (e instanceof Deno.errors.NotFound) {
console.log("Creating new auth database because it didn't exist.");
}
}
const auth = new Authenticator(store);
const user = await auth.getUser(username);
if (!user) {
console.error("User", username, "not found.");
Deno.exit(1);
}
await auth.deleteUser(username!);
await store.save(authFile);
}

42
cmd/user_passwd.ts
View File

@ -1,42 +0,0 @@
import getpass from "https://deno.land/x/getpass@0.3.1/mod.ts";
import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts";
import { Authenticator } from "../server/auth.ts";
export async function userPasswd(
options: any,
username?: string,
) {
const authFile = options.auth || ".auth.json";
console.log("Using auth file", authFile);
if (!username) {
username = prompt("Username:")!;
}
if (!username) {
return;
}
const store = new JSONKVStore();
try {
await store.load(authFile);
} catch (e: any) {
if (e instanceof Deno.errors.NotFound) {
console.log("Creating new auth database because it didn't exist.");
}
}
const auth = new Authenticator(store);
const user = await auth.getUser(username);
if (!user) {
console.error("User", username, "not found.");
Deno.exit(1);
}
const pw = getpass("New password: ");
if (!pw) {
return;
}
await auth.setPassword(username!, pw!);
await store.save(authFile);
}

6
plugos/hooks/endpoint.test.ts
View File

@ -26,7 +26,11 @@ Deno.test("Run a plugos endpoint server", async () => {
const app = new Application();
const port = 3123;
system.addHook(new EndpointHook(app, "/_/"));
const endpointHook = new EndpointHook("/_/");
app.use((context, next) => {
return endpointHook.handleRequest(system, context, next);
});
const controller = new AbortController();
app.listen({ port: port, signal: controller.signal });

139
plugos/hooks/endpoint.ts
View File

@ -1,6 +1,6 @@
import { Hook, Manifest } from "../types.ts";
import { System } from "../system.ts";
import { Application } from "../../server/deps.ts";
import { Application, Context, Next } from "../../server/deps.ts";
export type EndpointRequest = {
method: string;
@ -26,87 +26,90 @@ export type EndPointDef = {
};
export class EndpointHook implements Hook<EndpointHookT> {
private app: Application;
readonly prefix: string;
constructor(app: Application, prefix: string) {
this.app = app;
constructor(prefix: string) {
this.prefix = prefix;
}
apply(system: System<EndpointHookT>): void {
this.app.use(async (ctx, next) => {
const req = ctx.request;
const requestPath = ctx.request.url.pathname;
if (!requestPath.startsWith(this.prefix)) {
return next();
public async handleRequest(
system: System<EndpointHookT>,
ctx: Context,
next: Next,
) {
const req = ctx.request;
const requestPath = ctx.request.url.pathname;
if (!requestPath.startsWith(this.prefix)) {
return next();
}
console.log("Endpoint request", requestPath);
// Iterate over all loaded plugins
for (const [plugName, plug] of system.loadedPlugs.entries()) {
const manifest = plug.manifest;
if (!manifest) {
continue;
}
console.log("Endpoint request", requestPath);
// Iterate over all loaded plugins
for (const [plugName, plug] of system.loadedPlugs.entries()) {
const manifest = plug.manifest;
if (!manifest) {
const functions = manifest.functions;
// console.log("Checking plug", plugName);
const prefix = `${this.prefix}${plugName}`;
if (!requestPath.startsWith(prefix)) {
continue;
}
for (const [name, functionDef] of Object.entries(functions)) {
if (!functionDef.http) {
continue;
}
const functions = manifest.functions;
// console.log("Checking plug", plugName);
const prefix = `${this.prefix}${plugName}`;
if (!requestPath.startsWith(prefix)) {
continue;
}
for (const [name, functionDef] of Object.entries(functions)) {
if (!functionDef.http) {
continue;
}
// console.log("Got config", functionDef);
const endpoints = Array.isArray(functionDef.http)
? functionDef.http
: [functionDef.http];
// console.log(endpoints);
for (const { path, method } of endpoints) {
const prefixedPath = `${prefix}${path}`;
if (
prefixedPath === requestPath &&
((method || "GET") === req.method || method === "ANY")
) {
try {
const response: EndpointResponse = await plug.invoke(name, [
{
path: req.url.pathname,
method: req.method,
body: req.body(),
query: Object.fromEntries(
req.url.searchParams.entries(),
),
headers: Object.fromEntries(req.headers.entries()),
} as EndpointRequest,
]);
if (response.headers) {
for (
const [key, value] of Object.entries(
response.headers,
)
) {
ctx.response.headers.set(key, value);
}
// console.log("Got config", functionDef);
const endpoints = Array.isArray(functionDef.http)
? functionDef.http
: [functionDef.http];
// console.log(endpoints);
for (const { path, method } of endpoints) {
const prefixedPath = `${prefix}${path}`;
if (
prefixedPath === requestPath &&
((method || "GET") === req.method || method === "ANY")
) {
try {
const response: EndpointResponse = await plug.invoke(name, [
{
path: req.url.pathname,
method: req.method,
body: req.body(),
query: Object.fromEntries(
req.url.searchParams.entries(),
),
headers: Object.fromEntries(req.headers.entries()),
} as EndpointRequest,
]);
if (response.headers) {
for (
const [key, value] of Object.entries(
response.headers,
)
) {
ctx.response.headers.set(key, value);
}
ctx.response.status = response.status;
ctx.response.body = response.body;
// console.log("Sent result");
return;
} catch (e: any) {
console.error("Error executing function", e);
ctx.response.status = 500;
ctx.response.body = e.message;
return;
}
ctx.response.status = response.status;
ctx.response.body = response.body;
// console.log("Sent result");
return;
} catch (e: any) {
console.error("Error executing function", e);
ctx.response.status = 500;
ctx.response.body = e.message;
return;
}
}
}
}
// console.log("Shouldn't get here");
await next();
});
}
// console.log("Shouldn't get here");
await next();
}
apply(): void {
}
validateManifest(manifest: Manifest<EndpointHookT>): string[] {

4
plugos/lib/datastore.test.ts
View File

@ -2,11 +2,11 @@ import "https://esm.sh/fake-indexeddb@4.0.2/auto";
import { IndexedDBKvPrimitives } from "./indexeddb_kv_primitives.ts";
import { DataStore } from "./datastore.ts";
import { DenoKvPrimitives } from "./deno_kv_primitives.ts";
import { KvPrimitives } from "./kv_primitives.ts";
import { KvPrimitives, PrefixedKvPrimitives } from "./kv_primitives.ts";
import { assertEquals } from "https://deno.land/std@0.165.0/testing/asserts.ts";
async function test(db: KvPrimitives) {
const datastore = new DataStore(db, ["ds"], {
const datastore = new DataStore(new PrefixedKvPrimitives(db, ["ds"]), {
count: (arr: any[]) => arr.length,
});
await datastore.set(["user", "peter"], { name: "Peter" });

38
plugos/lib/datastore.ts
View File

@ -10,25 +10,16 @@ import { KvPrimitives } from "./kv_primitives.ts";
export class DataStore {
constructor(
readonly kv: KvPrimitives,
private prefix: KvKey = [],
private functionMap: FunctionMap = builtinFunctions,
) {
}
prefixed(prefix: KvKey): DataStore {
return new DataStore(
this.kv,
[...this.prefix, ...prefix],
this.functionMap,
);
}
async get<T = any>(key: KvKey): Promise<T | null> {
return (await this.batchGet([key]))[0];
}
batchGet<T = any>(keys: KvKey[]): Promise<(T | null)[]> {
return this.kv.batchGet(keys.map((key) => this.applyPrefix(key)));
return this.kv.batchGet(keys);
}
set(key: KvKey, value: any): Promise<void> {
@ -44,7 +35,7 @@ export class DataStore {
console.warn(`Duplicate key ${keyString} in batchSet, skipping`);
} else {
allKeyStrings.add(keyString);
uniqueEntries.push({ key: this.applyPrefix(key), value });
uniqueEntries.push({ key, value });
}
}
return this.kv.batchSet(uniqueEntries);
@ -55,7 +46,7 @@ export class DataStore {
}
batchDelete(keys: KvKey[]): Promise<void> {
return this.kv.batchDelete(keys.map((key) => this.applyPrefix(key)));
return this.kv.batchDelete(keys);
}
async query<T = any>(query: KvQuery): Promise<KV<T>[]> {
@ -63,15 +54,11 @@ export class DataStore {
let itemCount = 0;
// Accumulate results
let limit = Infinity;
const prefixedQuery: KvQuery = {
...query,
prefix: query.prefix ? this.applyPrefix(query.prefix) : undefined,
};
if (query.limit) {
limit = evalQueryExpression(query.limit, {}, this.functionMap);
}
for await (
const entry of this.kv.query(prefixedQuery)
const entry of this.kv.query(query)
) {
// Filter
if (
@ -89,29 +76,16 @@ export class DataStore {
}
}
// Apply order by, limit, and select
return applyQueryNoFilterKV(prefixedQuery, results, this.functionMap).map((
{ key, value },
) => ({ key: this.stripPrefix(key), value }));
return applyQueryNoFilterKV(query, results, this.functionMap);
}
async queryDelete(query: KvQuery): Promise<void> {
const keys: KvKey[] = [];
for (
const { key } of await this.query({
...query,
prefix: query.prefix ? this.applyPrefix(query.prefix) : undefined,
})
const { key } of await this.query(query)
) {
keys.push(key);
}
return this.batchDelete(keys);
}
private applyPrefix(key: KvKey): KvKey {
return [...this.prefix, ...(key ? key : [])];
}
private stripPrefix(key: KvKey): KvKey {
return key.slice(this.prefix.length);
}
}

44
plugos/lib/kv_primitives.ts
View File

@ -9,4 +9,48 @@ export interface KvPrimitives {
batchSet(entries: KV[]): Promise<void>;
batchDelete(keys: KvKey[]): Promise<void>;
query(options: KvQueryOptions): AsyncIterableIterator<KV>;
close(): void;
}
/**
* Turns any KvPrimitives into a KvPrimitives that automatically prefixes all keys (and removes them again when reading)
*/
export class PrefixedKvPrimitives implements KvPrimitives {
constructor(private wrapped: KvPrimitives, private prefix: KvKey) {
}
batchGet(keys: KvKey[]): Promise<any[]> {
return this.wrapped.batchGet(keys.map((key) => this.applyPrefix(key)));
}
batchSet(entries: KV[]): Promise<void> {
return this.wrapped.batchSet(
entries.map(({ key, value }) => ({ key: this.applyPrefix(key), value })),
);
}
batchDelete(keys: KvKey[]): Promise<void> {
return this.wrapped.batchDelete(keys.map((key) => this.applyPrefix(key)));
}
async *query(options: KvQueryOptions): AsyncIterableIterator<KV> {
for await (
const result of this.wrapped.query({
prefix: this.applyPrefix(options.prefix),
})
) {
yield { key: this.stripPrefix(result.key), value: result.value };
}
}
close(): void {
this.wrapped.close();
}
private applyPrefix(key?: KvKey): KvKey {
return [...this.prefix, ...(key ? key : [])];
}
private stripPrefix(key: KvKey): KvKey {
return key.slice(this.prefix.length);
}
}

56
plugos/lib/kv_store.json_file.ts
View File

@ -1,56 +0,0 @@
import { KV } from "$sb/types.ts";
export class JSONKVStore {
private data: { [key: string]: any } = {};
async load(path: string) {
this.loadString(await Deno.readTextFile(path));
}
loadString(jsonString: string) {
this.data = JSON.parse(jsonString);
}
async save(path: string) {
await Deno.writeTextFile(path, JSON.stringify(this.data));
}
del(key: string): Promise<void> {
delete this.data[key];
return Promise.resolve();
}
deletePrefix(prefix: string): Promise<void> {
for (const key in this.data) {
if (key.startsWith(prefix)) {
delete this.data[key];
}
}
return Promise.resolve();
}
deleteAll(): Promise<void> {
this.data = {};
return Promise.resolve();
}
set(key: string, value: any): Promise<void> {
this.data[key] = value;
return Promise.resolve();
}
get(key: string): Promise<any> {
return Promise.resolve(this.data[key]);
}
has(key: string): Promise<boolean> {
return Promise.resolve(key in this.data);
}
queryPrefix(keyPrefix: string): Promise<{ key: string; value: any }[]> {
const results: { key: string; value: any }[] = [];
for (const key in this.data) {
if (key.startsWith(keyPrefix)) {
results.push({ key, value: this.data[key] });
}
}
return Promise.resolve(results);
}
}

5
plugos/lib/mq.datastore.test.ts
View File

@ -3,12 +3,15 @@ import { assertEquals } from "../../test_deps.ts";
import { sleep } from "$sb/lib/async.ts";
import { DenoKvPrimitives } from "./deno_kv_primitives.ts";
import { DataStore } from "./datastore.ts";
import { PrefixedKvPrimitives } from "./kv_primitives.ts";
Deno.test("DataStore MQ", async () => {
const tmpFile = await Deno.makeTempFile();
const db = new DenoKvPrimitives(await Deno.openKv(tmpFile));
const mq = new DataStoreMQ(new DataStore(db, ["mq"]));
const mq = new DataStoreMQ(
new DataStore(new PrefixedKvPrimitives(db, ["mq"])),
);
await mq.send("test", "Hello World");
let messages = await mq.poll("test", 10);
assertEquals(messages.length, 1);

2
plugos/system.ts
View File

@ -141,7 +141,7 @@ export class System<HookT> extends EventEmitter<SystemEvents<HookT>> {
if (this.plugs.has(manifest.name)) {
this.unload(manifest.name);
}
console.log("Loaded plug", manifest.name);
console.log("Activated plug", manifest.name);
this.plugs.set(manifest.name, plug);
await this.emit("plugLoaded", plug);

120
server/auth.ts
View File

@ -1,120 +0,0 @@
import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts";
export type User = {
username: string;
passwordHash: string; // hashed password
salt: string;
groups: string[]; // special "admin"
};
async function createUser(
username: string,
password: string,
groups: string[],
salt = generateSalt(16),
): Promise<User> {
return {
username,
passwordHash: await hashSHA256(`${salt}${password}`),
salt,
groups,
};
}
const userPrefix = `u:`;
export class Authenticator {
constructor(private store: JSONKVStore) {
}
async register(
username: string,
password: string,
groups: string[],
salt?: string,
): Promise<void> {
await this.store.set(
`${userPrefix}${username}`,
await createUser(username, password, groups, salt),
);
}
async authenticateHashed(
username: string,
hashedPassword: string,
): Promise<boolean> {
const user = await this.store.get(`${userPrefix}${username}`) as User;
if (!user) {
return false;
}
return user.passwordHash === hashedPassword;
}
async authenticate(
username: string,
password: string,
): Promise<string | undefined> {
const user = await this.store.get(`${userPrefix}${username}`) as User;
if (!user) {
return undefined;
}
const hashedPassword = await hashSHA256(`${user.salt}${password}`);
return user.passwordHash === hashedPassword ? hashedPassword : undefined;
}
async getAllUsers(): Promise<User[]> {
return (await this.store.queryPrefix(userPrefix)).map((item) => item.value);
}
getUser(username: string): Promise<User | undefined> {
return this.store.get(`${userPrefix}${username}`);
}
async setPassword(username: string, password: string): Promise<void> {
const user = await this.getUser(username);
if (!user) {
throw new Error(`User does not exist`);
}
user.passwordHash = await hashSHA256(`${user.salt}${password}`);
await this.store.set(`${userPrefix}${username}`, user);
}
async deleteUser(username: string): Promise<void> {
const user = await this.getUser(username);
if (!user) {
throw new Error(`User does not exist`);
}
await this.store.del(`${userPrefix}${username}`);
}
async setGroups(username: string, groups: string[]): Promise<void> {
const user = await this.getUser(username);
if (!user) {
throw new Error(`User does not exist`);
}
user.groups = groups;
await this.store.set(`${userPrefix}${username}`, user);
}
}
async function hashSHA256(message: string): Promise<string> {
// Transform the string into an ArrayBuffer
const encoder = new TextEncoder();
const data = encoder.encode(message);
// Generate the hash
const hashBuffer = await window.crypto.subtle.digest("SHA-256", data);
// Transform the hash into a hex string
return Array.from(new Uint8Array(hashBuffer)).map((b) =>
b.toString(16).padStart(2, "0")
).join("");
}
function generateSalt(length: number): string {
const array = new Uint8Array(length / 2); // because two characters represent one byte in hex
crypto.getRandomValues(array);
return Array.from(array, (byte) => ("00" + byte.toString(16)).slice(-2)).join(
"",
);
}

13
server/crypto.ts Normal file
View File

@ -0,0 +1,13 @@
export async function hashSHA256(message: string): Promise<string> {
// Transform the string into an ArrayBuffer
const encoder = new TextEncoder();
const data = encoder.encode(message);
// Generate the hash
const hashBuffer = await window.crypto.subtle.digest("SHA-256", data);
// Transform the hash into a hex string
return Array.from(new Uint8Array(hashBuffer)).map((b) =>
b.toString(16).padStart(2, "0")
).join("");
}

44
server/db_backend.ts Normal file
View File

@ -0,0 +1,44 @@
import { DenoKvPrimitives } from "../plugos/lib/deno_kv_primitives.ts";
import { KvPrimitives } from "../plugos/lib/kv_primitives.ts";
import { path } from "./deps.ts";
/**
* Environment variables:
* - SB_DB_BACKEND: "denokv" or "off" (default: denokv)
* - SB_KV_DB (denokv only): path to the database file (default .silverbullet.db) or ":cloud:" for cloud storage
*/
export async function determineDatabaseBackend(
singleTenantFolder?: string,
): Promise<
KvPrimitives | undefined
> {
const backendConfig = Deno.env.get("SB_DB_BACKEND") || "denokv";
switch (backendConfig) {
case "denokv": {
let dbFile: string | undefined = Deno.env.get("SB_KV_DB") ||
".silverbullet.db";
if (singleTenantFolder) {
// If we're running in single tenant mode, we may as well use the tenant's space folder to keep the database
dbFile = path.resolve(singleTenantFolder, dbFile);
}
if (dbFile === ":cloud:") {
dbFile = undefined; // Deno Deploy will use the default KV store
}
const denoDb = await Deno.openKv(dbFile);
console.info(
`Using DenoKV as a database backend (${
dbFile || "cloud"
}), running in server-processing mode.`,
);
return new DenoKvPrimitives(denoDb);
}
default:
console.info(
"Running in databaseless mode: no server-side indexing and state keeping (beyond space files) will happen.",
);
return;
}
}

2
server/deps.ts
View File

@ -3,6 +3,8 @@ export type { Next } from "https://deno.land/x/oak@v12.4.0/mod.ts";
export {
Application,
Context,
Request,
Response,
Router,
} from "https://deno.land/x/oak@v12.4.0/mod.ts";
export * as etag from "https://deno.land/x/oak@v12.4.0/etag.ts";

329
server/http_server.ts
View File

@ -1,95 +1,155 @@
import { Application, Context, Next, oakCors, Router } from "./deps.ts";
import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
import { ensureSettingsAndIndex } from "../common/util.ts";
import { BuiltinSettings } from "../web/types.ts";
import { gitIgnoreCompiler } from "./deps.ts";
import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts";
import { Authenticator } from "./auth.ts";
import { FileMeta } from "$sb/types.ts";
import {
ShellRequest,
ShellResponse,
SyscallRequest,
SyscallResponse,
} from "./rpc.ts";
import { SilverBulletHooks } from "../common/manifest.ts";
import { System } from "../plugos/system.ts";
Application,
Context,
Next,
oakCors,
Request,
Router,
} from "./deps.ts";
import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
import { FileMeta } from "$sb/types.ts";
import { ShellRequest, SyscallRequest, SyscallResponse } from "./rpc.ts";
import { determineShellBackend } from "./shell_backend.ts";
import { SpaceServer, SpaceServerConfig } from "./instance.ts";
import {
KvPrimitives,
PrefixedKvPrimitives,
} from "../plugos/lib/kv_primitives.ts";
import { EndpointHook } from "../plugos/hooks/endpoint.ts";
import { hashSHA256 } from "./crypto.ts";
export type ServerOptions = {
app: Application;
hostname: string;
port: number;
pagesPath: string;
clientAssetBundle: AssetBundle;
authenticator: Authenticator;
pass?: string;
plugAssetBundle: AssetBundle;
baseKvPrimitives?: KvPrimitives;
syncOnly: boolean;
certFile?: string;
keyFile?: string;
configs: Map<string, SpaceServerConfig>;
};
export class HttpServer {
private hostname: string;
private port: number;
abortController?: AbortController;
clientAssetBundle: AssetBundle;
settings?: BuiltinSettings;
spacePrimitives: SpacePrimitives;
authenticator: Authenticator;
plugAssetBundle: AssetBundle;
hostname: string;
port: number;
app: Application<Record<string, any>>;
keyFile: string | undefined;
certFile: string | undefined;
constructor(
spacePrimitives: SpacePrimitives,
private app: Application,
private system: System<SilverBulletHooks> | undefined,
private options: ServerOptions,
) {
spaceServers = new Map<string, Promise<SpaceServer>>();
syncOnly: boolean;
baseKvPrimitives?: KvPrimitives;
configs: Map<string, SpaceServerConfig>;
constructor(options: ServerOptions) {
this.clientAssetBundle = options.clientAssetBundle;
this.plugAssetBundle = options.plugAssetBundle;
this.hostname = options.hostname;
this.port = options.port;
this.authenticator = options.authenticator;
this.clientAssetBundle = options.clientAssetBundle;
this.app = options.app;
this.keyFile = options.keyFile;
this.certFile = options.certFile;
this.syncOnly = options.syncOnly;
this.baseKvPrimitives = options.baseKvPrimitives;
this.configs = options.configs;
}
let fileFilterFn: (s: string) => boolean = () => true;
this.spacePrimitives = new FilteredSpacePrimitives(
spacePrimitives,
(meta) => fileFilterFn(meta.name),
async () => {
await this.reloadSettings();
if (typeof this.settings?.spaceIgnore === "string") {
fileFilterFn = gitIgnoreCompiler(this.settings.spaceIgnore).accepts;
} else {
fileFilterFn = () => true;
}
},
async bootSpaceServer(config: SpaceServerConfig): Promise<SpaceServer> {
const spaceServer = new SpaceServer(
config,
determineShellBackend(config.pagesPath),
this.plugAssetBundle,
this.baseKvPrimitives
? new PrefixedKvPrimitives(this.baseKvPrimitives, [
config.namespace,
])
: undefined,
);
await spaceServer.init();
return spaceServer;
}
determineConfig(req: Request): [string, SpaceServerConfig] {
let hostname = req.url.host; // hostname:port
// First try a full match
let config = this.configs.get(hostname);
if (config) {
return [hostname, config];
}
// Then rip off the port and try again
hostname = hostname.split(":")[0];
config = this.configs.get(hostname);
if (config) {
return [hostname, config];
}
// If all else fails, try the wildcard
config = this.configs.get("*");
if (config) {
return ["*", config];
}
throw new Error(`No space server config found for hostname ${hostname}`);
}
ensureSpaceServer(req: Request): Promise<SpaceServer> {
const [matchedHostname, config] = this.determineConfig(req);
const spaceServer = this.spaceServers.get(matchedHostname);
if (spaceServer) {
return spaceServer;
}
// And then boot the thing, async
const spaceServerPromise = this.bootSpaceServer(config);
// But immediately write the promise to the map so that we don't boot it twice
this.spaceServers.set(matchedHostname, spaceServerPromise);
return spaceServerPromise;
}
// Replaces some template variables in index.html in a rather ad-hoc manner, but YOLO
renderIndexHtml() {
renderIndexHtml(pagesPath: string) {
return this.clientAssetBundle.readTextFileSync(".client/index.html")
.replaceAll(
"{{SPACE_PATH}}",
this.options.pagesPath.replaceAll("\\", "\\\\"),
pagesPath.replaceAll("\\", "\\\\"),
// );
).replaceAll(
"{{SYNC_ONLY}}",
this.system ? "false" : "true",
this.syncOnly ? "true" : "false",
);
}
async start() {
await this.reloadSettings();
start() {
// Serve static files (javascript, css, html)
this.app.use(this.serveStatic.bind(this));
await this.addPasswordAuth(this.app);
const fsRouter = this.addFsRoutes(this.spacePrimitives);
const endpointHook = new EndpointHook("/_/");
this.app.use(async (context, next) => {
const spaceServer = await this.ensureSpaceServer(context.request);
return endpointHook.handleRequest(spaceServer.system!, context, next);
});
this.addPasswordAuth(this.app);
const fsRouter = this.addFsRoutes();
this.app.use(fsRouter.routes());
this.app.use(fsRouter.allowedMethods());
// Fallback, serve the UI index.html
this.app.use(({ response }) => {
this.app.use(async ({ request, response }) => {
response.headers.set("Content-type", "text/html");
response.body = this.renderIndexHtml();
response.headers.set("Cache-Control", "no-cache");
const spaceServer = await this.ensureSpaceServer(request);
response.body = this.renderIndexHtml(spaceServer.pagesPath);
});
this.abortController = new AbortController();
@ -98,11 +158,11 @@ export class HttpServer {
port: this.port,
signal: this.abortController.signal,
};
if (this.options.keyFile) {
listenOptions.key = Deno.readTextFileSync(this.options.keyFile);
if (this.keyFile) {
listenOptions.key = Deno.readTextFileSync(this.keyFile);
}
if (this.options.certFile) {
listenOptions.cert = Deno.readTextFileSync(this.options.certFile);
if (this.certFile) {
listenOptions.cert = Deno.readTextFileSync(this.certFile);
}
this.app.listen(listenOptions)
.catch((e: any) => {
@ -117,7 +177,7 @@ export class HttpServer {
);
}
serveStatic(
async serveStatic(
{ request, response }: Context<Record<string, any>, Record<string, any>>,
next: Next,
) {
@ -127,7 +187,9 @@ export class HttpServer {
// Serve the UI (index.html)
// Note: we're explicitly not setting Last-Modified and If-Modified-Since header here because this page is dynamic
response.headers.set("Content-type", "text/html");
response.body = this.renderIndexHtml();
response.headers.set("Cache-Control", "no-cache");
const spaceServer = await this.ensureSpaceServer(request);
response.body = this.renderIndexHtml(spaceServer.pagesPath);
return;
}
try {
@ -163,12 +225,7 @@ export class HttpServer {
}
}
async reloadSettings() {
// TODO: Throttle this?
this.settings = await ensureSettingsAndIndex(this.spacePrimitives);
}
private async addPasswordAuth(app: Application) {
private addPasswordAuth(app: Application) {
const excludedPaths = [
"/manifest.json",
"/favicon.png",
@ -192,14 +249,17 @@ export class HttpServer {
return;
} else if (request.method === "POST") {
const values = await request.body({ type: "form" }).value;
const username = values.get("username")!,
password = values.get("password")!,
refer = values.get("refer");
const hashedPassword = await this.authenticator.authenticate(
username,
password,
);
if (hashedPassword) {
const username = values.get("username")!;
const password = values.get("password")!;
const refer = values.get("refer");
const spaceServer = await this.ensureSpaceServer(request);
const hashedPassword = await hashSHA256(password);
const [expectedUser, expectedPassword] = spaceServer.auth!.split(":");
if (
username === expectedUser &&
hashedPassword === await hashSHA256(expectedPassword)
) {
await cookies.set(
authCookieName(host),
`${username}:${hashedPassword}`,
@ -223,33 +283,38 @@ export class HttpServer {
}
});
if ((await this.authenticator.getAllUsers()).length > 0) {
// Users defined, so enabling auth
app.use(async ({ request, response, cookies }, next) => {
const host = request.url.host;
if (!excludedPaths.includes(request.url.pathname)) {
const authCookie = await cookies.get(authCookieName(host));
if (!authCookie) {
response.redirect("/.auth");
return;
}
const [username, hashedPassword] = authCookie.split(":");
if (
!await this.authenticator.authenticateHashed(
username,
hashedPassword,
)
) {
response.redirect("/.auth");
return;
}
// Check auth
app.use(async ({ request, response, cookies }, next) => {
const spaceServer = await this.ensureSpaceServer(request);
if (!spaceServer.auth) {
// Auth disabled in this config, skip
return next();
}
const host = request.url.host;
if (!excludedPaths.includes(request.url.pathname)) {
const authCookie = await cookies.get(authCookieName(host));
if (!authCookie) {
response.redirect("/.auth");
return;
}
await next();
});
}
const spaceServer = await this.ensureSpaceServer(request);
const [username, hashedPassword] = authCookie.split(":");
const [expectedUser, expectedPassword] = spaceServer.auth!.split(
":",
);
if (
username !== expectedUser ||
hashedPassword !== await hashSHA256(expectedPassword)
) {
response.redirect("/.auth");
return;
}
}
await next();
});
}
private addFsRoutes(spacePrimitives: SpacePrimitives): Router {
private addFsRoutes(): Router {
const fsRouter = new Router();
const corsMiddleware = oakCors({
allowedHeaders: "*",
@ -264,11 +329,12 @@ export class HttpServer {
"/index.json",
// corsMiddleware,
async ({ request, response }) => {
const spaceServer = await this.ensureSpaceServer(request);
if (request.headers.has("X-Sync-Mode")) {
// Only handle direct requests for a JSON representation of the file list
response.headers.set("Content-type", "application/json");
response.headers.set("X-Space-Path", this.options.pagesPath);
const files = await spacePrimitives.fetchFileList();
response.headers.set("X-Space-Path", spaceServer.pagesPath);
const files = await spaceServer.spacePrimitives.fetchFileList();
response.body = JSON.stringify(files);
} else {
// Otherwise, redirect to the UI
@ -280,49 +346,29 @@ export class HttpServer {
// RPC
fsRouter.post("/.rpc", async ({ request, response }) => {
const spaceServer = await this.ensureSpaceServer(request);
const body = await request.body({ type: "json" }).value;
try {
switch (body.operation) {
case "shell": {
// TODO: Have a nicer way to do this
if (this.options.pagesPath.startsWith("s3://")) {
response.status = 500;
response.body = JSON.stringify({
stdout: "",
stderr: "Cannot run shell commands with S3 backend",
code: 500,
});
return;
}
const shellCommand: ShellRequest = body;
console.log(
"Running shell command:",
shellCommand.cmd,
shellCommand.args,
);
const p = new Deno.Command(shellCommand.cmd, {
args: shellCommand.args,
cwd: this.options.pagesPath,
stdout: "piped",
stderr: "piped",
});
const output = await p.output();
const stdout = new TextDecoder().decode(output.stdout);
const stderr = new TextDecoder().decode(output.stderr);
const shellResponse = await spaceServer.shellBackend.handle(
shellCommand,
);
response.headers.set("Content-Type", "application/json");
response.body = JSON.stringify({
stdout,
stderr,
code: output.code,
} as ShellResponse);
if (output.code !== 0) {
console.error("Error running shell command", stdout, stderr);
response.body = JSON.stringify(shellResponse);
if (shellResponse.code !== 0) {
console.error("Error running shell command", shellResponse);
}
return;
}
case "syscall": {
if (!this.system) {
if (this.syncOnly) {
response.headers.set("Content-Type", "text/plain");
response.status = 400;
response.body = "Unknown operation";
@ -330,7 +376,9 @@ export class HttpServer {
}
const syscallCommand: SyscallRequest = body;
try {
const plug = this.system.loadedPlugs.get(syscallCommand.ctx);
const plug = spaceServer.system!.loadedPlugs.get(
syscallCommand.ctx,
);
if (!plug) {
throw new Error(`Plug ${syscallCommand.ctx} not found`);
}
@ -372,6 +420,7 @@ export class HttpServer {
filePathRegex,
async ({ params, response, request }) => {
const name = params[0];
const spaceServer = await this.ensureSpaceServer(request);
console.log("Requested file", name);
if (!request.headers.has("X-Sync-Mode") && name.endsWith(".md")) {
// It can happen that during a sync, authentication expires, this may result in a redirect to the login page and then back to this particular file. This particular file may be an .md file, which isn't great to show so we're redirecting to the associated SB UI page.
@ -415,13 +464,15 @@ export class HttpServer {
try {
if (request.headers.has("X-Get-Meta")) {
// Getting meta via GET request
const fileData = await spacePrimitives.getFileMeta(name);
const fileData = await spaceServer.spacePrimitives.getFileMeta(
name,
);
response.status = 200;
this.fileMetaToHeaders(response.headers, fileData);
response.body = "";
return;
}
const fileData = await spacePrimitives.readFile(name);
const fileData = await spaceServer.spacePrimitives.readFile(name);
const lastModifiedHeader = new Date(fileData.meta.lastModified)
.toUTCString();
if (
@ -447,6 +498,7 @@ export class HttpServer {
filePathRegex,
async ({ request, response, params }) => {
const name = params[0];
const spaceServer = await this.ensureSpaceServer(request);
console.log("Saving file", name);
if (name.startsWith(".")) {
// Don't expose hidden files
@ -457,7 +509,7 @@ export class HttpServer {
const body = await request.body({ type: "bytes" }).value;
try {
const meta = await spacePrimitives.writeFile(
const meta = await spaceServer.spacePrimitives.writeFile(
name,
body,
);
@ -471,8 +523,9 @@ export class HttpServer {
}
},
)
.delete(filePathRegex, async ({ response, params }) => {
.delete(filePathRegex, async ({ request, response, params }) => {
const name = params[0];
const spaceServer = await this.ensureSpaceServer(request);
console.log("Deleting file", name);
if (name.startsWith(".")) {
// Don't expose hidden files
@ -480,7 +533,7 @@ export class HttpServer {
return;
}
try {
await spacePrimitives.deleteFile(name);
await spaceServer.spacePrimitives.deleteFile(name);
response.status = 200;
response.body = "OK";
} catch (e: any) {
@ -573,11 +626,3 @@ function utcDateString(mtime: number): string {
function authCookieName(host: string) {
return `auth:${host}`;
}
function headersToJson(headers: Headers) {
let headersObj: any = {};
for (const [key, value] of headers.entries()) {
headersObj[key] = value;
}
return JSON.stringify(headersObj);
}

87
server/instance.ts Normal file
View File

@ -0,0 +1,87 @@
import { SilverBulletHooks } from "../common/manifest.ts";
import { AssetBundlePlugSpacePrimitives } from "../common/spaces/asset_bundle_space_primitives.ts";
import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts";
import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
import { ensureSettingsAndIndex } from "../common/util.ts";
import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
import { KvPrimitives } from "../plugos/lib/kv_primitives.ts";
import { System } from "../plugos/system.ts";
import { BuiltinSettings } from "../web/types.ts";
import { gitIgnoreCompiler } from "./deps.ts";
import { ServerSystem } from "./server_system.ts";
import { ShellBackend } from "./shell_backend.ts";
import { determineStorageBackend } from "./storage_backend.ts";
export type SpaceServerConfig = {
hostname: string;
namespace: string;
auth?: string; // username:password
pagesPath: string;
};
export class SpaceServer {
public pagesPath: string;
auth?: string;
hostname: string;
private settings?: BuiltinSettings;
spacePrimitives: SpacePrimitives;
// Only set when syncOnly == false
private serverSystem?: ServerSystem;
system?: System<SilverBulletHooks>;
constructor(
config: SpaceServerConfig,
public shellBackend: ShellBackend,
plugAssetBundle: AssetBundle,
kvPrimitives?: KvPrimitives,
) {
this.pagesPath = config.pagesPath;
this.hostname = config.hostname;
this.auth = config.auth;
let fileFilterFn: (s: string) => boolean = () => true;
this.spacePrimitives = new FilteredSpacePrimitives(
new AssetBundlePlugSpacePrimitives(
determineStorageBackend(this.pagesPath),
plugAssetBundle,
),
(meta) => fileFilterFn(meta.name),
async () => {
await this.reloadSettings();
if (typeof this.settings?.spaceIgnore === "string") {
fileFilterFn = gitIgnoreCompiler(this.settings.spaceIgnore).accepts;
} else {
fileFilterFn = () => true;
}
},
);
// system = undefined in databaseless mode (no PlugOS instance on the server and no DB)
if (kvPrimitives) {
// Enable server-side processing
const serverSystem = new ServerSystem(
this.spacePrimitives,
kvPrimitives,
);
this.serverSystem = serverSystem;
}
}
async init() {
if (this.serverSystem) {
await this.serverSystem.init();
this.system = this.serverSystem.system;
}
await this.reloadSettings();
console.log("Booted server with hostname", this.hostname);
}
async reloadSettings() {
// TODO: Throttle this?
this.settings = await ensureSettingsAndIndex(this.spacePrimitives);
}
}

19
server/server_system.ts
View File

@ -25,7 +25,6 @@ import { shellSyscalls } from "../plugos/syscalls/shell.deno.ts";
import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
import { base64EncodedDataUrl } from "../plugos/asset_bundle/base64.ts";
import { Plug } from "../plugos/plug.ts";
import { DenoKvPrimitives } from "../plugos/lib/deno_kv_primitives.ts";
import { DataStore } from "../plugos/lib/datastore.ts";
import { dataStoreSyscalls } from "../plugos/syscalls/datastore.ts";
import { DataStoreMQ } from "../plugos/lib/mq.datastore.ts";
@ -34,6 +33,7 @@ import { handlebarsSyscalls } from "../common/syscalls/handlebars.ts";
import { codeWidgetSyscalls } from "../web/syscalls/code_widget.ts";
import { CodeWidgetHook } from "../web/hooks/code_widget.ts";
import { KVPrimitivesManifestCache } from "../plugos/manifest_cache.ts";
import { KvPrimitives } from "../plugos/lib/kv_primitives.ts";
const fileListInterval = 30 * 1000; // 30s
@ -42,27 +42,27 @@ const plugNameExtractRegex = /\/(.+)\.plug\.js$/;
export class ServerSystem {
system!: System<SilverBulletHooks>;
spacePrimitives!: SpacePrimitives;
denoKv!: Deno.Kv;
// denoKv!: Deno.Kv;
listInterval?: number;
ds!: DataStore;
constructor(
private baseSpacePrimitives: SpacePrimitives,
private dbPath: string,
private app: Application,
readonly kvPrimitives: KvPrimitives,
) {
}
// Always needs to be invoked right after construction
async init(awaitIndex = false) {
this.denoKv = await Deno.openKv(this.dbPath);
const kvPrimitives = new DenoKvPrimitives(this.denoKv);
this.ds = new DataStore(kvPrimitives);
this.ds = new DataStore(this.kvPrimitives);
this.system = new System(
"server",
{
manifestCache: new KVPrimitivesManifestCache(kvPrimitives, "manifest"),
manifestCache: new KVPrimitivesManifestCache(
this.kvPrimitives,
"manifest",
),
plugFlushTimeout: 5 * 60 * 1000, // 5 minutes
},
);
@ -75,9 +75,6 @@ export class ServerSystem {
const cronHook = new CronHook(this.system);
this.system.addHook(cronHook);
// Endpoint hook
this.system.addHook(new EndpointHook(this.app, "/_/"));
const mq = new DataStoreMQ(this.ds);
setInterval(() => {

64
server/shell_backend.ts Normal file
View File

@ -0,0 +1,64 @@
import { ShellRequest, ShellResponse } from "./rpc.ts";
/**
* Configuration via environment variables:
* - SB_SHELL_BACKEND: "local" or "off"
*/
export function determineShellBackend(path: string): ShellBackend {
const backendConfig = Deno.env.get("SB_SHELL_BACKEND") || "local";
switch (backendConfig) {
case "local":
return new LocalShell(path);
default:
console.info(
"Running in shellless mode, meaning shell commands are disabled",
);
return new NotSupportedShell();
}
}
export interface ShellBackend {
handle(shellRequest: ShellRequest): Promise<ShellResponse>;
}
class NotSupportedShell implements ShellBackend {
handle(): Promise<ShellResponse> {
return Promise.resolve({
stdout: "",
stderr: "Not supported",
code: 1,
});
}
}
class LocalShell implements ShellBackend {
constructor(private cwd: string) {
}
async handle(shellRequest: ShellRequest): Promise<ShellResponse> {
console.log(
"Running shell command:",
shellRequest.cmd,
shellRequest.args,
);
const p = new Deno.Command(shellRequest.cmd, {
cwd: this.cwd,
args: shellRequest.args,
stdout: "piped",
stderr: "piped",
});
const output = await p.output();
const stdout = new TextDecoder().decode(output.stdout);
const stderr = new TextDecoder().decode(output.stderr);
if (output.code !== 0) {
console.error("Error running shell command", stdout, stderr);
}
return {
stderr,
stdout,
code: output.code,
};
}
}

1
server/spaces/s3_space_primitives.test.ts
View File

@ -9,6 +9,7 @@ Deno.test("s3_space_primitives", async () => {
endPoint: "s3.eu-central-1.amazonaws.com",
region: "eu-central-1",
bucket: "zef-sb-space",
prefix: "test",
};
const primitives = new S3SpacePrimitives(options);

9
server/spaces/s3_space_primitives.ts
View File

@ -7,10 +7,15 @@ import { FileMeta } from "$sb/types.ts";
// TODO: IMPORTANT: This needs a different way to keep meta data (last modified and created dates)
export type S3SpacePrimitivesOptions = ClientOptions & { prefix: string };
export class S3SpacePrimitives implements SpacePrimitives {
client: S3Client;
constructor(options: ClientOptions) {
prefix: string;
constructor(options: S3SpacePrimitivesOptions) {
this.client = new S3Client(options);
// TODO: Use this
this.prefix = options.prefix;
}
private encodePath(name: string): string {
@ -18,7 +23,7 @@ export class S3SpacePrimitives implements SpacePrimitives {
}
private decodePath(encoded: string): string {
// AWS only returns ' replace dwith &apos;
// AWS only returns ' replace with &apos;
return encoded.replaceAll("&apos;", "'");
}

22
server/storage_backend.ts Normal file
View File

@ -0,0 +1,22 @@
import { DiskSpacePrimitives } from "../common/spaces/disk_space_primitives.ts";
import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
import { path } from "./deps.ts";
import { S3SpacePrimitives } from "./spaces/s3_space_primitives.ts";
export function determineStorageBackend(folder: string): SpacePrimitives {
if (folder === "s3://") {
console.info("Using S3 as a storage backend");
return new S3SpacePrimitives({
accessKey: Deno.env.get("AWS_ACCESS_KEY_ID")!,
secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!,
endPoint: Deno.env.get("AWS_ENDPOINT")!,
region: Deno.env.get("AWS_REGION")!,
bucket: Deno.env.get("AWS_BUCKET")!,
prefix: folder.slice(5),
});
} else {
folder = path.resolve(Deno.cwd(), folder);
console.info(`Using local disk as a storage backend: ${folder}`);
return new DiskSpacePrimitives(folder);
}
}

44
silverbullet.ts
View File

@ -7,10 +7,6 @@ import { upgradeCommand } from "./cmd/upgrade.ts";
import { versionCommand } from "./cmd/version.ts";
import { serveCommand } from "./cmd/server.ts";
import { plugCompileCommand } from "./cmd/plug_compile.ts";
import { userAdd } from "./cmd/user_add.ts";
import { userPasswd } from "./cmd/user_passwd.ts";
import { userDelete } from "./cmd/user_delete.ts";
import { userChgrp } from "./cmd/user_chgrp.ts";
import { plugRunCommand } from "./cmd/plug_run.ts";
await new Command()
@ -83,47 +79,7 @@ await new Command()
"Hostname or address to listen on",
)
.option("-p, --port <port:number>", "Port to listen on")
.option(
"--db <db:string>",
"Path to database file",
)
.action(plugRunCommand)
.command("user:add", "Add a new user to an authentication file")
.arguments("[username:string]")
.option(
"--auth <auth.json:string>",
"User authentication file to use",
)
.option("-G, --group <name:string>", "Add user to group", {
collect: true,
default: [] as string[],
})
.action(userAdd)
.command("user:delete", "Delete an existing user")
.arguments("[username:string]")
.option(
"--auth <auth.json:string>",
"User authentication file to use",
)
.action(userDelete)
.command("user:chgrp", "Update user groups")
.arguments("[username:string]")
.option(
"--auth <auth.json:string>",
"User authentication file to use",
)
.option("-G, --group <name:string>", "Groups to put user into", {
collect: true,
default: [] as string[],
})
.action(userChgrp)
.command("user:passwd", "Set the password for an existing user")
.arguments("[username:string]")
.option(
"--auth <auth.json:string>",
"User authentication file to use",
)
.action(userPasswd)
// upgrade
.command("upgrade", "Upgrade SilverBullet")
.action(upgradeCommand)

5
web/boot.ts
View File

@ -5,7 +5,10 @@ const syncMode = window.silverBulletConfig.syncOnly ||
!!localStorage.getItem("syncMode");
safeRun(async () => {
console.log("Booting SilverBullet...");
console.log(
"Booting SilverBullet client",
syncMode ? "in Sync Mode" : "in Online Mode",
);
const client = new Client(
document.getElementById("sb-root")!,

22
web/space.ts
View File

@ -71,6 +71,7 @@ export class Space {
}
});
eventHook.addLocalListener("file:listed", (files: FileMeta[]) => {
// console.log("Files listed", files);
this.cachedPageList = files.filter(this.isListedPage).map(
fileMetaToPageMeta,
);
@ -227,12 +228,17 @@ export class Space {
export function fileMetaToPageMeta(fileMeta: FileMeta): PageMeta {
const name = fileMeta.name.substring(0, fileMeta.name.length - 3);
return {
...fileMeta,
ref: name,
tags: ["page"],
name,
created: new Date(fileMeta.created).toISOString(),
lastModified: new Date(fileMeta.lastModified).toISOString(),
} as PageMeta;
try {
return {
...fileMeta,
ref: name,
tags: ["page"],
name,
created: new Date(fileMeta.created).toISOString(),
lastModified: new Date(fileMeta.lastModified).toISOString(),
} as PageMeta;
} catch (e) {
console.error("Failed to convert fileMeta to pageMeta", fileMeta, e);
throw e;
}
}

37
website/Authentication.md
View File

@ -1,8 +1,5 @@
SilverBullet supports simple authentication for one or many users.
SilverBullet supports simple authentication for a single user.
**Note**: This feature is experimental and will likely change significantly over time.
## Single User
By simply passing the `--user` flag with a username:password combination, you enable authentication for a single user. For instance:
```shell
@ -11,36 +8,10 @@ silverbullet --user pete:1234 .
Will let `pete` authenticate with password `1234`.
## Multiple users
Although multi-user support is still rudimentary, it is possible to have multiple users authenticate. These users can be configured using a JSON authentication file that SB can generate for you. It is usually named `.auth.json`.
You can enable authentication as follows:
Alternative, the same information can be passed in via the `SB_USER` environment variable, e.g.
```shell
silverbullet --auth /path/to/.auth.json
SB_USER=pete:1234 silverbullet .
```
To create and manage an `.auth.json` file, you can use the following commands:
* `silverbullet user:add --auth /path/to/.auth.json [username]` to add a user
* `silverbullet user:delete --auth /path/to/.auth.json [username]` to delete a user
* `silverbullet user:passwd --auth /path/to/.auth.json [username]` to update a password
If the `.auth.json` file does not yet exist, it will be created.
When SB is run with a `--auth` flag, this fill will automatically be reloaded upon change.
### Group management
While this functionality is not yet used, users can also be added to groups which can be arbitrarily named. The `admin` group will likely have a special meaning down the line.
When adding a user, you can add one more `-G` or `--group` flags:
```shell
silverbullet user:add --auth /path/to/.auth.json -G admin pete
```
And you can update these groups later with `silverbullet user:chgrp`:
```shell
silverbullet user:chgrp --auth /path/to/.auth.json -G admin pete
```
This is especially convenient when deploying using Docker

9
website/CHANGELOG.md
View File

@ -1,6 +1,15 @@
An attempt at documenting the changes/new features introduced in each
release.
---
## Next
* Removed built-in multi-user [[Authentication]], `SB_AUTH` is no longer supported, use `--user` or `SB_USER` instead, or an authentication layer such as [[Authelia]]
* Technical refactoring in preparation of multi-tenant deployment support (allowing you to run a single SB instance and serve multiple spaces and users at the same time)
* Lazy everything: plugs are now lazily loaded (after a first load, manifests are cached). On the server side, a whole lot of infrastructure is now only booted once the first HTTP request comes in
---
## 0.5.8
* Various bugfixes, primarily related to the new way of running docker containers, which broke things for some people. Be sure to have a look at the new [[Install/Local$env|environment variable]] configuration options

1
website/Install/Local.md
View File

@ -140,5 +140,4 @@ You can configure SB with environment variables instead of flags, which is proba
* `SB_HOSTNAME`: Set to the hostname to bind to (defaults to `127.0.0.0`, set to `0.0.0.0` to accept outside connections)
* `SB_PORT`: Sets the port to listen to, e.g. `SB_PORT=1234`
* `SB_FOLDER`: Sets the folder to expose, e.g. `SB_FOLDER=/space`
* `SB_AUTH`: Loads an [[Authentication]] database from a (JSON encoded) string, e.g. `SB_AUTH=$(cat /path/to/.auth.json)`
* `SB_SYNC_ONLY`: Runs the server in a "dumb" space store-only mode (not indexing content or keeping other state), e.g. `SB_SYNC_ONLY=1`. This will disable the Online [[Client Modes]] altogether (and not even show the sync icon in the top bar). Conceptually, [silverbullet.md](https://silverbullet.md) runs in this mode.