mingzailao
/
silverbullet
mirror of https://github.com/silverbulletmd/silverbullet.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Work on #508 (thin client)

pull/513/head
Zef Hemel 2023-08-26 08:31:51 +02:00
parent 3af0f180cd
commit 9ee9008bf2
30 changed files with 717 additions and 327 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

159
cli/plug_run.ts
View File

@ -1,38 +1,11 @@
import { path } from "../common/deps.ts"; import { path } from "../common/deps.ts";
import { PlugNamespaceHook } from "../common/hooks/plug_namespace.ts";
import { SilverBulletHooks } from "../common/manifest.ts";
import { loadMarkdownExtensions } from "../common/markdown_parser/markdown_ext.ts";
import buildMarkdown from "../common/markdown_parser/parser.ts";
import { DiskSpacePrimitives } from "../common/spaces/disk_space_primitives.ts"; import { DiskSpacePrimitives } from "../common/spaces/disk_space_primitives.ts";
import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitives.ts";
import { FileMetaSpacePrimitives } from "../common/spaces/file_meta_space_primitives.ts";
import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts";
import { AssetBundle } from "../plugos/asset_bundle/bundle.ts"; import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
import { createSandbox } from "../plugos/environments/deno_sandbox.ts";
import { CronHook } from "../plugos/hooks/cron.ts";
import { EventHook } from "../plugos/hooks/event.ts";
import { MQHook } from "../plugos/hooks/mq.ts";
import { DenoKVStore } from "../plugos/lib/kv_store.deno_kv.ts";
import { DexieMQ } from "../plugos/lib/mq.dexie.ts";
import assetSyscalls from "../plugos/syscalls/asset.ts";
import { eventSyscalls } from "../plugos/syscalls/event.ts";
import { sandboxFetchSyscalls } from "../plugos/syscalls/fetch.ts";
import { mqSyscalls } from "../plugos/syscalls/mq.dexie.ts";
import { shellSyscalls } from "../plugos/syscalls/shell.deno.ts";
import { storeSyscalls } from "../plugos/syscalls/store.ts";
import { System } from "../plugos/system.ts";
import { Space } from "../web/space.ts";
import { debugSyscalls } from "../web/syscalls/debug.ts";
import { pageIndexSyscalls } from "./syscalls/index.ts";
import { markdownSyscalls } from "../web/syscalls/markdown.ts";
import { systemSyscalls } from "../web/syscalls/system.ts";
import { yamlSyscalls } from "../web/syscalls/yaml.ts";
import { spaceSyscalls } from "./syscalls/space.ts";
import { IDBKeyRange, indexedDB } from "https://esm.sh/fake-indexeddb@4.0.2";
import { Application } from "../server/deps.ts"; import { Application } from "../server/deps.ts";
import { EndpointHook } from "../plugos/hooks/endpoint.ts";
import { sleep } from "../common/async_util.ts"; import { sleep } from "../common/async_util.ts";
import { ServerSystem } from "../server/server_system.ts";
import { AssetBundlePlugSpacePrimitives } from "../common/spaces/asset_bundle_space_primitives.ts";
export async function runPlug( export async function runPlug(
spacePath: string, spacePath: string,
@ -44,108 +17,44 @@ export async function runPlug(
httpHostname = "127.0.0.1", httpHostname = "127.0.0.1",
) { ) {
spacePath = path.resolve(spacePath); spacePath = path.resolve(spacePath);
const system = new System<SilverBulletHooks>("cli");
// Event hook
const eventHook = new EventHook();
system.addHook(eventHook);
// Cron hook
const cronHook = new CronHook(system);
system.addHook(cronHook);
const kvStore = new DenoKVStore();
const tempFile = Deno.makeTempFileSync({ suffix: ".db" }); const tempFile = Deno.makeTempFileSync({ suffix: ".db" });
await kvStore.init(tempFile); console.log("Tempt db file", tempFile);
// Endpoint hook
const app = new Application();
system.addHook(new EndpointHook(app, "/_"));
const serverController = new AbortController(); const serverController = new AbortController();
const app = new Application();
const serverSystem = new ServerSystem(
new AssetBundlePlugSpacePrimitives(
new DiskSpacePrimitives(spacePath),
builtinAssetBundle,
),
tempFile,
app,
);
await serverSystem.init();
app.listen({ app.listen({
hostname: httpHostname, hostname: httpHostname,
port: httpServerPort, port: httpServerPort,
signal: serverController.signal, signal: serverController.signal,
}); });
// Use DexieMQ for this, in memory
const mq = new DexieMQ("mq", indexedDB, IDBKeyRange);
const pageIndexCalls = pageIndexSyscalls(kvStore);
const plugNamespaceHook = new PlugNamespaceHook();
system.addHook(plugNamespaceHook);
system.addHook(new MQHook(system, mq));
const spacePrimitives = new FileMetaSpacePrimitives(
new EventedSpacePrimitives(
new PlugSpacePrimitives(
new DiskSpacePrimitives(spacePath),
plugNamespaceHook,
),
eventHook,
),
pageIndexCalls,
);
const space = new Space(spacePrimitives, kvStore);
// Add syscalls
system.registerSyscalls(
[],
eventSyscalls(eventHook),
spaceSyscalls(space),
assetSyscalls(system),
yamlSyscalls(),
storeSyscalls(kvStore),
systemSyscalls(undefined as any, system),
mqSyscalls(mq),
pageIndexCalls,
debugSyscalls(),
markdownSyscalls(buildMarkdown([])), // Will later be replaced with markdown extensions
);
// Syscalls that require some additional permissions
system.registerSyscalls(
["fetch"],
sandboxFetchSyscalls(),
);
system.registerSyscalls(
["shell"],
shellSyscalls("."),
);
await loadPlugsFromAssetBundle(system, builtinAssetBundle);
for (let plugPath of await space.listPlugs()) {
plugPath = path.resolve(spacePath, plugPath);
await system.load(
new URL(`file://${plugPath}`),
createSandbox,
);
}
// Load markdown syscalls based on all new syntax (if any)
system.registerSyscalls(
[],
markdownSyscalls(buildMarkdown(loadMarkdownExtensions(system))),
);
if (indexFirst) { if (indexFirst) {
await system.loadedPlugs.get("core")!.invoke("reindexSpace", []); await serverSystem.system.loadedPlugs.get("core")!.invoke(
"reindexSpace",
[],
);
} }
if (functionName) { if (functionName) {
const [plugName, funcName] = functionName.split("."); const [plugName, funcName] = functionName.split(".");
const plug = system.loadedPlugs.get(plugName); const plug = serverSystem.system.loadedPlugs.get(plugName);
if (!plug) { if (!plug) {
throw new Error(`Plug ${plugName} not found`); throw new Error(`Plug ${plugName} not found`);
} }
const result = await plug.invoke(funcName, args); const result = await plug.invoke(funcName, args);
await system.unloadAll(); await serverSystem.close();
await kvStore.delete(); await serverSystem.kvStore?.delete();
// await Deno.remove(tempFile);
serverController.abort(); serverController.abort();
return result; return result;
} else { } else {
@ -155,27 +64,3 @@ export async function runPlug(
} }
} }
} }
async function loadPlugsFromAssetBundle(
system: System<any>,
assetBundle: AssetBundle,
) {
const tempDir = await Deno.makeTempDir();
try {
for (const filePath of assetBundle.listFiles()) {
if (
filePath.endsWith(".plug.js") // && !filePath.includes("search.plug.js")
) {
const plugPath = path.join(tempDir, filePath);
await Deno.mkdir(path.dirname(plugPath), { recursive: true });
await Deno.writeFile(plugPath, assetBundle.readFileSync(filePath));
await system.load(
new URL(`file://${plugPath}`),
createSandbox,
);
}
}
} finally {
await Deno.remove(tempDir, { recursive: true });
}
}

1
cli/syscalls/index.test.ts
View File

@ -27,7 +27,6 @@ Deno.test("Test KV index", async () => {
}, { key: "random", value: "value3" }]); }, { key: "random", value: "value3" }]);
let results = await calls["index.queryPrefix"](ctx, "attr:"); let results = await calls["index.queryPrefix"](ctx, "attr:");
assertEquals(results.length, 4); assertEquals(results.length, 4);
console.log("here");
await calls["index.clearPageIndexForPage"](ctx, "page"); await calls["index.clearPageIndexForPage"](ctx, "page");
results = await calls["index.queryPrefix"](ctx, "attr:"); results = await calls["index.queryPrefix"](ctx, "attr:");
assertEquals(results.length, 2); assertEquals(results.length, 2);

28
cli/syscalls/index.ts
View File

@ -30,10 +30,18 @@ export function pageIndexSyscalls(kv: KVStore): SysCallMapping {
}], }],
); );
}, },
"index.batchSet": async (_ctx, page: string, kvs: KV[]) => { "index.batchSet": (_ctx, page: string, kvs: KV[]) => {
const batch: KV[] = [];
for (const { key, value } of kvs) { for (const { key, value } of kvs) {
await apiObj["index.set"](_ctx, page, key, value); batch.push({
key: `index${sep}${page}${sep}${key}`,
value,
}, {
key: `indexByKey${sep}${key}${sep}${page}`,
value,
});
} }
return kv.batchSet(batch);
}, },
"index.delete": (_ctx, page: string, key: string) => { "index.delete": (_ctx, page: string, key: string) => {
return kv.batchDelete([ return kv.batchDelete([
@ -62,20 +70,30 @@ export function pageIndexSyscalls(kv: KVStore): SysCallMapping {
await apiObj["index.deletePrefixForPage"](ctx, page, ""); await apiObj["index.deletePrefixForPage"](ctx, page, "");
}, },
"index.deletePrefixForPage": async (_ctx, page: string, prefix: string) => { "index.deletePrefixForPage": async (_ctx, page: string, prefix: string) => {
const allKeys: string[] = [];
for ( for (
const result of await kv.queryPrefix( const result of await kv.queryPrefix(
`index${sep}${page}${sep}${prefix}`, `index${sep}${page}${sep}${prefix}`,
) )
) { ) {
const [_ns, page, key] = result.key.split(sep); const [_ns, page, key] = result.key.split(sep);
await apiObj["index.delete"](_ctx, page, key); allKeys.push(
`index${sep}${page}${sep}${key}`,
`indexByKey${sep}${key}${sep}${page}`,
);
} }
return kv.batchDelete(allKeys);
}, },
"index.clearPageIndex": async (ctx) => { "index.clearPageIndex": async () => {
const allKeys: string[] = [];
for (const result of await kv.queryPrefix(`index${sep}`)) { for (const result of await kv.queryPrefix(`index${sep}`)) {
const [_ns, page, key] = result.key.split(sep); const [_ns, page, key] = result.key.split(sep);
await apiObj["index.delete"](ctx, page, key); allKeys.push(
`index${sep}${page}${sep}${key}`,
`indexByKey${sep}${key}${sep}${page}`,
);
} }
return kv.batchDelete(allKeys);
}, },
}; };
return apiObj; return apiObj;

51
cmd/server.ts
View File

@ -1,4 +1,4 @@
import { path } from "../server/deps.ts"; import { Application, path } from "../server/deps.ts";
import { HttpServer } from "../server/http_server.ts"; import { HttpServer } from "../server/http_server.ts";
import clientAssetBundle from "../dist/client_asset_bundle.json" assert { import clientAssetBundle from "../dist/client_asset_bundle.json" assert {
type: "json", type: "json",
@ -14,16 +14,34 @@ import { S3SpacePrimitives } from "../server/spaces/s3_space_primitives.ts";
import { Authenticator } from "../server/auth.ts"; import { Authenticator } from "../server/auth.ts";
import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts"; import { JSONKVStore } from "../plugos/lib/kv_store.json_file.ts";
import { sleep } from "../common/async_util.ts"; import { sleep } from "../common/async_util.ts";
import { ServerSystem } from "../server/server_system.ts";
import { SilverBulletHooks } from "../common/manifest.ts";
import { System } from "../plugos/system.ts";
export async function serveCommand( export async function serveCommand(
options: any, options: {
hostname?: string;
port?: number;
user?: string;
auth?: string;
cert?: string;
key?: string;
// Thin client mode
thinClient?: boolean;
reindex?: boolean;
db?: string;
},
folder?: string, folder?: string,
) { ) {
const hostname = options.hostname || Deno.env.get("SB_HOSTNAME") || const hostname = options.hostname || Deno.env.get("SB_HOSTNAME") ||
"127.0.0.1"; "127.0.0.1";
const port = options.port || const port = options.port ||
(Deno.env.get("SB_PORT") && +Deno.env.get("SB_PORT")!) || 3000; (Deno.env.get("SB_PORT") && +Deno.env.get("SB_PORT")!) || 3000;
const maxFileSizeMB = options.maxFileSizeMB || 20;
const thinClientMode = options.thinClient || Deno.env.has("SB_THIN_CLIENT");
let dbFile = options.db || Deno.env.get("SB_DB_FILE") || ".silverbullet.db";
const app = new Application();
if (!folder) { if (!folder) {
folder = Deno.env.get("SB_FOLDER"); folder = Deno.env.get("SB_FOLDER");
@ -55,16 +73,35 @@ To allow outside connections, pass -L 0.0.0.0 as a flag, and put a TLS terminato
bucket: Deno.env.get("AWS_BUCKET")!, bucket: Deno.env.get("AWS_BUCKET")!,
}); });
console.log("Running in S3 mode"); console.log("Running in S3 mode");
folder = Deno.cwd();
} else { } else {
// Regular disk mode // Regular disk mode
folder = path.resolve(Deno.cwd(), folder); folder = path.resolve(Deno.cwd(), folder);
spacePrimitives = new DiskSpacePrimitives(folder); spacePrimitives = new DiskSpacePrimitives(folder);
} }
spacePrimitives = new AssetBundlePlugSpacePrimitives( spacePrimitives = new AssetBundlePlugSpacePrimitives(
spacePrimitives, spacePrimitives,
new AssetBundle(plugAssetBundle as AssetJson), new AssetBundle(plugAssetBundle as AssetJson),
); );
let system: System<SilverBulletHooks> | undefined;
if (thinClientMode) {
dbFile = path.resolve(folder, dbFile);
console.log(`Running in thin client mode, keeping state in ${dbFile}`);
const serverSystem = new ServerSystem(spacePrimitives, dbFile, app);
await serverSystem.init();
spacePrimitives = serverSystem.spacePrimitives;
system = serverSystem.system;
if (options.reindex) {
console.log("Reindexing space (requested via --reindex flag)");
await serverSystem.system.loadedPlugs.get("core")!.invoke(
"reindexSpace",
[],
);
}
}
const authStore = new JSONKVStore(); const authStore = new JSONKVStore();
const authenticator = new Authenticator(authStore); const authenticator = new Authenticator(authStore);
@ -82,7 +119,7 @@ To allow outside connections, pass -L 0.0.0.0 as a flag, and put a TLS terminato
await authStore.load(authFile); await authStore.load(authFile);
(async () => { (async () => {
// Asynchronously kick off file watcher // Asynchronously kick off file watcher
for await (const _event of Deno.watchFs(options.auth)) { for await (const _event of Deno.watchFs(options.auth!)) {
console.log("Authentication file changed, reloading..."); console.log("Authentication file changed, reloading...");
await authStore.load(authFile); await authStore.load(authFile);
} }
@ -95,7 +132,7 @@ To allow outside connections, pass -L 0.0.0.0 as a flag, and put a TLS terminato
authStore.loadString(envAuth); authStore.loadString(envAuth);
} }
const httpServer = new HttpServer(spacePrimitives!, { const httpServer = new HttpServer(spacePrimitives!, app, system, {
hostname, hostname,
port: port, port: port,
pagesPath: folder!, pagesPath: folder!,
@ -103,11 +140,11 @@ To allow outside connections, pass -L 0.0.0.0 as a flag, and put a TLS terminato
authenticator, authenticator,
keyFile: options.key, keyFile: options.key,
certFile: options.cert, certFile: options.cert,
maxFileSizeMB: +maxFileSizeMB,
}); });
await httpServer.start(); await httpServer.start();
// Wait in an infinite loop (to keep the HTTP server running, only cancelable via Ctrl+C or other signal) // Wait in an infinite loop (to keep the HTTP server running, only cancelable via Ctrl+C or other signal)
while (true) { while (true) {
await sleep(1000); await sleep(10000);
} }
} }

14
deno.jsonc
View File

@ -7,10 +7,10 @@
"test": "deno test -A --unstable", "test": "deno test -A --unstable",
"build": "deno run -A build_plugs.ts && deno run -A --unstable build_web.ts", "build": "deno run -A build_plugs.ts && deno run -A --unstable build_web.ts",
"plugs": "deno run -A build_plugs.ts", "plugs": "deno run -A build_plugs.ts",
"server": "deno run -A --check silverbullet.ts", "server": "deno run -A --unstable --check silverbullet.ts",
"watch-web": "deno run -A --check build_web.ts --watch", "watch-web": "deno run -A --check build_web.ts --watch",
"watch-server": "deno run -A --check --watch silverbullet.ts", "watch-server": "deno run -A --unstable --check --watch silverbullet.ts",
"watch-plugs": "deno run -A --check build_plugs.ts -w", "watch-plugs": "deno run -A --check build_plugs.ts -w",
"bundle": "deno run -A build_bundle.ts", "bundle": "deno run -A build_bundle.ts",
@ -19,11 +19,11 @@
"generate": "lezer-generator common/markdown_parser/query.grammar -o common/markdown_parser/parse-query.js", "generate": "lezer-generator common/markdown_parser/query.grammar -o common/markdown_parser/parse-query.js",
// Compile // Compile
"compile": "deno task bundle && deno compile -A -o silverbullet dist/silverbullet.js", "compile": "deno task bundle && deno compile -A --unstable -o silverbullet dist/silverbullet.js",
"server:dist:linux-x86_64": "deno task bundle && deno compile -A --target x86_64-unknown-linux-gnu dist/silverbullet.js -o silverbullet && zip silverbullet-server-linux-x86_64.zip silverbullet", "server:dist:linux-x86_64": "deno task bundle && deno compile -A --unstable --target x86_64-unknown-linux-gnu dist/silverbullet.js -o silverbullet && zip silverbullet-server-linux-x86_64.zip silverbullet",
"server:dist:darwin-x86_64": "deno task bundle && deno compile -A --target x86_64-apple-darwin dist/silverbullet.js -o silverbullet && zip silverbullet-server-darwin-x86_64.zip silverbullet", "server:dist:darwin-x86_64": "deno task bundle && deno compile -A --unstable --target x86_64-apple-darwin dist/silverbullet.js -o silverbullet && zip silverbullet-server-darwin-x86_64.zip silverbullet",
"server:dist:darwin-aarch64": "deno task bundle && deno task bundle && deno compile -A --target aarch64-apple-darwin dist/silverbullet.js -o silverbullet && zip silverbullet-server-darwin-aarch64.zip silverbullet", "server:dist:darwin-aarch64": "deno task bundle && deno task bundle && deno compile -A --unstable --target aarch64-apple-darwin dist/silverbullet.js -o silverbullet && zip silverbullet-server-darwin-aarch64.zip silverbullet",
"server:dist:windows-x86_64": "deno compile -A --target x86_64-pc-windows-msvc dist/silverbullet.js -o silverbullet.exe && zip silverbullet-server-windows-x86_64.zip silverbullet.exe" "server:dist:windows-x86_64": "deno task bundle && deno compile -A --unstable --target x86_64-pc-windows-msvc dist/silverbullet.js -o silverbullet.exe && zip silverbullet-server-windows-x86_64.zip silverbullet.exe"
}, },
"compilerOptions": { "compilerOptions": {

65
plug-api/plugos-syscall/fetch.ts
View File

@ -1,4 +1,3 @@
import { init } from "https://esm.sh/v131/node_events.js";
import type { import type {
ProxyFetchRequest, ProxyFetchRequest,
ProxyFetchResponse, ProxyFetchResponse,
@ -8,21 +7,44 @@ import {
base64Encode, base64Encode,
} from "../../plugos/asset_bundle/base64.ts"; } from "../../plugos/asset_bundle/base64.ts";
async function readStream(
stream: ReadableStream<Uint8Array>,
): Promise<Uint8Array> {
const arrays: Uint8Array[] = [];
let totalRead = 0;
const reader = stream.getReader();
while (true) {
// The `read()` method returns a promise that
// resolves when a value has been received.
const { done, value } = await reader.read();
// Result objects contain two properties:
// `done` - `true` if the stream has already given you all its data.
// `value` - Some data. Always `undefined` when `done` is `true`.
if (done) {
const resultArray = new Uint8Array(totalRead);
let offset = 0;
for (const array of arrays) {
resultArray.set(array, offset);
offset += array.length;
}
return resultArray;
}
arrays.push(value);
totalRead += value.length;
}
}
export async function sandboxFetch( export async function sandboxFetch(
reqInfo: RequestInfo, reqInfo: RequestInfo,
options?: ProxyFetchRequest, options?: ProxyFetchRequest,
): Promise<ProxyFetchResponse> { ): Promise<ProxyFetchResponse> {
if (typeof reqInfo !== "string") { if (typeof reqInfo !== "string") {
// Request as first argument, let's deconstruct it const body = new Uint8Array(await reqInfo.arrayBuffer());
// console.log("fetch", reqInfo); const encodedBody = body.length > 0 ? base64Encode(body) : undefined;
options = { options = {
method: reqInfo.method, method: reqInfo.method,
headers: Object.fromEntries(reqInfo.headers.entries()), headers: Object.fromEntries(reqInfo.headers.entries()),
base64Body: reqInfo.body base64Body: encodedBody,
? base64Encode(
new Uint8Array(await (new Response(reqInfo.body)).arrayBuffer()),
)
: undefined,
}; };
reqInfo = reqInfo.url; reqInfo = reqInfo.url;
} }
@ -30,6 +52,21 @@ export async function sandboxFetch(
return syscall("sandboxFetch.fetch", reqInfo, options); return syscall("sandboxFetch.fetch", reqInfo, options);
} }
async function bodyInitToUint8Array(init: BodyInit): Promise<Uint8Array> {
if (init instanceof Blob) {
const buffer = await init.arrayBuffer();
return new Uint8Array(buffer);
} else if (init instanceof ArrayBuffer) {
return new Uint8Array(init);
} else if (init instanceof ReadableStream) {
return readStream(init);
} else if (typeof init === "string") {
return new TextEncoder().encode(init);
} else {
throw new Error("Unknown body init type");
}
}
export function monkeyPatchFetch() { export function monkeyPatchFetch() {
// @ts-ignore: monkey patching fetch // @ts-ignore: monkey patching fetch
globalThis.nativeFetch = globalThis.fetch; globalThis.nativeFetch = globalThis.fetch;
@ -38,16 +75,18 @@ export function monkeyPatchFetch() {
reqInfo: RequestInfo, reqInfo: RequestInfo,
init?: RequestInit, init?: RequestInit,
): Promise<Response> { ): Promise<Response> {
const encodedBody = init && init.body
? base64Encode(
new Uint8Array(await (new Response(init.body)).arrayBuffer()),
)
: undefined;
// console.log("Encoded this body", encodedBody);
const r = await sandboxFetch( const r = await sandboxFetch(
reqInfo, reqInfo,
init && { init && {
method: init.method, method: init.method,
headers: init.headers as Record<string, string>, headers: init.headers as Record<string, string>,
base64Body: init.body base64Body: encodedBody,
? base64Encode(
new Uint8Array(await (new Response(init.body)).arrayBuffer()),
)
: undefined,
}, },
); );
return new Response(r.base64Body ? base64Decode(r.base64Body) : null, { return new Response(r.base64Body ? base64Decode(r.base64Body) : null, {

13
plug-api/silverbullet-syscall/clientStore.ts Normal file
View File

@ -0,0 +1,13 @@
import { syscall } from "./syscall.ts";
export function set(key: string, value: any): Promise<void> {
return syscall("clientStore.set", key, value);
}
export function get(key: string): Promise<any> {
return syscall("clientStore.get", key);
}
export function del(key: string): Promise<void> {
return syscall("clientStore.delete", key);
}

3
plug-api/silverbullet-syscall/mod.ts
View File

@ -3,7 +3,6 @@ export * as index from "./index.ts";
export * as markdown from "./markdown.ts"; export * as markdown from "./markdown.ts";
export * as space from "./space.ts"; export * as space from "./space.ts";
export * as system from "./system.ts"; export * as system from "./system.ts";
// Legacy redirect, use "store" in $sb/plugos-syscall/mod.ts instead export * as clientStore from "./clientStore.ts";
export * as clientStore from "./store.ts";
export * as sync from "./sync.ts"; export * as sync from "./sync.ts";
export * as debug from "./debug.ts"; export * as debug from "./debug.ts";

6
plugos/compile.ts
View File

@ -4,9 +4,9 @@ import { bundleAssets } from "./asset_bundle/builder.ts";
import { Manifest } from "./types.ts"; import { Manifest } from "./types.ts";
import { version } from "../version.ts"; import { version } from "../version.ts";
// const workerRuntimeUrl = new URL("./worker_runtime.ts", import.meta.url); const workerRuntimeUrl = new URL("./worker_runtime.ts", import.meta.url);
const workerRuntimeUrl = // const workerRuntimeUrl =
`https://deno.land/x/silverbullet@${version}/plugos/worker_runtime.ts`; // `https://deno.land/x/silverbullet@${version}/plugos/worker_runtime.ts`;
export type CompileOptions = { export type CompileOptions = {
debug?: boolean; debug?: boolean;

27
plugos/lib/kv_store.deno_kv.test.ts
View File

@ -15,6 +15,25 @@ Deno.test("Test KV index", async () => {
{ key: "page:hello2", value: "Hello 2" }, { key: "page:hello2", value: "Hello 2" },
{ key: "page:hello3", value: "Hello 3" }, { key: "page:hello3", value: "Hello 3" },
{ key: "something", value: "Something" }, { key: "something", value: "Something" },
{ key: "something1", value: "Something" },
{ key: "something2", value: "Something" },
{ key: "something3", value: "Something" },
{ key: "something4", value: "Something" },
{ key: "something5", value: "Something" },
{ key: "something6", value: "Something" },
{ key: "something7", value: "Something" },
{ key: "something8", value: "Something" },
{ key: "something9", value: "Something" },
{ key: "something10", value: "Something" },
{ key: "something11", value: "Something" },
{ key: "something12", value: "Something" },
{ key: "something13", value: "Something" },
{ key: "something14", value: "Something" },
{ key: "something15", value: "Something" },
{ key: "something16", value: "Something" },
{ key: "something17", value: "Something" },
{ key: "something18", value: "Something" },
{ key: "something19", value: "Something" },
]); ]);
const results = await kv.queryPrefix("page:"); const results = await kv.queryPrefix("page:");
@ -25,5 +44,13 @@ Deno.test("Test KV index", async () => {
"Hello 3", "Hello 3",
]); ]);
await kv.deletePrefix("page:");
assertEquals(await kv.queryPrefix("page:"), []);
assertEquals((await kv.queryPrefix("")).length, 20);
await kv.deletePrefix("");
assertEquals(await kv.queryPrefix(""), []);
await kv.delete(); await kv.delete();
}); });

102
plugos/lib/kv_store.deno_kv.ts
View File

@ -2,6 +2,8 @@
import { KV, KVStore } from "./kv_store.ts"; import { KV, KVStore } from "./kv_store.ts";
const kvBatchSize = 10;
export class DenoKVStore implements KVStore { export class DenoKVStore implements KVStore {
kv!: Deno.Kv; kv!: Deno.Kv;
path: string | undefined; path: string | undefined;
@ -22,55 +24,75 @@ export class DenoKVStore implements KVStore {
} }
} }
async del(key: string): Promise<void> { del(key: string): Promise<void> {
const res = await this.kv.atomic() return this.batchDelete([key]);
.delete([key])
.commit();
if (!res.ok) {
throw res;
}
} }
async deletePrefix(prefix: string): Promise<void> { async deletePrefix(prefix: string): Promise<void> {
const allKeys: string[] = [];
for await ( for await (
const result of this.kv.list({ const result of this.kv.list(
start: [prefix], prefix
end: [endRange(prefix)], ? {
}) start: [prefix],
end: [endRange(prefix)],
}
: { prefix: [] },
)
) { ) {
await this.del(result.key[0] as string); allKeys.push(result.key[0] as string);
} }
return this.batchDelete(allKeys);
} }
async deleteAll(): Promise<void> { deleteAll(): Promise<void> {
for await ( return this.deletePrefix("");
const result of this.kv.list({ prefix: [] })
) {
await this.del(result.key[0] as string);
}
} }
async set(key: string, value: any): Promise<void> { set(key: string, value: any): Promise<void> {
const res = await this.kv.atomic() return this.batchSet([{ key, value }]);
.set([key], value)
.commit();
if (!res.ok) {
throw res;
}
} }
async batchSet(kvs: KV[]): Promise<void> { async batchSet(kvs: KV[]): Promise<void> {
for (const { key, value } of kvs) { // Split into batches of kvBatchSize
await this.set(key, value); const batches: KV[][] = [];
for (let i = 0; i < kvs.length; i += kvBatchSize) {
batches.push(kvs.slice(i, i + kvBatchSize));
}
for (const batch of batches) {
let batchOp = this.kv.atomic();
for (const { key, value } of batch) {
batchOp = batchOp.set([key], value);
}
const res = await batchOp.commit();
if (!res.ok) {
throw res;
}
} }
} }
async batchDelete(keys: string[]): Promise<void> { async batchDelete(keys: string[]): Promise<void> {
for (const key of keys) { const batches: string[][] = [];
await this.del(key); for (let i = 0; i < keys.length; i += kvBatchSize) {
batches.push(keys.slice(i, i + kvBatchSize));
}
for (const batch of batches) {
let batchOp = this.kv.atomic();
for (const key of batch) {
batchOp = batchOp.delete([key]);
}
const res = await batchOp.commit();
if (!res.ok) {
throw res;
}
} }
} }
batchGet(keys: string[]): Promise<any[]> { async batchGet(keys: string[]): Promise<any[]> {
const results: Promise<any>[] = []; const results: any[] = [];
for (const key of keys) { const batches: Deno.KvKey[][] = [];
results.push(this.get(key)); for (let i = 0; i < keys.length; i += kvBatchSize) {
batches.push(keys.slice(i, i + kvBatchSize).map((k) => [k]));
} }
return Promise.all(results); for (const batch of batches) {
const res = await this.kv.getMany(batch);
results.push(...res.map((r) => r.value));
}
return results;
} }
async get(key: string): Promise<any> { async get(key: string): Promise<any> {
return (await this.kv.get([key])).value; return (await this.kv.get([key])).value;
@ -81,10 +103,14 @@ export class DenoKVStore implements KVStore {
async queryPrefix(keyPrefix: string): Promise<{ key: string; value: any }[]> { async queryPrefix(keyPrefix: string): Promise<{ key: string; value: any }[]> {
const results: { key: string; value: any }[] = []; const results: { key: string; value: any }[] = [];
for await ( for await (
const result of (this.kv).list({ const result of this.kv.list(
start: [keyPrefix], keyPrefix
end: [endRange(keyPrefix)], ? {
}) start: [keyPrefix],
end: [endRange(keyPrefix)],
}
: { prefix: [] },
)
) { ) {
results.push({ results.push({
key: result.key[0] as string, key: result.key[0] as string,

11
plugs/core/editor.ts
View File

@ -1,21 +1,20 @@
import { editor } from "$sb/silverbullet-syscall/mod.ts"; import { clientStore, editor } from "$sb/silverbullet-syscall/mod.ts";
import { store } from "$sb/plugos-syscall/mod.ts";
// Run on "editor:init" // Run on "editor:init"
export async function setEditorMode() { export async function setEditorMode() {
if (await store.get("vimMode")) { if (await clientStore.get("vimMode")) {
await editor.setUiOption("vimMode", true); await editor.setUiOption("vimMode", true);
} }
if (await store.get("darkMode")) { if (await clientStore.get("darkMode")) {
await editor.setUiOption("darkMode", true); await editor.setUiOption("darkMode", true);
} }
} }
export async function toggleDarkMode() { export async function toggleDarkMode() {
let darkMode = await store.get("darkMode"); let darkMode = await clientStore.get("darkMode");
darkMode = !darkMode; darkMode = !darkMode;
await editor.setUiOption("darkMode", darkMode); await editor.setUiOption("darkMode", darkMode);
await store.set("darkMode", darkMode); await clientStore.set("darkMode", darkMode);
} }
export async function foldCommand() { export async function foldCommand() {

3
plugs/core/page.ts
View File

@ -152,12 +152,13 @@ export async function reindexSpace() {
} }
export async function processIndexQueue(messages: Message[]) { export async function processIndexQueue(messages: Message[]) {
// console.log("Processing batch of", messages.length, "pages to index");
for (const message of messages) { for (const message of messages) {
const name: string = message.body; const name: string = message.body;
console.log(`Indexing page ${name}`); console.log(`Indexing page ${name}`);
const text = await space.readPage(name); const text = await space.readPage(name);
// console.log("Going to parse markdown");
const parsed = await markdown.parseMarkdown(text); const parsed = await markdown.parseMarkdown(text);
// console.log("Dispatching ;age:index");
await events.dispatchEvent("page:index", { await events.dispatchEvent("page:index", {
name, name,
tree: parsed, tree: parsed,

3
plugs/markdown/markdown.plug.yaml
View File

@ -1,8 +1,6 @@
name: markdown name: markdown
assets: assets:
- "assets/*" - "assets/*"
requiredPermissions:
- fs
functions: functions:
toggle: toggle:
path: "./markdown.ts:togglePreview" path: "./markdown.ts:togglePreview"
@ -13,6 +11,7 @@ functions:
preview: preview:
path: "./preview.ts:updateMarkdownPreview" path: "./preview.ts:updateMarkdownPreview"
env: client
events: events:
- plug:load - plug:load
- editor:updated - editor:updated

6
plugs/markdown/markdown.ts
View File

@ -1,11 +1,11 @@
import { editor } from "$sb/silverbullet-syscall/mod.ts"; import { editor } from "$sb/silverbullet-syscall/mod.ts";
import { readSettings } from "$sb/lib/settings_page.ts"; import { readSettings } from "$sb/lib/settings_page.ts";
import { updateMarkdownPreview } from "./preview.ts"; import { updateMarkdownPreview } from "./preview.ts";
import { store } from "$sb/plugos-syscall/mod.ts"; import { clientStore } from "$sb/silverbullet-syscall/mod.ts";
export async function togglePreview() { export async function togglePreview() {
const currentValue = !!(await store.get("enableMarkdownPreview")); const currentValue = !!(await clientStore.get("enableMarkdownPreview"));
await store.set("enableMarkdownPreview", !currentValue); await clientStore.set("enableMarkdownPreview", !currentValue);
if (!currentValue) { if (!currentValue) {
await updateMarkdownPreview(); await updateMarkdownPreview();
} else { } else {

6
plugs/markdown/preview.ts
View File

@ -1,11 +1,11 @@
import { editor, system } from "$sb/silverbullet-syscall/mod.ts"; import { clientStore, editor, system } from "$sb/silverbullet-syscall/mod.ts";
import { asset, store } from "$sb/plugos-syscall/mod.ts"; import { asset } from "$sb/plugos-syscall/mod.ts";
import { parseMarkdown } from "$sb/silverbullet-syscall/markdown.ts"; import { parseMarkdown } from "$sb/silverbullet-syscall/markdown.ts";
import { renderMarkdownToHtml } from "./markdown_render.ts"; import { renderMarkdownToHtml } from "./markdown_render.ts";
import { resolvePath } from "$sb/lib/resolve.ts"; import { resolvePath } from "$sb/lib/resolve.ts";
export async function updateMarkdownPreview() { export async function updateMarkdownPreview() {
if (!(await store.get("enableMarkdownPreview"))) { if (!(await clientStore.get("enableMarkdownPreview"))) {
return; return;
} }
const currentPage = await editor.getCurrentPage(); const currentPage = await editor.getCurrentPage();

8
plugs/search/search.plug.yaml
View File

@ -1,9 +1,11 @@
name: search name: search
functions: functions:
indexPage: indexPage:
path: search.ts:indexPage path: search.ts:indexPage
events: # Only enable in client for now
- page:index env: client
events:
- page:index
clearIndex: clearIndex:
path: search.ts:clearIndex path: search.ts:clearIndex

82
server/http_server.ts
View File

@ -2,12 +2,19 @@ import { Application, Context, Next, oakCors, Router } from "./deps.ts";
import { SpacePrimitives } from "../common/spaces/space_primitives.ts"; import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
import { AssetBundle } from "../plugos/asset_bundle/bundle.ts"; import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
import { ensureSettingsAndIndex } from "../common/util.ts"; import { ensureSettingsAndIndex } from "../common/util.ts";
import { performLocalFetch } from "../common/proxy_fetch.ts";
import { BuiltinSettings } from "../web/types.ts"; import { BuiltinSettings } from "../web/types.ts";
import { gitIgnoreCompiler } from "./deps.ts"; import { gitIgnoreCompiler } from "./deps.ts";
import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts"; import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts";
import { Authenticator } from "./auth.ts"; import { Authenticator } from "./auth.ts";
import { FileMeta } from "$sb/types.ts"; import { FileMeta } from "$sb/types.ts";
import {
ShellRequest,
ShellResponse,
SyscallRequest,
SyscallResponse,
} from "./rpc.ts";
import { SilverBulletHooks } from "../common/manifest.ts";
import { System } from "../plugos/system.ts";
export type ServerOptions = { export type ServerOptions = {
hostname: string; hostname: string;
@ -18,11 +25,9 @@ export type ServerOptions = {
pass?: string; pass?: string;
certFile?: string; certFile?: string;
keyFile?: string; keyFile?: string;
maxFileSizeMB?: number;
}; };
export class HttpServer { export class HttpServer {
app: Application;
private hostname: string; private hostname: string;
private port: number; private port: number;
abortController?: AbortController; abortController?: AbortController;
@ -33,27 +38,19 @@ export class HttpServer {
constructor( constructor(
spacePrimitives: SpacePrimitives, spacePrimitives: SpacePrimitives,
private app: Application,
private system: System<SilverBulletHooks> | undefined,
private options: ServerOptions, private options: ServerOptions,
) { ) {
this.hostname = options.hostname; this.hostname = options.hostname;
this.port = options.port; this.port = options.port;
this.app = new Application();
this.authenticator = options.authenticator; this.authenticator = options.authenticator;
this.clientAssetBundle = options.clientAssetBundle; this.clientAssetBundle = options.clientAssetBundle;
let fileFilterFn: (s: string) => boolean = () => true; let fileFilterFn: (s: string) => boolean = () => true;
this.spacePrimitives = new FilteredSpacePrimitives( this.spacePrimitives = new FilteredSpacePrimitives(
spacePrimitives, spacePrimitives,
(meta) => { (meta) => fileFilterFn(meta.name),
// Don't list file exceeding the maximum file size
if (
options.maxFileSizeMB &&
meta.size / (1024 * 1024) > options.maxFileSizeMB
) {
return false;
}
return fileFilterFn(meta.name);
},
async () => { async () => {
await this.reloadSettings(); await this.reloadSettings();
if (typeof this.settings?.spaceIgnore === "string") { if (typeof this.settings?.spaceIgnore === "string") {
@ -71,7 +68,7 @@ export class HttpServer {
.replaceAll( .replaceAll(
"{{SPACE_PATH}}", "{{SPACE_PATH}}",
this.options.pagesPath.replaceAll("\\", "\\\\"), this.options.pagesPath.replaceAll("\\", "\\\\"),
); ).replaceAll("{{THIN_CLIENT_MODE}}", this.system ? "on" : "off");
} }
async start() { async start() {
@ -282,13 +279,6 @@ export class HttpServer {
const body = await request.body({ type: "json" }).value; const body = await request.body({ type: "json" }).value;
try { try {
switch (body.operation) { switch (body.operation) {
// case "fetch": {
// const result = await performLocalFetch(body.url, body.options);
// console.log("Proxying fetch request to", body.url);
// response.headers.set("Content-Type", "application/json");
// response.body = JSON.stringify(result);
// return;
// }
case "shell": { case "shell": {
// TODO: Have a nicer way to do this // TODO: Have a nicer way to do this
if (this.options.pagesPath.startsWith("s3://")) { if (this.options.pagesPath.startsWith("s3://")) {
@ -300,9 +290,14 @@ export class HttpServer {
}); });
return; return;
} }
console.log("Running shell command:", body.cmd, body.args); const shellCommand: ShellRequest = body;
const p = new Deno.Command(body.cmd, { console.log(
args: body.args, "Running shell command:",
shellCommand.cmd,
shellCommand.args,
);
const p = new Deno.Command(shellCommand.cmd, {
args: shellCommand.args,
cwd: this.options.pagesPath, cwd: this.options.pagesPath,
stdout: "piped", stdout: "piped",
stderr: "piped", stderr: "piped",
@ -316,12 +311,40 @@ export class HttpServer {
stdout, stdout,
stderr, stderr,
code: output.code, code: output.code,
}); } as ShellResponse);
if (output.code !== 0) { if (output.code !== 0) {
console.error("Error running shell command", stdout, stderr); console.error("Error running shell command", stdout, stderr);
} }
return; return;
} }
case "syscall": {
if (!this.system) {
response.headers.set("Content-Type", "text/plain");
response.status = 400;
response.body = "Unknown operation";
return;
}
const syscallCommand: SyscallRequest = body;
try {
const result = await this.system.localSyscall(
syscallCommand.ctx,
syscallCommand.name,
syscallCommand.args,
);
response.headers.set("Content-type", "application/json");
response.status = 200;
response.body = JSON.stringify({
result: result,
} as SyscallResponse);
} catch (e: any) {
response.headers.set("Content-type", "application/json");
response.status = 500;
response.body = JSON.stringify({
error: e.message,
} as SyscallResponse);
}
return;
}
default: default:
response.headers.set("Content-Type", "text/plain"); response.headers.set("Content-Type", "text/plain");
response.status = 400; response.status = 400;
@ -530,10 +553,3 @@ function utcDateString(mtime: number): string {
function authCookieName(host: string) { function authCookieName(host: string) {
return `auth:${host}`; return `auth:${host}`;
} }
function copyHeader(fromHeaders: Headers, toHeaders: Headers, header: string) {
const value = fromHeaders.get(header);
if (value) {
toHeaders.set(header, value);
}
}

21
server/rpc.ts Normal file
View File

@ -0,0 +1,21 @@
export type ShellRequest = {
cmd: string;
args: string[];
};
export type ShellResponse = {
stdout: string;
stderr: string;
code: number;
};
export type SyscallRequest = {
ctx: string; // Plug name requesting
name: string;
args: any[];
};
export type SyscallResponse = {
result?: any;
error?: string;
};

160
server/server_system.ts Normal file
View File

@ -0,0 +1,160 @@
import { PlugNamespaceHook } from "../common/hooks/plug_namespace.ts";
import { SilverBulletHooks } from "../common/manifest.ts";
import { loadMarkdownExtensions } from "../common/markdown_parser/markdown_ext.ts";
import buildMarkdown from "../common/markdown_parser/parser.ts";
import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitives.ts";
import { FileMetaSpacePrimitives } from "../common/spaces/file_meta_space_primitives.ts";
import { PlugSpacePrimitives } from "../common/spaces/plug_space_primitives.ts";
import { createSandbox } from "../plugos/environments/webworker_sandbox.ts";
import { CronHook } from "../plugos/hooks/cron.ts";
import { EndpointHook } from "../plugos/hooks/endpoint.ts";
import { EventHook } from "../plugos/hooks/event.ts";
import { MQHook } from "../plugos/hooks/mq.ts";
import { DenoKVStore } from "../plugos/lib/kv_store.deno_kv.ts";
import { DexieMQ } from "../plugos/lib/mq.dexie.ts";
import assetSyscalls from "../plugos/syscalls/asset.ts";
import { eventSyscalls } from "../plugos/syscalls/event.ts";
import { mqSyscalls } from "../plugos/syscalls/mq.dexie.ts";
import { storeSyscalls } from "../plugos/syscalls/store.ts";
import { System } from "../plugos/system.ts";
import { Space } from "../web/space.ts";
import { debugSyscalls } from "../web/syscalls/debug.ts";
import { pageIndexSyscalls } from "../cli/syscalls/index.ts";
import { markdownSyscalls } from "../web/syscalls/markdown.ts";
import { spaceSyscalls } from "../cli/syscalls/space.ts";
import { systemSyscalls } from "../web/syscalls/system.ts";
import { yamlSyscalls } from "../web/syscalls/yaml.ts";
import { Application, path } from "./deps.ts";
import { sandboxFetchSyscalls } from "../plugos/syscalls/fetch.ts";
import { shellSyscalls } from "../plugos/syscalls/shell.deno.ts";
import { IDBKeyRange, indexedDB } from "https://esm.sh/fake-indexeddb@4.0.2";
import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
export class ServerSystem {
system: System<SilverBulletHooks> = new System("server");
spacePrimitives!: SpacePrimitives;
requeueInterval?: number;
kvStore?: DenoKVStore;
constructor(
private baseSpacePrimitives: SpacePrimitives,
private dbPath: string,
private app: Application,
) {
}
// Always needs to be invoked right after construction
async init() {
// Event hook
const eventHook = new EventHook();
this.system.addHook(eventHook);
// Cron hook
const cronHook = new CronHook(this.system);
this.system.addHook(cronHook);
this.kvStore = new DenoKVStore();
await this.kvStore.init(this.dbPath);
// Endpoint hook
this.system.addHook(new EndpointHook(this.app, "/_/"));
// Use DexieMQ for this, in memory
const mq = new DexieMQ("mq", indexedDB, IDBKeyRange);
this.requeueInterval = setInterval(() => {
// Timeout after 5s, retries 3 times, otherwise drops the message (no DLQ)
mq.requeueTimeouts(5000, 3, true).catch(console.error);
}, 20000); // Look to requeue every 20s
const pageIndexCalls = pageIndexSyscalls(this.kvStore);
const plugNamespaceHook = new PlugNamespaceHook();
this.system.addHook(plugNamespaceHook);
this.system.addHook(new MQHook(this.system, mq));
this.spacePrimitives = new FileMetaSpacePrimitives(
new EventedSpacePrimitives(
new PlugSpacePrimitives(
this.baseSpacePrimitives,
plugNamespaceHook,
),
eventHook,
),
pageIndexCalls,
);
const space = new Space(this.spacePrimitives, this.kvStore);
// Add syscalls
this.system.registerSyscalls(
[],
eventSyscalls(eventHook),
spaceSyscalls(space),
assetSyscalls(this.system),
yamlSyscalls(),
storeSyscalls(this.kvStore),
systemSyscalls(undefined as any, this.system),
mqSyscalls(mq),
pageIndexCalls,
debugSyscalls(),
markdownSyscalls(buildMarkdown([])), // Will later be replaced with markdown extensions
);
// Syscalls that require some additional permissions
this.system.registerSyscalls(
["fetch"],
sandboxFetchSyscalls(),
);
this.system.registerSyscalls(
["shell"],
shellSyscalls("."),
);
await this.loadPlugs();
// for (let plugPath of await space.listPlugs()) {
// plugPath = path.resolve(this.spacePath, plugPath);
// await this.system.load(
// new URL(`file://${plugPath}`),
// createSandbox,
// );
// }
// Load markdown syscalls based on all new syntax (if any)
this.system.registerSyscalls(
[],
markdownSyscalls(buildMarkdown(loadMarkdownExtensions(this.system))),
);
}
async loadPlugs() {
const tempDir = await Deno.makeTempDir();
try {
for (const { name } of await this.spacePrimitives.fetchFileList()) {
if (
name.endsWith(".plug.js") // && !filePath.includes("search.plug.js")
) {
const plugPath = path.join(tempDir, name);
await Deno.mkdir(path.dirname(plugPath), { recursive: true });
await Deno.writeFile(
plugPath,
(await this.spacePrimitives.readFile(name)).data,
);
await this.system.load(
new URL(`file://${plugPath}`),
createSandbox,
);
}
}
} finally {
await Deno.remove(tempDir, { recursive: true });
}
}
async close() {
clearInterval(this.requeueInterval);
await this.system.unloadAll();
}
}

12
silverbullet.ts
View File

@ -45,8 +45,16 @@ await new Command()
"Path to TLS key", "Path to TLS key",
) )
.option( .option(
"--maxFileSize [type:number]", "-t [type:boolean], --thin-client [type:boolean]",
"Do not sync/expose files larger than this (in MB)", "Enable thin-client mode",
)
.option(
"--reindex [type:boolean]",
"Reindex space on startup (applies to thin-mode only)",
)
.option(
"--db <db:string>",
"Path to database file (applies to thin-mode only)",
) )
.action(serveCommand) .action(serveCommand)
// plug:compile // plug:compile

14
web/boot.ts
View File

@ -1,11 +1,13 @@
import { safeRun } from "../common/util.ts"; import { safeRun } from "../common/util.ts";
import { Client } from "./client.ts"; import { Client } from "./client.ts";
const thinClientMode = window.silverBulletConfig.thinClientMode === "on";
safeRun(async () => { safeRun(async () => {
console.log("Booting SilverBullet..."); console.log("Booting SilverBullet...");
const client = new Client( const client = new Client(
document.getElementById("sb-root")!, document.getElementById("sb-root")!,
thinClientMode,
); );
await client.init(); await client.init();
window.client = client; window.client = client;
@ -19,12 +21,14 @@ if (navigator.serviceWorker) {
.then(() => { .then(() => {
console.log("Service worker registered..."); console.log("Service worker registered...");
}); });
navigator.serviceWorker.ready.then((registration) => { if (!thinClientMode) {
registration.active!.postMessage({ navigator.serviceWorker.ready.then((registration) => {
type: "config", registration.active!.postMessage({
config: window.silverBulletConfig, type: "config",
config: window.silverBulletConfig,
});
}); });
}); }
} else { } else {
console.warn( console.warn(
"Not launching service worker, likely because not running from localhost or over HTTPs. This means SilverBullet will not be available offline.", "Not launching service worker, likely because not running from localhost or over HTTPs. This means SilverBullet will not be available offline.",

77
web/client.ts
View File

@ -36,6 +36,7 @@ import { MainUI } from "./editor_ui.tsx";
import { DexieMQ } from "../plugos/lib/mq.dexie.ts"; import { DexieMQ } from "../plugos/lib/mq.dexie.ts";
import { cleanPageRef } from "$sb/lib/resolve.ts"; import { cleanPageRef } from "$sb/lib/resolve.ts";
import { expandPropertyNames } from "$sb/lib/json.ts"; import { expandPropertyNames } from "$sb/lib/json.ts";
import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/; const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/;
const autoSaveInterval = 1000; const autoSaveInterval = 1000;
@ -45,6 +46,7 @@ declare global {
// Injected via index.html // Injected via index.html
silverBulletConfig: { silverBulletConfig: {
spaceFolderPath: string; spaceFolderPath: string;
thinClientMode: "on" | "off";
}; };
client: Client; client: Client;
} }
@ -53,15 +55,13 @@ declare global {
// TODO: Oh my god, need to refactor this // TODO: Oh my god, need to refactor this
export class Client { export class Client {
system: ClientSystem; system: ClientSystem;
editorView: EditorView; editorView: EditorView;
private pageNavigator!: PathPageNavigator; private pageNavigator!: PathPageNavigator;
private dbPrefix: string; private dbPrefix: string;
plugSpaceRemotePrimitives!: PlugSpacePrimitives; plugSpaceRemotePrimitives!: PlugSpacePrimitives;
localSpacePrimitives!: FilteredSpacePrimitives; // localSpacePrimitives!: FilteredSpacePrimitives;
remoteSpacePrimitives!: HttpSpacePrimitives; remoteSpacePrimitives!: HttpSpacePrimitives;
space!: Space; space!: Space;
@ -88,6 +88,7 @@ export class Client {
constructor( constructor(
parent: Element, parent: Element,
private thinClientMode = false,
) { ) {
// Generate a semi-unique prefix for the database so not to reuse databases for different space paths // Generate a semi-unique prefix for the database so not to reuse databases for different space paths
this.dbPrefix = "" + simpleHash(window.silverBulletConfig.spaceFolderPath); this.dbPrefix = "" + simpleHash(window.silverBulletConfig.spaceFolderPath);
@ -116,12 +117,13 @@ export class Client {
this.mq, this.mq,
this.dbPrefix, this.dbPrefix,
this.eventHook, this.eventHook,
this.thinClientMode,
); );
this.initSpace(); const localSpacePrimitives = this.initSpace();
this.syncService = new SyncService( this.syncService = new SyncService(
this.localSpacePrimitives, localSpacePrimitives,
this.plugSpaceRemotePrimitives, this.plugSpaceRemotePrimitives,
this.kvStore, this.kvStore,
this.eventHook, this.eventHook,
@ -133,6 +135,7 @@ export class Client {
// Except federated ones // Except federated ones
path.startsWith("!"); path.startsWith("!");
}, },
!this.thinClientMode,
); );
this.ui = new MainUI(this); this.ui = new MainUI(this);
@ -319,7 +322,7 @@ export class Client {
} }
} }
initSpace() { initSpace(): SpacePrimitives {
this.remoteSpacePrimitives = new HttpSpacePrimitives( this.remoteSpacePrimitives = new HttpSpacePrimitives(
location.origin, location.origin,
window.silverBulletConfig.spaceFolderPath, window.silverBulletConfig.spaceFolderPath,
@ -332,34 +335,40 @@ export class Client {
let fileFilterFn: (s: string) => boolean = () => true; let fileFilterFn: (s: string) => boolean = () => true;
this.localSpacePrimitives = new FilteredSpacePrimitives( let localSpacePrimitives: SpacePrimitives | undefined;
new FileMetaSpacePrimitives(
new EventedSpacePrimitives(
// Using fallback space primitives here to allow (by default) local reads to "fall through" to HTTP when files aren't synced yet
new FallbackSpacePrimitives(
new IndexedDBSpacePrimitives(
`${this.dbPrefix}_space`,
globalThis.indexedDB,
),
this.plugSpaceRemotePrimitives,
),
this.eventHook,
),
this.system.indexSyscalls,
),
(meta) => fileFilterFn(meta.name),
// Run when a list of files has been retrieved
async () => {
await this.loadSettings();
if (typeof this.settings?.spaceIgnore === "string") {
fileFilterFn = gitIgnoreCompiler(this.settings.spaceIgnore).accepts;
} else {
fileFilterFn = () => true;
}
},
);
this.space = new Space(this.localSpacePrimitives, this.kvStore); if (!this.thinClientMode) {
localSpacePrimitives = new FilteredSpacePrimitives(
new FileMetaSpacePrimitives(
new EventedSpacePrimitives(
// Using fallback space primitives here to allow (by default) local reads to "fall through" to HTTP when files aren't synced yet
new FallbackSpacePrimitives(
new IndexedDBSpacePrimitives(
`${this.dbPrefix}_space`,
globalThis.indexedDB,
),
this.plugSpaceRemotePrimitives,
),
this.eventHook,
),
this.system.indexSyscalls,
),
(meta) => fileFilterFn(meta.name),
// Run when a list of files has been retrieved
async () => {
await this.loadSettings();
if (typeof this.settings?.spaceIgnore === "string") {
fileFilterFn = gitIgnoreCompiler(this.settings.spaceIgnore).accepts;
} else {
fileFilterFn = () => true;
}
},
);
} else {
localSpacePrimitives = this.plugSpaceRemotePrimitives;
}
this.space = new Space(localSpacePrimitives, this.kvStore);
this.space.on({ this.space.on({
pageChanged: (meta) => { pageChanged: (meta) => {
@ -379,6 +388,8 @@ export class Client {
}); });
this.space.watch(); this.space.watch();
return localSpacePrimitives;
} }
async loadSettings(): Promise<BuiltinSettings> { async loadSettings(): Promise<BuiltinSettings> {

48
web/client_system.ts
View File

@ -33,6 +33,8 @@ import {
import { DexieMQ } from "../plugos/lib/mq.dexie.ts"; import { DexieMQ } from "../plugos/lib/mq.dexie.ts";
import { MQHook } from "../plugos/hooks/mq.ts"; import { MQHook } from "../plugos/hooks/mq.ts";
import { mqSyscalls } from "../plugos/syscalls/mq.dexie.ts"; import { mqSyscalls } from "../plugos/syscalls/mq.dexie.ts";
import { indexProxySyscalls } from "./syscalls/index.proxy.ts";
import { storeProxySyscalls } from "./syscalls/store.proxy.ts";
export class ClientSystem { export class ClientSystem {
system: System<SilverBulletHooks> = new System("client"); system: System<SilverBulletHooks> = new System("client");
@ -45,11 +47,12 @@ export class ClientSystem {
mdExtensions: MDExt[] = []; mdExtensions: MDExt[] = [];
constructor( constructor(
private editor: Client, private client: Client,
private kvStore: DexieKVStore, private kvStore: DexieKVStore,
private mq: DexieMQ, private mq: DexieMQ,
private dbPrefix: string, private dbPrefix: string,
private eventHook: EventHook, private eventHook: EventHook,
private thinClientMode: boolean,
) { ) {
this.system.addHook(this.eventHook); this.system.addHook(this.eventHook);
@ -61,11 +64,15 @@ export class ClientSystem {
const cronHook = new CronHook(this.system); const cronHook = new CronHook(this.system);
this.system.addHook(cronHook); this.system.addHook(cronHook);
this.indexSyscalls = pageIndexSyscalls( if (thinClientMode) {
`${dbPrefix}_page_index`, this.indexSyscalls = indexProxySyscalls(client);
globalThis.indexedDB, } else {
globalThis.IDBKeyRange, this.indexSyscalls = pageIndexSyscalls(
); `${dbPrefix}_page_index`,
globalThis.indexedDB,
globalThis.IDBKeyRange,
);
}
// Code widget hook // Code widget hook
this.codeWidgetHook = new CodeWidgetHook(); this.codeWidgetHook = new CodeWidgetHook();
@ -78,7 +85,7 @@ export class ClientSystem {
this.commandHook = new CommandHook(); this.commandHook = new CommandHook();
this.commandHook.on({ this.commandHook.on({
commandsUpdated: (commandMap) => { commandsUpdated: (commandMap) => {
this.editor.ui.viewDispatch({ this.client.ui.viewDispatch({
type: "update-commands", type: "update-commands",
commands: commandMap, commands: commandMap,
}); });
@ -87,7 +94,7 @@ export class ClientSystem {
this.system.addHook(this.commandHook); this.system.addHook(this.commandHook);
// Slash command hook // Slash command hook
this.slashCommandHook = new SlashCommandHook(this.editor); this.slashCommandHook = new SlashCommandHook(this.client);
this.system.addHook(this.slashCommandHook); this.system.addHook(this.slashCommandHook);
this.eventHook.addLocalListener("plug:changed", async (fileName) => { this.eventHook.addLocalListener("plug:changed", async (fileName) => {
@ -96,7 +103,7 @@ export class ClientSystem {
const plug = await this.system.load( const plug = await this.system.load(
new URL(`/${fileName}`, location.href), new URL(`/${fileName}`, location.href),
createSandbox, createSandbox,
this.editor.settings.plugOverrides, this.client.settings.plugOverrides,
); );
if ((plug.manifest! as Manifest).syntax) { if ((plug.manifest! as Manifest).syntax) {
// If there are syntax extensions, rebuild the markdown parser immediately // If there are syntax extensions, rebuild the markdown parser immediately
@ -108,19 +115,21 @@ export class ClientSystem {
} }
registerSyscalls() { registerSyscalls() {
const storeCalls = storeSyscalls(this.kvStore); const storeCalls = this.thinClientMode
? storeProxySyscalls(this.client)
: storeSyscalls(this.kvStore);
// Slash command hook // Slash command hook
this.slashCommandHook = new SlashCommandHook(this.editor); this.slashCommandHook = new SlashCommandHook(this.client);
this.system.addHook(this.slashCommandHook); this.system.addHook(this.slashCommandHook);
// Syscalls available to all plugs // Syscalls available to all plugs
this.system.registerSyscalls( this.system.registerSyscalls(
[], [],
eventSyscalls(this.eventHook), eventSyscalls(this.eventHook),
editorSyscalls(this.editor), editorSyscalls(this.client),
spaceSyscalls(this.editor), spaceSyscalls(this.client),
systemSyscalls(this.editor, this.system), systemSyscalls(this.client, this.system),
markdownSyscalls(buildMarkdown(this.mdExtensions)), markdownSyscalls(buildMarkdown(this.mdExtensions)),
assetSyscalls(this.system), assetSyscalls(this.system),
yamlSyscalls(), yamlSyscalls(),
@ -128,20 +137,19 @@ export class ClientSystem {
storeCalls, storeCalls,
this.indexSyscalls, this.indexSyscalls,
debugSyscalls(), debugSyscalls(),
syncSyscalls(this.editor), syncSyscalls(this.client),
// LEGACY clientStoreSyscalls(this.kvStore),
clientStoreSyscalls(storeCalls),
); );
// Syscalls that require some additional permissions // Syscalls that require some additional permissions
this.system.registerSyscalls( this.system.registerSyscalls(
["fetch"], ["fetch"],
sandboxFetchSyscalls(this.editor), sandboxFetchSyscalls(this.client),
); );
this.system.registerSyscalls( this.system.registerSyscalls(
["shell"], ["shell"],
shellSyscalls(this.editor), shellSyscalls(this.client),
); );
} }
@ -155,7 +163,7 @@ export class ClientSystem {
await this.system.load( await this.system.load(
new URL(plugName, location.origin), new URL(plugName, location.origin),
createSandbox, createSandbox,
this.editor.settings.plugOverrides, this.client.settings.plugOverrides,
); );
} catch (e: any) { } catch (e: any) {
console.error("Could not load plug", plugName, "error:", e.message); console.error("Could not load plug", plugName, "error:", e.message);

2
web/index.html
View File

@ -35,11 +35,13 @@
window.silverBulletConfig = { window.silverBulletConfig = {
// These {{VARIABLES}} are replaced by http_server.ts // These {{VARIABLES}} are replaced by http_server.ts
spaceFolderPath: "{{SPACE_PATH}}", spaceFolderPath: "{{SPACE_PATH}}",
thinClientMode: "{{THIN_CLIENT_MODE}}",
}; };
// But in case these variables aren't replaced by the server, fall back fully static mode (no sync) // But in case these variables aren't replaced by the server, fall back fully static mode (no sync)
if (window.silverBulletConfig.spaceFolderPath.includes("{{")) { if (window.silverBulletConfig.spaceFolderPath.includes("{{")) {
window.silverBulletConfig = { window.silverBulletConfig = {
spaceFolderPath: "", spaceFolderPath: "",
thinClientMode: "off",
}; };
} }
</script> </script>

44
web/sync_service.ts
View File

@ -45,6 +45,7 @@ export class SyncService {
private kvStore: KVStore, private kvStore: KVStore,
private eventHook: EventHook, private eventHook: EventHook,
private isSyncCandidate: (path: string) => boolean, private isSyncCandidate: (path: string) => boolean,
private enabled: boolean,
) { ) {
this.spaceSync = new SpaceSync( this.spaceSync = new SpaceSync(
this.localSpacePrimitives, this.localSpacePrimitives,
@ -74,6 +75,9 @@ export class SyncService {
} }
async isSyncing(): Promise<boolean> { async isSyncing(): Promise<boolean> {
if (!this.enabled) {
return false;
}
const startTime = await this.kvStore.get(syncStartTimeKey); const startTime = await this.kvStore.get(syncStartTimeKey);
if (!startTime) { if (!startTime) {
return false; return false;
@ -91,11 +95,19 @@ export class SyncService {
} }
hasInitialSyncCompleted(): Promise<boolean> { hasInitialSyncCompleted(): Promise<boolean> {
if (!this.enabled) {
return Promise.resolve(true);
}
// Initial sync has happened when sync progress has been reported at least once, but the syncStartTime has been reset (which happens after sync finishes) // Initial sync has happened when sync progress has been reported at least once, but the syncStartTime has been reset (which happens after sync finishes)
return this.kvStore.has(syncInitialFullSyncCompletedKey); return this.kvStore.has(syncInitialFullSyncCompletedKey);
} }
async registerSyncStart(fullSync: boolean): Promise<void> { async registerSyncStart(fullSync: boolean): Promise<void> {
if (!this.enabled) {
return;
}
// Assumption: this is called after an isSyncing() check // Assumption: this is called after an isSyncing() check
await this.kvStore.batchSet([ await this.kvStore.batchSet([
{ {
@ -116,6 +128,10 @@ export class SyncService {
} }
async registerSyncProgress(status?: SyncStatus): Promise<void> { async registerSyncProgress(status?: SyncStatus): Promise<void> {
if (!this.enabled) {
return;
}
// Emit a sync event at most every 2s // Emit a sync event at most every 2s
if (status && this.lastReportedSyncStatus < Date.now() - 2000) { if (status && this.lastReportedSyncStatus < Date.now() - 2000) {
this.eventHook.dispatchEvent("sync:progress", status); this.eventHook.dispatchEvent("sync:progress", status);
@ -126,6 +142,10 @@ export class SyncService {
} }
async registerSyncStop(isFullSync: boolean): Promise<void> { async registerSyncStop(isFullSync: boolean): Promise<void> {
if (!this.enabled) {
return;
}
await this.registerSyncProgress(); await this.registerSyncProgress();
await this.kvStore.del(syncStartTimeKey); await this.kvStore.del(syncStartTimeKey);
if (isFullSync) { if (isFullSync) {
@ -142,6 +162,10 @@ export class SyncService {
// Await a moment when the sync is no longer running // Await a moment when the sync is no longer running
async noOngoingSync(timeout: number): Promise<void> { async noOngoingSync(timeout: number): Promise<void> {
if (!this.enabled) {
return;
}
// Not completely safe, could have race condition on setting the syncStartTimeKey // Not completely safe, could have race condition on setting the syncStartTimeKey
const startTime = Date.now(); const startTime = Date.now();
while (await this.isSyncing()) { while (await this.isSyncing()) {
@ -155,6 +179,10 @@ export class SyncService {
filesScheduledForSync = new Set<string>(); filesScheduledForSync = new Set<string>();
async scheduleFileSync(path: string): Promise<void> { async scheduleFileSync(path: string): Promise<void> {
if (!this.enabled) {
return;
}
if (this.filesScheduledForSync.has(path)) { if (this.filesScheduledForSync.has(path)) {
// Already scheduled, no need to duplicate // Already scheduled, no need to duplicate
console.info(`File ${path} already scheduled for sync`); console.info(`File ${path} already scheduled for sync`);
@ -167,11 +195,19 @@ export class SyncService {
} }
async scheduleSpaceSync(): Promise<void> { async scheduleSpaceSync(): Promise<void> {
if (!this.enabled) {
return;
}
await this.noOngoingSync(5000); await this.noOngoingSync(5000);
await this.syncSpace(); await this.syncSpace();
} }
start() { start() {
if (!this.enabled) {
return;
}
this.syncSpace().catch(console.error); this.syncSpace().catch(console.error);
setInterval(async () => { setInterval(async () => {
@ -191,6 +227,10 @@ export class SyncService {
} }
async syncSpace(): Promise<number> { async syncSpace(): Promise<number> {
if (!this.enabled) {
return 0;
}
if (await this.isSyncing()) { if (await this.isSyncing()) {
console.log("Aborting space sync: already syncing"); console.log("Aborting space sync: already syncing");
return 0; return 0;
@ -218,6 +258,10 @@ export class SyncService {
// Syncs a single file // Syncs a single file
async syncFile(name: string) { async syncFile(name: string) {
if (!this.enabled) {
return;
}
// console.log("Checking if we can sync file", name); // console.log("Checking if we can sync file", name);
if (!this.isSyncCandidate(name)) { if (!this.isSyncCandidate(name)) {
console.info("Requested sync, but not a sync candidate", name); console.info("Requested sync, but not a sync candidate", name);

11
web/syscalls/clientStore.ts
View File

@ -1,14 +1,19 @@
import { KVStore } from "../../plugos/lib/kv_store.ts";
import { storeSyscalls } from "../../plugos/syscalls/store.ts";
import { proxySyscalls } from "../../plugos/syscalls/transport.ts"; import { proxySyscalls } from "../../plugos/syscalls/transport.ts";
import { SysCallMapping } from "../../plugos/system.ts"; import { SysCallMapping } from "../../plugos/system.ts";
// DEPRECATED, use store directly
export function clientStoreSyscalls( export function clientStoreSyscalls(
storeCalls: SysCallMapping, db: KVStore,
): SysCallMapping { ): SysCallMapping {
const localStoreCalls = storeSyscalls(db);
return proxySyscalls( return proxySyscalls(
["clientStore.get", "clientStore.set", "clientStore.delete"], ["clientStore.get", "clientStore.set", "clientStore.delete"],
(ctx, name, ...args) => { (ctx, name, ...args) => {
return storeCalls[name.replace("clientStore.", "store.")](ctx, ...args); return localStoreCalls[name.replace("clientStore.", "store.")](
ctx,
...args,
);
}, },
); );
} }

16
web/syscalls/index.proxy.ts Normal file
View File

@ -0,0 +1,16 @@
import { SysCallMapping } from "../../plugos/system.ts";
import { Client } from "../client.ts";
import { proxySyscalls } from "./util.ts";
export function indexProxySyscalls(client: Client): SysCallMapping {
return proxySyscalls(client, [
"index.set",
"index.batchSet",
"index.delete",
"index.get",
"index.queryPrefix",
"index.clearPageIndexForPage",
"index.deletePrefixForPage",
"index.clearPageIndex",
]);
}

18
web/syscalls/store.proxy.ts Normal file
View File

@ -0,0 +1,18 @@
import type { SysCallMapping } from "../../plugos/system.ts";
import type { Client } from "../client.ts";
import { proxySyscalls } from "./util.ts";
export function storeProxySyscalls(client: Client): SysCallMapping {
return proxySyscalls(client, [
"store.delete",
"store.deletePrefix",
"store.deleteAll",
"store.set",
"store.batchSet",
"store.batchDelete",
"store.batchGet",
"store.get",
"store.has",
"store.queryPrefix",
]);
}

33
web/syscalls/util.ts Normal file
View File

@ -0,0 +1,33 @@
import { SysCallMapping } from "../../plugos/system.ts";
import { SyscallResponse } from "../../server/rpc.ts";
import { Client } from "../client.ts";
export function proxySyscalls(client: Client, names: string[]): SysCallMapping {
const syscalls: SysCallMapping = {};
for (const name of names) {
syscalls[name] = async (_ctx, ...args: any[]) => {
if (!client.remoteSpacePrimitives) {
throw new Error("Not supported");
}
const resp = await client.remoteSpacePrimitives.authenticatedFetch(
`${client.remoteSpacePrimitives.url}/.rpc`,
{
method: "POST",
body: JSON.stringify({
operation: "syscall",
name,
args,
}),
},
);
const result: SyscallResponse = await resp.json();
if (result.error) {
console.error("Remote syscall error", result.error);
throw new Error(result.error);
} else {
return result.result;
}
};
}
return syscalls;
}