From 5a7a35c759b6d7396128d52ac3ea23e378a62bc4 Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Sun, 17 Dec 2023 11:46:18 +0100 Subject: [PATCH] E2E encryption (prototype) (#601) Prototype E2E encryption --- cmd/server.ts | 19 +- common/crypto/aes.test.ts | 33 -- common/crypto/aes.ts | 111 ----- .../spaces/encrypted_space_primitives.test.ts | 62 +++ common/spaces/encrypted_space_primitives.ts | 455 ++++++++++++++++++ common/spaces/space_primitives.test.ts | 4 +- common/util.ts | 12 +- server/http_server.ts | 52 +- server/instance.ts | 22 +- silverbullet.ts | 4 + web/client.ts | 115 +++-- web/editor_state.ts | 2 +- web/index.html | 2 + web/service_worker.ts | 3 +- web/syscalls/fetch.ts | 6 +- web/syscalls/shell.ts | 6 +- web/syscalls/system.ts | 2 +- web/syscalls/util.ts | 2 +- 18 files changed, 702 insertions(+), 210 deletions(-) delete mode 100644 common/crypto/aes.test.ts delete mode 100644 common/crypto/aes.ts create mode 100644 common/spaces/encrypted_space_primitives.test.ts create mode 100644 common/spaces/encrypted_space_primitives.ts diff --git a/cmd/server.ts b/cmd/server.ts index 36febf57..2d0ca2b4 100644 --- a/cmd/server.ts +++ b/cmd/server.ts @@ -22,6 +22,7 @@ export async function serveCommand( key?: string; reindex?: boolean; syncOnly?: boolean; + clientEncryption?: boolean; }, folder?: string, ) { @@ -29,7 +30,22 @@ export async function serveCommand( "127.0.0.1"; const port = options.port || (Deno.env.get("SB_PORT") && +Deno.env.get("SB_PORT")!) || 3000; + + const clientEncryption = options.clientEncryption || + !!Deno.env.get("SB_CLIENT_ENCRYPTION"); + + if (clientEncryption) { + console.log( + "Running in client encryption mode, this will implicitly enable sync-only mode", + ); + } + const syncOnly = options.syncOnly || !!Deno.env.get("SB_SYNC_ONLY"); + + if (syncOnly) { + console.log("Running in sync-only mode (no backend processing)"); + } + const app = new Application(); if (!folder) { @@ -69,6 +85,8 @@ To allow outside connections, pass -L 0.0.0.0 as a flag, and put a TLS terminato namespace: "*", auth: userCredentials, authToken: Deno.env.get("SB_AUTH_TOKEN"), + syncOnly, + clientEncryption, pagesPath: folder, }); @@ -79,7 +97,6 @@ To allow outside connections, pass -L 0.0.0.0 as a flag, and put a TLS terminato clientAssetBundle: new AssetBundle(clientAssetBundle as AssetJson), plugAssetBundle: new AssetBundle(plugAssetBundle as AssetJson), baseKvPrimitives, - syncOnly, keyFile: options.key, certFile: options.cert, configs, diff --git a/common/crypto/aes.test.ts b/common/crypto/aes.test.ts deleted file mode 100644 index eb66e9b2..00000000 --- a/common/crypto/aes.test.ts +++ /dev/null @@ -1,33 +0,0 @@ -import { assertEquals } from "../../test_deps.ts"; -import { - decryptAES, - decryptPath, - deriveKeyFromPassword, - encryptAES, - encryptPath, -} from "./aes.ts"; - -Deno.test("AES encryption and decryption", async () => { - const password = "YourPassword"; - const salt = "UniquePerUserSalt"; - const message = "Hello, World!"; - - const key = await deriveKeyFromPassword(password, salt); - const encrypted = await encryptAES(key, message); - - const decrypted = await decryptAES(key, encrypted); - assertEquals(decrypted, message); - - // Test that checks if a path is encrypted the same way every time and can be unencrypted - const path = - "this/is/a/long/path/that/needs/to/be/encrypted because that's what we do.md"; - const encryptedPath = await encryptPath(key, path); - const encryptedPath2 = await encryptPath(key, path); - // Assure two runs give the same result - assertEquals(encryptedPath, encryptedPath2); - - // Ensure decryption works - const decryptedPath = await decryptPath(key, encryptedPath); - console.log(encryptedPath); - assertEquals(decryptedPath, path); -}); diff --git a/common/crypto/aes.ts b/common/crypto/aes.ts deleted file mode 100644 index 041439e6..00000000 --- a/common/crypto/aes.ts +++ /dev/null @@ -1,111 +0,0 @@ -import { - base64Decode, - base64Encode, -} from "../../plugos/asset_bundle/base64.ts"; - -const encoder = new TextEncoder(); -const decoder = new TextDecoder(); - -export async function deriveKeyFromPassword( - password: string, - salt: string, -): Promise { - const baseKey = encoder.encode(password); - const importedKey = await window.crypto.subtle.importKey( - "raw", - baseKey, - { name: "PBKDF2" }, - false, - ["deriveKey"], - ); - return crypto.subtle.deriveKey( - { - name: "PBKDF2", - salt: encoder.encode(salt), - iterations: 10000, - hash: "SHA-256", - }, - importedKey, - { - name: "AES-GCM", - length: 256, - }, - true, - ["encrypt", "decrypt"], - ); -} - -export async function encryptAES( - key: CryptoKey, - message: string, -): Promise { - const iv = crypto.getRandomValues(new Uint8Array(12)); - const encodedMessage = encoder.encode(message); - const ciphertext = await window.crypto.subtle.encrypt( - { - name: "AES-GCM", - iv: iv, - }, - key, - encodedMessage, - ); - return appendBuffer(iv, ciphertext); -} - -export async function decryptAES( - key: CryptoKey, - data: ArrayBuffer, -): Promise { - const iv = data.slice(0, 12); - const ciphertext = data.slice(12); - const decrypted = await window.crypto.subtle.decrypt( - { - name: "AES-GCM", - iv: iv, - }, - key, - ciphertext, - ); - return decoder.decode(decrypted); -} - -function appendBuffer(buffer1: ArrayBuffer, buffer2: ArrayBuffer): ArrayBuffer { - const tmp = new Uint8Array(buffer1.byteLength + buffer2.byteLength); - tmp.set(new Uint8Array(buffer1), 0); - tmp.set(new Uint8Array(buffer2), buffer1.byteLength); - return tmp.buffer; -} - -// This is against security recommendations, but we need a way to always generate the same encrypted path for the same path and password -const pathIv = new Uint8Array(12); // 12 bytes of 0 - -export async function encryptPath( - key: CryptoKey, - path: string, -): Promise { - const encodedMessage = encoder.encode(path); - const ciphertext = await crypto.subtle.encrypt( - { - name: "AES-GCM", - iv: pathIv, - }, - key, - encodedMessage, - ); - return base64Encode(new Uint8Array(ciphertext)); -} - -export async function decryptPath( - key: CryptoKey, - data: string, -): Promise { - const decrypted = await crypto.subtle.decrypt( - { - name: "AES-GCM", - iv: pathIv, - }, - key, - base64Decode(data), - ); - return decoder.decode(decrypted); -} diff --git a/common/spaces/encrypted_space_primitives.test.ts b/common/spaces/encrypted_space_primitives.test.ts new file mode 100644 index 00000000..3421f340 --- /dev/null +++ b/common/spaces/encrypted_space_primitives.test.ts @@ -0,0 +1,62 @@ +import { MemoryKvPrimitives } from "../../plugos/lib/memory_kv_primitives.ts"; +import { assert, assertEquals } from "../../test_deps.ts"; +import { ChunkedKvStoreSpacePrimitives } from "./chunked_datastore_space_primitives.ts"; +import { EncryptedSpacePrimitives } from "./encrypted_space_primitives.ts"; +import { testSpacePrimitives } from "./space_primitives.test.ts"; + +Deno.test("Encrypted Space Primitives", async () => { + // Using an in-memory store for testing + const memoryKv = new MemoryKvPrimitives(); + const spacePrimitives = new EncryptedSpacePrimitives( + new ChunkedKvStoreSpacePrimitives( + memoryKv, + 1024 * 1024, + ), + ); + assertEquals(false, await spacePrimitives.init()); + await spacePrimitives.setup("password"); + assertEquals(await spacePrimitives.fetchFileList(), []); + await testSpacePrimitives(spacePrimitives); + + // Let's try an incorrect password + try { + await spacePrimitives.login("wronk"); + assert(false); + } catch (e: any) { + assertEquals(e.message, "Incorrect password"); + } + + // Now let's update the password + await spacePrimitives.updatePassword("password", "password2"); + + try { + await spacePrimitives.updatePassword("password", "password2"); + assert(false); + } catch (e: any) { + assertEquals(e.message, "Incorrect password"); + } + + await spacePrimitives.writeFile( + "test.txt", + new TextEncoder().encode("Hello World"), + ); + + // Let's do this again with the new password + + const spacePrimitives2 = new EncryptedSpacePrimitives( + new ChunkedKvStoreSpacePrimitives( + memoryKv, + 1024 * 1024, + ), + ); + assertEquals(true, await spacePrimitives2.init()); + await spacePrimitives2.login("password2"); + assertEquals( + new TextDecoder().decode( + (await spacePrimitives2.readFile("test.txt")).data, + ), + "Hello World", + ); + await spacePrimitives2.deleteFile("test.txt"); + await testSpacePrimitives(spacePrimitives2); +}); diff --git a/common/spaces/encrypted_space_primitives.ts b/common/spaces/encrypted_space_primitives.ts new file mode 100644 index 00000000..30be47fa --- /dev/null +++ b/common/spaces/encrypted_space_primitives.ts @@ -0,0 +1,455 @@ +import { FileMeta } from "../../plug-api/types.ts"; +import { SpacePrimitives } from "./space_primitives.ts"; + +export const encryptedFileExt = ".crypt"; +export const keyPath = "KEY"; +export const saltFile = "salt.crypt"; + +/** + * This class adds an (AES) based encryption layer on top of another SpacePrimitives implementation. + * It encrypts all file names and file contents. + * It uses a key file (default named _KEY) to store the encryption key, this file is encrypted with a key derived from the user's password. + * The reason to keep the actualy encryption key in a file is to allow the user to change their password without having to re-encrypt all files. + * Important note: FileMeta's size will reflect the underlying encrypted size, not the original size + */ +export class EncryptedSpacePrimitives implements SpacePrimitives { + private masterKey?: CryptoKey; + private encryptedKeyFileName?: string; + spaceSalt?: Uint8Array; + + constructor( + private wrapped: SpacePrimitives, + ) { + } + + /** + * Checks if the space is initialized by loading the salt file. + * @returns true if the space was initialized, false if it was not initialized yet + */ + async init(salt?: Uint8Array | undefined | null): Promise { + if (salt) { + this.spaceSalt = salt; + return true; + } + try { + this.spaceSalt = (await this.wrapped.readFile(saltFile)).data; + return true; + } catch (e: any) { + if (e.message === "Not found") { + console.warn("Space not initialized"); + return false; + } + throw e; + } + } + + /** + * Setup a fresh space with a new salt and master encryption key derived from a password + * @param password + */ + async setup(password: string): Promise { + if (this.spaceSalt) { + throw new Error("Space already initialized"); + } + this.spaceSalt = this.generateSalt(); + await this.wrapped.writeFile(saltFile, this.spaceSalt); + await this.createKey(password); + } + + /** + * Loads the encryption key from the master key based on the user's password + * @param password the user's password + */ + async login(password: string): Promise { + if (!this.spaceSalt) { + throw new Error("Space not initialized"); + } + // First derive an encryption key solely used for encrypting the key file from the user's password + const keyEncryptionKey = await this.deriveKeyFromPassword(password); + const encryptedKeyFileName = await this.encryptPath( + keyEncryptionKey, + keyPath, + ); + + try { + this.masterKey = await this.importKey( + await this.decryptAES( + keyEncryptionKey, + (await this.wrapped.readFile( + encryptedKeyFileName, + )).data, + ), + ); + this.encryptedKeyFileName = encryptedKeyFileName; + } catch (e: any) { + if (e.message === "Not found") { + throw new Error("Incorrect password"); + } + console.trace(); + throw e; + } + } + + private generateKey(): Promise { + return window.crypto.subtle.generateKey( + { + name: "AES-GCM", + length: 256, + }, + true, + ["encrypt", "decrypt"], + ); + } + + private async createKey(password: string): Promise { + const keyEncryptionKey = await this.deriveKeyFromPassword(password); + this.encryptedKeyFileName = await this.encryptPath( + keyEncryptionKey, + keyPath, + ); + this.masterKey = await this.generateKey(); + // And write it + await this.wrapped.writeFile( + this.encryptedKeyFileName, + await this.encryptAES( + keyEncryptionKey, + await this.exportKey(this.masterKey), + ), + ); + } + + async updatePassword(oldPassword: string, newPasword: string): Promise { + if (!this.masterKey) { + throw new Error("No key loaded"); + } + const oldPasswordKeyFileName = await this.encryptPath( + await this.deriveKeyFromPassword(oldPassword), + keyPath, + ); + + // Check if the old password is correct + try { + await this.wrapped.getFileMeta(oldPasswordKeyFileName); + } catch (e: any) { + if (e.message === "Not found") { + throw new Error("Incorrect password"); + } else { + throw e; + } + } + + // First derive an encryption key solely used for encrypting the key file from the user's password + const keyEncryptionKey = await this.deriveKeyFromPassword(newPasword); + + this.encryptedKeyFileName = await this.encryptPath( + keyEncryptionKey, + keyPath, + ); + // And write it + await this.wrapped.writeFile( + this.encryptedKeyFileName, + await this.encryptAES( + keyEncryptionKey, + await this.exportKey(this.masterKey), + ), + ); + + // Then delete the old key file based on the old password + await this.wrapped.deleteFile(oldPasswordKeyFileName); + } + + isUnencryptedPath(name: string) { + return name.startsWith("_plug/"); + } + + private generateSalt(): Uint8Array { + return crypto.getRandomValues(new Uint8Array(16)); + } + + private async exportKey(key: CryptoKey): Promise { + const arrayBuffer = await window.crypto.subtle.exportKey("raw", key); + return new Uint8Array(arrayBuffer); + } + + private importKey(key: Uint8Array): Promise { + return window.crypto.subtle.importKey( + "raw", + key, + { name: "AES-GCM" }, + true, + ["encrypt", "decrypt"], + ); + } + + private async deriveKeyFromPassword( + password: string, + ): Promise { + const baseKey = new TextEncoder().encode(password); + const importedKey = await window.crypto.subtle.importKey( + "raw", + baseKey, + { name: "PBKDF2" }, + false, + ["deriveKey"], + ); + return crypto.subtle.deriveKey( + { + name: "PBKDF2", + salt: this.spaceSalt!, + iterations: 10000, + hash: "SHA-256", + }, + importedKey, + { + name: "AES-GCM", + length: 256, + }, + true, + ["encrypt", "decrypt"], + ); + } + + /** + * Encrypts using AES-GCM and prepends the IV to the ciphertext + * @param key + * @param message + * @returns + */ + private async encryptAES( + key: CryptoKey, + message: Uint8Array, + ): Promise { + const iv = crypto.getRandomValues(new Uint8Array(12)); + const ciphertext = await window.crypto.subtle.encrypt( + { + name: "AES-GCM", + iv: iv, + }, + key, + message, + ); + return appendBuffer(iv, new Uint8Array(ciphertext)); + } + + /** + * Decrypts using AES-GCM and expects the IV to be prepended to the ciphertext + * @param key + * @param data + * @returns + */ + async decryptAES( + key: CryptoKey, + data: Uint8Array, + ): Promise { + const iv = data.slice(0, 12); + const ciphertext = data.slice(12); + const decrypted = await window.crypto.subtle.decrypt( + { + name: "AES-GCM", + iv: iv, + }, + key, + ciphertext, + ); + return new Uint8Array(decrypted); + } + + /** + * Left pads a string with zeros to a length of 32, encrypts it using AES-GCM and returns the base32 encoded ciphertext + * @param key + * @param path + * @returns + */ + async encryptPath( + key: CryptoKey, + path: string, + ): Promise { + if (!this.spaceSalt) { + throw new Error("Space not initialized"); + } + if (this.isUnencryptedPath(path)) { + return path; + } + + path = path.padEnd(32, "\0"); + const encodedMessage = new TextEncoder().encode(path); + + const ciphertext = await crypto.subtle.encrypt( + { + name: "AES-GCM", + iv: this.spaceSalt, + }, + key, + encodedMessage, + ); + const encodedPath = base32Encode(new Uint8Array(ciphertext)); + // console.log(new TextDecoder().decode(ciphertext)); + return encodedPath.slice(0, 3) + "/" + encodedPath.slice(3) + + encryptedFileExt; + } + + private async decryptPath( + key: CryptoKey, + encryptedPath: string, + ): Promise { + if (!this.spaceSalt) { + throw new Error("Space not initialized"); + } + if (this.isUnencryptedPath(encryptedPath)) { + return encryptedPath; + } + + if (!encryptedPath.endsWith(encryptedFileExt)) { + throw new Error("Invalid encrypted path"); + } + // Remove the extension and slashes + encryptedPath = encryptedPath.slice(0, -encryptedFileExt.length).replaceAll( + "/", + "", + ); + + // console.log("To decrypt", encryptedPath); + const decrypted = await crypto.subtle.decrypt( + { + name: "AES-GCM", + iv: this.spaceSalt, + }, + key, + base32Decode(encryptedPath), + ); + // Decode the buffer and remove the padding + return removePadding(new TextDecoder().decode(decrypted), "\0"); + } + + async fetchFileList(): Promise { + const files = await this.wrapped.fetchFileList(); + // console.log(files); + return Promise.all( + files.filter((fileMeta) => + fileMeta.name !== this.encryptedKeyFileName && + fileMeta.name !== saltFile + ) + .map(async (fileMeta) => { + return { + ...fileMeta, + name: await this.decryptPath(this.masterKey!, fileMeta.name), + }; + }), + ); + } + + async getFileMeta(name: string): Promise { + if (this.isUnencryptedPath(name)) { + return this.wrapped.getFileMeta(name); + } + const fileMeta = await this.wrapped.getFileMeta( + await this.encryptPath(this.masterKey!, name), + ); + return { + ...fileMeta, + name, + }; + } + + async readFile(name: string): Promise<{ data: Uint8Array; meta: FileMeta }> { + if (this.isUnencryptedPath(name)) { + return this.wrapped.readFile(name); + } + const { data, meta } = await this.wrapped.readFile( + await this.encryptPath(this.masterKey!, name), + ); + return { + data: await this.decryptAES(this.masterKey!, data), + meta: { + ...meta, + name, + }, + }; + } + + async writeFile( + name: string, + data: Uint8Array, + selfUpdate?: boolean | undefined, + meta?: FileMeta | undefined, + ): Promise { + if (this.isUnencryptedPath(name)) { + return this.wrapped.writeFile(name, data, selfUpdate, meta); + } + const newMeta = await this.wrapped.writeFile( + await this.encryptPath(this.masterKey!, name), + await this.encryptAES(this.masterKey!, data), + selfUpdate, + meta, + ); + return { + ...newMeta, + name, + }; + } + + async deleteFile(name: string): Promise { + if (this.isUnencryptedPath(name)) { + return this.wrapped.deleteFile(name); + } + return this.wrapped.deleteFile( + await this.encryptPath(this.masterKey!, name), + ); + } +} + +function removePadding(str: string, paddingChar: string): string { + // let startIndex = 0; + // while (startIndex < str.length && str[startIndex] === paddingChar) { + // startIndex++; + // } + // return str.substring(startIndex); + let endIndex = str.length - 1; + while (endIndex >= 0 && str[endIndex] === paddingChar) { + endIndex--; + } + return str.substring(0, endIndex + 1); +} + +function base32Encode(data: Uint8Array): string { + const alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; + let result = ""; + let bits = 0; + let value = 0; + for (const byte of data) { + value = (value << 8) | byte; + bits += 8; + while (bits >= 5) { + result += alphabet[(value >>> (bits - 5)) & 31]; + bits -= 5; + } + } + if (bits > 0) { + result += alphabet[(value << (5 - bits)) & 31]; + } + return result; +} + +function base32Decode(data: string): Uint8Array { + const alphabet = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567"; + const result = new Uint8Array(Math.floor(data.length * 5 / 8)); + let bits = 0; + let value = 0; + let index = 0; + for (const char of data) { + value = (value << 5) | alphabet.indexOf(char); + bits += 5; + if (bits >= 8) { + result[index++] = (value >>> (bits - 8)) & 255; + bits -= 8; + } + } + return result; +} + +function appendBuffer(buffer1: Uint8Array, buffer2: Uint8Array): Uint8Array { + const tmp = new Uint8Array(buffer1.byteLength + buffer2.byteLength); + tmp.set(new Uint8Array(buffer1), 0); + tmp.set(new Uint8Array(buffer2), buffer1.byteLength); + return tmp; +} diff --git a/common/spaces/space_primitives.test.ts b/common/spaces/space_primitives.test.ts index a78c96ab..0dfd2b1b 100644 --- a/common/spaces/space_primitives.test.ts +++ b/common/spaces/space_primitives.test.ts @@ -35,8 +35,8 @@ export async function testSpacePrimitives(spacePrimitives: SpacePrimitives) { buf.set([1, 2, 3, 4, 5]); // Write binary file await spacePrimitives.writeFile("test.bin", buf); - const fMeta = await spacePrimitives.getFileMeta("test.bin"); - assertEquals(fMeta.size, 1024 * 1024); + const fileData = await spacePrimitives.readFile("test.bin"); + assertEquals(fileData.data.length, 1024 * 1024); assertEquals((await spacePrimitives.fetchFileList()).length, 2); // console.log(spacePrimitives); diff --git a/common/util.ts b/common/util.ts index d85f906b..aefe5f86 100644 --- a/common/util.ts +++ b/common/util.ts @@ -1,6 +1,7 @@ import { SETTINGS_TEMPLATE } from "./settings_template.ts"; import { YAML } from "./deps.ts"; import { SpacePrimitives } from "./spaces/space_primitives.ts"; +import { expandPropertyNames } from "$sb/lib/json.ts"; export function safeRun(fn: () => Promise) { fn().catch((e) => { @@ -43,6 +44,7 @@ export async function ensureSettingsAndIndex( ); } catch (e: any) { if (e.message === "Not found") { + console.log("No settings found, creating default settings"); await space.writeFile( "SETTINGS.md", new TextEncoder().encode(SETTINGS_TEMPLATE), @@ -56,7 +58,11 @@ export async function ensureSettingsAndIndex( // Ok, then let's also check the index page try { await space.getFileMeta("index.md"); - } catch { + } catch (e: any) { + console.log( + "No index page found, creating default index page", + e.message, + ); await space.writeFile( "index.md", new TextEncoder().encode( @@ -71,5 +77,7 @@ page: "[[!silverbullet.md/Getting Started]]" } } - return parseYamlSettings(settingsText); + const settings = parseYamlSettings(settingsText); + expandPropertyNames(settings); + return settings; } diff --git a/server/http_server.ts b/server/http_server.ts index f5da61c0..1a343c91 100644 --- a/server/http_server.ts +++ b/server/http_server.ts @@ -14,6 +14,7 @@ import { SpaceServer, SpaceServerConfig } from "./instance.ts"; import { KvPrimitives } from "../plugos/lib/kv_primitives.ts"; import { EndpointHook } from "../plugos/hooks/endpoint.ts"; import { PrefixedKvPrimitives } from "../plugos/lib/prefixed_kv_primitives.ts"; +import { base64Encode } from "../plugos/asset_bundle/base64.ts"; const authenticationExpirySeconds = 60 * 60 * 24 * 7; // 1 week @@ -24,7 +25,6 @@ export type ServerOptions = { clientAssetBundle: AssetBundle; plugAssetBundle: AssetBundle; baseKvPrimitives: KvPrimitives; - syncOnly: boolean; certFile?: string; keyFile?: string; @@ -42,7 +42,6 @@ export class HttpServer { certFile: string | undefined; spaceServers = new Map>(); - syncOnly: boolean; baseKvPrimitives: KvPrimitives; configs: Map; @@ -54,7 +53,6 @@ export class HttpServer { this.app = options.app; this.keyFile = options.keyFile; this.certFile = options.certFile; - this.syncOnly = options.syncOnly; this.baseKvPrimitives = options.baseKvPrimitives; this.configs = options.configs; } @@ -67,7 +65,6 @@ export class HttpServer { new PrefixedKvPrimitives(this.baseKvPrimitives, [ config.namespace, ]), - this.syncOnly, ); await spaceServer.init(); @@ -114,15 +111,18 @@ export class HttpServer { } // Replaces some template variables in index.html in a rather ad-hoc manner, but YOLO - renderIndexHtml(pagesPath: string) { + renderIndexHtml(spaceServer: SpaceServer) { return this.clientAssetBundle.readTextFileSync(".client/index.html") .replaceAll( "{{SPACE_PATH}}", - pagesPath.replaceAll("\\", "\\\\"), + spaceServer.pagesPath.replaceAll("\\", "\\\\"), // ); ).replaceAll( "{{SYNC_ONLY}}", - this.syncOnly ? "true" : "false", + spaceServer.syncOnly ? "true" : "false", + ).replaceAll( + "{{CLIENT_ENCRYPTION}}", + spaceServer.clientEncryption ? "true" : "false", ); } @@ -149,7 +149,7 @@ export class HttpServer { response.headers.set("Content-type", "text/html"); response.headers.set("Cache-Control", "no-cache"); const spaceServer = await this.ensureSpaceServer(request); - response.body = this.renderIndexHtml(spaceServer.pagesPath); + response.body = this.renderIndexHtml(spaceServer); }); this.abortController = new AbortController(); @@ -181,6 +181,7 @@ export class HttpServer { { request, response }: Context, Record>, next: Next, ) { + const spaceServer = await this.ensureSpaceServer(request); if ( request.url.pathname === "/" ) { @@ -188,8 +189,7 @@ export class HttpServer { // Note: we're explicitly not setting Last-Modified and If-Modified-Since header here because this page is dynamic response.headers.set("Content-type", "text/html"); response.headers.set("Cache-Control", "no-cache"); - const spaceServer = await this.ensureSpaceServer(request); - response.body = this.renderIndexHtml(spaceServer.pagesPath); + response.body = this.renderIndexHtml(spaceServer); return; } try { @@ -197,7 +197,8 @@ export class HttpServer { if ( this.clientAssetBundle.has(assetName) && request.headers.get("If-Modified-Since") === - utcDateString(this.clientAssetBundle.getMtime(assetName)) + utcDateString(this.clientAssetBundle.getMtime(assetName)) && + assetName !== "service_worker.js" ) { response.status = 304; return; @@ -207,17 +208,34 @@ export class HttpServer { "Content-type", this.clientAssetBundle.getMimeType(assetName), ); - const data = this.clientAssetBundle.readFileSync( + let data: Uint8Array | string = this.clientAssetBundle.readFileSync( assetName, ); response.headers.set("Cache-Control", "no-cache"); response.headers.set("Content-length", "" + data.length); - response.headers.set( - "Last-Modified", - utcDateString(this.clientAssetBundle.getMtime(assetName)), - ); + if (assetName !== "service_worker.js") { + response.headers.set( + "Last-Modified", + utcDateString(this.clientAssetBundle.getMtime(assetName)), + ); + } if (request.method === "GET") { + if (assetName === "service_worker.js") { + const textData = new TextDecoder().decode(data); + // console.log( + // "Swapping out config hash in service worker", + // ); + data = textData.replaceAll( + "{{CONFIG_HASH}}", + base64Encode( + JSON.stringify([ + spaceServer.clientEncryption, + spaceServer.syncOnly, + ]), + ), + ); + } response.body = data; } } catch { @@ -384,7 +402,7 @@ export class HttpServer { return; } case "syscall": { - if (this.syncOnly) { + if (spaceServer.syncOnly) { response.headers.set("Content-Type", "text/plain"); response.status = 400; response.body = "Unknown operation"; diff --git a/server/instance.ts b/server/instance.ts index 295062c6..c73f75fb 100644 --- a/server/instance.ts +++ b/server/instance.ts @@ -21,6 +21,8 @@ export type SpaceServerConfig = { // Additional API auth token authToken?: string; pagesPath: string; + syncOnly?: boolean; + clientEncryption?: boolean; }; export class SpaceServer { @@ -37,18 +39,26 @@ export class SpaceServer { // Only set when syncOnly == false private serverSystem?: ServerSystem; system?: System; + clientEncryption: boolean; + syncOnly: boolean; constructor( config: SpaceServerConfig, public shellBackend: ShellBackend, private plugAssetBundle: AssetBundle, private kvPrimitives: KvPrimitives, - private syncOnly: boolean, ) { this.pagesPath = config.pagesPath; this.hostname = config.hostname; this.auth = config.auth; this.authToken = config.authToken; + this.clientEncryption = !!config.clientEncryption; + this.syncOnly = !!config.syncOnly; + if (this.clientEncryption) { + // Sync only will forced on when encryption is enabled + this.syncOnly = true; + } + this.jwtIssuer = new JWTIssuer(kvPrimitives); } @@ -100,7 +110,13 @@ export class SpaceServer { } async reloadSettings() { - // TODO: Throttle this? - this.settings = await ensureSettingsAndIndex(this.spacePrimitives); + if (!this.clientEncryption) { + // Only attempt this when the space is not encrypted + this.settings = await ensureSettingsAndIndex(this.spacePrimitives); + } else { + this.settings = { + indexPage: "index", + }; + } } } diff --git a/silverbullet.ts b/silverbullet.ts index 25a218e3..35b37249 100755 --- a/silverbullet.ts +++ b/silverbullet.ts @@ -41,6 +41,10 @@ await new Command() "--sync-only", "Run the server as a pure space (file) store only without any backend processing (this disables 'online mode' in the client)", ) + .option( + "--client-encryption", + "Enable client-side encryption for spaces", + ) .option( "--reindex", "Reindex space on startup", diff --git a/web/client.ts b/web/client.ts index 607cf7fe..2ca7c49e 100644 --- a/web/client.ts +++ b/web/client.ts @@ -9,7 +9,7 @@ import { } from "../common/deps.ts"; import { fileMetaToPageMeta, Space } from "./space.ts"; import { FilterOption } from "./types.ts"; -import { parseYamlSettings } from "../common/util.ts"; +import { ensureSettingsAndIndex } from "../common/util.ts"; import { EventHook } from "../plugos/hooks/event.ts"; import { AppCommand } from "./hooks/command.ts"; import { PathPageNavigator } from "./navigator.ts"; @@ -37,13 +37,16 @@ import { createEditorState } from "./editor_state.ts"; import { OpenPages } from "./open_pages.ts"; import { MainUI } from "./editor_ui.tsx"; import { cleanPageRef } from "$sb/lib/resolve.ts"; -import { expandPropertyNames } from "$sb/lib/json.ts"; import { SpacePrimitives } from "../common/spaces/space_primitives.ts"; import { FileMeta, PageMeta } from "$sb/types.ts"; import { DataStore } from "../plugos/lib/datastore.ts"; import { IndexedDBKvPrimitives } from "../plugos/lib/indexeddb_kv_primitives.ts"; import { DataStoreMQ } from "../plugos/lib/mq.datastore.ts"; import { DataStoreSpacePrimitives } from "../common/spaces/datastore_space_primitives.ts"; +import { + encryptedFileExt, + EncryptedSpacePrimitives, +} from "../common/spaces/encrypted_space_primitives.ts"; const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/; const autoSaveInterval = 1000; @@ -54,6 +57,7 @@ declare global { silverBulletConfig: { spaceFolderPath: string; syncOnly: boolean; + clientEncryption: boolean; }; client: Client; } @@ -69,7 +73,7 @@ export class Client { plugSpaceRemotePrimitives!: PlugSpacePrimitives; // localSpacePrimitives!: FilteredSpacePrimitives; - remoteSpacePrimitives!: HttpSpacePrimitives; + httpSpacePrimitives!: HttpSpacePrimitives; space!: Space; saveTimeout?: number; @@ -177,15 +181,14 @@ export class Client { this.focus(); - // This constructor will always be followed by an (async) invocatition of init() await this.system.init(); // Load settings - this.settings = await this.loadSettings(); + this.settings = await ensureSettingsAndIndex(localSpacePrimitives); // Pinging a remote space to ensure we're authenticated properly, if not will result in a redirect to auth page try { - await this.remoteSpacePrimitives.ping(); + await this.httpSpacePrimitives.ping(); } catch (e: any) { if (e.message === "Not authenticated") { console.warn("Not authenticated, redirecting to auth page"); @@ -346,13 +349,83 @@ export class Client { } async initSpace(): Promise { - this.remoteSpacePrimitives = new HttpSpacePrimitives( + this.httpSpacePrimitives = new HttpSpacePrimitives( location.origin, window.silverBulletConfig.spaceFolderPath, ); + let remoteSpacePrimitives: SpacePrimitives = this.httpSpacePrimitives; + + if (window.silverBulletConfig.clientEncryption) { + console.log("Enabling encryption"); + + const encryptedSpacePrimitives = new EncryptedSpacePrimitives( + this.httpSpacePrimitives, + ); + remoteSpacePrimitives = encryptedSpacePrimitives; + let loggedIn = false; + // First figure out if we're online & if the key file exists, if not we need to initialize the space + try { + if (!await encryptedSpacePrimitives.init()) { + console.log( + "Space not initialized, will ask for password to initialize", + ); + alert( + "You appear to be accessing a new space with encryption enabled, you will now be asked to create a password", + ); + const password = prompt("Choose a password"); + if (!password) { + alert("Cannot do anything without a password, reloading"); + location.reload(); + throw new Error("Not initialized"); + } + const password2 = prompt("Confirm password"); + if (password !== password2) { + alert("Passwords don't match, reloading"); + location.reload(); + throw new Error("Not initialized"); + } + await encryptedSpacePrimitives.setup(password); + // this.stateDataStore.set(["encryptionKey"], password); + await this.stateDataStore.set( + ["spaceSalt"], + encryptedSpacePrimitives.spaceSalt, + ); + loggedIn = true; + } + } catch (e: any) { + if (e.message === "Offline") { + console.log( + "Offline, will assume encryption space is initialized, fetching salt from data store", + ); + await encryptedSpacePrimitives.init( + await this.stateDataStore.get(["spaceSalt"]), + ); + } + } + if (!loggedIn) { + // Let's ask for the password + try { + await encryptedSpacePrimitives.login( + prompt("Password")!, + ); + await this.stateDataStore.set( + ["spaceSalt"], + encryptedSpacePrimitives.spaceSalt, + ); + } catch (e: any) { + console.log("Got this error", e); + if (e.message === "Incorrect password") { + alert("Incorrect password"); + location.reload(); + } + throw e; + } + } + } + this.plugSpaceRemotePrimitives = new PlugSpacePrimitives( - this.remoteSpacePrimitives, + remoteSpacePrimitives, this.system.namespaceHook, this.syncMode ? undefined : "client", ); @@ -380,7 +453,10 @@ export class Client { (meta) => fileFilterFn(meta.name), // Run when a list of files has been retrieved async () => { - await this.loadSettings(); + if (!this.settings) { + this.settings = await ensureSettingsAndIndex(localSpacePrimitives!); + } + if (typeof this.settings?.spaceIgnore === "string") { fileFilterFn = gitIgnoreCompiler(this.settings.spaceIgnore).accepts; } else { @@ -439,27 +515,6 @@ export class Client { return localSpacePrimitives; } - async loadSettings(): Promise { - let settingsText: string | undefined; - - try { - settingsText = (await this.space.readPage("SETTINGS")).text; - } catch (e: any) { - console.info("No SETTINGS page, falling back to default", e); - settingsText = '```yaml\nindexPage: "[[index]]"\n```\n'; - } - let settings = parseYamlSettings(settingsText!) as BuiltinSettings; - - settings = expandPropertyNames(settings); - - // console.log("Settings", settings); - - if (!settings.indexPage) { - settings.indexPage = "[[index]]"; - } - return settings; - } - get currentPage(): string | undefined { return this.ui.viewState.currentPage; } diff --git a/web/editor_state.ts b/web/editor_state.ts index 352cb0d9..af6477f1 100644 --- a/web/editor_state.ts +++ b/web/editor_state.ts @@ -166,6 +166,7 @@ export function createEditorState( { selector: "FrontMatter", class: "sb-frontmatter" }, ]), keymap.of([ + ...commandKeyBindings, ...smartQuoteKeymap, ...closeBracketsKeymap, ...standardKeymap, @@ -173,7 +174,6 @@ export function createEditorState( ...historyKeymap, ...completionKeymap, indentWithTab, - ...commandKeyBindings, { key: "Ctrl-k", mac: "Cmd-k", diff --git a/web/index.html b/web/index.html index 4c27b196..787463c3 100644 --- a/web/index.html +++ b/web/index.html @@ -39,12 +39,14 @@ // These {{VARIABLES}} are replaced by http_server.ts spaceFolderPath: "{{SPACE_PATH}}", syncOnly: "{{SYNC_ONLY}}" === "true", + clientEncryption: "{{CLIENT_ENCRYPTION}}" === "true", }; // But in case these variables aren't replaced by the server, fall back sync only mode if (window.silverBulletConfig.spaceFolderPath.includes("{{")) { window.silverBulletConfig = { spaceFolderPath: "", syncOnly: true, + clientEncryption: false, }; } diff --git a/web/service_worker.ts b/web/service_worker.ts index da538340..05e39c6b 100644 --- a/web/service_worker.ts +++ b/web/service_worker.ts @@ -3,7 +3,7 @@ import { simpleHash } from "../common/crypto.ts"; import { DataStore } from "../plugos/lib/datastore.ts"; import { IndexedDBKvPrimitives } from "../plugos/lib/indexeddb_kv_primitives.ts"; -const CACHE_NAME = "{{CACHE_NAME}}"; +const CACHE_NAME = "{{CACHE_NAME}}_{{CONFIG_HASH}}"; const precacheFiles = Object.fromEntries([ "/", @@ -61,7 +61,6 @@ self.addEventListener("activate", (event: any) => { }); let ds: DataStore | undefined; -const filesMetaPrefix = ["file", "meta"]; const filesContentPrefix = ["file", "content"]; self.addEventListener("fetch", (event: any) => { diff --git a/web/syscalls/fetch.ts b/web/syscalls/fetch.ts index ec036e5a..745585bb 100644 --- a/web/syscalls/fetch.ts +++ b/web/syscalls/fetch.ts @@ -28,15 +28,15 @@ export function sandboxFetchSyscalls( body: options.base64Body && base64Decode(options.base64Body), } : {}; - if (!client.remoteSpacePrimitives) { + if (!client.httpSpacePrimitives) { // No SB server to proxy the fetch available so let's execute the request directly return performLocalFetch(url, fetchOptions); } fetchOptions.headers = fetchOptions.headers ? { ...fetchOptions.headers, "X-Proxy-Request": "true" } : { "X-Proxy-Request": "true" }; - const resp = await client.remoteSpacePrimitives.authenticatedFetch( - `${client.remoteSpacePrimitives.url}/!${url}`, + const resp = await client.httpSpacePrimitives.authenticatedFetch( + `${client.httpSpacePrimitives.url}/!${url}`, fetchOptions, ); const body = await resp.arrayBuffer(); diff --git a/web/syscalls/shell.ts b/web/syscalls/shell.ts index 700c9dab..6585c0ae 100644 --- a/web/syscalls/shell.ts +++ b/web/syscalls/shell.ts @@ -10,11 +10,11 @@ export function shellSyscalls( cmd: string, args: string[], ): Promise<{ stdout: string; stderr: string; code: number }> => { - if (!client.remoteSpacePrimitives) { + if (!client.httpSpacePrimitives) { throw new Error("Not supported in fully local mode"); } - const resp = client.remoteSpacePrimitives.authenticatedFetch( - `${client.remoteSpacePrimitives.url}/.rpc`, + const resp = client.httpSpacePrimitives.authenticatedFetch( + `${client.httpSpacePrimitives.url}/.rpc`, { method: "POST", body: JSON.stringify({ diff --git a/web/syscalls/system.ts b/web/syscalls/system.ts index d32fa0c1..1a50e7e4 100644 --- a/web/syscalls/system.ts +++ b/web/syscalls/system.ts @@ -43,7 +43,7 @@ export function systemSyscalls( // Proxy to another environment return proxySyscall( ctx, - client.remoteSpacePrimitives, + client.httpSpacePrimitives, "system.invokeFunction", [fullName, ...args], ); diff --git a/web/syscalls/util.ts b/web/syscalls/util.ts index 38b5f265..26a4ce94 100644 --- a/web/syscalls/util.ts +++ b/web/syscalls/util.ts @@ -7,7 +7,7 @@ export function proxySyscalls(client: Client, names: string[]): SysCallMapping { const syscalls: SysCallMapping = {}; for (const name of names) { syscalls[name] = (ctx, ...args: any[]) => { - return proxySyscall(ctx, client.remoteSpacePrimitives, name, args); + return proxySyscall(ctx, client.httpSpacePrimitives, name, args); }; } return syscalls;