mingzailao
/
silverbullet
mirror of https://github.com/silverbulletmd/silverbullet.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Migrate to express

deno-express
Zef Hemel 2023-06-03 15:58:38 +02:00
parent eebf920a9d
commit 36acf29591
2 changed files with 239 additions and 299 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
server/collab.ts
View File

@ -1,5 +1,4 @@
import { Hocuspocus } from "npm:@hocuspocus/server@2.0.6"; import { Hocuspocus } from "npm:@hocuspocus/server@2.0.6";
import { getAvailablePortSync } from "https://deno.land/x/port@1.0.0/mod.ts";
import { nanoid } from "https://esm.sh/nanoid@4.0.0"; import { nanoid } from "https://esm.sh/nanoid@4.0.0";
import { race, timeout } from "../common/async_util.ts"; import { race, timeout } from "../common/async_util.ts";
import { Application } from "./deps.ts"; import { Application } from "./deps.ts";
@ -108,13 +107,8 @@ export class CollabServer {
} }
} }
route(app: Application) { route(app: Express.Application) {
// The way this works is that we spin up a separate WS server locally and then proxy requests to it
// This is the only way I could get Hocuspocus to work with Deno
const internalPort = getAvailablePortSync();
const hocuspocus = new Hocuspocus({ const hocuspocus = new Hocuspocus({
port: internalPort,
address: "127.0.0.1",
quiet: true, quiet: true,
onLoadDocument: async (doc) => { onLoadDocument: async (doc) => {
console.log("[Hocuspocus]", "Requesting doc load", doc.documentName); console.log("[Hocuspocus]", "Requesting doc load", doc.documentName);
@ -185,55 +179,10 @@ export class CollabServer {
}, },
}); });
hocuspocus.listen(); // hocuspocus.listen();
app.ws("/.ws-collab", (ws, req) => {
app.use((ctx) => { hocuspocus.handleConnection(ws, req);
if (ctx.request.url.pathname === "/.ws") {
const sock = ctx.upgrade();
sock.onmessage = (e) => {
console.log("WS: Got message", e.data);
};
}
// Websocket proxy to hocuspocus
if (ctx.request.url.pathname === "/.ws-collab") {
const sock = ctx.upgrade();
const ws = new WebSocket(`ws://localhost:${internalPort}`);
const wsReady = race([
new Promise<void>((resolve) => {
ws.onopen = () => {
resolve();
};
}),
timeout(1000),
]).catch(() => {
console.error("Timeout waiting for collab to open websocket");
sock.close();
});
sock.onmessage = (e) => {
// console.log("Got message", e);
wsReady.then(() => ws.send(e.data)).catch(console.error);
};
sock.onclose = () => {
if (ws.OPEN) {
ws.close();
}
};
ws.onmessage = (e) => {
if (sock.OPEN) {
sock.send(e.data);
} else {
console.error("Got message from websocket but socket is not open");
}
};
ws.onclose = () => {
if (sock.OPEN) {
sock.close();
}
};
}
}); });
}
} }
function splitCollabId(documentName: string): [string, string] { function splitCollabId(documentName: string): [string, string] {

479
server/http_server.ts
View File

@ -1,7 +1,6 @@
import { Application, Router } from "./deps.ts"; // import { Application, Router } from "./deps.ts";
import { SpacePrimitives } from "../common/spaces/space_primitives.ts"; import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
import { AssetBundle } from "../plugos/asset_bundle/bundle.ts"; import { AssetBundle } from "../plugos/asset_bundle/bundle.ts";
import { base64Decode } from "../plugos/asset_bundle/base64.ts";
import { ensureSettingsAndIndex } from "../common/util.ts"; import { ensureSettingsAndIndex } from "../common/util.ts";
import { performLocalFetch } from "../common/proxy_fetch.ts"; import { performLocalFetch } from "../common/proxy_fetch.ts";
import { BuiltinSettings } from "../web/types.ts"; import { BuiltinSettings } from "../web/types.ts";
@ -9,6 +8,17 @@ import { gitIgnoreCompiler } from "./deps.ts";
import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts"; import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts";
import { CollabServer } from "./collab.ts"; import { CollabServer } from "./collab.ts";
// @deno-types="npm:@types/express@4.17.15"
import express from "npm:express@4.18.2";
import cookieParser from "npm:cookie-parser";
// @deno-types="npm:@types/express-ws"
import expressWebsockets from "npm:express-ws@5.0.2";
import { Buffer } from "node:buffer";
// @deno-types="npm:body-parser@1.19.2"
import bodyParser from "npm:body-parser@1.19.2";
export type ServerOptions = { export type ServerOptions = {
hostname: string; hostname: string;
port: number; port: number;
@ -22,7 +32,7 @@ export type ServerOptions = {
}; };
export class HttpServer { export class HttpServer {
app: Application; private app: express.Express;
private hostname: string; private hostname: string;
private port: number; private port: number;
user?: string; user?: string;
@ -38,7 +48,7 @@ export class HttpServer {
) { ) {
this.hostname = options.hostname; this.hostname = options.hostname;
this.port = options.port; this.port = options.port;
this.app = new Application(); this.app = expressWebsockets(express()).app;
this.user = options.user; this.user = options.user;
this.clientAssetBundle = options.clientAssetBundle; this.clientAssetBundle = options.clientAssetBundle;
@ -82,92 +92,72 @@ export class HttpServer {
async start() { async start() {
await this.reloadSettings(); await this.reloadSettings();
// Serve static files (javascript, css, html) this.app.use(cookieParser());
this.app.use(async ({ request, response }, next) => { this.app.all(/^(\/((?!\.fs).)*)$/, (req, res) => {
if (request.url.pathname === "/") { // console.log(req.path, req.method);
// Note: we're explicitly not setting Last-Modified and If-Modified-Since header here because this page is dynamic
response.headers.set("Content-type", "text/html");
response.body = this.renderIndexHtml();
return;
}
try { try {
const assetName = request.url.pathname.slice(1); const assetName = req.path.slice(1);
if ( if (this.clientAssetBundle.has(assetName)) {
this.clientAssetBundle.has(assetName) && console.log("Serving asset", assetName);
request.headers.get("If-Modified-Since") === if (
utcDateString(this.clientAssetBundle.getMtime(assetName)) req.header("If-Modified-Since") ===
) { utcDateString(this.clientAssetBundle.getMtime(assetName))
response.status = 304; ) {
res.status(304);
res.send();
return;
}
res.status(200);
res.header(
"Content-type",
this.clientAssetBundle.getMimeType(assetName),
);
const data = this.clientAssetBundle.readFileSync(
assetName,
);
res.header("Cache-Control", "no-cache");
// res.header("Content-Length", "" + data.length);
res.header(
"Last-Modified",
utcDateString(this.clientAssetBundle.getMtime(assetName)),
);
if (req.method === "GET") {
const buf = Buffer.from(data);
console.log("Buffer length", buf.length, assetName);
res.send(buf);
// res.send
} else {
res.send();
}
return; return;
} }
response.status = 200; } catch (e: any) {
response.headers.set( console.error("Error", e);
"Content-type",
this.clientAssetBundle.getMimeType(assetName),
);
const data = this.clientAssetBundle.readFileSync(
assetName,
);
response.headers.set("Cache-Control", "no-cache");
response.headers.set("Content-length", "" + data.length);
response.headers.set(
"Last-Modified",
utcDateString(this.clientAssetBundle.getMtime(assetName)),
);
if (request.method === "GET") {
response.body = data;
}
} catch {
await next();
} }
});
// Fallback, serve index.html res.status(200);
this.app.use(({ request, response }, next) => { res.header("Content-Type", "text/html");
if ( res.send(this.renderIndexHtml());
!request.url.pathname.startsWith("/.fs") &&
request.url.pathname !== "/.auth" &&
!request.url.pathname.startsWith("/.ws")
) {
response.headers.set("Content-type", "text/html");
response.body = this.renderIndexHtml();
} else {
return next();
}
}); });
// Pages API // Pages API
const fsRouter = this.buildFsRouter(this.spacePrimitives); this.app.use(
this.addPasswordAuth(this.app); "/.fs",
this.app.use(fsRouter.routes()); // passwordMiddleware,
this.app.use(fsRouter.allowedMethods()); this.buildFsRouter(this.spacePrimitives),
this.collab.route(this.app);
this.abortController = new AbortController();
const listenOptions: any = {
hostname: this.hostname,
port: this.port,
signal: this.abortController.signal,
};
if (this.options.keyFile) {
listenOptions.key = Deno.readTextFileSync(this.options.keyFile);
}
if (this.options.certFile) {
listenOptions.cert = Deno.readTextFileSync(this.options.certFile);
}
this.app.listen(listenOptions)
.catch((e: any) => {
console.log("Server listen error:", e.message);
Deno.exit(1);
});
const visibleHostname = this.hostname === "0.0.0.0"
? "localhost"
: this.hostname;
console.log(
`SilverBullet is now running: http://${visibleHostname}:${this.port}`,
); );
this.app.listen(this.port, this.hostname, () => {
const visibleHostname = this.hostname === "0.0.0.0"
? "localhost"
: this.hostname;
console.log(
`SilverBullet is now running: http://${visibleHostname}:${this.port}`,
);
});
// return;
// this.collab.route(this.app);
} }
async reloadSettings() { async reloadSettings() {
@ -175,7 +165,11 @@ export class HttpServer {
this.settings = await ensureSettingsAndIndex(this.spacePrimitives); this.settings = await ensureSettingsAndIndex(this.spacePrimitives);
} }
private addPasswordAuth(app: Application) { private passwordAuth(
req: express.Request,
res: express.Response,
next: express.NextFunction,
) {
const excludedPaths = [ const excludedPaths = [
"/manifest.json", "/manifest.json",
"/favicon.png", "/favicon.png",
@ -184,127 +178,130 @@ export class HttpServer {
]; ];
if (this.user) { if (this.user) {
const b64User = btoa(this.user); const b64User = btoa(this.user);
app.use(async ({ request, response, cookies }, next) => { if (!excludedPaths.includes(req.path)) {
if (!excludedPaths.includes(request.url.pathname)) { const authCookie = req.cookies["auth"];
const authCookie = await cookies.get("auth"); if (!authCookie || authCookie !== b64User) {
if (!authCookie || authCookie !== b64User) { res.status(401);
response.status = 401; res.send("Unauthorized, please authenticate");
response.body = "Unauthorized, please authenticate"; return;
return;
}
} }
if (request.url.pathname === "/.auth") { }
if (request.method === "GET") { if (req.path === "/.auth") {
response.headers.set("Content-type", "text/html"); if (req.method === "GET") {
response.body = this.clientAssetBundle.readTextFileSync( res.header("Content-type", "text/html");
".client/auth.html", res.send(this.clientAssetBundle.readTextFileSync(
); ".client/auth.html",
return; ));
} else if (request.method === "POST") { return;
const values = await request.body({ type: "form" }).value; } else if (req.method === "POST") {
const username = values.get("username"), bodyParser.urlencoded({ extended: false })(req, res, next);
password = values.get("password"), // const values = req.body;
refer = values.get("refer"); const username = req.body.username,
if (this.user === `${username}:${password}`) { password = req.body.password,
await cookies.set("auth", b64User, { refer = req.body.refer;
expires: new Date(Date.now() + 1000 * 60 * 60 * 24 * 7), // in a week if (this.user === `${username}:${password}`) {
sameSite: "strict", res.cookie("auth", b64User, {
}); expires: new Date(Date.now() + 1000 * 60 * 60 * 24 * 7), // in a week
response.redirect(refer || "/"); sameSite: "strict",
// console.log("All headers", request.headers); });
} else { res.redirect(refer || "/");
response.redirect("/.auth?error=1"); // console.log("All headers", request.headers);
}
return;
} else { } else {
response.status = 401; res.redirect("/.auth?error=1");
response.body = "Unauthorized";
return;
} }
return;
} else { } else {
// Unauthenticated access to excluded paths res.status(401);
await next(); res.send("Unauthorized");
return;
} }
}); } else {
// Unauthenticated access to excluded paths
next();
}
} }
} }
private buildFsRouter(spacePrimitives: SpacePrimitives): Router { private buildFsRouter(spacePrimitives: SpacePrimitives): express.Router {
const fsRouter = new Router(); const fsRouter = express.Router();
// File list // File list
fsRouter.get("/", async ({ response }) => { fsRouter.route("/").get(async (_req, res) => {
response.headers.set("Content-type", "application/json"); res.header("X-Space-Path", this.options.pagesPath);
response.headers.set("X-Space-Path", this.options.pagesPath); res.json(await spacePrimitives.fetchFileList());
const files = await spacePrimitives.fetchFileList(); }).post(
response.body = JSON.stringify(files); bodyParser.json({
}); type: "*/*",
}),
// RPC async (req, res) => {
fsRouter.post("/", async ({ request, response }) => { // console.log("RPC", req.body);
const body = await request.body({ type: "json" }).value; const body = req.body;
try { try {
switch (body.operation) { switch (body.operation) {
case "fetch": { case "fetch": {
const result = await performLocalFetch(body.url, body.options); const result = await performLocalFetch(body.url, body.options);
console.log("Proxying fetch request to", body.url); console.log("Proxying fetch request to", body.url);
response.headers.set("Content-Type", "application/json"); res.header("Content-Type", "application/json");
response.body = JSON.stringify(result); res.send(JSON.stringify(result));
return;
}
case "shell": {
// TODO: Have a nicer way to do this
if (this.options.pagesPath.startsWith("s3://")) {
response.status = 500;
response.body = JSON.stringify({
stdout: "",
stderr: "Cannot run shell commands with S3 backend",
code: 500,
});
return; return;
} }
const p = new Deno.Command(body.cmd, { case "shell": {
args: body.args, // TODO: Have a nicer way to do this
cwd: this.options.pagesPath, if (this.options.pagesPath.startsWith("s3://")) {
stdout: "piped", res.status(500);
stderr: "piped", res.header("Content-Type", "application/json");
}); res.send(JSON.stringify({
const output = await p.output(); stdout: "",
const stdout = new TextDecoder().decode(output.stdout); stderr: "Cannot run shell commands with S3 backend",
const stderr = new TextDecoder().decode(output.stderr); code: 500,
}));
return;
}
const p = new Deno.Command(body.cmd, {
args: body.args,
cwd: this.options.pagesPath,
stdout: "piped",
stderr: "piped",
});
const output = await p.output();
const stdout = new TextDecoder().decode(output.stdout);
const stderr = new TextDecoder().decode(output.stderr);
response.headers.set("Content-Type", "application/json"); res.header("Content-Type", "application/json");
response.body = JSON.stringify({ res.send(JSON.stringify({
stdout, stdout,
stderr, stderr,
code: output.code, code: output.code,
}); }));
return; return;
}
case "ping": {
// RPC to check (for collab purposes) which client has what page open
res.header("Content-Type", "application/json");
// console.log("Got ping", body);
res.send(JSON.stringify(
this.collab.ping(body.clientId, body.page),
));
return;
}
default:
res.header("Content-Type", "text/plain");
res.status(400);
res.send("Unknown operation");
} }
case "ping": { } catch (e: any) {
// RPC to check (for collab purposes) which client has what page open console.log("Error", e);
response.headers.set("Content-Type", "application/json"); res.status(500);
// console.log("Got ping", body); res.send(e.message);
response.body = JSON.stringify( return;
this.collab.ping(body.clientId, body.page),
);
return;
}
default:
response.headers.set("Content-Type", "text/plain");
response.status = 400;
response.body = "Unknown operation";
} }
} catch (e: any) { },
console.log("Error", e); );
response.status = 500;
response.body = e.message;
return;
}
});
fsRouter fsRouter
.get("\/(.+)", async ({ params, response, request }) => { .route(/\/(.+)/)
const name = params[0]; .get(async (req, res) => {
const name = req.params[0];
console.log("Loading file", name); console.log("Loading file", name);
try { try {
const attachmentData = await spacePrimitives.readFile( const attachmentData = await spacePrimitives.readFile(
@ -312,89 +309,83 @@ export class HttpServer {
); );
const lastModifiedHeader = new Date(attachmentData.meta.lastModified) const lastModifiedHeader = new Date(attachmentData.meta.lastModified)
.toUTCString(); .toUTCString();
if (request.headers.get("If-Modified-Since") === lastModifiedHeader) { if (req.header("If-Modified-Since") === lastModifiedHeader) {
response.status = 304; res.status(304);
res.end();
return; return;
} }
response.status = 200; res.status(200);
response.headers.set( res.header(
"X-Last-Modified", "X-Last-Modified",
"" + attachmentData.meta.lastModified, "" + attachmentData.meta.lastModified,
); );
response.headers.set("Cache-Control", "no-cache"); res.header("Cache-Control", "no-cache");
response.headers.set("X-Permission", attachmentData.meta.perm); res.header("X-Permission", attachmentData.meta.perm);
response.headers.set( res.header(
"Last-Modified", "Last-Modified",
lastModifiedHeader, lastModifiedHeader,
); );
response.headers.set("Content-Type", attachmentData.meta.contentType); res.header("Content-Type", attachmentData.meta.contentType);
response.body = attachmentData.data; res.send(Buffer.from(attachmentData.data));
} catch { } catch {
// console.error("Error in main router", e); // console.error("Error in main router", e);
response.status = 404; res.status(404);
response.body = ""; res.end();
} }
}) })
.put("\/(.+)", async ({ request, response, params }) => { .put(
const name = params[0]; bodyParser.raw({ type: "*/*", limit: "100mb" }),
console.log("Saving file", name); async (req, res) => {
const name = req.params[0];
console.log("Saving file", name);
let body: Uint8Array; try {
if ( const meta = await spacePrimitives.writeFile(
request.headers.get("X-Content-Base64") name,
) { new Uint8Array(req.body as ArrayBuffer),
const content = await request.body({ type: "text" }).value; );
body = base64Decode(content); res.status(200);
} else { res.header("Content-Type", meta.contentType);
body = await request.body({ type: "bytes" }).value; res.header("X-Last-Modified", "" + meta.lastModified);
} res.header("X-Content-Length", "" + meta.size);
res.header("X-Permission", meta.perm);
try { res.send("OK");
const meta = await spacePrimitives.writeFile( } catch (err) {
name, res.status(500);
body, res.send("Write failed");
); console.error("Pipeline failed", err);
response.status = 200; }
response.headers.set("Content-Type", meta.contentType); },
response.headers.set("X-Last-Modified", "" + meta.lastModified); )
response.headers.set("X-Content-Length", "" + meta.size); .options(async (req, res) => {
response.headers.set("X-Permission", meta.perm); const name = req.params[0];
response.body = "OK";
} catch (err) {
response.status = 500;
response.body = "Write failed";
console.error("Pipeline failed", err);
}
})
.options("\/(.+)", async ({ response, params }) => {
const name = params[0];
try { try {
const meta = await spacePrimitives.getFileMeta(name); const meta = await spacePrimitives.getFileMeta(name);
response.status = 200; res.status(200);
response.headers.set("Content-Type", meta.contentType); res.header("Content-Type", meta.contentType);
response.headers.set("X-Last-Modified", "" + meta.lastModified); res.header("X-Last-Modified", "" + meta.lastModified);
response.headers.set("X-Content-Length", "" + meta.size); res.header("X-Content-Length", "" + meta.size);
response.headers.set("X-Permission", meta.perm); res.header("X-Permission", meta.perm);
} catch { } catch {
response.status = 404; res.status(404);
response.body = "Not found"; res.send("Not found");
// console.error("Options failed", err); // console.error("Options failed", err);
} }
}) })
.delete("\/(.+)", async ({ response, params }) => { .delete(async (req, res) => {
const name = params[0]; const name = req.params[0];
console.log("Deleting file", name); console.log("Deleting file", name);
try { try {
await spacePrimitives.deleteFile(name); await spacePrimitives.deleteFile(name);
response.status = 200; res.status(200);
response.body = "OK"; res.send("OK");
} catch (e: any) { } catch (e: any) {
console.error("Error deleting attachment", e); console.error("Error deleting attachment", e);
response.status = 200; res.status(200);
response.body = e.message; res.send(e.message);
} }
}); });
return new Router().use("/.fs", fsRouter.routes()); return fsRouter;
} }
stop() { stop() {