mingzailao
/
silverbullet
mirror of https://github.com/silverbulletmd/silverbullet.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Pre plug format change

pull/3/head
Zef Hemel 2022-03-27 09:55:29 +02:00
parent 08e6c3bad8
commit a382ab3baa
13 changed files with 222 additions and 110 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
package.json
View File

@ -1,21 +1,26 @@
{
"name": "silverbullet",
"name": "@zefhemel/silverbullet",
"version": "0.0.1",
"license": "MIT",
"resolutions": {
"@lezer/common": "https://github.com/zefhemel/common.git#046c880d1fcab713cadad327a5b7d8bb5de6522c"
},
"bin": {
"plugbox-bundle": "./dist/bundler/plugbox-bundle.js"
"plugbox-bundle": "./dist/plugbox/plugbox-bundle.js",
"plugbox-server": "./dist/plugbox/plugbox-server.js",
"silverbullet": "./dist/server/server.js"
},
"scripts": {
"watch": "rm -rf .parcel-cache && parcel watch",
"build": "parcel build",
"clean": "rm -rf dist",
"plugs": "node dist/bundler/plugbox-bundle.js plugs/core/core.plug.json plugs/dist/core.plug.json && node dist/bundler/plugbox-bundle.js plugs/git/git.plug.json plugs/dist/git.plug.json",
"plugs": "plugbox-bundle -w --dist plugs/dist plugs/core/core.plug.json plugs/git/git.plug.json",
"server": "nodemon -w dist/server dist/server/server.js pages",
"test": "jest"
},
"files": [
"dist"
],
"targets": {
"webapp": {
"source": [
@ -31,8 +36,11 @@
"isLibrary": true,
"context": "node"
},
"bundler": {
"source": "plugbox/bin/plugbox-bundle.ts",
"plugbox": {
"source": [
"plugbox/bin/plugbox-bundle.ts",
"plugbox/bin/plugbox-server.ts"
],
"outputFormat": "commonjs",
"isLibrary": true,
"context": "node"
@ -80,7 +88,7 @@
"knex": "^1.0.4",
"lodash": "^4.17.21",
"node-cron": "^3.0.0",
"node-fetch": "^3.2.3",
"node-fetch": "2",
"nodemon": "^2.0.15",
"parcel": "^2.3.2",
"react": "^17.0.2",
@ -90,6 +98,7 @@
"supertest": "^6.2.2",
"ts-jest": "^27.1.3",
"vm2": "^3.9.9",
"yaml": "^1.10.2",
"yargs": "^17.3.1"
},
"devDependencies": {
@ -107,6 +116,7 @@
"@types/react": "^17.0.39",
"@types/react-dom": "^17.0.11",
"@types/supertest": "^2.0.11",
"@types/yaml": "^1.9.7",
"@vscode/sqlite3": "^5.0.7",
"assert": "^2.0.0",
"crypto-browserify": "^3.12.0",

46
plugbox/bin/plugbox-bundle.ts
View File

@ -7,6 +7,7 @@ import path from "path";
import yargs from "yargs";
import { hideBin } from "yargs/helpers";
import { Manifest } from "../types";
import { watchFile } from "fs";
async function compile(filePath: string, functionName: string, debug: boolean) {
let outFile = "out.js";
@ -63,18 +64,49 @@ async function bundle(manifestPath: string, sourceMaps: boolean) {
}
return manifest;
}
async function buildManifest(
manifestPath: string,
distPath: string,
debug: boolean
) {
let generatedManifest = await bundle(manifestPath, debug);
const outPath = path.join(distPath, path.basename(manifestPath));
console.log("Emitting bundle to", outPath);
await writeFile(outPath, JSON.stringify(generatedManifest, null, 2));
return { generatedManifest, outPath };
}
async function run() {
let args = await yargs(hideBin(process.argv))
let args = yargs(hideBin(process.argv))
.option("debug", {
type: "boolean",
})
.option("watch", {
type: "boolean",
alias: "w",
})
.option("dist", {
type: "string",
default: ".",
})
.parse();
let generatedManifest = await bundle(args._[0] as string, !!args.debug);
await writeFile(
args._[1] as string,
JSON.stringify(generatedManifest, null, 2)
);
if (args._.length === 0) {
console.log(
"Usage: plugbox-bundle [--debug] [--dist <path>] <manifest.plug.json> <manifest2.plug.json> ..."
);
process.exit(1);
}
for (const plugManifestPath of args._) {
let manifestPath = plugManifestPath as string;
await buildManifest(manifestPath, args.dist, !!args.debug);
if (args.watch) {
watchFile(manifestPath, { interval: 1000 }, async () => {
console.log("Rebuilding", manifestPath);
await buildManifest(manifestPath, args.dist, !!args.debug);
});
}
}
}
run().catch((e) => {

66
plugbox/bin/plugbox-server.ts Executable file
View File

@ -0,0 +1,66 @@
#!/usr/bin/env node
import express from "express";
import yargs from "yargs";
import { hideBin } from "yargs/helpers";
import { DiskPlugLoader } from "../plug_loader";
import { CronHook, NodeCronFeature } from "../feature/node_cron";
import shellSyscalls from "../syscall/shell.node";
import { System } from "../system";
import { EndpointFeature, EndpointHook } from "../feature/endpoint";
import { safeRun } from "../util";
import knex from "knex";
import {
ensureTable,
storeReadSyscalls,
storeWriteSyscalls,
} from "../syscall/store.knex_node";
import { fetchSyscalls } from "../syscall/fetch.node";
let args = yargs(hideBin(process.argv))
.option("port", {
type: "number",
default: 1337,
})
.parse();
if (!args._.length) {
console.error("Usage: plugbox-server <path-to-plugs>");
process.exit(1);
}
const plugPath = args._[0] as string;
const app = express();
type ServerHook = EndpointHook & CronHook;
const system = new System<ServerHook>("server");
safeRun(async () => {
const db = knex({
client: "better-sqlite3",
connection: {
filename: "plugbox.db",
},
useNullAsDefault: true,
});
await ensureTable(db, "item");
let plugLoader = new DiskPlugLoader(system, plugPath);
await plugLoader.loadPlugs();
plugLoader.watcher();
system.addFeature(new NodeCronFeature());
system.addFeature(new EndpointFeature(app, ""));
system.registerSyscalls("shell", [], shellSyscalls("."));
system.registerSyscalls("fetch", [], fetchSyscalls());
system.registerSyscalls(
"store",
[],
storeWriteSyscalls(db, "item"),
storeReadSyscalls(db, "item")
);
app.listen(args.port, () => {
console.log(`Plugbox server listening on port ${args.port}`);
});
});

18
plugbox/environment/node_sandbox.ts
View File

@ -6,6 +6,8 @@ import workerCode from "bundle-text:./node_worker.ts";
import { Sandbox } from "../sandbox";
import { WorkerLike } from "./worker";
import { Plug } from "../plug";
import path from "path";
import fs from "fs";
class NodeWorkerWrapper implements WorkerLike {
onMessage?: (message: any) => Promise<void>;
@ -33,16 +35,16 @@ class NodeWorkerWrapper implements WorkerLike {
}
}
// Look for the node_modules directory, to be passed to the worker to find e.g. the vm2 module
let nodeModulesDir = __dirname;
while (!fs.existsSync(nodeModulesDir + "/node_modules")) {
nodeModulesDir = path.dirname(nodeModulesDir);
}
export function createSandbox(plug: Plug<any>) {
let worker = new Worker(workerCode, {
eval: true,
workerData: path.join(nodeModulesDir, "node_modules"),
});
return new Sandbox(
plug,
new NodeWorkerWrapper(
new Worker(workerCode, {
eval: true,
})
)
);
return new Sandbox(plug, new NodeWorkerWrapper(worker));
}

7
plugbox/environment/node_worker.ts
View File

@ -1,5 +1,8 @@
const { VM, VMScript } = require("vm2");
const { parentPort } = require("worker_threads");
const { parentPort, workerData } = require("worker_threads");
let vm2 = `${workerData}/vm2`;
const { VM, VMScript } = require(vm2);
// console.log("Process env", process.env);
let loadedFunctions = new Map<string, Function>();
let pendingRequests = new Map<

2
plugbox/feature/endpoint.test.ts
View File

@ -33,7 +33,7 @@ test("Run a plugbox endpoint server", async () => {
const app = express();
const port = 3123;
system.addFeature(new EndpointFeature(app));
system.addFeature(new EndpointFeature(app, "/_"));
let server = app.listen(port, () => {
console.log(`Listening on port ${port}`);

13
plugbox/feature/endpoint.ts
View File

@ -26,20 +26,21 @@ export type EndPointDef = {
handler: string; // function name
};
const endPointPrefix = "/_";
export class EndpointFeature implements Feature<EndpointHook> {
private app: Express;
private prefix: string;
constructor(app: Express) {
constructor(app: Express, prefix: string) {
this.app = app;
this.prefix = prefix;
}
apply(system: System<EndpointHook>): void {
this.app.use((req: Request, res: Response, next: NextFunction) => {
if (!req.path.startsWith(endPointPrefix)) {
if (!req.path.startsWith(this.prefix)) {
return next();
}
console.log("Endpoint request", req.path);
Promise.resolve()
.then(async () => {
for (const [plugName, plug] of system.loadedPlugs.entries()) {
@ -48,8 +49,10 @@ export class EndpointFeature implements Feature<EndpointHook> {
continue;
}
const endpoints = manifest.hooks?.endpoints;
console.log("Checking plug", plugName, endpoints);
if (endpoints) {
let prefix = `${endPointPrefix}/${plugName}`;
let prefix = `${this.prefix}/${plugName}`;
console.log("Need prefix", prefix, "got", req.path);
if (!req.path.startsWith(prefix)) {
continue;
}

2
plugs/core/task.ts
View File

@ -1,4 +1,4 @@
import { ClickEvent } from "../../webapp/src/app_event";
import type { ClickEvent } from "../../webapp/app_event";
import { syscall } from "../lib/syscall";
export async function taskToggle(event: ClickEvent) {

28
server/api_server.ts
View File

@ -119,21 +119,21 @@ export class SocketServer {
}
onCall(
"invokeFunction",
(plugName: string, name: string, ...args: any[]): Promise<any> => {
let plug = this.system.loadedPlugs.get(plugName);
if (!plug) {
throw new Error(`Plug ${plugName} not loaded`);
}
console.log(
"Invoking function",
name,
"for plug",
plugName,
"as requested over socket"
);
return plug.invoke(name, args);
"invokeFunction",
(plugName: string, name: string, ...args: any[]): Promise<any> => {
let plug = this.system.loadedPlugs.get(plugName);
if (!plug) {
throw new Error(`Plug ${plugName} not loaded`);
}
console.log(
"Invoking function",
name,
"for plug",
plugName,
"as requested over socket"
);
return plug.invoke(name, args);
}
);
console.log("Sending the sytem to the client");

2
server/express_server.ts
View File

@ -19,7 +19,7 @@ export class ExpressServer {
this.rootPath = rootPath;
this.system = system;
system.addFeature(new EndpointFeature(app));
system.addFeature(new EndpointFeature(app, "/_"));
// Fallback, serve index.html
let cachedIndex: string | undefined = undefined;

56
server/server.ts Normal file → Executable file
View File

@ -1,26 +1,30 @@
#!/usr/bin/env node
import express from "express";
import http from "http";
import { Server } from "socket.io";
import { SocketServer } from "./api_server";
import {Server} from "socket.io";
import {SocketServer} from "./api_server";
import yargs from "yargs";
import { hideBin } from "yargs/helpers";
import { SilverBulletHooks } from "../common/manifest";
import { ExpressServer } from "./express_server";
import { DiskPlugLoader } from "../plugbox/plug_loader";
import {hideBin} from "yargs/helpers";
import {SilverBulletHooks} from "../common/manifest";
import {ExpressServer} from "./express_server";
import {DiskPlugLoader} from "../plugbox/plug_loader";
import { NodeCronFeature } from "../plugbox/feature/node_cron";
import shellSyscalls from "../plugbox/syscall/shell.node";
import { System } from "../plugbox/system";
let args = yargs(hideBin(process.argv))
.option("debug", {
type: "boolean",
})
.option("port", {
type: "number",
default: 3000,
})
.parse();
if (!args._.length) {
console.error("Usage: silverbullet <path-to-pages>");
process.exit(1);
}
const pagesPath = args._[0] as string;
const app = express();
@ -41,25 +45,25 @@ app.use("/", express.static(distDir));
let socketServer = new SocketServer(pagesPath, io, system);
socketServer.init().catch((e) => {
console.error(e);
console.error(e);
});
const expressServer = new ExpressServer(app, pagesPath, distDir, system);
expressServer
.init()
.then(async () => {
let plugLoader = new DiskPlugLoader(
system,
`${__dirname}/../../plugs/dist`
);
await plugLoader.loadPlugs();
plugLoader.watcher();
system.registerSyscalls("shell", ["shell"], shellSyscalls(pagesPath));
system.addFeature(new NodeCronFeature());
server.listen(port, () => {
console.log(`Server listening on port ${port}`);
.init()
.then(async () => {
let plugLoader = new DiskPlugLoader(
system,
`${__dirname}/../../plugs/dist`
);
await plugLoader.loadPlugs();
plugLoader.watcher();
system.registerSyscalls("shell", ["shell"], shellSyscalls(pagesPath));
system.addFeature(new NodeCronFeature());
server.listen(port, () => {
console.log(`Server listening on port ${port}`);
});
})
.catch((e) => {
console.error(e);
});
})
.catch((e) => {
console.error(e);
});

3
webapp/boot.ts
View File

@ -3,8 +3,7 @@ import { Space } from "./space";
import { safeRun } from "./util";
import { io } from "socket.io-client";
let socket = io(`http://${location.hostname}:3000`);
let socket = io();
let editor = new Editor(new Space(socket), document.getElementById("root")!);
safeRun(async () => {

67
yarn.lock
View File

@ -1718,6 +1718,13 @@
dependencies:
"@types/superagent" "*"
"@types/yaml@^1.9.7":
version "1.9.7"
resolved "https://registry.yarnpkg.com/@types/yaml/-/yaml-1.9.7.tgz#2331f36e0aac91311a63d33eb026c21687729679"
integrity sha512-8WMXRDD1D+wCohjfslHDgICd2JtMATZU8CkhH8LVJqcJs6dyYj5TGptzP8wApbmEullGBSsCEzzap73DQ1HJaA==
dependencies:
yaml "*"
"@types/yargs-parser@*":
version "21.0.0"
resolved "https://registry.yarnpkg.com/@types/yargs-parser/-/yargs-parser-21.0.0.tgz#0c60e537fa790f5f9472ed2776c2b71ec117351b"
@ -2709,11 +2716,6 @@ csstype@^3.0.2:
resolved "https://registry.yarnpkg.com/csstype/-/csstype-3.0.11.tgz#d66700c5eacfac1940deb4e3ee5642792d85cd33"
integrity sha512-sa6P2wJ+CAbgyy4KFssIb/JNMLxFvKF1pCYCSXS8ZMuqZnMsrxqI2E5sPyoTpxoPU/gVZMzr2zjOfg8GIZOMsw==
data-uri-to-buffer@^4.0.0:
version "4.0.0"
resolved "https://registry.yarnpkg.com/data-uri-to-buffer/-/data-uri-to-buffer-4.0.0.tgz#b5db46aea50f6176428ac05b73be39a57701a64b"
integrity sha512-Vr3mLBA8qWmcuschSLAOogKgQ/Jwxulv3RNE4FXnYWRGujzrRWQI4m12fQqRkwX06C0KanhLr4hK+GydchZsaA==
data-urls@^2.0.0:
version "2.0.0"
resolved "https://registry.yarnpkg.com/data-urls/-/data-urls-2.0.0.tgz#156485a72963a970f5d5821aaf642bef2bf2db9b"
@ -3379,14 +3381,6 @@ fb-watchman@^2.0.0:
dependencies:
bser "2.1.1"
fetch-blob@^3.1.2, fetch-blob@^3.1.4:
version "3.1.5"
resolved "https://registry.yarnpkg.com/fetch-blob/-/fetch-blob-3.1.5.tgz#0077bf5f3fcdbd9d75a0b5362f77dbb743489863"
integrity sha512-N64ZpKqoLejlrwkIAnb9iLSA3Vx/kjgzpcDhygcqJ2KKjky8nCgUQ+dzXtbrLaWZGZNmNfQTsiQ0weZ1svglHg==
dependencies:
node-domexception "^1.0.0"
web-streams-polyfill "^3.0.3"
file-uri-to-path@1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/file-uri-to-path/-/file-uri-to-path-1.0.0.tgz#553a7b8446ff6f684359c445f1e37a05dacc33dd"
@ -3443,13 +3437,6 @@ form-data@^4.0.0:
combined-stream "^1.0.8"
mime-types "^2.1.12"
formdata-polyfill@^4.0.10:
version "4.0.10"
resolved "https://registry.yarnpkg.com/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz#24807c31c9d402e002ab3d8c720144ceb8848423"
integrity sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==
dependencies:
fetch-blob "^3.1.2"
formidable@^2.0.1:
version "2.0.1"
resolved "https://registry.yarnpkg.com/formidable/-/formidable-2.0.1.tgz#4310bc7965d185536f9565184dee74fbb75557ff"
@ -4966,19 +4953,12 @@ node-cron@^3.0.0:
dependencies:
moment-timezone "^0.5.31"
node-domexception@^1.0.0:
version "1.0.0"
resolved "https://registry.yarnpkg.com/node-domexception/-/node-domexception-1.0.0.tgz#6888db46a1f71c0b76b3f7555016b63fe64766e5"
integrity sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==
node-fetch@^3.2.3:
version "3.2.3"
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-3.2.3.tgz#a03c9cc2044d21d1a021566bd52f080f333719a6"
integrity sha512-AXP18u4pidSZ1xYXRDPY/8jdv3RAozIt/WLNR/MBGZAz+xjtlr90RvCnsvHQRiXyWliZF/CpytExp32UU67/SA==
node-fetch@2:
version "2.6.7"
resolved "https://registry.yarnpkg.com/node-fetch/-/node-fetch-2.6.7.tgz#24de9fba827e3b4ae44dc8b20256a379160052ad"
integrity sha512-ZjMPFEfVx5j+y2yF35Kzx5sF7kDzxuDj6ziH4FFbOp87zKDZNx8yExJIb05OGF4Nlt9IHFIMBkRl41VdvcNdbQ==
dependencies:
data-uri-to-buffer "^4.0.0"
fetch-blob "^3.1.4"
formdata-polyfill "^4.0.10"
whatwg-url "^5.0.0"
node-gyp-build@^4.2.3, node-gyp-build@^4.3.0:
version "4.3.0"
@ -6501,6 +6481,11 @@ tr46@^2.1.0:
dependencies:
punycode "^2.1.1"
tr46@~0.0.3:
version "0.0.3"
resolved "https://registry.yarnpkg.com/tr46/-/tr46-0.0.3.tgz#8184fd347dac9cdc185992f3a6622e14b9d9ab6a"
integrity sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=
ts-jest@^27.1.3:
version "27.1.3"
resolved "https://registry.yarnpkg.com/ts-jest/-/ts-jest-27.1.3.tgz#1f723e7e74027c4da92c0ffbd73287e8af2b2957"
@ -6740,10 +6725,10 @@ weak-lru-cache@^1.2.2:
resolved "https://registry.yarnpkg.com/weak-lru-cache/-/weak-lru-cache-1.2.2.tgz#fdbb6741f36bae9540d12f480ce8254060dccd19"
integrity sha512-DEAoo25RfSYMuTGc9vPJzZcZullwIqRDSI9LOy+fkCJPi6hykCnfKaXTuPBDuXAUcqHXyOgFtHNp/kB2FjYHbw==
web-streams-polyfill@^3.0.3:
version "3.2.0"
resolved "https://registry.yarnpkg.com/web-streams-polyfill/-/web-streams-polyfill-3.2.0.tgz#a6b74026b38e4885869fb5c589e90b95ccfc7965"
integrity sha512-EqPmREeOzttaLRm5HS7io98goBgZ7IVz79aDvqjD0kYXLtFZTc0T/U6wHTPKyIjb+MdN7DFIIX6hgdBEpWmfPA==
webidl-conversions@^3.0.0:
version "3.0.1"
resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-3.0.1.tgz#24534275e2a7bc6be7bc86611cc16ae0a5654871"
integrity sha1-JFNCdeKnvGvnvIZhHMFq4KVlSHE=
webidl-conversions@^4.0.2:
version "4.0.2"
@ -6772,6 +6757,14 @@ whatwg-mimetype@^2.3.0:
resolved "https://registry.yarnpkg.com/whatwg-mimetype/-/whatwg-mimetype-2.3.0.tgz#3d4b1e0312d2079879f826aff18dbeeca5960fbf"
integrity sha512-M4yMwr6mAnQz76TbJm914+gPpB/nCwvZbJU28cUD6dR004SAxDLOOSUaB1JDRqLtaOV/vi0IC5lEAGFgrjGv/g==
whatwg-url@^5.0.0:
version "5.0.0"
resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-5.0.0.tgz#966454e8765462e37644d3626f6742ce8b70965d"
integrity sha1-lmRU6HZUYuN2RNNib2dCzotwll0=
dependencies:
tr46 "~0.0.3"
webidl-conversions "^3.0.0"
whatwg-url@^8.0.0, whatwg-url@^8.4.0, whatwg-url@^8.5.0:
version "8.7.0"
resolved "https://registry.yarnpkg.com/whatwg-url/-/whatwg-url-8.7.0.tgz#656a78e510ff8f3937bc0bcbe9f5c0ac35941b77"
@ -6904,7 +6897,7 @@ yallist@^4.0.0:
resolved "https://registry.yarnpkg.com/yallist/-/yallist-4.0.0.tgz#9bb92790d9c0effec63be73519e11a35019a3a72"
integrity sha512-3wdGidZyq5PB084XLES5TpOSRA3wjXAlIWMhum2kRcv/41Sn2emQ0dycQW4uZXLejwKvg6EsvbdlVL+FYEct7A==
yaml@^1.10.0, yaml@^1.10.2:
yaml@*, yaml@^1.10.0, yaml@^1.10.2:
version "1.10.2"
resolved "https://registry.yarnpkg.com/yaml/-/yaml-1.10.2.tgz#2301c5ffbf12b467de8da2333a459e29e7920e4b"
integrity sha512-r3vXyErRCYJ7wg28yvBY5VSoAF8ZvlcW9/BwUzEtUsjvX/DKs24dIkuwjtuprwJJHsbyUbLApepYTR1BN4uHrg==