Dependency builds for plugos

pull/3/head
Zef Hemel 2022-05-13 14:36:26 +02:00
parent 8c974161c3
commit 3c5048ac25
29 changed files with 309 additions and 76 deletions

View File

@ -1,7 +0,0 @@
// These are the node modules that will be pre-bundled with SB
// as a result they will not be included into plugos bundles and assumed to be loadable
// via require() in the sandbox
// Candidate modules for this are larger modules
// When adding a module to this list, also manually add it to sandbox_worker.ts
export const preloadModules = ["@lezer/lr", "yaml", "handlebars"];

View File

@ -1,6 +1,6 @@
#!/usr/bin/env node
import { readFile, unlink, watch, writeFile } from "fs/promises";
import { readFile, watch, writeFile } from "fs/promises";
import path from "path";
import yargs from "yargs";
@ -8,7 +8,7 @@ import { hideBin } from "yargs/helpers";
import { Manifest } from "../types";
import YAML from "yaml";
import { mkdirSync } from "fs";
import { compile } from "../compile";
import { compile, sandboxCompileModule } from "../compile";
async function bundle(
manifestPath: string,
@ -24,7 +24,13 @@ async function bundle(
throw new Error(`Missing 'name' in ${manifestPath}`);
}
for (let [name, def] of Object.entries(manifest.functions)) {
let allModulesToExclude = excludeModules.slice();
for (let [name, moduleSpec] of Object.entries(manifest.dependencies || {})) {
manifest.dependencies![name] = await sandboxCompileModule(moduleSpec);
allModulesToExclude.push(name);
}
for (let [name, def] of Object.entries(manifest.functions || {})) {
let jsFunctionName = "default",
filePath = path.join(rootPath, def.path!);
if (filePath.indexOf(":") !== -1) {
@ -35,7 +41,7 @@ async function bundle(
filePath,
jsFunctionName,
sourceMaps,
excludeModules
allModulesToExclude
);
delete def.path;
}

View File

@ -1,6 +1,11 @@
import esbuild from "esbuild";
import { readFile, unlink, writeFile } from "fs/promises";
import { mkdir, readFile, rm, symlink, unlink, writeFile } from "fs/promises";
import path from "path";
import { tmpdir } from "os";
import { nodeModulesDir } from "./environments/node_sandbox";
import { promisify } from "util";
import { execFile } from "child_process";
const execFilePromise = promisify(execFile);
export async function compile(
filePath: string,
@ -22,8 +27,6 @@ export async function compile(
)}";export default ${functionName};`
);
}
// console.log("In:", inFile);
// console.log("Outfile:", outFile);
// TODO: Figure out how to make source maps work correctly with eval() code
let result = await esbuild.build({
@ -50,6 +53,81 @@ export async function compile(
if (inFile !== filePath) {
await unlink(inFile);
}
return `(() => { ${jsCode}
return mod;})()`;
return `(() => { ${jsCode} return mod;})()`;
}
export async function compileModule(
cwd: string,
moduleName: string
): Promise<string> {
let inFile = path.resolve(cwd, "_in.ts");
await writeFile(inFile, `export * from "${moduleName}";`);
let code = await compile(inFile);
await unlink(inFile);
return code;
}
// TODO: Reconsider this later
const exposedModules = [
"@silverbulletmd/plugos-silverbullet-syscall",
"@plugos/plugos-syscall",
];
export async function sandboxCompile(
filename: string,
code: string,
functionName?: string,
installModules: string[] = [],
globalModules: string[] = []
): Promise<string> {
let tmpDir = `${tmpdir()}/plugos-${Math.random()}`;
await mkdir(tmpDir, { recursive: true });
const srcNodeModules = `${nodeModulesDir}/node_modules`;
const targetNodeModules = `${tmpDir}/node_modules`;
await mkdir(`${targetNodeModules}/@silverbulletmd`, { recursive: true });
await mkdir(`${targetNodeModules}/@plugos`, { recursive: true });
for (const exposedModule of exposedModules) {
await symlink(
`${srcNodeModules}/${exposedModule}`,
`${targetNodeModules}/${exposedModule}`,
"dir"
);
}
for (let moduleName of installModules) {
await execFilePromise("npm", ["install", moduleName], {
cwd: tmpDir,
});
}
await writeFile(`${tmpDir}/${filename}`, code);
let jsCode = await compile(
`${tmpDir}/${filename}`,
functionName,
false,
globalModules
);
await rm(tmpDir, { recursive: true });
return jsCode;
}
export async function sandboxCompileModule(
moduleName: string,
globalModules: string[] = []
): Promise<string> {
let [modulePart, path] = moduleName.split(":");
let modulePieces = modulePart.split("@");
let cleanModulesName = modulePieces
.slice(0, modulePieces.length - 1)
.join("@");
return sandboxCompile(
"module.ts",
// `export * from "${cleanModulesName}${path ? path : ""}";`,
`module.exports = require("${cleanModulesName}${path ? path : ""}");`,
undefined,
[modulePart],
globalModules
);
}

View File

@ -43,6 +43,9 @@ export class ConsoleLogger {
case "number":
pieces.push("" + val);
break;
case "undefined":
pieces.push("undefined");
break;
default:
try {
let s = JSON.stringify(val, null, 2);

View File

@ -41,15 +41,11 @@ while (!fs.existsSync(nodeModulesDir + "/node_modules/vm2")) {
nodeModulesDir = path.dirname(nodeModulesDir);
}
export function createSandbox(
plug: Plug<any>,
preloadedModules: string[] = []
) {
export function createSandbox(plug: Plug<any>) {
let worker = new Worker(workerCode, {
eval: true,
workerData: {
nodeModulesPath: path.join(nodeModulesDir, "node_modules"),
preloadedModules,
},
});
return new Sandbox(plug, new NodeWorkerWrapper(worker));

View File

@ -2,7 +2,7 @@ import { ConsoleLogger } from "./custom_logger";
const {
parentPort,
workerData: { preloadedModules, nodeModulesPath },
workerData: { nodeModulesPath },
} = require("worker_threads");
const { VM, VMScript } = require(`${nodeModulesPath}/vm2`);
@ -26,6 +26,8 @@ let consoleLogger = new ConsoleLogger((level, message) => {
});
}, false);
let loadedModules = new Map<string, any>();
let vm = new VM({
sandbox: {
// Exposing some "safe" APIs
@ -39,9 +41,14 @@ let vm = new VM({
// This is only going to be called for pre-bundled modules, we won't allow
// arbitrary requiring of modules
require: (moduleName: string): any => {
// console.log("Loading", moduleName);
if (preloadedModules.includes(moduleName)) {
return require(`${nodeModulesPath}/${moduleName}`);
// console.log("Loading module", moduleName);
// if (preloadedModules.includes(moduleName)) {
// return require(`${nodeModulesPath}/${moduleName}`);
// } else
if (loadedModules.has(moduleName)) {
let mod = loadedModules.get(moduleName);
// console.log("And it has the value", mod);
return mod;
} else {
throw Error(`Cannot import arbitrary modules like ${moduleName}`);
}
@ -84,6 +91,20 @@ parentPort.on("message", (data: any) => {
name: data.name,
});
break;
case "load-dependency":
// console.log("Asked to load dep", data.name);
try {
let r = vm.run(data.code);
// console.log("Loaded dependency", r);
loadedModules.set(data.name, r);
parentPort.postMessage({
type: "dependency-inited",
name: data.name,
});
} catch (e: any) {
console.error("Could not load dependency", e.message);
}
break;
case "invoke":
let fn = loadedFunctions.get(data.name);
if (!fn) {

View File

@ -39,19 +39,12 @@ self.syscall = async (name: string, ...args: any[]) => {
});
};
const preloadedModules: { [key: string]: any } = {
"@lezer/lr": require("@lezer/lr"),
yaml: require("yaml"),
handlebars: require("handlebars/dist/handlebars"),
};
// for (const moduleName of preloadModules) {
// preloadedModules[moduleName] = require(moduleName);
// }
let loadedModules = new Map<string, any>();
// @ts-ignore
self.require = (moduleName: string): any => {
// console.log("Loading", moduleName, preloadedModules[moduleName]);
return preloadedModules[moduleName];
// console.log("Loading", moduleName, loadedModules.get(moduleName));
return loadedModules.get(moduleName);
};
// @ts-ignore
@ -75,6 +68,17 @@ self.addEventListener("message", (event: { data: WorkerMessage }) => {
name: data.name,
});
break;
case "load-dependency":
// console.log("Received dep", data.name);
let fn3 = new Function(`return ${data.code!}`);
let v = fn3();
loadedModules.set(data.name!, v);
// console.log("Dep val", v);
workerPostMessage({
type: "dependency-inited",
name: data.name,
});
break;
case "invoke":
let fn = loadedFunctions.get(data.name!);
if (!fn) {

View File

@ -1,6 +1,12 @@
import type { LogLevel } from "./custom_logger";
export type ControllerMessageType = "inited" | "result" | "syscall" | "log";
export type ControllerMessageType =
| "inited"
| "dependency-inited"
| "result"
| "syscall"
| "log";
export type ControllerMessage = {
type: ControllerMessageType;
id?: number;
@ -21,7 +27,11 @@ export interface WorkerLike {
terminate(): void;
}
export type WorkerMessageType = "load" | "invoke" | "syscall-response";
export type WorkerMessageType =
| "load"
| "load-dependency"
| "invoke"
| "syscall-response";
export type WorkerMessage = {
type: WorkerMessageType;

View File

@ -67,7 +67,9 @@ export class EventHook implements Hook<EventHookT> {
validateManifest(manifest: Manifest<EventHookT>): string[] {
let errors = [];
for (const [name, functionDef] of Object.entries(manifest.functions)) {
for (const [name, functionDef] of Object.entries(
manifest.functions || {}
)) {
if (functionDef.events && !Array.isArray(functionDef.events)) {
errors.push("'events' key must be an array of strings");
}

View File

@ -60,7 +60,8 @@
"vm2": "^3.9.9",
"ws": "^8.5.0",
"yaml": "^1.10.2",
"yargs": "^17.3.1"
"yargs": "^17.3.1",
"typescript": "^4.6.2"
},
"devDependencies": {
"@lezer/lr": "^0.15.0",
@ -82,7 +83,6 @@
"assert": "^2.0.0",
"events": "^3.3.0",
"parcel": "2.3.2",
"prettier": "^2.5.1",
"typescript": "^4.6.2"
"prettier": "^2.5.1"
}
}

View File

@ -27,6 +27,9 @@ export class Plug<HookT> {
this.manifest = manifest;
// TODO: These need to be explicitly granted, not just taken
this.grantedPermissions = manifest.requiredPermissions || [];
for (let [dep, code] of Object.entries(manifest.dependencies || {})) {
await this.sandbox.loadDependency(dep, code);
}
}
syscall(name: string, args: any[]): Promise<any> {

View File

@ -18,6 +18,7 @@ export class Sandbox {
protected worker: WorkerLike;
protected reqId = 0;
protected outstandingInits = new Map<string, () => void>();
protected outstandingDependencyInits = new Map<string, () => void>();
protected outstandingInvocations = new Map<
number,
{ resolve: (result: any) => void; reject: (e: any) => void }
@ -63,6 +64,22 @@ export class Sandbox {
});
}
async loadDependency(name: string, code: string): Promise<void> {
// console.log("Loading dependency", name);
this.worker.postMessage({
type: "load-dependency",
name: name,
code: code,
} as WorkerMessage);
return new Promise((resolve) => {
// console.log("Loaded dependency", name);
this.outstandingDependencyInits.set(name, () => {
this.outstandingDependencyInits.delete(name);
resolve();
});
});
}
async onMessage(data: ControllerMessage) {
switch (data.type) {
case "inited":
@ -70,6 +87,11 @@ export class Sandbox {
initCb && initCb();
this.outstandingInits.delete(data.name!);
break;
case "dependency-inited":
let depInitCb = this.outstandingDependencyInits.get(data.name!);
depInitCb && depInitCb();
this.outstandingDependencyInits.delete(data.name!);
break;
case "syscall":
try {
let result = await this.plug.syscall(data.name!, data.args!);

View File

@ -10,36 +10,95 @@ const exposedModules = [
"yaml",
];
import * as ts from "typescript";
type CompileError = {
message: string;
pos: number;
};
function checkTypeScript(scriptFile: string): void {
let program = ts.createProgram([scriptFile], {
noEmit: true,
allowJs: true,
});
let emitResult = program.emit();
let allDiagnostics = ts
.getPreEmitDiagnostics(program)
.concat(emitResult.diagnostics);
let errors: CompileError[] = [];
allDiagnostics.forEach((diagnostic) => {
if (diagnostic.file) {
let { line, character } = ts.getLineAndCharacterOfPosition(
diagnostic.file,
diagnostic.start!
);
let message = ts.flattenDiagnosticMessageText(
diagnostic.messageText,
"\n"
);
errors.push({
message: ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n"),
pos: diagnostic.start!,
});
// console.log(
// `${diagnostic.file.fileName} (${line + 1},${character + 1}): ${message}`
// );
} else {
console.log(
ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n")
);
}
});
let exitCode = emitResult.emitSkipped ? 1 : 0;
console.log(`Process exiting with code '${exitCode}'.`);
process.exit(exitCode);
}
export function esbuildSyscalls(): SysCallMapping {
return {
"tsc.analyze": async (
ctx,
filename: string,
code: string
): Promise<any> => {},
"esbuild.compile": async (
ctx,
filename: string,
code: string,
functionName?: string
): Promise<string> => {
let tmpDir = `${tmpdir()}/plugos-${Math.random()}`;
await mkdir(tmpDir, { recursive: true });
const srcNodeModules = `${nodeModulesDir}/node_modules`;
const targetNodeModules = `${tmpDir}/node_modules`;
await mkdir(`${targetNodeModules}/@silverbulletmd`, { recursive: true });
await mkdir(`${targetNodeModules}/@plugos`, { recursive: true });
for (const exposedModule of exposedModules) {
await symlink(
`${srcNodeModules}/${exposedModule}`,
`${targetNodeModules}/${exposedModule}`,
"dir"
);
}
await writeFile(`${tmpDir}/${filename}`, code);
let tmpDir = await prepareCompileEnv(filename, code);
let jsCode = await compile(`${tmpDir}/${filename}`, functionName, false, [
"yaml",
"handlebars",
]);
await rm(tmpDir, { recursive: true });
return jsCode;
},
};
}
async function prepareCompileEnv(filename: string, code: string) {
let tmpDir = `${tmpdir()}/plugos-${Math.random()}`;
await mkdir(tmpDir, { recursive: true });
const srcNodeModules = `${nodeModulesDir}/node_modules`;
const targetNodeModules = `${tmpDir}/node_modules`;
await mkdir(`${targetNodeModules}/@silverbulletmd`, { recursive: true });
await mkdir(`${targetNodeModules}/@plugos`, { recursive: true });
for (const exposedModule of exposedModules) {
await symlink(
`${srcNodeModules}/${exposedModule}`,
`${targetNodeModules}/${exposedModule}`,
"dir"
);
}
await writeFile(`${tmpDir}/${filename}`, code);
return tmpDir;
}

View File

@ -3,6 +3,9 @@ import { System } from "./system";
export interface Manifest<HookT> {
name: string;
requiredPermissions?: string[];
dependencies?: {
[key: string]: string;
};
functions: {
[key: string]: FunctionDef<HookT>;
};

View File

@ -422,6 +422,7 @@ function $9072202279b76d33$export$5884dae03c64f759(parsedQuery, records) {
}
async function $9072202279b76d33$export$b3c659c1456e61b0(parsedQuery, data) {
if (parsedQuery.render) {
console.log("Handlebars", ($parcel$interopDefault($hVExJ$handlebars)));
($parcel$interopDefault($hVExJ$handlebars)).registerHelper("json", (v)=>JSON.stringify(v)
);
($parcel$interopDefault($hVExJ$handlebars)).registerHelper("niceDate", (ts)=>$c3893eec0c49ec96$export$5dc1410f87262ed6(new Date(ts))

File diff suppressed because one or more lines are too long

View File

@ -19,6 +19,7 @@ syntax:
styles:
color: "#0330cb"
textDecoration: underline
cursor: pointer
functions:
clearPageIndex:
path: "./page.ts:clearPageIndex"

View File

@ -14,7 +14,7 @@ async function actionClickOrActionEnter(mdTree: ParseTree | null) {
if (!mdTree) {
return;
}
console.log("Attempting to navigate based on syntax node", mdTree);
// console.log("Attempting to navigate based on syntax node", mdTree);
switch (mdTree.type) {
case "WikiLinkPage":
let pageLink = mdTree.children![0].text!;

View File

@ -0,0 +1,5 @@
name: global
dependencies:
yaml: "yaml@2"
handlebars: "handlebars@4.7.7:/dist/handlebars"
"@lezer/lr": "@lezer/lr@0.15.4"

View File

@ -8,8 +8,8 @@
"license": "MIT",
"scripts": {
"generate": "lezer-generator query/query.grammar -o query/parse-query.js",
"watch": "plugos-bundle -w --dist dist --exclude @lezer/lr yaml handlebars -- */*.plug.yaml",
"build": "plugos-bundle --dist dist --exclude @lezer/lr yaml handlebars -- */*.plug.yaml",
"watch": "plugos-bundle --dist ../common/dist global.plug.yaml && plugos-bundle -w --dist dist --exclude @lezer/lr yaml handlebars -- */*.plug.yaml",
"build": "plugos-bundle --dist ../common/dist global.plug.yaml && plugos-bundle --dist dist --exclude @lezer/lr yaml handlebars -- */*.plug.yaml",
"test": "jest build/test"
},
"files": [

View File

@ -36,6 +36,7 @@ export async function compileCommand() {
);
console.log("Wrote this plug", manifest);
await hideBhs();
await reloadPlugs();
} catch (e: any) {
await showBhs(e.message);
@ -136,6 +137,7 @@ export async function updatePlugs() {
return;
}
let plugYaml = codeTextNode.children![0].text;
console.log("YAML", YAML);
let plugList = YAML.parse(plugYaml!);
console.log("Plug YAML", plugList);
let allPlugNames: string[] = [];

View File

@ -221,6 +221,7 @@ export async function renderQuery(
data: any[]
): Promise<string> {
if (parsedQuery.render) {
console.log("Handlebars", Handlebars);
Handlebars.registerHelper("json", (v) => JSON.stringify(v));
Handlebars.registerHelper("niceDate", (ts) => niceDate(new Date(ts)));
Handlebars.registerHelper("yaml", (v, prefix) => {

View File

@ -1,4 +1,7 @@
name: query
# dependencies:
# yaml: "yaml@2"
# "@lezer/lr": "@lezer/lr@0.15.4"
functions:
updateMaterializedQueriesOnPage:
path: ./materialized_queries.ts:updateMaterializedQueriesOnPage

View File

@ -27,9 +27,12 @@ import { systemSyscalls } from "./syscalls/system";
import { plugPrefix } from "@silverbulletmd/common/spaces/constants";
import { Authenticator } from "./auth";
import { nextTick } from "process";
import sandboxSyscalls from "@plugos/plugos/syscalls/sandbox";
import globalModules from "../common/dist/global.plug.json";
import { safeRun } from "./util";
const safeFilename = /^[a-zA-Z0-9_\-\.]+$/;
export type ServerOptions = {
@ -37,7 +40,6 @@ export type ServerOptions = {
pagesPath: string;
distDir: string;
builtinPlugDir: string;
preloadedModules: string[];
token?: string;
};
export class ExpressServer {
@ -50,7 +52,6 @@ export class ExpressServer {
private port: number;
private server?: Server;
builtinPlugDir: string;
preloadedModules: string[];
token?: string;
constructor(options: ServerOptions) {
@ -59,7 +60,6 @@ export class ExpressServer {
this.builtinPlugDir = options.builtinPlugDir;
this.distDir = options.distDir;
this.system = new System<SilverBulletHooks>("server");
this.preloadedModules = options.preloadedModules;
this.token = options.token;
// Setup system
@ -96,6 +96,18 @@ export class ExpressServer {
);
this.system.addHook(new EndpointHook(this.app, "/_"));
this.system.on({
plugLoaded: (plug) => {
safeRun(async () => {
for (let [modName, code] of Object.entries(
globalModules.dependencies
)) {
await plug.sandbox.loadDependency(modName, code);
}
});
},
});
this.eventHook.addLocalListener(
"get-plug:builtin",
async (plugName: string): Promise<Manifest> => {
@ -165,9 +177,7 @@ export class ExpressServer {
console.log("Reloading plugs");
for (let pageInfo of allPlugs) {
let { text } = await this.space.readPage(pageInfo.name);
await this.system.load(JSON.parse(text), (p) =>
createSandbox(p, this.preloadedModules)
);
await this.system.load(JSON.parse(text), createSandbox);
}
this.rebuildMdExtensions();
}

View File

@ -1,6 +1,5 @@
#!/usr/bin/env -S node --enable-source-maps
import { nodeModulesDir } from "@plugos/plugos/environments/node_sandbox";
import { preloadModules } from "@silverbulletmd/common/preload_modules";
import { realpathSync } from "fs";
import yargs from "yargs";
import { hideBin } from "yargs/helpers";
@ -39,7 +38,6 @@ console.log("Builtin plug dist dir", plugDistDir);
const expressServer = new ExpressServer({
port: port,
pagesPath: pagesPath,
preloadedModules: preloadModules,
distDir: webappDistDir,
builtinPlugDir: plugDistDir,
token: args.token,

View File

@ -1,8 +1,3 @@
declare global {
function syscall(name: string, ...args: any[]): Promise<any>;
// function require(moduleName: string): any;
}
window.addEventListener("message", (message) => {
const data = message.data;
switch (data.type) {

View File

@ -59,6 +59,7 @@ import { FilterList } from "./components/filter";
import { FilterOption } from "@silverbulletmd/common/types";
import { syntaxTree } from "@codemirror/language";
import sandboxSyscalls from "@plugos/plugos/syscalls/sandbox";
import globalModules from "../common/dist/global.plug.json";
class PageState {
constructor(
@ -131,6 +132,18 @@ export class Editor {
clientStoreSyscalls(),
sandboxSyscalls(this.system)
);
this.system.on({
plugLoaded: (plug) => {
safeRun(async () => {
for (let [modName, code] of Object.entries(
globalModules.dependencies
)) {
await plug.sandbox.loadDependency(modName, code);
}
});
},
});
}
get currentPage(): string | undefined {

View File

@ -1,5 +1,8 @@
import { Action, AppViewState } from "./types";
let m = new Map();
m.size;
export default function reducer(
state: AppViewState,
action: Action

View File

@ -171,6 +171,7 @@
cursor: pointer;
}
.wiki-link {
cursor: pointer;
color: #a8abbd;
}