Dependency builds for plugos
parent
8c974161c3
commit
3c5048ac25
|
@ -1,7 +0,0 @@
|
||||||
// These are the node modules that will be pre-bundled with SB
|
|
||||||
// as a result they will not be included into plugos bundles and assumed to be loadable
|
|
||||||
// via require() in the sandbox
|
|
||||||
// Candidate modules for this are larger modules
|
|
||||||
|
|
||||||
// When adding a module to this list, also manually add it to sandbox_worker.ts
|
|
||||||
export const preloadModules = ["@lezer/lr", "yaml", "handlebars"];
|
|
|
@ -1,6 +1,6 @@
|
||||||
#!/usr/bin/env node
|
#!/usr/bin/env node
|
||||||
|
|
||||||
import { readFile, unlink, watch, writeFile } from "fs/promises";
|
import { readFile, watch, writeFile } from "fs/promises";
|
||||||
import path from "path";
|
import path from "path";
|
||||||
|
|
||||||
import yargs from "yargs";
|
import yargs from "yargs";
|
||||||
|
@ -8,7 +8,7 @@ import { hideBin } from "yargs/helpers";
|
||||||
import { Manifest } from "../types";
|
import { Manifest } from "../types";
|
||||||
import YAML from "yaml";
|
import YAML from "yaml";
|
||||||
import { mkdirSync } from "fs";
|
import { mkdirSync } from "fs";
|
||||||
import { compile } from "../compile";
|
import { compile, sandboxCompileModule } from "../compile";
|
||||||
|
|
||||||
async function bundle(
|
async function bundle(
|
||||||
manifestPath: string,
|
manifestPath: string,
|
||||||
|
@ -24,7 +24,13 @@ async function bundle(
|
||||||
throw new Error(`Missing 'name' in ${manifestPath}`);
|
throw new Error(`Missing 'name' in ${manifestPath}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
for (let [name, def] of Object.entries(manifest.functions)) {
|
let allModulesToExclude = excludeModules.slice();
|
||||||
|
for (let [name, moduleSpec] of Object.entries(manifest.dependencies || {})) {
|
||||||
|
manifest.dependencies![name] = await sandboxCompileModule(moduleSpec);
|
||||||
|
allModulesToExclude.push(name);
|
||||||
|
}
|
||||||
|
|
||||||
|
for (let [name, def] of Object.entries(manifest.functions || {})) {
|
||||||
let jsFunctionName = "default",
|
let jsFunctionName = "default",
|
||||||
filePath = path.join(rootPath, def.path!);
|
filePath = path.join(rootPath, def.path!);
|
||||||
if (filePath.indexOf(":") !== -1) {
|
if (filePath.indexOf(":") !== -1) {
|
||||||
|
@ -35,7 +41,7 @@ async function bundle(
|
||||||
filePath,
|
filePath,
|
||||||
jsFunctionName,
|
jsFunctionName,
|
||||||
sourceMaps,
|
sourceMaps,
|
||||||
excludeModules
|
allModulesToExclude
|
||||||
);
|
);
|
||||||
delete def.path;
|
delete def.path;
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,6 +1,11 @@
|
||||||
import esbuild from "esbuild";
|
import esbuild from "esbuild";
|
||||||
import { readFile, unlink, writeFile } from "fs/promises";
|
import { mkdir, readFile, rm, symlink, unlink, writeFile } from "fs/promises";
|
||||||
import path from "path";
|
import path from "path";
|
||||||
|
import { tmpdir } from "os";
|
||||||
|
import { nodeModulesDir } from "./environments/node_sandbox";
|
||||||
|
import { promisify } from "util";
|
||||||
|
import { execFile } from "child_process";
|
||||||
|
const execFilePromise = promisify(execFile);
|
||||||
|
|
||||||
export async function compile(
|
export async function compile(
|
||||||
filePath: string,
|
filePath: string,
|
||||||
|
@ -22,8 +27,6 @@ export async function compile(
|
||||||
)}";export default ${functionName};`
|
)}";export default ${functionName};`
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
// console.log("In:", inFile);
|
|
||||||
// console.log("Outfile:", outFile);
|
|
||||||
|
|
||||||
// TODO: Figure out how to make source maps work correctly with eval() code
|
// TODO: Figure out how to make source maps work correctly with eval() code
|
||||||
let result = await esbuild.build({
|
let result = await esbuild.build({
|
||||||
|
@ -50,6 +53,81 @@ export async function compile(
|
||||||
if (inFile !== filePath) {
|
if (inFile !== filePath) {
|
||||||
await unlink(inFile);
|
await unlink(inFile);
|
||||||
}
|
}
|
||||||
return `(() => { ${jsCode}
|
return `(() => { ${jsCode} return mod;})()`;
|
||||||
return mod;})()`;
|
}
|
||||||
|
|
||||||
|
export async function compileModule(
|
||||||
|
cwd: string,
|
||||||
|
moduleName: string
|
||||||
|
): Promise<string> {
|
||||||
|
let inFile = path.resolve(cwd, "_in.ts");
|
||||||
|
await writeFile(inFile, `export * from "${moduleName}";`);
|
||||||
|
let code = await compile(inFile);
|
||||||
|
await unlink(inFile);
|
||||||
|
return code;
|
||||||
|
}
|
||||||
|
|
||||||
|
// TODO: Reconsider this later
|
||||||
|
const exposedModules = [
|
||||||
|
"@silverbulletmd/plugos-silverbullet-syscall",
|
||||||
|
"@plugos/plugos-syscall",
|
||||||
|
];
|
||||||
|
|
||||||
|
export async function sandboxCompile(
|
||||||
|
filename: string,
|
||||||
|
code: string,
|
||||||
|
functionName?: string,
|
||||||
|
installModules: string[] = [],
|
||||||
|
globalModules: string[] = []
|
||||||
|
): Promise<string> {
|
||||||
|
let tmpDir = `${tmpdir()}/plugos-${Math.random()}`;
|
||||||
|
await mkdir(tmpDir, { recursive: true });
|
||||||
|
|
||||||
|
const srcNodeModules = `${nodeModulesDir}/node_modules`;
|
||||||
|
const targetNodeModules = `${tmpDir}/node_modules`;
|
||||||
|
|
||||||
|
await mkdir(`${targetNodeModules}/@silverbulletmd`, { recursive: true });
|
||||||
|
await mkdir(`${targetNodeModules}/@plugos`, { recursive: true });
|
||||||
|
for (const exposedModule of exposedModules) {
|
||||||
|
await symlink(
|
||||||
|
`${srcNodeModules}/${exposedModule}`,
|
||||||
|
`${targetNodeModules}/${exposedModule}`,
|
||||||
|
"dir"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for (let moduleName of installModules) {
|
||||||
|
await execFilePromise("npm", ["install", moduleName], {
|
||||||
|
cwd: tmpDir,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
await writeFile(`${tmpDir}/${filename}`, code);
|
||||||
|
|
||||||
|
let jsCode = await compile(
|
||||||
|
`${tmpDir}/${filename}`,
|
||||||
|
functionName,
|
||||||
|
false,
|
||||||
|
globalModules
|
||||||
|
);
|
||||||
|
await rm(tmpDir, { recursive: true });
|
||||||
|
return jsCode;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function sandboxCompileModule(
|
||||||
|
moduleName: string,
|
||||||
|
globalModules: string[] = []
|
||||||
|
): Promise<string> {
|
||||||
|
let [modulePart, path] = moduleName.split(":");
|
||||||
|
let modulePieces = modulePart.split("@");
|
||||||
|
let cleanModulesName = modulePieces
|
||||||
|
.slice(0, modulePieces.length - 1)
|
||||||
|
.join("@");
|
||||||
|
return sandboxCompile(
|
||||||
|
"module.ts",
|
||||||
|
// `export * from "${cleanModulesName}${path ? path : ""}";`,
|
||||||
|
`module.exports = require("${cleanModulesName}${path ? path : ""}");`,
|
||||||
|
undefined,
|
||||||
|
[modulePart],
|
||||||
|
globalModules
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
|
@ -43,6 +43,9 @@ export class ConsoleLogger {
|
||||||
case "number":
|
case "number":
|
||||||
pieces.push("" + val);
|
pieces.push("" + val);
|
||||||
break;
|
break;
|
||||||
|
case "undefined":
|
||||||
|
pieces.push("undefined");
|
||||||
|
break;
|
||||||
default:
|
default:
|
||||||
try {
|
try {
|
||||||
let s = JSON.stringify(val, null, 2);
|
let s = JSON.stringify(val, null, 2);
|
||||||
|
|
|
@ -41,15 +41,11 @@ while (!fs.existsSync(nodeModulesDir + "/node_modules/vm2")) {
|
||||||
nodeModulesDir = path.dirname(nodeModulesDir);
|
nodeModulesDir = path.dirname(nodeModulesDir);
|
||||||
}
|
}
|
||||||
|
|
||||||
export function createSandbox(
|
export function createSandbox(plug: Plug<any>) {
|
||||||
plug: Plug<any>,
|
|
||||||
preloadedModules: string[] = []
|
|
||||||
) {
|
|
||||||
let worker = new Worker(workerCode, {
|
let worker = new Worker(workerCode, {
|
||||||
eval: true,
|
eval: true,
|
||||||
workerData: {
|
workerData: {
|
||||||
nodeModulesPath: path.join(nodeModulesDir, "node_modules"),
|
nodeModulesPath: path.join(nodeModulesDir, "node_modules"),
|
||||||
preloadedModules,
|
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
return new Sandbox(plug, new NodeWorkerWrapper(worker));
|
return new Sandbox(plug, new NodeWorkerWrapper(worker));
|
||||||
|
|
|
@ -2,7 +2,7 @@ import { ConsoleLogger } from "./custom_logger";
|
||||||
|
|
||||||
const {
|
const {
|
||||||
parentPort,
|
parentPort,
|
||||||
workerData: { preloadedModules, nodeModulesPath },
|
workerData: { nodeModulesPath },
|
||||||
} = require("worker_threads");
|
} = require("worker_threads");
|
||||||
|
|
||||||
const { VM, VMScript } = require(`${nodeModulesPath}/vm2`);
|
const { VM, VMScript } = require(`${nodeModulesPath}/vm2`);
|
||||||
|
@ -26,6 +26,8 @@ let consoleLogger = new ConsoleLogger((level, message) => {
|
||||||
});
|
});
|
||||||
}, false);
|
}, false);
|
||||||
|
|
||||||
|
let loadedModules = new Map<string, any>();
|
||||||
|
|
||||||
let vm = new VM({
|
let vm = new VM({
|
||||||
sandbox: {
|
sandbox: {
|
||||||
// Exposing some "safe" APIs
|
// Exposing some "safe" APIs
|
||||||
|
@ -39,9 +41,14 @@ let vm = new VM({
|
||||||
// This is only going to be called for pre-bundled modules, we won't allow
|
// This is only going to be called for pre-bundled modules, we won't allow
|
||||||
// arbitrary requiring of modules
|
// arbitrary requiring of modules
|
||||||
require: (moduleName: string): any => {
|
require: (moduleName: string): any => {
|
||||||
// console.log("Loading", moduleName);
|
// console.log("Loading module", moduleName);
|
||||||
if (preloadedModules.includes(moduleName)) {
|
// if (preloadedModules.includes(moduleName)) {
|
||||||
return require(`${nodeModulesPath}/${moduleName}`);
|
// return require(`${nodeModulesPath}/${moduleName}`);
|
||||||
|
// } else
|
||||||
|
if (loadedModules.has(moduleName)) {
|
||||||
|
let mod = loadedModules.get(moduleName);
|
||||||
|
// console.log("And it has the value", mod);
|
||||||
|
return mod;
|
||||||
} else {
|
} else {
|
||||||
throw Error(`Cannot import arbitrary modules like ${moduleName}`);
|
throw Error(`Cannot import arbitrary modules like ${moduleName}`);
|
||||||
}
|
}
|
||||||
|
@ -84,6 +91,20 @@ parentPort.on("message", (data: any) => {
|
||||||
name: data.name,
|
name: data.name,
|
||||||
});
|
});
|
||||||
break;
|
break;
|
||||||
|
case "load-dependency":
|
||||||
|
// console.log("Asked to load dep", data.name);
|
||||||
|
try {
|
||||||
|
let r = vm.run(data.code);
|
||||||
|
// console.log("Loaded dependency", r);
|
||||||
|
loadedModules.set(data.name, r);
|
||||||
|
parentPort.postMessage({
|
||||||
|
type: "dependency-inited",
|
||||||
|
name: data.name,
|
||||||
|
});
|
||||||
|
} catch (e: any) {
|
||||||
|
console.error("Could not load dependency", e.message);
|
||||||
|
}
|
||||||
|
break;
|
||||||
case "invoke":
|
case "invoke":
|
||||||
let fn = loadedFunctions.get(data.name);
|
let fn = loadedFunctions.get(data.name);
|
||||||
if (!fn) {
|
if (!fn) {
|
||||||
|
|
|
@ -39,19 +39,12 @@ self.syscall = async (name: string, ...args: any[]) => {
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
const preloadedModules: { [key: string]: any } = {
|
let loadedModules = new Map<string, any>();
|
||||||
"@lezer/lr": require("@lezer/lr"),
|
|
||||||
yaml: require("yaml"),
|
|
||||||
handlebars: require("handlebars/dist/handlebars"),
|
|
||||||
};
|
|
||||||
// for (const moduleName of preloadModules) {
|
|
||||||
// preloadedModules[moduleName] = require(moduleName);
|
|
||||||
// }
|
|
||||||
|
|
||||||
// @ts-ignore
|
// @ts-ignore
|
||||||
self.require = (moduleName: string): any => {
|
self.require = (moduleName: string): any => {
|
||||||
// console.log("Loading", moduleName, preloadedModules[moduleName]);
|
// console.log("Loading", moduleName, loadedModules.get(moduleName));
|
||||||
return preloadedModules[moduleName];
|
return loadedModules.get(moduleName);
|
||||||
};
|
};
|
||||||
|
|
||||||
// @ts-ignore
|
// @ts-ignore
|
||||||
|
@ -75,6 +68,17 @@ self.addEventListener("message", (event: { data: WorkerMessage }) => {
|
||||||
name: data.name,
|
name: data.name,
|
||||||
});
|
});
|
||||||
break;
|
break;
|
||||||
|
case "load-dependency":
|
||||||
|
// console.log("Received dep", data.name);
|
||||||
|
let fn3 = new Function(`return ${data.code!}`);
|
||||||
|
let v = fn3();
|
||||||
|
loadedModules.set(data.name!, v);
|
||||||
|
// console.log("Dep val", v);
|
||||||
|
workerPostMessage({
|
||||||
|
type: "dependency-inited",
|
||||||
|
name: data.name,
|
||||||
|
});
|
||||||
|
break;
|
||||||
case "invoke":
|
case "invoke":
|
||||||
let fn = loadedFunctions.get(data.name!);
|
let fn = loadedFunctions.get(data.name!);
|
||||||
if (!fn) {
|
if (!fn) {
|
||||||
|
|
|
@ -1,6 +1,12 @@
|
||||||
import type { LogLevel } from "./custom_logger";
|
import type { LogLevel } from "./custom_logger";
|
||||||
|
|
||||||
export type ControllerMessageType = "inited" | "result" | "syscall" | "log";
|
export type ControllerMessageType =
|
||||||
|
| "inited"
|
||||||
|
| "dependency-inited"
|
||||||
|
| "result"
|
||||||
|
| "syscall"
|
||||||
|
| "log";
|
||||||
|
|
||||||
export type ControllerMessage = {
|
export type ControllerMessage = {
|
||||||
type: ControllerMessageType;
|
type: ControllerMessageType;
|
||||||
id?: number;
|
id?: number;
|
||||||
|
@ -21,7 +27,11 @@ export interface WorkerLike {
|
||||||
terminate(): void;
|
terminate(): void;
|
||||||
}
|
}
|
||||||
|
|
||||||
export type WorkerMessageType = "load" | "invoke" | "syscall-response";
|
export type WorkerMessageType =
|
||||||
|
| "load"
|
||||||
|
| "load-dependency"
|
||||||
|
| "invoke"
|
||||||
|
| "syscall-response";
|
||||||
|
|
||||||
export type WorkerMessage = {
|
export type WorkerMessage = {
|
||||||
type: WorkerMessageType;
|
type: WorkerMessageType;
|
||||||
|
|
|
@ -67,7 +67,9 @@ export class EventHook implements Hook<EventHookT> {
|
||||||
|
|
||||||
validateManifest(manifest: Manifest<EventHookT>): string[] {
|
validateManifest(manifest: Manifest<EventHookT>): string[] {
|
||||||
let errors = [];
|
let errors = [];
|
||||||
for (const [name, functionDef] of Object.entries(manifest.functions)) {
|
for (const [name, functionDef] of Object.entries(
|
||||||
|
manifest.functions || {}
|
||||||
|
)) {
|
||||||
if (functionDef.events && !Array.isArray(functionDef.events)) {
|
if (functionDef.events && !Array.isArray(functionDef.events)) {
|
||||||
errors.push("'events' key must be an array of strings");
|
errors.push("'events' key must be an array of strings");
|
||||||
}
|
}
|
||||||
|
|
|
@ -60,7 +60,8 @@
|
||||||
"vm2": "^3.9.9",
|
"vm2": "^3.9.9",
|
||||||
"ws": "^8.5.0",
|
"ws": "^8.5.0",
|
||||||
"yaml": "^1.10.2",
|
"yaml": "^1.10.2",
|
||||||
"yargs": "^17.3.1"
|
"yargs": "^17.3.1",
|
||||||
|
"typescript": "^4.6.2"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@lezer/lr": "^0.15.0",
|
"@lezer/lr": "^0.15.0",
|
||||||
|
@ -82,7 +83,6 @@
|
||||||
"assert": "^2.0.0",
|
"assert": "^2.0.0",
|
||||||
"events": "^3.3.0",
|
"events": "^3.3.0",
|
||||||
"parcel": "2.3.2",
|
"parcel": "2.3.2",
|
||||||
"prettier": "^2.5.1",
|
"prettier": "^2.5.1"
|
||||||
"typescript": "^4.6.2"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
|
@ -27,6 +27,9 @@ export class Plug<HookT> {
|
||||||
this.manifest = manifest;
|
this.manifest = manifest;
|
||||||
// TODO: These need to be explicitly granted, not just taken
|
// TODO: These need to be explicitly granted, not just taken
|
||||||
this.grantedPermissions = manifest.requiredPermissions || [];
|
this.grantedPermissions = manifest.requiredPermissions || [];
|
||||||
|
for (let [dep, code] of Object.entries(manifest.dependencies || {})) {
|
||||||
|
await this.sandbox.loadDependency(dep, code);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
syscall(name: string, args: any[]): Promise<any> {
|
syscall(name: string, args: any[]): Promise<any> {
|
||||||
|
|
|
@ -18,6 +18,7 @@ export class Sandbox {
|
||||||
protected worker: WorkerLike;
|
protected worker: WorkerLike;
|
||||||
protected reqId = 0;
|
protected reqId = 0;
|
||||||
protected outstandingInits = new Map<string, () => void>();
|
protected outstandingInits = new Map<string, () => void>();
|
||||||
|
protected outstandingDependencyInits = new Map<string, () => void>();
|
||||||
protected outstandingInvocations = new Map<
|
protected outstandingInvocations = new Map<
|
||||||
number,
|
number,
|
||||||
{ resolve: (result: any) => void; reject: (e: any) => void }
|
{ resolve: (result: any) => void; reject: (e: any) => void }
|
||||||
|
@ -63,6 +64,22 @@ export class Sandbox {
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async loadDependency(name: string, code: string): Promise<void> {
|
||||||
|
// console.log("Loading dependency", name);
|
||||||
|
this.worker.postMessage({
|
||||||
|
type: "load-dependency",
|
||||||
|
name: name,
|
||||||
|
code: code,
|
||||||
|
} as WorkerMessage);
|
||||||
|
return new Promise((resolve) => {
|
||||||
|
// console.log("Loaded dependency", name);
|
||||||
|
this.outstandingDependencyInits.set(name, () => {
|
||||||
|
this.outstandingDependencyInits.delete(name);
|
||||||
|
resolve();
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
async onMessage(data: ControllerMessage) {
|
async onMessage(data: ControllerMessage) {
|
||||||
switch (data.type) {
|
switch (data.type) {
|
||||||
case "inited":
|
case "inited":
|
||||||
|
@ -70,6 +87,11 @@ export class Sandbox {
|
||||||
initCb && initCb();
|
initCb && initCb();
|
||||||
this.outstandingInits.delete(data.name!);
|
this.outstandingInits.delete(data.name!);
|
||||||
break;
|
break;
|
||||||
|
case "dependency-inited":
|
||||||
|
let depInitCb = this.outstandingDependencyInits.get(data.name!);
|
||||||
|
depInitCb && depInitCb();
|
||||||
|
this.outstandingDependencyInits.delete(data.name!);
|
||||||
|
break;
|
||||||
case "syscall":
|
case "syscall":
|
||||||
try {
|
try {
|
||||||
let result = await this.plug.syscall(data.name!, data.args!);
|
let result = await this.plug.syscall(data.name!, data.args!);
|
||||||
|
|
|
@ -10,36 +10,95 @@ const exposedModules = [
|
||||||
"yaml",
|
"yaml",
|
||||||
];
|
];
|
||||||
|
|
||||||
|
import * as ts from "typescript";
|
||||||
|
|
||||||
|
type CompileError = {
|
||||||
|
message: string;
|
||||||
|
pos: number;
|
||||||
|
};
|
||||||
|
|
||||||
|
function checkTypeScript(scriptFile: string): void {
|
||||||
|
let program = ts.createProgram([scriptFile], {
|
||||||
|
noEmit: true,
|
||||||
|
allowJs: true,
|
||||||
|
});
|
||||||
|
let emitResult = program.emit();
|
||||||
|
|
||||||
|
let allDiagnostics = ts
|
||||||
|
.getPreEmitDiagnostics(program)
|
||||||
|
.concat(emitResult.diagnostics);
|
||||||
|
|
||||||
|
let errors: CompileError[] = [];
|
||||||
|
allDiagnostics.forEach((diagnostic) => {
|
||||||
|
if (diagnostic.file) {
|
||||||
|
let { line, character } = ts.getLineAndCharacterOfPosition(
|
||||||
|
diagnostic.file,
|
||||||
|
diagnostic.start!
|
||||||
|
);
|
||||||
|
let message = ts.flattenDiagnosticMessageText(
|
||||||
|
diagnostic.messageText,
|
||||||
|
"\n"
|
||||||
|
);
|
||||||
|
errors.push({
|
||||||
|
message: ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n"),
|
||||||
|
pos: diagnostic.start!,
|
||||||
|
});
|
||||||
|
// console.log(
|
||||||
|
// `${diagnostic.file.fileName} (${line + 1},${character + 1}): ${message}`
|
||||||
|
// );
|
||||||
|
} else {
|
||||||
|
console.log(
|
||||||
|
ts.flattenDiagnosticMessageText(diagnostic.messageText, "\n")
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
|
||||||
|
let exitCode = emitResult.emitSkipped ? 1 : 0;
|
||||||
|
console.log(`Process exiting with code '${exitCode}'.`);
|
||||||
|
process.exit(exitCode);
|
||||||
|
}
|
||||||
|
|
||||||
export function esbuildSyscalls(): SysCallMapping {
|
export function esbuildSyscalls(): SysCallMapping {
|
||||||
return {
|
return {
|
||||||
|
"tsc.analyze": async (
|
||||||
|
ctx,
|
||||||
|
filename: string,
|
||||||
|
code: string
|
||||||
|
): Promise<any> => {},
|
||||||
"esbuild.compile": async (
|
"esbuild.compile": async (
|
||||||
ctx,
|
ctx,
|
||||||
filename: string,
|
filename: string,
|
||||||
code: string,
|
code: string,
|
||||||
functionName?: string
|
functionName?: string
|
||||||
): Promise<string> => {
|
): Promise<string> => {
|
||||||
let tmpDir = `${tmpdir()}/plugos-${Math.random()}`;
|
let tmpDir = await prepareCompileEnv(filename, code);
|
||||||
await mkdir(tmpDir, { recursive: true });
|
|
||||||
|
|
||||||
const srcNodeModules = `${nodeModulesDir}/node_modules`;
|
|
||||||
const targetNodeModules = `${tmpDir}/node_modules`;
|
|
||||||
|
|
||||||
await mkdir(`${targetNodeModules}/@silverbulletmd`, { recursive: true });
|
|
||||||
await mkdir(`${targetNodeModules}/@plugos`, { recursive: true });
|
|
||||||
for (const exposedModule of exposedModules) {
|
|
||||||
await symlink(
|
|
||||||
`${srcNodeModules}/${exposedModule}`,
|
|
||||||
`${targetNodeModules}/${exposedModule}`,
|
|
||||||
"dir"
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
await writeFile(`${tmpDir}/${filename}`, code);
|
|
||||||
let jsCode = await compile(`${tmpDir}/${filename}`, functionName, false, [
|
let jsCode = await compile(`${tmpDir}/${filename}`, functionName, false, [
|
||||||
"yaml",
|
"yaml",
|
||||||
|
"handlebars",
|
||||||
]);
|
]);
|
||||||
await rm(tmpDir, { recursive: true });
|
await rm(tmpDir, { recursive: true });
|
||||||
return jsCode;
|
return jsCode;
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function prepareCompileEnv(filename: string, code: string) {
|
||||||
|
let tmpDir = `${tmpdir()}/plugos-${Math.random()}`;
|
||||||
|
await mkdir(tmpDir, { recursive: true });
|
||||||
|
|
||||||
|
const srcNodeModules = `${nodeModulesDir}/node_modules`;
|
||||||
|
const targetNodeModules = `${tmpDir}/node_modules`;
|
||||||
|
|
||||||
|
await mkdir(`${targetNodeModules}/@silverbulletmd`, { recursive: true });
|
||||||
|
await mkdir(`${targetNodeModules}/@plugos`, { recursive: true });
|
||||||
|
for (const exposedModule of exposedModules) {
|
||||||
|
await symlink(
|
||||||
|
`${srcNodeModules}/${exposedModule}`,
|
||||||
|
`${targetNodeModules}/${exposedModule}`,
|
||||||
|
"dir"
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
await writeFile(`${tmpDir}/${filename}`, code);
|
||||||
|
return tmpDir;
|
||||||
|
}
|
||||||
|
|
|
@ -3,6 +3,9 @@ import { System } from "./system";
|
||||||
export interface Manifest<HookT> {
|
export interface Manifest<HookT> {
|
||||||
name: string;
|
name: string;
|
||||||
requiredPermissions?: string[];
|
requiredPermissions?: string[];
|
||||||
|
dependencies?: {
|
||||||
|
[key: string]: string;
|
||||||
|
};
|
||||||
functions: {
|
functions: {
|
||||||
[key: string]: FunctionDef<HookT>;
|
[key: string]: FunctionDef<HookT>;
|
||||||
};
|
};
|
||||||
|
|
|
@ -422,6 +422,7 @@ function $9072202279b76d33$export$5884dae03c64f759(parsedQuery, records) {
|
||||||
}
|
}
|
||||||
async function $9072202279b76d33$export$b3c659c1456e61b0(parsedQuery, data) {
|
async function $9072202279b76d33$export$b3c659c1456e61b0(parsedQuery, data) {
|
||||||
if (parsedQuery.render) {
|
if (parsedQuery.render) {
|
||||||
|
console.log("Handlebars", ($parcel$interopDefault($hVExJ$handlebars)));
|
||||||
($parcel$interopDefault($hVExJ$handlebars)).registerHelper("json", (v)=>JSON.stringify(v)
|
($parcel$interopDefault($hVExJ$handlebars)).registerHelper("json", (v)=>JSON.stringify(v)
|
||||||
);
|
);
|
||||||
($parcel$interopDefault($hVExJ$handlebars)).registerHelper("niceDate", (ts)=>$c3893eec0c49ec96$export$5dc1410f87262ed6(new Date(ts))
|
($parcel$interopDefault($hVExJ$handlebars)).registerHelper("niceDate", (ts)=>$c3893eec0c49ec96$export$5dc1410f87262ed6(new Date(ts))
|
||||||
|
|
File diff suppressed because one or more lines are too long
|
@ -19,6 +19,7 @@ syntax:
|
||||||
styles:
|
styles:
|
||||||
color: "#0330cb"
|
color: "#0330cb"
|
||||||
textDecoration: underline
|
textDecoration: underline
|
||||||
|
cursor: pointer
|
||||||
functions:
|
functions:
|
||||||
clearPageIndex:
|
clearPageIndex:
|
||||||
path: "./page.ts:clearPageIndex"
|
path: "./page.ts:clearPageIndex"
|
||||||
|
|
|
@ -14,7 +14,7 @@ async function actionClickOrActionEnter(mdTree: ParseTree | null) {
|
||||||
if (!mdTree) {
|
if (!mdTree) {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
console.log("Attempting to navigate based on syntax node", mdTree);
|
// console.log("Attempting to navigate based on syntax node", mdTree);
|
||||||
switch (mdTree.type) {
|
switch (mdTree.type) {
|
||||||
case "WikiLinkPage":
|
case "WikiLinkPage":
|
||||||
let pageLink = mdTree.children![0].text!;
|
let pageLink = mdTree.children![0].text!;
|
||||||
|
|
|
@ -0,0 +1,5 @@
|
||||||
|
name: global
|
||||||
|
dependencies:
|
||||||
|
yaml: "yaml@2"
|
||||||
|
handlebars: "handlebars@4.7.7:/dist/handlebars"
|
||||||
|
"@lezer/lr": "@lezer/lr@0.15.4"
|
|
@ -8,8 +8,8 @@
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"generate": "lezer-generator query/query.grammar -o query/parse-query.js",
|
"generate": "lezer-generator query/query.grammar -o query/parse-query.js",
|
||||||
"watch": "plugos-bundle -w --dist dist --exclude @lezer/lr yaml handlebars -- */*.plug.yaml",
|
"watch": "plugos-bundle --dist ../common/dist global.plug.yaml && plugos-bundle -w --dist dist --exclude @lezer/lr yaml handlebars -- */*.plug.yaml",
|
||||||
"build": "plugos-bundle --dist dist --exclude @lezer/lr yaml handlebars -- */*.plug.yaml",
|
"build": "plugos-bundle --dist ../common/dist global.plug.yaml && plugos-bundle --dist dist --exclude @lezer/lr yaml handlebars -- */*.plug.yaml",
|
||||||
"test": "jest build/test"
|
"test": "jest build/test"
|
||||||
},
|
},
|
||||||
"files": [
|
"files": [
|
||||||
|
|
|
@ -36,6 +36,7 @@ export async function compileCommand() {
|
||||||
);
|
);
|
||||||
console.log("Wrote this plug", manifest);
|
console.log("Wrote this plug", manifest);
|
||||||
await hideBhs();
|
await hideBhs();
|
||||||
|
|
||||||
await reloadPlugs();
|
await reloadPlugs();
|
||||||
} catch (e: any) {
|
} catch (e: any) {
|
||||||
await showBhs(e.message);
|
await showBhs(e.message);
|
||||||
|
@ -136,6 +137,7 @@ export async function updatePlugs() {
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
let plugYaml = codeTextNode.children![0].text;
|
let plugYaml = codeTextNode.children![0].text;
|
||||||
|
console.log("YAML", YAML);
|
||||||
let plugList = YAML.parse(plugYaml!);
|
let plugList = YAML.parse(plugYaml!);
|
||||||
console.log("Plug YAML", plugList);
|
console.log("Plug YAML", plugList);
|
||||||
let allPlugNames: string[] = [];
|
let allPlugNames: string[] = [];
|
||||||
|
|
|
@ -221,6 +221,7 @@ export async function renderQuery(
|
||||||
data: any[]
|
data: any[]
|
||||||
): Promise<string> {
|
): Promise<string> {
|
||||||
if (parsedQuery.render) {
|
if (parsedQuery.render) {
|
||||||
|
console.log("Handlebars", Handlebars);
|
||||||
Handlebars.registerHelper("json", (v) => JSON.stringify(v));
|
Handlebars.registerHelper("json", (v) => JSON.stringify(v));
|
||||||
Handlebars.registerHelper("niceDate", (ts) => niceDate(new Date(ts)));
|
Handlebars.registerHelper("niceDate", (ts) => niceDate(new Date(ts)));
|
||||||
Handlebars.registerHelper("yaml", (v, prefix) => {
|
Handlebars.registerHelper("yaml", (v, prefix) => {
|
||||||
|
|
|
@ -1,4 +1,7 @@
|
||||||
name: query
|
name: query
|
||||||
|
# dependencies:
|
||||||
|
# yaml: "yaml@2"
|
||||||
|
# "@lezer/lr": "@lezer/lr@0.15.4"
|
||||||
functions:
|
functions:
|
||||||
updateMaterializedQueriesOnPage:
|
updateMaterializedQueriesOnPage:
|
||||||
path: ./materialized_queries.ts:updateMaterializedQueriesOnPage
|
path: ./materialized_queries.ts:updateMaterializedQueriesOnPage
|
||||||
|
|
|
@ -27,9 +27,12 @@ import { systemSyscalls } from "./syscalls/system";
|
||||||
import { plugPrefix } from "@silverbulletmd/common/spaces/constants";
|
import { plugPrefix } from "@silverbulletmd/common/spaces/constants";
|
||||||
|
|
||||||
import { Authenticator } from "./auth";
|
import { Authenticator } from "./auth";
|
||||||
import { nextTick } from "process";
|
|
||||||
import sandboxSyscalls from "@plugos/plugos/syscalls/sandbox";
|
import sandboxSyscalls from "@plugos/plugos/syscalls/sandbox";
|
||||||
|
|
||||||
|
import globalModules from "../common/dist/global.plug.json";
|
||||||
|
|
||||||
|
import { safeRun } from "./util";
|
||||||
|
|
||||||
const safeFilename = /^[a-zA-Z0-9_\-\.]+$/;
|
const safeFilename = /^[a-zA-Z0-9_\-\.]+$/;
|
||||||
|
|
||||||
export type ServerOptions = {
|
export type ServerOptions = {
|
||||||
|
@ -37,7 +40,6 @@ export type ServerOptions = {
|
||||||
pagesPath: string;
|
pagesPath: string;
|
||||||
distDir: string;
|
distDir: string;
|
||||||
builtinPlugDir: string;
|
builtinPlugDir: string;
|
||||||
preloadedModules: string[];
|
|
||||||
token?: string;
|
token?: string;
|
||||||
};
|
};
|
||||||
export class ExpressServer {
|
export class ExpressServer {
|
||||||
|
@ -50,7 +52,6 @@ export class ExpressServer {
|
||||||
private port: number;
|
private port: number;
|
||||||
private server?: Server;
|
private server?: Server;
|
||||||
builtinPlugDir: string;
|
builtinPlugDir: string;
|
||||||
preloadedModules: string[];
|
|
||||||
token?: string;
|
token?: string;
|
||||||
|
|
||||||
constructor(options: ServerOptions) {
|
constructor(options: ServerOptions) {
|
||||||
|
@ -59,7 +60,6 @@ export class ExpressServer {
|
||||||
this.builtinPlugDir = options.builtinPlugDir;
|
this.builtinPlugDir = options.builtinPlugDir;
|
||||||
this.distDir = options.distDir;
|
this.distDir = options.distDir;
|
||||||
this.system = new System<SilverBulletHooks>("server");
|
this.system = new System<SilverBulletHooks>("server");
|
||||||
this.preloadedModules = options.preloadedModules;
|
|
||||||
this.token = options.token;
|
this.token = options.token;
|
||||||
|
|
||||||
// Setup system
|
// Setup system
|
||||||
|
@ -96,6 +96,18 @@ export class ExpressServer {
|
||||||
);
|
);
|
||||||
this.system.addHook(new EndpointHook(this.app, "/_"));
|
this.system.addHook(new EndpointHook(this.app, "/_"));
|
||||||
|
|
||||||
|
this.system.on({
|
||||||
|
plugLoaded: (plug) => {
|
||||||
|
safeRun(async () => {
|
||||||
|
for (let [modName, code] of Object.entries(
|
||||||
|
globalModules.dependencies
|
||||||
|
)) {
|
||||||
|
await plug.sandbox.loadDependency(modName, code);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
this.eventHook.addLocalListener(
|
this.eventHook.addLocalListener(
|
||||||
"get-plug:builtin",
|
"get-plug:builtin",
|
||||||
async (plugName: string): Promise<Manifest> => {
|
async (plugName: string): Promise<Manifest> => {
|
||||||
|
@ -165,9 +177,7 @@ export class ExpressServer {
|
||||||
console.log("Reloading plugs");
|
console.log("Reloading plugs");
|
||||||
for (let pageInfo of allPlugs) {
|
for (let pageInfo of allPlugs) {
|
||||||
let { text } = await this.space.readPage(pageInfo.name);
|
let { text } = await this.space.readPage(pageInfo.name);
|
||||||
await this.system.load(JSON.parse(text), (p) =>
|
await this.system.load(JSON.parse(text), createSandbox);
|
||||||
createSandbox(p, this.preloadedModules)
|
|
||||||
);
|
|
||||||
}
|
}
|
||||||
this.rebuildMdExtensions();
|
this.rebuildMdExtensions();
|
||||||
}
|
}
|
||||||
|
|
|
@ -1,6 +1,5 @@
|
||||||
#!/usr/bin/env -S node --enable-source-maps
|
#!/usr/bin/env -S node --enable-source-maps
|
||||||
import { nodeModulesDir } from "@plugos/plugos/environments/node_sandbox";
|
import { nodeModulesDir } from "@plugos/plugos/environments/node_sandbox";
|
||||||
import { preloadModules } from "@silverbulletmd/common/preload_modules";
|
|
||||||
import { realpathSync } from "fs";
|
import { realpathSync } from "fs";
|
||||||
import yargs from "yargs";
|
import yargs from "yargs";
|
||||||
import { hideBin } from "yargs/helpers";
|
import { hideBin } from "yargs/helpers";
|
||||||
|
@ -39,7 +38,6 @@ console.log("Builtin plug dist dir", plugDistDir);
|
||||||
const expressServer = new ExpressServer({
|
const expressServer = new ExpressServer({
|
||||||
port: port,
|
port: port,
|
||||||
pagesPath: pagesPath,
|
pagesPath: pagesPath,
|
||||||
preloadedModules: preloadModules,
|
|
||||||
distDir: webappDistDir,
|
distDir: webappDistDir,
|
||||||
builtinPlugDir: plugDistDir,
|
builtinPlugDir: plugDistDir,
|
||||||
token: args.token,
|
token: args.token,
|
||||||
|
|
|
@ -1,8 +1,3 @@
|
||||||
declare global {
|
|
||||||
function syscall(name: string, ...args: any[]): Promise<any>;
|
|
||||||
// function require(moduleName: string): any;
|
|
||||||
}
|
|
||||||
|
|
||||||
window.addEventListener("message", (message) => {
|
window.addEventListener("message", (message) => {
|
||||||
const data = message.data;
|
const data = message.data;
|
||||||
switch (data.type) {
|
switch (data.type) {
|
||||||
|
|
|
@ -59,6 +59,7 @@ import { FilterList } from "./components/filter";
|
||||||
import { FilterOption } from "@silverbulletmd/common/types";
|
import { FilterOption } from "@silverbulletmd/common/types";
|
||||||
import { syntaxTree } from "@codemirror/language";
|
import { syntaxTree } from "@codemirror/language";
|
||||||
import sandboxSyscalls from "@plugos/plugos/syscalls/sandbox";
|
import sandboxSyscalls from "@plugos/plugos/syscalls/sandbox";
|
||||||
|
import globalModules from "../common/dist/global.plug.json";
|
||||||
|
|
||||||
class PageState {
|
class PageState {
|
||||||
constructor(
|
constructor(
|
||||||
|
@ -131,6 +132,18 @@ export class Editor {
|
||||||
clientStoreSyscalls(),
|
clientStoreSyscalls(),
|
||||||
sandboxSyscalls(this.system)
|
sandboxSyscalls(this.system)
|
||||||
);
|
);
|
||||||
|
|
||||||
|
this.system.on({
|
||||||
|
plugLoaded: (plug) => {
|
||||||
|
safeRun(async () => {
|
||||||
|
for (let [modName, code] of Object.entries(
|
||||||
|
globalModules.dependencies
|
||||||
|
)) {
|
||||||
|
await plug.sandbox.loadDependency(modName, code);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
},
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
get currentPage(): string | undefined {
|
get currentPage(): string | undefined {
|
||||||
|
|
|
@ -1,5 +1,8 @@
|
||||||
import { Action, AppViewState } from "./types";
|
import { Action, AppViewState } from "./types";
|
||||||
|
|
||||||
|
let m = new Map();
|
||||||
|
m.size;
|
||||||
|
|
||||||
export default function reducer(
|
export default function reducer(
|
||||||
state: AppViewState,
|
state: AppViewState,
|
||||||
action: Action
|
action: Action
|
||||||
|
|
|
@ -171,6 +171,7 @@
|
||||||
cursor: pointer;
|
cursor: pointer;
|
||||||
}
|
}
|
||||||
.wiki-link {
|
.wiki-link {
|
||||||
|
cursor: pointer;
|
||||||
color: #a8abbd;
|
color: #a8abbd;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue