mingzailao
/
silverbullet
mirror of https://github.com/silverbulletmd/silverbullet.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Service workers fixed (I think)

pull/3/head
Zef Hemel 2022-04-25 19:46:08 +02:00
parent d52d81a909
commit cb2d3f8652
16 changed files with 350 additions and 93 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
package-lock.json generated
View File

@ -2627,6 +2627,7 @@
"node_modules/@vscode/sqlite3": {
"version": "5.0.8",
"dev": true,
"hasInstallScript": true,
"license": "BSD-3-Clause",
"dependencies": {
"node-addon-api": "^4.2.0"
@ -9771,7 +9772,7 @@
"@codemirror/stream-parser": "^0.19.9",
"@jest/globals": "^27.5.1",
"@lezer/markdown": "^0.15.0",
"@silverbulletmd/web": "^0.0.1",
"@silverbulletmd/web": "^0.0.2",
"better-sqlite3": "^7.5.0",
"body-parser": "^1.19.2",
"buffer": "^6.0.3",
@ -9815,7 +9816,7 @@
},
"packages/web": {
"name": "@silverbulletmd/web",
"version": "0.0.1",
"version": "0.0.2",
"license": "MIT",
"dependencies": {
"@codemirror/autocomplete": "^0.19.1",
@ -11351,7 +11352,7 @@
"@parcel/transformer-sass": "2.3.2",
"@parcel/transformer-webmanifest": "2.3.2",
"@parcel/validator-typescript": "2.3.2",
"@silverbulletmd/web": "^0.0.1",
"@silverbulletmd/web": "^0.0.2",
"@types/cors": "^2.8.12",
"@types/events": "^3.0.0",
"@types/express": "^4.17.13",

2
package.json
View File

@ -7,7 +7,7 @@
"@lezer/common": "https://github.com/zefhemel/common.git#046c880d1fcab713cadad327a5b7d8bb5de6522c"
},
"scripts": {
"watch": "rm -rf .parcel-cache && parcel watch packages/{web,server,plugos} desktop",
"watch": "rm -rf .parcel-cache && parcel watch --no-hmr packages/{web,server,plugos} desktop",
"clean": "rm -rf .parcel-cache packages/*/dist",
"build": "parcel build packages/{web,server,plugos}",
"plugs": "cd packages/plugs && npm run watch",

4
packages/common/spaces/http_space_primitives.ts
View File

@ -119,7 +119,11 @@ export class HttpSpacePrimitives implements SpacePrimitives {
if (req.headers.get("Content-length") === "0") {
return;
}
if (req.headers.get("Content-type") === "application/json") {
return await req.json();
} else {
return await req.text();
}
}
async getPageMeta(name: string): Promise<PageMeta> {

53
packages/plugos/bin/plugos-bundle.ts
View File

@ -1,6 +1,5 @@
#!/usr/bin/env node
import esbuild from "esbuild";
import { readFile, unlink, watch, writeFile } from "fs/promises";
import path from "path";
@ -9,53 +8,7 @@ import { hideBin } from "yargs/helpers";
import { Manifest } from "../types";
import YAML from "yaml";
import { mkdirSync } from "fs";
async function compile(
filePath: string,
functionName: string,
debug: boolean,
excludeModules: string[],
meta = false
) {
let outFile = "_out.tmp";
let inFile = filePath;
if (functionName) {
// Generate a new file importing just this one function and exporting it
inFile = "_in.js";
await writeFile(
inFile,
`import {${functionName}} from "./${filePath}";export default ${functionName};`
);
}
// TODO: Figure out how to make source maps work correctly with eval() code
let result = await esbuild.build({
entryPoints: [inFile],
bundle: true,
format: "iife",
globalName: "mod",
platform: "browser",
sourcemap: false, //sourceMap ? "inline" : false,
minify: !debug,
outfile: outFile,
metafile: true,
external: excludeModules,
});
if (meta) {
let text = await esbuild.analyzeMetafile(result.metafile);
console.log("Bundle info for", functionName, text);
}
let jsCode = (await readFile(outFile)).toString();
await unlink(outFile);
if (inFile !== filePath) {
await unlink(inFile);
}
return `(() => { ${jsCode}
return mod;})()`;
}
import { compile } from "../compile";
async function bundle(
manifestPath: string,
@ -155,8 +108,8 @@ async function run() {
})) {
if (
filename.endsWith(".plug.yaml") ||
filename.endsWith(".ts") ||
(filename.endsWith(".js") && !filename.endsWith("_in.js"))
filename.endsWith(".js") ||
(filename.endsWith(".ts") && !filename.endsWith("_in.ts"))
) {
console.log("Change detected", eventType, filename);
await buildAll();

51
packages/plugos/compile.ts Normal file
View File

@ -0,0 +1,51 @@
import esbuild from "esbuild";
import { readFile, unlink, writeFile } from "fs/promises";
import path from "path";
export async function compile(
filePath: string,
functionName: string = "",
debug: boolean = false,
excludeModules: string[] = [],
meta = false
): Promise<string> {
let outFile = path.join(path.dirname(filePath), "_out.tmp");
let inFile = filePath;
if (functionName) {
// Generate a new file importing just this one function and exporting it
inFile = "_in.ts";
await writeFile(
inFile,
`import {${functionName}} from "./${filePath}";export default ${functionName};`
);
}
// TODO: Figure out how to make source maps work correctly with eval() code
let result = await esbuild.build({
entryPoints: [path.basename(inFile)],
bundle: true,
format: "iife",
globalName: "mod",
platform: "browser",
sourcemap: false, //sourceMap ? "inline" : false,
minify: !debug,
outfile: outFile,
metafile: true,
external: excludeModules,
absWorkingDir: path.resolve(path.dirname(inFile)),
});
if (meta) {
let text = await esbuild.analyzeMetafile(result.metafile);
console.log("Bundle info for", functionName, text);
}
let jsCode = (await readFile(outFile)).toString();
await unlink(outFile);
if (inFile !== filePath) {
await unlink(inFile);
}
return `(() => { ${jsCode}
return mod;})()`;
}

44
packages/plugos/syscalls/esbuild.ts Normal file
View File

@ -0,0 +1,44 @@
import { compile } from "../compile";
import { SysCallMapping } from "../system";
import { tmpdir } from "os";
import { mkdir, rm, symlink, writeFile } from "fs/promises";
import { nodeModulesDir } from "../environments/node_sandbox";
const exposedModules = [
"@silverbulletmd/plugos-silverbullet-syscall",
"@plugos/plugos-syscall",
"yaml",
];
export function esbuildSyscalls(): SysCallMapping {
return {
"esbuild.compile": async (
ctx,
filename: string,
code: string
): Promise<string> => {
let tmpDir = `${tmpdir()}/plugos-${Math.random()}`;
await mkdir(tmpDir, { recursive: true });
const srcNodeModules = `${nodeModulesDir}/node_modules`;
const targetNodeModules = `${tmpDir}/node_modules`;
await mkdir(`${targetNodeModules}/@silverbulletmd`, { recursive: true });
await mkdir(`${targetNodeModules}/@plugos`, { recursive: true });
for (const exposedModule of exposedModules) {
await symlink(
`${srcNodeModules}/${exposedModule}`,
`${targetNodeModules}/${exposedModule}`,
"dir"
);
}
await writeFile(`${tmpDir}/${filename}`, code);
console.log("Dir", tmpDir);
let jsCode = await compile(`${tmpDir}/${filename}`, "", false, ["yaml"]);
// console.log("JS code", jsCode);
await rm(tmpDir, { recursive: true });
return jsCode;
},
};
}

113
packages/plugos/yarn-error.log Normal file
View File

@ -0,0 +1,113 @@
Arguments:
/Users/zef/.nvm/versions/node/v16.13.2/bin/node /Users/zef/.nvm/versions/node/v16.13.2/bin/yarn add --json @parcel/transformer-babel -D -W
PATH:
/Users/zef/git/silverbullet/packages/plugos/node_modules/.bin:/Users/zef/git/silverbullet/packages/node_modules/.bin:/Users/zef/git/silverbullet/node_modules/.bin:/Users/zef/git/node_modules/.bin:/Users/zef/node_modules/.bin:/Users/node_modules/.bin:/node_modules/.bin:/Users/zef/.nvm/versions/node/v16.13.2/lib/node_modules/npm/node_modules/@npmcli/run-script/lib/node-gyp-bin:/Users/zef/.nvm/versions/node/v16.13.2/bin:/Library/Frameworks/Python.framework/Versions/2.7/bin:/Users/zef/.local/share/solana/install/active_release/bin:/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin:/Users/zef/.nvm/versions/node/v16.13.2/bin:/Library/Frameworks/Python.framework/Versions/2.7/bin:/Users/zef/.local/share/solana/install/active_release/bin:/opt/homebrew/bin:/opt/homebrew/sbin:/Users/zef/.cargo/bin
Yarn version:
1.22.18
Node version:
16.13.2
Platform:
darwin arm64
Trace:
Error: ENOENT: no such file or directory, scandir '/Users/zef/git/silverbullet/node_modules/@plugos-silverbulet-syscall'
npm manifest:
{
"name": "@plugos/plugos",
"author": {
"name": "Zef Hemel",
"email": "zef@zef.me"
},
"version": "0.0.1",
"license": "MIT",
"bin": {
"plugos-bundle": "./dist/plugos/plugos-bundle.js",
"plugos-server": "./dist/plugos/plugos-server.js"
},
"scripts": {
"watch": "rm -rf .parcel-cache && parcel watch",
"build": "parcel build",
"clean": "rm -rf dist",
"test": "jest dist/test"
},
"targets": {
"plugos": {
"source": [
"bin/plugos-bundle.ts",
"bin/plugos-server.ts"
],
"outputFormat": "commonjs",
"isLibrary": true,
"context": "node"
},
"test": {
"source": [
"runtime.test.ts",
"hooks/endpoint.test.ts",
"syscalls/store.knex_node.test.ts",
"syscalls/store.dexie_browser.test.ts"
],
"outputFormat": "commonjs",
"isLibrary": true,
"context": "node"
}
},
"dependencies": {
"@jest/globals": "^27.5.1",
"@types/cors": "^2.8.12",
"@types/express": "^4.17.13",
"@types/jsonwebtoken": "^8.5.8",
"better-sqlite3": "^7.5.0",
"body-parser": "^1.19.2",
"cors": "^2.8.5",
"dexie": "^3.2.1",
"esbuild": "^0.14.27",
"express": "^4.17.3",
"fake-indexeddb": "^3.1.7",
"jest": "^27.5.1",
"jsonwebtoken": "^8.5.1",
"knex": "^1.0.4",
"node-cron": "^3.0.0",
"node-fetch": "2",
"node-watch": "^0.7.3",
"supertest": "^6.2.2",
"vm2": "^3.9.9",
"ws": "^8.5.0",
"yaml": "^1.10.2",
"yargs": "^17.3.1"
},
"devDependencies": {
"@lezer/lr": "^0.15.0",
"@parcel/optimizer-data-url": "2.3.2",
"@parcel/packager-raw-url": "2.3.2",
"@parcel/service-worker": "2.3.2",
"@parcel/transformer-inline-string": "2.3.2",
"@parcel/transformer-sass": "2.3.2",
"@parcel/transformer-webmanifest": "2.3.2",
"@parcel/validator-typescript": "2.3.2",
"@types/events": "^3.0.0",
"@types/jest": "^27.4.1",
"@types/node": "^17.0.21",
"@types/node-cron": "^3.0.1",
"@types/node-fetch": "^2.6.1",
"@types/supertest": "^2.0.11",
"@types/yaml": "^1.9.7",
"@vscode/sqlite3": "^5.0.7",
"assert": "^2.0.0",
"events": "^3.3.0",
"parcel": "2.3.2",
"prettier": "^2.5.1",
"typescript": "^4.6.2"
}
}
yarn manifest:
No manifest
Lockfile:
No lockfile

5
packages/plugs/draft/draft.plug.yaml
View File

@ -1,5 +0,0 @@
functions:
toggleDraftMode:
path: "./draft.ts:toggleMode"
command:
name: "Draft: Toggle Mode"

7
packages/plugs/draft/draft.ts
View File

@ -1,7 +0,0 @@
import * as clientStore from "@silverbulletmd/plugos-silverbullet-syscall/clientStore";
export async function toggleMode() {
let currentValue = !!(await clientStore.get("enableDraftMode"));
console.log("New draft mode", !currentValue);
await clientStore.set("enableDraftMode", !currentValue);
}

8
packages/plugs/plugger/plugger.plug.yaml Normal file
View File

@ -0,0 +1,8 @@
functions:
compile:
path: "./plugger.ts:compileCommand"
command:
name: "Plugger: Compile"
compileJS:
path: "./plugger.ts:compileJS"
env: server

85
packages/plugs/plugger/plugger.ts Normal file
View File

@ -0,0 +1,85 @@
import type { Manifest } from "@silverbulletmd/common/manifest";
import {
addParentPointers,
collectNodesOfType,
findNodeOfType,
} from "@silverbulletmd/common/tree";
import {
getCurrentPage,
getText,
} from "@silverbulletmd/plugos-silverbullet-syscall/editor";
import { parseMarkdown } from "@silverbulletmd/plugos-silverbullet-syscall/markdown";
import { writePage } from "@silverbulletmd/plugos-silverbullet-syscall/space";
import { invokeFunction } from "@silverbulletmd/plugos-silverbullet-syscall/system";
import YAML from "yaml";
import { extractMeta } from "../query/data";
export async function compileCommand() {
let text = await getText();
let tree = await parseMarkdown(text);
addParentPointers(tree);
let allHeaders = collectNodesOfType(tree, "ATXHeading2");
let manifest: Manifest = {
functions: {},
};
for (let t of allHeaders) {
let parent = t.parent!;
let headerIdx = parent.children!.indexOf(t);
let headerTitle = t.children![1].text!.trim();
if (!headerTitle.startsWith("function ")) {
continue;
}
let functionName = headerTitle
.substring("function ".length)
.replace(/[^\w]/g, "_");
let meta: any;
let code: string | undefined;
let language = "js";
for (let i = headerIdx + 1; i < parent.children!.length; i++) {
let child = parent.children![i];
if (child.type === "FencedCode") {
let codeInfo = findNodeOfType(child, "CodeInfo")!.children![0].text!;
let codeText = findNodeOfType(child, "CodeText")!.children![0].text!;
if (codeInfo === "yaml") {
meta = YAML.parse(codeText);
continue;
}
if (codeInfo === "typescript" || codeInfo === "ts") {
language = "ts";
}
code = codeText;
}
if (child.type?.startsWith("ATXHeading")) {
break;
}
}
if (code) {
let compiled = await invokeFunction(
"server",
"compileJS",
`file.${language}`,
code
);
manifest.functions[functionName] = meta;
manifest.functions[functionName].code = compiled;
}
}
let pageMeta = extractMeta(tree);
if (pageMeta.name) {
await writePage(
`_plug/${pageMeta.name}`,
JSON.stringify(manifest, null, 2)
);
console.log("Wrote this plug", manifest);
}
}
export async function compileJS(
filename: string,
code: string
): Promise<string> {
return self.syscall("esbuild.compile", filename, code);
}

13
packages/server/api_server.ts
View File

@ -23,6 +23,7 @@ import { jwtSyscalls } from "@plugos/plugos/syscalls/jwt";
import buildMarkdown from "@silverbulletmd/web/parser";
import { loadMarkdownExtensions } from "@silverbulletmd/web/markdown_ext";
import http, { Server } from "http";
import { esbuildSyscalls } from "@plugos/plugos/syscalls/esbuild";
export class ExpressServer {
app: Express;
@ -72,6 +73,7 @@ export class ExpressServer {
this.system.registerSyscalls([], spaceSyscalls(this.space));
this.system.registerSyscalls([], eventSyscalls(this.eventHook));
this.system.registerSyscalls([], markdownSyscalls(buildMarkdown([])));
this.system.registerSyscalls([], esbuildSyscalls());
this.system.registerSyscalls([], jwtSyscalls());
this.system.addHook(new EndpointHook(this.app, "/_/"));
@ -268,12 +270,13 @@ export class ExpressServer {
);
// Fallback, serve index.html
let cachedIndex: string | undefined = undefined;
// let cachedIndex: string | undefined = undefined;
this.app.get("/*", async (req, res) => {
if (!cachedIndex) {
cachedIndex = await readFile(`${this.distDir}/index.html`, "utf8");
}
res.status(200).header("Content-Type", "text/html").send(cachedIndex);
// if (!cachedIndex) {
// let cachedIndex = await readFile(`${this.distDir}/index.html`, "utf8");
// }
res.sendFile(`${this.distDir}/index.html`, {});
// res.status(200).header("Content-Type", "text/html").send(cachedIndex);
});
this.server = http.createServer(this.app);

9
packages/server/server.ts
View File

@ -5,6 +5,9 @@ import { hideBin } from "yargs/helpers";
import { ExpressServer } from "./api_server";
import { nodeModulesDir } from "@plugos/plugos/environments/node_sandbox";
import { preloadModules } from "@silverbulletmd/common/preload_modules";
import path from "path";
import { realpath } from "fs/promises";
import { realpathSync } from "fs";
let args = yargs(hideBin(process.argv))
.option("port", {
@ -20,7 +23,11 @@ if (!args._.length) {
const pagesPath = args._[0] as string;
const port = args.port;
const webappDistDir = `${nodeModulesDir}/node_modules/@silverbulletmd/web/dist`;
const webappDistDir = realpathSync(
`${nodeModulesDir}/node_modules/@silverbulletmd/web/dist`
);
console.log("Webapp dist dir", webappDistDir);
const expressServer = new ExpressServer(
port,

18
packages/web/boot.ts
View File

@ -1,17 +1,16 @@
import { Editor } from "./editor";
import { safeRun } from "../common/util";
import { safeRun } from "@silverbulletmd/common/util";
import { Space } from "@silverbulletmd/common/spaces/space";
import { HttpSpacePrimitives } from "@silverbulletmd/common/spaces/http_space_primitives";
// let localSpace = new Space(new IndexedDBSpacePrimitives("pages"), true);
// localSpace.watch();
// @ts-ignore
let isDesktop = typeof window.desktop !== "undefined";
let serverSpace = new Space(new HttpSpacePrimitives(""), true);
serverSpace.watch();
console.log("Booting...");
// // @ts-ignore
// window.syncer = async () => {
// let lastLocalSync = +(localStorage.getItem("lastLocalSync") || "0"),
@ -39,10 +38,13 @@ safeRun(async () => {
// @ts-ignore
window.editor = editor;
if (!isDesktop) {
navigator.serviceWorker
// if (!isDesktop) {
// if (localStorage.getItem("disable_sw") !== "true") {
navigator.serviceWorker
.register(new URL("service_worker.ts", import.meta.url), { type: "module" })
.then((r) => {
console.log("Service worker registered", r);
console.log("Service worker registered...");
});
}
// }
// }

6
packages/web/editor.tsx
View File

@ -158,12 +158,6 @@ export class Editor {
}, 100);
this.space.on({
pageCreated: (meta) => {
console.log("Page created", meta);
},
pageDeleted: (meta) => {
console.log("Page delete", meta);
},
pageChanged: (meta) => {
if (this.currentPage === meta.name) {
console.log("Page changed on disk, reloading");

10
packages/web/service_worker.ts
View File

@ -2,19 +2,23 @@ import { manifest, version } from "@parcel/service-worker";
async function install() {
const cache = await caches.open(version);
// console.log("Installing", manifest);
console.log("Installing", manifest, "version", version);
await cache.addAll(manifest);
// console.log("DOne");
// @ts-ignore
self.skipWaiting(); // This automatically enables the service worker, preventing from caching stuff forever if there's a page open
console.log("Installed");
}
//@ts-ignore
self.addEventListener("install", (e) => e.waitUntil(install()));
async function activate() {
const keys = await caches.keys();
// console.log("Activating");
// console.log("Activating for ", keys, "new version", version);
await Promise.all(keys.map((key) => key !== version && caches.delete(key)));
// console.log("DOne activating");
}
//@ts-ignore
self.addEventListener("activate", (e) => e.waitUntil(activate()));