mingzailao
/
silverbullet
mirror of https://github.com/silverbulletmd/silverbullet.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Runs-ish?

pull/87/head
Zef Hemel 2022-10-06 15:14:21 +02:00
parent 4bb89563a5
commit f051e3de6b
38 changed files with 805 additions and 518 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
.gitignore vendored
View File

@ -1,14 +1,10 @@
pages
test_space
.DS_Store
node_modules
.parcel-cache
dist
build
generated
.yarnrc.yml
*.test.js
*.js.map
.vscode
website_build
data.db
/index.json

1
.vscode/configurationCache.log vendored Normal file
View File

@ -0,0 +1 @@
{"buildTargets":[],"launchTargets":[],"customConfigurationProvider":{"workspaceBrowse":{"browsePath":[],"compilerArgs":[]},"fileIndex":[]}}

6
.vscode/dryrun.log vendored Normal file
View File

@ -0,0 +1,6 @@
make --dry-run --always-make --keep-going --print-directory
make: Entering directory `/Users/zef/git/silverbullet'
make: Leaving directory `/Users/zef/git/silverbullet'
make: *** No targets specified and no makefile found. Stop.

7
.vscode/settings.json vendored Normal file
View File

@ -0,0 +1,7 @@
{
"editor.formatOnSave": true,
"typescriptHero.imports.stringQuoteStyle": "\"",
"deno.enable": true,
"deno.importMap": "import_map.json",
"deno.config": "deno.json"
}

254
.vscode/targets.log vendored Normal file
View File

@ -0,0 +1,254 @@
make all --print-data-base --no-builtin-variables --no-builtin-rules --question
# GNU Make 3.81
# Copyright (C) 2006 Free Software Foundation, Inc.
# This is free software; see the source for copying conditions.
# There is NO warranty; not even for MERCHANTABILITY or FITNESS FOR A
# PARTICULAR PURPOSE.
# This program built for i386-apple-darwin11.3.0
make: *** No rule to make target `all'. Stop.
# Make data base, printed on Mon Jul 4 09:43:18 2022
# Variables
# automatic
<D = $(patsubst %/,%,$(dir $<))
# automatic
?F = $(notdir $?)
# environment
VSCODE_LOG_NATIVE = false
# environment
NVM_INC = /Users/zef/.nvm/versions/node/v18.4.0/include/node
# automatic
?D = $(patsubst %/,%,$(dir $?))
# automatic
@D = $(patsubst %/,%,$(dir $@))
# automatic
@F = $(notdir $@)
# makefile
CURDIR := /Users/zef/git/silverbullet
# makefile
SHELL = /bin/sh
# environment
VSCODE_NLS_CONFIG = {"locale":"en-us","availableLanguages":{},"_languagePackSupport":true}
# environment
_ = /usr/bin/make
# makefile
MAKEFILE_LIST :=
# environment
VSCODE_VERBOSE_LOGGING = true
# environment
__CFBundleIdentifier = com.microsoft.VSCode
# environment
INFOPATH = /opt/homebrew/share/info:
# environment
VSCODE_IPC_HOOK_EXTHOST = /var/folders/s2/4nqrw2192hngtxg672qzc0nr0000gn/T/vscode-ipc-15b47298-2e0d-4c34-99d9-3434e4f1806c.sock
# environment
VSCODE_CWD = /
# environment
PATH = /Users/zef/.nvm/versions/node/v18.4.0/bin:/Library/Frameworks/Python.framework/Versions/2.7/bin:/Users/zef/.local/share/solana/install/active_release/bin:/opt/homebrew/bin:/opt/homebrew/sbin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/Library/Apple/usr/bin:/Users/zef/.cargo/bin:/Users/zef/.fig/bin:/Users/zef/.local/bin
# environment
LSCOLORS = Gxfxcxdxbxegedabagacad
# environment
NVM_BIN = /Users/zef/.nvm/versions/node/v18.4.0/bin
# environment
VSCODE_LOG_STACK = false
# environment
ELECTRON_RUN_AS_NODE = 1
# default
.FEATURES := target-specific order-only second-expansion else-if archives jobserver check-symlink
# environment
SSH_AUTH_SOCK = /private/tmp/com.apple.launchd.qE7FAVvbDO/Listeners
# automatic
%F = $(notdir $%)
# environment
TTY = not a tty
# environment
VSCODE_PIPE_LOGGING = true
# environment
FIG_PID = 75947
# environment
PWD = /Users/zef/git/silverbullet
# environment
HOMEBREW_CELLAR = /opt/homebrew/Cellar
# environment
ORIGINAL_XDG_CURRENT_DESKTOP = undefined
# environment
MANPATH = /Users/zef/.nvm/versions/node/v18.4.0/share/man:/opt/homebrew/share/man::
# environment
VSCODE_AMD_ENTRYPOINT = vs/workbench/api/node/extensionHostProcess
# environment
HOME = /Users/zef
# default
MAKEFILEPATH := /Applications/Xcode.app/Contents/Developer/Makefiles
# environment
VSCODE_CODE_CACHE_PATH = /Users/zef/Library/Application Support/Code/CachedData/30d9c6cd9483b2cc586687151bcbcd635f373630
# environment
LOGNAME = zef
# environment
APPLICATION_INSIGHTS_NO_DIAGNOSTIC_CHANNEL = 1
# environment
NVM_CD_FLAGS = -q
# environment
ZSH = /Users/zef/.local/share/fig/plugins/ohmyzsh
# environment
VSCODE_HANDLES_UNCAUGHT_ERRORS = true
# automatic
^D = $(patsubst %/,%,$(dir $^))
# environment
XPC_FLAGS = 0x0
# default
MAKE = $(MAKE_COMMAND)
# default
MAKECMDGOALS := all
# environment
SHLVL = 1
# default
MAKE_VERSION := 3.81
# environment
USER = zef
# makefile
.DEFAULT_GOAL :=
# environment
LESS = -R
# automatic
%D = $(patsubst %/,%,$(dir $%))
# default
MAKE_COMMAND := /Applications/Xcode.app/Contents/Developer/usr/bin/make
# default
.VARIABLES :=
# environment
TMPDIR = /var/folders/s2/4nqrw2192hngtxg672qzc0nr0000gn/T/
# automatic
*F = $(notdir $*)
# environment
VSCODE_IPC_HOOK = /Users/zef/Library/Application Support/Code/1.68.1-main.sock
# makefile
MAKEFLAGS = Rrqp
# environment
MFLAGS = -Rrqp
# automatic
*D = $(patsubst %/,%,$(dir $*))
# environment
NVM_DIR = /Users/zef/.nvm
# environment
XPC_SERVICE_NAME = application.com.microsoft.VSCode.109834161.109834167
# environment
HOMEBREW_PREFIX = /opt/homebrew
# automatic
+D = $(patsubst %/,%,$(dir $+))
# automatic
+F = $(notdir $+)
# environment
HOMEBREW_REPOSITORY = /opt/homebrew
# environment
__CF_USER_TEXT_ENCODING = 0x1F5:0x0:0x0
# environment
COMMAND_MODE = unix2003
# default
MAKEFILES :=
# automatic
<F = $(notdir $<)
# environment
PAGER = less
# environment
LC_ALL = C
# automatic
^F = $(notdir $^)
# default
SUFFIXES :=
# environment
MAKELEVEL := 0
# environment
LANG = C
# environment
VSCODE_PID = 75884
# variable set hash-table stats:
# Load=76/1024=7%, Rehash=0, Collisions=1/96=1%
# Pattern-specific Variable Values
# No pattern-specific variable values.
# Directories
# . (device 16777232, inode 91684251): 20 files, no impossibilities.
# 20 files, no impossibilities in 1 directories.
# Implicit Rules
# No implicit rules.
# Files
# Not a target:
all:
# Command-line target.
# Implicit rule search has been done.
# File does not exist.
# File has not been updated.
# variable set hash-table stats:
# Load=0/32=0%, Rehash=0, Collisions=0/0=0%
# Not a target:
.SUFFIXES:
# Implicit rule search has not been done.
# Modification time never checked.
# File has not been updated.
# Not a target:
Makefile:
# A default, MAKEFILES, or -include/sinclude makefile.
# Implicit rule search has been done.
# File does not exist.
# File has been updated.
# Failed to be updated.
# variable set hash-table stats:
# Load=0/32=0%, Rehash=0, Collisions=0/0=0%
# Not a target:
makefile:
# A default, MAKEFILES, or -include/sinclude makefile.
# Implicit rule search has been done.
# File does not exist.
# File has been updated.
# Failed to be updated.
# variable set hash-table stats:
# Load=0/32=0%, Rehash=0, Collisions=0/0=0%
# Not a target:
.DEFAULT:
# Implicit rule search has not been done.
# Modification time never checked.
# File has not been updated.
# Not a target:
GNUmakefile:
# A default, MAKEFILES, or -include/sinclude makefile.
# Implicit rule search has been done.
# File does not exist.
# File has been updated.
# Failed to be updated.
# variable set hash-table stats:
# Load=0/32=0%, Rehash=0, Collisions=0/0=0%
# files hash-table stats:
# Load=6/1024=1%, Rehash=0, Collisions=0/17=0%
# VPATH Search Paths
# No `vpath' search paths.
# No general (`VPATH' variable) search path.
# # of strings in strcache: 0
# # of strcache buffers: 0
# strcache size: total = 0 / max = 0 / min = 4096 / avg = 0
# strcache free: total = 0 / max = 0 / min = 4096 / avg = 0
# Finished Make data base on Mon Jul 4 09:43:18 2022

16
.vscode/tasks.json vendored Normal file
View File

@ -0,0 +1,16 @@
{
"version": "2.0.0",
"tasks": [
{
"type": "npm",
"script": "test",
"problemMatcher": [],
"label": "npm: test",
"detail": "jest packages/*/{dist,build}/test",
"group": {
"kind": "test",
"isDefault": true
}
}
]
}

9
Makefile Normal file
View File

@ -0,0 +1,9 @@
plugs:
deno run -A --unstable packages/plugos/bin/plugos-bundle.ts --dist dist packages/plugs/global.plug.yaml
deno run -A --unstable packages/plugos/bin/plugos-bundle.ts --dist packages/plugs/dist --exclude=https://esm.sh/handlebars,https://deno.land/std@0.158.0/encoding/yaml.ts packages/plugs/*/*.plug.yaml
test:
deno test -A --unstable
watch:
deno task watch

9
build.ts
View File

@ -28,10 +28,13 @@ async function copyAssets(dest: string) {
await Deno.writeTextFile("dist/main.css", compiler.to_string() as string);
}
async function bundle(appFile: string): Promise<void> {
async function bundle(): Promise<void> {
await Promise.all([
esbuild.build({
entryPoints: [appFile],
entryPoints: {
"client": "packages/web/boot.ts",
"worker": "packages/plugos/environments/sandbox_worker.ts",
},
outdir: "./dist",
absWorkingDir: Deno.cwd(),
bundle: true,
@ -57,5 +60,5 @@ async function bundle(appFile: string): Promise<void> {
await copyAssets("dist");
console.log("Built!");
}
await bundle("packages/web/boot.ts");
await bundle();
// esbuild.stop();

3
deno.json
View File

@ -5,7 +5,8 @@
"importMap": "import_map.json",
"tasks": {
"test": "deno test -A --unstable",
"watch": "deno run -A build.ts"
"watch": "deno run -A build.ts",
"plugs": "deno run -A --unstable packages/plugos/bin/plugos-bundle.ts --dist packages/plugs/dist --exclude=https://esm.sh/handlebars,https://deno.land/std@0.158.0/encoding/yaml.ts packages/plugs/*/*.plug.yaml"
}
}

1
dep_server.ts
View File

@ -1,2 +1,3 @@
export * from "./dep_common.ts";
export { Database as SQLite } from "https://deno.land/x/sqlite3@0.6.1/mod.ts";
export { Application, Router } from "https://deno.land/x/oak@v11.1.0/mod.ts";

5
import_map.json
View File

@ -2,6 +2,9 @@
"imports": {
"@codemirror/state": "https://esm.sh/@codemirror/state",
"@codemirror/language": "https://esm.sh/@codemirror/language",
"@lezer/lr": "https://esm.sh/@lezer/lr@1.2.3"
"@lezer/lr": "https://esm.sh/@lezer/lr@1.2.3",
"yaml": "https://deno.land/std@0.158.0/encoding/yaml.ts",
"$sb/": "./packages/",
"handlebars": "https://esm.sh/handlebars"
}
}

2
packages/plugos/bin/plugos-bundle.ts
View File

@ -40,7 +40,7 @@ async function bundle(
jsFunctionName,
debug,
allModulesToExclude,
true,
false,
);
delete def.path;
}

22
packages/plugos/environments/sandbox_worker.ts
View File

@ -2,6 +2,22 @@ import { safeRun } from "../util.ts";
import { ConsoleLogger } from "./custom_logger.ts";
import { ControllerMessage, WorkerMessage } from "./worker.ts";
if (typeof Deno === "undefined") {
// @ts-ignore: Deno hack
self.Deno = {
args: [],
// @ts-ignore: Deno hack
build: {
arch: "x86_64",
},
env: {
// @ts-ignore: Deno hack
get() {
},
},
};
}
let loadedFunctions = new Map<string, Function>();
let pendingRequests = new Map<
number,
@ -43,7 +59,7 @@ let loadedModules = new Map<string, any>();
// @ts-ignore
self.require = (moduleName: string): any => {
console.log("Requiring", moduleName, loadedModules.get(moduleName));
// console.log("Requiring", moduleName, loadedModules.get(moduleName));
return loadedModules.get(moduleName);
};
@ -73,7 +89,7 @@ self.addEventListener("message", (event: { data: WorkerMessage }) => {
break;
case "load-dependency":
{
console.log("Received dep", data.name);
// console.log("Received dep", data.name);
let fn3 = new Function(`return ${data.code!}`);
let v = fn3();
loadedModules.set(data.name!, v);
@ -119,7 +135,7 @@ self.addEventListener("message", (event: { data: WorkerMessage }) => {
"Current outstanding requests",
pendingRequests,
"looking up",
syscallId
syscallId,
);
throw Error("Invalid request id");
}

10
packages/plugos/environments/webworker_sandbox.ts
View File

@ -29,9 +29,13 @@ class WebWorkerWrapper implements WorkerLike {
}
export function createSandbox(plug: Plug<any>) {
// ParcelJS will build this file into a worker.
let worker = new Worker(new URL("sandbox_worker.ts", import.meta.url), {
const worker = new Worker(
import.meta.url
? new URL("sandbox_worker.ts", import.meta.url)
: new URL("worker.js", location.origin),
{
type: "module",
});
},
);
return new Sandbox(plug, new WebWorkerWrapper(worker));
}

4
packages/plugos/hooks/endpoint.test.ts
View File

@ -3,8 +3,8 @@ import { Manifest } from "../types.ts";
import { EndpointHook, EndpointHookT } from "./endpoint.ts";
import { System } from "../system.ts";
import { Application } from "https://deno.land/x/oak/mod.ts";
import { assertEquals } from "https://deno.land/std@0.123.0/testing/asserts.ts";
import { Application } from "../../../dep_server.ts";
import { assertEquals } from "../../../test_dep.ts";
Deno.test("Run a plugos endpoint server", async () => {
let system = new System<EndpointHookT>("server");

4
packages/plugos/hooks/endpoint.ts
View File

@ -1,6 +1,6 @@
import { Hook, Manifest } from "../types.ts";
import { System } from "../system.ts";
import { Application } from "https://deno.land/x/oak@v10.2.1/application.ts";
import { Application } from "../../../dep_server.ts";
export type EndpointRequest = {
method: string;
@ -103,7 +103,7 @@ export class EndpointHook implements Hook<EndpointHookT> {
}
}
}
console.log("Shouldn't get here");
// console.log("Shouldn't get here");
next();
});
}

4
packages/plugos/sandbox.ts
View File

@ -65,7 +65,7 @@ export class Sandbox {
}
loadDependency(name: string, code: string): Promise<void> {
console.log("Loading dependency", name);
// console.log("Loading dependency", name);
this.worker.postMessage({
type: "load-dependency",
name: name,
@ -116,7 +116,7 @@ export class Sandbox {
if (data.error) {
resultCbs &&
resultCbs.reject(
new Error(`${data.error}\nStack trace: ${data.stack}`)
new Error(`${data.error}\nStack trace: ${data.stack}`),
);
} else {
resultCbs && resultCbs.resolve(data.result);

34
packages/plugos/syscalls/esbuild.ts
View File

@ -1,4 +1,4 @@
import { sandboxCompile, sandboxCompileModule } from "../compile";
import { sandboxCompileModule } from "../compile.ts";
import { SysCallMapping } from "../system.ts";
// TODO: FIgure out a better way to do this
@ -11,22 +11,22 @@ export function esbuildSyscalls(): SysCallMapping {
filename: string,
code: string,
): Promise<any> => {},
"esbuild.compile": async (
ctx,
filename: string,
code: string,
functionName?: string,
excludeModules: string[] = [],
): Promise<string> => {
return await sandboxCompile(
filename,
code,
functionName,
true,
[],
[...builtinModules, ...excludeModules],
);
},
// "esbuild.compile": async (
// ctx,
// filename: string,
// code: string,
// functionName?: string,
// excludeModules: string[] = [],
// ): Promise<string> => {
// return await sandboxCompile(
// filename,
// code,
// functionName,
// true,
// [],
// [...builtinModules, ...excludeModules],
// );
// },
"esbuild.compileModule": async (
ctx,
moduleName: string,

49
packages/plugos/syscalls/fulltext.knex_sqlite.ts
View File

@ -1,41 +1,56 @@
import { Knex } from "knex";
import { SysCallMapping } from "../system";
import { SQLite } from "../../../dep_server.ts";
import { SysCallMapping } from "../system.ts";
import { asyncExecute, asyncQuery } from "./store.deno.ts";
type Item = {
key: string;
value: string;
};
export async function ensureFTSTable(
db: Knex<any, unknown>,
tableName: string
export function ensureFTSTable(
db: SQLite,
tableName: string,
) {
if (!(await db.schema.hasTable(tableName))) {
await db.raw(`CREATE VIRTUAL TABLE ${tableName} USING fts5(key, value);`);
const stmt = db.prepare(
`SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
);
const result = stmt.all(tableName);
if (result.length === 0) {
asyncExecute(
db,
`CREATE VIRTUAL TABLE ${tableName} USING fts5(key, value);`,
);
console.log(`Created fts5 table ${tableName}`);
}
return Promise.resolve();
}
export function fullTextSearchSyscalls(
db: Knex<any, unknown>,
tableName: string
db: SQLite,
tableName: string,
): SysCallMapping {
return {
"fulltext.index": async (ctx, key: string, value: string) => {
await db<Item>(tableName).where({ key }).del();
await db<Item>(tableName).insert({ key, value });
await asyncExecute(db, `DELETE FROM ${tableName} WHERE key = ?`, key);
await asyncExecute(
db,
`INSERT INTO ${tableName} (key, value) VALUES (?, ?)`,
key,
value,
);
},
"fulltext.delete": async (ctx, key: string) => {
await db<Item>(tableName).where({ key }).del();
await asyncExecute(db, `DELETE FROM ${tableName} WHERE key = ?`, key);
},
"fulltext.search": async (ctx, phrase: string, limit: number) => {
return (
await db<any>(tableName)
.whereRaw(`value MATCH ?`, [phrase])
.select(["key", "rank"])
.orderBy("rank")
.limit(limit)
await asyncQuery<any>(
db,
`SELECT key, rank FROM ${tableName} WHERE value MATCH ? ORDER BY key, rank LIMIT ?`,
phrase,
limit,
)
).map((item) => ({ name: item.key, rank: item.rank }));
},
};

19
packages/plugos/syscalls/shell.node.ts
View File

@ -1,19 +1,22 @@
import { promisify } from "util";
import { execFile } from "child_process";
import type { SysCallMapping } from "../system";
const execFilePromise = promisify(execFile);
import type { SysCallMapping } from "../system.ts";
export default function (cwd: string): SysCallMapping {
return {
"shell.run": async (
ctx,
_ctx,
cmd: string,
args: string[]
args: string[],
): Promise<{ stdout: string; stderr: string }> => {
let { stdout, stderr } = await execFilePromise(cmd, args, {
const p = Deno.run({
cmd: [cmd, ...args],
cwd: cwd,
stdout: "piped",
stderr: "piped",
});
await p.status();
const stdout = new TextDecoder().decode(await p.output());
const stderr = new TextDecoder().decode(await p.stderrOutput());
return { stdout, stderr };
},
};

4
packages/plugos/syscalls/store.deno.ts
View File

@ -70,7 +70,7 @@ export function queryToSql(
};
}
function asyncQuery<T extends Record<string, unknown>>(
export function asyncQuery<T extends Record<string, unknown>>(
db: SQLite,
query: string,
...params: any[]
@ -79,7 +79,7 @@ function asyncQuery<T extends Record<string, unknown>>(
return Promise.resolve(db.prepare(query).all<T>(params));
}
function asyncExecute(
export function asyncExecute(
db: SQLite,
query: string,
...params: any[]

18
packages/plugs/core/core.plug.yaml
View File

@ -136,15 +136,15 @@ functions:
- page:complete
# Full text search
searchIndex:
path: ./search.ts:index
events:
- page:index
searchUnindex:
path: "./search.ts:unindex"
env: server
events:
- page:deleted
# searchIndex:
# path: ./search.ts:index
# events:
# - page:index
# searchUnindex:
# path: "./search.ts:unindex"
# env: server
# events:
# - page:deleted
searchQueryProvider:
path: ./search.ts:queryProvider
events:

5
packages/plugs/emoji/emoji.ts
View File

@ -1,6 +1,5 @@
// @ts-ignore
import emojis from "./emoji.json";
import { matchBefore } from "@silverbulletmd/plugos-silverbullet-syscall/editor";
import emojis from "./emoji.json" assert { type: "json" };
import { matchBefore } from "../../plugos-silverbullet-syscall/editor.ts";
export async function emojiCompleter() {
let prefix = await matchBefore(":[\\w]+");

9
packages/plugs/markdown/markdown.ts
View File

@ -1,8 +1,7 @@
import { hideRhs, hideLhs } from "@silverbulletmd/plugos-silverbullet-syscall/editor";
import { invokeFunction } from "@silverbulletmd/plugos-silverbullet-syscall/system";
import * as clientStore from "@silverbulletmd/plugos-silverbullet-syscall/clientStore";
import { readSettings, writeSettings } from "@silverbulletmd/plugs/lib/settings_page";;
import { hideLhs, hideRhs } from "../../plugos-silverbullet-syscall/editor.ts";
import { invokeFunction } from "../../plugos-silverbullet-syscall/system.ts";
import * as clientStore from "../../plugos-silverbullet-syscall/clientStore.ts";
import { readSettings } from "../lib/settings_page.ts";
export async function togglePreview() {
let currentValue = !!(await clientStore.get("enableMarkdownPreview"));

16
packages/plugs/markdown/preview.ts
View File

@ -1,14 +1,10 @@
import MarkdownIt from "markdown-it";
import MarkdownIt from "https://esm.sh/markdown-it@13.0.1";
import {
getText,
showPanel,
} from "@silverbulletmd/plugos-silverbullet-syscall/editor";
import * as clientStore from "@silverbulletmd/plugos-silverbullet-syscall/clientStore";
import { cleanMarkdown } from "./util";
import {
readSettings,
writeSettings,
} from "@silverbulletmd/plugs/lib/settings_page";
} from "../../plugos-silverbullet-syscall/editor.ts";
import * as clientStore from "../../plugos-silverbullet-syscall/clientStore.ts";
import { cleanMarkdown } from "./util.ts";
const css = `
<style>
@ -66,7 +62,7 @@ hr:after {
</style>
`;
var taskLists = require("markdown-it-task-lists");
import taskLists from "https://esm.sh/markdown-it-task-lists@2.1.1";
const md = new MarkdownIt({
linkify: true,
@ -83,6 +79,6 @@ export async function updateMarkdownPreview() {
await showPanel(
"rhs",
2,
`<html><head>${css}</head><body>${md.render(cleanMd)}</body></html>`
`<html><head>${css}</head><body>${md.render(cleanMd)}</body></html>`,
);
}

6
packages/plugs/markdown/util.ts
View File

@ -2,8 +2,8 @@ import {
findNodeOfType,
renderToText,
replaceNodesMatching,
} from "@silverbulletmd/common/tree";
import { parseMarkdown } from "@silverbulletmd/plugos-silverbullet-syscall/markdown";
} from "../../common/tree.ts";
import { parseMarkdown } from "../../plugos-silverbullet-syscall/markdown.ts";
export function encodePageUrl(name: string): string {
return name.replaceAll(" ", "_");
@ -11,7 +11,7 @@ export function encodePageUrl(name: string): string {
export async function cleanMarkdown(
text: string,
validPages?: string[]
validPages?: string[],
): Promise<string> {
let mdTree = await parseMarkdown(text);
replaceNodesMatching(mdTree, (n) => {

47
packages/plugs/package.json
View File

@ -1,47 +0,0 @@
{
"name": "@silverbulletmd/plugs",
"author": {
"name": "Zef Hemel",
"email": "zef@zef.me"
},
"version": "0.0.35",
"license": "MIT",
"scripts": {
"generate": "lezer-generator query/query.grammar -o query/parse-query.js",
"watch": "plugos-bundle --dist ../web/dist global.plug.yaml && plugos-bundle --debug -w --dist dist --exclude @lezer/lr yaml handlebars -- */*.plug.yaml",
"build": "plugos-bundle --dist ../web/dist global.plug.yaml && plugos-bundle --dist dist --exclude @lezer/lr yaml handlebars -- */*.plug.yaml",
"test": "jest build/test"
},
"files": [
"*"
],
"targets": {
"test": {
"source": [
"query/engine.test.ts"
],
"outputFormat": "commonjs",
"isLibrary": true,
"context": "node",
"includeNodeModules": [
"@silverbulletmd/common",
"@silverbulletmd/plugos-silverbullet-syscall"
],
"distDir": "build/test"
}
},
"dependencies": {
"@jest/globals": "^27.5.1",
"@lezer/generator": "1.0.0",
"@lezer/lr": "1.0.0",
"@silverbulletmd/common": "^0.0.35",
"@types/yaml": "^1.9.7",
"handlebars": "^4.7.7",
"markdown-it": "^12.3.2",
"markdown-it-task-lists": "^2.1.1",
"yaml": "^1.10.2"
},
"devDependencies": {
"@types/markdown-it": "^12.2.3"
}
}

25
packages/plugs/plugmd/plugmd.ts
View File

@ -1,24 +1,21 @@
import {
collectNodesOfType,
findNodeOfType,
} from "@silverbulletmd/common/tree";
import { collectNodesOfType, findNodeOfType } from "../../common/tree.ts";
import {
getText,
hideBhs,
showBhs,
} from "@silverbulletmd/plugos-silverbullet-syscall/editor";
import { parseMarkdown } from "@silverbulletmd/plugos-silverbullet-syscall/markdown";
} from "../../plugos-silverbullet-syscall/editor.ts";
import { parseMarkdown } from "../../plugos-silverbullet-syscall/markdown.ts";
import {
readPage,
writePage,
} from "@silverbulletmd/plugos-silverbullet-syscall/space";
} from "../../plugos-silverbullet-syscall/space.ts";
import {
invokeFunction,
reloadPlugs,
} from "@silverbulletmd/plugos-silverbullet-syscall/system";
import YAML from "yaml";
} from "../../plugos-silverbullet-syscall/system.ts";
import * as YAML from "yaml";
import type { Manifest } from "@silverbulletmd/common/manifest";
import type { Manifest } from "../../common/manifest.ts";
export async function compileCommand() {
let text = await getText();
@ -26,7 +23,7 @@ export async function compileCommand() {
let manifest = await compileDefinition(text);
await writePage(
`_plug/${manifest.name}`,
JSON.stringify(manifest, null, 2)
JSON.stringify(manifest, null, 2),
);
console.log("Wrote this plug", manifest);
await hideBhs();
@ -94,7 +91,7 @@ async function compileDefinition(text: string): Promise<Manifest> {
`file.${language}`,
code,
name,
Object.keys(manifest.dependencies)
Object.keys(manifest.dependencies),
);
func.code = compiled;
}
@ -108,7 +105,7 @@ export async function compileJS(
filename: string,
code: string,
functionName: string,
excludeModules: string[]
excludeModules: string[],
): Promise<string> {
// console.log("Compiling JS", filename, excludeModules);
return self.syscall(
@ -116,7 +113,7 @@ export async function compileJS(
filename,
code,
functionName,
excludeModules
excludeModules,
);
}

6
packages/plugs/query/complete.ts
View File

@ -1,6 +1,6 @@
import { listEvents } from "@plugos/plugos-syscall/event";
import { matchBefore } from "@silverbulletmd/plugos-silverbullet-syscall/editor";
import { listPages } from "@silverbulletmd/plugos-silverbullet-syscall/space";
import { listEvents } from "$sb/plugos-syscall/event.ts";
import { matchBefore } from "$sb/plugos-silverbullet-syscall/editor.ts";
import { listPages } from "$sb/plugos-silverbullet-syscall/space.ts";
export async function queryComplete() {
let prefix = await matchBefore("#query [\\w\\-_]*");

52
packages/plugs/query/materialized_queries.ts
View File

@ -2,23 +2,21 @@ import {
getCurrentPage,
reloadPage,
save,
} from "@silverbulletmd/plugos-silverbullet-syscall/editor";
import Handlebars from "https://esm.sh/handlebars";
} from "$sb/plugos-silverbullet-syscall/editor.ts";
import {
readPage,
writePage,
} from "@silverbulletmd/plugos-silverbullet-syscall/space";
import { invokeFunction } from "@silverbulletmd/plugos-silverbullet-syscall/system";
import { renderQuery } from "./engine";
import { parseQuery } from "./parser";
import { replaceTemplateVars } from "../core/template";
import { jsonToMDTable, queryRegex } from "./util";
import { dispatch } from "@plugos/plugos-syscall/event";
import { replaceAsync } from "../lib/util";
import { parseMarkdown } from "@silverbulletmd/plugos-silverbullet-syscall/markdown";
import { nodeAtPos, renderToText } from "@silverbulletmd/common/tree";
import { extractMeta } from "./data";
import Handlebars from "handlebars";
import { readPage, writePage } from "$sb/plugos-silverbullet-syscall/space.ts";
import { invokeFunction } from "$sb/plugos-silverbullet-syscall/system.ts";
import { renderQuery } from "./engine.ts";
import { parseQuery } from "./parser.ts";
import { replaceTemplateVars } from "../core/template.ts";
import { jsonToMDTable, queryRegex } from "./util.ts";
import { dispatch } from "$sb/plugos-syscall/event.ts";
import { replaceAsync } from "../lib/util.ts";
import { parseMarkdown } from "$sb/plugos-silverbullet-syscall/markdown.ts";
import { nodeAtPos, renderToText } from "$sb/common/tree.ts";
import { extractMeta } from "./data.ts";
export async function updateMaterializedQueriesCommand() {
const currentPage = await getCurrentPage();
@ -27,7 +25,7 @@ export async function updateMaterializedQueriesCommand() {
await invokeFunction(
"server",
"updateMaterializedQueriesOnPage",
currentPage
currentPage,
)
) {
await reloadPage();
@ -39,7 +37,7 @@ export const templateInstRegex =
async function updateTemplateInstantiations(
text: string,
pageName: string
pageName: string,
): Promise<string> {
return replaceAsync(
text,
@ -74,12 +72,12 @@ async function updateTemplateInstantiations(
templateText = renderToText(tree);
let templateFn = Handlebars.compile(
replaceTemplateVars(templateText, pageName),
{ noEscape: true }
{ noEscape: true },
);
newBody = templateFn(parsedArgs);
}
return `${startInst}\n${newBody.trim()}\n${endInst}`;
}
},
);
}
@ -95,20 +93,20 @@ async function cleanTemplateInstantiations(text: string): Promise<string> {
fullMatch: string,
startQuery: string,
query: string,
body: string
body: string,
) => {
return body.trim();
}
},
);
}
return `${startInst}${body}${endInst}`;
}
},
);
}
// Called from client, running on server
export async function updateMaterializedQueriesOnPage(
pageName: string
pageName: string,
): Promise<boolean> {
let text = "";
try {
@ -117,7 +115,7 @@ export async function updateMaterializedQueriesOnPage(
console.warn(
"Could not read page",
pageName,
"perhaps it doesn't yet exist"
"perhaps it doesn't yet exist",
);
return false;
}
@ -147,7 +145,7 @@ export async function updateMaterializedQueriesOnPage(
let results = await dispatch(
`query:${parsedQuery.table}`,
{ query: parsedQuery, pageName: pageName },
10 * 1000
10 * 1000,
);
if (results.length === 0) {
return `${startQuery}\n${endQuery}`;
@ -162,7 +160,7 @@ export async function updateMaterializedQueriesOnPage(
console.error("Too many query results", results);
return fullMatch;
}
}
},
);
newText = await cleanTemplateInstantiations(newText);
if (text !== newText) {

27
packages/plugs/tasks/task.ts
View File

@ -1,20 +1,17 @@
import type { ClickEvent, IndexTreeEvent } from "@silverbulletmd/web/app_event";
import type { ClickEvent, IndexTreeEvent } from "$sb/web/app_event.ts";
import {
batchSet,
queryPrefix,
} from "@silverbulletmd/plugos-silverbullet-syscall/index";
import {
readPage,
writePage,
} from "@silverbulletmd/plugos-silverbullet-syscall/space";
import { parseMarkdown } from "@silverbulletmd/plugos-silverbullet-syscall/markdown";
} from "$sb/plugos-silverbullet-syscall/index.ts";
import { readPage, writePage } from "$sb/plugos-silverbullet-syscall/space.ts";
import { parseMarkdown } from "$sb/plugos-silverbullet-syscall/markdown.ts";
import {
dispatch,
filterBox,
getCursor,
getText,
} from "@silverbulletmd/plugos-silverbullet-syscall/editor";
} from "$sb/plugos-silverbullet-syscall/editor.ts";
import {
addParentPointers,
collectNodesMatching,
@ -24,10 +21,10 @@ import {
ParseTree,
renderToText,
replaceNodesMatching,
} from "@silverbulletmd/common/tree";
import { removeQueries } from "../query/util";
import { applyQuery, QueryProviderEvent } from "../query/engine";
import { niceDate } from "../core/dates";
} from "$sb/common/tree.ts";
import { removeQueries } from "../query/util.ts";
import { applyQuery, QueryProviderEvent } from "../query/engine.ts";
import { niceDate } from "../core/dates.ts";
export type Task = {
name: string;
@ -111,7 +108,7 @@ async function toggleTaskMarker(node: ParseTree, moveToPos: number) {
let parentWikiLinks = collectNodesMatching(
node.parent!,
(n) => n.type === "WikiLinkPage"
(n) => n.type === "WikiLinkPage",
);
for (let wikiLink of parentWikiLinks) {
let ref = wikiLink.children![0].text!;
@ -126,7 +123,7 @@ async function toggleTaskMarker(node: ParseTree, moveToPos: number) {
if (!taskMarkerNode || taskMarkerNode.type !== "TaskMarker") {
console.error(
"Reference not a task marker, out of date?",
taskMarkerNode
taskMarkerNode,
);
return;
}
@ -177,7 +174,7 @@ export async function postponeCommand() {
{ name: "a week", orderId: 2 },
{ name: "following Monday", orderId: 3 },
],
"Select the desired time span to delay this task"
"Select the desired time span to delay this task",
);
if (!option) {
return;

440
packages/server/express_server.ts
View File

@ -1,60 +1,45 @@
import express, { Express } from "express";
import { Manifest, SilverBulletHooks } from "@silverbulletmd/common/manifest";
import { EndpointHook } from "@plugos/plugos/hooks/endpoint";
import { readdir, readFile, rm } from "fs/promises";
import { System } from "@plugos/plugos/system";
import { DiskSpacePrimitives } from "@silverbulletmd/common/spaces/disk_space_primitives";
import path from "path";
import bodyParser from "body-parser";
import { EventHook } from "@plugos/plugos/hooks/event";
import spaceSyscalls from "./syscalls/space";
import { eventSyscalls } from "@plugos/plugos/syscalls/event";
import { ensureTable as ensureIndexTable, pageIndexSyscalls } from "./syscalls";
import knex, { Knex } from "knex";
import shellSyscalls from "@plugos/plugos/syscalls/shell.node";
import { NodeCronHook } from "@plugos/plugos/hooks/node_cron";
import { markdownSyscalls } from "@silverbulletmd/common/syscalls/markdown";
import { EventedSpacePrimitives } from "@silverbulletmd/common/spaces/evented_space_primitives";
import { Space } from "@silverbulletmd/common/spaces/space";
import { Manifest, SilverBulletHooks } from "../common/manifest.ts";
import { EndpointHook } from "../plugos/hooks/endpoint.ts";
import { System } from "../plugos/system.ts";
import { DiskSpacePrimitives } from "../common/spaces/disk_space_primitives.ts";
import { path, SQLite } from "../../dep_server.ts";
import { EventHook } from "../plugos/hooks/event.ts";
import spaceSyscalls from "./syscalls/space.ts";
import { eventSyscalls } from "../plugos/syscalls/event.ts";
import {
createSandbox,
nodeModulesDir,
} from "@plugos/plugos/environments/node_sandbox";
import { jwtSyscalls } from "@plugos/plugos/syscalls/jwt";
import buildMarkdown from "@silverbulletmd/common/parser";
import { loadMarkdownExtensions } from "@silverbulletmd/common/markdown_ext";
import http, { Server } from "http";
import { esbuildSyscalls } from "@plugos/plugos/syscalls/esbuild";
import { systemSyscalls } from "./syscalls/system";
import { plugPrefix } from "@silverbulletmd/common/spaces/constants";
ensureTable as ensureIndexTable,
pageIndexSyscalls,
} from "./syscalls/index.ts";
import shellSyscalls from "../plugos/syscalls/shell.node.ts";
import { NodeCronHook } from "../plugos/hooks/node_cron.ts";
import { markdownSyscalls } from "../common/syscalls/markdown.ts";
import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitives.ts";
import { Space } from "../common/spaces/space.ts";
import { createSandbox } from "../plugos/environments/deno_sandbox.ts";
// import { jwtSyscalls } from "../plugos/syscalls/jwt.ts";
import buildMarkdown from "../common/parser.ts";
import { loadMarkdownExtensions } from "../common/markdown_ext.ts";
import { esbuildSyscalls } from "../plugos/syscalls/esbuild.ts";
import { systemSyscalls } from "./syscalls/system.ts";
import { plugPrefix } from "../common/spaces/constants.ts";
import sandboxSyscalls from "@plugos/plugos/syscalls/sandbox";
// @ts-ignore
import settingsTemplate from "bundle-text:./SETTINGS_template.md";
import { safeRun } from "./util";
import sandboxSyscalls from "../plugos/syscalls/sandbox.ts";
// import settingsTemplate from "bundle-text:./SETTINGS_template.md";
import { safeRun } from "./util.ts";
import {
ensureFTSTable,
fullTextSearchSyscalls,
} from "@plugos/plugos/syscalls/fulltext.knex_sqlite";
import { PlugSpacePrimitives } from "./hooks/plug_space_primitives";
import { PageNamespaceHook } from "./hooks/page_namespace";
import { readFileSync } from "fs";
import fileSystemSyscalls from "@plugos/plugos/syscalls/fs.node";
} from "../plugos/syscalls/fulltext.knex_sqlite.ts";
import { PlugSpacePrimitives } from "./hooks/plug_space_primitives.ts";
import { PageNamespaceHook } from "./hooks/page_namespace.ts";
import fileSystemSyscalls from "../plugos/syscalls/fs.deno.ts";
import {
ensureTable as ensureStoreTable,
storeSyscalls,
} from "@plugos/plugos/syscalls/store.knex_node";
import { parseYamlSettings } from "@silverbulletmd/common/util";
import { SpacePrimitives } from "@silverbulletmd/common/spaces/space_primitives";
import { version } from "./package.json";
const globalModules: any = JSON.parse(
readFileSync(
nodeModulesDir + "/node_modules/@silverbulletmd/web/dist/global.plug.json",
"utf-8"
)
);
} from "../plugos/syscalls/store.deno.ts";
import { parseYamlSettings } from "../common/util.ts";
import { SpacePrimitives } from "../common/spaces/space_primitives.ts";
import { Application, Router } from "../../dep_server.ts";
const safeFilename = /^[a-zA-Z0-9_\-\.]+$/;
@ -70,26 +55,31 @@ const storeVersionKey = "$silverBulletVersion";
const indexRequiredKey = "$spaceIndexed";
export class ExpressServer {
app: Express;
app: Application;
system: System<SilverBulletHooks>;
private space: Space;
private distDir: string;
private eventHook: EventHook;
private db: Knex<any, unknown[]>;
private db: SQLite;
private port: number;
private server?: Server;
builtinPlugDir: string;
password?: string;
settings: { [key: string]: any } = {};
spacePrimitives: SpacePrimitives;
abortController?: AbortController;
globalModules: Manifest;
constructor(options: ServerOptions) {
this.port = options.port;
this.app = express();
this.app = new Application();
this.builtinPlugDir = options.builtinPlugDir;
this.distDir = options.distDir;
this.password = options.password;
this.globalModules = JSON.parse(
Deno.readTextFileSync(`${this.distDir}/global.plug.json`),
);
// Set up the PlugOS System
this.system = new System<SilverBulletHooks>("server");
@ -105,20 +95,14 @@ export class ExpressServer {
this.spacePrimitives = new EventedSpacePrimitives(
new PlugSpacePrimitives(
new DiskSpacePrimitives(options.pagesPath),
namespaceHook
namespaceHook,
),
this.eventHook
this.eventHook,
);
this.space = new Space(this.spacePrimitives);
// The database used for persistence (SQLite)
this.db = knex({
client: "better-sqlite3",
connection: {
filename: path.join(options.pagesPath, "data.db"),
},
useNullAsDefault: true,
});
this.db = new SQLite(path.join(options.pagesPath, "data.db"));
// The cron hook
this.system.addHook(new NodeCronHook());
@ -135,7 +119,7 @@ export class ExpressServer {
esbuildSyscalls(),
systemSyscalls(this),
sandboxSyscalls(this.system),
jwtSyscalls()
// jwtSyscalls(),
);
// Danger zone
this.system.registerSyscalls(["shell"], shellSyscalls(options.pagesPath));
@ -148,9 +132,11 @@ export class ExpressServer {
plugLoaded: (plug) => {
// Automatically inject some modules into each plug
safeRun(async () => {
for (let [modName, code] of Object.entries(
globalModules.dependencies
)) {
for (
let [modName, code] of Object.entries(
this.globalModules.dependencies!,
)
) {
await plug.sandbox.loadDependency(modName, code as string);
}
});
@ -166,15 +152,14 @@ export class ExpressServer {
throw new Error(`Invalid plug name: ${plugName}`);
}
try {
let manifestJson = await readFile(
let manifestJson = await Deno.readTextFile(
path.join(this.builtinPlugDir, `${plugName}.plug.json`),
"utf8"
);
return JSON.parse(manifestJson);
} catch (e) {
} catch {
throw new Error(`No such builtin: ${plugName}`);
}
}
},
);
// Second, for loading plug JSON files with absolute or relative (from CWD) paths
@ -182,20 +167,20 @@ export class ExpressServer {
"get-plug:file",
async (plugPath: string): Promise<Manifest> => {
let resolvedPath = path.resolve(plugPath);
if (!resolvedPath.startsWith(process.cwd())) {
if (!resolvedPath.startsWith(Deno.cwd())) {
throw new Error(
`Plugin path outside working directory, this is disallowed: ${resolvedPath}`
`Plugin path outside working directory, this is disallowed: ${resolvedPath}`,
);
}
try {
let manifestJson = await readFile(resolvedPath, "utf8");
let manifestJson = await Deno.readTextFile(resolvedPath);
return JSON.parse(manifestJson);
} catch (e) {
throw new Error(
`No such file: ${resolvedPath} or could not parse as JSON`
`No such file: ${resolvedPath} or could not parse as JSON`,
);
}
}
},
);
// Rescan disk every 5s to detect any out-of-process file changes
@ -207,26 +192,25 @@ export class ExpressServer {
rebuildMdExtensions() {
this.system.registerSyscalls(
[],
markdownSyscalls(buildMarkdown(loadMarkdownExtensions(this.system)))
markdownSyscalls(buildMarkdown(loadMarkdownExtensions(this.system))),
);
}
// In case of a new space with no `PLUGS` file, generate a default one based on all built-in plugs
private async bootstrapBuiltinPlugs() {
let allPlugFiles = await readdir(this.builtinPlugDir);
let allPlugFiles = await Deno.readDir(this.builtinPlugDir);
let pluginNames = [];
for (let file of allPlugFiles) {
if (file.endsWith(".plug.json")) {
let manifestJson = await readFile(
path.join(this.builtinPlugDir, file),
"utf8"
for await (let file of allPlugFiles) {
if (file.name.endsWith(".plug.json")) {
let manifestJson = await Deno.readTextFile(
path.join(this.builtinPlugDir, file.name),
);
let manifest: Manifest = JSON.parse(manifestJson);
pluginNames.push(manifest.name);
await this.spacePrimitives.writeFile(
`${plugPrefix}${file}`,
`${plugPrefix}${file.name}`,
"string",
manifestJson
manifestJson,
);
}
}
@ -240,7 +224,7 @@ export class ExpressServer {
"PLUGS",
"This file lists all plugs that SilverBullet will load. Run the `Plugs: Update` command to update and reload this list of plugs.\n\n```yaml\n- " +
pluginNames.map((name) => `builtin:${name}`).join("\n- ") +
"\n```"
"\n```",
);
}
}
@ -250,31 +234,31 @@ export class ExpressServer {
let lastRunningVersion = await this.system.localSyscall(
"core",
"store.get",
[storeVersionKey]
[storeVersionKey],
);
let upgrading = false;
if (lastRunningVersion !== version) {
upgrading = true;
console.log("Version change detected!");
console.log("Going to re-bootstrap with the builtin set of plugs...");
console.log("First removing existing plug files...");
const existingPlugFiles = (
await this.spacePrimitives.fetchFileList()
).filter((meta) => meta.name.startsWith(plugPrefix));
for (let plugFile of existingPlugFiles) {
await this.spacePrimitives.deleteFile(plugFile.name);
}
console.log("Now writing the default set of plugs...");
await this.bootstrapBuiltinPlugs();
await this.system.localSyscall("core", "store.set", [
storeVersionKey,
version,
]);
await this.system.localSyscall("core", "store.set", [
"$spaceIndexed",
false,
]);
}
// if (lastRunningVersion !== version) {
// upgrading = true;
// console.log("Version change detected!");
// console.log("Going to re-bootstrap with the builtin set of plugs...");
// console.log("First removing existing plug files...");
// const existingPlugFiles = (
// await this.spacePrimitives.fetchFileList()
// ).filter((meta) => meta.name.startsWith(plugPrefix));
// for (let plugFile of existingPlugFiles) {
// await this.spacePrimitives.deleteFile(plugFile.name);
// }
// console.log("Now writing the default set of plugs...");
// await this.bootstrapBuiltinPlugs();
// await this.system.localSyscall("core", "store.set", [
// storeVersionKey,
// version,
// ]);
// await this.system.localSyscall("core", "store.set", [
// "$spaceIndexed",
// false,
// ]);
// }
await this.space.updatePageList();
@ -336,106 +320,128 @@ export class ExpressServer {
await ensureIndexTable(this.db);
await ensureStoreTable(this.db, "store");
await ensureFTSTable(this.db, "fts");
// await ensureFTSTable(this.db, "fts");
await this.ensureAndLoadSettings();
// Load plugs
this.reloadPlugs().catch(console.error);
// Serve static files (javascript, css, html)
this.app.use("/", express.static(this.distDir));
// Pages API
this.app.use(
"/fs",
passwordMiddleware,
buildFsRouter(this.spacePrimitives)
);
// Plug API
this.app.use("/plug", passwordMiddleware, this.buildPlugRouter());
// Fallback, serve index.html
this.app.get(/^(\/((?!fs\/).)+)$/, async (req, res) => {
res.sendFile(`${this.distDir}/index.html`, {});
});
this.server = http.createServer(this.app);
this.server.listen(this.port, () => {
console.log(
`Silver Bullet is now running: http://localhost:${this.port}`
);
console.log("--------------");
this.app.use(async (ctx, next) => {
if (ctx.request.url.pathname === "/") {
return ctx.send({
root: "/",
path: `${this.distDir}/index.html`,
});
}
const root = this.distDir;
try {
await ctx.send({ root });
} catch {
await next();
}
});
private buildPlugRouter() {
let plugRouter = express.Router();
// Pages API
const fsRouter = buildFsRouter(this.spacePrimitives);
this.app.use(fsRouter.routes());
this.app.use(fsRouter.allowedMethods());
// Plug API
const plugRouter = this.buildPlugRouter();
this.app.use(plugRouter.routes());
this.app.use(plugRouter.allowedMethods());
// Fallback, serve index.html
this.app.use((ctx) => {
console.log("Here!!");
return ctx.send({
root: "/",
path: `${this.distDir}/index.html`,
});
});
this.abortController = new AbortController();
this.app.listen({ port: this.port, signal: this.abortController.signal })
.catch(console.error);
console.log(
`Silver Bullet is now running: http://localhost:${this.port}`,
);
console.log("--------------");
}
private buildPlugRouter(): Router {
let plugRouter = new Router();
plugRouter.post(
"/:plug/syscall/:name",
bodyParser.json(),
async (req, res) => {
const name = req.params.name;
const plugName = req.params.plug;
const args = req.body as any;
async (ctx) => {
const name = ctx.params.name;
const plugName = ctx.params.plug;
const args = await ctx.request.body().value;
const plug = this.system.loadedPlugs.get(plugName);
if (!plug) {
res.status(404);
return res.send(`Plug ${plugName} not found`);
ctx.response.status = 404;
ctx.response.body = `Plug ${plugName} not found`;
return;
}
try {
const result = await this.system.syscallWithContext(
{ plug },
name,
args
args,
);
res.status(200);
res.header("Content-Type", "application/json");
res.send(JSON.stringify(result));
ctx.response.headers.set("Content-Type", "application/json");
ctx.response.body = JSON.stringify(result);
} catch (e: any) {
res.status(500);
return res.send(e.message);
}
ctx.response.status = 500;
ctx.response.body = e.message;
return;
}
},
);
plugRouter.post(
"/:plug/function/:name",
bodyParser.json(),
async (req, res) => {
const name = req.params.name;
const plugName = req.params.plug;
const args = req.body as any[];
async (ctx) => {
const name = ctx.params.name;
const plugName = ctx.params.plug;
const args = await ctx.request.body().value;
const plug = this.system.loadedPlugs.get(plugName);
if (!plug) {
res.status(404);
return res.send(`Plug ${plugName} not found`);
ctx.response.status = 404;
ctx.response.body = `Plug ${plugName} not found`;
return;
}
try {
const result = await plug.invoke(name, args);
res.status(200);
res.header("Content-Type", "application/json");
res.send(JSON.stringify(result));
ctx.response.headers.set("Content-Type", "application/json");
ctx.response.body = JSON.stringify(result);
} catch (e: any) {
res.status(500);
ctx.response.status = 500;
// console.log("Error invoking function", e);
return res.send(e.message);
}
ctx.response.body = e.message;
}
},
);
return plugRouter;
return new Router().use("/plug", plugRouter.routes());
}
async ensureAndLoadSettings() {
try {
await this.space.getPageMeta("SETTINGS");
} catch (e) {
await this.space.writePage("SETTINGS", settingsTemplate, true);
await this.space.writePage(
"SETTINGS",
await Deno.readTextFile(
new URL("SETTINGS_template.md", import.meta.url).pathname,
),
true,
);
}
let { text: settingsText } = await this.space.readPage("SETTINGS");
const { text: settingsText } = await this.space.readPage("SETTINGS");
this.settings = parseYamlSettings(settingsText);
if (!this.settings.indexPage) {
this.settings.indexPage = "index";
@ -446,106 +452,102 @@ export class ExpressServer {
} catch (e) {
await this.space.writePage(
this.settings.indexPage,
`Welcome to your new space!`
`Welcome to your new space!`,
);
}
}
async stop() {
if (this.server) {
if (this.abortController) {
console.log("Stopping");
await this.system.unloadAll();
console.log("Stopped plugs");
return new Promise<void>((resolve, reject) => {
this.server!.close((err) => {
this.server = undefined;
this.abortController.abort();
console.log("stopped server");
if (err) {
reject(err);
} else {
resolve();
}
});
});
}
}
}
function buildFsRouter(spacePrimitives: SpacePrimitives) {
let fsRouter = express.Router();
function buildFsRouter(spacePrimitives: SpacePrimitives): Router {
const fsRouter = new Router();
// File list
fsRouter.route("/").get(async (req, res, next) => {
res.json(await spacePrimitives.fetchFileList());
fsRouter.get("/", async ({ response }) => {
const list = await spacePrimitives.fetchFileList();
// console.log("List", list);
response.headers.set("Content-type", "application/json");
response.body = JSON.stringify(list);
});
fsRouter
.route(/\/(.+)/)
.get(async (req, res, next) => {
let name = req.params[0];
.get("\/(.+)", async ({ params, request, response }, next) => {
let name = params[0];
console.log("Loading file", name);
try {
let attachmentData = await spacePrimitives.readFile(
name,
"arraybuffer"
"arraybuffer",
);
res.status(200);
res.header("Last-Modified", "" + attachmentData.meta.lastModified);
res.header("X-Permission", attachmentData.meta.perm);
res.header("Content-Type", attachmentData.meta.contentType);
res.send(Buffer.from(attachmentData.data as ArrayBuffer));
} catch (e) {
response.status = 200;
response.headers.set(
"Last-Modified",
"" + attachmentData.meta.lastModified,
);
response.headers.set("X-Permission", attachmentData.meta.perm);
response.headers.set("Content-Type", attachmentData.meta.contentType);
response.body = attachmentData.data as ArrayBuffer;
} catch (e: any) {
console.error("Error in main router", e);
next();
}
})
.put(bodyParser.raw({ type: "*/*", limit: "100mb" }), async (req, res) => {
let name = req.params[0];
.put("\/(.+)", async ({ request, response, params }) => {
let name = params[0];
console.log("Saving file", name);
try {
let meta = await spacePrimitives.writeFile(
name,
"arraybuffer",
req.body,
false
await request.body().value,
false,
);
res.status(200);
res.header("Last-Modified", "" + meta.lastModified);
res.header("Content-Type", meta.contentType);
res.header("Content-Length", "" + meta.size);
res.header("X-Permission", meta.perm);
res.send("OK");
response.status = 200;
response.headers.set("Last-Modified", "" + meta.lastModified);
response.headers.set("Content-Type", meta.contentType);
response.headers.set("Content-Length", "" + meta.size);
response.headers.set("X-Permission", meta.perm);
response.body = "OK";
} catch (err) {
res.status(500);
res.send("Write failed");
response.status = 500;
response.body = "Write failed";
console.error("Pipeline failed", err);
}
})
.options(async (req, res, next) => {
let name = req.params[0];
.options("\/(.+)", async ({ request, response, params }, next) => {
let name = params[0];
try {
const meta = await spacePrimitives.getFileMeta(name);
res.status(200);
res.header("Last-Modified", "" + meta.lastModified);
res.header("X-Permission", meta.perm);
res.header("Content-Length", "" + meta.size);
res.header("Content-Type", meta.contentType);
res.send("");
} catch (e) {
response.status = 200;
response.headers.set("Last-Modified", "" + meta.lastModified);
response.headers.set("Content-Type", meta.contentType);
response.headers.set("Content-Length", "" + meta.size);
response.headers.set("X-Permission", meta.perm);
} catch {
next();
}
})
.delete(async (req, res) => {
let name = req.params[0];
.delete("\/(.+)", async ({ request, response, params }) => {
let name = params[0];
try {
await spacePrimitives.deleteFile(name);
res.status(200);
res.send("OK");
} catch (e) {
response.status = 200;
response.body = "OK";
} catch (e: any) {
console.error("Error deleting attachment", e);
res.status(500);
res.send("OK");
response.status = 200;
response.body = e.message;
}
});
return fsRouter;
return new Router().use("/fs", fsRouter.routes());
}

22
packages/server/hooks/plug_space_primitives.ts
View File

@ -1,20 +1,16 @@
import { Plug } from "@plugos/plugos/plug";
import { Plug } from "../../plugos/plug.ts";
import {
FileData,
FileEncoding,
SpacePrimitives,
} from "@silverbulletmd/common/spaces/space_primitives";
import {
AttachmentMeta,
FileMeta,
PageMeta,
} from "@silverbulletmd/common/types";
import { PageNamespaceHook, NamespaceOperation } from "./page_namespace";
} from "../../common/spaces/space_primitives.ts";
import { AttachmentMeta, FileMeta, PageMeta } from "../../common/types.ts";
import { NamespaceOperation, PageNamespaceHook } from "./page_namespace.ts";
export class PlugSpacePrimitives implements SpacePrimitives {
constructor(
private wrapped: SpacePrimitives,
private hook: PageNamespaceHook
private hook: PageNamespaceHook,
) {}
performOperation(
@ -52,7 +48,7 @@ export class PlugSpacePrimitives implements SpacePrimitives {
readFile(
name: string,
encoding: FileEncoding
encoding: FileEncoding,
): Promise<{ data: FileData; meta: FileMeta }> {
let result = this.performOperation("readFile", name);
if (result) {
@ -73,14 +69,14 @@ export class PlugSpacePrimitives implements SpacePrimitives {
name: string,
encoding: FileEncoding,
data: FileData,
selfUpdate?: boolean
selfUpdate?: boolean,
): Promise<FileMeta> {
let result = this.performOperation(
"writeFile",
name,
encoding,
data,
selfUpdate
selfUpdate,
);
if (result) {
return result;
@ -105,7 +101,7 @@ export class PlugSpacePrimitives implements SpacePrimitives {
plug: Plug<any>,
env: string,
name: string,
args: any[]
args: any[],
): Promise<any> {
return this.wrapped.invokeFunction(plug, env, name, args);
}

44
packages/server/server.ts
View File

@ -1,39 +1,29 @@
#!/usr/bin/env -S node --enable-source-maps
import { nodeModulesDir } from "@plugos/plugos/environments/node_sandbox";
import { realpathSync } from "fs";
import yargs from "yargs";
import { hideBin } from "yargs/helpers";
import * as flags from "https://deno.land/std@0.158.0/flags/mod.ts";
import * as path from "https://deno.land/std@0.158.0/path/mod.ts";
import { ExpressServer } from "./express_server.ts";
import { ExpressServer } from "./express_server";
let args = yargs(hideBin(process.argv))
.option("port", {
type: "number",
default: 3000,
})
.option("password", {
type: "string",
})
.parse();
type ServerArgs = {
_: string[];
port: number;
password: string;
};
let args: ServerArgs = flags.parse(Deno.args);
if (!args._.length) {
console.error(
"Usage: silverbullet [--port 3000] [--password mysecretpassword] <path-to-pages>"
"Usage: silverbullet [--port 3000] [--password mysecretpassword] <path-to-pages>",
);
process.exit(1);
Deno.exit(1);
}
const pagesPath = args._[0] as string;
const port = args.port;
const pagesPath = path.resolve(Deno.cwd(), args._[0] as string);
const port = args.port ? +args.port : 3000;
const webappDistDir = realpathSync(
`${nodeModulesDir}/node_modules/@silverbulletmd/web/dist`
);
// console.log("Webapp dist dir", webappDistDir);
const plugDistDir = realpathSync(
`${nodeModulesDir}/node_modules/@silverbulletmd/plugs/dist`
);
// console.log("Builtin plug dist dir", plugDistDir);
const webappDistDir = new URL("./../../dist", import.meta.url).pathname;
console.log("Webapp dist dir", webappDistDir);
const plugDistDir = new URL("./../plugs/dist", import.meta.url).pathname;
console.log("Pages dir", pagesPath);
const expressServer = new ExpressServer({
port: port,

98
packages/server/syscalls/index.ts
View File

@ -1,6 +1,12 @@
import { Knex } from "knex";
import { SysCallMapping } from "@plugos/plugos/system";
import { Query, queryToKnex } from "@plugos/plugos/syscalls/store.knex_node";
// import { Knex } from "knex";
import { SysCallMapping } from "../../plugos/system.ts";
import {
asyncExecute,
asyncQuery,
Query,
queryToSql,
} from "../../plugos/syscalls/store.deno.ts";
import { SQLite } from "../../../dep_server.ts";
type Item = {
page: string;
@ -15,32 +21,41 @@ export type KV = {
const tableName = "page_index";
export async function ensureTable(db: Knex<any, unknown>) {
if (!(await db.schema.hasTable(tableName))) {
await db.schema.createTable(tableName, (table) => {
table.string("page");
table.string("key");
table.text("value");
table.primary(["page", "key"]);
table.index(["key"]);
});
export function ensureTable(db: SQLite): Promise<void> {
const stmt = db.prepare(
`SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
);
const result = stmt.all(tableName);
if (result.length === 0) {
db.exec(
`CREATE TABLE ${tableName} (key STRING, page STRING, value TEXT, PRIMARY KEY (page, key));`,
);
db.exec(
`CREATE INDEX ${tableName}_idx ON ${tableName}(key);`,
);
console.log(`Created table ${tableName}`);
}
return Promise.resolve();
}
export function pageIndexSyscalls(db: Knex<any, unknown>): SysCallMapping {
export function pageIndexSyscalls(db: SQLite): SysCallMapping {
const apiObj: SysCallMapping = {
"index.set": async (ctx, page: string, key: string, value: any) => {
let changed = await db<Item>(tableName)
.where({ key, page })
.update("value", JSON.stringify(value));
if (changed === 0) {
await db<Item>(tableName).insert({
await asyncExecute(
db,
`UPDATE ${tableName} SET value = ? WHERE key = ? AND page = ?`,
JSON.stringify(value),
key,
page,
value: JSON.stringify(value),
});
);
if (db.changes === 0) {
await asyncExecute(
db,
`INSERT INTO ${tableName} (key, page, value) VALUES (?, ?, ?)`,
key,
page,
JSON.stringify(value),
);
}
},
"index.batchSet": async (ctx, page: string, kvs: KV[]) => {
@ -52,9 +67,12 @@ export function pageIndexSyscalls(db: Knex<any, unknown>): SysCallMapping {
await db<Item>(tableName).where({ key, page }).del();
},
"index.get": async (ctx, page: string, key: string) => {
let result = await db<Item>(tableName)
.where({ key, page })
.select("value");
const result = await asyncQuery<Item>(
db,
`SELECT value FROM ${tableName} WHERE key = ? AND page = ?`,
key,
page,
);
if (result.length) {
return JSON.parse(result[0].value);
} else {
@ -63,9 +81,11 @@ export function pageIndexSyscalls(db: Knex<any, unknown>): SysCallMapping {
},
"index.queryPrefix": async (ctx, prefix: string) => {
return (
await db<Item>(tableName)
.andWhereLike("key", `${prefix}%`)
.select("key", "value", "page")
await asyncQuery<Item>(
db,
`SELECT key, page, value FROM ${tableName} WHERE key LIKE "?%"`,
prefix,
)
).map(({ key, value, page }) => ({
key,
page,
@ -73,11 +93,12 @@ export function pageIndexSyscalls(db: Knex<any, unknown>): SysCallMapping {
}));
},
"index.query": async (ctx, query: Query) => {
const { sql, params } = queryToSql(query);
return (
await queryToKnex(db<Item>(tableName), query).select(
"key",
"value",
"page"
await asyncQuery<Item>(
db,
`SELECT key, value FROM ${tableName} ${sql}`,
...params,
)
).map(({ key, value, page }: any) => ({
key,
@ -89,13 +110,18 @@ export function pageIndexSyscalls(db: Knex<any, unknown>): SysCallMapping {
await apiObj["index.deletePrefixForPage"](ctx, page, "");
},
"index.deletePrefixForPage": async (ctx, page: string, prefix: string) => {
return db<Item>(tableName)
.where({ page })
.andWhereLike("key", `${prefix}%`)
.del();
await asyncExecute(
db,
`DELETE FROM ${tableName} WHERE key LIKE "?%" AND page = ?`,
prefix,
page,
);
},
"index.clearPageIndex": async (ctx) => {
await db<Item>(tableName).del();
await asyncExecute(
db,
`DELETE FROM ${tableName}`,
);
},
};
return apiObj;

18
packages/server/syscalls/space.ts
View File

@ -1,10 +1,10 @@
import { AttachmentMeta, PageMeta } from "@silverbulletmd/common/types";
import { SysCallMapping } from "@plugos/plugos/system";
import { Space } from "@silverbulletmd/common/spaces/space";
import { AttachmentMeta, PageMeta } from "../../common/types.ts";
import { SysCallMapping } from "../../plugos/system.ts";
import { Space } from "../../common/spaces/space.ts";
import {
FileData,
FileEncoding,
} from "@silverbulletmd/common/spaces/space_primitives";
} from "../../common/spaces/space_primitives.ts";
export default (space: Space): SysCallMapping => {
return {
@ -13,7 +13,7 @@ export default (space: Space): SysCallMapping => {
},
"space.readPage": async (
ctx,
name: string
name: string,
): Promise<{ text: string; meta: PageMeta }> => {
return space.readPage(name);
},
@ -23,7 +23,7 @@ export default (space: Space): SysCallMapping => {
"space.writePage": async (
ctx,
name: string,
text: string
text: string,
): Promise<PageMeta> => {
return space.writePage(name, text);
},
@ -38,13 +38,13 @@ export default (space: Space): SysCallMapping => {
},
"space.readAttachment": async (
ctx,
name: string
name: string,
): Promise<{ data: FileData; meta: AttachmentMeta }> => {
return await space.readAttachment(name, "dataurl");
},
"space.getAttachmentMeta": async (
ctx,
name: string
name: string,
): Promise<AttachmentMeta> => {
return await space.getAttachmentMeta(name);
},
@ -52,7 +52,7 @@ export default (space: Space): SysCallMapping => {
ctx,
name: string,
encoding: FileEncoding,
data: string
data: string,
): Promise<AttachmentMeta> => {
return await space.writeAttachment(name, encoding, data);
},

9
packages/server/syscalls/system.ts
View File

@ -1,10 +1,9 @@
import { SysCallMapping } from "@plugos/plugos/system";
import type { ExpressServer } from "../express_server";
import { version } from "../package.json";
import { SysCallMapping } from "../../plugos/system.ts";
import type { ExpressServer } from "../express_server.ts";
export function systemSyscalls(expressServer: ExpressServer): SysCallMapping {
return {
"system.invokeFunction": async (
"system.invokeFunction": (
ctx,
env: string,
name: string,
@ -15,7 +14,7 @@ export function systemSyscalls(expressServer: ExpressServer): SysCallMapping {
}
return ctx.plug.invoke(name, args);
},
"system.reloadPlugs": async () => {
"system.reloadPlugs": () => {
return expressServer.reloadPlugs();
},
};

2
packages/web/index.html
View File

@ -33,7 +33,7 @@
document.documentElement.dataset.theme = localStorage.theme ?? "light";
</script>
<link rel="stylesheet" href="main.css" />
<script type="module" src="boot.js"></script>
<script type="module" src="client.js"></script>
<!--link rel="manifest" href="manifest.json" /-->
<link rel="icon" type="image/x-icon" href="images/favicon.gif" />
</head>