diff --git a/cmd/publish.ts b/cmd/publish.ts deleted file mode 100755 index 1369c03f..00000000 --- a/cmd/publish.ts +++ /dev/null @@ -1,170 +0,0 @@ -#!/usr/bin/env node -import { createSandbox } from "../plugos/environments/deno_sandbox.ts"; -import { EventHook } from "../plugos/hooks/event.ts"; -import { eventSyscalls } from "../plugos/syscalls/event.ts"; -import fileSystemSyscalls from "../plugos/syscalls/fs.deno.ts"; -import { - ensureFTSTable, - fullTextSearchSyscalls, -} from "../plugos/syscalls/fulltext.sqlite.ts"; -import sandboxSyscalls from "../plugos/syscalls/sandbox.ts"; -import shellSyscalls from "../plugos/syscalls/shell.deno.ts"; -import { - ensureTable as ensureStoreTable, - storeSyscalls, -} from "../plugos/syscalls/store.deno.ts"; -import { System } from "../plugos/system.ts"; -import { SilverBulletHooks } from "../common/manifest.ts"; -import { loadMarkdownExtensions } from "../common/markdown_ext.ts"; -import buildMarkdown from "../common/parser.ts"; -import { DiskSpacePrimitives } from "../common/spaces/disk_space_primitives.ts"; -import { EventedSpacePrimitives } from "../common/spaces/evented_space_primitives.ts"; -import { Space } from "../common/spaces/space.ts"; -import { markdownSyscalls } from "../common/syscalls/markdown.ts"; -import { PageNamespaceHook } from "../server/hooks/page_namespace.ts"; -import { PlugSpacePrimitives } from "../server/hooks/plug_space_primitives.ts"; -import { - ensureTable as ensureIndexTable, - pageIndexSyscalls, -} from "../server/syscalls/index.ts"; -import spaceSyscalls from "../server/syscalls/space.ts"; - -import assetBundle from "../dist/asset_bundle.json" assert { type: "json" }; -import { AssetBundle, AssetJson } from "../plugos/asset_bundle/bundle.ts"; -import { path } from "../server/deps.ts"; -import { AsyncSQLite } from "../plugos/sqlite/async_sqlite.ts"; -import { AssetBundlePlugSpacePrimitives } from "../common/spaces/asset_bundle_space_primitives.ts"; -import assetSyscalls from "../plugos/syscalls/asset.ts"; - -export async function publishCommand(options: { - index: boolean; - watch: boolean; - output: string; -}, pagesPath: string) { - const assets = new AssetBundle(assetBundle as AssetJson); - // Set up the PlugOS System - const system = new System("server"); - - // Instantiate the event bus hook - const eventHook = new EventHook(); - system.addHook(eventHook); - - // And the page namespace hook - const namespaceHook = new PageNamespaceHook(); - system.addHook(namespaceHook); - - pagesPath = path.resolve(pagesPath); - - // The space - const space = new Space( - new AssetBundlePlugSpacePrimitives( - new EventedSpacePrimitives( - new PlugSpacePrimitives( - new DiskSpacePrimitives(pagesPath), - namespaceHook, - "server", - ), - eventHook, - ), - assets, - ), - ); - - await space.updatePageList(); - - // The database used for persistence (SQLite) - const db = new AsyncSQLite(path.join(pagesPath, "publish-data.db")); - db.init().catch((e) => { - console.error("Error initializing database", e); - }); - - // Register syscalls available on the server side - system.registerSyscalls( - [], - pageIndexSyscalls(db), - storeSyscalls(db, "store"), - fullTextSearchSyscalls(db, "fts"), - spaceSyscalls(space), - eventSyscalls(eventHook), - markdownSyscalls(buildMarkdown([])), - sandboxSyscalls(system), - assetSyscalls(system), - ); - // Danger zone - system.registerSyscalls(["shell"], shellSyscalls(pagesPath)); - system.registerSyscalls(["fs"], fileSystemSyscalls("/")); - - const globalModules = JSON.parse( - assets.readTextFileSync(`web/global.plug.json`), - ); - - system.on({ - sandboxInitialized: async (sandbox) => { - for ( - const [modName, code] of Object.entries( - globalModules.dependencies, - ) - ) { - await sandbox.loadDependency(modName, code as string); - } - }, - }); - - await space.updatePageList(); - - const allPlugs = await space.listPlugs(); - - console.log("Loading plugs", allPlugs); - await Promise.all((await space.listPlugs()).map(async (plugName) => { - const { data } = await space.readAttachment(plugName, "string"); - await system.load(JSON.parse(data as string), createSandbox); - })); - - const corePlug = system.loadedPlugs.get("core"); - if (!corePlug) { - console.error("Something went very wrong, 'core' plug not found"); - return; - } - - system.registerSyscalls( - [], - markdownSyscalls(buildMarkdown(loadMarkdownExtensions(system))), - ); - - await ensureIndexTable(db); - await ensureStoreTable(db, "store"); - await ensureFTSTable(db, "fts"); - - if (options.index) { - console.log("Now indexing space"); - await corePlug.invoke("reindexSpace", []); - } - - const outputDir = path.resolve(options.output); - - await Deno.mkdir(outputDir, { recursive: true }); - - const publishPlug = system.loadedPlugs.get("publish")!; - - await publishPlug.invoke("publishAll", [outputDir]); - - if (options.watch) { - console.log("Watching for changes"); - let building = false; - for await (const _event of Deno.watchFs(pagesPath, { recursive: true })) { - console.log("Change detected, republishing"); - if (building) { - continue; - } - building = true; - space.updatePageList().then(async () => { - await publishPlug.invoke("publishAll", [outputDir]); - building = false; - }); - } - } else { - console.log("Done!"); - Deno.exit(0); - // process.exit(0); - } -} diff --git a/plugs/publish/README.md b/plugs/publish/README.md deleted file mode 100644 index 64748200..00000000 --- a/plugs/publish/README.md +++ /dev/null @@ -1,45 +0,0 @@ -# Silver Bullet Publish -A simple tool to export a subset of your [SilverBullet](https://silverbullet.md) space as a static website. - -**Note:** this is highly experimental and not necessarily production ready code, use at your own risk. - -silverbullet-publish currentenly publishes a subset of a space in two formats: - -* Markdown (.md files) -* HTML (.html files based on currently hardcoded templates (see `page.hbs` and `style.css`) - -The tool can be run in two ways: - -1. As a Silver Bullet plug (via the `Silver Bullet Publish: Publish All` command) -2. As a stand-alone CLI tool (via `npx`) - -The latter allows for automatic deployments to e.g. environments like Netlify. - -## Configuration -SilverBullet Publish is configured via the `PUBLISH` page with the following properties: - - ```yaml - # Index page to use for public version - indexPage: Public - # Optional destination folder when used in plug mode - destDir: /Users/you/my-website - title: Name of the space - removeHashtags: true - removeUnpublishedLinks: false - # Publish all pages with specific tag - tags: - - "#pub" - # Publish all pages with a specifix prefix - prefixes: - - /public - ``` - -## Running via `npx` -The easiest way to run SilverBullet Publish is via `npx`, it takes a few optional arguments beyond the path to your SilverBullet space: - -* `-o` specifies where to write the output to (defaults to `./web`) -* `--index` runs a full space index (e.g. to index all hash tags) before publishing, this is primarily useful when run in a CI/CI pipeline (like Netlify) because there no `data.db` in your repo containing this index. - -```bash -npx @silverbulletmd/publish -o web_build --index . -``` \ No newline at end of file diff --git a/plugs/publish/assets/page.hbs b/plugs/publish/assets/page.hbs deleted file mode 100644 index 8ac793b8..00000000 --- a/plugs/publish/assets/page.hbs +++ /dev/null @@ -1,16 +0,0 @@ - - - - - - -{{#if pageName}}{{pageName}} — {{config.title}}{{else}}{{config.title}}{{/if}} - - - -

{{pageName}}

-{{{body}}} - - diff --git a/plugs/publish/assets/style.css b/plugs/publish/assets/style.css deleted file mode 100644 index 59ef3246..00000000 --- a/plugs/publish/assets/style.css +++ /dev/null @@ -1,53 +0,0 @@ -body { - font-family: georgia, times, serif; - font-size: 14pt; - max-width: 800px; - margin-left: auto; - margin-right: auto; - padding-left: 20px; - padding-right: 20px; -} - -table { - width: 100%; - border-spacing: 0; -} - -thead tr { - background-color: #333; - color: #eee; -} - -th, -td { - padding: 8px; -} - -tbody tr:nth-of-type(even) { - background-color: #f3f3f3; -} - -ul li p { - margin: 0; -} - -a[href] { - text-decoration: none; -} - -blockquote { - border-left: 1px solid #333; - margin-left: 2px; - padding-left: 10px; -} - -.footer { - border-top: 1px solid #000; - padding-top: 5px; - text-align: center; - font-size: 70%; -} - -img { - max-width: 90%; -} diff --git a/plugs/publish/publish.plug.yaml b/plugs/publish/publish.plug.yaml deleted file mode 100644 index 57630497..00000000 --- a/plugs/publish/publish.plug.yaml +++ /dev/null @@ -1,15 +0,0 @@ -name: publish -imports: - - https://get.silverbullet.md/global.plug.json -requiredPermissions: - - fs -assets: - - "assets/*" -functions: - publishAll: - path: "./publish.ts:publishAll" - env: server - publishAllCommand: - path: "./publish.ts:publishAllCommand" - command: - name: "Silver Bullet Publish: Publish All" diff --git a/plugs/publish/publish.ts b/plugs/publish/publish.ts deleted file mode 100644 index df1d1d86..00000000 --- a/plugs/publish/publish.ts +++ /dev/null @@ -1,208 +0,0 @@ -import { asset, fs } from "$sb/plugos-syscall/mod.ts"; -import { - editor, - index, - markdown, - space, - system, -} from "$sb/silverbullet-syscall/mod.ts"; -import { readYamlPage } from "$sb/lib/yaml_page.ts"; -import { renderMarkdownToHtml } from "../markdown/markdown_render.ts"; - -import Handlebars from "handlebars"; - -import { - collectNodesOfType, - findNodeOfType, - ParseTree, - renderToText, - replaceNodesMatching, -} from "$sb/lib/tree.ts"; - -type PublishConfig = { - destDir?: string; - title?: string; - indexPage?: string; - removeHashtags?: boolean; - publishAll?: boolean; - tags?: string[]; - prefixes?: string[]; - footerPage?: string; -}; - -async function generatePage( - pageName: string, - htmlPath: string, - mdPath: string, - publishedPages: string[], - publishConfig: PublishConfig, - destDir: string, - footerText: string, -) { - const pageTemplate = await asset.readAsset("assets/page.hbs"); - const pageCSS = await asset.readAsset("assets/style.css"); - const text = await space.readPage(pageName); - const renderPage = Handlebars.compile(pageTemplate); - console.log("Writing", pageName); - const mdTree = await markdown.parseMarkdown(`${text}\n${footerText}`); - const publishMd = cleanMarkdown( - mdTree, - publishConfig, - publishedPages, - ); - const attachments = await collectAttachments(mdTree); - for (const attachment of attachments) { - try { - const result = await space.readAttachment(attachment); - console.log("Writing", `${destDir}/${attachment}`); - await fs.writeFile(`${destDir}/${attachment}`, result, "dataurl"); - } catch (e: any) { - console.error("Error reading attachment", attachment, e.message); - } - } - // Write .md file - await fs.writeFile(mdPath, publishMd); - // Write .html file - await fs.writeFile( - htmlPath, - renderPage({ - pageName, - config: publishConfig, - css: pageCSS, - body: renderMarkdownToHtml(mdTree, { - smartHardBreak: true, - attachmentUrlPrefix: "/", - }), - }), - ); -} - -export async function publishAll(destDir?: string) { - const publishConfig: PublishConfig = await readYamlPage("PUBLISH"); - destDir = destDir || publishConfig.destDir || "."; - console.log("Publishing to", destDir); - let allPages: any[] = await space.listPages(); - let allPageMap: Map = new Map( - allPages.map((pm) => [pm.name, pm]), - ); - for (const { page, value } of await index.queryPrefix("meta:")) { - const p = allPageMap.get(page); - if (p) { - for (const [k, v] of Object.entries(value)) { - p[k] = v; - } - } - } - - allPages = [...allPageMap.values()]; - let publishedPages = new Set(); - if (publishConfig.publishAll) { - publishedPages = new Set(allPages.map((p) => p.name)); - } else { - for (const page of allPages) { - if (publishConfig.tags && page.tags) { - for (const tag of page.tags) { - if (publishConfig.tags.includes(tag)) { - publishedPages.add(page.name); - } - } - } - // Some sanity checking - if (typeof page.name !== "string") { - continue; - } - if (publishConfig.prefixes) { - for (const prefix of publishConfig.prefixes) { - if (page.name.startsWith(prefix)) { - publishedPages.add(page.name); - } - } - } - } - } - console.log("Starting this thing", [...publishedPages]); - - let footer = ""; - - if (publishConfig.footerPage) { - footer = await space.readPage(publishConfig.footerPage); - } - - const publishedPagesArray = [...publishedPages]; - for (const page of publishedPagesArray) { - await generatePage( - page, - `${destDir}/${page.replaceAll(" ", "_")}/index.html`, - `${destDir}/${page}.md`, - publishedPagesArray, - publishConfig, - destDir, - footer, - ); - } - - if (publishConfig.indexPage) { - console.log("Writing", publishConfig.indexPage); - await generatePage( - publishConfig.indexPage, - `${destDir}/index.html`, - `${destDir}/index.md`, - publishedPagesArray, - publishConfig, - destDir, - footer, - ); - } -} - -export async function publishAllCommand() { - await editor.flashNotification("Publishing..."); - await await system.invokeFunction("server", "publishAll"); - await editor.flashNotification("Done!"); -} - -export function encodePageUrl(name: string): string { - return name.replaceAll(" ", "_"); -} - -async function collectAttachments(tree: ParseTree) { - const attachments: string[] = []; - collectNodesOfType(tree, "URL").forEach((node) => { - let url = node.children![0].text!; - if (url.indexOf("://") === -1) { - attachments.push(url); - } - }); - return attachments; -} - -function cleanMarkdown( - mdTree: ParseTree, - publishConfig: PublishConfig, - validPages: string[], -): string { - replaceNodesMatching(mdTree, (n) => { - if (n.type === "WikiLink") { - let page = n.children![1].children![0].text!; - if (page.includes("@")) { - page = page.split("@")[0]; - } - if (!validPages.includes(page)) { - // Replace with just page text - return { - text: `_${page}_`, - }; - } - } - // Simply get rid of these - if (n.type === "CommentBlock" || n.type === "Comment") { - return null; - } - if (n.type === "Hashtag") { - if (publishConfig.removeHashtags) { - return null; - } - } - }); - return renderToText(mdTree).trim(); -} diff --git a/silverbullet.ts b/silverbullet.ts index 0992e246..159cc3c3 100755 --- a/silverbullet.ts +++ b/silverbullet.ts @@ -7,7 +7,6 @@ import { versionCommand } from "./cmd/version.ts"; import { fixCommand } from "./cmd/fix.ts"; import { serveCommand } from "./cmd/server.ts"; import { plugCompileCommand } from "./cmd/plug_compile.ts"; -import { publishCommand } from "./cmd/publish.ts"; import { invokeFunction } from "./cmd/invokeFunction.ts"; await new Command() @@ -54,14 +53,6 @@ await new Command() default: "data.db", }) .action(invokeFunction) - // publish - .command("publish") - .description("Publish a SilverBullet site") - .arguments("") - .option("--index [type:boolean]", "Index space first", { default: false }) - .option("--watch, -w [type:boolean]", "Watch for changes", { default: false }) - .option("--output, -o ", "Output directory", { default: "web" }) - .action(publishCommand) // upgrade .command("upgrade", "Upgrade Silver Bullet") .action(upgradeCommand)