From c6a45be4bb5fbc25a7f3d6f10fe858938a6a007a Mon Sep 17 00:00:00 2001 From: Zef Hemel Date: Mon, 29 May 2023 09:53:49 +0200 Subject: [PATCH] Fixes #409 --- cmd/server.ts | 36 +++++++++----------- common/deps.ts | 2 ++ common/spaces/disk_space_primitives.ts | 13 +------- common/spaces/filtered_space_primitives.ts | 35 +++++++++++++++++++ common/util.ts | 8 ++++- server/http_server.ts | 39 +++++++++++++++++++--- web/editor.tsx | 26 ++++++++++++--- web/types.ts | 2 ++ website/CHANGELOG.md | 5 +++ website/SETTINGS.md | 10 +++--- 10 files changed, 129 insertions(+), 47 deletions(-) create mode 100644 common/spaces/filtered_space_primitives.ts diff --git a/cmd/server.ts b/cmd/server.ts index 95294873..9ebd4cc2 100644 --- a/cmd/server.ts +++ b/cmd/server.ts @@ -43,37 +43,33 @@ To allow outside connections, pass -L 0.0.0.0 as a flag, and put a TLS terminato } let spacePrimitives: SpacePrimitives | undefined; if (folder === "s3://") { - spacePrimitives = new AssetBundlePlugSpacePrimitives( - new S3SpacePrimitives({ - accessKey: Deno.env.get("AWS_ACCESS_KEY_ID")!, - secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!, - endPoint: Deno.env.get("AWS_ENDPOINT")!, - region: Deno.env.get("AWS_REGION")!, - bucket: Deno.env.get("AWS_BUCKET")!, - }), - new AssetBundle(plugAssetBundle as AssetJson), - ); + spacePrimitives = new S3SpacePrimitives({ + accessKey: Deno.env.get("AWS_ACCESS_KEY_ID")!, + secretKey: Deno.env.get("AWS_SECRET_ACCESS_KEY")!, + endPoint: Deno.env.get("AWS_ENDPOINT")!, + region: Deno.env.get("AWS_REGION")!, + bucket: Deno.env.get("AWS_BUCKET")!, + }); console.log("Running in S3 mode"); } else { + // Regular disk mode folder = path.resolve(Deno.cwd(), folder); - spacePrimitives = new AssetBundlePlugSpacePrimitives( - new DiskSpacePrimitives(folder, { - maxFileSizeMB: options.maxFileSizeMB, - }), - new AssetBundle(plugAssetBundle as AssetJson), - ); + spacePrimitives = new DiskSpacePrimitives(folder); } - console.log("Serving pages from", folder); + spacePrimitives = new AssetBundlePlugSpacePrimitives( + spacePrimitives, + new AssetBundle(plugAssetBundle as AssetJson), + ); - const httpServer = new HttpServer(spacePrimitives, { + const httpServer = new HttpServer(spacePrimitives!, { hostname, port: port, - pagesPath: folder, + pagesPath: folder!, clientAssetBundle: new AssetBundle(clientAssetBundle as AssetJson), user: options.user ?? Deno.env.get("SB_USER"), keyFile: options.key, certFile: options.cert, maxFileSizeMB: +maxFileSizeMB, }); - httpServer.start().catch(console.error); + return httpServer.start(); } diff --git a/common/deps.ts b/common/deps.ts index 1fd8a941..508154a5 100644 --- a/common/deps.ts +++ b/common/deps.ts @@ -124,3 +124,5 @@ export { } from "https://esm.sh/@codemirror/lang-javascript@6.1.8?external=@codemirror/language,@codemirror/autocomplete,@codemirror/view,@codemirror/state,@codemirror/lint,@lezer/common,@lezer/lr,@lezer/javascript,@codemirror/commands"; export { mime } from "https://deno.land/x/mimetypes@v1.0.0/mod.ts"; + +export { compile as gitIgnoreCompiler } from "https://esm.sh/gitignore-parser@0.0.2"; diff --git a/common/spaces/disk_space_primitives.ts b/common/spaces/disk_space_primitives.ts index 9892c679..614a3593 100644 --- a/common/spaces/disk_space_primitives.ts +++ b/common/spaces/disk_space_primitives.ts @@ -16,14 +16,10 @@ function normalizeForwardSlashPath(path: string) { const excludedFiles = ["data.db", "data.db-journal", "sync.json"]; -export type DiskSpaceOptions = { - maxFileSizeMB?: number; -}; - export class DiskSpacePrimitives implements SpacePrimitives { rootPath: string; - constructor(rootPath: string, private options: DiskSpaceOptions = {}) { + constructor(rootPath: string) { this.rootPath = Deno.realPathSync(rootPath); } @@ -150,13 +146,6 @@ export class DiskSpacePrimitives implements SpacePrimitives { const fullPath = file.path; try { const s = await Deno.stat(fullPath); - // Don't list file exceeding the maximum file size - if ( - this.options.maxFileSizeMB && - s.size / (1024 * 1024) > this.options.maxFileSizeMB - ) { - continue; - } const name = fullPath.substring(this.rootPath.length + 1); if (excludedFiles.includes(name)) { continue; diff --git a/common/spaces/filtered_space_primitives.ts b/common/spaces/filtered_space_primitives.ts new file mode 100644 index 00000000..20b33c17 --- /dev/null +++ b/common/spaces/filtered_space_primitives.ts @@ -0,0 +1,35 @@ +import { FileMeta } from "../types.ts"; +import { SpacePrimitives } from "./space_primitives.ts"; + +export class FilteredSpacePrimitives implements SpacePrimitives { + constructor( + private wrapped: SpacePrimitives, + private filterFn: (name: FileMeta) => boolean, + private onFetchList?: () => Promise, + ) { + } + + async fetchFileList(): Promise { + if (this.onFetchList) { + await this.onFetchList(); + } + return (await this.wrapped.fetchFileList()).filter(this.filterFn); + } + readFile(name: string): Promise<{ data: Uint8Array; meta: FileMeta }> { + return this.wrapped.readFile(name); + } + getFileMeta(name: string): Promise { + return this.wrapped.getFileMeta(name); + } + writeFile( + name: string, + data: Uint8Array, + selfUpdate?: boolean | undefined, + lastModified?: number | undefined, + ): Promise { + return this.wrapped.writeFile(name, data, selfUpdate, lastModified); + } + deleteFile(name: string): Promise { + return this.wrapped.deleteFile(name); + } +} diff --git a/common/util.ts b/common/util.ts index 9624435e..4ce85fba 100644 --- a/common/util.ts +++ b/common/util.ts @@ -36,14 +36,18 @@ export function parseYamlSettings(settingsMarkdown: string): { export async function ensureSettingsAndIndex( space: SpacePrimitives, ): Promise { + let settingsText: string | undefined; try { - await space.getFileMeta("SETTINGS.md"); + settingsText = new TextDecoder().decode( + (await space.readFile("SETTINGS.md")).data, + ); } catch { await space.writeFile( "SETTINGS.md", new TextEncoder().encode(SETTINGS_TEMPLATE), true, ); + settingsText = SETTINGS_TEMPLATE; // Ok, then let's also write the index page try { await space.getFileMeta("index.md"); @@ -60,4 +64,6 @@ Loading some onboarding content for you (but doing so does require a working int ); } } + + return parseYamlSettings(settingsText); } diff --git a/server/http_server.ts b/server/http_server.ts index c6880cd7..620b0b96 100644 --- a/server/http_server.ts +++ b/server/http_server.ts @@ -4,6 +4,9 @@ import { AssetBundle } from "../plugos/asset_bundle/bundle.ts"; import { base64Decode } from "../plugos/asset_bundle/base64.ts"; import { ensureSettingsAndIndex } from "../common/util.ts"; import { performLocalFetch } from "../common/proxy_fetch.ts"; +import { BuiltinSettings } from "../web/types.ts"; +import { gitIgnoreCompiler } from "./deps.ts"; +import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts"; export type ServerOptions = { hostname: string; @@ -22,12 +25,13 @@ export class HttpServer { private hostname: string; private port: number; user?: string; - settings: { [key: string]: any } = {}; abortController?: AbortController; clientAssetBundle: AssetBundle; + settings?: BuiltinSettings; + spacePrimitives: SpacePrimitives; constructor( - private spacePrimitives: SpacePrimitives, + spacePrimitives: SpacePrimitives, private options: ServerOptions, ) { this.hostname = options.hostname; @@ -35,6 +39,29 @@ export class HttpServer { this.app = new Application(); this.user = options.user; this.clientAssetBundle = options.clientAssetBundle; + + let fileFilterFn: (s: string) => boolean = () => true; + this.spacePrimitives = new FilteredSpacePrimitives( + spacePrimitives, + (meta) => { + // Don't list file exceeding the maximum file size + if ( + options.maxFileSizeMB && + meta.size / (1024 * 1024) > options.maxFileSizeMB + ) { + return false; + } + return fileFilterFn(meta.name); + }, + async () => { + await this.reloadSettings(); + if (typeof this.settings?.spaceIgnore === "string") { + fileFilterFn = gitIgnoreCompiler(this.settings.spaceIgnore).accepts; + } else { + fileFilterFn = () => true; + } + }, + ); } // Replaces some template variables in index.html in a rather ad-hoc manner, but YOLO @@ -50,8 +77,7 @@ export class HttpServer { } async start() { - await ensureSettingsAndIndex(this.spacePrimitives); - + await this.reloadSettings(); // Serve static files (javascript, css, html) this.app.use(async ({ request, response }, next) => { if (request.url.pathname === "/") { @@ -137,6 +163,11 @@ export class HttpServer { ); } + async reloadSettings() { + // TODO: Throttle this? + this.settings = await ensureSettingsAndIndex(this.spacePrimitives); + } + private addPasswordAuth(app: Application) { const excludedPaths = [ "/manifest.json", diff --git a/web/editor.tsx b/web/editor.tsx index 626dffe1..dd8de568 100644 --- a/web/editor.tsx +++ b/web/editor.tsx @@ -15,6 +15,7 @@ import { EditorSelection, EditorState, EditorView, + gitIgnoreCompiler, highlightSpecialChars, history, historyKeymap, @@ -134,6 +135,8 @@ import { SyncStatus } from "../common/spaces/sync.ts"; import { HttpSpacePrimitives } from "../common/spaces/http_space_primitives.ts"; import { FallbackSpacePrimitives } from "../common/spaces/fallback_space_primitives.ts"; import { syncSyscalls } from "./syscalls/sync.ts"; +import { FilteredSpacePrimitives } from "../common/spaces/filtered_space_primitives.ts"; +import { globToRegExp } from "https://deno.land/std@0.189.0/path/glob.ts"; const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/; @@ -247,12 +250,24 @@ export class Editor { namespaceHook, ); - const localSpacePrimitives = new FileMetaSpacePrimitives( - new EventedSpacePrimitives( - plugSpacePrimitives, - this.eventHook, + let fileFilterFn: (s: string) => boolean = () => true; + const localSpacePrimitives = new FilteredSpacePrimitives( + new FileMetaSpacePrimitives( + new EventedSpacePrimitives( + plugSpacePrimitives, + this.eventHook, + ), + indexSyscalls, ), - indexSyscalls, + (meta) => fileFilterFn(meta.name), + async () => { + await this.loadSettings(); + if (typeof this.settings?.spaceIgnore === "string") { + fileFilterFn = gitIgnoreCompiler(this.settings.spaceIgnore).accepts; + } else { + fileFilterFn = () => true; + } + }, ); this.space = new Space(localSpacePrimitives); @@ -458,6 +473,7 @@ export class Editor { this.syncService.start(); this.eventHook.addLocalListener("sync:success", async (operations) => { + // console.log("Operations", operations); if (operations > 0) { // Update the page list await this.space.updatePageList(); diff --git a/web/types.ts b/web/types.ts index b8f3d570..762c572e 100644 --- a/web/types.ts +++ b/web/types.ts @@ -33,6 +33,8 @@ export type PanelMode = number; export type BuiltinSettings = { indexPage: string; + // Format: compatible with docker ignore + spaceIgnore?: string; }; export type PanelConfig = { diff --git a/website/CHANGELOG.md b/website/CHANGELOG.md index 3bee7af9..77eb8f12 100644 --- a/website/CHANGELOG.md +++ b/website/CHANGELOG.md @@ -1,6 +1,11 @@ An attempt at documenting the changes/new features introduced in each release. +## Next + +* Added `spaceIgnore` setting to not sync specific folders or file patterns to the client, see [[SETTINGS]] for documentation + + --- ## 0.3.1 diff --git a/website/SETTINGS.md b/website/SETTINGS.md index b4015281..25accf50 100644 --- a/website/SETTINGS.md +++ b/website/SETTINGS.md @@ -20,9 +20,9 @@ weeklyNoteMonday: false # Markdown previewOnRHS: true -# Sync -sync: - # Do not sync pages with a specific prefix - excludePrefixes: - - PLUGS +# Defines files to ignore in a format compatible with .gitignore +spaceIgnore: | + dist + largefolder + *.mp4 ```