diff --git a/common/spaces/evented_space_primitives.ts b/common/spaces/evented_space_primitives.ts index 1fb149fb..4836d797 100644 --- a/common/spaces/evented_space_primitives.ts +++ b/common/spaces/evented_space_primitives.ts @@ -1,7 +1,9 @@ import { FileMeta } from "$sb/types.ts"; +import { IndexEvent } from "$sb/app_event.ts"; import { EventHook } from "../../plugos/hooks/event.ts"; import type { SpacePrimitives } from "./space_primitives.ts"; +import { fileMetaToPageMeta } from "../../web/space.ts"; /** * Events exposed: @@ -135,8 +137,9 @@ export class EventedSpacePrimitives implements SpacePrimitives { await this.dispatchEvent("page:saved", pageName, newMeta); await this.dispatchEvent("page:index_text", { name: pageName, + meta: fileMetaToPageMeta(newMeta), text, - }); + } as IndexEvent); } return newMeta; } finally { diff --git a/plug-api/app_event.ts b/plug-api/app_event.ts index 5ed624f2..910277f2 100644 --- a/plug-api/app_event.ts +++ b/plug-api/app_event.ts @@ -1,6 +1,6 @@ import type { ParseTree } from "$sb/lib/tree.ts"; import { TextChange } from "$sb/lib/change.ts"; -import { Query } from "$sb/types.ts"; +import { PageMeta, Query } from "$sb/types.ts"; export type AppEvent = | "page:click" @@ -32,11 +32,13 @@ export type ClickEvent = { export type IndexEvent = { name: string; + meta: PageMeta; text: string; }; export type IndexTreeEvent = { name: string; + meta: PageMeta; tree: ParseTree; }; diff --git a/plug-api/lib/async.test.ts b/plug-api/lib/async.test.ts index 5ccdf5aa..b7e9288d 100644 --- a/plug-api/lib/async.test.ts +++ b/plug-api/lib/async.test.ts @@ -1,5 +1,5 @@ -import { assertEquals } from "../../test_deps.ts"; -import { PromiseQueue, sleep } from "./async.ts"; +import { assert, assertEquals } from "../../test_deps.ts"; +import { batchRequests, PromiseQueue, sleep } from "./async.ts"; Deno.test("PromiseQueue test", async () => { const q = new PromiseQueue(); @@ -24,3 +24,19 @@ Deno.test("PromiseQueue test", async () => { }); assertEquals(wasRun, true); }); + +Deno.test("Batch test", async () => { + // Generate an array with numbers up to 100 + const elements = Array.from(Array(100).keys()); + const multiplied = await batchRequests(elements, async (batch) => { + await sleep(2); + // Batches should be 9 or smaller (last batch will be smaller) + assert(batch.length <= 9); + return batch.map((e) => e * 2); + }, 9); + assertEquals(multiplied, elements.map((e) => e * 2)); + const multiplied2 = await batchRequests(elements, async (batch) => { + return batch.map((e) => e * 2); + }, 10000); + assertEquals(multiplied2, elements.map((e) => e * 2)); +}); diff --git a/plug-api/lib/async.ts b/plug-api/lib/async.ts index f0939b5c..e8b5ab41 100644 --- a/plug-api/lib/async.ts +++ b/plug-api/lib/async.ts @@ -67,3 +67,25 @@ export class PromiseQueue { this.run(); // Continue processing the next promise in the queue } } + +export async function batchRequests( + values: I[], + fn: (batch: I[]) => Promise, + batchSize: number, +): Promise { + const results: O[] = []; + // Split values into batches of batchSize + const batches: I[][] = []; + for (let i = 0; i < values.length; i += batchSize) { + batches.push(values.slice(i, i + batchSize)); + } + // Run fn on them in parallel + const batchResults = await Promise.all(batches.map(fn)); + // Flatten the results + for (const batchResult of batchResults) { + if (Array.isArray(batchResult)) { // If fn returns an array, collect them + results.push(...batchResult); + } + } + return results; +} diff --git a/plugs/index/anchor.ts b/plugs/index/anchor.ts index c28ae070..d70909ed 100644 --- a/plugs/index/anchor.ts +++ b/plugs/index/anchor.ts @@ -43,7 +43,10 @@ export async function anchorComplete(completeEvent: CompleteEvent) { // "bare" anchor, match any page for completion purposes filter = undefined; } - const allAnchors = await queryObjects("anchor", { filter }); + const allAnchors = await queryObjects("anchor", { + filter, + cacheSecs: 5, + }); return { from: completeEvent.pos - match[1].length, options: allAnchors.map((a) => ({ diff --git a/plugs/index/attributes.ts b/plugs/index/attributes.ts index bb59f931..18db8ade 100644 --- a/plugs/index/attributes.ts +++ b/plugs/index/attributes.ts @@ -42,23 +42,20 @@ export function determineType(v: any): string { export async function objectAttributeCompleter( attributeCompleteEvent: AttributeCompleteEvent, ): Promise { - const prefixFilter: QueryExpression = ["call", "startsWith", [[ - "attr", - "name", - ], ["string", attributeCompleteEvent.prefix]]]; const attributeFilter: QueryExpression | undefined = attributeCompleteEvent.source === "" - ? prefixFilter - : ["and", prefixFilter, ["=", ["attr", "tagName"], [ + ? undefined + : ["=", ["attr", "tagName"], [ "string", attributeCompleteEvent.source, - ]]]; + ]]; const allAttributes = await queryObjects("attribute", { filter: attributeFilter, distinct: true, - select: [{ name: "name" }, { name: "attributeType" }, { name: "tag" }, { + select: [{ name: "name" }, { name: "attributeType" }, { name: "tagName" }, { name: "readOnly", }], + cacheSecs: 5, }); return allAttributes.map((value) => { return { diff --git a/plugs/index/command.ts b/plugs/index/command.ts index 4fd8d984..98d2ccb8 100644 --- a/plugs/index/command.ts +++ b/plugs/index/command.ts @@ -1,6 +1,6 @@ import { editor, events, markdown, mq, space, system } from "$sb/syscalls.ts"; import { sleep } from "$sb/lib/async.ts"; -import { IndexEvent } from "$sb/app_event.ts"; +import { IndexEvent, IndexTreeEvent } from "$sb/app_event.ts"; import { MQMessage } from "$sb/types.ts"; import { isTemplate } from "$sb/lib/cheap_yaml.ts"; @@ -44,30 +44,34 @@ export async function processIndexQueue(messages: MQMessage[]) { await events.dispatchEvent("page:indexTemplate", { name, tree: parsed, - }); + } as IndexTreeEvent); } else { await events.dispatchEvent("page:index", { name, tree: parsed, - }); + } as IndexTreeEvent); } } } -export async function parseIndexTextRepublish({ name, text }: IndexEvent) { +export async function parseIndexTextRepublish( + { name, text, meta }: IndexEvent, +) { const parsed = await markdown.parseMarkdown(text); if (isTemplate(text)) { console.log("Indexing", name, "as template"); await events.dispatchEvent("page:indexTemplate", { name, + meta, tree: parsed, - }); + } as IndexTreeEvent); } else { console.log("Indexing", name, "as page"); await events.dispatchEvent("page:index", { name, + meta, tree: parsed, - }); + } as IndexTreeEvent); } } diff --git a/plugs/index/tags.ts b/plugs/index/tags.ts index 37e3fa3e..5062f7b5 100644 --- a/plugs/index/tags.ts +++ b/plugs/index/tags.ts @@ -73,6 +73,7 @@ export async function tagComplete(completeEvent: CompleteEvent) { filter: ["=", ["attr", "parent"], ["string", parent]], select: [{ name: "name" }], distinct: true, + cacheSecs: 5, }); if (parent === "page") { diff --git a/plugs/tasks/complete.ts b/plugs/tasks/complete.ts index abf2dd2e..fd47d8f4 100644 --- a/plugs/tasks/complete.ts +++ b/plugs/tasks/complete.ts @@ -9,7 +9,9 @@ export async function completeTaskState(completeEvent: CompleteEvent) { if (!taskMatch) { return null; } - const allStates = await queryObjects("taskstate", {}); + const allStates = await queryObjects("taskstate", { + cacheSecs: 5, + }); const states = [...new Set(allStates.map((s) => s.state))]; return { diff --git a/plugs/template/complete.ts b/plugs/template/complete.ts index 64c69633..571795dc 100644 --- a/plugs/template/complete.ts +++ b/plugs/template/complete.ts @@ -56,6 +56,7 @@ export async function templateSlashComplete( "boolean", false, ]]], + cacheSecs: 5, }); return allTemplates.map((template) => ({ label: template.trigger!, diff --git a/web/remote_datastore.ts b/web/remote_datastore.ts index de0ed563..e20b4177 100644 --- a/web/remote_datastore.ts +++ b/web/remote_datastore.ts @@ -3,6 +3,9 @@ import { KV, KvKey, KvQuery } from "$sb/types.ts"; import { DataStore } from "../plugos/lib/datastore.ts"; import { rpcCall } from "./syscalls/datastore.proxy.ts"; import { LimitedMap } from "../common/limited_map.ts"; +import { batchRequests } from "$sb/lib/async.ts"; + +const batchSize = 1000; export class RemoteDataStore implements DataStore { private cache = new LimitedMap(20); @@ -29,6 +32,7 @@ export class RemoteDataStore implements DataStore { return results[0]; } + // TODO: Batch these up batchGet(keys: KvKey[]): Promise<(T | null)[]> { return this.proxy("datastore.batchGet", keys); } @@ -37,16 +41,26 @@ export class RemoteDataStore implements DataStore { return this.batchSet([{ key, value }]); } - batchSet(entries: KV[]): Promise { - return this.proxy("datastore.batchSet", entries); + // TODO: Batch these up + async batchSet(entries: KV[]): Promise { + await batchRequests( + entries, + (entries) => this.proxy("datastore.batchSet", entries), + batchSize, + ); } delete(key: KvKey): Promise { return this.batchDelete([key]); } - batchDelete(keys: KvKey[]): Promise { - return this.proxy("datastore.batchDelete", keys); + // TODO: batch these up + async batchDelete(keys: KvKey[]): Promise { + await batchRequests( + keys, + (keys) => this.proxy("datastore.batchDelete", keys), + batchSize, + ); } /**