mingzailao
/
silverbullet
mirror of https://github.com/silverbulletmd/silverbullet.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Change how attribute indexing and completion works (#1061) 

* Creation of separate aspring-page objects for pages linked to, but not created
* Show "No results" instead of broken markdown table for no query results
* Show schema validation errors
* Deno upgrade
* Adds config support to plugs (see examples)
* Moves all builtin schemas to plug config
* Adds core plug just for builtin schemas
* Changes how attributes are indexed and completed, now attempts to derive a JSON schema for ad hoc attributes
pull/1062/head
Zef Hemel 2024-08-24 12:35:09 +02:00 committed by GitHub
parent 6f91b65457
commit 80f9c14b96
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
39 changed files with 1061 additions and 407 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.github/workflows/docker.yml vendored
View File

@ -33,7 +33,7 @@ jobs:
- name: Setup Deno
uses: denoland/setup-deno@v1
with:
deno-version: v1.45
deno-version: v1.46
- name: Run bundle build
run: |

2
.github/workflows/release.yml vendored
View File

@ -16,7 +16,7 @@ jobs:
- name: Setup Deno
uses: denoland/setup-deno@v1
with:
deno-version: v1.45
deno-version: v1.46
- name: Run build
run: deno task build
- name: Bundle

2
.github/workflows/server.yml vendored
View File

@ -16,7 +16,7 @@ jobs:
- name: Setup Deno
uses: denoland/setup-deno@v1
with:
deno-version: v1.45
deno-version: v1.46
- name: Build bundles
run: |

2
.github/workflows/test.yml vendored
View File

@ -20,7 +20,7 @@ jobs:
- name: Setup Deno
uses: denoland/setup-deno@v1
with:
deno-version: v1.45
deno-version: v1.46
- name: Run build
run: deno task build

2
.gitpod.Dockerfile vendored
View File

@ -1,6 +1,6 @@
FROM gitpod/workspace-full:latest
RUN curl -fsSL https://deno.land/x/install/install.sh | sh -s v1.45.3
RUN curl -fsSL https://deno.land/x/install/install.sh | sh -s v1.46.1
RUN /home/gitpod/.deno/bin/deno completions bash > /home/gitpod/.bashrc.d/90-deno && \
echo 'export DENO_INSTALL="/home/gitpod/.deno"' >> /home/gitpod/.bashrc.d/90-deno && \
echo 'export PATH="$DENO_INSTALL/bin:$PATH"' >> /home/gitpod/.bashrc.d/90-deno

2
Dockerfile
View File

@ -1,4 +1,4 @@
FROM denoland/deno:debian-1.45.4
FROM denoland/deno:debian-1.46.1
# The volume that will keep the space data

5
cmd/compile.ts
View File

@ -43,6 +43,11 @@ export async function compileManifest(
);
manifest.assets = assetsBundle.toJSON();
// Normalize the edge case of a plug with no functions
if (!manifest.functions) {
manifest.functions = {};
}
const jsFile = `
import { setupMessageListener } from "${
options.runtimeUrl || workerRuntimeUrl

10
common/config.ts
View File

@ -54,12 +54,20 @@ async function loadConfigsFromSystem(
console.warn("Index plug not loaded yet, falling back to default config");
return defaultConfig;
}
// We'll start witht the default config
let fullConfig: any = { ...defaultConfig };
// Then merge in all plug-based configs
for (const plugDef of system.loadedPlugs.values()) {
if (plugDef.manifest?.config) {
const plugConfig = cleanupJSON(plugDef.manifest.config);
fullConfig = deepObjectMerge(fullConfig, plugConfig);
}
}
// Query all space-configs
const allConfigs: ConfigObject[] = await system.invokeFunction(
"index.queryObjects",
["space-config", {}],
);
let fullConfig: any = { ...defaultConfig };
// Now let's intelligently merge them
for (const config of allConfigs) {
let configObject = { [config.key]: config.value };

4
common/syscalls/jsonschema.ts
View File

@ -25,6 +25,7 @@ export function jsonschemaSyscalls(): SysCallMapping {
schema: any,
object: any,
): undefined | string => {
try {
const validate = ajv.compile(schema);
if (validate(object)) {
return;
@ -34,6 +35,9 @@ export function jsonschemaSyscalls(): SysCallMapping {
text = text.replace(/^data[\.\s]/, "");
return text;
}
} catch (e) {
return e.message;
}
},
"jsonschema.validateSchema": (
_ctx,

5
lib/plugos/types.ts
View File

@ -29,6 +29,11 @@ export interface Manifest<HookT> {
* see: common/manifest.ts#SilverBulletHooks
*/
functions: Record<string, FunctionDef<HookT>>;
/**
* A map of configuration options for the plug (to be merged with the system configuration).
*/
config?: any;
}
/** Associates hooks with a function. This is the generic base structure, that identifies the function. Hooks are defined by the type parameter. */

4
plug-api/lib/frontmatter.ts
View File

@ -108,8 +108,8 @@ export async function extractFrontmatter(
) {
return null;
}
} catch (e: any) {
console.warn("Could not parse frontmatter", e.message);
} catch {
// console.warn("Could not parse frontmatter", e.message);
}
}

4
plug-api/syscalls/space.ts
View File

@ -158,3 +158,7 @@ export function writeFile(
export function deleteFile(name: string): Promise<void> {
return syscall("space.deleteFile", name);
}
export function fileExists(name: string): Promise<boolean> {
return syscall("space.fileExists", name);
}

1
plugs/builtin_plugs.ts
View File

@ -1,5 +1,6 @@
// TODO: Figure out how to keep this up-to-date automatically
export const builtinPlugNames = [
"core",
"editor",
"index",
"sync",

109
plugs/core/core.plug.yaml Normal file
View File

@ -0,0 +1,109 @@
name: core
config:
# Built-in schemas
schema.tag:
syscall:
type: object
properties:
ref:
type: string
readOnly: true
tag:
type: string
readOnly: true
name:
type: string
argCount:
type: number
requiredPermissions:
type: array
items:
type: string
command:
type: object
properties:
ref:
type: string
readOnly: true
tag:
type: string
readOnly: true
name:
type: string
priority:
type: number
key:
type: string
nullable: true
mac:
type: string
nullable: true
hide:
type: boolean
nullable: true
requireMode:
type: string
enum: ["ro", "rw"]
nullable: true
space-config:
type: object
properties:
ref:
type: string
readOnly: true
tag:
type: string
readOnly: true
key:
type: string
value:
type:
- number
- string
- boolean
- object
- array
- null
space-style:
type: object
properties:
ref:
type: string
readOnly: true
tag:
type: string
readOnly: true
style:
type: string
origin:
type: string
space-script:
type: object
properties:
ref:
type: string
readOnly: true
tag:
type: string
readOnly: true
script:
type: string
# Built-in configuration schemas
schema.config.properties:
indexPage:
type: string
format: page-ref
maximumAttachmentSize:
type: number
nullable: true
objectDecorators:
type: array
items:
type: object
required:
- where
- attributes
nullable: true
spaceIgnore:
type: string
nullable: true

19
plugs/editor/complete.ts
View File

@ -8,7 +8,7 @@ import type {
import { listFilesCached } from "../federation/federation.ts";
import { queryObjects } from "../index/plug_api.ts";
import { folderName } from "@silverbulletmd/silverbullet/lib/resolve";
import type { LinkObject } from "../index/page_links.ts";
import type { AspiringPageObject } from "../index/page_links.ts";
import { localDateString } from "$lib/dates.ts";
// A meta page is a page tagged with either #template or #meta
@ -79,19 +79,16 @@ export async function pageComplete(completeEvent: CompleteEvent) {
filter: ["!=~", ["attr", "name"], ["regexp", "^_", ""]],
}, 5),
// And all links to non-existing pages (to augment the existing ones)
queryObjects<LinkObject>("link", {
filter: ["and", ["attr", "toPage"], ["not", ["call", "pageExists", [[
"attr",
"toPage",
]]]]],
select: [{ name: "toPage" }],
}, 5).then((brokenLinks) =>
queryObjects<AspiringPageObject>("aspiring-page", {
distinct: true,
select: [{ name: "name" }],
}, 5).then((aspiringPages) =>
// Rewrite them to PageMeta shaped objects
brokenLinks.map((link): PageMeta => ({
ref: link.toPage!,
aspiringPages.map((aspiringPage): PageMeta => ({
ref: aspiringPage.name,
tag: "page",
tags: ["non-existing"], // Picked up later in completion
name: link.toPage!,
name: aspiringPage.name,
created: "",
lastModified: "",
perm: "rw",

62
plugs/editor/editor.plug.yaml
View File

@ -1,6 +1,68 @@
name: editor
requiredPermissions:
- fetch
config:
schema.config.properties:
shortcuts:
type: array
items:
type: object
properties:
command:
type: string
key:
type: string
nullable: true
mac:
type: string
nullable: true
slashCommand:
type: string
nullable: true
priority:
type: number
nullable: true
required:
- command
nullable: true
useSmartQuotes:
type: boolean
nullable: true
pwaOpenLastPage:
type: boolean
nullable: true
hideEditButton:
type: boolean
nullable: true
hideSyncButton:
type: boolean
nullable: true
actionButtons:
type: array
items:
type: object
properties:
icon:
type: string
description:
type: string
nullable: true
command:
type: string
args:
type: array
items:
type: object
nullable: true
mobile:
type: boolean
nullable: true
required:
- icon
- command
defaultLinkStyle:
type: string
nullable: true
functions:
setEditorMode:
path: "./editor.ts:setEditorMode"

12
plugs/emoji/emoji.plug.yaml
View File

@ -1,4 +1,16 @@
name: emoji
config:
schema.config.properties:
emoji:
type: object
properties:
aliases:
type: object
additionalProperties:
type: string
required:
- aliases
nullable: true
functions:
emojiCompleter:
path: "./emoji.ts:emojiCompleter"

19
plugs/federation/federation.plug.yaml
View File

@ -1,6 +1,25 @@
name: federation
requiredPermissions:
- fetch
config:
schema.config.properties:
libraries:
type: array
items:
type: object
properties:
import:
type: string
format: page-ref
exclude:
type: array
items:
type: string
format: page-ref
nullable: true
required:
- import
nullable: true
functions:
readFile:
path: ./federation.ts:readFile

87
plugs/index/api.ts
View File

@ -1,4 +1,4 @@
import { datastore } from "@silverbulletmd/silverbullet/syscalls";
import { datastore, system } from "@silverbulletmd/silverbullet/syscalls";
import type {
KV,
KvKey,
@ -7,8 +7,7 @@ import type {
ObjectValue,
} from "../../plug-api/types.ts";
import type { QueryProviderEvent } from "../../plug-api/types.ts";
import { builtins } from "./builtins.ts";
import { determineType } from "./attributes.ts";
import { determineType, type SimpleJSONType } from "./attributes.ts";
import { ttlCache } from "$lib/memory_cache.ts";
const indexKey = "idx";
@ -79,12 +78,13 @@ export async function clearIndex(): Promise<void> {
/**
* Indexes entities in the data store
*/
export function indexObjects<T>(
export async function indexObjects<T>(
page: string,
objects: ObjectValue<T>[],
): Promise<void> {
const kvs: KV<T>[] = [];
const allAttributes = new Map<string, string>(); // tag:name -> attributeType
const schema = await system.getSpaceConfig("schema");
const allAttributes = new Map<string, SimpleJSONType>();
for (const obj of objects) {
if (!obj.tag) {
console.error("Object has no tag", obj, "this shouldn't happen");
@ -92,6 +92,8 @@ export function indexObjects<T>(
}
// Index as all the tag + any additional tags specified
const allTags = [obj.tag, ...obj.tags || []];
const tagSchemaProperties =
schema.tag[obj.tag] && schema.tag[obj.tag].properties || {};
for (const tag of allTags) {
// The object itself
kvs.push({
@ -99,41 +101,32 @@ export function indexObjects<T>(
value: obj,
});
// Index attributes
const builtinAttributes = builtins[tag];
if (!builtinAttributes) {
// This is not a builtin tag, so we index all attributes (almost, see below)
attributeLabel: for (
const [attrName, attrValue] of Object.entries(
obj as Record<string, any>,
)
) {
if (attrName.startsWith("$")) {
continue;
}
// Check for all tags attached to this object if they're builtins
// If so: if `attrName` is defined in the builtin, use the attributeType from there (mostly to preserve readOnly aspects)
for (const otherTag of allTags) {
const builtinAttributes = builtins[otherTag];
if (builtinAttributes && builtinAttributes[attrName]) {
allAttributes.set(
`${tag}:${attrName}`,
builtinAttributes[attrName],
);
continue attributeLabel;
}
}
allAttributes.set(`${tag}:${attrName}`, determineType(attrValue));
}
} else if (tag !== "attribute") {
// For builtin tags, only index custom ones
const schemaAttributes = schema.tag[tag] && schema.tag[tag].properties;
if (!schemaAttributes) {
// There is no schema definition for this tag, so we index all attributes
for (
const [attrName, attrValue] of Object.entries(
obj as Record<string, any>,
)
) {
// console.log("Indexing", tag, attrName, attrValue);
// Skip builtins and internal attributes
if (builtinAttributes[attrName] || attrName.startsWith("$")) {
if (attrName.startsWith("$") || tagSchemaProperties[attrName]) {
continue;
}
allAttributes.set(`${tag}:${attrName}`, determineType(attrValue));
}
} else {
// For tags with schemas, only index attributes that are not in the schema
for (
const [attrName, attrValue] of Object.entries(
obj as Record<string, any>,
)
) {
// Skip schema-defined and internal attributes
if (
schemaAttributes[attrName] || tagSchemaProperties[attrName] ||
attrName.startsWith("$")
) {
continue;
}
allAttributes.set(`${tag}:${attrName}`, determineType(attrValue));
@ -144,17 +137,15 @@ export function indexObjects<T>(
if (allAttributes.size > 0) {
[...allAttributes].forEach(([key, value]) => {
const [tagName, name] = key.split(":");
const attributeType = value.startsWith("!") ? value.substring(1) : value;
kvs.push({
key: ["attribute", cleanKey(key, page)],
key: ["ah-attr", cleanKey(key, page)],
value: {
ref: key,
tag: "attribute",
tag: "ah-attr",
tagName,
name,
attributeType,
readOnly: value.startsWith("!"),
page,
schema: value,
} as T,
});
});
@ -188,6 +179,16 @@ export function queryObjects<T>(
}, ttlSecs);
}
export function queryDeleteObjects<T>(
tag: string,
query: ObjectQuery,
): Promise<void> {
return datastore.queryDelete({
...query,
prefix: [indexKey, tag],
});
}
export async function query(
query: KvQuery,
variables?: Record<string, any>,
@ -220,11 +221,15 @@ export async function objectSourceProvider({
}
export async function discoverSources() {
const schema = await system.getSpaceConfig("schema");
// Query all tags we indexed
return (await datastore.query({
prefix: [indexKey, "tag"],
select: [{ name: "name" }],
distinct: true,
})).map((
{ value },
) => value.name);
) => value.name)
// And concatenate all the tags from the schema
.concat(Object.keys(schema.tag));
}

64
plugs/index/attributes.test.ts Normal file
View File

@ -0,0 +1,64 @@
import { assertEquals } from "@std/assert/equals";
import { determineType, jsonTypeToString } from "./attributes.ts";
Deno.test("JSON Determine type", () => {
// Determine type tests
assertEquals(determineType(null), { type: "null" });
assertEquals(determineType(undefined), { type: "null" });
assertEquals(determineType("hello"), { type: "string" });
assertEquals(determineType(10), { type: "number" });
assertEquals(determineType(true), { type: "boolean" });
assertEquals(determineType({}), { type: "object", properties: {} });
assertEquals(determineType([]), { type: "array" });
assertEquals(determineType([1]), {
type: "array",
items: { type: "number" },
});
assertEquals(
determineType({ name: "Pete", age: 10, siblings: ["Sarah"] }),
{
type: "object",
properties: {
name: { type: "string" },
age: { type: "number" },
siblings: { type: "array", items: { type: "string" } },
},
},
);
});
Deno.test("Serialize JSON Type to string", () => {
assertEquals(jsonTypeToString({ type: "string" }), "string");
assertEquals(jsonTypeToString({ type: "null" }), "null");
assertEquals(jsonTypeToString({ type: "number" }), "number");
assertEquals(jsonTypeToString({ type: "boolean" }), "boolean");
assertEquals(jsonTypeToString({ type: "array" }), "any[]");
assertEquals(
jsonTypeToString({ type: "array", items: { type: "number" } }),
"number[]",
);
assertEquals(
jsonTypeToString({ type: "object", properties: {} }),
"{}",
);
assertEquals(
jsonTypeToString({
type: "object",
properties: {
name: { type: "string" },
age: { type: "number" },
},
}),
"{name: string; age: number;}",
);
assertEquals(
jsonTypeToString({
anyOf: [
{ type: "string" },
{ type: "number" },
{ type: "boolean" },
],
}),
"string | number | boolean",
);
});

145
plugs/index/attributes.ts
View File

@ -3,16 +3,23 @@ import type {
ObjectValue,
QueryExpression,
} from "../../plug-api/types.ts";
import { events } from "@silverbulletmd/silverbullet/syscalls";
import { events, system } from "@silverbulletmd/silverbullet/syscalls";
import { queryObjects } from "./api.ts";
import { determineTags } from "$lib/cheap_yaml.ts";
import type { TagObject } from "./tags.ts";
export type AttributeObject = ObjectValue<{
export type SimpleJSONType = {
type?: "string" | "number" | "boolean" | "any" | "array" | "object" | "null";
items?: SimpleJSONType;
properties?: Record<string, SimpleJSONType>;
anyOf?: SimpleJSONType[];
};
export type AdhocAttributeObject = ObjectValue<{
name: string;
attributeType: string;
schema: SimpleJSONType;
tagName: string;
page: string;
readOnly: boolean;
}>;
export type AttributeCompleteEvent = {
@ -23,20 +30,11 @@ export type AttributeCompleteEvent = {
export type AttributeCompletion = {
name: string;
source: string;
// String version of JSON schema
attributeType: string;
readOnly: boolean;
readOnly?: boolean;
};
export function determineType(v: any): string {
const t = typeof v;
if (t === "object") {
if (Array.isArray(v)) {
return "array";
}
}
return t;
}
/**
* Triggered by the `attribute:complete:*` event (that is: gimme all attribute completions)
* @param attributeCompleteEvent
@ -49,21 +47,59 @@ export async function objectAttributeCompleter(
attributeCompleteEvent.source === ""
? undefined
: ["=", ["attr", "tagName"], ["string", attributeCompleteEvent.source]];
const allAttributes = await queryObjects<AttributeObject>("attribute", {
const schema = await system.getSpaceConfig("schema");
const allAttributes = (await queryObjects<AdhocAttributeObject>("ah-attr", {
filter: attributeFilter,
distinct: true,
select: [{ name: "name" }, { name: "attributeType" }, { name: "tag" }, {
name: "readOnly",
}, { name: "tagName" }],
}, 5);
return allAttributes.map((value) => {
select: [{ name: "name" }, { name: "schema" }, { name: "tag" }, {
name: "tagName",
}],
}, 5)).map((value) => {
return {
name: value.name,
source: value.tagName,
attributeType: value.attributeType,
readOnly: value.readOnly,
attributeType: jsonTypeToString(value.schema),
} as AttributeCompletion;
});
// Add attributes from the direct schema
addAttributeCompletionsForTag(
schema,
attributeCompleteEvent.source,
allAttributes,
);
// Look up the tag so we can check the parent as well
const sourceTags = await queryObjects<TagObject>("tag", {
filter: ["=", ["attr", "name"], ["string", attributeCompleteEvent.source]],
});
if (sourceTags.length > 0) {
addAttributeCompletionsForTag(schema, sourceTags[0].parent, allAttributes);
}
return allAttributes;
}
function addAttributeCompletionsForTag(
schema: any,
tag: string,
allAttributes: AttributeCompletion[],
) {
if (schema.tag[tag]) {
for (
const [name, value] of Object.entries(
schema.tag[tag].properties as Record<
string,
any
>,
)
) {
allAttributes.push({
name,
source: tag,
attributeType: jsonTypeToString(value),
readOnly: value.readOnly,
});
}
}
}
/**
@ -151,3 +187,66 @@ export function attributeCompletionsToCMCompletion(
}),
);
}
/**
* Attempt some reasonable stringification of a JSON schema
* @param schema
* @returns
*/
export function jsonTypeToString(schema: SimpleJSONType): string {
if (schema.anyOf) {
return schema.anyOf.map(jsonTypeToString).join(" | ");
} else if (schema.type === "array") {
if (schema.items) {
return `${jsonTypeToString(schema.items)}[]`;
} else {
return "any[]";
}
} else if (schema.type === "object") {
if (schema.properties) {
return `{${
Object.entries(schema.properties).map(([k, v]) =>
`${k}: ${jsonTypeToString(v)};`
).join(" ")
}}`;
} else {
return "{}";
}
}
return schema.type!;
}
export function determineType(v: any): SimpleJSONType {
const t = typeof v;
if (t === "undefined" || v === null) {
return { type: "null" };
} else if (t === "object") {
if (Array.isArray(v)) {
if (v.length === 0) {
return {
type: "array",
};
} else {
return {
type: "array",
items: determineType(v[0]),
};
}
} else {
return {
type: "object",
properties: Object.fromEntries(
Object.entries(v).map(([k, v]) => [k, determineType(v)]),
),
};
}
} else if (t === "number") {
return { type: "number" };
} else if (t === "boolean") {
return { type: "boolean" };
} else if (t === "string") {
return { type: "string" };
} else {
return { type: "any" };
}
}

166
plugs/index/builtins.ts
View File

@ -1,172 +1,8 @@
import { system } from "@silverbulletmd/silverbullet/syscalls";
import { indexObjects } from "./api.ts";
import type {
ObjectValue,
QueryProviderEvent,
} from "@silverbulletmd/silverbullet/types";
import type { QueryProviderEvent } from "@silverbulletmd/silverbullet/types";
import { applyQuery } from "@silverbulletmd/silverbullet/lib/query";
import { builtinFunctions } from "$lib/builtin_query_functions.ts";
export const builtinPseudoPage = ":builtin:";
// Types marked with a ! are read-only, they cannot be set by the user
export const builtins: Record<string, Record<string, string>> = {
page: {
ref: "!string",
name: "!string",
displayName: "string",
aliases: "string[]",
created: "!date",
lastModified: "!date",
perm: "!rw|ro",
contentType: "!string",
size: "!number",
tags: "string[]",
},
attachment: {
ref: "!string",
name: "!string",
created: "!date",
lastModified: "!date",
perm: "!rw|ro",
contentType: "!string",
size: "!number",
},
task: {
ref: "!string",
name: "!string",
done: "!boolean",
page: "!string",
state: "!string",
deadline: "string",
pos: "!number",
tags: "string[]",
},
item: {
ref: "!string",
name: "!string",
page: "!string",
tags: "string[]",
},
taskstate: {
ref: "!string",
tags: "!string[]",
state: "!string",
count: "!number",
page: "!string",
},
tag: {
ref: "!string",
name: "!string",
page: "!string",
context: "!string",
},
attribute: {
ref: "!string",
name: "!string",
attributeType: "!string",
tagName: "!string",
page: "!string",
readOnly: "!boolean",
},
anchor: {
ref: "!string",
name: "!string",
page: "!string",
pos: "!number",
},
link: {
ref: "!string",
name: "!string",
page: "!string",
pos: "!number",
alias: "!string",
asTemplate: "!boolean",
},
header: {
ref: "!string",
name: "!string",
page: "!string",
level: "!number",
pos: "!number",
},
paragraph: {
text: "!string",
page: "!string",
pos: "!number",
},
template: {
ref: "!string",
page: "!string",
pageName: "string",
pos: "!number",
hooks: "hooksSpec",
},
table: {
ref: "!string",
page: "!string",
pos: "!number",
},
// System builtins
syscall: {
name: "!string",
requiredPermissions: "!string[]",
argCount: "!number",
},
command: {
name: "!string",
priority: "!number",
key: "!string",
mac: "!string",
hide: "!boolean",
requireMode: "!rw|ro",
},
"space-config": {
key: "!string",
value: "!any",
},
"space-style": {
style: "!string",
origin: "!string",
},
"space-script": {
script: "!string",
},
};
export async function loadBuiltinsIntoIndex() {
if (await system.getMode() === "ro") {
console.log("Running in read-only mode, not loading builtins");
return;
}
console.log("Loading builtins attributes into index");
const allObjects: ObjectValue<any>[] = [];
for (const [tagName, attributes] of Object.entries(builtins)) {
allObjects.push({
ref: tagName,
tag: "tag",
name: tagName,
page: builtinPseudoPage,
parent: "builtin",
});
allObjects.push(
...Object.entries(attributes).map(([name, attributeType]) => ({
ref: `${tagName}:${name}`,
tag: "attribute",
tagName,
name,
attributeType: attributeType.startsWith("!")
? attributeType.substring(1)
: attributeType,
readOnly: attributeType.startsWith("!"),
page: builtinPseudoPage,
})),
);
}
await indexObjects(builtinPseudoPage, allObjects);
}
export async function syscallSourceProvider({
query,
variables,

2
plugs/index/command.ts
View File

@ -28,8 +28,6 @@ export async function reindexSpace(noClear = false) {
// Executed this way to not have to embed the search plug code here
await system.invokeFunction("index.clearIndex");
}
// Load builtins
await system.invokeFunction("index.loadBuiltinsIntoIndex");
// Pre-index SETTINGS page to get useful settings
console.log("Indexing SETTINGS page");
await indexPage("SETTINGS");

401
plugs/index/index.plug.yaml
View File

@ -1,11 +1,5 @@
name: index
functions:
loadBuiltinsIntoIndex:
path: builtins.ts:loadBuiltinsIntoIndex
env: server
events:
- system:ready
# Public API
batchSet:
path: api.ts:batchSet
@ -21,7 +15,6 @@ functions:
getObjectByRef:
path: api.ts:getObjectByRef
env: server
objectSourceProvider:
path: api.ts:objectSourceProvider
events:
@ -239,3 +232,397 @@ functions:
path: builtins.ts:commandSourceProvider
events:
- query:command
config:
# Schema for the built-in tags indexed by this plug
schema.tag:
page:
type: object
additionalProperties: true
properties:
ref:
type: string
readOnly: true
tag:
type: string
readOnly: true
enum:
- page
tags:
anyOf:
- type: array
items:
type: string
- type: string
itags:
type: array
readOnly: true
items:
type: string
nullable: true
name:
type: string
readOnly: true
pageDecoration:
type: object
properties:
prefix:
type: string
nullable: true
cssClasses:
type: array
items:
type: string
nullable: true
hide:
type: boolean
nullable: true
renderWidgets:
type: boolean
nullable: true
nullable: true
displayName:
type: string
nullable: true
aliases:
type: array
items:
type: string
nullable: true
created:
type: string
readOnly: true
contentType:
type: string
readOnly: true
size:
type: number
readOnly: true
lastModified:
type: string
readOnly: true
perm:
type: string
readOnly: true
enum:
- ro
- rw
lastOpened:
type: number
readOnly: true
nullable: true
aspiring-page:
type: object
additionalProperties: true
properties:
ref:
type: string
readOnly: true
tag:
type: string
readOnly: true
enum:
- aspiring-page
name:
type: string
readOnly: true
page:
type: string
readOnly: true
pos:
type: number
readOnly: true
attachment:
type: object
additionalProperties: true
properties:
ref:
type: string
readOnly: true
tag:
type: string
readOnly: true
enum:
- attachment
tags:
type: array
readOnly: true
items:
type: string
nullable: true
itags:
type: array
readOnly: true
items:
type: string
nullable: true
name:
type: string
readOnly: true
created:
readOnly: true
type: string
contentType:
type: string
readOnly: true
size:
type: number
readOnly: true
lastModified:
type: string
readOnly: true
perm:
type: string
readOnly: true
enum:
- ro
- rw
item:
type: object
additionalProperties: true
properties:
ref:
type: string
readOnly: true
tag:
type: string
readOnly: true
enum:
- item
tags:
type: array
items:
type: string
nullable: true
itags:
type: array
readOnly: true
items:
type: string
nullable: true
name:
type: string
readOnly: true
page:
type: string
readOnly: true
parent:
type: string
readOnly: true
pos:
type: number
readOnly: true
text:
type: string
readOnly: true
tag:
type: object
properties:
ref:
type: string
readOnly: true
tag:
type: string
readOnly: true
enum:
- tag
tags:
type: array
readOnly: true
items:
type: string
nullable: true
itags:
type: array
readOnly: true
items:
type: string
nullable: true
name:
type: string
readOnly: true
page:
type: string
readOnly: true
parent:
type: string
readOnly: true
context:
type: string
readOnly: true
anchor:
type: object
properties:
ref:
type: string
readOnly: true
tag:
type: string
enum:
- anchor
tags:
type: array
items:
type: string
nullable: true
itags:
type: array
items:
type: string
nullable: true
name:
type: string
page:
type: string
pos:
type: number
link:
type: object
properties:
ref:
type: string
readOnly: true
tag:
type: string
enum:
- link
tags:
type: array
items:
type: string
nullable: true
itags:
type: array
items:
type: string
nullable: true
name:
type: string
page:
type: string
toFile:
type: string
nullable: true
toPage:
type: string
nullable: true
snippet:
type: string
pos:
type: number
alias:
type: string
asTemplate:
type: boolean
header:
type: object
properties:
ref:
type: string
readOnly: true
tag:
type: string
enum:
- header
tags:
type: array
items:
type: string
nullable: true
itags:
type: array
items:
type: string
nullable: true
name:
type: string
page:
type: string
pos:
type: number
level:
type: string
paragraph:
type: object
properties:
ref:
type: string
readOnly: true
tag:
type: string
enum:
- paragraph
tags:
type: array
items:
type: string
nullable: true
itags:
type: array
items:
type: string
nullable: true
text:
type: string
page:
type: string
pos:
type: number
template:
type: object
properties:
ref:
type: string
readOnly: true
tag:
type: string
enum:
- template
tags:
anyOf:
- type: array
items:
type: string
- type: string
itags:
type: array
items:
type: string
nullable: true
page:
type: string
pageName:
type: string
description:
type: string
pos:
type: number
hooks:
type: object
frontmatter:
anyOf:
- type: object
- type: string
table:
type: object
properties:
ref:
type: string
readOnly: true
tag:
type: string
enum:
- table
tags:
type: array
items:
type: string
nullable: true
itags:
type: array
items:
type: string
nullable: true
page:
type: string
pos:
type: number

6
plugs/index/lint.ts
View File

@ -11,7 +11,7 @@ import {
} from "@silverbulletmd/silverbullet/lib/tree";
import type { LintEvent } from "../../plug-api/types.ts";
import { queryObjects } from "./api.ts";
import type { AttributeObject } from "./attributes.ts";
import type { AdhocAttributeObject } from "./attributes.ts";
import { extractFrontmatter } from "@silverbulletmd/silverbullet/lib/frontmatter";
import {
cleanupJSON,
@ -26,8 +26,8 @@ export async function lintYAML({ tree }: LintEvent): Promise<LintDiagnostic[]> {
await traverseTreeAsync(tree, async (node) => {
if (node.type === "FrontMatterCode") {
// Query all readOnly attributes for pages with this tag set
const readOnlyAttributes = await queryObjects<AttributeObject>(
"attribute",
const readOnlyAttributes = await queryObjects<AdhocAttributeObject>(
"ah-attr",
{
filter: ["and", ["=", ["attr", "tagName"], [
"array",

12
plugs/index/page.ts
View File

@ -1,4 +1,4 @@
import type { IndexTreeEvent } from "../../plug-api/types.ts";
import type { IndexTreeEvent } from "@silverbulletmd/silverbullet/types";
import {
editor,
markdown,
@ -9,13 +9,14 @@ import {
import type { LintDiagnostic, PageMeta } from "../../plug-api/types.ts";
import { extractFrontmatter } from "@silverbulletmd/silverbullet/lib/frontmatter";
import { extractAttributes } from "@silverbulletmd/silverbullet/lib/attribute";
import { indexObjects } from "./api.ts";
import { indexObjects, queryDeleteObjects } from "./api.ts";
import {
findNodeOfType,
renderToText,
traverseTreeAsync,
} from "../../plug-api/lib/tree.ts";
} from "@silverbulletmd/silverbullet/lib/tree";
import { updateITags } from "@silverbulletmd/silverbullet/lib/tags";
import type { AspiringPageObject } from "./page_links.ts";
export async function indexPage({ name, tree }: IndexTreeEvent) {
if (name.startsWith("_")) {
@ -62,6 +63,11 @@ export async function indexPage({ name, tree }: IndexTreeEvent) {
// console.log("Page object", combinedPageMeta);
await indexObjects<PageMeta>(name, [combinedPageMeta]);
// Make sure this page is no (longer) in the aspiring pages list
await queryDeleteObjects<AspiringPageObject>("aspiring-page", {
filter: ["=", ["attr", "name"], ["string", name]],
});
}
export async function lintFrontmatter(): Promise<LintDiagnostic[]> {

41
plugs/index/page_links.ts
View File

@ -23,6 +23,7 @@ import {
mdLinkRegex,
wikiLinkRegex,
} from "$common/markdown_parser/constants.ts";
import { space } from "@silverbulletmd/silverbullet/syscalls";
export type LinkObject = ObjectValue<
{
@ -50,6 +51,19 @@ export type LinkObject = ObjectValue<
}
>;
/**
* Represents a page that does not yet exist, but is being linked to. A page "aspiring" to be created.
*/
export type AspiringPageObject = ObjectValue<{
// ref: page@pos
// The page the link appears on
page: string;
// And the position
pos: number;
// The page the link points to
name: string;
}>;
export async function indexLinks({ name, tree }: IndexTreeEvent) {
const links: ObjectValue<LinkObject>[] = [];
const frontmatter = await extractFrontmatter(tree);
@ -194,8 +208,35 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) {
}
return false;
});
// console.log("Found", links, "page link(s)");
if (links.length > 0) {
await indexObjects(name, links);
}
// Now let's check which are aspiring pages
const aspiringPages: ObjectValue<AspiringPageObject>[] = [];
for (const link of links) {
if (link.toPage) {
// No federated links, nothing with template directives
if (link.toPage.startsWith("!") || link.toPage.includes("{{")) {
continue;
}
if (!await space.fileExists(`${link.toPage}.md`)) {
aspiringPages.push({
ref: `${name}@${link.pos}`,
tag: "aspiring-page",
page: name,
pos: link.pos,
name: link.toPage,
} as AspiringPageObject);
}
}
}
if (aspiringPages.length > 0) {
await indexObjects(name, aspiringPages);
}
}
export async function getBackLinks(

4
plugs/query/widget.ts
View File

@ -44,7 +44,11 @@ export async function widget(
},
);
if (Array.isArray(results)) {
if (results.length === 0) {
resultMarkdown = "No results";
} else {
resultMarkdown = jsonToMDTable(results);
}
} else {
resultMarkdown = results;
}

65
plugs/tasks/tasks.plug.yaml
View File

@ -41,3 +41,68 @@ functions:
command:
name: "Task: Remove Completed"
requireMode: rw
config:
schema.tag:
task:
type: object
additionalProperties: true
properties:
ref:
type: string
readOnly: true
tag:
type: string
enum:
- task
tags:
type: array
items:
type: string
nullable: true
itags:
type: array
items:
type: string
nullable: true
name:
type: string
page:
type: string
parent:
type: string
pos:
type: number
text:
type: string
state:
type: string
done:
type: boolean
deadline:
type: string
nullable: true
taskstate:
type: object
properties:
ref:
type: string
readOnly: true
tag:
type: string
enum:
- taskstate
tags:
type: array
items:
type: string
nullable: true
itags:
type: array
items:
type: string
nullable: true
state:
type: string
count:
type: number

11
server/server_system.ts
View File

@ -120,7 +120,7 @@ export class ServerSystem extends CommonSystem {
this.system.registerSyscalls(
[],
eventSyscalls(this.eventHook),
spaceReadSyscalls(space),
spaceReadSyscalls(space, this.allKnownFiles),
assetSyscalls(this.system),
yamlSyscalls(),
systemSyscalls(
@ -169,6 +169,7 @@ export class ServerSystem extends CommonSystem {
this.eventHook.addLocalListener(
"file:changed",
async (path, localChange) => {
this.allKnownFiles.add(path);
if (!localChange) {
console.log("Outside file change: reindexing", path);
// Change made outside of editor, trigger reindex
@ -190,6 +191,7 @@ export class ServerSystem extends CommonSystem {
},
);
// Keep allKnownFiles up to date
this.eventHook.addLocalListener(
"file:listed",
(allFiles: FileMeta[]) => {
@ -202,6 +204,13 @@ export class ServerSystem extends CommonSystem {
});
},
);
this.eventHook.addLocalListener(
"file:deleted",
(path: string) => {
// Update list of known pages and attachments
this.allKnownFiles.delete(path);
},
);
// All setup, enable eventing
eventedSpacePrimitives.enabled = true;

8
server/syscalls/space.ts
View File

@ -9,7 +9,10 @@ import type { Space } from "../../common/space.ts";
/**
* Almost the same as web/syscalls/space.ts except leaving out client-specific stuff
*/
export function spaceReadSyscalls(space: Space): SysCallMapping {
export function spaceReadSyscalls(
space: Space,
allKnownFiles: Set<string>,
): SysCallMapping {
return {
"space.listPages": (): Promise<PageMeta[]> => {
return space.fetchPageList();
@ -46,6 +49,9 @@ export function spaceReadSyscalls(space: Space): SysCallMapping {
"space.readFile": async (_ctx, name: string): Promise<Uint8Array> => {
return (await space.spacePrimitives.readFile(name)).data;
},
"space.fileExists": (_ctx, name: string): boolean => {
return allKnownFiles.has(name);
},
};
}

102
type/config.ts
View File

@ -58,102 +58,6 @@ type SchemaConfig = {
config: Record<string, any>; // any = JSONSchema
};
const configSchema = {
type: "object",
properties: {
indexPage: { type: "string", format: "page-ref" },
shortcuts: {
type: "array",
items: {
type: "object",
properties: {
command: { type: "string" },
key: { type: "string", nullable: true },
mac: { type: "string", nullable: true },
slashCommand: { type: "string", nullable: true },
priority: { type: "number", nullable: true },
},
required: ["command"],
},
nullable: true,
},
useSmartQuotes: { type: "boolean", nullable: true },
maximumAttachmentSize: { type: "number", nullable: true },
pwaOpenLastPage: { type: "boolean", nullable: true },
hideEditButton: { type: "boolean", nullable: true },
hideSyncButton: { type: "boolean", nullable: true },
libraries: {
type: "array",
items: {
type: "object",
properties: {
import: { type: "string", format: "page-ref" },
exclude: {
type: "array",
items: { type: "string", format: "page-ref" },
nullable: true,
},
},
required: ["import"],
},
nullable: true,
},
actionButtons: {
type: "array",
items: {
type: "object",
properties: {
icon: { type: "string" },
description: { type: "string", nullable: true },
command: { type: "string" },
args: {
type: "array",
items: { type: "object" },
nullable: true,
},
mobile: { type: "boolean", nullable: true },
},
required: ["icon", "command"],
},
},
objectDecorators: {
type: "array",
items: {
type: "object",
required: ["where", "attributes"],
},
nullable: true,
},
spaceIgnore: { type: "string", nullable: true },
emoji: {
type: "object",
properties: {
aliases: {
type: "object",
additionalProperties: {
type: "string",
},
},
},
required: ["aliases"],
nullable: true,
},
customStyles: {
anyOf: [
{ type: "string" },
{
type: "array",
items: { type: "string" },
},
{ type: "null" },
],
},
defaultLinkStyle: { type: "string", nullable: true },
},
additionalProperties: true,
required: [],
};
export const defaultConfig: Config = {
indexPage: "index",
hideSyncButton: false,
@ -162,7 +66,11 @@ export const defaultConfig: Config = {
actionButtons: [], // Actually defaults to defaultActionButtons
schema: {
config: configSchema,
config: {
type: "object",
properties: {},
additionalProperties: true,
},
tag: {},
},
};

24
web/client.ts
View File

@ -73,7 +73,7 @@ import { LimitedMap } from "$lib/limited_map.ts";
import { plugPrefix } from "$common/spaces/constants.ts";
import { lezerToParseTree } from "$common/markdown_parser/parse_tree.ts";
import { findNodeMatching } from "@silverbulletmd/silverbullet/lib/tree";
import type { LinkObject } from "../plugs/index/page_links.ts";
import type { AspiringPageObject } from "../plugs/index/page_links.ts";
import type { Config, ConfigContainer } from "../type/config.ts";
import { editor } from "@silverbulletmd/silverbullet/syscalls";
@ -754,19 +754,15 @@ export class Client implements ConfigContainer {
return;
}
const allPages = await this.clientSystem.queryObjects<PageMeta>("page", {});
const allBrokenLinkPages = (await this.clientSystem.queryObjects<
LinkObject
>("link", {
filter: ["and", ["attr", "toPage"], ["not", ["call", "pageExists", [[
"attr",
"toPage",
]]]]],
select: [{ name: "toPage" }],
})).map((link): PageMeta => ({
ref: link.toPage!,
const allAspiringPages = (await this.clientSystem.queryObjects<
AspiringPageObject
>("aspiring-page", {
select: [{ name: "name" }],
})).map((aspiringPage): PageMeta => ({
ref: aspiringPage.name,
tag: "page",
_isBrokenLink: true,
name: link.toPage!,
_isAspiring: true,
name: aspiringPage.name,
created: "",
lastModified: "",
perm: "rw",
@ -774,7 +770,7 @@ export class Client implements ConfigContainer {
this.ui.viewDispatch({
type: "update-page-list",
allPages: allPages.concat(allBrokenLinkPages),
allPages: allPages.concat(allAspiringPages),
});
}

4
web/components/page_navigator.tsx
View File

@ -70,12 +70,12 @@ export function PageNavigator({
name: (pageMeta.pageDecoration?.prefix ?? "") + pageMeta.name,
description,
orderId: orderId,
hint: pageMeta._isBrokenLink ? "Create page" : undefined,
hint: pageMeta._isAspiring ? "Create page" : undefined,
cssClass,
});
} else if (mode === "meta") {
// Special behavior for #template and #meta pages
if (pageMeta._isBrokenLink) {
if (pageMeta._isAspiring) {
// Skip over broken links
continue;
}

3
web/syscalls/space.ts
View File

@ -42,6 +42,9 @@ export function spaceReadSyscalls(editor: Client): SysCallMapping {
"space.readFile": async (_ctx, name: string): Promise<Uint8Array> => {
return (await editor.space.spacePrimitives.readFile(name)).data;
},
"space.fileExists": (_ctx, name: string): boolean => {
return editor.clientSystem.allKnownFiles.has(name);
},
};
}

3
website/Aspiring Pages.md Normal file
View File

@ -0,0 +1,3 @@
An aspiring page is a [[Pages|page]] that does not yet exist, but is already linked to.
Aspiring pages appear in the [[Page Picker]] (with a `Create page` hint) as well as in auto complete when creating [[Links]].

2
website/CHANGELOG.md
View File

@ -208,7 +208,7 @@ So, what’s the fuss all about?
* Ah yes, you can also still take notes with SilverBullet. That still works. I think.
## 0.6.1
* Tag pages: when you click on a #tag you will now be directed to a page that shows all pages, tasks, items and paragraphs tagged with that tag.
* Tag pages: when you click on a `#tag` you will now be directed to a page that shows all pages, tasks, items and paragraphs tagged with that tag.
* Action buttons (top right buttons) can now be configured; see [[SETTINGS]] for how to do this.
* Headers are now indexed, meaning you can query them [[Objects#header]] and also reference them by name via page links using `#` that I just demonstrated 👈. See [[Links]] for more information on all the types of link formats that SilverBullet now supports.
* New {[Task: Remove Completed]} command to remove all completed tasks from a page

12
website/Library/Core/Page/Maintenance.md
View File

@ -2,19 +2,17 @@
We would like to keep our space clean, these are some tools that help you do that.
# Broken links
This shows all internal links that are broken.
# Aspiring pages
This shows all page links that link to a page that does not (yet) exist. These could be broken links or just pages _aspiring_ to be created.
```template
{{#let @brokenLinks = {
link where toPage and not pageExists(toPage)
}}}
{{#let @brokenLinks = {aspiring-page}}}
{{#if @brokenLinks}}
{{#each @brokenLinks}}
* [[{{ref}}]]: broken link to [[{{toPage}}]]
* [[{{ref}}]]: broken link to [[{{name}}]]
{{/each}}
{{else}}
No broken links, all good!
No aspiring pages, all good!
{{/if}}
{{/let}}
```

20
website/Objects.md
View File

@ -39,6 +39,14 @@ Note that you can also query this page using the `level/intermediate` directly:
level/intermediate
```
## aspiring-page
[[Aspiring Pages]] are pages that are linked to, but not yet created.
```query
aspiring-page
```
## table
Markdown table rows are indexed using the `table` tag, any additional tags can be added using [[Tags]] in any of its cells.
@ -48,7 +56,6 @@ Markdown table rows are indexed using the `table` tag, any additional tags can b
| Some Row | This is an example row in between two others |
| Another key | This time without a tag |
```query
table
```
@ -153,10 +160,10 @@ In addition, the `snippet` attribute attempts to capture a little bit of context
_Note_: this is the data source used for the {[Mentions: Toggle]} feature as well page {[Page: Rename]}.
Here is a query that shows all links that appear in this particular page:
Here is a query that shows some links that appear in this particular page:
```query
link where page = @page.name
link where page = @page.name limit 5
```
## anchor
@ -185,13 +192,6 @@ Here are the tags used/defined in this page:
tag where page = @page.name select name, parent
```
## attribute
This is another meta tag, which is used to index all [[Attributes]] used in your space. This is used by e.g. attribute completion in various contexts. You likely don’t need to use this tag directly, but it’s there.
```query
attribute where page = @page.name limit 1
```
## space-config
This stores all configuration picked up as part of [[Space Config]]