mingzailao
/
silverbullet
mirror of https://github.com/silverbulletmd/silverbullet.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'main' of github.com:silverbulletmd/silverbullet

pull/628/head
Zef Hemel 2024-01-11 13:21:50 +01:00
parent a115c71381 848211120c
commit 154d9a8ce9
29 changed files with 276 additions and 162 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

30
common/space_index.ts Normal file
View File

@ -0,0 +1,30 @@
import { DataStore } from "../plugos/lib/datastore.ts";
import { System } from "../plugos/system.ts";
const indexVersionKey = ["$indexVersion"];
// Bump this one every time a full reinxex is needed
const desiredIndexVersion = 2;
let indexOngoing = false;
export async function ensureSpaceIndex(ds: DataStore, system: System<any>) {
const currentIndexVersion = await ds.get(indexVersionKey);
console.info("Current space index version", currentIndexVersion);
if (currentIndexVersion !== desiredIndexVersion && !indexOngoing) {
console.info("Performing a full space reindex, this could take a while...");
indexOngoing = true;
await system.loadedPlugs.get("index")!.invoke("reindexSpace", []);
console.info("Full space index complete.");
await markFullSpaceIndexComplete(ds);
indexOngoing = false;
} else {
console.info("Space index is up to date");
}
}
export async function markFullSpaceIndexComplete(ds: DataStore) {
await ds.set(indexVersionKey, desiredIndexVersion);
}

3
plug-api/lib/cheap_yaml.ts
View File

@ -1,5 +1,6 @@
const yamlKvRegex = /^\s*(\w+):\s*["']?([^'"]*)["']?$/;
const yamlListItemRegex = /^\s*-\s+["']?([^'"]+)["']?$/;
const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/;
/**
* Cheap YAML parser to determine tags (ugly, regex based but fast)
@ -35,8 +36,6 @@ export function determineTags(yamlText: string): string[] {
return tags;
}
const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/;
/**
* Quick and dirty way to check if a page is a template or not
* @param pageText

72
plug-api/lib/frontmatter.ts
View File

@ -2,14 +2,13 @@ import { YAML } from "$sb/plugos-syscall/mod.ts";
import {
addParentPointers,
collectNodesOfType,
ParseTree,
renderToText,
replaceNodesMatchingAsync,
traverseTreeAsync,
} from "$sb/lib/tree.ts";
export type FrontMatter = { tags: string[] } & Record<string, any>;
export type FrontMatter = { tags?: string[] } & Record<string, any>;
export type FrontmatterExtractOptions = {
removeKeys?: string[];
@ -17,8 +16,11 @@ export type FrontmatterExtractOptions = {
removeFrontmatterSection?: boolean;
};
// Extracts front matter from a markdown document
// optionally removes certain keys from the front matter
/**
* Extracts front matter from a markdown document, as well as extracting tags that are to apply to the page
* optionally removes certain keys from the front matter
* Side effect: will add parent pointers
*/
export async function extractFrontmatter(
tree: ParseTree,
options: FrontmatterExtractOptions = {},
@ -26,29 +28,44 @@ export async function extractFrontmatter(
let data: FrontMatter = {
tags: [],
};
const tags: string[] = [];
addParentPointers(tree);
let paragraphCounter = 0;
await replaceNodesMatchingAsync(tree, async (t) => {
// Find tags in the first paragraph to attach to the page
if (t.type === "Paragraph") {
paragraphCounter++;
// Only attach hashtags in the first paragraph to the page
if (paragraphCounter !== 1) {
return;
// Find tags in paragraphs directly nested under the document where the only content is tags
if (t.type === "Paragraph" && t.parent?.type === "Document") {
let onlyTags = true;
const collectedTags = new Set<string>();
for (const child of t.children!) {
if (child.text) {
if (child.text.startsWith("\n") && child.text !== "\n") {
// Multi line paragraph, cut it off here
break;
}
if (child.text.trim()) {
// Text node with actual text (not just whitespace): not a page tag line!
onlyTags = false;
break;
}
} else if (child.type === "Hashtag") {
const tagname = child.children![0].text!.substring(1);
collectedTags.add(tagname);
if (
options.removeTags === true || options.removeTags?.includes(tagname)
) {
// Ugly hack to remove the hashtag
child.children![0].text = "";
}
} else if (child.type) {
// Found something else than tags, so... nope!
onlyTags = false;
break;
}
}
if (onlyTags) {
tags.push(...collectedTags);
}
collectNodesOfType(t, "Hashtag").forEach((h) => {
const tagname = h.children![0].text!.substring(1);
if (!data.tags.includes(tagname)) {
data.tags.push(tagname);
}
if (
options.removeTags === true || options.removeTags?.includes(tagname)
) {
// Ugly hack to remove the hashtag
h.children![0].text = "";
}
});
}
// Find FrontMatter and parse it
if (t.type === "FrontMatter") {
@ -65,11 +82,9 @@ export async function extractFrontmatter(
// Normalize tags to an array
// support "tag1, tag2" as well as "tag1 tag2" as well as "#tag1 #tag2" notations
if (typeof data.tags === "string") {
data.tags = (data.tags as string).split(/,\s*|\s+/);
tags.push(...(data.tags as string).split(/,\s*|\s+/));
}
// Strip # from tags
data.tags = data.tags.map((t) => t.replace(/^#/, ""));
if (options.removeKeys && options.removeKeys.length > 0) {
let removedOne = false;
@ -97,6 +112,11 @@ export async function extractFrontmatter(
return undefined;
});
// Strip # from tags
data.tags = [...new Set([...tags.map((t) => t.replace(/^#/, ""))])];
// console.log("Extracted tags", data.tags);
return data;
}

14
plug-api/lib/tags.ts Normal file
View File

@ -0,0 +1,14 @@
import { FrontMatter } from "$sb/lib/frontmatter.ts";
import { ObjectValue } from "$sb/types.ts";
export function updateITags<T>(obj: ObjectValue<T>, frontmatter: FrontMatter) {
const itags = [obj.tag, ...frontmatter.tags || []];
if (obj.tags) {
for (const tag of obj.tags) {
if (!itags.includes(tag)) {
itags.push(tag);
}
}
}
obj.itags = itags;
}

4
plug-api/types.ts
View File

@ -118,7 +118,9 @@ export type FunctionMap = Record<string, (...args: any[]) => any>;
*/
export type ObjectValue<T> = {
ref: string;
tags: string[];
tag: string; // main tag
tags?: string[];
itags?: string[]; // implicit or inherited tags (inherited from the page for instance)
} & T;
export type ObjectQuery = Omit<Query, "prefix">;

2
plugs/editor/complete.ts
View File

@ -83,7 +83,7 @@ function fileMetaToPageMeta(fileMeta: FileMeta): PageMeta {
return {
...fileMeta,
ref: fileMeta.name,
tags: ["page"],
tag: "page",
name,
created: new Date(fileMeta.created).toISOString(),
lastModified: new Date(fileMeta.lastModified).toISOString(),

2
plugs/index/anchor.ts
View File

@ -16,7 +16,7 @@ export async function indexAnchors({ name: pageName, tree }: IndexTreeEvent) {
const aName = n.children![0].text!.substring(1);
anchors.push({
ref: `${pageName}$${aName}`,
tags: ["anchor"],
tag: "anchor",
name: aName,
page: pageName,
pos: n.from!,

16
plugs/index/api.ts
View File

@ -71,7 +71,13 @@ export async function indexObjects<T>(
const kvs: KV<T>[] = [];
const allAttributes = new Map<string, string>(); // tag:name -> attributeType
for (const obj of objects) {
for (const tag of obj.tags) {
if (!obj.tag) {
console.error("Object has no tag", obj, "this shouldn't happen");
continue;
}
// Index as all the tag + any additional tags specified
const allTags = [obj.tag, ...obj.tags || []];
for (const tag of allTags) {
// The object itself
kvs.push({
key: [tag, cleanKey(obj.ref, page)],
@ -91,7 +97,7 @@ export async function indexObjects<T>(
}
// Check for all tags attached to this object if they're builtins
// If so: if `attrName` is defined in the builtin, use the attributeType from there (mostly to preserve readOnly aspects)
for (const otherTag of obj.tags) {
for (const otherTag of allTags) {
const builtinAttributes = builtins[otherTag];
if (builtinAttributes && builtinAttributes[attrName]) {
allAttributes.set(
@ -124,14 +130,14 @@ export async function indexObjects<T>(
await indexObjects<AttributeObject>(
page,
[...allAttributes].map(([key, value]) => {
const [tag, name] = key.split(":");
const [tagName, name] = key.split(":");
const attributeType = value.startsWith("!")
? value.substring(1)
: value;
return {
ref: key,
tags: ["attribute"],
tag,
tag: "attribute",
tagName,
name,
attributeType,
readOnly: value.startsWith("!"),

6
plugs/index/attributes.ts
View File

@ -7,7 +7,7 @@ import { determineTags } from "../../plug-api/lib/cheap_yaml.ts";
export type AttributeObject = ObjectValue<{
name: string;
attributeType: string;
tag: string;
tagName: string;
page: string;
readOnly: boolean;
}>;
@ -49,7 +49,7 @@ export async function objectAttributeCompleter(
const attributeFilter: QueryExpression | undefined =
attributeCompleteEvent.source === ""
? prefixFilter
: ["and", prefixFilter, ["=", ["attr", "tag"], [
: ["and", prefixFilter, ["=", ["attr", "tagName"], [
"string",
attributeCompleteEvent.source,
]]];
@ -63,7 +63,7 @@ export async function objectAttributeCompleter(
return allAttributes.map((value) => {
return {
name: value.name,
source: value.tag,
source: value.tagName,
attributeType: value.attributeType,
readOnly: value.readOnly,
} as AttributeCompletion;

22
plugs/index/builtins.ts
View File

@ -29,6 +29,12 @@ export const builtins: Record<string, Record<string, string>> = {
pos: "!number",
tags: "string[]",
},
item: {
ref: "!string",
name: "!string",
page: "!string",
tags: "string[]",
},
taskstate: {
ref: "!string",
tags: "!string[]",
@ -46,7 +52,7 @@ export const builtins: Record<string, Record<string, string>> = {
ref: "!string",
name: "!string",
attributeType: "!string",
type: "!string",
tagName: "!string",
page: "!string",
readOnly: "!boolean",
},
@ -84,11 +90,11 @@ export const builtins: Record<string, Record<string, string>> = {
export async function loadBuiltinsIntoIndex() {
console.log("Loading builtins attributes into index");
const allTags: ObjectValue<TagObject>[] = [];
for (const [tag, attributes] of Object.entries(builtins)) {
for (const [tagName, attributes] of Object.entries(builtins)) {
allTags.push({
ref: tag,
tags: ["tag"],
name: tag,
ref: tagName,
tag: "tag",
name: tagName,
page: builtinPseudoPage,
parent: "builtin",
});
@ -96,9 +102,9 @@ export async function loadBuiltinsIntoIndex() {
builtinPseudoPage,
Object.entries(attributes).map(([name, attributeType]) => {
return {
ref: `${tag}:${name}`,
tags: ["attribute"],
tag,
ref: `${tagName}:${name}`,
tag: "attribute",
tagName,
name,
attributeType: attributeType.startsWith("!")
? attributeType.substring(1)

13
plugs/index/data.ts
View File

@ -4,6 +4,8 @@ import { collectNodesOfType, findNodeOfType } from "$sb/lib/tree.ts";
import { ObjectValue } from "$sb/types.ts";
import { indexObjects } from "./api.ts";
import { TagObject } from "./tags.ts";
import { extractFrontmatter } from "$sb/lib/frontmatter.ts";
import { updateITags } from "$sb/lib/tags.ts";
type DataObject = ObjectValue<
{
@ -14,6 +16,7 @@ type DataObject = ObjectValue<
export async function indexData({ name, tree }: IndexTreeEvent) {
const dataObjects: ObjectValue<DataObject>[] = [];
const frontmatter = await extractFrontmatter(tree);
await Promise.all(
collectNodesOfType(tree, "FencedCode").map(async (t) => {
@ -41,19 +44,21 @@ export async function indexData({ name, tree }: IndexTreeEvent) {
continue;
}
const pos = t.from! + i;
dataObjects.push({
const dataObj = {
ref: `${name}@${pos}`,
tags: [dataType],
tag: dataType,
...doc,
pos,
page: name,
});
};
updateITags(dataObj, frontmatter);
dataObjects.push(dataObj);
}
// console.log("Parsed data", parsedData);
await indexObjects<TagObject>(name, [
{
ref: dataType,
tags: ["tag"],
tag: "tag",
name: dataType,
page: name,
parent: "data",

12
plugs/index/index.plug.yaml
View File

@ -87,13 +87,13 @@ functions:
indexParagraphs:
path: "./paragraph.ts:indexParagraphs"
events:
- page:index
- page:index
# Backlinks
indexLinks:
path: "./page_links.ts:indexLinks"
events:
- page:index
- page:index
attributeComplete:
path: "./attributes.ts:attributeComplete"
@ -109,13 +109,13 @@ functions:
indexItem:
path: "./item.ts:indexItems"
events:
- page:index
- page:index
# Anchors
indexAnchors:
path: "./anchor.ts:indexAnchors"
events:
- page:index
- page:index
anchorComplete:
path: "./anchor.ts:anchorComplete"
events:
@ -125,13 +125,13 @@ functions:
indexData:
path: data.ts:indexData
events:
- page:index
- page:index
# Hashtags
indexTags:
path: tags.ts:indexTags
events:
- page:index
- page:index
tagComplete:
path: tags.ts:tagComplete
events:

15
plugs/index/item.ts
View File

@ -5,6 +5,8 @@ import { extractAttributes } from "$sb/lib/attribute.ts";
import { rewritePageRefs } from "$sb/lib/resolve.ts";
import { ObjectValue } from "$sb/types.ts";
import { indexObjects } from "./api.ts";
import { updateITags } from "$sb/lib/tags.ts";
import { extractFrontmatter } from "$sb/lib/frontmatter.ts";
export type ItemObject = ObjectValue<
{
@ -17,7 +19,7 @@ export type ItemObject = ObjectValue<
export async function indexItems({ name, tree }: IndexTreeEvent) {
const items: ObjectValue<ItemObject>[] = [];
// console.log("Indexing items", name);
const frontmatter = await extractFrontmatter(tree);
const coll = collectNodesOfType(tree, "ListItem");
@ -30,11 +32,10 @@ export async function indexItems({ name, tree }: IndexTreeEvent) {
continue;
}
const tags = new Set<string>(["item"]);
const tags = new Set<string>();
const item: ItemObject = {
ref: `${name}@${n.from}`,
tags: [],
tag: "item",
name: "", // to be replaced
page: name,
pos: n.from!,
@ -62,7 +63,11 @@ export async function indexItems({ name, tree }: IndexTreeEvent) {
}
item.name = textNodes.map(renderToText).join("").trim();
item.tags = [...tags.values()];
if (tags.size > 0) {
item.tags = [...tags];
}
updateITags(item, frontmatter);
items.push(item);
}

2
plugs/index/lint.ts
View File

@ -16,7 +16,7 @@ export async function lintYAML({ tree }: LintEvent): Promise<LintDiagnostic[]> {
const tags = ["page", ...frontmatter.tags || []];
// Query all readOnly attributes for pages with this tag set
const readOnlyAttributes = await queryObjects<AttributeObject>("attribute", {
filter: ["and", ["=", ["attr", "tag"], [
filter: ["and", ["=", ["attr", "tagName"], [
"array",
tags.map((tag): QueryExpression => ["string", tag]),
]], [

20
plugs/index/page.ts
View File

@ -10,6 +10,7 @@ import {
renderToText,
traverseTreeAsync,
} from "$sb/lib/tree.ts";
import { updateITags } from "$sb/lib/tags.ts";
export async function indexPage({ name, tree }: IndexTreeEvent) {
if (name.startsWith("_")) {
@ -24,7 +25,7 @@ export async function indexPage({ name, tree }: IndexTreeEvent) {
// Note the order here, making sure that the actual page meta data overrules
// any attempt to manually set built-in attributes like 'name' or 'lastModified'
// pageMeta appears at the beginning and the end due to the ordering behavior of ojects in JS (making builtin attributes appear first)
const combinedPageMeta = {
const combinedPageMeta: PageMeta = {
...pageMeta,
...frontmatter,
...toplevelAttributes,
@ -33,18 +34,16 @@ export async function indexPage({ name, tree }: IndexTreeEvent) {
combinedPageMeta.tags = [
...new Set([
"page",
...frontmatter.tags || [],
...toplevelAttributes.tags || [],
]),
];
// if (pageMeta.tags.includes("template")) {
// // If this is a template, we don't want to index it as a page or anything else, just a template
// pageMeta.tags = ["template"];
// }
combinedPageMeta.tag = "page";
// console.log("Page object", pageObj);
updateITags(combinedPageMeta, frontmatter);
// console.log("Page object", combinedPageMeta);
await indexObjects<PageMeta>(name, [combinedPageMeta]);
}
@ -109,13 +108,6 @@ async function lintYaml(
const errorMatch = errorRegex.exec(e.message);
if (errorMatch) {
console.log("YAML error", e.message);
// const line = parseInt(errorMatch[1], 10) - 1;
// const yamlLines = yamlText.split("\n");
// let pos = posOffset;
// for (let i = 0; i < line; i++) {
// pos += yamlLines[i].length + 1;
// }
// const endPos = pos + yamlLines[line].length;
return {
from,

21
plugs/index/page_links.ts
View File

@ -3,12 +3,12 @@ import { IndexTreeEvent } from "$sb/app_event.ts";
import { resolvePath } from "$sb/lib/resolve.ts";
import { indexObjects, queryObjects } from "./api.ts";
import { ObjectValue } from "$sb/types.ts";
import { extractFrontmatter } from "$sb/lib/frontmatter.ts";
import { updateITags } from "$sb/lib/tags.ts";
const pageRefRegex = /\[\[([^\]]+)\]\]/g;
export type LinkObject = {
ref: string;
tags: string[];
export type LinkObject = ObjectValue<{
// The page the link points to
toPage: string;
// The page the link occurs in
@ -17,7 +17,7 @@ export type LinkObject = {
snippet: string;
alias?: string;
asTemplate: boolean;
};
}>;
export function extractSnippet(text: string, pos: number): string {
let prefix = "";
@ -47,7 +47,7 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) {
const links: ObjectValue<LinkObject>[] = [];
// [[Style Links]]
// console.log("Now indexing links for", name);
const frontmatter = await extractFrontmatter(tree);
const pageText = renderToText(tree);
traverseTree(tree, (n): boolean => {
@ -59,7 +59,7 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) {
toPage = toPage.split(/[@$]/)[0];
const link: LinkObject = {
ref: `${name}@${pos}`,
tags: ["link"],
tag: "link",
toPage: toPage,
snippet: extractSnippet(pageText, pos),
pos,
@ -69,6 +69,7 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) {
if (wikiLinkAlias) {
link.alias = wikiLinkAlias.children![0].text!;
}
updateITags(link, frontmatter);
links.push(link);
return true;
}
@ -90,15 +91,17 @@ export async function indexLinks({ name, tree }: IndexTreeEvent) {
for (const match of matches) {
const pageRefName = resolvePath(name, match[1]);
const pos = codeText.from! + match.index! + 2;
links.push({
const link = {
ref: `${name}@${pos}`,
tags: ["link"],
tag: "link",
toPage: pageRefName,
page: name,
snippet: extractSnippet(pageText, pos),
pos: pos,
asTemplate: true,
});
};
updateITags(link, frontmatter);
links.push(link);
}
}
}

47
plugs/index/paragraph.ts
View File

@ -9,6 +9,9 @@ import {
} from "$sb/lib/tree.ts";
import { extractAttributes } from "$sb/lib/attribute.ts";
import { ObjectValue } from "$sb/types.ts";
import a from "https://esm.sh/v135/node_process.js";
import { updateITags } from "$sb/lib/tags.ts";
import { extractFrontmatter } from "$sb/lib/frontmatter.ts";
/** ParagraphObject An index object for the top level text nodes */
export type ParagraphObject = ObjectValue<
@ -21,41 +24,53 @@ export type ParagraphObject = ObjectValue<
export async function indexParagraphs({ name: page, tree }: IndexTreeEvent) {
const objects: ParagraphObject[] = [];
addParentPointers(tree);
let paragraphCounter = 0;
const frontmatter = await extractFrontmatter(tree);
await traverseTreeAsync(tree, async (p) => {
if (p.type !== "Paragraph") {
return false;
}
paragraphCounter++;
if (findParentMatching(p, (n) => n.type === "ListItem")) {
// Not looking at paragraphs nested in a list
return false;
}
// So we're looking at indexable a paragraph now
const tags = new Set<string>(["paragraph"]);
if (paragraphCounter > 1) {
// Only attach hashtags to later paragraphs than the first
const attrs = await extractAttributes(p, true);
const tags = new Set<string>();
const text = renderToText(p);
// tag the paragraph with any hashtags inside it
collectNodesOfType(p, "Hashtag").forEach((tagNode) => {
tags.add(tagNode.children![0].text!.substring(1));
});
// So we're looking at indexable a paragraph now
collectNodesOfType(p, "Hashtag").forEach((tagNode) => {
tags.add(tagNode.children![0].text!.substring(1));
// Hacky way to remove the hashtag
tagNode.children = [];
});
const textWithoutTags = renderToText(p);
if (!textWithoutTags.trim()) {
// Empty paragraph, just tags and attributes maybe
return true;
}
const attrs = await extractAttributes(p, false);
const pos = p.from!;
objects.push({
const paragraph: ParagraphObject = {
ref: `${page}@${pos}`,
text: renderToText(p),
tags: [...tags.values()],
text,
tag: "paragraph",
page,
pos,
...attrs,
});
};
if (tags.size > 0) {
paragraph.tags = [...tags];
paragraph.itags = [...tags];
}
updateITags(paragraph, frontmatter);
objects.push(paragraph);
// stop on every element except document, including paragraphs
return true;

4
plugs/index/tags.ts
View File

@ -17,7 +17,7 @@ export type TagObject = ObjectValue<{
export async function indexTags({ name, tree }: IndexTreeEvent) {
const tags = new Set<string>(); // name:parent
addParentPointers(tree);
const pageTags: string[] = (await extractFrontmatter(tree)).tags;
const pageTags: string[] = (await extractFrontmatter(tree)).tags || [];
for (const pageTag of pageTags) {
tags.add(`${pageTag}:page`);
}
@ -41,7 +41,7 @@ export async function indexTags({ name, tree }: IndexTreeEvent) {
const [tagName, parent] = tag.split(":");
return {
ref: tag,
tags: ["tag"],
tag: "tag",
name: tagName,
page: name,
parent,

17
plugs/tasks/task.ts
View File

@ -18,6 +18,8 @@ import { extractAttributes } from "$sb/lib/attribute.ts";
import { rewritePageRefs } from "$sb/lib/resolve.ts";
import { ObjectValue } from "$sb/types.ts";
import { indexObjects, queryObjects } from "../index/plug_api.ts";
import { updateITags } from "$sb/lib/tags.ts";
import { extractFrontmatter } from "$sb/lib/frontmatter.ts";
export type TaskObject = ObjectValue<
{
@ -46,9 +48,8 @@ const incompleteStates = [" "];
export async function indexTasks({ name, tree }: IndexTreeEvent) {
const tasks: ObjectValue<TaskObject>[] = [];
const taskStates = new Map<string, { count: number; firstPos: number }>();
addParentPointers(tree);
// const allAttributes: AttributeObject[] = [];
// const allTags = new Set<string>();
const frontmatter = await extractFrontmatter(tree);
await traverseTreeAsync(tree, async (n) => {
if (n.type !== "Task") {
return false;
@ -65,7 +66,7 @@ export async function indexTasks({ name, tree }: IndexTreeEvent) {
const complete = completeStates.includes(state);
const task: TaskObject = {
ref: `${name}@${n.from}`,
tags: [],
tag: "task",
name: "",
done: complete,
page: name,
@ -84,10 +85,12 @@ export async function indexTasks({ name, tree }: IndexTreeEvent) {
if (tree.type === "Hashtag") {
// Push the tag to the list, removing the initial #
const tagName = tree.children![0].text!.substring(1);
if (!task.tags) {
task.tags = [];
}
task.tags.push(tagName);
}
});
task.tags = ["task", ...task.tags];
// Extract attributes and remove from tree
const extractedAttributes = await extractAttributes(n, true);
@ -97,6 +100,8 @@ export async function indexTasks({ name, tree }: IndexTreeEvent) {
task.name = n.children!.slice(1).map(renderToText).join("").trim();
updateITags(task, frontmatter);
tasks.push(task);
return true;
});
@ -107,7 +112,7 @@ export async function indexTasks({ name, tree }: IndexTreeEvent) {
name,
Array.from(taskStates.entries()).map(([state, { firstPos, count }]) => ({
ref: `${name}@${firstPos}`,
tags: ["taskstate"],
tag: "taskstate",
state,
count,
page: name,

6
plugs/tasks/tasks.plug.yaml
View File

@ -35,15 +35,11 @@ functions:
indexTasks:
path: "./task.ts:indexTasks"
events:
- page:index
- page:index
taskToggle:
path: "./task.ts:taskToggle"
events:
- page:click
# itemQueryProvider:
# path: ./task.ts:queryProvider
# events:
# - query:task
taskToggleCommand:
path: ./task.ts:taskCycleCommand
command:

6
plugs/template/template.ts
View File

@ -37,7 +37,7 @@ export async function newPageCommand(
const templateText = await space.readPage(templateName!);
const tempPageMeta: PageMeta = {
tags: ["page"],
tag: "page",
ref: "",
name: "",
created: "",
@ -169,7 +169,7 @@ export async function dailyNoteCommand() {
await space.writePage(
pageName,
await replaceTemplateVars(dailyNoteTemplateText, {
tags: ["page"],
tag: "page",
ref: pageName,
name: pageName,
created: "",
@ -218,7 +218,7 @@ export async function weeklyNoteCommand() {
await replaceTemplateVars(weeklyNoteTemplateText, {
name: pageName,
ref: pageName,
tags: ["page"],
tag: "page",
created: "",
lastModified: "",
perm: "rw",

2
plugs/template/util.ts
View File

@ -19,7 +19,7 @@ export function defaultJsonTransformer(v: any): string {
}
if (Array.isArray(v)) {
return v.map(defaultJsonTransformer).join(", ");
} else if (typeof v === "object") {
} else if (v && typeof v === "object") {
return Object.entries(v).map(([k, v]: [string, any]) =>
`${k}: ${defaultJsonTransformer(v)}`
).join(", ");

18
server/server_system.ts
View File

@ -33,6 +33,7 @@ import { CodeWidgetHook } from "../web/hooks/code_widget.ts";
import { KVPrimitivesManifestCache } from "../plugos/manifest_cache.ts";
import { KvPrimitives } from "../plugos/lib/kv_primitives.ts";
import { ShellBackend } from "./shell_backend.ts";
import { ensureSpaceIndex } from "../common/space_index.ts";
const fileListInterval = 30 * 1000; // 30s
@ -161,19 +162,10 @@ export class ServerSystem {
})().catch(console.error);
});
// Check if this space was ever indexed before
if (!await this.ds.get(["$initialIndexDone"])) {
console.log("Indexing space for the first time (in the background)");
const indexPromise = this.system.loadedPlugs.get("index")!.invoke(
"reindexSpace",
[],
).then(() => {
console.log("Initial index completed!");
this.ds.set(["$initialIndexDone"], true);
}).catch(console.error);
if (awaitIndex) {
await indexPromise;
}
// Ensure a valid index
const indexPromise = ensureSpaceIndex(this.ds, this.system);
if (awaitIndex) {
await indexPromise;
}
await eventHook.dispatchEvent("system:ready");

26
web/client.ts
View File

@ -47,6 +47,10 @@ import {
EncryptedSpacePrimitives,
} from "../common/spaces/encrypted_space_primitives.ts";
import { LimitedMap } from "../common/limited_map.ts";
import {
ensureSpaceIndex,
markFullSpaceIndexComplete,
} from "../common/space_index.ts";
const frontMatterRegex = /^---\n(([^\n]|\n)*?)---\n/;
const autoSaveInterval = 1000;
@ -216,7 +220,7 @@ export class Client {
await this.loadPlugs();
this.initNavigator();
this.initSync();
await this.initSync();
this.loadCustomStyles().catch(console.error);
@ -235,9 +239,12 @@ export class Client {
this.updatePageListCache().catch(console.error);
}
private initSync() {
private async initSync() {
this.syncService.start();
// We're still booting, if a initial sync has already been completed we know this is the initial sync
const initialSync = !await this.syncService.hasInitialSyncCompleted();
this.eventHook.addLocalListener("sync:success", async (operations) => {
// console.log("Operations", operations);
if (operations > 0) {
@ -247,13 +254,24 @@ export class Client {
if (operations !== undefined) {
// "sync:success" is called with a number of operations only from syncSpace(), not from syncing individual pages
this.fullSyncCompleted = true;
console.log("Full sync completed");
// A full sync just completed
if (!initialSync) {
// If this was NOT the initial sync let's check if we need to perform a space reindex
ensureSpaceIndex(this.stateDataStore, this.system.system).catch(
console.error,
);
} else {
// This was the initial sync, let's mark a full index as completed
await markFullSpaceIndexComplete(this.stateDataStore);
}
}
// if (this.system.plugsUpdated) {
if (operations) {
// Likely initial sync so let's show visually that we're synced now
this.showProgress(100);
}
// }
this.ui.viewDispatch({ type: "sync-change", syncSuccess: true });
});

6
web/components/page_navigator.tsx
View File

@ -53,11 +53,9 @@ export function PageNavigator({
if (aliases.length > 0) {
description = "(a.k.a. " + aliases.join(", ") + ") ";
}
if (pageMeta.tags.length > 1) {
// Every page has the "page" tag, so it only gets interesting beyond that
const interestingTags = pageMeta.tags.filter((tag) => tag !== "page");
if (pageMeta.tags) {
description = (description || "") +
interestingTags.map((tag) => `#${tag}`).join(" ");
pageMeta.tags.map((tag) => `#${tag}`).join(" ");
}
options.push({
...pageMeta,

2
web/space.ts
View File

@ -187,7 +187,7 @@ export function fileMetaToPageMeta(fileMeta: FileMeta): PageMeta {
return {
...fileMeta,
ref: name,
tags: ["page"],
tag: "page",
name,
created: new Date(fileMeta.created).toISOString(),
lastModified: new Date(fileMeta.lastModified).toISOString(),

2
web/styles/editor.scss
View File

@ -414,8 +414,6 @@
.sb-markdown-top-widget h1,
.sb-markdown-bottom-widget h1 {
border-top-right-radius: 5px;
border-top-left-radius: 5px;
margin: 0 0 5px 0;
padding: 10px !important;
background-color: var(--editor-widget-background-color);

5
website/CHANGELOG.md
View File

@ -6,6 +6,11 @@ release.
_Not yet released, this will likely become 0.6.0._
* **Directives have now been removed** from the code base. Please use [[Live Queries]] and [[Live Templates]] instead. If you hadn’t migrated yet and want to auto migrate, downgrade your SilverBullet version to 0.5.11 (e.g. using the `zefhemel/silverbullet:0.5.11` docker image) and run the {[Directive: Convert Entire Space to Live/Templates]} command with that version.
* (Hopefully subtle) **breaking change** in how tags work (see [[Objects]]):
* Every object now has a `tag` attribute, signifying the “main” tag for that object (e.g. `page`, `item`)
* The `tags` attribute will now _only_ contain explicitly assigned tags (so not the built-in tag, which moved to `tag`)
* The new `itags` attribute (available in many objects) includes both the `tag`, `tags` as well as any tags inherited from the page the object appears in.
* Page tags now no longer need to appear at the top of the page, but can appear anywhere as long as they are the only thing appearing in a paragraph with no additional text, see [[Objects$page]].
* New [[Markdown/Code Widgets|Code Widget]]: `toc` to manually include a [[Table of Contents]]
* New template type: [[Live Template Widgets]] allowing you to automatically add templates to the top or bottom of your pages (based on some criteria). Using this feature it possible to implement [[Table of Contents]] and [[Linked Mentions]] without having “hard coded” into SilverBullet itself.
* **“Breaking” change:** Two features are now no longer hardcoded into SilverBullet, but can be activated quite easily using [[Live Template Widgets]] (see their respective documentation pages on instructions on how to do this):

43
website/Objects.md
View File

@ -1,33 +1,31 @@
#core
SilverBullet automatically builds and maintains an index of _objects_ extracted from all markdown pages in your space. It subsequently allows you to [[Live Queries|query]] this database in (potentially) useful ways.
Some examples of things you can query for:
* Give me a list of all books that I have marked as _want to read_
* Give me a list of all tasks not yet completed that have today as a due date
* Give me a list of items tagged with `#quote`
* Give me a list of not-completed tasks that reference the current page
By design, the truth remains in the markdown: all data indexed as objects will have a representation in markdown text as well. The index can be flushed at any time and be rebuilt from its source markdown files kept in your space.
By design, the truth remains in the markdown: all data indexed as objects will have a representation in markdown text as well. This index can be flushed at any time and be rebuilt from its source markdown files kept in your space (and you can do so on demand if you like using the {[Space: Reindex]} command).
# Object representation
Every object has a set of [[Attributes]].
Every object has a set of [[Attributes]], some predefined, but you can add any additional custom attributes that you like.
At the very least:
* `ref`: a unique _identifier_ (unique to the page, at least), often represented as a pointer to the place (page, position) in your space where the object is defined. For instance, a _page_ object will use the page name as its `ref` attribute, and a `task` will use `page@pos` (where `pos` is the location the task appears in `page`).
* `tags`: an array of type(s) of an object, see [[$tags]].
The following attributes are predefined, and you can expect all objects to have them:
* `ref`: a globally unique _identifier_, often represented as a pointer to the place (page, position) in your space where the object is defined. For instance, a _page_ object will use the page name as its `ref` attribute, and a `task` will use `page@pos` (where `pos` is the location the task appears in `page`).
* `tag`: the main type, or “tag” of the page, usually a built-in type of the object (see below).
In addition, any number of additional tag-specific and custom [[Attributes]] can be defined (see below).
In addition, many objects will also contain:
* `tags`: an optional set of additional, explicitly assigned tags.
* `itags`: a set of _implicit_ or _inherited_ tags: including the object’s `tag`, `tags` as well as any tags _assigned to its containing page_. This is useful to answer queries like, “give me all tasks on pages where that page is tagged with `person`“, which would be expressed as `task where itags = "person"` (although technically that would also match any tags that have the `#person` explicitly assigned).
Beside these, any number of additional tag-specific and custom [[Attributes]] can be defined (see below).
# Tags
$tags
Every object has one or more tags, defining the _types_ of an object. Some tags are built-in (as described below), but you can easily define new tags by simply using the #hashtag notation in strategic locations (more on these locations later).
Every object has a main `tag`, which signifies the type of object being described. In addition, any number of additional tags can be assigned as well via the `tags` attribute. You can use either the main `tag` or any of the `tags` as query sources in [[Live Queries]] — examples below.
Here are the currently built-in tags:
## page
$page
Every page in your space is available via the `page` tag. You can attach _additional tags_ to a page, by either specifying them in the `tags` attribute [[Frontmatter]], or by putting additional [[Tags]] in the _first paragraph of your page_, as is done with the #core tag at the beginning of this page.
Every page in your space is available via the `page` tag. You can attach _additional_ tags to a page, by either specifying them in the `tags` attribute [[Frontmatter]], or by putting additional [[Tags]] in a stand alone paragraph with no other (textual) content in them, e.g.:
#example-tag #another-tag
In addition to `ref` and `tags`, the `page` tag defines a bunch of additional attributes as can be seen in this example query:
@ -35,6 +33,12 @@ In addition to `ref` and `tags`, the `page` tag defines a bunch of additional at
page where name = "{{@page.name}}"
```
Note that you can also query this page using the `example-tag` directly:
```query
example-tag
```
## task
$task
Every task in your space is tagged with the `task` tag by default. You tag it with additional tags by using [[Tags]] in the task name, e.g.
@ -48,6 +52,7 @@ The following query shows all attributes available for tasks:
```query
upnext
```
Although you may want to render it using a template such as [[template/task]] instead:
```query
@ -71,13 +76,13 @@ $template
Indexes all pages tagged with `#template`. See [[Templates]] for more information on templates.
```query
template select name
template select name limit 5
```
## item
$item
List items (both bullet point and numbered items) are indexed by default with the `item` tag, and additional tags can be added using [[Tags]].
List items (both bullet point and numbered items) are indexed with the `item` tag, and additional tags can be added using [[Tags]].
Here is an example of a #quote item using a custom [[Attributes|attribute]]:
@ -86,7 +91,7 @@ Here is an example of a #quote item using a custom [[Attributes|attribute]]:
And then queried via the #quote tag:
```query
quote where tags = "item" select name, by
quote where page = "{{@page.name}}" and tag = "item" select name, by
```
## paragraph