Full text refactor step 1

pull/513/head
Zef Hemel 2023-08-30 21:12:33 +02:00
parent 75da8e7ca9
commit 79242a524c
3 changed files with 27 additions and 17 deletions

View File

@ -1,22 +1,32 @@
import { assertEquals } from "../../test_deps.ts"; import { assertEquals } from "../../test_deps.ts";
import { BatchKVStore, SimpleSearchEngine } from "./engine.ts"; import { BatchKVStore, SimpleSearchEngine } from "./engine.ts";
class InMemoryBatchKVStore<K, V> implements BatchKVStore<K, V> { class InMemoryBatchKVStore implements BatchKVStore {
private store = new Map<K, V>(); private store = new Map<string, any>();
get(keys: K[]): Promise<(V | undefined)[]> { get(keys: string[]): Promise<(any | undefined)[]> {
const results: (V | undefined)[] = keys.map((key) => this.store.get(key)); const results: (any | undefined)[] = keys.map((key) => this.store.get(key));
return Promise.resolve(results); return Promise.resolve(results);
} }
set(entries: Map<K, V>): Promise<void> { queryPrefix(prefix: string): Promise<[string, any][]> {
const results: [string, any][] = [];
for (const [key, value] of this.store.entries()) {
if (key.startsWith(prefix)) {
results.push([key, value]);
}
}
return Promise.resolve(results);
}
set(entries: Map<string, any>): Promise<void> {
for (const [key, value] of entries) { for (const [key, value] of entries) {
this.store.set(key, value); this.store.set(key, value);
} }
return Promise.resolve(); return Promise.resolve();
} }
delete(keys: K[]): Promise<void> { delete(keys: string[]): Promise<void> {
for (const key of keys) { for (const key of keys) {
this.store.delete(key); this.store.delete(key);
} }

View File

@ -5,10 +5,10 @@ export type Document = {
text: string; text: string;
}; };
export interface BatchKVStore<K, V> { export interface BatchKVStore {
get(keys: K[]): Promise<(V | undefined)[]>; get(keys: string[]): Promise<(any | undefined)[]>;
set(entries: Map<K, V>): Promise<void>; set(entries: Map<string, any>): Promise<void>;
delete(keys: K[]): Promise<void>; delete(keys: string[]): Promise<void>;
} }
type ResultObject = { type ResultObject = {
@ -20,8 +20,8 @@ export class SimpleSearchEngine {
private stopWords = ["and", "or", "the", "a", "an"]; private stopWords = ["and", "or", "the", "a", "an"];
constructor( constructor(
public index: BatchKVStore<string, string[]>, public index: BatchKVStore,
public reverseIndex: BatchKVStore<string, string[]>, public reverseIndex: BatchKVStore,
) { ) {
} }
@ -86,7 +86,7 @@ export class SimpleSearchEngine {
const filteredWords = this.removeStopWords(words); const filteredWords = this.removeStopWords(words);
const stemmedWords = filteredWords.map((word) => this.stem(word)); const stemmedWords = filteredWords.map((word) => this.stem(word));
const wordIdsArray = await this.index.get(stemmedWords); const wordIdsArray: string[][] = await this.index.get(stemmedWords);
const matchCounts: Map<string, number> = new Map(); const matchCounts: Map<string, number> = new Map();
wordIdsArray.forEach((wordIds) => { wordIdsArray.forEach((wordIds) => {
@ -110,13 +110,13 @@ export class SimpleSearchEngine {
// Delete a document from the index // Delete a document from the index
public async deleteDocument(documentId: string): Promise<void> { public async deleteDocument(documentId: string): Promise<void> {
const words = await this.reverseIndex.get([documentId]); const words: string[][] = await this.reverseIndex.get([documentId]);
if (words && words[0]) { if (words && words[0]) {
const currentIdsArray = await this.index.get(words[0]); const currentIdsArray: string[][] = await this.index.get(words[0]);
const deleteKeys: string[] = []; const deleteKeys: string[] = [];
const updateMap = new Map<string, string[]>(); const updateMap = new Map<string, string[]>();
words[0].forEach((word, i) => { words[0].forEach((word: string, i: number) => {
const currentIds = currentIdsArray[i]; const currentIds = currentIdsArray[i];
if (currentIds) { if (currentIds) {
const updatedIds = currentIds.filter((id) => id !== documentId); const updatedIds = currentIds.filter((id) => id !== documentId);

View File

@ -8,7 +8,7 @@ import { PromiseQueue } from "$sb/lib/async.ts";
const searchPrefix = "🔍 "; const searchPrefix = "🔍 ";
class StoreKVStore implements BatchKVStore<string, string[]> { class StoreKVStore implements BatchKVStore {
constructor(private prefix: string) { constructor(private prefix: string) {
} }
get(keys: string[]): Promise<(string[] | undefined)[]> { get(keys: string[]): Promise<(string[] | undefined)[]> {