nandi.uk
This repository has no description
refactor: improve similarity search with shared word frequency algorithm
Co-authored-by: aider (anthropic/claude-sonnet-4-20250514) <aider@aider.chat>
author
Wesley Finck
co-author
aider (anthropic/claude-sonnet-4-20250514)
date
(Oct 23, 2025, 3:17 PM -0700)
commit
2c8f20d8
2c8f20d8b2bfab62b5190396e07ebdfc9241dda5
parent
d8a2165f
d8a2165f5258df341a259a64620f1d726406762e
+42
-10
+42
-10
src/modules/search/infrastructure/InMemoryVectorDatabase.ts
+42
-10
src/modules/search/infrastructure/InMemoryVectorDatabase.ts
···
60
60
): Promise<Result<UrlSearchResult[]>> {
61
61
try {
62
62
console.log('all urls to compare', this.urls);
63
-
const threshold = params.threshold || 0.3;
63
+
const threshold = params.threshold || 0.1; // Lower default threshold for more matches
64
64
const results: UrlSearchResult[] = [];
65
65
66
66
// Get the query URL's content for comparison
67
67
const queryUrl = this.urls.get(params.url);
68
68
const queryContent = queryUrl?.content || params.url;
69
+
70
+
console.log('Query content for similarity:', queryContent);
69
71
70
72
for (const [url, indexed] of this.urls.entries()) {
71
73
// Skip the query URL itself
···
76
78
indexed.content,
77
79
);
78
80
81
+
console.log(`Similarity between "${queryContent}" and "${indexed.content}": ${similarity}`);
82
+
79
83
if (similarity >= threshold) {
80
84
results.push({
81
85
url: indexed.url,
···
89
93
results.sort((a, b) => b.similarity - a.similarity);
90
94
const limitedResults = results.slice(0, params.limit);
91
95
96
+
console.log(`Found ${limitedResults.length} similar URLs above threshold ${threshold}`);
97
+
92
98
return ok(limitedResults);
93
99
} catch (error) {
94
100
return err(
···
117
123
}
118
124
119
125
/**
120
-
* Simple text similarity calculation using Jaccard similarity
121
-
* In a real implementation, this would use proper vector embeddings
126
+
* Simple text similarity calculation based on shared words
127
+
* Uses a more lenient scoring system to increase likelihood of matches
122
128
*/
123
129
private calculateSimilarity(text1: string, text2: string): number {
124
130
const words1 = this.tokenize(text1);
125
131
const words2 = this.tokenize(text2);
126
132
127
-
const set1 = new Set(words1);
128
-
const set2 = new Set(words2);
133
+
if (words1.length === 0 && words2.length === 0) return 1;
134
+
if (words1.length === 0 || words2.length === 0) return 0;
129
135
130
-
const intersection = new Set([...set1].filter((word) => set2.has(word)));
131
-
const union = new Set([...set1, ...set2]);
136
+
// Count shared words (with frequency)
137
+
const freq1 = this.getWordFrequency(words1);
138
+
const freq2 = this.getWordFrequency(words2);
132
139
133
-
if (union.size === 0) return 0;
140
+
let sharedWords = 0;
141
+
let totalWords = 0;
134
142
135
-
return intersection.size / union.size;
143
+
// Count shared words based on minimum frequency
144
+
for (const word of new Set([...words1, ...words2])) {
145
+
const count1 = freq1.get(word) || 0;
146
+
const count2 = freq2.get(word) || 0;
147
+
148
+
if (count1 > 0 && count2 > 0) {
149
+
sharedWords += Math.min(count1, count2);
150
+
}
151
+
totalWords += Math.max(count1, count2);
152
+
}
153
+
154
+
// Return ratio of shared words to total words
155
+
// This is more lenient than Jaccard similarity
156
+
return totalWords > 0 ? sharedWords / totalWords : 0;
157
+
}
158
+
159
+
/**
160
+
* Get word frequency map
161
+
*/
162
+
private getWordFrequency(words: string[]): Map<string, number> {
163
+
const freq = new Map<string, number>();
164
+
for (const word of words) {
165
+
freq.set(word, (freq.get(word) || 0) + 1);
166
+
}
167
+
return freq;
136
168
}
137
169
138
170
private tokenize(text: string): string[] {
···
140
172
.toLowerCase()
141
173
.replace(/[^\w\s]/g, ' ')
142
174
.split(/\s+/)
143
-
.filter((word) => word.length > 2); // Filter out very short words
175
+
.filter((word) => word.length > 1); // Allow shorter words for more matches
144
176
}
145
177
146
178
/**