Sneed-Group/agsamantha
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

good job chcuklenuts

Browse source
This commit is contained in:
The Ghost of FOSS' Past 2024-10-03 10:33:07 -05:00
parent 9d0ad93176
commit 4e76fa1f8b
1 changed files with 1 additions and 2 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
index.mjs
View file

@ -150,11 +150,10 @@ async function siteCrawler(hostname) {
const crawled = generateSearchTerm(hostname); const crawled = generateSearchTerm(hostname);
const loader = new RecursiveUrlLoader(crawled, { const loader = new RecursiveUrlLoader(crawled, {
extractor: new Readability(craweled).parse(),
maxDepth: maxDepthCount, maxDepth: maxDepthCount,
excludeDirs: ["https://doubleclick.net", "https://paypal.com", "https://archive.org", "*.css"], excludeDirs: ["https://doubleclick.net", "https://paypal.com", "https://archive.org", "*.css"],
}); });
const webContents = await loader.load(); const webContents = await Readability(loader.load()).parse();
webContents.forEach(content => contexts.push(content)); webContents.forEach(content => contexts.push(content));
return webContents; return webContents;
} catch (error) { } catch (error) {