Sneed-Group/agsamantha
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Update index.mjs

Browse source
This commit is contained in:
The Ghost of FOSS' Future 2024-09-15 19:29:45 -05:00 committed by GitHub
parent 93141df1d3
commit fe59ffa0a3
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 10 additions and 9 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
index.mjs
View file

@ -117,17 +117,18 @@ function generateSearchTerm(hostname) {
const contexts = [];
async function siteCrawler(hostname) {
const crawled = generateSearchTerm(hostname);
const compiledConvert = compile({ wordwrap: 130 });
const loader = new RecursiveUrlLoader(crawled, {
extractor: compiledConvert,
maxDepth: 6,
excludeDirs: ["https://search.sparksammy.com/", "https://search.sparksammy.com/search.php", "https://archive.org", "https://doubleclick.net", "https://paypal.com"],
});
try {
const crawled = generateSearchTerm(hostname);
const compiledConvert = compile({ wordwrap: 130 });
const loader = new RecursiveUrlLoader(crawled, {
extractor: compiledConvert,
maxDepth: 6,
excludeDirs: ["https://search.sparksammy.com/", "https://search.sparksammy.com/search.php", "https://archive.org", "https://doubleclick.net", "https://paypal.com"],
});
const webContents = await loader.load();
webContents.forEach(content => contexts.push(content));
return webContents;