Sneed-Group/agsamantha
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

update/create maxDepthCount so that it hopefully wont run out of heapmem

Browse source
This commit is contained in:
The Ghost of FOSS' Future 2024-09-16 08:27:46 -05:00 committed by GitHub
parent fe0fa62361
commit 864e2657dc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 2 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
index.mjs
View file

@ -11,6 +11,7 @@ const execPromise = promisify(exec);
const modelfilePath = "I:rolling-agi-mf" const modelfilePath = "I:rolling-agi-mf"
const maxDepthCount = 8
// Function to run the publishing commands // Function to run the publishing commands
async function runPublishCommands() { async function runPublishCommands() {
@ -124,7 +125,7 @@ async function siteCrawler(hostname) {
const loader = new RecursiveUrlLoader(crawled, { const loader = new RecursiveUrlLoader(crawled, {
extractor: compiledConvert, extractor: compiledConvert,
maxDepth: 14, maxDepth: maxDepthCount,
excludeDirs: ["https://search.sparksammy.com/", "https://search.sparksammy.com/search.php", "https://archive.org", "https://doubleclick.net", "https://paypal.com"], excludeDirs: ["https://search.sparksammy.com/", "https://search.sparksammy.com/search.php", "https://archive.org", "https://doubleclick.net", "https://paypal.com"],
}); });
const webContents = await loader.load(); const webContents = await loader.load();