atreyabain/safeImport
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[fix] ignore clone failures

Browse Source
This commit is contained in:
Atreya Bain
2025-08-25 23:45:34 +01:00
parent 09adbb78f1
commit 01b6902677
3 changed files with 82 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src_dataset/index.mjs
View File

@@ -30,15 +30,15 @@ const intermediateRepoList = await cacheFunctionOutput('repos.n2.json', async fu
// const packageMap = new Map(packageList)
console.log(`Total repos`,intermediateRepoList.length)
const intermediateRepoListSmaller = intermediateRepoList.slice(0,20000);
const intermediateRepoListSmaller = intermediateRepoList.slice(0,60000);
const repoStatus = await processPromisesBatch(intermediateRepoListSmaller,40,cloneRepoAndCheck)
const repoStatusString = csv.stringify(repoStatus);
await fsp.writeFile('repostatus.csv', repoStatusString);
await fsp.writeFile('repostatus2.csv', repoStatusString);
const minableRepositories = repoStatus.filter(e=>(e!==null && e?.[1]));
const output = csv.stringify(minableRepositories);
await fsp.writeFile('minableRepositories2.csv', output);
await fsp.writeFile('minableRepositories3.csv', output);
// console.log("written results")