atreyabain/safeImport
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[doc]

Browse Source
This commit is contained in:
Atreya Bain
2025-08-03 15:55:42 +01:00
parent ef886bf610
commit 47937e08fb
3 changed files with 18 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
src_dataset/index.mjs
View File

@@ -22,15 +22,15 @@ const intermediateRepoList = await cacheFunctionOutput('repos.json', async funct
*/
const withRepos = packageList.map(e => [e[0], packageRepos[e[0]], e[1]])
console.log('withrepos', withRepos.length);
const withExactRepos = withRepos.filter(e => ((e[1]) !== null && (e[1]) !== undefined && (e[1]) !== ""))
const withExactRepos = withRepos.filter(e => ((e[1]) !== null && (e[1]) !== undefined && (e[1]) !== "")) // filter out repos that are not available
console.log('withreposCleaned', withExactRepos.length);
withExactRepos.sort((a,b)=>(-a[2]+b[2]))
withExactRepos.sort((a,b)=>(-a[2]+b[2])) // sort by download count
return withExactRepos;
})
// const packageMap = new Map(packageList)
console.log(intermediateRepoList.length)
const intermediateRepoListSmaller = intermediateRepoList.slice(0,2000);
const intermediateRepoListSmaller = intermediateRepoList.slice(0,250);
const repoStatus = await processPromisesBatch(intermediateRepoListSmaller,15,cloneRepoAndCheck)

15
src_dataset/mine.mjs
View File

@@ -7,7 +7,7 @@ import { FILTER_LIST } from './FILTER_LIST.mjs';
/**
*
* @param {[string,string,number]} param0
* @returns {Promise<[string,string|null]>}
* @returns {Promise<[string,string|null]>} second argument is null if ineligible for slicing
*/
export async function cloneRepoAndCheck([repoName, repoGitUrl, downloadCount]) {
const repoPath = resolve('cache/repos', repoName)
@@ -31,14 +31,25 @@ export async function cloneRepoAndCheck([repoName, repoGitUrl, downloadCount]) {
// console.log(packageJSONContentsString);
const packageJSONContents = JSON.parse(packageJSONContentsString)
// console.log(repoName, packageJSONContents.license)
if(!hasAnyActualDependencies(packageJSONContents, repoName)) {
console.log("[git] skipping", repoName, "has no dependencies");
return [repoName, null];
}
const hasDependencies = checkTestingDependencies(packageJSONContents, repoName);
if (hasDependencies)
return [repoName, ((packageJSONContents?.scripts?.test))]
else return [repoName, null]
}
function hasAnyActualDependencies(packageJSONContents, repoName) {
if (packageJSONContents.dependencies !== undefined && Object.keys(packageJSONContents.dependencies).length > 0) {
return true;
}
return false;
}
function checkTestingDependencies(packageJSONContents, repoName) {
const testingLibraries = new Set(['jest', 'mocha', 'chai', 'istanbul', 'vitest']);
const testingLibraries = new Set(['mocha', 'istanbul']);
const dependencies = new Set();
if (packageJSONContents.dependencies !== undefined) {
for (const dep of Object.keys(packageJSONContents.dependencies)) {