[add] implement vulnerability checking and advisory fetching, enhance repo processing, and add utility functions

src_dataset/FILTER_LIST.mjs

@@ -41,7 +41,8 @@ const FILTER_LIST = [
     "https://github.com/foliojs-fork/linebreaker",
     "https://github.com/segmentio/analytics.js-video-plugins",
     "https://github.com/cucumber/cucumber-expressions-javascript",
-    "https://github.com/jakwings/node-temp-fs"
+    "https://github.com/jakwings/node-temp-fs",
+    "https://github.com/bower/bower/tree/master/packages/*"
 ];
 
 const FILTER_LIST_REGEX = FILTER_LIST.map(GlobToRegExp)

src_dataset/cache.mjs

@@ -9,9 +9,9 @@ import { resolve } from "node:path";
  * @param {()=>Promise<T>} asyncCallback 
  * @returns {Promise<T>}
  */
-export async function cacheFunctionOutput(fileName, asyncCallback, silent=false) {
+export async function cacheFunctionOutput(fileName, asyncCallback, silent=false,passthrough=false) {
     const fileLoc = resolve('../cache-repos', fileName);
-    if (existsSync(fileLoc)) {
+    if (!passthrough && existsSync(fileLoc)) {
         !silent && console.log("[cacher] Using cached ", fileLoc);
         const fileContents = (await readFile(fileLoc)).toString();
         return JSON.parse(fileContents);

src_dataset/index.mjs

@@ -30,9 +30,9 @@ const intermediateRepoList = await cacheFunctionOutput('repos.n2.json', async fu
 // const packageMap = new Map(packageList)
 
 console.log(`Total repos`,intermediateRepoList.length)
-const intermediateRepoListSmaller = intermediateRepoList.slice(0,10000);
+const intermediateRepoListSmaller = intermediateRepoList.slice(0,20000);
 
-const repoStatus = await processPromisesBatch(intermediateRepoListSmaller,20,cloneRepoAndCheck)
+const repoStatus = await processPromisesBatch(intermediateRepoListSmaller,40,cloneRepoAndCheck)
 
 const repoStatusString = csv.stringify(repoStatus);
 await fsp.writeFile('repostatus.csv', repoStatusString);

src_dataset/mine.mjs

@@ -43,6 +43,15 @@ export async function cloneRepoAndCheck([repoName, repoGitUrl, downloadCount]) {
     // console.log(repoName, packageJSONContents.license)
     if (!hasAnyActualDependencies(packageJSONContents, repoName)) {
         // console.log("[git] skipping", repoName, "has no dependencies");
+        await removeUnnecessaryClone(repoPath);
+        // console.log("Cleaned up ", repoPath);
+        return [repoName, null];
+    }
+
+    if(isLikelyTypescriptProject(packageJSONContents)) {
+        await removeUnnecessaryClone(repoPath);
+        // console.warn("[git] Ignoring ", repoName, "because it is a typescript project.");
+        // console.log("Cleaned up ", repoPath);
         return [repoName, null];
     }
 
@@ -58,15 +67,37 @@ export async function cloneRepoAndCheck([repoName, repoGitUrl, downloadCount]) {
         }
         const packageFile = resolve(repoPath, 'package.json')
         if (!existsSync(packageFile)){
-            console.warn("[git] Unexpected package.json not found in", repoName, "at", packageFile);
+            // console.warn("[git] Unexpected package.json not found in", repoName, "at", packageFile);
             return [repoName, null];}
 
         // finally, return the test script if it exists
         return [repoName, ((packageJSONContents?.scripts?.test))]
     }
-    else return [repoName, null]
+    else{
+        await removeUnnecessaryClone(repoPath);
+
+        return [repoName, null]
+    }
 }
 
+function isLikelyTypescriptProject(packageJSONContents) {
+    if (packageJSONContents.devDependencies !== undefined) {
+        if (Object.keys(packageJSONContents.devDependencies).some(e => e.startsWith('typescript'))) {
+            return true;
+        }
+        if (Object.keys(packageJSONContents.dependencies).some(e => e.startsWith('typescript'))) {
+            return true;
+        }
+    }
+    return false;
+}
+async function removeUnnecessaryClone(repoPath) {
+    if(existsSync(repoPath)){
+         console.log("[git] unnecessary clone, removing", repoPath) ;
+        //  while(true){}
+        await rm(repoPath, { recursive: true, force: true });
+    }
+}
 
 function filterRepo(repoGitUrl) {
     return matchFilterList(repoGitUrl);