[doc]

src_dataset/index.mjs

@@ -22,15 +22,15 @@ const intermediateRepoList = await cacheFunctionOutput('repos.json', async funct
      */
     const withRepos = packageList.map(e => [e[0], packageRepos[e[0]], e[1]])
     console.log('withrepos', withRepos.length);
-    const withExactRepos = withRepos.filter(e => ((e[1]) !== null && (e[1]) !== undefined && (e[1]) !== ""))
+    const withExactRepos = withRepos.filter(e => ((e[1]) !== null && (e[1]) !== undefined && (e[1]) !== "")) // filter out repos that are not available
     console.log('withreposCleaned', withExactRepos.length);
-    withExactRepos.sort((a,b)=>(-a[2]+b[2]))
+    withExactRepos.sort((a,b)=>(-a[2]+b[2])) // sort by download count
     return withExactRepos;
 })
 // const packageMap = new Map(packageList)
 
 console.log(intermediateRepoList.length)
-const intermediateRepoListSmaller = intermediateRepoList.slice(0,2000);
+const intermediateRepoListSmaller = intermediateRepoList.slice(0,250);
 
 const repoStatus = await processPromisesBatch(intermediateRepoListSmaller,15,cloneRepoAndCheck)
 

src_dataset/mine.mjs

@@ -7,7 +7,7 @@ import { FILTER_LIST } from './FILTER_LIST.mjs';
 /**
  * 
  * @param {[string,string,number]} param0 
- * @returns {Promise<[string,string|null]>}
+ * @returns {Promise<[string,string|null]>} second argument is null if ineligible for slicing
  */
 export async function cloneRepoAndCheck([repoName, repoGitUrl, downloadCount]) {
     const repoPath = resolve('cache/repos', repoName)
@@ -31,14 +31,25 @@ export async function cloneRepoAndCheck([repoName, repoGitUrl, downloadCount]) {
     // console.log(packageJSONContentsString);
     const packageJSONContents = JSON.parse(packageJSONContentsString)
     // console.log(repoName, packageJSONContents.license)
+    if(!hasAnyActualDependencies(packageJSONContents, repoName)) {
+        console.log("[git] skipping", repoName, "has no dependencies");
+        return [repoName, null];
+    }
+
     const hasDependencies = checkTestingDependencies(packageJSONContents, repoName);
     if (hasDependencies)
         return [repoName, ((packageJSONContents?.scripts?.test))]
     else return [repoName, null]
 }
+function hasAnyActualDependencies(packageJSONContents, repoName) {
+    if (packageJSONContents.dependencies !== undefined && Object.keys(packageJSONContents.dependencies).length > 0) {
+        return true;
+    }
+    return false;
+}
 
 function checkTestingDependencies(packageJSONContents, repoName) {
-    const testingLibraries = new Set(['jest', 'mocha', 'chai', 'istanbul', 'vitest']);
+    const testingLibraries = new Set(['mocha', 'istanbul']);
     const dependencies = new Set();
     if (packageJSONContents.dependencies !== undefined) {
         for (const dep of Object.keys(packageJSONContents.dependencies)) {

test_src/index.cjs

@@ -5,7 +5,7 @@ var _var = require('process');
 var {sum, div,sad} = require('./arithmetic.cjs');
 // var {sum, div} = require('../output/lodash.bundle.js');
 var ms = require('./ms.bundle.cjs')
-var cn = require('./classnames.bundle.cjs')
+var cn = require('classnames')
 
 
 let cwd = process.cwd;
@@ -25,6 +25,6 @@ console.log(`Read some data`,newLocal,
     div(7,0),
     div(32,3),
     // ceil(10.24),
-    cn('fooo','foobar'),
+    cn('fooo','foobar',{tempo:true}),
     ms('1000y',{long:true})
 );