[update] enhance wpCompress logging, add rootModule parameter to sliceAndWriteCalls, and improve dependency checks

src_dataset/FILTER_LIST.mjs

@@ -17,7 +17,6 @@ const FILTER_LIST = [
     "https://github.com/nodelib/nodelib/tree/master/packages/fs/fs.macchiato",
     "https://github.com/paulmillr/async-each",
     "https://github.com/yarnpkg/yarn/blob/master/packages",
-    "https://github.com/emotion-js/emotion/tree/master/packages/stylis",
     "https://github.com/kogosoftwarellc/open-api/tree/master/packages/openapi-types",
     "https://github.com/thenativeweb/boolean",
     "https://github.com/zkochan/packages/tree/master/read-yaml-file",
@@ -25,7 +24,6 @@ const FILTER_LIST = [
     "https://github.com/adobe/react-spectrum/tree/main/packages/@internationalized/date",
     "https://github.com/pnpm/pnpm/blob/main/packages",
     "https://github.com/jhermsmeier/node-scuid",
-    "https://github.com/emotion-js/emotion/tree/master/packages/babel-plugin-emotion",
     "https://github.com/emotion-js/emotion/tree/master/removed-packages/core",
     "https://github.com/babel/babel/tree/master/packages/*",
     "https://github.com/pugjs/pug/tree/master/packages/*",
@@ -33,7 +31,11 @@ const FILTER_LIST = [
     "https://github.com/Marak/Faker.js",
     "https://github.com/ethanent/phin",
     "https://github.com/Popmotion/popmotion/tree/master/packages/*",
-    "https://github.com/gulpjs/copy-prop"
+    "https://github.com/gulpjs/copy-prop",
+    "https://github.com/netlify/serverless-functions-api",
+    "https://github.com/igoradamenko/esbuild-plugin-alias",
+    "https://github.com/emotion-js/emotion/tree/master/packages/*",
+    "https://github.com/jhermsmeier/node-http-link-header"
 ];
 
 const FILTER_LIST_REGEX = FILTER_LIST.map(GlobToRegExp)

src_dataset/index.mjs

@@ -6,7 +6,7 @@ import { processPromisesBatch } from './batch.mjs';
 
 
 
-const intermediateRepoList = await cacheFunctionOutput('repos.json', async function () {
+const intermediateRepoList = await cacheFunctionOutput('repos.n2.json', async function () {
     const [packagesM, packageReposM] = await Promise.all([
         import('download-counts', { with:{type: 'json'}}),
         import('all-the-package-repos', { with: { type: 'json' } })
@@ -15,7 +15,7 @@ const intermediateRepoList = await cacheFunctionOutput('repos.json', async funct
     const packageRepos = packageReposM.default;
 
     const packageList = Object.keys(packages).map(e => [e, packages[e]])
-        .filter(e => e[1] > 100).filter(e => !e[0].startsWith("@types/"))
+        .filter(e => e[1] > 100_000).filter(e => !e[0].startsWith("@types/")).filter(e => !e[0].startsWith("@webassemblyjs/")) // filter out typescript packages and @types packages
     console.log('packagelist', packageList.length)
     /**
      * @type {[string,string,number][]} repo, link count
@@ -30,9 +30,9 @@ const intermediateRepoList = await cacheFunctionOutput('repos.json', async funct
 // const packageMap = new Map(packageList)
 
 console.log(`Total repos`,intermediateRepoList.length)
-const intermediateRepoListSmaller = intermediateRepoList.slice(0,5000);
+const intermediateRepoListSmaller = intermediateRepoList.slice(0,6000);
 
-const repoStatus = await processPromisesBatch(intermediateRepoListSmaller,15,cloneRepoAndCheck)
+const repoStatus = await processPromisesBatch(intermediateRepoListSmaller,10,cloneRepoAndCheck)
 
 const repoStatusString = csv.stringify(repoStatus);
 await fsp.writeFile('repostatus.csv', repoStatusString);

src_dataset/mine.mjs

@@ -28,6 +28,7 @@ export async function cloneRepoAndCheck([repoName, repoGitUrl, downloadCount]) {
 
     const packageFile = resolve(repoPath, 'package.json')
     if (!existsSync(packageFile)) return [repoName, null];
+    // console.log("[git] checking", repoName, "for dependencies at ", packageFile);
     const packageJSONContentsString = (await readFile(packageFile)).toString()
 
     // console.log(packageJSONContentsString);
@@ -58,7 +59,7 @@ function hasAnyActualDependencies(packageJSONContents, repoName) {
 }
 
 function checkTestingDependencies(packageJSONContents, repoName) {
-    const testingLibraries = new Set(['mocha']);
+    const testingLibraries = new Set(['mocha','jest']);
     const dependencies = new Set();
     if (packageJSONContents.dependencies !== undefined) {
         for (const dep of Object.keys(packageJSONContents.dependencies)) {