[add] filter by npm api before repo fetch

2025-08-15 04:01:08 +01:00
parent 7f2f0b9240
commit 52d0c7b649
7 changed files with 794 additions and 157 deletions
--- a/minableRepositories2.csv
+++ b/minableRepositories2.csv
--- a/package-lock.json
+++ b/package-lock.json
@@ -26,6 +26,7 @@
        "glob-to-regexp": "^0.4.1",
        "jalangi2": "^0.2.6",
        "minimatch": "^10.0.3",
+        "npm-api": "^1.0.1",
        "semver": "^7.7.2",
        "set.prototype.intersection": "^1.1.7",
        "slice-js": "file:/home/atreyab/Documents/Docs/SlicingImport/repos-js/slice-js",
@@ -648,6 +649,15 @@
        "url": "https://github.com/sponsors/ljharb"
      }
    },
+    "node_modules/axios": {
+      "version": "0.21.4",
+      "resolved": "https://registry.npmjs.org/axios/-/axios-0.21.4.tgz",
+      "integrity": "sha512-ut5vewkiu8jjGBdqpM44XxjuCjq9LAKeHVmoVfHVzy8eHgxxq8SbAVQNovDA8mVi05kP0Ea/n/UzcSHcTJQfNg==",
+      "license": "MIT",
+      "dependencies": {
+        "follow-redirects": "^1.14.0"
+      }
+    },
    "node_modules/babel-code-frame": {
      "version": "6.26.0",
      "resolved": "https://registry.npmjs.org/babel-code-frame/-/babel-code-frame-6.26.0.tgz",
@@ -1447,6 +1457,20 @@
        "url": "https://github.com/chalk/wrap-ansi?sponsor=1"
      }
    },
+    "node_modules/clone-deep": {
+      "version": "4.0.1",
+      "resolved": "https://registry.npmjs.org/clone-deep/-/clone-deep-4.0.1.tgz",
+      "integrity": "sha512-neHB9xuzh/wk0dIHweyAXv2aPGZIVk3pLMe+/RNzINf17fe0OG96QroktYAUm7SM1PBnzTabaLboqqxDyMU+SQ==",
+      "license": "MIT",
+      "dependencies": {
+        "is-plain-object": "^2.0.4",
+        "kind-of": "^6.0.2",
+        "shallow-clone": "^3.0.0"
+      },
+      "engines": {
+        "node": ">=6"
+      }
+    },
    "node_modules/code-block-writer": {
      "version": "13.0.3",
      "resolved": "https://registry.npmjs.org/code-block-writer/-/code-block-writer-13.0.3.tgz",
@@ -1758,6 +1782,20 @@
      "integrity": "sha512-37tirWLAIKWITTaFK1FUkNRk7W7qxN2riCc51ambmFmBrtDcUvf0umvCPQ/a1p5ZC4CaiCfdtqY4vXD2HRr3Cw==",
      "license": "MIT"
    },
+    "node_modules/download-stats": {
+      "version": "0.3.4",
+      "resolved": "https://registry.npmjs.org/download-stats/-/download-stats-0.3.4.tgz",
+      "integrity": "sha512-ic2BigbyUWx7/CBbsfGjf71zUNZB4edBGC3oRliSzsoNmvyVx3Ycfp1w3vp2Y78Ee0eIIkjIEO5KzW0zThDGaA==",
+      "license": "MIT",
+      "dependencies": {
+        "JSONStream": "^1.2.1",
+        "lazy-cache": "^2.0.1",
+        "moment": "^2.15.1"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
    "node_modules/dunder-proto": {
      "version": "1.0.1",
      "resolved": "https://registry.npmjs.org/dunder-proto/-/dunder-proto-1.0.1.tgz",
@@ -2230,6 +2268,26 @@
        "flat": "cli.js"
      }
    },
+    "node_modules/follow-redirects": {
+      "version": "1.15.11",
+      "resolved": "https://registry.npmjs.org/follow-redirects/-/follow-redirects-1.15.11.tgz",
+      "integrity": "sha512-deG2P0JfjrTxl50XGCDyfI97ZGVCxIpfKYmfyrQ54n5FO/0gfIES8C/Psl6kWVDolizcaaxZJnTS0QSMxvnsBQ==",
+      "funding": [
+        {
+          "type": "individual",
+          "url": "https://github.com/sponsors/RubenVerborgh"
+        }
+      ],
+      "license": "MIT",
+      "engines": {
+        "node": ">=4.0"
+      },
+      "peerDependenciesMeta": {
+        "debug": {
+          "optional": true
+        }
+      }
+    },
    "node_modules/for-each": {
      "version": "0.3.5",
      "resolved": "https://registry.npmjs.org/for-each/-/for-each-0.3.5.tgz",
@@ -2717,6 +2775,12 @@
        "url": "https://github.com/sponsors/ljharb"
      }
    },
+    "node_modules/is-buffer": {
+      "version": "1.1.6",
+      "resolved": "https://registry.npmjs.org/is-buffer/-/is-buffer-1.1.6.tgz",
+      "integrity": "sha512-NcdALwpXkTm5Zvvbk7owOUSvVvBKDgKP5/ewfXEznmQFfs4ZRmanOeKBTjRVjka3QFoN6XJ+9F3USqfHqTaU5w==",
+      "license": "MIT"
+    },
    "node_modules/is-callable": {
      "version": "1.2.7",
      "resolved": "https://registry.npmjs.org/is-callable/-/is-callable-1.2.7.tgz",
@@ -2893,6 +2957,18 @@
        "node": ">=8"
      }
    },
+    "node_modules/is-plain-object": {
+      "version": "2.0.4",
+      "resolved": "https://registry.npmjs.org/is-plain-object/-/is-plain-object-2.0.4.tgz",
+      "integrity": "sha512-h5PpgXkWitc38BBMYawTYMWJHFZJVnBquFE57xFpjB8pJFiF6gZ+bU+WyI/yqXiFR5mdLsgYNaPe8uao6Uv9Og==",
+      "license": "MIT",
+      "dependencies": {
+        "isobject": "^3.0.1"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
    "node_modules/is-regex": {
      "version": "1.2.1",
      "resolved": "https://registry.npmjs.org/is-regex/-/is-regex-1.2.1.tgz",
@@ -3054,6 +3130,15 @@
      "integrity": "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw==",
      "license": "ISC"
    },
+    "node_modules/isobject": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
+      "integrity": "sha512-WhB9zCku7EGTj/HQQRz5aUQEUeoQZH2bWcltRErOpymJ4boYE6wL9Tbr23krRPSZ+C5zqNSrSw+Cc7sZZ4b7vg==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
    "node_modules/iterate-iterator": {
      "version": "1.0.2",
      "resolved": "https://registry.npmjs.org/iterate-iterator/-/iterate-iterator-1.0.2.tgz",
@@ -3204,6 +3289,52 @@
      "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==",
      "license": "MIT"
    },
+    "node_modules/jsonparse": {
+      "version": "1.3.1",
+      "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz",
+      "integrity": "sha512-POQXvpdL69+CluYsillJ7SUhKvytYjW9vG/GKpnf+xP8UWgYEM/RaMzHHofbALDiKbbP1W8UEYmgGl39WkPZsg==",
+      "engines": [
+        "node >= 0.2.0"
+      ],
+      "license": "MIT"
+    },
+    "node_modules/JSONStream": {
+      "version": "1.3.5",
+      "resolved": "https://registry.npmjs.org/JSONStream/-/JSONStream-1.3.5.tgz",
+      "integrity": "sha512-E+iruNOY8VV9s4JEbe1aNEm6MiszPRr/UfcHMz0TQh1BXSxHK+ASV1R6W4HpjBhSeS+54PIsAMCBmwD06LLsqQ==",
+      "license": "(MIT OR Apache-2.0)",
+      "dependencies": {
+        "jsonparse": "^1.2.0",
+        "through": ">=2.2.7 <3"
+      },
+      "bin": {
+        "JSONStream": "bin.js"
+      },
+      "engines": {
+        "node": "*"
+      }
+    },
+    "node_modules/kind-of": {
+      "version": "6.0.3",
+      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-6.0.3.tgz",
+      "integrity": "sha512-dcS1ul+9tmeD95T+x28/ehLgd9mENa3LsvDTtzm3vyBEO7RPptvAD+t44WVXaUjTBRcrpFeFlC8WCruUR456hw==",
+      "license": "MIT",
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/lazy-cache": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/lazy-cache/-/lazy-cache-2.0.2.tgz",
+      "integrity": "sha512-7vp2Acd2+Kz4XkzxGxaB1FWOi8KjWIWsgdfD5MCb86DWvlLqhRPM+d6Pro3iNEL5VT9mstz5hKAlcd+QR6H3aA==",
+      "license": "MIT",
+      "dependencies": {
+        "set-getter": "^0.1.0"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
    "node_modules/loader-runner": {
      "version": "4.3.0",
      "resolved": "https://registry.npmjs.org/loader-runner/-/loader-runner-4.3.0.tgz",
@@ -3519,6 +3650,15 @@
      "integrity": "sha512-pEk4ECWQXV6z2zjhRZUongnLJNUeGQJ3w6OQ5ctGwD+i5o93qjRQUk2Rt6VdNeu3sEP0AB4LcfvdebpxBRVr4g==",
      "license": "MIT"
    },
+    "node_modules/moment": {
+      "version": "2.30.1",
+      "resolved": "https://registry.npmjs.org/moment/-/moment-2.30.1.tgz",
+      "integrity": "sha512-uEmtNhbDOrWPFS+hdjFCBfy9f2YoyzRpwcl+DqpC6taX21FzsTLQVbMV/W7PzNSX6x/bhC1zA3c2UQ5NzH6how==",
+      "license": "MIT",
+      "engines": {
+        "node": "*"
+      }
+    },
    "node_modules/ms": {
      "version": "2.1.3",
      "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz",
@@ -3552,12 +3692,49 @@
      "integrity": "sha512-Yd3UES5mWCSqR+qNT93S3UoYUkqAZ9lLg8a7g9rimsWmYGK8cVToA4/sF3RrshdyV3sAGMXVUmpMYOw+dLpOuw==",
      "license": "MIT"
    },
+    "node_modules/node-fetch": {
+      "version": "2.7.0",
+      "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.7.0.tgz",
+      "integrity": "sha512-c4FRfUm/dbcWZ7U+1Wq0AwCyFL+3nt2bEw05wfxSz+DWpWsitgmSgYmy2dQdWyKC1694ELPqMs/YzUSNozLt8A==",
+      "license": "MIT",
+      "dependencies": {
+        "whatwg-url": "^5.0.0"
+      },
+      "engines": {
+        "node": "4.x || >=6.0.0"
+      },
+      "peerDependencies": {
+        "encoding": "^0.1.0"
+      },
+      "peerDependenciesMeta": {
+        "encoding": {
+          "optional": true
+        }
+      }
+    },
    "node_modules/node-releases": {
      "version": "2.0.19",
      "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.19.tgz",
      "integrity": "sha512-xxOWJsBKtzAq7DY0J+DTzuz58K8e7sJbdgwkbMWQe8UYB6ekmsQ45q0M/tJDsGaZmbC+l7n57UV8Hl5tHxO9uw==",
      "license": "MIT"
    },
+    "node_modules/npm-api": {
+      "version": "1.0.1",
+      "resolved": "https://registry.npmjs.org/npm-api/-/npm-api-1.0.1.tgz",
+      "integrity": "sha512-4sITrrzEbPcr0aNV28QyOmgn6C9yKiF8k92jn4buYAK8wmA5xo1qL3II5/gT1r7wxbXBflSduZ2K3FbtOrtGkA==",
+      "license": "MIT",
+      "dependencies": {
+        "clone-deep": "^4.0.1",
+        "download-stats": "^0.3.4",
+        "JSONStream": "^1.3.5",
+        "moment": "^2.24.0",
+        "node-fetch": "^2.6.0",
+        "paged-request": "^2.0.1"
+      },
+      "engines": {
+        "node": ">=10.0"
+      }
+    },
    "node_modules/object-assign": {
      "version": "4.1.1",
      "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz",
@@ -3687,6 +3864,18 @@
      "integrity": "sha512-UEZIS3/by4OC8vL3P2dTXRETpebLI2NiI5vIrjaD/5UtrkFX/tNbwjTSRAGC/+7CAo2pIcBaRgWmcBBHcsaCIw==",
      "license": "BlueOak-1.0.0"
    },
+    "node_modules/paged-request": {
+      "version": "2.0.2",
+      "resolved": "https://registry.npmjs.org/paged-request/-/paged-request-2.0.2.tgz",
+      "integrity": "sha512-NWrGqneZImDdcMU/7vMcAOo1bIi5h/pmpJqe7/jdsy85BA/s5MSaU/KlpxwW/IVPmIwBcq2uKPrBWWhEWhtxag==",
+      "license": "MIT",
+      "dependencies": {
+        "axios": "^0.21.1"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
    "node_modules/parse5": {
      "version": "2.1.5",
      "resolved": "https://registry.npmjs.org/parse5/-/parse5-2.1.5.tgz",
@@ -4156,6 +4345,18 @@
        "node": ">= 0.4"
      }
    },
+    "node_modules/set-getter": {
+      "version": "0.1.1",
+      "resolved": "https://registry.npmjs.org/set-getter/-/set-getter-0.1.1.tgz",
+      "integrity": "sha512-9sVWOy+gthr+0G9DzqqLaYNA7+5OKkSmcqjL9cBpDEaZrr3ShQlyX2cZ/O/ozE41oxn/Tt0LGEM/w4Rub3A3gw==",
+      "license": "MIT",
+      "dependencies": {
+        "to-object-path": "^0.3.0"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
    "node_modules/set-proto": {
      "version": "1.0.0",
      "resolved": "https://registry.npmjs.org/set-proto/-/set-proto-1.0.0.tgz",
@@ -4191,6 +4392,18 @@
        "url": "https://github.com/sponsors/ljharb"
      }
    },
+    "node_modules/shallow-clone": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/shallow-clone/-/shallow-clone-3.0.1.tgz",
+      "integrity": "sha512-/6KqX+GVUdqPuPPd2LxDDxzX6CAbjJehAAOKlNpqqUpAqPM6HeL8f+o3a+JsyGjn2lv0WY8UsTgUJjU9Ok55NA==",
+      "license": "MIT",
+      "dependencies": {
+        "kind-of": "^6.0.2"
+      },
+      "engines": {
+        "node": ">=8"
+      }
+    },
    "node_modules/shebang-command": {
      "version": "2.0.0",
      "resolved": "https://registry.npmjs.org/shebang-command/-/shebang-command-2.0.0.tgz",
@@ -4612,6 +4825,12 @@
        "node": ">=0.8"
      }
    },
+    "node_modules/through": {
+      "version": "2.3.8",
+      "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz",
+      "integrity": "sha512-w89qg7PI8wAdvX60bMDP+bFoD5Dvhm9oLheFp5O4a2QF0cSBGsBX4qZmadPMvVqlLJBBci+WqGGOAPvcDeNSVg==",
+      "license": "MIT"
+    },
    "node_modules/to-fast-properties": {
      "version": "1.0.3",
      "resolved": "https://registry.npmjs.org/to-fast-properties/-/to-fast-properties-1.0.3.tgz",
@@ -4621,6 +4840,30 @@
        "node": ">=0.10.0"
      }
    },
+    "node_modules/to-object-path": {
+      "version": "0.3.0",
+      "resolved": "https://registry.npmjs.org/to-object-path/-/to-object-path-0.3.0.tgz",
+      "integrity": "sha512-9mWHdnGRuh3onocaHzukyvCZhzvr6tiflAy/JRFXcJX0TjgfWA9pk9t8CMbzmBE4Jfw58pXbkngtBtqYxzNEyg==",
+      "license": "MIT",
+      "dependencies": {
+        "kind-of": "^3.0.2"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
+    "node_modules/to-object-path/node_modules/kind-of": {
+      "version": "3.2.2",
+      "resolved": "https://registry.npmjs.org/kind-of/-/kind-of-3.2.2.tgz",
+      "integrity": "sha512-NOW9QQXMoZGg/oqnVNoNTTIFEIid1627WCffUBJEdMxYApq7mNE7CpzucIPc+ZQg25Phej7IJSmX3hO+oblOtQ==",
+      "license": "MIT",
+      "dependencies": {
+        "is-buffer": "^1.1.5"
+      },
+      "engines": {
+        "node": ">=0.10.0"
+      }
+    },
    "node_modules/to-regex-range": {
      "version": "5.0.1",
      "resolved": "https://registry.npmjs.org/to-regex-range/-/to-regex-range-5.0.1.tgz",
@@ -4633,6 +4876,12 @@
        "node": ">=8.0"
      }
    },
+    "node_modules/tr46": {
+      "version": "0.0.3",
+      "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz",
+      "integrity": "sha512-N3WMsuqV66lT30CrXNbEjx4GEwlow3v6rr4mCcv6prnfwhS01rkgyFdjPNBYd9br7LpXV1+Emh01fHnq2Gdgrw==",
+      "license": "MIT"
+    },
    "node_modules/ts-morph": {
      "version": "26.0.0",
      "resolved": "https://registry.npmjs.org/ts-morph/-/ts-morph-26.0.0.tgz",
@@ -4813,6 +5062,12 @@
        "node": ">=10.13.0"
      }
    },
+    "node_modules/webidl-conversions": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/webidl-conversions/-/webidl-conversions-3.0.1.tgz",
+      "integrity": "sha512-2JAn3z8AR6rjK8Sm8orRC0h/bcl/DqL7tRPdGZ4I1CjdF+EaMLmYxBHyXuKL849eucPFhvBoxMsflfOb8kxaeQ==",
+      "license": "BSD-2-Clause"
+    },
    "node_modules/webpack": {
      "version": "5.99.9",
      "resolved": "https://registry.npmjs.org/webpack/-/webpack-5.99.9.tgz",
@@ -4901,6 +5156,16 @@
        "node": ">=4.0"
      }
    },
+    "node_modules/whatwg-url": {
+      "version": "5.0.0",
+      "resolved": "https://registry.npmjs.org/whatwg-url/-/whatwg-url-5.0.0.tgz",
+      "integrity": "sha512-saE57nupxk6v3HY35+jzBwYa0rKSy0XR8JSxZPwgLr7ys0IBzhGviA1/TUGJLmSVqs8pb9AnvICXEuOHLprYTw==",
+      "license": "MIT",
+      "dependencies": {
+        "tr46": "~0.0.3",
+        "webidl-conversions": "^3.0.0"
+      }
+    },
    "node_modules/which": {
      "version": "1.0.9",
      "resolved": "https://registry.npmjs.org/which/-/which-1.0.9.tgz",
--- a/package.json
+++ b/package.json
@@ -28,6 +28,7 @@
    "glob-to-regexp": "^0.4.1",
    "jalangi2": "^0.2.6",
    "minimatch": "^10.0.3",
+    "npm-api": "^1.0.1",
    "semver": "^7.7.2",
    "set.prototype.intersection": "^1.1.7",
    "slice-js": "file:/home/atreyab/Documents/Docs/SlicingImport/repos-js/slice-js",
--- a/src_dataset/FILTER_LIST.mjs
+++ b/src_dataset/FILTER_LIST.mjs
@@ -35,7 +35,13 @@ const FILTER_LIST = [
    "https://github.com/netlify/serverless-functions-api",
    "https://github.com/igoradamenko/esbuild-plugin-alias",
    "https://github.com/emotion-js/emotion/tree/master/packages/*",
-    "https://github.com/jhermsmeier/node-http-link-header"
+    "https://github.com/jhermsmeier/node-http-link-header",
+    "https://github.com/serverless/utils",
+    "https://github.com/serverless/dashboard-plugin",
+    "https://github.com/foliojs-fork/linebreaker",
+    "https://github.com/segmentio/analytics.js-video-plugins",
+    "https://github.com/cucumber/cucumber-expressions-javascript",
+    "https://github.com/jakwings/node-temp-fs"
 ];

 const FILTER_LIST_REGEX = FILTER_LIST.map(GlobToRegExp)
--- a/src_dataset/cache.mjs
+++ b/src_dataset/cache.mjs
@@ -9,18 +9,18 @@ import { resolve } from "node:path";
 * @param {()=>Promise<T>} asyncCallback 
 * @returns {Promise<T>}
 */
-export async function cacheFunctionOutput(fileName, asyncCallback) {
+export async function cacheFunctionOutput(fileName, asyncCallback, silent=false) {
    const fileLoc = resolve('../cache-repos', fileName);
    if (existsSync(fileLoc)) {
-        console.log("[cacher] Using cached ", fileLoc);
+        !silent && console.log("[cacher] Using cached ", fileLoc);
        const fileContents = (await readFile(fileLoc)).toString();
        return JSON.parse(fileContents);
    } else {
-        console.log("[cacher] cache miss")
+        !silent && console.log("[cacher] cache miss")
        const returnRes = await asyncCallback();
        const fileContents = JSON.stringify(returnRes);
        await writeFile(fileLoc,fileContents);
-        console.log("[cacher] saved ",fileLoc)
+        !silent && console.log("[cacher] saved ",fileLoc)
        return returnRes;
    }
 }
--- a/src_dataset/index.mjs
+++ b/src_dataset/index.mjs
@@ -30,9 +30,9 @@ const intermediateRepoList = await cacheFunctionOutput('repos.n2.json', async fu
 // const packageMap = new Map(packageList)

 console.log(`Total repos`,intermediateRepoList.length)
-const intermediateRepoListSmaller = intermediateRepoList.slice(0,6000);
+const intermediateRepoListSmaller = intermediateRepoList.slice(0,10000);

-const repoStatus = await processPromisesBatch(intermediateRepoListSmaller,10,cloneRepoAndCheck)
+const repoStatus = await processPromisesBatch(intermediateRepoListSmaller,20,cloneRepoAndCheck)

 const repoStatusString = csv.stringify(repoStatus);
 await fsp.writeFile('repostatus.csv', repoStatusString);
--- a/src_dataset/mine.mjs
+++ b/src_dataset/mine.mjs
@@ -4,6 +4,8 @@ import git from 'git-client'
 import { resolve } from 'path'
 import int from 'set.prototype.intersection';
 import { matchFilterList } from './FILTER_LIST.mjs';
+import npmapi from 'npm-api'
+import { cacheFunctionOutput } from './cache.mjs';
 /**
 * 
 * @param {[string,string,number]} param0 
@@ -17,19 +19,24 @@ export async function cloneRepoAndCheck([repoName, repoGitUrl, downloadCount]) {
        return [repoName, null]
    };
    // console.log('[git] fetching',repoName, repoGitUrl);
-    await cacheCloneIdempotently(repoPath, repoName, repoGitUrl)
+    let api = new npmapi();
+    const repo = api.repo(repoName);

-    const tsConfigFileLocation = resolve(repoPath, 'tsconfig.json');
-    const tsConfigFileExists = existsSync(tsConfigFileLocation);
-    if (tsConfigFileExists){
-         return [repoName, null];
+    let packageJSONContentsString = null;
+
+
+    try{
+        packageJSONContentsString = await cacheFunctionOutput(`cache-repo-package-json-${repoName.replaceAll('/',"_sl_")}.json`,async ()=> JSON.stringify(await repo.package()),true);
+        // console.log("[git] fetched package.json for", repoName);
+    }catch(e){
+        throw new Error(`Failed to fetch package.json for ${repoName} from npm: ${e.message}`);
    }

-
-    const packageFile = resolve(repoPath, 'package.json')
-    if (!existsSync(packageFile)) return [repoName, null];
+    if (packageJSONContentsString === undefined || packageJSONContentsString === null) {
+        throw new Error(`Failed to fetch package.json for ${repoName} from npm`);
    // console.log("[git] checking", repoName, "for dependencies at ", packageFile);
-    const packageJSONContentsString = (await readFile(packageFile)).toString()
+    // const packageJSONContentsString = (await readFile(packageFile)).toString()
+    }

    // console.log(packageJSONContentsString);
    const packageJSONContents = JSON.parse(packageJSONContentsString)
@@ -41,6 +48,20 @@ export async function cloneRepoAndCheck([repoName, repoGitUrl, downloadCount]) {

    const hasDependencies = checkTestingDependencies(packageJSONContents, repoName);
    if (hasDependencies) {
+        await cacheCloneIdempotently(repoPath, repoName, repoGitUrl);
+
+        const tsConfigFileLocation = resolve(repoPath, 'tsconfig.json');
+        const tsConfigFileExists = existsSync(tsConfigFileLocation);
+        if (tsConfigFileExists){
+            // console.warn("[git] Ignoring ", repoName, "because it has a tsconfig.json file.");
+            return [repoName, null];
+        }
+        const packageFile = resolve(repoPath, 'package.json')
+        if (!existsSync(packageFile)){
+            console.warn("[git] Unexpected package.json not found in", repoName, "at", packageFile);
+            return [repoName, null];}
+
+        // finally, return the test script if it exists
        return [repoName, ((packageJSONContents?.scripts?.test))]
    }
    else return [repoName, null]
@@ -88,7 +109,7 @@ async function cacheCloneIdempotently(repoPath, repoName, repoGitUrl) {
        if (!isDir) throw new Error(repoName, " is mangled. delete directory and re-clone.")
        else {
            // const path = await git.status({ $cwd: repoPath })
-
+            // console.log("[git] already cloned", repoName, "at", repoPath);
        }
    } else {
        console.log("[git] cloning", repoGitUrl);