[add] more repos and generic filtering

minableRepositories2.csv

@@ -3,7 +3,6 @@ glob-parent,nyc mocha --async-only
 js-yaml,npm run lint && mocha
 fs-extra,npm run lint && npm run unit && npm run unit-esm
 iconv-lite,mocha --reporter spec --grep .
-form-data,npm run tests-only
 braces,mocha -r ./test/mocha-initialization.js
 fill-range,npm run lint && npm run mocha
 micromatch,mocha
@@ -24,3 +23,417 @@ regenerator-runtime,test/run.sh
 eslint,node Makefile.js test
 is-extendable,mocha
 fast-levenshtein,mocha
+@eslint/eslintrc,mocha -R progress -c 'tests/lib/*.cjs' && c8 mocha -R progress -c 'tests/lib/**/*.js'
+esquery,nyc npm run mocha && npm run lint
+on-finished,mocha --reporter spec --check-leaks test/
+error-ex,mocha --compilers coffee:coffee-script/register
+raw-body,mocha --trace-deprecation --reporter spec --bail --check-leaks test/
+finalhandler,mocha --reporter spec --check-leaks test/
+send,mocha --check-leaks --reporter spec
+body-parser,mocha --reporter spec --check-leaks test/
+accepts,mocha --reporter spec --check-leaks --bail test/
+serve-static,mocha --reporter spec --bail --check-leaks test/
+@eslint/js,node Makefile.js test
+express,mocha --require test/support/env --reporter spec --check-leaks test/ test/acceptance/
+@sinonjs/fake-timers,npm run test-node && npm run test-headless
+type-is,mocha --reporter spec --check-leaks --bail test/
+terser,node test/compress.js && mocha test/mocha
+through2,npm run lint && npm run test:node && npm run test:browser
+css-tree,mocha lib/__tests --require lib/__tests/helpers/setup.js --reporter progress
+psl,mocha test/*.spec.js
+proxy-addr,mocha --reporter spec --bail --check-leaks test/
+babel-plugin-istanbul,cross-env NODE_ENV=test nyc --reporter=lcov --reporter=text mocha --timeout 5000 test/*.js
+@humanwhocodes/config-array,mocha -r esm tests/ --recursive
+jsdom,npm-run-all --continue-on-error test:*
+makeerror,NODE_PATH=./lib mocha --ui exports
+walker,NODE_PATH=./lib mocha --ui exports
+watchpack,mocha
+eslint-plugin-import,npm run tests-only
+eslint-module-utils,npm run tests-only
+eslint-import-resolver-node,npm run tests-only
+eslint-utils,"run-s ""format:prettier -- --check"" lint build test:mocha"
+has-values,mocha
+has-value,mocha
+@webassemblyjs/ast,make test
+@webassemblyjs/helper-wasm-bytecode,make test
+color,xo && tsd && mocha
+@webassemblyjs/wast-printer,make test
+@webassemblyjs/helper-api-error,make test
+@webassemblyjs/wasm-parser,make test
+@webassemblyjs/wasm-gen,make test
+@webassemblyjs/wasm-edit,make test
+@webassemblyjs/wasm-opt,make test
+@webassemblyjs/helper-wasm-section,make test
+@webassemblyjs/helper-buffer,make test
+@webassemblyjs/leb128,make test
+@webassemblyjs/ieee754,make test
+@webassemblyjs/utf8,make test
+@webassemblyjs/floating-point-hex-parser,make test
+url-parse,c8 --reporter=lcov --reporter=text mocha test/test.js
+@cspotcode/source-map-support,mocha
+simple-swizzle,mocha --compilers coffee:coffee-script/register
+global-prefix,mocha
+enquirer,mocha && tsc -p ./test/types
+compression,mocha --check-leaks --reporter spec
+@aws-crypto/util,lerna run test
+compressible,mocha --reporter spec --bail --check-leaks test/
+regenerator-transform,test/run.sh
+@aws-crypto/sha256-js,lerna run test
+url,npm run tests-only
+ejs,npx jake test
+global-modules,mocha
+@webassemblyjs/helper-numbers,make test
+thenify,mocha --reporter spec
+ecdsa-sig-formatter,istanbul cover --root src _mocha -- spec
+mz,mocha --reporter spec
+thenify-all,mocha --reporter spec
+rechoir,nyc mocha --async-only
+eslint-plugin-prettier,pnpm lint && mocha
+hoist-non-react-statics,nyc mocha tests/unit/ --recursive --reporter spec --require=@babel/register
+clone-deep,mocha
+shallow-clone,mocha
+handlebars,npm run test:mocha
+jsonwebtoken,npm run lint && npm run coverage && cost-of-modules
+jake,./bin/cli.js test
+@aws-crypto/supports-web-crypto,lerna run test
+prettier-linter-helpers,npm run lint && mocha
+@aws-crypto/sha256-browser,lerna run test
+cors,npm run lint && npm run test-ci
+expand-brackets,mocha
+node-gyp,cross-env NODE_GYP_NULL_LOGGER=true mocha --timeout 15000 test/test-download.js test/test-*
+extglob,mocha
+fd-slicer,mocha --reporter spec --check-leaks
+http-proxy,nyc --reporter=text --reporter=lcov npm run mocha
+promise,mocha --bail --timeout 200 --slow 99999 -R dot && npm run test-memory-leak
+set-value,mocha
+csso,mocha test --reporter ${REPORTER:-progress}
+promise-retry,mocha --bail -t 10000
+object.pick,mocha
+language-tags,c8 mocha
+d3-array,mocha 'test/**/*-test.js' && eslint src test
+@img/sharp-linuxmusl-x64,npm run test-lint && npm run test-unit && npm run test-licensing && npm run test-types
+registry-auth-token,mocha --require test/global-hooks.js
+ua-parser-js,./script/test-all.sh
+union-value,mocha
+to-object-path,mocha
+unset-value,mocha
+archiver-utils,mocha --reporter dot
+object-visit,mocha
+collection-visit,mocha
+map-visit,mocha
+pascalcase,mocha
+object-copy,mocha
+static-extend,mocha
+memory-fs,mocha
+fragment-cache,mocha
+cache-base,nyc --reporter=text --reporter=html mocha
+pinkie-promise,mocha
+regex-not,mocha
+to-regex,mocha
+base,mocha
+snapdragon,mocha
+nanomatch,mocha
+class-utils,mocha
+selfsigned,mocha -t 5000
+snapdragon-util,mocha
+hash.js,mocha --reporter=spec test/*-test.js && npm run lint
+lie,npm run jshint && mocha -R nyan ./test/cover.js && tsc --noEmit ./test/types.ts
+elliptic,npm run lint && npm run unit
+asn1.js,mocha --reporter spec test/*-test.js && cd rfc/2560 && npm i && npm test && cd ../../rfc/5280 && npm i && npm test && cd ../../ && npm run lint
+serve-index,mocha --reporter spec --bail --check-leaks test/
+d3-shape,mocha 'test/**/*-test.js' && eslint src test
+relateurl,c8 mocha test.js --bail --check-leaks
+d3-interpolate,mocha 'test/**/*-test.js' && eslint src test
+pretty-error,"mocha ""test/**/*.coffee"""
+spdy,mocha --reporter=spec test/*-test.js
+spdy-transport,mocha --reporter=spec test/**/*-test.js test/**/**/*-test.js
+renderkid,"mocha ""test/**/*.coffee"""
+hpack.js,mocha test/*-test.js
+wbuf,mocha test/**/*-test.js
+browserify-zlib,npm run test:node && npm run test:browser
+d3-time,TZ=America/Los_Angeles mocha 'test/**/*-test.js' && eslint src test
+@aws-crypto/crc32,lerna run test
+sharp,npm run test-lint && npm run test-unit && npm run test-licensing && npm run test-types
+hmac-drbg,mocha --reporter=spec test/*-test.js
+des.js,NODE_OPTIONS=--openssl-legacy-provider mocha --reporter=spec test/*-test.js
+d3-time-format,TZ=America/Los_Angeles mocha 'test/**/*-test.js' && eslint src test
+d3-scale,TZ=America/Los_Angeles mocha 'test/**/*-test.js' && eslint src test
+json-bigint,./node_modules/mocha/bin/mocha -R spec --check-leaks test/*-test.js
+dom-converter,"mocha ""test/**/*.coffee"""
+expand-tilde,mocha
+archiver,mocha --reporter dot
+homedir-polyfill,mocha
+zip-stream,mocha --reporter dot
+compress-commons,mocha --reporter dot
+crc32-stream,mocha --reporter dot
+superagent,npm run build && npm run lint && make test
+miller-rabin,mocha --reporter=spec test/**/*-test.js
+linkify-it,npm run lint && npm run build && c8 --exclude build --exclude test -r text -r html -r lcov mocha
+one-time,nyc --reporter=text --reporter=json-summary npm run test:runner
+colorspace,mocha test.js
+resolve-dir,mocha
+@dabh/diagnostics,nyc --reporter=text --reporter=lcov npm run test:runner
+@parcel/watcher,mocha
+findup-sync,nyc mocha --async-only
+duplexer2,mocha -R tap
+compare-func,npm run-script lint && mocha
+markdown-it,npm run lint && CJS_ONLY=1 npm run build && c8 --exclude dist --exclude test -r text -r html -r lcov mocha && node support/specsplit.mjs
+notifications-node-client,"mocha ""spec/**/*.js"" && npm run test:markdown:standard"
+basic-auth,mocha --reporter spec --bail --check-leaks test/
+@img/sharp-linux-x64,npm run test-lint && npm run test-unit && npm run test-licensing && npm run test-types
+connect,mocha --require test/support/env --reporter spec --bail --check-leaks test/
+needle,mocha test
+ts-loader,git clean -xfd test/comparison-tests && git clean -xfd test/execution-tests && node test/run-tests.js
+@aws-crypto/crc32c,lerna run test
+@aws-crypto/sha1-browser,lerna run test
+aws-sdk,node ./scripts/composite-test.js
+for-own,mocha
+sinon,npm run test-node && npm run test-headless && npm run test-webworker
+md5,mocha
+nodemon,npm run lint && npm run spec
+sane,npm run format && eslint src/ test/ index.js && mocha --bail test/test.js && mocha --bail test/utils-test.js && mocha --bail 'test/watchexec_*-test.js'
+@parcel/watcher-linux-x64-glibc,mocha
+is-relative,mocha
+pgpass,"""$npm_execpath"" run _hint && ""$npm_execpath"" run _covered_test"
+capture-exit,mocha test
+is-absolute,mocha
+is-unc-path,mocha
+supertest,nyc --reporter=html --reporter=text mocha --exit --require should --reporter spec --check-leaks
+write,mocha
+coa,nyc mocha
+multer,mocha --reporter spec --exit --check-leaks test/
+redis-parser,nyc --cache --preserve-comments mocha -- -R spec && nyc report --reporter=html
+eslint-plugin-es,npm run -s test:mocha
+@parcel/watcher-linux-x64-musl,mocha
+temp,mocha test/temp-test.js
+recursive-readdir,mocha test/
+path-root,mocha
+lazy-cache,mocha
+nise,mocha lib/**/*.test.js
+router,mocha --reporter spec --check-leaks test/
+parse-filepath,mocha
+vinyl,nyc mocha --async-only
+css,mocha --require should --reporter spec test/*.js
+node-dir,./node_modules/.bin/mocha --reporter spec
+request-progress,mocha --bail
+streamroller,nyc --check-coverage mocha
+currently-unhandled,xo && nyc ava
+@webassemblyjs/wast-parser,make test
+@webassemblyjs/helper-code-frame,make test
+@webassemblyjs/helper-fsm,make test
+@webassemblyjs/helper-module-context,make test
+isomorphic-fetch,jshint `npm run -s files` && lintspaces -i js-comments -e .editorconfig `npm run -s files` && mocha
+d3-transition,mocha 'test/**/*-test.js' && eslint src test
+@cypress/xvfb,eslint **/*.js && mocha
+morgan,mocha --check-leaks --reporter spec
+d3-drag,mocha 'test/**/*-test.js' && eslint src test
+d3-geo,mocha 'test/**/*-test.js' && eslint src test
+webpack-hot-middleware,mocha
+eslint-plugin-flowtype,mocha --require @babel/register ./tests/rules/index.js
+eslint-plugin-node,nyc npm run -s test:_mocha
+browser-resolve,node scripts/setup-symlinks.js && mocha --reporter list test/*.js
+wait-on,npm run lint && npm run test:mocha
+chromium-bidi,wireit
+d3-zoom,mocha 'test/**/*-test.js' && eslint src test
+babel-eslint,npm run lint && npm run test-only
+@aws-crypto/ie11-detection,lerna run test
+nock,nyc --reporter=lcov --reporter=text mocha --recursive tests
+stacktrace-parser,tsd && mocha --require ./mocha-babel-hook 'test/**/*.spec.js'
+zen-observable-ts,mocha tests/bundle.cjs
+watchpack-chokidar2,mocha
+randexp,istanbul cover node_modules/.bin/_mocha -- test/*-test.js
+mnemonist,mocha
+find-yarn-workspace-root,mocha tests/**/*-test.js
+unixify,mocha
+@surma/rollup-plugin-off-main-thread,node ./run_tests.js
+cli-progress,mocha test/**/*.test.js
+npm-run-all,nyc --require babel-register npm run _mocha
+@wry/equality,nx run-many --target=test --all --parallel
+@wry/trie,nx run-many --target=test --all --parallel
+d3-scale-chromatic,mocha 'test/**/*-test.js' && eslint src test
+cookie-parser,mocha --reporter spec --bail --check-leaks test/
+d3-force,mocha 'test/**/*-test.js' && eslint src test
+d3-dsv,TZ=America/Los_Angeles mocha 'test/**/*-test.js' && eslint src test
+localforage,"node -e ""require('grunt').cli()"" null test"
+@mui/icons-material,node scripts/test.mjs
+@zkochan/js-yaml,npm run lint && mocha
+d3-delaunay,mocha 'test/**/*-test.js' && eslint src test
+d3,mocha 'test/**/*-test.js' && eslint src test
+window-size,mocha
+sockjs-client,mocha tests/node.js
+seek-bzip,mocha
+cookies,mocha --require test/support/env --reporter spec --bail --check-leaks test/
+keygrip,mocha --reporter spec test/
+@wry/context,nx run-many --target=test --all --parallel
+marked-terminal,cross-env FORCE_HYPERLINK=0 mocha tests/*.js --reporter spec
+get-pkg-repo,eslint src/ && nyc mocha --opts mocha.opts src/**/*.spec.js
+d3-brush,mocha 'test/**/*-test.js' && eslint src test
+brotli,mocha
+lolex,npm run test-node && npm run test-headless
+contains-path,mocha
+d3-chord,mocha 'test/**/*-test.js' && eslint src test
+fined,nyc mocha --async-only
+d3-contour,mkdir -p test/output && mocha -r module-alias/register 'test/**/*-test.js' test/snapshot.js && eslint src test
+canvas,mocha test/*.test.js
+unicode-trie,mocha
+passport,node_modules/.bin/mocha --reporter spec --require test/bootstrap/node test/*.test.js test/**/*.test.js
+plist,make test
+object.defaults,mocha
+d3-fetch,mocha 'test/**/*-test.js' && eslint src test
+@datadog/native-appsec,mocha
+klaw-sync,npm run lint && npm run unit
+dd-trace,SERVICES=* yarn services && mocha --expose-gc 'packages/dd-trace/test/setup/node.js' 'packages/*/test/**/*.spec.js'
+nearley,mocha test/*.test.js
+@datadog/native-iast-taint-tracking,mocha --recursive
+stylus,mocha test/ test/middleware/ --require chai --bail --check-leaks --reporter dot
+@datadog/native-metrics,mocha -n expose-gc 'test/**/*.spec.js' && node test/main
+is-color-stop,nyc --reporter=lcov --reporter=text mocha
+gonzales-pe,bash ./scripts/test.sh
+bufferutil,mocha
+make-iterator,mocha
+glob-base,mocha
+c8,cross-env TS_NODE_SKIP_PROJECT=true node ./bin/c8.js mocha --timeout=10000 ./test/*.js
+liftoff,nyc mocha --async-only
+http-assert,mocha --reporter spec --check-leaks test/
+uid-safe,mocha --trace-deprecation --reporter spec --bail --check-leaks test/
+karma,npm run test:unit && npm run test:e2e && npm run test:client
+fancy-log,nyc mocha --async-only
+object.map,mocha
+sanitize-html,npx eslint . && mocha test/test.js
+find-file-up,mocha
+object.omit,mocha
+find-pkg,mocha
+koa-convert,mocha index.spec.js --exit
+ssf,make test
+karma-chrome-launcher,npm run unit-test && npm run integration-test
+cfb,make test
+utf-8-validate,mocha
+glob-stream,nyc mocha --async-only
+expand-range,mocha
+jsonpath,mocha -u tdd test && jscs lib && jshint lib
+xss,export DEBUG=xss:* && mocha -t 5000
+mquery,mocha --exit test/index.js test/*.test.js
+mv,mocha test/test.js --reporter spec
+eslint-plugin-eslint-comments,nyc npm run -s test:mocha
+vinyl-fs,nyc mocha --async-only
+ethers,npm run test-esm
+easy-table,mocha -R dot --check-leaks
+eslint-plugin-es-x,npm run -s test:mocha
+wait-port,DEBUG=wait-port nyc --report-dir 'artifacts/coverage' -x 'lib/**/*.spec.js' --reporter=html --reporter=text mocha --recursive -t 10000 'lib/**/*.spec.js'
+is-dotfile,mocha
+parse-glob,mocha
+plugin-error,nyc mocha --async-only
+ansi-to-html,nyc mocha --reporter tap
+unicode-properties,npm run build && mocha
+lmdb,mocha test/**.test.js --expose-gc --recursive
+tdigest,mocha specs
+tslint,npm-run-all test:pre -p test:mocha test:rules
+serve-favicon,mocha --reporter spec --bail --check-leaks test/
+is-equal-shallow,mocha
+regex-cache,mocha
+git-log-parser,mocha
+node-environment-flags,mocha
+css-selector-tokenizer,mocha
+detective-typescript,npm run lint && npm run mocha
+underscore.string,npm run test:lint && npm run test:unit && npm run coverage
+precinct,npm run lint && npm run mocha
+tmp-promise,npm run mocha && npm run check-types
+detective-es6,npm run lint && npm run mocha
+module-definition,npm run lint && npm run mocha
+json2mq,./node_modules/.bin/mocha test
+create-error-class,mocha
+detective-amd,npm run lint && npm run mocha
+get-amd-module-type,npm run lint && npm run mocha
+detective-cjs,npm run lint && npm run mocha
+should,mocha -R mocha-better-spec-reporter --require ./cjs/should --color --check-leaks ./test/*.test.js ./test/**/*.test.js
+gray-matter,mocha
+should-format,mocha --harmony_simd -R mocha-better-spec-reporter tests/test.js
+should-equal,mocha --ui bdd -R mocha-better-spec-reporter test.js
+@apidevtools/swagger-parser,npm run test:node && npm run test:typescript
+now-and-later,nyc mocha --async-only
+@lmdb/lmdb-linux-x64,mocha test/**.test.js --expose-gc --recursive
+libmime,grunt
+react-focus-lock,yarn test:pick -- '_tests/**/*spec.js'
+clipboard,karma start --single-run
+fs-mkdirp-stream,nyc mocha --async-only
+to-through,nyc mocha --async-only
+resolve-options,nyc mocha --async-only
+vinyl-sourcemap,nyc mocha --async-only
+cls-hooked,mocha test/*.js & tap test/tap/*.tap.js
+ansi-gray,mocha
+pm2,bash test/unit.sh && bash test/e2e.sh
+mixin-object,mocha
+good-listener,karma start --single-run
+express-session,./test/support/gencert.sh && mocha --require test/support/env --check-leaks --bail --no-exit --reporter spec test/
+isurl,nyc --silent mocha test.js --bail --check-leaks
+@supabase/node-fetch,cross-env BABEL_ENV=test mocha --require babel-register --throw-deprecation test/test.js
+@ethersproject/abi,npm run test-esm
+react-clientside-effect,mocha
+load-yaml-file,standard && mocha && ts-readme-generator --check
+@wry/caches,nx run-many --target=test --all --parallel
+ordered-read-streams,nyc mocha --async-only
+errorhandler,mocha --reporter spec --bail --check-leaks test/
+react-helmet,karma start karma.config.js
+@ardatan/sync-fetch,mocha --throw-deprecation test/spec.js
+to-absolute-glob,mocha
+glogg,nyc mocha --async-only
+section-matter,mocha
+unique-stream,mocha
+align-text,mocha
+swagger-ui-express,./node_modules/.bin/mocha test/*.spec.js
+gulplog,nyc mocha --async-only
+broadcast-channel,"echo ""RUN ALL:"" && npm run test:node && npm run test:browser && npm run test:e2e && npm run test:modules"
+css-vendor,cross-env NODE_ENV=test karma start --single-run 
+@ethersproject/bytes,npm run test-esm
+@ethersproject/address,npm run test-esm
+@ethersproject/logger,npm run test-esm
+blob,mocha
+center-align,mocha
+right-align,mocha
+@ethersproject/bignumber,npm run test-esm
+@ethersproject/keccak256,npm run test-esm
+cwd,mocha
+@parcel/watcher-linux-arm64-glibc,mocha
+@parcel/watcher-win32-x64,mocha
+karma-coverage,mocha
+@ethersproject/rlp,npm run test-esm
+wkx,jshint . && nyc mocha
+@parcel/watcher-linux-arm64-musl,mocha
+@ethersproject/abstract-signer,npm run test-esm
+@ethersproject/strings,npm run test-esm
+@ethersproject/constants,npm run test-esm
+sync-fetch,mocha --throw-deprecation test/spec.js
+@ethersproject/transactions,npm run test-esm
+fontkit,run-s build mocha
+@ethersproject/hash,npm run test-esm
+@ethersproject/properties,npm run test-esm
+@ethersproject/networks,npm run test-esm
+multipipe,prettier-standard '**/*.js' 'test/*.js' && standard && mocha --reporter spec --timeout 300
+@ethersproject/signing-key,npm run test-esm
+chai-as-promised,c8 mocha
+@ethersproject/web,npm run test-esm
+gulp,nyc mocha --async-only
+glob-watcher,nyc mocha test test/lib --async-only
+@ethersproject/abstract-provider,npm run test-esm
+@sinonjs/formatio,mocha 'lib/**/*.test.js'
+@ethersproject/base64,npm run test-esm
+stream-to-array,mocha --reporter spec --bail
+react-modal,cross-env NODE_ENV=test karma start
+@parcel/watcher-darwin-arm64,mocha
+axios-mock-adapter,mocha && npm run clean && npm run build:umd:min && npm run test:types
+cz-conventional-changelog,mocha *.test.js
+json-pointer,nyc make test
+async-done,nyc mocha --async-only
+has-glob,mocha
+@parcel/watcher-darwin-x64,mocha
+gulp-cli,mocha --async-only --timeout 5000 test/lib test
+semver-greatest-satisfied-range,nyc mocha --async-only
+merge-deep,mocha
+bach,nyc mocha --async-only
+async-settle,nyc mocha --async-only
+@parcel/watcher-win32-arm64,mocha
+each-props,nyc mocha --async-only
+@parcel/watcher-linux-arm-glibc,mocha
+@parcel/watcher-android-arm64,mocha
+@parcel/watcher-win32-ia32,mocha
+is2,./node_modules/.bin/mocha -C --reporter list tests.js

package-lock.json

@@ -22,7 +22,9 @@
         "find-package-json": "^1.2.0",
         "git-client": "^1.9.3",
         "glob": "^11.0.3",
+        "glob-to-regexp": "^0.4.1",
         "jalangi2": "^0.2.6",
+        "minimatch": "^10.0.3",
         "semver": "^7.7.2",
         "set.prototype.intersection": "^1.1.7",
         "slice-js": "file:/home/atreyab/Documents/Docs/SlicingImport/repos-js/slice-js",

package.json

@@ -24,7 +24,9 @@
     "find-package-json": "^1.2.0",
     "git-client": "^1.9.3",
     "glob": "^11.0.3",
+    "glob-to-regexp": "^0.4.1",
     "jalangi2": "^0.2.6",
+    "minimatch": "^10.0.3",
     "semver": "^7.7.2",
     "set.prototype.intersection": "^1.1.7",
     "slice-js": "file:/home/atreyab/Documents/Docs/SlicingImport/repos-js/slice-js",

src/libcalls.mjs

@@ -80,7 +80,11 @@ export class LibraryTypesRecorder {
             return literalValue;
         } else if (type.isUndefined()) {
             return undefined;
-        } else if (type.isString()) {
+        } else if(type.isNull()){
+            return null;
+        } else if(type.isBigInt()){
+            return simpleFaker.number.bigInt();
+        }else if (type.isString()) {
             
             return simpleFaker.string.alphanumeric();
         } else if (type.isNumber()) {

src_dataset/FILTER_LIST.mjs

@@ -1,10 +1,10 @@
-export const FILTER_LIST = [
+import GlobToRegExp from "glob-to-regexp";
+
+const FILTER_LIST = [
+    "https://github.com/substack/*",
     "https://gitlab.com/contexttesting/zoroaster.git",
     "https://github.com/Eternity-Bots",
     "https://github.com/node-x-extras/x-path",
-    "https://github.com/substack/node-x256",
-    "https://github.com/substack/node-wordwrap",
-
     "https://github.com/zkochan/packages/blob/main/which-pm-runs",
     "https://github.com/webpack-contrib/webpack-addons",
     "https://github.com/zznoillusion1026/MyImage",
@@ -15,23 +15,34 @@ export const FILTER_LIST = [
     "https://github.com/nodelib/nodelib/tree/master/packages/fs/fs.scandir",
     "https://github.com/nodelib/nodelib/tree/master/packages/fs/fs.walk",
     "https://github.com/nodelib/nodelib/tree/master/packages/fs/fs.macchiato",
-    "https://github.com/substack/text-table",
-    "https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-object-rest-spread",
-    "https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-optional-catch-binding",
-    "https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-async-generators",
-    "https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-optional-chaining",
-    "https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-json-strings",
-    "https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-nullish-coalescing-operator",
-    "https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-bigint",
-    "https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-dynamic-import",
-    "https://github.com/substack/node-commondir",
-    "https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-export-namespace-from",
-    "https://github.com/substack/https-browserify",
-    "https://github.com/babel/babel/tree/master/packages/babel-runtime",
     "https://github.com/paulmillr/async-each",
     "https://github.com/yarnpkg/yarn/blob/master/packages",
-    "https://github.com/substack/semver-compare",
+    "https://github.com/emotion-js/emotion/tree/master/packages/stylis",
-    "https://github.com/substack/node-archy",
+    "https://github.com/kogosoftwarellc/open-api/tree/master/packages/openapi-types",
-    "https://github.com/substack/github-from-package",
+    "https://github.com/thenativeweb/boolean",
-    "https://github.com/babel/babel/tree/master/packages/babel-core"
+    "https://github.com/zkochan/packages/tree/master/read-yaml-file",
+    "https://github.com/johnotander/rgba-regex",
+    "https://github.com/adobe/react-spectrum/tree/main/packages/@internationalized/date",
+    "https://github.com/pnpm/pnpm/blob/main/packages",
+    "https://github.com/jhermsmeier/node-scuid",
+    "https://github.com/emotion-js/emotion/tree/master/packages/babel-plugin-emotion",
+    "https://github.com/emotion-js/emotion/tree/master/removed-packages/core",
+    "https://github.com/babel/babel/tree/master/packages/*",
+    "https://github.com/pugjs/pug/tree/master/packages/*",
+    "https://github.com/zkochan/packages/tree/master/*",
+    "https://github.com/Marak/Faker.js",
+    "https://github.com/ethanent/phin",
+    "https://github.com/Popmotion/popmotion/tree/master/packages/*",
+    "https://github.com/gulpjs/copy-prop"
 ];
+
+const FILTER_LIST_REGEX = FILTER_LIST.map(GlobToRegExp)
+
+/**
+ * 
+ * @param {string} repoUrl 
+ * @returns 
+ */
+export function matchFilterList(repoUrl) {
+    return FILTER_LIST_REGEX.some(filter => filter.test(repoUrl));
+}

src_dataset/index.mjs

@@ -30,9 +30,9 @@ const intermediateRepoList = await cacheFunctionOutput('repos.json', async funct
 // const packageMap = new Map(packageList)
 
 console.log(`Total repos`,intermediateRepoList.length)
-const intermediateRepoListSmaller = intermediateRepoList.slice(0,4000);
+const intermediateRepoListSmaller = intermediateRepoList.slice(0,5000);
 
-const repoStatus = await processPromisesBatch(intermediateRepoListSmaller,20,cloneRepoAndCheck)
+const repoStatus = await processPromisesBatch(intermediateRepoListSmaller,15,cloneRepoAndCheck)
 
 const repoStatusString = csv.stringify(repoStatus);
 await fsp.writeFile('repostatus.csv', repoStatusString);

src_dataset/mine.mjs

@@ -3,7 +3,7 @@ import { lstat, readFile,rm } from 'fs/promises'
 import git from 'git-client'
 import { resolve } from 'path'
 import int from 'set.prototype.intersection';
-import { FILTER_LIST } from './FILTER_LIST.mjs';
+import { matchFilterList } from './FILTER_LIST.mjs';
 /**
  * 
  * @param {[string,string,number]} param0 
@@ -12,7 +12,7 @@ import { FILTER_LIST } from './FILTER_LIST.mjs';
 export async function cloneRepoAndCheck([repoName, repoGitUrl, downloadCount]) {
     const repoPath = resolve('../cache-repos/repos', repoName)
 
-    if (FILTER_LIST.includes(repoGitUrl)) {
+    if (filterRepo(repoGitUrl)) {
         console.log("[git] ignoring ", repoName)
         return [repoName, null]
     };
@@ -44,6 +44,12 @@ export async function cloneRepoAndCheck([repoName, repoGitUrl, downloadCount]) {
     }
     else return [repoName, null]
 }
+
+
+function filterRepo(repoGitUrl) {
+    return matchFilterList(repoGitUrl);
+}
+
 function hasAnyActualDependencies(packageJSONContents, repoName) {
     if (packageJSONContents.dependencies !== undefined && Object.keys(packageJSONContents.dependencies).length > 0) {
         return true;

test/filterlist.spec.js

@@ -0,0 +1,105 @@
+import { assert, expect } from "chai";
+import { matchFilterList } from "../src_dataset/FILTER_LIST.mjs";
+
+describe('Filter List', function (){
+    it('should not match URL in filter list',function(){
+        const repoUrls = ["",
+            "https://github.com/babel/babel/tree/master/packages",
+            "https://github.com/babel/babel/tree/master/",
+            "https://github.com/substact/node-x256"
+        ];
+        const result = repoUrls.map(e=> matchFilterList(e));
+        result.forEach((e,i)=>{
+            assert.isFalse(e, `Expected ${repoUrls[i]} to not match filter list`);
+        })
+    })
+
+    it('should not match old URL filter list',function(){
+        const OLD_FILTER_LIST = [
+            "https://gitlab.com/contexttesting/zoroaster.git",
+            "https://github.com/Eternity-Bots",
+            "https://github.com/node-x-extras/x-path",
+            "https://github.com/substack/node-x256",
+            "https://github.com/substack/node-wordwrap",
+            "https://github.com/zkochan/packages/blob/main/which-pm-runs",
+            "https://github.com/webpack-contrib/webpack-addons",
+            "https://github.com/zznoillusion1026/MyImage",
+            "https://codehub.devcloud.huaweicloud.com/jsztxm00001/zzb-vue-ui.git",
+            "https://github.com/DZSF",
+            "https://github.com/chuzhixin/zx-count",
+            "https://github.com/nodelib/nodelib/tree/master/packages/fs/fs.stat",
+            "https://github.com/nodelib/nodelib/tree/master/packages/fs/fs.scandir",
+            "https://github.com/nodelib/nodelib/tree/master/packages/fs/fs.walk",
+            "https://github.com/nodelib/nodelib/tree/master/packages/fs/fs.macchiato",
+            "https://github.com/substack/text-table",
+            "https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-object-rest-spread",
+            "https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-optional-catch-binding",
+            "https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-async-generators",
+            "https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-optional-chaining",
+            "https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-json-strings",
+            "https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-nullish-coalescing-operator",
+            "https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-bigint",
+            "https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-dynamic-import",
+            "https://github.com/substack/node-commondir",
+            "https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-export-namespace-from",
+            "https://github.com/substack/https-browserify",
+            "https://github.com/babel/babel/tree/master/packages/babel-runtime",
+            "https://github.com/paulmillr/async-each",
+            "https://github.com/yarnpkg/yarn/blob/master/packages",
+            "https://github.com/substack/semver-compare",
+            "https://github.com/substack/node-archy",
+            "https://github.com/substack/github-from-package",
+            "https://github.com/babel/babel/tree/master/packages/babel-core",
+            "https://github.com/babel/babel/tree/master/packages/babel-code-frame",
+            "https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-trailing-function-commas",
+            "https://github.com/emotion-js/emotion/tree/master/packages/stylis",
+            "https://github.com/babel/babel/tree/master/packages/babel-types",
+            "https://github.com/babel/babel/tree/master/packages/babel-plugin-syntax-jsx",
+            "https://github.com/babel/babel/tree/master/packages/babel-messages",
+            "https://github.com/substack/node-chainsaw",
+            "https://github.com/substack/node-buffers",
+            "https://github.com/babel/babel/tree/master/packages/babel-traverse",
+            "https://github.com/substack/node-binary",
+            "https://github.com/substack/stream-combiner2",
+            "https://github.com/kogosoftwarellc/open-api/tree/master/packages/openapi-types",
+            "https://github.com/babel/babel/tree/master/packages/babel-template",
+            "https://github.com/substack/node-optimist",
+            "https://github.com/thenativeweb/boolean",
+            "https://github.com/zkochan/packages/tree/master/read-yaml-file",
+            "https://github.com/babel/babel/tree/master/packages/babel-generator",
+            "https://github.com/johnotander/rgba-regex",
+            "https://github.com/adobe/react-spectrum/tree/main/packages/@internationalized/date",
+            "https://github.com/pnpm/pnpm/blob/main/packages",
+            "https://github.com/jhermsmeier/node-scuid",
+            "https://github.com/emotion-js/emotion/tree/master/packages/babel-plugin-emotion",
+            "https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-es2015-modules-commonjs",
+            "https://github.com/babel/babel/tree/master/packages/babel-plugin-transform-strict-mode",
+            "https://github.com/babel/babel/tree/master/packages/babel-register",
+            "https://github.com/babel/babel/tree/master/packages/babel-helpers",
+            "https://github.com/emotion-js/emotion/tree/master/removed-packages/core",
+            "https://github.com/babel/babel/tree/master/packages/babel-helper-get-function-arity",
+            "https://github.com/babel/babel/tree/master/packages/babel-helper-function-name",
+            "https://github.com/babel/babel/tree/master/packages/*"
+        ];
+        const result = OLD_FILTER_LIST.map(e=> matchFilterList(e));
+        result.forEach((e,i)=>{
+            assert.isTrue(e, `Expected ${OLD_FILTER_LIST[i]} to not match filter list`);
+        })
+    })
+
+
+    it('should match URL in filter list',function(){
+        const repoUrl = ["https://github.com/babel/babel/tree/master/packages/babel-helpers"];
+        const result = matchFilterList(repoUrl);
+        assert.isTrue(result);
+    })
+
+    it('should match URL in filter list',function(){
+        const repoUrls = ["https://github.com/babel/babel/tree/master/packages/babel-helper",
+            "https://github.com/babel/babel/tree/master/packages/babel-helper",
+            "https://github.com/babel/babel/tree/master/packages/something"
+        ];
+        const result = repoUrls.map(e=> matchFilterList(e));
+        assert.isTrue(result.every(e=>e));
+    })
+});