diff --git a/README.md b/README.md index cc5b026..94ea9b4 100644 --- a/README.md +++ b/README.md @@ -12,4 +12,39 @@ | `src_dataset` | Source for batch cloning public repositories | | `test_src` | Example targets for development | | `output` | Webpack-ed dependencies | -| `dist` | Sliced dependencies | \ No newline at end of file +| `dist` | Sliced dependencies | + + + +### How to rerun the program + +1. Collect the mineable repositories + + ```bash + node src_dataset/index.mjs + ``` + + Look at the code and you can tweak the number of repos that are being used. + + They will get copied to `../cache-repos/repos`. + `package.json` of each of the files are fetched from npm (so that we dont clone the whole repo) and cached in `../cache-repos`. +2. Get a list of candidate repos that can slice. + + ```bash + ./script.sh + ``` + + If something goes wrong, add it to the ignore list. + This also moves the repos into a `candidates/` folder. + It will create `success.txt` with a list of repos that work with slicing. + +3. Slice and get test results + + Go through `success.txt`, and run `master.sh` with each repo. + + `./master.sh cov` -> Get test coverage and pre-test count + `./master.sh exec` -> Slice and post coverage + `./master.sh cloc` -> Sliced statistics + + + Note that in some cases, the scripts may attempt to execute tests in the backup node modules folder (`.node_modules`). In this case, move it out, and run the tests individually. You can get the commands from the output from the master script (which calls the `script-placer.sh` script). \ No newline at end of file diff --git a/master.sh b/master.sh index 41cae8c..592df49 100755 --- a/master.sh +++ b/master.sh @@ -1,7 +1,7 @@ -REPO_BASE="mime-types" +REPO_BASE="des.js" REPO_FOLDER="../candidates-repos/$REPO_BASE" @@ -27,12 +27,12 @@ echo "Input function: $INPUT_FUNCTION"; case "$INPUT_FUNCTION" in "pre-test") pushd "$REPO_FOLDER" - read -p "Deleting old node_modules_2 directory?" choice + read -p "Deleting old .node_modules directory?" choice if [[ "$choice" == "y" || "$choice" == "Y" ]]; then - rm -rf node_modules_2 + rm -rf .node_modules node_modules_2 else - echo "Skipping deletion of node_modules_2" + echo "Skipping deletion of .node_modules" fi npm i --silent || fail "Failed to install dependencies" npm run test @@ -40,20 +40,20 @@ case "$INPUT_FUNCTION" in ;; "rollback") pushd "$REPO_FOLDER" - # Move node_modules_2 back to node_modules - if [[ -d "node_modules_2" ]]; then - echo "Restoring node_modules from node_modules_2" + # Move .node_modules back to node_modules + if [[ -d ".node_modules" ]]; then + echo "Restoring node_modules from .node_modules" rm -irf node_modules - mv node_modules_2 node_modules + mv .node_modules node_modules else - fail "node_modules_2 does not exist, cannot rollback" + fail ".node_modules does not exist, cannot rollback" fi popd ;; "indeps") - # Inactivate the repo by moving node_modules to node_modules_2 + # Inactivate the repo by moving node_modules to .node_modules pushd "$REPO_FOLDER" - mv node_modules node_modules_2 + mv node_modules .node_modules popd ;; "stat") @@ -64,19 +64,29 @@ case "$INPUT_FUNCTION" in else echo "node_modules directory exists" fi - if [[ ! -d "node_modules_2" ]]; then - echo "node_modules_2 directory does not exist" + if [[ ! -d ".node_modules" ]]; then + echo ".node_modules directory does not exist" else - echo "node_modules_2 directory exists" + echo ".node_modules directory exists" fi popd >> /dev/null ;; "cov") pushd "$REPO_FOLDER" + rm -rf .node_modules node_modules_2 npm i --silent || fail "Failed to install dependencies" npm i -D @types/node || fail "Failed to install node types" nyc npm run test ;; + "slice") + pushd "$REPO_FOLDER" + rm -rf .node_modules node_modules_2 + npm i --silent || fail "Failed to install dependencies" + npm i -D @types/node || fail "Failed to install node types" + + popd + node src/index.mjs "$REPO_FOLDER" || fail "Failed to execute src/index.mjs" + ;; "exec") ./script-placer.sh "$REPO_FOLDER" @@ -87,8 +97,16 @@ case "$INPUT_FUNCTION" in echo "script-placer.sh executed successfully." fi ;; + "cloc") + echo "--: PRE :--" + cloc "output/$REPO_BASE" + echo "--: POST :--" + cloc "dist/$REPO_BASE" --exclude-ext=json + echo "--: DEPS :--" + echo "Dependencies: " `ls -1 dist/$REPO_BASE | wc -l` + ;; *) - fail "Unknown function: $INPUT_FUNCTION. Supported functions: test, build." + fail "Unknown function: $INPUT_FUNCTION. Supported functions: pre-test, rollback, indeps, stat, cov, exec, sloc" ;; esac \ No newline at end of file diff --git a/script-placer-successes.sh b/script-placer-successes.sh index 460e73f..88e7bad 100755 --- a/script-placer-successes.sh +++ b/script-placer-successes.sh @@ -1,11 +1,237 @@ #! /bin/bash +fail() { + echo "Error: $1" + exit "${2-1}" ## Return a code specified by $2, or 1 by default. +} ## Run the script-placer.sh for each repository in the success.txt file +IGNORE_REPOS=( +"glob-parent" +"is-glob" +"mime-types" +"doctrine" +"define-property" +"jsonfile" +"optionator" +"http-errors" +"type-check" +"levn" +"is-extendable" +"esquery" +"on-finished" +"error-ex" +"finalhandler" +"content-disposition" +"terser" +"proxy-addr" +"prop-types" +"has-values" +"has-value" +"url-parse" +"simple-swizzle" +"compressible" +"global-modules" +"thenify" +"mz" +"clone-deep" +"shallow-clone" +"prettier-linter-helpers" +"cors" +"fd-slicer" +"object.pick" +"language-tags" +"union-value" +"object-copy" +"static-extend" +"memory-fs" +"pinkie-promise" +"hash.js" +"pretty-error" +"renderkid" +"wbuf" +"browserify-zlib" +"hmac-drbg" +"des.js" +"dom-converter" +"expand-tilde" +"homedir-polyfill" +"zip-stream" +"crc32-stream" +"one-time" +"resolve-dir" +"yargs-unparser" +"# warning" +"bplist-parser" +"for-own" +"md5" +"is-relative" +"is-absolute" +"is-unc-path" +"redis-parser" +"recursive-readdir" +"path-root" +"lazy-cache" +"# css-to-react-native" +"parse-filepath" +"request-progress" +"jest-junit" +"postcss-initial" +"unixify" +"cookie-parser" +"saslprep" +"window-size" +"cookies" +"keygrip" +"contains-path" +"fined" +"object.defaults" +"is-color-stop" +"gonzales-pe" +"bufferutil" +"make-iterator" +"glob-base" +"uid-safe" +"fancy-log" +"object.map" +"object.omit" +"find-babel-config" +"utf-8-validate" +"mquery" +"xlsx" +"json-to-pretty-yaml" +"easy-table" +"named-placeholders" +"is-dotfile" +"parse-glob" +"plugin-error" +"is-equal-shallow" +"original" +"detective-typescript" +"detective-es6" +"json2mq" +"create-error-class" +"detective-cjs" +"to-through" +"resolve-options" +"ansi-gray" +"bcrypt" +"mixin-object" +"optimize-css-assets-webpack-plugin" +"ordered-read-streams" +"sync-fetch" +"to-absolute-glob" +"glogg" +"unique-stream" +"align-text" +"gulplog" +"blob" +"center-align" +"right-align" +"wkx" +"chai-as-promised" +"json-pointer" +"has-glob" +"# promptly" +"hot-shots" +"semver-greatest-satisfied-range" +"each-props" +"is2" +"levenary" +"airbnb-prop-types" +"remove-bom-stream" +"remove-bom-buffer" +"dotenv-defaults" +"rework" +"vizion" +"array-sort" +"default-compare" +"pad-right" +"passport-local" +"console.table" +"cli-tableau" +"condense-newlines" +"requireg" +"object.reduce" +"properties-reader" +"array-initial" +"default-resolution" +"collection-map" +"ansi-red" +"broccoli-merge-trees" +"eslint-plugin-react-native" +"is-valid-path" +"strip-hex-prefix" +"uglify-es" +"ansi-cyan" +"method-override" +"# readline2" +"number-allocator" +"has-gulplog" +"ethjs-util" +"unescape" +"validate.io-integer" +"stream-parser" +"compute-gcd" +"validate.io-integer-array" +"compute-lcm" +"set-getter" +"passport-oauth2" +"i18n-iso-countries" +"sha1" +"json-diff" +"dreamopt" +"highlight-es" +"basic-auth-connect" +"glob2base" +"third-party-capital" +"new-date" +"webrtc-adapter" +"xhr-request-promise" +"contentful-resolve-response" +"jest-sonar-reporter" +"parse-author" +"amd-name-resolver" +"mocha-multi-reporters" +"eslint-plugin-filenames" +"apache-crypt" +"semver-intersect" +"fetch-ponyfill" +"karma-mocha" +"is-odd" +"babel-plugin-ember-modules-api-polyfill" +"csurf" +"taketalk" +) while read -r repo; do + # if repo starts with a #, skip it + if [[ "${IGNORE_REPOS[@]}" =~ "$repo" ]]; then + echo "Skipping ignored repository: $repo" + continue + fi + if [[ "$repo" =~ ^# ]]; then + echo "Skipping comment line: $repo" + continue + fi if [[ -n "$repo" ]]; then # echo "Running script-placer.sh for repository: candidates/$repo" + if [[ ! -d "../candidates-repos/$repo" ]]; then + if [[ -d "candidates/$repo" ]]; then + echo "Repository $repo exists in candidates, copying" + pushd "candidates/$repo" + rm -rf node_modules + popd + echo "Copying $repo to candidates-repos" + cp -r "candidates/$repo" "../candidates-repos/$repo" || exit 1 + else + fail "Repository $repo does not exist in candidates or candidates-repos" + fi + continue + fi + echo "Analysing repository: $repo" + echo "$repo" > current-processing.log ./script-placer.sh "../candidates-repos/$repo" || echo "Failed to process $repo" + echo "Finished processing repository: $repo" else echo "Skipping empty repository entry" fi