[update] docs

README.md

@@ -12,4 +12,39 @@
 | `src_dataset` | Source for batch cloning public repositories |
 | `test_src`    | Example targets for development              |
 | `output`      | Webpack-ed dependencies                      |
-| `dist`        | Sliced dependencies                          |
+| `dist`        | Sliced dependencies                          |
+
+
+
+### How to rerun the program
+
+1. Collect the mineable repositories
+
+    ```bash
+    node src_dataset/index.mjs
+    ```
+
+    Look at the code and you can tweak the number of repos that are being used.
+    
+    They will get copied to `../cache-repos/repos`.
+    `package.json` of each of the files are fetched from npm (so that we dont clone the whole repo) and cached in `../cache-repos`.
+2. Get a list of candidate repos that can slice.
+
+    ```bash
+    ./script.sh
+    ```
+
+    If something goes wrong, add it to the ignore list.
+    This also moves the repos into a `candidates/` folder.
+    It will create `success.txt` with a list of repos that work with slicing.
+
+3. Slice and get test results
+
+    Go through `success.txt`, and run `master.sh` with each repo.
+
+    `./master.sh cov` -> Get test coverage and pre-test count
+    `./master.sh exec` -> Slice and post coverage
+    `./master.sh cloc` -> Sliced statistics
+
+
+    Note that in some cases, the scripts may attempt to execute tests in the backup node modules folder (`.node_modules`). In this case, move it out, and run the tests individually. You can get the commands from the output from the master script (which calls the `script-placer.sh` script).

master.sh

@@ -1,7 +1,7 @@
 
 
 
-REPO_BASE="mime-types"
+REPO_BASE="des.js"
 
 REPO_FOLDER="../candidates-repos/$REPO_BASE"
 
@@ -27,12 +27,12 @@ echo "Input function: $INPUT_FUNCTION";
 case "$INPUT_FUNCTION" in
     "pre-test")
         pushd "$REPO_FOLDER"
-        read -p "Deleting old node_modules_2 directory?" choice
+        read -p "Deleting old .node_modules directory?" choice
         if [[ "$choice" == "y" || "$choice" == "Y" ]];
         then
-            rm -rf node_modules_2
+            rm -rf .node_modules node_modules_2
         else
-            echo "Skipping deletion of node_modules_2"
+            echo "Skipping deletion of .node_modules"
         fi
         npm i --silent || fail "Failed to install dependencies"
         npm run test
@@ -40,20 +40,20 @@ case "$INPUT_FUNCTION" in
         ;;
     "rollback")
         pushd "$REPO_FOLDER"
-        # Move node_modules_2 back to node_modules
+        # Move .node_modules back to node_modules
-        if [[ -d "node_modules_2" ]]; then
+        if [[ -d ".node_modules" ]]; then
-            echo "Restoring node_modules from node_modules_2"
+            echo "Restoring node_modules from .node_modules"
             rm -irf node_modules
-            mv node_modules_2 node_modules
+            mv .node_modules node_modules
         else
-            fail "node_modules_2 does not exist, cannot rollback"
+            fail ".node_modules does not exist, cannot rollback"
         fi
         popd
         ;;
     "indeps")
-        # Inactivate the repo by moving node_modules to node_modules_2
+        # Inactivate the repo by moving node_modules to .node_modules
         pushd "$REPO_FOLDER"
-        mv node_modules node_modules_2
+        mv node_modules .node_modules
         popd
         ;;
     "stat")
@@ -64,19 +64,29 @@ case "$INPUT_FUNCTION" in
         else
             echo "node_modules directory exists"
         fi
-        if [[ ! -d "node_modules_2" ]]; then
+        if [[ ! -d ".node_modules" ]]; then
-            echo "node_modules_2 directory does not exist"
+            echo ".node_modules directory does not exist"
             else
-            echo "node_modules_2 directory exists"
+            echo ".node_modules directory exists"
         fi
         popd >> /dev/null
         ;;
     "cov")
         pushd "$REPO_FOLDER"
+        rm -rf .node_modules node_modules_2
         npm i --silent || fail "Failed to install dependencies"
         npm i -D @types/node || fail "Failed to install node types"
         nyc npm run test 
         ;;
+    "slice")
+        pushd "$REPO_FOLDER"
+        rm -rf .node_modules node_modules_2
+        npm i --silent || fail "Failed to install dependencies"
+        npm i -D @types/node || fail "Failed to install node types"
+        
+        popd
+        node src/index.mjs "$REPO_FOLDER" || fail "Failed to execute src/index.mjs"
+        ;;
     "exec")
 
         ./script-placer.sh "$REPO_FOLDER"
@@ -87,8 +97,16 @@ case "$INPUT_FUNCTION" in
             echo "script-placer.sh executed successfully."
         fi
         ;;
+    "cloc")
+        echo "--: PRE :--"
+        cloc "output/$REPO_BASE"
+        echo "--: POST :--"
+        cloc "dist/$REPO_BASE" --exclude-ext=json
+        echo "--: DEPS :--"
+        echo "Dependencies: " `ls -1 dist/$REPO_BASE | wc -l`
+        ;;
     *)
-        fail "Unknown function: $INPUT_FUNCTION. Supported functions: test, build."
+        fail "Unknown function: $INPUT_FUNCTION. Supported functions: pre-test, rollback, indeps, stat, cov, exec, sloc"
         ;;
 
 esac

script-placer-successes.sh

@@ -1,11 +1,237 @@
 #! /bin/bash
 
+fail() {
+    echo "Error: $1"
+    exit "${2-1}" ## Return a code specified by $2, or 1 by default.
+}
 ## Run the script-placer.sh for each repository in the success.txt file
+IGNORE_REPOS=( 
+"glob-parent"
+"is-glob"
+"mime-types"
+"doctrine"
+"define-property"
+"jsonfile"
+"optionator"
+"http-errors"
+"type-check"
+"levn"
+"is-extendable"
+"esquery"
+"on-finished"
+"error-ex"
+"finalhandler"
+"content-disposition"
+"terser"
+"proxy-addr"
+"prop-types"
+"has-values"
+"has-value"
+"url-parse"
+"simple-swizzle"
+"compressible"
+"global-modules"
+"thenify"
+"mz"
+"clone-deep"
+"shallow-clone"
+"prettier-linter-helpers"
+"cors"
+"fd-slicer"
+"object.pick"
+"language-tags"
+"union-value"
+"object-copy"
+"static-extend"
+"memory-fs"
+"pinkie-promise"
+"hash.js"
+"pretty-error"
+"renderkid"
+"wbuf"
+"browserify-zlib"
+"hmac-drbg"
+"des.js"
+"dom-converter"
+"expand-tilde"
+"homedir-polyfill"
+"zip-stream"
+"crc32-stream"
+"one-time"
+"resolve-dir"
+"yargs-unparser"
+"# warning"
+"bplist-parser"
+"for-own"
+"md5"
+"is-relative"
+"is-absolute"
+"is-unc-path"
+"redis-parser"
+"recursive-readdir"
+"path-root"
+"lazy-cache"
+"# css-to-react-native"
+"parse-filepath"
+"request-progress"
+"jest-junit"
+"postcss-initial"
+"unixify"
+"cookie-parser"
+"saslprep"
+"window-size"
+"cookies"
+"keygrip"
+"contains-path"
+"fined"
+"object.defaults"
+"is-color-stop"
+"gonzales-pe"
+"bufferutil"
+"make-iterator"
+"glob-base"
+"uid-safe"
+"fancy-log"
+"object.map"
+"object.omit"
+"find-babel-config"
+"utf-8-validate"
+"mquery"
+"xlsx"
+"json-to-pretty-yaml"
+"easy-table"
+"named-placeholders"
+"is-dotfile"
+"parse-glob"
+"plugin-error"
+"is-equal-shallow"
+"original"
+"detective-typescript"
+"detective-es6"
+"json2mq"
+"create-error-class"
+"detective-cjs"
+"to-through"
+"resolve-options"
+"ansi-gray"
+"bcrypt"
+"mixin-object"
+"optimize-css-assets-webpack-plugin"
+"ordered-read-streams"
+"sync-fetch"
+"to-absolute-glob"
+"glogg"
+"unique-stream"
+"align-text"
+"gulplog"
+"blob"
+"center-align"
+"right-align"
+"wkx"
+"chai-as-promised"
+"json-pointer"
+"has-glob"
+"# promptly"
+"hot-shots"
+"semver-greatest-satisfied-range"
+"each-props"
+"is2"
+"levenary"
+"airbnb-prop-types"
+"remove-bom-stream"
+"remove-bom-buffer"
+"dotenv-defaults"
+"rework"
+"vizion"
+"array-sort"
+"default-compare"
+"pad-right"
+"passport-local"
+"console.table"
+"cli-tableau"
+"condense-newlines"
+"requireg"
+"object.reduce"
+"properties-reader"
+"array-initial"
+"default-resolution"
+"collection-map"
+"ansi-red"
+"broccoli-merge-trees"
+"eslint-plugin-react-native"
+"is-valid-path"
+"strip-hex-prefix"
+"uglify-es"
+"ansi-cyan"
+"method-override"
+"# readline2"
+"number-allocator"
+"has-gulplog"
+"ethjs-util"
+"unescape"
+"validate.io-integer"
+"stream-parser"
+"compute-gcd"
+"validate.io-integer-array"
+"compute-lcm"
+"set-getter"
+"passport-oauth2"
+"i18n-iso-countries"
+"sha1"
+"json-diff"
+"dreamopt"
+"highlight-es"
+"basic-auth-connect"
+"glob2base"
+"third-party-capital"
+"new-date"
+"webrtc-adapter"
+"xhr-request-promise"
+"contentful-resolve-response"
+"jest-sonar-reporter"
+"parse-author"
+"amd-name-resolver"
+"mocha-multi-reporters"
+"eslint-plugin-filenames"
+"apache-crypt"
+"semver-intersect"
+"fetch-ponyfill"
+"karma-mocha"
+"is-odd"
+"babel-plugin-ember-modules-api-polyfill"
+"csurf"
+"taketalk"
+)
 
 while read -r repo; do
+    # if repo starts with a #, skip it
+    if [[ "${IGNORE_REPOS[@]}" =~ "$repo" ]]; then
+        echo "Skipping ignored repository: $repo"
+        continue
+    fi
+    if [[ "$repo" =~ ^# ]]; then
+        echo "Skipping comment line: $repo"
+        continue
+    fi
     if [[ -n "$repo" ]]; then
         # echo "Running script-placer.sh for repository: candidates/$repo"
+        if [[ ! -d "../candidates-repos/$repo" ]]; then
+            if [[ -d "candidates/$repo" ]]; then
+                echo "Repository $repo  exists in candidates, copying"
+                pushd "candidates/$repo" 
+                rm -rf node_modules
+                popd
+                echo "Copying $repo to candidates-repos"
+                cp -r "candidates/$repo" "../candidates-repos/$repo" || exit 1
+            else
+                fail "Repository $repo does not exist in candidates or candidates-repos"
+            fi
+            continue
+        fi
+        echo "Analysing repository: $repo"
+        echo "$repo" > current-processing.log
         ./script-placer.sh "../candidates-repos/$repo" || echo "Failed to process $repo"
+        echo "Finished processing repository: $repo"
     else
         echo "Skipping empty repository entry"
     fi