[update] docs
This commit is contained in:
35
README.md
35
README.md
@@ -13,3 +13,38 @@
|
||||
| `test_src` | Example targets for development |
|
||||
| `output` | Webpack-ed dependencies |
|
||||
| `dist` | Sliced dependencies |
|
||||
|
||||
|
||||
|
||||
### How to rerun the program
|
||||
|
||||
1. Collect the mineable repositories
|
||||
|
||||
```bash
|
||||
node src_dataset/index.mjs
|
||||
```
|
||||
|
||||
Look at the code and you can tweak the number of repos that are being used.
|
||||
|
||||
They will get copied to `../cache-repos/repos`.
|
||||
`package.json` of each of the files are fetched from npm (so that we dont clone the whole repo) and cached in `../cache-repos`.
|
||||
2. Get a list of candidate repos that can slice.
|
||||
|
||||
```bash
|
||||
./script.sh
|
||||
```
|
||||
|
||||
If something goes wrong, add it to the ignore list.
|
||||
This also moves the repos into a `candidates/` folder.
|
||||
It will create `success.txt` with a list of repos that work with slicing.
|
||||
|
||||
3. Slice and get test results
|
||||
|
||||
Go through `success.txt`, and run `master.sh` with each repo.
|
||||
|
||||
`./master.sh cov` -> Get test coverage and pre-test count
|
||||
`./master.sh exec` -> Slice and post coverage
|
||||
`./master.sh cloc` -> Sliced statistics
|
||||
|
||||
|
||||
Note that in some cases, the scripts may attempt to execute tests in the backup node modules folder (`.node_modules`). In this case, move it out, and run the tests individually. You can get the commands from the output from the master script (which calls the `script-placer.sh` script).
|
||||
48
master.sh
48
master.sh
@@ -1,7 +1,7 @@
|
||||
|
||||
|
||||
|
||||
REPO_BASE="mime-types"
|
||||
REPO_BASE="des.js"
|
||||
|
||||
REPO_FOLDER="../candidates-repos/$REPO_BASE"
|
||||
|
||||
@@ -27,12 +27,12 @@ echo "Input function: $INPUT_FUNCTION";
|
||||
case "$INPUT_FUNCTION" in
|
||||
"pre-test")
|
||||
pushd "$REPO_FOLDER"
|
||||
read -p "Deleting old node_modules_2 directory?" choice
|
||||
read -p "Deleting old .node_modules directory?" choice
|
||||
if [[ "$choice" == "y" || "$choice" == "Y" ]];
|
||||
then
|
||||
rm -rf node_modules_2
|
||||
rm -rf .node_modules node_modules_2
|
||||
else
|
||||
echo "Skipping deletion of node_modules_2"
|
||||
echo "Skipping deletion of .node_modules"
|
||||
fi
|
||||
npm i --silent || fail "Failed to install dependencies"
|
||||
npm run test
|
||||
@@ -40,20 +40,20 @@ case "$INPUT_FUNCTION" in
|
||||
;;
|
||||
"rollback")
|
||||
pushd "$REPO_FOLDER"
|
||||
# Move node_modules_2 back to node_modules
|
||||
if [[ -d "node_modules_2" ]]; then
|
||||
echo "Restoring node_modules from node_modules_2"
|
||||
# Move .node_modules back to node_modules
|
||||
if [[ -d ".node_modules" ]]; then
|
||||
echo "Restoring node_modules from .node_modules"
|
||||
rm -irf node_modules
|
||||
mv node_modules_2 node_modules
|
||||
mv .node_modules node_modules
|
||||
else
|
||||
fail "node_modules_2 does not exist, cannot rollback"
|
||||
fail ".node_modules does not exist, cannot rollback"
|
||||
fi
|
||||
popd
|
||||
;;
|
||||
"indeps")
|
||||
# Inactivate the repo by moving node_modules to node_modules_2
|
||||
# Inactivate the repo by moving node_modules to .node_modules
|
||||
pushd "$REPO_FOLDER"
|
||||
mv node_modules node_modules_2
|
||||
mv node_modules .node_modules
|
||||
popd
|
||||
;;
|
||||
"stat")
|
||||
@@ -64,19 +64,29 @@ case "$INPUT_FUNCTION" in
|
||||
else
|
||||
echo "node_modules directory exists"
|
||||
fi
|
||||
if [[ ! -d "node_modules_2" ]]; then
|
||||
echo "node_modules_2 directory does not exist"
|
||||
if [[ ! -d ".node_modules" ]]; then
|
||||
echo ".node_modules directory does not exist"
|
||||
else
|
||||
echo "node_modules_2 directory exists"
|
||||
echo ".node_modules directory exists"
|
||||
fi
|
||||
popd >> /dev/null
|
||||
;;
|
||||
"cov")
|
||||
pushd "$REPO_FOLDER"
|
||||
rm -rf .node_modules node_modules_2
|
||||
npm i --silent || fail "Failed to install dependencies"
|
||||
npm i -D @types/node || fail "Failed to install node types"
|
||||
nyc npm run test
|
||||
;;
|
||||
"slice")
|
||||
pushd "$REPO_FOLDER"
|
||||
rm -rf .node_modules node_modules_2
|
||||
npm i --silent || fail "Failed to install dependencies"
|
||||
npm i -D @types/node || fail "Failed to install node types"
|
||||
|
||||
popd
|
||||
node src/index.mjs "$REPO_FOLDER" || fail "Failed to execute src/index.mjs"
|
||||
;;
|
||||
"exec")
|
||||
|
||||
./script-placer.sh "$REPO_FOLDER"
|
||||
@@ -87,8 +97,16 @@ case "$INPUT_FUNCTION" in
|
||||
echo "script-placer.sh executed successfully."
|
||||
fi
|
||||
;;
|
||||
"cloc")
|
||||
echo "--: PRE :--"
|
||||
cloc "output/$REPO_BASE"
|
||||
echo "--: POST :--"
|
||||
cloc "dist/$REPO_BASE" --exclude-ext=json
|
||||
echo "--: DEPS :--"
|
||||
echo "Dependencies: " `ls -1 dist/$REPO_BASE | wc -l`
|
||||
;;
|
||||
*)
|
||||
fail "Unknown function: $INPUT_FUNCTION. Supported functions: test, build."
|
||||
fail "Unknown function: $INPUT_FUNCTION. Supported functions: pre-test, rollback, indeps, stat, cov, exec, sloc"
|
||||
;;
|
||||
|
||||
esac
|
||||
@@ -1,11 +1,237 @@
|
||||
#! /bin/bash
|
||||
|
||||
fail() {
|
||||
echo "Error: $1"
|
||||
exit "${2-1}" ## Return a code specified by $2, or 1 by default.
|
||||
}
|
||||
## Run the script-placer.sh for each repository in the success.txt file
|
||||
IGNORE_REPOS=(
|
||||
"glob-parent"
|
||||
"is-glob"
|
||||
"mime-types"
|
||||
"doctrine"
|
||||
"define-property"
|
||||
"jsonfile"
|
||||
"optionator"
|
||||
"http-errors"
|
||||
"type-check"
|
||||
"levn"
|
||||
"is-extendable"
|
||||
"esquery"
|
||||
"on-finished"
|
||||
"error-ex"
|
||||
"finalhandler"
|
||||
"content-disposition"
|
||||
"terser"
|
||||
"proxy-addr"
|
||||
"prop-types"
|
||||
"has-values"
|
||||
"has-value"
|
||||
"url-parse"
|
||||
"simple-swizzle"
|
||||
"compressible"
|
||||
"global-modules"
|
||||
"thenify"
|
||||
"mz"
|
||||
"clone-deep"
|
||||
"shallow-clone"
|
||||
"prettier-linter-helpers"
|
||||
"cors"
|
||||
"fd-slicer"
|
||||
"object.pick"
|
||||
"language-tags"
|
||||
"union-value"
|
||||
"object-copy"
|
||||
"static-extend"
|
||||
"memory-fs"
|
||||
"pinkie-promise"
|
||||
"hash.js"
|
||||
"pretty-error"
|
||||
"renderkid"
|
||||
"wbuf"
|
||||
"browserify-zlib"
|
||||
"hmac-drbg"
|
||||
"des.js"
|
||||
"dom-converter"
|
||||
"expand-tilde"
|
||||
"homedir-polyfill"
|
||||
"zip-stream"
|
||||
"crc32-stream"
|
||||
"one-time"
|
||||
"resolve-dir"
|
||||
"yargs-unparser"
|
||||
"# warning"
|
||||
"bplist-parser"
|
||||
"for-own"
|
||||
"md5"
|
||||
"is-relative"
|
||||
"is-absolute"
|
||||
"is-unc-path"
|
||||
"redis-parser"
|
||||
"recursive-readdir"
|
||||
"path-root"
|
||||
"lazy-cache"
|
||||
"# css-to-react-native"
|
||||
"parse-filepath"
|
||||
"request-progress"
|
||||
"jest-junit"
|
||||
"postcss-initial"
|
||||
"unixify"
|
||||
"cookie-parser"
|
||||
"saslprep"
|
||||
"window-size"
|
||||
"cookies"
|
||||
"keygrip"
|
||||
"contains-path"
|
||||
"fined"
|
||||
"object.defaults"
|
||||
"is-color-stop"
|
||||
"gonzales-pe"
|
||||
"bufferutil"
|
||||
"make-iterator"
|
||||
"glob-base"
|
||||
"uid-safe"
|
||||
"fancy-log"
|
||||
"object.map"
|
||||
"object.omit"
|
||||
"find-babel-config"
|
||||
"utf-8-validate"
|
||||
"mquery"
|
||||
"xlsx"
|
||||
"json-to-pretty-yaml"
|
||||
"easy-table"
|
||||
"named-placeholders"
|
||||
"is-dotfile"
|
||||
"parse-glob"
|
||||
"plugin-error"
|
||||
"is-equal-shallow"
|
||||
"original"
|
||||
"detective-typescript"
|
||||
"detective-es6"
|
||||
"json2mq"
|
||||
"create-error-class"
|
||||
"detective-cjs"
|
||||
"to-through"
|
||||
"resolve-options"
|
||||
"ansi-gray"
|
||||
"bcrypt"
|
||||
"mixin-object"
|
||||
"optimize-css-assets-webpack-plugin"
|
||||
"ordered-read-streams"
|
||||
"sync-fetch"
|
||||
"to-absolute-glob"
|
||||
"glogg"
|
||||
"unique-stream"
|
||||
"align-text"
|
||||
"gulplog"
|
||||
"blob"
|
||||
"center-align"
|
||||
"right-align"
|
||||
"wkx"
|
||||
"chai-as-promised"
|
||||
"json-pointer"
|
||||
"has-glob"
|
||||
"# promptly"
|
||||
"hot-shots"
|
||||
"semver-greatest-satisfied-range"
|
||||
"each-props"
|
||||
"is2"
|
||||
"levenary"
|
||||
"airbnb-prop-types"
|
||||
"remove-bom-stream"
|
||||
"remove-bom-buffer"
|
||||
"dotenv-defaults"
|
||||
"rework"
|
||||
"vizion"
|
||||
"array-sort"
|
||||
"default-compare"
|
||||
"pad-right"
|
||||
"passport-local"
|
||||
"console.table"
|
||||
"cli-tableau"
|
||||
"condense-newlines"
|
||||
"requireg"
|
||||
"object.reduce"
|
||||
"properties-reader"
|
||||
"array-initial"
|
||||
"default-resolution"
|
||||
"collection-map"
|
||||
"ansi-red"
|
||||
"broccoli-merge-trees"
|
||||
"eslint-plugin-react-native"
|
||||
"is-valid-path"
|
||||
"strip-hex-prefix"
|
||||
"uglify-es"
|
||||
"ansi-cyan"
|
||||
"method-override"
|
||||
"# readline2"
|
||||
"number-allocator"
|
||||
"has-gulplog"
|
||||
"ethjs-util"
|
||||
"unescape"
|
||||
"validate.io-integer"
|
||||
"stream-parser"
|
||||
"compute-gcd"
|
||||
"validate.io-integer-array"
|
||||
"compute-lcm"
|
||||
"set-getter"
|
||||
"passport-oauth2"
|
||||
"i18n-iso-countries"
|
||||
"sha1"
|
||||
"json-diff"
|
||||
"dreamopt"
|
||||
"highlight-es"
|
||||
"basic-auth-connect"
|
||||
"glob2base"
|
||||
"third-party-capital"
|
||||
"new-date"
|
||||
"webrtc-adapter"
|
||||
"xhr-request-promise"
|
||||
"contentful-resolve-response"
|
||||
"jest-sonar-reporter"
|
||||
"parse-author"
|
||||
"amd-name-resolver"
|
||||
"mocha-multi-reporters"
|
||||
"eslint-plugin-filenames"
|
||||
"apache-crypt"
|
||||
"semver-intersect"
|
||||
"fetch-ponyfill"
|
||||
"karma-mocha"
|
||||
"is-odd"
|
||||
"babel-plugin-ember-modules-api-polyfill"
|
||||
"csurf"
|
||||
"taketalk"
|
||||
)
|
||||
|
||||
while read -r repo; do
|
||||
# if repo starts with a #, skip it
|
||||
if [[ "${IGNORE_REPOS[@]}" =~ "$repo" ]]; then
|
||||
echo "Skipping ignored repository: $repo"
|
||||
continue
|
||||
fi
|
||||
if [[ "$repo" =~ ^# ]]; then
|
||||
echo "Skipping comment line: $repo"
|
||||
continue
|
||||
fi
|
||||
if [[ -n "$repo" ]]; then
|
||||
# echo "Running script-placer.sh for repository: candidates/$repo"
|
||||
if [[ ! -d "../candidates-repos/$repo" ]]; then
|
||||
if [[ -d "candidates/$repo" ]]; then
|
||||
echo "Repository $repo exists in candidates, copying"
|
||||
pushd "candidates/$repo"
|
||||
rm -rf node_modules
|
||||
popd
|
||||
echo "Copying $repo to candidates-repos"
|
||||
cp -r "candidates/$repo" "../candidates-repos/$repo" || exit 1
|
||||
else
|
||||
fail "Repository $repo does not exist in candidates or candidates-repos"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
echo "Analysing repository: $repo"
|
||||
echo "$repo" > current-processing.log
|
||||
./script-placer.sh "../candidates-repos/$repo" || echo "Failed to process $repo"
|
||||
echo "Finished processing repository: $repo"
|
||||
else
|
||||
echo "Skipping empty repository entry"
|
||||
fi
|
||||
|
||||
Reference in New Issue
Block a user