119 lines
4.4 KiB
JavaScript
119 lines
4.4 KiB
JavaScript
import { existsSync, } from 'fs'
|
|
import { lstat, readFile,rm } from 'fs/promises'
|
|
import git from 'git-client'
|
|
import { resolve } from 'path'
|
|
import int from 'set.prototype.intersection';
|
|
import { matchFilterList } from './FILTER_LIST.mjs';
|
|
import npmapi from 'npm-api'
|
|
import { cacheFunctionOutput } from './cache.mjs';
|
|
/**
|
|
*
|
|
* @param {[string,string,number]} param0
|
|
* @returns {Promise<[string,string|null]>} second argument is null if ineligible for slicing
|
|
*/
|
|
export async function cloneRepoAndCheck([repoName, repoGitUrl, downloadCount]) {
|
|
const repoPath = resolve('../cache-repos/repos', repoName)
|
|
|
|
if (filterRepo(repoGitUrl)) {
|
|
console.log("[git] ignoring ", repoName)
|
|
return [repoName, null]
|
|
};
|
|
// console.log('[git] fetching',repoName, repoGitUrl);
|
|
let api = new npmapi();
|
|
const repo = api.repo(repoName);
|
|
|
|
let packageJSONContentsString = null;
|
|
|
|
|
|
try{
|
|
packageJSONContentsString = await cacheFunctionOutput(`cache-repo-package-json-${repoName.replaceAll('/',"_sl_")}.json`,async ()=> JSON.stringify(await repo.package()),true);
|
|
// console.log("[git] fetched package.json for", repoName);
|
|
}catch(e){
|
|
throw new Error(`Failed to fetch package.json for ${repoName} from npm: ${e.message}`);
|
|
}
|
|
|
|
if (packageJSONContentsString === undefined || packageJSONContentsString === null) {
|
|
throw new Error(`Failed to fetch package.json for ${repoName} from npm`);
|
|
// console.log("[git] checking", repoName, "for dependencies at ", packageFile);
|
|
// const packageJSONContentsString = (await readFile(packageFile)).toString()
|
|
}
|
|
|
|
// console.log(packageJSONContentsString);
|
|
const packageJSONContents = JSON.parse(packageJSONContentsString)
|
|
// console.log(repoName, packageJSONContents.license)
|
|
if (!hasAnyActualDependencies(packageJSONContents, repoName)) {
|
|
// console.log("[git] skipping", repoName, "has no dependencies");
|
|
return [repoName, null];
|
|
}
|
|
|
|
const hasDependencies = checkTestingDependencies(packageJSONContents, repoName);
|
|
if (hasDependencies) {
|
|
await cacheCloneIdempotently(repoPath, repoName, repoGitUrl);
|
|
|
|
const tsConfigFileLocation = resolve(repoPath, 'tsconfig.json');
|
|
const tsConfigFileExists = existsSync(tsConfigFileLocation);
|
|
if (tsConfigFileExists){
|
|
// console.warn("[git] Ignoring ", repoName, "because it has a tsconfig.json file.");
|
|
return [repoName, null];
|
|
}
|
|
const packageFile = resolve(repoPath, 'package.json')
|
|
if (!existsSync(packageFile)){
|
|
console.warn("[git] Unexpected package.json not found in", repoName, "at", packageFile);
|
|
return [repoName, null];}
|
|
|
|
// finally, return the test script if it exists
|
|
return [repoName, ((packageJSONContents?.scripts?.test))]
|
|
}
|
|
else return [repoName, null]
|
|
}
|
|
|
|
|
|
function filterRepo(repoGitUrl) {
|
|
return matchFilterList(repoGitUrl);
|
|
}
|
|
|
|
function hasAnyActualDependencies(packageJSONContents, repoName) {
|
|
if (packageJSONContents.dependencies !== undefined && Object.keys(packageJSONContents.dependencies).length > 0) {
|
|
return true;
|
|
}
|
|
return false;
|
|
}
|
|
|
|
function checkTestingDependencies(packageJSONContents, repoName) {
|
|
const testingLibraries = new Set(['mocha','jest']);
|
|
const dependencies = new Set();
|
|
if (packageJSONContents.dependencies !== undefined) {
|
|
for (const dep of Object.keys(packageJSONContents.dependencies)) {
|
|
dependencies.add(dep)
|
|
}
|
|
}
|
|
if (packageJSONContents.devDependencies !== undefined) {
|
|
for (const dep of Object.keys(packageJSONContents.devDependencies)) {
|
|
dependencies.add(dep)
|
|
}
|
|
}
|
|
|
|
// console.log(dependencies)
|
|
/**
|
|
* @type {Set}
|
|
*/
|
|
const x = int(testingLibraries, dependencies);
|
|
// console.log(`join`, x)
|
|
return x.size > 0;
|
|
|
|
}
|
|
|
|
async function cacheCloneIdempotently(repoPath, repoName, repoGitUrl) {
|
|
if (existsSync(repoPath)) {
|
|
const isDir = (await lstat(repoPath)).isDirectory()
|
|
if (!isDir) throw new Error(repoName, " is mangled. delete directory and re-clone.")
|
|
else {
|
|
// const path = await git.status({ $cwd: repoPath })
|
|
// console.log("[git] already cloned", repoName, "at", repoPath);
|
|
}
|
|
} else {
|
|
console.log("[git] cloning", repoGitUrl);
|
|
await git.clone(repoGitUrl, repoPath, { 'single-branch': true, depth: 1 })
|
|
}
|
|
}
|