[add] implement vulnerability checking and advisory fetching, enhance repo processing, and add utility functions

src_vuln/index.mjs

@@ -0,0 +1,65 @@
+import { cacheFunctionOutput } from "../src_dataset/cache.mjs";
+import { bifurcateArray, getGithubTokenFromEnvironment } from "./lib.mjs";
+import { checkForParentDep, findSlicedDeps } from "./slicedeps.mjs";
+import { basename } from "path";
+
+const githubToken = getGithubTokenFromEnvironment();
+
+const vulnTargets = await findSlicedDeps();
+const affects = [...vulnTargets].join(',');
+
+// console.log(query)
+
+const res = await cacheFunctionOutput('advisories.json', async () => {
+    const query = `?ecosystem=npm&affects=${affects}`;
+    const res = await fetch('https://api.github.com/advisories'+query,
+        {
+            headers:{
+                Authorization: `Bearer ${githubToken}`,
+            }
+        }
+    );
+    const x = await res.json();
+    return x;
+},true, false);
+
+const cveMap = res.map(e=>({
+        summary: e.summary,
+        source: e.source_code_location,
+        severity: e.severity,
+        repo_name: basename(e.source_code_location),
+        cve: e.cve_id,
+        identifiers: e.identifiers,
+        cvss: e.cvss,
+    }));
+
+const [fullMaps,emptyMap]= bifurcateArray(cveMap, (e)=>e.source)
+
+
+// const slicedReposSoFar = await findSlicedDepsSoFar();
+const depMap = new Map();
+for(const depo of fullMaps){
+    if(!depMap.has(depo.repo_name)) {
+        depMap.set(depo.repo_name, []);
+    }
+    depMap.get(depo.repo_name).push(depo);
+}
+const depKeys = ([...depMap.keys()])
+console.log(depKeys)
+const repoKeys = await checkForParentDep(depKeys);
+console.log(repoKeys);
+// for(const repo of slicedReposSoFar) {
+//     const deps = await getDepsOfRepo(repo);
+//     console.log(repo,deps);
+//     const depCVEs = fullMaps.filter(e=>(deps).includes(e.repo_name));
+//     depMap.set(repo, depCVEs);
+// }
+console.log(cveMap.length, "advisories found");
+console.log(fullMaps.length, "advisories found");
+console.log(emptyMap.length, "advisories found");
+// what is pending
+// see what's been sliced so far. Find their dependencies, link back to 
+
+
+
+

src_vuln/lib.mjs

@@ -0,0 +1,26 @@
+import assert from "assert";
+
+/**
+ * Bifurcate an array into two arrays based on a predicate function.
+ * @template T
+ * @param {T[]} arr 
+ * @param {(T)=>boolean} predicate 
+ * @returns {[T[], T[]]}
+ */
+export function bifurcateArray(arr, predicate) {
+    const truthy = [];
+    const falsy = [];
+    for (const item of arr) {
+        if (predicate(item)) {
+            truthy.push(item);
+        } else {
+            falsy.push(item);
+        }
+    }
+    return [truthy, falsy];
+}
+export function getGithubTokenFromEnvironment() {
+    assert(process.env.GITHUB_TOKEN, "No token :(");
+    const githubToken = process.env.GITHUB_TOKEN;
+    return githubToken;
+}

src_vuln/slicedeps.mjs

@@ -0,0 +1,97 @@
+import { readdir, opendir } from 'node:fs/promises'
+import path, { basename, dirname } from 'node:path';
+
+
+
+/**
+ * Finds all dependencies that are sliced by the slicer.
+ * this function will search find the list of dependencies that are sliced by the slicer.
+ * in the dist folder, theres a folder of repos. in that, there is a folder for each dependency.
+ * collate into a set and return it.
+ * Eg.
+ * dist/
+ *  └── align-text/
+ *     └── kind-of
+ *    └── longest
+ * 
+ * it will return kind-of and longest as sliced deps.
+ */
+export async function findSlicedDeps(){
+    /**
+     * @type {Set<string>}
+     */
+    const slicedDeps = new Set();
+    const distPath = path.resolve('dist');
+    
+    for await(const p of walk(distPath)) {
+        if(p.endsWith("package.json")) {slicedDeps.add(basename(dirname(p)))}
+        else continue;
+    }
+    return slicedDeps;
+}
+
+/**
+ * Given a list of deps, find the repos that have these
+ * @param {string[]} dependencies 
+ */
+export async function checkForParentDep(dependencies){
+    // dep -> main
+    const map = await getReverseDeps();
+    const reposet = dependencies.flatMap(dep => (map.get(dep)??[]));
+    const repos = new Set(reposet);
+    return repos;
+}
+
+// for a given dep, find the list of main repo that has this dep. return map.
+async function getReverseDeps() {
+    const x = new Map();
+    const distPath = path.resolve('dist');
+
+    for await(const p of walk(distPath)) {
+        if (p.endsWith("package.json")) {
+            const repo = basename(dirname(dirname(p)));
+            const depName = basename(dirname(p));
+
+            if (!x.has(depName)) {
+                x.set(depName, []);
+            }
+            x.get(depName).push(repo);
+            // console.log(p,repo, depName);
+        }
+        else continue;
+    }
+    return x;
+}
+
+export async function findSlicedDepsSoFar() {
+    // return all folder names in the output directory.
+    const distPath = path.resolve('output');
+    const dirEntries = await readdir(distPath, { withFileTypes: true });
+    const repos = dirEntries.filter(dirent => dirent.isDirectory()).map(dirent => dirent.name);
+    return repos;
+}
+/**
+ * 
+ * @param {string} repo 
+ */
+export async function getDepsOfRepo(repo){
+    const distPath = path.resolve('output', repo);
+    const dirEntry = await readdir(distPath, { withFileTypes: true });
+    const deps = dirEntry.filter(dirent => dirent.isFile()).map(dirent => dirent.name.replace('.bundle.cjs',''));
+    return deps;
+}
+
+/**
+ * FS walk primitive
+ * Ref: https://gist.github.com/lovasoa/8691344
+ * @param {string} dir 
+ * @returns {AsyncGenerator<string>}
+ */
+async function* walk(dir) {
+    for await (const d of await opendir(dir)) {
+        const entry = path.join(dir, d.name);
+        if (d.isDirectory()) yield* walk(entry);
+        else if (d.isFile()) yield entry;
+    }
+}
+// checkForParentDep('thenify', 'glob-parent', 'terser', 'url-parse').then(console.log).catch(console.error);