[add] libcalls

.vscode/launch.json

@@ -15,4 +15,4 @@
             "args": ["${workspaceFolder}/test_src/index.mjs"],
         }
     ]
-}
+}

src/index.mjs

@@ -1,84 +1,66 @@
 
-import assert from 'node:assert';
-import { getASTAndScope } from './ast/analysis.mjs';
 
-import { getRequireCallsAndConstantArgs } from './calls.mjs';
 import { readFileSync ,realpathSync ,mkdirSync} from 'node:fs';
 import { writeFile } from 'node:fs/promises';
-import tsc, { Project, SyntaxKind } from 'ts-morph';
+import tsm, { Project, SyntaxKind ,ts} from 'ts-morph';
 import {getSliceAndInfoSync} from 'slice-js/src/slice-code/test/helpers/utils.js';
 import path, { dirname,join } from 'node:path';
-// import tsc from 'typescript'
+import { getImportCallsAndArgumentTypes } from './tsCalls.mjs';
+import { LibraryCallsRecorder } from './libcalls.mjs';
+import { wpCompress } from '../src_bundle/index.mjs';
 /**
- * Call parameter generation
+ * 
+ * @param {LibraryCallsRecorder['calls']} calls 
+ * @param {string} FILE_PATH 
  */
-function main() {
+export async function sliceAndWriteCalls(calls, FILE_PATH) {
-    const FILE_PATH = './test_src/index.cjs';
-    const { scopeManager, _parsedModAST } = getASTAndScope(FILE_PATH);
-    assert(scopeManager.scopes.length >= 2, "expected atleast global and module scope");
-    assert(scopeManager.scopes[1].type === 'function', "expected the 'module' scope to have function scope");
-
-    const calls = getRequireCallsAndConstantArgs(scopeManager);
-
-    logCallList(calls);
-
     const writePromises = [];
+
     for (const [moduleName, callBox] of calls) {
-        if (!isRelativeModule(moduleName)) { // not relative module
+        if (isRelativeModule(moduleName) || isNodeModule(moduleName)) { // not relative module
+            console.warn(`Skipping module ${moduleName} - relative or inbuilt Node.js module`);
             continue;
         }
-
+        console.log(`Slicing module ${moduleName} - ${callBox.size} calls`);
-        const relatedModuleNamePath = join(realpathSync(dirname(FILE_PATH)) ,moduleName);
+        
+        // const relatedModuleNamePath = import.meta.resolve(moduleName);
+        // console.log(`Related module path`, relatedModuleNamePath);
+        
+        const relatedModuleNamePath = await wpCompress(moduleName)
         const fileSource = readFileSync(relatedModuleNamePath).toString('utf-8');
-        const {slicedCode} = getSliceAndInfoSync(fileSource, (moduleExports) => {
+        // continue; // TODO - handle relative modules
-            return [...callBox.entries()].flatMap(([methodName, methodArgsList])=>{
+        const { slicedCode } = getSliceAndInfoSync(fileSource, (moduleExports) => {
+            return [...callBox.entries()].flatMap(([methodName, methodArgsList]) => {
                 const methodNameNormed = methodName.substring(1);
-                console.log("Calls for ",methodNameNormed,methodArgsList)
+                console.log("Calls for ", methodNameNormed, methodArgsList);
-                return methodArgsList.map(methodArgsList=>{
+                return methodArgsList.map(methodArgsList => {
-                    const methodObj = methodNameNormed===''?moduleExports:moduleExports[methodNameNormed];
+                    const methodObj = (methodNameNormed === '') ? moduleExports : moduleExports[methodNameNormed];
-                    methodObj.apply(moduleExports[methodNameNormed],methodArgsList)
+                    methodObj.apply(moduleExports[methodNameNormed], methodArgsList);
                 });
-            })
+            });
-        },relatedModuleNamePath);
+        }, relatedModuleNamePath);
-        // console.log(`Sliced code ${moduleName}\n`,slicedCode);
+        
+        console.log(`Sliced code ${moduleName}\n`,slicedCode);
+        continue;
         const writePath = path.resolve('./dist', moduleName);
-        if(writePath===moduleName){
+        if (writePath === moduleName) {
-            throw Error("Will overwrite!!!!");
+            throw Error("Unexpected Directory rewrite. Not allowed.");
         }
-        mkdirSync(path.dirname(writePath),{recursive: true});
+        mkdirSync(path.dirname(writePath), { recursive: true });
-        console.log(`Writing to`,writePath);
+        console.log(`Writing to`, writePath);
 
-        writePromises.push(writeFile(writePath,slicedCode));
+        writePromises.push(writeFile(writePath, slicedCode));
 
     }
 
-    Promise.all(writePromises).then(p=>{
+    Promise.all(writePromises).then(p => {
-        console.log("write finished")
+        console.log("write finished");
     }).catch(console.log);
 }
 
-class ImportCall{
+function main() {
-    /**
+    // const FILE_PATH = './test_src/index.cjs';
-     * @type {'import'|'importExpr'|'require'}
+    const FILE_PATH = './test_src/index.mjs';
-     */
-    importType;
-    /**
-     * @type {string}
-     */
-    importSyntax;
-    /**
-     * 
-     * @param {'import'|'importExpr'|'require'} importType 
-     * @param {string} importSyntax 
-     */
-    constructor(importType, importSyntax){
-        this.importSyntax = importSyntax;
-        this.importType = importType;
-    }
-}
-
-function main2() {
-    const FILE_PATH = './test_src/index.cjs';
 
     const project = new Project({compilerOptions:{allowJs: true, checkJs: false,}});
     project.addSourceFileAtPathIfExists(FILE_PATH);
@@ -87,45 +69,29 @@ function main2() {
     const checker = project.getTypeChecker();
 
     const sourceFile = project.getSourceFile(FILE_PATH)
-    
+
     const importDecls = sourceFile.getImportStringLiterals()
-    for(const importStringDecl of importDecls){
+    // foreach library, get a list of import calls
-        console.log(importStringDecl);
-        const importDecl = importStringDecl.getFirstAncestor();
-        if(importDecl.isKind(SyntaxKind.CallExpression)){
-            // the declaration is callExpression. Verify its based an identifier aliasing import or require
-            const importExpr = importDecl.getExpression();
-            const type = checker.getTypeAtLocation(importExpr);
-            console.log("Type of import expression",checker.compilerObject.resolveName());
-            console.log(importExpr);
-            if(importExpr.isKind(SyntaxKind.Identifier)){
-                // import is a require or import
-                const importName = importExpr.getText();
-                if(importName==='require' || importName==='import'){
-                    console.log("Found require/import call",importExpr);
-                }
-            }
-            
-        
-        }else if(importDecl.isKind(SyntaxKind.ImportDeclaration)){
-            // TODO pending extract the calls.
-        }else{
-            console.error("Unexpected import specifier",SyntaxKind[importDecl]);
-        }
-        const importThing = importStringDecl.getParent()
-    }
     
-    console.log(importDecls);
+    const calls = getImportCallsAndArgumentTypes(importDecls,checker,FILE_PATH);
+
+    const callMap = calls.generateAllArgumentsForRecordedCalls();
+    
+    logCallList(callMap);
+    sliceAndWriteCalls(callMap, FILE_PATH).then(()=>{
+        console.log("Slicing and writing calls done");
+    });
 }
 
 if (process.argv[1] === import.meta.filename) {
     console.log("[SafeImport] started");
-    main2();
+    main();
     console.log("done");
 }
 
 
-function logCallList(calls) {
+export function logCallList(calls) {
+    console.log(`[Call Log] Call List for ${calls.size} modules`);
     for (const [moduleName, callBoxes] of calls.entries()) {
         if (isRelativeModule(moduleName)) {
             console.log('Importing', moduleName, callBoxes);
@@ -134,8 +100,21 @@ function logCallList(calls) {
         }
     }
     console.log(`Call List`, calls);
+    console.log(`[Call Log] End List for ${calls.size} modules`);
+
 }
 
 function isRelativeModule(moduleName) {
     return moduleName.startsWith('.');
 }
+
+/**
+ * True if an inbuilt Node.js module.
+ * @param {string} moduleName 
+ * @returns 
+ */
+function isNodeModule(moduleName) {
+    if(moduleName.startsWith('node:')) return true;
+    const nodeModules = ['fs', 'fs/promises', 'path', 'http', 'https', 'os', 'crypto']
+    return nodeModules.includes(moduleName);
+}

src/js/calls.mjs

@@ -1,10 +1,12 @@
 import { Syntax } from 'esprima';
 import esquery from 'esquery';
-import { getSetOfIdentifierReferencesForRequireUses } from './ast/analysis.mjs';
+import { getASTAndScope, getSetOfIdentifierReferencesForRequireUses } from '../ast/analysis.mjs';
-import { LibraryCallsRecorder } from './libcalls.mjs';
+import { LibraryCallsRecorder } from '../libcalls.mjs';
-import { tagASTNode, getTagKey, untagASTNode } from './ast/tag.mjs';
+import { tagASTNode, getTagKey, untagASTNode } from '../ast/tag.mjs';
-import { ExpressionArrayVisitor } from './ast/visitors.mjs';
+import { ExpressionArrayVisitor } from '../ast/visitors.mjs';
 import assert from 'assert';
+import assert from 'assert';
+import { logCallList, sliceAndWriteCalls } from '../index.mjs';
 /**
  *
  * @param {import('eslint').Scope.ScopeManager} scopeManager
@@ -48,7 +50,10 @@ export function getRequireCallsAndConstantArgs(scopeManager) {
         }
     }
     return callRecorder.calls;
-}/**
+}
+
+
+/**
  *
  * @param {import('eslint').Scope.Definition} declaratorDefinition
  */
@@ -92,4 +97,19 @@ export function getModuleNameFromRequireAssignDeclaration(requireUsingReference)
 
     return moduleImported;
 }
+// import tsc from 'typescript'
+/**
+ * Call parameter generation
+ */
+function mainOld() {
+    const FILE_PATH = './test_src/index.cjs';
+    const { scopeManager, _parsedModAST } = getASTAndScope(FILE_PATH);
+    assert(scopeManager.scopes.length >= 2, "expected atleast global and module scope");
+    assert(scopeManager.scopes[1].type === 'function', "expected the 'module' scope to have function scope");
+
+    const calls = getRequireCallsAndConstantArgs(scopeManager);
+    logCallList(calls);
+
+    sliceAndWriteCalls(calls, FILE_PATH);
+}
 

src/libcalls.mjs

@@ -4,6 +4,8 @@
 * @typedef {import('estree').Literal["value"]} GenericLiteralType
 */
 
+import tsm, { Type } from 'ts-morph';
+
 /**
  * Record library calls
  */
@@ -32,4 +34,103 @@ export class LibraryCallsRecorder{
         return this.#calls;
     }
     
+}
+
+export class LibraryTypesRecorder{
+    /**
+     * @type {Map<string,Map<string,Type[][]>>}
+     */
+    #calls = new Map();
+    /**
+     * @param {tsm.TypeChecker} checker
+     */
+    checker;
+    /**
+     * 
+     * @param {string} moduleName 
+     * @param {string} libraryFunctionSegment 
+     * @param {Type[]} argumentsCalled 
+     */
+    pushToMap(moduleName, libraryFunctionSegment, argumentsCalled){
+        const modulePortion = this.#calls.get(moduleName)?? new Map();
+        
+        const defArgs = modulePortion.get(libraryFunctionSegment) ?? [];
+        defArgs.push(argumentsCalled);
+
+        modulePortion.set(libraryFunctionSegment,defArgs);
+        this.#calls.set(moduleName, modulePortion);
+    }
+
+    get calls(){
+        return this.#calls;
+    }
+
+    generateAllArgumentsForRecordedCalls(){
+        const callMap = new Map();
+        for(const [moduleName, modulePortion] of this.#calls){
+            /**
+             * @type {Map<string,(GenericLiteralType|null|undefined|{})[][]>}
+             */
+            const moduleCallMap = new Map();// todo refactor
+            for(const [libraryFunctionSegment, argsList] of modulePortion){
+                const argsForFunction = argsList.map(args=>args.map(arg=>this.instantiateType(arg)));
+                moduleCallMap.set(libraryFunctionSegment,argsForFunction);
+            }
+            callMap.set(moduleName,moduleCallMap);
+        }
+        return callMap;
+    }
+
+    /**
+     * If the the arguments types are available in the map, instantiate set of arguments matching the types.
+     * @param {string} moduleName 
+     * @param {string} libraryFunctionSegment 
+     * @returns {(GenericLiteralType|null|undefined|{})[][]|undefined}
+     */
+    generateArgumentsForCall(moduleName, libraryFunctionSegment){
+        const modulePortion = this.#calls.get(moduleName);
+        if(modulePortion===undefined){
+            return undefined;
+        }
+        
+        const argsTypesForFunctionCalls = modulePortion.get(libraryFunctionSegment);
+        if(argsTypesForFunctionCalls===undefined){
+            return undefined;
+        }
+        return argsTypesForFunctionCalls.map(argTypeForSingleCall=>{
+            return argTypeForSingleCall.map(type=>{
+                return LibraryTypesRecorder.instantiateType(type);
+                
+            });
+        });
+
+    }
+    /**
+     * 
+     * @param {Type} type 
+     * @returns 
+     */
+    instantiateType(type){
+        if(type.isStringLiteral()){
+            return type.getLiteralValue();
+        }else if(type.isNumberLiteral()){
+            return Number(type.getText());
+        }else if(type.isBooleanLiteral()){
+            return type.getText() === 'true';
+        }else if(type.isString()){
+            return "";
+        }else if(type.isNumber()){
+            return 0;
+        }else if(type.isBoolean()){
+            return false;// BAD IDEA
+        }else if(type.isArray()){
+            return [];
+        }else if(type.isObject()){
+            // TODO - handle functions
+            return {};
+        }else{
+            console.warn("Unknown type to instantiate",type.getText());
+            return undefined;
+        }
+    }
 }

src/tsCalls.mjs

@@ -0,0 +1,157 @@
+// @ts-check
+import path from 'path';
+import tsm, { Identifier, ImportSpecifier, StringLiteral, SyntaxKind, ts, } from 'ts-morph';
+import { LibraryTypesRecorder } from './libcalls.mjs';
+
+/**
+ * 
+ * @param {tsm.StringLiteral[]} importDecls
+ * @param {tsm.TypeChecker} checker
+ * @param {string} mainFilePath Main file path for the script being analyzed
+ * @returns {LibraryTypesRecorder} instance of recorded library calls
+ */
+export function getImportCallsAndArgumentTypes(importDecls, checker, mainFilePath) {
+    const libraryCallsRecorder = new LibraryTypesRecorder();
+    for (const importStringDecl of importDecls) {
+        // console.log(importStringDecl);
+        const importDecl = importStringDecl.getFirstAncestor();
+        if (importDecl === undefined) {
+            console.error("Import declaration is undefined for", importStringDecl.getText());
+            continue;
+        }
+        if (importDecl.isKind(SyntaxKind.CallExpression)) {
+            // the declaration is callExpression. Verify its based an identifier aliasing import or require
+            const importExpr = importDecl.getExpression();
+            const type = checker.getTypeAtLocation(importExpr);
+            console.log("Type of import expression", checker.getTypeText(type));
+            // console.log(importExpr);
+            if (importExpr.isKind(SyntaxKind.Identifier)) {
+                // import is a require or import
+                const importName = importExpr.getText();
+                const importId = importExpr;
+
+                // check if the require is from node
+                if (importName === 'require') {
+                    const importSymbol = importId.getType().getSymbol();
+                    if (importSymbol === undefined) {
+                        console.error("Import identifier has no symbol", importId.getText());
+                    } else {
+                        const importSymbolFullyQualifiedName = checker.getFullyQualifiedName(importSymbol);
+                        if (importSymbolFullyQualifiedName !== 'global.NodeJS.Require') {
+                            console.warn("Found require call but not from NodeJS global require");
+                        }
+                    }
+
+
+                    console.log("Found require/import call", importExpr);
+                    // extract the variables imported from the callexpression
+                    const importArgs = importDecl.getArguments();
+
+                    const parent = importDecl.getParent();
+                    if (parent?.isKind(SyntaxKind.VariableDeclaration)) {
+                        // this is a variable declaration
+                        const varDecl = parent;
+                        const varName = varDecl.getName();
+                        console.log("Variable name", varName);
+                        // check if declaration is identifier or object pattern
+                    }
+                    throw Error("Not implemented yet");
+                }
+            }
+
+
+        } else if (importDecl.isKind(SyntaxKind.ImportDeclaration)) {// import {x,z} from 'module'; 
+            console.log("Found import declaration", importDecl.getPos());
+            console.log("Named imports", importDecl.getNamedImports().length);
+            const namedImports = importDecl.getNamedImports();
+
+            for (const namedImport of namedImports) {
+                // TODO handle aliases
+                handleImportForGivenImport(importStringDecl,namedImport, mainFilePath, libraryCallsRecorder);
+
+            }
+            const defaultImportIdentifier = importDecl.getDefaultImport();
+            console.log("Default import",defaultImportIdentifier);
+            if( defaultImportIdentifier !== undefined) {
+                recordImportedIdentifierUsage(defaultImportIdentifier, mainFilePath, libraryCallsRecorder, importStringDecl, true);
+            }
+            // console.log("Namespace import",importDecl.getNamespaceImport());
+            // recordImportedIdentifierUsage(defaultImportIdentifier, mainFilePath, libraryCallsRecorder, importStringDecl, true);
+
+            console.log("STOP");
+
+        } else {
+            console.error("Unexpected import specifier", SyntaxKind[importDecl.getKind()]);
+        }
+        const importThing = importStringDecl.getParent()
+
+    }
+    // throw Error("Not implemented yet");
+    return libraryCallsRecorder;
+}
+
+/**
+ * 
+ * @param {tsm.StringLiteral} importStringLiteral
+ * @param {ImportSpecifier} namedImport 
+ * @param {string} mainFilePath 
+ * @param {LibraryTypesRecorder} libraryCallsRecorder
+ */
+function handleImportForGivenImport(importStringLiteral,namedImport, mainFilePath, libraryCallsRecorder) {
+    const aliasNode = namedImport.getAliasNode();
+    if (aliasNode !== undefined) {
+        console.error("Unhandled named import alias", aliasNode.getText());
+
+    }
+    console.log("Named import", namedImport.getNameNode().getText());
+    const importNode = namedImport.getNameNode();
+    if (importNode.isKind(SyntaxKind.StringLiteral)) {
+        throw Error("Unexpected string literal import node. Expected identifier");
+    }
+
+    recordImportedIdentifierUsage(importNode, mainFilePath, libraryCallsRecorder, importStringLiteral);
+}
+/**
+ * 
+ * @param {Identifier} importNode 
+ * @param {string} mainFilePath 
+ * @param {LibraryTypesRecorder} libraryCallsRecorder 
+ * @param {StringLiteral} importStringLiteral 
+ * @param {boolean} [isDefaultImport=false]
+ */
+function recordImportedIdentifierUsage(importNode, mainFilePath, libraryCallsRecorder, importStringLiteral, isDefaultImport = false) {
+    const importRefs = importNode.findReferences();
+    for (const importRef of importRefs) {
+        const referenceSourceFile = importRef.getDefinition().getSourceFile();
+        const comparePath = path.relative(mainFilePath, referenceSourceFile.getFilePath());
+        if (comparePath !== '') {
+            console.warn("Skipping import reference from other file", referenceSourceFile.getFilePath());
+            continue;
+        }
+        console.log("Compare path", comparePath === '');
+        // const filePath = referenceSourceFile.getFilePath();
+        // console.log("Refset for import",filePath);
+        for (const ref of importRef.getReferences()) {
+            if (ref.isDefinition()) {
+                continue;
+            }
+            // console.log("I am ",ref.isDefinition());
+            const callExpression = ref.getNode().getFirstAncestorByKind(SyntaxKind.CallExpression);
+
+            const callExpressionArguments = callExpression?.getArguments();
+            if (callExpressionArguments === undefined || callExpressionArguments.length === 0) {
+                console.warn("No call expressions found for import reference", ref.getNode().getText());
+                continue;
+            }
+
+            // for(const argument of callExpressionArguments){
+            //     console.log(`Arg ${idx} is ${arg.getText()}, type is ${arg.getType()}`);
+            // }
+            const getImportSection = '.' + (isDefaultImport? 'default':importNode.getText());
+            libraryCallsRecorder.pushToMap(importStringLiteral.getLiteralValue(), getImportSection, callExpressionArguments.map(arg => arg.getType()));
+
+            console.log("I am ", callExpression?.getText());
+        }
+    }
+}
+

src_bundle/index.mjs

@@ -1,44 +1,65 @@
 import wp from 'webpack';
 import path from 'node:path'
-const outputPath = path.resolve('./output/');
-console.log(outputPath);
 
-const ls = [
-'classnames',
-'semver',
-'ansi-styles',
-'debug',
-'supports-color',
-'chalk',
-'ms',
-'minimatch',
-'strip-ansi',
-'tslib',
-'has-flag',
-'ansi-regex',
-'color-convert',
-'color-name',
-// 'type-fest',
-'string-width'
-]
 
-ls.forEach(l=>{
+if (process.argv[1] === import.meta.filename) {
-    
+    console.log("[SafePack] started");
+    main();
+    console.log("done");
+}
+
+function main() {
+    const ls = [
+        'classnames',
+        'semver',
+        'ansi-styles',
+        // 'debug',
+        // 'supports-color',
+        'chalk',
+        'ms',
+        'minimatch',
+        'strip-ansi',
+        'tslib',
+        'has-flag',
+        'ansi-regex',
+        'color-convert',
+        'color-name',
+        // 'type-fest',
+        'string-width'
+    ];
+
+    ls.forEach(l => {
+
+        wpCompress(l).then(outputFileLocation => {
+            console.log("[wp] success", outputFileLocation);
+        }).catch(err => {
+            console.error("[failed wp]", l);
+            console.error("[wp] error");
+
+        });
+
+
+    });
+}
+
+export function wpCompress(l, outputPath = path.resolve('./output/')) {
+    return new Promise((resolve, reject) => {
     const libraryLocation = import.meta.resolve(l);
     console.log(libraryLocation);
+    const outputFile = l + '.bundle.cjs';
     // throw Error("5");
     wp({
-        entry:libraryLocation,
+        entry: libraryLocation,
         mode: 'production',
-        optimization:{
+        optimization: {
             mangleExports: false,
             avoidEntryIife: true,
-            minimize: false
+            minimize: false,
             
         },
         output: {
             path: outputPath,
-            filename: l+'.bundle.cjs',
+            filename: outputFile,
             clean: false,
             iife: false,
             library: {
@@ -46,15 +67,17 @@ ls.forEach(l=>{
                 // name: l
             }
             // module: true
-        }
+        },
-        ,
+    }, (err, stats) => {
-    },(err,stats)=>{
         if (err || stats.hasErrors()) {
-            console.log(err?.stack)
+            // console.log(err?.stack);
-            console.log(stats?.hasErrors())
+            // console.log(stats?.hasErrors());
-            console.log(stats?.toJson());
+            // console.log(stats?.toJson());
+            reject(err || stats);
+        }else{
+            resolve(path.resolve(outputPath, outputFile));
         }
-    })
+    });
+});
+}
 
-
-})

test_src/index.mjs

@@ -1,5 +1,12 @@
-import {readFile} from 'node:fs'
+import fs,{readFile} from 'node:fs'
+
+import classnames from 'classnames'
+// import {neq} from 'semver'
+import {sum, div,sad} from './arithmetic.cjs';
 
 readFile('a',(err)=>{
     if(err){return;}
-})
+})
+console.log(classnames('a', 'b', 'c',{a:5})); // $ExpectType string
+console.log(sum(2, 3)); 
+// console.log(neq('1.0.0', '1.0.1')); // $ExpectType boolean