From 4c40360bf620e8664b1ed4bf617f541bea6cd0e0 Mon Sep 17 00:00:00 2001 From: Atreya Bain Date: Fri, 1 Aug 2025 20:19:24 +0100 Subject: [PATCH] [add] libcalls --- .vscode/launch.json | 2 +- src/index.mjs | 151 +++++++++++++++++---------------------- src/{ => js}/calls.mjs | 30 ++++++-- src/libcalls.mjs | 101 ++++++++++++++++++++++++++ src/tsCalls.mjs | 157 +++++++++++++++++++++++++++++++++++++++++ src_bundle/index.mjs | 93 +++++++++++++++--------- test_src/index.mjs | 11 ++- 7 files changed, 416 insertions(+), 129 deletions(-) rename src/{ => js}/calls.mjs (82%) create mode 100644 src/tsCalls.mjs diff --git a/.vscode/launch.json b/.vscode/launch.json index 9b89ddd..db25979 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -15,4 +15,4 @@ "args": ["${workspaceFolder}/test_src/index.mjs"], } ] -} \ No newline at end of file +} diff --git a/src/index.mjs b/src/index.mjs index d14985c..28cb091 100644 --- a/src/index.mjs +++ b/src/index.mjs @@ -1,84 +1,66 @@ -import assert from 'node:assert'; -import { getASTAndScope } from './ast/analysis.mjs'; -import { getRequireCallsAndConstantArgs } from './calls.mjs'; import { readFileSync ,realpathSync ,mkdirSync} from 'node:fs'; import { writeFile } from 'node:fs/promises'; -import tsc, { Project, SyntaxKind } from 'ts-morph'; +import tsm, { Project, SyntaxKind ,ts} from 'ts-morph'; import {getSliceAndInfoSync} from 'slice-js/src/slice-code/test/helpers/utils.js'; import path, { dirname,join } from 'node:path'; -// import tsc from 'typescript' +import { getImportCallsAndArgumentTypes } from './tsCalls.mjs'; +import { LibraryCallsRecorder } from './libcalls.mjs'; +import { wpCompress } from '../src_bundle/index.mjs'; /** - * Call parameter generation + * + * @param {LibraryCallsRecorder['calls']} calls + * @param {string} FILE_PATH */ -function main() { - const FILE_PATH = './test_src/index.cjs'; - const { scopeManager, _parsedModAST } = getASTAndScope(FILE_PATH); - assert(scopeManager.scopes.length >= 2, "expected atleast global and module scope"); - assert(scopeManager.scopes[1].type === 'function', "expected the 'module' scope to have function scope"); - - const calls = getRequireCallsAndConstantArgs(scopeManager); - - logCallList(calls); - +export async function sliceAndWriteCalls(calls, FILE_PATH) { const writePromises = []; + for (const [moduleName, callBox] of calls) { - if (!isRelativeModule(moduleName)) { // not relative module + if (isRelativeModule(moduleName) || isNodeModule(moduleName)) { // not relative module + console.warn(`Skipping module ${moduleName} - relative or inbuilt Node.js module`); continue; } - - const relatedModuleNamePath = join(realpathSync(dirname(FILE_PATH)) ,moduleName); + console.log(`Slicing module ${moduleName} - ${callBox.size} calls`); + + // const relatedModuleNamePath = import.meta.resolve(moduleName); + // console.log(`Related module path`, relatedModuleNamePath); + + const relatedModuleNamePath = await wpCompress(moduleName) const fileSource = readFileSync(relatedModuleNamePath).toString('utf-8'); - const {slicedCode} = getSliceAndInfoSync(fileSource, (moduleExports) => { - return [...callBox.entries()].flatMap(([methodName, methodArgsList])=>{ + // continue; // TODO - handle relative modules + const { slicedCode } = getSliceAndInfoSync(fileSource, (moduleExports) => { + return [...callBox.entries()].flatMap(([methodName, methodArgsList]) => { const methodNameNormed = methodName.substring(1); - console.log("Calls for ",methodNameNormed,methodArgsList) - return methodArgsList.map(methodArgsList=>{ - const methodObj = methodNameNormed===''?moduleExports:moduleExports[methodNameNormed]; - methodObj.apply(moduleExports[methodNameNormed],methodArgsList) + console.log("Calls for ", methodNameNormed, methodArgsList); + return methodArgsList.map(methodArgsList => { + const methodObj = (methodNameNormed === '') ? moduleExports : moduleExports[methodNameNormed]; + methodObj.apply(moduleExports[methodNameNormed], methodArgsList); }); - }) - },relatedModuleNamePath); - // console.log(`Sliced code ${moduleName}\n`,slicedCode); + }); + }, relatedModuleNamePath); + + console.log(`Sliced code ${moduleName}\n`,slicedCode); + continue; const writePath = path.resolve('./dist', moduleName); - if(writePath===moduleName){ - throw Error("Will overwrite!!!!"); + if (writePath === moduleName) { + throw Error("Unexpected Directory rewrite. Not allowed."); } - mkdirSync(path.dirname(writePath),{recursive: true}); - console.log(`Writing to`,writePath); + mkdirSync(path.dirname(writePath), { recursive: true }); + console.log(`Writing to`, writePath); - writePromises.push(writeFile(writePath,slicedCode)); + writePromises.push(writeFile(writePath, slicedCode)); } - Promise.all(writePromises).then(p=>{ - console.log("write finished") + Promise.all(writePromises).then(p => { + console.log("write finished"); }).catch(console.log); } -class ImportCall{ - /** - * @type {'import'|'importExpr'|'require'} - */ - importType; - /** - * @type {string} - */ - importSyntax; - /** - * - * @param {'import'|'importExpr'|'require'} importType - * @param {string} importSyntax - */ - constructor(importType, importSyntax){ - this.importSyntax = importSyntax; - this.importType = importType; - } -} - -function main2() { - const FILE_PATH = './test_src/index.cjs'; +function main() { + // const FILE_PATH = './test_src/index.cjs'; + const FILE_PATH = './test_src/index.mjs'; const project = new Project({compilerOptions:{allowJs: true, checkJs: false,}}); project.addSourceFileAtPathIfExists(FILE_PATH); @@ -87,45 +69,29 @@ function main2() { const checker = project.getTypeChecker(); const sourceFile = project.getSourceFile(FILE_PATH) - + const importDecls = sourceFile.getImportStringLiterals() - for(const importStringDecl of importDecls){ - console.log(importStringDecl); - const importDecl = importStringDecl.getFirstAncestor(); - if(importDecl.isKind(SyntaxKind.CallExpression)){ - // the declaration is callExpression. Verify its based an identifier aliasing import or require - const importExpr = importDecl.getExpression(); - const type = checker.getTypeAtLocation(importExpr); - console.log("Type of import expression",checker.compilerObject.resolveName()); - console.log(importExpr); - if(importExpr.isKind(SyntaxKind.Identifier)){ - // import is a require or import - const importName = importExpr.getText(); - if(importName==='require' || importName==='import'){ - console.log("Found require/import call",importExpr); - } - } - - - }else if(importDecl.isKind(SyntaxKind.ImportDeclaration)){ - // TODO pending extract the calls. - }else{ - console.error("Unexpected import specifier",SyntaxKind[importDecl]); - } - const importThing = importStringDecl.getParent() - } + // foreach library, get a list of import calls - console.log(importDecls); + const calls = getImportCallsAndArgumentTypes(importDecls,checker,FILE_PATH); + + const callMap = calls.generateAllArgumentsForRecordedCalls(); + + logCallList(callMap); + sliceAndWriteCalls(callMap, FILE_PATH).then(()=>{ + console.log("Slicing and writing calls done"); + }); } if (process.argv[1] === import.meta.filename) { console.log("[SafeImport] started"); - main2(); + main(); console.log("done"); } -function logCallList(calls) { +export function logCallList(calls) { + console.log(`[Call Log] Call List for ${calls.size} modules`); for (const [moduleName, callBoxes] of calls.entries()) { if (isRelativeModule(moduleName)) { console.log('Importing', moduleName, callBoxes); @@ -134,8 +100,21 @@ function logCallList(calls) { } } console.log(`Call List`, calls); + console.log(`[Call Log] End List for ${calls.size} modules`); + } function isRelativeModule(moduleName) { return moduleName.startsWith('.'); } + +/** + * True if an inbuilt Node.js module. + * @param {string} moduleName + * @returns + */ +function isNodeModule(moduleName) { + if(moduleName.startsWith('node:')) return true; + const nodeModules = ['fs', 'fs/promises', 'path', 'http', 'https', 'os', 'crypto'] + return nodeModules.includes(moduleName); +} \ No newline at end of file diff --git a/src/calls.mjs b/src/js/calls.mjs similarity index 82% rename from src/calls.mjs rename to src/js/calls.mjs index 78be56d..aa811ef 100644 --- a/src/calls.mjs +++ b/src/js/calls.mjs @@ -1,10 +1,12 @@ import { Syntax } from 'esprima'; import esquery from 'esquery'; -import { getSetOfIdentifierReferencesForRequireUses } from './ast/analysis.mjs'; -import { LibraryCallsRecorder } from './libcalls.mjs'; -import { tagASTNode, getTagKey, untagASTNode } from './ast/tag.mjs'; -import { ExpressionArrayVisitor } from './ast/visitors.mjs'; +import { getASTAndScope, getSetOfIdentifierReferencesForRequireUses } from '../ast/analysis.mjs'; +import { LibraryCallsRecorder } from '../libcalls.mjs'; +import { tagASTNode, getTagKey, untagASTNode } from '../ast/tag.mjs'; +import { ExpressionArrayVisitor } from '../ast/visitors.mjs'; import assert from 'assert'; +import assert from 'assert'; +import { logCallList, sliceAndWriteCalls } from '../index.mjs'; /** * * @param {import('eslint').Scope.ScopeManager} scopeManager @@ -48,7 +50,10 @@ export function getRequireCallsAndConstantArgs(scopeManager) { } } return callRecorder.calls; -}/** +} + + +/** * * @param {import('eslint').Scope.Definition} declaratorDefinition */ @@ -92,4 +97,19 @@ export function getModuleNameFromRequireAssignDeclaration(requireUsingReference) return moduleImported; } +// import tsc from 'typescript' +/** + * Call parameter generation + */ +function mainOld() { + const FILE_PATH = './test_src/index.cjs'; + const { scopeManager, _parsedModAST } = getASTAndScope(FILE_PATH); + assert(scopeManager.scopes.length >= 2, "expected atleast global and module scope"); + assert(scopeManager.scopes[1].type === 'function', "expected the 'module' scope to have function scope"); + + const calls = getRequireCallsAndConstantArgs(scopeManager); + logCallList(calls); + + sliceAndWriteCalls(calls, FILE_PATH); +} diff --git a/src/libcalls.mjs b/src/libcalls.mjs index a53a69e..12c647b 100644 --- a/src/libcalls.mjs +++ b/src/libcalls.mjs @@ -4,6 +4,8 @@ * @typedef {import('estree').Literal["value"]} GenericLiteralType */ +import tsm, { Type } from 'ts-morph'; + /** * Record library calls */ @@ -32,4 +34,103 @@ export class LibraryCallsRecorder{ return this.#calls; } +} + +export class LibraryTypesRecorder{ + /** + * @type {Map>} + */ + #calls = new Map(); + /** + * @param {tsm.TypeChecker} checker + */ + checker; + /** + * + * @param {string} moduleName + * @param {string} libraryFunctionSegment + * @param {Type[]} argumentsCalled + */ + pushToMap(moduleName, libraryFunctionSegment, argumentsCalled){ + const modulePortion = this.#calls.get(moduleName)?? new Map(); + + const defArgs = modulePortion.get(libraryFunctionSegment) ?? []; + defArgs.push(argumentsCalled); + + modulePortion.set(libraryFunctionSegment,defArgs); + this.#calls.set(moduleName, modulePortion); + } + + get calls(){ + return this.#calls; + } + + generateAllArgumentsForRecordedCalls(){ + const callMap = new Map(); + for(const [moduleName, modulePortion] of this.#calls){ + /** + * @type {Map} + */ + const moduleCallMap = new Map();// todo refactor + for(const [libraryFunctionSegment, argsList] of modulePortion){ + const argsForFunction = argsList.map(args=>args.map(arg=>this.instantiateType(arg))); + moduleCallMap.set(libraryFunctionSegment,argsForFunction); + } + callMap.set(moduleName,moduleCallMap); + } + return callMap; + } + + /** + * If the the arguments types are available in the map, instantiate set of arguments matching the types. + * @param {string} moduleName + * @param {string} libraryFunctionSegment + * @returns {(GenericLiteralType|null|undefined|{})[][]|undefined} + */ + generateArgumentsForCall(moduleName, libraryFunctionSegment){ + const modulePortion = this.#calls.get(moduleName); + if(modulePortion===undefined){ + return undefined; + } + + const argsTypesForFunctionCalls = modulePortion.get(libraryFunctionSegment); + if(argsTypesForFunctionCalls===undefined){ + return undefined; + } + return argsTypesForFunctionCalls.map(argTypeForSingleCall=>{ + return argTypeForSingleCall.map(type=>{ + return LibraryTypesRecorder.instantiateType(type); + + }); + }); + + } + /** + * + * @param {Type} type + * @returns + */ + instantiateType(type){ + if(type.isStringLiteral()){ + return type.getLiteralValue(); + }else if(type.isNumberLiteral()){ + return Number(type.getText()); + }else if(type.isBooleanLiteral()){ + return type.getText() === 'true'; + }else if(type.isString()){ + return ""; + }else if(type.isNumber()){ + return 0; + }else if(type.isBoolean()){ + return false;// BAD IDEA + }else if(type.isArray()){ + return []; + }else if(type.isObject()){ + // TODO - handle functions + return {}; + }else{ + console.warn("Unknown type to instantiate",type.getText()); + return undefined; + } + } } \ No newline at end of file diff --git a/src/tsCalls.mjs b/src/tsCalls.mjs new file mode 100644 index 0000000..7c75968 --- /dev/null +++ b/src/tsCalls.mjs @@ -0,0 +1,157 @@ +// @ts-check +import path from 'path'; +import tsm, { Identifier, ImportSpecifier, StringLiteral, SyntaxKind, ts, } from 'ts-morph'; +import { LibraryTypesRecorder } from './libcalls.mjs'; + +/** + * + * @param {tsm.StringLiteral[]} importDecls + * @param {tsm.TypeChecker} checker + * @param {string} mainFilePath Main file path for the script being analyzed + * @returns {LibraryTypesRecorder} instance of recorded library calls + */ +export function getImportCallsAndArgumentTypes(importDecls, checker, mainFilePath) { + const libraryCallsRecorder = new LibraryTypesRecorder(); + for (const importStringDecl of importDecls) { + // console.log(importStringDecl); + const importDecl = importStringDecl.getFirstAncestor(); + if (importDecl === undefined) { + console.error("Import declaration is undefined for", importStringDecl.getText()); + continue; + } + if (importDecl.isKind(SyntaxKind.CallExpression)) { + // the declaration is callExpression. Verify its based an identifier aliasing import or require + const importExpr = importDecl.getExpression(); + const type = checker.getTypeAtLocation(importExpr); + console.log("Type of import expression", checker.getTypeText(type)); + // console.log(importExpr); + if (importExpr.isKind(SyntaxKind.Identifier)) { + // import is a require or import + const importName = importExpr.getText(); + const importId = importExpr; + + // check if the require is from node + if (importName === 'require') { + const importSymbol = importId.getType().getSymbol(); + if (importSymbol === undefined) { + console.error("Import identifier has no symbol", importId.getText()); + } else { + const importSymbolFullyQualifiedName = checker.getFullyQualifiedName(importSymbol); + if (importSymbolFullyQualifiedName !== 'global.NodeJS.Require') { + console.warn("Found require call but not from NodeJS global require"); + } + } + + + console.log("Found require/import call", importExpr); + // extract the variables imported from the callexpression + const importArgs = importDecl.getArguments(); + + const parent = importDecl.getParent(); + if (parent?.isKind(SyntaxKind.VariableDeclaration)) { + // this is a variable declaration + const varDecl = parent; + const varName = varDecl.getName(); + console.log("Variable name", varName); + // check if declaration is identifier or object pattern + } + throw Error("Not implemented yet"); + } + } + + + } else if (importDecl.isKind(SyntaxKind.ImportDeclaration)) {// import {x,z} from 'module'; + console.log("Found import declaration", importDecl.getPos()); + console.log("Named imports", importDecl.getNamedImports().length); + const namedImports = importDecl.getNamedImports(); + + for (const namedImport of namedImports) { + // TODO handle aliases + handleImportForGivenImport(importStringDecl,namedImport, mainFilePath, libraryCallsRecorder); + + } + const defaultImportIdentifier = importDecl.getDefaultImport(); + console.log("Default import",defaultImportIdentifier); + if( defaultImportIdentifier !== undefined) { + recordImportedIdentifierUsage(defaultImportIdentifier, mainFilePath, libraryCallsRecorder, importStringDecl, true); + } + // console.log("Namespace import",importDecl.getNamespaceImport()); + // recordImportedIdentifierUsage(defaultImportIdentifier, mainFilePath, libraryCallsRecorder, importStringDecl, true); + + console.log("STOP"); + + } else { + console.error("Unexpected import specifier", SyntaxKind[importDecl.getKind()]); + } + const importThing = importStringDecl.getParent() + + } + // throw Error("Not implemented yet"); + return libraryCallsRecorder; +} + +/** + * + * @param {tsm.StringLiteral} importStringLiteral + * @param {ImportSpecifier} namedImport + * @param {string} mainFilePath + * @param {LibraryTypesRecorder} libraryCallsRecorder + */ +function handleImportForGivenImport(importStringLiteral,namedImport, mainFilePath, libraryCallsRecorder) { + const aliasNode = namedImport.getAliasNode(); + if (aliasNode !== undefined) { + console.error("Unhandled named import alias", aliasNode.getText()); + + } + console.log("Named import", namedImport.getNameNode().getText()); + const importNode = namedImport.getNameNode(); + if (importNode.isKind(SyntaxKind.StringLiteral)) { + throw Error("Unexpected string literal import node. Expected identifier"); + } + + recordImportedIdentifierUsage(importNode, mainFilePath, libraryCallsRecorder, importStringLiteral); +} +/** + * + * @param {Identifier} importNode + * @param {string} mainFilePath + * @param {LibraryTypesRecorder} libraryCallsRecorder + * @param {StringLiteral} importStringLiteral + * @param {boolean} [isDefaultImport=false] + */ +function recordImportedIdentifierUsage(importNode, mainFilePath, libraryCallsRecorder, importStringLiteral, isDefaultImport = false) { + const importRefs = importNode.findReferences(); + for (const importRef of importRefs) { + const referenceSourceFile = importRef.getDefinition().getSourceFile(); + const comparePath = path.relative(mainFilePath, referenceSourceFile.getFilePath()); + if (comparePath !== '') { + console.warn("Skipping import reference from other file", referenceSourceFile.getFilePath()); + continue; + } + console.log("Compare path", comparePath === ''); + // const filePath = referenceSourceFile.getFilePath(); + // console.log("Refset for import",filePath); + for (const ref of importRef.getReferences()) { + if (ref.isDefinition()) { + continue; + } + // console.log("I am ",ref.isDefinition()); + const callExpression = ref.getNode().getFirstAncestorByKind(SyntaxKind.CallExpression); + + const callExpressionArguments = callExpression?.getArguments(); + if (callExpressionArguments === undefined || callExpressionArguments.length === 0) { + console.warn("No call expressions found for import reference", ref.getNode().getText()); + continue; + } + + // for(const argument of callExpressionArguments){ + // console.log(`Arg ${idx} is ${arg.getText()}, type is ${arg.getType()}`); + // } + const getImportSection = '.' + (isDefaultImport? 'default':importNode.getText()); + libraryCallsRecorder.pushToMap(importStringLiteral.getLiteralValue(), getImportSection, callExpressionArguments.map(arg => arg.getType())); + + console.log("I am ", callExpression?.getText()); + } + } +} + diff --git a/src_bundle/index.mjs b/src_bundle/index.mjs index 0596231..3f32eb8 100644 --- a/src_bundle/index.mjs +++ b/src_bundle/index.mjs @@ -1,44 +1,65 @@ import wp from 'webpack'; import path from 'node:path' -const outputPath = path.resolve('./output/'); -console.log(outputPath); -const ls = [ -'classnames', -'semver', -'ansi-styles', -'debug', -'supports-color', -'chalk', -'ms', -'minimatch', -'strip-ansi', -'tslib', -'has-flag', -'ansi-regex', -'color-convert', -'color-name', -// 'type-fest', -'string-width' -] -ls.forEach(l=>{ - +if (process.argv[1] === import.meta.filename) { + console.log("[SafePack] started"); + main(); + console.log("done"); +} + +function main() { + const ls = [ + 'classnames', + 'semver', + 'ansi-styles', + // 'debug', + // 'supports-color', + 'chalk', + 'ms', + 'minimatch', + 'strip-ansi', + 'tslib', + 'has-flag', + 'ansi-regex', + 'color-convert', + 'color-name', + // 'type-fest', + 'string-width' + ]; + + ls.forEach(l => { + + wpCompress(l).then(outputFileLocation => { + console.log("[wp] success", outputFileLocation); + }).catch(err => { + console.error("[failed wp]", l); + console.error("[wp] error"); + + }); + + + }); +} + +export function wpCompress(l, outputPath = path.resolve('./output/')) { + return new Promise((resolve, reject) => { const libraryLocation = import.meta.resolve(l); console.log(libraryLocation); + const outputFile = l + '.bundle.cjs'; // throw Error("5"); wp({ - entry:libraryLocation, + entry: libraryLocation, mode: 'production', - optimization:{ + optimization: { mangleExports: false, avoidEntryIife: true, - minimize: false + minimize: false, }, output: { path: outputPath, - filename: l+'.bundle.cjs', + filename: outputFile, clean: false, iife: false, library: { @@ -46,15 +67,17 @@ ls.forEach(l=>{ // name: l } // module: true - } - , - },(err,stats)=>{ + }, + }, (err, stats) => { if (err || stats.hasErrors()) { - console.log(err?.stack) - console.log(stats?.hasErrors()) - console.log(stats?.toJson()); + // console.log(err?.stack); + // console.log(stats?.hasErrors()); + // console.log(stats?.toJson()); + reject(err || stats); + }else{ + resolve(path.resolve(outputPath, outputFile)); } - }) + }); +}); +} - -}) diff --git a/test_src/index.mjs b/test_src/index.mjs index 596a8c9..5461bbd 100644 --- a/test_src/index.mjs +++ b/test_src/index.mjs @@ -1,5 +1,12 @@ -import {readFile} from 'node:fs' +import fs,{readFile} from 'node:fs' + +import classnames from 'classnames' +// import {neq} from 'semver' +import {sum, div,sad} from './arithmetic.cjs'; readFile('a',(err)=>{ if(err){return;} -}) \ No newline at end of file +}) +console.log(classnames('a', 'b', 'c',{a:5})); // $ExpectType string +console.log(sum(2, 3)); +// console.log(neq('1.0.0', '1.0.1')); // $ExpectType boolean \ No newline at end of file