├── .editorconfig ├── .gitignore ├── LICENSE ├── README.md ├── bin └── js-cpa.js ├── client ├── .babelrc ├── components │ ├── Collapse.css │ ├── Collapse.jsx │ ├── Content.css │ ├── Content.jsx │ ├── Header.css │ ├── Header.jsx │ ├── PrettyPrint.css │ ├── PrettyPrint.jsx │ ├── Reporter.css │ ├── Reporter.jsx │ ├── Sidebar.css │ └── Sidebar.jsx ├── index.jsx ├── plugins │ ├── Foldable.css │ └── Foldable.js ├── selection.js ├── style │ ├── colors.css │ ├── index.css │ └── variables.css ├── views │ └── index.ejs └── webpack.config.js ├── lib ├── cli.js ├── cpa.js ├── dfs.js ├── hash.js ├── html-reporter.js ├── index.js ├── reporter.js └── unpad.js ├── package.json ├── test ├── __snapshots__ │ └── cpa.test.js.snap ├── cpa.test.js └── fixtures │ ├── match.js │ ├── modules.js │ └── nomatch.js └── yarn.lock /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | end_of_line = lf 5 | insert_final_newline = true 6 | charset = utf-8 7 | trim_trailing_whitespace = true 8 | indent_style = space 9 | indent_size = 2 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | public 3 | js-cpa-report.html 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2017 [Vignesh Shanmugam](https://vigneshh.in) 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # js-cpa 2 | 3 | Identify structurally similar functions that are duplicated across a JavaScript bundle/file seamlessly. 4 | 5 | Running it on [Inferno](https://github.com/infernojs/inferno) 6 | 7 | CPA on Inferno 8 | 9 | ### Features 10 | 11 | + Works only across function boundaries 12 | + Matches the longest common subsequence and ignores the children 13 | + Ignores comments on the output 14 | + HTML report generation 15 | 16 | ### Related 17 | 18 | + [bundle-duplicates-plugin](https://github.com/vigneshshanmugam/bundle-duplicates-plugin) - Identify duplicate functions across all webpack JS bundles. 19 | 20 | ### Installation 21 | 22 | ```sh 23 | npm install -g js-cpa 24 | ``` 25 | 26 | ### CLI 27 | ```sh 28 | Usage: js-cpa [options] 29 | 30 | 31 | Options: 32 | 33 | -V, --version output the version number 34 | -f, --filelist read filelist from STDIN stream - if/when you cross ARG_MAX. eg: ls *.js | js-cpa -f 35 | -m, --module parse files with sourceType=module 36 | -l, --language language (js|ts|flow) 37 | -t, --threshold threshold (in bytes) 38 | -C, --no-colors disable colors in output 39 | -R, --report generate reports (html|term) 40 | -o, --report-file path for report generation 41 | -h, --help output usage information 42 | ``` 43 | 44 | ### API 45 | 46 | ```js 47 | const { findDuplicates, stringify }= require('js-cpa'); 48 | const fs = require("fs"); 49 | const code = fs.readFileSync(filePath, "utf-8"); 50 | const duplicates = findDuplicates(code, { 51 | filename: "test" 52 | }); 53 | 54 | process.stdout.write(stringify(duplicates)); // prints to stdout 55 | ``` 56 | 57 | ##### Options 58 | ------ 59 | + filename - name of the file used in the output 60 | + sourceType - denotes the mode the code should be parsed in. eg - script|module 61 | + language - denotes the language. eg - (js|ts|flow) 62 | + threshold - threshold in bytes 63 | 64 | ##### findDuplicates(code, opts) 65 | ------ 66 | Finds the optimal duplicate functions that are structurally similar. It tries to find the longest matching subexpression across the AST which ignores the children(inner function) if the parent(outer function) is already mached 67 | 68 | ##### findAllDuplicates(code, opts) 69 | ------ 70 | Finds all duplicate functions that are structurally similar. 71 | 72 | ##### stringify(duplicates, options) 73 | ------ 74 | Gets the output in a more presentable way 75 | 76 | Options 77 | + colors - enable colors on the stdout 78 | + newline - prints newline after each duplicates 79 | -------------------------------------------------------------------------------- /bin/js-cpa.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | require("../lib/cli"); 3 | -------------------------------------------------------------------------------- /client/.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "presets": [ 3 | [ 4 | "env", 5 | { 6 | "targets": { 7 | "browsers": ["> 5%", "last 2 versions"] 8 | } 9 | } 10 | ] 11 | ], 12 | "plugins": [ 13 | ["transform-react-jsx", { "pragma": "h" }] 14 | ] 15 | } 16 | -------------------------------------------------------------------------------- /client/components/Collapse.css: -------------------------------------------------------------------------------- 1 | @import '../style/colors.css'; 2 | @import '../style/variables.css'; 3 | 4 | .collapse { 5 | list-style-type: none; 6 | padding: 0; 7 | } 8 | -------------------------------------------------------------------------------- /client/components/Collapse.jsx: -------------------------------------------------------------------------------- 1 | import { h, Component } from "preact"; 2 | import cx from "classnames"; 3 | 4 | import styles from "./Collapse.css"; 5 | 6 | export default class Collapse extends Component { 7 | constructor(...args) { 8 | super(...args); 9 | this.state = { 10 | activeIndex: 0 11 | }; 12 | } 13 | 14 | render(props, state) { 15 | return
    Hello
; 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /client/components/Content.css: -------------------------------------------------------------------------------- 1 | @import "../style/colors.css"; 2 | @import "../style/variables.css"; 3 | 4 | :root { 5 | --content-padding-left: 20px; 6 | --section-padding: 10px; 7 | } 8 | 9 | .content { 10 | position: relative; 11 | display: inline-block; 12 | overflow-y: auto; 13 | overflow-x: hidden; 14 | flex-grow: 1; 15 | z-index: 1; 16 | -webkit-overflow-scrolling: touch; 17 | margin-left: var(--sidebar-width); 18 | padding: var(--content-padding-left); 19 | } 20 | 21 | .printWrapper { 22 | display: grid; 23 | list-style-type: none; 24 | padding: 0; 25 | margin: 0; 26 | grid-gap: 10px; 27 | grid-template-columns: 100%; 28 | } 29 | 30 | .printTitle { 31 | font-size: 14px; 32 | font-weight: var(--font-weight-bold); 33 | line-height: 24px; 34 | letter-spacing: 0; 35 | margin: 0; 36 | padding: 0; 37 | } 38 | 39 | .section { 40 | border-bottom: 1px solid #ddd; 41 | } 42 | 43 | .sectionHeader { 44 | cursor: pointer; 45 | -webkit-tap-highlight-color: transparent; 46 | line-height: 1.5; 47 | padding: var(--section-padding) 0; 48 | border-bottom: 1px solid #ddd; 49 | } 50 | 51 | .sectionBody { 52 | padding: var(--section-padding) 0; 53 | display: none; 54 | } 55 | 56 | .section.active .sectionBody { 57 | display: block; 58 | } 59 | 60 | .code mark { 61 | background-color: transparent; 62 | border-bottom: solid 1px white; 63 | } 64 | -------------------------------------------------------------------------------- /client/components/Content.jsx: -------------------------------------------------------------------------------- 1 | import { h, Component } from "preact"; 2 | import cx from "classnames"; 3 | import PrettyPrint from "./PrettyPrint"; 4 | import styles from "./Content.css"; 5 | 6 | export function getLineInfo({ sourceLines, loc, baseLine, margin }) { 7 | const { end: { line: endLine }, start: { line: startLine } } = loc; 8 | let highlightStart = startLine; 9 | let highlightEnd = endLine; 10 | let dataStart = baseLine; 11 | 12 | let realStartLine = startLine - margin - 1; 13 | if (realStartLine < 0) { 14 | realStartLine = 0; 15 | } else { 16 | highlightStart -= realStartLine; 17 | highlightEnd -= realStartLine; 18 | dataStart += realStartLine; 19 | } 20 | let realEndLine = endLine + margin; 21 | if (realEndLine >= sourceLines.length) { 22 | realEndLine = sourceLines.length; 23 | } 24 | 25 | return { 26 | dataStart, 27 | highlightStart, 28 | highlightEnd, 29 | realStartLine, 30 | realEndLine 31 | }; 32 | } 33 | 34 | function strSplice(str, idx, rem, ins) { 35 | return str.slice(0, idx) + ins + str.slice(idx + rem); 36 | } 37 | 38 | export const Code = ({ sourceLines, baseLine, margin, loc }) => { 39 | const { 40 | dataStart, 41 | highlightEnd, 42 | highlightStart, 43 | realStartLine, 44 | realEndLine 45 | } = getLineInfo({ sourceLines, baseLine, margin, loc }); 46 | 47 | let dataLine = `${highlightStart}-${highlightEnd}`; 48 | 49 | const outputLines = sourceLines.slice(realStartLine, realEndLine); 50 | 51 | const startLineNumber = margin; 52 | const endLineNumber = outputLines.length - margin - 1; 53 | 54 | if (startLineNumber === endLineNumber) { 55 | outputLines[margin] = strSplice( 56 | outputLines[margin], 57 | loc.start.column, 58 | 0, 59 | "" 60 | ); 61 | 62 | outputLines[endLineNumber] = strSplice( 63 | outputLines[endLineNumber], 64 | loc.end.column + "".length, 65 | 0, 66 | "" 67 | ); 68 | } 69 | 70 | let outputCode = outputLines.join("\n"); 71 | 72 | return ( 73 | 78 | {outputCode} 79 | 80 | ); 81 | }; 82 | 83 | export class File extends Component { 84 | constructor(...args) { 85 | super(...args); 86 | this.state = { 87 | isActive: this.isActive(this.props) 88 | }; 89 | 90 | this.toggleActive = () => this.setState({ isActive: !this.state.isActive }); 91 | } 92 | 93 | isActive(props) { 94 | let lines = 0; 95 | for (let i = 0; i < props.file.sourceCode.length; i++) { 96 | if (props.file.sourceCode[i] === "\n") lines++; 97 | } 98 | return lines < 50; 99 | } 100 | 101 | componentWillReceiveProps(nextProps) { 102 | this.setState({ 103 | isActive: this.isActive(nextProps) 104 | }); 105 | } 106 | 107 | shouldComponentUpdate(nextProps, nextState) { 108 | // FIXME (boopathi) 109 | return true; 110 | 111 | return ( 112 | this.props.data.matchingCode !== nextProps.data.matchingCode || 113 | nextState.isActive !== this.state.isActive 114 | ); 115 | } 116 | 117 | render({ file, baseLine, margin }, { isActive }) { 118 | const sourceLines = file.sourceCode.split("\n"); 119 | return ( 120 |
  • 125 |
    126 |
    {file.filename}
    127 |
    128 |
    129 | {file.nodes.map(node => ( 130 | 136 | ))} 137 |
    138 |
  • 139 | ); 140 | } 141 | } 142 | 143 | // Content 144 | export default ({ 145 | data, 146 | // number of lines to print before and after the highlight 147 | margin = 1, 148 | baseLine = 1 149 | }) => { 150 | const getData = data => console.log(data); 151 | return ( 152 |
    153 |
      154 | {data.map((file, idx) => ( 155 | 163 | ))} 164 |
    165 |
    166 | ); 167 | }; 168 | -------------------------------------------------------------------------------- /client/components/Header.css: -------------------------------------------------------------------------------- 1 | @import '../style/colors.css'; 2 | @import '../style/variables.css'; 3 | 4 | :root { 5 | --header-height: 40px; 6 | } 7 | 8 | .header { 9 | position: fixed; 10 | z-index: 10; 11 | width: 100%; 12 | top: 0; 13 | background: var(--color-primary); 14 | color: var(--color-white); 15 | height: var(--header-height); 16 | padding: 0 10px; 17 | 18 | box-sizing: border-box; 19 | font-family: var(--preferred-font); 20 | -webkit-font-smoothing: antialiased; 21 | font-smoothing: antialiased; 22 | text-size-adjust: 100%; 23 | 24 | & *, 25 | & *::after, 26 | & *::before { 27 | box-sizing: border-box; 28 | -webkit-font-smoothing: antialiased; 29 | font-smoothing: antialiased; 30 | text-size-adjust: 100%; 31 | -webkit-touch-callout: none; 32 | } 33 | } 34 | 35 | .headerRow { 36 | display: flex; 37 | flex-direction: row; 38 | flex-wrap: nowrap; 39 | flex-shrink: 0; 40 | box-sizing: border-box; 41 | align-self: stretch; 42 | align-items: center; 43 | margin: 0; 44 | height: var(--header-height); 45 | } 46 | 47 | .headerTitle { 48 | font-size: 20px; 49 | font-weight: 500; 50 | line-height: 1; 51 | letter-spacing: 0.1px; 52 | display: block; 53 | position: relative; 54 | 55 | font-weight: var(--font-weight-bold); 56 | box-sizing: border-box; 57 | } 58 | -------------------------------------------------------------------------------- /client/components/Header.jsx: -------------------------------------------------------------------------------- 1 | import { h } from "preact"; 2 | import cx from "classnames"; 3 | 4 | import styles from "./Header.css"; 5 | 6 | export default ({ title, className }) => { 7 | return ( 8 |
    9 |
    10 | {title} 11 |
    12 |
    13 | ); 14 | }; 15 | -------------------------------------------------------------------------------- /client/components/PrettyPrint.css: -------------------------------------------------------------------------------- 1 | :global .line-highlight { 2 | background: rgba(124, 242, 241, .3); 3 | } 4 | -------------------------------------------------------------------------------- /client/components/PrettyPrint.jsx: -------------------------------------------------------------------------------- 1 | import { h, Component } from "preact"; 2 | import cx from "classnames"; 3 | import Prism from "prismjs"; 4 | 5 | // Import prism plugins 6 | 7 | import "prismjs/plugins/line-numbers/prism-line-numbers"; 8 | import "prismjs/plugins/line-highlight/prism-line-highlight"; 9 | import "prismjs/plugins/keep-markup/prism-keep-markup"; 10 | // import "../plugins/Foldable"; 11 | 12 | import "prismjs/themes/prism-okaidia.css"; 13 | import "prismjs/plugins/line-numbers/prism-line-numbers.css"; 14 | import "prismjs/plugins/line-highlight/prism-line-highlight.css"; 15 | // import "../plugins/Foldable.css"; 16 | 17 | import styles from "./PrettyPrint.css"; 18 | 19 | export default class PrettyPrint extends Component { 20 | constructor(...args) { 21 | super(...args); 22 | this._timer1 = null; 23 | this._timer2 = null; 24 | } 25 | componentDidMount() { 26 | this.highlight(this.props); 27 | } 28 | shouldComponentUpdate(nextProps) { 29 | return false; 30 | } 31 | componentWillReceiveProps(nextProps) { 32 | this.highlight(nextProps); 33 | } 34 | highlight(props) { 35 | this._pre.dataset.line = props.dataLine; 36 | this._pre.dataset.start = props.dataStart; 37 | // this._pre.dataset.open = "2-4,5-7,11-25"; 38 | 39 | clearInterval(this._timer1); 40 | clearInterval(this._timer2); 41 | 42 | this._timer1 = setTimeout(() => { 43 | this._code.innerHTML = props.children[0]; 44 | this._timer2 = setTimeout(() => Prism.highlightElement(this._code), 1); 45 | }, 1); 46 | } 47 | render({ className }) { 48 | return ( 49 |
     (this._pre = ref)} className={styles.pre}>
    50 |          (this._code = ref)} />
    51 |       
    52 | ); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /client/components/Reporter.css: -------------------------------------------------------------------------------- 1 | @import '../style/colors.css'; 2 | @import '../style/variables.css'; 3 | 4 | .jsCpa { 5 | /* position: absolute; 6 | width: 100%; 7 | height: 100%; */ 8 | } 9 | 10 | .layout { 11 | width: 100%; 12 | height: 100%; 13 | display: flex; 14 | flex-direction: column; 15 | margin-top: 40px; 16 | overflow-y: auto; 17 | overflow-x: hidden; 18 | position: relative; 19 | -webkit-overflow-scrolling: touch; 20 | } 21 | -------------------------------------------------------------------------------- /client/components/Reporter.jsx: -------------------------------------------------------------------------------- 1 | import { h, Component } from "preact"; 2 | 3 | import Header from "./Header"; 4 | import Sidebar from "./Sidebar"; 5 | import Content from "./Content"; 6 | 7 | import styles from "./Reporter.css"; 8 | 9 | export default class Reporter extends Component { 10 | constructor(...args) { 11 | super(...args); 12 | this.state = { 13 | reportType: "count", 14 | activeIndex: 0 15 | }; 16 | 17 | this.handleItemChange = this.handleItemChange.bind(this); 18 | } 19 | 20 | handleItemChange(index) { 21 | this.setState({ activeIndex: index }); 22 | } 23 | 24 | render({ data }, { reportType, activeIndex }) { 25 | const sidebarProps = { 26 | data, 27 | reportType, 28 | handleItemChange: this.handleItemChange, 29 | activeIndex 30 | }; 31 | const contentProps = { 32 | data: data[activeIndex] 33 | }; 34 | return ( 35 |
    36 |
    37 |
    38 | 39 | 40 |
    41 |
    42 | ); 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /client/components/Sidebar.css: -------------------------------------------------------------------------------- 1 | @import "../style/colors.css"; 2 | @import "../style/variables.css"; 3 | 4 | :root { 5 | --sidebar-title-line-height: 30px; 6 | --sidebar-title-padding-horizontal: 20px; 7 | --sidebar-title-padding-vertical: 20px; 8 | 9 | --list-item-min-height: 40px; 10 | --list-item-height: 50px; 11 | --list-item-min-padding: 0 20px; 12 | } 13 | 14 | .sidebar { 15 | display: flex; 16 | flex-direction: column; 17 | flex-wrap: nowrap; 18 | position: fixed; 19 | width: var(--sidebar-width); 20 | height: 100%; 21 | max-height: 100%; 22 | box-sizing: border-box; 23 | border-right: 1px solid var(--palette-100); 24 | background: var(--palette-50); 25 | color: var(--color-black); 26 | overflow-y: auto; 27 | } 28 | 29 | .sidebar-title { 30 | font-size: 20px; 31 | font-weight: 500; 32 | line-height: 1; 33 | letter-spacing: 0.1px; 34 | display: block; 35 | position: relative; 36 | 37 | font-weight: var(--font-weight-bold); 38 | box-sizing: border-box; 39 | line-height: var(--sidebar-title-line-height); 40 | padding-left: var(--sidebar-title-padding-horizontal); 41 | padding-right: var(--sidebar-title-padding-horizontal); 42 | padding-top: var(--sidebar-title-padding-vertical); 43 | font-weight: var(--font-weight-normal); 44 | } 45 | 46 | .sidebar-list { 47 | display: block; 48 | padding: 0; 49 | list-style: none; 50 | } 51 | 52 | .sidebar-list-item { 53 | font-size: 16px; 54 | font-weight: 400; 55 | line-height: 24px; 56 | letter-spacing: 0.1px; 57 | line-height: 1; 58 | display: flex; 59 | min-height: var(--list-item-min-height); 60 | height: var(--list-item-height); 61 | box-sizing: border-box; 62 | flex-direction: row; 63 | flex-wrap: nowrap; 64 | align-items: center; 65 | padding: var(--list-item-min-padding); 66 | cursor: default; 67 | color: var(--color-black); 68 | overflow: hidden; 69 | 70 | cursor: pointer; 71 | } 72 | 73 | .active { 74 | background-color: var(--palette-100); 75 | } 76 | 77 | .sidebar-list-item-content { 78 | height: 48px; 79 | line-height: 20px; 80 | display: block; 81 | } 82 | 83 | .sidebar-list-item-content-sub { 84 | font-size: 12px; 85 | font-weight: 400; 86 | line-height: 24px; 87 | letter-spacing: 0; 88 | line-height: 18px; 89 | color: var(--palette-500); 90 | display: block; 91 | padding: 0; 92 | } 93 | -------------------------------------------------------------------------------- /client/components/Sidebar.jsx: -------------------------------------------------------------------------------- 1 | import { h } from "preact"; 2 | import cx from "classnames"; 3 | 4 | import styles from "./Sidebar.css"; 5 | 6 | const SidebarItem = ({ title, subtitle, isActive, onChange }) => ( 7 |
  • 13 | 14 | {title} 15 | 16 | {subtitle} 17 | 18 | 19 |
  • 20 | ); 21 | 22 | export default ({ data, reportType, handleItemChange, activeIndex }) => { 23 | const numMatches = dups => 24 | dups.map(file => file.nodes.length).reduce((acc, cur) => acc + cur, 0); 25 | 26 | const getFnLength = dups => dups[0].nodes[0].match.length; 27 | 28 | return ( 29 |
    30 |
    31 | Duplicates by {`${reportType}`} 32 |
    33 |
      34 | {data.map((value, index) => ( 35 | handleItemChange(index)} 41 | /> 42 | ))} 43 |
    44 |
    45 | ); 46 | }; 47 | -------------------------------------------------------------------------------- /client/index.jsx: -------------------------------------------------------------------------------- 1 | import { h, render } from "preact"; 2 | import Reporter from "./components/Reporter"; 3 | import Selection from "./selection"; 4 | import "./style/index.css"; 5 | 6 | render(, document.getElementById("reporterApp")); 7 | 8 | new Selection().disableDoubleClicks(); 9 | -------------------------------------------------------------------------------- /client/plugins/Foldable.css: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vigneshshanmugam/js-cpa/04fe147406e538fe73e5585e53c2bf00350eb56e/client/plugins/Foldable.css -------------------------------------------------------------------------------- /client/plugins/Foldable.js: -------------------------------------------------------------------------------- 1 | Prism.hooks.add("complete", env => { 2 | const pre = env.element.parentNode; 3 | const dataOpen = pre.dataset.open; 4 | 5 | if (!dataOpen) return; 6 | 7 | const element = env.element; 8 | 9 | const lines = []; 10 | 11 | // for (let i = 0; i < element.childNodes.length; i++) { 12 | // const node = element.childNodes[i]; 13 | // if (node.nodeType === Node.TEXT_NODE) { 14 | // const split = node.nodeValue.split("\n"); 15 | // if (split.length > 1) { 16 | // const previousSibling = node; 17 | // const parent = node.parentNode; 18 | // node.nodeValue = split[0]; 19 | // for (let j = 1; j < split.length; j++) {} 20 | // } 21 | // lines.push(...node.nodeValue.split("\n")); 22 | // } 23 | // } 24 | 25 | // console.log(lines); 26 | 27 | // console.log(getLinesShown(dataOpen, 0, 100)); 28 | }); 29 | 30 | function getLinesShown(dataOpen, start, end) { 31 | const parts = dataOpen 32 | .split(",") 33 | .map(part => part.split("-").map(it => parseInt(it, 10))); 34 | 35 | let lines = Array.from({ length: end - start }, () => ({ 36 | type: "HIDE" 37 | })); 38 | 39 | for (const part of parts) { 40 | for (let i = part[0]; i < part[1]; i++) { 41 | lines[i].type = "SHOW"; 42 | } 43 | } 44 | 45 | return lines; 46 | } 47 | -------------------------------------------------------------------------------- /client/selection.js: -------------------------------------------------------------------------------- 1 | // Source: https://stackoverflow.com/a/4117520 2 | 3 | const timeout = 500; 4 | export default class Selection { 5 | constructor(d = document, w = window) { 6 | this.tcTimer = 0; 7 | this.mouseDown = false; 8 | this.document = d; 9 | this.window = w; 10 | 11 | this.clickCallback = this.clickCallback.bind(this); 12 | } 13 | 14 | clickCallback() { 15 | this.clearSelection(); 16 | } 17 | 18 | disableDoubleClicks() { 19 | this.document.addEventListener("dblclick", () => this.clearSelection()); 20 | } 21 | 22 | disableAll() { 23 | this.document.addEventListener("mousedown", () => (this.mouseDown = true)); 24 | this.document.addEventListener("mouseup", () => (this.mouseDown = false)); 25 | 26 | this.document.addEventListener("dblclick", e => { 27 | this.clearSelection(); 28 | this.window.clearTimeout(this.tcTimer); 29 | this.document.addEventListener("click", this.clickCallback); 30 | this.tcTimer = this.window.setTimeout( 31 | () => this.unregisterClick(), 32 | timeout 33 | ); 34 | }); 35 | } 36 | 37 | unregisterClick() { 38 | if (!this.mouseDown) { 39 | this.document.removeEventListener("click", this.clickCallback); 40 | return true; 41 | } 42 | this.tcTimer = this.window.setTimeout( 43 | () => this.unregisterClick(), 44 | timeout 45 | ); 46 | return false; 47 | } 48 | 49 | clearSelection() { 50 | if (this.window.getSelection) { 51 | this.window.getSelection().removeAllRanges(); 52 | } 53 | if (this.document.selection) { 54 | this.document.selection.empty(); 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /client/style/colors.css: -------------------------------------------------------------------------------- 1 | :root { 2 | --palette-50: rgb(224, 242, 241); 3 | --palette-100: rgb(178, 223, 219); 4 | --palette-200: rgb(128, 203, 196); 5 | --palette-300: rgb(77, 182, 172); 6 | --palette-400: rgb(38, 166, 154); 7 | --palette-500: rgb(0, 150, 136); 8 | --palette-600: rgb(0, 137, 123); 9 | --palette-700: rgb(0, 121, 107); 10 | --palette-800: rgb(0, 105, 92); 11 | --palette-900: rgb(0, 77, 64); 12 | --palette-a100: rgb(167, 255, 235); 13 | --palette-a200: rgb(100, 255, 218); 14 | --palette-a400: rgb(29, 233, 182); 15 | --palette-a700: rgb(0, 191, 165); 16 | 17 | --color-black: rgb(0, 0, 0); 18 | --color-white: rgb(255, 255, 255); 19 | --color-dark-contrast: var(--color-white); 20 | --color-light-contrast: var(--color-black); 21 | } 22 | -------------------------------------------------------------------------------- /client/style/index.css: -------------------------------------------------------------------------------- 1 | @import './colors.css'; 2 | @import './variables.css'; 3 | :global { 4 | @import 'normalize.css'; 5 | } 6 | 7 | * { 8 | font-size: var(--font-size); 9 | box-sizing: border-box; 10 | font-family: var(--preferred-font); 11 | -webkit-font-smoothing: antialiased; 12 | font-smoothing: antialiased; 13 | text-size-adjust: 100%; 14 | 15 | & *, 16 | & *::after, 17 | & *::before { 18 | box-sizing: border-box; 19 | -webkit-font-smoothing: antialiased; 20 | font-smoothing: antialiased; 21 | text-size-adjust: 100%; 22 | -webkit-touch-callout: none; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /client/style/variables.css: -------------------------------------------------------------------------------- 1 | :root { 2 | --unit: 10px; 3 | 4 | --color-background: var(--color-white); 5 | --color-text: var(--color-black); 6 | 7 | --color-primary: var(--palette-500); 8 | --color-primary-dark: var(--palette-700); 9 | 10 | --preferred-font: -apple-system, BlinkMacSystemFont, "Segoe UI", "Roboto", 11 | "Oxygen", "Ubuntu", "Cantarell", "Fira Sans", "Droid Sans", "Helvetica Neue", 12 | sans-serif; 13 | 14 | --font-size: calc(1.6 * var(--unit)); 15 | --font-weight-thin: 300; 16 | --font-weight-normal: 400; 17 | --font-weight-bold: 700; 18 | 19 | --sidebar-width: 200px; 20 | 21 | --reset { 22 | box-sizing: border-box; 23 | font-family: var(--preferred-font); 24 | -webkit-font-smoothing: antialiased; 25 | font-smoothing: antialiased; 26 | text-size-adjust: 100%; 27 | 28 | & *, 29 | & *::after, 30 | & *::before { 31 | box-sizing: border-box; 32 | -webkit-font-smoothing: antialiased; 33 | font-smoothing: antialiased; 34 | text-size-adjust: 100%; 35 | -webkit-touch-callout: none; 36 | } 37 | } 38 | } 39 | -------------------------------------------------------------------------------- /client/views/index.ejs: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | JS-CPA Report 9 | 10 | 11 | 14 | 15 | 16 | 17 |
    18 | 21 | 22 | 26 | 27 | 28 | 29 | -------------------------------------------------------------------------------- /client/webpack.config.js: -------------------------------------------------------------------------------- 1 | const webpack = require("webpack"); 2 | const path = require("path"); 3 | 4 | const MinifyPlugin = require("babel-minify-webpack-plugin"); 5 | const ExtractTextPlugin = require("extract-text-webpack-plugin"); 6 | 7 | const isDev = process.env.NODE_ENV !== "production"; 8 | module.exports = { 9 | stats: { 10 | modules: false 11 | }, 12 | context: __dirname, 13 | entry: { 14 | reporter: "./index" 15 | }, 16 | output: { 17 | path: path.join(__dirname, "..", "public"), 18 | filename: "[name].js", 19 | publicPath: "/" 20 | }, 21 | resolve: { 22 | extensions: [".js", ".jsx"] 23 | }, 24 | devtool: isDev ? "cheap-module-inline-source-map" : "source-map", 25 | module: { 26 | rules: [ 27 | { 28 | test: /\.jsx?$/, 29 | exclude: /node_modules/, 30 | loader: "babel-loader" 31 | }, 32 | { 33 | test: /\.css$/, 34 | include: /node_modules/, 35 | use: ExtractTextPlugin.extract({ 36 | fallback: "style-loader", 37 | use: [ 38 | { 39 | loader: "css-loader", 40 | options: { 41 | minimize: !isDev 42 | } 43 | } 44 | ] 45 | }) 46 | }, 47 | { 48 | test: /\.css$/, 49 | exclude: /node_modules/, 50 | use: ExtractTextPlugin.extract({ 51 | fallback: "style-loader", 52 | use: [ 53 | { 54 | loader: "css-loader", 55 | options: { 56 | minimize: !isDev, 57 | modules: true, 58 | importLoaders: 1, 59 | localIdentName: isDev 60 | ? "[name]__[local]___[hash:base64:5]" 61 | : "[hash:base64:8]" 62 | } 63 | } 64 | ] 65 | }) 66 | } 67 | ] 68 | }, 69 | plugins: [new ExtractTextPlugin("[name].css")].concat( 70 | isDev 71 | ? [] 72 | : [ 73 | new MinifyPlugin(), 74 | new webpack.optimize.ModuleConcatenationPlugin(), 75 | new webpack.optimize.AggressiveMergingPlugin() 76 | ] 77 | ) 78 | }; 79 | -------------------------------------------------------------------------------- /lib/cli.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const fs = require("fs"); 4 | const path = require("path"); 5 | const chalk = require("chalk"); 6 | const readline = require("readline"); 7 | const program = require("commander"); 8 | const mkdirp = require("mkdirp"); 9 | 10 | const CPA = require("./cpa"); 11 | const Reporter = require("./reporter"); 12 | const HTMLReporter = require("./html-reporter"); 13 | 14 | program 15 | .version(require("../package.json").version) 16 | .usage("[options] ") 17 | .option( 18 | "-f, --filelist", 19 | "read filelist from STDIN stream - if/when you cross ARG_MAX. eg: ls *.js | js-cpa -f" 20 | ) 21 | .option("-m, --module", "parse files with sourceType=module") 22 | .option( 23 | "-l, --language ", 24 | "language (js|ts|flow)", 25 | /^(js|ts|flow)$/i, 26 | "js" 27 | ) 28 | .option( 29 | "-t, --threshold ", 30 | "Threshold (in bytes)", 31 | x => parseInt(x, 10), 32 | 100 33 | ) 34 | .option("-C, --no-colors", "Disable colors in output") 35 | .option( 36 | "-R, --report ", 37 | "Generate HTML report", 38 | /^(html|term)$/i, 39 | "term" 40 | ) 41 | .option( 42 | "-o, --report-file ", 43 | "/path/to/report.html", 44 | "./js-cpa-report.html" 45 | ) 46 | .parse(process.argv); 47 | 48 | const cpaOpts = { 49 | sourceType: program.module ? "module" : "script", 50 | language: program.language, 51 | threshold: program.threshold 52 | }; 53 | 54 | if (program.filelist) { 55 | const rl = readline.createInterface({ 56 | input: process.stdin, 57 | output: process.stdout, 58 | terminal: false 59 | }); 60 | 61 | const cpa = new CPA(cpaOpts); 62 | const errors = []; 63 | 64 | rl.on("line", input => { 65 | clearBuffer(); 66 | currentProgress(input); 67 | try { 68 | cpa.add(fs.readFileSync(input, "utf-8").toString(), { filename: input }); 69 | } catch (err) { 70 | errors.push({ file: input, err }); 71 | } 72 | }); 73 | 74 | rl.on("close", () => { 75 | clearBuffer(); 76 | try { 77 | if (program.report === "html") { 78 | generateHTMLReport(cpa, program.reportFile); 79 | } else { 80 | print(cpa); 81 | } 82 | } catch (err) { 83 | errors.push({ err }); 84 | } 85 | 86 | if (errors.length > 0) { 87 | process.stderr.write( 88 | "The following files were ignored because of errors:\n" 89 | ); 90 | 91 | errors.map(e => 92 | process.stderr.write(e.file + "\n" + e.err.message + "\n") 93 | ); 94 | 95 | const otherErrors = errors.filter(e => !e.file); 96 | 97 | if (otherErrors.length > 0) { 98 | process.stderr.write("Other errors:\n"); 99 | otherErrors.map(e => { 100 | process.stderr.write(e.message + "\n"); 101 | }); 102 | } 103 | 104 | process.exit(1); 105 | } 106 | }); 107 | } else if (process.stdin.isTTY) { 108 | const files = program.args; 109 | if (files.length < 1) { 110 | throw new Error("Need at least 1 file to process"); 111 | } 112 | const cpa = new CPA(cpaOpts); 113 | for (const file of files) { 114 | cpa.add(fs.readFileSync(file, "utf-8").toString(), { filename: file }); 115 | } 116 | if (program.report === "html") { 117 | generateHTMLReport(cpa, program.reportFile); 118 | } else { 119 | print(cpa); 120 | } 121 | } else { 122 | // handle piped content 123 | const inputChunks = []; 124 | process.stdin.resume(); 125 | process.stdin.setEncoding("utf-8"); 126 | process.stdin.on("data", chunk => inputChunks.push(chunk)); 127 | process.stdin.on("end", () => { 128 | const code = inputChunks.join(); 129 | const cpa = new CPA(cpaOpts); 130 | cpa.add(code, { filename: "dummy" + Math.ceil(Math.random() * 10) }); 131 | if (program.report === "html") { 132 | generateHTMLReport(cpa, program.reportFile); 133 | } else { 134 | print(cpa); 135 | } 136 | }); 137 | } 138 | 139 | function print(cpa) { 140 | const duplicates = cpa.findOptimalDuplicates(); 141 | const report = new Reporter(duplicates); 142 | process.stdout.write(report.toString({ colors: program.colors })); 143 | process.exit(0); 144 | } 145 | 146 | function generateHTMLReport(cpa, reportFile) { 147 | const duplicates = cpa.findOptimalDuplicates(); 148 | const sources = cpa.getSource(); 149 | const report = new HTMLReporter(duplicates, sources); 150 | 151 | report 152 | .renderFile() 153 | .then(({ type, warning, html }) => { 154 | if (type === HTMLReporter.TYPE_WARNING) { 155 | warn(warning); 156 | process.exit(1); 157 | } else if (type === HTMLReporter.TYPE_RESULT) { 158 | mkdirp.sync(path.dirname(reportFile)); 159 | fs.writeFileSync(reportFile, html, "utf-8"); 160 | write("CPA report generated at " + reportFile); 161 | process.exit(0); 162 | } else { 163 | error("Unknown Error from HTML Reporter"); 164 | process.exit(1); 165 | } 166 | }) 167 | .catch(err => { 168 | error(err.message); 169 | process.exit(1); 170 | }); 171 | 172 | function warn(message) { 173 | process.stderr.write( 174 | program.colors ? chalk.yellow(chalk.bold(message)) : message 175 | ); 176 | } 177 | function write(message) { 178 | process.stderr.write(program.colors ? chalk.green(message) : message); 179 | } 180 | function error(message) { 181 | process.stderr.write(program.colors ? chalk.red(message) : message); 182 | } 183 | } 184 | 185 | function clearBuffer() { 186 | process.stderr.clearLine(); 187 | } 188 | 189 | function currentProgress(file) { 190 | process.stderr.write(`Current File: ${file}\r`); 191 | } 192 | -------------------------------------------------------------------------------- /lib/cpa.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const { parse } = require("@babel/parser"); 4 | const t = require("@babel/types"); 5 | const dfs = require("./dfs"); 6 | const hashcons = require("./hash"); 7 | 8 | // Used to mark a node as Duplicate 9 | // refer below findOptimalDuplicates description for more info 10 | const DUPLICATE = Symbol("DUPLICATE"); 11 | 12 | module.exports = class CPA { 13 | constructor({ 14 | sourceType = "script", 15 | language = "js", 16 | threshold = 100 17 | } = {}) { 18 | // hash consing 19 | this.hashes = new Map(); 20 | // to find optimal duplicates 21 | this.ids = new Map(); 22 | this.sources = new Map(); 23 | 24 | this.sourceType = sourceType; 25 | this.language = language; 26 | this.threshold = threshold; 27 | } 28 | 29 | // Exposing it from here for using in reporter 30 | // Avoids reading from FS again 31 | getSource() { 32 | return this.sources; 33 | } 34 | 35 | add( 36 | code, 37 | { 38 | filename = "", 39 | sourceType = this.sourceType, 40 | language = this.language 41 | } = {} 42 | ) { 43 | this.sources.set(filename, code); 44 | 45 | const languagePlugins = []; 46 | switch (language) { 47 | case "flow": 48 | languagePlugins.push("flow"); 49 | break; 50 | case "ts": 51 | languagePlugins.push("typescript"); 52 | break; 53 | } 54 | 55 | const root = parse(code, { 56 | sourceFilename: filename, 57 | sourceType: sourceType, 58 | plugins: [ 59 | ...languagePlugins, 60 | "jsx", 61 | "doExpressions", 62 | "objectRestSpread", 63 | "decorators2", 64 | "classProperties", 65 | "classPrivateProperties", 66 | "exportExtensions", 67 | "asyncGenerators", 68 | "functionBind", 69 | "functionSent", 70 | "dynamicImport", 71 | "numericSeparator", 72 | "optionalChaining", 73 | "importMeta", 74 | "bigInt" 75 | ] 76 | }).program; 77 | 78 | this.addNode(root, filename); 79 | } 80 | 81 | /** 82 | * Performs a DFS Post Order traversal from the root and 83 | * hashconses to `this.hashes` 84 | * 85 | * @param {ASTNode} root 86 | * @param {String} id 87 | */ 88 | addNode(root, id /* for example filename */) { 89 | this.ids.set(id, root); 90 | 91 | dfs(root, { 92 | // post order traversal callback 93 | exit: node => { 94 | if (!t.isFunction(node)) { 95 | return; 96 | } 97 | 98 | if (node.end - node.start < this.threshold) { 99 | return; 100 | } 101 | 102 | if (!node.hash) { 103 | node.hash = hashcons(node); 104 | } 105 | 106 | if (!node.sourceCode) { 107 | node.sourceCode = this.sources.get(id).slice(node.start, node.end); 108 | } 109 | 110 | if (this.hashes.has(node.hash)) { 111 | // set the current node as a duplicate 112 | node[DUPLICATE] = true; 113 | const duplicateNode = this.hashes.get(node.hash); 114 | // set the previously pushed node as duplicate 115 | duplicateNode[0].node[DUPLICATE] = true; 116 | duplicateNode.push({ node, id }); 117 | } else { 118 | this.hashes.set(node.hash, [{ node, id }]); 119 | } 120 | } 121 | }); 122 | } 123 | 124 | /** 125 | * From the computed hashes `this.hashes`, filters the nodes that are 126 | * duplicates - i.e at least 2 entries for the same hash 127 | * 128 | * @return [[duplicateNode, ...], ...] 129 | */ 130 | findAllDuplicates() { 131 | return Array.from(this.hashes.values()).filter(dups => dups.length > 1); 132 | } 133 | 134 | /** 135 | * This does a DFS Pre Order traversal from the root till a duplicate node 136 | * is reached and ignores traversing the nodes inside this duplicate node 137 | * 138 | * This is because, 139 | * 140 | * function foo() { 141 | * function dup1() {} 142 | * function dup2() {} 143 | * } 144 | * 145 | * function bar() { 146 | * function dup1() {} 147 | * function dup2() {} 148 | * } 149 | * 150 | * Here function dup1, dup2, dup3 and dup4 are duplicates but since, 151 | * foo and bar are already duplicates, it is not required to extract the 152 | * nodes inside the duplicates as it creates more noise 153 | * 154 | * @return [[duplicateNode, ...], ...] 155 | */ 156 | findOptimalDuplicates() { 157 | const duplicates = {}; 158 | for (const [id, root] of this.ids.entries()) { 159 | dfs(root, { 160 | // pre-order traversal callback 161 | enter(node) { 162 | if (node[DUPLICATE]) { 163 | if (hop(duplicates, node.hash)) { 164 | duplicates[node.hash].push({ node, id }); 165 | } else { 166 | duplicates[node.hash] = [{ node, id }]; 167 | } 168 | 169 | // skip traversing the current node's children 170 | // once we found that the node itself is a duplicate 171 | node.shouldSkip = true; 172 | } 173 | } 174 | }); 175 | } 176 | return Object.keys(duplicates).map(key => duplicates[key]); 177 | } 178 | }; 179 | 180 | function hop(o, key) { 181 | return Object.prototype.hasOwnProperty.call(o, key); 182 | } 183 | -------------------------------------------------------------------------------- /lib/dfs.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const t = require("@babel/types"); 4 | 5 | module.exports = function dfs( 6 | root, 7 | /** 8 | * Callbacks: 9 | * 10 | * enter: pre-order callback 11 | * exit: post-order callback 12 | */ 13 | { enter = () => {}, exit = () => {} } = {} 14 | ) { 15 | return _dfs(root); 16 | 17 | function _dfs(node, parent = null, parentKey = "", isContainer = false) { 18 | if (!node || node.shouldSkip) { 19 | return; 20 | } 21 | 22 | // don't update the parent info every single time 23 | if (node.parent !== parent) { 24 | Object.assign(node, { 25 | parent, 26 | parentKey, 27 | isContainer 28 | }); 29 | } 30 | 31 | // pre-order call 32 | enter(node); 33 | 34 | // after entering if the traversor decides to skip the node, 35 | // just call the exit and skip traversing the children 36 | if (!node.shouldSkip) { 37 | const keys = getFields(node); 38 | if (!keys) return; 39 | 40 | for (const key of keys) { 41 | const subNode = node[key]; 42 | 43 | if (Array.isArray(subNode)) { 44 | for (const child of subNode) { 45 | _dfs(child, node, key, true); 46 | } 47 | } else { 48 | _dfs(subNode, node, key, false); 49 | } 50 | } 51 | } 52 | 53 | // post-order call 54 | exit(node); 55 | } 56 | }; 57 | 58 | function getFields(node) { 59 | return t.VISITOR_KEYS[node.type]; 60 | } 61 | -------------------------------------------------------------------------------- /lib/hash.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const t = require("@babel/types"); 4 | 5 | module.exports = hashcons; 6 | 7 | /** 8 | * Hashconsing - return the hash of an ASTNode computed recursively 9 | * 10 | * @param {ASTNode} node 11 | */ 12 | function hashcons(node) { 13 | const keys = t.VISITOR_KEYS[node.type]; 14 | if (!keys) return; 15 | 16 | let hash = hashcode(node); 17 | 18 | for (const key of keys) { 19 | const subNode = node[key]; 20 | 21 | if (Array.isArray(subNode)) { 22 | for (const child of subNode) { 23 | if (child) { 24 | hash += hashcons(child); 25 | } 26 | } 27 | } else if (subNode) { 28 | hash += hashcons(subNode); 29 | } 30 | } 31 | 32 | return hash; 33 | } 34 | 35 | function hashcode(node) { 36 | if (node.hash) { 37 | return node.hash; 38 | } 39 | 40 | let ret = ""; 41 | 42 | if (t.isFunctionDeclaration(node) || t.isFunctionExpression(node)) { 43 | ret += "Function-"; 44 | } else { 45 | ret += node.type + "-"; 46 | } 47 | 48 | if (t.isLiteral(node)) { 49 | ret += node.value + "-"; 50 | } else if ( 51 | t.isIdentifier(node) && 52 | (((t.isObjectProperty(node.parent) || 53 | t.isObjectMethod(node.parent) || 54 | t.isClassMethod(node.parent)) && 55 | node.parentKey === "key") || 56 | (t.isMemberExpression(node.parent) && node.parentKey === "property")) 57 | ) { 58 | ret += node.name; 59 | } else if (t.isFunction(node)) { 60 | ret += node.generator ? "generator-" : ""; 61 | ret += node.async ? "async-" : ""; 62 | } 63 | 64 | return ret; 65 | } 66 | -------------------------------------------------------------------------------- /lib/html-reporter.js: -------------------------------------------------------------------------------- 1 | const path = require("path"); 2 | const { readFileSync, writeFileSync } = require("fs"); 3 | const chalk = require("chalk"); 4 | const ejs = require("ejs"); 5 | 6 | const projectRoot = path.join(__dirname, ".."); 7 | const publicDir = path.join(projectRoot, "public"); 8 | const viewsDir = path.join(projectRoot, "client/views"); 9 | const defaultOptions = { 10 | appFileName: "reporter.js", 11 | appStyleName: "reporter.css" 12 | }; 13 | 14 | function getAssetsContent(filename) { 15 | return readFileSync(path.join(publicDir, filename), "utf-8"); 16 | } 17 | 18 | function hop(o, key) { 19 | return Object.prototype.hasOwnProperty.call(o, key); 20 | } 21 | 22 | class DupSerializer { 23 | constructor(duplicates, sources) { 24 | this.duplicates = duplicates; 25 | this.sources = sources; 26 | } 27 | 28 | serialize() { 29 | return JSON.stringify(this.getDuplicates(), null, 2); 30 | } 31 | 32 | getDuplicates() { 33 | return this.duplicates.map(dup => this.getFileDupsMapping(dup)); 34 | } 35 | 36 | getFileInfo(filename) { 37 | return { 38 | filename, 39 | sourceCode: this.sources.get(filename) 40 | }; 41 | } 42 | 43 | nodeSerializer({ loc, sourceCode: match }) { 44 | // we just need location from node 45 | return { 46 | loc, 47 | match 48 | }; 49 | } 50 | 51 | getFileDupsMapping(dupList, nodeSerializer = this.nodeSerializer) { 52 | // create a ListMultiMap 53 | const map = dupList.reduce((acc, cur) => { 54 | if (hop(acc, cur.id)) { 55 | acc[cur.id].push(nodeSerializer(cur.node)); 56 | } else { 57 | acc[cur.id] = [nodeSerializer(cur.node)]; 58 | } 59 | return acc; 60 | }, {}); 61 | 62 | // convert that to List 63 | return Object.keys(map).map(filename => 64 | Object.assign( 65 | { 66 | // sort nodes by location 67 | nodes: map[filename].sort((a, b) => { 68 | if (a.loc.start.line === b.loc.start.line) { 69 | return a.loc.start.column - b.loc.start.column; 70 | } 71 | return a.loc.start.line - b.loc.start.line; 72 | }) 73 | }, 74 | this.getFileInfo(filename) 75 | ) 76 | ); 77 | } 78 | } 79 | 80 | class HTMLReporter { 81 | static get TYPE_RESULT() { 82 | return "RESULT"; 83 | } 84 | 85 | static get TYPE_WARNING() { 86 | return "WARNING"; 87 | } 88 | 89 | constructor(duplicates, sources, opts = {}) { 90 | this.duplicates = duplicates; 91 | this.sources = sources; 92 | this.opts = Object.assign({}, defaultOptions, opts); 93 | } 94 | 95 | printError(err) { 96 | process.stderr.write(chalk.red(err)); 97 | } 98 | 99 | removeWhiteSpaces(value) { 100 | return value.replace(/ /g, ""); 101 | } 102 | 103 | renderFile() { 104 | return new Promise((resolve, reject) => { 105 | if (this.duplicates.lenght < 1) { 106 | return resolve({ 107 | type: HTMLReporter.TYPE_WARNING, 108 | warning: "There are NO duplicates. Report is not generated" 109 | }); 110 | } 111 | 112 | const duplicates = new DupSerializer( 113 | this.duplicates, 114 | this.sources 115 | ).serialize(); 116 | 117 | ejs.renderFile( 118 | path.join(viewsDir, "index.ejs"), 119 | { 120 | mode: "static", 121 | appFileName: this.opts.appFileName, 122 | appStyleName: this.opts.appStyleName, 123 | dumpAssets: getAssetsContent, 124 | data: duplicates 125 | }, 126 | (err, reportHtml) => { 127 | if (err) return reject(err); 128 | 129 | resolve({ 130 | type: HTMLReporter.TYPE_RESULT, 131 | html: reportHtml 132 | }); 133 | } 134 | ); 135 | }); 136 | } 137 | } 138 | 139 | module.exports = HTMLReporter; 140 | -------------------------------------------------------------------------------- /lib/index.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const CPA = require("./cpa"); 4 | const Reporter = require("./reporter"); 5 | 6 | module.exports = { 7 | CPA, 8 | findDuplicates(code, opts) { 9 | const cpa = new CPA(opts); 10 | cpa.add(code, { filename: opts.filename }); 11 | return cpa.findOptimalDuplicates(); 12 | }, 13 | findAllDuplicates(code, opts) { 14 | const cpa = new CPA(opts); 15 | cpa.add(code, { filename: opts.filename }); 16 | return cpa.findAllDuplicates(); 17 | }, 18 | Reporter, 19 | stringify(duplicates, opts) { 20 | const r = new Reporter(duplicates); 21 | return r.toString(opts); 22 | } 23 | }; 24 | -------------------------------------------------------------------------------- /lib/reporter.js: -------------------------------------------------------------------------------- 1 | const chalk = require("chalk"); 2 | const unpad = require("./unpad"); 3 | 4 | module.exports = class Reporter { 5 | constructor(duplicates) { 6 | this.duplicates = duplicates; 7 | } 8 | 9 | toString({ colors = true, newline = true } = {}) { 10 | const dups = this.duplicates; 11 | let str = ""; 12 | 13 | if (dups.length === 0) { 14 | str += "No matches found\n"; 15 | return str; 16 | } 17 | 18 | str += title(`Found ${dups.length} matches\n`); 19 | 20 | for (const [i, val] of dups.entries()) { 21 | str += title2(`\nDuplicate ${i + 1}\n\n`); 22 | for (const { id, node } of val) { 23 | str += subtitle(`${id}:${this.getLocation(node)}\n`); 24 | str += this.getCode(node) + "\n\n"; 25 | } 26 | if (newline && i !== dups.length - 1) str += this.newline(); 27 | } 28 | 29 | return str; 30 | 31 | function title(message) { 32 | return colors ? chalk.cyan(chalk.bold(message)) : message; 33 | } 34 | function title2(message) { 35 | return colors ? chalk.yellow(chalk.bold(message)) : message; 36 | } 37 | function subtitle(message) { 38 | return colors ? chalk.dim(message) : message; 39 | } 40 | } 41 | 42 | getLocation(node) { 43 | const { loc: { start, end } } = node; 44 | return `${start.line}:${end.line}`; 45 | } 46 | 47 | getCode(node) { 48 | return unpad(node.sourceCode); 49 | } 50 | 51 | newline() { 52 | let ret = ""; 53 | for (let i = 0; i < process.stdout.columns; i++) { 54 | ret += "-"; 55 | } 56 | return ret; 57 | } 58 | }; 59 | -------------------------------------------------------------------------------- /lib/unpad.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | module.exports = function unpad(str) { 4 | const lines = str.split("\n"); 5 | if (lines.length < 2) { 6 | return str; 7 | } 8 | const lastline = lines[lines.length - 1]; 9 | const match = lastline.match(/^\s+/); 10 | 11 | if (!match) return str; 12 | 13 | const pad = match[0].length; 14 | 15 | if (pad < 1) return str; 16 | 17 | const searcher = new RegExp(`^\\s{${pad}}`); 18 | 19 | return lines.map(line => line.replace(searcher, "")).join("\n").trim(); 20 | }; 21 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "js-cpa", 3 | "version": "0.2.1", 4 | "description": "Code pattern analysis for JS", 5 | "main": "lib/index.js", 6 | "bin": { 7 | "js-cpa": "bin/js-cpa.js" 8 | }, 9 | "scripts": { 10 | "test": "jest", 11 | "start": "webpack --config client/webpack.config.js --watch", 12 | "build": "NODE_ENV=production webpack --config client/webpack.config.js", 13 | "dev": "bash -c 'cat <(find ./public/*) <(find ./lib/*) | entr -s \"./bin/js-cpa.js -m -t 1 -R html test/fixtures/*.js\"'", 14 | "prepublish": "yarn build" 15 | }, 16 | "keywords": [ 17 | "code", 18 | "repetition", 19 | "pattern", 20 | "duplication", 21 | "javascript" 22 | ], 23 | "author": "Vignesh Shanmugam (https://vigneshh.in)", 24 | "contributors": [ 25 | "Boopathi Rajaa", 26 | "Vignesh Shanmugam" 27 | ], 28 | "homepage": "https://github.com/vigneshshanmugam/js-cpa", 29 | "license": "MIT", 30 | "repository": { 31 | "type": "git", 32 | "url": "git+https://github.com/vigneshshanmugam/js-cpa.git" 33 | }, 34 | "dependencies": { 35 | "@babel/parser": "^7.5.5", 36 | "@babel/types": "^7.5.5", 37 | "chalk": "^2.0.1", 38 | "commander": "^3.0.1", 39 | "ejs": "^2.5.7", 40 | "mkdirp": "^0.5.1" 41 | }, 42 | "devDependencies": { 43 | "babel-core": "6.25.0", 44 | "babel-loader": "7.1.5", 45 | "babel-minify-webpack-plugin": "^0.2.0", 46 | "babel-plugin-transform-react-jsx": "6.24.1", 47 | "babel-preset-env": "1.6.0", 48 | "classnames": "2.2.6", 49 | "css-loader": "0.28.4", 50 | "extract-text-webpack-plugin": "3.0.0", 51 | "jest": "^20.0.4", 52 | "normalize.css": "7.0.0", 53 | "preact": "8.5.2", 54 | "prismjs": "1.18.0", 55 | "style-loader": "0.18.2", 56 | "webpack": "3.4.1" 57 | }, 58 | "files": [ 59 | "bin", 60 | "lib", 61 | "client/views", 62 | "public/reporter.js", 63 | "public/reporter.css", 64 | "LICENSE", 65 | "README.md" 66 | ] 67 | } 68 | -------------------------------------------------------------------------------- /test/__snapshots__/cpa.test.js.snap: -------------------------------------------------------------------------------- 1 | // Jest Snapshot v1, https://goo.gl/fbAQLP 2 | 3 | exports[`fixtures match.js 1`] = ` 4 | "Found 2 matches 5 | 6 | Duplicate 1 7 | 8 | match.js:5:5 9 | function f() {} 10 | 11 | match.js:6:6 12 | function aa() {} 13 | 14 | 15 | Duplicate 2 16 | 17 | match.js:10:12 18 | function test() { 19 | return function g1() {}; 20 | } 21 | 22 | match.js:20:22 23 | function test1() { 24 | return function g2() {}; 25 | } 26 | 27 | match.js:25:27 28 | function test2() { 29 | return function g3() {}; 30 | } 31 | 32 | " 33 | `; 34 | 35 | exports[`fixtures modules.js 1`] = ` 36 | "Found 1 matches 37 | 38 | Duplicate 1 39 | 40 | modules.js:1:3 41 | function foo() { 42 | return 10; 43 | } 44 | 45 | modules.js:5:7 46 | function bar() { 47 | return 10; 48 | } 49 | 50 | " 51 | `; 52 | 53 | exports[`fixtures nomatch.js 1`] = ` 54 | "No matches found 55 | " 56 | `; 57 | -------------------------------------------------------------------------------- /test/cpa.test.js: -------------------------------------------------------------------------------- 1 | const { findDuplicates, stringify } = require("../"); 2 | const path = require("path"); 3 | const fs = require("fs"); 4 | 5 | const fixturesDir = path.join(__dirname, "fixtures"); 6 | 7 | describe("fixtures", () => { 8 | fs.readdirSync(fixturesDir).forEach(filename => 9 | test(filename, () => { 10 | const fixture = fs 11 | .readFileSync(path.join(fixturesDir, filename)) 12 | .toString(); 13 | expect( 14 | stringify( 15 | findDuplicates(fixture, { 16 | filename, 17 | sourceType: "module", 18 | threshold: 0 19 | }), 20 | { 21 | colors: false, 22 | newline: false 23 | } 24 | ) 25 | ).toMatchSnapshot(); 26 | }) 27 | ); 28 | }); 29 | -------------------------------------------------------------------------------- /test/fixtures/match.js: -------------------------------------------------------------------------------- 1 | function a() { 2 | const a = 20; 3 | function foo() { 4 | function aaa() { 5 | const asdasdsa = function f() {}; 6 | function aa() {} 7 | } 8 | } 9 | 10 | function test() { 11 | return function g1() {}; 12 | } 13 | return `213`; 14 | } 15 | 16 | function c() { 17 | function foo() { 18 | const b = 20; 19 | } 20 | function test1() { 21 | return function g2() {}; 22 | } 23 | } 24 | 25 | function test2() { 26 | return function g3() {}; 27 | } 28 | -------------------------------------------------------------------------------- /test/fixtures/modules.js: -------------------------------------------------------------------------------- 1 | export function foo() { 2 | return 10; 3 | } 4 | 5 | export function bar() { 6 | return 10; 7 | } 8 | -------------------------------------------------------------------------------- /test/fixtures/nomatch.js: -------------------------------------------------------------------------------- 1 | function foo() { 2 | function aaa() {} 3 | function d() { 4 | function e() { 5 | return "asd"; 6 | } 7 | } 8 | } 9 | 10 | function bar() { 11 | async function aaa() {} 12 | function d() { 13 | function e() { 14 | return "10"; 15 | } 16 | } 17 | } 18 | --------------------------------------------------------------------------------