├── .gitignore ├── CHANGELOG.md ├── LICENSE ├── README.md ├── doc └── intro.md ├── project.clj ├── resources └── sniper │ └── sniper.el ├── src └── sniper │ ├── core.clj │ ├── graph.clj │ ├── scope.clj │ └── snarf.clj └── test └── sniper └── snarf_test.clj /.gitignore: -------------------------------------------------------------------------------- 1 | /target 2 | /classes 3 | /checkouts 4 | pom.xml 5 | pom.xml.asc 6 | *.jar 7 | *.class 8 | /.lein-* 9 | /.nrepl-port 10 | .sniper-analysis-cache.clj 11 | .sniper-strong.clj 12 | -------------------------------------------------------------------------------- /CHANGELOG.md: -------------------------------------------------------------------------------- 1 | ## 0.1.0 2 | * Initial release -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | THE ACCOMPANYING PROGRAM IS PROVIDED UNDER THE TERMS OF THIS ECLIPSE PUBLIC 2 | LICENSE ("AGREEMENT"). ANY USE, REPRODUCTION OR DISTRIBUTION OF THE PROGRAM 3 | CONSTITUTES RECIPIENT'S ACCEPTANCE OF THIS AGREEMENT. 4 | 5 | 1. DEFINITIONS 6 | 7 | "Contribution" means: 8 | 9 | a) in the case of the initial Contributor, the initial code and 10 | documentation distributed under this Agreement, and 11 | 12 | b) in the case of each subsequent Contributor: 13 | 14 | i) changes to the Program, and 15 | 16 | ii) additions to the Program; 17 | 18 | where such changes and/or additions to the Program originate from and are 19 | distributed by that particular Contributor. A Contribution 'originates' from 20 | a Contributor if it was added to the Program by such Contributor itself or 21 | anyone acting on such Contributor's behalf. Contributions do not include 22 | additions to the Program which: (i) are separate modules of software 23 | distributed in conjunction with the Program under their own license 24 | agreement, and (ii) are not derivative works of the Program. 25 | 26 | "Contributor" means any person or entity that distributes the Program. 27 | 28 | "Licensed Patents" mean patent claims licensable by a Contributor which are 29 | necessarily infringed by the use or sale of its Contribution alone or when 30 | combined with the Program. 31 | 32 | "Program" means the Contributions distributed in accordance with this 33 | Agreement. 34 | 35 | "Recipient" means anyone who receives the Program under this Agreement, 36 | including all Contributors. 37 | 38 | 2. GRANT OF RIGHTS 39 | 40 | a) Subject to the terms of this Agreement, each Contributor hereby grants 41 | Recipient a non-exclusive, worldwide, royalty-free copyright license to 42 | reproduce, prepare derivative works of, publicly display, publicly perform, 43 | distribute and sublicense the Contribution of such Contributor, if any, and 44 | such derivative works, in source code and object code form. 45 | 46 | b) Subject to the terms of this Agreement, each Contributor hereby grants 47 | Recipient a non-exclusive, worldwide, royalty-free patent license under 48 | Licensed Patents to make, use, sell, offer to sell, import and otherwise 49 | transfer the Contribution of such Contributor, if any, in source code and 50 | object code form. This patent license shall apply to the combination of the 51 | Contribution and the Program if, at the time the Contribution is added by the 52 | Contributor, such addition of the Contribution causes such combination to be 53 | covered by the Licensed Patents. The patent license shall not apply to any 54 | other combinations which include the Contribution. No hardware per se is 55 | licensed hereunder. 56 | 57 | c) Recipient understands that although each Contributor grants the licenses 58 | to its Contributions set forth herein, no assurances are provided by any 59 | Contributor that the Program does not infringe the patent or other 60 | intellectual property rights of any other entity. Each Contributor disclaims 61 | any liability to Recipient for claims brought by any other entity based on 62 | infringement of intellectual property rights or otherwise. As a condition to 63 | exercising the rights and licenses granted hereunder, each Recipient hereby 64 | assumes sole responsibility to secure any other intellectual property rights 65 | needed, if any. For example, if a third party patent license is required to 66 | allow Recipient to distribute the Program, it is Recipient's responsibility 67 | to acquire that license before distributing the Program. 68 | 69 | d) Each Contributor represents that to its knowledge it has sufficient 70 | copyright rights in its Contribution, if any, to grant the copyright license 71 | set forth in this Agreement. 72 | 73 | 3. REQUIREMENTS 74 | 75 | A Contributor may choose to distribute the Program in object code form under 76 | its own license agreement, provided that: 77 | 78 | a) it complies with the terms and conditions of this Agreement; and 79 | 80 | b) its license agreement: 81 | 82 | i) effectively disclaims on behalf of all Contributors all warranties and 83 | conditions, express and implied, including warranties or conditions of title 84 | and non-infringement, and implied warranties or conditions of merchantability 85 | and fitness for a particular purpose; 86 | 87 | ii) effectively excludes on behalf of all Contributors all liability for 88 | damages, including direct, indirect, special, incidental and consequential 89 | damages, such as lost profits; 90 | 91 | iii) states that any provisions which differ from this Agreement are offered 92 | by that Contributor alone and not by any other party; and 93 | 94 | iv) states that source code for the Program is available from such 95 | Contributor, and informs licensees how to obtain it in a reasonable manner on 96 | or through a medium customarily used for software exchange. 97 | 98 | When the Program is made available in source code form: 99 | 100 | a) it must be made available under this Agreement; and 101 | 102 | b) a copy of this Agreement must be included with each copy of the Program. 103 | 104 | Contributors may not remove or alter any copyright notices contained within 105 | the Program. 106 | 107 | Each Contributor must identify itself as the originator of its Contribution, 108 | if any, in a manner that reasonably allows subsequent Recipients to identify 109 | the originator of the Contribution. 110 | 111 | 4. COMMERCIAL DISTRIBUTION 112 | 113 | Commercial distributors of software may accept certain responsibilities with 114 | respect to end users, business partners and the like. While this license is 115 | intended to facilitate the commercial use of the Program, the Contributor who 116 | includes the Program in a commercial product offering should do so in a 117 | manner which does not create potential liability for other Contributors. 118 | Therefore, if a Contributor includes the Program in a commercial product 119 | offering, such Contributor ("Commercial Contributor") hereby agrees to defend 120 | and indemnify every other Contributor ("Indemnified Contributor") against any 121 | losses, damages and costs (collectively "Losses") arising from claims, 122 | lawsuits and other legal actions brought by a third party against the 123 | Indemnified Contributor to the extent caused by the acts or omissions of such 124 | Commercial Contributor in connection with its distribution of the Program in 125 | a commercial product offering. The obligations in this section do not apply 126 | to any claims or Losses relating to any actual or alleged intellectual 127 | property infringement. In order to qualify, an Indemnified Contributor must: 128 | a) promptly notify the Commercial Contributor in writing of such claim, and 129 | b) allow the Commercial Contributor tocontrol, and cooperate with the 130 | Commercial Contributor in, the defense and any related settlement 131 | negotiations. The Indemnified Contributor may participate in any such claim 132 | at its own expense. 133 | 134 | For example, a Contributor might include the Program in a commercial product 135 | offering, Product X. That Contributor is then a Commercial Contributor. If 136 | that Commercial Contributor then makes performance claims, or offers 137 | warranties related to Product X, those performance claims and warranties are 138 | such Commercial Contributor's responsibility alone. Under this section, the 139 | Commercial Contributor would have to defend claims against the other 140 | Contributors related to those performance claims and warranties, and if a 141 | court requires any other Contributor to pay any damages as a result, the 142 | Commercial Contributor must pay those damages. 143 | 144 | 5. NO WARRANTY 145 | 146 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, THE PROGRAM IS PROVIDED ON 147 | AN "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, EITHER 148 | EXPRESS OR IMPLIED INCLUDING, WITHOUT LIMITATION, ANY WARRANTIES OR 149 | CONDITIONS OF TITLE, NON-INFRINGEMENT, MERCHANTABILITY OR FITNESS FOR A 150 | PARTICULAR PURPOSE. Each Recipient is solely responsible for determining the 151 | appropriateness of using and distributing the Program and assumes all risks 152 | associated with its exercise of rights under this Agreement , including but 153 | not limited to the risks and costs of program errors, compliance with 154 | applicable laws, damage to or loss of data, programs or equipment, and 155 | unavailability or interruption of operations. 156 | 157 | 6. DISCLAIMER OF LIABILITY 158 | 159 | EXCEPT AS EXPRESSLY SET FORTH IN THIS AGREEMENT, NEITHER RECIPIENT NOR ANY 160 | CONTRIBUTORS SHALL HAVE ANY LIABILITY FOR ANY DIRECT, INDIRECT, INCIDENTAL, 161 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING WITHOUT LIMITATION 162 | LOST PROFITS), HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 163 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 164 | ARISING IN ANY WAY OUT OF THE USE OR DISTRIBUTION OF THE PROGRAM OR THE 165 | EXERCISE OF ANY RIGHTS GRANTED HEREUNDER, EVEN IF ADVISED OF THE POSSIBILITY 166 | OF SUCH DAMAGES. 167 | 168 | 7. GENERAL 169 | 170 | If any provision of this Agreement is invalid or unenforceable under 171 | applicable law, it shall not affect the validity or enforceability of the 172 | remainder of the terms of this Agreement, and without further action by the 173 | parties hereto, such provision shall be reformed to the minimum extent 174 | necessary to make such provision valid and enforceable. 175 | 176 | If Recipient institutes patent litigation against any entity (including a 177 | cross-claim or counterclaim in a lawsuit) alleging that the Program itself 178 | (excluding combinations of the Program with other software or hardware) 179 | infringes such Recipient's patent(s), then such Recipient's rights granted 180 | under Section 2(b) shall terminate as of the date such litigation is filed. 181 | 182 | All Recipient's rights under this Agreement shall terminate if it fails to 183 | comply with any of the material terms or conditions of this Agreement and 184 | does not cure such failure in a reasonable period of time after becoming 185 | aware of such noncompliance. If all Recipient's rights under this Agreement 186 | terminate, Recipient agrees to cease use and distribution of the Program as 187 | soon as reasonably practicable. However, Recipient's obligations under this 188 | Agreement and any licenses granted by Recipient relating to the Program shall 189 | continue and survive. 190 | 191 | Everyone is permitted to copy and distribute copies of this Agreement, but in 192 | order to avoid inconsistency the Agreement is copyrighted and may only be 193 | modified in the following manner. The Agreement Steward reserves the right to 194 | publish new versions (including revisions) of this Agreement from time to 195 | time. No one other than the Agreement Steward has the right to modify this 196 | Agreement. The Eclipse Foundation is the initial Agreement Steward. The 197 | Eclipse Foundation may assign the responsibility to serve as the Agreement 198 | Steward to a suitable separate entity. Each new version of the Agreement will 199 | be given a distinguishing version number. The Program (including 200 | Contributions) may always be distributed subject to the version of the 201 | Agreement under which it was received. In addition, after a new version of 202 | the Agreement is published, Contributor may elect to distribute the Program 203 | (including its Contributions) under the new version. Except as expressly 204 | stated in Sections 2(a) and 2(b) above, Recipient receives no rights or 205 | licenses to the intellectual property of any Contributor under this 206 | Agreement, whether expressly, by implication, estoppel or otherwise. All 207 | rights in the Program not expressly granted under this Agreement are 208 | reserved. 209 | 210 | This Agreement is governed by the laws of the State of New York and the 211 | intellectual property laws of the United States of America. No party to this 212 | Agreement will bring a legal action under this Agreement more than one year 213 | after the cause of action arose. Each party waives its rights to a jury trial 214 | in any resulting litigation. 215 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # sniper 2 | 3 | A Clojure library designed to help you find and delete dead code, including an emacs mode. 4 | 5 | See the description below and sniper.scope's namespace docstring for details for how to use it. 6 | 7 | The analysis is currently far from perfect, but we've used it to successfully delete about 10% of our 160KLOC codebase. Contributions welcome. 8 | 9 | ## Motivation 10 | 11 | Over the years we've accumulated lots of dead code mixed into our namespaces, and getting rid of it manually is a painful job. At the same time, a fully automated solution won't work since there are lots of functions that aren't used that we want to keep. 12 | 13 | My ideal workflow for this would be a tool-assisted, interactive loop, where: 14 | 1. the tool shows the user a form that appears to be unused (except possibly by tests) 15 | 2. the user decides to keep the form and marks it as used, or deletes it and does accompanying cleanup of the code 16 | 3. the tool updates its internal dependency graph based on the user action, and goes back to (1). 17 | 18 | I couldn't find any tools that met these criteria (and worked on our 160KLOC codebase), so I wrote sniper. 19 | 20 | ## How to use it 21 | 22 | See sniper.scope's namespace docstring for details, but the basic workflow is: 23 | 24 | 1. Start a REPL that has all your code on the classpath, in addition to sniper (leiningen `[w01fe/sniper "0.1.0"]`). 25 | 26 | 1. Require `sniper.scope` and call `sniper.scope/start!` with appropriate arguments. For example, 27 | 28 | ```clojure 29 | (require '[sniper.scope :as ss]) 30 | (sniper.scope/start! 31 | "/Users/w01fe/my-repo" 32 | [#"-main" 33 | #"/\$"] 34 | #"/test/") 35 | ``` 36 | starts an interactive session removing unused code from `/users/w01fe/my-repo`, where all vars/classes 37 | whose name matches `#"/-main"` or `#"/\$"` (e.g., fnhouse handlers) are considered "strong", 38 | and all forms with a `test` in their file path are marked as "shadow" (i.e., supporting). 39 | 40 | - First, `start!` uses clojure.tools.analyzer to analyze the code, and find definitions and references from each form. 41 | - This may take awhile the first time, but results are cached so it should be much faster if you call it again 42 | - This phase still has some errors, which may lead to false positives or negatives later. 43 | - Then, it runs a graph analysis on the full set of forms to find candidates for removal. In this process: 44 | - "strong" forms matching your regexes (or entries in the `.sniper-strong.clj` cache), as well as any forms 45 | they depend on, are never considered as candidates for deletion. 46 | - "shadow" forms matching your file regex are not counted as references for the purpose of deciding if a 47 | form is dead or alive. however, `sniper` still tracks them so that, e.g. if you delete a form, it guides you 48 | through deleting its test as well. 49 | 50 | 1. Then, you enter an interactive loop that takes you through the above workflow. This is best experienced by loading the attached `resources/sniper.el` emacs mode and activating `M-x sniper-mode`, but can also be done at the repl: 51 | - Type `C-M-'` to jump to the current target (or call `(ss/aim!)` at the repl and navigate there manually). 52 | - Then either: 53 | - Delete the code, do any additional cleanup desired, then press `C-M-backspace` (or call `(ss/fired!)`) to 54 | tell sniper about your decision. If using `sniper.el`, it will immediately aim at the next target. 55 | - Decide to keep the form, and press `C-M-=` (or call `(ss/spare!)` to tell sniper about it. This decision will be cached to the `.sniper-strong.clj` file, and sniper will aim at the next target. 56 | - Repeat until there is nothing left to remove. Didn't that feel good?! 57 | - If at any point you want to pause, or something gets confused, you can always call `start!` again with the same arguments to pick back up. Both the analysis and your decisions are cached on disk. 58 | 59 | 60 | ## Features 61 | 62 | Sniper understands shadow (e.g., test) forms, which behave as weak references, and can also identify dead code cycles (a calls b, b calls a, nothing else calls either) which might appear live from a local perspective. 63 | 64 | When you delete a form, if there are dependents such as shadow forms or forms involved in a reference cycle, sniper walks you through removing this collateral damage as well. 65 | 66 | Sniper caches the results of analysis and the forms you mark as live, so while the first setup run on a project may be very slow (largely time in the analyzer), after making changes you can typically reanalyze a large codebase in seconds or less. 67 | 68 | ## License 69 | 70 | Copyright © 2015 Jason Wolfe 71 | 72 | Distributed under the Eclipse Public License either version 1.0 or (at 73 | your option) any later version. 74 | -------------------------------------------------------------------------------- /doc/intro.md: -------------------------------------------------------------------------------- 1 | # Introduction to sniper 2 | 3 | TODO: write [great documentation](http://jacobian.org/writing/what-to-write/) 4 | -------------------------------------------------------------------------------- /project.clj: -------------------------------------------------------------------------------- 1 | (defproject w01fe/sniper "0.1.1-SNAPSHOT" 2 | :description "Snipe at dead code" 3 | :url "https://github.com/w01fe/sniper" 4 | :license {:name "Eclipse Public License" 5 | :url "http://www.eclipse.org/legal/epl-v10.html"} 6 | :profiles {:dev {:dependencies [[org.clojure/clojure "1.7.0"]]}} 7 | :dependencies [[org.clojure/java.classpath "0.2.2"] 8 | [org.clojure/tools.analyzer.jvm "0.6.7"] 9 | [org.clojure/tools.namespace "0.2.4"] 10 | [prismatic/plumbing "0.4.4"]] 11 | :lein-release {:deploy-via :shell 12 | :shell ["lein" "deploy" "clojars"]} 13 | :signing {:gpg-key "4F31760D1FB590AE"}) 14 | -------------------------------------------------------------------------------- /resources/sniper/sniper.el: -------------------------------------------------------------------------------- 1 | (defun sniper-mode-aim-response-handler (buffer) 2 | "Make a handler for evaluating and printing result in BUFFER." 3 | (nrepl-make-response-handler 4 | buffer 5 | (lambda (buffer value) 6 | (let* ((v (first (read-from-string value))) 7 | (file (nth 0 v)) 8 | (line (nth 1 v)) 9 | (col (nth 2 v)) 10 | (reason (nth 3 v))) 11 | (find-file file) 12 | (goto-line line) 13 | (move-to-column (- col 1)) 14 | (message (prin1-to-string reason)))) 15 | '() 16 | (lambda (buffer err) 17 | (message "%s" err)) 18 | '())) 19 | 20 | (defun sniper-mode-command (command) 21 | (let ((buffer (current-buffer))) 22 | (nrepl-send-string 23 | (format "(when-let [a %s] (sniper.scope/aim->el a))" command) 24 | (sniper-mode-aim-response-handler buffer) 25 | "user"))) 26 | 27 | (defun sniper-mode-aim () 28 | (interactive) 29 | (sniper-mode-command "(sniper.scope/aim)")) 30 | 31 | (defun sniper-mode-fired () 32 | (interactive) 33 | (save-buffer) 34 | (kill-buffer) 35 | (sniper-mode-command "(sniper.scope/fired!)")) 36 | 37 | (defun sniper-mode-spare () 38 | (interactive) 39 | (kill-buffer) 40 | (sniper-mode-command "(sniper.scope/spare!)")) 41 | 42 | (define-minor-mode sniper-mode 43 | "Snipe that dead code" 44 | :lighter " sniper" 45 | :global true 46 | :keymap (let ((map (make-sparse-keymap))) 47 | (define-key map (kbd "C-M-'") 'sniper-mode-aim) 48 | (define-key map (kbd "") 'sniper-mode-fired) 49 | (define-key map (kbd "C-M-=") 'sniper-mode-spare) 50 | map)) 51 | -------------------------------------------------------------------------------- /src/sniper/core.clj: -------------------------------------------------------------------------------- 1 | (ns sniper.core 2 | "Defines the core `Form` data type for sniper, which represents a top-level 3 | Clojure form, and its dependencies and dependents." 4 | (:use plumbing.core) 5 | (:require 6 | [schema.core :as s])) 7 | 8 | (s/defschema SourceInfo 9 | {:file String 10 | :ns clojure.lang.Symbol 11 | :line s/Int 12 | :column s/Int 13 | (s/optional-key :end-column) s/Int 14 | (s/optional-key :end-line) s/Int}) 15 | 16 | (s/defschema Klass (s/named String "class name")) 17 | (s/defschema Var (s/named clojure.lang.Symbol "namespace-qualified symbol")) 18 | 19 | (s/defschema Form 20 | "Represents a single clojure form. Source-info describes where to find it, 21 | - class-defs and class-refs are Strings representing class names that are 22 | defined in and referenced in this form (respectively), 23 | - var-defs and var-refs are symbols representing vars that are defined 24 | in and referenced (respectively) 25 | - shadow? indicates whether this form is an form that only exists to support 26 | its dependents (e.g., a test). 27 | 28 | It is intended that class-defs and var-defs also include things that are not 29 | strictly defined in this form, but for which this form contributes to their 30 | definition (e.g. defmethod contributes to a the definition of the multimethod 31 | var. (Although this is not perfectly captured by the implementation currently.)" 32 | {:source-info SourceInfo 33 | :class-defs [Klass] 34 | :class-refs [Klass] 35 | :var-defs [Var] 36 | :var-refs [Var] 37 | :shadow? Boolean}) 38 | 39 | (s/defschema Ref (s/either Var Klass)) 40 | 41 | (defn shadow? [f] 42 | (safe-get f :shadow?)) 43 | 44 | (defnk definitions :- [Ref] 45 | [class-defs var-defs :as form] 46 | (concat class-defs var-defs)) 47 | 48 | (defnk references :- [Ref] 49 | [class-refs var-refs :as form] 50 | (concat class-refs var-refs)) 51 | -------------------------------------------------------------------------------- /src/sniper/graph.clj: -------------------------------------------------------------------------------- 1 | (ns sniper.graph 2 | "Build and maintain a dependency graph on sniper.core/Forms." 3 | (:refer-clojure :exclude [ancestors descendants]) 4 | (:use plumbing.core) 5 | (:require 6 | [clojure.pprint :as pprint] 7 | [clojure.set :as set] 8 | [schema.core :as s] 9 | [sniper.core :as sniper]) 10 | (:import 11 | [java.util Map HashMap LinkedHashMap Stack])) 12 | 13 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 14 | ;;; Protocol 15 | 16 | (defprotocol PDependencyGraph 17 | (forms [this]) 18 | (add-form [this form] "add a Form to the graph") 19 | (remove-form [this form] "remove a Form from the graph") 20 | (callees [this form] "all Forms that this Form directly depends on") 21 | (callers [this form] "all Forms that direction depend on this Form") 22 | (minify [this] "build a new graph (with new forms) that excludes all references to refs not defined in g.") 23 | (strongify [this ref] "remove all forms contribute towards generating a ref from the graph")) 24 | 25 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 26 | ;;; Private: helpers 27 | 28 | (s/defschema RefMap 29 | {sniper/Ref #{sniper/Form}}) 30 | 31 | (defn add-refs [rm form refs] 32 | (reduce 33 | (fn [rm r] 34 | (update rm r (fnil conj #{}) form)) 35 | rm 36 | refs)) 37 | 38 | (defn remove-refs [rm form refs] 39 | (reduce 40 | (fn [rm r] 41 | (update rm r (fnil disj #{}) form)) 42 | rm 43 | refs)) 44 | 45 | (defn dfs [init f] 46 | (let [visited (LinkedHashMap.)] 47 | ((fn visit [n] 48 | (when-not (.get visited n) 49 | (.put visited n true) 50 | (doseq [c (f n)] 51 | (visit c)))) 52 | init) 53 | (vec (keys visited)))) 54 | 55 | (defn scc-graph 56 | "Take an edge list and return [edge-list node-set-map] for graph of strongly 57 | connected components. Pretends every node has self-loop. Clusters returned 58 | will be in topological order. 59 | 60 | ex: (= (scc-graph [[1 2] [2 3] [2 4] [4 2]]) 61 | [[([0 1] [0 0] [1 2] [1 1] [2 2]) ; meta-edges 62 | {0 (1), 1 (2 4), 2 (3)}]]) ; meta-nodes --> old nodes" 63 | [edges] 64 | (let [edges (distinct (concat edges (map (fn [x] [x x]) (apply concat edges)))) 65 | pe (merge (into {} (map vector (map second edges) (repeat nil))) 66 | (map-vals #(map second %) (group-by first edges))) 67 | e (HashMap. ^Map pe) 68 | re (HashMap. ^Map (map-vals #(map first %) (group-by second edges))) 69 | s (Stack.)] 70 | (while (not (.isEmpty e)) 71 | ((fn dfs1 [n] 72 | (when (.containsKey e n) 73 | (let [nns (.get e n)] 74 | (.remove e n) 75 | (doseq [nn nns] (dfs1 nn))) 76 | (.push s n))) 77 | (first (keys e)))) 78 | (let [sccs (into (sorted-map) 79 | (indexed 80 | (remove empty? 81 | (for [n (reverse (seq s))] 82 | ((fn dfs2 [n] 83 | (when (.containsKey re n) 84 | (let [nns (.get re n)] 85 | (.remove re n) 86 | (cons n (apply concat (doall (map dfs2 nns))))))) 87 | n))))) 88 | rev-sccs (into {} (for [[k vs] sccs, v vs] [v k]))] 89 | [(distinct 90 | (for [[scc nodes] sccs 91 | node nodes 92 | outgoing (get pe node) 93 | :let [n-scc (get rev-sccs outgoing), 94 | _ (assert (<= scc n-scc))]] 95 | [scc n-scc])) 96 | sccs]))) 97 | 98 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 99 | ;;; Public: Implementation 100 | 101 | (declare dependency-graph ancestors descendants) 102 | 103 | (def sort-fn (comp (juxt :file :line) :source-info)) 104 | 105 | (s/defrecord DependencyGraph 106 | [forms :- #{sniper/Form} 107 | definers :- RefMap 108 | referers :- RefMap] 109 | PDependencyGraph 110 | 111 | (forms [this] 112 | (sort-by sort-fn forms)) 113 | 114 | (add-form [this f] 115 | (DependencyGraph. 116 | (conj forms f) 117 | (add-refs definers f (sniper/definitions f)) 118 | (add-refs referers f (sniper/references f)))) 119 | 120 | (remove-form [this f] 121 | (assert (contains? forms f) 122 | (str f)) 123 | (DependencyGraph. 124 | (disj forms f) 125 | (remove-refs definers f (sniper/definitions f)) 126 | (remove-refs referers f (sniper/references f)))) 127 | 128 | (callees [this f] 129 | (->> f 130 | sniper/references 131 | (map definers) 132 | (apply set/union) 133 | (<- (disj f)))) 134 | 135 | (callers [this f] 136 | (->> f 137 | sniper/definitions 138 | (map referers) 139 | (apply set/union) 140 | (<- (disj f)))) 141 | 142 | (strongify [this r] 143 | (->> (definers r) 144 | (mapcat #(descendants this %)) 145 | distinct 146 | (reduce remove-form this))) 147 | 148 | (minify [this] 149 | (let [valid-refs (->> definers (filter (comp seq val)) (map key) set)] 150 | (dependency-graph 151 | (for [f forms] 152 | (-> f 153 | (update :var-refs #(vec (filter valid-refs %))) 154 | (update :class-refs #(vec (filter valid-refs %))))))))) 155 | 156 | (defn ancestors 157 | "depth-first graph traversal of ancestors starting at f" 158 | [g f] 159 | (dfs f #(callers g %))) 160 | 161 | (defn descendants 162 | "depth-first graph traversal of descendants starting at f" 163 | [g f] 164 | (dfs f #(callees g %))) 165 | 166 | (defn unused? 167 | "Is this unused, except possibly by shadow forms? Note that this may 168 | return false for forms that particpate in a dependency cycle, none 169 | of which are used outside that cycle. To break these cycles, see 170 | `leaf-components`." 171 | [g f] 172 | (every? sniper/shadow? (callers g f))) 173 | 174 | (s/defn leaf-components :- [[sniper/Form]] 175 | "A list of strongly connected non-shadow leaf components." 176 | [g] 177 | (let [[scc-edges scc-nodes] (->> g 178 | forms 179 | (remove sniper/shadow?) 180 | (mapcat (fn [f] (for [c (cons :none (callees g f))] [f c]))) 181 | scc-graph) 182 | non-loop-scc-edges (remove (fn [[s d]] (= s d)) scc-edges) 183 | leaf-sccs (apply disj (set (map first non-loop-scc-edges)) 184 | (map second non-loop-scc-edges))] 185 | (map #(safe-get scc-nodes %) leaf-sccs))) 186 | 187 | (defn dependency-graph 188 | ([] (->DependencyGraph #{} {} {})) 189 | ([forms] 190 | (reduce add-form (dependency-graph) forms))) 191 | 192 | (def pprint-graph (comp pprint/pprint forms)) 193 | 194 | (defmethod print-method DependencyGraph [g writer] 195 | (print-method (forms g) writer)) 196 | -------------------------------------------------------------------------------- /src/sniper/scope.clj: -------------------------------------------------------------------------------- 1 | (ns sniper.scope 2 | "Driver for exploring the dead code in a project, one form at a time. 3 | 4 | To set things up, call start!. The basic flow accomplished by this is: 5 | - It first reads in a set of forms with sniper.snarf/classpath-ns-forms 6 | - Forms are marked as shadow if they have no definitions, or match 7 | a test regex. This prevents us from considering all tests as 8 | dead, and from considering forms as used just because they are tested. 9 | - A dependency graph is constructed, an initial set of forms are 10 | marked as `strong` (based on regexes and symbols in the .sniper-strong.clj 11 | file), removing them and all dependencies from the graph, and then 12 | the graph is minified. 13 | - An initial set of potentially dead forms is identified, and a first 14 | form is returned. 15 | 16 | At this point, you can load sniper.el and M-x sniper-mode, and hit C-M-' to 17 | jump to the first dead form that was returned (or navigate their manually). 18 | 19 | Then, you repeatedly either: 20 | - delete the form, do any additional desired cleanup, and call 21 | fired! or press C-M-Backspace. Any tests or other forms 22 | broken by this deletion will be added to the stack to be 23 | killed next (they are not spareable, and wil print as 24 | :collateral). OR, 25 | - don't delete the form, and call spare! or press C-M-= to mark 26 | it as strong. 27 | - At any point, you can call (aim) or press C-M-' to see the current 28 | target. 29 | 30 | If you make other modifications to the code, sniper may not pick 31 | them up so you may have to restart with start!. Because of caching, 32 | this should generally be fast. 33 | 34 | Note that sniper currently makes errors, especially if you have 35 | some code missing from your classpath, or on forms that don't 36 | directly define something but instead perform a mutation on 37 | another definition. 38 | 39 | Some forms that are currently often erroneously considered dead: 40 | - ^:const 41 | - defprotocol 42 | - extend-schema 43 | 44 | Other nice-to-haves (TODO): 45 | - identify orphaned test forms 46 | - identify untested forms 47 | - see paths to strong roots in emacs, color forms by liveness, etc. 48 | - watch files and auto-update 49 | - find entire dead namespaces." 50 | (:use plumbing.core) 51 | (:require 52 | [clojure.java.shell :as shell] 53 | [schema.core :as s] 54 | [sniper.core :as sniper] 55 | [sniper.snarf :as snarf] 56 | [sniper.graph :as graph])) 57 | 58 | 59 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 60 | ;;; Private: helpers for maintaining strong set 61 | 62 | (def +strong-set-file+ 63 | "A file with one ref (class/var) per line, indicating forms that should never be considered 64 | dead. The file is automatically updated as you interactively mark forms as strong." 65 | ".sniper-strong.clj") 66 | 67 | (def +manual-strong-set+ 68 | (atom (set (try (map read-string (.split (slurp +strong-set-file+) "\n")) (catch Exception e))))) 69 | 70 | (defn strongify! [s] 71 | (swap! +manual-strong-set+ conj s) 72 | (spit +strong-set-file+ (str (pr-str s) "\n") :append true)) 73 | 74 | (defn strong-ref? 75 | "Is this reference marked strong according to the manual strong set, or one of the provided 76 | regexes?" 77 | [regexes r] 78 | (or (@+manual-strong-set+ r) 79 | (and (symbol? r) 80 | (let [s (str r)] 81 | (some #(re-find % s) regexes))))) 82 | 83 | 84 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 85 | ;;; Private: maintaining scope state 86 | 87 | (s/defschema KillableForm 88 | {:type (s/enum :leaf :leaf-cycle :collateral) 89 | :form sniper/Form 90 | (s/optional-key :cause) (s/named sniper/Form "Killed form that made this collateral")}) 91 | 92 | (s/defschema State 93 | "State of the sniping process." 94 | {:repo-root String 95 | :graph sniper.graph.DependencyGraph 96 | :stack [KillableForm]}) 97 | 98 | (def +state+ (atom nil)) 99 | 100 | 101 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 102 | ;;; Private: info about the next form. 103 | 104 | (defn aim 105 | "Return the next form to snipe." 106 | [] 107 | (first (:stack @+state+))) 108 | 109 | (defn ref-count [s] 110 | (letk [[exit ^String out] (shell/sh "ag" "-Q" s (:repo-root @+state+))] 111 | (if (= exit 1) 112 | 0 113 | (count (.split out "\n"))))) 114 | 115 | (defnk ref-counts 116 | "Show reference counts for literal matches to defs form using 'ag', to help judge if sniper 117 | missed any key references." 118 | [var-defs class-defs] 119 | (for-map [s (concat 120 | (map name var-defs) 121 | (map #(last (.split ^String % "\\.")) class-defs)) 122 | :let [c (ref-count s)] 123 | :when (pos? c)] 124 | s c)) 125 | 126 | (defnk aim->el 127 | "Convert the result of `aim` into a format that can be read by sniper.el." 128 | [[:form [:source-info ^String file line column] var-defs :as form] type {cause nil}] 129 | (assert (.startsWith file "file:")) 130 | (list (subs file 5) 131 | line 132 | column 133 | (concat [type (first var-defs) (ref-counts form)] (when cause [cause])))) 134 | 135 | 136 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 137 | ;;; Public: main driver loop. 138 | 139 | (defn- prepare-forms [test-regex forms] 140 | (for [f forms] 141 | (if (or (empty? (sniper/definitions f)) 142 | (re-find test-regex (safe-get-in f [:source-info :file]))) 143 | (assoc f :shadow? true) 144 | f))) 145 | 146 | (defn- init-stack [g] 147 | (->> (graph/leaf-components g) 148 | (keep (fn [forms] 149 | (when-let [f (first (filter #(seq (sniper/definitions %)) forms))] 150 | {:form f 151 | :type (if (next forms) :leaf-cycle :leaf)}))) 152 | (sort-by (comp graph/sort-fn :form)) 153 | reverse)) 154 | 155 | (defn start! 156 | "Start a sniper session, by identifying a repo root, set of regexes to consider strong 157 | from the get-go, and test-regex to identify shadow forms. 158 | 159 | Returns the first form to snipe. After this, either delete the form (and do any other 160 | desired cleanup) then call fired!, or call spare! to mark it as strong. Either way 161 | the result will be an aim at the next target. 162 | 163 | By default, traverses the repo bottom-up, jumping to immediately kill collateral 164 | damage (e.g. tests of deleted forms)." 165 | [repo-root strong-regexes test-regex] 166 | (let [forms (prepare-forms 167 | test-regex 168 | (snarf/classpath-ns-forms (java.util.regex.Pattern/compile (str "^" repo-root)))) 169 | g (graph/dependency-graph forms) 170 | g (->> (safe-get g :definers) 171 | keys 172 | (filter #(strong-ref? strong-regexes %)) 173 | (reduce graph/strongify g) 174 | graph/minify)] 175 | (reset! 176 | +state+ 177 | {:repo-root repo-root 178 | :graph g 179 | :stack (init-stack g)})) 180 | (aim)) 181 | 182 | (defn spare! [] 183 | (letk [[graph stack :as state] @+state+ 184 | [form type] (first stack) 185 | defs (sniper/definitions form) 186 | new-graph (reduce graph/strongify graph defs)] 187 | (assert (not= type :collateral) "Cannot spare form (already deleted a dependency)") 188 | (doseq [d defs] (strongify! d)) 189 | (reset! +state+ 190 | (assoc state 191 | :graph new-graph 192 | :stack (vec (filter (comp (set (graph/forms new-graph)) :form) (next stack)))))) 193 | (aim)) 194 | 195 | (defn fired! [] 196 | (letk [[graph stack :as state] @+state+ 197 | form (:form (first stack)) 198 | new-g (graph/remove-form graph form)] 199 | (reset! +state+ 200 | (assoc state 201 | :graph new-g 202 | :stack (distinct-by 203 | :form 204 | (concat 205 | (for [f (next (graph/ancestors graph form))] 206 | {:type :collateral 207 | :form f 208 | :cause form}) 209 | ;; TODO: this won't find new leaf cycles, for now just assume you restart 210 | ;; now and again for that purpose. 211 | (keep #(when (graph/unused? new-g %) 212 | {:type :leaf :form %}) 213 | (graph/callees graph form)) 214 | (next stack)))))) 215 | (aim)) 216 | 217 | 218 | 219 | 220 | (comment 221 | (sniper.scope/start! 222 | "/Users/w01fe/prismatic" 223 | [#"deploy/prod" 224 | #"/\$" 225 | #"crane.task/" 226 | #"topic-specs.admin/" 227 | #"plumbing.schema.generative/" 228 | #"data-warehouse.domain-schemas/" 229 | #".test-utils/" 230 | #"html-learn.readability/" 231 | #"social-scores.repl/" 232 | #"^hiphip."] 233 | #"/test/") 234 | ) 235 | -------------------------------------------------------------------------------- /src/sniper/snarf.clj: -------------------------------------------------------------------------------- 1 | (ns sniper.snarf 2 | "Snarf in namespaces and extract sniper.core/Forms with dependency info. 3 | Main entry point is `classpath-ns-forms`." 4 | (:use plumbing.core) 5 | (:require 6 | [clojure.java.classpath :as classpath] 7 | [clojure.java.io :as java-io] 8 | [clojure.tools.analyzer.ast :as ast] 9 | [clojure.tools.analyzer.env :as env] 10 | [clojure.tools.analyzer.jvm :as jvm] 11 | [clojure.tools.analyzer.jvm.utils :as jvm-utils] 12 | [clojure.tools.namespace.find :as namespace-find] 13 | [clojure.tools.reader :as reader] 14 | [clojure.tools.reader.reader-types :as reader-types] 15 | [schema.core :as s] 16 | [sniper.core :as sniper])) 17 | 18 | 19 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 20 | ;; Copied and modified from tools.analyzer.jvm master 21 | 22 | (defn try-analyze [form env opts] 23 | (try (jvm/analyze form env opts) 24 | (catch Throwable t 25 | (println t) 26 | (println "WARNING: skipping form: " form)))) 27 | 28 | (defn analyze-ns 29 | "Analyzes a whole namespace. returns a vector of the ASTs for all the 30 | top-level ASTs of that file. 31 | Evaluates all the forms. 32 | Disables wrong-tag-handler, and fixes bug with opts shadowing, 33 | and doesn't eval." 34 | ([ns] (analyze-ns ns (jvm/empty-env))) 35 | ([ns env] (analyze-ns ns env {:passes-opts 36 | (merge 37 | jvm/default-passes-opts 38 | {:validate/wrong-tag-handler 39 | (fn [_ ast] 40 | #_(println "Wrong tag: " (-> ast :name meta :tag) 41 | " in def: " (:name ast)))})})) 42 | ([ns env opts] 43 | (println "Analyzing ns" ns) 44 | (env/ensure 45 | (jvm/global-env) 46 | (let [res ^java.net.URL (jvm-utils/ns-url ns)] 47 | (assert res (str "Can't find " ns " in classpath")) 48 | (let [filename (str res) 49 | path (.getPath res)] 50 | (when-not (get-in (env/deref-env) [::analyzed-clj path]) 51 | (binding [*ns* (the-ns ns) 52 | *file* filename] 53 | (with-open [rdr (java-io/reader res)] 54 | (let [pbr (reader-types/indexing-push-back-reader 55 | (java.io.PushbackReader. rdr) 1 filename) 56 | eof (Object.) 57 | read-opts {:eof eof :features #{:clj :t.a.jvm}} 58 | read-opts (if (.endsWith filename "cljc") 59 | (assoc read-opts :read-cond :allow) 60 | read-opts)] 61 | (loop [ret []] 62 | (let [form (reader/read read-opts pbr)] 63 | ;;(println "\n\n" form) 64 | (if (identical? form eof) 65 | (remove nil? ret) 66 | (recur 67 | (conj ret (try-analyze form (assoc env :ns (ns-name *ns*)) opts))))))))))))))) 68 | 69 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 70 | ;; Private helpers: extracting definitions and references from forms 71 | 72 | (defn class-name [^Class c] (.getName c)) 73 | 74 | (defn gen-interface-class [f] 75 | (when (= (first f) 'clojure.core/gen-interface) 76 | [(name (nth f 2))])) 77 | 78 | (defn protocol-gen-interface-form [node] 79 | (when (= (:type node) :class) (-> node :raw-forms first))) 80 | 81 | (defnk class-defs 82 | [op :as node] 83 | (case op 84 | (:deftype) [(class-name (safe-get node :class-name))] 85 | (:import) (-> node :raw-forms first next next first gen-interface-class) ;; possibly definterface 86 | (:const) (-> node protocol-gen-interface-form gen-interface-class) ;; possibly defprotocol 87 | nil)) 88 | 89 | (defnk class-refs [op :as node] 90 | (case op 91 | (:const) (when (= (:type node) :class) [(class-name (safe-get node :val))]) 92 | nil)) 93 | 94 | (defn var->symbol [v] 95 | (letk [[ns [:name :as var-name]] (meta v)] 96 | (symbol (name (ns-name ns)) (name var-name)))) 97 | 98 | (defn defprotocol-vars [f ns] 99 | (when (= (first f) 'clojure.core/gen-interface) 100 | (for [[m] (nth f 4)] 101 | (symbol (name ns) (name m))))) 102 | 103 | (defnk var-defs [op :as node] 104 | (case op 105 | (:def) [(var->symbol (safe-get node :var))] 106 | (:const) (-> node protocol-gen-interface-form (defprotocol-vars (safe-get-in node [:env :ns]))) 107 | (:instance-call) (when (and (= 'addMethod (:method node)) ;; defmethod is part of multi var def. 108 | (= clojure.lang.MultiFn (:class node))) 109 | [(var->symbol (safe-get-in node [:instance :var]))]) 110 | nil)) 111 | 112 | (defnk var-refs [op :as node] 113 | (case op 114 | (:var :the-var) [(var->symbol (safe-get node :var))] 115 | nil)) 116 | 117 | ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;; 118 | ;; Public 119 | 120 | (s/defn ^:always-validate normalized-form :- (s/maybe sniper/Form) 121 | "Convert a top-level analyzer AST node into a sniper/Form. 122 | May return nil for comment forms (which are often missing source info)." 123 | [ast-node] 124 | (let [nodes (ast/nodes ast-node) 125 | unique (fn [f] (sort-by str (distinct (mapcat f nodes))))] 126 | (if-not (get-in ast-node [:env :line]) 127 | (do (assert (= (ffirst (:raw-forms ast-node)) 'comment) 128 | (str "MISSING line" (:raw-forms ast-node) (:env ast-node))) 129 | nil) 130 | {:source-info (select-keys (:env ast-node) [:file :ns :line :column :end-column :end-line]) 131 | :class-defs (unique class-defs) 132 | :class-refs (unique class-refs) 133 | :var-defs (unique var-defs) 134 | :var-refs (unique var-refs) 135 | :shadow? false}))) 136 | 137 | (def +analysis-cache-file+ ".sniper-analysis-cache.clj") 138 | 139 | (defonce +analysis-cache+ 140 | (atom (try (read-string (slurp +analysis-cache-file+)) (catch Exception e {})))) 141 | 142 | (defn cached-ns-forms 143 | [ns] 144 | (let [c (slurp (jvm-utils/ns-url ns))] 145 | (or (@+analysis-cache+ c) 146 | (let [res (vec (keep normalized-form (analyze-ns ns)))] 147 | (swap! +analysis-cache+ assoc c res) 148 | res)))) 149 | 150 | (s/defn ns-forms :- [sniper/Form] 151 | "Get a flat sequence of forms for all namespaces in nss." 152 | [& nss :- [clojure.lang.Symbol]] 153 | (apply concat (pmap cached-ns-forms nss))) 154 | 155 | 156 | (defn classpath-namespaces [dir-regex] 157 | "Get a sequence of all namespaces on classpath that match dir-regex." 158 | (->> (classpath/classpath) 159 | (mapcat file-seq) 160 | distinct 161 | (filter (fn [^java.io.File f] 162 | (and (.isDirectory f) 163 | (re-find dir-regex (.getPath f))))) 164 | (mapcat #(namespace-find/find-namespaces-in-dir %)) 165 | distinct 166 | sort)) 167 | 168 | (s/defn classpath-ns-forms :- [sniper/Form] 169 | [dir-regex] 170 | "Get a flat sequence of forms for all namespaces on the classpath matching 171 | dir-regex." 172 | (let [nss (classpath-namespaces dir-regex)] 173 | (println "Requiring" nss) 174 | (apply require nss) 175 | (let [res (apply ns-forms nss)] 176 | (future (spit +analysis-cache-file+ @+analysis-cache+)) 177 | res))) 178 | -------------------------------------------------------------------------------- /test/sniper/snarf_test.clj: -------------------------------------------------------------------------------- 1 | (ns sniper.snarf-test 2 | (:use clojure.test) 3 | (:require 4 | [clojure.tools.analyzer.jvm :as jvm] 5 | [sniper.snarf :as snarf])) 6 | 7 | (deftest definterface-test 8 | (binding [*ns* (the-ns 'sniper.snarf-test)] 9 | (is (= {:class-defs ["sniper.snarf_test.Foo"] :var-defs []} 10 | (select-keys 11 | (snarf/normalized-form 12 | (jvm/analyze '(definterface Foo (bar [this])))) 13 | [:class-defs :var-defs]))))) 14 | 15 | (deftest defprotocol-test 16 | (binding [*ns* (the-ns 'sniper.snarf-test)] 17 | (is (= {:class-defs ["sniper.snarf_test.Foo"], 18 | :var-defs ['sniper.snarf-test/Foo 'sniper.snarf-test/bar]} 19 | (select-keys 20 | (snarf/normalized-form 21 | (jvm/analyze '(defprotocol Foo (bar [this])))) 22 | [:class-defs :var-defs]))))) 23 | --------------------------------------------------------------------------------