├── README.md ├── pom.xml ├── src └── main │ ├── java │ └── org │ │ └── yinwang │ │ └── pysonar │ │ ├── $.java │ │ ├── Analyzer.java │ │ ├── AstCache.java │ │ ├── Binder.java │ │ ├── Binding.java │ │ ├── Builtins.java │ │ ├── CallStack.java │ │ ├── Constants.java │ │ ├── Diagnostic.java │ │ ├── JSONDump.java │ │ ├── Options.java │ │ ├── Outliner.java │ │ ├── Pair.java │ │ ├── Parser.java │ │ ├── Progress.java │ │ ├── State.java │ │ ├── Stats.java │ │ ├── Test.java │ │ ├── TypeStack.java │ │ ├── ast │ │ ├── Alias.java │ │ ├── Assert.java │ │ ├── Assign.java │ │ ├── Attribute.java │ │ ├── Await.java │ │ ├── BinOp.java │ │ ├── Block.java │ │ ├── Break.java │ │ ├── Bytes.java │ │ ├── Call.java │ │ ├── ClassDef.java │ │ ├── Comprehension.java │ │ ├── Continue.java │ │ ├── Delete.java │ │ ├── Dict.java │ │ ├── DictComp.java │ │ ├── Dummy.java │ │ ├── Ellipsis.java │ │ ├── Exec.java │ │ ├── Expr.java │ │ ├── ExtSlice.java │ │ ├── For.java │ │ ├── FunctionDef.java │ │ ├── GeneratorExp.java │ │ ├── Global.java │ │ ├── Handler.java │ │ ├── If.java │ │ ├── IfExp.java │ │ ├── Import.java │ │ ├── ImportFrom.java │ │ ├── Index.java │ │ ├── Keyword.java │ │ ├── ListComp.java │ │ ├── Module.java │ │ ├── Name.java │ │ ├── NameType.java │ │ ├── Node.java │ │ ├── NodeType.java │ │ ├── Op.java │ │ ├── Pass.java │ │ ├── Print.java │ │ ├── PyComplex.java │ │ ├── PyFloat.java │ │ ├── PyInt.java │ │ ├── PyList.java │ │ ├── PySet.java │ │ ├── Raise.java │ │ ├── Repr.java │ │ ├── Return.java │ │ ├── Sequence.java │ │ ├── SetComp.java │ │ ├── Slice.java │ │ ├── Starred.java │ │ ├── Str.java │ │ ├── Subscript.java │ │ ├── Try.java │ │ ├── Tuple.java │ │ ├── UnaryOp.java │ │ ├── Unsupported.java │ │ ├── Url.java │ │ ├── While.java │ │ ├── With.java │ │ ├── Withitem.java │ │ ├── Yield.java │ │ └── YieldFrom.java │ │ ├── demos │ │ ├── Demo.java │ │ ├── HtmlOutline.java │ │ ├── Linker.java │ │ ├── Style.java │ │ └── StyleApplier.java │ │ ├── hash │ │ ├── EqualFunction.java │ │ ├── FunTypeEqualFunction.java │ │ ├── GenericEqualFunction.java │ │ ├── GenericHashFunction.java │ │ ├── HashFunction.java │ │ ├── MyHashMap.java │ │ └── MyHashSet.java │ │ ├── types │ │ ├── BoolType.java │ │ ├── ClassType.java │ │ ├── ComplexType.java │ │ ├── DictType.java │ │ ├── FloatType.java │ │ ├── FunType.java │ │ ├── InstanceType.java │ │ ├── IntType.java │ │ ├── ListType.java │ │ ├── ModuleType.java │ │ ├── StrType.java │ │ ├── SymbolType.java │ │ ├── TupleType.java │ │ ├── Type.java │ │ └── UnionType.java │ │ └── visitor │ │ ├── TypeInferencer.java │ │ ├── Visitor0.java │ │ ├── Visitor1.java │ │ └── Visitor2.java │ └── resources │ └── org │ └── yinwang │ └── pysonar │ ├── css │ └── demo.css │ ├── javascript │ ├── highlight-debug.js │ └── highlight.js │ ├── models │ └── __init__.py │ └── python │ └── dump_python.py └── tests ├── bom.test ├── bom.py └── refs.json ├── call.test ├── refs.json └── test1.py ├── global.test ├── refs.json └── test1.py ├── import ├── import-from.test │ ├── import-oven.py │ ├── import-pizza-from-oven.py │ ├── import-star-from-init.py │ ├── import-star-from-oven.py │ ├── kitchen │ │ ├── __init__.py │ │ └── oven.py │ └── refs.json ├── multi-level.test │ ├── bedroom.py │ ├── kitchen │ │ ├── __init__.py │ │ └── oven.py │ └── refs.json └── same-level.test │ ├── mod1.py │ ├── mod2.py │ └── refs.json ├── references-multi.test ├── multi.py └── refs.json ├── references.test ├── basic.py └── refs.json ├── relative-import.test ├── __init__.py ├── baz │ ├── __init__.py │ └── rel.py ├── foo │ ├── __init__.py │ └── bar.py └── refs.json └── unicode.test ├── refs.json └── test1.py /README.md: -------------------------------------------------------------------------------- 1 | ### PySonar2 - a type inferencer and indexer for Python 2 | 3 | PySonar2 is a type inferencer and indexer for Python, which performs 4 | sophisticated interprocedural analysis to infer types. It is one of the 5 | underlying technologies that power the code search engine Sourcegraph, where it has been used to 7 | index hundreds of thousands of open source Python repositories, producing a 8 | globally connected network of Python code. An older version of PySonar is used 9 | internally at Google, producing high-quality semantic code index for millions of 10 | lines of Python code. 11 | 12 | To understand its properties, please refer to my blog post: 13 | 14 | - http://yinwang0.wordpress.com/2010/09/12/pysonar 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | #### How to build 23 | 24 | mvn package 25 | 26 | 27 | 28 | #### How to use 29 | 30 | PySonar2 is mainly designed as a library for Python IDEs, other developer 31 | tools and code search engines, so its interface may not be as appealing as an 32 | end-user tool, but for your understanding of the library's capabilities, a 33 | reasonably nice demo program has been built. 34 | 35 | You can build a simple "code-browser" of the Python 2.7 standard library with 36 | the following command line: 37 | 38 | java -jar target/pysonar-.jar /usr/lib/python2.7 ./html 39 | 40 | This will take a few minutes. You should find some interactive HTML files inside 41 | the _html_ directory after this process. 42 | 43 | 44 | 45 | #### System requirements 46 | 47 | * Python 2.7.x 48 | * Python 3.x if you have Python3 files 49 | * Java 8 50 | * maven 51 | 52 | 53 | 54 | ##### Environment variables 55 | 56 | PySonar2 uses CPython's ast package to parse Python code, so please make sure 57 | you have `python` or `python3` installed and pointed to by the `PATH` 58 | environment variable. If you have them in different names, please make symbol 59 | links. 60 | 61 | `PYTHONPATH` environment variable is used for locating the Python standard 62 | libraries. It is important to point it to the correct Python library, for 63 | example 64 | 65 | export PYTHONPATH=/usr/lib/python2.7 66 | 67 | If this is not set up correctly, you may find suboptimal results. 68 | 69 | 70 | 71 | ##### Memory usage 72 | 73 | PySonar2 doesn't need much memory to do analysis compared to other static 74 | analysis tool of its class. 1.5Gb is probably enough for analyzing a medium 75 | sized project such as Python's standard library or Django. But for generating 76 | the HTML files, you may need quite some memory (~2.5Gb for Python 2.7 standard 77 | lib). This is due to the highlighting code is putting all code and their HTML 78 | tags into the memory. 79 | 80 | 81 | 82 | #### License (BSD Style) 83 | 84 | PySonar - a type inferencer and indexer for Python 85 | 86 | Copyright (c) 2013-2016 Yin Wang 87 | 88 | Redistribution and use in source and binary forms, with or without modification, 89 | are permitted provided that the following conditions are met: 90 | 91 | 1. Redistributions of source code must retain the above copyright notice, this 92 | list of conditions and the following disclaimer. 93 | 94 | 1. Redistributions in binary form must reproduce the above copyright notice, 95 | this list of conditions and the following disclaimer in the documentation 96 | and/or other materials provided with the distribution. 97 | 98 | 1. The name of the author may not be used to endorse or promote products derived 99 | from this software without specific prior written permission. 100 | 101 | THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED 102 | WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 103 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT 104 | SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 105 | EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT 106 | OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 107 | INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 108 | CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING 109 | IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY 110 | OF SUCH DAMAGE. 111 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 4.0.0 6 | 7 | org.yinwang 8 | pysonar 9 | 2.1 10 | jar 11 | pysonar2 12 | 13 | 14 | UTF-8 15 | UTF-8 16 | 3.1 17 | 2.1 18 | 2.2.4 19 | 2.2.3 20 | 21 | 22 | 23 | 24 | 25 | com.google.code.gson 26 | gson 27 | 2.2.4 28 | 29 | 30 | com.intellij 31 | annotations 32 | 5.1 33 | 34 | 35 | 36 | com.google.guava 37 | guava 38 | 15.0 39 | 40 | 41 | 42 | 43 | com.fasterxml.jackson.core 44 | jackson-core 45 | ${jackson-2-version} 46 | 47 | 48 | 50 | 51 | com.fasterxml.jackson.core 52 | jackson-annotations 53 | ${jackson-2-version} 54 | 55 | 56 | 57 | 58 | com.fasterxml.jackson.core 59 | jackson-databind 60 | ${jackson-2-version} 61 | 62 | 63 | 64 | commons-io 65 | commons-io 66 | 2.4 67 | 68 | 69 | 70 | org.apache.commons 71 | commons-lang3 72 | 3.0 73 | 74 | 75 | 76 | 77 | ${project.artifactId}-${project.version} 78 | 79 | 80 | maven-compiler-plugin 81 | ${maven-compiler-plugin.version} 82 | 83 | 1.8 84 | 1.8 85 | 86 | 87 | 88 | org.apache.maven.plugins 89 | maven-shade-plugin 90 | ${maven-shade-plugin.version} 91 | 92 | 93 | 94 | 95 | *:* 96 | 97 | META-INF/*.SF 98 | META-INF/*.DFA 99 | META-INF/*.RSA 100 | 101 | 102 | 103 | 104 | 105 | 106 | package 107 | 108 | shade 109 | 110 | 111 | 112 | 113 | 114 | org.yinwang.pysonar.demos.Demo 115 | 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | 125 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/AstCache.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | import org.jetbrains.annotations.Nullable; 5 | import org.yinwang.pysonar.ast.Module; 6 | import org.yinwang.pysonar.ast.Node; 7 | 8 | import java.io.*; 9 | import java.util.HashMap; 10 | import java.util.Map; 11 | import java.util.logging.Level; 12 | import java.util.logging.Logger; 13 | 14 | 15 | /** 16 | * Provides a factory for python source ASTs. Maintains configurable on-disk and 17 | * in-memory caches to avoid re-parsing files during analysis. 18 | */ 19 | public class AstCache { 20 | 21 | private static final Logger LOG = Logger.getLogger(AstCache.class.getCanonicalName()); 22 | 23 | private static AstCache INSTANCE; 24 | 25 | @NotNull 26 | private Map cache = new HashMap<>(); 27 | @NotNull 28 | private static Parser parser = new Parser();; 29 | 30 | public AstCache() { 31 | } 32 | 33 | 34 | /** 35 | * Clears the memory cache. 36 | */ 37 | public void clear() { 38 | cache.clear(); 39 | } 40 | 41 | 42 | /** 43 | * Removes all serialized ASTs from the on-disk cache. 44 | * 45 | * @return {@code true} if all cached AST files were removed 46 | */ 47 | public boolean clearDiskCache() { 48 | try { 49 | $.deleteDirectory(new File(Analyzer.self.cacheDir)); 50 | return true; 51 | } catch (Exception x) { 52 | LOG.log(Level.SEVERE, "Failed to clear disk cache: " + x); 53 | return false; 54 | } 55 | } 56 | 57 | 58 | public void close() { 59 | parser.close(); 60 | // clearDiskCache(); 61 | } 62 | 63 | 64 | /** 65 | * Returns the syntax tree for {@code path}. May find and/or create a 66 | * cached copy in the mem cache or the disk cache. 67 | * 68 | * @param path absolute path to a source file 69 | * @return the AST, or {@code null} if the parse failed for any reason 70 | */ 71 | @Nullable 72 | public Node getAST(@NotNull String path) { 73 | // Cache stores null value if the parse failed. 74 | if (cache.containsKey(path)) { 75 | return cache.get(path); 76 | } 77 | 78 | // Might be cached on disk but not in memory. 79 | Node node = getSerializedModule(path); 80 | if (node != null) { 81 | LOG.log(Level.FINE, "reusing " + path); 82 | cache.put(path, node); 83 | return node; 84 | } 85 | 86 | node = null; 87 | try { 88 | LOG.log(Level.FINE, "parsing " + path); 89 | node = parser.parseFile(path); 90 | } finally { 91 | cache.put(path, node); // may be null 92 | } 93 | 94 | if (node != null) { 95 | serialize(node); 96 | } 97 | 98 | return node; 99 | } 100 | 101 | 102 | /** 103 | * Each source file's AST is saved in an object file named for the MD5 104 | * checksum of the source file. All that is needed is the MD5, but the 105 | * file's base name is included for ease of debugging. 106 | */ 107 | @NotNull 108 | public String getCachePath(@NotNull String sourcePath) { 109 | return $.makePathString(Analyzer.self.cacheDir, $.getFileHash(sourcePath)); 110 | } 111 | 112 | 113 | // package-private for testing 114 | void serialize(@NotNull Node ast) { 115 | String path = getCachePath(ast.file); 116 | ObjectOutputStream oos = null; 117 | FileOutputStream fos = null; 118 | try { 119 | fos = new FileOutputStream(path); 120 | oos = new ObjectOutputStream(fos); 121 | oos.writeObject(ast); 122 | } catch (Exception e) { 123 | $.msg("Failed to serialize: " + path); 124 | } finally { 125 | try { 126 | if (oos != null) { 127 | oos.close(); 128 | } else if (fos != null) { 129 | fos.close(); 130 | } 131 | } catch (Exception e) { 132 | } 133 | } 134 | } 135 | 136 | 137 | // package-private for testing 138 | @Nullable 139 | Module getSerializedModule(String sourcePath) { 140 | if (!new File(sourcePath).canRead()) { 141 | return null; 142 | } 143 | File cached = new File(getCachePath(sourcePath)); 144 | if (!cached.canRead()) { 145 | return null; 146 | } 147 | return deserialize(sourcePath); 148 | } 149 | 150 | 151 | // package-private for testing 152 | @Nullable 153 | Module deserialize(@NotNull String sourcePath) { 154 | String cachePath = getCachePath(sourcePath); 155 | FileInputStream fis = null; 156 | ObjectInputStream ois = null; 157 | try { 158 | fis = new FileInputStream(cachePath); 159 | ois = new ObjectInputStream(fis); 160 | return (Module) ois.readObject(); 161 | } catch (Exception e) { 162 | return null; 163 | } finally { 164 | try { 165 | if (ois != null) { 166 | ois.close(); 167 | } else if (fis != null) { 168 | fis.close(); 169 | } 170 | } catch (Exception e) { 171 | 172 | } 173 | } 174 | } 175 | } 176 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/Binder.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | import org.yinwang.pysonar.ast.*; 5 | import org.yinwang.pysonar.types.*; 6 | 7 | import java.util.List; 8 | import java.util.Set; 9 | 10 | 11 | /** 12 | * Handles binding names to scopes, including destructuring assignment. 13 | */ 14 | public class Binder { 15 | 16 | 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/Binding.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | import org.jetbrains.annotations.Nullable; 5 | import org.yinwang.pysonar.ast.*; 6 | import org.yinwang.pysonar.types.ModuleType; 7 | import org.yinwang.pysonar.types.Type; 8 | import org.yinwang.pysonar.types.UnionType; 9 | 10 | import java.util.LinkedHashSet; 11 | import java.util.Set; 12 | 13 | 14 | public class Binding implements Comparable { 15 | 16 | public enum Kind { 17 | ATTRIBUTE, // attr accessed with "." on some other object 18 | CLASS, // class definition 19 | CONSTRUCTOR, // __init__ functions in classes 20 | FUNCTION, // plain function 21 | METHOD, // static or instance method 22 | MODULE, // file 23 | PARAMETER, // function param 24 | SCOPE, // top-level variable ("scope" means we assume it can have attrs) 25 | VARIABLE // local variable 26 | } 27 | 28 | 29 | private boolean isStatic = false; // static fields/methods 30 | private boolean isSynthetic = false; // auto-generated bindings 31 | private boolean isBuiltin = false; // not from a source file 32 | 33 | @NotNull 34 | public String name; // unqualified name 35 | @NotNull 36 | public Node node; 37 | @NotNull 38 | public String qname; // qualified name 39 | public Type type; // inferred type 40 | public Kind kind; // name usage context 41 | 42 | public Set refs = new LinkedHashSet<>(1); 43 | 44 | // fields from Def 45 | public int start = -1; 46 | public int end = -1; 47 | public int bodyStart = -1; 48 | public int bodyEnd = -1; 49 | 50 | @Nullable 51 | public String fileOrUrl; 52 | 53 | 54 | public Binding(@NotNull String id, @NotNull Node node, @NotNull Type type, @NotNull Kind kind) { 55 | this.name = id; 56 | this.qname = type.table.path; 57 | this.type = type; 58 | this.kind = kind; 59 | this.node = node; 60 | 61 | if (node instanceof Url) { 62 | String url = ((Url) node).url; 63 | if (url.startsWith("file://")) { 64 | fileOrUrl = url.substring("file://".length()); 65 | } else { 66 | fileOrUrl = url; 67 | } 68 | } else { 69 | fileOrUrl = node.file; 70 | if (node instanceof Name) { 71 | name = ((Name) node).id; 72 | } 73 | } 74 | 75 | initLocationInfo(node); 76 | Analyzer.self.registerBinding(this); 77 | } 78 | 79 | 80 | private void initLocationInfo(Node node) { 81 | start = node.start; 82 | end = node.end; 83 | 84 | Node parent = node.parent; 85 | if ((parent instanceof FunctionDef && ((FunctionDef) parent).name == node) || 86 | (parent instanceof ClassDef && ((ClassDef) parent).name == node)) 87 | { 88 | bodyStart = parent.start; 89 | bodyEnd = parent.end; 90 | } else if (node instanceof Module) { 91 | name = ((Module) node).name; 92 | start = 0; 93 | end = 0; 94 | bodyStart = node.start; 95 | bodyEnd = node.end; 96 | } else { 97 | bodyStart = node.start; 98 | bodyEnd = node.end; 99 | } 100 | } 101 | 102 | 103 | public Str getDocstring() { 104 | Node parent = node.parent; 105 | if ((parent instanceof FunctionDef && ((FunctionDef) parent).name == node) || 106 | (parent instanceof ClassDef && ((ClassDef) parent).name == node)) 107 | { 108 | return parent.getDocString(); 109 | } else { 110 | return node.getDocString(); 111 | } 112 | } 113 | 114 | 115 | public void setQname(@NotNull String qname) { 116 | this.qname = qname; 117 | } 118 | 119 | 120 | public void addRef(Node node) { 121 | refs.add(node); 122 | } 123 | 124 | 125 | // merge one more type into the type 126 | // used by stateful assignments which we can't track down the control flow 127 | public void addType(Type t) { 128 | type = UnionType.union(type, t); 129 | } 130 | 131 | 132 | public void setType(Type type) { 133 | this.type = type; 134 | } 135 | 136 | 137 | public void setKind(Kind kind) { 138 | this.kind = kind; 139 | } 140 | 141 | 142 | public void markStatic() { 143 | isStatic = true; 144 | } 145 | 146 | 147 | public boolean isStatic() { 148 | return isStatic; 149 | } 150 | 151 | 152 | public void markSynthetic() { 153 | isSynthetic = true; 154 | } 155 | 156 | 157 | public boolean isSynthetic() { 158 | return isSynthetic; 159 | } 160 | 161 | 162 | public boolean isBuiltin() { 163 | return isBuiltin; 164 | } 165 | 166 | 167 | @NotNull 168 | public String getFirstFile() { 169 | Type bt = type; 170 | if (bt instanceof ModuleType) { 171 | String file = bt.asModuleType().file; 172 | return file != null ? file : ""; 173 | } 174 | 175 | String file = getFile(); 176 | if (file != null) { 177 | return file; 178 | } 179 | 180 | return ""; 181 | } 182 | 183 | 184 | @Nullable 185 | public String getFile() { 186 | return isURL() ? null : fileOrUrl; 187 | } 188 | 189 | 190 | @Nullable 191 | public String getURL() { 192 | return isURL() ? fileOrUrl : null; 193 | } 194 | 195 | 196 | public boolean isURL() { 197 | return fileOrUrl != null && fileOrUrl.startsWith("http://"); 198 | } 199 | 200 | 201 | /** 202 | * Bindings can be sorted by their location for outlining purposes. 203 | */ 204 | public int compareTo(@NotNull Object o) { 205 | return start - ((Binding) o).start; 206 | } 207 | 208 | 209 | @NotNull 210 | @Override 211 | public String toString() { 212 | StringBuilder sb = new StringBuilder(); 213 | sb.append("(binding:"); 214 | sb.append(":kind=").append(kind); 215 | sb.append(":node=").append(node); 216 | sb.append(":type=").append(type); 217 | sb.append(":qname=").append(qname); 218 | sb.append(":refs="); 219 | if (refs.size() > 10) { 220 | sb.append("["); 221 | sb.append(refs.iterator().next()); 222 | sb.append(", ...("); 223 | sb.append(refs.size() - 1); 224 | sb.append(" more)]"); 225 | } else { 226 | sb.append(refs); 227 | } 228 | sb.append(">"); 229 | return sb.toString(); 230 | } 231 | 232 | 233 | @Override 234 | public int hashCode() { 235 | return node.hashCode(); 236 | } 237 | 238 | } 239 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/CallStack.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | import org.yinwang.pysonar.ast.Node; 5 | import org.yinwang.pysonar.types.Type; 6 | 7 | import java.util.HashSet; 8 | import java.util.Set; 9 | 10 | 11 | public class CallStack { 12 | 13 | @NotNull 14 | private Set stack = new HashSet<>(); 15 | 16 | 17 | public void push(Node call, Type type) { 18 | stack.add(call); 19 | } 20 | 21 | 22 | public void pop(Node call, Type type) { 23 | stack.remove(call); 24 | } 25 | 26 | 27 | public boolean contains(Node call, Type type) { 28 | return stack.contains(call); 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/Constants.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar; 2 | 3 | public class Constants { 4 | public static final String rbSelfName = "self"; 5 | } 6 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/Diagnostic.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | 6 | public class Diagnostic { 7 | public enum Category { 8 | INFO, WARNING, ERROR 9 | } 10 | 11 | 12 | public String file; 13 | public Category category; 14 | public int start; 15 | public int end; 16 | public String msg; 17 | 18 | 19 | public Diagnostic(String file, Category category, int start, int end, String msg) { 20 | this.category = category; 21 | this.file = file; 22 | this.start = start; 23 | this.end = end; 24 | this.msg = msg; 25 | } 26 | 27 | 28 | @NotNull 29 | @Override 30 | public String toString() { 31 | return ""; 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/Options.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar; 2 | 3 | import java.util.ArrayList; 4 | import java.util.LinkedHashMap; 5 | import java.util.List; 6 | import java.util.Map; 7 | 8 | public class Options { 9 | 10 | private Map optionsMap = new LinkedHashMap<>(); 11 | 12 | 13 | private List args = new ArrayList<>(); 14 | 15 | 16 | public Options(String[] args) { 17 | for (int i = 0; i < args.length; i++) { 18 | String key = args[i]; 19 | if (key.startsWith("--")) { 20 | if (i + 1 >= args.length) { 21 | $.die("option needs a value: " + key); 22 | } else { 23 | key = key.substring(2); 24 | String value = args[i + 1]; 25 | if (!value.startsWith("-")) { 26 | optionsMap.put(key, value); 27 | i++; 28 | } 29 | } 30 | } else if (key.startsWith("-")) { 31 | key = key.substring(1); 32 | optionsMap.put(key, true); 33 | } else { 34 | this.args.add(key); 35 | } 36 | } 37 | } 38 | 39 | 40 | public Object get(String key) { 41 | return optionsMap.get(key); 42 | } 43 | 44 | 45 | public boolean hasOption(String key) { 46 | Object v = optionsMap.get(key); 47 | if (v instanceof Boolean) { 48 | return (boolean) v; 49 | } else { 50 | return false; 51 | } 52 | } 53 | 54 | 55 | public void put(String key, Object value) { 56 | optionsMap.put(key, value); 57 | } 58 | 59 | 60 | public List getArgs() { 61 | return args; 62 | } 63 | 64 | 65 | public Map getOptionsMap() { 66 | return optionsMap; 67 | } 68 | 69 | 70 | public static void main(String[] args) { 71 | Options options = new Options(args); 72 | for (String key : options.optionsMap.keySet()) { 73 | System.out.println(key + " = " + options.get(key)); 74 | } 75 | } 76 | 77 | } 78 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/Outliner.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | import org.jetbrains.annotations.Nullable; 5 | import org.yinwang.pysonar.types.ClassType; 6 | import org.yinwang.pysonar.types.Type; 7 | import org.yinwang.pysonar.types.UnionType; 8 | 9 | import java.util.ArrayList; 10 | import java.util.List; 11 | import java.util.Set; 12 | import java.util.TreeSet; 13 | 14 | 15 | /** 16 | * Generates a file outline from the index: a structure representing the 17 | * variable and attribute definitions in a file. 18 | */ 19 | public class Outliner { 20 | 21 | public static abstract class Entry { 22 | @Nullable 23 | public String qname; // entry qualified name 24 | public int offset; // file offset of referenced declaration 25 | @Nullable 26 | public Binding.Kind kind; // binding kind of outline entry 27 | 28 | 29 | public Entry() { 30 | } 31 | 32 | 33 | public Entry(String qname, int offset, Binding.Kind kind) { 34 | this.qname = qname; 35 | this.offset = offset; 36 | this.kind = kind; 37 | } 38 | 39 | 40 | public abstract boolean isLeaf(); 41 | 42 | 43 | @NotNull 44 | public Leaf asLeaf() { 45 | return (Leaf) this; 46 | } 47 | 48 | 49 | public abstract boolean isBranch(); 50 | 51 | 52 | @NotNull 53 | public Branch asBranch() { 54 | return (Branch) this; 55 | } 56 | 57 | 58 | public abstract boolean hasChildren(); 59 | 60 | 61 | public abstract List getChildren(); 62 | 63 | 64 | public abstract void setChildren(List children); 65 | 66 | 67 | @Nullable 68 | public String getQname() { 69 | return qname; 70 | } 71 | 72 | 73 | public void setQname(@Nullable String qname) { 74 | if (qname == null) { 75 | throw new IllegalArgumentException("qname param cannot be null"); 76 | } 77 | this.qname = qname; 78 | } 79 | 80 | 81 | /** 82 | * Returns the file offset of the beginning of the identifier referenced 83 | * by this outline entry. 84 | */ 85 | public int getOffset() { 86 | return offset; 87 | } 88 | 89 | 90 | public void setOffset(int offset) { 91 | this.offset = offset; 92 | } 93 | 94 | 95 | public void setKind(@Nullable Binding.Kind kind) { 96 | if (kind == null) { 97 | throw new IllegalArgumentException("kind param cannot be null"); 98 | } 99 | this.kind = kind; 100 | } 101 | 102 | 103 | /** 104 | * Returns the simple (unqualified) name of the identifier. 105 | */ 106 | public String getName() { 107 | String[] parts = qname.split("[.&@%]"); 108 | return parts[parts.length - 1]; 109 | } 110 | 111 | 112 | @Override 113 | public String toString() { 114 | StringBuilder sb = new StringBuilder(); 115 | toString(sb, 0); 116 | return sb.toString().trim(); 117 | } 118 | 119 | 120 | public void toString(@NotNull StringBuilder sb, int depth) { 121 | for (int i = 0; i < depth; i++) { 122 | sb.append(" "); 123 | } 124 | sb.append(kind); 125 | sb.append(" "); 126 | sb.append(getName()); 127 | sb.append("\n"); 128 | if (hasChildren()) { 129 | for (Entry e : getChildren()) { 130 | e.toString(sb, depth + 1); 131 | } 132 | } 133 | } 134 | } 135 | 136 | 137 | /** 138 | * An outline entry with children. 139 | */ 140 | public static class Branch extends Entry { 141 | private List children = new ArrayList<>(); 142 | 143 | 144 | public Branch() { 145 | } 146 | 147 | 148 | public Branch(String qname, int start, Binding.Kind kind) { 149 | super(qname, start, kind); 150 | } 151 | 152 | 153 | public boolean isLeaf() { 154 | return false; 155 | } 156 | 157 | 158 | public boolean isBranch() { 159 | return true; 160 | } 161 | 162 | 163 | public boolean hasChildren() { 164 | return children != null && !children.isEmpty(); 165 | } 166 | 167 | 168 | public List getChildren() { 169 | return children; 170 | } 171 | 172 | 173 | public void setChildren(List children) { 174 | this.children = children; 175 | } 176 | } 177 | 178 | 179 | /** 180 | * An entry with no children. 181 | */ 182 | public static class Leaf extends Entry { 183 | public boolean isLeaf() { 184 | return true; 185 | } 186 | 187 | 188 | public boolean isBranch() { 189 | return false; 190 | } 191 | 192 | 193 | public Leaf() { 194 | } 195 | 196 | 197 | public Leaf(String qname, int start, Binding.Kind kind) { 198 | super(qname, start, kind); 199 | } 200 | 201 | 202 | public boolean hasChildren() { 203 | return false; 204 | } 205 | 206 | 207 | @NotNull 208 | public List getChildren() { 209 | return new ArrayList<>(); 210 | } 211 | 212 | 213 | public void setChildren(List children) { 214 | throw new UnsupportedOperationException("Leaf nodes cannot have children."); 215 | } 216 | } 217 | 218 | 219 | /** 220 | * Create an outline for a file in the index. 221 | * 222 | * @param scope the file scope 223 | * @param path the file for which to build the outline 224 | * @return a list of entries constituting the file outline. 225 | * Returns an empty list if the analyzer hasn't analyzed that path. 226 | */ 227 | @NotNull 228 | public List generate(@NotNull Analyzer idx, @NotNull String abspath) { 229 | Type mt = idx.loadFile(abspath); 230 | if (mt == null) { 231 | return new ArrayList<>(); 232 | } 233 | return generate(mt.table, abspath); 234 | } 235 | 236 | 237 | /** 238 | * Create an outline for a symbol table. 239 | * 240 | * @param state the file state 241 | * @param path the file for which we're building the outline 242 | * @return a list of entries constituting the outline 243 | */ 244 | @NotNull 245 | public List generate(@NotNull State state, @NotNull String path) { 246 | List result = new ArrayList<>(); 247 | 248 | Set entries = new TreeSet<>(); 249 | for (Binding b : state.values()) { 250 | if (!b.isSynthetic() 251 | && !b.isBuiltin() 252 | && path.equals(b.getFile())) 253 | { 254 | entries.add(b); 255 | } 256 | } 257 | 258 | for (Binding nb : entries) { 259 | List kids = null; 260 | 261 | if (nb.kind == Binding.Kind.CLASS) { 262 | Type realType = nb.type; 263 | if (realType instanceof UnionType) { 264 | for (Type t : ((UnionType) realType).types) { 265 | if (t instanceof ClassType) { 266 | realType = t; 267 | break; 268 | } 269 | } 270 | } 271 | kids = generate(realType.table, path); 272 | } 273 | 274 | Entry kid = kids != null ? new Branch() : new Leaf(); 275 | kid.setOffset(nb.start); 276 | kid.setQname(nb.qname); 277 | kid.setKind(nb.kind); 278 | 279 | if (kids != null) { 280 | kid.setChildren(kids); 281 | } 282 | result.add(kid); 283 | } 284 | return result; 285 | } 286 | } 287 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/Pair.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar; 2 | 3 | import java.util.Objects; 4 | 5 | public class Pair { 6 | public Object first; 7 | public Object second; 8 | 9 | public Pair(Object first, Object second) { 10 | this.first = first; 11 | this.second = second; 12 | } 13 | 14 | public boolean equals(Object first, Object second) { 15 | return this.first == first && this.second == second || 16 | this.first == second && this.second == first; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/Progress.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar; 2 | 3 | public class Progress { 4 | 5 | private static final int MAX_SPEED_DIGITS = 5; 6 | 7 | long startTime; 8 | long lastTickTime; 9 | long lastCount; 10 | int lastRate; 11 | int lastAvgRate; 12 | long total; 13 | long count; 14 | long width; 15 | long segSize; 16 | 17 | 18 | public Progress(long total, long width) { 19 | this.startTime = System.currentTimeMillis(); 20 | this.lastTickTime = System.currentTimeMillis(); 21 | this.lastCount = 0; 22 | this.lastRate = 0; 23 | this.lastAvgRate = 0; 24 | this.total = total; 25 | this.width = width; 26 | this.segSize = total / width; 27 | if (segSize == 0) { 28 | segSize = 1; 29 | } 30 | } 31 | 32 | 33 | public void tick(int n) { 34 | count += n; 35 | if (count > total) { 36 | total = count; 37 | } 38 | 39 | long elapsed = System.currentTimeMillis() - lastTickTime; 40 | 41 | if (elapsed > 500 || count == total) { 42 | $.msg_("\r"); 43 | int dlen = (int) Math.ceil(Math.log10((double) total)); 44 | $.msg_($.percent(count, total) + " (" + 45 | $.formatNumber(count, dlen) + 46 | " of " + $.formatNumber(total, dlen) + ")"); 47 | 48 | int rate; 49 | if (elapsed > 1) { 50 | rate = (int) ((count - lastCount) / (elapsed / 1000.0)); 51 | } else { 52 | rate = lastRate; 53 | } 54 | 55 | lastRate = rate; 56 | $.msg_(" SPEED: " + $.formatNumber(rate, MAX_SPEED_DIGITS) + "/s"); 57 | 58 | long totalElapsed = System.currentTimeMillis() - startTime; 59 | int avgRate; 60 | 61 | if (totalElapsed > 1) { 62 | avgRate = (int) (count / (totalElapsed / 1000.0)); 63 | } else { 64 | avgRate = lastAvgRate; 65 | } 66 | avgRate = avgRate == 0 ? 1 : avgRate; 67 | 68 | $.msg_(" AVG SPEED: " + $.formatNumber(avgRate, MAX_SPEED_DIGITS) + "/s"); 69 | 70 | long remain = total - count; 71 | long remainTime = remain / avgRate * 1000; 72 | $.msg_(" ETA: " + $.formatTime(remainTime)); 73 | 74 | $.msg_(" PARSE ERRS: " + Analyzer.self.failedToParse.size()); 75 | 76 | $.msg_(" "); // overflow area 77 | 78 | lastTickTime = System.currentTimeMillis(); 79 | lastAvgRate = avgRate; 80 | lastCount = count; 81 | } 82 | } 83 | 84 | 85 | public void tick() { 86 | if (!Analyzer.self.hasOption("quiet")) { 87 | tick(1); 88 | } 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/Stats.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | 7 | public class Stats { 8 | Map contents = new HashMap<>(); 9 | 10 | 11 | public void putInt(String key, long value) { 12 | contents.put(key, value); 13 | } 14 | 15 | 16 | public void inc(String key, long x) { 17 | Long old = getInt(key); 18 | 19 | if (old == null) { 20 | contents.put(key, 1); 21 | } else { 22 | contents.put(key, old + x); 23 | } 24 | } 25 | 26 | 27 | public void inc(String key) { 28 | inc(key, 1); 29 | } 30 | 31 | 32 | public Long getInt(String key) { 33 | Long ret = (Long) contents.get(key); 34 | if (ret == null) { 35 | return 0L; 36 | } else { 37 | return ret; 38 | } 39 | } 40 | 41 | 42 | public String print() { 43 | StringBuilder sb = new StringBuilder(); 44 | 45 | for (Map.Entry e : contents.entrySet()) { 46 | sb.append("\n- " + e.getKey() + ": " + e.getValue()); 47 | } 48 | 49 | return sb.toString(); 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/Test.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar; 2 | 3 | import com.google.gson.Gson; 4 | import com.google.gson.GsonBuilder; 5 | import org.yinwang.pysonar.ast.Dummy; 6 | import org.yinwang.pysonar.ast.Node; 7 | 8 | import java.io.File; 9 | import java.util.*; 10 | 11 | public class Test { 12 | 13 | static Gson gson = new GsonBuilder().setPrettyPrinting().create(); 14 | Analyzer analyzer; 15 | String inputDir; 16 | boolean exp; 17 | String expecteRefsFile; 18 | String failedRefsFile; 19 | 20 | 21 | public Test(String inputDir, boolean exp) { 22 | // make a quiet analyzer 23 | Map options = new HashMap<>(); 24 | options.put("quiet", true); 25 | this.analyzer = new Analyzer(options); 26 | 27 | this.inputDir = inputDir; 28 | this.exp = exp; 29 | if (new File(inputDir).isDirectory()) { 30 | expecteRefsFile = $.makePathString(inputDir, "refs.json"); 31 | failedRefsFile = $.makePathString(inputDir, "failed_refs.json"); 32 | } else { 33 | expecteRefsFile = $.makePathString(inputDir + ".refs.json"); 34 | failedRefsFile = $.makePathString(inputDir, ".failed_refs.json"); 35 | } 36 | } 37 | 38 | 39 | public void runAnalysis(String dir) { 40 | analyzer.analyze(dir); 41 | analyzer.finish(); 42 | } 43 | 44 | 45 | public void generateRefs() { 46 | 47 | List> refs = new ArrayList<>(); 48 | for (Map.Entry> e : analyzer.getReferences().entrySet()) { 49 | 50 | String file = e.getKey().file; 51 | 52 | // only record those in the inputDir 53 | if (file != null && file.startsWith(Analyzer.self.projectDir)) { 54 | file = $.projRelPath(file); 55 | Map writeout = new LinkedHashMap<>(); 56 | 57 | Map ref = new LinkedHashMap<>(); 58 | ref.put("name", e.getKey().name); 59 | ref.put("file", file); 60 | ref.put("start", e.getKey().start); 61 | ref.put("end", e.getKey().end); 62 | 63 | List> dests = new ArrayList<>(); 64 | for (Binding b : e.getValue()) { 65 | String destFile = b.getFile(); 66 | if (destFile != null && destFile.startsWith(Analyzer.self.projectDir)) { 67 | destFile = $.projRelPath(destFile); 68 | Map dest = new LinkedHashMap<>(); 69 | dest.put("name", b.name); 70 | dest.put("file", destFile); 71 | dest.put("start", b.start); 72 | dest.put("end", b.end); 73 | dests.add(dest); 74 | } 75 | } 76 | if (!dests.isEmpty()) { 77 | writeout.put("ref", ref); 78 | writeout.put("dests", dests); 79 | refs.add(writeout); 80 | } 81 | } 82 | } 83 | 84 | String json = gson.toJson(refs); 85 | $.writeFile(expecteRefsFile, json); 86 | } 87 | 88 | 89 | public boolean checkRefs() { 90 | List> failedRefs = new ArrayList<>(); 91 | String json = $.readFile(expecteRefsFile); 92 | if (json == null) { 93 | $.msg("Expected refs not found in: " + expecteRefsFile + 94 | "Please run Test with -exp to generate"); 95 | return false; 96 | } 97 | List> expectedRefs = gson.fromJson(json, List.class); 98 | for (Map r : expectedRefs) { 99 | Map refMap = (Map) r.get("ref"); 100 | Dummy dummy = makeDummy(refMap); 101 | 102 | List> dests = (List>) r.get("dests"); 103 | List actualDests = analyzer.getReferences().get(dummy); 104 | List> failedDests = new ArrayList<>(); 105 | 106 | for (Map d : dests) { 107 | // names are ignored, they are only for human readers 108 | String file = $.projAbsPath((String) d.get("file")); 109 | int start = (int) Math.floor((double) d.get("start")); 110 | int end = (int) Math.floor((double) d.get("end")); 111 | 112 | if (!checkBindingExist(actualDests, file, start, end)) { 113 | failedDests.add(d); 114 | } 115 | } 116 | 117 | // record the ref & failed dests if any 118 | if (!failedDests.isEmpty()) { 119 | Map failedRef = new LinkedHashMap<>(); 120 | failedRef.put("ref", refMap); 121 | failedRef.put("dests", failedDests); 122 | failedRefs.add(failedRef); 123 | } 124 | } 125 | 126 | if (failedRefs.isEmpty()) { 127 | return true; 128 | } else { 129 | String failedJson = gson.toJson(failedRefs); 130 | $.testmsg("Failed to find refs: " + failedJson); 131 | $.writeFile(failedRefsFile, failedJson); 132 | return false; 133 | } 134 | } 135 | 136 | 137 | boolean checkBindingExist(List bs, String file, int start, int end) { 138 | if (bs == null) { 139 | return false; 140 | } 141 | 142 | for (Binding b : bs) { 143 | if (((b.getFile() == null && file == null) || 144 | (b.getFile() != null && file != null && b.getFile().equals(file))) && 145 | b.start == start && b.end == end) 146 | { 147 | return true; 148 | } 149 | } 150 | 151 | return false; 152 | } 153 | 154 | 155 | public static Dummy makeDummy(Map m) { 156 | String file = $.projAbsPath((String) m.get("file")); 157 | int start = (int) Math.floor((double) m.get("start")); 158 | int end = (int) Math.floor((double) m.get("end")); 159 | return new Dummy(file, start, end); 160 | } 161 | 162 | 163 | public void generateTest() { 164 | runAnalysis(inputDir); 165 | generateRefs(); 166 | $.testmsg(" * " + inputDir); 167 | } 168 | 169 | 170 | public boolean runTest() { 171 | runAnalysis(inputDir); 172 | $.testmsg(" * " + inputDir); 173 | return checkRefs(); 174 | } 175 | 176 | 177 | // ------------------------- static part ----------------------- 178 | 179 | 180 | public static void testAll(String path, boolean exp) { 181 | List failed = new ArrayList<>(); 182 | if (exp) { 183 | $.testmsg("generating tests"); 184 | } else { 185 | $.testmsg("verifying tests"); 186 | } 187 | 188 | testRecursive(path, exp, failed); 189 | 190 | if (exp) { 191 | $.testmsg("all tests generated"); 192 | } else if (failed.isEmpty()) { 193 | $.testmsg("all tests passed!"); 194 | } else { 195 | $.testmsg("failed some tests: "); 196 | for (String f : failed) { 197 | $.testmsg(" * " + f); 198 | } 199 | } 200 | } 201 | 202 | 203 | public static void testRecursive(String path, boolean exp, List failed) { 204 | File file_or_dir = new File(path); 205 | 206 | if (file_or_dir.isDirectory()) { 207 | if (path.endsWith(".test")) { 208 | Test test = new Test(path, exp); 209 | if (exp) { 210 | test.generateTest(); 211 | } else { 212 | if (!test.runTest()) { 213 | failed.add(path); 214 | } 215 | } 216 | } else { 217 | for (File file : file_or_dir.listFiles()) { 218 | testRecursive(file.getPath(), exp, failed); 219 | } 220 | } 221 | } 222 | } 223 | 224 | 225 | public static void main(String[] args) throws Exception { 226 | Options options = new Options(args); 227 | List argsList = options.getArgs(); 228 | String inputDir = $.unifyPath(argsList.get(0)); 229 | 230 | // generate expected file? 231 | boolean exp = options.hasOption("exp"); 232 | testAll(inputDir, exp); 233 | } 234 | } 235 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/TypeStack.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | 8 | 9 | public class TypeStack { 10 | 11 | class Pair { 12 | public Object first; 13 | public Object second; 14 | 15 | 16 | public Pair(Object first, Object second) { 17 | this.first = first; 18 | this.second = second; 19 | } 20 | } 21 | 22 | 23 | @NotNull 24 | private List stack = new ArrayList<>(); 25 | 26 | 27 | public void push(Object first, Object second) { 28 | stack.add(new Pair(first, second)); 29 | } 30 | 31 | 32 | public void pop(Object first, Object second) { 33 | stack.remove(stack.size() - 1); 34 | } 35 | 36 | 37 | public boolean contains(Object first, Object second) { 38 | for (Pair p : stack) { 39 | if (p.first == first && p.second == second || 40 | p.first == second && p.second == first) 41 | { 42 | return true; 43 | } 44 | } 45 | return false; 46 | } 47 | 48 | } 49 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Alias.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.List; 6 | 7 | public class Alias extends Node { 8 | 9 | public List name; 10 | public Name asname; 11 | 12 | public Alias(List name, Name asname, String file, int start, int end) { 13 | super(NodeType.ALIAS, file, start, end); 14 | this.name = name; 15 | this.asname = asname; 16 | addChildren(name); 17 | addChildren(asname); 18 | } 19 | 20 | @NotNull 21 | @Override 22 | public String toString() { 23 | return ""; 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Assert.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class Assert extends Node { 6 | 7 | public Node test; 8 | public Node msg; 9 | 10 | public Assert(Node test, Node msg, String file, int start, int end) { 11 | super(NodeType.ASSERT, file, start, end); 12 | this.test = test; 13 | this.msg = msg; 14 | addChildren(test, msg); 15 | } 16 | 17 | @NotNull 18 | @Override 19 | public String toString() { 20 | return ""; 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Assign.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class Assign extends Node { 6 | 7 | @NotNull 8 | public Node target; 9 | @NotNull 10 | public Node value; 11 | 12 | public Assign(@NotNull Node target, @NotNull Node value, String file, int start, int end) { 13 | super(NodeType.ASSIGN, file, start, end); 14 | this.target = target; 15 | this.value = value; 16 | addChildren(target); 17 | addChildren(value); 18 | } 19 | 20 | @NotNull 21 | @Override 22 | public String toString() { 23 | return "(" + target + " = " + value + ")"; 24 | } 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Attribute.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class Attribute extends Node { 6 | 7 | @NotNull 8 | public Node target; 9 | @NotNull 10 | public Name attr; 11 | 12 | public Attribute(@NotNull Node target, @NotNull Name attr, String file, int start, int end) { 13 | super(NodeType.ATTRIBUTE, file, start, end); 14 | this.target = target; 15 | this.attr = attr; 16 | addChildren(target, attr); 17 | } 18 | 19 | @NotNull 20 | @Override 21 | public String toString() { 22 | return ""; 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Await.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class Await extends Node { 6 | 7 | public Node value; 8 | 9 | public Await(Node n, String file, int start, int end) { 10 | super(NodeType.AWAIT, file, start, end); 11 | this.value = n; 12 | addChildren(n); 13 | } 14 | 15 | @NotNull 16 | @Override 17 | public String toString() { 18 | return ""; 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/BinOp.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class BinOp extends Node { 6 | 7 | @NotNull 8 | public Node left; 9 | @NotNull 10 | public Node right; 11 | @NotNull 12 | public Op op; 13 | 14 | public BinOp(@NotNull Op op, @NotNull Node left, @NotNull Node right, String file, int start, int end) { 15 | super(NodeType.BINOP, file, start, end); 16 | this.left = left; 17 | this.right = right; 18 | this.op = op; 19 | addChildren(left, right); 20 | } 21 | 22 | @NotNull 23 | @Override 24 | public String toString() { 25 | return "(" + left + " " + op + " " + right + ")"; 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Block.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.List; 6 | 7 | public class Block extends Node { 8 | 9 | @NotNull 10 | public List seq; 11 | 12 | public Block(@NotNull List seq, String file, int start, int end) { 13 | super(NodeType.BLOCK, file, start, end); 14 | this.seq = seq; 15 | addChildren(seq); 16 | } 17 | 18 | @NotNull 19 | @Override 20 | public String toString() { 21 | return "(block:" + seq + ")"; 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Break.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class Break extends Node { 6 | 7 | public Break(String file, int start, int end) { 8 | super(NodeType.BREAK, file, start, end); 9 | } 10 | 11 | @NotNull 12 | @Override 13 | public String toString() { 14 | return "(break)"; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Bytes.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class Bytes extends Node { 6 | 7 | public Object value; 8 | 9 | public Bytes(@NotNull Object value, String file, int start, int end) { 10 | super(NodeType.BYTES, file, start, end); 11 | this.value = value.toString(); 12 | } 13 | 14 | @NotNull 15 | @Override 16 | public String toString() { 17 | return "(bytes: " + value + ")"; 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Call.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | import org.jetbrains.annotations.Nullable; 5 | 6 | import java.util.List; 7 | 8 | public class Call extends Node { 9 | 10 | public Node func; 11 | public List args; 12 | @Nullable 13 | public List keywords; 14 | public Node kwargs; 15 | public Node starargs; 16 | 17 | public Call(Node func, List args, @Nullable List keywords, 18 | Node kwargs, Node starargs, String file, int start, int end) { 19 | super(NodeType.CALL, file, start, end); 20 | this.func = func; 21 | this.args = args; 22 | this.keywords = keywords; 23 | this.kwargs = kwargs; 24 | this.starargs = starargs; 25 | addChildren(func, kwargs, starargs); 26 | addChildren(args); 27 | addChildren(keywords); 28 | } 29 | 30 | @NotNull 31 | @Override 32 | public String toString() { 33 | return "(call:" + func + ":" + args + ":" + start + ")"; 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/ClassDef.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | import org.yinwang.pysonar.Binding; 5 | import org.yinwang.pysonar.Builtins; 6 | import org.yinwang.pysonar.State; 7 | import org.yinwang.pysonar.types.Type; 8 | 9 | import java.util.List; 10 | 11 | public class ClassDef extends Node { 12 | 13 | @NotNull 14 | public Name name; 15 | public List bases; 16 | public Node body; 17 | 18 | public ClassDef(@NotNull Name name, List bases, Node body, String file, int start, int end) { 19 | super(NodeType.CLASSDEF, file, start, end); 20 | this.name = name; 21 | this.bases = bases; 22 | this.body = body; 23 | addChildren(name, this.body); 24 | addChildren(bases); 25 | } 26 | 27 | public void addSpecialAttribute(@NotNull State s, String name, Type proptype) { 28 | Binding b = new Binding(name, Builtins.newTutUrl("classes.html"), proptype, Binding.Kind.ATTRIBUTE); 29 | s.update(name, b); 30 | b.markSynthetic(); 31 | b.markStatic(); 32 | 33 | } 34 | 35 | @NotNull 36 | @Override 37 | public String toString() { 38 | return "(class:" + name.id + ":" + start + ")"; 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Comprehension.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.List; 6 | 7 | public class Comprehension extends Node { 8 | 9 | public Node target; 10 | public Node iter; 11 | public List ifs; 12 | 13 | public Comprehension(Node target, Node iter, List ifs, String file, int start, int end) { 14 | super(NodeType.COMPREHENSION, file, start, end); 15 | this.target = target; 16 | this.iter = iter; 17 | this.ifs = ifs; 18 | addChildren(target, iter); 19 | addChildren(ifs); 20 | } 21 | 22 | @NotNull 23 | @Override 24 | public String toString() { 25 | return ""; 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Continue.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class Continue extends Node { 6 | 7 | public Continue(String file, int start, int end) { 8 | super(NodeType.CONTINUE, file, start, end); 9 | } 10 | 11 | @NotNull 12 | @Override 13 | public String toString() { 14 | return "(continue)"; 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Delete.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.List; 6 | 7 | public class Delete extends Node { 8 | 9 | public List targets; 10 | 11 | public Delete(List elts, String file, int start, int end) { 12 | super(NodeType.DELETE, file, start, end); 13 | this.targets = elts; 14 | addChildren(elts); 15 | } 16 | 17 | @NotNull 18 | @Override 19 | public String toString() { 20 | return ""; 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Dict.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.List; 6 | 7 | public class Dict extends Node { 8 | 9 | public List keys; 10 | public List values; 11 | 12 | public Dict(List keys, List values, String file, int start, int end) { 13 | super(NodeType.DICT, file, start, end); 14 | this.keys = keys; 15 | this.values = values; 16 | addChildren(keys); 17 | addChildren(values); 18 | } 19 | 20 | @NotNull 21 | @Override 22 | public String toString() { 23 | return ""; 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/DictComp.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.List; 6 | 7 | public class DictComp extends Node { 8 | 9 | public Node key; 10 | public Node value; 11 | public List generators; 12 | 13 | public DictComp(Node key, Node value, List generators, String file, int start, int end) { 14 | super(NodeType.DICTCOMP, file, start, end); 15 | this.key = key; 16 | this.value = value; 17 | this.generators = generators; 18 | addChildren(key); 19 | addChildren(generators); 20 | } 21 | 22 | @NotNull 23 | @Override 24 | public String toString() { 25 | return ""; 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Dummy.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | /** 4 | * dummy node for locating purposes only 5 | * rarely used 6 | */ 7 | public class Dummy extends Node { 8 | 9 | public Dummy(String file, int start, int end) { 10 | super(NodeType.DUMMY, file, start, end); 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Ellipsis.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class Ellipsis extends Node { 6 | 7 | public Ellipsis(String file, int start, int end) { 8 | super(NodeType.ELLIPSIS, file, start, end); 9 | } 10 | 11 | @NotNull 12 | @Override 13 | public String toString() { 14 | return "..."; 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Exec.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class Exec extends Node { 6 | 7 | public Node body; 8 | public Node globals; 9 | public Node locals; 10 | 11 | public Exec(Node body, Node globals, Node locals, String file, int start, int end) { 12 | super(NodeType.EXEC, file, start, end); 13 | this.body = body; 14 | this.globals = globals; 15 | this.locals = locals; 16 | addChildren(body, globals, locals); 17 | } 18 | 19 | @NotNull 20 | @Override 21 | public String toString() { 22 | return ""; 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Expr.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | /** 6 | * Expression statement. 7 | */ 8 | public class Expr extends Node { 9 | 10 | public Node value; 11 | 12 | public Expr(Node n, String file, int start, int end) { 13 | super(NodeType.EXPR, file, start, end); 14 | this.value = n; 15 | addChildren(n); 16 | } 17 | 18 | @NotNull 19 | @Override 20 | public String toString() { 21 | return ""; 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/ExtSlice.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.List; 6 | 7 | public class ExtSlice extends Node { 8 | 9 | public List dims; 10 | 11 | public ExtSlice(List dims, String file, int start, int end) { 12 | super(NodeType.EXTSLICE, file, start, end); 13 | this.dims = dims; 14 | addChildren(dims); 15 | } 16 | 17 | @NotNull 18 | @Override 19 | public String toString() { 20 | return ""; 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/For.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class For extends Node { 6 | 7 | public Node target; 8 | public Node iter; 9 | public Block body; 10 | public Block orelse; 11 | public boolean isAsync = false; 12 | 13 | public For(Node target, Node iter, Block body, Block orelse, boolean isAsync, 14 | String file, int start, int end) { 15 | super(NodeType.FOR, file, start, end); 16 | this.target = target; 17 | this.iter = iter; 18 | this.body = body; 19 | this.orelse = orelse; 20 | this.isAsync = isAsync; 21 | addChildren(target, iter, body, orelse); 22 | } 23 | 24 | @NotNull 25 | @Override 26 | public String toString() { 27 | return ""; 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/FunctionDef.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.List; 6 | 7 | public class FunctionDef extends Node { 8 | 9 | public Name name; 10 | public List args; 11 | public List defaults; 12 | public Name vararg; // *args 13 | public Name kwarg; // **kwarg 14 | public List afterRest = null; // after rest arg of Ruby 15 | public Node body; 16 | public boolean called = false; 17 | public boolean isLamba = false; 18 | public boolean isAsync = false; 19 | 20 | public FunctionDef(Name name, List args, Node body, List defaults, 21 | Name vararg, Name kwarg, String file, boolean isAsync, int start, int end) { 22 | super(NodeType.FUNCTIONDEF, file, start, end); 23 | if (name != null) { 24 | this.name = name; 25 | } else { 26 | isLamba = true; 27 | String fn = genLambdaName(); 28 | this.name = new Name(fn, file, start, start + "lambda".length()); 29 | addChildren(this.name); 30 | } 31 | 32 | this.args = args; 33 | this.body = body; 34 | this.defaults = defaults; 35 | this.vararg = vararg; 36 | this.kwarg = kwarg; 37 | this.isAsync = isAsync; 38 | addChildren(name); 39 | addChildren(args); 40 | addChildren(defaults); 41 | addChildren(vararg, kwarg, this.body); 42 | } 43 | 44 | public String getArgumentExpr() { 45 | StringBuilder argExpr = new StringBuilder(); 46 | argExpr.append("("); 47 | boolean first = true; 48 | 49 | for (Node n : args) { 50 | if (!first) { 51 | argExpr.append(", "); 52 | } 53 | first = false; 54 | argExpr.append(n.toDisplay()); 55 | } 56 | 57 | if (vararg != null) { 58 | if (!first) { 59 | argExpr.append(", "); 60 | } 61 | first = false; 62 | argExpr.append("*" + vararg.toDisplay()); 63 | } 64 | 65 | if (kwarg != null) { 66 | if (!first) { 67 | argExpr.append(", "); 68 | } 69 | argExpr.append("**" + kwarg.toDisplay()); 70 | } 71 | 72 | argExpr.append(")"); 73 | return argExpr.toString(); 74 | } 75 | 76 | private static int lambdaCounter = 0; 77 | 78 | @NotNull 79 | public static String genLambdaName() { 80 | lambdaCounter = lambdaCounter + 1; 81 | return "lambda%" + lambdaCounter; 82 | } 83 | 84 | @NotNull 85 | @Override 86 | public String toString() { 87 | return "(func:" + start + ":" + name + ")"; 88 | } 89 | 90 | } 91 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/GeneratorExp.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.List; 6 | 7 | public class GeneratorExp extends Node { 8 | 9 | public Node elt; 10 | public List generators; 11 | 12 | public GeneratorExp(Node elt, List generators, String file, int start, int end) { 13 | super(NodeType.GENERATOREXP, file, start, end); 14 | this.elt = elt; 15 | this.generators = generators; 16 | addChildren(elt); 17 | addChildren(generators); 18 | } 19 | 20 | @NotNull 21 | @Override 22 | public String toString() { 23 | return ""; 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Global.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.List; 6 | 7 | public class Global extends Node { 8 | 9 | public List names; 10 | 11 | public Global(List names, String file, int start, int end) { 12 | super(NodeType.GLOBAL, file, start, end); 13 | this.names = names; 14 | addChildren(names); 15 | } 16 | 17 | @NotNull 18 | @Override 19 | public String toString() { 20 | return ""; 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Handler.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.List; 6 | 7 | public class Handler extends Node { 8 | 9 | public List exceptions; 10 | public Node binder; 11 | public Block body; 12 | 13 | public Handler(List exceptions, Node binder, Block body, String file, int start, int end) { 14 | super(NodeType.HANDLER, file, start, end); 15 | this.binder = binder; 16 | this.exceptions = exceptions; 17 | this.body = body; 18 | addChildren(binder, body); 19 | addChildren(exceptions); 20 | } 21 | 22 | @NotNull 23 | @Override 24 | public String toString() { 25 | return "(handler:" + start + ":" + exceptions + ":" + binder + ")"; 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/If.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class If extends Node { 6 | 7 | @NotNull 8 | public Node test; 9 | public Node body; 10 | public Node orelse; 11 | 12 | public If(@NotNull Node test, Node body, Node orelse, String file, int start, int end) { 13 | super(NodeType.IF, file, start, end); 14 | this.test = test; 15 | this.body = body; 16 | this.orelse = orelse; 17 | addChildren(test, body, orelse); 18 | } 19 | 20 | @NotNull 21 | @Override 22 | public String toString() { 23 | return ""; 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/IfExp.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class IfExp extends Node { 6 | 7 | public Node test; 8 | public Node body; 9 | public Node orelse; 10 | 11 | public IfExp(Node test, Node body, Node orelse, String file, int start, int end) { 12 | super(NodeType.IFEXP, file, start, end); 13 | this.test = test; 14 | this.body = body; 15 | this.orelse = orelse; 16 | addChildren(test, body, orelse); 17 | } 18 | 19 | @NotNull 20 | @Override 21 | public String toString() { 22 | return ""; 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Import.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.List; 6 | 7 | public class Import extends Node { 8 | 9 | public List names; 10 | 11 | public Import(List names, String file, int start, int end) { 12 | super(NodeType.IMPORT, file, start, end); 13 | this.names = names; 14 | addChildren(names); 15 | } 16 | 17 | @NotNull 18 | @Override 19 | public String toString() { 20 | return ""; 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/ImportFrom.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | import org.jetbrains.annotations.Nullable; 5 | import org.yinwang.pysonar.Analyzer; 6 | import org.yinwang.pysonar.Binding; 7 | import org.yinwang.pysonar.State; 8 | import org.yinwang.pysonar.types.ListType; 9 | import org.yinwang.pysonar.types.Type; 10 | 11 | import java.util.ArrayList; 12 | import java.util.List; 13 | import java.util.Map.Entry; 14 | import java.util.Set; 15 | 16 | public class ImportFrom extends Node { 17 | 18 | public List module; 19 | public List names; 20 | public int level; 21 | 22 | public ImportFrom(List module, List names, int level, String file, int start, int end) { 23 | super(NodeType.IMPORTFROM, file, start, end); 24 | this.module = module; 25 | this.level = level; 26 | this.names = names; 27 | addChildren(names); 28 | } 29 | 30 | public boolean isImportStar() { 31 | return names.size() == 1 && "*".equals(names.get(0).name.get(0).id); 32 | } 33 | 34 | public void importStar(@NotNull State s, @Nullable Type mt) { 35 | if (mt == null || mt.file == null) { 36 | return; 37 | } 38 | 39 | Node node = Analyzer.self.getAstForFile(mt.file); 40 | if (node == null) { 41 | return; 42 | } 43 | 44 | List names = new ArrayList<>(); 45 | Type allType = mt.table.lookupType("__all__"); 46 | 47 | if (allType != null && allType instanceof ListType) { 48 | ListType lt = (ListType) allType; 49 | 50 | for (Object o : lt.values) { 51 | if (o instanceof String) { 52 | names.add((String) o); 53 | } 54 | } 55 | } 56 | 57 | if (!names.isEmpty()) { 58 | int start = this.start; 59 | 60 | for (String name : names) { 61 | Set b = mt.table.lookupLocal(name); 62 | if (b != null) { 63 | s.update(name, b); 64 | } else { 65 | List m2 = new ArrayList<>(module); 66 | Name fakeName = new Name(name, this.file, start, start + name.length()); 67 | m2.add(fakeName); 68 | Type type = Analyzer.self.loadModule(m2, s); 69 | if (type != null) { 70 | start += name.length(); 71 | s.insert(name, fakeName, type, Binding.Kind.VARIABLE); 72 | } 73 | } 74 | } 75 | } else { 76 | // Fall back to importing all names not starting with "_". 77 | for (Entry> e : mt.table.entrySet()) { 78 | if (!e.getKey().startsWith("_")) { 79 | s.update(e.getKey(), e.getValue()); 80 | } 81 | } 82 | } 83 | } 84 | 85 | @NotNull 86 | @Override 87 | public String toString() { 88 | return ""; 89 | } 90 | 91 | } 92 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Index.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class Index extends Node { 6 | 7 | public Node value; 8 | 9 | public Index(Node n, String file, int start, int end) { 10 | super(NodeType.INDEX, file, start, end); 11 | this.value = n; 12 | addChildren(n); 13 | } 14 | 15 | @NotNull 16 | @Override 17 | public String toString() { 18 | return ""; 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Keyword.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | /** 6 | * Represents a keyword argument (name=value) in a function call. 7 | */ 8 | public class Keyword extends Node { 9 | 10 | public String arg; 11 | @NotNull 12 | public Node value; 13 | 14 | public Keyword(String arg, @NotNull Node value, String file, int start, int end) { 15 | super(NodeType.KEYWORD, file, start, end); 16 | this.arg = arg; 17 | this.value = value; 18 | addChildren(value); 19 | } 20 | 21 | @NotNull 22 | @Override 23 | public String toString() { 24 | return "(keyword:" + arg + ":" + value + ")"; 25 | } 26 | 27 | @NotNull 28 | @Override 29 | public String toDisplay() { 30 | return arg; 31 | } 32 | 33 | } 34 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/ListComp.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.List; 6 | 7 | public class ListComp extends Node { 8 | 9 | public Node elt; 10 | public List generators; 11 | 12 | public ListComp(Node elt, List generators, String file, int start, int end) { 13 | super(NodeType.LISTCOMP, file, start, end); 14 | this.elt = elt; 15 | this.generators = generators; 16 | addChildren(elt); 17 | addChildren(generators); 18 | } 19 | 20 | @NotNull 21 | @Override 22 | public String toString() { 23 | return ""; 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Module.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | import org.yinwang.pysonar.$; 5 | 6 | public class Module extends Node { 7 | 8 | public Block body; 9 | 10 | public Module(Block body, String file, int start, int end) { 11 | super(NodeType.MODULE, file, start, end); 12 | this.name = $.moduleName(file); 13 | this.body = body; 14 | addChildren(this.body); 15 | } 16 | 17 | @NotNull 18 | @Override 19 | public String toString() { 20 | return "(module:" + file + ")"; 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Name.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class Name extends Node { 6 | 7 | @NotNull 8 | public final String id; // identifier 9 | public NameType type; 10 | 11 | public Name(String id) { 12 | // generated name 13 | this(id, null, -1, -1); 14 | } 15 | 16 | public Name(@NotNull String id, String file, int start, int end) { 17 | super(NodeType.NAME, file, start, end); 18 | this.id = id; 19 | this.name = id; 20 | this.type = NameType.LOCAL; 21 | } 22 | 23 | public Name(@NotNull String id, NameType type, String file, int start, int end) { 24 | super(NodeType.NAME, file, start, end); 25 | this.id = id; 26 | this.type = type; 27 | } 28 | 29 | /** 30 | * Returns {@code true} if this name node is the {@code attr} child 31 | * (i.e. the attribute being accessed) of an {@link Attribute} node. 32 | */ 33 | public boolean isAttribute() { 34 | return parent instanceof Attribute 35 | && ((Attribute) parent).attr == this; 36 | } 37 | 38 | @NotNull 39 | @Override 40 | public String toString() { 41 | return "(" + id + ":" + start + ")"; 42 | } 43 | 44 | @NotNull 45 | @Override 46 | public String toDisplay() { 47 | return id; 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/NameType.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | public enum NameType { 4 | LOCAL, 5 | INSTANCE, 6 | CLASS, 7 | GLOBAL 8 | } 9 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Node.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | import org.jetbrains.annotations.Nullable; 5 | import org.yinwang.pysonar.$; 6 | import org.yinwang.pysonar.Analyzer; 7 | 8 | import java.util.Collection; 9 | 10 | /** 11 | * A Node is a junction in the program. 12 | * Since there is no way to put different things in the same segment of the same file, 13 | * a node is uniquely identified by a file, a start and end point. 14 | */ 15 | public abstract class Node implements java.io.Serializable, Comparable { 16 | 17 | public NodeType nodeType; 18 | public String file; 19 | public int start; 20 | public int end; 21 | 22 | public String name; 23 | public Node parent = null; 24 | 25 | public Node() { 26 | } 27 | 28 | public Node(NodeType nodeType, String file, int start, int end) { 29 | this.nodeType = nodeType; 30 | this.file = file; 31 | this.start = start; 32 | this.end = end; 33 | } 34 | 35 | public String getFullPath() { 36 | if (!file.startsWith("/")) { 37 | return $.makePathString(Analyzer.self.projectDir, file); 38 | } else { 39 | return file; 40 | } 41 | } 42 | 43 | public void setParent(Node parent) { 44 | this.parent = parent; 45 | } 46 | 47 | @NotNull 48 | public Node getAstRoot() { 49 | if (parent == null) { 50 | return this; 51 | } 52 | return parent.getAstRoot(); 53 | } 54 | 55 | public int length() { 56 | return end - start; 57 | } 58 | 59 | public void addChildren(@Nullable Node... nodes) { 60 | if (nodes != null) { 61 | for (Node n : nodes) { 62 | if (n != null) { 63 | n.setParent(this); 64 | } 65 | } 66 | } 67 | } 68 | 69 | public void addChildren(@Nullable Collection nodes) { 70 | if (nodes != null) { 71 | for (Node n : nodes) { 72 | if (n != null) { 73 | n.setParent(this); 74 | } 75 | } 76 | } 77 | } 78 | 79 | @Nullable 80 | public Str getDocString() { 81 | Node body = null; 82 | if (this instanceof FunctionDef) { 83 | body = ((FunctionDef) this).body; 84 | } else if (this instanceof ClassDef) { 85 | body = ((ClassDef) this).body; 86 | } else if (this instanceof Module) { 87 | body = ((Module) this).body; 88 | } 89 | 90 | if (body instanceof Block && ((Block) body).seq.size() >= 1) { 91 | Node firstExpr = ((Block) body).seq.get(0); 92 | if (firstExpr instanceof Expr) { 93 | Node docstrNode = ((Expr) firstExpr).value; 94 | if (docstrNode != null && docstrNode instanceof Str) { 95 | return (Str) docstrNode; 96 | } 97 | } 98 | } 99 | return null; 100 | } 101 | 102 | // nodes are equal if they are from the same file and same starting point 103 | @Override 104 | public boolean equals(Object obj) { 105 | if (!(obj instanceof Node)) { 106 | return false; 107 | } else { 108 | Node node = (Node) obj; 109 | String file = this.file; 110 | return (start == node.start && 111 | end == node.end && 112 | $.same(file, node.file)); 113 | } 114 | } 115 | 116 | @Override 117 | public int hashCode() { 118 | return (file + ":" + start + ":" + end).hashCode(); 119 | } 120 | 121 | @Override 122 | public int compareTo(@NotNull Object o) { 123 | if (o instanceof Node) { 124 | return start - ((Node) o).start; 125 | } else { 126 | return -1; 127 | } 128 | } 129 | 130 | public String toDisplay() { 131 | return ""; 132 | } 133 | 134 | @NotNull 135 | @Override 136 | public String toString() { 137 | return "(node:" + file + ":" + name + ":" + start + ")"; 138 | } 139 | 140 | } 141 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/NodeType.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | public enum NodeType { 4 | ALIAS, 5 | ASSERT, 6 | ASSIGN, 7 | ATTRIBUTE, 8 | AWAIT, 9 | BINOP, 10 | BLOCK, 11 | BREAK, 12 | BYTES, 13 | CALL, 14 | CLASSDEF, 15 | COMPREHENSION, 16 | CONTINUE, 17 | DELETE, 18 | DICT, 19 | DICTCOMP, 20 | DUMMY, 21 | ELLIPSIS, 22 | EXEC, 23 | EXPR, 24 | EXTSLICE, 25 | FOR, 26 | FUNCTIONDEF, 27 | GENERATOREXP, 28 | GLOBAL, 29 | HANDLER, 30 | IF, 31 | IFEXP, 32 | IMPORT, 33 | IMPORTFROM, 34 | INDEX, 35 | KEYWORD, 36 | LISTCOMP, 37 | MODULE, 38 | NAME, 39 | NAMETYPE, 40 | NODE, 41 | OP, 42 | PASS, 43 | PRINT, 44 | PYCOMPLEX, 45 | PYFLOAT, 46 | PYINT, 47 | PYLIST, 48 | PYSET, 49 | RAISE, 50 | REPR, 51 | RETURN, 52 | SEQUENCE, 53 | SETCOMP, 54 | SLICE, 55 | STARRED, 56 | STR, 57 | SUBSCRIPT, 58 | TRY, 59 | TUPLE, 60 | UNARYOP, 61 | UNSUPPORTED, 62 | URL, 63 | WHILE, 64 | WITH, 65 | WITHITEM, 66 | YIELD, 67 | YIELDFROM, 68 | } 69 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Op.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.Nullable; 4 | import org.yinwang.pysonar.$; 5 | 6 | public enum Op { 7 | // numeral 8 | Add("+", "__add__"), 9 | Sub("-", "__sub__"), 10 | Mul("*", "__mul__"), 11 | MatMult("@", "__matmult__"), 12 | Div("/", "__div__"), 13 | Mod("%", "__mod__"), 14 | Pow("**", "__pow__"), 15 | FloorDiv("//", "__floordiv__"), 16 | 17 | // comparison 18 | Eq("is"), 19 | Equal("==", "__eq__"), 20 | Lt("<", "__lt__"), 21 | Gt(">", "__gt__"), 22 | 23 | // bit 24 | BitAnd("&", "__and__"), 25 | BitOr("|", "__or__"), 26 | BitXor("^", "__xor__"), 27 | In("in"), 28 | LShift("<<", "__lshift__"), 29 | RShift(">>", "__rshift__"), 30 | Invert("~", "__invert__"), 31 | 32 | // boolean 33 | And("and"), 34 | Or("or"), 35 | Not("not"), 36 | 37 | // synthetic 38 | NotEqual("!=", "__neq__"), 39 | NotEq("is not"), 40 | LtE("<=", "__lte__"), 41 | GtE(">=", "__gte__"), 42 | NotIn("not in"), 43 | 44 | // unsupported new operator 45 | Unsupported("??"); 46 | 47 | private String rep; 48 | 49 | @Nullable 50 | private String method; 51 | 52 | Op(String rep, @Nullable String method) { 53 | this.rep = rep; 54 | this.method = method; 55 | } 56 | 57 | Op(String rep) { 58 | this.rep = rep; 59 | this.method = null; 60 | } 61 | 62 | public String getRep() { 63 | return rep; 64 | } 65 | 66 | @Nullable 67 | public String getMethod() { 68 | return method; 69 | } 70 | 71 | public static Op invert(Op op) { 72 | if (op == Op.Lt) { 73 | return Op.Gt; 74 | } 75 | 76 | if (op == Op.Gt) { 77 | return Op.Lt; 78 | } 79 | 80 | if (op == Op.Eq) { 81 | return Op.Eq; 82 | } 83 | 84 | if (op == Op.And) { 85 | return Op.Or; 86 | } 87 | 88 | if (op == Op.Or) { 89 | return Op.And; 90 | } 91 | 92 | $.die("invalid operator name for invert: " + op); 93 | return null; // unreacheable 94 | } 95 | 96 | public static boolean isBoolean(Op op) { 97 | return op == Eq || 98 | op == Equal || 99 | op == Lt || 100 | op == Gt || 101 | op == NotEqual || 102 | op == NotEq || 103 | op == LtE || 104 | op == GtE || 105 | op == In || 106 | op == NotIn; 107 | } 108 | } 109 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Pass.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class Pass extends Node { 6 | 7 | public Pass(String file, int start, int end) { 8 | super(NodeType.PASS, file, start, end); 9 | } 10 | 11 | @NotNull 12 | @Override 13 | public String toString() { 14 | return ""; 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Print.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.List; 6 | 7 | public class Print extends Node { 8 | 9 | public Node dest; 10 | public List values; 11 | 12 | public Print(Node dest, List elts, String file, int start, int end) { 13 | super(NodeType.PRINT, file, start, end); 14 | this.dest = dest; 15 | this.values = elts; 16 | addChildren(dest); 17 | addChildren(elts); 18 | } 19 | 20 | @NotNull 21 | @Override 22 | public String toString() { 23 | return ""; 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/PyComplex.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class PyComplex extends Node { 6 | 7 | public double real; 8 | public double imag; 9 | 10 | public PyComplex(double real, double imag, String file, int start, int end) { 11 | super(NodeType.PYCOMPLEX, file, start, end); 12 | this.real = real; 13 | this.imag = imag; 14 | } 15 | 16 | @NotNull 17 | @Override 18 | public String toString() { 19 | return "(" + real + "+" + imag + "j)"; 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/PyFloat.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class PyFloat extends Node { 6 | 7 | public double value; 8 | 9 | public PyFloat(String s, String file, int start, int end) { 10 | super(NodeType.PYFLOAT, file, start, end); 11 | s = s.replaceAll("_", ""); 12 | if (s.equals("inf")) { 13 | this.value = Double.POSITIVE_INFINITY; 14 | } else if (s.equals("-inf")) { 15 | this.value = Double.NEGATIVE_INFINITY; 16 | } else { 17 | this.value = Double.parseDouble(s); 18 | } 19 | } 20 | 21 | @NotNull 22 | @Override 23 | public String toString() { 24 | return "(float:" + value + ")"; 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/PyInt.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.math.BigInteger; 6 | 7 | public class PyInt extends Node { 8 | 9 | public BigInteger value; 10 | 11 | public PyInt(String s, String file, int start, int end) { 12 | super(NodeType.PYINT, file, start, end); 13 | 14 | s = s.replaceAll("_", ""); 15 | int sign = 1; 16 | 17 | if (s.startsWith("+")) { 18 | s = s.substring(1); 19 | } else if (s.startsWith("-")) { 20 | s = s.substring(1); 21 | sign = -1; 22 | } 23 | 24 | int base; 25 | if (s.startsWith("0b")) { 26 | base = 2; 27 | s = s.substring(2); 28 | } else if (s.startsWith("0x")) { 29 | base = 16; 30 | s = s.substring(2); 31 | } else if (s.startsWith("x")) { 32 | base = 16; 33 | s = s.substring(1); 34 | } else if (s.startsWith("0o")) { 35 | base = 8; 36 | s = s.substring(2); 37 | } else if (s.startsWith("0") && s.length() >= 2) { 38 | base = 8; 39 | s = s.substring(1); 40 | } else { 41 | base = 10; 42 | } 43 | 44 | value = new BigInteger(s, base); 45 | if (sign == -1) { 46 | value = value.negate(); 47 | } 48 | } 49 | 50 | @NotNull 51 | @Override 52 | public String toString() { 53 | return "(int:" + value + ")"; 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/PyList.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.List; 6 | 7 | public class PyList extends Sequence { 8 | 9 | public PyList(@NotNull List elts, String file, int start, int end) { 10 | super(NodeType.PYLIST, elts, file, start, end); 11 | } 12 | 13 | @NotNull 14 | @Override 15 | public String toString() { 16 | return ""; 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/PySet.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.List; 6 | 7 | public class PySet extends Sequence { 8 | 9 | public PySet(List elts, String file, int start, int end) { 10 | super(NodeType.PYSET, elts, file, start, end); 11 | } 12 | 13 | @NotNull 14 | @Override 15 | public String toString() { 16 | return ""; 17 | } 18 | 19 | } 20 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Raise.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class Raise extends Node { 6 | 7 | public Node exceptionType; 8 | public Node inst; 9 | public Node traceback; 10 | 11 | public Raise(Node exceptionType, Node inst, Node traceback, String file, int start, int end) { 12 | super(NodeType.RAISE, file, start, end); 13 | this.exceptionType = exceptionType; 14 | this.inst = inst; 15 | this.traceback = traceback; 16 | addChildren(exceptionType, inst, traceback); 17 | } 18 | 19 | @NotNull 20 | @Override 21 | public String toString() { 22 | return ""; 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Repr.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class Repr extends Node { 6 | 7 | public Node value; 8 | 9 | public Repr(Node n, String file, int start, int end) { 10 | super(NodeType.REPR, file, start, end); 11 | this.value = n; 12 | addChildren(n); 13 | } 14 | 15 | @NotNull 16 | @Override 17 | public String toString() { 18 | return ""; 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Return.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class Return extends Node { 6 | 7 | public Node value; 8 | 9 | public Return(Node n, String file, int start, int end) { 10 | super(NodeType.RETURN, file, start, end); 11 | this.value = n; 12 | addChildren(n); 13 | } 14 | 15 | @NotNull 16 | @Override 17 | public String toString() { 18 | return ""; 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Sequence.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.List; 6 | 7 | public abstract class Sequence extends Node { 8 | 9 | @NotNull 10 | public List elts; 11 | 12 | public Sequence(NodeType nodeType, @NotNull List elts, String file, int start, int end) { 13 | super(nodeType, file, start, end); 14 | this.elts = elts; 15 | addChildren(elts); 16 | } 17 | 18 | } 19 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/SetComp.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.List; 6 | 7 | public class SetComp extends Node { 8 | 9 | public Node elt; 10 | public List generators; 11 | 12 | public SetComp(Node elt, List generators, String file, int start, int end) { 13 | super(NodeType.SETCOMP, file, start, end); 14 | this.elt = elt; 15 | this.generators = generators; 16 | addChildren(elt); 17 | addChildren(generators); 18 | } 19 | 20 | @NotNull 21 | @Override 22 | public String toString() { 23 | return ""; 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Slice.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class Slice extends Node { 6 | 7 | public Node lower; 8 | public Node step; 9 | public Node upper; 10 | 11 | public Slice(Node lower, Node step, Node upper, String file, int start, int end) { 12 | super(NodeType.SLICE, file, start, end); 13 | this.lower = lower; 14 | this.step = step; 15 | this.upper = upper; 16 | addChildren(lower, step, upper); 17 | } 18 | 19 | @NotNull 20 | @Override 21 | public String toString() { 22 | return ""; 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Starred.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class Starred extends Node { 6 | 7 | public Node value; 8 | 9 | public Starred(Node n, String file, int start, int end) { 10 | super(NodeType.STARRED, file, start, end); 11 | this.value = n; 12 | addChildren(n); 13 | } 14 | 15 | @NotNull 16 | @Override 17 | public String toString() { 18 | return ""; 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Str.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class Str extends Node { 6 | 7 | public String value; 8 | 9 | public Str(@NotNull Object value, String file, int start, int end) { 10 | super(NodeType.STR, file, start, end); 11 | this.value = value.toString(); 12 | } 13 | 14 | @NotNull 15 | @Override 16 | public String toString() { 17 | String summary; 18 | if (value.length() > 10) { 19 | summary = value.substring(0, 10); 20 | } else { 21 | summary = value; 22 | } 23 | return "'" + summary + "'"; 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Subscript.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | import org.jetbrains.annotations.Nullable; 5 | 6 | public class Subscript extends Node { 7 | 8 | @NotNull 9 | public Node value; 10 | @Nullable 11 | public Node slice; // an NIndex or NSlice 12 | 13 | public Subscript(@NotNull Node value, @Nullable Node slice, String file, int start, int end) { 14 | super(NodeType.SUBSCRIPT, file, start, end); 15 | this.value = value; 16 | this.slice = slice; 17 | addChildren(value, slice); 18 | } 19 | 20 | @NotNull 21 | @Override 22 | public String toString() { 23 | return ""; 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Try.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.List; 6 | 7 | public class Try extends Node { 8 | 9 | public List handlers; 10 | public Block body; 11 | public Block orelse; 12 | public Block finalbody; 13 | 14 | public Try(List handlers, Block body, Block orelse, Block finalbody, 15 | String file, int start, int end) { 16 | super(NodeType.TRY, file, start, end); 17 | this.handlers = handlers; 18 | this.body = body; 19 | this.orelse = orelse; 20 | this.finalbody = finalbody; 21 | addChildren(handlers); 22 | addChildren(body, orelse); 23 | } 24 | 25 | @NotNull 26 | @Override 27 | public String toString() { 28 | return ""; 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Tuple.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.List; 6 | 7 | public class Tuple extends Sequence { 8 | 9 | public Tuple(List elts, String file, int start, int end) { 10 | super(NodeType.TUPLE, elts, file, start, end); 11 | } 12 | 13 | @NotNull 14 | @Override 15 | public String toString() { 16 | return ""; 17 | } 18 | 19 | @NotNull 20 | @Override 21 | public String toDisplay() { 22 | StringBuilder sb = new StringBuilder(); 23 | sb.append("("); 24 | 25 | int idx = 0; 26 | for (Node n : elts) { 27 | if (idx != 0) { 28 | sb.append(", "); 29 | } 30 | idx++; 31 | sb.append(n.toDisplay()); 32 | } 33 | 34 | sb.append(")"); 35 | return sb.toString(); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/UnaryOp.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class UnaryOp extends Node { 6 | 7 | public Op op; 8 | public Node operand; 9 | 10 | public UnaryOp(Op op, Node operand, String file, int start, int end) { 11 | super(NodeType.UNARYOP, file, start, end); 12 | this.op = op; 13 | this.operand = operand; 14 | addChildren(operand); 15 | } 16 | 17 | @NotNull 18 | @Override 19 | public String toString() { 20 | return "(" + op + " " + operand + ")"; 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Unsupported.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class Unsupported extends Node { 6 | 7 | public Unsupported(String file, int start, int end) { 8 | super(NodeType.UNSUPPORTED, file, start, end); 9 | } 10 | 11 | @NotNull 12 | @Override 13 | public String toString() { 14 | return "(unsupported)"; 15 | } 16 | } 17 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Url.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | /** 6 | * virtual-AST node used to represent virtual source locations for builtins 7 | * as external urls. 8 | */ 9 | public class Url extends Node { 10 | 11 | public String url; 12 | 13 | public Url(String url) { 14 | this.url = url; 15 | } 16 | 17 | @NotNull 18 | @Override 19 | public String toString() { 20 | return ""; 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/While.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class While extends Node { 6 | 7 | public Node test; 8 | public Node body; 9 | public Node orelse; 10 | 11 | public While(Node test, Node body, Node orelse, String file, int start, int end) { 12 | super(NodeType.WHILE, file, start, end); 13 | this.test = test; 14 | this.body = body; 15 | this.orelse = orelse; 16 | addChildren(test, body, orelse); 17 | } 18 | 19 | @NotNull 20 | @Override 21 | public String toString() { 22 | return ""; 23 | } 24 | 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/With.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | import java.util.List; 6 | 7 | public class With extends Node { 8 | 9 | @NotNull 10 | public List items; 11 | public Block body; 12 | public boolean isAsync = false; 13 | 14 | public With(@NotNull List items, Block body, String file, boolean isAsync, int start, int end) { 15 | super(NodeType.WITH, file, start, end); 16 | this.items = items; 17 | this.body = body; 18 | this.isAsync = isAsync; 19 | addChildren(items); 20 | addChildren(body); 21 | } 22 | 23 | @NotNull 24 | @Override 25 | public String toString() { 26 | return ""; 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Withitem.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | import org.jetbrains.annotations.Nullable; 5 | 6 | /** 7 | * A name alias. Used for the components of import and import-from statements. 8 | */ 9 | public class Withitem extends Node { 10 | 11 | @Nullable 12 | public Node optional_vars; 13 | @NotNull 14 | public Node context_expr; 15 | 16 | public Withitem(@NotNull Node context_expr, @Nullable Node optional_vars, String file, int start, int end) { 17 | super(NodeType.WITHITEM, file, start, end); 18 | this.context_expr = context_expr; 19 | this.optional_vars = optional_vars; 20 | addChildren(context_expr, optional_vars); 21 | } 22 | 23 | @NotNull 24 | @Override 25 | public String toString() { 26 | return "(withitem:" + context_expr + " as " + optional_vars + ")"; 27 | } 28 | 29 | } 30 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/Yield.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class Yield extends Node { 6 | 7 | public Node value; 8 | 9 | public Yield(Node n, String file, int start, int end) { 10 | super(NodeType.YIELD, file, start, end); 11 | this.value = n; 12 | addChildren(n); 13 | } 14 | 15 | @NotNull 16 | @Override 17 | public String toString() { 18 | return ""; 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/ast/YieldFrom.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.ast; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | 5 | public class YieldFrom extends Node { 6 | 7 | public Node value; 8 | 9 | public YieldFrom(Node n, String file, int start, int end) { 10 | super(NodeType.YIELDFROM, file, start, end); 11 | this.value = n; 12 | addChildren(n); 13 | } 14 | 15 | @NotNull 16 | @Override 17 | public String toString() { 18 | return ""; 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/demos/Demo.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.demos; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | import org.yinwang.pysonar.Analyzer; 5 | import org.yinwang.pysonar.Options; 6 | import org.yinwang.pysonar.Progress; 7 | import org.yinwang.pysonar.$; 8 | 9 | import java.io.File; 10 | import java.util.ArrayList; 11 | import java.util.List; 12 | import java.util.Map; 13 | 14 | 15 | public class Demo { 16 | 17 | private static File OUTPUT_DIR; 18 | 19 | private static final String CSS = $.readResource("org/yinwang/pysonar/css/demo.css"); 20 | private static final String JS = $.readResource("org/yinwang/pysonar/javascript/highlight.js"); 21 | private static final String JS_DEBUG = $.readResource("org/yinwang/pysonar/javascript/highlight-debug.js"); 22 | 23 | private Analyzer analyzer; 24 | private String rootPath; 25 | private Linker linker; 26 | 27 | 28 | private void makeOutputDir() { 29 | if (!OUTPUT_DIR.exists()) { 30 | OUTPUT_DIR.mkdirs(); 31 | $.msg("Created directory: " + OUTPUT_DIR.getAbsolutePath()); 32 | } 33 | } 34 | 35 | 36 | private void start(@NotNull String fileOrDir, Map options) throws Exception { 37 | File f = new File(fileOrDir); 38 | File rootDir = f.isFile() ? f.getParentFile() : f; 39 | try { 40 | rootPath = $.unifyPath(rootDir); 41 | } catch (Exception e) { 42 | $.die("File not found: " + f); 43 | } 44 | 45 | analyzer = new Analyzer(options); 46 | $.msg("Loading and analyzing files"); 47 | analyzer.analyze(f.getPath()); 48 | analyzer.finish(); 49 | 50 | generateHtml(); 51 | analyzer.close(); 52 | } 53 | 54 | 55 | private void generateHtml() { 56 | $.msg("\nGenerating HTML"); 57 | makeOutputDir(); 58 | 59 | linker = new Linker(rootPath, OUTPUT_DIR); 60 | linker.findLinks(analyzer); 61 | 62 | int rootLength = rootPath.length(); 63 | 64 | int total = 0; 65 | for (String path : analyzer.getLoadedFiles()) { 66 | if (path.startsWith(rootPath)) { 67 | total++; 68 | } 69 | } 70 | 71 | Progress progress = new Progress(total, 50); 72 | 73 | for (String path : analyzer.getLoadedFiles()) { 74 | if (path.startsWith(rootPath)) { 75 | progress.tick(); 76 | File destFile = $.joinPath(OUTPUT_DIR, path.substring(rootLength)); 77 | destFile.getParentFile().mkdirs(); 78 | String destPath = destFile.getAbsolutePath() + ".html"; 79 | String html = markup(path); 80 | try { 81 | $.writeFile(destPath, html); 82 | } catch (Exception e) { 83 | $.msg("Failed to write: " + destPath); 84 | } 85 | } 86 | } 87 | 88 | $.msg("\nWrote " + analyzer.getLoadedFiles().size() + " files to " + OUTPUT_DIR); 89 | } 90 | 91 | 92 | @NotNull 93 | private String markup(String path) { 94 | String source; 95 | 96 | try { 97 | source = $.readFile(path); 98 | } catch (Exception e) { 99 | $.die("Failed to read file: " + path); 100 | return ""; 101 | } 102 | 103 | List\n") 116 | .append("\n") 119 | .append("\n\n") 120 | .append("
") 121 | .append(outline) 122 | .append("") 123 | .append("
")
124 |                 .append(addLineNumbers(styledSource))
125 |                 .append("
") 126 | .append("
"); 127 | return sb.toString(); 128 | } 129 | 130 | 131 | @NotNull 132 | private String addLineNumbers(@NotNull String source) { 133 | StringBuilder result = new StringBuilder((int) (source.length() * 1.2)); 134 | int count = 1; 135 | for (String line : source.split("\n")) { 136 | result.append(""); 137 | result.append(String.format("%1$4d", count++)); 138 | result.append(" "); 139 | result.append(line); 140 | result.append("\n"); 141 | } 142 | return result.toString(); 143 | } 144 | 145 | 146 | private static void usage() { 147 | $.msg("Usage: java -jar pysonar-2.0-SNAPSHOT.jar "); 148 | $.msg("Example that generates an index for Python 2.7 standard library:"); 149 | $.msg(" java -jar pysonar-2.0-SNAPSHOT.jar /usr/lib/python2.7 ./html"); 150 | System.exit(0); 151 | } 152 | 153 | 154 | @NotNull 155 | private static File checkFile(String path) { 156 | File f = new File(path); 157 | if (!f.canRead()) { 158 | $.die("Path not found or not readable: " + path); 159 | } 160 | return f; 161 | } 162 | 163 | 164 | public static void main(@NotNull String[] args) throws Exception { 165 | Options options = new Options(args); 166 | 167 | List argsList = options.getArgs(); 168 | String fileOrDir = argsList.get(0); 169 | OUTPUT_DIR = new File(argsList.get(1)); 170 | 171 | // System.out.println("options: " + options.getOptionsMap()); 172 | new Demo().start(fileOrDir, options.getOptionsMap()); 173 | $.msg($.getGCStats()); 174 | } 175 | } 176 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/demos/HtmlOutline.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.demos; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | import org.jetbrains.annotations.Nullable; 5 | import org.yinwang.pysonar.Analyzer; 6 | import org.yinwang.pysonar.Outliner; 7 | import org.yinwang.pysonar.$; 8 | 9 | import java.util.List; 10 | 11 | 12 | class HtmlOutline { 13 | 14 | private Analyzer analyzer; 15 | @Nullable 16 | private StringBuilder buffer; 17 | 18 | 19 | public HtmlOutline(Analyzer idx) { 20 | this.analyzer = idx; 21 | } 22 | 23 | 24 | @NotNull 25 | public String generate(String path) { 26 | buffer = new StringBuilder(1024); 27 | List entries = generateOutline(analyzer, path); 28 | addOutline(entries); 29 | String html = buffer.toString(); 30 | buffer = null; 31 | return html; 32 | } 33 | 34 | 35 | @NotNull 36 | public List generateOutline(Analyzer analyzer, @NotNull String file) { 37 | return new Outliner().generate(analyzer, file); 38 | } 39 | 40 | 41 | private void addOutline(@NotNull List entries) { 42 | add("
    \n"); 43 | for (Outliner.Entry e : entries) { 44 | addEntry(e); 45 | } 46 | add("
\n"); 47 | } 48 | 49 | 50 | private void addEntry(@NotNull Outliner.Entry e) { 51 | add("
  • "); 52 | 53 | String style = null; 54 | switch (e.kind) { 55 | case FUNCTION: 56 | case METHOD: 57 | case CONSTRUCTOR: 58 | style = "function"; 59 | break; 60 | case CLASS: 61 | style = "type-name"; 62 | break; 63 | case PARAMETER: 64 | style = "parameter"; 65 | break; 66 | case VARIABLE: 67 | case SCOPE: 68 | style = "identifier"; 69 | break; 70 | } 71 | 72 | add(""); 75 | 76 | if (style != null) { 77 | add(""); 80 | } 81 | add(e.getName()); 82 | if (style != null) { 83 | add(""); 84 | } 85 | 86 | add(""); 87 | 88 | if (e.isBranch()) { 89 | addOutline(e.getChildren()); 90 | } 91 | add("
    • "); 92 | } 93 | 94 | 95 | private void add(String text) { 96 | buffer.append(text); 97 | } 98 | } 99 | -------------------------------------------------------------------------------- /src/main/java/org/yinwang/pysonar/demos/Style.java: -------------------------------------------------------------------------------- 1 | package org.yinwang.pysonar.demos; 2 | 3 | import org.jetbrains.annotations.NotNull; 4 | import org.jetbrains.annotations.Nullable; 5 | import org.yinwang.pysonar.$; 6 | 7 | import java.util.List; 8 | 9 | 10 | /** 11 | * Represents a simple style run for purposes of source highlighting. 12 | */ 13 | public class Style implements Comparable