├── .gitattributes ├── .gitignore ├── .travis.yml ├── CMakeLists.txt ├── COPYING ├── Makefile.am ├── README.md ├── autogen.sh ├── configure.ac ├── grammar ├── m4 ├── .gitkeep ├── ax_check_cxxflags.m4 └── ax_cxx_compile_stdcxx_11.m4 ├── pypa.config ├── pypa.creator ├── pypa.creator.user ├── pypa.files ├── pypa.includes ├── src ├── CMakeLists.txt ├── Makefile.am ├── double-conversion │ ├── .gitignore │ ├── AUTHORS │ ├── CMakeLists.txt │ ├── COPYING │ ├── Changelog │ ├── LICENSE │ ├── README │ ├── SConstruct │ ├── double-conversionBuildTreeSettings.cmake.in │ ├── double-conversionConfig.cmake.in │ ├── double-conversionConfigVersion.cmake.in │ ├── src │ │ ├── CMakeLists.txt │ │ ├── SConscript │ │ ├── bignum-dtoa.cc │ │ ├── bignum-dtoa.h │ │ ├── bignum.cc │ │ ├── bignum.h │ │ ├── cached-powers.cc │ │ ├── cached-powers.h │ │ ├── diy-fp.cc │ │ ├── diy-fp.h │ │ ├── double-conversion.cc │ │ ├── double-conversion.h │ │ ├── fast-dtoa.cc │ │ ├── fast-dtoa.h │ │ ├── fixed-dtoa.cc │ │ ├── fixed-dtoa.h │ │ ├── ieee.h │ │ ├── strtod.cc │ │ ├── strtod.h │ │ └── utils.h │ └── test │ │ ├── CMakeLists.txt │ │ └── cctest │ │ ├── CMakeLists.txt │ │ ├── SConscript │ │ ├── cctest.cc │ │ ├── cctest.h │ │ ├── checks.h │ │ ├── gay-fixed.cc │ │ ├── gay-fixed.h │ │ ├── gay-precision.cc │ │ ├── gay-precision.h │ │ ├── gay-shortest-single.cc │ │ ├── gay-shortest-single.h │ │ ├── gay-shortest.cc │ │ ├── gay-shortest.h │ │ ├── test-bignum-dtoa.cc │ │ ├── test-bignum.cc │ │ ├── test-conversions.cc │ │ ├── test-diy-fp.cc │ │ ├── test-dtoa.cc │ │ ├── test-fast-dtoa.cc │ │ ├── test-fixed-dtoa.cc │ │ ├── test-ieee.cc │ │ └── test-strtod.cc ├── pypa │ ├── ast │ │ ├── Grammar │ │ ├── ast.cc │ │ ├── ast.hh │ │ ├── ast_type.inl │ │ ├── base.hh │ │ ├── context_assign.hh │ │ ├── dump.cc │ │ ├── dump.hh │ │ ├── macros.hh │ │ ├── tree_walker.hh │ │ ├── types.hh │ │ └── visitor.hh │ ├── filebuf.cc │ ├── filebuf.hh │ ├── lexer │ │ ├── delim.hh │ │ ├── keyword.hh │ │ ├── lexer.cc │ │ ├── lexer.hh │ │ ├── op.hh │ │ ├── test.cc │ │ ├── tokendef.hh │ │ └── tokens.hh │ ├── parser │ │ ├── apply.hh │ │ ├── error.hh │ │ ├── future_features.hh │ │ ├── make_string.cc │ │ ├── parser.cc │ │ ├── parser.hh │ │ ├── parser_fwd.hh │ │ ├── state.hh │ │ ├── symbol_table.cc │ │ ├── symbol_table.hh │ │ ├── symbol_table_visitor.hh │ │ └── test.cc │ └── types.hh └── run-tests.sh ├── test ├── CMakeLists.txt └── tests │ ├── assignment.py │ ├── complex.py │ ├── dict_comp.py │ ├── exec_test.py │ ├── fail_fun.py │ ├── fun.py │ ├── hello_world.py │ ├── import_test.py │ ├── index_test.py │ ├── list.py │ ├── list_comp.py │ ├── list_comp2.py │ ├── literals.py │ ├── no_newline.py │ ├── no_newline2.py │ ├── number_integer.py │ ├── pass.py │ └── test.py └── tools └── astdump.py /.gitattributes: -------------------------------------------------------------------------------- 1 | *.c text eol=lf 2 | *.cc text eol=lf 3 | *.cxx text eol=lf 4 | *.cpp text eol=lf 5 | *.py text eol=lf 6 | *.h text eol=lf 7 | *.hh text eol=lf 8 | *.hpp text eol=lf 9 | *.hxx text eol=lf 10 | *.rc text eol=lf 11 | *.txt text eol=lf 12 | *.bat text eol=crlf 13 | *.vc*proj* text eol=crlf 14 | *.ini text eol=crlf 15 | *.sln text eol=crlf 16 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.a 2 | *.la 3 | *.lo 4 | *.m4 5 | *.o 6 | *.sw[po] 7 | .deps 8 | .dirstamp 9 | .libs 10 | ar-lib 11 | Makefile 12 | Makefile.in 13 | Makefile.in 14 | autom4te.cache/* 15 | compile 16 | config.guess 17 | config.log 18 | config.status 19 | config.sub 20 | configure 21 | depcomp 22 | install-sh 23 | libtool 24 | ltmain.sh 25 | missing 26 | src/lexer-test 27 | src/parser-test 28 | 29 | CMakeCache.txt 30 | CMakeFiles 31 | Makefile 32 | cmake_install.cmake 33 | install_manifest.txt 34 | 35 | src/CTestTestfile.cmake 36 | src/Testing/ 37 | src/double-conversion/double-conversionBuildTreeSettings.cmake 38 | src/double-conversion/double-conversionConfig.cmake 39 | src/double-conversion/double-conversionConfigVersion.cmake 40 | src/double-conversion/double-conversionLibraryDepends.cmake 41 | 42 | Testing/ 43 | test/CTestTestfile.cmake 44 | test/Testing/ 45 | 46 | .ninja_deps 47 | .ninja_log 48 | build.ninja 49 | rules.ninja 50 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: cpp 2 | 3 | compiler: 4 | - clang 5 | - gcc 6 | 7 | # use travis-ci docker based infrastructure 8 | sudo: false 9 | 10 | cache: 11 | directories: 12 | - $HOME/.ccache 13 | 14 | env: 15 | global: 16 | # The next declaration is the encrypted COVERITY_SCAN_TOKEN, created 17 | # via the "travis encrypt" command using the project repo's public key 18 | - secure: "a3CnWMzTKAied4cDb/vW3wBrZyqkMtavYTA6ptbv2fZxaG5maZg2RGiDMmAkgdpa9VJTyF7qnZHn+eGpa2svYs3hafX8kF6h0BHOMlNh+tKKPIh3gsK9BeZoXR4h2r25av/0FAK96f7zkpEhGEENDaQZ7dDNKn/mb85DWV43+nw=" 19 | 20 | addons: 21 | apt: 22 | sources: 23 | - llvm-toolchain-precise-3.5 24 | - ubuntu-toolchain-r-test 25 | packages: 26 | - ccache 27 | - clang-3.5 28 | - cmake 29 | - g++-4.8 30 | - gcc-4.8 31 | - libgmp3-dev 32 | coverity_scan: 33 | project: 34 | name: "vinzenz/libpypa" 35 | description: "Build submitted via Travis CI" 36 | notification_email: evilissimo@gmail.com 37 | build_command_prepend: "cmake . && make clean" 38 | build_command: "make -j 4" 39 | branch_pattern: coverity_scan 40 | 41 | before_install: 42 | - if [ "$CC" = "clang" ]; then export CC="clang-3.5" CXX="clang++-3.5"; fi 43 | - if [ "$CC" = "gcc" ]; then export CC="gcc-4.8" CXX="g++-4.8"; fi 44 | 45 | script: 46 | - cmake . && make 47 | -------------------------------------------------------------------------------- /CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8) 2 | project(libpypa) 3 | 4 | add_subdirectory(src) 5 | add_subdirectory(test) 6 | 7 | # check 8 | add_custom_target(check-libpypa COMMAND ${CMAKE_CTEST_COMMAND} --output-on-failure DEPENDS pypa parser-test lexer-test WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/test) 9 | -------------------------------------------------------------------------------- /Makefile.am: -------------------------------------------------------------------------------- 1 | AUTOMAKE_OPTIONS = subdir-objects foreign 2 | ACLOCAL_AMFLAGS = -I m4 3 | 4 | SUBDIRS=src 5 | 6 | dist_noinst_SCRIPTS = autogen.sh 7 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # libpypa - A Python Parser Library in C++ 2 | 3 | - [Introduction](#introduction) 4 | - [Motivation](#introduction-motivation) 5 | - [Goal](#introduction-goal) 6 | - [Example](#example) 7 | - [Error Reporting](#error-reporting) 8 | - [Requirements](#requirements) 9 | - [Structure](#structure) 10 | - [Lexer](#structure-lexer) 11 | - [Parser](#structure-parser) 12 | - [AST](#structure-ast) 13 | - [License](#license) 14 | 15 | 16 | ## Introduction 17 | **libpypa** is a Python parser implemented in pure *C++*. It neither uses any 18 | tools like [flex][1], [yacc][2], [bison][3] etc, nor is it using any parser 19 | framework like [Boost.Spirit][4]. It's implementation is pure C++ code. 20 | 21 | 22 | ### Motivation 23 | I started getting involved into the [pyston project][5] where it had an entry 24 | in their getting involved list for implementing a parser for Python. Never 25 | having properly tackled the problem of creating a parser library for any 26 | language, I decided it might be worth a try, since most of the libraries I 27 | found, where basically just using the builtin Python parser or where 28 | implemented in Python itself. 29 | 30 | 31 | ### Goal 32 | The first goal of the library is to support python 2.7 syntax, later on 3.x 33 | syntax might be added. 34 | 35 | 36 | ## Example 37 | 38 | An example file: 39 | 40 | $cat hello_world.py 41 | #! /usr/bin/env python 42 | # -*- coding: utf-8 -*- 43 | # 44 | 45 | """ 46 | A "Hello World" example for the pypa parser 47 | """ 48 | import sys 49 | 50 | print >> sys.stdout, "Hello", "World!" 51 | 52 | 53 | And here the output of the test parser: 54 | 55 | $ ./parser-test hello_world.py 56 | Parsing successfull 57 | 58 | [Module] 59 | - body: 60 | [Suite] 61 | - items: [ 62 | [DocString] 63 | - doc: 64 | A "Hello World" example for the pypa parser 65 | 66 | 67 | [Import] 68 | - names: 69 | [Alias] 70 | - as_name: 71 | - name: 72 | [Name] 73 | - context: Load 74 | - dotted: False 75 | - id: sys 76 | 77 | [Print] 78 | - destination: 79 | [Attribute] 80 | - attribute: 81 | [Name] 82 | - context: Load 83 | - dotted: False 84 | - id: stdout 85 | - context: Load 86 | - value: 87 | [Name] 88 | - context: Load 89 | - dotted: False 90 | - id: sys 91 | - newline: True 92 | - values: [ 93 | [Str] 94 | - value: Hello 95 | 96 | [Str] 97 | - value: World! 98 | ] 99 | ] 100 | - kind: Module 101 | 102 | And here the parse tree of python: (astdump.py can be found in tools) 103 | 104 | [Module] 105 | - body: [ 106 | 107 | [Expr] 108 | - value: 109 | [Str] 110 | - s: 111 | A "Hello World" example for the pypa parser 112 | 113 | 114 | [Import] 115 | - names: [ 116 | 117 | [alias] 118 | - asname: None 119 | - name: sys 120 | ] 121 | 122 | [Print] 123 | - dest: 124 | [Attribute] 125 | - attr: stdout 126 | - ctx: Load 127 | - value: 128 | [Name] 129 | - ctx: Load 130 | - id: sys 131 | - nl: True 132 | - values: [ 133 | 134 | [Str] 135 | - s: Hello 136 | 137 | [Str] 138 | - s: World! 139 | ] 140 | ] 141 | 142 | 143 | 144 | ## Error Reporting 145 | The parser supports also SyntaxError and IndentionError reporting: 146 | 147 | Let's take a look at this file `syntax_error.py` which clearly has a 148 | syntax error: 149 | 150 | #! /usr/bin/env python 151 | # -*- coding: utf-8 -*- 152 | """ 153 | Syntax error example 154 | """ 155 | 156 | print x y z 157 | 158 | This is the output of the test parser: 159 | 160 | $./parser-test syntax_error.py 161 | File "syntax_error.py", line 7 162 | print x y z 163 | ^ 164 | SyntaxError: Expected new line after statement 165 | -> Reported @pypa/parser/parser.cc:944 in bool pypa::simple_stmt(pypa::{anonymous}::State&, pypa::AstStmt&) 166 | 167 | Parsing failed 168 | 169 | And this of cpython 2.7: 170 | 171 | $ python syntax_error.py 172 | File "syntax_error.py", line 7 173 | print x y z 174 | ^ 175 | SyntaxError: invalid syntax 176 | 177 | **libpypa** uses different error messages than python, however in the hopes that 178 | that would increase the clarity. 179 | 180 | 181 | ## Requirements 182 | To be able using **libpypa**, you have to have a *C++11* compiler available. 183 | **libpypa** was developed on top of *g++ 4.8.2* and it heavily uses *C++11* 184 | features where seen fit. 185 | 186 | **libpypa** currently does not depend on any other libraries than the *C++11* 187 | standard library with the exception of the `class FileBuf` which currently 188 | uses system libraries, but might be changed to just use `fopen`/`fread`/ 189 | `fclose`. 190 | 191 | 192 | ## Structure 193 | **libpypa** currently consists of 3 major parts: 194 | 195 | 1. `Lexer` 196 | 2. `Parser` 197 | 3. `AST` 198 | 199 | 200 | ### Lexer 201 | The `Lexer` portion of the library tokenizes the input for the `Parser` and 202 | distinguishes the different types of tokens for the `Parser`. 203 | 204 | 205 | ### Parser 206 | The `Parser` utilizes the `Lexer` to parse the input and generates a 207 | preliminary `AST` from the input. 208 | 209 | 210 | ### AST 211 | The AST contains the definition of all syntax elements in the code. The main 212 | parts of the definition are in `pypa/ast/ast.hh` which makes heavily use of 213 | preprocessor macros to define typedefs, mappings for compile time type lookups 214 | by AstType (enum class), and an implementation for a switch based visitor. 215 | 216 | The AST types do not implement any methods, they are just structures with data. 217 | The only thing which is in there for some of the bases is the constructor, to 218 | set the type id value and initialize the line and column values. 219 | 220 | 221 | ## License 222 | Copyright 2014 Vinzenz Feenstra 223 | 224 | Licensed under the Apache License, Version 2.0 (the "License"); 225 | you may not use this file except in compliance with the License. 226 | You may obtain a copy of the License at 227 | 228 | http://www.apache.org/licenses/LICENSE-2.0 229 | 230 | Unless required by applicable law or agreed to in writing, software 231 | distributed under the License is distributed on an "AS IS" BASIS, 232 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 233 | See the License for the specific language governing permissions and 234 | limitations under the License. 235 | 236 | 237 | ### License for src/double-conversion 238 | Copyright 2006-2011, the V8 project authors. All rights reserved. 239 | Redistribution and use in source and binary forms, with or without 240 | modification, are permitted provided that the following conditions are 241 | met: 242 | 243 | * Redistributions of source code must retain the above copyright 244 | notice, this list of conditions and the following disclaimer. 245 | * Redistributions in binary form must reproduce the above 246 | copyright notice, this list of conditions and the following 247 | disclaimer in the documentation and/or other materials provided 248 | with the distribution. 249 | * Neither the name of Google Inc. nor the names of its 250 | contributors may be used to endorse or promote products derived 251 | from this software without specific prior written permission. 252 | 253 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 254 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 255 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 256 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 257 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 258 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 259 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 260 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 261 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 262 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 263 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 264 | 265 | 266 | [1]: http://flex.sourceforge.net/ 267 | [2]: http://invisible-island.net/byacc/byacc.html 268 | [3]: http://www.gnu.org/s/bison/ 269 | [4]: http://boost-spirit.com 270 | [5]: http://github.com/dropbox/pyston 271 | 272 | -------------------------------------------------------------------------------- /autogen.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | autoreconf -ivf 4 | -------------------------------------------------------------------------------- /configure.ac: -------------------------------------------------------------------------------- 1 | AC_INIT([A python 2 parser], 2 | [0.1], 3 | [evilissimo@gmail.com], 4 | [pypa], 5 | [http://github.com/vinzenz/pypa]) 6 | AC_PREREQ([2.59]) 7 | AC_CONFIG_MACRO_DIR([m4]) 8 | 9 | m4_pattern_allow([AM_PROG_AR]) 10 | 11 | AM_INIT_AUTOMAKE([1.10 -Wall no-define subdir-objects]) 12 | 13 | AC_LANG_CPLUSPLUS 14 | AC_PROG_CXX 15 | AX_CXX_COMPILE_STDCXX_11 16 | 17 | AM_PROG_AR 18 | AC_PROG_LIBTOOL 19 | 20 | # check for extra compiler flags 21 | AX_ADD_CXXFLAGS([-Werror]) # must be called first to trigger an error for the other unsupported options 22 | AX_ADD_CXXFLAGS([-W]) 23 | AX_ADD_CXXFLAGS([-Wall]) 24 | AX_ADD_CXXFLAGS([-pedantic]) 25 | AX_ADD_CXXFLAGS([-Wno-unused-parameter]) 26 | AX_ADD_CXXFLAGS([-Wno-unused-local-typedefs]) 27 | 28 | 29 | # check for cpython sources 30 | AC_ARG_WITH([cpython-src], 31 | [AS_HELP_STRING([--with-cpython-src], 32 | [path to cpython sources, to feed the test suite])]) 33 | AC_SUBST([CPYTHON_SRC], [$with_cpython_src]) 34 | 35 | AC_CONFIG_FILES([ 36 | Makefile 37 | src/Makefile 38 | ]) 39 | AC_OUTPUT 40 | -------------------------------------------------------------------------------- /grammar: -------------------------------------------------------------------------------- 1 | NEWLINE: '\r' | "\r\n" 2 | ENDMARKER: EOF 3 | NAME: [_A-Za-z][0-9A-Za-z_]* 4 | 5 | single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE 6 | file_input: (NEWLINE | stmt)* ENDMARKER 7 | eval_input: testlist NEWLINE* ENDMARKER 8 | 9 | decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE 10 | decorators: decorator+ 11 | decorated: decorators (classdef | funcdef) 12 | funcdef: 'def' NAME parameters ':' suite 13 | parameters: '(' [varargslist] ')' 14 | varargslist: ((fpdef ['=' test] ',')* 15 | ('*' NAME [',' '**' NAME] | '**' NAME) | 16 | fpdef ['=' test] (',' fpdef ['=' test])* [',']) 17 | fpdef: NAME | '(' fplist ')' 18 | fplist: fpdef (',' fpdef)* [','] 19 | 20 | stmt: simple_stmt | compound_stmt 21 | simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE 22 | small_stmt: (expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt | 23 | import_stmt | global_stmt | exec_stmt | assert_stmt) 24 | expr_stmt: testlist (augassign (yield_expr|testlist) | 25 | ('=' (yield_expr|testlist))*) 26 | augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | 27 | '<<=' | '>>=' | '**=' | '//=') 28 | # For normal assignments, additional restrictions enforced by the interpreter 29 | print_stmt: 'print' ( [ test (',' test)* [','] ] | 30 | '>>' test [ (',' test)+ [','] ] ) 31 | del_stmt: 'del' exprlist 32 | pass_stmt: 'pass' 33 | flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt 34 | break_stmt: 'break' 35 | continue_stmt: 'continue' 36 | return_stmt: 'return' [testlist] 37 | yield_stmt: yield_expr 38 | raise_stmt: 'raise' [test [',' test [',' test]]] 39 | import_stmt: import_name | import_from 40 | import_name: 'import' dotted_as_names 41 | import_from: ('from' ('.'* dotted_name | '.'+) 42 | 'import' ('*' | '(' import_as_names ')' | import_as_names)) 43 | import_as_name: NAME ['as' NAME] 44 | dotted_as_name: dotted_name ['as' NAME] 45 | import_as_names: import_as_name (',' import_as_name)* [','] 46 | dotted_as_names: dotted_as_name (',' dotted_as_name)* 47 | dotted_name: NAME ('.' NAME)* 48 | global_stmt: 'global' NAME (',' NAME)* 49 | exec_stmt: 'exec' expr ['in' test [',' test]] 50 | assert_stmt: 'assert' test [',' test] 51 | 52 | compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated 53 | if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] 54 | while_stmt: 'while' test ':' suite ['else' ':' suite] 55 | for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] 56 | try_stmt: ('try' ':' suite 57 | ((except_clause ':' suite)+ 58 | ['else' ':' suite] 59 | ['finally' ':' suite] | 60 | 'finally' ':' suite)) 61 | with_stmt: 'with' with_item (',' with_item)* ':' suite 62 | with_item: test ['as' expr] 63 | # NB compile.c makes sure that the default except clause is last 64 | except_clause: 'except' [test [('as' | ',') test]] 65 | suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT 66 | 67 | # Backward compatibility cruft to support: 68 | # [ x for x in lambda: True, lambda: False if x() ] 69 | # even while also allowing: 70 | # lambda x: 5 if x else 2 71 | # (But not a mix of the two) 72 | testlist_safe: old_test [(',' old_test)+ [',']] 73 | old_test: or_test | old_lambdef 74 | old_lambdef: 'lambda' [varargslist] ':' old_test 75 | 76 | test: or_test ['if' or_test 'else' test] | lambdef 77 | or_test: and_test ('or' and_test)* 78 | and_test: not_test ('and' not_test)* 79 | not_test: 'not' not_test | comparison 80 | comparison: expr (comp_op expr)* 81 | comp_op: '<'|'>'|'=='|'>='|'<='|'<>'|'!='|'in'|'not' 'in'|'is'|'is' 'not' 82 | expr: xor_expr ('|' xor_expr)* 83 | xor_expr: and_expr ('^' and_expr)* 84 | and_expr: shift_expr ('&' shift_expr)* 85 | shift_expr: arith_expr (('<<'|'>>') arith_expr)* 86 | arith_expr: term (('+'|'-') term)* 87 | term: factor (('*'|'/'|'%'|'//') factor)* 88 | factor: ('+'|'-'|'~') factor | power 89 | power: atom trailer* ['**' factor] 90 | atom: ('(' [yield_expr|testlist_comp] ')' | 91 | '[' [listmaker] ']' | 92 | '{' [dictorsetmaker] '}' | 93 | '`' testlist1 '`' | 94 | NAME | NUMBER | STRING+) 95 | listmaker: test ( list_for | (',' test)* [','] ) 96 | testlist_comp: test ( comp_for | (',' test)* [','] ) 97 | lambdef: 'lambda' [varargslist] ':' test 98 | trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME 99 | subscriptlist: subscript (',' subscript)* [','] 100 | subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop] 101 | sliceop: ':' [test] 102 | exprlist: expr (',' expr)* [','] 103 | testlist: test (',' test)* [','] 104 | dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) | 105 | (test (comp_for | (',' test)* [','])) ) 106 | 107 | classdef: 'class' NAME ['(' [testlist] ')'] ':' suite 108 | 109 | arglist: (argument ',')* (argument [','] 110 | |'*' test (',' argument)* [',' '**' test] 111 | |'**' test) 112 | # The reason that keywords are test nodes instead of NAME is that using NAME 113 | # results in an ambiguity. ast.c makes sure it's a NAME. 114 | argument: test [comp_for] | test '=' test 115 | 116 | list_iter: list_for | list_if 117 | list_for: 'for' exprlist 'in' testlist_safe [list_iter] 118 | list_if: 'if' old_test [list_iter] 119 | 120 | comp_iter: comp_for | comp_if 121 | comp_for: 'for' exprlist 'in' or_test [comp_iter] 122 | comp_if: 'if' old_test [comp_iter] 123 | 124 | testlist1: test (',' test)* 125 | 126 | # not used in grammar, but may appear in "node" passed from Parser to Compiler 127 | encoding_decl: NAME 128 | 129 | yield_expr: 'yield' [testlist] 130 | 131 | -------------------------------------------------------------------------------- /m4/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/dropbox/libpypa/5bba0f386b67f864683438aac65705dfcd695fbf/m4/.gitkeep -------------------------------------------------------------------------------- /m4/ax_check_cxxflags.m4: -------------------------------------------------------------------------------- 1 | dnl check if the compiler supports $1 as an extra flag 2 | dnl if so, add it to the CXXFLAGS 3 | AC_DEFUN([AX_ADD_CXXFLAGS], 4 | [pypa_save_CXXFLAGS="$CXXFLAGS" 5 | CXXFLAGS="$CXXFLAGS $1" 6 | AC_MSG_CHECKING([if $CXX supports $1]) 7 | AC_COMPILE_IFELSE([AC_LANG_SOURCE([int pypa = 1;])], 8 | [AC_MSG_RESULT([yes])], 9 | [CXXFLAGS=$pypa_save_CXXFLAGS 10 | AC_MSG_RESULT([no])]) 11 | ]) 12 | -------------------------------------------------------------------------------- /m4/ax_cxx_compile_stdcxx_11.m4: -------------------------------------------------------------------------------- 1 | # ============================================================================ 2 | # http://www.gnu.org/software/autoconf-archive/ax_cxx_compile_stdcxx_11.html 3 | # ============================================================================ 4 | # 5 | # SYNOPSIS 6 | # 7 | # AX_CXX_COMPILE_STDCXX_11([ext|noext],[mandatory|optional]) 8 | # 9 | # DESCRIPTION 10 | # 11 | # Check for baseline language coverage in the compiler for the C++11 12 | # standard; if necessary, add switches to CXXFLAGS to enable support. 13 | # 14 | # The first argument, if specified, indicates whether you insist on an 15 | # extended mode (e.g. -std=gnu++11) or a strict conformance mode (e.g. 16 | # -std=c++11). If neither is specified, you get whatever works, with 17 | # preference for an extended mode. 18 | # 19 | # The second argument, if specified 'mandatory' or if left unspecified, 20 | # indicates that baseline C++11 support is required and that the macro 21 | # should error out if no mode with that support is found. If specified 22 | # 'optional', then configuration proceeds regardless, after defining 23 | # HAVE_CXX11 if and only if a supporting mode is found. 24 | # 25 | # LICENSE 26 | # 27 | # Copyright (c) 2008 Benjamin Kosnik 28 | # Copyright (c) 2012 Zack Weinberg 29 | # Copyright (c) 2013 Roy Stogner 30 | # Copyright (c) 2014 Alexey Sokolov 31 | # 32 | # Copying and distribution of this file, with or without modification, are 33 | # permitted in any medium without royalty provided the copyright notice 34 | # and this notice are preserved. This file is offered as-is, without any 35 | # warranty. 36 | 37 | #serial 4 38 | 39 | m4_define([_AX_CXX_COMPILE_STDCXX_11_testbody], [[ 40 | template 41 | struct check 42 | { 43 | static_assert(sizeof(int) <= sizeof(T), "not big enough"); 44 | }; 45 | 46 | struct Base { 47 | virtual void f() {} 48 | }; 49 | struct Child : public Base { 50 | virtual void f() override {} 51 | }; 52 | 53 | typedef check> right_angle_brackets; 54 | 55 | int a; 56 | decltype(a) b; 57 | 58 | typedef check check_type; 59 | check_type c; 60 | check_type&& cr = static_cast(c); 61 | 62 | auto d = a; 63 | auto l = [](){}; 64 | ]]) 65 | 66 | AC_DEFUN([AX_CXX_COMPILE_STDCXX_11], [dnl 67 | m4_if([$1], [], [], 68 | [$1], [ext], [], 69 | [$1], [noext], [], 70 | [m4_fatal([invalid argument `$1' to AX_CXX_COMPILE_STDCXX_11])])dnl 71 | m4_if([$2], [], [ax_cxx_compile_cxx11_required=true], 72 | [$2], [mandatory], [ax_cxx_compile_cxx11_required=true], 73 | [$2], [optional], [ax_cxx_compile_cxx11_required=false], 74 | [m4_fatal([invalid second argument `$2' to AX_CXX_COMPILE_STDCXX_11])]) 75 | AC_LANG_PUSH([C++])dnl 76 | ac_success=no 77 | AC_CACHE_CHECK(whether $CXX supports C++11 features by default, 78 | ax_cv_cxx_compile_cxx11, 79 | [AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_11_testbody])], 80 | [ax_cv_cxx_compile_cxx11=yes], 81 | [ax_cv_cxx_compile_cxx11=no])]) 82 | if test x$ax_cv_cxx_compile_cxx11 = xyes; then 83 | ac_success=yes 84 | fi 85 | 86 | m4_if([$1], [noext], [], [dnl 87 | if test x$ac_success = xno; then 88 | for switch in -std=gnu++11 -std=gnu++0x; do 89 | cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx11_$switch]) 90 | AC_CACHE_CHECK(whether $CXX supports C++11 features with $switch, 91 | $cachevar, 92 | [ac_save_CXXFLAGS="$CXXFLAGS" 93 | CXXFLAGS="$CXXFLAGS $switch" 94 | AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_11_testbody])], 95 | [eval $cachevar=yes], 96 | [eval $cachevar=no]) 97 | CXXFLAGS="$ac_save_CXXFLAGS"]) 98 | if eval test x\$$cachevar = xyes; then 99 | CXXFLAGS="$CXXFLAGS $switch" 100 | ac_success=yes 101 | break 102 | fi 103 | done 104 | fi]) 105 | 106 | m4_if([$1], [ext], [], [dnl 107 | if test x$ac_success = xno; then 108 | for switch in -std=c++11 -std=c++0x; do 109 | cachevar=AS_TR_SH([ax_cv_cxx_compile_cxx11_$switch]) 110 | AC_CACHE_CHECK(whether $CXX supports C++11 features with $switch, 111 | $cachevar, 112 | [ac_save_CXXFLAGS="$CXXFLAGS" 113 | CXXFLAGS="$CXXFLAGS $switch" 114 | AC_COMPILE_IFELSE([AC_LANG_SOURCE([_AX_CXX_COMPILE_STDCXX_11_testbody])], 115 | [eval $cachevar=yes], 116 | [eval $cachevar=no]) 117 | CXXFLAGS="$ac_save_CXXFLAGS"]) 118 | if eval test x\$$cachevar = xyes; then 119 | CXXFLAGS="$CXXFLAGS $switch" 120 | ac_success=yes 121 | break 122 | fi 123 | done 124 | fi]) 125 | AC_LANG_POP([C++]) 126 | if test x$ax_cxx_compile_cxx11_required = xtrue; then 127 | if test x$ac_success = xno; then 128 | AC_MSG_ERROR([*** A compiler with support for C++11 language features is required.]) 129 | fi 130 | else 131 | if test x$ac_success = xno; then 132 | HAVE_CXX11=0 133 | AC_MSG_NOTICE([No compiler with C++11 support was found]) 134 | else 135 | HAVE_CXX11=1 136 | AC_DEFINE(HAVE_CXX11,1, 137 | [define if the compiler supports basic C++11 syntax]) 138 | fi 139 | 140 | AC_SUBST(HAVE_CXX11) 141 | fi 142 | ]) 143 | -------------------------------------------------------------------------------- /pypa.config: -------------------------------------------------------------------------------- 1 | // ADD PREDEFINED MACROS HERE! 2 | -------------------------------------------------------------------------------- /pypa.creator: -------------------------------------------------------------------------------- 1 | [General] 2 | -------------------------------------------------------------------------------- /pypa.files: -------------------------------------------------------------------------------- 1 | src/pypa/ast/ast.cc 2 | src/pypa/ast/ast.hh 3 | src/pypa/lexer/delim.hh 4 | src/pypa/lexer/keyword.hh 5 | src/pypa/lexer/lexer.cc 6 | src/pypa/lexer/lexer.hh 7 | src/pypa/lexer/op.hh 8 | src/pypa/lexer/tokendef.hh 9 | src/pypa/lexer/tokens.hh 10 | src/pypa/parser/parser.cc 11 | src/pypa/parser/parser.hh 12 | src/pypa/parser/test.cc 13 | src/pypa/filebuf.cc 14 | src/pypa/filebuf.hh 15 | src/pypa/ast/ast_type.inl 16 | src/pypa/ast/visitor.hh 17 | src/pypa/ast/types.hh 18 | src/pypa/ast/macros.hh 19 | src/pypa/ast/dump.cc 20 | src/pypa/ast/base.hh 21 | src/pypa/parser/state.hh 22 | src/pypa/parser/apply.hh 23 | src/pypa/parser/parser2.cc 24 | src/pypa/parser/parser_fwd.hh 25 | src/pypa/ast/tree_walker.hh 26 | src/pypa/ast/context_assign.hh 27 | src/pypa/parser/symbol_table.hh 28 | src/pypa/types.hh 29 | src/pypa/parser/future_features.hh 30 | src/pypa/parser/symbol_table.cc 31 | src/pypa/parser/symbol_table_visitor.hh 32 | src/pypa/ast/dump.hh 33 | src/pypa/parser/make_string.cc 34 | src/pypa/parser/error.hh 35 | -------------------------------------------------------------------------------- /pypa.includes: -------------------------------------------------------------------------------- 1 | src 2 | src/pypa/parser 3 | -------------------------------------------------------------------------------- /src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | include(CheckCXXCompilerFlag) 2 | 3 | set(CMAKE_INCLUDE_CURRENT_DIR ON) 4 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=c++11 -W -Wall -Werror -pedantic -Wno-unused-parameter -Wno-unused-const-variable") 5 | 6 | check_cxx_compiler_flag(-Wno-unused-local-typedefs CXX_NO_UNUSED_LOCAL_TYPEDEFS) 7 | if(CXX_NO_UNUSED_LOCAL_TYPEDEFS) 8 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-local-typedefs") 9 | endif() 10 | 11 | set(CLANG_FLAGS "-Qunused-arguments -fcolor-diagnostics" CACHE STRING "Clang specific C and CXX flags") 12 | if("${CMAKE_CXX_COMPILER_ID}" STREQUAL "Clang") 13 | set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CLANG_FLAGS}") 14 | endif() 15 | 16 | add_subdirectory(double-conversion) 17 | 18 | add_library(pypa pypa/ast/ast.cc 19 | pypa/ast/dump.cc 20 | pypa/filebuf.cc 21 | pypa/lexer/lexer.cc 22 | pypa/parser/parser.cc 23 | pypa/parser/make_string.cc 24 | pypa/parser/symbol_table.cc) 25 | 26 | # lexer_test 27 | add_executable(lexer-test EXCLUDE_FROM_ALL pypa/parser/test.cc) 28 | add_dependencies(lexer-test pypa) 29 | target_link_libraries(lexer-test pypa gmp double-conversion) 30 | 31 | # parser_test 32 | add_executable(parser-test EXCLUDE_FROM_ALL pypa/parser/test.cc) 33 | add_dependencies(parser-test pypa) 34 | target_link_libraries(parser-test pypa gmp double-conversion) 35 | 36 | # install 37 | install(TARGETS pypa ARCHIVE DESTINATION lib) 38 | install(DIRECTORY pypa DESTINATION include FILES_MATCHING PATTERN "*.hh" PATTERN "*.inl") 39 | -------------------------------------------------------------------------------- /src/Makefile.am: -------------------------------------------------------------------------------- 1 | AM_CPPFLAGS=-DIEEE_8087 2 | 3 | lib_LTLIBRARIES=libpypa.la 4 | libpypa_la_LDFLAGS=$(PYPA_LDFLAGS) -lgmp 5 | libpypa_la_SOURCES=\ 6 | pypa/ast/ast.cc \ 7 | pypa/ast/dump.cc \ 8 | pypa/filebuf.cc \ 9 | pypa/lexer/lexer.cc \ 10 | pypa/parser/parser.cc \ 11 | pypa/parser/make_string.cc \ 12 | pypa/parser/symbol_table.cc \ 13 | double-conversion/src/bignum-dtoa.cc \ 14 | double-conversion/src/bignum.cc \ 15 | double-conversion/src/cached-powers.cc \ 16 | double-conversion/src/diy-fp.cc \ 17 | double-conversion/src/double-conversion.cc \ 18 | double-conversion/src/fast-dtoa.cc \ 19 | double-conversion/src/fixed-dtoa.cc \ 20 | double-conversion/src/strtod.cc \ 21 | $(NULL) 22 | 23 | noinst_PROGRAMS=lexer-test parser-test 24 | lexer_test_SOURCES=\ 25 | pypa/lexer/test.cc \ 26 | $(NULL) 27 | lexer_test_LDADD=libpypa.la 28 | 29 | parser_test_SOURCES=\ 30 | pypa/parser/test.cc \ 31 | $(NULL) 32 | parser_test_LDADD=libpypa.la 33 | 34 | check-local:lexer-test parser-test $(srcdir)/run-tests.sh 35 | CPYTHON_SRC=$(CPYTHON_SRC) $(srcdir)/run-tests.sh 36 | 37 | pypadir=$(includedir)/pypa 38 | pypa_HEADERS=\ 39 | pypa/filebuf.hh \ 40 | pypa/types.hh \ 41 | $(NULL) 42 | 43 | 44 | pypaparserdir=$(includedir)/pypa/parser 45 | pypaparser_HEADERS=\ 46 | pypa/parser/apply.hh \ 47 | pypa/parser/error.hh \ 48 | pypa/parser/future_features.hh \ 49 | pypa/parser/parser.hh \ 50 | pypa/parser/parser_fwd.hh \ 51 | pypa/parser/state.hh \ 52 | pypa/parser/symbol_table.hh \ 53 | pypa/parser/symbol_table_visitor.hh \ 54 | $(NULL) 55 | 56 | pypaastdir=$(includedir)/pypa/ast 57 | pypaast_HEADERS=\ 58 | pypa/ast/ast.hh \ 59 | pypa/ast/ast_type.inl \ 60 | pypa/ast/base.hh \ 61 | pypa/ast/context_assign.hh \ 62 | pypa/ast/dump.hh \ 63 | pypa/ast/macros.hh \ 64 | pypa/ast/tree_walker.hh \ 65 | pypa/ast/types.hh \ 66 | pypa/ast/visitor.hh \ 67 | $(NULL) 68 | 69 | pypalexerdir=$(includedir)/pypa/lexer 70 | pypalexer_HEADERS=\ 71 | pypa/lexer/delim.hh \ 72 | pypa/lexer/keyword.hh \ 73 | pypa/lexer/lexer.hh \ 74 | pypa/lexer/op.hh \ 75 | pypa/lexer/tokendef.hh \ 76 | pypa/lexer/tokens.hh \ 77 | $(NULL) 78 | 79 | EXTRA_DIST=double-conversion 80 | -------------------------------------------------------------------------------- /src/double-conversion/.gitignore: -------------------------------------------------------------------------------- 1 | .sconsign.dblite 2 | run_tests 3 | *.o 4 | -------------------------------------------------------------------------------- /src/double-conversion/AUTHORS: -------------------------------------------------------------------------------- 1 | # Below is a list of people and organizations that have contributed 2 | # to the double-conversion project. Names should be added to the 3 | # list like so: 4 | # 5 | # Name/Organization 6 | 7 | Google Inc. 8 | Mozilla Foundation 9 | 10 | Jeff Muizelaar 11 | Mike Hommey 12 | Martin Olsson 13 | Kent Williams 14 | Elan Ruusamäe 15 | -------------------------------------------------------------------------------- /src/double-conversion/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION 2.8) 2 | project(double-conversion) 3 | 4 | # pick a version # 5 | set(double-conversion_VERSION 2.0.1) 6 | set(double-conversion_SOVERSION_MAJOR 1) 7 | set(double-conversion_SOVERSION_MINOR 0) 8 | set(double-conversion_SOVERSION_PATCH 0) 9 | set(double-conversion_SOVERSION 10 | ${double-conversion_SOVERSION_MAJOR}.${double-conversion_SOVERSION_MINOR}.${double-conversion_SOVERSION_PATCH}) 11 | 12 | # set paths for install -- empty initially 13 | # Offer the user the choice of overriding the installation directories 14 | set(INSTALL_BIN_DIR CACHE PATH "Installation directory for libraries") 15 | set(INSTALL_LIB_DIR CACHE PATH "Installation directory for libraries") 16 | set(INSTALL_INCLUDE_DIR CACHE PATH "Installation directory for include") 17 | # set suffix for CMake files used for packaging 18 | if(WIN32 AND NOT CYGWIN) 19 | set(INSTALL_CMAKE_DIR CMake) 20 | else() 21 | set(INSTALL_CMAKE_DIR lib/CMake/double-conversion) 22 | endif() 23 | 24 | # Make relative paths absolute (needed later) 25 | foreach(p LIB BIN INCLUDE CMAKE) 26 | set(var INSTALL_${p}_DIR) 27 | if(NOT IS_ABSOLUTE "${${var}}") 28 | set(${var} "${CMAKE_INSTALL_PREFIX}/${${var}}") 29 | endif() 30 | endforeach() 31 | 32 | # 33 | # set up include dirs 34 | include_directories("${PROJECT_SOURCE_DIR}/src" 35 | "${PROJECT_BINARY_DIR}" 36 | ) 37 | 38 | # Add src subdirectory 39 | add_subdirectory(src) 40 | 41 | # 42 | # set up testing if requested 43 | option(BUILD_TESTING "Build test programs" OFF) 44 | if(BUILD_TESTING) 45 | enable_testing() 46 | include(CTest) 47 | add_subdirectory(test) 48 | endif() 49 | 50 | # 51 | # mention the library target as export library 52 | export(TARGETS double-conversion 53 | FILE "${PROJECT_BINARY_DIR}/double-conversionLibraryDepends.cmake") 54 | 55 | # 56 | # set this build as an importable package 57 | export(PACKAGE double-conversion) 58 | 59 | # 60 | # make a cmake file -- in this case, all that needs defining 61 | # is double-conversion_INCLUDE_DIRS 62 | configure_file(double-conversionBuildTreeSettings.cmake.in 63 | "${PROJECT_BINARY_DIR}/double-conversionBuildTreeSettings.cmake" 64 | @ONLY) 65 | 66 | # 67 | # determine where include is relative to the CMake dir in 68 | # in installed tree 69 | file(RELATIVE_PATH CONF_REL_INCLUDE_DIR "${INSTALL_CMAKE_DIR}" 70 | "${INSTALL_INCLUDE_DIR}") 71 | 72 | # 73 | # sets up config to be used by CMake find_package 74 | configure_file(double-conversionConfig.cmake.in 75 | "${PROJECT_BINARY_DIR}/double-conversionConfig.cmake" 76 | @ONLY) 77 | # 78 | # Export version # checked by find_package 79 | configure_file(double-conversionConfigVersion.cmake.in 80 | "${PROJECT_BINARY_DIR}/double-conversionConfigVersion.cmake" 81 | @ONLY) 82 | # 83 | # install config files for find_package 84 | install(FILES 85 | "${PROJECT_BINARY_DIR}/double-conversionConfig.cmake" 86 | "${PROJECT_BINARY_DIR}/double-conversionConfigVersion.cmake" 87 | DESTINATION "${INSTALL_CMAKE_DIR}" COMPONENT dev) 88 | 89 | 90 | # 91 | # generates install cmake files to find libraries in installation. 92 | install(EXPORT double-conversionLibraryDepends DESTINATION 93 | "${INSTALL_CMAKE_DIR}" COMPONENT dev) 94 | -------------------------------------------------------------------------------- /src/double-conversion/COPYING: -------------------------------------------------------------------------------- 1 | Copyright 2006-2011, the V8 project authors. All rights reserved. 2 | Redistribution and use in source and binary forms, with or without 3 | modification, are permitted provided that the following conditions are 4 | met: 5 | 6 | * Redistributions of source code must retain the above copyright 7 | notice, this list of conditions and the following disclaimer. 8 | * Redistributions in binary form must reproduce the above 9 | copyright notice, this list of conditions and the following 10 | disclaimer in the documentation and/or other materials provided 11 | with the distribution. 12 | * Neither the name of Google Inc. nor the names of its 13 | contributors may be used to endorse or promote products derived 14 | from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | -------------------------------------------------------------------------------- /src/double-conversion/Changelog: -------------------------------------------------------------------------------- 1 | 2014-03-08: 2 | Update version number for cmake. 3 | Support shared libraries with cmake. 4 | Add build instructions to the README. 5 | 6 | 2014-01-12: 7 | Tagged v2.0.1. 8 | Fix compilation for ARMv8 64bit (used wrong define). 9 | Improved SConstruct file. Thanks to Milan Bouchet-Valat and Elan Ruusamäe. 10 | Fixed lots of warnings (especially on Windows). Thanks to Greg Smith. 11 | 12 | 2013-11-09: 13 | Tagged v2.0.0. 14 | String-to-Double|Float: ALLOW_LEADING_SPACES and similar flags now include 15 | new-lines, tabs and all Unicode whitespace characters. 16 | 17 | 2013-11-09: 18 | Tagged v1.1.2. 19 | Add support for ARM 64 and OsX ppc. 20 | Rewrite tests so they pass under Visual Studio. 21 | Add CMake build system support. 22 | Fix warnings. 23 | 24 | 2012-06-10: 25 | Tagged v1.1.1. 26 | Null terminate exponent buffer (only an issue when asserts are enabled). 27 | Support more architectures. 28 | 29 | 2012-02-05: 30 | Merged in Single-branch with single-precision support. 31 | Tagged v1.1 (based on b28450f33e1db493948a535d8f84e88fa211bd10). 32 | 33 | 2012-02-05: 34 | Tagged v1.0 (based on eda0196e9ac8fcdf59e92cb62885ee0af5391969). 35 | -------------------------------------------------------------------------------- /src/double-conversion/LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2006-2011, the V8 project authors. All rights reserved. 2 | Redistribution and use in source and binary forms, with or without 3 | modification, are permitted provided that the following conditions are 4 | met: 5 | 6 | * Redistributions of source code must retain the above copyright 7 | notice, this list of conditions and the following disclaimer. 8 | * Redistributions in binary form must reproduce the above 9 | copyright notice, this list of conditions and the following 10 | disclaimer in the documentation and/or other materials provided 11 | with the distribution. 12 | * Neither the name of Google Inc. nor the names of its 13 | contributors may be used to endorse or promote products derived 14 | from this software without specific prior written permission. 15 | 16 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 | "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 | LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 | A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 | OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 | SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 | LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | -------------------------------------------------------------------------------- /src/double-conversion/README: -------------------------------------------------------------------------------- 1 | http://code.google.com/p/double-conversion 2 | 3 | This project (double-conversion) provides binary-decimal and decimal-binary 4 | routines for IEEE doubles. 5 | 6 | The library consists of efficient conversion routines that have been extracted 7 | from the V8 JavaScript engine. The code has been refactored and improved so that 8 | it can be used more easily in other projects. 9 | 10 | There is extensive documentation in src/double-conversion.h. Other examples can 11 | be found in test/cctest/test-conversions.cc. 12 | 13 | 14 | Building 15 | ======== 16 | 17 | This library can be built with scons [0] or cmake [1]. 18 | The checked-in Makefile simply forwards to scons, and provides a 19 | shortcut to run all tests: 20 | 21 | make 22 | make test 23 | 24 | Scons 25 | ----- 26 | 27 | The easiest way to install this library is to use `scons`. It builds 28 | the static and shared library, and is set up to install those at the 29 | correct locations: 30 | 31 | scons install 32 | 33 | Use the `DESTDIR` option to change the target directory: 34 | 35 | scons DESTDIR=alternative_directory install 36 | 37 | Cmake 38 | ----- 39 | 40 | To use cmake run `cmake .` in the root directory. This overwrites the 41 | existing Makefile. 42 | 43 | Use `-DBUILD_SHARED_LIBS=ON` to enable the compilation of shared libraries. 44 | Note that this disables static libraries. There is currently no way to 45 | build both libraries at the same time with cmake. 46 | 47 | Use `-DBUILD_TESTING=ON` to build the test executable. 48 | 49 | cmake . -DBUILD_TESTING=ON 50 | make 51 | test/cctest/cctest --list | tr -d '<' | xargs test/cctest/cctest 52 | 53 | [0]: http://www.scons.org 54 | [1]: http://www.cmake.org 55 | -------------------------------------------------------------------------------- /src/double-conversion/SConstruct: -------------------------------------------------------------------------------- 1 | # vim:ft=python 2 | import os 3 | 4 | double_conversion_sources = ['src/' + x for x in SConscript('src/SConscript')] 5 | double_conversion_test_sources = ['test/cctest/' + x for x in SConscript('test/cctest/SConscript')] 6 | 7 | DESTDIR = ARGUMENTS.get('DESTDIR', '') 8 | prefix = ARGUMENTS.get('prefix', '/usr/local') 9 | lib = ARGUMENTS.get('libsuffix', 'lib') 10 | libdir = os.path.join(DESTDIR + prefix, lib) 11 | 12 | env = Environment(CPPPATH='#/src', LIBS=['m', 'stdc++'], 13 | CXXFLAGS=ARGUMENTS.get('CXXFLAGS', '')) 14 | debug = ARGUMENTS.get('debug', 0) 15 | optimize = ARGUMENTS.get('optimize', 0) 16 | env.Replace(CXX = ARGUMENTS.get('CXX', 'g++')) 17 | 18 | # for shared lib, requires scons 2.3.0 19 | env['SHLIBVERSION'] = '1.0.0' 20 | 21 | CCFLAGS = [] 22 | if int(debug): 23 | CCFLAGS.append(ARGUMENTS.get('CXXFLAGS', '-g -Wall -Werror')) 24 | if int(optimize): 25 | CCFLAGS.append(ARGUMENTS.get('CXXFLAGS', '-O3')) 26 | 27 | env.Append(CCFLAGS = " ".join(CCFLAGS)) 28 | 29 | double_conversion_shared_objects = [ 30 | env.SharedObject(src) for src in double_conversion_sources] 31 | double_conversion_static_objects = [ 32 | env.StaticObject(src) for src in double_conversion_sources] 33 | 34 | library_name = 'double-conversion' 35 | 36 | static_lib = env.StaticLibrary(library_name, double_conversion_static_objects) 37 | static_lib_pic = env.StaticLibrary(library_name + '_pic', double_conversion_shared_objects) 38 | shared_lib = env.SharedLibrary(library_name, double_conversion_shared_objects) 39 | 40 | env.Program('run_tests', double_conversion_test_sources, LIBS=[static_lib]) 41 | 42 | env.InstallVersionedLib(libdir, shared_lib) 43 | env.Install(libdir, static_lib) 44 | env.Install(libdir, static_lib_pic) 45 | 46 | env.Alias('install', libdir) 47 | -------------------------------------------------------------------------------- /src/double-conversion/double-conversionBuildTreeSettings.cmake.in: -------------------------------------------------------------------------------- 1 | set(double-conversion_INCLUDE_DIRS 2 | "@PROJECT_SOURCE_DIR@/src") 3 | -------------------------------------------------------------------------------- /src/double-conversion/double-conversionConfig.cmake.in: -------------------------------------------------------------------------------- 1 | # - Config file for the double-conversion package 2 | # It defines the following variables 3 | # double-conversion_INCLUDE_DIRS 4 | # double-conversion_LIBRARIES 5 | 6 | get_filename_component(double-conversion_CMAKE_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH) 7 | 8 | if(EXISTS "${double-conversion_CMAKE_DIR}/CMakeCache.txt") 9 | include("${double-conversion_CMAKE_DIR}/double-conversionBuildTreeSettings.cmake") 10 | else() 11 | set(double-conversion_INCLUDE_DIRS 12 | "${double-conversion_CMAKE_DIR}/@CONF_REL_INCLUDE_DIR@/include/double-conversion") 13 | endif() 14 | 15 | include("${double-conversion_CMAKE_DIR}/double-conversionLibraryDepends.cmake") 16 | 17 | set(double-conversion_LIBRARIES double-conversion) 18 | -------------------------------------------------------------------------------- /src/double-conversion/double-conversionConfigVersion.cmake.in: -------------------------------------------------------------------------------- 1 | set(PACKAGE_VERSION "@double-conversion_VERSION@") 2 | 3 | # Check whether the requested PACKAGE_FIND_VERSION is compatible 4 | if("${PACKAGE_VERSION}" VERSION_LESS "${PACKAGE_FIND_VERSION}") 5 | set(PACKAGE_VERSION_COMPATIBLE FALSE) 6 | else() 7 | set(PACKAGE_VERSION_COMPATIBLE TRUE) 8 | if ("${PACKAGE_VERSION}" VERSION_EQUAL "${PACKAGE_FIND_VERSION}") 9 | set(PACKAGE_VERSION_EXACT TRUE) 10 | endif() 11 | endif() 12 | -------------------------------------------------------------------------------- /src/double-conversion/src/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | set(headers 3 | bignum.h 4 | cached-powers.h 5 | diy-fp.h 6 | double-conversion.h 7 | fast-dtoa.h 8 | fixed-dtoa.h 9 | ieee.h 10 | strtod.h 11 | utils.h 12 | ) 13 | 14 | add_library(double-conversion 15 | bignum.cc 16 | bignum-dtoa.cc 17 | cached-powers.cc 18 | diy-fp.cc 19 | double-conversion.cc 20 | fast-dtoa.cc 21 | fixed-dtoa.cc 22 | strtod.cc 23 | ${headers} 24 | ) 25 | 26 | # 27 | # associates the list of headers with the library 28 | # for the purposes of installation/import into other projects 29 | set_target_properties(double-conversion 30 | PROPERTIES PUBLIC_HEADER "${headers}") 31 | 32 | if (BUILD_SHARED_LIBS) 33 | set_target_properties(double-conversion 34 | PROPERTIES VERSION ${double-conversion_SOVERSION} 35 | SOVERSION ${double-conversion_SOVERSION_MAJOR}) 36 | endif() 37 | 38 | # 39 | # install command to set up library install 40 | # given the above PUBLIC_HEADER property set, this 41 | # pulls along all the header files with the library. 42 | install(TARGETS double-conversion 43 | EXPORT double-conversionLibraryDepends 44 | RUNTIME DESTINATION "${INSTALL_BIN_DIR}" COMPONENT bin 45 | LIBRARY DESTINATION "${INSTALL_LIB_DIR}/lib" COMPONENT shlib 46 | ARCHIVE DESTINATION "${INSTALL_LIB_DIR}/lib" COMPONENT lib 47 | PUBLIC_HEADER DESTINATION "${INSTALL_INCLUDE_DIR}/include/double-conversion" 48 | COMPONENT dev) 49 | -------------------------------------------------------------------------------- /src/double-conversion/src/SConscript: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | double_conversion_sources = [ 3 | 'bignum.cc', 4 | 'bignum-dtoa.cc', 5 | 'cached-powers.cc', 6 | 'diy-fp.cc', 7 | 'double-conversion.cc', 8 | 'fast-dtoa.cc', 9 | 'fixed-dtoa.cc', 10 | 'strtod.cc' 11 | ] 12 | Return('double_conversion_sources') 13 | -------------------------------------------------------------------------------- /src/double-conversion/src/bignum-dtoa.h: -------------------------------------------------------------------------------- 1 | // Copyright 2010 the V8 project authors. All rights reserved. 2 | // Redistribution and use in source and binary forms, with or without 3 | // modification, are permitted provided that the following conditions are 4 | // met: 5 | // 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above 9 | // copyright notice, this list of conditions and the following 10 | // disclaimer in the documentation and/or other materials provided 11 | // with the distribution. 12 | // * Neither the name of Google Inc. nor the names of its 13 | // contributors may be used to endorse or promote products derived 14 | // from this software without specific prior written permission. 15 | // 16 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | 28 | #ifndef DOUBLE_CONVERSION_BIGNUM_DTOA_H_ 29 | #define DOUBLE_CONVERSION_BIGNUM_DTOA_H_ 30 | 31 | #include "utils.h" 32 | 33 | namespace double_conversion { 34 | 35 | enum BignumDtoaMode { 36 | // Return the shortest correct representation. 37 | // For example the output of 0.299999999999999988897 is (the less accurate but 38 | // correct) 0.3. 39 | BIGNUM_DTOA_SHORTEST, 40 | // Same as BIGNUM_DTOA_SHORTEST but for single-precision floats. 41 | BIGNUM_DTOA_SHORTEST_SINGLE, 42 | // Return a fixed number of digits after the decimal point. 43 | // For instance fixed(0.1, 4) becomes 0.1000 44 | // If the input number is big, the output will be big. 45 | BIGNUM_DTOA_FIXED, 46 | // Return a fixed number of digits, no matter what the exponent is. 47 | BIGNUM_DTOA_PRECISION 48 | }; 49 | 50 | // Converts the given double 'v' to ascii. 51 | // The result should be interpreted as buffer * 10^(point-length). 52 | // The buffer will be null-terminated. 53 | // 54 | // The input v must be > 0 and different from NaN, and Infinity. 55 | // 56 | // The output depends on the given mode: 57 | // - SHORTEST: produce the least amount of digits for which the internal 58 | // identity requirement is still satisfied. If the digits are printed 59 | // (together with the correct exponent) then reading this number will give 60 | // 'v' again. The buffer will choose the representation that is closest to 61 | // 'v'. If there are two at the same distance, than the number is round up. 62 | // In this mode the 'requested_digits' parameter is ignored. 63 | // - FIXED: produces digits necessary to print a given number with 64 | // 'requested_digits' digits after the decimal point. The produced digits 65 | // might be too short in which case the caller has to fill the gaps with '0's. 66 | // Example: toFixed(0.001, 5) is allowed to return buffer="1", point=-2. 67 | // Halfway cases are rounded up. The call toFixed(0.15, 2) thus returns 68 | // buffer="2", point=0. 69 | // Note: the length of the returned buffer has no meaning wrt the significance 70 | // of its digits. That is, just because it contains '0's does not mean that 71 | // any other digit would not satisfy the internal identity requirement. 72 | // - PRECISION: produces 'requested_digits' where the first digit is not '0'. 73 | // Even though the length of produced digits usually equals 74 | // 'requested_digits', the function is allowed to return fewer digits, in 75 | // which case the caller has to fill the missing digits with '0's. 76 | // Halfway cases are again rounded up. 77 | // 'BignumDtoa' expects the given buffer to be big enough to hold all digits 78 | // and a terminating null-character. 79 | void BignumDtoa(double v, BignumDtoaMode mode, int requested_digits, 80 | Vector buffer, int* length, int* point); 81 | 82 | } // namespace double_conversion 83 | 84 | #endif // DOUBLE_CONVERSION_BIGNUM_DTOA_H_ 85 | -------------------------------------------------------------------------------- /src/double-conversion/src/bignum.h: -------------------------------------------------------------------------------- 1 | // Copyright 2010 the V8 project authors. All rights reserved. 2 | // Redistribution and use in source and binary forms, with or without 3 | // modification, are permitted provided that the following conditions are 4 | // met: 5 | // 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above 9 | // copyright notice, this list of conditions and the following 10 | // disclaimer in the documentation and/or other materials provided 11 | // with the distribution. 12 | // * Neither the name of Google Inc. nor the names of its 13 | // contributors may be used to endorse or promote products derived 14 | // from this software without specific prior written permission. 15 | // 16 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | 28 | #ifndef DOUBLE_CONVERSION_BIGNUM_H_ 29 | #define DOUBLE_CONVERSION_BIGNUM_H_ 30 | 31 | #include "utils.h" 32 | 33 | namespace double_conversion { 34 | 35 | class Bignum { 36 | public: 37 | // 3584 = 128 * 28. We can represent 2^3584 > 10^1000 accurately. 38 | // This bignum can encode much bigger numbers, since it contains an 39 | // exponent. 40 | static const int kMaxSignificantBits = 3584; 41 | 42 | Bignum(); 43 | void AssignUInt16(uint16_t value); 44 | void AssignUInt64(uint64_t value); 45 | void AssignBignum(const Bignum& other); 46 | 47 | void AssignDecimalString(Vector value); 48 | void AssignHexString(Vector value); 49 | 50 | void AssignPowerUInt16(uint16_t base, int exponent); 51 | 52 | void AddUInt16(uint16_t operand); 53 | void AddUInt64(uint64_t operand); 54 | void AddBignum(const Bignum& other); 55 | // Precondition: this >= other. 56 | void SubtractBignum(const Bignum& other); 57 | 58 | void Square(); 59 | void ShiftLeft(int shift_amount); 60 | void MultiplyByUInt32(uint32_t factor); 61 | void MultiplyByUInt64(uint64_t factor); 62 | void MultiplyByPowerOfTen(int exponent); 63 | void Times10() { return MultiplyByUInt32(10); } 64 | // Pseudocode: 65 | // int result = this / other; 66 | // this = this % other; 67 | // In the worst case this function is in O(this/other). 68 | uint16_t DivideModuloIntBignum(const Bignum& other); 69 | 70 | bool ToHexString(char* buffer, int buffer_size) const; 71 | 72 | // Returns 73 | // -1 if a < b, 74 | // 0 if a == b, and 75 | // +1 if a > b. 76 | static int Compare(const Bignum& a, const Bignum& b); 77 | static bool Equal(const Bignum& a, const Bignum& b) { 78 | return Compare(a, b) == 0; 79 | } 80 | static bool LessEqual(const Bignum& a, const Bignum& b) { 81 | return Compare(a, b) <= 0; 82 | } 83 | static bool Less(const Bignum& a, const Bignum& b) { 84 | return Compare(a, b) < 0; 85 | } 86 | // Returns Compare(a + b, c); 87 | static int PlusCompare(const Bignum& a, const Bignum& b, const Bignum& c); 88 | // Returns a + b == c 89 | static bool PlusEqual(const Bignum& a, const Bignum& b, const Bignum& c) { 90 | return PlusCompare(a, b, c) == 0; 91 | } 92 | // Returns a + b <= c 93 | static bool PlusLessEqual(const Bignum& a, const Bignum& b, const Bignum& c) { 94 | return PlusCompare(a, b, c) <= 0; 95 | } 96 | // Returns a + b < c 97 | static bool PlusLess(const Bignum& a, const Bignum& b, const Bignum& c) { 98 | return PlusCompare(a, b, c) < 0; 99 | } 100 | private: 101 | typedef uint32_t Chunk; 102 | typedef uint64_t DoubleChunk; 103 | 104 | static const int kChunkSize = sizeof(Chunk) * 8; 105 | static const int kDoubleChunkSize = sizeof(DoubleChunk) * 8; 106 | // With bigit size of 28 we loose some bits, but a double still fits easily 107 | // into two chunks, and more importantly we can use the Comba multiplication. 108 | static const int kBigitSize = 28; 109 | static const Chunk kBigitMask = (1 << kBigitSize) - 1; 110 | // Every instance allocates kBigitLength chunks on the stack. Bignums cannot 111 | // grow. There are no checks if the stack-allocated space is sufficient. 112 | static const int kBigitCapacity = kMaxSignificantBits / kBigitSize; 113 | 114 | void EnsureCapacity(int size) { 115 | if (size > kBigitCapacity) { 116 | UNREACHABLE(); 117 | } 118 | } 119 | void Align(const Bignum& other); 120 | void Clamp(); 121 | bool IsClamped() const; 122 | void Zero(); 123 | // Requires this to have enough capacity (no tests done). 124 | // Updates used_digits_ if necessary. 125 | // shift_amount must be < kBigitSize. 126 | void BigitsShiftLeft(int shift_amount); 127 | // BigitLength includes the "hidden" digits encoded in the exponent. 128 | int BigitLength() const { return used_digits_ + exponent_; } 129 | Chunk BigitAt(int index) const; 130 | void SubtractTimes(const Bignum& other, int factor); 131 | 132 | Chunk bigits_buffer_[kBigitCapacity]; 133 | // A vector backed by bigits_buffer_. This way accesses to the array are 134 | // checked for out-of-bounds errors. 135 | Vector bigits_; 136 | int used_digits_; 137 | // The Bignum's value equals value(bigits_) * 2^(exponent_ * kBigitSize). 138 | int exponent_; 139 | 140 | DISALLOW_COPY_AND_ASSIGN(Bignum); 141 | }; 142 | 143 | } // namespace double_conversion 144 | 145 | #endif // DOUBLE_CONVERSION_BIGNUM_H_ 146 | -------------------------------------------------------------------------------- /src/double-conversion/src/cached-powers.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2006-2008 the V8 project authors. All rights reserved. 2 | // Redistribution and use in source and binary forms, with or without 3 | // modification, are permitted provided that the following conditions are 4 | // met: 5 | // 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above 9 | // copyright notice, this list of conditions and the following 10 | // disclaimer in the documentation and/or other materials provided 11 | // with the distribution. 12 | // * Neither the name of Google Inc. nor the names of its 13 | // contributors may be used to endorse or promote products derived 14 | // from this software without specific prior written permission. 15 | // 16 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | 28 | #include 29 | #include 30 | #include 31 | 32 | #include "utils.h" 33 | 34 | #include "cached-powers.h" 35 | 36 | namespace double_conversion { 37 | 38 | struct CachedPower { 39 | uint64_t significand; 40 | int16_t binary_exponent; 41 | int16_t decimal_exponent; 42 | }; 43 | 44 | static const CachedPower kCachedPowers[] = { 45 | {UINT64_2PART_C(0xfa8fd5a0, 081c0288), -1220, -348}, 46 | {UINT64_2PART_C(0xbaaee17f, a23ebf76), -1193, -340}, 47 | {UINT64_2PART_C(0x8b16fb20, 3055ac76), -1166, -332}, 48 | {UINT64_2PART_C(0xcf42894a, 5dce35ea), -1140, -324}, 49 | {UINT64_2PART_C(0x9a6bb0aa, 55653b2d), -1113, -316}, 50 | {UINT64_2PART_C(0xe61acf03, 3d1a45df), -1087, -308}, 51 | {UINT64_2PART_C(0xab70fe17, c79ac6ca), -1060, -300}, 52 | {UINT64_2PART_C(0xff77b1fc, bebcdc4f), -1034, -292}, 53 | {UINT64_2PART_C(0xbe5691ef, 416bd60c), -1007, -284}, 54 | {UINT64_2PART_C(0x8dd01fad, 907ffc3c), -980, -276}, 55 | {UINT64_2PART_C(0xd3515c28, 31559a83), -954, -268}, 56 | {UINT64_2PART_C(0x9d71ac8f, ada6c9b5), -927, -260}, 57 | {UINT64_2PART_C(0xea9c2277, 23ee8bcb), -901, -252}, 58 | {UINT64_2PART_C(0xaecc4991, 4078536d), -874, -244}, 59 | {UINT64_2PART_C(0x823c1279, 5db6ce57), -847, -236}, 60 | {UINT64_2PART_C(0xc2109436, 4dfb5637), -821, -228}, 61 | {UINT64_2PART_C(0x9096ea6f, 3848984f), -794, -220}, 62 | {UINT64_2PART_C(0xd77485cb, 25823ac7), -768, -212}, 63 | {UINT64_2PART_C(0xa086cfcd, 97bf97f4), -741, -204}, 64 | {UINT64_2PART_C(0xef340a98, 172aace5), -715, -196}, 65 | {UINT64_2PART_C(0xb23867fb, 2a35b28e), -688, -188}, 66 | {UINT64_2PART_C(0x84c8d4df, d2c63f3b), -661, -180}, 67 | {UINT64_2PART_C(0xc5dd4427, 1ad3cdba), -635, -172}, 68 | {UINT64_2PART_C(0x936b9fce, bb25c996), -608, -164}, 69 | {UINT64_2PART_C(0xdbac6c24, 7d62a584), -582, -156}, 70 | {UINT64_2PART_C(0xa3ab6658, 0d5fdaf6), -555, -148}, 71 | {UINT64_2PART_C(0xf3e2f893, dec3f126), -529, -140}, 72 | {UINT64_2PART_C(0xb5b5ada8, aaff80b8), -502, -132}, 73 | {UINT64_2PART_C(0x87625f05, 6c7c4a8b), -475, -124}, 74 | {UINT64_2PART_C(0xc9bcff60, 34c13053), -449, -116}, 75 | {UINT64_2PART_C(0x964e858c, 91ba2655), -422, -108}, 76 | {UINT64_2PART_C(0xdff97724, 70297ebd), -396, -100}, 77 | {UINT64_2PART_C(0xa6dfbd9f, b8e5b88f), -369, -92}, 78 | {UINT64_2PART_C(0xf8a95fcf, 88747d94), -343, -84}, 79 | {UINT64_2PART_C(0xb9447093, 8fa89bcf), -316, -76}, 80 | {UINT64_2PART_C(0x8a08f0f8, bf0f156b), -289, -68}, 81 | {UINT64_2PART_C(0xcdb02555, 653131b6), -263, -60}, 82 | {UINT64_2PART_C(0x993fe2c6, d07b7fac), -236, -52}, 83 | {UINT64_2PART_C(0xe45c10c4, 2a2b3b06), -210, -44}, 84 | {UINT64_2PART_C(0xaa242499, 697392d3), -183, -36}, 85 | {UINT64_2PART_C(0xfd87b5f2, 8300ca0e), -157, -28}, 86 | {UINT64_2PART_C(0xbce50864, 92111aeb), -130, -20}, 87 | {UINT64_2PART_C(0x8cbccc09, 6f5088cc), -103, -12}, 88 | {UINT64_2PART_C(0xd1b71758, e219652c), -77, -4}, 89 | {UINT64_2PART_C(0x9c400000, 00000000), -50, 4}, 90 | {UINT64_2PART_C(0xe8d4a510, 00000000), -24, 12}, 91 | {UINT64_2PART_C(0xad78ebc5, ac620000), 3, 20}, 92 | {UINT64_2PART_C(0x813f3978, f8940984), 30, 28}, 93 | {UINT64_2PART_C(0xc097ce7b, c90715b3), 56, 36}, 94 | {UINT64_2PART_C(0x8f7e32ce, 7bea5c70), 83, 44}, 95 | {UINT64_2PART_C(0xd5d238a4, abe98068), 109, 52}, 96 | {UINT64_2PART_C(0x9f4f2726, 179a2245), 136, 60}, 97 | {UINT64_2PART_C(0xed63a231, d4c4fb27), 162, 68}, 98 | {UINT64_2PART_C(0xb0de6538, 8cc8ada8), 189, 76}, 99 | {UINT64_2PART_C(0x83c7088e, 1aab65db), 216, 84}, 100 | {UINT64_2PART_C(0xc45d1df9, 42711d9a), 242, 92}, 101 | {UINT64_2PART_C(0x924d692c, a61be758), 269, 100}, 102 | {UINT64_2PART_C(0xda01ee64, 1a708dea), 295, 108}, 103 | {UINT64_2PART_C(0xa26da399, 9aef774a), 322, 116}, 104 | {UINT64_2PART_C(0xf209787b, b47d6b85), 348, 124}, 105 | {UINT64_2PART_C(0xb454e4a1, 79dd1877), 375, 132}, 106 | {UINT64_2PART_C(0x865b8692, 5b9bc5c2), 402, 140}, 107 | {UINT64_2PART_C(0xc83553c5, c8965d3d), 428, 148}, 108 | {UINT64_2PART_C(0x952ab45c, fa97a0b3), 455, 156}, 109 | {UINT64_2PART_C(0xde469fbd, 99a05fe3), 481, 164}, 110 | {UINT64_2PART_C(0xa59bc234, db398c25), 508, 172}, 111 | {UINT64_2PART_C(0xf6c69a72, a3989f5c), 534, 180}, 112 | {UINT64_2PART_C(0xb7dcbf53, 54e9bece), 561, 188}, 113 | {UINT64_2PART_C(0x88fcf317, f22241e2), 588, 196}, 114 | {UINT64_2PART_C(0xcc20ce9b, d35c78a5), 614, 204}, 115 | {UINT64_2PART_C(0x98165af3, 7b2153df), 641, 212}, 116 | {UINT64_2PART_C(0xe2a0b5dc, 971f303a), 667, 220}, 117 | {UINT64_2PART_C(0xa8d9d153, 5ce3b396), 694, 228}, 118 | {UINT64_2PART_C(0xfb9b7cd9, a4a7443c), 720, 236}, 119 | {UINT64_2PART_C(0xbb764c4c, a7a44410), 747, 244}, 120 | {UINT64_2PART_C(0x8bab8eef, b6409c1a), 774, 252}, 121 | {UINT64_2PART_C(0xd01fef10, a657842c), 800, 260}, 122 | {UINT64_2PART_C(0x9b10a4e5, e9913129), 827, 268}, 123 | {UINT64_2PART_C(0xe7109bfb, a19c0c9d), 853, 276}, 124 | {UINT64_2PART_C(0xac2820d9, 623bf429), 880, 284}, 125 | {UINT64_2PART_C(0x80444b5e, 7aa7cf85), 907, 292}, 126 | {UINT64_2PART_C(0xbf21e440, 03acdd2d), 933, 300}, 127 | {UINT64_2PART_C(0x8e679c2f, 5e44ff8f), 960, 308}, 128 | {UINT64_2PART_C(0xd433179d, 9c8cb841), 986, 316}, 129 | {UINT64_2PART_C(0x9e19db92, b4e31ba9), 1013, 324}, 130 | {UINT64_2PART_C(0xeb96bf6e, badf77d9), 1039, 332}, 131 | {UINT64_2PART_C(0xaf87023b, 9bf0ee6b), 1066, 340}, 132 | }; 133 | 134 | static const int kCachedPowersLength = ARRAY_SIZE(kCachedPowers); 135 | static const int kCachedPowersOffset = 348; // -1 * the first decimal_exponent. 136 | static const double kD_1_LOG2_10 = 0.30102999566398114; // 1 / lg(10) 137 | // Difference between the decimal exponents in the table above. 138 | const int PowersOfTenCache::kDecimalExponentDistance = 8; 139 | const int PowersOfTenCache::kMinDecimalExponent = -348; 140 | const int PowersOfTenCache::kMaxDecimalExponent = 340; 141 | 142 | void PowersOfTenCache::GetCachedPowerForBinaryExponentRange( 143 | int min_exponent, 144 | int max_exponent, 145 | DiyFp* power, 146 | int* decimal_exponent) { 147 | int kQ = DiyFp::kSignificandSize; 148 | double k = ceil((min_exponent + kQ - 1) * kD_1_LOG2_10); 149 | int foo = kCachedPowersOffset; 150 | int index = 151 | (foo + static_cast(k) - 1) / kDecimalExponentDistance + 1; 152 | ASSERT(0 <= index && index < kCachedPowersLength); 153 | CachedPower cached_power = kCachedPowers[index]; 154 | ASSERT(min_exponent <= cached_power.binary_exponent); 155 | (void) max_exponent; // Mark variable as used. 156 | ASSERT(cached_power.binary_exponent <= max_exponent); 157 | *decimal_exponent = cached_power.decimal_exponent; 158 | *power = DiyFp(cached_power.significand, cached_power.binary_exponent); 159 | } 160 | 161 | 162 | void PowersOfTenCache::GetCachedPowerForDecimalExponent(int requested_exponent, 163 | DiyFp* power, 164 | int* found_exponent) { 165 | ASSERT(kMinDecimalExponent <= requested_exponent); 166 | ASSERT(requested_exponent < kMaxDecimalExponent + kDecimalExponentDistance); 167 | int index = 168 | (requested_exponent + kCachedPowersOffset) / kDecimalExponentDistance; 169 | CachedPower cached_power = kCachedPowers[index]; 170 | *power = DiyFp(cached_power.significand, cached_power.binary_exponent); 171 | *found_exponent = cached_power.decimal_exponent; 172 | ASSERT(*found_exponent <= requested_exponent); 173 | ASSERT(requested_exponent < *found_exponent + kDecimalExponentDistance); 174 | } 175 | 176 | } // namespace double_conversion 177 | -------------------------------------------------------------------------------- /src/double-conversion/src/cached-powers.h: -------------------------------------------------------------------------------- 1 | // Copyright 2010 the V8 project authors. All rights reserved. 2 | // Redistribution and use in source and binary forms, with or without 3 | // modification, are permitted provided that the following conditions are 4 | // met: 5 | // 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above 9 | // copyright notice, this list of conditions and the following 10 | // disclaimer in the documentation and/or other materials provided 11 | // with the distribution. 12 | // * Neither the name of Google Inc. nor the names of its 13 | // contributors may be used to endorse or promote products derived 14 | // from this software without specific prior written permission. 15 | // 16 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | 28 | #ifndef DOUBLE_CONVERSION_CACHED_POWERS_H_ 29 | #define DOUBLE_CONVERSION_CACHED_POWERS_H_ 30 | 31 | #include "diy-fp.h" 32 | 33 | namespace double_conversion { 34 | 35 | class PowersOfTenCache { 36 | public: 37 | 38 | // Not all powers of ten are cached. The decimal exponent of two neighboring 39 | // cached numbers will differ by kDecimalExponentDistance. 40 | static const int kDecimalExponentDistance; 41 | 42 | static const int kMinDecimalExponent; 43 | static const int kMaxDecimalExponent; 44 | 45 | // Returns a cached power-of-ten with a binary exponent in the range 46 | // [min_exponent; max_exponent] (boundaries included). 47 | static void GetCachedPowerForBinaryExponentRange(int min_exponent, 48 | int max_exponent, 49 | DiyFp* power, 50 | int* decimal_exponent); 51 | 52 | // Returns a cached power of ten x ~= 10^k such that 53 | // k <= decimal_exponent < k + kCachedPowersDecimalDistance. 54 | // The given decimal_exponent must satisfy 55 | // kMinDecimalExponent <= requested_exponent, and 56 | // requested_exponent < kMaxDecimalExponent + kDecimalExponentDistance. 57 | static void GetCachedPowerForDecimalExponent(int requested_exponent, 58 | DiyFp* power, 59 | int* found_exponent); 60 | }; 61 | 62 | } // namespace double_conversion 63 | 64 | #endif // DOUBLE_CONVERSION_CACHED_POWERS_H_ 65 | -------------------------------------------------------------------------------- /src/double-conversion/src/diy-fp.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2010 the V8 project authors. All rights reserved. 2 | // Redistribution and use in source and binary forms, with or without 3 | // modification, are permitted provided that the following conditions are 4 | // met: 5 | // 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above 9 | // copyright notice, this list of conditions and the following 10 | // disclaimer in the documentation and/or other materials provided 11 | // with the distribution. 12 | // * Neither the name of Google Inc. nor the names of its 13 | // contributors may be used to endorse or promote products derived 14 | // from this software without specific prior written permission. 15 | // 16 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | 28 | 29 | #include "diy-fp.h" 30 | #include "utils.h" 31 | 32 | namespace double_conversion { 33 | 34 | void DiyFp::Multiply(const DiyFp& other) { 35 | // Simply "emulates" a 128 bit multiplication. 36 | // However: the resulting number only contains 64 bits. The least 37 | // significant 64 bits are only used for rounding the most significant 64 38 | // bits. 39 | const uint64_t kM32 = 0xFFFFFFFFU; 40 | uint64_t a = f_ >> 32; 41 | uint64_t b = f_ & kM32; 42 | uint64_t c = other.f_ >> 32; 43 | uint64_t d = other.f_ & kM32; 44 | uint64_t ac = a * c; 45 | uint64_t bc = b * c; 46 | uint64_t ad = a * d; 47 | uint64_t bd = b * d; 48 | uint64_t tmp = (bd >> 32) + (ad & kM32) + (bc & kM32); 49 | // By adding 1U << 31 to tmp we round the final result. 50 | // Halfway cases will be round up. 51 | tmp += 1U << 31; 52 | uint64_t result_f = ac + (ad >> 32) + (bc >> 32) + (tmp >> 32); 53 | e_ += other.e_ + 64; 54 | f_ = result_f; 55 | } 56 | 57 | } // namespace double_conversion 58 | -------------------------------------------------------------------------------- /src/double-conversion/src/diy-fp.h: -------------------------------------------------------------------------------- 1 | // Copyright 2010 the V8 project authors. All rights reserved. 2 | // Redistribution and use in source and binary forms, with or without 3 | // modification, are permitted provided that the following conditions are 4 | // met: 5 | // 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above 9 | // copyright notice, this list of conditions and the following 10 | // disclaimer in the documentation and/or other materials provided 11 | // with the distribution. 12 | // * Neither the name of Google Inc. nor the names of its 13 | // contributors may be used to endorse or promote products derived 14 | // from this software without specific prior written permission. 15 | // 16 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | 28 | #ifndef DOUBLE_CONVERSION_DIY_FP_H_ 29 | #define DOUBLE_CONVERSION_DIY_FP_H_ 30 | 31 | #include "utils.h" 32 | 33 | namespace double_conversion { 34 | 35 | // This "Do It Yourself Floating Point" class implements a floating-point number 36 | // with a uint64 significand and an int exponent. Normalized DiyFp numbers will 37 | // have the most significant bit of the significand set. 38 | // Multiplication and Subtraction do not normalize their results. 39 | // DiyFp are not designed to contain special doubles (NaN and Infinity). 40 | class DiyFp { 41 | public: 42 | static const int kSignificandSize = 64; 43 | 44 | DiyFp() : f_(0), e_(0) {} 45 | DiyFp(uint64_t f, int e) : f_(f), e_(e) {} 46 | 47 | // this = this - other. 48 | // The exponents of both numbers must be the same and the significand of this 49 | // must be bigger than the significand of other. 50 | // The result will not be normalized. 51 | void Subtract(const DiyFp& other) { 52 | ASSERT(e_ == other.e_); 53 | ASSERT(f_ >= other.f_); 54 | f_ -= other.f_; 55 | } 56 | 57 | // Returns a - b. 58 | // The exponents of both numbers must be the same and this must be bigger 59 | // than other. The result will not be normalized. 60 | static DiyFp Minus(const DiyFp& a, const DiyFp& b) { 61 | DiyFp result = a; 62 | result.Subtract(b); 63 | return result; 64 | } 65 | 66 | 67 | // this = this * other. 68 | void Multiply(const DiyFp& other); 69 | 70 | // returns a * b; 71 | static DiyFp Times(const DiyFp& a, const DiyFp& b) { 72 | DiyFp result = a; 73 | result.Multiply(b); 74 | return result; 75 | } 76 | 77 | void Normalize() { 78 | ASSERT(f_ != 0); 79 | uint64_t f = f_; 80 | int e = e_; 81 | 82 | // This method is mainly called for normalizing boundaries. In general 83 | // boundaries need to be shifted by 10 bits. We thus optimize for this case. 84 | const uint64_t k10MSBits = UINT64_2PART_C(0xFFC00000, 00000000); 85 | while ((f & k10MSBits) == 0) { 86 | f <<= 10; 87 | e -= 10; 88 | } 89 | while ((f & kUint64MSB) == 0) { 90 | f <<= 1; 91 | e--; 92 | } 93 | f_ = f; 94 | e_ = e; 95 | } 96 | 97 | static DiyFp Normalize(const DiyFp& a) { 98 | DiyFp result = a; 99 | result.Normalize(); 100 | return result; 101 | } 102 | 103 | uint64_t f() const { return f_; } 104 | int e() const { return e_; } 105 | 106 | void set_f(uint64_t new_value) { f_ = new_value; } 107 | void set_e(int new_value) { e_ = new_value; } 108 | 109 | private: 110 | static const uint64_t kUint64MSB = UINT64_2PART_C(0x80000000, 00000000); 111 | 112 | uint64_t f_; 113 | int e_; 114 | }; 115 | 116 | } // namespace double_conversion 117 | 118 | #endif // DOUBLE_CONVERSION_DIY_FP_H_ 119 | -------------------------------------------------------------------------------- /src/double-conversion/src/fast-dtoa.h: -------------------------------------------------------------------------------- 1 | // Copyright 2010 the V8 project authors. All rights reserved. 2 | // Redistribution and use in source and binary forms, with or without 3 | // modification, are permitted provided that the following conditions are 4 | // met: 5 | // 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above 9 | // copyright notice, this list of conditions and the following 10 | // disclaimer in the documentation and/or other materials provided 11 | // with the distribution. 12 | // * Neither the name of Google Inc. nor the names of its 13 | // contributors may be used to endorse or promote products derived 14 | // from this software without specific prior written permission. 15 | // 16 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | 28 | #ifndef DOUBLE_CONVERSION_FAST_DTOA_H_ 29 | #define DOUBLE_CONVERSION_FAST_DTOA_H_ 30 | 31 | #include "utils.h" 32 | 33 | namespace double_conversion { 34 | 35 | enum FastDtoaMode { 36 | // Computes the shortest representation of the given input. The returned 37 | // result will be the most accurate number of this length. Longer 38 | // representations might be more accurate. 39 | FAST_DTOA_SHORTEST, 40 | // Same as FAST_DTOA_SHORTEST but for single-precision floats. 41 | FAST_DTOA_SHORTEST_SINGLE, 42 | // Computes a representation where the precision (number of digits) is 43 | // given as input. The precision is independent of the decimal point. 44 | FAST_DTOA_PRECISION 45 | }; 46 | 47 | // FastDtoa will produce at most kFastDtoaMaximalLength digits. This does not 48 | // include the terminating '\0' character. 49 | static const int kFastDtoaMaximalLength = 17; 50 | // Same for single-precision numbers. 51 | static const int kFastDtoaMaximalSingleLength = 9; 52 | 53 | // Provides a decimal representation of v. 54 | // The result should be interpreted as buffer * 10^(point - length). 55 | // 56 | // Precondition: 57 | // * v must be a strictly positive finite double. 58 | // 59 | // Returns true if it succeeds, otherwise the result can not be trusted. 60 | // There will be *length digits inside the buffer followed by a null terminator. 61 | // If the function returns true and mode equals 62 | // - FAST_DTOA_SHORTEST, then 63 | // the parameter requested_digits is ignored. 64 | // The result satisfies 65 | // v == (double) (buffer * 10^(point - length)). 66 | // The digits in the buffer are the shortest representation possible. E.g. 67 | // if 0.099999999999 and 0.1 represent the same double then "1" is returned 68 | // with point = 0. 69 | // The last digit will be closest to the actual v. That is, even if several 70 | // digits might correctly yield 'v' when read again, the buffer will contain 71 | // the one closest to v. 72 | // - FAST_DTOA_PRECISION, then 73 | // the buffer contains requested_digits digits. 74 | // the difference v - (buffer * 10^(point-length)) is closest to zero for 75 | // all possible representations of requested_digits digits. 76 | // If there are two values that are equally close, then FastDtoa returns 77 | // false. 78 | // For both modes the buffer must be large enough to hold the result. 79 | bool FastDtoa(double d, 80 | FastDtoaMode mode, 81 | int requested_digits, 82 | Vector buffer, 83 | int* length, 84 | int* decimal_point); 85 | 86 | } // namespace double_conversion 87 | 88 | #endif // DOUBLE_CONVERSION_FAST_DTOA_H_ 89 | -------------------------------------------------------------------------------- /src/double-conversion/src/fixed-dtoa.h: -------------------------------------------------------------------------------- 1 | // Copyright 2010 the V8 project authors. All rights reserved. 2 | // Redistribution and use in source and binary forms, with or without 3 | // modification, are permitted provided that the following conditions are 4 | // met: 5 | // 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above 9 | // copyright notice, this list of conditions and the following 10 | // disclaimer in the documentation and/or other materials provided 11 | // with the distribution. 12 | // * Neither the name of Google Inc. nor the names of its 13 | // contributors may be used to endorse or promote products derived 14 | // from this software without specific prior written permission. 15 | // 16 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | 28 | #ifndef DOUBLE_CONVERSION_FIXED_DTOA_H_ 29 | #define DOUBLE_CONVERSION_FIXED_DTOA_H_ 30 | 31 | #include "utils.h" 32 | 33 | namespace double_conversion { 34 | 35 | // Produces digits necessary to print a given number with 36 | // 'fractional_count' digits after the decimal point. 37 | // The buffer must be big enough to hold the result plus one terminating null 38 | // character. 39 | // 40 | // The produced digits might be too short in which case the caller has to fill 41 | // the gaps with '0's. 42 | // Example: FastFixedDtoa(0.001, 5, ...) is allowed to return buffer = "1", and 43 | // decimal_point = -2. 44 | // Halfway cases are rounded towards +/-Infinity (away from 0). The call 45 | // FastFixedDtoa(0.15, 2, ...) thus returns buffer = "2", decimal_point = 0. 46 | // The returned buffer may contain digits that would be truncated from the 47 | // shortest representation of the input. 48 | // 49 | // This method only works for some parameters. If it can't handle the input it 50 | // returns false. The output is null-terminated when the function succeeds. 51 | bool FastFixedDtoa(double v, int fractional_count, 52 | Vector buffer, int* length, int* decimal_point); 53 | 54 | } // namespace double_conversion 55 | 56 | #endif // DOUBLE_CONVERSION_FIXED_DTOA_H_ 57 | -------------------------------------------------------------------------------- /src/double-conversion/src/strtod.h: -------------------------------------------------------------------------------- 1 | // Copyright 2010 the V8 project authors. All rights reserved. 2 | // Redistribution and use in source and binary forms, with or without 3 | // modification, are permitted provided that the following conditions are 4 | // met: 5 | // 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above 9 | // copyright notice, this list of conditions and the following 10 | // disclaimer in the documentation and/or other materials provided 11 | // with the distribution. 12 | // * Neither the name of Google Inc. nor the names of its 13 | // contributors may be used to endorse or promote products derived 14 | // from this software without specific prior written permission. 15 | // 16 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | 28 | #ifndef DOUBLE_CONVERSION_STRTOD_H_ 29 | #define DOUBLE_CONVERSION_STRTOD_H_ 30 | 31 | #include "utils.h" 32 | 33 | namespace double_conversion { 34 | 35 | // The buffer must only contain digits in the range [0-9]. It must not 36 | // contain a dot or a sign. It must not start with '0', and must not be empty. 37 | double Strtod(Vector buffer, int exponent); 38 | 39 | // The buffer must only contain digits in the range [0-9]. It must not 40 | // contain a dot or a sign. It must not start with '0', and must not be empty. 41 | float Strtof(Vector buffer, int exponent); 42 | 43 | } // namespace double_conversion 44 | 45 | #endif // DOUBLE_CONVERSION_STRTOD_H_ 46 | -------------------------------------------------------------------------------- /src/double-conversion/test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | add_subdirectory(cctest) -------------------------------------------------------------------------------- /src/double-conversion/test/cctest/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | 2 | set(CCTEST_SRC 3 | cctest.cc 4 | gay-fixed.cc 5 | gay-precision.cc 6 | gay-shortest.cc 7 | gay-shortest-single.cc 8 | test-bignum.cc 9 | test-bignum-dtoa.cc 10 | test-conversions.cc 11 | test-diy-fp.cc 12 | test-dtoa.cc 13 | test-fast-dtoa.cc 14 | test-fixed-dtoa.cc 15 | test-ieee.cc 16 | test-strtod.cc 17 | ) 18 | 19 | add_executable(cctest ${CCTEST_SRC}) 20 | target_link_libraries(cctest double-conversion) 21 | 22 | add_test(NAME test_bignum 23 | WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} 24 | COMMAND $ test-bignum) 25 | 26 | add_test(NAME test_bignum_dtoa 27 | WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} 28 | COMMAND $ test-bignum-dtoa) 29 | 30 | add_test(NAME test_conversions 31 | WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} 32 | COMMAND $ test-conversions) 33 | add_test(NAME test_diy_fp 34 | WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} 35 | COMMAND $ test-diy-fp) 36 | add_test(NAME test_dtoa 37 | WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} 38 | COMMAND $ test-dtoa) 39 | add_test(NAME test_fast_dtoa 40 | WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} 41 | COMMAND $ test-fast-dtoa) 42 | add_test(NAME test_fixed_dtoa 43 | WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} 44 | COMMAND $ test-fixed-dtoa) 45 | add_test(NAME test_ieee 46 | WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} 47 | COMMAND $ test-ieee) 48 | add_test(NAME test_strtod 49 | WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR} 50 | COMMAND $ test-strtod) 51 | -------------------------------------------------------------------------------- /src/double-conversion/test/cctest/SConscript: -------------------------------------------------------------------------------- 1 | double_conversion_test_sources = [ 2 | 'cctest.cc', 3 | 'gay-fixed.cc', 4 | 'gay-precision.cc', 5 | 'gay-shortest.cc', 6 | 'gay-shortest-single.cc', 7 | 'test-bignum.cc', 8 | 'test-bignum-dtoa.cc', 9 | 'test-conversions.cc', 10 | 'test-diy-fp.cc', 11 | 'test-dtoa.cc', 12 | 'test-fast-dtoa.cc', 13 | 'test-fixed-dtoa.cc', 14 | 'test-ieee.cc', 15 | 'test-strtod.cc', 16 | ] 17 | Return('double_conversion_test_sources') 18 | -------------------------------------------------------------------------------- /src/double-conversion/test/cctest/cctest.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2008 the V8 project authors. All rights reserved. 2 | // Redistribution and use in source and binary forms, with or without 3 | // modification, are permitted provided that the following conditions are 4 | // met: 5 | // 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above 9 | // copyright notice, this list of conditions and the following 10 | // disclaimer in the documentation and/or other materials provided 11 | // with the distribution. 12 | // * Neither the name of Google Inc. nor the names of its 13 | // contributors may be used to endorse or promote products derived 14 | // from this software without specific prior written permission. 15 | // 16 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | 28 | #include "cctest.h" 29 | #include 30 | #include 31 | #include 32 | 33 | 34 | CcTest* CcTest::last_ = NULL; 35 | 36 | 37 | CcTest::CcTest(TestFunction* callback, const char* file, const char* name, 38 | const char* dependency, bool enabled) 39 | : callback_(callback), name_(name), dependency_(dependency), prev_(last_) { 40 | // Find the base name of this test (const_cast required on Windows). 41 | char *basename = strrchr(const_cast(file), '/'); 42 | if (!basename) { 43 | basename = strrchr(const_cast(file), '\\'); 44 | } 45 | if (!basename) { 46 | basename = strdup(file); 47 | } else { 48 | basename = strdup(basename + 1); 49 | } 50 | // Drop the extension, if there is one. 51 | char *extension = strrchr(basename, '.'); 52 | if (extension) *extension = 0; 53 | // Install this test in the list of tests 54 | file_ = basename; 55 | enabled_ = enabled; 56 | prev_ = last_; 57 | last_ = this; 58 | } 59 | 60 | 61 | static void PrintTestList(CcTest* current) { 62 | if (current == NULL) return; 63 | PrintTestList(current->prev()); 64 | if (current->dependency() != NULL) { 65 | printf("%s/%s<%s\n", 66 | current->file(), current->name(), current->dependency()); 67 | } else { 68 | printf("%s/%s<\n", current->file(), current->name()); 69 | } 70 | } 71 | 72 | 73 | int main(int argc, char* argv[]) { 74 | int tests_run = 0; 75 | bool print_run_count = true; 76 | for (int i = 1; i < argc; i++) { 77 | char* arg = argv[i]; 78 | if (strcmp(arg, "--list") == 0) { 79 | PrintTestList(CcTest::last()); 80 | print_run_count = false; 81 | 82 | } else { 83 | char* arg_copy = strdup(arg); 84 | char* testname = strchr(arg_copy, '/'); 85 | if (testname) { 86 | // Split the string in two by nulling the slash and then run 87 | // exact matches. 88 | *testname = 0; 89 | char* file = arg_copy; 90 | char* name = testname + 1; 91 | CcTest* test = CcTest::last(); 92 | while (test != NULL) { 93 | if (test->enabled() 94 | && strcmp(test->file(), file) == 0 95 | && strcmp(test->name(), name) == 0) { 96 | test->Run(); 97 | tests_run++; 98 | } 99 | test = test->prev(); 100 | } 101 | 102 | } else { 103 | // Run all tests with the specified file or test name. 104 | char* file_or_name = arg_copy; 105 | CcTest* test = CcTest::last(); 106 | while (test != NULL) { 107 | if (test->enabled() 108 | && (strcmp(test->file(), file_or_name) == 0 109 | || strcmp(test->name(), file_or_name) == 0)) { 110 | test->Run(); 111 | tests_run++; 112 | } 113 | test = test->prev(); 114 | } 115 | } 116 | delete[] arg_copy; 117 | } 118 | } 119 | if (print_run_count && tests_run != 1) 120 | printf("Ran %i tests.\n", tests_run); 121 | return 0; 122 | } 123 | -------------------------------------------------------------------------------- /src/double-conversion/test/cctest/cctest.h: -------------------------------------------------------------------------------- 1 | // Copyright 2008 the V8 project authors. All rights reserved. 2 | // Redistribution and use in source and binary forms, with or without 3 | // modification, are permitted provided that the following conditions are 4 | // met: 5 | // 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above 9 | // copyright notice, this list of conditions and the following 10 | // disclaimer in the documentation and/or other materials provided 11 | // with the distribution. 12 | // * Neither the name of Google Inc. nor the names of its 13 | // contributors may be used to endorse or promote products derived 14 | // from this software without specific prior written permission. 15 | // 16 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | 28 | #ifndef CCTEST_H_ 29 | #define CCTEST_H_ 30 | 31 | #include 32 | #include 33 | 34 | #include "utils.h" 35 | 36 | #ifndef TEST 37 | #define TEST(Name) \ 38 | static void Test##Name(); \ 39 | CcTest register_test_##Name(Test##Name, __FILE__, #Name, NULL, true); \ 40 | static void Test##Name() 41 | #endif 42 | 43 | #ifndef DEPENDENT_TEST 44 | #define DEPENDENT_TEST(Name, Dep) \ 45 | static void Test##Name(); \ 46 | CcTest register_test_##Name(Test##Name, __FILE__, #Name, #Dep, true); \ 47 | static void Test##Name() 48 | #endif 49 | 50 | #ifndef DISABLED_TEST 51 | #define DISABLED_TEST(Name) \ 52 | static void Test##Name(); \ 53 | CcTest register_test_##Name(Test##Name, __FILE__, #Name, NULL, false); \ 54 | static void Test##Name() 55 | #endif 56 | 57 | #define CHECK(condition) CheckHelper(__FILE__, __LINE__, #condition, condition) 58 | #define CHECK_GE(a, b) CHECK((a) >= (b)) 59 | 60 | static inline void CheckHelper(const char* file, 61 | int line, 62 | const char* source, 63 | bool condition) { 64 | if (!condition) { 65 | printf("%s:%d:\n CHECK(%s) failed\n", file, line, source); 66 | abort(); 67 | } 68 | } 69 | 70 | #define CHECK_EQ(a, b) CheckEqualsHelper(__FILE__, __LINE__, #a, a, #b, b) 71 | 72 | static inline void CheckEqualsHelper(const char* file, int line, 73 | const char* expected_source, 74 | const char* expected, 75 | const char* value_source, 76 | const char* value) { 77 | if ((expected == NULL && value != NULL) || 78 | (expected != NULL && value == NULL) || 79 | (expected != NULL && value != NULL && strcmp(expected, value) != 0)) { 80 | printf("%s:%d:\n CHECK_EQ(%s, %s) failed\n" 81 | "# Expected: %s\n" 82 | "# Found: %s\n", 83 | file, line, expected_source, value_source, expected, value); 84 | abort(); 85 | } 86 | } 87 | 88 | static inline void CheckEqualsHelper(const char* file, int line, 89 | const char* expected_source, 90 | int expected, 91 | const char* value_source, 92 | int value) { 93 | if (expected != value) { 94 | printf("%s:%d:\n CHECK_EQ(%s, %s) failed\n" 95 | "# Expected: %d\n" 96 | "# Found: %d\n", 97 | file, line, expected_source, value_source, expected, value); 98 | abort(); 99 | } 100 | } 101 | 102 | static inline void CheckEqualsHelper(const char* file, int line, 103 | const char* expected_source, 104 | double expected, 105 | const char* value_source, 106 | double value) { 107 | // If expected and value are NaNs then expected != value. 108 | if (expected != value && (expected == expected || value == value)) { 109 | printf("%s:%d:\n CHECK_EQ(%s, %s) failed\n" 110 | "# Expected: %.30e\n" 111 | "# Found: %.30e\n", 112 | file, line, expected_source, value_source, expected, value); 113 | abort(); 114 | } 115 | } 116 | 117 | 118 | class CcTest { 119 | public: 120 | typedef void (TestFunction)(); 121 | CcTest(TestFunction* callback, const char* file, const char* name, 122 | const char* dependency, bool enabled); 123 | void Run() { callback_(); } 124 | static int test_count(); 125 | static CcTest* last() { return last_; } 126 | CcTest* prev() { return prev_; } 127 | const char* file() { return file_; } 128 | const char* name() { return name_; } 129 | const char* dependency() { return dependency_; } 130 | bool enabled() { return enabled_; } 131 | private: 132 | TestFunction* callback_; 133 | const char* file_; 134 | const char* name_; 135 | const char* dependency_; 136 | bool enabled_; 137 | static CcTest* last_; 138 | CcTest* prev_; 139 | }; 140 | 141 | #endif // ifndef CCTEST_H_ 142 | -------------------------------------------------------------------------------- /src/double-conversion/test/cctest/gay-fixed.h: -------------------------------------------------------------------------------- 1 | // Copyright 2006-2008 the V8 project authors. All rights reserved. 2 | // Redistribution and use in source and binary forms, with or without 3 | // modification, are permitted provided that the following conditions are 4 | // met: 5 | // 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above 9 | // copyright notice, this list of conditions and the following 10 | // disclaimer in the documentation and/or other materials provided 11 | // with the distribution. 12 | // * Neither the name of Google Inc. nor the names of its 13 | // contributors may be used to endorse or promote products derived 14 | // from this software without specific prior written permission. 15 | // 16 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | 28 | #ifndef GAY_FIXED_H_ 29 | #define GAY_FIXED_H_ 30 | 31 | namespace double_conversion { 32 | 33 | struct PrecomputedFixed { 34 | double v; 35 | int number_digits; 36 | const char* representation; 37 | int decimal_point; 38 | }; 39 | 40 | // Returns precomputed values of dtoa. The strings have been generated using 41 | // Gay's dtoa in mode "fixed". 42 | Vector PrecomputedFixedRepresentations(); 43 | 44 | } // namespace double_conversion 45 | 46 | #endif // GAY_FIXED_H_ 47 | -------------------------------------------------------------------------------- /src/double-conversion/test/cctest/gay-precision.h: -------------------------------------------------------------------------------- 1 | // Copyright 2006-2008 the V8 project authors. All rights reserved. 2 | // Redistribution and use in source and binary forms, with or without 3 | // modification, are permitted provided that the following conditions are 4 | // met: 5 | // 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above 9 | // copyright notice, this list of conditions and the following 10 | // disclaimer in the documentation and/or other materials provided 11 | // with the distribution. 12 | // * Neither the name of Google Inc. nor the names of its 13 | // contributors may be used to endorse or promote products derived 14 | // from this software without specific prior written permission. 15 | // 16 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | 28 | #ifndef GAY_PRECISION_H_ 29 | #define GAY_PRECISION_H_ 30 | 31 | namespace double_conversion { 32 | 33 | struct PrecomputedPrecision { 34 | double v; 35 | int number_digits; 36 | const char* representation; 37 | int decimal_point; 38 | }; 39 | 40 | // Returns precomputed values of dtoa. The strings have been generated using 41 | // Gay's dtoa in mode "precision". 42 | Vector PrecomputedPrecisionRepresentations(); 43 | 44 | } // namespace double_conversion 45 | 46 | #endif // GAY_PRECISION_H_ 47 | -------------------------------------------------------------------------------- /src/double-conversion/test/cctest/gay-shortest-single.h: -------------------------------------------------------------------------------- 1 | // Copyright 2011, the V8 project authors. All rights reserved. 2 | // Redistribution and use in source and binary forms, with or without 3 | // modification, are permitted provided that the following conditions are 4 | // met: 5 | // 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above 9 | // copyright notice, this list of conditions and the following 10 | // disclaimer in the documentation and/or other materials provided 11 | // with the distribution. 12 | // * Neither the name of Google Inc. nor the names of its 13 | // contributors may be used to endorse or promote products derived 14 | // from this software without specific prior written permission. 15 | // 16 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | 28 | #ifndef GAY_SHORTEST_SINGLE_H_ 29 | #define GAY_SHORTEST_SINGLE_H_ 30 | 31 | namespace double_conversion { 32 | 33 | struct PrecomputedShortestSingle { 34 | float v; 35 | const char* representation; 36 | int decimal_point; 37 | }; 38 | 39 | Vector 40 | PrecomputedShortestSingleRepresentations(); 41 | 42 | } // namespace double_conversion 43 | 44 | #endif // GAY_SHORTEST_SINGLE_H_ 45 | -------------------------------------------------------------------------------- /src/double-conversion/test/cctest/gay-shortest.h: -------------------------------------------------------------------------------- 1 | // Copyright 2006-2008 the V8 project authors. All rights reserved. 2 | // Redistribution and use in source and binary forms, with or without 3 | // modification, are permitted provided that the following conditions are 4 | // met: 5 | // 6 | // * Redistributions of source code must retain the above copyright 7 | // notice, this list of conditions and the following disclaimer. 8 | // * Redistributions in binary form must reproduce the above 9 | // copyright notice, this list of conditions and the following 10 | // disclaimer in the documentation and/or other materials provided 11 | // with the distribution. 12 | // * Neither the name of Google Inc. nor the names of its 13 | // contributors may be used to endorse or promote products derived 14 | // from this software without specific prior written permission. 15 | // 16 | // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 17 | // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 18 | // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 19 | // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 20 | // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 21 | // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 22 | // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 23 | // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 24 | // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 25 | // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 26 | // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 27 | 28 | #ifndef GAY_SHORTEST_H_ 29 | #define GAY_SHORTEST_H_ 30 | 31 | namespace double_conversion { 32 | 33 | struct PrecomputedShortest { 34 | double v; 35 | const char* representation; 36 | int decimal_point; 37 | }; 38 | 39 | Vector PrecomputedShortestRepresentations(); 40 | 41 | } // namespace double_conversion 42 | 43 | #endif // GAY_SHORTEST_H_ 44 | -------------------------------------------------------------------------------- /src/double-conversion/test/cctest/test-diy-fp.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2006-2008 the V8 project authors. All rights reserved. 2 | 3 | #include 4 | 5 | #include "cctest.h" 6 | #include "diy-fp.h" 7 | #include "utils.h" 8 | 9 | 10 | using namespace double_conversion; 11 | 12 | 13 | TEST(Subtract) { 14 | DiyFp diy_fp1 = DiyFp(3, 0); 15 | DiyFp diy_fp2 = DiyFp(1, 0); 16 | DiyFp diff = DiyFp::Minus(diy_fp1, diy_fp2); 17 | 18 | CHECK(2 == diff.f()); // NOLINT 19 | CHECK_EQ(0, diff.e()); 20 | diy_fp1.Subtract(diy_fp2); 21 | CHECK(2 == diy_fp1.f()); // NOLINT 22 | CHECK_EQ(0, diy_fp1.e()); 23 | } 24 | 25 | 26 | TEST(Multiply) { 27 | DiyFp diy_fp1 = DiyFp(3, 0); 28 | DiyFp diy_fp2 = DiyFp(2, 0); 29 | DiyFp product = DiyFp::Times(diy_fp1, diy_fp2); 30 | 31 | CHECK(0 == product.f()); // NOLINT 32 | CHECK_EQ(64, product.e()); 33 | diy_fp1.Multiply(diy_fp2); 34 | CHECK(0 == diy_fp1.f()); // NOLINT 35 | CHECK_EQ(64, diy_fp1.e()); 36 | 37 | diy_fp1 = DiyFp(UINT64_2PART_C(0x80000000, 00000000), 11); 38 | diy_fp2 = DiyFp(2, 13); 39 | product = DiyFp::Times(diy_fp1, diy_fp2); 40 | CHECK(1 == product.f()); // NOLINT 41 | CHECK_EQ(11 + 13 + 64, product.e()); 42 | 43 | // Test rounding. 44 | diy_fp1 = DiyFp(UINT64_2PART_C(0x80000000, 00000001), 11); 45 | diy_fp2 = DiyFp(1, 13); 46 | product = DiyFp::Times(diy_fp1, diy_fp2); 47 | CHECK(1 == product.f()); // NOLINT 48 | CHECK_EQ(11 + 13 + 64, product.e()); 49 | 50 | diy_fp1 = DiyFp(UINT64_2PART_C(0x7fffffff, ffffffff), 11); 51 | diy_fp2 = DiyFp(1, 13); 52 | product = DiyFp::Times(diy_fp1, diy_fp2); 53 | CHECK(0 == product.f()); // NOLINT 54 | CHECK_EQ(11 + 13 + 64, product.e()); 55 | 56 | // Halfway cases are allowed to round either way. So don't check for it. 57 | 58 | // Big numbers. 59 | diy_fp1 = DiyFp(UINT64_2PART_C(0xFFFFFFFF, FFFFFFFF), 11); 60 | diy_fp2 = DiyFp(UINT64_2PART_C(0xFFFFFFFF, FFFFFFFF), 13); 61 | // 128bit result: 0xfffffffffffffffe0000000000000001 62 | product = DiyFp::Times(diy_fp1, diy_fp2); 63 | CHECK(UINT64_2PART_C(0xFFFFFFFF, FFFFFFFe) == product.f()); 64 | CHECK_EQ(11 + 13 + 64, product.e()); 65 | } 66 | -------------------------------------------------------------------------------- /src/pypa/ast/Grammar: -------------------------------------------------------------------------------- 1 | # Grammar for Python 2 | 3 | # Note: Changing the grammar specified in this file will most likely 4 | # require corresponding changes in the parser module 5 | # (../Modules/parsermodule.c). If you can't make the changes to 6 | # that module yourself, please co-ordinate the required changes 7 | # with someone who can; ask around on python-dev for help. Fred 8 | # Drake will probably be listening there. 9 | 10 | # NOTE WELL: You should also follow all the steps listed in PEP 306, 11 | # "How to Change Python's Grammar" 12 | 13 | # Start symbols for the grammar: 14 | # single_input is a single interactive statement; 15 | # file_input is a module or sequence of commands read from an input file; 16 | # eval_input is the input for the eval() and input() functions. 17 | # NB: compound_stmt in single_input is followed by extra NEWLINE! 18 | single_input: NEWLINE | simple_stmt | compound_stmt NEWLINE 19 | file_input: (NEWLINE | stmt)* ENDMARKER 20 | eval_input: testlist NEWLINE* ENDMARKER 21 | 22 | decorator: '@' dotted_name [ '(' [arglist] ')' ] NEWLINE 23 | decorators: decorator+ 24 | decorated: decorators (classdef | funcdef) 25 | funcdef: 'def' NAME parameters ':' suite 26 | parameters: '(' [varargslist] ')' 27 | varargslist: ((fpdef ['=' test] ',')* 28 | ('*' NAME [',' '**' NAME] | '**' NAME) | 29 | fpdef ['=' test] (',' fpdef ['=' test])* [',']) 30 | fpdef: NAME | '(' fplist ')' 31 | fplist: fpdef (',' fpdef)* [','] 32 | 33 | stmt: simple_stmt | compound_stmt 34 | simple_stmt: small_stmt (';' small_stmt)* [';'] NEWLINE 35 | small_stmt: (expr_stmt | print_stmt | del_stmt | pass_stmt | flow_stmt | 36 | import_stmt | global_stmt | exec_stmt | assert_stmt) 37 | expr_stmt: testlist (augassign (yield_expr|testlist) | 38 | ('=' (yield_expr|testlist))*) 39 | augassign: ('+=' | '-=' | '*=' | '/=' | '%=' | '&=' | '|=' | '^=' | 40 | '<<=' | '>>=' | '**=' | '//=') 41 | # For normal assignments, additional restrictions enforced by the interpreter 42 | print_stmt: 'print' ( [ test (',' test)* [','] ] | 43 | '>>' test [ (',' test)+ [','] ] ) 44 | del_stmt: 'del' exprlist 45 | pass_stmt: 'pass' 46 | flow_stmt: break_stmt | continue_stmt | return_stmt | raise_stmt | yield_stmt 47 | break_stmt: 'break' 48 | continue_stmt: 'continue' 49 | return_stmt: 'return' [testlist] 50 | yield_stmt: yield_expr 51 | raise_stmt: 'raise' [test [',' test [',' test]]] 52 | import_stmt: import_name | import_from 53 | import_name: 'import' dotted_as_names 54 | import_from: ('from' ('.'* dotted_name | '.'+) 55 | 'import' ('*' | '(' import_as_names ')' | import_as_names)) 56 | import_as_name: NAME ['as' NAME] 57 | dotted_as_name: dotted_name ['as' NAME] 58 | import_as_names: import_as_name (',' import_as_name)* [','] 59 | dotted_as_names: dotted_as_name (',' dotted_as_name)* 60 | dotted_name: NAME ('.' NAME)* 61 | global_stmt: 'global' NAME (',' NAME)* 62 | exec_stmt: 'exec' expr ['in' test [',' test]] 63 | assert_stmt: 'assert' test [',' test] 64 | 65 | compound_stmt: if_stmt | while_stmt | for_stmt | try_stmt | with_stmt | funcdef | classdef | decorated 66 | if_stmt: 'if' test ':' suite ('elif' test ':' suite)* ['else' ':' suite] 67 | while_stmt: 'while' test ':' suite ['else' ':' suite] 68 | for_stmt: 'for' exprlist 'in' testlist ':' suite ['else' ':' suite] 69 | try_stmt: ('try' ':' suite 70 | ((except_clause ':' suite)+ 71 | ['else' ':' suite] 72 | ['finally' ':' suite] | 73 | 'finally' ':' suite)) 74 | with_stmt: 'with' with_item (',' with_item)* ':' suite 75 | with_item: test ['as' expr] 76 | # NB compile.c makes sure that the default except clause is last 77 | except_clause: 'except' [test [('as' | ',') test]] 78 | suite: simple_stmt | NEWLINE INDENT stmt+ DEDENT 79 | 80 | # Backward compatibility cruft to support: 81 | # [ x for x in lambda: True, lambda: False if x() ] 82 | # even while also allowing: 83 | # lambda x: 5 if x else 2 84 | # (But not a mix of the two) 85 | testlist_safe: old_test [(',' old_test)+ [',']] 86 | old_test: or_test | old_lambdef 87 | old_lambdef: 'lambda' [varargslist] ':' old_test 88 | 89 | test: or_test ['if' or_test 'else' test] | lambdef 90 | or_test: and_test ('or' and_test)* 91 | and_test: not_test ('and' not_test)* 92 | not_test: 'not' not_test | comparison 93 | comparison: expr (comp_op expr)* 94 | comp_op: '<'|'>'|'=='|'>='|'<='|'!='|'in'|'not' 'in'|'is'|'is' 'not' 95 | expr: xor_expr ('|' xor_expr)* 96 | xor_expr: and_expr ('^' and_expr)* 97 | and_expr: shift_expr ('&' shift_expr)* 98 | shift_expr: arith_expr (('<<'|'>>') arith_expr)* 99 | arith_expr: term (('+'|'-') term)* 100 | term: factor (('*'|'/'|'%'|'//') factor)* 101 | factor: ('+'|'-'|'~') factor | power 102 | power: atom trailer* ['**' factor] 103 | atom: ('(' [yield_expr|testlist_comp] ')' | 104 | '[' [listmaker] ']' | 105 | '{' [dictorsetmaker] '}' | 106 | '`' testlist1 '`' | 107 | NAME | NUMBER | STRING+) 108 | listmaker: test ( list_for | (',' test)* [','] ) 109 | testlist_comp: test ( comp_for | (',' test)* [','] ) 110 | lambdef: 'lambda' [varargslist] ':' test 111 | trailer: '(' [arglist] ')' | '[' subscriptlist ']' | '.' NAME 112 | subscriptlist: subscript (',' subscript)* [','] 113 | subscript: '.' '.' '.' | test | [test] ':' [test] [sliceop] 114 | sliceop: ':' [test] 115 | exprlist: expr (',' expr)* [','] 116 | testlist: test (',' test)* [','] 117 | dictorsetmaker: ( (test ':' test (comp_for | (',' test ':' test)* [','])) | 118 | (test (comp_for | (',' test)* [','])) ) 119 | 120 | classdef: 'class' NAME ['(' [testlist] ')'] ':' suite 121 | 122 | arglist: (argument ',')* (argument [','] 123 | |'*' test (',' argument)* [',' '**' test] 124 | |'**' test) 125 | # The reason that keywords are test nodes instead of NAME is that using NAME 126 | # results in an ambiguity. ast.c makes sure it's a NAME. 127 | argument: test [comp_for] | test '=' test 128 | 129 | list_iter: list_for | list_if 130 | list_for: 'for' exprlist 'in' testlist_safe [list_iter] 131 | list_if: 'if' old_test [list_iter] 132 | 133 | comp_iter: comp_for | comp_if 134 | comp_for: 'for' exprlist 'in' or_test [comp_iter] 135 | comp_if: 'if' old_test [comp_iter] 136 | 137 | testlist1: test (',' test)* 138 | 139 | # not used in grammar, but may appear in "node" passed from Parser to Compiler 140 | encoding_decl: NAME 141 | 142 | yield_expr: 'yield' [testlist] 143 | -------------------------------------------------------------------------------- /src/pypa/ast/ast.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #include "ast.hh" 15 | #include "visitor.hh" 16 | namespace pypa { 17 | namespace detail { 18 | void visit_dump_internal(int depth, Ast const & value) { 19 | pypa::visit(ast_member_dump_revisit(depth), value); 20 | } 21 | } 22 | } 23 | -------------------------------------------------------------------------------- /src/pypa/ast/ast_type.inl: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | PYPA_AST_TYPE(Alias) 15 | PYPA_AST_TYPE(Arguments) 16 | PYPA_AST_TYPE(Assert) 17 | PYPA_AST_TYPE(Assign) 18 | PYPA_AST_TYPE(Attribute) 19 | PYPA_AST_TYPE(AugAssign) 20 | PYPA_AST_TYPE(BinOp) 21 | PYPA_AST_TYPE(Bool) 22 | PYPA_AST_TYPE(BoolOp) 23 | PYPA_AST_TYPE(Break) 24 | PYPA_AST_TYPE(Call) 25 | PYPA_AST_TYPE(ClassDef) 26 | PYPA_AST_TYPE(Compare) 27 | PYPA_AST_TYPE(Complex) 28 | PYPA_AST_TYPE(Comprehension) 29 | PYPA_AST_TYPE(Continue) 30 | PYPA_AST_TYPE(Delete) 31 | PYPA_AST_TYPE(Dict) 32 | PYPA_AST_TYPE(DictComp) 33 | PYPA_AST_TYPE(DocString) 34 | PYPA_AST_TYPE(Ellipsis) 35 | PYPA_AST_TYPE(EllipsisObject) 36 | PYPA_AST_TYPE(Except) 37 | PYPA_AST_TYPE(Exec) 38 | PYPA_AST_TYPE(Expression) 39 | PYPA_AST_TYPE(ExpressionStatement) 40 | PYPA_AST_TYPE(ExtSlice) 41 | PYPA_AST_TYPE(For) 42 | PYPA_AST_TYPE(FunctionDef) 43 | PYPA_AST_TYPE(Generator) 44 | PYPA_AST_TYPE(Global) 45 | PYPA_AST_TYPE(If) 46 | PYPA_AST_TYPE(IfExpr) 47 | PYPA_AST_TYPE(Import) 48 | PYPA_AST_TYPE(ImportFrom) 49 | PYPA_AST_TYPE(Index) 50 | PYPA_AST_TYPE(Keyword) 51 | PYPA_AST_TYPE(Lambda) 52 | PYPA_AST_TYPE(List) 53 | PYPA_AST_TYPE(ListComp) 54 | PYPA_AST_TYPE(Module) 55 | PYPA_AST_TYPE(Name) 56 | PYPA_AST_TYPE(None) 57 | PYPA_AST_TYPE(Number) 58 | PYPA_AST_TYPE(Pass) 59 | PYPA_AST_TYPE(Print) 60 | PYPA_AST_TYPE(Raise) 61 | PYPA_AST_TYPE(Repr) 62 | PYPA_AST_TYPE(Return) 63 | PYPA_AST_TYPE(Set) 64 | PYPA_AST_TYPE(SetComp) 65 | PYPA_AST_TYPE(Slice) 66 | PYPA_AST_TYPE(Statement) 67 | PYPA_AST_TYPE(Str) 68 | PYPA_AST_TYPE(Subscript) 69 | PYPA_AST_TYPE(Suite) 70 | PYPA_AST_TYPE(TryExcept) 71 | PYPA_AST_TYPE(TryFinally) 72 | PYPA_AST_TYPE(Tuple) 73 | PYPA_AST_TYPE(UnaryOp) 74 | PYPA_AST_TYPE(While) 75 | PYPA_AST_TYPE(With) 76 | PYPA_AST_TYPE(Yield) 77 | PYPA_AST_TYPE(YieldExpr) 78 | // PYPA_AST_TYPE(DottedName) 79 | // PYPA_AST_TYPE(Parameters) 80 | // PYPA_AST_TYPE(VarArgList) 81 | -------------------------------------------------------------------------------- /src/pypa/ast/base.hh: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef GUARD_PYPA_AST_BASE_HH_INCLUDED 15 | #define GUARD_PYPA_AST_BASE_HH_INCLUDED 16 | 17 | #include 18 | #include 19 | 20 | namespace pypa { 21 | 22 | PYPA_AST_TYPE_DECL_ALIAS(Ast, AstPtr, AstPtrList) { 23 | static constexpr AstType TYPE = AstType::Invalid; 24 | Ast(AstType type) : type(type), line(0), column(0) {} 25 | AstType type; 26 | uint32_t line; 27 | uint32_t column; 28 | }; 29 | 30 | template 31 | struct AstT : Base { 32 | static constexpr AstType TYPE = Type; 33 | AstT(AstType type = Type) : Base{type} {} 34 | }; 35 | template<> 36 | struct AstIDByType { static constexpr AstType Id = AstType::Invalid; }; 37 | 38 | template 39 | constexpr AstType AstT::TYPE; 40 | 41 | PYPA_AST_TYPE_DECL_DERIVED_ALIAS(Expression, AstExpr, AstExprList) { 42 | using AstT::AstT; 43 | }; 44 | DEF_AST_TYPE_BY_ID1(Expression); 45 | PYPA_AST_MEMBERS0(Expression); 46 | 47 | template 48 | struct AstExprT : AstT { 49 | AstExprT() 50 | : AstT(Type) 51 | {} 52 | }; 53 | 54 | PYPA_AST_TYPE_DECL_DERIVED_ALIAS(Statement, AstStmt, AstStmtList) { 55 | using AstT::AstT; 56 | }; 57 | DEF_AST_TYPE_BY_ID1(Statement); 58 | PYPA_AST_MEMBERS0(Statement); 59 | 60 | template 61 | struct AstStmtT : AstT { 62 | AstStmtT() : AstT(Type){} 63 | }; 64 | 65 | PYPA_AST_STMT(Suite) { 66 | AstStmtList items; 67 | }; 68 | PYPA_AST_MEMBERS1(Suite, items); 69 | 70 | } 71 | #endif // GUARD_PYPA_AST_BASE_HH_INCLUDED 72 | -------------------------------------------------------------------------------- /src/pypa/ast/context_assign.hh: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef GUARD_PYPA_AST_CONTEXT_ASSIGN_HH_INCLUDED 15 | #define GUARD_PYPA_AST_CONTEXT_ASSIGN_HH_INCLUDED 16 | 17 | #include 18 | 19 | namespace pypa { 20 | 21 | struct context_assign { 22 | AstContext context; 23 | 24 | template< typename T > 25 | void operator() (std::shared_ptr p) { 26 | if(p) (*this)(*p); 27 | } 28 | 29 | void operator() (AstTuple & tuple) { 30 | // If AstContext::Param then tuple and elements should be AstContext::Store 31 | // That's the behaviour of Python 2.7 32 | tuple.context = context == AstContext::Param ? AstContext::Store : context; 33 | for(auto & e : tuple.elements) { 34 | visit(context_assign{tuple.context}, e); 35 | } 36 | } 37 | 38 | void operator() (AstList & lst) { 39 | lst.context = context == AstContext::Param ? AstContext::Store : context; 40 | for(auto & e : lst.elements) { 41 | visit(context_assign{lst.context}, e); 42 | } 43 | } 44 | 45 | void operator() (AstName & name) { 46 | name.context = context; 47 | } 48 | 49 | void operator() (AstKeyword & k) { 50 | visit(context_assign{AstContext::Store}, k.name); 51 | } 52 | 53 | void operator() (AstAttribute & attribute) { 54 | attribute.context = context; 55 | visit(*this, attribute.attribute); 56 | } 57 | 58 | void operator() (AstSubscript & subscript) { 59 | subscript.context = context; 60 | } 61 | 62 | template< typename T > 63 | void operator() (T) { 64 | // Ignored 65 | } 66 | 67 | }; 68 | 69 | } 70 | 71 | #endif // GUARD_PYPA_AST_CONTEXT_ASSIGN_HH_INCLUDED 72 | -------------------------------------------------------------------------------- /src/pypa/ast/dump.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #include 15 | #include 16 | #include 17 | 18 | namespace pypa { 19 | 20 | void dump(AstPtr p) { 21 | if(p) { 22 | pypa::detail::visit_dump_internal(0, *p); 23 | } 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /src/pypa/ast/dump.hh: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef GUARD_PYPA_AST_DUMP_HH_INCLUDED 15 | #define GUARD_PYPA_AST_DUMP_HH_INCLUDED 16 | 17 | #include 18 | 19 | namespace pypa { 20 | 21 | template 22 | struct ast_member_dump; 23 | 24 | struct ast_member_dump_revisit { 25 | int depth_; 26 | ast_member_dump_revisit(int depth) : depth_(depth) {} 27 | template< typename T > 28 | bool operator() ( T const & v ) const { 29 | ast_member_dump::dump(depth_, v); 30 | return true; 31 | } 32 | }; 33 | 34 | // Forward declaration for the dump visit implementation 35 | struct Ast; 36 | 37 | namespace detail { 38 | void visit_dump_internal(int depth, Ast const & v); 39 | 40 | template< typename T, typename V, typename F> 41 | void apply_member(T & t, V T::*member, F f) { 42 | f(t.*member); 43 | } 44 | 45 | template< typename T, typename V, typename F> 46 | void apply_member(std::shared_ptr t, V T::*member, F f) { 47 | f((*t).*member); 48 | } 49 | 50 | inline void print_padding(int depth) { 51 | while(depth--) printf(" "); 52 | } 53 | 54 | inline void dump_member_value(int depth, Ast const & v) { 55 | print_padding(depth); 56 | 57 | // This is used to dispatch it to the right ast_member_dump 58 | // implementation using ast_member_dump_revisit 59 | visit_dump_internal(depth, v); 60 | } 61 | 62 | inline void dump_member_value(int depth, String const & v) { 63 | printf("%s\n", v.c_str()); 64 | } 65 | 66 | inline void dump_member_value(int depth, char const * v) { 67 | printf("RAW BUFFER: %p\n", v); 68 | } 69 | 70 | inline void dump_member_value(int depth, bool v) { 71 | printf("%s\n", v ? "True" : "False"); 72 | } 73 | 74 | inline void dump_member_value(int depth, int const & v) { 75 | printf("%d\n", int(v)); 76 | } 77 | 78 | inline void dump_member_value(int depth, int64_t const & v) { 79 | long long int p = v; 80 | printf("%lld\n", p); 81 | } 82 | 83 | inline void dump_member_value(int depth, double const & v) { 84 | printf("%g\n", v); 85 | } 86 | 87 | inline void dump_member_value(int depth, AstBoolOpType const & v) { 88 | printf("%s\n", to_string(v)); 89 | } 90 | 91 | inline void dump_member_value(int depth, AstUnaryOpType const & v) { 92 | printf("%s\n", to_string(v)); 93 | } 94 | 95 | inline void dump_member_value(int depth, AstCompareOpType const & v) { 96 | printf("%s\n", to_string(v)); 97 | } 98 | 99 | inline void dump_member_value(int depth, AstBinOpType const & v) { 100 | printf("%s\n", to_string(v)); 101 | } 102 | 103 | inline void dump_member_value(int depth, AstModuleKind const & v) { 104 | printf("%s\n", to_string(v)); 105 | } 106 | 107 | inline void dump_member_value(int depth, AstContext const & v) { 108 | printf("%s\n", to_string(v)); 109 | } 110 | 111 | template< typename T > 112 | inline void dump_member_value(int depth, std::shared_ptr const & v) { 113 | if(!v) { 114 | printf("\n"); 115 | } 116 | else { 117 | T const & v_ = *v; 118 | dump_member_value(int(depth), v_); 119 | } 120 | } 121 | 122 | template< typename T > 123 | inline void dump_padded_member(int depth, std::shared_ptr const & v) { 124 | if(!v) { printf("\n"); print_padding(depth+4); } 125 | dump_member_value(depth+4, v); 126 | } 127 | 128 | template< typename T > 129 | inline void dump_padded_member(int depth, T const & v) { 130 | dump_member_value(depth+4, v); 131 | } 132 | 133 | template< typename T > 134 | inline void dump_member_value(int depth, std::vector const & v) { 135 | if(v.empty()) { 136 | printf("[]\n"); 137 | } 138 | else { 139 | printf("["); 140 | for(auto const & e : v) { 141 | dump_padded_member(depth, e); 142 | } 143 | print_padding(depth+4); 144 | printf("]\n"); 145 | } 146 | } 147 | } 148 | } 149 | 150 | #endif // GUARD_PYPA_AST_DUMP_HH_INCLUDED 151 | -------------------------------------------------------------------------------- /src/pypa/ast/tree_walker.hh: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef GUARD_PYPA_AST_TREE_WALKER_HH_INCLUDED 15 | #define GUARD_PYPA_AST_TREE_WALKER_HH_INCLUDED 16 | 17 | #include 18 | #include 19 | 20 | namespace pypa { 21 | 22 | template< typename AstT, typename F > 23 | void walk_tree(AstT & t, F f, int depth = 0); 24 | 25 | namespace detail { 26 | template< typename F > 27 | struct tree_walk_visitor { 28 | struct each { 29 | F * f_; 30 | int depth_; 31 | each(F * f, int depth) : f_(f), depth_(depth) {} 32 | 33 | void next(Ast & t) { 34 | visit(detail::tree_walk_visitor(f_, depth_ + 1), t); 35 | } 36 | 37 | template< typename T > 38 | void next(std::shared_ptr t) { 39 | if(t) visit(detail::tree_walk_visitor(f_, depth_ + 1), *t); 40 | } 41 | 42 | template< typename T > 43 | void next(T) {} 44 | 45 | template< typename T > 46 | bool operator() (T t) { 47 | if((*f_)(t)) { 48 | next(t); 49 | return true; 50 | } 51 | return false; 52 | } 53 | 54 | template< typename T > 55 | void operator() (std::vector & t) { 56 | for(auto & e : t) { 57 | next(e); 58 | } 59 | } 60 | }; 61 | 62 | F * f_; 63 | int depth_; 64 | tree_walk_visitor(F * f, int depth) : f_(f), depth_(depth){} 65 | template< typename T > 66 | void operator() (T t) { 67 | ast_member_visit::Id>::apply(t, each(f_, depth_ + 1)); 68 | } 69 | }; 70 | } 71 | 72 | template< typename AstT, typename F > 73 | void walk_tree(AstT & t, F f, int depth) { 74 | visit(detail::tree_walk_visitor(&f, depth), t); 75 | } 76 | 77 | template< typename AstT, typename F > 78 | void walk_tree(std::vector & t, F f, int depth = 0) { 79 | for(auto & e : t) { 80 | visit(detail::tree_walk_visitor(&f, depth), e); 81 | } 82 | } 83 | 84 | } 85 | 86 | #endif //GUARD_PYPA_AST_TREE_WALKER_HH_INCLUDED 87 | -------------------------------------------------------------------------------- /src/pypa/ast/types.hh: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef GUARD_PYPA_AST_TYPES_HH_INCLUDED 15 | #define GUARD_PYPA_AST_TYPES_HH_INCLUDED 16 | 17 | #include 18 | 19 | #include 20 | #include 21 | #include 22 | 23 | namespace pypa { 24 | 25 | enum class AstContext { 26 | Load, 27 | Store, 28 | Del, 29 | AugLoad, 30 | AugStore, 31 | Param 32 | }; 33 | 34 | inline char const * to_string(AstContext o) { 35 | switch(o) { 36 | case AstContext::Load: return "Load"; 37 | case AstContext::Store: return "Store"; 38 | case AstContext::Del: return "Del"; 39 | case AstContext::AugLoad: return "AugLoad"; 40 | case AstContext::AugStore: return "AugStore"; 41 | case AstContext::Param: return "Param"; 42 | } 43 | return "UNKNOWN AstContext"; 44 | } 45 | 46 | enum class AstBinOpType { 47 | Undefined, 48 | Add, 49 | BitAnd, 50 | BitOr, 51 | BitXor, 52 | Div, 53 | FloorDiv, 54 | LeftShift, 55 | Mod, 56 | Mult, 57 | Power, 58 | RightShift, 59 | Sub, 60 | }; 61 | 62 | inline char const * to_string(AstBinOpType o) { 63 | switch(o) { 64 | case AstBinOpType::Undefined: return "UNDEFINED"; 65 | case AstBinOpType::Add: return "+"; 66 | case AstBinOpType::BitAnd: return "&"; 67 | case AstBinOpType::BitOr: return "|"; 68 | case AstBinOpType::BitXor: return "^"; 69 | case AstBinOpType::Div: return "/"; 70 | case AstBinOpType::FloorDiv: return "//"; 71 | case AstBinOpType::LeftShift: return "<<"; 72 | case AstBinOpType::Mod: return "%"; 73 | case AstBinOpType::Mult: return "*"; 74 | case AstBinOpType::Power: return "**"; 75 | case AstBinOpType::RightShift: return ">>"; 76 | case AstBinOpType::Sub: return "-"; 77 | } 78 | return "UNKNOWN AstBinOpType"; 79 | } 80 | 81 | enum class AstUnaryOpType { 82 | Undefined, 83 | Add, 84 | Invert, 85 | Not, 86 | Sub, 87 | }; 88 | 89 | inline char const * to_string(AstUnaryOpType o) { 90 | switch(o) { 91 | case AstUnaryOpType::Undefined: return "UNDEFINED"; 92 | case AstUnaryOpType::Add: return "+"; 93 | case AstUnaryOpType::Invert: return "~"; 94 | case AstUnaryOpType::Not: return "not"; 95 | case AstUnaryOpType::Sub: return "-"; 96 | } 97 | return "UNKNOWN AstUnaryOpType"; 98 | } 99 | 100 | enum class AstBoolOpType { 101 | Undefined, 102 | And, 103 | Or 104 | }; 105 | 106 | inline char const * to_string(AstBoolOpType o) { 107 | switch(o) { 108 | case AstBoolOpType::Undefined: return "UNDEFINED"; 109 | case AstBoolOpType::And: return "and"; 110 | case AstBoolOpType::Or: return "or"; 111 | } 112 | return "UNKNOWN AstBoolOpType"; 113 | } 114 | 115 | enum class AstCompareOpType { 116 | Undefined, 117 | Equals, 118 | In, 119 | Is, 120 | IsNot, 121 | Less, 122 | LessEqual, 123 | More, 124 | MoreEqual, 125 | NotEqual, 126 | NotIn, 127 | }; 128 | 129 | inline char const * to_string(AstCompareOpType o) { 130 | switch(o) { 131 | case AstCompareOpType::Undefined: return "UNDEFINED"; 132 | case AstCompareOpType::Equals: return "=="; 133 | case AstCompareOpType::In: return "in"; 134 | case AstCompareOpType::Is: return "is"; 135 | case AstCompareOpType::IsNot: return "is not"; 136 | case AstCompareOpType::Less: return "<"; 137 | case AstCompareOpType::LessEqual: return "<="; 138 | case AstCompareOpType::More: return ">"; 139 | case AstCompareOpType::MoreEqual: return ">="; 140 | case AstCompareOpType::NotEqual: return "!="; 141 | case AstCompareOpType::NotIn: return "not in"; 142 | } 143 | return "UNKNOWN AstCompareOpType"; 144 | } 145 | 146 | enum class AstModuleKind { 147 | Module, 148 | Expression, 149 | Interactive, 150 | Suite 151 | }; 152 | 153 | inline char const * to_string(AstModuleKind o) { 154 | switch(o) { 155 | case AstModuleKind::Module: return "Module"; 156 | case AstModuleKind::Expression: return "Expression"; 157 | case AstModuleKind::Interactive: return "Interactive"; 158 | case AstModuleKind::Suite: return "Suite"; 159 | } 160 | return "UNKNOWN AstModuleKind"; 161 | } 162 | 163 | enum class AstType { 164 | Invalid = -1, 165 | #undef PYPA_AST_TYPE 166 | #define PYPA_AST_TYPE(X) X, 167 | # include 168 | #undef PYPA_AST_TYPE 169 | }; 170 | 171 | template 172 | struct AstTypeByID; 173 | 174 | template 175 | struct AstIDByType; 176 | 177 | template 178 | struct AstIDByType< std::shared_ptr > : AstIDByType {}; 179 | 180 | template 181 | struct AstIDByType< T const > : AstIDByType {}; 182 | 183 | template 184 | struct AstTypePtrByID { 185 | typedef std::shared_ptr::Type> Type; 186 | }; 187 | 188 | template 189 | struct ast_member_visit; 190 | 191 | } 192 | 193 | #endif // GUARD_PYPA_AST_TYPES_HH_INCLUDED 194 | -------------------------------------------------------------------------------- /src/pypa/ast/visitor.hh: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef GUARD_PYPA_AST_VISITOR_HH_INCLUDED 15 | #define GUARD_PYPA_AST_VISITOR_HH_INCLUDED 16 | 17 | #include 18 | #include 19 | #include 20 | 21 | namespace pypa { 22 | 23 | template 24 | inline void visit(F visitor, AstPtr v) { 25 | if(!v) return; 26 | switch(v->type) { 27 | #undef PYPA_AST_TYPE 28 | #define PYPA_AST_TYPE(X) case AstType::X: visitor(std::static_pointer_cast::Type>(v)); break; 29 | # include 30 | #undef PYPA_AST_TYPE 31 | default: 32 | assert("Invalid AST type received" && false); 33 | break; 34 | } 35 | } 36 | 37 | template 38 | inline R visit(F visitor, AstPtr v) { 39 | if(!v) { assert("Visit called with null pointer" && false); return R(); } 40 | switch(v->type) { 41 | #undef PYPA_AST_TYPE 42 | #define PYPA_AST_TYPE(X) case AstType::X: return visitor(std::static_pointer_cast::Type>(v)); 43 | # include 44 | #undef PYPA_AST_TYPE 45 | default: 46 | assert("Invalid AST type received" && false); 47 | break; 48 | } 49 | return R(); 50 | } 51 | 52 | template 53 | inline R visit(F visitor, Ast & v) { 54 | switch(v.type) { 55 | #undef PYPA_AST_TYPE 56 | #define PYPA_AST_TYPE(X) case AstType::X: return visitor(static_cast::Type&>(v)); 57 | # include 58 | #undef PYPA_AST_TYPE 59 | default: 60 | assert("Invalid AST type received" && false); 61 | break; 62 | } 63 | return R(); 64 | } 65 | 66 | template 67 | inline void visit(F visitor, Ast & v) { 68 | switch(v.type) { 69 | #undef PYPA_AST_TYPE 70 | #define PYPA_AST_TYPE(X) case AstType::X: visitor(static_cast::Type&>(v)); break; 71 | # include 72 | #undef PYPA_AST_TYPE 73 | default: 74 | assert("Invalid AST type received" && false); 75 | break; 76 | } 77 | } 78 | 79 | template 80 | inline R visit(F visitor, Ast const & v) { 81 | switch(v.type) { 82 | #undef PYPA_AST_TYPE 83 | #define PYPA_AST_TYPE(X) case AstType::X: return visitor(static_cast::Type const &>(v)); 84 | # include 85 | #undef PYPA_AST_TYPE 86 | default: 87 | assert("Invalid AST type received" && false); 88 | break; 89 | } 90 | return R(); 91 | } 92 | 93 | template 94 | inline void visit(F visitor, Ast const & v) { 95 | switch(v.type) { 96 | #undef PYPA_AST_TYPE 97 | #define PYPA_AST_TYPE(X) case AstType::X: visitor(static_cast::Type const &>(v)); break; 98 | # include 99 | #undef PYPA_AST_TYPE 100 | default: 101 | assert("Invalid AST type received" && false); 102 | break; 103 | } 104 | } 105 | 106 | } 107 | 108 | #endif // GUARD_PYPA_AST_VISITOR_HH_INCLUDED 109 | -------------------------------------------------------------------------------- /src/pypa/filebuf.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #include 15 | 16 | #if defined(WIN32) 17 | #include 18 | #else 19 | #include 20 | #include 21 | #include 22 | #include 23 | #endif 24 | #include 25 | 26 | namespace pypa { 27 | 28 | inline file_handle_t open_file(char const * file_path) { 29 | file_handle_t v{}; 30 | #if defined(WIN32) 31 | v = ::CreateFileA(file_path, GENERIC_READ, 0, 0, OPEN_EXISTING, 0, 0); 32 | #else 33 | v = ::open(file_path, O_RDONLY); 34 | #endif 35 | return v; 36 | } 37 | 38 | FileBuf::FileBuf(char const * file_path) 39 | : handle_(open_file(file_path)) 40 | , buffer_{} 41 | , position_{0} 42 | , length_{0} 43 | , line_{1} 44 | , current_{EOF} 45 | , eof_{true} 46 | , utf8_{false} 47 | { 48 | eof_ = !fill_buffer(); 49 | if(length_ >= 3) { 50 | utf8_ = buffer_[0] == '\xEF' 51 | && buffer_[1] == '\xBB' 52 | && buffer_[2] == '\xBF'; 53 | position_ += utf8_ ? 3 : 0; 54 | } 55 | } 56 | 57 | FileBuf::~FileBuf() { 58 | #if defined(WIN32) 59 | if (handle_!= INVALID_HANDLE_VALUE) { 60 | CloseHandle(handle_); 61 | } 62 | #else 63 | ::close(handle_); 64 | #endif 65 | } 66 | 67 | bool FileBuf::utf8() const { 68 | return utf8_; 69 | } 70 | 71 | char FileBuf::next() { 72 | if(position_ >= length_) { 73 | if(length_ == 0 || !fill_buffer()) { 74 | current_ = -1; 75 | eof_ = true; 76 | } 77 | } 78 | if (position_ < length_) { 79 | eof_ = false; 80 | current_ = unsigned(buffer_[position_++]); 81 | if(current_ == '\n' || current_ == '\x0c') line_++; 82 | } 83 | return current_; 84 | } 85 | 86 | char FileBuf::current() const { 87 | return current_; 88 | } 89 | 90 | bool FileBuf::eof() const { 91 | return eof_; 92 | } 93 | 94 | bool FileBuf::fill_buffer() { 95 | #if defined(WIN32) 96 | int n = -1; 97 | if (handle_!= INVALID_HANDLE_VALUE) { 98 | DWORD bytes_read = 0; 99 | if (::ReadFile(handle_, buffer_, BufferSize, &bytes_read, 0)) { 100 | n = int(bytes_read); 101 | } 102 | } 103 | #else 104 | int n = read(handle_, buffer_, BufferSize); 105 | #endif 106 | position_ = 0; 107 | length_ = n <= 0 ? 0 : unsigned(n); 108 | return length_ != 0; 109 | } 110 | } 111 | 112 | 113 | -------------------------------------------------------------------------------- /src/pypa/filebuf.hh: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef GUARD_PYPA_FILEBUF_HH_INCLUDED 15 | #define GUARD_PYPA_FILEBUF_HH_INCLUDED 16 | 17 | #include 18 | 19 | namespace pypa { 20 | 21 | #if defined(WIN32) 22 | typedef void* file_handle_t; 23 | #else 24 | typedef int file_handle_t; 25 | #endif 26 | 27 | class FileBuf { 28 | static const unsigned BufferSize = 32; 29 | file_handle_t handle_; 30 | char buffer_[BufferSize]; 31 | unsigned position_; 32 | unsigned length_; 33 | unsigned line_; 34 | char current_; 35 | bool eof_; 36 | bool utf8_; 37 | public: 38 | FileBuf(char const * file_path); 39 | ~FileBuf(); 40 | 41 | unsigned line() const { return line_; } 42 | char next(); 43 | char current() const; 44 | bool eof() const; 45 | bool utf8() const; 46 | private: 47 | bool fill_buffer(); 48 | }; 49 | 50 | } 51 | 52 | #endif //GUARD_PYPA_FILEBUF_HH_INCLUDED 53 | -------------------------------------------------------------------------------- /src/pypa/lexer/delim.hh: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef GUARD_PYPA_TOKENIZER_DELIM_HH_INCLUDED 15 | #define GUARD_PYPA_TOKENIZER_DELIM_HH_INCLUDED 16 | 17 | #include 18 | #include 19 | #include 20 | 21 | namespace pypa { 22 | TokenDef DelimTokens[] = { 23 | TokenDef(Token::DelimBraceOpen, TokenString("{"), TokenKind::LeftBrace, TokenClass::Delimiter), 24 | TokenDef(Token::DelimBraceClose, TokenString("}"), TokenKind::RightBrace, TokenClass::Delimiter), 25 | TokenDef(Token::DelimComma, TokenString(","), TokenKind::Comma, TokenClass::Delimiter), 26 | TokenDef(Token::DelimColon, TokenString(":"), TokenKind::Colon, TokenClass::Delimiter), 27 | TokenDef(Token::DelimPeriod, TokenString("."), TokenKind::Dot, TokenClass::Delimiter), 28 | TokenDef(Token::DelimSemiColon, TokenString(";"), TokenKind::SemiColon, TokenClass::Delimiter), 29 | TokenDef(Token::DelimBracketOpen, TokenString("["), TokenKind::LeftBracket, TokenClass::Delimiter), 30 | TokenDef(Token::DelimBracketClose, TokenString("]"), TokenKind::RightBracket, TokenClass::Delimiter), 31 | TokenDef(Token::DelimParenOpen, TokenString("("), TokenKind::LeftParen, TokenClass::Delimiter), 32 | TokenDef(Token::DelimParenClose, TokenString(")"), TokenKind::RightParen, TokenClass::Delimiter), 33 | TokenDef(Token::DelimAt, TokenString("@"), TokenKind::At, TokenClass::Delimiter) 34 | }; 35 | 36 | inline ConstArray Delims() { 37 | return { DelimTokens }; 38 | } 39 | } 40 | #endif //GUARD_PYPA_TOKENIZER_DELIM_HH_INCLUDED 41 | 42 | -------------------------------------------------------------------------------- /src/pypa/lexer/keyword.hh: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef GUARD_PYPA_TOKENIZER_KEYWORD_HH_INCLUDED 15 | #define GUARD_PYPA_TOKENIZER_KEYWORD_HH_INCLUDED 16 | 17 | #include 18 | #include 19 | 20 | 21 | namespace pypa { 22 | static const TokenDef KeywordTokens[] = { 23 | TokenDef(Token::KeywordAnd, TokenString("and"), TokenKind::Name, TokenClass::Keyword), 24 | TokenDef(Token::KeywordAs, TokenString("as"), TokenKind::Name, TokenClass::Keyword), 25 | TokenDef(Token::KeywordAssert, TokenString("assert"), TokenKind::Name, TokenClass::Keyword), 26 | TokenDef(Token::KeywordBreak, TokenString("break"), TokenKind::Name, TokenClass::Keyword), 27 | TokenDef(Token::KeywordClass, TokenString("class"), TokenKind::Name, TokenClass::Keyword), 28 | TokenDef(Token::KeywordContinue, TokenString("continue"), TokenKind::Name, TokenClass::Keyword), 29 | TokenDef(Token::KeywordDef, TokenString("def"), TokenKind::Name, TokenClass::Keyword), 30 | TokenDef(Token::KeywordDel, TokenString("del"), TokenKind::Name, TokenClass::Keyword), 31 | TokenDef(Token::KeywordElIf, TokenString("elif"), TokenKind::Name, TokenClass::Keyword), 32 | TokenDef(Token::KeywordElse, TokenString("else"), TokenKind::Name, TokenClass::Keyword), 33 | TokenDef(Token::KeywordExcept, TokenString("except"), TokenKind::Name, TokenClass::Keyword), 34 | TokenDef(Token::KeywordExec, TokenString("exec"), TokenKind::Name, TokenClass::Keyword), 35 | TokenDef(Token::KeywordFinally, TokenString("finally"), TokenKind::Name, TokenClass::Keyword), 36 | TokenDef(Token::KeywordFor, TokenString("for"), TokenKind::Name, TokenClass::Keyword), 37 | TokenDef(Token::KeywordFrom, TokenString("from"), TokenKind::Name, TokenClass::Keyword), 38 | TokenDef(Token::KeywordGlobal, TokenString("global"), TokenKind::Name, TokenClass::Keyword), 39 | TokenDef(Token::KeywordIf, TokenString("if"), TokenKind::Name, TokenClass::Keyword), 40 | TokenDef(Token::KeywordImport, TokenString("import"), TokenKind::Name, TokenClass::Keyword), 41 | TokenDef(Token::KeywordIn, TokenString("in"), TokenKind::Name, TokenClass::Keyword), 42 | TokenDef(Token::KeywordIs, TokenString("is"), TokenKind::Name, TokenClass::Keyword), 43 | TokenDef(Token::KeywordLambda, TokenString("lambda"), TokenKind::Name, TokenClass::Keyword), 44 | TokenDef(Token::KeywordNonLocal, TokenString("nonlocal"), TokenKind::Name, TokenClass::Keyword), 45 | TokenDef(Token::KeywordNot, TokenString("not"), TokenKind::Name, TokenClass::Keyword), 46 | TokenDef(Token::KeywordOr, TokenString("or"), TokenKind::Name, TokenClass::Keyword), 47 | TokenDef(Token::KeywordPass, TokenString("pass"), TokenKind::Name, TokenClass::Keyword), 48 | TokenDef(Token::KeywordRaise, TokenString("raise"), TokenKind::Name, TokenClass::Keyword), 49 | TokenDef(Token::KeywordReturn, TokenString("return"), TokenKind::Name, TokenClass::Keyword), 50 | TokenDef(Token::KeywordTry, TokenString("try"), TokenKind::Name, TokenClass::Keyword), 51 | TokenDef(Token::KeywordWhile, TokenString("while"), TokenKind::Name, TokenClass::Keyword), 52 | TokenDef(Token::KeywordWith, TokenString("with"), TokenKind::Name, TokenClass::Keyword), 53 | TokenDef(Token::KeywordYield, TokenString("yield"), TokenKind::Name, TokenClass::Keyword), 54 | // Starting with capital 55 | // TokenDef(Token::KeywordFalse, TokenString("False"), TokenKind::Name, TokenClass::Keyword), 56 | // TokenDef(Token::KeywordNone, TokenString("None"), TokenKind::Name, TokenClass::Keyword), 57 | // TokenDef(Token::KeywordTrue, TokenString("True"), TokenKind::Name, TokenClass::Keyword) 58 | }; 59 | 60 | inline ConstArray Keywords() { 61 | return { KeywordTokens }; 62 | } 63 | } 64 | #endif // GUARD_PYPA_TOKENIZER_KEYWORD_HH_INCLUDED 65 | -------------------------------------------------------------------------------- /src/pypa/lexer/lexer.hh: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef GUARD_PYPA_TOKENIZER_LEXER_HH_INCLUDED 15 | #define GUARD_PYPA_TOKENIZER_LEXER_HH_INCLUDED 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | #include 23 | #include 24 | 25 | namespace pypa { 26 | 27 | struct TokenInfo { 28 | TokenIdent ident; 29 | uint64_t line; 30 | uint64_t column; 31 | std::string value; 32 | }; 33 | 34 | enum class LexerInfoLevel { 35 | Information, 36 | Warning, 37 | Error 38 | }; 39 | 40 | struct LexerInfo { 41 | TokenInfo info; 42 | LexerInfoLevel level; 43 | std::string value; 44 | }; 45 | 46 | class Lexer { 47 | enum { 48 | TabSize = 8, 49 | AltTabSize = 1, 50 | }; 51 | 52 | pypa::FileBuf file_; 53 | std::string input_path_; 54 | 55 | uint32_t column_; 56 | int level_; 57 | int indent_; 58 | std::vector indent_stack_; 59 | std::vector alt_indent_stack_; 60 | std::deque lex_buffer_; 61 | 62 | std::list info_; 63 | char first_indet_char; 64 | std::deque token_buffer_; 65 | public: 66 | Lexer(char const * file_path); 67 | ~Lexer(); 68 | 69 | std::string get_name() const; 70 | std::string get_line(int idx); 71 | 72 | std::list const & info(); 73 | 74 | TokenInfo next(); 75 | 76 | private: 77 | char skip(); 78 | char skip_comment(); 79 | unsigned line() const { return file_.line(); } 80 | char next_char(); 81 | void put_char(char c); 82 | TokenInfo get_string(TokenInfo & tok, char first, char prefix=0); 83 | TokenInfo get_number(TokenInfo & tok, char first); 84 | TokenInfo get_number_binary(TokenInfo & tok, char first); 85 | TokenInfo get_number_hex(TokenInfo & tok, char first); 86 | TokenInfo get_number_octal(TokenInfo & tok, char first); 87 | TokenInfo get_number_float(TokenInfo & tok, char first); 88 | TokenInfo get_number_integer(TokenInfo & tok, char first); 89 | TokenInfo get_number_complex(TokenInfo & tok, char first); 90 | 91 | bool handle_indentation(bool continuation = false); 92 | void handle_whitespace(char c); 93 | 94 | TokenInfo make_token(TokenInfo & tok, Token id, TokenKind kind, 95 | TokenClass cls = TokenClass::Default); 96 | TokenInfo make_token(TokenInfo & tok, TokenIdent ident); 97 | TokenInfo make_invalid(char const * begin, char const * end, 98 | Token id = Token::Invalid, 99 | TokenKind kind = TokenKind::Error, 100 | TokenClass cls = TokenClass::Default); 101 | 102 | bool add_indent_error(bool dedent = false); 103 | void add_info_item(LexerInfoLevel level, TokenInfo const & t); 104 | 105 | inline void add_information(TokenInfo const & t) { 106 | add_info_item(LexerInfoLevel::Information, t); 107 | } 108 | 109 | inline void add_warning(TokenInfo const & t) { 110 | add_info_item(LexerInfoLevel::Warning, t); 111 | } 112 | 113 | inline void add_error(TokenInfo const & t) { 114 | add_info_item(LexerInfoLevel::Error, t); 115 | } 116 | }; 117 | 118 | } 119 | 120 | #endif //GUARD_PYPA_TOKENIZER_LEXER_HH_INCLUDED 121 | 122 | -------------------------------------------------------------------------------- /src/pypa/lexer/op.hh: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef GUARD_PYPA_TOKENIZER_OP_HH_INCLUDED 15 | #define GUARD_PYPA_TOKENIZER_OP_HH_INCLUDED 16 | 17 | #include 18 | #include 19 | #include 20 | 21 | namespace pypa { 22 | TokenDef OpTokens[] = { 23 | TokenDef(Token::OpAddAssign, TokenString("+="), TokenKind::PlusEqual, TokenClass::Operator), 24 | TokenDef(Token::OpSubAssign, TokenString("-="), TokenKind::MinusEqual, TokenClass::Operator), 25 | TokenDef(Token::OpMulAssign, TokenString("*="), TokenKind::StarEqual, TokenClass::Operator), 26 | TokenDef(Token::OpDivAssign, TokenString("/="), TokenKind::SlashEqual, TokenClass::Operator), 27 | TokenDef(Token::OpDivDivAssign, TokenString("//="), TokenKind::DoubleSlashEqual, TokenClass::Operator), 28 | TokenDef(Token::OpModAssign, TokenString("%="), TokenKind::PercentEqual, TokenClass::Operator), 29 | TokenDef(Token::OpAndAssign, TokenString("&="), TokenKind::BinAndEqual, TokenClass::Operator), 30 | TokenDef(Token::OpOrAssign, TokenString("|="), TokenKind::BinOrEqual, TokenClass::Operator), 31 | TokenDef(Token::OpXorAssign, TokenString("^="), TokenKind::CircumFlexEqual, TokenClass::Operator), 32 | TokenDef(Token::OpShiftLeftAssign, TokenString("<<="), TokenKind::LeftShiftEqual, TokenClass::Operator), 33 | TokenDef(Token::OpShiftRightAssign, TokenString(">>="), TokenKind::RightShiftEqual, TokenClass::Operator), 34 | TokenDef(Token::OpExpAssign, TokenString("**="), TokenKind::DoubleStarEqual, TokenClass::Operator), 35 | TokenDef(Token::OpExp, TokenString("**"), TokenKind::DoubleStar, TokenClass::Operator), 36 | TokenDef(Token::OpDivDiv, TokenString("//"), TokenKind::DoubleSlash, TokenClass::Operator), 37 | TokenDef(Token::OpInv, TokenString("~"), TokenKind::Tilde, TokenClass::Operator), 38 | TokenDef(Token::OpShiftLeft, TokenString("<<"), TokenKind::LeftShift, TokenClass::Operator), 39 | TokenDef(Token::OpShiftRight, TokenString(">>"), TokenKind::RightShift, TokenClass::Operator), 40 | TokenDef(Token::OpEqual, TokenString("=="), TokenKind::EqualEqual, TokenClass::Operator), 41 | TokenDef(Token::OpNotEqual, TokenString("!="), TokenKind::NotEqual, TokenClass::Operator), 42 | TokenDef(Token::OpNotEqual, TokenString("<>"), TokenKind::NotEqual, TokenClass::Operator), 43 | TokenDef(Token::OpMoreEqual, TokenString(">="), TokenKind::GreaterEqual, TokenClass::Operator), 44 | TokenDef(Token::OpLessEqual, TokenString("<="), TokenKind::LessEqual, TokenClass::Operator), 45 | TokenDef(Token::OpAssign, TokenString("="), TokenKind::Equal, TokenClass::Operator), 46 | TokenDef(Token::OpMore, TokenString(">"), TokenKind::Greater, TokenClass::Operator), 47 | TokenDef(Token::OpLess, TokenString("<"), TokenKind::Less, TokenClass::Operator), 48 | TokenDef(Token::OpAdd, TokenString("+"), TokenKind::Plus, TokenClass::Operator), 49 | TokenDef(Token::OpSub, TokenString("-"), TokenKind::Minus, TokenClass::Operator), 50 | TokenDef(Token::OpMul, TokenString("*"), TokenKind::Star, TokenClass::Operator), 51 | TokenDef(Token::OpDiv, TokenString("/"), TokenKind::Slash, TokenClass::Operator), 52 | TokenDef(Token::OpMod, TokenString("%"), TokenKind::Percent, TokenClass::Operator), 53 | TokenDef(Token::OpAnd, TokenString("&"), TokenKind::BinAnd, TokenClass::Operator), 54 | TokenDef(Token::OpOr, TokenString("|"), TokenKind::BinOr, TokenClass::Operator), 55 | TokenDef(Token::OpXor, TokenString("^"), TokenKind::CircumFlex, TokenClass::Operator), 56 | TokenDef(Token::DelimArrow, TokenString("->"), TokenKind::Arrow, TokenClass::Default) 57 | }; 58 | 59 | inline ConstArray Ops() { 60 | return { OpTokens }; 61 | } 62 | } 63 | #endif //GUARD_PYPA_TOKENIZER_OP_HH_INCLUDED 64 | 65 | -------------------------------------------------------------------------------- /src/pypa/lexer/test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #include 15 | #include 16 | #include 17 | 18 | 19 | 20 | int main(int argc, char const ** argv) { 21 | int errors = 0; 22 | int warnings = 0; 23 | for(int i = 0 + (argc == 1 ? 0 : 1); i < argc; ++i) { 24 | char const * file = i != 0 ? argv[i] : "test.py"; 25 | pypa::Lexer l(file); 26 | for(;;) { 27 | auto t = l.next(); 28 | if(t.ident.id() == pypa::Token::End) break; 29 | if(t.ident.id() == pypa::Token::Invalid) { 30 | // printf("########### INVALID TOKEN: (%s:%d:%d) %d - %s\n", file, t.line, t.column, int(t.ident.id()), t.value.c_str()); 31 | ++errors; 32 | } 33 | // printf("%s Token[%d] %s\n", file, int(t.ident.id()), t.value.c_str()); 34 | } 35 | if(!l.info().empty()) { 36 | printf("LexerInfo:\n"); 37 | char const *level[] = {"Info", "Warn", "Error"}; 38 | for(auto const & i : l.info()) { 39 | ++warnings; 40 | printf("\t%s: %d:%d %s\n", level[int(i.level)], int(i.info.line), int(i.info.column), i.value.c_str()); 41 | } 42 | } 43 | // printf("%s current line: %d\n", file, l.line_); 44 | if(warnings || errors) return 0; 45 | } 46 | printf("%d lexing errors\n", errors); 47 | printf("%d lexing warnings\n", warnings); 48 | } 49 | -------------------------------------------------------------------------------- /src/pypa/lexer/tokendef.hh: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef GUARD_PYPA_TOKENIZER_TOKENDEF_HH_INCLUDED 15 | #define GUARD_PYPA_TOKENIZER_TOKENDEF_HH_INCLUDED 16 | 17 | #include 18 | #include 19 | #include 20 | 21 | namespace pypa { 22 | 23 | template< typename ValueType, size_t InitSizeDiff = 0> 24 | class ConstArray { 25 | private: 26 | ValueType * const data_; 27 | const std::size_t size_; 28 | public: 29 | template 30 | ConstArray(ValueType(&data)[N]) 31 | : data_(data) 32 | , size_(N-InitSizeDiff) 33 | {} 34 | 35 | char operator[](std::size_t index) const { 36 | return index < size_ ? data_[index] : throw std::out_of_range(""); 37 | } 38 | 39 | std::size_t size() const { 40 | return size_; 41 | } 42 | 43 | ValueType const * data() const { 44 | return data_; 45 | } 46 | 47 | bool empty() const { 48 | return size_ == 0; 49 | } 50 | 51 | ValueType const * begin() const { 52 | return data(); 53 | } 54 | 55 | ValueType const * end() const { 56 | return data() + size(); 57 | } 58 | }; 59 | 60 | class TokenString : public ConstArray 61 | { 62 | public: 63 | template 64 | TokenString(char const (&u)[N]) 65 | : ConstArray(u) 66 | {} 67 | 68 | char const * c_str() const { 69 | return data(); 70 | } 71 | 72 | bool empty() const { 73 | return size() == 0 || data()[0] == '\0'; 74 | } 75 | }; 76 | enum class TokenClass { 77 | Default, 78 | Operator, 79 | Keyword, 80 | Delimiter, 81 | Literal 82 | }; 83 | 84 | enum class TokenKind { 85 | End = 0, 86 | Name = 1, 87 | Number = 2, 88 | String = 3, 89 | NewLine = 4, 90 | Indent = 5, 91 | Dedent = 6, 92 | LeftParen = 7, 93 | RightParen = 8, 94 | LeftBracket = 9, 95 | RightBracket = 10, 96 | Colon = 11, 97 | Comma = 12, 98 | SemiColon = 13, 99 | Plus = 14, 100 | Minus = 15, 101 | Star = 16, 102 | Slash = 17, 103 | BinOr = 18, 104 | BinAnd = 19, 105 | Less = 20, 106 | Greater = 21, 107 | Equal = 22, 108 | Dot = 23, 109 | Percent = 24, 110 | BackQuote = 25, 111 | LeftBrace = 26, 112 | RightBrace = 27, 113 | EqualEqual = 28, 114 | NotEqual = 29, 115 | LessEqual = 30, 116 | GreaterEqual = 31, 117 | Tilde = 32, 118 | CircumFlex = 33, 119 | LeftShift = 34, 120 | RightShift = 35, 121 | DoubleStar = 36, 122 | PlusEqual = 37, 123 | MinusEqual = 38, 124 | StarEqual = 39, 125 | SlashEqual = 40, 126 | PercentEqual = 41, 127 | BinAndEqual = 42, 128 | BinOrEqual = 43, 129 | CircumFlexEqual = 44, 130 | LeftShiftEqual = 45, 131 | RightShiftEqual = 46, 132 | DoubleStarEqual = 47, 133 | DoubleSlash = 48, 134 | DoubleSlashEqual= 49, 135 | At = 50, 136 | Operator = 51, 137 | Error = 52, 138 | 139 | // Let this be at the end 140 | KindCount, 141 | Arrow 142 | }; 143 | 144 | class TokenIdent { 145 | private: 146 | Token id_; 147 | TokenKind kind_; 148 | TokenClass cls_; 149 | public: 150 | TokenIdent(Token id, TokenKind kind, TokenClass cls) 151 | : id_(id) 152 | , kind_(kind) 153 | , cls_(cls) 154 | {} 155 | 156 | TokenIdent() 157 | : TokenIdent(Token::Invalid, TokenKind::Error, TokenClass::Default) 158 | {} 159 | 160 | Token id() const { 161 | return id_; 162 | } 163 | 164 | TokenKind kind() const { 165 | return kind_; 166 | } 167 | 168 | TokenClass cls() const { 169 | return cls_; 170 | } 171 | }; 172 | 173 | class TokenDef { 174 | private: 175 | TokenIdent ident_; 176 | TokenString value_; 177 | public: 178 | TokenDef(Token id, TokenString value, TokenKind kind, TokenClass cls) 179 | : ident_(id, kind, cls) 180 | , value_(value) 181 | {} 182 | 183 | TokenIdent ident() const { 184 | return ident_; 185 | } 186 | 187 | TokenString value() const { 188 | return value_; 189 | } 190 | 191 | inline bool match3(char c0, char c1, char c2) const { 192 | return (value().size() == 3 && (c0 == value()[0] && c1 == value()[1] && c2 == value()[2])) 193 | || (value().size() == 2 && (c0 == value()[0] && c1 == value()[1])) 194 | || (value().size() == 1 && (c0 == value()[0])); 195 | } 196 | 197 | inline bool match2(char c0, char c1) const { 198 | return (value().size() == 2 && (c0 == value()[0] && c1 == value()[1])) 199 | || (value().size() == 1 && (c0 == value()[0])); 200 | } 201 | }; 202 | } 203 | #endif // GUARD_PYPA_TOKENIZER_TOKENDEF_HH_INCLUDED 204 | 205 | -------------------------------------------------------------------------------- /src/pypa/lexer/tokens.hh: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef GUARD_PYPA_TOKENIZER_HH_INCLUDED 15 | #define GUARD_PYPA_TOKENIZER_HH_INCLUDED 16 | 17 | namespace pypa { 18 | enum class Token { 19 | Invalid, 20 | End, 21 | NewLine, 22 | Indent, 23 | Dedent, 24 | 25 | NotMultipleOfFourIndentError, 26 | MixedIndentError, 27 | IndentationError, 28 | DedentationError, 29 | LineContinuationError, 30 | UnterminatedStringError, 31 | 32 | Comment, 33 | BackQuote, 34 | Identifier, 35 | NumberFloat, 36 | NumberInteger, 37 | NumberHex, 38 | NumberOct, 39 | NumberBinary, 40 | NumberComplex, 41 | String, 42 | 43 | KeywordAnd, 44 | KeywordAs, 45 | KeywordAssert, 46 | KeywordBreak, 47 | KeywordClass, 48 | KeywordContinue, 49 | KeywordDef, 50 | KeywordDel, 51 | KeywordElIf, 52 | KeywordElse, 53 | KeywordExcept, 54 | KeywordExec, 55 | //KeywordFalse, 56 | KeywordFinally, 57 | KeywordFor, 58 | KeywordFrom, 59 | KeywordGlobal, 60 | KeywordIf, 61 | KeywordImport, 62 | KeywordIn, 63 | KeywordIs, 64 | KeywordLambda, 65 | KeywordNonLocal, 66 | KeywordNone, 67 | KeywordNot, 68 | KeywordOr, 69 | KeywordPass, 70 | KeywordRaise, 71 | KeywordReturn, 72 | //KeywordTrue, 73 | KeywordTry, 74 | KeywordWhile, 75 | KeywordWith, 76 | KeywordYield, 77 | 78 | Ellipsis, 79 | 80 | OpAdd, 81 | OpSub, 82 | OpMul, 83 | OpDiv, 84 | OpExp, 85 | OpMod, 86 | OpAnd, 87 | OpOr, 88 | OpXor, 89 | OpDivDiv, 90 | OpInv, 91 | OpShiftLeft, 92 | OpShiftRight, 93 | OpEqual, 94 | OpNotEqual, 95 | OpMoreEqual, 96 | OpLessEqual, 97 | OpMore, 98 | OpLess, 99 | OpAssign, 100 | OpAddAssign, 101 | OpSubAssign, 102 | OpMulAssign, 103 | OpDivAssign, 104 | OpDivDivAssign, 105 | OpModAssign, 106 | OpAndAssign, 107 | OpOrAssign, 108 | OpXorAssign, 109 | OpShiftLeftAssign, 110 | OpShiftRightAssign, 111 | OpExpAssign, 112 | 113 | DelimBraceOpen, 114 | DelimBraceClose, 115 | DelimComma, 116 | DelimColon, 117 | DelimPeriod, 118 | DelimSemiColon, 119 | DelimBracketOpen, 120 | DelimBracketClose, 121 | DelimParenOpen, 122 | DelimParenClose, 123 | DelimAt, 124 | DelimArrow, 125 | }; 126 | } 127 | 128 | #endif //GUARD_PYPA_TOKENIZER_HH_INCLUDED 129 | 130 | -------------------------------------------------------------------------------- /src/pypa/parser/apply.hh: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef GUARD_PYPA_PARSER_APPLY_HH_INCLUDED 15 | #define GUARD_PYPA_PARSER_APPLY_HH_INCLUDED 16 | 17 | #include 18 | 19 | namespace pypa { 20 | 21 | template< typename TargetT, typename BaseT> 22 | bool apply(State & s, BaseT & b, bool(*f)(State &, TargetT&)) 23 | { 24 | TargetT t; 25 | if(f(s, t)) { 26 | b = t; 27 | return true; 28 | } 29 | return false; 30 | } 31 | 32 | template< typename TargetT, typename ContainerT> 33 | bool push_apply(State & s, ContainerT & c, bool(*f)(State &, TargetT&)) { 34 | typename ContainerT::value_type v; 35 | if(apply(s, v, f)) { 36 | c.push_back(v); 37 | return true; 38 | } 39 | return false; 40 | } 41 | 42 | } 43 | 44 | #endif // GUARD_PYPA_PARSER_APPLY_HH_INCLUDED 45 | -------------------------------------------------------------------------------- /src/pypa/parser/error.hh: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef GUARD_PYPA_PARSER_ERROR_HH_INCLUDED 15 | #define GUARD_PYPA_PARSER_ERROR_HH_INCLUDED 16 | 17 | #include 18 | #include 19 | 20 | namespace pypa { 21 | 22 | enum class ErrorType { 23 | SyntaxError, 24 | SyntaxWarning, 25 | IndentationError 26 | }; 27 | 28 | struct Error { 29 | ErrorType type; 30 | String message; 31 | String file_name; 32 | TokenInfo cur; 33 | AstPtr ast; 34 | std::string line; 35 | int detected_line; 36 | char const * detected_file; 37 | char const * detected_function; 38 | }; 39 | } 40 | 41 | #endif // GUARD_PYPA_PARSER_ERROR_HH_INCLUDED 42 | -------------------------------------------------------------------------------- /src/pypa/parser/future_features.hh: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef GUARD_PYPA_PARSER_FUTURE_FEATURES_HH_INCLUDED 15 | #define GUARD_PYPA_PARSER_FUTURE_FEATURES_HH_INCLUDED 16 | 17 | namespace pypa { 18 | 19 | struct FutureFeatures { 20 | bool nested_scopes; 21 | bool generators; 22 | bool division; 23 | bool absolute_imports; 24 | bool with_statement; 25 | bool print_function; 26 | bool unicode_literals; 27 | int last_line; // last line with a future feature statement 28 | 29 | FutureFeatures() 30 | : nested_scopes() 31 | , generators() 32 | , division() 33 | , absolute_imports() 34 | , with_statement() 35 | , print_function() 36 | , unicode_literals() 37 | , last_line() 38 | {} 39 | }; 40 | 41 | } 42 | 43 | #endif // GUARD_PYPA_PARSER_FUTURE_FEATURES_HH_INCLUDED 44 | -------------------------------------------------------------------------------- /src/pypa/parser/make_string.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | 15 | #include 16 | #include 17 | 18 | namespace pypa { 19 | 20 | inline bool isxdigit(char c) { 21 | return (c >= '0' && c <= '9') 22 | || (c >= 'a' && c <= 'f') 23 | || (c >= 'A' && c <= 'F'); 24 | } 25 | 26 | inline bool islower(char c) { 27 | return (c >= 'a' && c <= 'z'); 28 | } 29 | 30 | String make_string(String const & input, bool & unicode, bool & raw) { 31 | String result; 32 | size_t first_quote = input.find_first_of("\"'"); 33 | assert(first_quote != String::npos); 34 | char quote = input[first_quote]; 35 | size_t string_start = input.find_first_not_of(quote, first_quote); 36 | 37 | char const * qst = input.c_str() + first_quote; 38 | char const * tmp = input.c_str(); 39 | 40 | // bool bytes = false; 41 | raw = false; 42 | while(tmp != qst) { 43 | switch(*tmp) { 44 | case 'u': case 'U': 45 | unicode = true; 46 | break; 47 | case 'r': case 'R': 48 | raw = true; 49 | break; 50 | case 'b': case 'B': 51 | // bytes = true; 52 | unicode = false; 53 | break; 54 | default: 55 | assert("Unknown character prefix" && false); 56 | break; 57 | } 58 | ++tmp; 59 | } 60 | 61 | 62 | if(string_start == String::npos) { 63 | // Empty string 64 | return result; 65 | } 66 | 67 | assert((string_start - first_quote) < input.size()); 68 | size_t string_end = input.size() - (string_start - first_quote); 69 | assert(string_end != String::npos && string_start <= string_end); 70 | 71 | char const * s = input.c_str() + string_start; 72 | char const * end = input.c_str() + string_end; 73 | 74 | std::back_insert_iterator p((result)); 75 | 76 | while(s < end) { 77 | char c = *s; 78 | if(raw || unicode) { 79 | *p++ = *s++; 80 | } 81 | else { // !raw 82 | switch(*s) { 83 | case '\\': 84 | ++s; 85 | assert(s < end); 86 | c = *s; 87 | ++s; 88 | switch(c) { 89 | case '\n': break; 90 | case '\\': case '\'': case '\"': *p++ = c; break; 91 | case 'b': *p++ = '\b'; break; 92 | case 'f': *p++ = '\014'; break; /* FF */ 93 | case 't': *p++ = '\t'; break; 94 | case 'n': *p++ = '\n'; break; 95 | case 'r': *p++ = '\r'; break; 96 | case 'v': *p++ = '\013'; break; 97 | case 'a': *p++ = '\007'; break; 98 | case '0': case '1': case '2': case '3': 99 | case '4': case '5': case '6': case '7': 100 | c = c - '0'; 101 | if (s < end && '0' <= *s && *s <= '7') { 102 | c = (c<<3) + *s++ - '0'; 103 | if (s < end && '0' <= *s && *s <= '7') { 104 | c = (c<<3) + *s++ - '0'; 105 | } 106 | } 107 | *p++ = c; 108 | break; 109 | case 'x': 110 | if (s+1 < end && isxdigit(s[0]) && isxdigit(s[1])) 111 | { 112 | unsigned int x = 0; 113 | c = *s; 114 | s++; 115 | if (isdigit(c)) 116 | x = c - '0'; 117 | else if (islower(c)) 118 | x = 10 + c - 'a'; 119 | else 120 | x = 10 + c - 'A'; 121 | x = x << 4; 122 | c = *s; 123 | s++; 124 | if (isdigit(c)) 125 | x += c - '0'; 126 | else if (islower(c)) 127 | x += 10 + c - 'a'; 128 | else 129 | x += 10 + c - 'A'; 130 | *p++ = x; 131 | break; 132 | } 133 | /* skip \x */ 134 | if (s < end && isxdigit(s[0])) 135 | s++; /* and a hexdigit */ 136 | break; 137 | default: 138 | *p++ = '\\'; 139 | s--; 140 | } 141 | break; 142 | default: 143 | *p++ = *s++; 144 | break; 145 | } 146 | } // else !raw 147 | } 148 | return result; 149 | } 150 | 151 | 152 | } 153 | -------------------------------------------------------------------------------- /src/pypa/parser/parser.hh: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef GUARD_PYPA_PARSER_PARSER_HH_INCLUDED 15 | #define GUARD_PYPA_PARSER_PARSER_HH_INCLUDED 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | namespace pypa { 24 | 25 | struct ParserOptions { 26 | ParserOptions() 27 | : python3only(false) 28 | , python3allowed(false) 29 | , docstrings(true) 30 | , printerrors(true) 31 | , printdbgerrors(false) 32 | , handle_future_errors(true) 33 | , error_handler() 34 | {} 35 | 36 | bool python3only; // If it is parsing python3 37 | bool python3allowed; // Whether or not python3 constructs are allowed 38 | bool docstrings; // Converts docstring strings to AstDocString 39 | // entries 40 | bool printerrors; // Prints errors itself 41 | bool printdbgerrors; // Prints internal debug information 42 | bool handle_future_errors; // Handles unknown __future__ features 43 | // by reporting an error 44 | std::function error_handler; 45 | std::function unicode_escape_handler; 46 | }; 47 | 48 | bool parse(Lexer & lexer, 49 | AstModulePtr & ast, 50 | SymbolTablePtr & symbols, 51 | ParserOptions options = ParserOptions()); 52 | 53 | } 54 | 55 | #endif // GUARD_PYPA_PARSER_PARSER_HH_INCLUDED 56 | 57 | -------------------------------------------------------------------------------- /src/pypa/parser/parser_fwd.hh: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef GUARD_PYPA_PARSER_PARSER_FWD_HH_INCLUDED 15 | #define GUARD_PYPA_PARSER_PARSER_FWD_HH_INCLUDED 16 | 17 | #include 18 | 19 | namespace pypa { 20 | 21 | bool and_expr(State & s, AstExpr & ast); 22 | bool and_test(State & s, AstExpr & ast); 23 | bool arglist(State & s, AstArguments & ast); 24 | bool argument(State & s, AstExpr & ast); 25 | bool arith_expr(State & s, AstExpr & ast); 26 | bool assert_stmt(State & s, AstStmt & ast); 27 | bool atom(State & s, AstExpr & ast); 28 | #if 0 29 | bool augassign(State & s, AstExpr & ast); 30 | #endif 31 | bool break_stmt(State & s, AstStmt & ast); 32 | bool classdef(State & s, AstStmt & ast); 33 | bool comp_for(State & s, AstExprList & ast); 34 | bool comp_if(State & s, AstExpr & ast); 35 | bool comp_op(State & s, AstCompareOpType & op); 36 | bool comparison(State & s, AstExpr & ast); 37 | bool compound_stmt(State & s, AstStmt & ast); 38 | bool continue_stmt(State & s, AstStmt & ast); 39 | bool decorated(State & s, AstStmt & ast); 40 | bool decorator(State & s, AstExpr & ast); 41 | bool decorators(State & s, AstExprList & decorators); 42 | bool del_stmt(State & s, AstStmt & ast); 43 | bool dictorsetmaker(State & s, AstExpr & ast); 44 | bool dotted_as_name(State & s, AstExpr & ast); 45 | bool dotted_as_names(State & s, AstExpr & ast); 46 | bool dotted_name(State & s, AstExpr & ast, bool as_dotted_name); 47 | #if 0 48 | bool eval_input(State & s, AstModulePtr & ast); 49 | #endif 50 | bool except_clause(State & s, AstExpr & ast); 51 | bool exec_stmt(State & s, AstStmt & ast); 52 | bool expr(State & s, AstExpr & ast); 53 | bool expr_stmt(State & s, AstStmt & ast); 54 | bool exprlist(State & s, AstExpr & ast); 55 | bool factor(State & s, AstExpr & ast); 56 | bool file_input(State & s, AstModulePtr & ast); 57 | bool flow_stmt(State & s, AstStmt & ast); 58 | bool for_stmt(State & s, AstStmt & ast); 59 | bool fpdef(State & s, AstExpr & ast); 60 | bool fplist(State & s, AstExpr & ast); 61 | bool funcdef(State & s, AstStmt & ast); 62 | bool global_stmt(State & s, AstStmt & ast); 63 | bool if_stmt(State & s, AstStmt & ast); 64 | bool import_as_name(State & s, AstExpr & ast); 65 | bool import_as_names(State & s, AstExpr & ast); 66 | bool import_from(State & s, AstStmt & ast); 67 | bool import_name(State & s, AstStmt & ast); 68 | bool import_stmt(State & s, AstStmt & ast); 69 | bool lambdef(State & s, AstExpr & ast); 70 | bool list_for(State & s, AstExprList & ast); 71 | bool list_if(State & s, AstExpr & ast); 72 | bool listmaker(State & s, AstExpr & ast); 73 | bool not_test(State & s, AstExpr & ast); 74 | bool old_lambdef(State & s, AstExpr & ast); 75 | bool old_test(State & s, AstExpr & ast); 76 | bool or_test(State & s, AstExpr & ast); 77 | bool parameters(State & s, AstArguments & ast); 78 | bool pass_stmt(State & s, AstStmt & ast); 79 | bool power(State & s, AstExpr & ast); 80 | bool print_stmt(State & s, AstStmt & ast); 81 | bool raise_stmt(State & s, AstStmt & ast); 82 | bool return_stmt(State & s, AstStmt & ast); 83 | bool shift_expr(State & s, AstExpr & ast); 84 | bool simple_stmt(State & s, AstStmt & ast); 85 | #if 0 86 | bool single_input(State & s, AstModulePtr & ast); 87 | #endif 88 | bool sliceop(State & s, AstExpr & ast); 89 | bool small_stmt(State & s, AstStmt & ast); 90 | bool stmt(State & s, AstStmt & ast); 91 | bool subscript(State & s, AstExpr & ast); 92 | bool subscriptlist(State & s, AstExtSlice & ast); 93 | bool suite(State & s, AstStmt & ast); 94 | bool term(State & s, AstExpr & ast); 95 | bool test(State & s, AstExpr & ast); 96 | bool testlist(State & s, AstExpr & ast); 97 | bool testlist1(State & s, AstExpr & ast); 98 | #if 0 99 | bool testlist_comp(State & s, AstExpr & ast); 100 | #endif 101 | bool testlist_safe(State & s, AstExpr & ast); 102 | bool trailer(State & s, AstExpr & ast, AstExpr target); 103 | bool try_stmt(State & s, AstStmt & ast); 104 | bool varargslist(State & s, AstArguments & ast); 105 | bool while_stmt(State & s, AstStmt & ast); 106 | bool with_stmt(State & s, AstStmt & ast); 107 | bool xor_expr(State & s, AstExpr & ast); 108 | bool yield_expr(State & s, AstExpr & ast); 109 | bool yield_stmt(State & s, AstStmt & ast); 110 | 111 | } 112 | 113 | #endif //GUARD_PYPA_PARSER_PARSER_FWD_HH_INCLUDED 114 | -------------------------------------------------------------------------------- /src/pypa/parser/state.hh: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef GUARD_PYPA_PARSER_STATE_HH_INCLUDED 15 | #define GUARD_PYPA_PARSER_STATE_HH_INCLUDED 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | 22 | namespace pypa { 23 | namespace { 24 | struct State { 25 | Lexer * lexer; 26 | std::stack tokens; 27 | std::stack popped; 28 | std::stack savepoints; 29 | TokenInfo tok_cur; 30 | std::stack errors; 31 | ParserOptions options; 32 | FutureFeatures future_features; 33 | }; 34 | 35 | inline TokenInfo pop(State & s) { 36 | s.popped.push(s.tok_cur); 37 | if(s.tokens.empty()) { 38 | s.tok_cur = s.lexer->next(); 39 | } 40 | else { 41 | s.tok_cur = s.tokens.top(); 42 | s.tokens.pop(); 43 | } 44 | 45 | return s.tok_cur; 46 | } 47 | 48 | inline void unpop(State & s) { 49 | s.tokens.push(s.tok_cur); 50 | s.tok_cur = s.popped.top(); 51 | s.popped.pop(); 52 | } 53 | 54 | inline void save(State & s) { 55 | s.savepoints.push(s.popped.size()); 56 | } 57 | 58 | inline void revert(State & s) { 59 | if(!s.savepoints.empty()) { 60 | while(s.popped.size() != s.savepoints.top()) { 61 | unpop(s); 62 | } 63 | s.savepoints.pop(); 64 | } 65 | } 66 | 67 | inline void pop_savepoint(State & s) { 68 | if(!s.savepoints.empty()) { 69 | s.savepoints.pop(); 70 | } 71 | } 72 | 73 | struct StateGuard { 74 | StateGuard(State & s) : reset_(), s_(&s) { save(s); } 75 | template< typename T > 76 | StateGuard(State & s, std::shared_ptr & r) : reset_([&r](){r.reset();}), s_(&s) { save(s); } 77 | ~StateGuard() { if(s_) revert(*s_); if(reset_) reset_(); } 78 | bool commit() { if(s_) pop_savepoint(*s_); s_ = 0; reset_ = {}; return true; } 79 | private: 80 | std::function reset_; 81 | State * s_; 82 | }; 83 | 84 | inline void commit(State & s) { 85 | s.popped = std::stack(); 86 | } 87 | 88 | inline TokenInfo const & top(State & s) { 89 | return s.tok_cur; 90 | } 91 | 92 | inline TokenKind kind(TokenInfo const & tok) { 93 | return tok.ident.kind(); 94 | } 95 | 96 | inline TokenClass cls(TokenInfo const & tok) { 97 | return tok.ident.cls(); 98 | } 99 | 100 | inline Token token(TokenInfo const & tok) { 101 | return tok.ident.id(); 102 | } 103 | 104 | template< typename T > 105 | inline std::shared_ptr & create(std::shared_ptr & t) { 106 | return (t = std::make_shared()); 107 | } 108 | 109 | template< typename U, typename T > 110 | inline std::shared_ptr create(std::shared_ptr & t) { 111 | return std::static_pointer_cast(t = std::make_shared()); 112 | } 113 | 114 | inline void location(State & s, AstPtr a) { 115 | a->line = top(s).line; 116 | a->column = top(s).column; 117 | } 118 | 119 | inline void location(State & s, Ast & a) { 120 | a.line = top(s).line; 121 | a.column = top(s).column; 122 | } 123 | 124 | inline void clone_location(Ast & source, Ast & target) { 125 | target.column = source.column; 126 | target.line = source.line; 127 | } 128 | 129 | inline void clone_location(AstPtr source, AstPtr target) { 130 | if(source && target) { 131 | clone_location(*source, *target); 132 | } 133 | } 134 | 135 | inline bool is(TokenInfo const & info, Token tok) { 136 | return token(info) == tok; 137 | } 138 | 139 | inline bool is(TokenInfo const & info, TokenKind k) { 140 | return kind(info) == k; 141 | } 142 | 143 | inline bool is(TokenInfo const & info, TokenClass c) { 144 | return cls(info) == c; 145 | } 146 | 147 | inline bool is(State & s, Token tok) { 148 | return is(top(s), tok); 149 | } 150 | 151 | inline bool is(State & s, TokenKind k) { 152 | return is(top(s), k); 153 | } 154 | 155 | inline bool is(State & s, TokenClass c) { 156 | return is(top(s), c); 157 | } 158 | 159 | inline bool end(State & s) { 160 | return is(s, Token::End); 161 | } 162 | 163 | template< typename T > 164 | inline bool expect(State & s, T t) { 165 | if(is(top(s), t)) { 166 | pop(s); 167 | return true; 168 | } 169 | return false; 170 | } 171 | 172 | template< typename T > 173 | inline bool consume_value(State & s, T t, String & v) { 174 | if(is(s, t)) { 175 | v = top(s).value; 176 | pop(s); 177 | return true; 178 | } 179 | return false; 180 | } 181 | }} 182 | 183 | 184 | #endif // GUARD_PYPA_PARSER_STATE_HH_INCLUDED 185 | -------------------------------------------------------------------------------- /src/pypa/parser/symbol_table.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #include 15 | #include 16 | #include 17 | #include 18 | 19 | namespace pypa { 20 | void SymbolTable::enter_block(BlockType type, String const & name, Ast & a) { 21 | auto e = std::make_shared(); 22 | e->id = &a; 23 | e->type = type; 24 | e->name = name; 25 | e->is_nested = false; 26 | e->start_line = a.line; 27 | symbols[e->id] = e; 28 | 29 | if(type == BlockType::Module) { 30 | module = e; 31 | } 32 | 33 | if(current && (current->is_nested || current->type == BlockType::Function)) { 34 | e->is_nested = true; 35 | } 36 | if(current) { 37 | stack.push(current); 38 | } 39 | current = e; 40 | } 41 | 42 | void SymbolTable::leave_block() { 43 | if(!stack.empty()) { 44 | current = stack.top(); 45 | stack.pop(); 46 | } 47 | else { 48 | current = module; // This should never be necessary but who knows 49 | } 50 | } 51 | void create_from_ast(SymbolTablePtr p, Ast const & a, SymbolErrorReportFun add_err) { 52 | walk_tree(a, symbol_table_visitor{p, add_err}); 53 | } 54 | } 55 | -------------------------------------------------------------------------------- /src/pypa/parser/symbol_table.hh: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef GUARD_PYPA_PARSER_SYMBOL_TABLE_HH_INCLUDED 15 | #define GUARD_PYPA_PARSER_SYMBOL_TABLE_HH_INCLUDED 16 | 17 | #include 18 | #include 19 | #include 20 | #include 21 | #include 22 | 23 | #include 24 | #include 25 | #include 26 | 27 | namespace pypa { 28 | 29 | enum class BlockType { 30 | Function, 31 | Class, 32 | Module 33 | }; 34 | 35 | enum SymbolFlags { 36 | SymbolFlag_GlobalExplicit = 1 << 0, 37 | SymbolFlag_GlobalImplicit = 1 << 1, 38 | SymbolFlag_Global = (SymbolFlag_GlobalImplicit | SymbolFlag_GlobalExplicit), 39 | SymbolFlag_Local = 1 << 2, 40 | SymbolFlag_Param = 1 << 3, 41 | SymbolFlag_Used = 1 << 4, 42 | SymbolFlag_Free = 1 << 5, 43 | SymbolFlag_FreeClass = 1 << 6, 44 | SymbolFlag_Import = 1 << 7, 45 | SymbolFlag_Cell = 1 << 8, 46 | SymbolFlag_Bound = (SymbolFlag_Import | SymbolFlag_Local | SymbolFlag_Param) 47 | }; 48 | 49 | enum OptimizeFlags { 50 | OptimizeFlag_ImportStar = 1 << 0, 51 | OptimizeFlag_Exec = 1 << 1, 52 | OptimizeFlag_BareExec = 1 << 2, 53 | OptimizeFlag_TopLevel = 1 << 3, // Top level names including eval and exec 54 | }; 55 | 56 | typedef std::shared_ptr SymbolTableEntryPtr; 57 | struct SymbolTableEntry { 58 | void * id; 59 | BlockType type; 60 | String name; 61 | 62 | std::unordered_map symbols; 63 | std::unordered_set variables; 64 | std::list children; 65 | 66 | bool is_nested; // true if nested 67 | bool returns_value; // true if namespace uses return with an argument 68 | bool has_varargs; 69 | bool has_varkw; 70 | bool is_generator; // true if namespace is a generator 71 | 72 | bool has_free_vars; // true if block has free variables 73 | bool child_has_free_vars; // true if a child block has free vars, 74 | // including free refs to globals 75 | int start_line; // Line number where the block starts 76 | int temp_name_count; // number of temporary variables (e.g. in list comp) 77 | 78 | uint32_t unoptimized; // optimization flags 79 | int opt_last_line; // last line of exec or import 80 | }; 81 | 82 | struct SymbolTable { 83 | String file_name; 84 | 85 | String current_class; 86 | std::unordered_map symbols; 87 | std::stack stack; 88 | 89 | SymbolTableEntryPtr module; 90 | SymbolTableEntryPtr current; 91 | 92 | FutureFeatures future_features; 93 | 94 | void enter_block(BlockType type, String const & name, Ast &); 95 | void leave_block(); 96 | }; 97 | 98 | typedef std::shared_ptr SymbolTablePtr; 99 | typedef std::function SymbolErrorReportFun; 100 | void create_from_ast(SymbolTablePtr p, Ast const & a, SymbolErrorReportFun add_err); 101 | } 102 | 103 | #endif // GUARD_PYPA_PARSER_SYMBOL_TABLE_HH_INCLUDED 104 | -------------------------------------------------------------------------------- /src/pypa/parser/test.cc: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #include 15 | #include 16 | 17 | namespace pypa { 18 | void dump(AstPtr); 19 | } 20 | 21 | int main(int argc, char const ** argv) { 22 | if(argc != 2) { 23 | return 1; 24 | } 25 | pypa::AstModulePtr ast; 26 | pypa::SymbolTablePtr symbols; 27 | pypa::ParserOptions options; 28 | // options.python3allowed = true; 29 | options.printerrors = true; 30 | options.printdbgerrors = true; 31 | pypa::Lexer lexer(argv[1]); 32 | if(pypa::parse(lexer, ast, symbols, options)) { 33 | printf("Parsing successfull\n"); 34 | dump(ast); 35 | } 36 | else { 37 | printf("Parsing failed\n"); 38 | return 1; 39 | } 40 | return 0; 41 | } 42 | -------------------------------------------------------------------------------- /src/pypa/types.hh: -------------------------------------------------------------------------------- 1 | // Copyright 2014 Vinzenz Feenstra 2 | // 3 | // Licensed under the Apache License, Version 2.0 (the "License"); 4 | // you may not use this file except in compliance with the License. 5 | // You may obtain a copy of the License at 6 | // 7 | // http://www.apache.org/licenses/LICENSE-2.0 8 | // 9 | // Unless required by applicable law or agreed to in writing, software 10 | // distributed under the License is distributed on an "AS IS" BASIS, 11 | // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | // See the License for the specific language governing permissions and 13 | // limitations under the License. 14 | #ifndef GUARD_PYPA_TYPES_HH_INCLUDED 15 | #define GUARD_PYPA_TYPES_HH_INCLUDED 16 | 17 | #include 18 | #include 19 | 20 | namespace pypa { 21 | 22 | typedef std::string String; 23 | typedef std::vector StringList; 24 | 25 | } 26 | 27 | #endif // GUARD_PYPA_TYPES_HH_INCLUDED 28 | -------------------------------------------------------------------------------- /src/run-tests.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | # 3 | # run-tests.sh 4 | # Copyright (C) 2014 evilissimo 5 | # 6 | # Distributed under terms of the MIT license. 7 | # 8 | 9 | CPYTHON_SRC=${CPYTHON_SRC:-$HOME/devel/apps/cpython} 10 | test -d $CPYTHON_SRC/Python || { echo "CPYTHON_SRC not correctly set: $CPYTHON_SOURCE_DIR/Python not found" && exit 1; } 11 | mkdir -p results 12 | for file in `find $CPYTHON_SRC -name "*.py"` 13 | do 14 | echo "Parsing $file" 15 | ./parser-test $file &> ./results/`basename $file`.ast.out && rm -f ./results/`basename $file`.ast.out 16 | done 17 | 18 | 19 | BADSYN3=0 20 | BADSYN8=0 21 | BADSYN9=0 22 | PY3TEST=0 23 | FAILURE=0 24 | for result in `ls ./results`; 25 | do 26 | case $result in 27 | badsyntax_future3.py.ast.out) 28 | BADSYN3=1 29 | continue 30 | ;; 31 | badsyntax_future8.py.ast.out) 32 | BADSYN8=1 33 | continue 34 | ;; 35 | badsyntax_future9.py.ast.out) 36 | BADSYN9=1 37 | continue 38 | ;; 39 | py3_test_grammar.py.ast.out) 40 | PY3TEST=1 41 | continue 42 | ;; 43 | esac 44 | FAILURE=1 45 | echo "Failed to parse $result" 46 | done; 47 | 48 | SUCCESS=1 49 | if [ $FAILURE = 0 ] && [ $BADSYN3 = 1 ] && [ $BADSYN8 = 1 ] && [ $BADSYN9 = 1 ] && [ $PY3TEST = 1 ]; 50 | then 51 | SUCCESS=0 52 | fi 53 | exit $SUCCESS 54 | -------------------------------------------------------------------------------- /test/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | enable_testing() 2 | file(GLOB PYTHON_SRCS ${CMAKE_CURRENT_SOURCE_DIR}/tests/*.py) 3 | foreach(PYTHON_SRC ${PYTHON_SRCS}) 4 | get_filename_component(BASEFILENAME ${PYTHON_SRC} NAME_WE) 5 | add_test(NAME parser-test_${BASEFILENAME} COMMAND ./parser-test "${PYTHON_SRC}" WORKING_DIRECTORY ${CMAKE_BINARY_DIR}/src) 6 | endforeach() 7 | -------------------------------------------------------------------------------- /test/tests/assignment.py: -------------------------------------------------------------------------------- 1 | a = 1 2 | b.x = {} 3 | c.foo.bar = None 4 | -------------------------------------------------------------------------------- /test/tests/complex.py: -------------------------------------------------------------------------------- 1 | -1.5-3.5e-20j 2 | 0b0011100111000+3.5e-20j 3 | -0b0011100111000+3.5e-20j 4 | -07776+3.5e-20j 5 | -1.5 -3.5e-20j 6 | -07776+3.5e-20j 7 | -4094+3.5e-20j 8 | -30582+3.5e-20j 9 | 1848+3.5e-20j 10 | -1.5-3.5e-20j 11 | 0b0011100111000 + 3.5e-20j 12 | -0b0011100111000 + 3.5e-20j 13 | -07776 + 3.5e-20j 14 | -1.5 -3.5e-20j 15 | -07776 + 3.5e-20j 16 | -4094 + 3.5e-20j 17 | -30582 + 3.5e-20j 18 | 1848 + 3.5e-20j 19 | -------------------------------------------------------------------------------- /test/tests/dict_comp.py: -------------------------------------------------------------------------------- 1 | {x:x*2 for x in y if x is not None} 2 | -------------------------------------------------------------------------------- /test/tests/exec_test.py: -------------------------------------------------------------------------------- 1 | def run(self, cmd, globals=None, locals=None): 2 | if globals is None: 3 | import __main__ 4 | globals = __main__.__dict__ 5 | if locals is None: 6 | locals = globals 7 | self.reset() 8 | sys.settrace(self.trace_dispatch) 9 | if not isinstance(cmd, types.CodeType): 10 | cmd = cmd+'\n' 11 | try: 12 | exec cmd in globals, locals 13 | except BdbQuit: 14 | pass 15 | finally: 16 | self.quitting = 1 17 | sys.settrace(None) 18 | -------------------------------------------------------------------------------- /test/tests/fail_fun.py: -------------------------------------------------------------------------------- 1 | def fun(a, b, k=None, *argc, **kwargs): 2 | pass 3 | 4 | def fun(a, b, k=None, *argc, fail=failure, **kwargs): 5 | pass 6 | -------------------------------------------------------------------------------- /test/tests/fun.py: -------------------------------------------------------------------------------- 1 | def fun(a, b, k=None, *argc, **kwargs): 2 | pass 3 | -------------------------------------------------------------------------------- /test/tests/hello_world.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # 4 | 5 | """ 6 | A "Hello World" example for the pypa parser 7 | """ 8 | import sys 9 | 10 | print >> sys.stdout, "Hello", "World!" 11 | -------------------------------------------------------------------------------- /test/tests/import_test.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # vim:fenc=utf-8 4 | # 5 | # Copyright © 2014 vfeenstr 6 | # 7 | # Distributed under terms of the MIT license. 8 | 9 | import foo as bar 10 | from bar2 import baz as blah 11 | from bar3 import baz 12 | from ... import blahblub as blubblah 13 | from ...something import blahblub as blubblah2 14 | from blu.bla.bli import foo as blublabli 15 | import blu.bla.bli as totally_different 16 | 17 | @property(abc) 18 | @property 19 | def foobar(): 20 | from sys import * 21 | 22 | -------------------------------------------------------------------------------- /test/tests/index_test.py: -------------------------------------------------------------------------------- 1 | def reverse(self): 2 | 'S.reverse() -- reverse *IN PLACE*' 3 | n = len(self) 4 | for i in range(n//2): 5 | self[i], self[n-i-1] = self[n-i-1], self[i] 6 | 7 | -------------------------------------------------------------------------------- /test/tests/list.py: -------------------------------------------------------------------------------- 1 | [1, 2, 3, 4, 5, 6] 2 | ['a', 'b', 'c', 'd', 'e'] 3 | [None, None, None, None, None] 4 | -------------------------------------------------------------------------------- /test/tests/list_comp.py: -------------------------------------------------------------------------------- 1 | y = range(5) 2 | [x for x in y if x is not None] 3 | -------------------------------------------------------------------------------- /test/tests/list_comp2.py: -------------------------------------------------------------------------------- 1 | nums = [1, 2, 3, 4, 5] 2 | strs = ["Apple", "Banana", "Coconut"] 3 | spcs = [" Apple", " Banana ", "Coco nut "] 4 | 5 | print [(i, c) for i in nums for s in strs if len(s) % 2 != 0 for c in s] 6 | -------------------------------------------------------------------------------- /test/tests/literals.py: -------------------------------------------------------------------------------- 1 | print {'s', 'e', 't'} 2 | print ['l', 'i', 's', 't'] 3 | print ('t', 'u', 'p', 'l', 'e') 4 | print {'d': '1', 'i': '2', 'c': '3', 't': '4'} 5 | print 123456789 6 | print 0x75BCD15 7 | print 0b111010110111100110100010101 8 | print 0726746425 9 | print 0o726746425 10 | print "123456789" "foobar" "foobar" "foobar" 11 | print True 12 | print False 13 | print None 14 | -------------------------------------------------------------------------------- /test/tests/no_newline.py: -------------------------------------------------------------------------------- 1 | import os 2 | import os.path 3 | 4 | 5 | def cs_path_exists(fspath): 6 | if not os.path.exists(fspath): 7 | return False 8 | # make absolute so we always have a directory 9 | abspath = os.path.abspath(fspath) 10 | directory, filename = os.path.split(abspath) 11 | return filename in os.listdir(directory) -------------------------------------------------------------------------------- /test/tests/no_newline2.py: -------------------------------------------------------------------------------- 1 | from .core import where -------------------------------------------------------------------------------- /test/tests/number_integer.py: -------------------------------------------------------------------------------- 1 | 123456789 2 | -------------------------------------------------------------------------------- /test/tests/pass.py: -------------------------------------------------------------------------------- 1 | pass 2 | -------------------------------------------------------------------------------- /test/tests/test.py: -------------------------------------------------------------------------------- 1 | def test(a, b, c, *args, **kwargs): 2 | with 1.22: 3 | print 1.2 4 | print {'s', 'e', 't'} 5 | print ['l', 'i', 's', 't'] 6 | print ('t', 'u', 'p', 'l', 'e') 7 | print {'d': '1', 'i': '2', 'c': '3', 't': '4'} 8 | print [x for x in []] 9 | print [x for x in [] if x is not None] 10 | print 123456789 11 | print 0x75BCD15 12 | print 0b111010110111100110100010101 13 | print 0726746425 14 | print 0o726746425 15 | print "123456789" 16 | pass 17 | pass 18 | yield 1 19 | a(a, b, *args, c=d, x=y, **kwargs) 20 | subscript[idx] 21 | slicesimpl[slow:supper] 22 | sliceext[elow:eupper:estep] 23 | 24 | 25 | class Foobar1: 26 | pass 27 | 28 | class Foobar2(SomeBase): 29 | pass 30 | 31 | class Foobar3(Base1, Base2): 32 | pass 33 | 34 | class Foobar4(Base1, Base2): 35 | def __init__(self, *args, **kwargs): 36 | self.arg = args 37 | -------------------------------------------------------------------------------- /tools/astdump.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | # -*- coding: utf-8 -*- 3 | # vim:fenc=utf-8 4 | 5 | import ast 6 | PADDING = ' ' 7 | 8 | 9 | def member_filter(item): 10 | if item in ('col_offset', 'lineno'): 11 | return False 12 | return not item.startswith("_") 13 | 14 | 15 | def print_ast(a, pad=''): 16 | if isinstance(a, (list, tuple)): 17 | print '[' 18 | for i in a: 19 | print_ast(i, pad + PADDING) 20 | print '%s]' % pad 21 | elif isinstance(a, (basestring, int, float, long, complex)) or a is None: 22 | print a 23 | elif isinstance(a, (ast.Load, ast.Store, ast.AugLoad, ast.AugStore, 24 | ast.Param, ast.Del)): 25 | print a.__class__.__name__ 26 | else: 27 | print '\n%s[%s]' % (pad, a.__class__.__name__) 28 | items = [i for i in dir(a) if member_filter(i)] 29 | for i in items: 30 | print '%s%s- %s:' % (pad, PADDING, i), 31 | print_ast(getattr(a, i), pad + PADDING) 32 | # else: 33 | # print a.__class__.__name__, [i for i in dir(a) if member_filter(i)] 34 | 35 | 36 | if __name__ == '__main__': 37 | import sys 38 | print_ast(ast.parse(open(sys.argv[1], 'r').read())) 39 | --------------------------------------------------------------------------------