├── .codeclimate.yml ├── .gitignore ├── .travis.yml ├── CHANGELOG.txt ├── LICENSE.txt ├── MANIFEST.in ├── README.md ├── TODO.txt ├── cyther ├── __init__.py ├── __main__.py ├── aberdeen.py ├── arguments.py ├── commands.py ├── configuration.py ├── core.py ├── definitions.py ├── direct.py ├── extractor.py ├── instructions.py ├── launcher.py ├── parser.py ├── pathway.py ├── processing.py ├── project.py ├── searcher.py ├── system.py ├── test.py ├── test │ ├── example_file.pyx │ ├── makefile │ ├── random.c │ ├── random.h │ ├── randomtreetest.c │ ├── readme.txt │ ├── tree.c │ └── tree.h └── tools.py ├── setup.cfg └── setup.py /.codeclimate.yml: -------------------------------------------------------------------------------- 1 | engines: 2 | pep8: 3 | enabled: true 4 | 5 | radon: 6 | enabled: true 7 | 8 | duplication: 9 | enabled: true 10 | config: 11 | languages: 12 | python: 13 | python_version: 3 14 | 15 | ratings: 16 | paths: 17 | - "cyther/**" 18 | - "**.py" -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Custom 2 | __cythercache__/ 3 | .idea/ 4 | .cyther 5 | /DO_NOT_DISTRIBUTE.txt 6 | /pypi.py 7 | /import_cyther.py 8 | /script.py 9 | /example_file.c 10 | /example_file.html 11 | /example_file.lis 12 | /xubuntu_testing_instructions.txt 13 | /packages_diagram.png 14 | 15 | # Byte-compiled / optimized / DLL files 16 | __pycache__/ 17 | *.py[cod] 18 | *$py.class 19 | 20 | # C extensions 21 | *.so 22 | *.dll 23 | *.pyd 24 | 25 | # Distribution / packaging 26 | .Python 27 | env/ 28 | build/ 29 | develop-eggs/ 30 | dist/ 31 | downloads/ 32 | eggs/ 33 | .eggs/ 34 | lib/ 35 | lib64/ 36 | parts/ 37 | sdist/ 38 | var/ 39 | *.egg-info/ 40 | .installed.cfg 41 | *.egg 42 | 43 | # PyInstaller 44 | # Usually these files are written by a python script from a template 45 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 46 | *.manifest 47 | *.spec 48 | 49 | # Installer logs 50 | pip-log.txt 51 | pip-delete-this-directory.txt 52 | 53 | # Unit test / coverage reports 54 | htmlcov/ 55 | .tox/ 56 | .coverage 57 | .coverage.* 58 | .cache 59 | nosetests.xml 60 | coverage.xml 61 | *,cover 62 | .hypothesis/ 63 | 64 | # Sphinx documentation 65 | docs/_build/ 66 | 67 | # PyBuilder 68 | target/ 69 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | os: 2 | - linux 3 | 4 | language: python 5 | 6 | python: 7 | - 3.6 8 | - 3.5 9 | - 3.4 10 | - 3.3 11 | - nightly 12 | 13 | install: 14 | - pip install . 15 | 16 | script: 17 | - cyther test utilities 18 | -------------------------------------------------------------------------------- /CHANGELOG.txt: -------------------------------------------------------------------------------- 1 | 0.0.0 - 0.4.0 2 | Private development, not documented. 3 | 4 | 0.4.1 5 | 6 | Creation of everything needed to push to PYPI, and a few bug fixes. 7 | 8 | 0.4.2 9 | Reformatting and renaming of key functions to make everything more understandable 10 | A few bug fixes involving directories to include 11 | Started using setuptools if possible 12 | Fixed and elaborated on setup.py 13 | 14 | 0.4.3 15 | Recreation of the file / directory finding system to make sure that it works in virtualenv 16 | Made the __cytherinfo__ give more helpful information 17 | Fixed the assumptions 18 | Wrote a testing script 19 | Encapsulated and reordered cyther.py 'CONSTANTS' 20 | 21 | 0.5.0 22 | Creation of a distutils-like function to find the include and libs directories in almost any installation 23 | Now passes builds on Travis 24 | Cyther is now in beta version 25 | Took out auto-deployment 26 | Changed the where and which function to be the crawl and where functions, respectively 27 | 28 | 0.5.1 29 | Fixed many of the unordered badges 30 | Updated the operating system finder to be safer 31 | Fixed the hardcoding of the python environment structure to not be primary 32 | 33 | 0.5.2 34 | Transferred the include_string and runtime_string to be partially defined outside the function 35 | 36 | 0.5.3 37 | Added auto deployment 38 | Added and implemented the run function, to extract code from a file and automatically run it 39 | 40 | 0.6.1 41 | Gave travis a secure password in the yaml file 42 | Fixed a few bugs with the run() function 43 | Added a readme generator system to have the readme.md and readme.rst 44 | Changed the '-t' option to '-s' 45 | Added --execute and --timer to the commandline argument system 46 | Made timer and execute be '-t', and '-x' respectively 47 | Did some README updates 48 | Added to the testing script significantly, and got travis to run that AND nosetests 49 | Properly encapsulated the 'cytherize' and 'core' functions 50 | Added the -X and -T options, meant for super powerful debugging and building 51 | Gave the option to print the commands passed off only if wanted to 52 | Formatted the '-h' text to be even more helpful! 53 | Deleted the current useless documentation, adding new docs soon 54 | Did some serious encapsulation and reorganization of entire compilation system 55 | Fixed the error management system to make sure it is reliable 56 | All communications from cyther to the user are located in a single place, and done through stdout 57 | 58 | 0.6.2 59 | Fixed an encoding bug 60 | Deleted the test_cyther null command in the travis yaml 61 | Took README.md out of the .gitignore 62 | Fixed the distribution settings 63 | Implemented some PEP8 rules 64 | Added the __all__ attribute to cyther.py 65 | Reordered all functions to make more sense 66 | Properly encapsulated the 'script' code 67 | Finally added proper documentation! 68 | 69 | 0.7.0 70 | Dealt with pypi nonsense 71 | Took out auto deployment 72 | 73 | 0.7.1 74 | Updated the README to include the current cytherize -h text 75 | Added linereader to the packages included 76 | 77 | 0.8.0 78 | A bunch of bug fixes and internal problem solving: 79 | 80 | Cut out unnecssary junk in the help text (gcc configurations) 81 | Fixed some PEP8 issues 82 | Split cyther.py into multiple modules to make everything easier to understand 83 | Updated setup.cfg to include the correct README 84 | Implemented entry_points on setup tools to make an executable script instead of doing it manually 85 | Temporarily disabled the -p presets system 86 | Cyther now works in three stages: cython, compile, link 87 | The help text now prints the directories for 'include' and 'runtime' used by cyther 88 | Updated test_cyther to correctly call cytherize 89 | Reorganized the structure of the package 90 | Reworked the 'package_data' 91 | Cyther is now being locally tested on xubuntu 92 | Stopped deploying the rst readme 93 | Included cython in the install_requirements 94 | Deleted the 'super-flags'. Probably not a very good practice... 95 | Changed the signature of argparser. Now vastly superior to the previous model. 96 | Reformatted the arguments.py file to be easier to read 97 | Implemented an 'actions' file for the arguments to call directly! 98 | Updated README badges to include links to build utilities and display more useful stuff 99 | Made the error recognition system better when calling 'cyther.tools.call' 100 | TODO (not yet done) 101 | Implemented a 'makefile' system. This is not the primary method of compilation. 102 | Instead of directly calling commands, it will make a 'makefile', for later modification if desired -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2016 Nicholas C. Pandolfi ALL RIGHTS RESERVED (MIT) 2 | 3 | Permission is hereby granted, free of charge, to any person 4 | obtaining a copy of this software and associated documentation 5 | files (the "Software"), to deal in the Software without 6 | restriction, including without limitation the rights to use, 7 | copy, modify, merge, publish, distribute, sublicense, and/or 8 | sell copies of the Software, and to permit persons to whom the 9 | Software is furnished to do so, subject to the following conditions: 10 | 11 | The above copyright notice and this permission notice shall 12 | be included in all copies or substantial portions of the Software. 13 | 14 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 15 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 16 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 17 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 18 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 19 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 20 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 21 | OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | include *.py 2 | include *.txt 3 | include *.md 4 | include *.yml 5 | include cyther/test/* 6 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | 2 | # Cyther: The Cross-Platform Cython/Python/C Auto-Compiler 3 | 4 | ## Important 5 | 6 | Cyther is currently under temporary renovation and this README does not apply just yet. It represents the future `0.8.0` version, offering true cross-platform compatibility and the new and improved intermediate `makefile` system. I am making the README first, then developing the code based around that. Why you ask? Because I have no idea what I'm doing. That's why. 7 | 8 | [![Repository](https://badge.fury.io/py/cyther.svg)](https://pypi.python.org/pypi/Cyther) 9 | [![Github](https://img.shields.io/github/stars/nickpandolfi/cyther.svg?style=social&label=Star)](https://github.com/nickpandolfi/Cyther) 10 | [![Downloads](https://img.shields.io/github/downloads/nickpandolfi/Cyther/total.svg)](https://github.com/nickpandolfi/Cyther/releases) 11 | 12 | [![Travis](https://secure.travis-ci.org/nickpandolfi/Cyther.png)](https://travis-ci.org/nickpandolfi/Cyther) 13 | ![Status](https://img.shields.io/badge/Status-Alpha-orange.svg?style=flat) 14 | [![GPA](https://img.shields.io/codeclimate/github/nickpandolfi/Cyther.svg)](https://codeclimate.com/github/nickpandolfi/Cyther) 15 | [![Issues](https://img.shields.io/codeclimate/issues/github/nickpandolfi/Cyther.svg)](https://codeclimate.com/github/nickpandolfi/Cyther/issues) 16 | 17 | ![Versions](https://img.shields.io/pypi/pyversions/cyther.svg?maxAge=2592000) 18 | ![Implementation](https://img.shields.io/pypi/implementation/cyther.svg?maxAge=2592000) 19 | ![License](https://img.shields.io/pypi/l/cyther.svg?maxAge=2592000) 20 | ![Format](https://img.shields.io/pypi/format/cyther.svg?maxAge=2592000) 21 | 22 | ## What is Cyther? 23 | 24 | Cyther is a tool used to elegantly compile C and Cython *(and Python for that matter)*. Cyther is what I'd like to call an 'auto-compiler', because it makes the compilation process significantly easier and less cryptic than Python's standard `distutils.build_ext` does. Very similar to GNU's `make` utility, Cyther offers a three step system to compilation: 25 | 26 | ###1. Configuration 27 | 28 | Cyther will guide you through the configuration of your own development environment. It will check to make sure that all of it's dependencies are installed correctly, and are *automatically* accessible. 29 | 30 | ###2. Initialization 31 | 32 | Cyther will construct a single file that holds the commands that it will pass-off to the actual underlying compilers to handle. It is here where you can see what is happening to compile your source code. This can be used as an educational tool. 33 | 34 | ###3. Compilation 35 | 36 | Cyther will then proceed to execute that exact set of commands. This phase also takes several very intelligent arguments to dynamically run and time the code as if the compiled code were 'interpretable'. This process is also extremely fast with very minimal overhead. 37 | 38 | ## A few reasons why you may want to use Cyther 39 | 40 | 1. Cyther's method for finding the 'include' and 'runtime' libraries is very sophisticated. Combining regex with a `which` like utility, Cyther will systematically search for the correct directories to use. 41 | 2. The commands used to set your project up and then compile your project are completely different, allowing for better bug fixing and quicker environment diagnostics. 42 | 3. All the errors ever produced by Cyther are helpful, tell you what to fix, and exactly how to do it. 43 | 4. Cyther is windows friendly 44 | 5. You do not have to use a full setup.py system to compile one file, or ten, or even a whole project. 45 | 6. You will never, *ever*, see the infamous `vcvarsall.bat not found` error. Not only is it cryptic, but it has many different underlying causes, meaning the user is very lucky if they've solved it. 46 | 7. Cyther can be used from the Python level, as it has a full API and can be imported. 47 | 48 | ### What Cyther is NOT 49 | 50 | * Cyther is not a replacement for distutils' or setuptools' `build_ext` systems 51 | * Cyther is not meant to be used in a setup.py script for developers to use it to install source files on a user's machine 52 | * Cyther should not be used for important / critical pieces of software, ***yet*** 53 | 54 | ## How to use 55 | 56 | Cyther is extremely easy to use. One can call ``cyther`` from the command line, or import `cyther` and 57 | call `cyther.core` from the module level. 58 | 59 | from cyther import core 60 | core('example_file.pyx') 61 | 62 | same can be done with: 63 | 64 | $ cytherize example_file.pyx 65 | 66 | And as expected, one can call `$ cyther -h` for all the argument help they need. See below. 67 | 68 | ### Raw examples (to be better explained later) 69 | 70 | `$ cyther build example.pyx` 71 | Constructs an importable dll (pyd|so) of name `example.pyd` or `example.so`, depending on your machine 72 | 73 | `$ cyther build example.pyx>{c}` 74 | Compiles `example.pyx` into the c file of the same name: `example.c` 75 | 76 | `$ cyther build example.pyx>example.c` 77 | This does the same thing as the previous command, just more clear 78 | 79 | `$ cyther build example.pyx>{@output_name}` 80 | Creates a dll and calls it `output_name.pyd` 81 | 82 | `$ cyther build (example.o)big_program.py>{pyd} example.pyx>{o}` 83 | Compile `example.pyx` into an object file of name `example.o` 84 | Use this object file and link to big_program.o once you get there 85 | Then make dll by linking 86 | 87 | `$ cyther build (?example.o)big_program.py>{pyd}` 88 | If you were given the example.o file you would use this command 89 | 90 | `$ cyther build example.pyx>example.o{^locally}` 91 | `$ cyther build example.pyx>{o}{^locally}` 92 | `$ cyther build example.pyx>{o,^locally}` 93 | These three commands do exactly the same thing 94 | The last one makes the most sense 95 | 96 | Notes: 97 | 98 | Dependent files can be in any position relative to the user: 99 | 100 | >The structure of the {} operator: 101 | > {type,@name,^where,/directory} 102 | > where: local, cache, obfuscate 103 | > /directory could use both forward-slashes or backslashes 104 | 105 | You can also write something like this to execute tests directly after the build procedure 106 | 107 | # example_file.pyx 108 | from math import sqrt 109 | 110 | cdef int triangular(int n): 111 | cdef: 112 | double q 113 | int r 114 | q = (n * (n + 1)) / 2 115 | r = int(q) 116 | return r 117 | 118 | def inverse_triangular(n): 119 | x = (sqrt(8 * n + 1) - 1) / 2 120 | n = int(x) 121 | if x - n > 0: 122 | return False 123 | return int(x) 124 | 125 | ''' 126 | @Cyther 127 | a = ''.join([str(x) for x in range(10)]) 128 | print(a) 129 | ''' 130 | 131 | The `@Cyther` line tells Cyther that it should extract the code after it in 132 | the single quoted multi-line string and execute it if the build passed. One 133 | can also tell Cyther to time the `@Cyther` code, returning an IPython-esque 134 | timing message. Here are a few examples of how to use these features. 135 | 136 | The wonderful `-x` option, and its output to `stdout` 137 | 138 | $ cytherize example_file.pyx -x 139 | 0123456789 140 | 141 | The `-t` option is also super helpful 142 | 143 | $ cytherize example_file.pyx -t 144 | 10000 loops, best of 3: (2.94e-06) sec per loop 145 | 146 | ## The help text of Cyther 147 | 148 | $ cyther --help 149 | 150 | ### Assumptions Cyther makes about your system 151 | 152 | Cyther, like everything else, isn't perfect. Currently for it to function 153 | properly, it needs to make a few assumptions of your development environment. 154 | All these quirks are listed below. I strongly recommend that you look them 155 | over before using Cyther. In the near future I hope to make Cyther as polished 156 | as possible, and bring the list of assumptions listed below to a minimum. 157 | Future plans can be found in the to-do file I have included with the project. 158 | 159 | 1. gcc is installed, and accessible from the terminal 160 | 2. Your Python version supports `shutil.which` 161 | 3. Your environment path variable is able to be found by `shutil.which` 162 | 4. `distutils` is able to find the Python runtime static library 163 | (usually `libpythonXY.a` or `libpythonXY.so`) 164 | 5. Windows will support gcc compiled C code 165 | 166 | ### The environment used to develop Cyther 167 | 168 | 1. Windows 10, 64 bit 169 | 2. I use the latest Python 3 version available 170 | 3. Linux platforms tested using Travis (cloud based) 171 | 4. Ubuntu platforms tested using a local Xubuntu machine 172 | 5. GCC version 4.9.3 173 | 174 | ### Miscellaneous 175 | 176 | #### Backstory 177 | 178 | Python's standard distutils library is weakly defined when it comes to it's 179 | `build_ext` functionality *(big claim? Look at the code for yourself)*. 180 | On windows specifically, there are many errors that never seem to be addressed 181 | regarding finding and using Microsoft Visual C++ redistributables. 182 | StackOverflow is littered with these kinds of errors, piling duplicate 183 | question on duplicate question. In many instances, as I believe I had 184 | mentioned before, there are many underlying causes to these errors, and the 185 | individual errors tell you absolutely nothing about the problem. Getting to 186 | the bottom of it was what inspired me to write Cyther. 187 | 188 | Cyther is my humble attempt* at bridging this gap, and still offering a piece 189 | of software that a beginner to Python can use. I intend for Cyther to be used 190 | by people who simply want to compile a set of files for their own use. Cyther 191 | explicitly avoids distutils' esoteric compilation system. 192 | *(I know, I really live on the edge huh?)* 193 | 194 | #### Contact + Reporting Info 195 | 196 | If you notice any bugs or peculiarities, please report them to our bug 197 | tracker, it will help us out a lot! 198 | 199 | https://github.com/nickpandolfi/Cyther/issues 200 | 201 | If you have any questions or concerns, or even any suggestions, 202 | don't hesitate to email me at: 203 | 204 | npandolfi@wpi.edu 205 | 206 | *Happy compiling! - Nick* 207 | 208 | ###### *Choking hazard. Small parts. Not for children under 3 years of age.* 209 | -------------------------------------------------------------------------------- /TODO.txt: -------------------------------------------------------------------------------- 1 | 2 | TO-DO (before next version) 3 | 4 | NON-CRITICAL 5 | Figure out how to do a github release! 6 | Installation instructions into README 7 | Fix the help text in the readme to not have 'Nothing here yet' 8 | Get 'where' to find libpython!! (Polymorphesize this function to be awesome) 9 | where (identifier, start=None, isexec, isfile, isdir) 10 | 11 | CRITICAL 12 | GENERAL 13 | Look how Cython builds the commands on xubuntu and figure out what you are missing 14 | What is the use of the vcruntime140.dll on windows installations? Does it need to be included? 15 | 16 | ORDERED 17 | Figure out how to minimally represent an instruction (what is necessary once extraneous info is processed??) 18 | Make sure that an 'Instruction' instance has enough information in the baseline info to work with dependencies and other instructions 19 | Finish setDefaults() of Instruction once Instruction baseline info is known 20 | Now Instruction is done, work on InstructionManager 21 | Integrate the generateBatches command into it 22 | What do we go to after this is done? Do we compile commands then and there?? 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | MAINTAIN 33 | Keep __all__ up to date with all the useful functions 34 | Have an __all__ for each module 35 | Privatize all objects not in __all__ 36 | Comment and properly docstring everything 37 | Keep the -h following all the other common -h practices 38 | Make sure that the -h is up to date, including all the version info of all the external features (gcc/Cython) 39 | Make sure that you constantly search for the assumptions cyther makes 40 | Make my getIncludeAndRuntime() function is up to date with build_ext in distutils 41 | Any name cool badges to add? Make sure you got all the recent and useful ones (http://shields.io/) 42 | Make sure the test file is updated to include the latest features 43 | 44 | 45 | FUTURE UPDATES 46 | Make the 'make_config' utilities not overwrite the config file if found, or ask smartly instead... 47 | Make utilities to check if the fields in a config file are valid before it is fully loaded and used. 48 | Make 'path' add a pathsep at the end of the path if it is a directory. Safe? 49 | Figure out how to include a C file AS A DATA FILE 50 | Travis OSX tests are not working. It seems to be their issue. Keep an eye on it. 51 | Make CytherError only apply if the error was the user's fault 52 | Where can we find a reliable source of code in build_ext (getIncludeandRuntime()) 53 | Have cyther's help text print its version! (Its needs to know its own version) 54 | Update Cyther's github page to be a full tutorial (make readme readable by formatting) 55 | Theres an odd bug where "INSTALL_REQUIRES = ['cython']" will install the non-latest version of cython. 56 | Find and follow as many conventions as possible in the bookmarks set 57 | Make sure a new window doesnt get spawned on the subprocess calls (on some systems it does, some it doesnt) 58 | Fix this stupid line: platform.platform().strip().lower().startswith('windows') 59 | Incorporate the format of '----- >> watch stats << ------' into watch 60 | Get cyther to modify distutils imported at runtime to compile everything instead of distutils 61 | Make 'path(root='.')' work 62 | 63 | Q) Cache the results of 'find' when compiling without a config file? 64 | Q) Include an entry point that sits outside of 'cyther/' that is a simple argparse starter (called cyther_manual.py or something) 65 | Q) What to do about type checking in non-user functions! 66 | Q) Can I use the Python source code from PSF to literally figure out where the include/runtime dirs were installed? 67 | Q) Even bother getting Cyther to work with python 2 or pypy(3)? 68 | Q) Whats up with pypi version badge doing '?.?.?' 69 | Q) Can ar recompile static libraries? (http://www.linux.org/threads/gnu-binutils.6544/) 70 | Q) Any other commands to `entry_points` other than `console-script`? 71 | Q) What do the platform names even look like? Is their structure reliable? Does it need to be? 72 | Q) Is it necessary to make sure that os.pardir will never be a problem?? 73 | Q) What is the difference between os.name and sys.platform and platform.platform? 74 | Q) Can error info be sent automatically in bytes? 75 | Q) C interpreter for cyther to check if a file is C? Probably not... 76 | 77 | 78 | BEFORE DEPLOY (for me) 79 | Make sure that the version specified makes sense (major.minor.patch) 80 | Make sure that the README reflects the new interface changes 81 | Make sure that the CHANGELOG is up to date and includes the current version change 82 | Delete the comments that are no longer needed. Any junk or temporary commented code should be extracted 83 | Update testing script to cover any new features 84 | 85 | 86 | BIG QUESTIONS 87 | Is it useful at all to always back up the previous compile? 88 | Erase Cyther's usage of distutils entirely? 89 | What about packaging the libraries with cyther?? Is cross compiling possible / a good idea? 90 | Find the best format for the docstrings that is the most universal (What does sphinx use?) 91 | 92 | SORT THESE CHANGES 93 | Have the CYTHER_CONFIG_FILE hold the paths to the compiler executables! 94 | Then, it may be ok to not have gcc in the path, and thus a seperate and private gcc can be used 95 | So for this ^ we need a massive function to search through the drive as far as possible and implement regex patterns 96 | BUT HOW TO THE INDIVIDUAL CYTHER OPTIONS FIT INTO THE MAKEFILE SYSTEM? 97 | Make something like an 'instance' object to create independantly of core, THEN pass to core 98 | What it will do it be an object holding the attribute values of the different arg_parse flags. 99 | Get the arguments of run(timer=True) to work properly, or be automatic, like IPython 100 | Instead of the include option, have the system automatically detect the python {}.get_include() modules 101 | Say the search for the directories necessary for the runtime compilation fails, and returns nothing. Then run the 102 | crawl algorithm over the whole DRIVE to find a lib file that matches the REGEX definition 103 | Make cyther able to use the microsoft compiler cl; to mesh well with the windows system 104 | However, we need a powerful searching algorithm to find all the correct executables... 105 | Make the example_file.pyx a prototype for the SuperObject system. Implementing live testing of it? 106 | DLLTOOL regex algorithm launched in a separate thread 107 | -------------------------------------------------------------------------------- /cyther/__init__.py: -------------------------------------------------------------------------------- 1 | # I'm just a small-town (girl) initialization file 2 | 3 | # TODO RE-ENABLE! 4 | # from .processing import core, CytherError, run 5 | # from .core import info, configure, test, setup, make, clean, purge 6 | 7 | 8 | try: 9 | from shutil import which 10 | except ImportError: 11 | raise ValueError("The current version of Python doesn't support the" 12 | "function 'which', normally located in shutil") 13 | 14 | 15 | __author__ = "Nicholas C. Pandolfi" 16 | 17 | __copyright__ = "Copyright (c) 2016 Nicholas C. Pandolfi" 18 | 19 | __credits__ = "Stack Exchange" 20 | 21 | __license__ = "MIT" 22 | 23 | __email__ = "npandolfi@wpi.edu" 24 | 25 | __status__ = "Development" 26 | -------------------------------------------------------------------------------- /cyther/__main__.py: -------------------------------------------------------------------------------- 1 | 2 | """ 3 | This module is the console entry point when setup.py was used to install 4 | cyther. This module is also run with the -m option from the terminal 5 | """ 6 | 7 | import sys 8 | from .arguments import parser 9 | 10 | 11 | def main(args=None): 12 | """ 13 | Entry point for cyther-script, generated by setup.py on installation 14 | """ 15 | if args is None: 16 | args = sys.argv[1:] 17 | 18 | if not args: 19 | args = ['-h'] 20 | 21 | namespace = parser.parse_args(args) 22 | entry_function = namespace.func 23 | del namespace.func 24 | kwargs = namespace.__dict__ 25 | return entry_function(**kwargs) 26 | -------------------------------------------------------------------------------- /cyther/aberdeen.py: -------------------------------------------------------------------------------- 1 | 2 | """ 3 | A module that defines the different testing procedures to be used by test.py 4 | """ 5 | 6 | import os 7 | 8 | 9 | def display_resources(): 10 | """ 11 | Displays a resource name and checks that it exists so we know that C code 12 | to test cyther (randomtreetest.c) was included 13 | """ 14 | from .tools import find_resource 15 | print("Resource 'randomtreetest.c': {}" 16 | "".format(find_resource('randomtreetest.c'))) 17 | 18 | 19 | def display_configure(): 20 | """ 21 | Displays (does not test) config data generated by 'configuration.py' 22 | """ 23 | 24 | from .configuration import generate_configurations 25 | print(generate_configurations(save=True)) 26 | 27 | # '/opt/python/3.6.0/include/python3.6m/Python.h' 28 | 29 | 30 | def test_find(): 31 | """ 32 | Tests the 'find' function from cyther.searcher 33 | """ 34 | from .searcher import find 35 | #assert find('') 36 | 37 | 38 | def test_extract(): 39 | """ 40 | Tests some extraction procedures to make sure they return the correct 41 | amounts of values, and nothing when appropriate 42 | """ 43 | 44 | from .extractor import extract, extractAtCyther, extractVersion, \ 45 | NONE, MULTIPLE 46 | 47 | string = \ 48 | """ 49 | test test test 50 | 51 | This is a word 52 | ''' 53 | @cyther Hello 54 | ''' 55 | #@Cyther import penguins 56 | version 3.4.5.100 57 | 58 | Hi 59 | """ 60 | 61 | assert extract('(?Ptest)', string) == ['test', 'test', 'test'] 62 | assert extract('(?Ptest)', string, one=True) == MULTIPLE 63 | a = extract('(?Ptest)', string, one=True, condense=True) 64 | assert a == 'test' 65 | assert extract('(?Pflounder)', string, one=True) == NONE 66 | assert extract('(?Pword)', string, one=True) == 'word' 67 | assert extractVersion(string) == '3.4.5.100' 68 | 69 | string2 = \ 70 | """ 71 | Version 3.4 72 | """ 73 | 74 | assert extractVersion(string2) == '3.4' 75 | string2 += "\nversion: 3.5\n" 76 | assert extractVersion(string2) == '?' 77 | 78 | a = extractAtCyther(string) 79 | assert a == 'import penguins\nHello' 80 | 81 | 82 | def test_dict_file(): 83 | """ 84 | Tests 'write_dict_to_file' and 'read_dict_from_file' from tools.py 85 | """ 86 | 87 | from .tools import write_dict_to_file, read_dict_from_file 88 | 89 | file_path = os.path.abspath('abcd_test_dbca.txt') 90 | dictionary = {'key1': 'value1', 91 | 'key2': ('value2', 'value3'), 92 | 'key3': 'value4'} 93 | write_dict_to_file(file_path, dictionary) 94 | extracted_dict = read_dict_from_file(file_path) 95 | assert dictionary == extracted_dict 96 | os.remove(file_path) 97 | 98 | 99 | def test_path(): 100 | """ 101 | Tests 'path' function in 'pathway.py' to make sure it's working correctly 102 | """ 103 | 104 | import os 105 | from .pathway import ISFILE, ISDIR, path, get_dir, \ 106 | OverwriteError, normalize 107 | 108 | # Turn sep appending off for testing purposes 109 | import cyther.pathway 110 | cyther.pathway.APPEND_SEP_TO_DIRS = False 111 | 112 | cwd = os.getcwd() 113 | assert path() == os.getcwd() 114 | p1 = path(['one', 'two', 'three']) 115 | p2 = os.path.normpath(os.path.join(cwd, 'one', 'two', 'three')) 116 | assert p1 == p2 117 | 118 | cwd_and_file = os.path.abspath('test.py') 119 | assert normalize(cwd) == (os.path.normpath(cwd), ISDIR) 120 | assert normalize(cwd_and_file) == (os.path.normpath(cwd_and_file), ISFILE) 121 | 122 | user_and_file = os.path.join('~', 'test.py') 123 | expanded_file = os.path.expanduser(user_and_file) 124 | assert normalize(user_and_file) == (expanded_file, ISFILE) 125 | 126 | example_path = os.path.abspath('test.o') 127 | assert path(example_path) == example_path 128 | assert path(example_path, relpath=True) != example_path 129 | 130 | assert path('test.o') != 'test.o' 131 | assert path('test.o') == os.path.abspath('test.o') 132 | assert path('test.o') == path(name='test', ext='o') 133 | 134 | assert path('.tester') == os.path.abspath('.tester') 135 | assert path('.tester') == path(ext='tester') 136 | assert path('.tester') == path(ext='.tester') 137 | assert path('.tester') == path(name='.tester') 138 | assert path('.tester') != path(name='tester') 139 | assert path('.config.yaml') == path(name='.config', ext='yaml') 140 | assert path(os.path.join('p', 'test.o')) == path('test.o', inject='p') 141 | 142 | fake_file_name = 'abcd' * 3 + '.guyfieri' 143 | fake_path = os.path.abspath(fake_file_name) 144 | 145 | try: 146 | path(fake_path, name='is', ext='dumb', inject='nick') 147 | raise AssertionError("This command shouldn't have worked") 148 | except OverwriteError: 149 | pass 150 | 151 | another_fake = os.path.abspath(os.path.join("nick", "says", 152 | "cyther", "is.dumb")) 153 | faker_root = os.path.dirname(os.path.dirname(get_dir(path(another_fake)))) 154 | p1 = path(os.path.join('says', 'cyther'), root=faker_root, name='is.dumb') 155 | assert p1 == another_fake 156 | 157 | assert path('test', ISFILE) == os.path.abspath('test') 158 | example_path = os.path.abspath(os.path.join('a.b.c', 'test.o')) 159 | calculated_path = path('a.b.c', ISDIR, name='test.o') 160 | assert calculated_path == example_path 161 | 162 | 163 | def test_generateBatches(): 164 | """ 165 | Tests functionality of the generateBatches function 166 | """ 167 | 168 | from .tools import generateBatches 169 | 170 | t = { 171 | 'a': ['b'], 172 | 'b': ['c'], 173 | 'c': ['d', 'e', 'f', 'g'], 174 | 'd': ['e', 'g', 'j'], 175 | 'e': ['f'], 176 | 'f': ['i', 'j'], 177 | 'g': ['j'], 178 | 'h': ['i'], 179 | 'i': ['j'], 180 | 'j': ['q'], 181 | } 182 | g = ['q'] 183 | batches = generateBatches(t, g) 184 | assert batches == [{'j'}, {'i', 'g'}, {'f', 'h'}, {'e'}, 185 | {'d'}, {'c'}, {'b'}, {'a'}] 186 | -------------------------------------------------------------------------------- /cyther/arguments.py: -------------------------------------------------------------------------------- 1 | 2 | """ 3 | This module defines how cyther can be used from the command line, and how to 4 | process the arguments passed in. It connects hooks to cyther.core to provide 5 | an entry point that makes sense. 6 | """ 7 | 8 | import argparse 9 | from .core import info, configure, setup, make, clean, purge 10 | from .test import test_all, test_compiler, test_utilities 11 | 12 | 13 | help_info = "Prints the information regarding cyther's installation and " \ 14 | "environment. Returns helpful info on the compilers used and" \ 15 | "all the current build settings, then exits" 16 | help_configure = "A very important command to run once in a while. This will" \ 17 | "trigger cyther's search for the compilers it will use on" \ 18 | "your system" 19 | help_test = "Binary flag to run the included 'test.py' script" 20 | help_setup = "Constructs the standard 'cytherize' file necessary to use" \ 21 | "cyther. Takes an optional 'preset' argument to control the" \ 22 | "style of commands generated and injected into the" \ 23 | "'cytherize' file" 24 | help_make = "Similar to GNU's 'make' command. Runs a file called" \ 25 | "'cytherize' in your local directory. Optionally takes an" \ 26 | "argument of a path to a specific 'cytherize' formatted file" \ 27 | "that may not normally run." 28 | help_clean = "Cleans the current directory of anything not in use. Will" \ 29 | "ask explicit permission for anything to be deleted. Empties" \ 30 | "the '__cythercache__' of independent files. This command is" \ 31 | "similar to GNU's conventional '$make clean' for use with" \ 32 | "makefiles" 33 | help_purge = "Cleans the current directory of EVERYTHING cyther related." \ 34 | "Will ask explicit permission for anything" \ 35 | "to be deleted. Deletes the '__cythercache__'" 36 | 37 | description_text = "Auto compile and build .pyx, .py, or .c files in-place" 38 | formatter = argparse.RawDescriptionHelpFormatter 39 | 40 | # Any others to use? Why did I choose this one? 41 | parser = argparse.ArgumentParser(description=description_text, 42 | formatter_class=formatter) 43 | 44 | 45 | commands = parser.add_subparsers() 46 | 47 | # $$$$$$$$$$ COMMANDS FOR INFO $$$$$$$$$$ 48 | info_parser = commands.add_parser('info', help=help_info) 49 | info_parser.set_defaults(func=info) 50 | # Empty as of now 51 | 52 | 53 | # $$$$$$$$$$ COMMANDS FOR CONFIGURE $$$$$$$$$$ 54 | configure_parser = commands.add_parser('configure', help=help_configure) 55 | configure_parser.set_defaults(func=configure) 56 | # Empty as of now 57 | 58 | 59 | # $$$$$$$$$$ COMMANDS FOR TEST $$$$$$$$$$ 60 | test_parser = commands.add_parser('test', help=help_test) 61 | 62 | help_test_all = "Run all tests Cyther has to offer" 63 | help_test_compiler = "Run tests dealing with Cyther's core functionality" 64 | help_test_utilities = "Run tests dealing with Cyther's tools used to function" 65 | 66 | test_commands = test_parser.add_subparsers() 67 | test_all_parser = test_commands.add_parser('all', help=help_test_all) 68 | test_all_parser.set_defaults(func=lambda: print('yolo')) 69 | 70 | test_compiler_parser = test_commands.add_parser('compiler', 71 | help=help_test_compiler) 72 | test_compiler_parser.set_defaults(func=test_compiler) 73 | 74 | test_utilities_parser = test_commands.add_parser('utilities', 75 | help=help_test_utilities) 76 | test_utilities_parser.set_defaults(func=test_utilities) 77 | 78 | 79 | # $$$$$$$$$$ COMMANDS FOR SETUP $$$$$$$$$$ 80 | setup_parser = commands.add_parser('setup', help=help_setup) 81 | setup_parser.set_defaults(func=setup) 82 | help_filenames = "The Cython source file(s)" 83 | setup_parser.add_argument('filenames', action='store', 84 | nargs='+', help=help_filenames) 85 | help_preset = 'The preset options for using cython and' \ 86 | 'gcc (ninja, beast, minimal, swift)' 87 | setup_parser.add_argument('--preset', action='store', 88 | default='', help=help_preset) 89 | help_output = 'Change the name of the output file,' \ 90 | 'default is basename plus .pyd' 91 | setup_parser.add_argument('--output', action='store', help=help_output) 92 | help_include = 'The names of the python modules that have an include' \ 93 | 'library that needs to be passed to gcc' 94 | setup_parser.add_argument('--include', action='store', 95 | default='', help=help_include) 96 | help_gcc = "Arguments to pass to gcc" 97 | setup_parser.add_argument('--gcc', action='store', nargs='+', 98 | dest='gcc_args', default=[], help=help_gcc) 99 | help_cython = "Arguments to pass to Cython" 100 | setup_parser.add_argument('--cython', action='store', nargs='+', 101 | dest='cython_args', default=[], help=help_cython) 102 | 103 | 104 | # $$$$$$$$$$ COMMANDS FOR MAKE $$$$$$$$$$ 105 | make_parser = commands.add_parser('make', help=help_make) 106 | make_parser.set_defaults(func=make) 107 | help_concise = "Get cyther to NOT print what it is thinking. Only use if" \ 108 | "you like to live on the edge" 109 | make_parser.add_argument('--concise', action='store_true', help=help_concise) 110 | help_local = 'When not flagged, builds in __cythercache__, when flagged,' \ 111 | 'it builds locally in the same directory' 112 | make_parser.add_argument('--local', action='store_true', help=help_local) 113 | help_watch = "When given, cyther will watch the directory with the 't'" \ 114 | "option implied and compile, when necessary, the files given" 115 | make_parser.add_argument('--watch', action='store_true', help=help_watch) 116 | help_error = "Raise a CytherError exception instead of printing out stderr" \ 117 | "when -w is not specified" 118 | make_parser.add_argument('--error', action='store_true', help=help_error) 119 | execution_system = make_parser.add_mutually_exclusive_group() 120 | help_execute = "Run the @Cyther code in multi-line single quoted strings," \ 121 | "and comments" 122 | execution_system.add_argument('--execute', action='store_true', 123 | dest='execute', help=help_execute) 124 | help_timer = "Time the @Cyther code in multi-line single quoted strings," \ 125 | "and comments" 126 | execution_system.add_argument('--timeit', action='store_true', 127 | dest='timer', help=help_timer) 128 | 129 | 130 | # $$$$$$$$$$ COMMANDS FOR CLEAN $$$$$$$$$$ 131 | clean_parser = commands.add_parser('clean', help=help_clean) 132 | clean_parser.set_defaults(func=clean) 133 | # Empty as of now 134 | 135 | 136 | # $$$$$$$$$$ COMMANDS FOR PURGE $$$$$$$$$$ 137 | purge_parser = commands.add_parser('purge', help=help_purge) 138 | purge_parser.set_defaults(func=purge) 139 | # Empty as of now 140 | -------------------------------------------------------------------------------- /cyther/commands.py: -------------------------------------------------------------------------------- 1 | 2 | import argparse 3 | 4 | from .system import * 5 | from .arguments import parser 6 | 7 | 8 | COMMAND_FILENAME = '.cyther' 9 | 10 | 11 | class SimpleCommand: 12 | def __init__(self): 13 | self.runtime_names = [] 14 | self.include_names = [] 15 | self.cython_name = None 16 | self.python_name = None 17 | self.c_name = None 18 | self.o_name = None 19 | self.dll_name = None 20 | 21 | def getCythonFileName(self): 22 | return self.cython_name 23 | 24 | def setCythonFileName(self, obj): 25 | self.cython_name = obj 26 | 27 | def getPythonFileName(self): 28 | return self.python_name 29 | 30 | def setPythonFileName(self, obj): 31 | self.python_name = obj 32 | 33 | def getCName(self): 34 | return self.c_name 35 | 36 | def setCName(self, obj): 37 | self.c_name = obj 38 | 39 | def getOName(self): 40 | return self.o_name 41 | 42 | def setOName(self, obj): 43 | self.o_name = obj 44 | 45 | def getDLLName(self): 46 | return self.dll_name 47 | 48 | def setDLLName(self, obj): 49 | self.dll_name = obj 50 | 51 | def getRuntimeNames(self): 52 | return self.runtime_names 53 | 54 | def setRuntimeNames(self, obj): 55 | self.runtime_names = obj 56 | 57 | def getIncludeNames(self): 58 | return self.include_names 59 | 60 | def setIncludeNames(self, obj): 61 | self.include_names = obj 62 | 63 | 64 | 65 | def furtherArgsProcessing(args): 66 | """ 67 | Converts args, and deals with incongruities that argparse couldn't handle 68 | """ 69 | if isinstance(args, str): 70 | unprocessed = args.strip().split(' ') 71 | if unprocessed[0] == 'cyther': 72 | del unprocessed[0] 73 | args = parser.parse_args(unprocessed).__dict__ 74 | elif isinstance(args, argparse.Namespace): 75 | args = args.__dict__ 76 | elif isinstance(args, dict): 77 | pass 78 | else: 79 | raise CytherError( 80 | "Args must be a instance of str or argparse.Namespace, not '{}'".format( 81 | str(type(args)))) 82 | 83 | if args['watch']: 84 | args['timestamp'] = True 85 | 86 | args['watch_stats'] = {'counter': 0, 'errors': 0, 'compiles': 0, 87 | 'polls': 0} 88 | args['print_args'] = True 89 | 90 | return args 91 | 92 | 93 | def processFiles(args): 94 | """ 95 | Generates and error checks each file's information before the compilation actually starts 96 | """ 97 | to_process = [] 98 | 99 | for filename in args['filenames']: 100 | file = dict() 101 | 102 | if args['include']: 103 | file['include'] = INCLUDE_STRING + ''.join( 104 | ['-I' + item for item in args['include']]) 105 | else: 106 | file['include'] = INCLUDE_STRING 107 | 108 | file['file_path'] = getPath(filename) 109 | file['file_base_name'] = \ 110 | os.path.splitext(os.path.basename(file['file_path']))[0] 111 | file['no_extension'], file['extension'] = os.path.splitext( 112 | file['file_path']) 113 | if file['extension'] not in CYTHONIZABLE_FILE_EXTS: 114 | raise CytherError( 115 | "The file '{}' is not a designated cython file".format( 116 | file['file_path'])) 117 | base_path = os.path.dirname(file['file_path']) 118 | local_build = args['local'] 119 | if not local_build: 120 | cache_name = os.path.join(base_path, '__cythercache__') 121 | os.makedirs(cache_name, exist_ok=True) 122 | file['c_name'] = os.path.join(cache_name, 123 | file['file_base_name']) + '.c' 124 | else: 125 | file['c_name'] = file['no_extension'] + '.c' 126 | file['object_file_name'] = os.path.splitext(file['c_name'])[0] + '.o' 127 | output_name = args['output_name'] 128 | if args['watch']: 129 | file['output_name'] = file['no_extension']+DEFAULT_OUTPUT_EXTENSION 130 | elif output_name: 131 | if os.path.exists(output_name) and os.path.isfile(output_name): 132 | file['output_name'] = output_name 133 | else: 134 | dirname = os.path.dirname(output_name) 135 | if not dirname: 136 | dirname = os.getcwd() 137 | if os.path.exists(dirname): 138 | file['output_name'] = output_name 139 | else: 140 | raise CytherError('The directory specified to write' 141 | 'the output file in does not exist') 142 | else: 143 | file['output_name'] = file['no_extension']+DEFAULT_OUTPUT_EXTENSION 144 | 145 | file['stamp_if_error'] = 0 146 | to_process.append(file) 147 | return to_process 148 | 149 | 150 | """ 151 | Each file gets read and information gets determined 152 | Depencencies are determined 153 | Dependencies are matched and a heirarchy is created 154 | In order from lowest to highest, those commands get created 155 | Put them into cytherize 156 | """ 157 | 158 | 159 | class Commands: 160 | """ 161 | Class to hold the data and methods for processing a manager of instructions 162 | into commands to execute in order to do what should be accomplished 163 | """ 164 | def __init__(self): 165 | self.__unprocessed = [] 166 | 167 | def toFile(self, filename=None): 168 | if not filename: 169 | filename = COMMAND_FILENAME 170 | 171 | commands = self.generateCommands() 172 | string = str() 173 | for command in commands: 174 | string += (' '.join(command) + '\n') 175 | 176 | # TODO What is the best file permission? 177 | with open(filename, 'w+') as file: 178 | chars = file.write(string) 179 | 180 | return chars 181 | 182 | @staticmethod 183 | def fromFile(filename=None): 184 | if not filename: 185 | filename = COMMAND_FILENAME 186 | 187 | with open(filename) as file: 188 | lines = file.readlines() 189 | 190 | output = [] 191 | for line in lines: 192 | output.append(line.split()) 193 | 194 | return output 195 | 196 | def generateCommands(self): 197 | """ 198 | Generate a list of lists of commands from the internal manager object 199 | """ 200 | # 1) Sort the commands 201 | # 2) Return the commands in the form of a list 202 | # 3) This does what makeCommands does right now 203 | pass 204 | 205 | 206 | def makeCommands(file): 207 | """ 208 | Given a high level preset, it will construct the basic args to pass over. 209 | 'ninja', 'beast', 'minimal', 'swift' 210 | """ 211 | commands = [['cython', '-a', '-p', '-o', 212 | file['c_name'], file['file_path']], 213 | ['gcc', '-DNDEBUG', '-g', '-fwrapv', '-O3', '-Wall', '-Wextra', 214 | '-pthread', '-fPIC', '-c', file['include'], '-o', 215 | file['object_file_name'], file['c_name']], 216 | ['gcc', '-g', '-Wall', '-Wextra', '-pthread', '-shared', 217 | RUNTIME_STRING, '-o', file['output_name'], 218 | file['object_file_name'], L_OPTION]] 219 | 220 | return commands 221 | -------------------------------------------------------------------------------- /cyther/configuration.py: -------------------------------------------------------------------------------- 1 | 2 | """ 3 | This module holds the necessary definitions to make and find config files which 4 | hold critical data about where different compile-critical directories exist. 5 | """ 6 | 7 | import os 8 | 9 | from .pathway import path, USER 10 | from .searcher import find 11 | from .extractor import extractMajorMinor 12 | from .definitions import CONFIG_FILE_NAME, VER, DOT_VER 13 | 14 | from .tools import read_dict_from_file, write_dict_to_file,\ 15 | get_input, get_choice 16 | 17 | 18 | class DirectoryError(Exception): 19 | """A custom error used to denote an error with a system of directories""" 20 | should_be_guided = "If this is the case, please DO run in guided mode so" \ 21 | " you can manually choose which one you wish to use" 22 | no_include_dirs = "No include directory found for this version of Python" 23 | multiple_include = "Multiple include directories were found for this " \ 24 | "version of Python. " + should_be_guided 25 | 26 | no_runtime_dirs = "No runtime library search directories were found for" \ 27 | " this version of Python" 28 | multiple_runtime_dirs = "Multiple runtime search directories were found " \ 29 | "for this version of Python. " + should_be_guided 30 | 31 | def __init__(self, *args, **kwargs): 32 | super(DirectoryError, self).__init__(*args, **kwargs) 33 | 34 | 35 | INCLUDE_DIRS_KEY = 'include_search_directory' 36 | RUNTIME_DIRS_KEY = 'runtime_search_directory' 37 | RUNTIME_KEY = 'runtime_libraries' 38 | 39 | 40 | # TODO What about the __file__ attribute 41 | # TODO Make this automatic if 'numpy' is seen in the source code? 42 | def getDirsToInclude(string): 43 | """ 44 | Given a string of module names, it will return the 'include' directories 45 | essential to their compilation as long as the module has the conventional 46 | 'get_include' function. 47 | """ 48 | dirs = [] 49 | a = string.strip() 50 | obj = a.split('-') 51 | 52 | if len(obj) == 1 and obj[0]: 53 | for module in obj: 54 | try: 55 | exec('import {}'.format(module)) 56 | except ImportError: 57 | raise FileNotFoundError("The module '{}' does not" 58 | "exist".format(module)) 59 | try: 60 | dirs.append('-I{}'.format(eval(module).get_include())) 61 | except AttributeError: 62 | print(NOT_NEEDED_MESSAGE.format(module)) 63 | return dirs 64 | 65 | 66 | def purge_configs(): 67 | """ 68 | These will delete any configs found in either the current directory or the 69 | user's home directory 70 | """ 71 | user_config = path(CONFIG_FILE_NAME, root=USER) 72 | inplace_config = path(CONFIG_FILE_NAME) 73 | 74 | if os.path.isfile(user_config): 75 | os.remove(user_config) 76 | 77 | if os.path.isfile(inplace_config): 78 | os.remove(inplace_config) 79 | 80 | 81 | def write_config_file(file_path, data): 82 | """ 83 | Writes a config data structure (dict for now) to the file path specified 84 | """ 85 | write_dict_to_file(file_path, data) 86 | 87 | 88 | def read_config_file(file_path): 89 | """ 90 | Reads a config data structure (dict for now) from the file path specified 91 | """ 92 | return read_dict_from_file(file_path) 93 | 94 | 95 | def find_config_file(): 96 | """ 97 | Returns the path to the config file if found in either the current working 98 | directory, or the user's home directory. If a config file is not found, 99 | the function will return None. 100 | """ 101 | local_config_name = path(CONFIG_FILE_NAME) 102 | if os.path.isfile(local_config_name): 103 | return local_config_name 104 | else: 105 | user_config_name = path(CONFIG_FILE_NAME, root=USER) 106 | if os.path.isfile(user_config_name): 107 | return user_config_name 108 | else: 109 | return None 110 | 111 | 112 | CONFIG_NOT_FOUND = 'cnf' 113 | CONFIG_NOT_VALID = 'cnv' 114 | CONFIG_VALID = 'cv' 115 | 116 | 117 | def get_config(): 118 | """ 119 | Get the config data structure that is supposed to exist in the form: 120 | (Status, Data) 121 | """ 122 | config_path = find_config_file() 123 | if not config_path: 124 | return CONFIG_NOT_FOUND, None 125 | 126 | try: 127 | config_data = read_config_file(config_path) 128 | except Exception as error: 129 | return CONFIG_NOT_VALID, error 130 | 131 | return CONFIG_VALID, config_data 132 | 133 | 134 | BOTH_CONFIGS_EXIST = "Config files exist both in the current directory, and " \ 135 | "the user's directory. Specifying either would result " \ 136 | "in one being overwriten\n" 137 | 138 | USER_CONFIG_EXISTS = "There is a config file in the user's directory, but " \ 139 | "not in the current directory; making a config in " \ 140 | "user's would overwrite the existing one\n" 141 | 142 | INPLACE_CONFIG_EXISTS = "There is a config file in the current directory, " \ 143 | "but not in the user's directory; making a config " \ 144 | "inplace would overwrite the existing one\n" 145 | 146 | NO_CONFIGS_EXIST = "No configs were found, it's safe " \ 147 | "to make the config file anywhere\n" 148 | 149 | COMPLEX_PROMPT = "Where do you want to make the config file?" 150 | 151 | 152 | def _complex_decision(*, guided): 153 | user = path(CONFIG_FILE_NAME, root=USER) 154 | inplace = path(CONFIG_FILE_NAME) 155 | 156 | if os.path.isfile(user): 157 | if os.path.isfile(inplace): 158 | code = 0 159 | default = inplace 160 | else: 161 | code = 1 162 | default = user 163 | else: 164 | if os.path.isfile(inplace): 165 | code = 2 166 | default = inplace 167 | else: 168 | code = 3 169 | default = user 170 | 171 | if guided: 172 | # Code used to preface the situation to the user on the command line 173 | if code == 0: 174 | print(BOTH_CONFIGS_EXIST) 175 | elif code == 1: 176 | print(USER_CONFIG_EXISTS) 177 | elif code == 2: 178 | print(INPLACE_CONFIG_EXISTS) 179 | else: 180 | print(NO_CONFIGS_EXIST) 181 | 182 | # Get the user's response to said situation ^ 183 | response = get_input(COMPLEX_PROMPT,('user', 'inplace', 'default', '')) 184 | 185 | # Decide what to do based on the user's error checked response 186 | if response == 'user': 187 | result = user 188 | elif response == 'inplace': 189 | result = inplace 190 | elif response == 'default': 191 | result = default 192 | else: 193 | exit() 194 | return 195 | else: 196 | result = default 197 | 198 | return result, code 199 | 200 | 201 | SIMPLE_PROMPT = "Do you want to overwrite '{}'?" 202 | 203 | 204 | def _simple_decision(directory, *, guided): 205 | config_name = path(CONFIG_FILE_NAME, root=directory) 206 | if os.path.isfile(config_name): 207 | if guided: 208 | response = get_input(SIMPLE_PROMPT.format(config_name), ('y', 'n')) 209 | if response == 'n': 210 | exit() 211 | return config_name 212 | 213 | 214 | def _make_config_location(*, guided): 215 | current = path(CONFIG_FILE_NAME) 216 | 217 | if os.path.isdir(path(USER)): 218 | if path() == path(USER): 219 | result = _simple_decision(current, guided=guided) 220 | else: 221 | result, code = _complex_decision(guided=guided) 222 | else: 223 | result = _simple_decision(current, guided=guided) 224 | 225 | return result 226 | 227 | 228 | def _check_include_dir_identity(include_path): 229 | a = VER in include_path or DOT_VER in include_path 230 | b = 'include' in include_path 231 | return a and b 232 | 233 | 234 | def _filter_include_dirs(include_dirs): 235 | filtered_dirs = [] 236 | for include_path in include_dirs: 237 | if _check_include_dir_identity(include_path): 238 | filtered_dirs.append(os.path.dirname(include_path)) 239 | 240 | if len(filtered_dirs) > 1: 241 | refiltered_dirs = [] 242 | for include_path in filtered_dirs: 243 | if extractMajorMinor(include_path) == [DOT_VER]: 244 | refiltered_dirs.append(include_path) 245 | filtered_dirs = refiltered_dirs 246 | return filtered_dirs 247 | 248 | 249 | INCLUDE_PROMPT = "Choose one of the listed include directories above (by " \ 250 | "entering the number), or enter nothing to exit the process" 251 | 252 | 253 | def _make_include_dirs(*, guided): 254 | unfiltered_dirs = find('Python.h', content="Py_PYTHON_H") 255 | include_dirs = _filter_include_dirs(unfiltered_dirs) 256 | 257 | if not include_dirs: 258 | raise DirectoryError(DirectoryError.no_include_dirs) 259 | elif len(include_dirs) == 1: 260 | return include_dirs[0] 261 | elif not guided: 262 | # Be indecisive if multiple valid directories are found 263 | raise DirectoryError(DirectoryError.multiple_include) 264 | else: 265 | return get_choice(INCLUDE_PROMPT, include_dirs) 266 | 267 | 268 | def _filter_runtime_dirs(runtime_dirs): 269 | filtered_dirs = [] 270 | for include_path in runtime_dirs: 271 | filtered_dirs.append(os.path.dirname(include_path)) 272 | return filtered_dirs 273 | 274 | 275 | RUNTIME_DIRS_PROMPT = "Choose one of the listed runtime search directories " \ 276 | "above (by entering the number), or enter nothing to " \ 277 | "exit the process" 278 | 279 | 280 | def _make_runtime_dirs(*, guided): 281 | # Dont need to filter on this one 282 | print("Calculated runtime name: '{}'".format(_make_full_runtime())) 283 | unfiltered_dirs = find(_make_full_runtime()) 284 | print("Unfiltered: '{}'".format(unfiltered_dirs)) 285 | runtime_dirs = _filter_runtime_dirs(unfiltered_dirs) 286 | print("Filtered dirs: '{}'".format(runtime_dirs)) 287 | 288 | if not runtime_dirs: 289 | raise DirectoryError(DirectoryError.no_runtime_dirs) 290 | elif len(runtime_dirs) == 1: 291 | return runtime_dirs[0] 292 | elif not guided: 293 | # Be indecisive if multiple valid dirs are found 294 | raise DirectoryError(DirectoryError.multiple_runtime_dirs) 295 | else: 296 | return get_choice(RUNTIME_DIRS_PROMPT, runtime_dirs) 297 | 298 | 299 | def _make_full_runtime(): 300 | return 'lib' + _make_runtime() + '.a' 301 | 302 | 303 | def _make_runtime(): 304 | name = 'python' + VER 305 | return name 306 | 307 | 308 | def make_config_data(*, guided): 309 | """ 310 | Makes the data necessary to construct a functional config file 311 | """ 312 | config_data = {} 313 | config_data[INCLUDE_DIRS_KEY] = _make_include_dirs(guided=guided) 314 | config_data[RUNTIME_DIRS_KEY] = _make_runtime_dirs(guided=guided) 315 | config_data[RUNTIME_KEY] = _make_runtime() 316 | 317 | return config_data 318 | 319 | 320 | def make_config_file(guided=False): 321 | """ 322 | Options: --auto, --guided, --manual 323 | Places for the file: --inplace, --user 324 | """ 325 | config_path = _make_config_location(guided=guided) 326 | 327 | config_data = make_config_data(guided=guided) 328 | 329 | write_config_file(config_path, config_data) 330 | 331 | 332 | # TODO Make errors cascade out to the outside (this is why travis !catching it) 333 | def generate_configurations(*, guided=False, fresh_start=False, save=False): 334 | """ 335 | If a config file is found in the standard locations, it will be loaded and 336 | the config data would be retuned. If not found, then generate the data on 337 | the fly, and return it 338 | """ 339 | 340 | if fresh_start: 341 | purge_configs() 342 | 343 | loaded_status, loaded_data = get_config() 344 | if loaded_status != CONFIG_VALID: 345 | if save: 346 | make_config_file(guided=guided) 347 | status, config_data = get_config() 348 | else: 349 | config_data = make_config_data(guided=guided) 350 | else: 351 | config_data = loaded_data 352 | 353 | return config_data 354 | 355 | 356 | def test(): 357 | from pprint import pprint 358 | pprint(generate_configurations(fresh_start=True, guided=True, save=True)) 359 | -------------------------------------------------------------------------------- /cyther/core.py: -------------------------------------------------------------------------------- 1 | 2 | """ 3 | The heart of Cyther 4 | """ 5 | 6 | from .system import INFO 7 | from .project import purge_project, clean_project 8 | 9 | """ 10 | Each function must have the parameter 'args', even if they do not use it. 11 | This is because they can be called by the parser, and parser always passes a 12 | 'args' variable in. 13 | """ 14 | 15 | 16 | def info(**kwargs): 17 | print(INFO) 18 | 19 | 20 | def configure(**kwargs): 21 | pass 22 | 23 | 24 | def setup(**kwargs): 25 | print(kwargs) 26 | 27 | 28 | def make(**kwargs): 29 | pass 30 | 31 | 32 | def build(**kwargs): 33 | pass 34 | 35 | 36 | def clean(**kwargs): 37 | clean_project() 38 | 39 | 40 | def purge(**kwargs): 41 | purge_project() 42 | -------------------------------------------------------------------------------- /cyther/definitions.py: -------------------------------------------------------------------------------- 1 | 2 | import sys 3 | 4 | CACHE_NAME = "__cythercache__" 5 | CONFIG_FILE_NAME = '.cyther' 6 | 7 | MAJOR = str(sys.version_info.major) 8 | MINOR = str(sys.version_info.minor) 9 | VER = MAJOR + MINOR 10 | DOT_VER = MAJOR + '.' + MINOR 11 | 12 | ############################################################################### 13 | 14 | FINE = 0 15 | ERROR_PASSOFF = 1 16 | SKIPPED_COMPILATION = 1337 17 | WAIT_FOR_FIX = 42 18 | 19 | INTERVAL = .25 20 | 21 | WATCH_STATS_TEMPLATE = "\n......\n" 23 | 24 | SETUP_TEMPLATE = """ 25 | # high level importing to extract whats necessary from your '@cyther' code 26 | import sys 27 | sys.path.insert(0, '{0}') 28 | import {1} 29 | 30 | # bringing everything into your local namespace 31 | extract = ', '.join([name for name in dir({1}) if not name.startswith('__')]) 32 | exec('from {1} import ' + extract) 33 | 34 | # freshening up your namespace 35 | del {1} 36 | del sys.path[0] 37 | del sys 38 | 39 | # this is the end of the setup actions 40 | """ 41 | 42 | TIMER_TEMPLATE = """ 43 | import timeit 44 | 45 | setup_string = '''{0}''' 46 | 47 | code_string = '''{1}''' 48 | 49 | repeat = {2} 50 | number = {3} 51 | precision = {4} 52 | 53 | exec(setup_string) 54 | 55 | timer_obj = timeit.Timer(code_string, 56 | setup="from __main__ import {5}".format(extract)) 57 | 58 | try: 59 | result = min(timer_obj.repeat(repeat, number)) / number 60 | rounded = format(result, '.{5}e'.format(precision)) 61 | print("{5} loops, best of {5}: ({5}) sec per loop".format(number, repeat, 62 | rounded)) 63 | except: 64 | timer_obj.print_exc() 65 | 66 | """ 67 | 68 | MISSING_INCLUDE_DIRS = """ 69 | Cyther could not find any include directories that the 70 | current Python installation was built off of. 71 | 72 | This is eiher a bug or you don't have Python correctly installed. 73 | """ 74 | 75 | MISSING_RUNTIME_DIRS = """ 76 | Cyther could not find any runtime libraries that the 77 | current Python installation was built off of. 78 | 79 | This is eiher a bug or you don't have Python correctly installed. 80 | """ 81 | 82 | NOT_NEEDED_MESSAGE = "Module '{}' does not have to be included," \ 83 | "or has no .get_include() method" 84 | -------------------------------------------------------------------------------- /cyther/direct.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | import sys 4 | 5 | import site 6 | import distutils.sysconfig 7 | import distutils.msvccompiler 8 | 9 | from .tools import CytherError 10 | from .definitions import DOT_VER 11 | 12 | PLATFORM = sys.platform 13 | BASENAME = "python" + DOT_VER 14 | 15 | 16 | def get_dir_contents(directory): 17 | return '\n'.join([','.join(os.listdir(d)) for d in directory]) 18 | 19 | 20 | def display_direct(): 21 | """ 22 | Displays the output of 'get_direct_config', formatted nicely 23 | """ 24 | 25 | include_dirs, runtime_dirs, runtime = get_direct_config() 26 | print("Include Search Dirs: {}".format(include_dirs)) 27 | print("\tContents: {}\n".format(get_dir_contents(include_dirs))) 28 | print("Runtime Search Dirs: {}".format(runtime_dirs)) 29 | print("\tContents: {}\n".format(get_dir_contents(runtime_dirs))) 30 | print("Runtime Libs: '{}'".format(runtime)) 31 | 32 | 33 | def get_direct_config(): 34 | """ 35 | Get the basic config data to compile python, by generating it directly 36 | """ 37 | include_dirs, runtime_dirs = getIncludeAndRuntime() 38 | runtime = BASENAME 39 | return include_dirs, runtime_dirs, runtime 40 | 41 | 42 | def dealWithMissingStaticLib(message): 43 | """ 44 | Deal with the python missing static lib (.a|.so). Currently all this does 45 | is raises a helpful error... 46 | """ 47 | raise CytherError(message) 48 | 49 | 50 | def getIncludeAndRuntime(): 51 | """ 52 | A function from distutils' build_ext.py that was updated and changed 53 | to ACTUALLY WORK 54 | """ 55 | include_dirs, library_dirs = [], [] 56 | 57 | py_include = distutils.sysconfig.get_python_inc() 58 | plat_py_include = distutils.sysconfig.get_python_inc(plat_specific=1) 59 | 60 | include_dirs.append(py_include) 61 | if plat_py_include != py_include: 62 | include_dirs.append(plat_py_include) 63 | 64 | if os.name == 'nt': 65 | library_dirs.append(os.path.join(sys.exec_prefix, 'libs')) 66 | include_dirs.append(os.path.join(sys.exec_prefix, 'PC')) 67 | 68 | MSVC_VERSION = int(distutils.msvccompiler.get_build_version()) 69 | if MSVC_VERSION == 14: 70 | library_dirs.append(os.path.join(sys.exec_prefix, 'PC', 'VS14', 71 | 'win32release')) 72 | elif MSVC_VERSION == 9: 73 | suffix = '' if PLATFORM == 'win32' else PLATFORM[4:] 74 | new_lib = os.path.join(sys.exec_prefix, 'PCbuild') 75 | if suffix: 76 | new_lib = os.path.join(new_lib, suffix) 77 | library_dirs.append(new_lib) 78 | elif MSVC_VERSION == 8: 79 | library_dirs.append(os.path.join(sys.exec_prefix, 'PC', 'VS8.0', 80 | 'win32release')) 81 | elif MSVC_VERSION == 7: 82 | library_dirs.append(os.path.join(sys.exec_prefix, 'PC', 'VS7.1')) 83 | else: 84 | library_dirs.append(os.path.join(sys.exec_prefix, 'PC', 'VC6')) 85 | 86 | if os.name == 'os2': 87 | library_dirs.append(os.path.join(sys.exec_prefix, 'Config')) 88 | 89 | is_cygwin = sys.platform[:6] == 'cygwin' 90 | is_atheos = sys.platform[:6] == 'atheos' 91 | is_shared = distutils.sysconfig.get_config_var('Py_ENABLE_SHARED') 92 | is_linux = sys.platform.startswith('linux') 93 | is_gnu = sys.platform.startswith('gnu') 94 | is_sunos = sys.platform.startswith('sunos') 95 | 96 | if is_cygwin or is_atheos: 97 | if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): 98 | library_dirs.append(os.path.join(sys.prefix, "lib", BASENAME, 99 | "config")) 100 | else: 101 | library_dirs.append(os.getcwd()) 102 | 103 | if (is_linux or is_gnu or is_sunos) and is_shared: 104 | if sys.executable.startswith(os.path.join(sys.exec_prefix, "bin")): 105 | library_dirs.append(distutils.sysconfig.get_config_var('LIBDIR')) 106 | else: 107 | library_dirs.append(os.getcwd()) 108 | 109 | user_include = os.path.join(site.USER_BASE, "include") 110 | user_lib = os.path.join(site.USER_BASE, "lib") 111 | if os.path.isdir(user_include): 112 | include_dirs.append(user_include) 113 | if os.path.isdir(user_lib): 114 | library_dirs.append(user_lib) 115 | 116 | ret_object = (include_dirs, library_dirs) 117 | _filter_non_existing_dirs(ret_object) 118 | 119 | return ret_object 120 | 121 | 122 | def _filter_non_existing_dirs(ret_object): 123 | for x, obj in enumerate(ret_object): 124 | for y, item in enumerate(obj): 125 | if not os.path.isdir(item): 126 | del ret_object[x][y] 127 | -------------------------------------------------------------------------------- /cyther/extractor.py: -------------------------------------------------------------------------------- 1 | 2 | """ 3 | This module provides tools to extract different patterns from raw text or files 4 | """ 5 | 6 | import os 7 | import re 8 | 9 | # Although NONE and MULTIPLE arent used here, they may be used BY THE USER 10 | from .tools import process_output, assert_output, NONE, MULTIPLE 11 | 12 | 13 | def get_content(pattern, string, tag='content'): 14 | """ 15 | Finds the 'content' tag from a 'pattern' in the provided 'string' 16 | """ 17 | output = [] 18 | for match in re.finditer(pattern, string): 19 | output.append(match.group(tag)) 20 | return output 21 | 22 | 23 | def extract(pattern, string, *, assert_equal=False, one=False, 24 | condense=False, default=None, default_if_multiple=True, 25 | default_if_none=True): 26 | """ 27 | Used to extract a given regex pattern from a string, given several options 28 | """ 29 | 30 | if isinstance(pattern, str): 31 | output = get_content(pattern, string) 32 | else: 33 | # Must be a linear container 34 | output = [] 35 | for p in pattern: 36 | output += get_content(p, string) 37 | 38 | output = process_output(output, one=one, condense=condense, 39 | default=default, 40 | default_if_multiple=default_if_multiple, 41 | default_if_none=default_if_none) 42 | 43 | if assert_equal: 44 | assert_output(output, assert_equal) 45 | else: 46 | return output 47 | 48 | 49 | RUNTIME_PATTERN = r"(?<=lib)(?P.+?)(?=\.so|\.a)" 50 | 51 | 52 | def extractRuntime(runtime_dirs): 53 | """ 54 | Used to find the correct static lib name to pass to gcc 55 | """ 56 | names = [str(item) for name in runtime_dirs for item in os.listdir(name)] 57 | string = '\n'.join(names) 58 | result = extract(RUNTIME_PATTERN, string, condense=True) 59 | return result 60 | 61 | 62 | POUND_PATTERN = r"#\s*@\s?[Cc]yther +(?P.+?)\s*?(\n|$)" 63 | TRIPPLE_PATTERN = r"(?P'{3}|\"{3})(.|\n)+?@[Cc]yther\s+" \ 64 | r"(?P(.|\n)+?)\s*(?P=quote)" 65 | 66 | 67 | def extractAtCyther(string): 68 | """ 69 | Extracts the '@cyther' code to be run as a script after compilation 70 | """ 71 | if isinstance(string, str) and os.path.isfile(string): 72 | with open(string) as file: 73 | string = file.read() 74 | 75 | found_pound = extract(POUND_PATTERN, string) 76 | found_tripple = extract(TRIPPLE_PATTERN, string) 77 | all_found = found_pound + found_tripple 78 | code = '\n'.join([item for item in all_found]) 79 | 80 | return code 81 | 82 | 83 | VERSION_PATTERN = r"[Vv]((\s*)|(ersion:?\s+))" \ 84 | r"(?P(\d+\.){1,}((dev)?\d+))" 85 | 86 | 87 | def extractVersion(string, default='?'): 88 | """ 89 | Extracts a three digit standard format version number 90 | """ 91 | return extract(VERSION_PATTERN, string, condense=True, default=default, 92 | one=True) 93 | 94 | 95 | MAJORMINOR_PATTERN = r"(?<=[^\d\.])(?P\d\.\d)" 96 | 97 | 98 | def extractMajorMinor(dirpath): 99 | """ 100 | Extracts the version number (excluding the patch number) from a path 101 | Ex) python/thing/3.5.1/include -> '3.5' 102 | """ 103 | return extract(MAJORMINOR_PATTERN, dirpath, condense=True, one=True) 104 | -------------------------------------------------------------------------------- /cyther/instructions.py: -------------------------------------------------------------------------------- 1 | 2 | """ 3 | This module holds the necessary functions and object definitions to handle and 4 | process basic instructions, whether they originate from an api usage or from 5 | the terminal. This is where both functionalities merge. Serious error checking. 6 | """ 7 | 8 | from .pathway import File 9 | from .parser import parseString 10 | 11 | 12 | INCORRECT_INSTRUCTION_INIT = "Instruction doesn't accept arguments " \ 13 | "of type '{}', only string or individual " \ 14 | "parameter setting" 15 | NO_INPUT_FILE = "Must have an input file specified for each instruction" 16 | 17 | 18 | # TODO Privatize all of the attributes 19 | class Instruction: 20 | """ 21 | Holds the necessary information and utilities to process and default check 22 | the different fields. Provides a merging for the api and terminal 23 | functionality 24 | """ 25 | def __init__(self, init=None): 26 | self.input = None 27 | # Steps inbetween the input and output need their own individual 28 | # instructions to be able to elaborate on them! 29 | self.output = None 30 | self.buildable_dependencies = [] 31 | self.given_dependencies = [] 32 | 33 | # Not critical information, information is obtained from ^ attributes 34 | # These attribs below are 'overwriters' in a sense 35 | self.output_format = None 36 | self.build_directory = None 37 | 38 | if init: 39 | if isinstance(init, str): 40 | ret = parseString(init) 41 | self.input = ret['input_name'] 42 | self.output = ret['output_name'] 43 | self.output_format = ret['output_format'] 44 | self.buildable_dependencies = ret['buildable_dependencies'] 45 | self.given_dependencies = ret['given_dependencies'] 46 | self.building_directory = ret['building_directory'] 47 | self.output_directory = ret['output_directory'] 48 | else: 49 | raise ValueError(INCORRECT_INSTRUCTION_INIT.format(type(init))) 50 | else: 51 | pass # If init is not specified, the user must use the methods! 52 | 53 | def processAndSetDefaults(self): 54 | """ 55 | The heart of the 'Instruction' object. This method will make sure that 56 | all fields not entered will be defaulted to a correct value. Also 57 | checks for incongruities in the data entered, if it was by the user. 58 | """ 59 | # INPUT, OUTPUT, GIVEN + BUILDABLE DEPS 60 | if not self.input: 61 | raise ValueError(NO_INPUT_FILE) 62 | 63 | if not self.output: 64 | # Build directory must exist, right? 65 | if not self.build_directory: 66 | File() 67 | pass # Can it be built? / reference self.output_format for this 68 | else: 69 | pass # if it is not congruent with other info provided 70 | 71 | if not self.build_directory: 72 | pass # Initialize it 73 | 74 | for dependency in self.given_dependencies: 75 | pass # Check if the dependcy exists 76 | 77 | if self.output_format != self.output.getType(): 78 | raise ValueError("") 79 | # Given dependencies must actually exist! 80 | # output_name must be at a lower extenion level than input_name 81 | # The build directory 82 | return 83 | 84 | def setBuildableDependencies(self, dependencies): 85 | self.buildable_dependencies = dependencies 86 | 87 | def setGivenDependencies(self, dependencies): 88 | self.given_dependencies = dependencies 89 | 90 | def setBuildDirectory(self, directory): 91 | self.build_directory = directory 92 | 93 | def setInput(self, input_name, **kwargs): 94 | self.input = File(input_name, **kwargs) 95 | 96 | def setOutput(self, output_name, **kwargs): 97 | self.output = File(output_name, **kwargs) 98 | 99 | 100 | 101 | class InstructionManager: 102 | def parseInstruction(self, instruction): 103 | # This will parse a given string and automatically add an instruction! 104 | pass 105 | 106 | def parseInstructions(self, instructions): 107 | for instruction in instructions: 108 | self.parseInstruction(instruction) 109 | 110 | def to_file(self): 111 | pass 112 | 113 | def from_file(self): 114 | pass 115 | -------------------------------------------------------------------------------- /cyther/launcher.py: -------------------------------------------------------------------------------- 1 | 2 | """ 3 | This module contains definitions for launching sets of commands into a 4 | subprocess and handling its output correctly and efficiently 5 | """ 6 | 7 | import subprocess 8 | import traceback 9 | import sys 10 | import multiprocessing 11 | 12 | from .extractor import extract, extractVersion 13 | 14 | import dill 15 | 16 | 17 | class Result: 18 | """ 19 | A class to hold the results of a command call. Holds stderr and stdout 20 | Contains useful functions to process them 21 | """ 22 | def __init__(self, returncode=0, stdout='', stderr=''): 23 | self.returncode = returncode 24 | self.stdout = stdout 25 | self.stderr = stderr 26 | 27 | def __str__(self): 28 | return self.getOutput() 29 | 30 | def extract(self, pattern, **kwargs): 31 | """ 32 | Given a regex pattern, it will find all of the occurences in the output 33 | """ 34 | return extract(pattern, self.getOutput(), **kwargs) 35 | 36 | def extractVersion(self, default='?'): 37 | """ 38 | Extracts a version number from the standard outputs 39 | """ 40 | return extractVersion(self.getOutput(), default=default) 41 | 42 | def getStdout(self): 43 | """ 44 | Returns stdout 45 | """ 46 | return self.stdout 47 | 48 | def getStderr(self): 49 | """ 50 | Returns stderr 51 | """ 52 | return self.stderr 53 | 54 | def getOutput(self): 55 | """ 56 | Returns the combined output of stdout and stderr 57 | """ 58 | output = self.stdout 59 | if self.stdout: 60 | output += '\r\n' 61 | output += self.stderr 62 | return output 63 | 64 | def extendInformation(self, response): 65 | """ 66 | This extends the objects stdout and stderr by 67 | 'response's stdout and stderr 68 | """ 69 | if response.stdout: 70 | self.stdout += '\r\n' + response.stdout 71 | if response.stderr: 72 | self.stderr += '\r\n' + response.stderr 73 | 74 | 75 | def _get_encodings(): 76 | """ 77 | Just a simple function to return the system encoding (defaults to utf-8) 78 | """ 79 | stdout_encoding = sys.stdout.encoding if sys.stdout.encoding else 'utf-8' 80 | stderr_encoding = sys.stderr.encoding if sys.stderr.encoding else 'utf-8' 81 | return stdout_encoding, stderr_encoding 82 | 83 | 84 | def _print_commands(*several_commands): 85 | for commands in several_commands: 86 | print(' '.join(commands).strip()) 87 | 88 | 89 | def _extract_output(process, print_result, raise_exception): 90 | stdout_bytes, stderr_bytes = process.communicate() 91 | stdout_encoding, stderr_encoding = _get_encodings() 92 | stdout = stdout_bytes.decode(stdout_encoding) 93 | stderr = stderr_bytes.decode(stderr_encoding) 94 | result = Result(process.returncode, stdout, stderr) 95 | 96 | if print_result and not raise_exception: 97 | if stdout: 98 | print(stdout, file=sys.stdout) 99 | if stderr: 100 | print(stderr, file=sys.stderr) 101 | return result 102 | 103 | 104 | # TODO An option to raise a Exception as well? Is that useful? 105 | def call(commands, *, print_result=False, raise_exception=False, 106 | print_commands=False): 107 | """ 108 | Will call a set of commands and wrangle the output how you choose 109 | """ 110 | if isinstance(commands, str): 111 | commands = commands.split() 112 | 113 | if not (isinstance(commands, tuple) or 114 | isinstance(commands, list)): 115 | raise ValueError("Function 'call' does not accept a 'commands'" 116 | "argument of type '{}'".format(type(commands))) 117 | 118 | if raise_exception: 119 | print_result = False 120 | try: 121 | process = subprocess.Popen(commands, 122 | stdout=subprocess.PIPE, 123 | stderr=subprocess.PIPE) 124 | if print_commands: 125 | _print_commands(commands) 126 | 127 | except: 128 | # TODO Why couldn't we just do 'except Exception as output' 129 | output = traceback.format_exc() 130 | result = Result(1, stderr=output) 131 | if print_result and not raise_exception: 132 | print(output, file=sys.stderr) 133 | 134 | else: 135 | result = _extract_output(process, print_result, raise_exception) 136 | 137 | if raise_exception and (result.returncode == 1): 138 | message = "An error occurred in an external process:\n\n{}" 139 | raise Exception(message.format(result.getStderr())) 140 | return result 141 | 142 | 143 | # TODO Should I pass on the argument 'raise_exception' to call? 144 | # TODO This can be done with '**kwargs' 145 | def multiCall(*commands, dependent=True, bundle=False, 146 | print_result=False, print_commands=False): 147 | """ 148 | Calls the function 'call' multiple times, given sets of commands 149 | """ 150 | results = [] 151 | dependent_failed = False 152 | 153 | for command in commands: 154 | if not dependent_failed: 155 | response = call(command, print_result=print_result, 156 | print_commands=print_commands) 157 | # TODO Will an error ever return a code other than '1'? 158 | if (response.returncode == 1) and dependent: 159 | dependent_failed = True 160 | else: 161 | response = None 162 | results.append(response) 163 | 164 | if bundle: 165 | result = Result() 166 | for response in results: 167 | if not response: 168 | continue 169 | elif response.returncode == 1: 170 | result.returncode = 1 171 | 172 | result.extendInformation(response) 173 | processed_response = result 174 | else: 175 | processed_response = results 176 | 177 | return processed_response 178 | 179 | 180 | def _run_pickled(pickled): 181 | function, item = dill.loads(pickled) 182 | return function(item) 183 | 184 | 185 | def distribute(function, iterable, *, workers=4): 186 | """ 187 | A version of multiprocessing.Pool.map that works using dill to pickle the 188 | function and iterable 189 | """ 190 | with multiprocessing.Pool(workers) as pool: 191 | processes = [] 192 | for item in iterable: 193 | pickled = dill.dumps((function, item)) 194 | process = pool.apply_async(_run_pickled, (pickled,)) 195 | processes.append(process) 196 | 197 | results = [process.get() for process in processes] 198 | return results 199 | 200 | 201 | def test(): 202 | def f(x): 203 | return x ** 2 204 | 205 | print(distribute(f, [1, 2, 3])) 206 | -------------------------------------------------------------------------------- /cyther/parser.py: -------------------------------------------------------------------------------- 1 | 2 | """ 3 | This module contains the definitions necessary to parse sets of instructions in 4 | string form and return a simple attribute object holding everything 5 | """ 6 | 7 | 8 | INSTRUCTION_HAS_WHITESPACE = "Cannot parse an instruction with whitespace" 9 | INCOMPLETE_SET = "There must be one set of '{},{}' to finish " \ 10 | "the dependency definition" 11 | INCORRECT_SET_CONSTITUENT = "There can only be one '{}' in a " \ 12 | "dependency definition" 13 | MORE_THAN_ONE_SET = "An instruction can only have a maximum of one set of " \ 14 | "'{},{}', denoting the dependencies" 15 | DEPENDENCIES_NOT_FIRST = "An instruction with dependencies must have " \ 16 | "them defined first" 17 | OPTIONS_NOT_LAST = "An instruction with building options must have them " \ 18 | "defined last" 19 | MULTIPLE_CONFLICTING_BUILD_OPTIONS = "There can only be one of each " \ 20 | "building option per instruction" 21 | EMPTY_PARAMETER = "Cannot have an empty parameter" 22 | EMPTY_KEYWORD_PARAMETER = "Cannot have an empty keyword parameter" 23 | MULTIPLE_FLOW_OPERATORS = "An instruction can only have one '>' in the " \ 24 | "task name" 25 | FLOW_OPERATOR_ON_ENDS = "Greater than character (flow operator) must " \ 26 | "seperate filenames" 27 | 28 | 29 | def _get_contents_between(string, opener, closer): 30 | """ 31 | Get the contents of a string between two characters 32 | """ 33 | opener_location = string.index(opener) 34 | closer_location = string.index(closer) 35 | content = string[opener_location + 1:closer_location] 36 | return content 37 | 38 | 39 | def _check_whitespace(string): 40 | """ 41 | Make sure thre is no whitespace in the given string. Will raise a 42 | ValueError if whitespace is detected 43 | """ 44 | if string.count(' ') + string.count('\t') + string.count('\n') > 0: 45 | raise ValueError(INSTRUCTION_HAS_WHITESPACE) 46 | 47 | 48 | def _check_enclosing_characters(string, opener, closer): 49 | """ 50 | Makes sure that the enclosing characters for a definition set make sense 51 | 1) There is only one set 52 | 2) They are in the right order (opening, then closing) 53 | """ 54 | opener_count = string.count(opener) 55 | closer_count = string.count(closer) 56 | total = opener_count + closer_count 57 | if total > 2: 58 | msg = MORE_THAN_ONE_SET.format(opener, closer) 59 | raise ValueError(msg) 60 | elif total == 1: 61 | msg = INCOMPLETE_SET.format(opener, closer) 62 | raise ValueError(msg) 63 | elif opener_count > 1: 64 | msg = INCORRECT_SET_CONSTITUENT.format(opener) 65 | raise ValueError(msg) 66 | elif closer_count > 1: 67 | msg = INCORRECT_SET_CONSTITUENT.format(closer) 68 | raise ValueError(msg) 69 | 70 | 71 | def _check_parameters(parameters, symbols): 72 | """ 73 | Checks that the parameters given are not empty. Ones with prefix symbols 74 | can be denoted by including the prefix in symbols 75 | """ 76 | for param in parameters: 77 | if not param: 78 | raise ValueError(EMPTY_PARAMETER) 79 | elif (param[0] in symbols) and (not param[1:]): 80 | print(param) 81 | raise ValueError(EMPTY_KEYWORD_PARAMETER) 82 | 83 | 84 | def _check_dependencies(string): 85 | """ 86 | Checks the dependencies constructor. Looks to make sure that the 87 | dependencies are the first things defined 88 | """ 89 | opener, closer = '(', ')' 90 | _check_enclosing_characters(string, opener, closer) 91 | if opener in string: 92 | if string[0] != opener: 93 | raise ValueError(DEPENDENCIES_NOT_FIRST) 94 | ret = True 95 | else: 96 | ret = False 97 | return ret 98 | 99 | 100 | def _check_building_options(string): 101 | """ 102 | Checks the building options to make sure that they are defined last, 103 | after the task name and the dependencies 104 | """ 105 | opener, closer = '{', '}' 106 | _check_enclosing_characters(string, opener, closer) 107 | if opener in string: 108 | if string[-1] != closer: 109 | raise ValueError(OPTIONS_NOT_LAST) 110 | ret = True 111 | else: 112 | ret = False 113 | return ret 114 | 115 | 116 | def _check_flow_operator(string): 117 | """ 118 | Checks the flow operator ('>') to make sure that it: 119 | 1) Is non empty 120 | 2) There is only one of them 121 | """ 122 | greater_than_count = string.count('>') 123 | if greater_than_count > 1: 124 | raise ValueError(MULTIPLE_FLOW_OPERATORS) 125 | elif (string[0] == '>') or (string[-1] == '>'): 126 | raise ValueError(FLOW_OPERATOR_ON_ENDS) 127 | else: 128 | if greater_than_count == 1: 129 | ret = True 130 | else: 131 | ret = False 132 | return ret 133 | 134 | 135 | def _parse_dependencies(string): 136 | """ 137 | This function actually parses the dependencies are sorts them into 138 | the buildable and given dependencies 139 | """ 140 | contents = _get_contents_between(string, '(', ')') 141 | unsorted_dependencies = contents.split(',') 142 | _check_parameters(unsorted_dependencies, ('?',)) 143 | 144 | buildable_dependencies = [] 145 | given_dependencies = [] 146 | for dependency in unsorted_dependencies: 147 | if dependency[0] == '?': 148 | given_dependencies.append(dependency[1:]) 149 | else: 150 | buildable_dependencies.append(dependency) 151 | 152 | string = string[string.index(')') + 1:] 153 | return buildable_dependencies, given_dependencies, string 154 | 155 | 156 | def _parse_building_options(string): 157 | """ 158 | This will parse and sort the building options defined in the '{}' 159 | constructor. Will only allow one of each argument 160 | """ 161 | contents = _get_contents_between(string, '{', '}') 162 | unsorted_options = contents.split(',') 163 | _check_parameters(unsorted_options, ('@', '/', '\\', '^')) 164 | 165 | output_directory = None 166 | output_format = None 167 | building_directory = None 168 | for option in unsorted_options: 169 | if option[0] == '@': 170 | if output_format: 171 | raise ValueError(MULTIPLE_CONFLICTING_BUILD_OPTIONS) 172 | output_format = option[1:] 173 | elif option[0] in ('/', '\\'): 174 | if output_directory: 175 | raise ValueError(MULTIPLE_CONFLICTING_BUILD_OPTIONS) 176 | output_directory = option[1:] 177 | elif option[0] == '^': 178 | if building_directory: 179 | raise ValueError(MULTIPLE_CONFLICTING_BUILD_OPTIONS) 180 | building_directory = option[1:] 181 | 182 | string = string[:string.index('{')] 183 | return output_directory, output_format, building_directory, string 184 | 185 | 186 | """ 187 | (example_file.o)[yolo.pyx]{^local} example_file.pyx{o} 188 | 189 | Starting Point (task_name is the filename!) 190 | [Intermediate steps] 191 | Endpoint 192 | """ 193 | 194 | 195 | def parseString(string): 196 | """ 197 | This function takes an entire instruction in the form of a string, and 198 | will parse the entire string and return a dictionary of the fields 199 | gathered from the parsing 200 | """ 201 | buildable_dependencies = [] 202 | given_dependencies = [] 203 | output_directory = None 204 | output_format = None 205 | building_directory = None 206 | output_name = None 207 | 208 | _check_whitespace(string) 209 | 210 | there_are_dependencies = _check_dependencies(string) 211 | if there_are_dependencies: 212 | buildable_dependencies, \ 213 | given_dependencies, \ 214 | string = _parse_dependencies(string) 215 | 216 | there_are_options = _check_building_options(string) 217 | if there_are_options: 218 | output_directory, \ 219 | output_format, \ 220 | building_directory, string = _parse_building_options(string) 221 | 222 | if string[0] == '>': 223 | string = string[1:] 224 | if string[-1] == '>': 225 | string = string[:-1] 226 | 227 | is_a_flow_operator = _check_flow_operator(string) 228 | if is_a_flow_operator: 229 | greater_than_location = string.index('>') 230 | output_name = string[greater_than_location + 1:] 231 | string = string[:greater_than_location] 232 | 233 | ret = object() 234 | ret.input_name = string 235 | ret.output_name = output_name 236 | ret.buildable_dependencies = buildable_dependencies 237 | ret.given_dependencies = given_dependencies 238 | ret.output_format = output_format 239 | ret.building_directory = building_directory 240 | ret.output_directory = output_directory 241 | return ret 242 | -------------------------------------------------------------------------------- /cyther/pathway.py: -------------------------------------------------------------------------------- 1 | 2 | """ 3 | This module holds the utilities necessary to process full path names and hold 4 | their parsed information, so other tools can easily extract this information 5 | (i.e. extension), and hold the information in a containers better designed 6 | for it 7 | """ 8 | 9 | import os 10 | import time 11 | 12 | APPEND_SEP_TO_DIRS = True 13 | 14 | DRIVE = 0 15 | REST = 1 16 | 17 | NAME = 0 18 | EXTENSION = 1 19 | 20 | ISFILE = True 21 | ISDIR = False 22 | 23 | EXT = '.' 24 | USER = '~' 25 | 26 | 27 | class OverwriteError(Exception): 28 | """ 29 | Denotes if the user tried to overwrite a part of a file without giving 30 | explicit permission 31 | """ 32 | def __init__(self, *args, **kwargs): 33 | super(OverwriteError, self).__init__(*args, **kwargs) 34 | 35 | 36 | def normalize(path_name, override=None): 37 | """ 38 | Prepares a path name to be worked with. Path name must not be empty. This 39 | function will return the 'normpath'ed path and the identity of the path. 40 | This function takes an optional overriding argument for the identity. 41 | 42 | ONLY PROVIDE OVERRIDE IF: 43 | 1) YOU AREWORKING WITH A FOLDER THAT HAS AN EXTENSION IN THE NAME 44 | 2) YOU ARE MAKING A FILE WITH NO EXTENSION 45 | """ 46 | 47 | identity = identify(path_name, override=override) 48 | new_path_name = os.path.normpath(os.path.expanduser(path_name)) 49 | 50 | return new_path_name, identity 51 | 52 | 53 | def identify(path_name, *, override=None, check_exists=True, default=ISDIR): 54 | """ 55 | Identify the type of a given path name (file or directory). If check_exists 56 | is specified to be false, then the function will not set the identity based 57 | on the path existing or not. If override is specified as either ISDIR or 58 | ISFILE, then the function will try its best to over-ride the identity to be 59 | what you have specified. The 'default' parameter is what the function will 60 | default the identity to if 'override' is not specified, and the path looks 61 | like it could be both a directory and a file. 62 | """ 63 | 64 | head, tail = os.path.split(path_name) 65 | 66 | if check_exists and os.path.exists(path_name): 67 | if os.path.isfile(path_name): 68 | if override == ISDIR: 69 | raise ValueError("Cannot override a path as a directory if it " 70 | "is a file that already exists") 71 | result = ISFILE 72 | elif os.path.isdir(path_name): 73 | if override == ISFILE: 74 | raise ValueError("Cannot override a path as a file if it is " 75 | "a directory that already exists") 76 | result = ISDIR 77 | else: 78 | raise Exception("Path exists but isn't a file or a directory...") 79 | elif not tail: 80 | if override == ISFILE: 81 | raise ValueError("Cannot interpret a path with a slash at the end " 82 | "to be a file") 83 | result = ISDIR 84 | elif has_ext(tail, if_all_ext=True): 85 | if override is None: 86 | result = ISFILE 87 | else: 88 | result = override 89 | else: 90 | if override is None: 91 | result = default 92 | else: 93 | result = override 94 | 95 | return result 96 | 97 | 98 | def join_ext(name, extension): 99 | """ 100 | Joins a given name with an extension. If the extension doesn't have a '.' 101 | it will add it for you 102 | """ 103 | if extension[0] == EXT: 104 | ret = name + extension 105 | else: 106 | ret = name + EXT + extension 107 | return ret 108 | 109 | 110 | def get_drive(path_name): 111 | """ 112 | Gets the part of the path that specified the drive. If this cannot be found 113 | it will return an empty string 114 | """ 115 | return os.path.splitdrive(path_name)[DRIVE] 116 | 117 | 118 | def has_ext(path_name, *, multiple=None, if_all_ext=False): 119 | """ 120 | Determine if the given path name has an extension 121 | """ 122 | base = os.path.basename(path_name) 123 | count = base.count(EXT) 124 | 125 | if not if_all_ext and base[0] == EXT and count != 0: 126 | count -= 1 127 | 128 | if multiple is None: 129 | return count >= 1 130 | elif multiple: 131 | return count > 1 132 | else: 133 | return count == 1 134 | 135 | 136 | def get_ext(path_name, *, if_all_ext=False): 137 | """ 138 | Get an extension from the given path name. If an extension cannot be found, 139 | it will return an empty string 140 | """ 141 | if has_ext(path_name): 142 | return os.path.splitext(path_name)[EXTENSION] 143 | elif if_all_ext and has_ext(path_name, if_all_ext=True): 144 | return os.path.splitext(path_name)[NAME] 145 | else: 146 | return '' 147 | 148 | 149 | def has_dir(path_name): 150 | """ 151 | Determine if the given path name contains a directory part, even if the 152 | path refers to a file 153 | """ 154 | return bool(os.path.dirname(path_name)) 155 | 156 | 157 | def get_dir(path_name, *, greedy=False, override=None, identity=None): 158 | """ 159 | Gets the directory path of the given path name. If the argument 'greedy' 160 | is specified as True, then if the path name represents a directory itself, 161 | the function will return the whole path 162 | """ 163 | if identity is None: 164 | identity = identify(path_name, override=override) 165 | 166 | path_name = os.path.normpath(path_name) 167 | 168 | if greedy and identity == ISDIR: 169 | return path_name 170 | else: 171 | return os.path.dirname(path_name) 172 | 173 | 174 | def get_parent(path_name): 175 | """ 176 | Gets the parent directory of the given path name 177 | """ 178 | return os.path.basename(os.path.dirname(path_name)) 179 | 180 | 181 | def get_name(path_name, *, ext=True, override=None, identity=None): 182 | """ 183 | Gets the name par of the path name given. By 'name' I mean the basename of 184 | a filename's path, such as 'test.o' in the path: 'C:/test/test.o' 185 | """ 186 | if identity is None: 187 | identity = identify(path_name, override=override) 188 | 189 | if identity == ISFILE: 190 | if ext: 191 | r = os.path.basename(path_name) 192 | else: 193 | r = os.path.splitext(os.path.basename(path_name))[NAME] 194 | else: 195 | r = '' 196 | return r 197 | 198 | 199 | def disintegrate(path_name): 200 | """ 201 | Disintegrates the path name by splitting all of the components apart 202 | """ 203 | return os.path.normpath(path_name).split(os.sep) 204 | 205 | 206 | def get_system_drives(): 207 | """ 208 | Get the available drive names on the system. Always returns a list. 209 | """ 210 | drives = [] 211 | if os.name == 'nt': 212 | import ctypes 213 | bitmask = ctypes.windll.kernel32.GetLogicalDrives() 214 | letter = ord('A') 215 | while bitmask > 0: 216 | if bitmask & 1: 217 | name = chr(letter) + ':' + os.sep 218 | if os.path.isdir(name): 219 | drives.append(name) 220 | bitmask >>= 1 221 | letter += 1 222 | else: 223 | current_drive = get_drive(os.getcwd()) 224 | if current_drive: 225 | drive = current_drive 226 | else: 227 | drive = os.sep 228 | drives.append(drive) 229 | 230 | return drives 231 | 232 | 233 | def has_suffix(path_name, suffix): 234 | """ 235 | Determines if path_name has a suffix of at least 'suffix' 236 | """ 237 | if isinstance(suffix, str): 238 | suffix = disintegrate(suffix) 239 | 240 | components = disintegrate(path_name) 241 | 242 | for i in range(-1, -(len(suffix) + 1), -1): 243 | if components[i] != suffix[i]: 244 | break 245 | else: 246 | return True 247 | return False 248 | 249 | ########################################################################### 250 | 251 | 252 | def _initialize(path_name, override, root, inject): 253 | if path_name: 254 | if isinstance(path_name, list) or isinstance(path_name, tuple): 255 | path_name = os.path.join(*path_name) 256 | elif not isinstance(path_name, str): 257 | raise ValueError("Parameter 'path_name' must be of " 258 | "type str, list, tuple, or NoneType, " 259 | "not {}".format(type(path_name))) 260 | path_name, identity = normalize(path_name, override) 261 | else: 262 | identity = None 263 | 264 | if root: 265 | root, root_identity = normalize(root) 266 | if root_identity == ISFILE: 267 | raise ValueError("Parameter 'root' cannot be a file") 268 | elif not os.path.isabs(root): 269 | raise ValueError("The root must be an absolute directory if " 270 | "specified") 271 | elif path_name and os.path.isabs(path_name): 272 | raise ValueError("The path cannot be absolute as well as the r" 273 | "oot; cannot add two absolute paths together") 274 | 275 | if inject and identify(inject) == ISFILE: 276 | raise ValueError("Parameter 'inject' must be a directory") 277 | 278 | return path_name, identity, root 279 | 280 | 281 | def _process_name(path_name, identity, name, ext): 282 | if name or ext or (identity == ISFILE): 283 | if identity == ISFILE: 284 | if name: 285 | raise OverwriteError("The path supplied must be a directory, " 286 | "if you provide a name") 287 | else: 288 | file_name = get_name(path_name, identity=ISFILE, ext=True) 289 | if ext: 290 | new_name = join_ext(file_name, ext) 291 | else: 292 | new_name = file_name 293 | else: 294 | if name: 295 | if ext: 296 | new_name = join_ext(name, ext) 297 | else: 298 | new_name = name 299 | else: 300 | new_name = join_ext('', ext) 301 | else: 302 | new_name = '' 303 | 304 | return new_name 305 | 306 | 307 | def _process_directory(path_name, identity, root, inject): 308 | if root: 309 | if path_name: 310 | new_directory = os.path.join(root, get_dir(path_name, 311 | identity=identity, 312 | greedy=True)) 313 | else: 314 | new_directory = root 315 | else: 316 | cwd = os.getcwd() 317 | if path_name and (has_dir(path_name) or identity == ISDIR): 318 | dir_extension = get_dir(path_name, identity=identity, greedy=True) 319 | if os.path.isabs(path_name): 320 | new_directory = dir_extension 321 | else: 322 | new_directory = os.path.join(cwd, dir_extension) 323 | else: 324 | new_directory = cwd 325 | 326 | if inject: 327 | new_directory = os.path.join(new_directory, inject) 328 | 329 | return new_directory 330 | 331 | 332 | def _format_path(path_name, root, relpath, reduce): 333 | if reduce: 334 | relpath = True 335 | 336 | if not relpath: 337 | result = path_name 338 | else: 339 | if isinstance(relpath, str): 340 | if not os.path.isabs(relpath): 341 | raise ValueError("If relpath is manually specified, it must " 342 | "be an absolute path") 343 | start = relpath 344 | elif root: 345 | start = root 346 | else: 347 | start = os.getcwd() 348 | 349 | if get_drive(path_name) != get_drive(start): 350 | raise ValueError("Calculating relpath requires that the " 351 | "comparator path is of the same drive") 352 | 353 | new_path = os.path.relpath(path_name, start=start) 354 | 355 | if reduce and (len(new_path) >= len(path_name)): 356 | result = path_name 357 | else: 358 | result = new_path 359 | 360 | return result 361 | 362 | 363 | # TODO Will '~' mean anything on a system with no user specified? Possible? 364 | def path(path_name=None, override=None, *, root=None, name=None, ext=None, 365 | inject=None, relpath=None, reduce=False): 366 | """ 367 | Path manipulation black magic 368 | """ 369 | path_name, identity, root = _initialize(path_name, override, root, inject) 370 | new_name = _process_name(path_name, identity, name, ext) 371 | new_directory = _process_directory(path_name, identity, root, inject) 372 | full_path = os.path.normpath(os.path.join(new_directory, new_name)) 373 | if APPEND_SEP_TO_DIRS and not new_name and full_path[-1] != os.sep: 374 | full_path += os.sep 375 | final_path = _format_path(full_path, root, relpath, reduce) 376 | return final_path 377 | 378 | 379 | class File: 380 | """ 381 | Holds all of the information and methods necessary to process full paths. 382 | Takes a path name and an optional constructor to build a file name on 383 | if it does not exist already. 384 | """ 385 | 386 | def __init__(self, path_name=None, **kwargs): 387 | self.__path = path(path_name, **kwargs) 388 | self.__stamp = None 389 | 390 | def __str__(self): 391 | return self.getPath() 392 | 393 | def __repr__(self): 394 | return "File('{}')".format(str(self)) 395 | 396 | def getmtime(self): 397 | """ 398 | Works the same as os.path.getmtime, but on the full internal path 399 | """ 400 | return os.path.getmtime(self.getPath()) 401 | 402 | def exists(self): 403 | """ 404 | Method that works the same as os.path.exists, but on the internal path 405 | """ 406 | return os.path.exists(self.getPath()) 407 | 408 | def path(self, **kwargs): 409 | """ 410 | Returns a different object with the specified changes applied to 411 | it. This object is not changed in the process. 412 | """ 413 | new_path = path(self.getPath(), **kwargs) 414 | return File(new_path) 415 | 416 | def isOutDated(self, output_file): 417 | """ 418 | Figures out if Cyther should compile the given FileInfo object by 419 | checking the both of the modified times 420 | """ 421 | if output_file.exists(): 422 | source_time = self.getmtime() 423 | output_time = output_file.getmtime() 424 | return source_time > output_time 425 | else: 426 | return True 427 | 428 | def stampError(self): 429 | """ 430 | Sets the current error point to the 431 | """ 432 | self.__stamp = time.time() 433 | 434 | def isUpdated(self): 435 | """ 436 | Figures out if the file had previously errored and hasn't 437 | been fixed since given a numerical time 438 | """ 439 | modified_time = self.getmtime() 440 | valid = modified_time > self.__stamp 441 | return valid 442 | 443 | def getName(self, ext=True): 444 | """ 445 | Gets the name of the file as the parent directory sees it 446 | (ex. 'example.py') 447 | """ 448 | return get_name(self.getPath(), ext=ext) 449 | 450 | def getExtension(self): 451 | """ 452 | Returns the type of the file (its extension) with the '.' 453 | """ 454 | return get_ext(self.getPath()) 455 | 456 | def getDirectory(self): 457 | """ 458 | Returns the parent directory of the file 459 | """ 460 | return get_dir(self.getPath()) 461 | 462 | def getPath(self): 463 | """ 464 | Returns the full file path to the file, including the drive 465 | """ 466 | return self.__path 467 | -------------------------------------------------------------------------------- /cyther/processing.py: -------------------------------------------------------------------------------- 1 | import time 2 | 3 | from .launcher import multiCall 4 | from .commands import furtherArgsProcessing, processFiles, makeCommands 5 | from .definitions import WAIT_FOR_FIX, SKIPPED_COMPILATION, INTERVAL, \ 6 | ERROR_PASSOFF, FINE, WATCH_STATS_TEMPLATE, \ 7 | SETUP_TEMPLATE, TIMER_TEMPLATE 8 | from .extractor import extractAtCyther 9 | from .system import * 10 | 11 | 12 | def cueExtractAndRun(args, file): 13 | """ 14 | Cues the @cyther code execution procedure 15 | """ 16 | filename = file['file_path'] 17 | if args['execute']: 18 | holla = run(filename) 19 | else: 20 | holla = run(filename, True) 21 | return holla 22 | 23 | 24 | def initiateCompilation(args, file): 25 | """ 26 | Starts the entire compilation procedure 27 | """ 28 | ####commands = finalizeCommands(args, file) 29 | commands = makeCommands(0, file) 30 | if not args['concise'] and args['print_args']: 31 | print_commands = bool(args['watch']) 32 | response = multiCall(*commands, print_commands=print_commands) 33 | return response 34 | 35 | 36 | def cytherize(args, file): 37 | """ 38 | Used by core to integrate all the pieces of information, and to interface 39 | with the user. Compiles and cleans up. 40 | """ 41 | if isOutDated(file): 42 | if isUpdated(file): 43 | response = initiateCompilation(args, file) 44 | else: 45 | response = {'returncode': WAIT_FOR_FIX, 'output': ''} 46 | else: 47 | if args['timestamp']: 48 | response = {'returncode': SKIPPED_COMPILATION, 'output': ''} 49 | else: 50 | response = initiateCompilation(args, file) 51 | 52 | ########################################################################### 53 | 54 | time.sleep(INTERVAL) 55 | if response['returncode'] == ERROR_PASSOFF: 56 | file['stamp_if_error'] = time.time() 57 | if args['watch']: 58 | if len(args['filenames']) > 1: 59 | output = "Error in file: '{}'; Cyther will wait until it is" \ 60 | "fixed...\n".format(file['file_path']) 61 | else: 62 | output = "Cyther will wait for you to fix this error before" \ 63 | "it tries to compile again...\n" 64 | else: 65 | output = "Error in source file, see above\n" 66 | 67 | elif response['returncode'] == SKIPPED_COMPILATION: 68 | if not args['watch']: 69 | output = 'Skipping compilation: source file not updated since' \ 70 | 'last compile\n' 71 | else: 72 | output = '' 73 | 74 | elif response['returncode'] == WAIT_FOR_FIX: 75 | output = '' 76 | 77 | elif response['returncode'] == FINE: 78 | if args['watch']: 79 | if len(args['filenames']) > 1: 80 | output = "Compiled the file '{}'\n".format(file['file_path']) 81 | else: 82 | output = 'Compiled the file\n' 83 | else: 84 | if not args['concise']: 85 | output = 'Compilation complete\n' 86 | else: 87 | output = '' 88 | 89 | else: 90 | raise CytherError("Unrecognized return value '{}'" 91 | "".format(response['returncode'])) 92 | 93 | response['output'] += output 94 | 95 | ########################################################################### 96 | 97 | condition = response['returncode'] == SKIPPED_COMPILATION and not args[ 98 | 'watch'] 99 | if (args['execute'] or args['timer']) and response[ 100 | 'returncode'] == FINE or condition: 101 | ret = cueExtractAndRun(args, file) 102 | response['output'] += ret['output'] 103 | 104 | ########################################################################### 105 | 106 | if args['watch']: 107 | if response['returncode'] == FINE or response[ 108 | 'returncode'] == ERROR_PASSOFF: 109 | if response['returncode'] == FINE: 110 | args['watch_stats']['compiles'] += 1 111 | else: 112 | args['watch_stats']['errors'] += 1 113 | args['watch_stats']['counter'] += 1 114 | response['output'] += \ 115 | WATCH_STATS_TEMPLATE.format(args['watch_stats']['counter'], 116 | args['watch_stats']['compiles'], 117 | args['watch_stats']['errors'], 118 | args['watch_stats']['polls']) 119 | else: 120 | args['watch_stats']['polls'] += 1 121 | 122 | ########################################################################### 123 | 124 | if args['watch']: 125 | if response['returncode'] == 1: 126 | print(response['output'] + '\n') 127 | else: 128 | if response['output']: 129 | print(response['output']) 130 | else: 131 | if response['returncode'] == 1: 132 | if args['error']: 133 | raise CytherError(response['output']) 134 | else: 135 | print(response['output']) 136 | else: 137 | print(response['output']) 138 | 139 | 140 | def run(path, timer=False, repeat=3, number=10000, precision=2): 141 | """ 142 | Extracts and runs the '@cyther' code from the given file 'path' name 143 | """ 144 | code = extractAtCyther(path) 145 | if not code: 146 | output = "There was no '@cyther' code collected from the " \ 147 | "file '{}'\n".format(path) 148 | # TODO This should use a result, right? 149 | return {'returncode': 0, 'output': output} 150 | 151 | module_directory = os.path.dirname(path) 152 | module_name = os.path.splitext(os.path.basename(path))[0] 153 | setup_string = SETUP_TEMPLATE.format(module_directory, module_name, '{}') 154 | 155 | if timer: 156 | string = TIMER_TEMPLATE.format(setup_string, code, repeat, 157 | number, precision, '{}') 158 | else: 159 | string = setup_string + code 160 | 161 | script = os.path.join(os.path.dirname(__file__), 'script.py') 162 | with open(script, 'w+') as file: 163 | file.write(string) 164 | 165 | response = call(['python', script]) 166 | return response 167 | 168 | 169 | def core(args): 170 | """ 171 | The heart of Cyther, this function controls the main loop, and can be 172 | used to perform any Cyther action. You can call if using Cyther 173 | from the module level 174 | """ 175 | args = furtherArgsProcessing(args) 176 | 177 | numfiles = len(args['filenames']) 178 | interval = INTERVAL / numfiles 179 | files = processFiles(args) 180 | while True: 181 | for file in files: 182 | cytherize(args, file) 183 | if not args['watch']: 184 | break 185 | else: 186 | time.sleep(interval) 187 | 188 | 189 | if __name__ == '__main__': 190 | raise CytherError('This module is not meant to be run as a script.' 191 | 'Try \'cyther make\' for this functionality') 192 | -------------------------------------------------------------------------------- /cyther/project.py: -------------------------------------------------------------------------------- 1 | 2 | """ 3 | This module deals with operations regarding individual cyther projects 4 | """ 5 | 6 | import os 7 | 8 | from .tools import get_input 9 | from .pathway import path, ISDIR 10 | from .definitions import CACHE_NAME 11 | 12 | 13 | def assure_cache(project_path=None): 14 | """ 15 | Assure that a project directory has a cache folder. 16 | If not, it will create it. 17 | """ 18 | 19 | project_path = path(project_path, ISDIR) 20 | cache_path = os.path.join(project_path, CACHE_NAME) 21 | 22 | if not os.path.isdir(cache_path): 23 | os.mkdir(cache_path) 24 | 25 | 26 | def clean_project(): 27 | """ 28 | Clean a project of anything cyther related that is not essential to a build 29 | """ 30 | pass 31 | 32 | 33 | def purge_project(): 34 | """ 35 | Purge a directory of anything cyther related 36 | """ 37 | print('Current Directory: {}'.format(os.getcwd())) 38 | directories = os.listdir(os.getcwd()) 39 | if CACHE_NAME in directories: 40 | response = get_input("Would you like to delete the cache and" 41 | "everything in it? [y/n]: ", ('y', 'n')) 42 | if response == 'y': 43 | print("Listing local '__cythercache__':") 44 | cache_dir = os.path.join(os.getcwd(), "__cythercache__") 45 | to_delete = [] 46 | contents = os.listdir(cache_dir) 47 | if contents: 48 | for filename in contents: 49 | print('\t' + filename) 50 | filepath = os.path.join(cache_dir, filename) 51 | to_delete.append(filepath) 52 | else: 53 | print("\tNothing was found in the cache") 54 | 55 | check_response = get_input("Delete all these files? (^)" 56 | "[y/n]: ", ('y', 'n')) 57 | if check_response == 'y': 58 | for filepath in to_delete: 59 | os.remove(filepath) 60 | os.rmdir(cache_dir) 61 | else: 62 | print("Skipping the deletion... all files are fine!") 63 | else: 64 | print("Skipping deletion of the cache") 65 | else: 66 | print("Couldn't find a cache file ('{}') in this " 67 | "directory".format(CACHE_NAME)) 68 | -------------------------------------------------------------------------------- /cyther/searcher.py: -------------------------------------------------------------------------------- 1 | """ 2 | This module holds utilities to search for items, whether they be files or 3 | textual patterns, that cyther will use for compilation. This module is 4 | designed to be relatively easy to use and make a very complex task much less so 5 | """ 6 | 7 | import os 8 | import re 9 | import shutil 10 | 11 | # For testing purposes 12 | from time import time 13 | 14 | from .tools import isIterable, process_output 15 | from .pathway import get_system_drives, has_suffix, disintegrate 16 | from .launcher import distribute 17 | 18 | 19 | def _is_exe(fpath): 20 | return os.path.isfile(fpath) and os.access(fpath, os.X_OK) 21 | 22 | 23 | def where(cmd, path=None): 24 | """ 25 | A function to wrap shutil.which for universal usage 26 | """ 27 | raw_result = shutil.which(cmd, os.X_OK, path) 28 | if raw_result: 29 | return os.path.abspath(raw_result) 30 | else: 31 | raise ValueError("Could not find '{}' in the path".format(cmd)) 32 | 33 | 34 | def search_file(pattern, file_path): 35 | """ 36 | Search a given file's contents for the regex pattern given as 'pattern' 37 | """ 38 | try: 39 | with open(file_path) as file: 40 | string = file.read() 41 | except PermissionError: 42 | return [] 43 | 44 | matches = re.findall(pattern, string) 45 | 46 | return matches 47 | 48 | 49 | def _find_init(init, start): 50 | if not init: 51 | raise ValueError("Parameter 'init' must not be empty") 52 | elif isinstance(init, str): 53 | target = init 54 | suffix = None 55 | elif isIterable(init): 56 | target = init.pop() 57 | if init: 58 | suffix = init 59 | else: 60 | suffix = None 61 | else: 62 | raise TypeError("Parameter 'init' cannot be type " 63 | "'{}'".format(type(init))) 64 | 65 | if not start: 66 | start = get_system_drives() 67 | elif isinstance(start, str) and os.path.isdir(start): 68 | start = [start] 69 | else: 70 | raise TypeError("Parameter 'start' must be None, tuple, or list") 71 | 72 | return start, target, suffix 73 | 74 | 75 | def breadth(dirs): 76 | """ 77 | Crawl through directories like os.walk, but use a 'breadth first' approach 78 | (os.walk uses 'depth first') 79 | """ 80 | while dirs: 81 | next_dirs = [] 82 | print("Dirs: '{}'".format(dirs)) 83 | for d in dirs: 84 | next_dirs = [] 85 | try: 86 | for name in os.listdir(d): 87 | p = os.path.join(d, name) 88 | if os.path.isdir(p): 89 | print(p) 90 | next_dirs.append(p) 91 | except PermissionError as nallowed: 92 | print(nallowed) 93 | dirs = next_dirs 94 | if dirs: 95 | yield dirs 96 | 97 | 98 | def _get_starting_points(base_start): 99 | return base_start, [], [] 100 | 101 | 102 | # TODO Make it possible to find multiple things at once (saves crazy time) 103 | # TODO It turns out that process_args might not be necesssary at all... ('one') 104 | def find(init, start=None, one=False, is_exec=False, content=None, 105 | parallelize=True, workers=None): 106 | """ 107 | Finds a given 'target' (filename string) in the file system 108 | """ 109 | base_start, target, suffix = _find_init(init, start) 110 | 111 | def _condition(file_path, dirpath, filenames): 112 | if target in filenames or is_exec and os.access(file_path, os.X_OK): 113 | if not suffix or has_suffix(dirpath, suffix): 114 | if not content or search_file(content, file_path): 115 | return True 116 | return False 117 | 118 | starting_points, watch_dirs, excludes = _get_starting_points(base_start) 119 | disintegrated_excludes = [disintegrate(e) for e in excludes] 120 | 121 | def _filter(dirnames, dirpath): 122 | if disintegrate(dirpath) in watch_dirs: 123 | for e in disintegrated_excludes: 124 | if e[-1] in dirnames: 125 | if disintegrate(dirpath) == e[:-1]: 126 | dirnames.remove(e[-1]) 127 | 128 | def _fetch(top): 129 | results = [] 130 | for dirpath, dirnames, filenames in os.walk(top, topdown=True): 131 | # This if-statement is designed to save time 132 | _filter(dirnames, dirpath) 133 | 134 | file_path = os.path.normpath(os.path.join(dirpath, target)) 135 | if _condition(file_path, dirpath, filenames): 136 | results.append(file_path) 137 | return results 138 | 139 | st = time() 140 | if parallelize: 141 | unzipped_results = distribute(_fetch, starting_points, workers=workers) 142 | else: 143 | unzipped_results = [_fetch(point) for point in base_start] 144 | et = time() 145 | #print(et - st) 146 | 147 | zipped_results = [i for item in unzipped_results for i in item] 148 | processed_results = process_output(zipped_results, one=one) 149 | 150 | return processed_results 151 | 152 | 153 | def bloop(p): 154 | start = time() 155 | i = find(['include', 'Python.h'], parallelize=p) 156 | end = time() 157 | return end - start 158 | 159 | 160 | def cachunk(prit=False): 161 | t1 = bloop(True) 162 | t2 = bloop(False) 163 | if prit: 164 | print("Time (parallelized): '{}'".format(t1)) 165 | print("Time (not parallelized): '{}'".format(t2)) 166 | 167 | 168 | def test(): 169 | function = lambda i: [x ** 2 for x in list(range(1000000))] 170 | t1s = time() 171 | a = distribute(lambda x: x ** 2, to_process) 172 | t1e = time() 173 | print("Time (parallelized): '{}'".format(t1e - t1s)) 174 | t2s = time() 175 | b = [x ** 2 for x in to_process] 176 | t2e = time() 177 | print("Time (not parallelized): '{}'".format(t2e - t2s)) 178 | assert a == b 179 | -------------------------------------------------------------------------------- /cyther/system.py: -------------------------------------------------------------------------------- 1 | 2 | """ 3 | This module holds most of the 'constants' dealing with information on the 4 | user's system that doesn't change throughout the compilation process 5 | 6 | Holds tons of important low level functions and info 7 | """ 8 | 9 | import platform 10 | import sys 11 | import os 12 | import textwrap 13 | 14 | from .tools import CytherError 15 | from .searcher import where 16 | from .launcher import call 17 | from .definitions import MISSING_INCLUDE_DIRS, MISSING_RUNTIME_DIRS 18 | from .direct import getIncludeAndRuntime 19 | 20 | 21 | MAJOR = str(sys.version_info.major) 22 | MINOR = str(sys.version_info.minor) 23 | VER = MAJOR + MINOR 24 | 25 | CYTHONIZABLE_FILE_EXTS = ('.pyx', '.py') 26 | 27 | DRIVE, _ = os.path.splitdrive(sys.exec_prefix) 28 | if not DRIVE: 29 | DRIVE = os.path.normpath(os.sep) 30 | 31 | OPERATING_SYSTEM = platform.platform() 32 | 33 | IS_WINDOWS = OPERATING_SYSTEM.lower().startswith('windows') 34 | 35 | DEFAULT_OUTPUT_EXTENSION = '.pyd' if IS_WINDOWS else '.so' 36 | 37 | 38 | PYTHON_EXECUTABLE = where('python') 39 | ''' 40 | PYTHON_VERSION = call(['python', '--version'], 41 | raise_exception=True).extractVersion() 42 | ''' 43 | CYTHON_EXECUTABLE = where('cython') 44 | GCC_EXECUTABLE = where('gcc') 45 | 46 | gcc_output = call(['gcc', '-v'], raise_exception=True) 47 | #print("gcc output: '{}'".format(gcc_output)) 48 | GCC_INFO = gcc_output.getOutput() 49 | GCC_VERSION = gcc_output.extractVersion() 50 | 51 | cython_output = call(['cython', '-V'], raise_exception=True) 52 | #print("cython output: '{}'".format(cython_output)) 53 | CYTHON_OUTPUT = cython_output.getOutput() 54 | CYTHON_VERSION = cython_output.extractVersion() 55 | 56 | INFO = str() 57 | INFO += "\nSystem:" 58 | 59 | # TODO There must be a better way to do this... 60 | INFO += "\n\tPython ({}):".format(PYTHON_EXECUTABLE) 61 | INFO += "\n\t\tVersion: {}".format('.'.join(list(VER))) 62 | INFO += "\n\t\tOperating System: {}".format(OPERATING_SYSTEM) 63 | INFO += "\n\t\t\tOS is Windows: {}".format(IS_WINDOWS) 64 | INFO += "\n\t\tDefault Output Extension: {}".format(DEFAULT_OUTPUT_EXTENSION) 65 | INFO += "\n\t\tInstallation Directory: {}".format(sys.exec_prefix) 66 | INFO += '\n' 67 | INFO += "\n\tCython ({}) ({}):".format(CYTHON_VERSION, CYTHON_EXECUTABLE) 68 | INFO += "\n\t{}".format(textwrap.indent(CYTHON_OUTPUT, '\t')) 69 | 70 | INFO += "\n\tCyther:" 71 | INFO += "\n\t\tIncludable Header Search Command: {}".format('') 72 | INFO += "\n\t\tRuntime Library Search Command: {}".format('') 73 | INFO += "\n\t\tRuntime Library Name(s): {}".format('') 74 | INFO += "\n" 75 | INFO += "\n\tGCC ({}) ({}):".format(GCC_VERSION, GCC_EXECUTABLE) 76 | 77 | INFO += "\n{}".format(textwrap.indent(GCC_INFO.splitlines()[-1], '\t\t')) 78 | INFO += "\n" 79 | -------------------------------------------------------------------------------- /cyther/test.py: -------------------------------------------------------------------------------- 1 | 2 | """ 3 | This module holds the different functions for testing different aspects of 4 | Cyther. It tets Cyther's main operation as well as the underlying utilities 5 | Cyther uses. 6 | """ 7 | 8 | 9 | def test_all(): 10 | """ 11 | Tests everything 12 | """ 13 | test_utilities() 14 | test_compiler() 15 | 16 | 17 | def test_compiler(): 18 | """ 19 | Tests Cyther's entire core operation 20 | """ 21 | import subprocess 22 | import cyther 23 | 24 | cyther.info(None) 25 | cyther.core('cyther info') 26 | subprocess.call(['cyther', 'info']) 27 | 28 | print('<@test.py> All compilation tests have been passed') 29 | 30 | 31 | def test_utilities(): 32 | """ 33 | A function to test cyther's internal compilation and helper tools 34 | """ 35 | from .aberdeen import test_generateBatches, test_path, test_dict_file, \ 36 | test_extract, test_find, display_configure, display_resources 37 | from .direct import display_direct 38 | 39 | test_generateBatches() 40 | test_path() 41 | test_dict_file() 42 | test_extract() 43 | #test_find() 44 | display_direct() 45 | display_configure() 46 | display_resources() 47 | print('<@test.py> All utility tests have been passed') 48 | 49 | 50 | if __name__ == '__main__': 51 | test_all() 52 | -------------------------------------------------------------------------------- /cyther/test/example_file.pyx: -------------------------------------------------------------------------------- 1 | from math import sqrt 2 | 3 | cdef int triangular(int n): 4 | cdef: 5 | double q 6 | int r 7 | q = (n * (n + 1)) / 2 8 | r = int(q) 9 | return r 10 | 11 | def inverse_triangular(n): 12 | x = (sqrt(8 * n + 1) - 1) / 2 13 | n = int(x) 14 | if x - n > 0: 15 | return False 16 | return int(x) 17 | 18 | 19 | ''' 20 | @cyther 21 | 22 | a = ''.join([str(x) for x in range(10)]) 23 | ''' -------------------------------------------------------------------------------- /cyther/test/makefile: -------------------------------------------------------------------------------- 1 | all: randomtreetest 2 | 3 | randomtreetest: randomtreetest.o tree.o random.o 4 | gcc -g --std=gnu89 randomtreetest.o tree.o random.o -o randomtreetest 5 | 6 | randomtreetest.o: randomtreetest.c tree.h random.h 7 | gcc -g -c --std=gnu89 randomtreetest.c 8 | 9 | 10 | random.o: random.c random.h 11 | gcc -g -c random.c 12 | 13 | tree.o: tree.c tree.h 14 | gcc -g -c tree.c 15 | 16 | 17 | 18 | clean: 19 | rm -f randomtreetest 20 | rm -f *.o 21 | -------------------------------------------------------------------------------- /cyther/test/random.c: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | 4 | #include "random.h" 5 | 6 | /** Helper functions to aid with the random generation of data 7 | * @author Nicholas C. Pandolfi 8 | */ 9 | 10 | 11 | /** The function will produce a single printable character (ascii range 33 to 126) 12 | * @return A single randomized character 13 | */ 14 | 15 | char random_character(){ 16 | char c; 17 | int i; 18 | 19 | // Generate a random lowercase character (ASCII range 97 - 122 inclusive) 20 | i = random_number(97, 123); 21 | 22 | // Get the character that the int i represents 23 | c = (char)i; 24 | 25 | return c; 26 | } 27 | 28 | 29 | /** This function will generate a string of random printable characters in the range of 33 - 126 30 | * @param length The length of the string you want to return. Filled entirely with random characters. 31 | * @return The pointer to the null terminated string 32 | */ 33 | 34 | char *random_string(size_t length){ 35 | char *string; 36 | char c; 37 | 38 | // The memoy to allocate for the string 39 | string = (char *)malloc(length + 1); 40 | 41 | int i; 42 | // Create the string, character by character 43 | for (i = 0; i < length; i++){ 44 | c = random_character(); 45 | string[i] = c; 46 | } 47 | 48 | // Tack on the null terminator to finish construction of the string 49 | string[length] = '\0'; 50 | 51 | return string; 52 | } 53 | 54 | 55 | /** Generates a random number (int) 56 | * @param start The starting number to generate from (inclusive) 57 | * @param end The maximum number to generate to (exclusive) 58 | * @return The randomly generated numer (int) 59 | */ 60 | 61 | int random_number(int start, int end){ 62 | int random_num; 63 | 64 | // Generates the random number "[start, end)" 65 | random_num = (rand() % ((end - start))) + start; 66 | 67 | return random_num; 68 | 69 | } 70 | -------------------------------------------------------------------------------- /cyther/test/random.h: -------------------------------------------------------------------------------- 1 | /** Just some function prototypes of the random.c source code 2 | * @author Nicholas C. Pandolfi 3 | */ 4 | 5 | // Function prototypes 6 | 7 | char random_character(); 8 | char *random_string(size_t length); 9 | int random_number(int start, int end); 10 | -------------------------------------------------------------------------------- /cyther/test/randomtreetest.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "tree.h" 6 | #include "random.h" 7 | 8 | // The maximum/minimum length of the randomly generated strings 9 | #define MAX_LENGTH (20) 10 | #define MIN_LENGTH (10) 11 | 12 | // The amount of nodes to construct 13 | #define NODES (50) 14 | 15 | 16 | /** This main program will generate random strings of random length and feed them into a tree 17 | * one by one. It will then print the tree, which is already sorted. The traversal is done 18 | * 'inorder'. The tree is then freed. 19 | * @author Nicholas C. Pandolfi 20 | */ 21 | 22 | int main(){ 23 | // Initialize the random generator 24 | srand(time(NULL)); 25 | 26 | int length; 27 | char *value; 28 | 29 | // Initialize the first node (root of the tree) 30 | length = random_number(0, MAX_LENGTH); 31 | value = random_string(length); 32 | Tnode *tree = add_tnode(NULL, value); 33 | 34 | // Run the loop to add more nodes 35 | int i; 36 | for (i = 0; i < NODES; i++){ 37 | // Calculate the length of the random string (random) 38 | length = random_number(MIN_LENGTH, MAX_LENGTH); 39 | 40 | // Get the random string of 'length' length 41 | value = random_string(length); 42 | 43 | // Generate a new node with that new randomized string 44 | add_tnode(tree, value); 45 | } 46 | 47 | // Print the full tree generated 48 | printf("\n"); 49 | print_tree_inorder(tree); 50 | printf("\n"); 51 | 52 | // Free the tree and all the nodes in it ('1' means to also free the values inside of the nodes) 53 | free_tree(tree, 1); 54 | } 55 | -------------------------------------------------------------------------------- /cyther/test/readme.txt: -------------------------------------------------------------------------------- 1 | Author: Nicholas C. Pandolfi 2 | 3 | randomtreetest 4 | -------------- 5 | 6 | This main program does the same as 'treetest', except it creates its own random strings, composed 7 | of a random amount of random lowercase letters. It will generate 50 of these strings, and then 8 | make a node to hold each one. These nodes will get passed into the tree, and once fully 9 | constructed, will be printed in reverse alphabetical order. 10 | 11 | Example Usage: 12 | 13 | $ ./randomtreetest 14 | 15 | zuflxplpebhjtumjdc 16 | zsrympfovumnxyqgdr 17 | ygvfgfbuqwcmvysgxjt 18 | xykubxyeaocp 19 | xpjnetonctrcnwfblz 20 | wzigozzoapr 21 | vlnraxreqockczexvtz 22 | upailpxwhgab 23 | tfxhy 24 | stghknvfjhvghlmkeee 25 | rhbswpiyesvwsovsv 26 | qacsvuynggqdf 27 | pwiowppkrgc 28 | pacjogpqjj 29 | oxwzsqmbvqkwwiben 30 | ojqbnlpcpkvrztz 31 | ohauqgqmibwx 32 | obswvynkklmjwvoozys 33 | nrrrejuwpdisymmwobw 34 | npuxblbdbowpdyzvu 35 | ndaqarbwpjzuycudez 36 | nbfsjocveyyqxhknvad 37 | mosqjyrjootabob 38 | mfoohajjfcyuf 39 | matrnjszqweoh 40 | kmpubkxjrnlqq 41 | kmaenqbrgimb 42 | kikqoulwfsuznlcuunn 43 | kchackszet 44 | jtjuttilqcaqnsmmckq 45 | jktktsxjckqsmj 46 | ijmvgseppkycyjgdpc 47 | ihbtjojcaufk 48 | hmziznqinf 49 | gysdyclobmecdy 50 | gkhvwcdtjbqtu 51 | fnhumfbuhfac 52 | fegjpvbihqaxaytm 53 | fbfddiwpmoig 54 | emettinbnpdp 55 | elczhtermegaaucy 56 | ebmmhhxpfnmsvezgg 57 | cuvznarbeddeidzdc 58 | cnhmbkzspoqfs 59 | cimrtmnrqkwujv 60 | calggvlfypxlg 61 | broxzuyezdpwsbmcjcz 62 | bnjqytbmeajuojug 63 | atixxheghvilewuvod 64 | afadtvdlebxxqlum 65 | adkegjdwwaiipnjf 66 | 67 | 68 | 69 | 70 | -------------------------------------------------------------------------------- /cyther/test/tree.c: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "tree.h" 6 | 7 | 8 | /** Definitions for the creation and modification of a linked tree data structure 9 | * @author Nicholas C. Pandolfi 10 | */ 11 | 12 | 13 | /** This function adds a node to an already existing tree, or creates the first node to a new tree 14 | * @param current_tnode A pointer to the node to create off of. NULL can be used to denote a new tree 15 | * @param value The pointer to the string that the node will hold 16 | * @return The pointer to the Tnode created 17 | */ 18 | 19 | Tnode *add_tnode(Tnode *current_tnode, char *value){ 20 | if (!current_tnode){ 21 | // Allocate a new node 22 | Tnode *new_node = (Tnode *) malloc(sizeof(Tnode)); 23 | 24 | // Put the value into the new node, and initialize the pointers of both children to NULL 25 | new_node->value = value; 26 | new_node->left_child = NULL; 27 | new_node->right_child = NULL; 28 | 29 | // Return a pointer to that new node 30 | return new_node; 31 | 32 | } else { 33 | // Compare the value given vs the value of current_tnode 34 | int cmp; 35 | cmp = strcmp(current_tnode->value, value); 36 | 37 | // Branch according to the value (negative to left, positive to right) 38 | Tnode *return_value; 39 | if (cmp <= 0){ 40 | return_value = add_tnode(current_tnode->left_child, value); 41 | if (return_value){ 42 | current_tnode->left_child = return_value; 43 | } else return NULL; 44 | } else { 45 | return_value = add_tnode(current_tnode->right_child, value); 46 | if (return_value){ 47 | current_tnode->right_child = return_value; 48 | } else return NULL; 49 | } 50 | } 51 | } 52 | 53 | 54 | /** Print the tree given in order, treating the node passed in as the root node 55 | * @param current_tnode The root node to print 56 | */ 57 | 58 | void print_tree_inorder(Tnode *current_tnode){ 59 | 60 | // Visit the left child 61 | if (current_tnode->left_child){ 62 | print_tree_inorder(current_tnode->left_child); 63 | } 64 | 65 | // Print the value in the current node 66 | printf("%s\n", current_tnode->value); 67 | 68 | // Visit the right child 69 | if (current_tnode->right_child){ 70 | print_tree_inorder(current_tnode->right_child); 71 | } 72 | } 73 | 74 | 75 | /** Frees all nodes in the tree, but not the 'value' parameter in each node 76 | * @param current_tnode The node to tret as the root of a tree to be freed 77 | * @param free_value An int hat acts like a boolean to denote if user wants to ALSO free 78 | * value field in each node 79 | */ 80 | 81 | void free_tree(Tnode *current_tnode, int free_value){ 82 | // Free the left child 83 | if (current_tnode->left_child){ 84 | free_tree(current_tnode->left_child, free_value); 85 | } 86 | 87 | // Free the right child 88 | if (current_tnode->right_child){ 89 | free_tree(current_tnode->right_child, free_value); 90 | } 91 | 92 | // Free the current node 93 | if (free_value){ 94 | free(current_tnode->value); 95 | } 96 | free(current_tnode); 97 | } 98 | 99 | -------------------------------------------------------------------------------- /cyther/test/tree.h: -------------------------------------------------------------------------------- 1 | 2 | /** Just some function prototypes for the tree.c source code 3 | * @author Nicholas C. Pandolfi 4 | */ 5 | 6 | 7 | /** Struct to define a tree. Holds a c style string pointer and pointers to left and right children. 8 | */ 9 | 10 | struct tnode { 11 | char *value; 12 | struct tnode *left_child; 13 | struct tnode *right_child; 14 | }; 15 | 16 | typedef struct tnode Tnode; 17 | 18 | 19 | 20 | // Function prototypes 21 | 22 | Tnode *add_tnode(Tnode *current_tnode, char *value); 23 | void print_tree_inorder(Tnode *current_tnode); 24 | void free_tree(Tnode *current_tnode, int free_value); 25 | -------------------------------------------------------------------------------- /cyther/tools.py: -------------------------------------------------------------------------------- 1 | 2 | """ 3 | Provides the tools for Cyther's basic operations (functions that dont have 4 | a specific theme to them; miscellaneous 5 | """ 6 | 7 | import os 8 | import pkg_resources 9 | 10 | 11 | class CytherError(Exception): 12 | """A custom error used to denote that an exception was Cyther related""" 13 | def __init__(self, *args, **kwargs): 14 | super(CytherError, self).__init__(*args, **kwargs) 15 | 16 | 17 | def find_resource(r, *, pkg='cyther'): 18 | """ 19 | Finds a given cyther resource in the 'test' subdirectory in 20 | 'cyther' package 21 | """ 22 | file_path = pkg_resources.resource_filename(pkg, os.path.join('test', r)) 23 | if not os.path.isfile(file_path): 24 | msg = "Resource '{}' does not exist" 25 | raise FileNotFoundError(msg.format(file_path)) 26 | return file_path 27 | 28 | 29 | def isIterable(obj): 30 | """ 31 | Returns a boolean denoting if the object passed in is iterable 32 | """ 33 | try: 34 | _ = iter(obj) 35 | except TypeError: 36 | return False 37 | else: 38 | return True 39 | 40 | 41 | MULTIPLE = 'multiple' 42 | NONE = 'none' 43 | 44 | 45 | def process_output(output, *, condense=False, one=False, default=None, 46 | default_if_multiple=True, default_if_none=True): 47 | """ 48 | Taking a iterative container (list, tuple), this function will process its 49 | contents and condense if necessary. It also has functionality to try to 50 | assure that the output has only one item in it if desired. If this is not 51 | the case, then it will return a 'default' in place of the output, if 52 | specified. 53 | """ 54 | if condense: 55 | output = list(set(output)) 56 | 57 | if one: 58 | if len(output) > 1: 59 | if default and default_if_multiple: 60 | output = default 61 | else: 62 | return MULTIPLE 63 | elif len(output) == 1: 64 | output = output[0] 65 | else: 66 | if default and default_if_none: 67 | return default 68 | else: 69 | return NONE 70 | 71 | return output 72 | 73 | 74 | ASSERT_ERROR = "The search result:\n\t{}\nIs not equivalent to the assert " \ 75 | "test provided:\n\t{}" 76 | 77 | 78 | def assert_output(output, assert_equal): 79 | """ 80 | Check that two outputs have the same contents as one another, even if they 81 | aren't sorted yet 82 | """ 83 | sorted_output = sorted(output) 84 | sorted_assert = sorted(assert_equal) 85 | if sorted_output != sorted_assert: 86 | raise ValueError(ASSERT_ERROR.format(sorted_output, sorted_assert)) 87 | 88 | 89 | def write_dict_to_file(file_path, obj): 90 | """ 91 | Write a dictionary of string keys to a file 92 | """ 93 | lines = [] 94 | for key, value in obj.items(): 95 | lines.append(key + ':' + repr(value) + '\n') 96 | 97 | with open(file_path, 'w+') as file: 98 | file.writelines(lines) 99 | 100 | return None 101 | 102 | 103 | def read_dict_from_file(file_path): 104 | """ 105 | Read a dictionary of strings from a file 106 | """ 107 | with open(file_path) as file: 108 | lines = file.read().splitlines() 109 | 110 | obj = {} 111 | for line in lines: 112 | key, value = line.split(':', maxsplit=1) 113 | obj[key] = eval(value) 114 | 115 | return obj 116 | 117 | 118 | RESPONSES_ERROR = "Argument 'acceptableResponses' cannot be of type: '{}'" 119 | 120 | 121 | def get_input(prompt, check, *, redo_prompt=None, repeat_prompt=False): 122 | """ 123 | Ask the user to input something on the terminal level, check their response 124 | and ask again if they didn't answer correctly 125 | """ 126 | if isinstance(check, str): 127 | check = (check,) 128 | 129 | to_join = [] 130 | for item in check: 131 | if item: 132 | to_join.append(str(item)) 133 | else: 134 | to_join.append("''") 135 | 136 | prompt += " [{}]: ".format('/'.join(to_join)) 137 | 138 | if repeat_prompt: 139 | redo_prompt = prompt 140 | elif not redo_prompt: 141 | redo_prompt = "Incorrect input, please choose from {}: " \ 142 | "".format(str(check)) 143 | 144 | if callable(check): 145 | def _checker(r): return check(r) 146 | elif isinstance(check, tuple): 147 | def _checker(r): return r in check 148 | else: 149 | raise ValueError(RESPONSES_ERROR.format(type(check))) 150 | 151 | response = input(prompt) 152 | while not _checker(response): 153 | print(response, type(response)) 154 | response = input(redo_prompt if redo_prompt else prompt) 155 | return response 156 | 157 | 158 | def get_choice(prompt, choices): 159 | """ 160 | Asks for a single choice out of multiple items. 161 | Given those items, and a prompt to ask the user with 162 | """ 163 | print() 164 | checker = [] 165 | for offset, choice in enumerate(choices): 166 | number = offset + 1 167 | print("\t{}): '{}'\n".format(number, choice)) 168 | checker.append(str(number)) 169 | 170 | response = get_input(prompt, tuple(checker) + ('',)) 171 | if not response: 172 | print("Exiting...") 173 | exit() 174 | 175 | offset = int(response) - 1 176 | selected = choices[offset] 177 | 178 | return selected 179 | 180 | 181 | def _removeGivensFromTasks(tasks, givens): 182 | for given in givens: 183 | if given in tasks: 184 | raise Exception("Task '{}' is not supposed to be a" 185 | " given".format(given)) 186 | for task in tasks: 187 | if given in tasks[task]: 188 | tasks[task].remove(given) 189 | 190 | 191 | def generateBatches(tasks, givens): 192 | """ 193 | A function to generate a batch of commands to run in a specific order as to 194 | meet all the dependencies for each command. For example, the commands with 195 | no dependencies are run first, and the commands with the most deep 196 | dependencies are run last 197 | """ 198 | _removeGivensFromTasks(tasks, givens) 199 | 200 | batches = [] 201 | while tasks: 202 | batch = set() 203 | for task, dependencies in tasks.items(): 204 | if not dependencies: 205 | batch.add(task) 206 | 207 | if not batch: 208 | _batchErrorProcessing(tasks) 209 | 210 | for task in batch: 211 | del tasks[task] 212 | 213 | for task, dependencies in tasks.items(): 214 | for item in batch: 215 | if item in dependencies: 216 | tasks[task].remove(item) 217 | 218 | batches.append(batch) 219 | return batches 220 | 221 | 222 | GIVENS_NOT_SPECIFIED = "The dependencies '{}' should be givens if not " \ 223 | "specified as tasks" 224 | 225 | 226 | def _batchErrorProcessing(tasks): 227 | should_be_givens = [] 228 | total_deps = {dep for deps in tasks.values() for dep in deps} 229 | for dep in total_deps: 230 | if dep not in tasks: 231 | should_be_givens.append(dep) 232 | 233 | if should_be_givens: 234 | string = ', '.join(should_be_givens) 235 | message = GIVENS_NOT_SPECIFIED.format(string) 236 | else: 237 | message = "Circular dependency found:\n\t" 238 | msg = [] 239 | for task, dependencies in tasks.items(): 240 | for parent in dependencies: 241 | line = "{} -> {}".format(task, parent) 242 | msg.append(line) 243 | message += "\n\t".join(msg) 244 | 245 | raise ValueError(message) 246 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | universal = 1 3 | 4 | [metadata] 5 | description-file = README.rst 6 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | 2 | from setuptools import setup 3 | 4 | NAME = 'cyther' 5 | AUTHOR = 'Nicholas C. Pandolfi' 6 | AUTHOR_EMAIL = 'npandolfi@wpi.edu' 7 | URL = 'https://github.com/nickpandolfi/Cyther' 8 | LICENSE = 'MIT' 9 | 10 | 11 | SHORT_DESCRIPTION = "Cyther: The Cross-Platform Cython/Python/C Auto-Compiler" 12 | VERSION = '0.8.dev37' 13 | INSTALL_REQUIRES = ['cython', 'dill'] 14 | 15 | try: 16 | LONG_DESCRIPTION = open('README.txt').read() 17 | except FileNotFoundError: 18 | LONG_DESCRIPTION = SHORT_DESCRIPTION 19 | 20 | 21 | PACKAGES = ['cyther'] 22 | 23 | PACKAGE_DATA = {'cyther': ['test/*']} 24 | 25 | ENTRY_POINTS = {'console_scripts': ['cyther = cyther.__main__:main']} 26 | 27 | 28 | PLATFORMS = ['Windows', 'MacOS', 'POSIX', 'Unix'] 29 | 30 | KEYWORDS = ['Cyther', 'Cython', 'Python', 'MinGW32', 'vcvarsall.bat', 31 | 'vcvarsall not found', 'setup.py', 'gcc', 'Python 3', 32 | 'user-friendly', 'command-line', 'script', 'auto-compiler', 33 | 'compiler', 'integration', 'api'] 34 | 35 | CLASSIFIERS = ['Development Status :: 3 - Alpha', 36 | 'Environment :: Console', 37 | 'Topic :: Software Development :: Compilers', 38 | 'Topic :: Software Development :: Build Tools', 39 | 'Topic :: Desktop Environment :: File Managers', 40 | 'Intended Audience :: Developers', 41 | 'Intended Audience :: Science/Research', 42 | 'Operating System :: OS Independent', 43 | 'Programming Language :: Python', 44 | 'Programming Language :: Python :: 3.6', 45 | 'Programming Language :: Python :: 3.5', 46 | 'Programming Language :: Python :: 3.4', 47 | 'Programming Language :: Python :: 3.3', 48 | 'Programming Language :: Cython', 49 | 'License :: OSI Approved :: MIT License'] 50 | 51 | setup(name=NAME, 52 | version=VERSION, 53 | install_requires=INSTALL_REQUIRES, 54 | description=SHORT_DESCRIPTION, 55 | long_description=LONG_DESCRIPTION, 56 | packages=PACKAGES, 57 | 58 | package_data=PACKAGE_DATA, 59 | include_package_data=True, 60 | zip_safe=False, 61 | 62 | entry_points=ENTRY_POINTS, 63 | platforms=PLATFORMS, 64 | author=AUTHOR, 65 | author_email=AUTHOR_EMAIL, 66 | url=URL, 67 | license=LICENSE, 68 | keywords=KEYWORDS, 69 | classifiers=CLASSIFIERS) 70 | --------------------------------------------------------------------------------