├── .gitignore ├── .pytest_cache └── v │ └── cache │ ├── lastfailed │ └── nodeids ├── LICENSE.txt ├── README.md ├── gslab_fill ├── __init__.py ├── tablefill.py ├── tablefill_info.py ├── tests │ ├── __init__.py │ ├── input │ │ ├── alternative_prefix.log │ │ ├── legal.log │ │ ├── logs_for_textfill.do │ │ ├── tablefill_template.lyx │ │ ├── tablefill_template.tex │ │ ├── tablefill_template_breaks.lyx │ │ ├── tablefill_template_breaks.tex │ │ ├── tablefill_template_filled.lyx │ │ ├── tablefill_template_filled.tex │ │ ├── tables_appendix.txt │ │ ├── tables_appendix_two.txt │ │ ├── tags_dont_match.log │ │ ├── tags_incorrectly_named.log │ │ ├── tags_not_closed.log │ │ ├── textfill_template.lyx │ │ └── textfill_template_filled.lyx │ ├── log │ │ └── make.log │ ├── test_tablefill.py │ └── test_textfill.py ├── textfill.py └── textfill_info.py ├── gslab_make ├── __init__.py ├── dir_mod.py ├── get_externals.py ├── get_externals_github.py ├── make_link_logs.py ├── make_links.py ├── make_log.py ├── private │ ├── __init__.py │ ├── exceptionclasses.py │ ├── getexternalsdirectives.py │ ├── linkdirectives.py │ ├── linkslist.py │ ├── messages.py │ ├── metadata.py │ ├── preliminaries.py │ └── runprogramdirective.py ├── run_program.py └── tests │ ├── __init__.py │ └── nostderrout.py ├── gslab_misc ├── SaveData │ ├── SaveData.py │ ├── __init__.py │ └── tests │ │ ├── __init__.py │ │ ├── data │ │ └── data.csv │ │ └── test_main.py ├── __init__.py └── gencat │ ├── __init__.py │ ├── gencat.py │ └── tests │ ├── __init__.py │ ├── log │ ├── make.log │ └── test.log │ ├── run_all_tests.py │ ├── test_checkDicts.py │ ├── test_cleanDir.py │ ├── test_main.py │ ├── test_unzipFiles.py │ ├── test_writeDict.py │ └── test_zipFile.py ├── gslab_scons ├── README.md ├── __init__.py ├── _exception_classes.py ├── _release_tools.py ├── builders │ ├── __init__.py │ ├── build_anything.py │ ├── build_latex.py │ ├── build_lyx.py │ ├── build_mathematica.py │ ├── build_matlab.py │ ├── build_python.py │ ├── build_r.py │ ├── build_stata.py │ ├── build_tables.py │ └── gslab_builder.py ├── check_prereq.py ├── log.py ├── log_paths_dict.py ├── misc.py ├── release.py ├── scons_debrief.py └── tests │ ├── __init__.py │ ├── _side_effects.py │ ├── _test_helpers.py │ ├── assets_listing.txt │ ├── config_user.yaml │ ├── input │ └── lyx_test_dependencies.lyx │ ├── test_build_latex.py │ ├── test_build_lyx.py │ ├── test_build_mathematica.py │ ├── test_build_matlab.py │ ├── test_build_python.py │ ├── test_build_r.py │ ├── test_build_stata.py │ ├── test_build_tables.py │ ├── test_configuration_tests.py │ ├── test_log.py │ ├── test_misc.py │ ├── test_release_function.py │ ├── test_release_tools.py │ └── test_size_warning.py ├── setup.cfg ├── setup.py └── test.log /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *.ipynb* 3 | *.egg* 4 | .cache 5 | .coverage 6 | .DS_Store 7 | *.Rhistory 8 | *.coveragerc 9 | *.ipynb 10 | .ipynb_checkpoints/ 11 | build/ 12 | issue* 13 | -------------------------------------------------------------------------------- /.pytest_cache/v/cache/lastfailed: -------------------------------------------------------------------------------- 1 | { 2 | "gslab_scons/tests/test_build_latex.py": true, 3 | "gslab_scons/tests/test_build_lyx.py": true, 4 | "gslab_scons/tests/test_build_matlab.py": true, 5 | "gslab_scons/tests/test_build_python.py": true, 6 | "gslab_scons/tests/test_build_r.py": true, 7 | "gslab_scons/tests/test_build_stata.py": true, 8 | "gslab_scons/tests/test_build_tables.py": true, 9 | "gslab_scons/tests/test_configuration_tests.py": true, 10 | "gslab_scons/tests/test_log.py": true, 11 | "gslab_scons/tests/test_misc.py": true, 12 | "gslab_scons/tests/test_release_function.py": true, 13 | "gslab_scons/tests/test_release_tools.py": true, 14 | "gslab_scons/tests/test_size_warning.py": true 15 | } -------------------------------------------------------------------------------- /.pytest_cache/v/cache/nodeids: -------------------------------------------------------------------------------- 1 | [ 2 | "gslab_fill/tests/test_tablefill.py::testTablefill::testArgumentOrder", 3 | "gslab_fill/tests/test_tablefill.py::testTablefill::testBreaksRoundingString", 4 | "gslab_fill/tests/test_tablefill.py::testTablefill::testIllegalSyntax", 5 | "gslab_fill/tests/test_tablefill.py::testTablefill::testInput", 6 | "gslab_fill/tests/test_textfill.py::testTextfill::test_alternative_prefix", 7 | "gslab_fill/tests/test_textfill.py::testTextfill::test_argument_order", 8 | "gslab_fill/tests/test_textfill.py::testTextfill::test_illegal_syntax", 9 | "gslab_fill/tests/test_textfill.py::testTextfill::test_input", 10 | "gslab_fill/tests/test_textfill.py::testTextfill::test_remove_echoes", 11 | "gslab_fill/tests/test_textfill.py::testTextfill::test_tags_dont_match", 12 | "gslab_fill/tests/test_textfill.py::testTextfill::test_tags_incorrectly_specified", 13 | "gslab_fill/tests/test_textfill.py::testTextfill::test_tags_not_closed", 14 | "gslab_misc/gencat/tests/test_checkDicts.py::test_checkDicts::test_default", 15 | "gslab_misc/gencat/tests/test_checkDicts.py::test_checkDicts::test_notAllDicts", 16 | "gslab_misc/gencat/tests/test_checkDicts.py::test_checkDicts::test_notAllTuple", 17 | "gslab_misc/gencat/tests/test_cleanDir.py::test_cleanDir::test_cleanDir", 18 | "gslab_misc/gencat/tests/test_cleanDir.py::test_cleanDir::test_makeDir", 19 | "gslab_misc/gencat/tests/test_cleanDir.py::test_cleanDir::test_noNew", 20 | "gslab_misc/gencat/tests/test_cleanDir.py::test_cleanDir::test_recursiveClear", 21 | "gslab_misc/gencat/tests/test_cleanDir.py::test_cleanDir::test_recursiveMake", 22 | "gslab_misc/gencat/tests/test_cleanDir.py::test_cleanDir::test_strangeFlag", 23 | "gslab_misc/gencat/tests/test_main.py::test_main::test_default", 24 | "gslab_misc/gencat/tests/test_unzipFiles.py::test_unzipFiles::test_aZipFile", 25 | "gslab_misc/gencat/tests/test_unzipFiles.py::test_unzipFiles::test_blankZipFile", 26 | "gslab_misc/gencat/tests/test_unzipFiles.py::test_unzipFiles::test_noFile", 27 | "gslab_misc/gencat/tests/test_unzipFiles.py::test_unzipFiles::test_noZipFile", 28 | "gslab_misc/gencat/tests/test_unzipFiles.py::test_unzipFiles::test_twoFile", 29 | "gslab_misc/gencat/tests/test_unzipFiles.py::test_unzipFiles::test_twoZipFile", 30 | "gslab_misc/gencat/tests/test_writeDict.py::test_writeDict::test_noPath", 31 | "gslab_misc/gencat/tests/test_writeDict.py::test_writeDict::test_onePath", 32 | "gslab_misc/gencat/tests/test_writeDict.py::test_writeDict::test_twoFile", 33 | "gslab_misc/gencat/tests/test_writeDict.py::test_writeDict::test_twopPath", 34 | "gslab_misc/gencat/tests/test_zipFile.py::test_zipFiles::test_oneFile", 35 | "gslab_misc/gencat/tests/test_zipFile.py::test_zipFiles::test_twoConcatsOneZip", 36 | "gslab_misc/gencat/tests/test_zipFile.py::test_zipFiles::test_twoFile", 37 | "gslab_misc/gencat/tests/test_zipFile.py::test_zipFiles::test_twoZips" 38 | ] -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2016 Matthew Gentzkow, Jesse Shapiro 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # GSLab Python Library Collection 4.1.3 2 | 3 | Overview 4 | -------- 5 | This repository contains the following GSLab Python libraries: 6 | - gslab_make 7 | - gslab_fill 8 | - gslab_scons 9 | - gencat 10 | 11 | Information about each of these packages is available in its internal documentation. 12 | 13 | Requirements 14 | ------------ 15 | - Python 2.7 16 | - `setuptools` ([installation instructions](https://packaging.python.org/installing/)) 17 | - `mock` (> 2.0.0) and `coverage` in order to run the unit tests 18 | 19 | Installation 20 | ------------ 21 | 22 | The preferred installation method is to use [pip](https://pypi.python.org/pypi/pip): 23 | ``` 24 | pip install git+ssh://git@github.com/gslab-econ/gslab_python.git@master 25 | ``` 26 | or 27 | ``` 28 | pip install git+https://git@github.com/gslab-econ/gslab_python.git@master 29 | ``` 30 | which are the SSH and HTTPS protocol versions. 31 | 32 | The package at any tagged release, branch, or commit can be installed with the same commands, just changing `master` to the desirved target e.g., 33 | ``` 34 | pip install git+ssh://git@github.com/gslab-econ/gslab_python.git@ 35 | ``` 36 | 37 | 38 | Note that this installation procedure may require obtaining machine privileges through, 39 | say, a `sudo` command. 40 | 41 | 42 | Alternatively, one may install the local version of gslab_python by running (from the root of the repository) 43 | 44 | ``` 45 | pip install . 46 | ``` 47 | 48 | We do not reccommend that these packages be installed by executing 49 | ```bash 50 | python setup.py install 51 | ``` 52 | This method of installation uses egg files rather than Wheels, which can cause conflicts with previous versions of `gslab_tools`. If this method of installation is executed, some files need to be removed from the directory with a `clean` argument. `clean` removes `/build`,`/dist`, and `GSLab_Tools.egg-info`, which are built upon installation. This argument can be called by executing 53 | 54 | ```bash 55 | python setup.py clean 56 | ``` 57 | 58 | 59 | Testing 60 | ------- 61 | 62 | We recommend that users use [coverage](https://pypi.python.org/pypi/coverage/) 63 | to run this repository's unit tests. Upon installing coverage (this can be done with 64 | pip using the command `pip install coverage`), one may test `gslab_python`'s contents 65 | and then produce a code coverage report the commands: 66 | 67 | ```bash 68 | python setup.py test [--include=] 69 | ``` 70 | 71 | Here, the optional `--include=` argument specifies the files whose test results 72 | should be included in the coverage report produced by the command. 73 | It works as `coverage`'s argument of the same name does. The command should be 74 | run without this option before committing to `gslab_python`. 75 | 76 | 77 | License 78 | ------- 79 | See [here](https://github.com/gslab-econ/gslab_python/blob/master/LICENSE.txt). 80 | 81 | FAQs 82 | ------- 83 | 84 | Q: What if I want to install a different branch called `dev` of `gslab_python` rather than `master`? 85 | A: Either `git checkout dev` that branch of the repo before installing, or change `@master` to `@dev` in the `pip install` instruction. 86 | 87 | -------------------------------------------------------------------------------- /gslab_fill/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | gslab_fill: A library for filling LyX templates with tablular data and Stata output 3 | =================================================================================== 4 | 5 | gslab_fill provides two functions for filling LyX template files with data. 6 | These are `tablefill` and `textfill`. Please see their docstrings for informations 7 | on their use and functionalities. 8 | ''' 9 | 10 | from tablefill import tablefill 11 | from textfill import textfill 12 | -------------------------------------------------------------------------------- /gslab_fill/tests/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | This directory contains unit tests for `gslab_fill`. These tests can be run using 3 | `python -m unittest discover` 4 | from `gslab_fill/` or from `gslab_fill/tests/`. To run the tests with logging, use: 5 | `python run_all_tests.py` 6 | from `gslab_fill/tests/`. 7 | 8 | The following inputs of these tests were created by running logs_for_textfill.do, 9 | a Stata do file that is now stored in the subdirectory. 10 | - legal.log 11 | - alternative_prefix.log 12 | - tags_not_closed.log 13 | - tags_incorrectly_named.log 14 | - tags_dont_match.log 15 | ''' 16 | -------------------------------------------------------------------------------- /gslab_fill/tests/input/logs_for_textfill.do: -------------------------------------------------------------------------------- 1 | /********************************************************** 2 | * 3 | * LOGS_FOR_TEXTFILL.DO 4 | * Outputs example stata logs tagged for textfill 5 | * 6 | **********************************************************/ 7 | 8 | version 11 9 | set more off 10 | adopath + ../../external/gslab_misc 11 | preliminaries 12 | 13 | program main 14 | setup_data 15 | legal 16 | tags_dont_match 17 | tag_not_closed 18 | tags_incorrectly_named 19 | alternative_prefix 20 | * I run what would be small_table outside main, so that echoes show up on the logfile 21 | end 22 | 23 | program setup_data 24 | set obs 100 25 | gen x = uniform() 26 | gen y = uniform() 27 | end 28 | 29 | program legal 30 | log using ../../log/stata_output_for_textfill/legal.log, replace name(legal_long) 31 | insert_tag test_long, open 32 | summ x, det 33 | summ y, det 34 | summ 35 | regress y x 36 | regress x y 37 | regress x y, robust 38 | insert_tag test_long, close 39 | log close legal_long 40 | 41 | log using ../../log/stata_output_for_textfill/legal.log, append name(legal_short) 42 | insert_tag test_small, open 43 | regress y x 44 | insert_tag test_small, close 45 | log close legal_short 46 | end 47 | 48 | program tags_dont_match 49 | log using ../../log/stata_output_for_textfill/tags_dont_match.log, replace name(nomatch) 50 | insert_tag start, open 51 | tab x 52 | tab y 53 | insert_tag end, close 54 | log close nomatch 55 | end 56 | 57 | program tag_not_closed 58 | log using ../../log/stata_output_for_textfill/tags_not_closed.log, replace name(noend) 59 | insert_tag tag, open 60 | tab x 61 | tab y 62 | log close noend 63 | end 64 | 65 | program tags_incorrectly_named 66 | log using ../../log/stata_output_for_textfill/tags_incorrectly_named.log, replace name(badnames) 67 | insert_tag test_long, open prefix(different) 68 | tab x 69 | tab y 70 | insert_tag test_long, close prefix(different) 71 | log close badnames 72 | end 73 | 74 | program alternative_prefix 75 | log using ../../log/stata_output_for_textfill/alternative_prefix.log, replace name(prefix_long) 76 | insert_tag test_long, open prefix(prefix) 77 | summ x, det 78 | summ y, det 79 | summ 80 | regress y x 81 | regress x y 82 | regress x y, robust 83 | insert_tag test_long, close prefix(prefix) 84 | log close prefix_long 85 | 86 | log using ../../log/stata_output_for_textfill/alternative_prefix.log, append name(prefix_short) 87 | insert_tag test_small, open prefix(prefix) 88 | regress y x 89 | insert_tag test_small, close prefix(prefix) 90 | log close prefix_short 91 | end 92 | 93 | * EXECUTE 94 | main 95 | 96 | -------------------------------------------------------------------------------- /gslab_fill/tests/input/tablefill_template.tex: -------------------------------------------------------------------------------- 1 | \documentclass[english]{article} 2 | \usepackage[T1]{fontenc} 3 | \usepackage[latin9]{inputenc} 4 | \usepackage{array} 5 | \usepackage{float} 6 | 7 | \makeatletter 8 | 9 | \providecommand{\tabularnewline}{\\} 10 | 11 | \makeatother 12 | 13 | \usepackage{babel} 14 | \begin{document} 15 | \begin{table}[H] 16 | \caption{Determinants of Newspaper Affiliation\label{tab:panel_supply}} 17 | \smallskip{} 18 | 19 | Dependent variable: Dummy for newspaper choosing Republican affiliation 20 | \begin{raggedright} 21 | \begin{tabular}{lccc} 22 | \hline 23 | & (1) & (2) & (3)\tabularnewline 24 | \hline 25 | Republican vote share & #4,# & #4,# & #4,#\tabularnewline 26 | & (#4,#) & (###) & (###)\tabularnewline 27 | Number of Republican incumbents & #0# & #4# & #4#\tabularnewline 28 | & (#4#) & (#4#) & (#4#)\tabularnewline 29 | Number of Democratic incumbents & #4# & #4# & #4#\tabularnewline 30 | & (#4#) & (#4#) & (#4#)\tabularnewline 31 | Lag Republican vote share & & & #4#\tabularnewline 32 | & & & (#4#)\tabularnewline 33 | Instrument with lag vote share? & & X & \tabularnewline 34 | \hline 35 | R2 & #4# & #4# & #4#\tabularnewline 36 | Number of markets & #0# & #0# & #0#\tabularnewline 37 | Number of newspapers & #0# & #0# & #0#\tabularnewline 38 | \hline 39 | \end{tabular} 40 | \par\end{raggedright} 41 | \raggedright{}{\footnotesize{}}% 42 | \begin{tabular}{>{\raggedright}p{6in}} 43 | \noalign{\vskip\doublerulesep} 44 | \raggedright{}\tabularnewline 45 | \end{tabular}{\footnotesize \par} 46 | \end{table} 47 | 48 | \begin{table}[H] 49 | \caption{Sensitivity of Parameter Estimates to Omitting Unobservables From 50 | Model\label{tab:unobservables}} 51 | \smallskip{} 52 | 53 | \centering{}% 54 | \begin{tabular}{lcc} 55 | \hline 56 | & Baseline & No Unobservables\tabularnewline 57 | \hline 58 | \noalign{\vskip\doublerulesep} 59 | Demand parameters & & \tabularnewline 60 | & & \tabularnewline 61 | \qquad{}$\underline{\beta}$ & ### & #2#\tabularnewline 62 | & (#0#) & (#0#)\tabularnewline[\doublerulesep] 63 | \qquad{}$\overline{\beta}$ & #0# & #0#\tabularnewline 64 | & (#10#) & (#10#)\tabularnewline[\doublerulesep] 65 | \qquad{}$\Gamma$ & #4# & #4#\tabularnewline 66 | & (#4#) & (#4#)\tabularnewline[\doublerulesep] 67 | & & \tabularnewline 68 | Supply parameters & & \tabularnewline 69 | & & \tabularnewline 70 | \qquad{}$a_{l}$ & #4# & #4#\tabularnewline 71 | & (#4#) & (#4#)\tabularnewline[\doublerulesep] 72 | \qquad{}$\sigma_{\xi}$ & #4# & #4#\tabularnewline 73 | & (#4#) & (#4#)\tabularnewline[\doublerulesep] 74 | \hline 75 | \end{tabular} 76 | \end{table} 77 | 78 | \begin{table} 79 | \begin{centering} 80 | \caption{Determinants of Equilibrium Diversity\label{tab:Diversity}} 81 | \smallskip{} 82 | \par\end{centering} 83 | \centering{}% 84 | \begin{tabular}{lccc} 85 | \hline 86 | & Markets with & Share of hhlds & Share of hhlds\tabularnewline 87 | & diverse & in mkt with & reading\tabularnewline 88 | & papers & diverse papers & diverse papers\tabularnewline 89 | \hline 90 | \noalign{\vskip0.01\textheight} 91 | Baseline & #0# & #2# & #3#\tabularnewline 92 | When choosing affiliation, newspapers: & & & \tabularnewline 93 | \noalign{\vskip\doublerulesep} 94 | \quad{}Ignore competitors' choices & #0# & #2# & #3#\tabularnewline[\doublerulesep] 95 | \noalign{\vskip\doublerulesep} 96 | \quad{}Ignore household ideology & #0# & #2# & #3#\tabularnewline[\doublerulesep] 97 | \noalign{\vskip\doublerulesep} 98 | \quad{}Ignore idiosyncratic cost shocks ($\xi$) & #0# & #2# & #3#\tabularnewline 99 | Owners chosen at random from & & & \tabularnewline 100 | local households and newspaper & & & \tabularnewline 101 | type equals owner type & #0# & #2# & #3#\tabularnewline[\doublerulesep] 102 | \hline 103 | \noalign{\vskip0.01\textheight} 104 | \end{tabular} 105 | \end{table} 106 | 107 | \end{document} 108 | -------------------------------------------------------------------------------- /gslab_fill/tests/input/tablefill_template_breaks.tex: -------------------------------------------------------------------------------- 1 | \documentclass[english]{article} 2 | \usepackage[T1]{fontenc} 3 | \usepackage[latin9]{inputenc} 4 | \usepackage{array} 5 | \usepackage{float} 6 | 7 | \makeatletter 8 | 9 | \providecommand{\tabularnewline}{\\} 10 | 11 | \makeatother 12 | 13 | \usepackage{babel} 14 | \begin{document} 15 | \begin{table}[H] 16 | \caption{Determinants of Newspaper Affiliation\label{tab:panel_supply}} 17 | \smallskip{} 18 | 19 | Dependent variable: Dummy for newspaper choosing Republican affiliation 20 | \begin{raggedright} 21 | \begin{tabular}{lccc} 22 | \hline 23 | & (1) & (2) & (3)\tabularnewline 24 | \hline 25 | Republican vote share & #4,# & #4,# & #4,#\tabularnewline 26 | & (#4,#) & (###) & (###)\tabularnewline 27 | Number of Republican incumbents & #0# & #4# & #4#\tabularnewline 28 | & (#4#) & (#4#) & (#4#)\tabularnewline 29 | Number of Democratic incumbents & #4# & #4# & #4#\tabularnewline 30 | & (#4#) & (#4#) & (#4#)\tabularnewline 31 | Lag Republican vote share & & & #4#\tabularnewline 32 | & & & (#4#)\tabularnewline 33 | Instrument with lag vote share? & & X & \tabularnewline 34 | \hline 35 | R2 & #4# & #4# & #4#\tabularnewline 36 | Number of markets & #0# & #0# & #0#\tabularnewline 37 | Number of newspapers & #0# & #0# & #0#\tabularnewline 38 | \hline 39 | \end{tabular} 40 | \par\end{raggedright} 41 | \raggedright{}{\footnotesize{}}% 42 | \begin{tabular}{>{\raggedright}p{6in}} 43 | \noalign{\vskip\doublerulesep} 44 | \raggedright{}\tabularnewline 45 | \end{tabular}{\footnotesize \par} 46 | \end{table} 47 | 48 | \begin{table}[H] 49 | \caption{Sensitivity of Parameter Estimates to Omitting Unobservables From 50 | Model\label{tab:unobservables}} 51 | \smallskip{} 52 | 53 | \centering{}% 54 | \begin{tabular}{lcc} 55 | \hline 56 | & Baseline & No Unobservables\tabularnewline 57 | \hline 58 | \noalign{\vskip\doublerulesep} 59 | Demand parameters & & \tabularnewline 60 | & & \tabularnewline 61 | \qquad{}$\underline{\beta}$ & #2# & #2#\tabularnewline 62 | & (#0#) & (#0#)\tabularnewline[\doublerulesep] 63 | \qquad{}$\overline{\beta}$ & #0# & #0#\tabularnewline 64 | & (#10#) & (#10#)\tabularnewline[\doublerulesep] 65 | \qquad{}$\Gamma$ & #4# & #4#\tabularnewline 66 | & (#4#) & (#4#)\tabularnewline[\doublerulesep] 67 | & & \tabularnewline 68 | Supply parameters & & \tabularnewline 69 | & & \tabularnewline 70 | \qquad{}$a_{l}$ & #4# & #4#\tabularnewline 71 | & (#4#) & (#4#)\tabularnewline[\doublerulesep] 72 | \qquad{}$\sigma_{\xi}$ & #4# & #4#\tabularnewline 73 | & (#4#) & (#4#)\tabularnewline[\doublerulesep] 74 | \hline 75 | \end{tabular} 76 | \end{table} 77 | 78 | \begin{table} 79 | \begin{centering} 80 | \caption{Determinants of Equilibrium Diversity\label{tab:Diversity}} 81 | \smallskip{} 82 | \par\end{centering} 83 | \centering{}% 84 | \begin{tabular}{lccc} 85 | \hline 86 | & Markets with & Share of hhlds & Share of hhlds\tabularnewline 87 | & diverse & in mkt with & reading\tabularnewline 88 | & papers & diverse papers & diverse papers\tabularnewline 89 | \hline 90 | \noalign{\vskip0.01\textheight} 91 | Baseline & #0# & #2# & #3#\tabularnewline 92 | When choosing affiliation, newspapers: & & & \tabularnewline 93 | \noalign{\vskip\doublerulesep} 94 | \quad{}Ignore competitors' choices & #0# & #2# & #3#\tabularnewline[\doublerulesep] 95 | \noalign{\vskip\doublerulesep} 96 | \quad{}Ignore household ideology & #0# & #2# & #3#\tabularnewline[\doublerulesep] 97 | \noalign{\vskip\doublerulesep} 98 | \quad{}Ignore idiosyncratic cost shocks ($\xi$) & #0# & #2# & #3#\tabularnewline 99 | Owners chosen at random from & & & \tabularnewline 100 | local households and newspaper & & & \tabularnewline 101 | type equals owner type & #0# & #2# & #3#\tabularnewline[\doublerulesep] 102 | \hline 103 | \noalign{\vskip0.01\textheight} 104 | \end{tabular} 105 | \end{table} 106 | 107 | \end{document} 108 | -------------------------------------------------------------------------------- /gslab_fill/tests/input/tablefill_template_filled.tex: -------------------------------------------------------------------------------- 1 | \documentclass[english]{article} 2 | \usepackage[T1]{fontenc} 3 | \usepackage[latin9]{inputenc} 4 | \usepackage{array} 5 | \usepackage{float} 6 | 7 | \makeatletter 8 | 9 | \providecommand{\tabularnewline}{\\} 10 | 11 | \makeatother 12 | 13 | \usepackage{babel} 14 | \begin{document} 15 | \begin{table}[H] 16 | \caption{Determinants of Newspaper Affiliation\label{tab:panel_supply}} 17 | \smallskip{} 18 | 19 | Dependent variable: Dummy for newspaper choosing Republican affiliation 20 | \begin{raggedright} 21 | \begin{tabular}{lccc} 22 | \hline 23 | & (1) & (2) & (3)\tabularnewline 24 | \hline 25 | Republican vote share & 2,000.1355 & 20,000.2353 & 10,000.9424\tabularnewline 26 | & (10,558,000.0000) & (0.0703) & (0.1021)\tabularnewline 27 | Number of Republican incumbents & 100 & -0.0813 & -0.0757\tabularnewline 28 | & (0.0124) & (0.0128) & (0.0123)\tabularnewline 29 | Number of Democratic incumbents & 0.0644 & 0.0707 & 0.0644\tabularnewline 30 | & (0.0120) & (0.0124) & (0.0120)\tabularnewline 31 | Lag Republican vote share & & & 0.2034\tabularnewline 32 | & & & (0.0869)\tabularnewline 33 | Instrument with lag vote share? & & X & \tabularnewline 34 | \hline 35 | R2 & 0.2860 & 0.2854 & 0.2871\tabularnewline 36 | Number of markets & 1336 & 1336 & 1336\tabularnewline 37 | Number of newspapers & 3177 & 3177 & 3177\tabularnewline 38 | \hline 39 | \end{tabular} 40 | \par\end{raggedright} 41 | \raggedright{}{\footnotesize{}}% 42 | \begin{tabular}{>{\raggedright}p{6in}} 43 | \noalign{\vskip\doublerulesep} 44 | \raggedright{}\tabularnewline 45 | \end{tabular}{\footnotesize \par} 46 | \end{table} 47 | 48 | \begin{table}[H] 49 | \caption{Sensitivity of Parameter Estimates to Omitting Unobservables From 50 | Model\label{tab:unobservables}} 51 | \smallskip{} 52 | 53 | \centering{}% 54 | \begin{tabular}{lcc} 55 | \hline 56 | & Baseline & No Unobservables\tabularnewline 57 | \hline 58 | \noalign{\vskip\doublerulesep} 59 | Demand parameters & & \tabularnewline 60 | & & \tabularnewline 61 | \qquad{}$\underline{\beta}$ & String & -0.13\tabularnewline 62 | & (0) & (0)\tabularnewline[\doublerulesep] 63 | \qquad{}$\overline{\beta}$ & 1 & 1\tabularnewline 64 | & (0.0664110000) & (0.0528430000)\tabularnewline[\doublerulesep] 65 | \qquad{}$\Gamma$ & 0.2438 & 0.1532\tabularnewline 66 | & (0.0561) & (0.0471)\tabularnewline[\doublerulesep] 67 | & & \tabularnewline 68 | Supply parameters & & \tabularnewline 69 | & & \tabularnewline 70 | \qquad{}$a_{l}$ & 6.5121 & 6.5974\tabularnewline 71 | & (0.8944) & (0.8917)\tabularnewline[\doublerulesep] 72 | \qquad{}$\sigma_{\xi}$ & 0.2005 & 0.1826\tabularnewline 73 | & (0.0267) & (0.0238)\tabularnewline[\doublerulesep] 74 | \hline 75 | \end{tabular} 76 | \end{table} 77 | 78 | \begin{table} 79 | \begin{centering} 80 | \caption{Determinants of Equilibrium Diversity\label{tab:Diversity}} 81 | \smallskip{} 82 | \par\end{centering} 83 | \centering{}% 84 | \begin{tabular}{lccc} 85 | \hline 86 | & Markets with & Share of hhlds & Share of hhlds\tabularnewline 87 | & diverse & in mkt with & reading\tabularnewline 88 | & papers & diverse papers & diverse papers\tabularnewline 89 | \hline 90 | \noalign{\vskip0.01\textheight} 91 | Baseline & 140 & 0.22 & 0.036\tabularnewline 92 | When choosing affiliation, newspapers: & & & \tabularnewline 93 | \noalign{\vskip\doublerulesep} 94 | \quad{}Ignore competitors' choices & 87 & 0.14 & 0.022\tabularnewline[\doublerulesep] 95 | \noalign{\vskip\doublerulesep} 96 | \quad{}Ignore household ideology & 208 & 0.30 & 0.048\tabularnewline[\doublerulesep] 97 | \noalign{\vskip\doublerulesep} 98 | \quad{}Ignore idiosyncratic cost shocks ($\xi$) & 106 & 0.17 & 0.030\tabularnewline 99 | Owners chosen at random from & & & \tabularnewline 100 | local households and newspaper & & & \tabularnewline 101 | type equals owner type & 150 & 0.23 & 0.038\tabularnewline[\doublerulesep] 102 | \hline 103 | \noalign{\vskip0.01\textheight} 104 | \end{tabular} 105 | \end{table} 106 | 107 | \end{document} 108 | -------------------------------------------------------------------------------- /gslab_fill/tests/input/tables_appendix.txt: -------------------------------------------------------------------------------- 1 | 2 | 2000.1355 20000.2353 10000.9424 3 | 1.0558e+7 0.0703 0.1021 4 | 100 -0.0813 -0.0757 5 | 0.0124 0.0128 0.0123 6 | 0.0644 0.0707 0.0644 7 | 0.0120 0.0124 0.0120 8 | . . 0.2034 9 | . . 0.0869 10 | 0.2860 0.2854 0.2871 11 | 1336.0000 1336.0000 1336.0000 12 | 3177.0000 3177.0000 3177.0000 13 | 14 | 139.6000 0.2171 0.0355 15 | 87.0000 0.1425 0.0224 16 | 208.4000 0.3048 0.0482 17 | 106.0000 0.1743 0.0295 18 | 150.0000 0.2299 0.0377 -------------------------------------------------------------------------------- /gslab_fill/tests/input/tables_appendix_two.txt: -------------------------------------------------------------------------------- 1 | 2 | String -0.12976 3 | 0.05923 0.047828 4 | 0.76394 0.69839 5 | 0.066411 0.052843 6 | 0.24382 0.15318 7 | 0.056142 0.047061 8 | 6.5121 6.5974 9 | 0.89442 0.8917 10 | 0.20046 0.18256 11 | 0.026702 0.023789 12 | -------------------------------------------------------------------------------- /gslab_fill/tests/input/textfill_template.lyx: -------------------------------------------------------------------------------- 1 | #LyX 2.0 created this file. For more info see http://www.lyx.org/ 2 | \lyxformat 413 3 | \begin_document 4 | \begin_header 5 | \textclass article 6 | \use_default_options true 7 | \begin_modules 8 | text 9 | \end_modules 10 | \maintain_unincluded_children false 11 | \language english 12 | \language_package default 13 | \inputencoding auto 14 | \fontencoding global 15 | \font_roman default 16 | \font_sans default 17 | \font_typewriter default 18 | \font_default_family default 19 | \use_non_tex_fonts false 20 | \font_sc false 21 | \font_osf false 22 | \font_sf_scale 100 23 | \font_tt_scale 100 24 | 25 | \graphics default 26 | \default_output_format default 27 | \output_sync 0 28 | \bibtex_command default 29 | \index_command default 30 | \paperfontsize default 31 | \spacing single 32 | \use_hyperref false 33 | \papersize default 34 | \use_geometry false 35 | \use_amsmath 1 36 | \use_esint 1 37 | \use_mhchem 1 38 | \use_mathdots 1 39 | \cite_engine basic 40 | \use_bibtopic false 41 | \use_indices false 42 | \paperorientation portrait 43 | \suppress_date false 44 | \use_refstyle 1 45 | \index Index 46 | \shortcut idx 47 | \color #008000 48 | \end_index 49 | \secnumdepth 3 50 | \tocdepth 3 51 | \paragraph_separation indent 52 | \paragraph_indentation default 53 | \quotes_language english 54 | \papercolumns 1 55 | \papersides 1 56 | \paperpagestyle default 57 | \tracking_changes false 58 | \output_changes false 59 | \html_math_output 0 60 | \html_css_as_file 0 61 | \html_be_strict false 62 | \end_header 63 | 64 | \begin_body 65 | 66 | \begin_layout Section 67 | Test file for textfill. 68 | \end_layout 69 | 70 | \begin_layout Standard 71 | This is a test file for textfill. 72 | Below we include the smaller test table: 73 | \end_layout 74 | 75 | \begin_layout Standard 76 | \align center 77 | 78 | \size small 79 | \begin_inset Flex Text 80 | status open 81 | 82 | \begin_layout Plain Layout 83 | 84 | \size small 85 | \begin_inset CommandInset label 86 | LatexCommand label 87 | name "text:test_small" 88 | 89 | \end_inset 90 | 91 | 92 | \end_layout 93 | 94 | \end_inset 95 | 96 | 97 | \end_layout 98 | 99 | \begin_layout Standard 100 | The sample table should be included above. 101 | 102 | \end_layout 103 | 104 | \begin_layout Standard 105 | Below should be printed the longer selection of log output from Stata: 106 | \end_layout 107 | 108 | \begin_layout Standard 109 | \align center 110 | 111 | \size small 112 | \begin_inset Flex Text 113 | status open 114 | 115 | \begin_layout Plain Layout 116 | 117 | \size small 118 | \begin_inset CommandInset label 119 | LatexCommand label 120 | name "text:test_long" 121 | 122 | \end_inset 123 | 124 | 125 | \end_layout 126 | 127 | \end_inset 128 | 129 | 130 | \end_layout 131 | 132 | \begin_layout Standard 133 | 134 | \end_layout 135 | 136 | \end_body 137 | \end_document 138 | -------------------------------------------------------------------------------- /gslab_fill/tests/log/make.log: -------------------------------------------------------------------------------- 1 | testArgumentOrder (test_tablefill.testTablefill) ... ok 2 | testBreaksRoundingString (test_tablefill.testTablefill) ... ok 3 | testIllegalSyntax (test_tablefill.testTablefill) ... ok 4 | testInput (test_tablefill.testTablefill) ... ok 5 | test_alternative_prefix (test_textfill.testTextfill) ... ok 6 | test_argument_order (test_textfill.testTextfill) ... ok 7 | test_illegal_syntax (test_textfill.testTextfill) ... ok 8 | test_input (test_textfill.testTextfill) ... ok 9 | test_remove_echoes (test_textfill.testTextfill) ... ok 10 | test_tags_dont_match (test_textfill.testTextfill) ... ok 11 | test_tags_incorrectly_specified (test_textfill.testTextfill) ... ok 12 | test_tags_not_closed (test_textfill.testTextfill) ... ok 13 | 14 | ---------------------------------------------------------------------- 15 | Ran 12 tests in 0.063s 16 | 17 | OK 18 | -------------------------------------------------------------------------------- /gslab_fill/tests/test_textfill.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | import unittest 3 | import sys 4 | import os 5 | import re 6 | import types 7 | import HTMLParser 8 | import shutil 9 | 10 | sys.path.append('../..') 11 | from gslab_fill.textfill import (textfill, read_text, 12 | remove_trailing_leading_blanklines) 13 | from gslab_make.tests import nostderrout 14 | 15 | class testTextfill(unittest.TestCase): 16 | 17 | def setUp(self): 18 | if not os.path.exists('./build/'): 19 | os.mkdir('./build/') 20 | 21 | def test_input(self): 22 | with nostderrout(): 23 | message = textfill(input = '../../gslab_fill/tests/input/legal.log', 24 | template = '../../gslab_fill/tests/input/textfill_template.lyx', 25 | output = './build/textfill_template_filled.lyx') 26 | self.assertIn('filled successfully', message) 27 | log_remove_string = '. insert_tag' 28 | log = '../../gslab_fill/tests/input/legal.log' 29 | self.check_log_in_LyX(log, log_remove_string, "textfill_") 30 | 31 | def test_alternative_prefix(self): 32 | with nostderrout(): 33 | message = textfill(input = '../../gslab_fill/tests/input/alternative_prefix.log', 34 | template = '../../gslab_fill/tests/input/textfill_template.lyx', 35 | output = './build/textfill_template_filled.lyx', 36 | prefix = 'prefix') 37 | self.assertIn('filled successfully', message) 38 | log_remove_string = '. insert_tag' 39 | log = '../../gslab_fill/tests/input/alternative_prefix.log' 40 | self.check_log_in_LyX(log, log_remove_string, "prefix_") 41 | 42 | def test_remove_echoes(self): 43 | with nostderrout(): 44 | textfill(input = '../../gslab_fill/tests/input/legal.log', 45 | template = '../../gslab_fill/tests/input/textfill_template.lyx', 46 | output = './build/textfill_template_filled.lyx', 47 | remove_echoes = True) 48 | log_remove_string = '. ' 49 | log = '../../gslab_fill/tests/input/legal.log' 50 | self.check_log_in_LyX(log, log_remove_string, "textfill_") 51 | 52 | def check_log_in_LyX(self, log, log_remove_string, prefix): 53 | raw_lyx = open("../../gslab_fill/tests/input/textfill_template_filled.lyx", 'rU').readlines() 54 | raw_lyx = [re.sub(r'\\end_layout\n$', '', x) for x in raw_lyx] 55 | 56 | text = read_text(log, prefix) 57 | self.assertEqual( len(text.results),2 ) 58 | self.assertEqual(text.results.keys(), ['test_small', 'test_long']) 59 | 60 | for key in text.results: 61 | raw_table = text.results[key].split('\n') 62 | raw_table = filter(lambda x: not x.startswith(log_remove_string), raw_table) 63 | raw_table = remove_trailing_leading_blanklines(raw_table) 64 | for n in range( len(raw_table) ): 65 | self.assertIn(raw_table[n], raw_lyx) 66 | 67 | def test_tags_dont_match(self): 68 | with nostderrout(): 69 | error = textfill(input = '../../gslab_fill/tests/input/tags_dont_match.log', 70 | template = '../../gslab_fill/tests/input/textfill_template.lyx', 71 | output = './build/textfill_template_filled.lyx') 72 | self.assertIn('ValueError', error) 73 | 74 | def test_tags_not_closed(self): 75 | with nostderrout(): 76 | error = textfill(input = '../../gslab_fill/tests/input/tags_not_closed.log', 77 | template = '../../gslab_fill/tests/input/textfill_template.lyx', 78 | output = './build/textfill_template_filled.lyx') 79 | self.assertIn('HTMLParseError', error) 80 | 81 | def test_tags_incorrectly_specified(self): 82 | with nostderrout(): 83 | textfill(input = '../../gslab_fill/tests/input/tags_incorrectly_named.log', 84 | template = '../../gslab_fill/tests/input/textfill_template.lyx', 85 | output = './build/textfill_template_filled.lyx') 86 | 87 | log_remove_string = '. insert_tag' 88 | 89 | with self.assertRaises(AssertionError): 90 | log = '../../gslab_fill/tests/input/tags_incorrectly_named.log' 91 | self.check_log_in_LyX(log, log_remove_string, "textfill_") 92 | 93 | def test_illegal_syntax(self): 94 | # missing arguments 95 | with nostderrout(): 96 | error = textfill(input = '../../gslab_fill/tests/input/legal.log', 97 | template = '../../gslab_fill/tests/input/textfill_template.lyx') 98 | self.assertIn('KeyError', error) 99 | 100 | # non-existent input 1 101 | with nostderrout(): 102 | error = textfill(input = '../../gslab_fill/tests/input/fake_file.log', 103 | template = '../../gslab_fill/tests/input/textfill_template.lyx', 104 | output = './build/textfill_template_filled.lyx') 105 | 106 | 107 | self.assertIn('IOError', error) 108 | 109 | # non-existent input 2 110 | with nostderrout(): 111 | error = textfill(input = './log/stata.log ./input/fake_file.log', 112 | template = '../../gslab_fill/tests/input/textfill_template.lyx', 113 | output = './build/textfill_template_filled.lyx') 114 | 115 | self.assertIn('IOError', error) 116 | 117 | def test_argument_order(self): 118 | 119 | with nostderrout(): 120 | message = textfill(input = '../../gslab_fill/tests/input/legal.log', 121 | output = '../../gslab_fill/tests/input/textfill_template_filled.lyx', 122 | template = '../../gslab_fill/tests/input/textfill_template.lyx') 123 | 124 | self.assertIn('filled successfully', message) 125 | 126 | with nostderrout(): 127 | message = textfill(template = '../../gslab_fill/tests/input/textfill_template.lyx', 128 | output = './build/textfill_template_filled.lyx', 129 | input = '../../gslab_fill/tests/input/legal.log') 130 | 131 | self.assertIn('filled successfully', message) 132 | 133 | def tearDown(self): 134 | if os.path.exists('./build/'): 135 | shutil.rmtree('./build/') 136 | 137 | 138 | if __name__ == '__main__': 139 | os.getcwd() 140 | unittest.main() 141 | -------------------------------------------------------------------------------- /gslab_fill/textfill.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | import os 4 | import argparse 5 | import types 6 | import traceback 7 | import textfill_info 8 | from HTMLParser import HTMLParser, HTMLParseError 9 | 10 | 11 | def textfill(**kwargs): 12 | try: 13 | args = parse_arguments(kwargs) 14 | text = parse_text(args) 15 | insert_text(args, text) 16 | exitmessage = args['template'] + ' filled successfully by textfill' 17 | print exitmessage 18 | return exitmessage 19 | 20 | except: 21 | print 'Error Found' 22 | exitmessage = traceback.format_exc() 23 | print exitmessage 24 | return exitmessage 25 | 26 | # Set textfill's docstring as the text in "textfill_info.py" 27 | textfill.__doc__ = textfill_info.__doc__ 28 | 29 | 30 | def parse_arguments(kwargs): 31 | args = dict() 32 | if 'input' in kwargs.keys(): 33 | input_list = kwargs['input'].split() 34 | args['input'] = input_list 35 | if 'template' in kwargs.keys(): 36 | args['template'] = kwargs['template'] 37 | if 'output' in kwargs.keys(): 38 | args['output'] = kwargs['output'] 39 | if 'remove_echoes' in kwargs.keys(): 40 | args['remove_echoes'] = kwargs['remove_echoes'] 41 | else: 42 | args['remove_echoes'] = False 43 | if 'size' in kwargs.keys(): 44 | args['size'] = kwargs['size'] 45 | else: 46 | args['size'] = 'Default' 47 | if 'prefix' in kwargs.keys(): 48 | args['prefix'] = kwargs['prefix'] + "_" 49 | else: 50 | args['prefix'] = 'textfill_' 51 | 52 | return args 53 | 54 | 55 | def parse_text(args): 56 | text = read_text(args['input'], args['prefix']) 57 | text = clean_text(text, args['remove_echoes']) 58 | 59 | return text 60 | 61 | 62 | def read_text(input, prefix): 63 | data = '' 64 | if isinstance(input, types.StringTypes): 65 | input = [input] 66 | for file in input: 67 | data += open(file, 'rU').read() 68 | text = text_parser(prefix) 69 | text.feed(data) 70 | text.close() 71 | 72 | return text 73 | 74 | 75 | class text_parser(HTMLParser): 76 | def __init__(self, prefix): 77 | HTMLParser.__init__(self) 78 | self.recording = False 79 | self.results = {} 80 | self.open = [] 81 | self.closed = [] 82 | self.prefix = prefix 83 | 84 | def handle_starttag(self, tag, attrs): 85 | if tag.startswith(self.prefix): 86 | tag_name = tag.replace(self.prefix, '', 1) 87 | self.recording = True 88 | self.results[tag_name] = '' 89 | self.open.append(tag_name) 90 | 91 | def handle_data(self, data): 92 | if self.recording: 93 | self.results[self.open[-1]]+=data 94 | 95 | def handle_endtag(self, tag): 96 | if tag.startswith(self.prefix): 97 | tag_name = tag.replace(self.prefix, '', 1) 98 | self.open.remove(tag_name) 99 | self.closed.append(tag_name) 100 | if not self.open: 101 | self.recording = False 102 | 103 | def close(self): 104 | for tag in self.results.keys(): 105 | if tag not in self.closed: 106 | raise HTMLParseError('Tag %s is not closed' % tag) 107 | 108 | 109 | def clean_text(text, remove_echoes): 110 | for key in text.results: 111 | data = text.results[key].split('\n') 112 | if remove_echoes: 113 | data = filter(lambda x: not x.startswith('.'), data) 114 | else: 115 | data = filter(lambda x: not x.startswith('. insert_tag'), data) 116 | data = remove_trailing_leading_blanklines(data) 117 | text.results[key] = '\n'.join(data) 118 | 119 | return text 120 | 121 | 122 | def remove_trailing_leading_blanklines(list): 123 | while list and not list[0]: 124 | del list[0] 125 | while list and not list[-1]: 126 | del list[-1] 127 | 128 | return list 129 | 130 | 131 | def insert_text(args,text): 132 | lyx_text = open(args['template'], 'rU').readlines() 133 | # Loop over (expanding) raw LyX text 134 | n = 0 135 | loop = True 136 | while loop==True: 137 | n+=1 138 | if n` 60 | 61 | The user should now add lines to the do file which print the output they want to add to 62 | the tagged section, followed by the line: 63 | insert_tag tag_name, close 64 | 65 | This inserts the following line to the log file, indicating the end of the tagged section: 66 | `` 67 | 68 | 69 | ########################### 70 | Template LyX Format: 71 | ########################### 72 | 73 | The LyX template file contains labels which determine where the tagged sections of the 74 | input files are inserted. To insert a log section tagged as 'tag_name', in a particular 75 | place in the LyX file, the user inserts a label object with the value 'text:tag_name' 76 | inside a 'Text' custom inset. The 'text:' part of the label is mandatory. When textfill 77 | is run, the tagged section of the input files will be inserted as text input at the 78 | location of corresponding label in the LyX file. 79 | 80 | Note that the 'Text' custom inset object is available from 'Insert > Custom Insets' when 81 | Lyx had been reconfigured with the custom module text.module. This module is available on 82 | the repo at /admin/Computer Build Sheet/, and can be installed according to the instructions 83 | in /admin/Computer Build Sheet/standard_build.pdf. 84 | 85 | Note that label/tag names cannot be duplicated. For a single template file, each block of 86 | text to be inserted must have a unique label, and there must be one, and only one, section 87 | of the input files tagged with that same label. Having multiple sections of the input files 88 | or multiple labels in the template file with the same name will cause errors. 89 | 90 | Note also that when a LyX file with a 'text:' label is opened in LyX, or when textfill.py is 91 | run on it, LyX may issue a warning: 92 | "The module text has been requested by this document but has not been found..." 93 | 94 | This warning means that the custom module text.module has not been installed - see above. 95 | 96 | 97 | ##################### 98 | # Example 99 | ##################### 100 | 101 | The following is an example of code, which could appear in a Stata do file, used to produce 102 | input for textfill. 103 | ``` 104 | insert_tag example_log, open 105 | display "test" 106 | insert_tag example_log, close 107 | ``` 108 | 109 | Suppose output from Stata is being logged in stata.log. This code adds the following lines 110 | to stata.log: 111 | 112 | ``` 113 | . insert_tag example_log, open 114 | 115 | 116 | . display "test" 117 | test 118 | 119 | . insert_tag example_log, close 120 | 121 | ``` 122 | 123 | Suppose we have a LyX file, template.lyx, which contains a label with the value 124 | "text:example_log" (without the ""). The following textfill command, 125 | `textfill( input = 'stata.log', template = 'template.lyx', output = 'output.lyx' )` 126 | 127 | would produce a file, output.lyx, identical to template.lyx, but with the label 128 | "text:example.log" replaced with the verbatim input: 129 | 130 | ``` 131 | . display "test" 132 | test 133 | ``` 134 | 135 | The following command, 136 | `textfill( input = 'stata.log', template = 'template.lyx', 137 | output = 'output.lyx', remove_echoes = True )` 138 | 139 | would produce output.lyx replacing the label with the verbatim input (removing Stata command echoes): 140 | 141 | 142 | `test` 143 | 144 | 145 | ###################### 146 | # Error Logging 147 | ###################### 148 | 149 | If an error occurs during the call to text, it will be displayed in the command window. 150 | When make.py finishes, the user will be able to scroll up through the output and examine 151 | any error messages. Error messages, which include a description of the error type 152 | and a traceback to the line of code where the error occurred, can also be returned as a 153 | string object using the following syntax: 154 | 155 | ``` 156 | exitmessage = textfill( input = 'input_file(s)', template = 'template_file', output = 'output_file', 157 | [size = 'size'], [remove_echoes = 'True/False'] ) 158 | ``` 159 | 160 | Lines can then be added to make.py to output this string to a log file using standard 161 | Python and built in gslab_make commands. 162 | ''' 163 | -------------------------------------------------------------------------------- /gslab_make/get_externals.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | import os 4 | import private.preliminaries as prelim 5 | import private.metadata as metadata 6 | import private.messages as messages 7 | 8 | from private.getexternalsdirectives import SystemDirective 9 | 10 | def get_externals(externals_file, 11 | external_dir = '@DEFAULTVALUE@', 12 | makelog = '@DEFAULTVALUE@', 13 | quiet = False): 14 | '''Fetch external files 15 | 16 | Description: 17 | This function interprets a formatted text document listing files 18 | to be exported via SVN or a system copy command. 19 | 20 | Syntax: 21 | get_externals(externals_file [, externals_dir [, makelog [, quiet]]]) 22 | 23 | Usage: 24 | The `externals_file` argument should be the path of a tab-delimited text 25 | file containing information on the external files that the function call 26 | should retrieve. This file needs to rows of numbers or characters, delimited 27 | by either tabs or 4 spaces,one for each file to be exported via svn. 28 | The proper format is: rev dir file outdir outfile notes 29 | 30 | ### Column descriptions: 31 | * rev 32 | * Revision number of the file/directory in integer format. 33 | If left blank along with directory column, get_externals.py will 34 | read the last specified revision number. If copying from a shared 35 | drive rather than the repository, list revision number as COPY. 36 | * dir 37 | * Directory of the file/directory requested. As described above, 38 | %xxx% placemarkers are substituted in from predefined values in 39 | metadata.py. If left blank along with revision column, 40 | get_externals.py will read the last specified directory. 41 | * file 42 | * Name of the file requested. If entire directory is required, leave 43 | column as a single *. If a file name wildcard is required place 44 | single * within filename. get_externals.py will attempt to screen 45 | out bad file names. Cannot be left blank. 46 | * outdir 47 | * Desired output directory of the exported file/directory. 48 | Typically of the form ./subdir/. If left blank, will be 49 | filled with the first level of the externals relative path. 50 | * outfile 51 | * Desired output name of the exported file/directory. If left as 52 | double quotes, indicates that it should have the same name. 53 | Adding a directory name that is different from the default """" 54 | will place this subdirectory within the outdir. Additionally, 55 | get_externals can assign a prefix tag to exported file collections, 56 | either through a folder export, or a wildcard call; it does so 57 | when the outfile column contains text of the pattern '[prefix]*', 58 | where the prefix [prefix] will be attached to exported files. 59 | * notes 60 | * Optional column with notes on the export. get_externals.py ignores this, 61 | but logs it. 62 | 63 | Example of externals.txt: 64 | ``` 65 | rev dir file outdir outfile notes 66 | 2 %svn%/directory/ * ./destination_directory/ """" 67 | COPY %svn%/other_directory/ my_file.txt . """" 68 | ``` 69 | 70 | The destination directory is specified by an optional second 71 | parameter whose default value is "../external". The log file produced by 72 | get_externals is automatically added to an optional third parameter 73 | whose default value is '../output/make.log'. 74 | 75 | The fourth argument, quiet, is by default False. Setting this argument to 76 | True suppresses standard output and errors from SVN. 77 | ''' 78 | try: 79 | LOGFILE = prelim.start_logging(metadata.settings['externalslog_file'], 'get_externals.py') 80 | makelog, externals, last_dir, last_rev = \ 81 | prelim.externals_preliminaries(makelog, externals_file, LOGFILE) 82 | 83 | for line in externals: 84 | try: 85 | directive = SystemDirective(line, LOGFILE, last_dir, last_rev) 86 | directive.error_check() 87 | directive.clean(external_dir) 88 | directive.issue_sys_command(quiet) 89 | 90 | # Save rev/dir for next line 91 | last_dir = directive.dir 92 | last_rev = directive.rev 93 | except: 94 | prelim.print_error(LOGFILE) 95 | prelim.end_logging(LOGFILE, makelog, 'get_externals.py') 96 | 97 | except Exception as errmsg: 98 | print "Error with get_external: \n", errmsg 99 | -------------------------------------------------------------------------------- /gslab_make/get_externals_github.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | import os 4 | import getpass 5 | import private.preliminaries as prelim 6 | import private.metadata as metadata 7 | import private.messages as messages 8 | from private.getexternalsdirectives import SystemDirective 9 | 10 | 11 | def get_externals_github(externals_file, external_dir = '@DEFAULTVALUE@', 12 | makelog = '@DEFAULTVALUE@', quiet = False): 13 | '''Fetch external files from GitHub 14 | 15 | Description: 16 | This function retrieves files from GitHub as specified by a formatted text 17 | document indicated as an argument. 18 | get_externals_github is only intended to download assets from releases. 19 | It cannot download the release source code or other assets not contained 20 | within a release. 21 | 22 | Usage: 23 | - This function's usage largely follows that of get_externals(). Instead of 24 | rading 25 | - get_externals_github is only intended to download assets from releases. 26 | It cannot download the release source code or other assets not contained 27 | within a release. 28 | 29 | File format of a `externals_file`: 30 | This file needs to rows of numbers or characters, delimited by either tabs or 31 | 4 spaces, one for each file to be exported via GitHub. 32 | The proper column format is: url outdir outfile notes 33 | 34 | Column descriptions: 35 | * url 36 | * Download url for the file in a specific GitHub release. The url given 37 | should be the complete url. 38 | * outdir 39 | * Desired output directory of the exported file/directory. Typically of the form 40 | ./subdir/. If left blank, will be filled with the first level of the externals 41 | relative path. 42 | * outfile 43 | * Desired output name of the exported file/directory. If left as double quotes, 44 | indicates that it should have the same name as the asset name in GitHub. 45 | * notes 46 | * Optional column with notes on the export. get_externals_github.py ignores this, 47 | but logs it. 48 | 49 | Example of externals_github.txt: 50 | 51 | ``` 52 | url outdir outfile notes 53 | https://github.com/TestUser/TestRepo/releases/download/v1.0/document.pdf ./ """" 54 | https://github.com/TestUser/TestRepo/releases/download/AlternativeVersion/other_document.pdf ./ """" 55 | ``` 56 | ''' 57 | try: 58 | # Request Token 59 | token = getpass.getpass("\nEnter a valid GitHub token and then press enter: ") 60 | 61 | LOGFILE = prelim.start_logging(metadata.settings['githublog_file'], 62 | 'get_externals_github.py') 63 | makelog, externals, last_dir, last_rev = \ 64 | prelim.externals_preliminaries(makelog, externals_file, LOGFILE) 65 | 66 | for line in externals: 67 | try: 68 | directive = SystemDirective(line, LOGFILE, last_dir, last_rev, 69 | token = token) 70 | directive.error_check() 71 | directive.clean(external_dir) 72 | directive.issue_sys_command(quiet) 73 | except: 74 | prelim.print_error(LOGFILE) 75 | 76 | prelim.end_logging(LOGFILE, makelog, 'get_externals_github.py') 77 | 78 | except Exception as errmsg: 79 | print "Error with get_externals_github: \n", errmsg 80 | -------------------------------------------------------------------------------- /gslab_make/make_link_logs.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | import os 4 | import re 5 | import datetime 6 | import private.metadata as metadata 7 | 8 | from make_log import make_stats_log, make_heads_log 9 | from private.linkslist import LinksList 10 | 11 | 12 | def make_link_logs (links_files, 13 | links_dir = '@DEFAULTVALUE@', 14 | link_logs_dir = '@DEFAULTVALUE@', 15 | link_stats_file = '@DEFAULTVALUE@', 16 | link_heads_file = '@DEFAULTVALUE@', 17 | link_orig_file = '@DEFAULTVALUE@', 18 | recur_lim = 2): 19 | '''Import the make_link_logs function from the make_link_logs.py module. 20 | 21 | Description: 22 | This function parses and interprets links files and uses that input to create logs 23 | of useful information regarding where local shortcuts point to and information regarding 24 | when and how the linked files were last changed 25 | 26 | Syntax: 27 | make_link_logs(links_files [, links_dir [, link_logs_dir [, link_stats_file 28 | [, link_heads_file [, link_orig_file [, recur_lim]]]]]]) 29 | 30 | Parameters: 31 | - links_files: See `make_links`'s docstring. 32 | - links_dir: See `make_links`'s docstring. 33 | - link_logs_dir: A string containing the name of the directory into which the log files will 34 | be saved. The default argument for this parameter is '../log/'. 35 | - link_stats_file: A string containing the name of the link stats file (see below for 36 | full explanation of what this file contains). To prevent this log from being 37 | made, set this argument to an empty string (i.e. ''). This is the name of the 38 | file only; the directory name is determined by link_logs_dir. 39 | The default argument for this parameter is 'link_stats.log'. 40 | - link_head_file: Same as link_stats_file above except for the link heads file. 41 | The default argument for this parameter is 'link_heads.log'. 42 | - link_orig_file: Same as link_stats_file above except for link origins file. 43 | The default argument for this parameter is 'link_orig.log' 44 | - recur_lim: An integer which determines the directory depth to which the log files will 45 | search for a list in the links_dir. By default, this argument is 2, which searches 46 | links_dir and one level of subdirectories. If the argument is changed to 1, the log 47 | files would only search in links_dir, and if it was changed to 3 the log files would 48 | search in links_dir and 2 levels of subdirectories (and so on). If the argument is 49 | set to 0, there is no depth limit. The default argument for this parameter is 2. 50 | 51 | 52 | Description of log files: 53 | - link_stats_file: stores all file names, date and time last modified, and file sizes 54 | - link_heads_file: stores the first ten lines of all readable files 55 | - link_orig_file: stores the local names of all symlinks and the original files to 56 | which they point; if a directory is a symlink, only the directory 57 | will be included in the mapping, even though it's contents will 58 | technically be links, too 59 | ''' 60 | 61 | if link_logs_dir == '@DEFAULTVALUE@': 62 | link_logs_dir = metadata.settings['link_logs_dir'] 63 | if link_stats_file == '@DEFAULTVALUE@': 64 | link_stats_file = metadata.settings['link_stats_file'] 65 | if link_heads_file == '@DEFAULTVALUE@': 66 | link_heads_file = metadata.settings['link_heads_file'] 67 | if link_orig_file == '@DEFAULTVALUE@': 68 | link_orig_file = metadata.settings['link_orig_file'] 69 | 70 | links_list = LinksList(links_files, links_dir) 71 | sorted_files, links_dict = links_list.link_files_and_dict(recur_lim) 72 | 73 | make_stats_log(link_logs_dir, link_stats_file, sorted_files) 74 | make_heads_log(link_logs_dir, link_heads_file, sorted_files) 75 | make_link_orig_log(link_logs_dir, link_orig_file, links_dict) 76 | 77 | 78 | def make_link_orig_log (link_logs_dir, link_orig_file, links_dict): 79 | """ 80 | Using the mappings from the `links_dict` argument, create a log file at 81 | `link_orig_file` in `link_logs_dir` that reports the local names 82 | of symlinks and the original files to which they link. 83 | """ 84 | 85 | # Return if no link_orig_file is specified 86 | if link_orig_file == '': 87 | return 88 | 89 | # Return if the links_dict is empty 90 | num_links = len(links_dict.keys()) 91 | if num_links == 0: 92 | return 93 | 94 | link_orig_path = os.path.join(link_logs_dir, link_orig_file) 95 | link_orig_path = re.sub('\\\\', '/', link_orig_path) 96 | 97 | header = "local\tlinked" 98 | 99 | if not os.path.isdir(os.path.dirname(link_orig_path)): 100 | os.makedirs(os.path.dirname(link_orig_path)) 101 | ORIGFILE = open(link_orig_path, 'w+') 102 | print >> ORIGFILE, header 103 | 104 | links_dict_it = iter(sorted(links_dict.iteritems())) 105 | for i in range(num_links): 106 | print >> ORIGFILE, "%s\t%s" % links_dict_it.next() 107 | 108 | 109 | -------------------------------------------------------------------------------- /gslab_make/make_links.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | import os 4 | 5 | from dir_mod import remove_dir 6 | from private.linkslist import LinksList 7 | from private.preliminaries import start_logging, end_logging 8 | import private.metadata as metadata 9 | 10 | def make_links (links_files, 11 | links_dir = '@DEFAULTVALUE@', 12 | makelog = '@DEFAULTVALUE@', 13 | quiet = False): 14 | '''Import the make_links function from the make_links.py module. 15 | 16 | Description: 17 | This function parses and interprets links files and uses that input to create symlinks 18 | to large data files (symlinks allow shortcuts to be made locally so that programs will 19 | act as if the files are stored locally, but no local hard drive space is taken up) 20 | 21 | Syntax: 22 | make_links(links_files [, links_dir [, makelog [, quiet]]]) 23 | 24 | Parameters: 25 | - links_files: A string containing the name of a valid links file, a string containing 26 | multiple names of valid links files (delimited by spaces), or a Python list of 27 | strings in which each element is a valid links file. Each string may also contain one 28 | wildcard (as an asterisk). For example: 29 | `make_links('./links.txt')`, or 30 | `make_links('../external/links1.txt ../external/links2.txt')`, or 31 | `make_links(['../external/links1.txt', '../external/links2.txt'])`, or 32 | `make_links('../external/links*.txt')`. 33 | 34 | To get every file in a directory, simply end the string with a wild card: 35 | 36 | `make_links('../external/links_files/*')` 37 | 38 | There is no default argument for this parameter; links_files must be included in 39 | every call to make_links(). 40 | 41 | - links_dir: A string containing the name of the local directory into which the symlinks 42 | will be created. The default argument for this parameter is '../external_links/'. 43 | 44 | - makelog: A string containing the name of the makelog that will store make_links() 45 | output. If makelog is an empty string (i.e. ''), the default log file will be 46 | unchanged. The default argument for this parameter is '../output/make.log'. 47 | 48 | - quiet: A boolean. Setting this argument to True suppresses standard output and errors 49 | from svn. The default argument for this parameter is False. 50 | 51 | Caution: 52 | 53 | Take care when dealing with symlinks. Most programs will treat the links as if they are 54 | the file being linked to. For instance, if ../data_links/some_data.dta is a link to 55 | \external_source\some_data.dta and Stata opens ../data_links/some_data.dta, edits it, 56 | and saves it to the same location, the original file (\external_source\some_data.dta) 57 | will be changed. In such a case, it would probably be preferable to save a local copy of 58 | the data and make changes to that file. 59 | ''' 60 | try: 61 | if makelog == '@DEFAULTVALUE@': 62 | makelog = metadata.settings['makelog_file'] 63 | 64 | # Run preliminaries 65 | LOGFILE = start_logging(metadata.settings['linkslog_file'], 'make_links.py') 66 | 67 | list = LinksList(links_files, links_dir) 68 | 69 | if os.path.exists(list.links_dir): 70 | remove_dir(list.links_dir) 71 | os.makedirs(list.links_dir) 72 | 73 | list.issue_sys_command(LOGFILE, quiet) 74 | 75 | end_logging(LOGFILE, makelog, 'make_links.py') 76 | 77 | except Exception as errmsg: 78 | print "Error with make_links: \n", errmsg 79 | -------------------------------------------------------------------------------- /gslab_make/private/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | This directory contains functions used internally within 3 | the gslab_make Python module. 4 | ''' 5 | from .exceptionclasses import CustomError, CritError, SyntaxError, LogicError 6 | -------------------------------------------------------------------------------- /gslab_make/private/exceptionclasses.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | class CustomError(Exception): 4 | def __init__(self, value): 5 | self.value = value 6 | def __str__(self): 7 | return repr(self.value) 8 | 9 | class CritError(CustomError): 10 | pass 11 | 12 | class SyntaxError(CustomError): 13 | pass 14 | 15 | class LogicError(CustomError): 16 | pass -------------------------------------------------------------------------------- /gslab_make/private/linkdirectives.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | import os 4 | import re 5 | import subprocess 6 | 7 | import messages as messages 8 | import metadata as metadata 9 | from exceptionclasses import SyntaxError, LogicError 10 | 11 | class LinkDirectives(object): 12 | 13 | def __init__(self, line, links_dir): 14 | self.line = line.replace(' ','\t') 15 | list_line = ((self.line.split('\t')+[None])[:2]) 16 | for i in xrange(len(list_line)): 17 | list_line[i] = list_line[i].strip() 18 | local, link = list_line 19 | 20 | self.linkeddir, self.linkedfile = os.path.split(link) 21 | if self.linkedfile == '': 22 | self.linkedfile = '*' 23 | 24 | self.localdir, self.localfile = os.path.split(local) 25 | if self.localdir == '': 26 | self.localdir = '.' 27 | if self.localfile == '': 28 | self.localfile = '""""' 29 | 30 | self.links_dir = links_dir 31 | 32 | self.error_check() 33 | self.clean() 34 | self.create_flag_list() 35 | 36 | # Error Check 37 | def error_check(self): 38 | # Check syntax 39 | if self.linkeddir == '' or self.localdir=='': 40 | raise SyntaxError(messages.syn_error_noname) 41 | if not re.search('\*',self.linkedfile) and re.search('\*',self.localfile): 42 | raise SyntaxError(messages.syn_error_wildfilename1.replace('\n','')) 43 | if re.search('\*',self.localfile) and not re.search('\*$',self.localfile): 44 | raise SyntaxError(messages.syn_error_wildlocalfile.replace('\n','')) 45 | 46 | # Clean 47 | def clean(self): 48 | self.clean_syntax() 49 | self.clean_logic() 50 | 51 | def clean_syntax(self): 52 | # Clean slashes 53 | if self.localdir=='.' or self.localdir=='.\\': 54 | self.localdir = './' 55 | self.localdir = re.sub('^\.\\.*\\$', '^/.*/$', self.localdir) 56 | if not re.search('/$',self.localdir): 57 | self.localdir = self.localdir+'/' 58 | 59 | self.linkeddir = os.path.abspath(self.linkeddir) 60 | self.linkeddir = re.sub('^\.\\.*\\$', '^/.*/$', self.linkeddir) 61 | if not re.search('/$', self.linkeddir): 62 | self.linkeddir = self.linkeddir + '/' 63 | 64 | def clean_logic(self): 65 | # Prepare prefix 66 | self.outprefix = '' 67 | if re.search('\*',self.localfile): 68 | self.outprefix = self.localfile[0:-1] 69 | 70 | # If localfile is left as a double quote, keep the same file name 71 | if self.localfile=='""""': 72 | self.localfile = self.linkedfile 73 | 74 | # Set file/localfile to empty if entire directory is operated on 75 | if self.linkedfile=='*': 76 | self.linkedfile = '' 77 | self.localfile = '' 78 | 79 | # Replace '.' in links.txt with inputted links directory location 80 | if self.localdir=='./': 81 | if self.linkedfile == '': 82 | dirname = os.path.split(os.path.dirname(self.linkeddir))[1] 83 | self.localdir = os.path.join(self.links_dir, dirname) 84 | else: 85 | self.localdir = self.links_dir 86 | else: 87 | self.localdir = self.links_dir + self.localdir 88 | 89 | def create_flag_list (self): 90 | self.flag_list = False 91 | if re.search('\*.+',self.linkedfile) or re.search('^.+\*',self.linkedfile) or (re.search('\*',self.localfile)): 92 | # Make wildcard list 93 | self.flag_list = True 94 | wildfirst,wildlast = self.linkedfile.split('*') 95 | self.LIST = [] 96 | for element in os.listdir(self.linkeddir): 97 | if re.match('%s.*%s$' % (wildfirst,wildlast),element): 98 | self.LIST.append(element.rstrip('\n')) 99 | 100 | # Check wildcard directory non-empty 101 | if self.flag_list and not self.LIST: 102 | raise LogicError(messages.crit_error_emptylist) 103 | 104 | # Issue System Command 105 | def issue_sys_command(self, logfile, quiet): 106 | if self.flag_list: 107 | print >> logfile, messages.note_array % self.LIST 108 | for element in self.LIST: 109 | #Add prefix to localfile name (if none, then no change) 110 | self.localfile = self.outprefix+element 111 | self.linkedfile = element 112 | self.command(logfile, quiet) 113 | else: 114 | self.command(logfile, quiet) 115 | 116 | def command(self, logfile, quiet): 117 | if self.localfile == '': 118 | self.localdir = self.localdir.rstrip('/.') 119 | if self.linkeddir == '': 120 | self.linkeddir = self.linkeddir.rstrip('/.') 121 | if os.name == 'posix': 122 | command = metadata.commands['makelinkunix'] 123 | options = (self.linkeddir, self.linkedfile, self.localdir, self.localfile) 124 | else: 125 | if self.linkedfile == '': 126 | option = '/d' 127 | else: 128 | option = '' 129 | command = metadata.commands['makelinkwin'] 130 | options = (option, self.localdir, self.localfile, self.linkeddir, self.linkedfile) 131 | 132 | # Following command contains try/except clause 133 | if quiet: 134 | subprocess.check_call(command % options, shell=True, 135 | stdout=open(os.devnull, 'w'), 136 | stderr=open(os.devnull, 'w')) 137 | else: 138 | subprocess.check_call(command % options, shell=True) 139 | 140 | print >> logfile, messages.success_makelink % (self.linkeddir, self.linkedfile, 141 | self.localdir, self.localfile) 142 | 143 | def add_to_dict(self, links_dict): 144 | if self.flag_list: 145 | for element in self.LIST: 146 | #Add prefix to localfile name (if none, then no change) 147 | self.localfile = self.outprefix+element 148 | self.linkedfile = element 149 | 150 | local = os.path.relpath(os.path.join(self.localdir, self.localfile)) 151 | dest = os.path.join(self.linkeddir, self.linkedfile) 152 | links_dict[local] = dest 153 | else: 154 | local = os.path.relpath(os.path.join(self.localdir, self.localfile)) 155 | dest = os.path.join(self.linkeddir, self.linkedfile) 156 | links_dict[local] = dest 157 | return links_dict 158 | -------------------------------------------------------------------------------- /gslab_make/private/linkslist.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | import os 4 | import re 5 | 6 | from linkdirectives import * 7 | from preliminaries import * 8 | import metadata as metadata 9 | 10 | class LinksList(object): 11 | 12 | def __init__(self, file_list, links_dir = '@DEFAULTVALUE@'): 13 | if links_dir == '@DEFAULTVALUE@': 14 | links_dir = metadata.settings['links_dir'] 15 | if links_dir[-1] != '/': 16 | links_dir = links_dir + '/' 17 | self.links_dir = links_dir 18 | 19 | if type(file_list) is str: 20 | file_list = file_list.split(' ') 21 | 22 | self.links_files = [] 23 | for f1 in file_list: 24 | if re.search('\*.+', f1) or re.search('^.+\*', f1) or re.search('\*', f1): 25 | # Make wildcard list 26 | wildfirst,wildlast = f1.split('*') 27 | file_dir,wildfirst = os.path.split(wildfirst) 28 | file_dir = os.path.abspath(file_dir) 29 | files = [ f2 for f2 in os.listdir(file_dir) 30 | if os.path.isfile(os.path.join(file_dir, f2)) ] 31 | for f3 in files: 32 | if re.match('%s.*%s$' % (wildfirst,wildlast), f3): 33 | self.links_files.append(os.path.join(file_dir, f3)) 34 | else: 35 | self.links_files.append(f1) 36 | 37 | self.linkdirectives_list = [] 38 | for f in self.links_files: 39 | lines = input_to_array(f) 40 | for line in lines: 41 | directive = LinkDirectives(line, self.links_dir) 42 | self.linkdirectives_list.append(directive) 43 | 44 | def issue_sys_command(self, logfile, quiet): 45 | for link in self.linkdirectives_list: 46 | try: 47 | link.issue_sys_command(logfile, quiet) 48 | except: 49 | print_error(logfile) 50 | 51 | def link_files_and_dict(self, recur_lim): 52 | links_dict = {} 53 | for link in self.linkdirectives_list: 54 | links_dict = link.add_to_dict(links_dict) 55 | 56 | links = links_dict.values() 57 | sorted_files = [ f for f in links if os.path.isfile(f) ] 58 | if recur_lim > 1: 59 | dirs = [ d for d in links if os.path.isdir(d) ] 60 | for d in dirs: 61 | new_files = files_list(d, recur_lim - 1) 62 | sorted_files = sorted_files + new_files 63 | elif not recur_lim: 64 | dirs = [ d for d in links if os.path.isdir(d) ] 65 | for d in dirs: 66 | new_files = files_list(d, recur_lim) 67 | sorted_files = sorted_files + new_files 68 | sorted_files.sort() 69 | 70 | return sorted_files, links_dict 71 | 72 | 73 | 74 | 75 | 76 | -------------------------------------------------------------------------------- /gslab_make/private/messages.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | ###################################################### 4 | # Define Messages 5 | ###################################################### 6 | 7 | # 1) Critical Errors 8 | crit_error_log = 'ERROR! Cannot open logfile %s' 9 | crit_error_nomakelog = 'ERROR! Makelog %s not found (either not started or deleted)' 10 | crit_error_log_header = 'ERROR! %s is not a log of %s (first line should be "%s")' 11 | crit_error_links_header = 'ERROR! %s is not a links file (first line should be "%s")' 12 | crit_error_extfile = 'ERROR! Cannot open externals file %s' 13 | crit_error_file = 'ERROR! Cannot open file %s' 14 | crit_error_emptylist = '''ERROR! Source directory, and/or matched list, is empty. Please check.''' 15 | crit_error_copyincomplete = 'COPY ERROR! Copy not complete on %s%s.' 16 | crit_error_extension = 'ERROR! %s%s does not have the right program extension' 17 | crit_error_no_file = 'ERROR! File %s not found' 18 | crit_error_unknown_system = 'ERROR! The program syntax for system %s is not defined' 19 | crit_error_bad_command = 'ERROR! Command %s executed with errors' 20 | crit_error_no_directory = 'ERROR! Directory %s does not exist' 21 | crit_error_no_dta_file = 'Error! DTA output file %s is not listed in the manifest file %s.' 22 | crit_error_option_overlap = 'ERROR! Option %s and command line option %s overlap' 23 | crit_error_no_package = 'ERROR! R package %s not found' 24 | crit_error_github = 'ERROR! Unable to reach github API at %s. Check token and url.' 25 | crit_error_assetid = 'ERROR! Unable to determine asset id for %s in release %s. Check url and token.' 26 | 27 | # 2) Syntax Errors 28 | syn_error_revnum = 'ERROR! Revision number %s for %s%s is not a valid number.' 29 | syn_error_noname = 'ERROR! No file/outfile/outdir name given.' 30 | syn_error_wildfilename = '''ERROR! In externals text file, a prefix can only be set for lines with 31 | a wildcard in the file column (single files can simply be renamed).''' 32 | syn_error_wildoutfile = '''ERROR! When specifying an outfile wildcard, \'*\' must occur at end 33 | of column (i.e., we only allow prefixes).''' 34 | syn_error_noprogram = '''ERROR! This call must take an argument program = "program_name"''' 35 | syn_error_nopackage = '''ERROR! This call must take an argument package = "package_name"''' 36 | syn_error_nocommand = '''ERROR! An argument command = "" must be specified to run the Shell command''' 37 | syn_error_manifest = r'Error in %s! All lines that start with "File:" should have a valid file path' 38 | syn_error_url = 'ERROR! Incorrect url format. Please check externals_github and see associated readme.' 39 | 40 | # 3) Logical Errors 41 | logic_error_revdir = '''ERROR! At line %s%s - source path and revision number must both be defined 42 | or be undefined. Currently, %s%s.''' 43 | logic_error_firstrevdir = 'ERROR! First rev/dir pair cannot be left blank.' 44 | 45 | # 4) Notes & Warnings 46 | note_logstart = '\n %s started:' 47 | note_makelogstart = '\n make.py started:' 48 | note_logend = '\n %s ended:' 49 | note_makelogend = '\n make.py ended:' 50 | note_extfilename = 'Note: not using filename externals.txt.\n' 51 | note_extdir_nofound = 'Note: %s not found.\n' 52 | note_input = '\n Input was: "%s".' 53 | note_svnfail = 'Svn Export unsuccessful on %s%s. Please check externals file.' 54 | note_array = '''Subset of folder, prefixed group of files, or entire folder if rev=COPY, to be exported. 55 | The array is: %s.''' 56 | note_nofile = '\n File %s does not exist.\n' 57 | note_no_csv_file = '\n Warning! CSV output file %s is not listed in the manifest file %s.' 58 | note_no_txt_file = '\n Warning! TXT output file %s is not listed in the manifest file %s.' 59 | note_option_replaced = '\n Note: replacing command line option %s with option %s.\n' 60 | 61 | # 5) Successes 62 | success_del_extdir = '%s successfully deleted.\n' 63 | success_create_extdir = '%s successfully (re)created.\n' 64 | success_svn = 'SVN command passed: %s%s @%s exported to %s%s.' 65 | success_makelink = 'Symlink successfully created. Source: %s%s\tLocal location: %s%s.' 66 | success_copy = 'COPY command passed: %s%s copied to %s%s.' 67 | success_github = 'GitHub command passed: %s exported to %s%s.' 68 | -------------------------------------------------------------------------------- /gslab_make/private/metadata.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | ###################################################### 4 | # Define Metadata 5 | ###################################################### 6 | 7 | makelog_started = False 8 | 9 | # Commands 10 | commands = { 11 | 'svnexport' : 'svn export --force -r%s \"%s%s@%s\" \"%s%s\"', 12 | 'makelinkwin' : 'mklink %s \"%s%s\" \"%s%s\"', 13 | 'makelinkunix' : 'ln -s \"%s%s\" \"%s%s\"', 14 | 'rmdirwin' : 'rmdir %s \"%s\"', 15 | 'rmdirunix' : 'rm %s \"%s\"', 16 | 'stata' : '%s %s do %s', 17 | 'matlab' : '%s -r %s -logfile %s %s', 18 | 'perl' : '%s %s %s %s', 19 | 'python' : '%s %s %s %s', 20 | 'math' : '%s < %s %s', 21 | 'st' : '%s %s', 22 | 'lyx' : '%s %s %s', 23 | 'rbatch' : '%s %s %s %s', 24 | 'rinstall' : '%s %s %s %s', 25 | 'sas' : '%s %s %s' 26 | } 27 | 28 | default_options = { 29 | 'rmdirwin' : '/s /q', 30 | 'rmdirunix' : '-rf', 31 | 'matlabwin' : '-nosplash -minimize -wait', 32 | 'matlabunix' : '-nosplash -nodesktop', 33 | 'statawin' : '/e', 34 | 'stataunix' : '-e', 35 | 'rbatch' : '--no-save', 36 | 'rinstall' : '--no-multiarch', 37 | 'saswin' : '-nosplash', 38 | 'math' : '-noprompt', 39 | 'lyx' : '-e pdf2' 40 | } 41 | 42 | option_overlaps = { 43 | 'matlab' : {'log': '-logfile'}, 44 | 'sas' : {'log': '-log', 'lst': '-print'} 45 | } 46 | 47 | default_executables = { 48 | 'statawin' : '%STATAEXE%', 49 | 'stataunix' : 'statamp', 50 | 'matlab' : 'matlab', 51 | 'perl' : 'perl', 52 | 'python' : 'python', 53 | 'math' : 'math', 54 | 'st' : 'st', 55 | 'lyx' : 'lyx', 56 | 'rbatch' : 'R CMD BATCH', 57 | 'rinstall' : 'R CMD INSTALL', 58 | 'sas' : 'sas' 59 | } 60 | 61 | extensions = { 62 | 'stata' : '.do', 63 | 'matlab' : '.m', 64 | 'perl' : '.pl', 65 | 'python' : '.py', 66 | 'math' : '.m', 67 | 'stc' : '.stc', 68 | 'stcmd' : '.stcmd', 69 | 'lyx' : '.lyx', 70 | 'rbatch' : '.R', 71 | 'rinstall' : '', 72 | 'sas' : '.sas', 73 | 'other' : '' 74 | } 75 | 76 | option_start_chars = ['-', '+'] 77 | 78 | # Locals 79 | file_loc = { 80 | 'svn' : 'https://econ-gentzkow-svn.stanford.edu/repos/main/trunk', 81 | 'svnbranch' : 'https://econ-gentzkow-svn.stanford.edu/repos/main/branches', 82 | 'svn_retail2' : 'file:///data/svn/repository/retailer2/trunk', 83 | 'svnbranch_retail2' : 'file:///data/svn/repository/retailer2/branches', 84 | 'gslab_l' : r'//Gentzkow-dt1/GSLAB_L' 85 | } 86 | 87 | # Settings (directory keys must end in 'dir' and file keys must end in 'file') 88 | settings = { 89 | 'external_dir' : '../external/', 90 | 'links_dir' : '../external_links/', 91 | 'externalslog_file' : './get_externals.log', 92 | 'githublog_file' : './get_externals_github.log', 93 | 'linkslog_file' : './make_links.log', 94 | 'output_dir' : '../output/', 95 | 'output_local_dir' : '../output_local/', 96 | 'temp_dir' : '../temp/', 97 | 'makelog_file' : '../output/make.log', 98 | 'manifest_file' : '../output/data_file_manifest.log', 99 | 'link_logs_dir' : '../log/', 100 | 'link_stats_file' : 'link_stats.log', 101 | 'link_heads_file' : 'link_heads.log', 102 | 'link_orig_file' : 'link_orig.log', 103 | 'stats_file' : 'stats.log', 104 | 'heads_file' : 'heads.log' 105 | } 106 | -------------------------------------------------------------------------------- /gslab_make/private/preliminaries.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | import sys 4 | import os 5 | import datetime 6 | import re 7 | import traceback 8 | import shutil 9 | 10 | import messages as messages 11 | import metadata as metadata 12 | 13 | from exceptionclasses import CustomError, CritError 14 | 15 | #== Logging =============================================== 16 | def start_logging(log, logtype): 17 | try: 18 | LOGFILE = open(log,'wb') 19 | except: 20 | raise CustomError.crit(messages.crit_error_log % log) 21 | time_begin = datetime.datetime.now().replace(microsecond=0) 22 | orig_stderr = sys.stderr 23 | sys.stderr = LOGFILE 24 | working_dir = os.getcwd() 25 | print >> LOGFILE, messages.note_logstart % logtype, time_begin, working_dir 26 | return LOGFILE 27 | 28 | def end_logging(LOGFILE, makelog, logtype): 29 | time_end = datetime.datetime.now().replace(microsecond=0) 30 | print >> LOGFILE, messages.note_logend % logtype,time_end 31 | LOGFILE.close() 32 | if not makelog: return 33 | if not (metadata.makelog_started and os.path.isfile(makelog)): 34 | raise CritError(messages.crit_error_nomakelog % makelog) 35 | MAKE_LOGFILE = open(makelog, 'ab') 36 | MAKE_LOGFILE.write( open(LOGFILE.name, 'rU').read() ) 37 | MAKE_LOGFILE.close() 38 | os.remove(LOGFILE.name) 39 | #========================================================== 40 | 41 | def input_to_array(filename): 42 | # Import file 43 | try: 44 | FILENAME = open(filename, 'rU') 45 | except: 46 | raise CritError(messages.crit_error_file % filename) 47 | 48 | # Delete header 49 | filearray = [] 50 | for line in FILENAME: 51 | if ( not re.match('rev', line) and not re.match('linkpath', line) 52 | and not re.match('\s*\#',line) and not re.match('\s*$',line) and not re.match('url', line)): 53 | filearray.append(line.rstrip('\n')) 54 | FILENAME.close() 55 | 56 | return filearray 57 | 58 | #== Print error =========================================== 59 | def print_error(LOGFILE): 60 | print '\n' 61 | print >> LOGFILE, '\n' 62 | print 'Error Found' 63 | traceback.print_exc(file = LOGFILE) 64 | traceback.print_exc(file = sys.stdout) 65 | 66 | def add_error_to_log(makelog): 67 | if not makelog: return 68 | if not (metadata.makelog_started and os.path.isfile(makelog)): 69 | raise CritError(messages.crit_error_nomakelog % makelog) 70 | LOGFILE = open(makelog, 'ab') 71 | print_error(LOGFILE) 72 | LOGFILE.close() 73 | 74 | 75 | 76 | #== Walk through directories ============================== 77 | def files_list (read_dir, recur_lim): 78 | """Generate a list of all files in "read_dir" and its subdirectories (up to 79 | a depth of "recur_lim" -- if recur_lim is 0, there is no depth limit).""" 80 | 81 | all_files = [] 82 | 83 | walk = walk_dir(read_dir, recur_lim) 84 | 85 | for dir_name, file_list in walk: 86 | for file_name in file_list: 87 | all_files.append(os.path.join(dir_name, file_name)) 88 | 89 | all_files.sort() 90 | return all_files 91 | 92 | def walk_dir(read_dir, recur_lim): 93 | """ Yields a matching of all non-hidden subdirectories of "read_dir" to the 94 | files in the subdirectories up to a depth of "recur_lim" -- if recur_lim is 95 | 0 (or False), there is no depth limit. """ 96 | 97 | if in_current_drive(read_dir): 98 | read_dir = os.path.abspath(read_dir) 99 | 100 | if recur_lim: 101 | dir_files = walk_lim(read_dir, 1, recur_lim) 102 | for i in range(len(dir_files)): 103 | yield dir_files[i] 104 | else: 105 | for root, dirs, files in os.walk(read_dir): 106 | if in_current_drive(root): 107 | root = os.path.abspath(root) 108 | this_dir = os.path.basename(root) 109 | if not this_dir.startswith('.'): 110 | files = [ f for f in files if not f.startswith('.') ] 111 | yield root, files 112 | else: 113 | del dirs[:] 114 | 115 | def walk_lim (read_dir, current_depth, recur_lim): 116 | """Recursively match all non-hidden files and subdirectories of "read_dir", 117 | where read_dir is "current_depth" directories deep from the original 118 | directory, and there is a maximum depth of "recur_lim" """ 119 | 120 | if in_current_drive(read_dir): 121 | read_dir = os.path.abspath(read_dir) 122 | dir_list = os.listdir(read_dir) 123 | 124 | files = [ f for f in dir_list if os.path.isfile(os.path.join(read_dir, f)) ] 125 | files = [ f for f in files if not f.startswith('.') ] 126 | 127 | output = [[read_dir, files]] 128 | 129 | current_depth = current_depth + 1 130 | 131 | if current_depth <= recur_lim: 132 | dirs = [ d for d in dir_list if os.path.isdir(os.path.join(read_dir, d)) ] 133 | dirs = [ d for d in dirs if not d.startswith('.') ] 134 | for d in dirs: 135 | walk = walk_lim(os.path.join(read_dir, d), 136 | current_depth, 137 | recur_lim) 138 | for w in walk: 139 | output.append(w) 140 | 141 | return output 142 | 143 | def in_current_drive (dir): 144 | current_drive = os.path.splitdrive(os.getcwd())[0] 145 | other_drive = os.path.splitdrive(dir)[0] 146 | return current_drive == other_drive 147 | 148 | #========================================================== 149 | 150 | def externals_preliminaries(makelog, externals_file, LOGFILE): 151 | if makelog == '@DEFAULTVALUE@': 152 | makelog = metadata.settings['makelog_file'] 153 | if externals_file!='externals.txt': 154 | print >> LOGFILE, messages.note_extfilename 155 | externals = input_to_array(externals_file) 156 | 157 | # Prepare last rev/dir variables 158 | last_dir = '' 159 | last_rev = '' 160 | return([makelog, externals, last_dir, last_rev]) 161 | 162 | -------------------------------------------------------------------------------- /gslab_make/tests/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | This directory contains the `gslab_make` library's unit tests. 3 | These tests can be run using 4 | `python -m unittest discover` 5 | or 6 | `python run_all_tests.py` 7 | from `gslab_make/tests/`. The latter command stores the test 8 | results in `gslab_make/tests/log/make.log`. 9 | ''' 10 | 11 | from nostderrout import nostderrout 12 | -------------------------------------------------------------------------------- /gslab_make/tests/nostderrout.py: -------------------------------------------------------------------------------- 1 | import contextlib 2 | import sys 3 | 4 | @contextlib.contextmanager 5 | def nostderrout(): 6 | savestderr = sys.stderr 7 | savestdout = sys.stdout 8 | class Devnull(object): 9 | def write(self, _): pass 10 | sys.stderr = Devnull() 11 | sys.stdout = Devnull() 12 | yield 13 | sys.stderr = savestderr 14 | sys.stdout = savestdout -------------------------------------------------------------------------------- /gslab_misc/SaveData/SaveData.py: -------------------------------------------------------------------------------- 1 | # -*- coding: utf-8 -*- 2 | 3 | import pandas as pd 4 | import hashlib 5 | import re 6 | import pathlib 7 | 8 | def SaveData(df, keys, out_file, log_file = '', append = False, sortbykey = True): 9 | extension = CheckExtension(out_file) 10 | CheckColumnsNotList(df) 11 | CheckKeys(df, keys) 12 | # reorder df so keys are on the left 13 | cols_reordered = keys + [col for col in df.columns if col not in keys] 14 | df = df[cols_reordered] 15 | df_hash = hashlib.md5(pd.util.hash_pandas_object(df).values).hexdigest() 16 | summary_stats = GetSummaryStats(df) 17 | SaveDf(df, keys, out_file, sortbykey, extension) 18 | SaveLog(df_hash, keys, summary_stats, out_file, append, log_file) 19 | 20 | 21 | def CheckExtension(out_file): 22 | if type(out_file) == str: 23 | extension = re.findall(r'\.[a-z]+$', out_file) 24 | elif type(out_file) == pathlib.PosixPath: 25 | extension = [out_file.suffix] 26 | else: 27 | raise ValueError('Output file format must either be string or pathlib.PosixPath') 28 | if not extension[0] in ['.csv', '.dta']: 29 | raise ValueError("File extension should be one of .csv or .dta.") 30 | return extension[0] 31 | 32 | def CheckColumnsNotList(df): 33 | type_list = [any(df[col].apply(lambda x: type(x) == list)) for col in df.columns] 34 | if any(type_list): 35 | type_list_columns = df.columns[type_list] 36 | raise TypeError("No column can be of type list - check the following columns: " + ", ".join(type_list_columns)) 37 | 38 | 39 | 40 | def CheckKeys(df, keys): 41 | if not isinstance(keys, list): 42 | raise TypeError("Keys must be specified as a list.") 43 | 44 | for key in keys: 45 | if not key in df.columns: 46 | print('%s is not a column name.' % (key)) 47 | raise ValueError('One of the keys you specified is not among the columns.') 48 | 49 | df_keys = df[keys] 50 | 51 | keys_with_missing = df_keys.columns[df_keys.isnull().any()] 52 | if keys_with_missing.any(): 53 | missings_string = ', '.join(keys_with_missing) 54 | raise ValueError(f'The following keys are missing in some rows: {missings_string}.') 55 | 56 | 57 | 58 | 59 | type_list = any([any(df[keycol].apply(lambda x: type(x) == list)) for keycol in keys]) 60 | if type_list: 61 | raise TypeError("No key can contain keys of type list") 62 | 63 | 64 | if not all(df.groupby(keys).size() == 1): 65 | raise ValueError("Keys do not uniquely identify the observations.") 66 | 67 | 68 | def GetSummaryStats(df): 69 | var_types = df.dtypes 70 | with pd.option_context("future.no_silent_downcasting", True): 71 | var_stats = df.describe(include='all').transpose().fillna('').infer_objects(copy=False) 72 | 73 | var_stats['count'] = df.notnull().sum() 74 | var_stats = var_stats.drop(columns=['top', 'freq'], errors='ignore') 75 | 76 | summary_stats = pd.DataFrame({'type': var_types}).\ 77 | merge(var_stats, how = 'left', left_index = True, right_index = True) 78 | summary_stats = summary_stats.round(4) 79 | 80 | return summary_stats 81 | 82 | 83 | def SaveDf(df, keys, out_file, sortbykey, extension): 84 | if sortbykey: 85 | df.sort_values(keys, inplace = True) 86 | 87 | if extension == '.csv': 88 | df.to_csv(out_file, index = False) 89 | if extension == '.dta': 90 | df.to_stata(out_file, write_index = False) 91 | 92 | print(f"File '{out_file}' saved successfully.") 93 | 94 | 95 | def SaveLog(df_hash, keys, summary_stats, out_file, append, log_file): 96 | if log_file: 97 | if append: 98 | with open(log_file, 'a') as f: 99 | f.write('\n\n') 100 | f.write('File: %s\n\n' % (out_file)) 101 | f.write('MD5 hash: %s\n\n' % (df_hash)) 102 | f.write('Keys: ') 103 | for item in keys: 104 | f.write('%s ' % (item)) 105 | f.write('\n\n') 106 | f.write(summary_stats.to_string(header = True, index = True)) 107 | f.write("\n\n") 108 | else: 109 | with open(log_file, 'w') as f: 110 | f.write('File: %s\n\n' % (out_file)) 111 | f.write('MD5 hash: %s\n\n' % (df_hash)) 112 | f.write('Keys: ') 113 | for item in keys: 114 | f.write('%s ' % (item)) 115 | f.write('\n\n') 116 | f.write(summary_stats.to_string(header = True, index = True)) 117 | f.write("\n\n") 118 | 119 | f.close() 120 | else: 121 | pass 122 | -------------------------------------------------------------------------------- /gslab_misc/SaveData/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | SaveData: a Python library for saving data in a normalized format. 3 | ===================================================== 4 | 5 | ''' 6 | 7 | from SaveData import SaveData 8 | -------------------------------------------------------------------------------- /gslab_misc/SaveData/tests/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | This directory contains the `SaveData` library's unit tests. These tests can be run using 3 | `python -m unittest discover` 4 | from `SaveData/` or from `SaveData/tests/`. To run the tests with logging, use: 5 | `python run_all_tests.py` 6 | from `SaveData/tests/`. 7 | ''' 8 | -------------------------------------------------------------------------------- /gslab_misc/SaveData/tests/data/data.csv: -------------------------------------------------------------------------------- 1 | id,partid1,partid2,name,num 2 | 1,a,1,aa,10 3 | 2,a,2,bb,9 4 | 3,a,3,cc,8 5 | 4,b,1,aa,7 6 | 5,b,2,bb,6 7 | 6,b,3,cc,11 8 | 7,c,1,aa,20 9 | 8,c,2,bb,19 10 | 9,c,3,cc,4500 11 | 10,d,1,cc,NA 12 | -------------------------------------------------------------------------------- /gslab_misc/SaveData/tests/test_main.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import sys 3 | import pandas as pd 4 | import os 5 | from pathlib import Path 6 | 7 | sys.path.append('..') 8 | 9 | from SaveData import SaveData 10 | 11 | pd.set_option('future.no_silent_downcasting', True) 12 | 13 | class TestSaveData(unittest.TestCase): 14 | 15 | def test_wrong_extension(self): 16 | df = pd.read_csv('data/data.csv') 17 | with self.assertRaises(ValueError): 18 | SaveData(df, ['id'], 'dfs.pdf') 19 | 20 | def test_wrong_keytype(self): 21 | df = pd.read_csv('data/data.csv') 22 | with self.assertRaises(TypeError): 23 | SaveData(df, 'id', 'dfs.csv') 24 | 25 | def test_wrong_key_column_list(self): 26 | df = pd.read_csv('data/data.csv') 27 | df['list_id'] = df['id'].apply(lambda x: [x]) 28 | with self.assertRaises(TypeError): 29 | SaveData(df, 'list_id', 'dfs.csv') 30 | 31 | def test_wrong_key_column_containing_list(self): 32 | df = pd.read_csv('data/data.csv') 33 | df['list_id'] = df['id'].apply(lambda x: [x] if x == 1 else x) 34 | with self.assertRaises(TypeError): 35 | SaveData(df, 'list_id', 'dfs.csv') 36 | 37 | def test_wrong_non_key_column_containing_list(self): 38 | df = pd.read_csv('data/data.csv') 39 | df['list_column'] = df['id'].apply(lambda x: [x] if x == 1 else x) 40 | with self.assertRaises(TypeError): 41 | SaveData(df, 'id', 'dfs.csv') 42 | 43 | def test_key_on_left(self): 44 | df = pd.read_csv('data/data.csv') 45 | df['id2'] = df['id'] 46 | SaveData(df, ['id2'], 'df.csv') 47 | df_saved = pd.read_csv('df.csv') 48 | self.assertEqual(df_saved.columns[0], 'id2') 49 | os.remove('df.csv') 50 | 51 | def test_two_keys_on_left(self): 52 | df = pd.read_csv('data/data.csv') 53 | df['id2'] = df['id'] 54 | SaveData(df, ['id2', 'id'], 'df.csv') 55 | df_saved = pd.read_csv('df.csv') 56 | self.assertEqual(True, all([x == y for x, y in zip(['id2', 'id'], df_saved.columns[:2])])) 57 | os.remove('df.csv') 58 | 59 | def test_missingkeys(self): 60 | df = pd.read_csv('data/data.csv') 61 | with self.assertRaises(ValueError): 62 | SaveData(df, ['num'], 'dfs.csv') 63 | 64 | def test_duplicate_keys(self): 65 | df = pd.read_csv('data/data.csv') 66 | with self.assertRaises(ValueError): 67 | SaveData(df, ['partid1'], 'dfs.csv') 68 | 69 | def test_multiple_keys(self): 70 | df = pd.read_csv('data/data.csv') 71 | SaveData(df, ['id', 'partid1','partid2'], 'df.csv') 72 | df_saved = pd.read_csv('df.csv') 73 | self.assertEqual(True, df.compare(df_saved).shape==(0,0)) 74 | os.remove('df.csv') 75 | 76 | def test_saves_desired_file_dta(self): 77 | df = pd.read_csv('data/data.csv') 78 | SaveData(df, ['id'], 'df.dta') 79 | df_saved = pd.read_stata('df.dta') 80 | self.assertEqual(True, df.compare(df_saved).shape==(0,0)) 81 | os.remove('df.dta') 82 | 83 | def test_saves_desired_file_without_log(self): 84 | df = pd.read_csv('data/data.csv') 85 | SaveData(df, ['id'], 'df.csv') 86 | exists = os.path.isfile('df.csv') 87 | df_saved = pd.read_csv('df.csv') 88 | self.assertEqual(True, df.compare(df_saved).shape==(0,0)) 89 | os.remove('df.csv') 90 | 91 | def test_saves_with_log(self): 92 | df = pd.read_csv('data/data.csv') 93 | SaveData(df, ['id'], 'df.csv', 'df.log') 94 | exists = os.path.isfile('df.log') 95 | self.assertEqual(exists, True) 96 | os.remove('df.log') 97 | os.remove('df.csv') 98 | 99 | def test_saves_when_append_given(self): 100 | df = pd.read_csv('data/data.csv') 101 | SaveData(df, ['id'], 'df.csv') 102 | SaveData(df, ['id'], 'df.csv', 'df.log', append = True) 103 | exists = os.path.isfile('df.log') 104 | self.assertEqual(exists, True) 105 | os.remove('df.log') 106 | os.remove('df.csv') 107 | 108 | def test_saves_when_sort_is_false(self): 109 | df = pd.read_csv('data/data.csv') 110 | SaveData(df, ['id'], 'df.csv', 'df.log', sortbykey = False) 111 | exists = os.path.isfile('df.csv') 112 | self.assertEqual(exists, True) 113 | os.remove('df.log') 114 | os.remove('df.csv') 115 | 116 | def test_saves_with_path(self): 117 | indir = Path('data/data.csv') 118 | outdir_csv = Path('data.csv') 119 | outdir_log = Path('data.log') 120 | df = pd.read_csv(indir) 121 | SaveData(df, ['id'], outdir_csv, outdir_log) 122 | exists = os.path.isfile(str(outdir_log)) 123 | self.assertEqual(exists, True) 124 | os.remove(str(outdir_csv)) 125 | os.remove(str(outdir_log)) 126 | 127 | if __name__ == '__main__': 128 | unittest.main() 129 | -------------------------------------------------------------------------------- /gslab_misc/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | gslab_misc: A library for miscellaneous python tools 3 | ============================================================================================ 4 | Current tools: 5 | gencat: a Python library for concatenating text files. 6 | SaveData: a Python library for saving data in a normalized format. 7 | ''' 8 | 9 | from .gencat.gencat import gencat 10 | -------------------------------------------------------------------------------- /gslab_misc/gencat/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | gencat: a Python library for concatenating text files 3 | ===================================================== 4 | 5 | The gencat (General Concatenator) library defines an abstract class for 6 | concatenating text files. Please see this class's docstring for additional 7 | information on its structure and functionalities. 8 | ''' 9 | 10 | from gencat import gencat 11 | -------------------------------------------------------------------------------- /gslab_misc/gencat/gencat.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import os 3 | import shutil 4 | import zipfile 5 | import zlib 6 | from abc import ABCMeta, abstractmethod 7 | 8 | class gencat(object): 9 | ''' 10 | Tool for concatenating text files stored in .zip files 11 | 12 | gencat (General Concatenation) is an abstract class that concatenates 13 | files stored in a .zip file in a user-specified structure and stores 14 | the new files in a .zip file. Its constructor takes the following 15 | as arguments: 16 | - path_in: the path to the directory containing the .zip files holding the 17 | text files that we wish to concatenate. 18 | - path_temp: the path to a temporary directory used in the intermediate 19 | stages of file concatenation. This directory is created and destroyed 20 | by the class's main method. 21 | - path_out: the path to the directory to which a gencat object will save 22 | its final output. 23 | ''' 24 | 25 | __metaclass__ = ABCMeta 26 | 27 | def __init__(self, path_in, path_temp, path_out): 28 | self.path_in = os.path.join(path_in, '') 29 | self.path_temp = os.path.join(path_temp, '') 30 | self.path_out = os.path.join(path_out, '') 31 | self.concat_dict = {} 32 | self.zip_dict = {} 33 | 34 | 35 | def main(self): 36 | ''' 37 | Run all methods in order to produce fresh output. 38 | Begins by wiping the path_temp and path_out directories. 39 | ''' 40 | self.cleanDir(self.path_temp) 41 | self.cleanDir(self.path_out) 42 | self.unzipFiles() 43 | self.makeConcatDict() 44 | self.makeZipDict() 45 | self.checkDicts() 46 | self.writeDict(self.concat_dict, 'concatDict.txt', self.path_temp) 47 | self.writeDict(self.zip_dict, 'zipDict.txt', '.') 48 | self.zipFiles() 49 | self.cleanDir(self.path_temp, new_dir = False) 50 | 51 | 52 | def cleanDir(self, path, new_dir = True): 53 | ''' 54 | Remove path and all subdirectories below. 55 | Recreates path directory unless new_dir is False 56 | ''' 57 | if path != self.path_in: 58 | shutil.rmtree(path, ignore_errors = True) 59 | if new_dir != False: 60 | os.makedirs(path) 61 | 62 | def unzipFiles(self): 63 | ''' 64 | Unzips files from path_in to path_temp 65 | ''' 66 | infilenames = os.listdir(self.path_in) 67 | 68 | for infilename in infilenames: 69 | infile = os.path.join(self.path_in, infilename) 70 | 71 | if zipfile.is_zipfile(infile): 72 | with zipfile.ZipFile(infile, 'r') as zf: 73 | zf.extractall(self.path_temp) 74 | 75 | @abstractmethod 76 | def makeConcatDict(self): 77 | ''' 78 | This method should assign a dictionary to self.concat_dict where each key is a distinct concatenated 79 | filename and the values for the key are all raw files to be concatenated. 80 | ''' 81 | pass 82 | 83 | @abstractmethod 84 | def makeZipDict(self): 85 | ''' 86 | This method should assign a dictionary to self.zip_dict where each key is a distinct zipfile and the 87 | values for the key are all concatenated files to be contained in the zipfile. 88 | ''' 89 | pass 90 | 91 | def checkDicts(self): 92 | ''' 93 | Raises an exception if ZipDict or ConcatDict is empty or has non-tuple values. 94 | ''' 95 | for d in [self.concat_dict, self.zip_dict]: 96 | if not d: 97 | raise Exception('THe dictionary %s must be non-empty' % (d)) 98 | else: 99 | for key in d: 100 | if not type(d[key]) is tuple: 101 | raise TypeError('All keys in dictionary %s must be tuples. Check key %s, and try again.' % (d, key)) 102 | 103 | 104 | def writeDict(self, dict, name, rel_path): 105 | ''' 106 | Write the dictionary to output as a |-delimited text file. The elements of each tuple are 107 | shortened to their filenames for writing only. 108 | ''' 109 | outfile_path = os.path.join(self.path_out, name) 110 | with open(outfile_path, 'wb') as outfile: 111 | 112 | for key in sorted(dict.keys()): 113 | outfile.write(key) 114 | 115 | for val in dict[key]: 116 | write = os.path.relpath(val, rel_path) 117 | outfile.write('|' + write) 118 | 119 | outfile.write('\n') 120 | 121 | 122 | def zipFiles(self): 123 | ''' 124 | Concatenates all files in a dictionary values to a new file named for the corresponding key. 125 | Files are concatenated in the order in which they appear in the dictionary value. 126 | Places NEWFILE\nFILENAME: before each new file in the concatenation. 127 | Stores all concatenated files to .zip file(s) with ZIP64 compression in path_out. 128 | ''' 129 | for zip_key in self.zip_dict.keys(): 130 | catdirpath = os.path.join(self.path_temp, zip_key, '') 131 | os.makedirs(catdirpath) 132 | inzippath = os.path.join('..', zip_key, '') 133 | self.cleanDir(inzippath) 134 | 135 | outzipname = zip_key + '.zip' 136 | outzippath = os.path.join(self.path_out, outzipname) 137 | zf = zipfile.ZipFile(outzippath, 'a', zipfile.ZIP_DEFLATED, True) 138 | 139 | for zip_val in self.zip_dict[zip_key]: 140 | catfilename = zip_val + '.txt' 141 | catfilepath = os.path.join(catdirpath, catfilename) 142 | with open(catfilepath, 'ab') as catfile: 143 | concat_key = zip_val 144 | for concat_val in self.concat_dict[concat_key]: 145 | catfile.write('\nNEWFILE\nFILENAME: %s\n\n' % (os.path.basename(concat_val))) 146 | with open(concat_val, 'rU') as f: 147 | for line in f: 148 | catfile.write(line) 149 | 150 | inzipfile = os.path.join(inzippath, catfilename) 151 | shutil.copyfile(catfilepath, inzipfile) 152 | zf.write(inzipfile) 153 | 154 | self.cleanDir(inzippath, new_dir = False) 155 | -------------------------------------------------------------------------------- /gslab_misc/gencat/tests/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | This directory contains the `gencat` library's unit tests. These tests can be run using 3 | `python -m unittest discover` 4 | from `gencat/` or from `gencat/tests/`. To run the tests with logging, use: 5 | `python run_all_tests.py` 6 | from `gencat/tests/`. 7 | ''' 8 | -------------------------------------------------------------------------------- /gslab_misc/gencat/tests/log/make.log: -------------------------------------------------------------------------------- 1 | test_default (test_checkDicts.test_checkDicts) ... ok 2 | test_notAllDicts (test_checkDicts.test_checkDicts) ... ok 3 | test_notAllTuple (test_checkDicts.test_checkDicts) ... ok 4 | test_cleanDir (test_cleanDir.test_cleanDir) ... ok 5 | test_makeDir (test_cleanDir.test_cleanDir) ... ok 6 | test_noNew (test_cleanDir.test_cleanDir) ... ok 7 | test_recursiveClear (test_cleanDir.test_cleanDir) ... ok 8 | test_recursiveMake (test_cleanDir.test_cleanDir) ... ok 9 | test_strangeFlag (test_cleanDir.test_cleanDir) ... ok 10 | test_default (test_main.test_main) ... ok 11 | test_aZipFile (test_unzipFiles.test_unzipFiles) ... ok 12 | test_blankZipFile (test_unzipFiles.test_unzipFiles) ... ok 13 | test_noFile (test_unzipFiles.test_unzipFiles) ... ok 14 | test_noZipFile (test_unzipFiles.test_unzipFiles) ... ok 15 | test_twoFile (test_unzipFiles.test_unzipFiles) ... ok 16 | test_twoZipFile (test_unzipFiles.test_unzipFiles) ... ok 17 | test_noPath (test_writeDict.test_writeDict) ... ok 18 | test_onePath (test_writeDict.test_writeDict) ... ok 19 | test_twoFile (test_writeDict.test_writeDict) ... ok 20 | test_twopPath (test_writeDict.test_writeDict) ... ok 21 | test_oneFile (test_zipFile.test_zipFiles) ... ok 22 | test_twoConcatsOneZip (test_zipFile.test_zipFiles) ... ok 23 | test_twoFile (test_zipFile.test_zipFiles) ... ok 24 | test_twoZips (test_zipFile.test_zipFiles) ... ok 25 | 26 | ---------------------------------------------------------------------- 27 | Ran 24 tests in 0.073s 28 | 29 | OK 30 | -------------------------------------------------------------------------------- /gslab_misc/gencat/tests/log/test.log: -------------------------------------------------------------------------------- 1 | test_default (test_checkDicts.test_checkDicts) ... ok 2 | test_notAllDicts (test_checkDicts.test_checkDicts) ... ok 3 | test_notAllTuple (test_checkDicts.test_checkDicts) ... ok 4 | test_cleanDir (test_cleanDir.test_cleanDir) ... ok 5 | test_makeDir (test_cleanDir.test_cleanDir) ... ok 6 | test_noNew (test_cleanDir.test_cleanDir) ... ok 7 | test_recursiveClear (test_cleanDir.test_cleanDir) ... ok 8 | test_recursiveMake (test_cleanDir.test_cleanDir) ... ok 9 | test_strangeFlag (test_cleanDir.test_cleanDir) ... ok 10 | test_default (test_main.test_main) ... ok 11 | test_aZipFile (test_unzipFiles.test_unzipFiles) ... ok 12 | test_blankZipFile (test_unzipFiles.test_unzipFiles) ... ok 13 | test_noFile (test_unzipFiles.test_unzipFiles) ... ok 14 | test_noZipFile (test_unzipFiles.test_unzipFiles) ... ok 15 | test_twoFile (test_unzipFiles.test_unzipFiles) ... ok 16 | test_twoZipFile (test_unzipFiles.test_unzipFiles) ... ok 17 | test_noPath (test_writeDict.test_writeDict) ... ok 18 | test_onePath (test_writeDict.test_writeDict) ... ok 19 | test_twoFile (test_writeDict.test_writeDict) ... ok 20 | test_twopPath (test_writeDict.test_writeDict) ... ok 21 | test_oneFile (test_zipFile.test_zipFiles) ... ok 22 | test_twoConcatsOneZip (test_zipFile.test_zipFiles) ... ok 23 | test_twoFile (test_zipFile.test_zipFiles) ... ok 24 | test_twoZips (test_zipFile.test_zipFiles) ... ok 25 | 26 | ---------------------------------------------------------------------- 27 | Ran 24 tests in 0.079s 28 | 29 | OK 30 | -------------------------------------------------------------------------------- /gslab_misc/gencat/tests/run_all_tests.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | import unittest 4 | 5 | loader = unittest.TestLoader() 6 | tests = loader.discover('.') 7 | 8 | with open('./log/make.log', 'wb') as log: 9 | testRunner = unittest.TextTestRunner(stream = log, verbosity = 2) 10 | testRunner.run(tests) 11 | 12 | with open('./log/make.log', 'rU') as log: 13 | print '\n=== Test results ' + '='*53 14 | print log.read() 15 | 16 | -------------------------------------------------------------------------------- /gslab_misc/gencat/tests/test_checkDicts.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | import shutil 4 | import zipfile 5 | import sys 6 | 7 | # Ensure that Python can find and load gencat.py 8 | os.chdir(os.path.dirname(os.path.realpath(__file__))) 9 | sys.path.append('..') 10 | 11 | from gencat import gencat 12 | 13 | class test_checkDicts(unittest.TestCase): 14 | 15 | def setUp(self): 16 | paths = ['./test_data', './test_temp', './test_out'] 17 | for path in paths: 18 | try: 19 | os.makedirs(path) 20 | except: 21 | shutil.rmtree(path, ignore_errors = True) 22 | os.makedirs(path) 23 | 24 | def test_default(self): 25 | ''' 26 | Test that no exception is raised when dictionaries exist and all 27 | keys are tuples. 28 | ''' 29 | class MockCat(gencat): 30 | def makeZipDict(self): 31 | self.zip_dict = {'a': ('tuple1', )} 32 | def makeConcatDict(self): 33 | self.concat_dict = {'b': ('tuple2', )} 34 | testcat = MockCat('./test_data', './test_temp', './test_out') 35 | testcat.makeConcatDict() 36 | testcat.makeZipDict() 37 | 38 | testcat.checkDicts() 39 | 40 | def test_notAllTuple(self): 41 | ''' 42 | Test that TypeError is raised when dictionaries exist but do not 43 | have tuple-valued keys. 44 | ''' 45 | class MockCat(gencat): 46 | def makeZipDict(self): 47 | self.zip_dict = {'a': 'not_a_tuple_tuple1'} 48 | def makeConcatDict(self): 49 | self.concat_dict = {'b', ('tuple2', )} 50 | testcat = MockCat('./test_data', './test_temp', './test_out') 51 | testcat.makeConcatDict() 52 | testcat.makeZipDict() 53 | 54 | with self.assertRaises(TypeError): 55 | testcat.checkDicts() 56 | 57 | def test_notAllDicts(self): 58 | ''' 59 | Test that Exception is raised when a dictionary does not exist. 60 | ''' 61 | class MockCat(gencat): 62 | def makeZipDict(self): 63 | self.zip_dict = {} 64 | def makeConcatDict(self): 65 | self.concat_dict = {'b', ('tuple2', )} 66 | testcat = MockCat('./test_data', './test_temp', './test_out') 67 | testcat.makeConcatDict() 68 | testcat.makeZipDict() 69 | 70 | with self.assertRaises(Exception): 71 | testcat.checkDicts() 72 | 73 | def tearDown(self): 74 | paths = ['./test_data', './test_temp', './test_out'] 75 | for path in paths: 76 | shutil.rmtree(path, ignore_errors = True) 77 | 78 | 79 | if __name__ == '__main__': 80 | unittest.main() 81 | -------------------------------------------------------------------------------- /gslab_misc/gencat/tests/test_cleanDir.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | import shutil 4 | import sys 5 | 6 | # Ensure that Python can find and load gencat.py 7 | os.chdir(os.path.dirname(os.path.realpath(__file__))) 8 | sys.path.append('..') 9 | 10 | from gencat import gencat 11 | 12 | 13 | class MockCat(gencat): 14 | def makeZipDict(self): 15 | pass 16 | 17 | def makeConcatDict(self): 18 | pass 19 | 20 | 21 | testcat = MockCat('./test_data', './test_temp', './out_temp') 22 | 23 | 24 | class test_cleanDir(unittest.TestCase): 25 | 26 | def setUp(self): 27 | paths = ['./test_data', './test_temp', './test_out'] 28 | for path in paths: 29 | try: 30 | os.makedirs(path) 31 | except: 32 | shutil.rmtree(path, ignore_errors = True) 33 | os.makedirs(path) 34 | 35 | def test_cleanDir(self): 36 | ''' 37 | Test that files in directory are cleared. 38 | ''' 39 | with open('test_data/test_file', 'wb') as f: 40 | f.write('test') 41 | testcat.cleanDir('./test_data') 42 | self.assertFalse(os.listdir('./test_data')) 43 | 44 | def test_makeDir(self): 45 | ''' 46 | Test that method creates a new directory. 47 | ''' 48 | try: 49 | shutil.rmtree('./test') 50 | except: 51 | pass 52 | testcat.cleanDir('./test') 53 | self.assertTrue(os.path.isdir('./test')) 54 | 55 | def test_strangeFlag(self): 56 | ''' 57 | Test that new_dir flag does not affect output unless False is entered. 58 | ''' 59 | testcat.cleanDir('./test', new_dir = 'gibberish1!') 60 | self.assertTrue(os.path.isdir('./test')) 61 | 62 | def test_noNew(self): 63 | ''' 64 | Test that a new directory is not created if the new_dir flag is False. 65 | ''' 66 | testcat.cleanDir('./test', new_dir = False) 67 | self.assertFalse(os.path.isdir('./test')) 68 | 69 | def test_recursiveMake(self): 70 | ''' 71 | Test that the cleanDir method functions creates directories recursively. 72 | ''' 73 | testcat.cleanDir('./a/deep/test/dir') 74 | self.assertTrue(os.path.isdir('./a/deep/test/dir')) 75 | shutil.rmtree('./a') 76 | 77 | def test_recursiveClear(self): 78 | ''' 79 | Test that the cleanDir method clears directories recursively. 80 | ''' 81 | os.makedirs('./a/deep/test/dir') 82 | testcat.cleanDir('./a') 83 | self.assertTrue(os.path.isdir('./a')) 84 | self.assertFalse(os.path.isdir('./a/deep')) 85 | os.removedirs('./a') 86 | 87 | def tearDown(self): 88 | paths = ['./test_data', './test_temp', './test_out', './test'] 89 | for path in paths: 90 | shutil.rmtree(path, ignore_errors = True) 91 | 92 | 93 | if __name__ == '__main__': 94 | unittest.main() -------------------------------------------------------------------------------- /gslab_misc/gencat/tests/test_main.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | import shutil 4 | import zipfile 5 | import sys 6 | 7 | # Ensure that Python can find and load gencat.py 8 | os.chdir(os.path.dirname(os.path.realpath(__file__))) 9 | sys.path.append('../') 10 | 11 | from gencat import gencat 12 | 13 | 14 | class MockCat(gencat): 15 | 16 | def makeZipDict(self): 17 | self.zip_dict = {} 18 | self.zip_dict['zip1'] = ('concat1', ) 19 | 20 | def makeConcatDict(self): 21 | self.concat_dict = {} 22 | self.concat_dict['concat1'] = ('./test_data/file1.txt', ) + ('./test_data/file2.txt', ) 23 | 24 | 25 | class test_main(unittest.TestCase): 26 | 27 | def setUp(self): 28 | paths = ['./test_data'] 29 | for path in paths: 30 | try: 31 | os.makedirs(path) 32 | except: 33 | shutil.rmtree(path, ignore_errors = True) 34 | os.makedirs(path) 35 | count = 1 36 | for FILE in ['./test_data/file1.txt', './test_data/file2.txt']: 37 | with open(FILE, 'wb') as f: 38 | f.write('THIS IS TEST FILE %s.\n' % (count)) 39 | count = count + 1 40 | 41 | def test_default(self): 42 | ''' 43 | Test that the lines in main run in the intended order and produce predictable output 44 | when given simple input. 45 | ''' 46 | testcat = MockCat('./test_data', './test_temp', './test_out') 47 | testcat.main() 48 | 49 | self.assertFalse(os.path.isdir('./test_temp')) 50 | self.assertTrue(os.path.isfile('./test_out/concatDict.txt')) 51 | self.assertTrue(os.path.isfile('./test_out/zipDict.txt')) 52 | self.assertTrue(os.path.isfile('./test_out/zip1.zip')) 53 | self.assertTrue(zipfile.is_zipfile('./test_out/zip1.zip')) 54 | with zipfile.ZipFile('./test_out/zip1.zip', 'r') as zf: 55 | zf.extractall('./test_out/') 56 | 57 | with open('./test_out/zip1/concat1.txt', 'rU') as f: 58 | text = f.read() 59 | 60 | test_text = '\nNEWFILE\nFILENAME: file1.txt\n\nTHIS IS TEST FILE 1.' + \ 61 | '\n\nNEWFILE\nFILENAME: file2.txt\n\nTHIS IS TEST FILE 2.\n' 62 | self.assertEqual(text, test_text) 63 | 64 | def tearDown(self): 65 | paths = ['./test_data', './test_out'] 66 | for path in paths: 67 | shutil.rmtree(path, ignore_errors = True) 68 | 69 | 70 | if __name__ == '__main__': 71 | unittest.main() 72 | 73 | -------------------------------------------------------------------------------- /gslab_misc/gencat/tests/test_unzipFiles.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | import shutil 4 | import zipfile 5 | import sys 6 | 7 | # Ensure that Python can find and load gencat.py 8 | os.chdir(os.path.dirname(os.path.realpath(__file__))) 9 | sys.path.append('../') 10 | 11 | from gencat import gencat 12 | 13 | class MockCat(gencat): 14 | def makeZipDict(self): 15 | pass 16 | def makeConcatDict(self): 17 | pass 18 | 19 | 20 | testcat = MockCat('./test_data', './test_temp', './out_temp') 21 | 22 | 23 | class test_unzipFiles(unittest.TestCase): 24 | 25 | def setUp(self): 26 | paths = ['./test_data', './test_temp', './test_out'] 27 | for path in paths: 28 | try: 29 | os.makedirs(path) 30 | except: 31 | shutil.rmtree(path, ignore_errors = True) 32 | os.makedirs(path) 33 | 34 | def test_noFile(self): 35 | ''' 36 | Test that an empty list is returned when there is no file in the input directory. 37 | ''' 38 | testcat.unzipFiles() 39 | l = os.listdir('./test_out') 40 | self.assertEqual(l, []) 41 | 42 | def test_noZipFile(self): 43 | ''' 44 | Test that an empty list is returned when there is no zip file in the input directory. 45 | ''' 46 | with open('./test_data/test.txt', 'wb') as f: 47 | f.write('test') 48 | 49 | testcat.unzipFiles() 50 | l = os.listdir('./test_out') 51 | self.assertEqual(l, []) 52 | 53 | def test_blankZipFile(self): 54 | ''' 55 | Test that nothing is returned by a zip file without content. 56 | ''' 57 | # Set up 58 | inzip = zipfile.ZipFile('test_temp/test_zip.zip', 'w', zipfile.ZIP_DEFLATED, True) 59 | inzip.close() 60 | 61 | # Test Zipping 62 | self.assertTrue(os.path.isfile('test_temp/test_zip.zip')) 63 | self.assertTrue(zipfile.is_zipfile('test_temp/test_zip.zip')) 64 | 65 | # Test Unzipping 66 | testcat.unzipFiles() 67 | outzip = zipfile.ZipFile('test_temp/test_zip.zip') 68 | outzip.extractall('test_temp') 69 | 70 | # Test Content 71 | self.assertEqual(os.listdir('test_temp/'), ['test_zip.zip']) 72 | 73 | def test_aZipFile(self): 74 | ''' 75 | Test that a single file in a single zip file is unzipped and that content is preserved. 76 | ''' 77 | # Set up 78 | inzip = zipfile.ZipFile('test_temp/test_zip.zip', 'w', zipfile.ZIP_DEFLATED, True) 79 | with open('test_data/test_text.txt', 'wb') as f: 80 | f.write('test\ntest') 81 | inzip.write('test_data/test_text.txt') 82 | inzip.close() 83 | 84 | # Test Zipping 85 | self.assertTrue(os.path.isfile('test_temp/test_zip.zip')) 86 | self.assertTrue(zipfile.is_zipfile('test_temp/test_zip.zip')) 87 | 88 | # Test Unzipping 89 | testcat.unzipFiles() 90 | outzip = zipfile.ZipFile('test_temp/test_zip.zip') 91 | outzip.extractall('test_temp') 92 | self.assertTrue(os.path.isfile('test_temp/test_data/test_text.txt')) 93 | 94 | # Test Content 95 | with open('test_temp/test_data/test_text.txt', 'rU') as f: 96 | lines = f.readlines() 97 | 98 | count = 0 99 | for line in lines: 100 | line = line.strip() 101 | self.assertEqual(line, 'test') 102 | count = count + 1 103 | self.assertEqual(count, 2) 104 | 105 | def test_twoFile(self): 106 | ''' 107 | Test that a single zip file containing two text files is unzipped and that content is preserved. 108 | ''' 109 | # Set up 110 | files = ['test1', 'test2'] 111 | inzip = zipfile.ZipFile('test_temp/test_zip.zip', 'w', zipfile.ZIP_DEFLATED, True) 112 | for f in files: 113 | with open('test_data/%s_text.txt' % f, 'wb') as fi: 114 | fi.write('%s\n%s' % (f, f)) 115 | inzip.write('test_data/%s_text.txt' % f) 116 | inzip.close() 117 | 118 | # Test Zipping 119 | self.assertTrue(os.path.isfile('test_temp/test_zip.zip')) 120 | self.assertTrue(zipfile.is_zipfile('test_temp/test_zip.zip')) 121 | 122 | # Test Unzipping 123 | testcat.unzipFiles() 124 | outzip = zipfile.ZipFile('test_temp/test_zip.zip') 125 | outzip.extractall('test_temp') 126 | 127 | for f in files: 128 | self.assertTrue(os.path.isfile('test_temp/test_data/%s_text.txt' % f)) 129 | 130 | # Test Content 131 | for f in files: 132 | with open('test_temp/test_data/%s_text.txt' % f, 'rU') as fi: 133 | lines = fi.readlines() 134 | 135 | count = 0 136 | for line in lines: 137 | line = line.strip() 138 | self.assertEqual(line, f) 139 | count = count + 1 140 | self.assertEqual(count, 2) 141 | 142 | def test_twoZipFile(self): 143 | ''' 144 | Test that two zip files containing one text file apiece are unzipped and that content is preserved. 145 | ''' 146 | # Set up 147 | files = ['test1', 'test2'] 148 | inzip1 = zipfile.ZipFile('test_temp/test1_zip.zip', 'w', zipfile.ZIP_DEFLATED, True) 149 | inzip2 = zipfile.ZipFile('test_temp/test2_zip.zip', 'w', zipfile.ZIP_DEFLATED, True) 150 | for f in files: 151 | with open('test_data/%s_text.txt' % f, 'wb') as fi: 152 | fi.write('%s\n%s' % (f, f)) 153 | if f == 'test1': 154 | inzip1.write('test_data/%s_text.txt' % f) 155 | inzip1.close() 156 | elif f == 'test2': 157 | inzip2.write('test_data/%s_text.txt' % f) 158 | inzip2.close() 159 | 160 | # Test Zipping 161 | for zf in ['test1_zip.zip', 'test2_zip.zip']: 162 | self.assertTrue(os.path.isfile('test_temp/%s' % zf)) 163 | self.assertTrue(zipfile.is_zipfile('test_temp/%s' % zf)) 164 | 165 | # Test Unzipping 166 | testcat.unzipFiles() 167 | outzip1 = zipfile.ZipFile('test_temp/test1_zip.zip') 168 | outzip1.extractall('test_temp') 169 | outzip2 = zipfile.ZipFile('test_temp/test2_zip.zip') 170 | outzip2.extractall('test_temp') 171 | 172 | for f in files: 173 | self.assertTrue(os.path.isfile('test_temp/test_data/%s_text.txt' % f)) 174 | 175 | # Test Content 176 | for f in files: 177 | with open('test_temp/test_data/%s_text.txt' % f, 'rU') as fi: 178 | lines = fi.readlines() 179 | 180 | count = 0 181 | for line in lines: 182 | line = line.strip() 183 | self.assertEqual(line, f) 184 | count = count + 1 185 | self.assertEqual(count, 2) 186 | 187 | def tearDown(self): 188 | paths = ['./test_data', './test_temp', './test_out'] 189 | for path in paths: 190 | shutil.rmtree(path, ignore_errors = True) 191 | 192 | 193 | if __name__ == '__main__': 194 | unittest.main() 195 | -------------------------------------------------------------------------------- /gslab_misc/gencat/tests/test_writeDict.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | import shutil 4 | import zipfile 5 | import sys 6 | 7 | # Ensure theat Python can find and load gencaat.py 8 | os.chdir(os.path.dirname(os.path.realpath(__file__))) 9 | sys.path.append('..') 10 | 11 | from gencat import gencat 12 | 13 | class MockCat(gencat): 14 | def makeZipDict(self): 15 | pass 16 | def makeConcatDict(self): 17 | pass 18 | 19 | 20 | testcat = MockCat('./test_data', './test_temp', './test_out') 21 | 22 | 23 | class test_writeDict(unittest.TestCase): 24 | 25 | def setUp(self): 26 | paths = ['./test_data', './test_temp', './test_out'] 27 | for path in paths: 28 | try: 29 | os.makedirs(path) 30 | except: 31 | shutil.rmtree(path, ignore_errors = True) 32 | os.makedirs(path) 33 | 34 | def test_onePath(self): 35 | ''' 36 | Test that dictionary with one key paired to a single-element tuple value is printed as intended. 37 | ''' 38 | d = {'path': ('path/to/file.txt', )} 39 | testcat.writeDict(d, 'test_dict.txt', 'path/to/') 40 | 41 | self.assertTrue(os.path.isfile('./test_out/test_dict.txt')) 42 | with open('./test_out/test_dict.txt', 'rU') as f: 43 | lines = f.readline().strip() 44 | self.assertEqual(lines, 'path|file.txt') 45 | 46 | def test_twopPath(self): 47 | ''' 48 | Test that dictionary with two keys paired to single-element tuple values is printed as intended. 49 | ''' 50 | d = {'path1': ('path/to/file1.txt', ), 'path2': ('path/to/file2.txt', )} 51 | testcat.writeDict(d, 'test_dict.txt', 'path/to/') 52 | 53 | self.assertTrue(os.path.isfile('./test_out/test_dict.txt')) 54 | with open('./test_out/test_dict.txt', 'rU') as f: 55 | lines = [] 56 | for line in f: 57 | lines.append(line.strip()) 58 | self.assertEqual(lines[0], 'path1|file1.txt') 59 | self.assertEqual(lines[1], 'path2|file2.txt') 60 | 61 | def test_twoFile(self): 62 | ''' 63 | Test that dictionary with one key paired to two-element tuple value is printed as intended. 64 | ''' 65 | d = {'path': ('path/to/file1.txt', 'path/to/file2.txt')} 66 | testcat.writeDict(d, 'test_dict.txt', 'path/to/') 67 | 68 | self.assertTrue(os.path.isfile('./test_out/test_dict.txt')) 69 | with open('./test_out/test_dict.txt', 'rU') as f: 70 | lines = f.readline().strip() 71 | self.assertEqual(lines, 'path|file1.txt|file2.txt') 72 | 73 | def test_noPath(self): 74 | ''' 75 | Test that printed path is not modified when rel_path flag is blank. 76 | ''' 77 | d = {'path': ('a/path/file.txt', )} 78 | testcat.writeDict(d, 'test_dict.txt', '') 79 | 80 | with open('./test_out/test_dict.txt', 'rU') as f: 81 | lines = f.readline() 82 | lines = lines.strip() 83 | self.assertEqual(lines, 'path|a/path/file.txt') 84 | 85 | def tearDown(self): 86 | paths = ['./test_data', './test_temp', './test_out'] 87 | for path in paths: 88 | shutil.rmtree(path, ignore_errors = True) 89 | 90 | 91 | if __name__ == '__main__': 92 | unittest.main() -------------------------------------------------------------------------------- /gslab_misc/gencat/tests/test_zipFile.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import os 3 | import shutil 4 | import zipfile 5 | import sys 6 | 7 | # Ensure the script is run from its own directory 8 | os.chdir(os.path.dirname(os.path.realpath(__file__))) 9 | 10 | sys.path.append('../../') 11 | from gencat import gencat 12 | 13 | 14 | class MockCat(gencat): 15 | def makeZipDict(self): 16 | pass 17 | 18 | def makeConcatDict(self): 19 | pass 20 | 21 | class test_zipFiles(unittest.TestCase): 22 | 23 | def setUp(self): 24 | paths = ['./test_data', './test_temp', './test_out'] 25 | for path in paths: 26 | try: 27 | os.makedirs(path) 28 | except: 29 | shutil.rmtree(path, ignore_errors = True) 30 | os.makedirs(path) 31 | for FILE in ['./test_data/file1.txt', './test_data/file2.txt']: 32 | with open(FILE, 'wb') as f: 33 | f.write('''THIS IS A TEST FILE.\n''') 34 | 35 | def test_oneFile(self): 36 | ''' 37 | Test that contentation functions for a single file. 38 | ''' 39 | testcat = MockCat('./test_data', './test_temp', './test_out') 40 | testcat.zip_dict = {} 41 | testcat.zip_dict['zip1'] = ('concat1', ) 42 | testcat.concat_dict = {} 43 | testcat.concat_dict['concat1'] = ('./test_data/file1.txt', ) 44 | 45 | testcat.zipFiles() 46 | 47 | self.assertTrue(os.path.isfile('./test_out/zip1.zip')) 48 | self.assertTrue(zipfile.is_zipfile('./test_out/zip1.zip')) 49 | 50 | with zipfile.ZipFile('./test_out/zip1.zip', 'r') as zf: 51 | zf.extractall('./test_out/') 52 | 53 | with open('./test_out/zip1/concat1.txt', 'rU') as f: 54 | text = f.read() 55 | 56 | self.assertEqual(text, '\nNEWFILE\nFILENAME: file1.txt\n\nTHIS IS A TEST FILE.\n') 57 | 58 | def test_twoFile(self): 59 | ''' 60 | Test that two text files are concatenated into one without loss of content. 61 | ''' 62 | testcat = MockCat('./test_data', './test_temp', './test_out') 63 | testcat.zip_dict = {} 64 | testcat.zip_dict['zip1'] = ('concat1', ) 65 | testcat.concat_dict = {} 66 | testcat.concat_dict['concat1'] = ('./test_data/file1.txt', ) + ('./test_data/file2.txt', ) 67 | 68 | testcat.zipFiles() 69 | 70 | self.assertTrue(os.path.isfile('./test_out/zip1.zip')) 71 | self.assertTrue(zipfile.is_zipfile('./test_out/zip1.zip')) 72 | 73 | with zipfile.ZipFile('./test_out/zip1.zip', 'r') as zf: 74 | zf.extractall('./test_out/') 75 | 76 | with open('./test_out/zip1/concat1.txt', 'rU') as f: 77 | text = f.read() 78 | 79 | test_text = '\nNEWFILE\nFILENAME: file1.txt\n\nTHIS IS A TEST FILE.' + \ 80 | '\n\nNEWFILE\nFILENAME: file2.txt\n\nTHIS IS A TEST FILE.\n' 81 | self.assertEqual(text, test_text) 82 | 83 | def test_twoZips(self): 84 | ''' 85 | Test that two files can be concatenated to different text files and stored in separate zip files. 86 | ''' 87 | testcat = MockCat('./test_data', './test_temp', './test_out') 88 | testcat.zip_dict = {} 89 | testcat.zip_dict['zip1'] = ('concat1', ) 90 | testcat.zip_dict['zip2'] = ('concat2', ) 91 | testcat.concat_dict = {} 92 | testcat.concat_dict['concat1'] = ('./test_data/file1.txt', ) 93 | testcat.concat_dict['concat2'] = ('./test_data/file2.txt', ) 94 | 95 | testcat.zipFiles() 96 | 97 | self.assertTrue(os.path.isfile('./test_out/zip1.zip')) 98 | self.assertTrue(os.path.isfile('./test_out/zip2.zip')) 99 | self.assertTrue(zipfile.is_zipfile('./test_out/zip1.zip')) 100 | self.assertTrue(zipfile.is_zipfile('./test_out/zip2.zip')) 101 | 102 | with zipfile.ZipFile('./test_out/zip1.zip', 'r') as zf: 103 | zf.extractall('./test_out/') 104 | with zipfile.ZipFile('./test_out/zip2.zip', 'r') as zf: 105 | zf.extractall('./test_out/') 106 | 107 | with open('./test_out/zip1/concat1.txt', 'rU') as f: 108 | text1 = f.read() 109 | with open('./test_out/zip2/concat2.txt', 'rU') as f: 110 | text2 = f.read() 111 | 112 | self.assertEqual(text1, '\nNEWFILE\nFILENAME: file1.txt\n\nTHIS IS A TEST FILE.\n') 113 | self.assertEqual(text2, '\nNEWFILE\nFILENAME: file2.txt\n\nTHIS IS A TEST FILE.\n') 114 | 115 | def test_twoConcatsOneZip(self): 116 | ''' 117 | Test that two files can be concatenated to different text files and stored in the same zip file. 118 | ''' 119 | testcat = MockCat('./test_data', './test_temp', './test_out') 120 | testcat.zip_dict = {} 121 | testcat.zip_dict['zip1'] = ('concat1', ) + ('concat2', ) 122 | testcat.concat_dict = {} 123 | testcat.concat_dict['concat1'] = ('./test_data/file1.txt', ) 124 | testcat.concat_dict['concat2'] = ('./test_data/file2.txt', ) 125 | 126 | testcat.zipFiles() 127 | 128 | self.assertTrue(os.path.isfile('./test_out/zip1.zip')) 129 | self.assertTrue(zipfile.is_zipfile('./test_out/zip1.zip')) 130 | 131 | with zipfile.ZipFile('./test_out/zip1.zip', 'r') as zf: 132 | zf.extractall('./test_out/') 133 | 134 | with open('./test_out/zip1/concat1.txt', 'rU') as f: 135 | text1 = f.read() 136 | with open('./test_out/zip1/concat2.txt', 'rU') as f: 137 | text2 = f.read() 138 | 139 | self.assertEqual(text1, '\nNEWFILE\nFILENAME: file1.txt\n\nTHIS IS A TEST FILE.\n') 140 | self.assertEqual(text2, '\nNEWFILE\nFILENAME: file2.txt\n\nTHIS IS A TEST FILE.\n') 141 | 142 | def tearDown(self): 143 | paths = ['./test_data', './test_temp', './test_out'] 144 | for path in paths: 145 | shutil.rmtree(path, ignore_errors = True) 146 | 147 | 148 | if __name__ == '__main__': 149 | unittest.main() 150 | -------------------------------------------------------------------------------- /gslab_scons/README.md: -------------------------------------------------------------------------------- 1 | ## Notes on release.py 2 | 3 | Make a release from a SCons directory by running the module `release`. 4 | 5 | ### Basic use 6 | 7 | A user can either run the following from the command line within the directory of interest (e.g. `/paper_slides`) or write a Python script to call the module (see "Python wrapper" section below). If using the command line, run the following: 8 | 9 | ```sh 10 | python -m gslab_scons.release version= readme= 11 | ``` 12 | 13 | where `` is the name of the version that will be released, and `` is the path from the current directory to the repository readme file (default to `./README.md`). As an example, to release version v1.2.1 of a directory with the README file one level up, navigate to the root of the directory and run: 14 | 15 | ```sh 16 | python -m gslab_scons.release version=v1.2.1 readme=../README.md 17 | ``` 18 | 19 | An automatic location for release must be specified in `config_user.yaml` with 20 | 21 | ```yaml 22 | release_directory: 23 | ``` 24 | 25 | For example, to automatically release to Dropbox set 26 | 27 | ``` 28 | release_directory: /Users/you/Dropbox/release 29 | ``` 30 | 31 | ### Check stability 32 | 33 | Before releasing, the module checks that your repository is up-to-date by executing (in order) a 34 | * `git status` 35 | * SCons dry run 36 | 37 | By default, we run SCons from `run.py`, but you can specify another script through `scons_local_path=`. Set this argument to `None` or `False` if you want to run `scons` using your global installation. Executing a dry run may update log files packed into the SConstruct, which could affect the `git status` in repeated runs. 38 | 39 | ### Other options 40 | 41 | Including the option `no_zip` will prevent files from being zipped before they are released to the specified location. 42 | 43 | This release procedure will warn you when a versioned file is larger than 2MB and when the directory's versioned content is larger than 500MB in total. 44 | 45 | Instead of entering the GitHub token as a password when using `release`, you can store it in `config_user.yaml` as 46 | 47 | ```yaml 48 | github_token: 49 | ``` 50 | 51 | ### Python wrapper 52 | 53 | If you wish to wrap the `release` module in a Python script to help pass arguments, use the syntax 54 | 55 | ```python 56 | from gslab_scons import release 57 | release.main(version = '', 58 | readme = '../readme.md') 59 | ``` 60 | 61 | For a complete list of arguments that you can pass to `release.main()`, visit `release.py`. 62 | -------------------------------------------------------------------------------- /gslab_scons/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | gslab_scons - a SCons builder library 3 | ===================================== 4 | 5 | gslab_scons is a Python library containing general-purpose SCons builders 6 | for various software packages. Its builders work on both Unix and Windows 7 | platforms. 8 | 9 | Please consult the docstrings of the gslab_scons builders belonging to 10 | this module for additonal information on their functionalities. 11 | ''' 12 | import os 13 | import misc 14 | from .log import start_log, end_log 15 | from .log_paths_dict import log_paths_dict, record_dir 16 | from .scons_debrief import scons_debrief 17 | from .check_prereq import check_prereq 18 | from .builders.build_r import build_r 19 | from .builders.build_latex import build_latex 20 | from .builders.build_lyx import build_lyx 21 | from .builders.build_stata import build_stata 22 | from .builders.build_tables import build_tables 23 | from .builders.build_python import build_python 24 | from .builders.build_mathematica import build_mathematica 25 | from .builders.build_matlab import build_matlab 26 | from .builders.build_anything import build_anything 27 | -------------------------------------------------------------------------------- /gslab_scons/_exception_classes.py: -------------------------------------------------------------------------------- 1 | class ExecCallError(Exception): 2 | def __init__(self, message = ''): 3 | print 'Error: ' + message 4 | 5 | class BadExtensionError(Exception): 6 | pass 7 | 8 | class LFSError(Exception): 9 | pass 10 | 11 | class ReleaseError(Exception): 12 | pass 13 | 14 | class PrerequisiteError(Exception): 15 | pass 16 | 17 | class TargetNonexistenceError(Exception): 18 | pass -------------------------------------------------------------------------------- /gslab_scons/builders/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | This directory contains gslab_scons's builder functions. 3 | 4 | In general, the source code file (e.g. `.do` for Stata) must be listed as 5 | the first argument in source when calling builders from SConscript. 6 | ''' 7 | -------------------------------------------------------------------------------- /gslab_scons/builders/build_anything.py: -------------------------------------------------------------------------------- 1 | import os 2 | import copy 3 | import warnings 4 | 5 | from gslab_builder import GSLabBuilder 6 | import gslab_scons.misc as misc 7 | 8 | 9 | def build_anything(target, source, action, env, warning = True, **kw): 10 | ''' 11 | Anything builder-generator. The generator will create a custom builder 12 | that runs `action` and add it as a SCons node, similar to the native env.Command. 13 | Using gslab_scons.build_anything will utilize our logging mechanism 14 | and error catching similar to our other builders. 15 | ` 16 | Parameters 17 | target: string or list 18 | The target(s) of the SCons command. The log ends up in the 19 | directory of the first specified target. 20 | source: string or list 21 | The source(s) of the SCons command. 22 | action: string 23 | The code to be run by the generated builder. 24 | env: SCons construction environment, see SCons user guide 7.2. 25 | You *** MUST *** manually pass `env = env` when calling this in SConscript, 26 | since this is not a Scons.env method like env.Command. 27 | Special parameters that can be added when using the builder 28 | log_ext: string 29 | Instead of logging to `sconscript.log` in the target dir, 30 | the builder will log to `sconscript_.log`. 31 | origin_log_file: string 32 | Sometimes, your command may produce a log file in an undesirable location. 33 | Specifying the that location in this argument leads the builder to append 34 | the content of origin_log_file to log_file and delete origin_log_file. 35 | The builder will crash if this file doesn't exist at the end of the command. 36 | warning: Boolean 37 | Turns off warnings if warning = False. 38 | ''' 39 | import SCons.Builder 40 | builder_attributes = { 41 | 'name': 'Anything Builder' 42 | } 43 | target = [t for t in misc.make_list_if_string(target) if t] 44 | source = [s for s in misc.make_list_if_string(source) if s] 45 | local_env = env.Clone() 46 | for k, v in kw.items(): 47 | local_env[k] = v 48 | builder = AnythingBuilder(target, source, action, local_env, warning, **builder_attributes) 49 | bkw = { 50 | 'action': builder.build_anything, 51 | 'target_factory' : local_env.fs.Entry, 52 | 'source_factory': local_env.fs.Entry, 53 | } 54 | bld = SCons.Builder.Builder(**bkw) 55 | return bld(local_env, target, source) 56 | 57 | 58 | class AnythingBuilder(GSLabBuilder): 59 | ''' 60 | ''' 61 | def __init__(self, target, source, action, env, warning = True, name = ''): 62 | ''' 63 | ''' 64 | target = [self.to_str(t) for t in target] 65 | source = [self.to_str(s) for s in source] 66 | self.action = action 67 | super(AnythingBuilder, self).__init__(target, source, env, name = name) 68 | try: 69 | origin_log_file = env['origin_log_file'] 70 | except KeyError: 71 | origin_log_file = None 72 | self.origin_log_file = origin_log_file 73 | if '>' in action and warning == True: 74 | warning_message = '\nThere is a redirection operator > in ' \ 75 | 'your prescribed action key.\n' \ 76 | 'The Anything Builder\'s logging mechanism '\ 77 | 'may not work as intended.' 78 | warnings.warn(warning_message) 79 | 80 | 81 | @staticmethod 82 | def to_str(s): 83 | ''' 84 | Convert s to string and drop leading `#` if it exists. 85 | ''' 86 | s = str(s) 87 | if s and s[0] == '#': 88 | s = s[1:] 89 | return s 90 | 91 | 92 | def add_call_args(self): 93 | ''' 94 | ''' 95 | args = '%s > %s 2>&1' % (self.action, os.path.normpath(self.log_file)) 96 | self.call_args = args 97 | return None 98 | 99 | 100 | def build_anything(self, **kwargs): 101 | ''' 102 | Just a GSLabBuilder execute_system_call method, 103 | but given a nice name for printing. 104 | ''' 105 | super(AnythingBuilder, self).execute_system_call() 106 | if self.origin_log_file is not None: 107 | with open(log_file, 'ab') as sconscript_log: 108 | with open(origin_log_file, 'rU') as origin_log: 109 | sconscript_log.write(origin_log.read()) 110 | os.remove(origin_log_file) 111 | return None 112 | -------------------------------------------------------------------------------- /gslab_scons/builders/build_latex.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from gslab_builder import GSLabBuilder 4 | 5 | 6 | def build_latex(target, source, env): 7 | ''' 8 | Compile a pdf from a LaTeX file 9 | 10 | This function is a SCons builder that compiles a .tex file 11 | as a pdf and places it at the path specified by target. 12 | 13 | Parameters 14 | ---------- 15 | target: string or list 16 | The target of the SCons command. This should be the path 17 | of the pdf that the builder is instructed to compile. 18 | source: string or list 19 | The source of the SCons command. This should 20 | be the .tex file that the function will compile as a PDF. 21 | env: SCons construction environment, see SCons user guide 7.2 22 | ''' 23 | builder_attributes = { 24 | 'name': 'LaTeX', 25 | 'valid_extensions': ['.tex'], 26 | 'exec_opts': '-interaction nonstopmode -jobname' 27 | } 28 | builder = LatexBuilder(target, source, env, **builder_attributes) 29 | builder.execute_system_call() 30 | return None 31 | 32 | class LatexBuilder(GSLabBuilder): 33 | ''' 34 | ''' 35 | def add_call_args(self): 36 | ''' 37 | ''' 38 | target_name = os.path.splitext(self.target[0])[0] 39 | args = '%s %s %s > %s' % (self.cl_arg, target_name, os.path.normpath(self.source_file), os.path.normpath(self.log_file)) 40 | self.call_args = args 41 | return None 42 | -------------------------------------------------------------------------------- /gslab_scons/builders/build_lyx.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | 4 | from gslab_builder import GSLabBuilder 5 | 6 | 7 | def build_lyx(target, source, env): 8 | '''Compile a pdf from a LyX file 9 | 10 | This function is a SCons builder that compiles a .lyx file 11 | as a pdf and places it at the path specified by target. 12 | 13 | Parameters 14 | ---------- 15 | target: string or list 16 | The target of the SCons command. This should be the path 17 | of the pdf that the builder is instructed to compile. 18 | source: string or list 19 | The source of the SCons command. This should 20 | be the .lyx file that the function will compile as a PDF. 21 | env: SCons construction environment, see SCons user guide 7.2 22 | ''' 23 | builder_attributes = { 24 | 'name': 'LyX', 25 | 'valid_extensions': ['.lyx'], 26 | 'exec_opts': '-e pdf2' 27 | } 28 | builder = LyxBuilder(target, source, env, **builder_attributes) 29 | builder.execute_system_call() 30 | return None 31 | 32 | class LyxBuilder(GSLabBuilder): 33 | ''' 34 | ''' 35 | def add_call_args(self): 36 | ''' 37 | ''' 38 | args = '%s %s > %s' % (self.cl_arg, os.path.normpath(self.source_file), os.path.normpath(self.log_file)) 39 | self.call_args = args 40 | return None 41 | 42 | 43 | def do_call(self): 44 | ''' 45 | ''' 46 | super(LyxBuilder, self).do_call() 47 | new_pdf = os.path.splitext(self.source_file)[0] + '.pdf' 48 | new_pdf_path = os.path.normpath('%s/%s' % (self.target_dir, os.path.basename(new_pdf))) 49 | shutil.move(new_pdf, new_pdf_path) 50 | return None 51 | -------------------------------------------------------------------------------- /gslab_scons/builders/build_mathematica.py: -------------------------------------------------------------------------------- 1 | import os 2 | import subprocess 3 | 4 | from gslab_builder import GSLabBuilder 5 | 6 | 7 | def build_mathematica(target, source, env): 8 | ''' 9 | Build targets with a Mathematica command 10 | 11 | This function executes a Mathematica function to build objects 12 | specified by target using the objects specified by source. 13 | It requires Mathematica to be callable from the command line 14 | via `math` (or `MathKernel` for OS X). 15 | ''' 16 | builder_attributes = { 17 | 'name': 'Mathematica', 18 | 'valid_extensions': ['.m'], 19 | 'exec_opts': '-script' 20 | } 21 | builder = MathematicaBuilder(target, source, env, **builder_attributes) 22 | builder.execute_system_call() 23 | return None 24 | 25 | class MathematicaBuilder(GSLabBuilder): 26 | ''' 27 | ''' 28 | def add_call_args(self): 29 | ''' 30 | ''' 31 | args = '%s %s > %s' % (os.path.normpath(self.source_file), 32 | self.cl_arg, 33 | os.path.normpath(self.log_file)) 34 | self.call_args = args 35 | return None 36 | 37 | -------------------------------------------------------------------------------- /gslab_scons/builders/build_matlab.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import hashlib 4 | import sys 5 | 6 | import gslab_scons.misc as misc 7 | from gslab_builder import GSLabBuilder 8 | 9 | 10 | def build_matlab(target, source, env): 11 | ''' 12 | Build targets with a MATLAB command 13 | 14 | This function executes a MATLAB function to build objects 15 | specified by target using the objects specified by source. 16 | It requires MATLAB to be callable from the command line 17 | via `matlab`. 18 | 19 | Accessing command line arguments from within matlab is 20 | possible via the `command_line_arg = getenv('CL_ARG')`. 21 | ''' 22 | builder_attributes = { 23 | 'name': 'MATLAB', 24 | 'valid_extensions': ['.m'], 25 | } 26 | builder = MatlabBuilder(target, source, env, **builder_attributes) 27 | builder.execute_system_call() 28 | return None 29 | 30 | class MatlabBuilder(GSLabBuilder): 31 | ''' 32 | ''' 33 | def __init__(self, target, source, env, name = '', valid_extensions = []): 34 | ''' 35 | ''' 36 | exec_opts = self.add_executable_options() 37 | super(MatlabBuilder, self).__init__(target, source, env, name = name, 38 | exec_opts = exec_opts, 39 | valid_extensions = valid_extensions) 40 | 41 | 42 | def add_executable_options(self): 43 | ''' 44 | ''' 45 | if misc.is_unix(): 46 | platform_option = '-nodesktop ' 47 | elif sys.platform == 'win32': 48 | platform_option = '-minimize -wait ' 49 | else: 50 | message = 'Cannot find MATLAB command line syntax for platform.' 51 | raise PrerequisiteError(message) 52 | options = ' -nosplash %s -r' % platform_option 53 | return options 54 | 55 | 56 | def add_call_args(self): 57 | ''' 58 | ''' 59 | source_hash = hashlib.sha1(self.source_file).hexdigest() 60 | source_exec = 'source_%s' % source_hash 61 | exec_file = source_exec + '.m' 62 | shutil.copy(self.source_file, exec_file) 63 | args = '%s > %s' % (os.path.normpath(source_exec), os.path.normpath(self.log_file)) 64 | self.call_args = args 65 | self.exec_file = os.path.normpath(exec_file) 66 | return None 67 | 68 | 69 | def execute_system_call(self): 70 | ''' 71 | ''' 72 | os.environ['CL_ARG'] = self.cl_arg 73 | super(MatlabBuilder, self).execute_system_call() 74 | os.remove(self.exec_file) 75 | return None 76 | -------------------------------------------------------------------------------- /gslab_scons/builders/build_python.py: -------------------------------------------------------------------------------- 1 | import os 2 | 3 | from gslab_builder import GSLabBuilder 4 | 5 | def build_python(target, source, env): 6 | ''' 7 | Build SCons targets using a Python script 8 | 9 | This function executes a Python script to build objects specified 10 | by target using the objects specified by source. 11 | 12 | Parameters 13 | ---------- 14 | target: string or list 15 | The target(s) of the SCons command. 16 | source: string or list 17 | The source(s) of the SCons command. The first source specified 18 | should be the Python script that the builder is intended to execute. 19 | env: SCons construction environment, see SCons user guide 7.2 20 | ''' 21 | builder_attributes = { 22 | 'name': 'Python', 23 | 'valid_extensions': ['.py'], 24 | } 25 | builder = PythonBuilder(target, source, env, **builder_attributes) 26 | builder.execute_system_call() 27 | return None 28 | 29 | class PythonBuilder(GSLabBuilder): 30 | ''' 31 | ''' 32 | def add_call_args(self): 33 | ''' 34 | ''' 35 | args = '%s %s > %s' % (os.path.normpath(self.source_file), self.cl_arg, os.path.normpath(self.log_file)) 36 | self.call_args = args 37 | return None 38 | -------------------------------------------------------------------------------- /gslab_scons/builders/build_r.py: -------------------------------------------------------------------------------- 1 | import os 2 | from gslab_builder import GSLabBuilder 3 | 4 | 5 | def build_r(target, source, env): 6 | ''' 7 | Build SCons targets using an R script 8 | 9 | This function executes an R script to build objects specified 10 | by target using the objects specified by source. 11 | 12 | Parameters 13 | ---------- 14 | target: string or list 15 | The target(s) of the SCons command. 16 | source: string or list 17 | The source(s) of the SCons command. The first source specified 18 | should be the R script that the builder is intended to execute. 19 | env: SCons construction environment, see SCons user guide 7.2 20 | ''' 21 | builder_attributes = { 22 | 'name': 'R', 23 | 'valid_extensions': ['.r'], 24 | 'exec_opts': '--no-save --no-restore --verbose' 25 | } 26 | builder = RBuilder(target, source, env, **builder_attributes) 27 | builder.execute_system_call() 28 | return None 29 | 30 | class RBuilder(GSLabBuilder): 31 | ''' 32 | ''' 33 | def add_call_args(self): 34 | ''' 35 | ''' 36 | args = '%s %s > %s 2>&1' % (os.path.normpath(self.source_file), self.cl_arg, os.path.normpath(self.log_file)) 37 | self.call_args = args 38 | return None 39 | -------------------------------------------------------------------------------- /gslab_scons/builders/build_stata.py: -------------------------------------------------------------------------------- 1 | import os 2 | import shutil 3 | import sys 4 | 5 | import gslab_scons.misc as misc 6 | from gslab_builder import GSLabBuilder 7 | 8 | def build_stata(target, source, env): 9 | ''' 10 | Build targets with a Stata command 11 | 12 | This function executes a Stata script to build objects specified 13 | by target using the objects specified by source. 14 | 15 | Parameters 16 | ---------- 17 | target: string or list 18 | The target(s) of the SCons command. 19 | source: string or list 20 | The source(s) of the SCons command. The first source specified 21 | should be the Stata .do script that the builder is intended to execute. 22 | env: SCons construction environment, see SCons user guide 7.2 23 | ''' 24 | builder_attributes = { 25 | 'name': 'Stata', 26 | 'valid_extensions': ['.do'] 27 | } 28 | builder = StataBuilder(target, source, env, **builder_attributes) 29 | builder.execute_system_call() 30 | return None 31 | 32 | class StataBuilder(GSLabBuilder): 33 | ''' 34 | ''' 35 | def __init__(self, target, source, env, name = '', valid_extensions = []): 36 | ''' 37 | ''' 38 | exec_opts = self.add_executable_options() 39 | super(StataBuilder, self).__init__(target, source, env, name = name, 40 | exec_opts = exec_opts, 41 | valid_extensions = valid_extensions) 42 | 43 | 44 | def add_log_file(self): 45 | super(StataBuilder, self).add_log_file() 46 | self.final_sconscript_log = os.path.normpath(self.log_file) 47 | log_file = os.path.splitext(os.path.basename(self.source_file))[0] 48 | log_file = '%s.log' % log_file 49 | self.log_file = os.path.normpath(log_file) 50 | return None 51 | 52 | 53 | def add_executable_options(self): 54 | platform_options = { 55 | 'darwin': ' -e' , 56 | 'linux': ' -b' , 57 | 'linux2': ' -b' , 58 | 'win32': ' /e do ' 59 | } 60 | try: 61 | options = platform_options[sys.platform] 62 | except KeyError: 63 | message = 'Cannot find Stata command line syntax for platform %s.' % sys.platform 64 | raise PrerequisiteError(message) 65 | return options 66 | 67 | 68 | def add_call_args(self): 69 | ''' 70 | ''' 71 | args = '%s %s' % (os.path.normpath(self.source_file), self.cl_arg) 72 | self.call_args = args 73 | return None 74 | 75 | 76 | def execute_system_call(self): 77 | ''' 78 | ''' 79 | super(StataBuilder, self).execute_system_call() 80 | shutil.move(self.log_file, self.final_sconscript_log) 81 | return None 82 | 83 | -------------------------------------------------------------------------------- /gslab_scons/builders/build_tables.py: -------------------------------------------------------------------------------- 1 | import os 2 | from gslab_builder import GSLabBuilder 3 | 4 | from gslab_fill import tablefill 5 | 6 | 7 | def build_tables(target, source, env): 8 | '''Build a SCons target by filling a table 9 | 10 | This function uses the tablefill function from gslab_fill to produced a 11 | filled table from (i) an empty table in a LyX/Tex file and (ii) text files 12 | containing data to be used in filling the table. 13 | 14 | Parameters 15 | ---------- 16 | target: string or list 17 | The target(s) of the SCons command. 18 | source: string or list 19 | The source(s) of the SCons command. The first source specified 20 | should be the LyX/Tex file specifying the table format. The subsequent 21 | sources should be the text files containing the data with which the 22 | tables are to be filled. 23 | env: SCons construction environment, see SCons user guide 7.2 24 | ''' 25 | builder_attributes = { 26 | 'name': 'Tablefill', 27 | 'valid_extensions': ['.lyx', '.tex'], 28 | 'exec_opts': '-interaction nonstopmode -jobname' 29 | } 30 | builder = TableBuilder(target, source, env, **builder_attributes) 31 | builder.execute_system_call() 32 | return None 33 | 34 | class TableBuilder(GSLabBuilder): 35 | ''' 36 | ''' 37 | def __init__(self, target, source, env, name = '', valid_extensions = [], exec_opts = ''): 38 | ''' 39 | ''' 40 | super(TableBuilder, self).__init__(target, source, env, name = name, 41 | valid_extensions = valid_extensions, 42 | exec_opts = exec_opts) 43 | self.input_string = ' '.join([str(i) for i in source[1:]]) 44 | self.target_file = os.path.normpath(self.target[0]) 45 | 46 | 47 | def add_call_args(self): 48 | self.call_args = None 49 | return None 50 | 51 | def do_call(self): 52 | ''' 53 | ''' 54 | output = tablefill(input = self.input_string, 55 | template = os.path.normpath(self.source_file), 56 | output = os.path.normpath(self.target_file)) 57 | with open(self.log_file, 'wb') as f: 58 | f.write(output) 59 | f.write('\n\n') 60 | if 'traceback' in str.lower(output): # if tablefill.py returns an error 61 | command = 'tablefill(input = %s,\n' \ 62 | ' template = %s,\n' \ 63 | ' output = %s)' \ 64 | % (self.input_string, self.source_file, self.target_file) 65 | self.raise_system_call_exception(command = command) 66 | return None 67 | -------------------------------------------------------------------------------- /gslab_scons/check_prereq.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import subprocess 3 | import pkg_resources 4 | 5 | import misc 6 | from _exception_classes import PrerequisiteError 7 | 8 | 9 | def check_prereq(prereq, manual_execs = {}, gslab_vers = None): 10 | ''' 11 | Check if the prerequisites for prereq are satisfied. 12 | If prereq is a program, check that its executable is in the path. 13 | If prereq is (gslab_)python, check that it is the appropriate version. 14 | If prereq is git_lfs, check that it has been installed. 15 | ''' 16 | prereq_clean = str(prereq).lower().strip() 17 | path_checkers = ['r', 'stata', 'matlab', 'mathematica', 'lyx', 'latex'] 18 | if prereq_clean in path_checkers: 19 | executable = misc.get_executable(prereq_clean, manual_execs) 20 | if not misc.is_in_path(executable): 21 | message = 'Cannot find executable for %s in PATH.' % prereq_clean 22 | raise PrerequisiteError(message) 23 | elif prereq_clean == 'python': 24 | if sys.version_info[0] != 2: 25 | raise PrerequisiteError('Please use Python 2') 26 | elif prereq_clean == 'gslab_python': 27 | required_version = process_gslab_version(gslab_vers) 28 | installed_version = pkg_resources.get_distribution('gslab_tools').version 29 | installed_version = process_gslab_version(installed_version) 30 | if check_gslab_version(required_version, installed_version): 31 | message = 'Your version of gslab_python (%s) is outdated. ' \ 32 | 'This repository requires gslab_python (%s) or higher to run' \ 33 | % ('.'.join(str(v) for v in installed_version), '.'.join(str(v) for v in required_version)) 34 | raise PrerequisiteError(message) 35 | elif prereq_clean == 'git_lfs': 36 | check_git_lfs() 37 | else: 38 | message = 'Cannot find prerequisite check for %s' % prereq 39 | raise PrerequisiteError(message) 40 | return None 41 | 42 | 43 | def process_gslab_version(gslab_version): 44 | ''' 45 | Split semantically versioned gslab_version number at `.`. 46 | ''' 47 | try: 48 | vers = gslab_version.split('.') 49 | except: 50 | message = 'You must pass gslab_version as a string value.' 51 | raise PrerequisiteError(message) 52 | if len(vers) != 3: 53 | message = 'The gslab_version argument must have exactly two periods `.` ' \ 54 | 'to correspond to semantic versioning.' 55 | raise PrerequisiteError(message) 56 | try: 57 | vers = [int(val) for val in vers] 58 | except ValueError: 59 | message = 'All components of gslab_version between periods ' \ 60 | 'must be expressable as integers.' 61 | raise PrerequisiteError(message) 62 | return vers 63 | 64 | 65 | def check_gslab_version(required, installed): 66 | ''' 67 | Check (recursively) that installed gslab_python version meets or exceeds the required. 68 | ''' 69 | # Base case 70 | if required == installed: 71 | return False 72 | else: 73 | required_val = required[0] 74 | installed_val = installed[0] 75 | # More base cases 76 | if required_val > installed_val: 77 | out_dated = True 78 | else: 79 | out_dated = False 80 | # Recursive case 81 | if required_val == installed_val: 82 | out_dated = check_gslab_version(required[1:], installed[1:]) 83 | return out_dated 84 | 85 | 86 | def check_git_lfs(): 87 | ''' 88 | Check that a valid git-lfs is installed by trying to start it. 89 | ''' 90 | try: 91 | _ = subprocess.check_output('git lfs install', shell = True) 92 | except subprocess.CalledProcessError: 93 | try: 94 | _ = subprocess.check_output('git lfs init', shell = True) 95 | except subprocess.CalledProcessError: 96 | message = "Either Git LFS is not installed " \ 97 | "or your Git LFS settings need to be updated. " \ 98 | "Please install Git LFS or run " \ 99 | "'git lfs install --force' if prompted above." 100 | raise PrerequisiteError(message) 101 | return None 102 | -------------------------------------------------------------------------------- /gslab_scons/log.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import glob 4 | from datetime import datetime 5 | import subprocess 6 | import shutil 7 | import gslab_scons.misc as misc 8 | 9 | 10 | def start_log(mode, cl_args_list = sys.argv, log = 'sconstruct.log'): 11 | '''Begins logging a build process''' 12 | 13 | if not (mode in ['develop', 'cache']): 14 | raise Exception("Error: %s is not a defined mode" % mode) 15 | elif misc.is_scons_dry_run(cl_args_list = cl_args_list): 16 | return None 17 | 18 | start_message = "*** New build: {%s} ***\n" % misc.current_time() 19 | with open(log, "w") as f: 20 | f.write(start_message) 21 | 22 | if misc.is_unix(): 23 | sys.stdout = os.popen('tee -a %s' % log, 'wb') 24 | elif sys.platform == 'win32': 25 | sys.stdout = open(log, 'ab') 26 | 27 | sys.stderr = sys.stdout 28 | 29 | return None 30 | 31 | 32 | def end_log(cl_args_list = sys.argv, log = 'sconstruct.log', excluded_dirs = [], 33 | release_dir = './release/'): 34 | '''Complete the log of a build process.''' 35 | if misc.is_scons_dry_run(cl_args_list = cl_args_list): 36 | return None 37 | 38 | end_message = "*** Build completed: {%s} ***\n \n \n" % misc.current_time() 39 | with open(log, "a") as f: 40 | f.write(end_message) 41 | 42 | # scan sconstruct.log for start time 43 | with open(log, "rU") as f: 44 | s = f.readline() 45 | s = s[s.find('{') + 1: s.find('}')] 46 | start_time = datetime.strptime(s, "%Y-%m-%d %H:%M:%S") 47 | 48 | # gather all sconscript logs 49 | parent_dir = os.getcwd() 50 | builder_logs = collect_builder_logs(parent_dir, excluded_dirs = excluded_dirs) 51 | 52 | # keep only builder logs from this run OR is broken (value == beginning_of_time) 53 | beginning_of_time = datetime.min # to catch broken logs (see collect_builder_logs) 54 | this_run_dict = {key:value for key, value in builder_logs.items() if (value > start_time) or value == beginning_of_time} 55 | this_run_list = sorted(this_run_dict, key=this_run_dict.get, reverse=True) 56 | 57 | with open(log, "a") as sconstruct: 58 | for f in this_run_list: 59 | with open(f, 'rU') as sconscript: 60 | if this_run_dict[f] == beginning_of_time: 61 | warning_string = "*** Warning!!! The log below does not have timestamps," + \ 62 | " the Sconscript may not have finished.\n" 63 | sconstruct.write(warning_string) 64 | sconstruct.write(f + '\n') 65 | sconstruct.write(sconscript.read()) 66 | 67 | # move top level logs to /release/ directory. 68 | if not os.path.exists(release_dir): 69 | os.makedirs(release_dir) 70 | for file in glob.glob("*.log"): 71 | shutil.move('./' + file, release_dir + file) 72 | return None 73 | 74 | 75 | def collect_builder_logs(parent_dir, excluded_dirs = []): 76 | ''' Recursively return dictionary of files named sconscript*.log 77 | in parent_dir and nested directories. 78 | Also return timestamp from those sconscript.log 79 | (snippet from SO 3964681) 80 | 81 | excluded_dirs (str or list of str): 82 | list of directories to be excluded from the search 83 | ''' 84 | builder_log_collect = {} 85 | 86 | # Store paths to logs in a list, found from platform-specific command line tool 87 | rel_parent_dir = os.path.relpath(parent_dir) 88 | log_name = '*sconscript*.log' 89 | excluded_dirs = misc.make_list_if_string(excluded_dirs) 90 | 91 | log_paths = misc.finder(rel_parent_dir, log_name, excluded_dirs) 92 | 93 | # Read the file at each path to a log and store output complete-time in a dict at filename 94 | for log_path in log_paths: 95 | with open(log_path, 'rU') as f: 96 | try: 97 | s = f.readlines()[1] # line 0 = log start time, line 1 = log end time 98 | except IndexError: 99 | s = '' 100 | s = s[s.find('{') + 1: s.find('}')] # find {} time identifier 101 | try: 102 | builder_log_end_time = datetime.strptime(s, "%Y-%m-%d %H:%M:%S") 103 | except ValueError: # if the code breaks, there's no time identifier 104 | beginning_of_time = datetime.min 105 | builder_log_end_time = beginning_of_time 106 | builder_log_collect[log_path] = builder_log_end_time 107 | 108 | return builder_log_collect 109 | 110 | -------------------------------------------------------------------------------- /gslab_scons/log_paths_dict.py: -------------------------------------------------------------------------------- 1 | import os 2 | import sys 3 | import scandir 4 | import pymmh3 as mmh3 5 | 6 | import misc 7 | 8 | def log_paths_dict(d, record_key = 'input', nest_depth = 1, sep = ':', 9 | cl_args_list = sys.argv): 10 | ''' 11 | Records contents of dictionary d at record_key on nest_depth. 12 | Assumes unnested elements of d follow human-name: file-path. 13 | Values of d at record_key can be string or (nested) dict. 14 | ''' 15 | if misc.is_scons_dry_run(cl_args_list = cl_args_list): 16 | return None 17 | record_dict = misc.flatten_dict(d) 18 | record_dict = [(key, val) for key, val in sorted(record_dict.items()) 19 | if key.count(sep) >= nest_depth and val not in [None, 'None', '']] 20 | for name, path in record_dict: 21 | if record_key == name.split(sep)[nest_depth]: 22 | record_dir(path, name) 23 | return None 24 | 25 | def record_dir(inpath, name, 26 | include_checksum = False, 27 | file_limit = 5000, 28 | outpath = 'state_of_input.log'): 29 | ''' 30 | Record relative path, size, and (optionally) checksum of all files within inpath. 31 | Relative paths are from inpath. 32 | Append info in |-delimited format to outpath below a heading made from inpath. 33 | ''' 34 | inpath, name, this_file_only, do_walk = check_inpath(inpath, name) 35 | if do_walk: 36 | files_info = walk(inpath, include_checksum, file_limit, this_file_only) 37 | else: 38 | files_info = None 39 | check_outpath(outpath) 40 | write_log(name, files_info, outpath) 41 | return None 42 | 43 | def check_inpath(inpath, name): 44 | ''' 45 | Check that inpath exists as file or directory. 46 | If file, make inpath the file's directory and only record info for that file. 47 | ''' 48 | this_file_only = None 49 | do_walk = True 50 | if os.path.isfile(inpath): 51 | this_file_only = inpath 52 | inpath = os.path.dirname(inpath) 53 | elif os.path.isdir(inpath): 54 | pass 55 | else: 56 | name = name + ', could not find at runtime.' 57 | do_walk = False 58 | return inpath, name, this_file_only, do_walk 59 | 60 | def check_outpath(outpath): 61 | ''' 62 | Ensure that the directory for outpath exists. 63 | ''' 64 | dirname = os.path.dirname(outpath) 65 | if dirname and not os.path.isdir(dirname): 66 | os.makedirs(dirname) 67 | return None 68 | 69 | def walk(inpath, include_checksum, file_limit, this_file_only): 70 | ''' 71 | Walk through inpath and grab paths to all subdirs and info on all files. 72 | Walk in same order as os.walk. 73 | Keep walking until there are no more subdirs or there's info on file_limit files. 74 | ''' 75 | dirs = [inpath] 76 | files_info, file_limit = prep_files_info(include_checksum, file_limit) 77 | while dirs and do_more_files(files_info, file_limit): 78 | dirs, files_info = scan_dir_wrapper( 79 | dirs, files_info, inpath, include_checksum, file_limit, this_file_only) 80 | return files_info 81 | 82 | def prep_files_info(include_checksum, file_limit): 83 | ''' 84 | Create a header for the file characteristics to grab. 85 | Adjusts file_limit for existence of header. 86 | ''' 87 | files_info = [['file path', 'file size in bytes']] 88 | if include_checksum: 89 | files_info[0].append('MurmurHash3') 90 | file_limit += 1 91 | return files_info, file_limit 92 | 93 | def do_more_files(files_info, file_limit): 94 | ''' 95 | True if files_info has fewer then file_limit elements. 96 | ''' 97 | return bool(len(files_info) < file_limit) 98 | 99 | def scan_dir_wrapper(dirs, files_info, inpath, include_checksum, file_limit, 100 | this_file_only): 101 | ''' 102 | Drop down access and output management for scan_dir. 103 | Keep running the while loop in walk as directories are removed and added. 104 | ''' 105 | dir_to_scan = dirs.pop(0) 106 | subdirs, files_info = scan_dir( 107 | dir_to_scan, files_info, inpath, include_checksum, file_limit, this_file_only) 108 | dirs += subdirs 109 | return dirs, files_info 110 | 111 | def scan_dir(dir_to_scan, files_info, inpath, include_checksum, file_limit, 112 | this_file_only): 113 | ''' 114 | Collect names of all subdirs and all information on files. 115 | ''' 116 | subdirs = [] 117 | entries = scandir.scandir(dir_to_scan) 118 | for entry in entries: 119 | if entry.is_dir(follow_symlinks = False): 120 | if '.git' in entry.path or '.svn' in entry.path: 121 | continue 122 | else: 123 | subdirs.append(entry.path) 124 | elif entry.is_file() and (this_file_only is None or this_file_only == entry.path): 125 | f_info = get_file_information(entry, inpath, include_checksum) 126 | files_info.append(f_info) 127 | if not do_more_files(files_info, file_limit): 128 | break 129 | return subdirs, files_info 130 | 131 | def get_file_information(f, inpath, include_checksum): 132 | ''' 133 | Grabs path and size from scandir file object. 134 | Will compute file's checksum if asked. 135 | ''' 136 | f_path = os.path.relpath(f.path, inpath).strip() 137 | f_size = str(f.stat().st_size) 138 | f_info = [f_path, f_size] 139 | if include_checksum: 140 | with open(f.path, 'rU') as infile: 141 | f_checksum = str(mmh3.hash128(infile.read(), 2017)) 142 | f_info.append(f_checksum) 143 | return f_info 144 | 145 | def write_log(name, files_info, outpath): 146 | ''' 147 | Write file information to outpath under a nice header. 148 | ''' 149 | out_name = misc.make_heading(name) 150 | if files_info is not None: 151 | out_files_info = ['|'.join(l) for l in files_info] 152 | out_files_info = '\n'.join(out_files_info) 153 | else: 154 | out_files_info = '' 155 | with open(outpath, 'ab') as f: 156 | f.write(out_name) 157 | f.write(out_files_info) 158 | f.write('\n\n') 159 | return None 160 | 161 | -------------------------------------------------------------------------------- /gslab_scons/release.py: -------------------------------------------------------------------------------- 1 | import re 2 | import os 3 | import sys 4 | import _release_tools 5 | from _exception_classes import ReleaseError 6 | from misc import load_yaml_value, check_and_expand_path 7 | 8 | def main(version = None, 9 | user_yaml = 'config_user.yaml', 10 | release_files = [], 11 | dont_zip = False, 12 | readme = None, 13 | scons_local_path = 'run.py'): 14 | 15 | # Check if user specified a scons_local_path 16 | if scons_local_path == 'run.py': 17 | check_none = lambda scons_local_path, regex: bool(re.match(regex, scons_local_path, re.IGNORECASE)) 18 | try: 19 | scons_local_path = next(arg for arg in sys.argv if re.search('^scons_local_path=', arg)) 20 | scons_local_path = re.sub('^scons_local_path=', '', scons_local_path) 21 | if check_none(scons_local_path, 'None') or check_none(scons_local_path, 'False'): 22 | scons_local_path = None 23 | except: 24 | pass 25 | 26 | # Check if repository is up-to-date and ready for release. Stop if not. 27 | # Order matters because SCons check changes logs. 28 | if not _release_tools.git_up_to_date(): 29 | raise ReleaseError('Git working tree not clean.') 30 | elif not _release_tools.scons_up_to_date(scons_local_path): 31 | raise ReleaseError('SCons targets not up to date.') 32 | 33 | # Extract information about the clone from its .git directory 34 | try: 35 | repo, organisation, branch = _release_tools.extract_dot_git() 36 | except: 37 | try: 38 | repo, organisation, branch = _release_tools.extract_dot_git(path = '../.git') 39 | except: 40 | raise ReleaseError("Could not find .git/config in the current directory or parent directory.") 41 | 42 | # Determine the version number 43 | if version is None: 44 | try: 45 | version = next(arg for arg in sys.argv if re.search("^version=", arg)) 46 | except: 47 | raise ReleaseError('No version specified.') 48 | version = re.sub('^version=', '', version) 49 | 50 | # Determine whether the user has specified the no_zip option 51 | if dont_zip == False: 52 | dont_zip = 'no_zip' in sys.argv 53 | zip_release = not dont_zip 54 | 55 | # Read a list of files to release to release_dir 56 | if release_files == []: 57 | for root, _, files in os.walk('./release'): 58 | for file_name in files: 59 | # Do not release .DS_Store 60 | if not re.search("\.DS_Store", file_name): 61 | release_files.append(os.path.join(root, file_name)) 62 | 63 | # Specify the local release directory 64 | release_dir = load_yaml_value(user_yaml, 'release_directory') 65 | release_dir = check_and_expand_path(release_dir) 66 | 67 | if branch == 'master': 68 | name = repo 69 | branch = '' 70 | else: 71 | name = "%s-%s" % (repo, branch) 72 | local_release = '%s/%s/' % (release_dir, name) 73 | local_release = local_release + version + '/' 74 | 75 | # Get GitHub token: 76 | github_token = load_yaml_value(user_yaml, 'github_token') 77 | 78 | _release_tools.release(vers = version, 79 | DriveReleaseFiles = release_files, 80 | local_release = local_release, 81 | org = organisation, 82 | repo = repo, 83 | target_commitish = branch, 84 | zip_release = zip_release, 85 | github_token = github_token) 86 | 87 | if __name__ == '__main__': 88 | main() 89 | -------------------------------------------------------------------------------- /gslab_scons/tests/__init__.py: -------------------------------------------------------------------------------- 1 | ''' 2 | This directory contains the `gslab_scons` library's unit tests. These tests can be run using 3 | `python -m unittest discover` 4 | from `gslab_scons/` or from `gslab_scons/tests/`. To run the tests with logging, use: 5 | `python run_all_tests.py` 6 | from `gslab_scons/tests/`. 7 | ''' 8 | -------------------------------------------------------------------------------- /gslab_scons/tests/assets_listing.txt: -------------------------------------------------------------------------------- 1 | release: release/repo-test_branch/test_version/release_content.zip 2 | paper.pdf 3 | plot.pdf -------------------------------------------------------------------------------- /gslab_scons/tests/config_user.yaml: -------------------------------------------------------------------------------- 1 | stata_executable: -------------------------------------------------------------------------------- /gslab_scons/tests/input/lyx_test_dependencies.lyx: -------------------------------------------------------------------------------- 1 | #LyX 2.2 created this file. For more info see http://www.lyx.org/ 2 | \lyxformat 508 3 | \begin_document 4 | \begin_header 5 | \save_transient_properties true 6 | \origin unavailable 7 | \textclass article 8 | \use_default_options true 9 | \maintain_unincluded_children false 10 | \language english 11 | \language_package default 12 | \inputencoding auto 13 | \fontencoding global 14 | \font_roman "default" "default" 15 | \font_sans "default" "default" 16 | \font_typewriter "default" "default" 17 | \font_math "auto" "auto" 18 | \font_default_family default 19 | \use_non_tex_fonts false 20 | \font_sc false 21 | \font_osf false 22 | \font_sf_scale 100 100 23 | \font_tt_scale 100 100 24 | \graphics default 25 | \default_output_format default 26 | \output_sync 0 27 | \bibtex_command default 28 | \index_command default 29 | \paperfontsize default 30 | \use_hyperref false 31 | \papersize default 32 | \use_geometry false 33 | \use_package amsmath 1 34 | \use_package amssymb 1 35 | \use_package cancel 0 36 | \use_package esint 1 37 | \use_package mathdots 1 38 | \use_package mathtools 0 39 | \use_package mhchem 1 40 | \use_package stackrel 0 41 | \use_package stmaryrd 0 42 | \use_package undertilde 0 43 | \cite_engine basic 44 | \cite_engine_type default 45 | \biblio_style plain 46 | \use_bibtopic false 47 | \use_indices false 48 | \paperorientation portrait 49 | \suppress_date false 50 | \justification true 51 | \use_refstyle 1 52 | \index Index 53 | \shortcut idx 54 | \color #008000 55 | \end_index 56 | \secnumdepth 3 57 | \tocdepth 3 58 | \paragraph_separation indent 59 | \paragraph_indentation default 60 | \quotes_language english 61 | \papercolumns 1 62 | \papersides 1 63 | \paperpagestyle default 64 | \tracking_changes false 65 | \output_changes false 66 | \html_math_output 0 67 | \html_css_as_file 0 68 | \html_be_strict false 69 | \end_header 70 | 71 | \begin_body 72 | 73 | \begin_layout Standard 74 | \begin_inset CommandInset include 75 | LatexCommand include 76 | filename "lyx_test_file.lyx" 77 | 78 | \end_inset 79 | 80 | 81 | \end_layout 82 | 83 | \begin_layout Standard 84 | \begin_inset CommandInset include 85 | LatexCommand include 86 | filename "tables_appendix.txt" 87 | 88 | \end_inset 89 | 90 | 91 | \end_layout 92 | 93 | \end_body 94 | \end_document 95 | -------------------------------------------------------------------------------- /gslab_scons/tests/test_build_latex.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | import unittest 3 | import sys 4 | import os 5 | import shutil 6 | import mock 7 | import re 8 | # Import gslab_scons testing helper modules 9 | import _test_helpers as helpers 10 | import _side_effects as fx 11 | 12 | # Ensure that Python can find and load the GSLab libraries 13 | os.chdir(os.path.dirname(os.path.realpath(__file__))) 14 | sys.path.append('../..') 15 | 16 | import gslab_scons.builders.build_latex as gs 17 | from gslab_scons._exception_classes import BadExtensionError, ExecCallError 18 | 19 | # Define path to the builder for use in patching 20 | path = 'gslab_scons.builders.build_latex' 21 | 22 | 23 | class TestBuildLateX(unittest.TestCase): 24 | 25 | def setUp(self): 26 | if not os.path.exists('./build/'): 27 | os.mkdir('./build/') 28 | 29 | @mock.patch('%s.subprocess.check_output' % path) 30 | def test_default(self, mock_system): 31 | ''' 32 | Test that build_latex() behaves correctly when provided with 33 | standard inputs. 34 | ''' 35 | mock_system.side_effect = fx.latex_side_effect 36 | target = './build/latex.pdf' 37 | helpers.standard_test(self, gs.build_latex, 'tex', 38 | system_mock = mock_system, 39 | target = target) 40 | self.assertTrue(os.path.isfile(target)) 41 | 42 | @mock.patch('%s.subprocess.check_output' % path) 43 | def test_list_arguments(self, mock_system): 44 | ''' 45 | Check that build_latex() works when its source and target 46 | arguments are lists 47 | ''' 48 | mock_system.side_effect = fx.latex_side_effect 49 | target = ['./build/latex.pdf'] 50 | helpers.standard_test(self, gs.build_latex, 'tex', 51 | system_mock = mock_system, 52 | source = ['./test_script.tex'], 53 | target = target) 54 | self.assertTrue(os.path.isfile(target[0])) 55 | 56 | def test_bad_extension(self): 57 | '''Test that build_latex() recognises an improper file extension''' 58 | helpers.bad_extension(self, gs.build_latex, good = 'test.tex') 59 | 60 | @mock.patch('%s.os.system' % path) 61 | def test_env_argument(self, mock_system): 62 | ''' 63 | Test that numerous types of objects can be passed to 64 | build_latex() without affecting the function's operation. 65 | ''' 66 | mock_system.side_effect = fx.latex_side_effect 67 | target = './build/latex.pdf' 68 | source = ['./input/latex_test_file.tex'] 69 | log = './build/sconscript.log' 70 | 71 | for env in [True, [1, 2, 3], ('a', 'b'), None, TypeError]: 72 | with self.assertRaises(TypeError): 73 | gs.build_latex(target, source, env = env) 74 | 75 | @mock.patch('%s.os.system' % path) 76 | def test_nonexistent_source(self, mock_system): 77 | ''' 78 | Test build_latex()'s behaviour when the source file 79 | does not exist. 80 | ''' 81 | mock_system.side_effect = fx.latex_side_effect 82 | # i) Directory doesn't exist 83 | with self.assertRaises(ExecCallError): 84 | gs.build_latex('./build/latex.pdf', 85 | ['./bad_dir/latex_test_file.tex'], env = {}) 86 | # ii) Directory exists, but file doesn't 87 | with self.assertRaises(ExecCallError): 88 | gs.build_latex('./build/latex.pdf', 89 | ['./input/nonexistent_file.tex'], env = {}) 90 | 91 | @mock.patch('%s.os.system' % path) 92 | def test_nonexistent_target_directory(self, mock_system): 93 | ''' 94 | Test build_latex()'s behaviour when the target file's 95 | directory does not exist. 96 | ''' 97 | mock_system.side_effect = fx.latex_side_effect 98 | with self.assertRaises(TypeError): 99 | gs.build_latex('./nonexistent_directory/latex.pdf', 100 | ['./input/latex_test_file.tex'], env = True) 101 | 102 | def tearDown(self): 103 | if os.path.exists('./build/'): 104 | shutil.rmtree('./build/') 105 | 106 | 107 | if __name__ == '__main__': 108 | unittest.main() 109 | -------------------------------------------------------------------------------- /gslab_scons/tests/test_build_lyx.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | import unittest 3 | import sys 4 | import os 5 | import shutil 6 | import mock 7 | import re 8 | # Import gslab_scons testing helper modules 9 | import _test_helpers as helpers 10 | import _side_effects as fx 11 | 12 | # Ensure that Python can find and load the GSLab libraries 13 | os.chdir(os.path.dirname(os.path.realpath(__file__))) 14 | sys.path.append('../..') 15 | 16 | import gslab_scons.builders.build_lyx as gs 17 | from gslab_scons._exception_classes import BadExtensionError, ExecCallError 18 | 19 | # Define path to the builder for use in patching 20 | path = 'gslab_scons.builders.build_lyx' 21 | 22 | 23 | class TestBuildLyX(unittest.TestCase): 24 | 25 | def setUp(self): 26 | if not os.path.exists('./build/'): 27 | os.mkdir('./build/') 28 | 29 | @mock.patch('%s.subprocess.check_output' % path) 30 | def test_default(self, mock_system): 31 | ''' 32 | Test that build_lyx() behaves correctly when provided with 33 | standard inputs. 34 | ''' 35 | mock_system.side_effect = fx.lyx_side_effect 36 | target = './build/lyx.pdf' 37 | helpers.standard_test(self, gs.build_lyx, 'lyx', 38 | system_mock = mock_system, 39 | target = target) 40 | self.assertTrue(os.path.isfile(target)) 41 | 42 | @mock.patch('%s.subprocess.check_output' % path) 43 | def test_list_arguments(self, mock_system): 44 | ''' 45 | Check that build_lyx() works when its source and target 46 | arguments are lists 47 | ''' 48 | mock_system.side_effect = fx.lyx_side_effect 49 | target = ['./build/lyx.pdf'] 50 | helpers.standard_test(self, gs.build_lyx, 'lyx', 51 | system_mock = mock_system, 52 | source = ['./test_script.lyx'], 53 | target = target) 54 | self.assertTrue(os.path.isfile(target[0])) 55 | 56 | def test_bad_extension(self): 57 | '''Test that build_lyx() recognises an improper file extension''' 58 | helpers.bad_extension(self, gs.build_lyx, good = 'test.lyx') 59 | 60 | @mock.patch('%s.os.system' % path) 61 | def test_env_argument(self, mock_system): 62 | ''' 63 | Test that numerous types of objects can be passed to 64 | build_lyx() without affecting the function's operation. 65 | ''' 66 | mock_system.side_effect = fx.lyx_side_effect 67 | target = './build/lyx.pdf' 68 | source = ['./input/lyx_test_file.lyx'] 69 | log = './build/sconscript.log' 70 | 71 | for env in [True, [1, 2, 3], ('a', 'b'), None, TypeError]: 72 | with self.assertRaises(TypeError): 73 | gs.build_lyx(target, source, env = env) 74 | 75 | @mock.patch('%s.os.system' % path) 76 | def test_nonexistent_source(self, mock_system): 77 | ''' 78 | Test build_lyx()'s behaviour when the source file 79 | does not exist. 80 | ''' 81 | mock_system.side_effect = fx.lyx_side_effect 82 | # i) Directory doesn't exist 83 | with self.assertRaises(ExecCallError): 84 | gs.build_lyx('./build/lyx.pdf', 85 | ['./bad_dir/lyx_test_file.lyx'], env = {}) 86 | # ii) Directory exists, but file doesn't 87 | with self.assertRaises(ExecCallError): 88 | gs.build_lyx('./build/lyx.pdf', 89 | ['./input/nonexistent_file.lyx'], env = {}) 90 | 91 | @mock.patch('%s.os.system' % path) 92 | def test_nonexistent_target_directory(self, mock_system): 93 | ''' 94 | Test build_lyx()'s behaviour when the target file's 95 | directory does not exist. 96 | ''' 97 | mock_system.side_effect = fx.lyx_side_effect 98 | with self.assertRaises(TypeError): 99 | gs.build_lyx('./nonexistent_directory/lyx.pdf', 100 | ['./input/lyx_test_file.lyx'], env = True) 101 | 102 | def tearDown(self): 103 | if os.path.exists('./build/'): 104 | shutil.rmtree('./build/') 105 | 106 | 107 | if __name__ == '__main__': 108 | unittest.main() 109 | -------------------------------------------------------------------------------- /gslab_scons/tests/test_build_mathematica.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | import unittest 3 | import sys 4 | import os 5 | import shutil 6 | import mock 7 | import re 8 | # Import gslab_scons testing helper modules 9 | import _test_helpers as helpers 10 | import _side_effects as fx 11 | 12 | # Ensure that Python can find and load the GSLab libraries 13 | os.chdir(os.path.dirname(os.path.realpath(__file__))) 14 | sys.path.append('../..') 15 | 16 | import gslab_scons as gs 17 | from gslab_scons._exception_classes import (BadExtensionError, 18 | ExecCallError) 19 | from gslab_make.tests import nostderrout 20 | 21 | system_patch = mock.patch('gslab_scons.builders.build_mathematica.subprocess.check_output') 22 | 23 | 24 | class TestBuildMathematica(unittest.TestCase): 25 | 26 | def setUp(self): 27 | if not os.path.exists('./build/'): 28 | os.mkdir('./build/') 29 | 30 | @system_patch 31 | def test_standard(self, mock_check_output): 32 | '''Test build_mathematica()'s behaviour when given standard inputs.''' 33 | mock_check_output.side_effect = fx.make_mathematica_side_effect(True) 34 | helpers.standard_test(self, gs.build_mathematica, 'm', 35 | system_mock = mock_check_output) 36 | # With a list of targets 37 | targets = ['./test_output.txt'] 38 | helpers.standard_test(self, gs.build_mathematica, 'm', 39 | system_mock = mock_check_output, 40 | target = targets) 41 | 42 | @system_patch 43 | def test_cl_arg(self, mock_check_output): 44 | mock_check_output.side_effect = fx.make_mathematica_side_effect(True) 45 | helpers.test_cl_args(self, gs.build_mathematica, mock_check_output, 'm') 46 | 47 | def test_bad_extension(self): 48 | '''Test that build_mathematica() recognises an inappropriate file extension''' 49 | helpers.bad_extension(self, gs.build_mathematica, good = 'test.m') 50 | 51 | @system_patch 52 | def test_no_executable(self, mock_check_output): 53 | ''' 54 | Check build_mathematica()'s behaviour when math (or MathKernel for OS X) 55 | is not recognised as an executable. 56 | ''' 57 | mock_check_output.side_effect = \ 58 | fx.make_mathematica_side_effect(recognized = False) 59 | with self.assertRaises(ExecCallError): 60 | helpers.standard_test(self, gs.build_mathematica, 'm', 61 | system_mock = mock_check_output) 62 | 63 | @system_patch 64 | def test_unintended_inputs(self, mock_check_output): 65 | # We expect build_mathematica() to raise an error if its env 66 | # argument does not support indexing by strings. 67 | mock_check_output.side_effect = fx.make_mathematica_side_effect(True) 68 | 69 | check = lambda **kwargs: helpers.input_check(self, gs.build_mathematica, 70 | 'm', **kwargs) 71 | 72 | for bad_env in [True, (1, 2), TypeError]: 73 | check(env = bad_env, error = TypeError) 74 | 75 | def tearDown(self): 76 | if os.path.exists('./build/'): 77 | shutil.rmtree('./build/') 78 | if os.path.isfile('./test_output.txt'): 79 | os.remove('./test_output.txt') 80 | 81 | if __name__ == '__main__': 82 | unittest.main() 83 | 84 | -------------------------------------------------------------------------------- /gslab_scons/tests/test_build_matlab.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import sys 3 | import os 4 | import shutil 5 | import mock 6 | import re 7 | # Import gslab_scons testing helper modules 8 | import _test_helpers as helpers 9 | import _side_effects as fx 10 | 11 | sys.path.append('../..') 12 | import gslab_scons as gs 13 | from gslab_scons._exception_classes import (BadExtensionError, 14 | ExecCallError, 15 | PrerequisiteError) 16 | 17 | # Define main test patch 18 | path = 'gslab_scons.builders.build_matlab' 19 | check_output_patch = mock.patch('%s.subprocess.check_output' % path) 20 | copy_patch = mock.patch('%s.shutil.copy' % path) 21 | main_patch = lambda f: check_output_patch(copy_patch(f)) 22 | 23 | 24 | class TestBuildMatlab(unittest.TestCase): 25 | 26 | def setUp(self): 27 | if not os.path.exists('./build/'): 28 | os.mkdir('./build/') 29 | 30 | @helpers.platform_patch('darwin', path) 31 | @main_patch 32 | def test_unix(self, mock_copy, mock_check_output): 33 | ''' 34 | Test that build_matlab() creates a log and properly submits 35 | a matlab system command on a Unix machine. 36 | ''' 37 | # Mock copy so that it just creates the destination file 38 | mock_copy.side_effect = fx.matlab_copy_effect 39 | mock_check_output.side_effect = fx.make_matlab_side_effect(True) 40 | 41 | helpers.standard_test(self, gs.build_matlab, 'm') 42 | self.check_call(mock_check_output, ['-nosplash', '-nodesktop']) 43 | 44 | def check_call(self, mock_check_output, options): 45 | ''' 46 | Check that build_matlab() called Matlab correctly. 47 | mock_system should be the mock of os.system in build_matlab(). 48 | ''' 49 | # Extract the system command 50 | command = mock_check_output.call_args[0][0] 51 | # Look for the expected executable and options 52 | self.assertTrue(re.search('^matlab', command)) 53 | for option in options: 54 | self.assertIn(option, command.split(' ')) 55 | 56 | @helpers.platform_patch('win32', path) 57 | @main_patch 58 | def test_windows(self, mock_copy, mock_check_output): 59 | ''' 60 | Test that build_matlab() creates a log and properly submits 61 | a matlab system command on a Windows machine. 62 | ''' 63 | mock_copy.side_effect = fx.matlab_copy_effect 64 | mock_check_output.side_effect = fx.make_matlab_side_effect(True) 65 | 66 | helpers.standard_test(self, gs.build_matlab, 'm') 67 | self.check_call(mock_check_output, ['-nosplash', '-minimize', '-wait']) 68 | 69 | @helpers.platform_patch('riscos', path) 70 | @main_patch 71 | def test_other_os(self, mock_copy, mock_check_output): 72 | ''' 73 | Test that build_matlab() raises an exception when run on a 74 | non-Unix, non-Windows operating system. 75 | ''' 76 | mock_copy.side_effect = fx.matlab_copy_effect 77 | mock_check_output.side_effect = fx.make_matlab_side_effect(True) 78 | with self.assertRaises(PrerequisiteError): 79 | gs.build_matlab(target = './build/test.mat', 80 | source = './input/matlab_test_script.m', 81 | env = {}) 82 | 83 | @main_patch 84 | def test_clarg(self, mock_copy, mock_check_output): 85 | ''' 86 | Test that build_matlab() properly sets command-line arguments 87 | in its env argument as system environment variables. 88 | ''' 89 | mock_copy.side_effect = fx.matlab_copy_effect 90 | mock_check_output.side_effect = fx.make_matlab_side_effect(True) 91 | 92 | env = {'CL_ARG': 'COMMANDLINE'} 93 | helpers.standard_test(self, gs.build_matlab, 'm', 94 | system_mock = mock_check_output, env = env) 95 | self.assertEqual(os.environ['CL_ARG'], env['CL_ARG']) 96 | 97 | def test_bad_extension(self): 98 | '''Test that build_matlab() recognises an improper file extension''' 99 | helpers.bad_extension(self, gs.build_matlab, good = 'test.m') 100 | 101 | @main_patch 102 | def test_no_executable(self, mock_copy, mock_check_output): 103 | mock_copy.side_effect = fx.matlab_copy_effect 104 | mock_check_output.side_effect = \ 105 | fx.make_matlab_side_effect(recognized = False) 106 | 107 | with self.assertRaises(ExecCallError): 108 | gs.build_matlab(target = './build/test.mat', 109 | source = './input/matlab_test_script.m', 110 | env = {}) 111 | def tearDown(self): 112 | if os.path.exists('./build/'): 113 | shutil.rmtree('./build/') 114 | if os.path.isfile('./test_output.txt'): 115 | os.remove('./test_output.txt') 116 | 117 | if __name__ == '__main__': 118 | unittest.main() 119 | -------------------------------------------------------------------------------- /gslab_scons/tests/test_build_python.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import sys 3 | import os 4 | import shutil 5 | import mock 6 | import subprocess 7 | import re 8 | # Import gslab_scons testing helper modules 9 | import _test_helpers as helpers 10 | import _side_effects as fx 11 | 12 | sys.path.append('../..') 13 | import gslab_scons.builders.build_python as gs 14 | from gslab_scons._exception_classes import BadExtensionError, ExecCallError 15 | from gslab_make.tests import nostderrout 16 | 17 | # Define path to the builder for use in patching 18 | path = 'gslab_scons.builders.build_python' 19 | 20 | class TestBuildPython(unittest.TestCase): 21 | 22 | def setUp(self): 23 | if not os.path.exists('./build/'): 24 | os.mkdir('./build/') 25 | 26 | @mock.patch('%s.subprocess.check_output' % path) 27 | def test_log_creation(self, mock_check_output): 28 | '''Test build_python()'s behaviour when given standard inputs.''' 29 | mock_check_output.side_effect = fx.python_side_effect 30 | helpers.standard_test(self, gs.build_python, 'py', 31 | system_mock = mock_check_output) 32 | 33 | def test_bad_extension(self): 34 | '''Test that build_python() recognises an improper file extension''' 35 | helpers.bad_extension(self, gs.build_python, good = 'test.py') 36 | 37 | @mock.patch('%s.subprocess.check_output' % path) 38 | def test_cl_arg(self, mock_check_output): 39 | mock_check_output.side_effect = fx.python_side_effect 40 | helpers.test_cl_args(self, gs.build_python, mock_check_output, 'py') 41 | 42 | @mock.patch('%s.subprocess.check_output' % path) 43 | def test_unintended_inputs(self, mock_check_output): 44 | ''' 45 | Test that build_python() handles unintended inputs 46 | as expected. 47 | ''' 48 | mock_check_output.side_effect = fx.python_side_effect 49 | 50 | check = lambda **kwargs: helpers.input_check(self, gs.build_python, 51 | 'py', **kwargs) 52 | 53 | # env's class must support indexing by strings 54 | check(env = None, error = TypeError) 55 | check(env = 'env', error = TypeError) 56 | 57 | test_source = ['./test_script.py', 'nonexistent_data.txt'] 58 | check(source = test_source, error = None) 59 | test_source.reverse() 60 | check(source = test_source, error = BadExtensionError) 61 | 62 | def test_nonexistent_input(self): 63 | ''' 64 | Test build_python()'s behaviour when the source script doesn't exist. 65 | ''' 66 | if os.path.exists('test.py'): 67 | os.remove('test.py') 68 | 69 | with self.assertRaises(ExecCallError): 70 | helpers.standard_test(self, gs.build_python, source = 'test.py') 71 | 72 | def tearDown(self): 73 | if os.path.exists('./build/'): 74 | shutil.rmtree('./build/') 75 | if os.path.isfile('./test_output.txt'): 76 | os.remove('./test_output.txt') 77 | 78 | 79 | if __name__ == '__main__': 80 | unittest.main() 81 | -------------------------------------------------------------------------------- /gslab_scons/tests/test_build_r.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | import unittest 3 | import sys 4 | import os 5 | import shutil 6 | import mock 7 | import re 8 | # Import gslab_scons testing helper modules 9 | import _test_helpers as helpers 10 | import _side_effects as fx 11 | 12 | # Ensure that Python can find and load the GSLab libraries 13 | os.chdir(os.path.dirname(os.path.realpath(__file__))) 14 | sys.path.append('../..') 15 | 16 | import gslab_scons as gs 17 | from gslab_scons._exception_classes import (BadExtensionError, 18 | ExecCallError) 19 | from gslab_make.tests import nostderrout 20 | 21 | system_patch = mock.patch('gslab_scons.builders.build_r.subprocess.check_output') 22 | 23 | 24 | class TestBuildR(unittest.TestCase): 25 | 26 | def setUp(self): 27 | if not os.path.exists('./build/'): 28 | os.mkdir('./build/') 29 | 30 | @system_patch 31 | def test_standard(self, mock_check_output): 32 | '''Test build_r()'s behaviour when given standard inputs.''' 33 | mock_check_output.side_effect = fx.make_r_side_effect(True) 34 | helpers.standard_test(self, gs.build_r, 'R', 35 | system_mock = mock_check_output) 36 | # With a list of targets 37 | targets = ['./test_output.txt'] 38 | helpers.standard_test(self, gs.build_r, 'R', 39 | system_mock = mock_check_output, 40 | target = targets) 41 | 42 | @system_patch 43 | def test_cl_arg(self, mock_check_output): 44 | mock_check_output.side_effect = fx.make_r_side_effect(True) 45 | helpers.test_cl_args(self, gs.build_r, mock_check_output, 'R') 46 | 47 | def test_bad_extension(self): 48 | '''Test that build_r() recognises an inappropriate file extension''' 49 | helpers.bad_extension(self, gs.build_r, good = 'test.r') 50 | 51 | @system_patch 52 | def test_no_executable(self, mock_check_output): 53 | ''' 54 | Check build_r()'s behaviour when R is not recognised as 55 | an executable. 56 | ''' 57 | mock_check_output.side_effect = \ 58 | fx.make_r_side_effect(recognized = False) 59 | with self.assertRaises(ExecCallError): 60 | helpers.standard_test(self, gs.build_r, 'R', 61 | system_mock = mock_check_output) 62 | 63 | @system_patch 64 | def test_unintended_inputs(self, mock_check_output): 65 | # We expect build_r() to raise an error if its env 66 | # argument does not support indexing by strings. 67 | mock_check_output.side_effect = fx.make_r_side_effect(True) 68 | 69 | check = lambda **kwargs: helpers.input_check(self, gs.build_r, 70 | 'r', **kwargs) 71 | 72 | for bad_env in [True, (1, 2), TypeError]: 73 | check(env = bad_env, error = TypeError) 74 | 75 | def tearDown(self): 76 | if os.path.exists('./build/'): 77 | shutil.rmtree('./build/') 78 | if os.path.isfile('./test_output.txt'): 79 | os.remove('./test_output.txt') 80 | 81 | if __name__ == '__main__': 82 | unittest.main() 83 | -------------------------------------------------------------------------------- /gslab_scons/tests/test_release_tools.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | import sys 3 | import os 4 | import re 5 | import mock 6 | import tempfile 7 | import shutil 8 | # Import module containing gslab_scons testing side effects 9 | import _side_effects as fx 10 | 11 | # Ensure that Python can find and load the GSLab libraries 12 | os.chdir(os.path.dirname(os.path.realpath(__file__))) 13 | sys.path.append('../..') 14 | 15 | import gslab_scons 16 | import gslab_scons._release_tools as tools 17 | from gslab_scons._exception_classes import ReleaseError 18 | from gslab_make.tests import nostderrout 19 | 20 | 21 | class TestReleaseTools(unittest.TestCase): 22 | 23 | @mock.patch('gslab_scons._release_tools.requests.session') 24 | @mock.patch('gslab_scons._release_tools.open') 25 | @mock.patch('gslab_scons._release_tools.os.path.isfile') 26 | def test_upload_asset_standard(self, mock_isfile, mock_open, mock_session): 27 | ''' 28 | Test that upload_asset() correctly prepares a request 29 | to upload a release asset to GitHub. 30 | ''' 31 | # Allow upload_asset() to work without an actual release asset file 32 | mock_isfile.return_value = True 33 | mock_open.return_value = 'file_object' 34 | 35 | # There are three connected requests-related mocks at play here: 36 | # i) mock_session: the requests.session() function 37 | # ii) the session object returned by requests.session 38 | # iii) the mocked post() method of the mocked session object 39 | mock_session.return_value = mock.MagicMock(post = mock.MagicMock()) 40 | 41 | tools.upload_asset(github_token = 'test_token', 42 | org = 'gslab-econ', 43 | repo = 'gslab_python', 44 | release_id = 'test_release', 45 | file_name = 'release.txt', 46 | content_type = 'text/markdown') 47 | 48 | # Check that upload_asset called a session object's post() method 49 | # once and with the correct arguments. 50 | mock_session.return_value.post.assert_called_once() 51 | 52 | keyword_args = mock_session.return_value.post.call_args[1] 53 | positional_args = mock_session.return_value.post.call_args[0] 54 | 55 | self.assertEqual(keyword_args['files']['file'], 'file_object') 56 | self.assertEqual(keyword_args['headers']['Authorization'], 'token test_token') 57 | self.assertEqual(keyword_args['headers']['Content-Type'], 'text/markdown') 58 | 59 | # Check that the first positional argument matches the desired upload path 60 | desired_upload_path = ''.join(['https://uploads.github.com/repos/', 61 | 'gslab-econ/gslab_python/releases/', 62 | 'test_release/assets?name=release.txt']) 63 | self.assertEqual(positional_args[0], desired_upload_path) 64 | 65 | @mock.patch('gslab_scons._release_tools.requests.session') 66 | def test_upload_asset_bad_file(self, mock_session): 67 | ''' 68 | Test that upload_asset() raises an error when its file_name 69 | argument isn't valid. 70 | ''' 71 | mock_session.return_value = mock.MagicMock(post = mock.MagicMock()) 72 | 73 | with self.assertRaises(ReleaseError), nostderrout(): 74 | tools.upload_asset(github_token = 'test_token', 75 | org = 'gslab-econ', 76 | repo = 'gslab_python', 77 | release_id = 'test_release', 78 | file_name = 'nonexistent_file', 79 | content_type = 'text/markdown') 80 | 81 | @mock.patch('gslab_scons._release_tools.subprocess.call') 82 | def test_up_to_date(self, mock_call): 83 | ''' 84 | Test that up_to_date() correctly recognises 85 | an SCons directory as up-to-date or out of date. 86 | ''' 87 | # The mode argument needs to be one of the valid options 88 | with self.assertRaises(ReleaseError), nostderrout(): 89 | gslab_scons._release_tools.up_to_date(mode = 'invalid') 90 | 91 | # The mock of subprocess call should write pre-specified text 92 | # to stdout. This mock prevents us from having to set up real 93 | # SCons and git directories. 94 | mock_call.side_effect = \ 95 | fx.make_call_side_effect('Your branch is up-to-date') 96 | self.assertTrue(gslab_scons._release_tools.up_to_date(mode = 'git')) 97 | 98 | mock_call.side_effect = \ 99 | fx.make_call_side_effect('modified: .sconsign.dblite') 100 | self.assertFalse(gslab_scons._release_tools.up_to_date(mode = 'git')) 101 | 102 | mock_call.side_effect = \ 103 | fx.make_call_side_effect("scons: `.' is up to date.") 104 | self.assertTrue(gslab_scons._release_tools.up_to_date(mode = 'scons')) 105 | 106 | mock_call.side_effect = \ 107 | fx.make_call_side_effect('python some_script.py') 108 | self.assertFalse(gslab_scons._release_tools.up_to_date(mode = 'scons')) 109 | 110 | # The up_to_date() function shouldn't work in SCons or git mode 111 | # when it is called outside of a SCons directory or a git 112 | # repository, respectively. 113 | mock_call.side_effect = \ 114 | fx.make_call_side_effect("Not a git repository") 115 | with self.assertRaises(ReleaseError), nostderrout(): 116 | gslab_scons._release_tools.up_to_date(mode = 'git') 117 | 118 | mock_call.side_effect = \ 119 | fx.make_call_side_effect("No SConstruct file found") 120 | with self.assertRaises(ReleaseError), nostderrout(): 121 | gslab_scons._release_tools.up_to_date(mode = 'scons') 122 | 123 | 124 | @mock.patch('gslab_scons._release_tools.open') 125 | def test_extract_dot_git(self, mock_open): 126 | ''' 127 | Test that extract_dot_git() correctly extracts repository 128 | information from a .git folder's config file. 129 | ''' 130 | 131 | mock_open.side_effect = fx.dot_git_open_side_effect() 132 | 133 | repo_info = tools.extract_dot_git('.git') 134 | self.assertEqual(repo_info[0], 'repo') 135 | self.assertEqual(repo_info[1], 'org') 136 | self.assertEqual(repo_info[2], 'branch') 137 | 138 | # Ensure that extract_dot_git() raises an error when the directory 139 | # argument is not a .git folder. 140 | # i) The directory argument identifies an empty folder 141 | with self.assertRaises(ReleaseError): 142 | repo_info = tools.extract_dot_git('not/git') 143 | 144 | # ii) Mock the .git/config file so that url information is missing 145 | # from its "[remote "origin"]" section. (We parse organisaton, 146 | # repo, and branch information from this url.) 147 | mock_open.side_effect = fx.dot_git_open_side_effect(url = False) 148 | with self.assertRaises(ReleaseError): 149 | repo_info = tools.extract_dot_git('.git') 150 | 151 | 152 | if __name__ == '__main__': 153 | unittest.main() 154 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | description-file = README.md 3 | 4 | [aliases] 5 | test1=pytest 6 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | import os 2 | import re 3 | import sys 4 | import shutil 5 | import site 6 | from setuptools import setup, find_packages 7 | from setuptools.command.build_py import build_py 8 | from setuptools.command.install import install 9 | from glob import glob 10 | 11 | # Determine if the user has specified which paths to report coverage for 12 | is_include_arg = map(lambda x: bool(re.search('^--include=', x)), 13 | sys.argv) 14 | 15 | if True in is_include_arg: 16 | include_arg = sys.argv[is_include_arg.index(True)] 17 | include_arg = sys.argv[is_include_arg.index(True)] 18 | del sys.argv[is_include_arg.index(True)] 19 | else: 20 | include_arg = None 21 | 22 | # Additional build commands 23 | class TestRepo(build_py): 24 | '''Build command for running tests in repo''' 25 | def run(self): 26 | if include_arg: 27 | coverage_command = 'coverage report -m %s' % include_arg 28 | else: 29 | coverage_command = 'coverage report -m --omit=setup.py,*/__init__.py,.eggs/*' 30 | 31 | 32 | if sys.platform != 'win32': 33 | os.system("coverage run --branch --source ./ setup.py test1 2>&1 " 34 | "| tee test.log") 35 | # http://unix.stackexchange.com/questions/80707/ 36 | # how-to-output-text-to-both-screen-and-file-inside-a-shell-script 37 | os.system("%s 2>&1 | tee -a test.log" % coverage_command) 38 | else: 39 | os.system("coverage run --branch --source ./ setup.py " 40 | "> test.log") 41 | os.system("%s >> test.log" % coverage_command) 42 | 43 | sys.exit() 44 | 45 | 46 | class CleanRepo(build_py): 47 | '''Build command for clearing setup directories after installation''' 48 | def run(self): 49 | # i) Remove the .egg-info or .dist-info folders 50 | egg_directories = glob('./*.egg-info') 51 | map(shutil.rmtree, egg_directories) 52 | dist_directories = glob('./*.dist-info') 53 | map(shutil.rmtree, dist_directories) 54 | # ii) Remove the ./build and ./dist directories 55 | if os.path.isdir('./build'): 56 | shutil.rmtree('./build') 57 | if os.path.isdir('./dist'): 58 | shutil.rmtree('./dist') 59 | 60 | # Requirements 61 | requirements = ['requests', 'scandir', 'pymmh3', 'pandas'] 62 | 63 | setup(name = 'GSLab_Tools', 64 | version = '4.1.2', 65 | description = 'Python tools for GSLab', 66 | url = 'https://github.com/gslab-econ/gslab_python', 67 | author = 'Matthew Gentzkow, Jesse Shapiro', 68 | author_email = 'gentzkow@stanford.edu, jesse_shapiro_1@brown.edu', 69 | license = 'MIT', 70 | packages = find_packages(), 71 | install_requires = requirements, 72 | zip_safe = False, 73 | cmdclass = {'test': TestRepo, 'clean': CleanRepo}, 74 | setup_requires = ['pytest-runner', 'coverage'], 75 | tests_require = ['pytest', 'coverage']) 76 | 77 | --------------------------------------------------------------------------------