├── .Rbuildignore ├── .github ├── ISSUE_TEMPLATE │ ├── discussion-issue-template.md │ └── proposal-issue-template.md └── workflows │ └── deploy_bookdown.yml ├── .gitignore ├── .words ├── CITATION.cff ├── CODE_OF_CONDUCT.md ├── CONTRIBUTING.md ├── DESCRIPTION ├── LICENSE-MIT.md ├── LICENSE.md ├── Makefile ├── README.md ├── _bookdown.yml ├── _common.R ├── _output.yml ├── appendix.Rmd ├── author-flyer-2021-07.pdf ├── bin ├── captions.py ├── chapters.py ├── chunks.py ├── citations.py ├── crossrefs.py ├── dedication.R ├── exercises.py ├── glossarize.py ├── glossary-merge.py ├── glosscheck.py ├── images.py ├── line-length.py ├── linkcheck.py ├── list-keypoints-objectives.R ├── nbspref.py ├── right-trim.py ├── rstrip.py ├── test_chunks.py └── util.py ├── blog-posts └── carpentries.md ├── blurb.txt ├── book.bib ├── chapters ├── anaconda.Rmd ├── automate.Rmd ├── bash-advanced.Rmd ├── bash-basics.Rmd ├── bash-tools.Rmd ├── chapter-abstracts.md ├── config.Rmd ├── documentation.Rmd ├── errors.Rmd ├── finale.Rmd ├── getting-started.Rmd ├── git-advanced.Rmd ├── git-cmdline.Rmd ├── keypoints.Rmd ├── objectives.Rmd ├── packaging.Rmd ├── provenance.Rmd ├── scripting.Rmd ├── solutions.Rmd ├── ssh.Rmd ├── style.Rmd ├── teams.Rmd ├── testing.Rmd ├── tree.Rmd └── yaml.Rmd ├── code-review └── stop-words.py ├── css └── style.css ├── favicon.ico ├── figures ├── .gitkeep ├── FIXME.png ├── anaconda │ ├── cloud-windspharm-ajdawson.png │ └── cloud-windspharm-search.png ├── automate │ ├── automate.drawio │ ├── collated.png │ ├── concept.pdf │ ├── concept.png │ ├── concept.svg │ └── make-dependency-graph.png ├── bash-basics │ ├── exercise-filesystem.drawio │ ├── exercise-filesystem.pdf │ ├── exercise-filesystem.png │ ├── exercise-filesystem.svg │ ├── man-callouts.png │ ├── nano-editor.png │ ├── sample-filesystem.drawio │ ├── sample-filesystem.pdf │ ├── sample-filesystem.png │ ├── sample-filesystem.svg │ └── the-shell.png ├── bash-tools │ ├── pipe.drawio │ ├── pipe.pdf │ ├── pipe.png │ ├── pipe.svg │ ├── standard-io.drawio │ ├── standard-io.pdf │ ├── standard-io.png │ └── standard-io.svg ├── config │ ├── jane-eyre-big-labels.png │ └── jane-eyre-default.png ├── errors │ ├── exceptions.drawio │ ├── exceptions.pdf │ ├── exceptions.png │ └── exceptions.svg ├── git-advanced │ ├── after-fork.png │ ├── after-sami-pushes.png │ ├── dracula-fit.png │ ├── fill-in-pull-request.png │ ├── fork-button.png │ ├── new-pull-request.png │ ├── open-pull-request-detail.png │ ├── open-pull-request.png │ ├── pr-changes.png │ ├── pr-comment-marker.png │ ├── pr-conflict.png │ ├── pr-details.png │ ├── pr-list.png │ ├── pr-successful-merge.png │ ├── pr-with-comment.png │ ├── pr-with-fix.png │ ├── pr-writing-comment.png │ └── viewing-new-pull-request.png ├── git-cmdline │ ├── git-remote.drawio │ ├── git-remote.pdf │ ├── git-remote.png │ ├── git-remote.svg │ ├── phd-comics.gif │ ├── phd-comics.png │ ├── plot-initial.png │ ├── plot-loglog.png │ ├── repo-history.png │ ├── repo-link.png │ ├── staging-area.drawio │ ├── staging-area.pdf │ ├── staging-area.png │ └── staging-area.svg ├── packaging │ ├── concept.pdf │ ├── concept.png │ ├── concept.svg │ ├── landing-page-original.png │ ├── landing-page.png │ ├── module-countwords.png │ ├── module-index.png │ ├── rse-package-py.drawio │ └── testpypi.png ├── project │ ├── noble.pdf │ ├── noble.png │ ├── noble.svg │ └── project.drawio ├── provenance │ └── release.png ├── scripting │ ├── error-message.png │ └── jane-eyre.png ├── ssh │ ├── ssh.drawio │ ├── ssh.pdf │ ├── ssh.png │ └── ssh.svg ├── style │ ├── code-review-completed-comment.png │ ├── code-review-file-view.png │ ├── code-review-start-comment.png │ └── code-review-writing-comment.png ├── teams │ ├── bug-report.png │ ├── effort-impact-matrix.pdf │ ├── effort-impact-matrix.svg │ ├── issue-labels.png │ ├── lifecycle.drawio │ ├── lifecycle.pdf │ ├── lifecycle.png │ └── lifecycle.svg └── testing │ ├── concept.pdf │ ├── concept.png │ ├── concept.svg │ ├── python-coverage.png │ ├── rse-correct-concept.drawio │ ├── travis-add-repo.png │ ├── travis-build-pass.png │ └── travis-list-repos.png ├── footnote.sty ├── glossary ├── glossary-html.lua ├── glossary-pdf.lua ├── glossary-slugs.txt ├── glossary.md └── glossary.yml ├── includes ├── after_body.tex ├── before_body.tex ├── dedication.md ├── header.html └── preamble.tex ├── index.Rmd ├── keypoints ├── automate.md ├── bash-advanced.md ├── bash-basics.md ├── bash-tools.md ├── config.md ├── errors.md ├── getting-started.md ├── git-advanced.md ├── git-cmdline.md ├── packaging.md ├── provenance.md ├── scripting.md ├── teams.md └── testing.md ├── krantz.cls ├── links.md ├── objectives ├── automate.md ├── bash-advanced.md ├── bash-basics.md ├── bash-tools.md ├── config.md ├── errors.md ├── getting-started.md ├── git-advanced.md ├── git-cmdline.md ├── packaging.md ├── provenance.md ├── scripting.md ├── teams.md └── testing.md ├── py-rse.Rproj ├── references.Rmd ├── requirements.txt ├── reviews ├── review-01+2020-11-02.md ├── review-02+2020-11-11.md └── review-03+2020-11-30.md ├── src └── package-py │ ├── 10 │ ├── README.rst │ ├── bin │ │ └── check_zipf.py │ ├── docs │ │ ├── Makefile │ │ ├── conf.py │ │ └── index.rst │ ├── requirements.txt │ ├── requirements_docs.txt │ ├── setup.py │ ├── test_zipf.py │ └── zipfpy │ │ ├── __init__.py │ │ ├── check.py │ │ └── generate.py │ ├── 01 │ ├── use.py │ └── zipf.py │ ├── 02 │ ├── use.py │ └── zipf.py │ ├── 03a │ ├── test_zipf.py │ └── zipf.py │ ├── 03b │ ├── test_zipf.py │ └── zipfpy │ │ ├── __init__.py │ │ └── zipf.py │ ├── 03c │ ├── test_zipf.py │ └── zipfpy │ │ ├── __init__.py │ │ ├── check.py │ │ └── generate.py │ ├── 04 │ ├── test_zipf.py │ └── zipfpy │ │ ├── __init__.py │ │ ├── check.py │ │ └── generate.py │ ├── 05 │ └── showpath.py │ ├── 06 │ ├── setup.py │ ├── test_zipf.py │ └── zipfpy │ │ ├── __init__.py │ │ ├── check.py │ │ └── generate.py │ ├── 07 │ ├── requirements.txt │ ├── setup.py │ ├── test_zipf.py │ └── zipfpy │ │ ├── __init__.py │ │ ├── check.py │ │ └── generate.py │ ├── 08 │ ├── bin │ │ └── check_zipf.py │ ├── requirements.txt │ ├── setup.py │ ├── test_zipf.py │ └── zipfpy │ │ ├── __init__.py │ │ ├── check.py │ │ └── generate.py │ └── 09 │ ├── README.rst │ ├── bin │ └── check_zipf.py │ ├── requirements.txt │ ├── setup.py │ ├── test_zipf.py │ └── zipfpy │ ├── __init__.py │ ├── check.py │ └── generate.py ├── taylor-francis ├── abstracts-orcids.pdf ├── alt-text │ ├── Alternative Text (alt text).pptx │ ├── Author+Guide+to+Writing+Alt+Text.pdf │ └── SAMPLE_AltTextFigures.doc ├── back-cover-copy.txt ├── bios.txt ├── crc-author-questionnaire.docx ├── notes-for-the-copyeditor.txt ├── permissions-declaration-form.docx ├── permissions-declaration-form.pdf ├── permissions-guidelines │ ├── STM_Permissions_Guidelines_2014.pdf │ ├── ccc-guidelines.pdf │ ├── tf-figure-image-flow-chart.pdf │ ├── tf-permission-flow-chart.pdf │ └── tf-permission-guidelines-and-faqs.pdf ├── permissions-request-letter.docx ├── proposal-guidelines.pdf ├── taylor-francis-contract-2020-08-10.docx ├── taylor-francis-contract-2020-08-10.pdf ├── taylor-francis-proposal-2020-07.md └── tf-latex-manuscript-preparation-guidelines.pdf ├── tex-packages.txt ├── tugboats-3264x2448.jpg ├── tugboats-800x600.jpg ├── unused ├── links-unused.md ├── manning │ ├── manning-contract-2020-08-07.pdf │ └── manning-proposal-2020-07.md └── unused-glossary.md └── zipf ├── .gitignore ├── CITATION.md ├── CONDUCT.md ├── CONTRIBUTING.md ├── KhanVirtanen2020.md ├── LICENSE.md ├── Makefile ├── README.rst ├── bin ├── book_summary.sh ├── collate.py ├── countwords.py ├── plotcounts.py ├── plotparams.yml ├── script_template.py ├── test_zipfs.py └── utilities.py ├── data ├── README.md ├── dracula.txt ├── frankenstein.txt ├── jane_eyre.txt ├── moby_dick.txt ├── sense_and_sensibility.txt ├── sherlock_holmes.txt └── time_machine.txt ├── docs ├── Makefile ├── conf.py ├── index.rst └── source │ ├── collate.rst │ ├── countwords.rst │ ├── modules.rst │ ├── plotcounts.rst │ ├── test_zipfs.rst │ └── utilities.rst ├── environment.yml ├── requirements.txt ├── requirements_docs.txt ├── results ├── collated.csv ├── collated.png ├── dracula.csv ├── dracula.png ├── frankenstein.csv ├── jane_eyre.csv ├── jane_eyre.png ├── moby_dick.csv ├── sense_and_sensibility.csv ├── sherlock_holmes.csv └── time_machine.csv ├── setup.py └── test_data ├── random_words.txt └── risk.txt /.Rbuildignore: -------------------------------------------------------------------------------- 1 | ^\.travis\.yml$ 2 | ^.*\.Rproj$ 3 | ^\.Rproj\.user$ 4 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/discussion-issue-template.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Discussion issue template 3 | about: Template for starting discussions. 4 | title: 'Discussion: ' 5 | labels: discussion 6 | assignees: '' 7 | 8 | --- 9 | 10 | 11 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/proposal-issue-template.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Proposal issue template 3 | about: Template to follow when creating a proposal for the book 4 | title: 'Proposal: ' 5 | labels: proposal 6 | assignees: '' 7 | 8 | --- 9 | 10 | ## Proposal 11 | 12 | What precisely is being proposed (longer than issue subject line, but no more than 2-3 sentences). 13 | 14 | ## Background 15 | 16 | Forces at play, previous discussion, etc. 17 | 18 | ## Pros and Cons 19 | 20 | **Pro:** 21 | - One 22 | - Two 23 | - Three 24 | 25 | **Con:** 26 | - Four 27 | - Five 28 | - Six 29 | 30 | ## Alternatives 31 | 32 | Other options and why they are less good than the proposal. 33 | -------------------------------------------------------------------------------- /.github/workflows/deploy_bookdown.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: 4 | - book 5 | 6 | name: renderbook 7 | 8 | jobs: 9 | bookdown: 10 | name: Render-Book 11 | runs-on: macOS-latest 12 | steps: 13 | - name: Checkout repo 14 | uses: actions/checkout@v2 15 | 16 | - name: Setup R 17 | uses: r-lib/actions/setup-r@master 18 | 19 | 20 | - uses: r-lib/actions/setup-pandoc@v1 21 | with: 22 | pandoc-version: '2.7.3' # The pandoc version to download (if necessary) and use. 23 | 24 | - name: Query dependencies 25 | run: | 26 | install.packages('remotes') 27 | saveRDS(remotes::dev_package_deps(dependencies = TRUE), ".github/depends.Rds", version = 2) 28 | shell: Rscript {0} 29 | 30 | - name: Cache R packages 31 | uses: actions/cache@v1 32 | with: 33 | path: ${{ env.R_LIBS_USER }} 34 | key: macOS-r-4.0-1-${{ hashFiles('.github/depends.Rds') }} 35 | restore-keys: macOS-r-4.0-1- 36 | 37 | - name: Install dependencies 38 | run: | 39 | install.packages("remotes") 40 | remotes::install_deps(dependencies = TRUE) 41 | # tinytex::install_tinytex() 42 | shell: Rscript {0} 43 | 44 | - name: Set up Python 3.6 45 | uses: actions/setup-python@v2 46 | with: 47 | python-version: 3.6 48 | 49 | - name: Install dependencies 50 | run: pip3 install --user -r requirements.txt 51 | 52 | - name: Render Book 53 | run: make html 54 | 55 | - name: Cache bookdown results 56 | uses: actions/cache@v1 57 | with: 58 | path: _bookdown_files 59 | key: bookdown-${{ hashFiles('**/*Rmd') }} 60 | restore-keys: bookdown- 61 | 62 | - uses: actions/upload-artifact@v1 63 | with: 64 | name: _book 65 | path: _book/ 66 | 67 | # Need to first create an empty gh-pages branch 68 | # see https://pkgdown.r-lib.org/reference/deploy_site_github.html 69 | # and also add secrets for a UNIQUE_PAT and EMAIL to the repository 70 | # gh-action from Cecilapp/GitHub-Pages-deploy 71 | checkout-and-deploy: 72 | runs-on: ubuntu-latest 73 | needs: bookdown 74 | steps: 75 | - name: Checkout 76 | uses: actions/checkout@master 77 | - name: Download artifact 78 | uses: actions/download-artifact@v1.0.0 79 | with: 80 | # Artifact name 81 | name: _book # optional 82 | # Destination path 83 | path: _book # optional 84 | - name: Deploy to GitHub Pages 85 | uses: Cecilapp/GitHub-Pages-deploy@v3 86 | env: 87 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # https://github.com/settings/tokens 88 | with: 89 | build_dir: _book/ 90 | email: lwjohnst@gmail.com 91 | 92 | 93 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.aux 2 | *.lof 3 | *.log 4 | *.lot 5 | *.out 6 | *.swp 7 | *.toc 8 | *~ 9 | .DS_Store 10 | .Rhistory 11 | .Rproj.user 12 | .coverage 13 | .pytest_cache 14 | .~lock.* 15 | __pycache__ 16 | _book 17 | _bookdown_files 18 | settings.json 19 | py-rse.Rmd 20 | py-rse.pdf 21 | py-rse.tex 22 | -------------------------------------------------------------------------------- /.words: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/.words -------------------------------------------------------------------------------- /CITATION.cff: -------------------------------------------------------------------------------- 1 | cff-version: 1.2.0 2 | message: "Please cite this paper as shown." 3 | authors: 4 | - family-names: "Irving" 5 | given-names: "Damien" 6 | orcid: "https://orcid.org/0000-0003-1258-5002" 7 | - family-names: "Hertweck" 8 | given-names: "Kate" 9 | orcid: "https://orcid.org/0000-0002-4026-4612" 10 | - family-names: "Johnston" 11 | given-names: "Luke" 12 | orcid: "https://orcid.org/0000-0003-4169-2616" 13 | - family-names: "Ostblom" 14 | given-names: "Joel" 15 | orcid: "https://orcid.org/0000-0003-0051-3239" 16 | - family-names: "Wickham" 17 | given-names: "Charlotte" 18 | orcid: "https://orcid.org/0000-0002-6365-5499" 19 | - family-names: "Wilson" 20 | given-names: "Greg" 21 | orcid: "https://orcid.org/0000-0001-8659-8979" 22 | title: "Research Software Engineering with Python: Building Software that Makes Research Possible" 23 | preferred-citation: 24 | type: book 25 | authors: 26 | - family-names: "Irving" 27 | given-names: "Damien" 28 | orcid: "https://orcid.org/0000-0003-1258-5002" 29 | - family-names: "Hertweck" 30 | given-names: "Kate" 31 | orcid: "https://orcid.org/0000-0002-4026-4612" 32 | - family-names: "Johnston" 33 | given-names: "Luke" 34 | orcid: "https://orcid.org/0000-0003-4169-2616" 35 | - family-names: "Ostblom" 36 | given-names: "Joel" 37 | orcid: "https://orcid.org/0000-0003-0051-3239" 38 | - family-names: "Wickham" 39 | given-names: "Charlotte" 40 | orcid: "https://orcid.org/0000-0002-6365-5499" 41 | - family-names: "Wilson" 42 | given-names: "Greg" 43 | orcid: "https://orcid.org/0000-0001-8659-8979" 44 | isbn: "978-0367698348" 45 | title: "Research Software Engineering with Python: Building Software that Makes Research Possible" 46 | year: "2021" 47 | publisher: "Chapman & Hall/CRC Press" 48 | -------------------------------------------------------------------------------- /CODE_OF_CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Code of Conduct 2 | 3 | In the interest of fostering an open and welcoming environment, we as 4 | contributors and maintainers pledge to making participation in our project and 5 | our community a harassment-free experience for everyone, regardless of age, body 6 | size, disability, ethnicity, gender identity and expression, level of 7 | experience, education, socio-economic status, nationality, personal appearance, 8 | race, religion, or sexual identity and orientation. 9 | 10 | ## Our Standards {#conduct-standards} 11 | 12 | Examples of behavior that contributes to creating a positive environment 13 | include: 14 | 15 | * using welcoming and inclusive language, 16 | * being respectful of differing viewpoints and experiences, 17 | * gracefully accepting constructive criticism, 18 | * focusing on what is best for the community, and 19 | * showing empathy towards other community members. 20 | 21 | Examples of unacceptable behavior by participants include: 22 | 23 | * the use of sexualized language or imagery and unwelcome sexual 24 | attention or advances, 25 | * trolling, insulting/derogatory comments, and personal or political 26 | attacks, 27 | * public or private harassment, 28 | * publishing others' private information, such as a physical or 29 | electronic address, without explicit permission, and 30 | * other conduct which could reasonably be considered inappropriate in 31 | a professional setting 32 | 33 | ## Our Responsibilities {#conduct-responsibilities} 34 | 35 | Project maintainers are responsible for clarifying the standards of acceptable 36 | behavior and are expected to take appropriate and fair corrective action in 37 | response to any instances of unacceptable behavior. 38 | 39 | Project maintainers have the right and responsibility to remove, edit, or reject 40 | comments, commits, code, wiki edits, issues, and other contributions that are 41 | not aligned to this Code of Conduct, or to ban temporarily or permanently any 42 | contributor for other behaviors that they deem inappropriate, threatening, 43 | offensive, or harmful. 44 | 45 | ## Scope {#conduct-scope} 46 | 47 | This Code of Conduct applies both within project spaces and in public spaces 48 | when an individual is representing the project or its community. Examples of 49 | representing a project or community include using an official project e-mail 50 | address, posting via an official social media account, or acting as an appointed 51 | representative at an online or offline event. Representation of a project may be 52 | further defined and clarified by project maintainers. 53 | 54 | ## Enforcement {#conduct-enforcement} 55 | 56 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 57 | reported by [emailing the project team](mailto:gvwilson@third-bit.com). All 58 | complaints will be reviewed and investigated and will result in a response that 59 | is deemed necessary and appropriate to the circumstances. The project team is 60 | obligated to maintain confidentiality with regard to the reporter of an 61 | incident. Further details of specific enforcement policies may be posted 62 | separately. 63 | 64 | Project maintainers who do not follow or enforce the Code of Conduct in good 65 | faith may face temporary or permanent repercussions as determined by other 66 | members of the project's leadership. 67 | 68 | ## Attribution {#conduct-attribution} 69 | 70 | This Code of Conduct is adapted from the 71 | [Contributor Covenant](https://www.contributor-covenant.org) version 1.4. 72 | -------------------------------------------------------------------------------- /DESCRIPTION: -------------------------------------------------------------------------------- 1 | Package: merely.useful 2 | Type: Book 3 | Title: Merely Useful 4 | Version: 1.0.0 5 | Imports: 6 | tinytex, 7 | bookdown, 8 | tools, 9 | reticulate 10 | -------------------------------------------------------------------------------- /LICENSE-MIT.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Authors 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation of the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | # License {#license} 2 | 3 | *This is a human-readable summary of (and not a substitute for) the license. 4 | Please see for the full legal text.* 5 | 6 | This work is licensed under the Creative Commons Attribution 4.0 7 | International license (CC-BY-4.0). 8 | 9 | **You are free to:** 10 | 11 | - **Share**---copy and redistribute the material in any medium or 12 | format 13 | 14 | - **Remix**---remix, transform, and build upon the material for any 15 | purpose, even commercially. 16 | 17 | The licensor cannot revoke these freedoms as long as you follow the 18 | license terms. 19 | 20 | **Under the following terms:** 21 | 22 | - **Attribution**---You must give appropriate credit, provide a link 23 | to the license, and indicate if changes were made. You may do so in 24 | any reasonable manner, but not in any way that suggests the licensor 25 | endorses you or your use. 26 | 27 | - **No additional restrictions**---You may not apply legal terms or 28 | technological measures that legally restrict others from doing 29 | anything the license permits. 30 | 31 | **Notices:** 32 | 33 | You do not have to comply with the license for elements of the 34 | material in the public domain or where your use is permitted by an 35 | applicable exception or limitation. 36 | 37 | No warranties are given. The license may not give you all of the 38 | permissions necessary for your intended use. For example, other rights 39 | such as publicity, privacy, or moral rights may limit how you use the 40 | material. 41 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Research Software Engineering with Python: Building software that makes research possible 2 | 3 | 4 | ![renderbook](https://github.com/merely-useful/py-rse/workflows/renderbook/badge.svg) 5 | 6 | 7 | > I understand that students enjoy it, but it's merely useful. 8 | > 9 | > -- A research professor speaking about an early version of this material. 10 | 11 | This repository contains material for a semester-long course on computing skills for researchers. 12 | Please see for the rendered version. 13 | 14 | For contributors, please follow our [contributing guidelines](.github/CONTRIBUTING.md) 15 | as well as our [Code of Conduct](CODE_OF_CONDUCT.md). 16 | 17 | ``` 18 | @book{rse-py, 19 | author = {Damien Irving and Kate Hertweck and Luke Johnston and Joel Ostblom and Charlotte Wickham and Greg Wilson}, 20 | title = {Research Software Engineering with Python: Building Software that Makes Research Possible}, 21 | publisher = {CRC Press/Taylor and Francis}, 22 | year = {2021}, 23 | isbn = {978-0367698348}, 24 | link = {http://third-bit.com/py-rse/} 25 | } 26 | ``` 27 | -------------------------------------------------------------------------------- /_bookdown.yml: -------------------------------------------------------------------------------- 1 | book_filename: "py-rse" 2 | language: 3 | label: 4 | fig: "Figure " 5 | tab: "Table " 6 | ui: 7 | chapter_name: "Chapter " 8 | output_dir: "_book/" 9 | delete_merged_file: false 10 | before_chapter_script: "_common.R" 11 | rmd_files: 12 | - index.Rmd 13 | - chapters/getting-started.Rmd 14 | - chapters/bash-basics.Rmd 15 | - chapters/bash-tools.Rmd 16 | - chapters/bash-advanced.Rmd 17 | - chapters/scripting.Rmd 18 | - chapters/git-cmdline.Rmd 19 | - chapters/git-advanced.Rmd 20 | - chapters/teams.Rmd 21 | - chapters/automate.Rmd 22 | - chapters/config.Rmd 23 | - chapters/testing.Rmd 24 | - chapters/errors.Rmd 25 | - chapters/provenance.Rmd 26 | - chapters/packaging.Rmd 27 | - chapters/finale.Rmd 28 | 29 | # End files 30 | - appendix.Rmd 31 | - chapters/solutions.Rmd 32 | - chapters/objectives.Rmd 33 | - chapters/keypoints.Rmd 34 | - chapters/tree.Rmd 35 | - chapters/ssh.Rmd 36 | - chapters/style.Rmd 37 | - chapters/documentation.Rmd 38 | - chapters/yaml.Rmd 39 | - chapters/anaconda.Rmd 40 | - glossary/glossary.md 41 | - references.Rmd 42 | - links.md 43 | -------------------------------------------------------------------------------- /_common.R: -------------------------------------------------------------------------------- 1 | #' Insert graphics sensibly into an R Markdown document. 2 | #' 3 | #' @param file The path to the figure. 4 | #' 5 | #' @return Inserts image as necessary for the output format. 6 | #' 7 | #' @examples 8 | #' 9 | #' insert_graphic("figures/automate/concept.pdf") 10 | insert_graphic <- function(file) { 11 | filename_no_ext <- tools::file_path_sans_ext(file) 12 | file_extension <- tools::file_ext(file) 13 | 14 | if (file_extension %in% c("pdf", "svg")) { 15 | pdf_file <- paste0(filename_no_ext, ".pdf") 16 | svg_file <- paste0(filename_no_ext, ".svg") 17 | 18 | pdf_check <- file.exists(pdf_file) 19 | svg_check <- file.exists(svg_file) 20 | 21 | if (!all(pdf_check, svg_check)) 22 | stop("Make sure both a svg and pdf version of the graphic exists.") 23 | 24 | if (knitr::is_latex_output()) { 25 | knitr::include_graphics(pdf_file) 26 | } else { 27 | knitr::include_graphics(svg_file) 28 | } 29 | } else { 30 | knitr::include_graphics(file) 31 | } 32 | } 33 | 34 | # To use Python inside the bookdown R Markdown files. 35 | library(reticulate) 36 | 37 | knitr::opts_chunk$set(fig.align = "center", 38 | out.width = "100%") 39 | -------------------------------------------------------------------------------- /_output.yml: -------------------------------------------------------------------------------- 1 | bookdown::gitbook: 2 | includes: 3 | in_header: includes/header.html 4 | css: css/style.css 5 | keep_md: true 6 | split_bib: false 7 | highlight: haddock 8 | config: 9 | toc: 10 | collapse: section 11 | before: | 12 |
  • Merely Useful
    • 13 |
    • 14 | after: | 15 |
  • Published with bookdown
    • 16 | edit: https://github.com/merely-useful/py-rse/edit/book/%s 17 | view: https://github.com/merely-useful/py-rse/blob/book/%s 18 | sharing: null 19 | pandoc_args: ['--lua-filter=glossary/glossary-html.lua'] 20 | bookdown::pdf_book: 21 | includes: 22 | in_header: includes/preamble.tex 23 | before_body: includes/before_body.tex 24 | after_body: includes/after_body.tex 25 | keep_tex: true 26 | latex_engine: "xelatex" 27 | template: null 28 | pandoc_args: ['--lua-filter=glossary/glossary-pdf.lua', '--top-level-division=chapter'] 29 | toc_depth: 2 30 | toc_unnumbered: false 31 | toc_appendix: true 32 | 33 | -------------------------------------------------------------------------------- /appendix.Rmd: -------------------------------------------------------------------------------- 1 | # (APPENDIX) Appendix {-} 2 | -------------------------------------------------------------------------------- /author-flyer-2021-07.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/author-flyer-2021-07.pdf -------------------------------------------------------------------------------- /bin/captions.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | captions.py source_file [source_file...] 5 | 6 | Check the captions of figures. 7 | ''' 8 | 9 | import sys 10 | import re 11 | from util import read_all_files 12 | 13 | 14 | PAT_USE = re.compile(r'fig.cap="(.+?)"') 15 | 16 | 17 | def main(source_files): 18 | '''Main driver.''' 19 | 20 | captions = read_all_files(source_files, find_captions) 21 | captions = [c for c in captions if not c.endswith('.')] 22 | if captions: 23 | for c in captions: 24 | print(c) 25 | 26 | 27 | def find_captions(filename, reader): 28 | '''Find all link uses in input stream.''' 29 | 30 | return set(PAT_USE.findall(reader.read())) 31 | 32 | 33 | if __name__ == '__main__': 34 | if len(sys.argv) < 2: 35 | print('Usage: captions.py source_file [source_file...]', file=sys.stderr) 36 | sys.exit(1) 37 | main(sys.argv[1:]) 38 | -------------------------------------------------------------------------------- /bin/chapters.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | Check that chapters are consistent in configuration YAML, Makefile, keypoints, 5 | and objectives. 6 | 7 | Usage: make settings | bin/chapters.py _something.yml MAKEFILE_KEY ...include_files... 8 | ''' 9 | 10 | 11 | import sys 12 | import yaml 13 | import re 14 | import os 15 | 16 | 17 | CONFIG_KEY = 'rmd_files' 18 | INCLUDE_RE = re.compile(r'```{r,\s+child="(.+?)"}') 19 | BOILERPLATE = 'CONDUCT.md CONTRIBUTING.md LICENSE.md appendix.Rmd gloss.md index.Rmd links.md references.Rmd'.split() 20 | 21 | 22 | def main(args): 23 | assert len(args) >= 2, 'Expected at least 2 arguments' 24 | config_file, makefile_key, include_files = args[0], args[1], args[2:] 25 | makefile = read_makefile_names(sys.stdin, makefile_key) 26 | config = read_config(config_file) - sanitize(BOILERPLATE) 27 | includes = [read_include(f) for f in include_files] 28 | check('config and makefile', config, makefile) 29 | for (filename, values) in zip(include_files, includes): 30 | check('config and {}'.format(filename), config, values) 31 | 32 | 33 | def read_makefile_names(reader, makefile_key): 34 | lines = [x for x in reader.readlines() if x.startswith(makefile_key)] 35 | assert len(lines) == 1, 'No match for {}'.format(makefile_key) 36 | return sanitize(lines[0].split(':')[1].strip().split()) 37 | 38 | 39 | def read_config(config_file): 40 | with open(config_file, 'r') as reader: 41 | config = yaml.safe_load(reader) 42 | assert CONFIG_KEY in config, '{} not found in {}'.format(CONFIG_KEY, config_file) 43 | return sanitize(config[CONFIG_KEY]) 44 | 45 | 46 | def read_include(filename): 47 | with open(filename, 'r') as reader: 48 | raw = [x.split('/') for x in INCLUDE_RE.findall(reader.read())] 49 | assert len(set([x[0] for x in raw])) == 1, 'Inconsistent directory names in {}'.format(filename) 50 | return sanitize([x[1] for x in raw]) 51 | 52 | 53 | def sanitize(names): 54 | return set([os.path.splitext(n)[0] for n in names]) 55 | 56 | 57 | def check(title, left, right): 58 | for (subtitle, items) in (('missing', left-right), ('extra', right-left)): 59 | if items: 60 | print('{}: {}'.format(title, subtitle)) 61 | for i in sorted(items): 62 | print(' {}'.format(i)) 63 | 64 | 65 | if __name__ == '__main__': 66 | main(sys.argv[1:]) 67 | -------------------------------------------------------------------------------- /bin/chunks.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | chunks.py [source_file...] 5 | 6 | Check that all chunks are either naked (no language spec) or have a recognizable 7 | language and a label. If no source files are given, reads from standard input. 8 | ''' 9 | 10 | 11 | import sys 12 | import re 13 | from util import LANGUAGES, read_all_files, report 14 | 15 | 16 | MARKER = '```' 17 | RICH_MARKER = re.compile(r'^```\{(.+?)\s+([^=,]+)(,\s*.+=.+)*\s*}$') 18 | 19 | 20 | def main(source_files): 21 | '''Main driver.''' 22 | 23 | chunks = read_all_files(source_files, get_chunks) 24 | chunks = {c for c in chunks if bad_chunk(c)} 25 | report('Bad Chunks', chunks) 26 | 27 | 28 | def get_chunks(filename, reader): 29 | '''Extract chunk headers.''' 30 | 31 | result = set() 32 | in_chunk = False 33 | for (i, line) in enumerate(reader): 34 | if line.startswith(MARKER): 35 | marker = line.rstrip() 36 | if in_chunk: 37 | assert marker == MARKER, \ 38 | f'Badly-formed end of chunk {filename}:{i}' 39 | in_chunk = False 40 | else: 41 | result.add((filename, i+1, marker)) 42 | in_chunk = True 43 | 44 | return result 45 | 46 | 47 | def bad_chunk(chunk): 48 | '''Is this a badly formed chunk?''' 49 | 50 | filename, line_number, marker = chunk 51 | 52 | # Naked chunk (plain text). 53 | if marker == MARKER: 54 | return False 55 | 56 | # Doesn't match pattern for rich chunk header. 57 | match = RICH_MARKER.search(marker) 58 | if not match: 59 | return True 60 | 61 | # Unknown language. 62 | language, label = match.group(1), match.group(2) 63 | if language not in LANGUAGES: 64 | return True 65 | 66 | # No reason to reject. 67 | return False 68 | 69 | 70 | if __name__ == '__main__': 71 | main(sys.argv[1:]) 72 | -------------------------------------------------------------------------------- /bin/citations.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | '''Check that all entries in the bibliography are used.''' 4 | 5 | import sys 6 | import re 7 | from util import read_all_files, report 8 | 9 | 10 | BIB_KEY = re.compile('^@\w+\{(\w+),') 11 | BIB_REF = re.compile('@(\w+)', re.DOTALL + re.MULTILINE) 12 | FALSE_POSITIVES = {'ref'} 13 | 14 | 15 | def main(): 16 | '''Main driver.''' 17 | bib_file, chapters = sys.argv[1], sys.argv[2:] 18 | defined = get_keys(bib_file) 19 | used = read_all_files(chapters, get_references, True) 20 | report('Defined but not used', defined - used) 21 | report('Used but not defined', used - defined) 22 | 23 | 24 | def get_keys(filename): 25 | '''Get bibliography keys from .bib file.''' 26 | 27 | with open(filename, 'r') as reader: 28 | lines = reader.readlines() 29 | matches = [BIB_KEY.search(x) for x in lines] 30 | matches = [x.group(1) for x in matches if x] 31 | return set(matches) 32 | 33 | 34 | def get_references(name, reader): 35 | '''Get bibliography references.''' 36 | 37 | return set(BIB_REF.findall(reader.read())) - FALSE_POSITIVES 38 | 39 | 40 | if __name__ == '__main__': 41 | main() 42 | -------------------------------------------------------------------------------- /bin/crossrefs.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | Check cross-references. 5 | Usage: bin/crossrefs.py title file file file 6 | ''' 7 | 8 | import sys 9 | import re 10 | from util import report 11 | 12 | HEADING = re.compile(r'^#+\s*.+\{#(.+)\}', re.MULTILINE) 13 | REF = re.compile(r'\\@ref\((.+?)\)') 14 | FIGURE = re.compile(r'```\{r\s+([^,]+),.+fig\.cap=".+"\}') 15 | 16 | 17 | def main(title, filenames): 18 | text = [open(f, 'r').read() for f in filenames] 19 | headings = set([x for sublist in text for x in HEADING.findall(sublist)]) 20 | figures = set(['fig:{}'.format(x) for sublist in text for x in FIGURE.findall(sublist)]) 21 | refs = set([r for sublist in text for r in REF.findall(sublist)]) 22 | report('{}: used but not defined'.format(title), refs - (headings | figures)) 23 | 24 | 25 | if __name__ == '__main__': 26 | assert len(sys.argv) > 2, 'Need title and filenames' 27 | main(sys.argv[1], sys.argv[2:]) 28 | -------------------------------------------------------------------------------- /bin/dedication.R: -------------------------------------------------------------------------------- 1 | library(tidyverse) 2 | library(here) 3 | 4 | tmp_md_file <- tempfile(fileext = ".md") 5 | 6 | rmarkdown::pandoc_convert( 7 | here("includes/before_body.tex"), 8 | to = "markdown", 9 | output = tmp_md_file 10 | ) 11 | 12 | tmp_md_file %>% 13 | read_lines() %>% 14 | str_replace_all("\\\\", " ") %>% 15 | write_lines(here("includes/dedication.md")) 16 | -------------------------------------------------------------------------------- /bin/exercises.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | Check that exercises have solutions and solutions have exercises, 5 | all in the correct order. 6 | ''' 7 | 8 | 9 | import sys 10 | import os 11 | import re 12 | 13 | 14 | PAT_EXERCISE = re.compile(r'^###\s+.+?\{#(.+?)\}', re.MULTILINE + re.DOTALL) 15 | PAT_SOLUTION = re.compile(r'^###\s+Exercise.+?\((.+?)\)', re.MULTILINE + re.DOTALL) 16 | 17 | 18 | def main(): 19 | '''Main driver.''' 20 | allSolutions = getSolutions(sys.argv[1]) 21 | for filename in sys.argv[2:]: 22 | exercises = getExercises(filename) 23 | if not exercises: 24 | continue 25 | subset = subsetSolutions(filename, allSolutions) 26 | for (ex, sol) in zip(exercises, subset): 27 | if ex != sol: 28 | print(ex, sol) 29 | 30 | 31 | def getSolutions(filename): 32 | with open(filename, 'r') as reader: 33 | text = reader.read() 34 | return [f'{t}\n' for t in PAT_SOLUTION.findall(text)] 35 | 36 | 37 | def getExercises(filename): 38 | with open(filename, 'r') as reader: 39 | text = reader.read() 40 | return [f'{d}\n' for d in PAT_EXERCISE.findall(text) if '-ex-' in d] 41 | 42 | 43 | def subsetSolutions(filename, allSolutions): 44 | stem = os.path.splitext(os.path.basename(filename))[0] 45 | return [x for x in allSolutions if x.startswith(stem)] 46 | 47 | 48 | if __name__ == '__main__': 49 | main() 50 | -------------------------------------------------------------------------------- /bin/glossarize.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | '''Convert YAML glossary to required Markdown format.''' 4 | 5 | 6 | import sys 7 | import re 8 | import yaml 9 | 10 | 11 | MARKDOWN_REF = re.compile(r'\[(.+?)\]\(#(.+?)\)', re.DOTALL + re.MULTILINE) 12 | 13 | 14 | def main(): 15 | '''Main driver.''' 16 | with open(sys.argv[1], 'r') as reader: 17 | terms = {x.strip() for x in reader} 18 | glossary = yaml.load(sys.stdin, Loader=yaml.FullLoader) 19 | crossref = {entry['slug']:entry for entry in glossary if entry['slug'] in terms} 20 | slugs = list(sorted(crossref.keys())) 21 | for slug in slugs: 22 | show(crossref, crossref[slug]) 23 | 24 | 25 | def show(glossary, entry): 26 | '''Print required Markdown.''' 27 | term = entry['en']['term'] 28 | slug = entry['slug'] 29 | body = make_body(entry['en']['def']) 30 | ref = make_ref(glossary, entry) 31 | print('**{term}**'.format(term=term, slug=slug)) 32 | print(f': {body}{ref}\n') 33 | 34 | 35 | def make_body_replace(match): 36 | '''Replace a single Markdown-style interlink in the glossary.''' 37 | text = match.group(1) 38 | link = match.group(2) 39 | return f'\\gref{{{text}}}{{{link}}}' 40 | 41 | 42 | def make_body(raw): 43 | '''Convert glossary YAML body into required Markdown text.''' 44 | body = ' '.join([x.strip() for x in raw.split('\n')]) 45 | return MARKDOWN_REF.sub(make_body_replace, body) 46 | 47 | 48 | def make_ref(glossary, entry): 49 | '''Build cross-references if any.''' 50 | if 'ref' not in entry: 51 | return '' 52 | 53 | terms = [glossary[r] for r in entry['ref'] if r in glossary] 54 | if not terms: 55 | return '' 56 | 57 | slugs = [e['slug'] for e in terms] 58 | words = [e['en']['term'] for e in terms] 59 | links = [f'[{word}](#{slug})' for (word, slug) in zip(words, slugs)] 60 | joined = ", ".join(links) 61 | return f' See also: {joined}' 62 | 63 | 64 | if __name__ == '__main__': 65 | main() 66 | -------------------------------------------------------------------------------- /bin/glossary-merge.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | '''Merge glossary files, optionally keeping only selected terms.''' 4 | 5 | import sys 6 | import getopt 7 | import yaml 8 | 9 | 10 | USAGE = 'merge [-h] [-w wordlist | -] glossary [glossary...]' 11 | 12 | 13 | def main(): 14 | '''Main driver.''' 15 | wordList, glossaryFiles = parseArgs() 16 | glossary = readAndMerge(glossaryFiles) 17 | if wordList: 18 | glossary = keepOnly(glossary, wordList) 19 | glossary = listSort(glossary) 20 | yaml.dump(glossary, sys.stdout, encoding='utf-8') 21 | 22 | 23 | def parseArgs(): 24 | '''Parse command-line arguments, reading keep list if provided.''' 25 | 26 | wordList = None 27 | options, filenames = getopt.getopt(sys.argv[1:], 'hw:') 28 | for (opt, arg) in options: 29 | if opt == '-h': 30 | print(USAGE) 31 | sys.exit(0) 32 | elif opt == '-w': 33 | wordList = arg 34 | else: 35 | print(USAGE, sys.stderr) 36 | sys.exit(1) 37 | 38 | if not filenames: 39 | print(USAGE, sys.stderr) 40 | sys.exit(1) 41 | 42 | if wordList == '-': 43 | wordList = [x.strip() for x in sys.stdin] 44 | elif wordList is not None: 45 | with open(wordList, 'r') as reader: 46 | wordList = [x.strip() for x in reader] 47 | 48 | return wordList, filenames 49 | 50 | 51 | def readAndMerge(filenames): 52 | '''Read all glossaries, returning merged version.''' 53 | 54 | combined = {} 55 | for f in filenames: 56 | with open(f, 'r', encoding='utf-8') as reader: 57 | new = yaml.load(reader, Loader=yaml.FullLoader) 58 | for entry in new: 59 | for key in entry: 60 | if (type(entry[key]) == dict) and ('def' in entry[key]): 61 | entry[key]['def'] = entry[key]['def'].strip() 62 | combined[entry['slug']] = entry 63 | return combined 64 | 65 | 66 | def keepOnly(glossary, wordList): 67 | '''Keep only the listed entries.''' 68 | 69 | result, missing = {}, False 70 | for word in wordList: 71 | if word not in glossary: 72 | print(f'Unknown word {word}', file=sys.stderr) 73 | missing = True 74 | else: 75 | result[word] = glossary[word] 76 | 77 | if missing: 78 | sys.exit(1) 79 | return result 80 | 81 | 82 | def listSort(glossary): 83 | '''Convert glossary to list sorted by slug.''' 84 | glossary = [x for x in glossary.values()] 85 | glossary.sort(key=lambda x: x['slug']) 86 | return glossary 87 | 88 | 89 | if __name__ == '__main__': 90 | main() 91 | -------------------------------------------------------------------------------- /bin/glosscheck.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | glosscheck.py gloss_file source_file [source_file...] 5 | 6 | Check the glossary entries in the given files (all defined, all used). 7 | ''' 8 | 9 | 10 | import sys 11 | import re 12 | from util import read_all_files, report 13 | 14 | 15 | def main(gloss_file, source_files): 16 | '''Main driver.''' 17 | 18 | term_def = get_defined_terms(gloss_file) 19 | term_used = read_all_files(source_files, find_uses, remove_code=False) 20 | report('Undefined terms', term_used - term_def) 21 | report('Unused links', term_def - term_used) 22 | 23 | 24 | def get_defined_terms(gloss_file): 25 | '''Get IDs of all defined glossary entries.''' 26 | pat = re.compile(r'', re.MULTILINE + re.DOTALL) 27 | with open(gloss_file, 'r') as reader: 28 | return set(pat.findall(reader.read())) 29 | 30 | 31 | LINK_USE_PAT = re.compile(r'\\gref\{.+?\}\{(.+?)\}') 32 | def find_uses(filename, reader): 33 | '''Find all link uses in input stream.''' 34 | 35 | return set(LINK_USE_PAT.findall(reader.read())) 36 | 37 | 38 | if __name__ == '__main__': 39 | if len(sys.argv) < 3: 40 | print('Usage: glosscheck.py gloss_file source_file [source_file...]', file=sys.stderr) 41 | sys.exit(1) 42 | main(sys.argv[1], sys.argv[2:]) 43 | -------------------------------------------------------------------------------- /bin/images.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | images.py figures_dir [source_file...] 5 | 6 | Check that all of the images used in the given source files are present and that 7 | all images are used. If no source files are given, reads from standard input 8 | (but image directory must still be named). 9 | ''' 10 | 11 | import sys 12 | import os 13 | import glob 14 | import re 15 | from util import read_all_files, report 16 | 17 | 18 | SUFFIXES = ('jpg', 'pdf', 'png', 'svg') 19 | 20 | 21 | PAT_USE = re.compile(r'knitr::include_graphics\("(.+?)"\)') 22 | 23 | 24 | def main(links_dir, source_files): 25 | '''Main driver.''' 26 | 27 | defined = set() 28 | for suffix in SUFFIXES: 29 | defined.update(set(os.path.normpath(p) for p in glob.glob(f'{links_dir}/**/*.{suffix}', 30 | recursive=True))) 31 | used = read_all_files(source_files, find_uses) 32 | report('Links: undefined', used - defined) 33 | report('Links: unused', defined - used) 34 | 35 | 36 | def find_uses(filename, reader): 37 | '''Find all link uses in input stream.''' 38 | 39 | return set(PAT_USE.findall(reader.read())) 40 | 41 | 42 | if __name__ == '__main__': 43 | if len(sys.argv) < 2: 44 | print('Usage: images.py links_file [source_file...]', file=sys.stderr) 45 | sys.exit(1) 46 | main(sys.argv[1], sys.argv[2:]) 47 | -------------------------------------------------------------------------------- /bin/line-length.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | 5 | 6 | def main(): 7 | limit = int(sys.argv[1]) 8 | problems = find_problems(limit, sys.argv[2:]) 9 | if (problems): 10 | report(limit, problems) 11 | 12 | 13 | def find_problems(limit, filenames): 14 | result = [] 15 | for filename in filenames: 16 | with open(filename, 'r') as reader: 17 | for (i, line) in enumerate(reader): 18 | line = line.rstrip() 19 | if len(line) > limit: 20 | result.append([filename, str(i), line]) 21 | return result 22 | 23 | 24 | def report(limit, problems): 25 | first = max([len(entry[0]) for entry in problems]) 26 | second = max([len(entry[1]) for entry in problems]) 27 | fmt = '{{0:{0}s}} {{1:{1}s}} {{2:{2}s}}'.format(first, second, limit) 28 | print(fmt.format(' ' * first, ' ' * second, '-' * limit)) 29 | for problem in problems: 30 | print(fmt.format(*problem)) 31 | 32 | 33 | if __name__ == '__main__': 34 | main() 35 | -------------------------------------------------------------------------------- /bin/linkcheck.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | linkcheck.py links_file source_file [source_file...] 5 | 6 | Check the links in the given files (all defined, all used). 7 | ''' 8 | 9 | 10 | import sys 11 | import re 12 | from util import read_all_files, report 13 | 14 | 15 | def main(links_file, source_files): 16 | '''Main driver.''' 17 | 18 | link_def = get_defined_links(links_file) 19 | link_used = read_all_files(source_files, find_uses, remove_code=False) 20 | report('Undefined links', link_used - link_def) 21 | report('Unused links', link_def - link_used) 22 | 23 | 24 | def get_defined_links(links_file): 25 | '''Get aliases of all defined links.''' 26 | pat = re.compile(r'^\[(.+?)\]:', re.MULTILINE + re.DOTALL) 27 | with open(links_file, 'r') as reader: 28 | return set(pat.findall(reader.read())) 29 | 30 | 31 | LINK_USE_PAT = re.compile(r'\[.+?\]\[(.+?)\]') 32 | def find_uses(filename, reader): 33 | '''Find all link uses in input stream.''' 34 | 35 | return set(LINK_USE_PAT.findall(reader.read())) 36 | 37 | 38 | if __name__ == '__main__': 39 | if len(sys.argv) < 3: 40 | print('Usage: linkcheck.py links_file source_file [source_file...]', file=sys.stderr) 41 | sys.exit(1) 42 | main(sys.argv[1], sys.argv[2:]) 43 | -------------------------------------------------------------------------------- /bin/list-keypoints-objectives.R: -------------------------------------------------------------------------------- 1 | library(commonmark) 2 | library(tidyverse) 3 | library(xml2) 4 | library(glue) 5 | library(fs) 6 | 7 | # Extract chapter info ---------------------------------------------------- 8 | 9 | chapter_files <- dir_ls("chapters/") 10 | 11 | extract_h1_text <- function(.file) { 12 | .file %>% 13 | read_lines() %>% 14 | markdown_xml() %>% 15 | read_xml() %>% 16 | xml_find_first(".//d1:heading") %>% 17 | xml_text() %>% 18 | str_remove(" \\{.*$") 19 | } 20 | 21 | chapter_name <- chapter_files %>% 22 | map_chr(extract_h1_text) 23 | 24 | chapter_df <- tibble( 25 | id = chapter_files %>% 26 | str_remove("^.*\\/") %>% 27 | str_remove("\\.R?md$"), 28 | chapter_file = chapter_files, 29 | chapter_name = chapter_name 30 | ) 31 | 32 | # Extract keypoint info --------------------------------------------------- 33 | 34 | keypoint_files <- dir_ls("keypoints") 35 | 36 | keypoint_df <- tibble( 37 | id = keypoint_files %>% 38 | str_remove("^.*\\/") %>% 39 | str_remove("\\.md$"), 40 | keypoint_files = keypoint_files 41 | ) 42 | 43 | # Extract objectives info ------------------------------------------------- 44 | 45 | objectives_files <- dir_ls("objectives") 46 | 47 | objectives_df <- tibble( 48 | id = objectives_files %>% 49 | str_remove("^.*\\/") %>% 50 | str_remove("\\.md$"), 51 | objectives_files = objectives_files 52 | ) 53 | 54 | # Get chapter order ------------------------------------------------------- 55 | 56 | chapter_order <- yaml::read_yaml("_bookdown.yml")$rmd_files %>% 57 | str_match("py-rse\\/.*") %>% 58 | na.omit() 59 | 60 | order_df <- tibble( 61 | ordering = 1:length(chapter_order), 62 | id = chapter_order %>% 63 | str_remove("^.*\\/") %>% 64 | str_remove("\\.R?md$") 65 | ) 66 | 67 | # Combine, prepare, and paste --------------------------------------------- 68 | 69 | # To look it over 70 | reduce(list(chapter_df, keypoint_df, objectives_df, order_df), 71 | full_join, 72 | by = "id") %>% 73 | View() 74 | 75 | combined_df <- 76 | reduce(list(chapter_df, keypoint_df, objectives_df, order_df), 77 | full_join, 78 | by = "id") %>% 79 | arrange(ordering) %>% 80 | # To get rid of unnecessary info 81 | na.omit() 82 | 83 | keypoint_list_to_paste <- combined_df %>% 84 | glue_data(" 85 | 86 | ## {chapter_name} 87 | 88 | ```{{r, child='{keypoint_files}'}} 89 | ``` 90 | 91 | ") %>% 92 | glue_collapse() 93 | 94 | # Send to clipboard to paste 95 | clipr::write_clip(keypoint_list_to_paste) 96 | 97 | objective_list_to_paste <- combined_df %>% 98 | glue_data(" 99 | 100 | ## {chapter_name} 101 | 102 | ```{{r, child='{objectives_files}'}} 103 | ``` 104 | 105 | ") %>% 106 | glue_collapse() 107 | 108 | clipr::write_clip(objective_list_to_paste) 109 | -------------------------------------------------------------------------------- /bin/nbspref.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | Check that all \@ref() uses have a non-breaking space in front of them. 5 | Usage: nbspref.py filenames... 6 | ''' 7 | 8 | 9 | import sys 10 | import re 11 | 12 | 13 | REF_RE = re.compile(r'.\\@ref\(.+?\)') 14 | 15 | 16 | def main(filenames): 17 | for f in filenames: 18 | with open(f, 'r') as reader: 19 | data = reader.read() 20 | for m in REF_RE.findall(data): 21 | if (not m.startswith(' ')): 22 | print('{}: {}'.format(f, m)) 23 | 24 | 25 | if __name__ == '__main__': 26 | main(sys.argv[1:]) 27 | -------------------------------------------------------------------------------- /bin/right-trim.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import sys 4 | 5 | 6 | def main(): 7 | for filename in sys.argv[1:]: 8 | with open(filename, 'r') as reader: 9 | lines = reader.readlines() 10 | lines = [x.rstrip() for x in lines] 11 | with open(filename, 'w') as writer: 12 | for line in lines: 13 | print(line, file=writer) 14 | 15 | 16 | if __name__ == '__main__': 17 | main() 18 | -------------------------------------------------------------------------------- /bin/rstrip.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | Strip trailing whitespaces from lines. 5 | Usage: rstrip.py file file... 6 | ''' 7 | 8 | 9 | import sys 10 | 11 | 12 | def main(filenames): 13 | for f in filenames: 14 | with open(f, 'r') as reader: 15 | lines = reader.readlines() 16 | lines = [x.rstrip() + '\n' for x in lines] 17 | with open(f, 'w') as writer: 18 | writer.writelines(lines) 19 | 20 | 21 | if __name__ == '__main__': 22 | main(sys.argv[1:]) 23 | -------------------------------------------------------------------------------- /bin/test_chunks.py: -------------------------------------------------------------------------------- 1 | from chunks import bad_chunk 2 | 3 | def wrap_bad(chunk): 4 | return bad_chunk(('testfile', 0, chunk)) 5 | 6 | def test_naked_chunk(): 7 | assert not wrap_bad('```') 8 | 9 | def test_r_label(): 10 | assert not wrap_bad('```{r label}') 11 | 12 | def test_r_label_param(): 13 | assert not wrap_bad('```{r label, param=value}') 14 | 15 | def test_r_label_multi_param(): 16 | assert not wrap_bad('```{r label-2, param=value, param=value}') 17 | 18 | def test_r_param_only(): 19 | assert wrap_bad('```{r param=value}') 20 | 21 | def test_py_label(): 22 | assert not wrap_bad('```{python label}') 23 | 24 | def test_py_label_param(): 25 | assert not wrap_bad('```{python label, param=value}') 26 | 27 | def test_py_param_only(): 28 | assert wrap_bad('```{python param=value}') 29 | 30 | def test_unknown_lang(): 31 | assert wrap_bad('```{unknown label}') 32 | -------------------------------------------------------------------------------- /bin/util.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | ''' 4 | Utilities. 5 | ''' 6 | 7 | import sys, re 8 | from io import StringIO 9 | 10 | 11 | LANGUAGES = {'r', 'python'} 12 | PAT_BLOCK = re.compile(r'^```.+?```$', re.DOTALL + re.MULTILINE) 13 | PAT_INLINE = re.compile(r'`.+?`') 14 | 15 | 16 | def report(title, values): 17 | '''Print values if any.''' 18 | 19 | if values: 20 | print(title) 21 | for v in sorted(values): 22 | print(' ', v) 23 | 24 | 25 | def read_all_files(filenames, handler, remove_code=False): 26 | '''Read information from stdin or all source files.''' 27 | 28 | if not filenames: 29 | _handle_single_file('-', handler, sys.stdin, remove_code) 30 | 31 | else: 32 | result = set() 33 | for name in filenames: 34 | with open(name, 'r') as reader: 35 | result |= _handle_single_file(name, handler, reader, remove_code) 36 | 37 | return result 38 | 39 | 40 | def _handle_single_file(name, handler, stream, remove_code): 41 | '''Read and handle a single file.''' 42 | 43 | raw = stream.read() 44 | if remove_code: 45 | raw = _remove_ticked_code(raw) 46 | wrapped = StringIO(raw) 47 | return handler(name, wrapped) 48 | 49 | 50 | def _remove_ticked_code(text): 51 | '''Remove `inline` and ```triple quoted``` code blocks.''' 52 | 53 | text = PAT_BLOCK.sub('', text) 54 | text = PAT_INLINE.sub('', text) 55 | return text 56 | -------------------------------------------------------------------------------- /blog-posts/carpentries.md: -------------------------------------------------------------------------------- 1 | ## New book: Research Software Engineering with Python 2 | 3 | *Damien Irving, Kate Hertweck, Luke Johnston, Joel Ostblom, Charlotte Wickham, Greg Wilson* 4 | 5 | A big part of the Carpentries success is the two-day workshop format, 6 | but that hasn't stopped countless instructors over the years speculating 7 | about the great things we could teach if only we had more time. 8 | With an entire semester, for instance, 9 | you could take researchers through the entire lifecycle of a data analysis project, 10 | from the initial setup and code development 11 | through to a fully automated data processing pipeline and published software package. 12 | A small group of us decided to act on that speculation a few years ago, 13 | and set about writing *Research Software Engineering with Python*. 14 | It effectively represents our attempt at lesson materials for a Software Carpentry workshop 15 | that runs for the duration of a university semester course. 16 | 17 | The book follows Amira and Sami as they work together to write a software package to address a real research question. 18 | The data analysis task relates to a fascinating result in the field of quantitative linguistics. 19 | Zipf’s Law states that the second most common word in a large body of text appears half as often as the most common, 20 | the third most common appears a third as often, and so on. 21 | To test whether Zipf’s Law holds for a collection of classic novels that are freely available from Project Gutenberg, 22 | Amira and Sami write a software package that counts and analyses the word frequency distribution in any arbitrary body of text. 23 | 24 | In the process of writing and publishing this Python package to verify Zipf’s Law, the book covers how to do the following: 25 | 26 | - Organise small and medium-sized data science projects. 27 | - Use the Unix shell to efficiently manage your data and code. 28 | - Write Python programs that can be used on the command line. 29 | - Use Git and GitHub to track and share your work. 30 | - Work productively in a small team where everyone is welcome. 31 | - Use Make to automate complex workflows. 32 | - Enable users to configure your software without modifying it directly. 33 | - Test your software and know which parts have not yet been tested. 34 | - Find, handle, and fix errors in your code. 35 | - Publish your code and research in open and reproducible ways. 36 | - Create Python packages that can be installed in standard ways. 37 | 38 | The book is available for 39 | [purchase now](https://www.routledge.com/Research-Software-Engineering-with-Python-Building-software-that-makes/Irving-Hertweck-Johnston-Ostblom-Wickham-Wilson/p/book/9780367698324) 40 | (all proceeds go to The Carpentries) 41 | and the content and associated code is licensed under a CC-BY 4.0 and MIT License, 42 | so there’s also a freely available [web version](https://merely-useful.tech/py-rse/). 43 | A corresponding book for R users is also currently under development 44 | (see the live development version [here](https://merely-useful.tech/r-rse/)). 45 | 46 | Much of the content in the early chapters of the book borrows from existing Software Carpentry lessons (e.g. unix shell, version control), 47 | but the later chapters cover a number of topics that either aren't included in the current offerings of Carpentries lessons at all 48 | or that are currently only at the Incubator stage of development 49 | (e.g. elements of the program configuration, error handling, provenance tracking and python packaging chapters). 50 | The book might therefore be a useful reference for people developing new lessons. 51 | 52 | Comments and suggestions are more than welcome at the book’s [GitHub repository](https://github.com/merely-useful/py-rse) – 53 | we’d be particularly keen to hear from anyone (before, during or after) 54 | who uses it as a textbook for a university semester course or an extended Carpentries-style workshop. 55 | -------------------------------------------------------------------------------- /blurb.txt: -------------------------------------------------------------------------------- 1 | Writing and running software is now as much a part of science as telescopes and test tubes, but most researchers are never taught how to do either well. As a result, it takes them longer to accomplish simple tasks than it should, and it is harder for them to share their work with others than it needs to be. 2 | 3 | This book introduces the concepts, tools, and skills that researchers need to get more done in less time and with less pain. Based on the practical experiences of its authors, who collectively have spent several decades teaching software skills to scientists, it covers everything graduate-level researchers need to automate their workflows, collaborate with colleagues, ensure that their results are trustworthy, and publish what they have built so that others can build on it. The book assumes only a basic knowledge of Python as a starting point, and shows readers how it, the Unix shell, Git, Make, and related tools can give them more time to focus on the research they actually want to do. 4 | 5 | "Research Software Engineering with Python" can be used as the main text in a one-semester course or for self-guided study. A running example shows how to organize a small research project step by step; over a hundred exercises give readers a chance to practice these skills themselves, while a glossary defining over two hundred terms will help readers find their way through the terminology. All of the material can be read at https://merely-useful.tech/py-rse/ and re-used under a Creative Commons license, and all royalties from sales of the book will be donated to The Carpentries, an organization that teaches foundational coding and data science skills to researchers worldwide. 6 | -------------------------------------------------------------------------------- /chapters/finale.Rmd: -------------------------------------------------------------------------------- 1 | # Finale {#finale} 2 | 3 | We have come a long way since we first met Amira, Jun, and Sami in Section \@ref(intro-personas). 4 | Amira now has her scripts, datasets and reports organized, 5 | in version control, 6 | and on GitHub. 7 | Her work has already paid off: 8 | a colleague spotted a problem in an analysis step, 9 | that impacted five figures in one of her reports. 10 | Because Amira has embraced GitHub, 11 | the colleague could easily suggest a fix through a pull request. 12 | It took Amira some time to regenerate all of the 13 | affected figures, because she had to recall 14 | which code needed to be rerun. 15 | She recognized this as a great reason to embrace Make, 16 | and has added implementing it to her to-do list. 17 | 18 | She used to be intimidated by the Unix shell, 19 | but now Amira finds it an essential part of her everyday work: 20 | she uses shell scripts to automate data processing tasks, 21 | she runs her own command-line tools she wrote in Python, 22 | and issues countless commands to Git. 23 | Her comfort with the shell is also helping 24 | as she learns how to run her analyses on a remote computing cluster 25 | from Sami. 26 | 27 | Sami had experience with Git and Make in software projects from 28 | their undergraduate studies, 29 | but they have a new appreciation for their importance in research projects. 30 | They've reworked their Git and Make workshops to use examples of data pipelines, 31 | and have been getting rave reviews from participants. 32 | Sami has also gained an appreciation 33 | for the importance of provenance 34 | of both data and code in research. 35 | They've been helping their users 36 | by suggesting ways they can make their research more accessible, inspectable and reproducible. 37 | 38 | Jun is now working on his first Python package. 39 | He's added better error handling so he and his 40 | users get better information when something goes wrong, 41 | he's implemented some testing strategies to give him and his users confidence that his code works, and 42 | he's improved his documentation. 43 | Jun has also added a license, a Code of Conduct and contributing guidelines to his project repo, 44 | and has already had a contribution that fixes some typos in the documentation. 45 | 46 | ## Why We Wrote This Book 47 | 48 | Shell scripts, 49 | branching, 50 | automated workflows, 51 | healthy team dynamics---they all take time to learn, 52 | but they enable researchers to get more done in less time and with less pain, 53 | and that's why we wrote this book. 54 | The climate crisis is the greatest threat our species has faced since civilization began. 55 | The COVID-19 pandemic has shown just how ill-prepared we are to deal with problems of that magnitude, 56 | or with the suspicion and disinformation that now poisons every public discussion online. 57 | 58 | Every hour that a researcher *doesn't* waste wrestling with software 59 | is an hour they can spend solving a new problem; 60 | every meeting that ends early with a clear decision 61 | frees up time to explain to the public what we actually know and why it matters. 62 | We don't expect this book to change the world, 63 | but we hope that knowing how to write, test, document, package, and share your work 64 | will give you a slightly better chance of doing so. 65 | We hope you have enjoyed reading what we have written; 66 | if so, 67 | we would enjoy hearing from you. 68 | 69 | - Damien Irving () 70 | - Kate Hertweck () 71 | - Luke Johnston () 72 | - Joel Ostblom () 73 | - Charlotte Wickham () 74 | - Greg Wilson () 75 | 76 | 77 | > So much universe, and so little time. 78 | > 79 | > --- Terry Pratchett\index{Pratchett, Terry} 80 | -------------------------------------------------------------------------------- /chapters/keypoints.Rmd: -------------------------------------------------------------------------------- 1 | # Key Points {#keypoints} 2 | 3 | This appendix lists the key points for each chapter. 4 | 5 | ## Getting Started 6 | 7 | ```{r, child='keypoints/getting-started.md'} 8 | ``` 9 | 10 | ## The Basics of the Unix Shell 11 | 12 | ```{r, child='keypoints/bash-basics.md'} 13 | ``` 14 | 15 | ## Building Tools with the Unix Shell 16 | 17 | ```{r, child='keypoints/bash-tools.md'} 18 | ``` 19 | 20 | ## Going Further with the Unix Shell 21 | 22 | ```{r, child='keypoints/bash-advanced.md'} 23 | ``` 24 | 25 | ## Building Command-Line Programs in Python 26 | 27 | ```{r, child='keypoints/scripting.md'} 28 | ``` 29 | 30 | ## Using Git at the Command Line 31 | 32 | ```{r, child='keypoints/git-cmdline.md'} 33 | ``` 34 | 35 | ## Going Further with Git 36 | 37 | ```{r, child='keypoints/git-advanced.md'} 38 | ``` 39 | 40 | ## Working in Teams 41 | 42 | ```{r, child='keypoints/teams.md'} 43 | ``` 44 | 45 | ## Automating Analyses with Make 46 | 47 | ```{r, child='keypoints/automate.md'} 48 | ``` 49 | 50 | ## Configuring Programs 51 | 52 | ```{r, child='keypoints/config.md'} 53 | ``` 54 | 55 | ## Testing Software 56 | 57 | ```{r, child='keypoints/testing.md'} 58 | ``` 59 | 60 | ## Handling Errors 61 | 62 | ```{r, child='keypoints/errors.md'} 63 | ``` 64 | 65 | ## Tracking Provenance 66 | 67 | ```{r, child='keypoints/provenance.md'} 68 | ``` 69 | 70 | ## Creating Packages with Python 71 | 72 | ```{r, child='keypoints/packaging.md'} 73 | ``` 74 | -------------------------------------------------------------------------------- /chapters/objectives.Rmd: -------------------------------------------------------------------------------- 1 | # Learning Objectives {#objectives} 2 | 3 | This appendix lists the learning objectives for each chapter, 4 | and is intended to help instructors who want to use this curriculum. 5 | 6 | 7 | 8 | ## Getting Started 9 | 10 | ```{r, child='objectives/getting-started.md'} 11 | ``` 12 | 13 | ## The Basics of the Unix Shell 14 | 15 | ```{r, child='objectives/bash-basics.md'} 16 | ``` 17 | 18 | ## Building Tools with the Unix Shell 19 | 20 | ```{r, child='objectives/bash-tools.md'} 21 | ``` 22 | 23 | ## Going Further with the Unix Shell 24 | 25 | ```{r, child='objectives/bash-advanced.md'} 26 | ``` 27 | 28 | ## Building Command-Line Tools with Python 29 | 30 | ```{r, child='objectives/scripting.md'} 31 | ``` 32 | 33 | ## Using Git at the Command Line 34 | 35 | ```{r, child='objectives/git-cmdline.md'} 36 | ``` 37 | 38 | ## Going Further with Git 39 | 40 | ```{r, child='objectives/git-advanced.md'} 41 | ``` 42 | 43 | ## Working in Teams 44 | 45 | ```{r, child='objectives/teams.md'} 46 | ``` 47 | 48 | ## Automating Analyses with Make 49 | 50 | ```{r, child='objectives/automate.md'} 51 | ``` 52 | 53 | ## Configuring Programs 54 | 55 | ```{r, child='objectives/config.md'} 56 | ``` 57 | 58 | ## Testing Software 59 | 60 | ```{r, child='objectives/testing.md'} 61 | ``` 62 | 63 | ## Handling Errors 64 | 65 | ```{r, child='objectives/errors.md'} 66 | ``` 67 | 68 | ## Tracking Provenance 69 | 70 | ```{r, child='objectives/provenance.md'} 71 | ``` 72 | 73 | ## Creating Packages with Python 74 | 75 | ```{r, child='objectives/packaging.md'} 76 | ``` 77 | -------------------------------------------------------------------------------- /chapters/yaml.Rmd: -------------------------------------------------------------------------------- 1 | # YAML {#yaml} 2 | 3 | \gref{YAML}{yaml_glossary}\index{YAML} is a way to write nested data structures in plain text 4 | that is often used to specify configuration options for software. 5 | The acronym stands for "YAML Ain't Markup Language," 6 | but that's misleading: 7 | YAML doesn't use tags like HTML, 8 | but can still be quite fussy about what is allowed to appear where. 9 | 10 | Throughout this book we use YAML to 11 | configure our plotting script (Chapter \@ref(config)), 12 | continuous integration with Travis CI (Section \@ref(testing-ci)), 13 | software environment with `conda` (Section \@ref(provenance-code-environment)), 14 | and a documentation website with Read the Docs (Section \@ref(packaging-rtd)). 15 | While you don't need to be an expert in YAML to complete tasks like these, 16 | this appendix outlines some basics that can help make things easier. 17 | 18 | A simple YAML file has one key-value pair on each line 19 | with a colon separating the key from the value: 20 | 21 | ```yaml 22 | project-name: planet earth 23 | purpose: science fair 24 | moons: 1 25 | ``` 26 | 27 | Here, 28 | the keys are `"project-name"`, `"purpose"`, and `"moons"`, 29 | and the values are `"planet earth"`, 30 | `"science fair"`, 31 | and (hopefully) the number 1, 32 | since most YAML implementations try to guess the type of data. 33 | 34 | If we want to create a list of values without keys, 35 | we can write it either using square brackets (like a Python array) 36 | or dashed items (like a Markdown\index{Markdown} list), 37 | so: 38 | 39 | ```yaml 40 | rotation-time: ["1 year", "12 months", "365.25 days"] 41 | ``` 42 | 43 | and: 44 | 45 | ```yaml 46 | rotation-time: 47 | - 1 year 48 | - 12 months 49 | - 365.25 days 50 | ``` 51 | 52 | are equivalent. 53 | (The indentation isn't absolutely required in this case, 54 | but helps make the intention clear.) 55 | If we want to write entire paragraphs, 56 | we can use a marker to show that a value spans multiple lines: 57 | 58 | ```yaml 59 | feedback: | 60 | Neat molten core concept. 61 | Too much water. 62 | Could have used more imaginative ending. 63 | ``` 64 | 65 | We can also add comments using `#` just as we do in many programming languages. 66 | 67 | YAML is easy to understand when used this way, 68 | but it starts to get tricky as soon as sub-lists and sub-keys appear. 69 | For example, 70 | this is part of the YAML configuration file for formatting this book: 71 | 72 | ```yaml 73 | bookdown::gitbook: 74 | highlight: tango 75 | config: 76 | download: ["pdf", "epub"] 77 | toc: 78 | collapse: section 79 | before: | 80 |
  • Merely Useful
    • 81 | sharing: no 82 | ``` 83 | 84 | It corresponds to the following Python data structure: 85 | 86 | ```python 87 | { 88 | 'bookdown::gitbook': { 89 | 'highlight': 'tango', 90 | 'config': { 91 | 'download': [ 92 | 'pdf', 93 | 'epub' 94 | ], 95 | 'toc': { 96 | 'collapse': 'section', 97 | 'before': '
  • Merely Useful
    • \n' 98 | } 99 | 'sharing': False 100 | } 101 | } 102 | } 103 | ``` 104 | -------------------------------------------------------------------------------- /code-review/stop-words.py: -------------------------------------------------------------------------------- 1 | import sys 2 | 3 | 4 | stops = ['a', 'A', 'the', 'The', 'and'] 5 | 6 | 7 | def count(ln): 8 | n = 0 9 | for i in range(len(ln)): 10 | line = ln[i] 11 | stuff = line.split() 12 | for word in stuff: 13 | # print(word) 14 | j = stops.count(word) 15 | if j > 0: 16 | n = n + 1 17 | return n 18 | 19 | 20 | lines = sys.stdin.readlines() 21 | # print('number of lines', len(lines)) 22 | n = count(lines) 23 | print('number', n) 24 | -------------------------------------------------------------------------------- /css/style.css: -------------------------------------------------------------------------------- 1 | p.caption { 2 | color: #777; 3 | margin-top: 10px; 4 | } 5 | p code { 6 | white-space: inherit; 7 | } 8 | pre { 9 | line-height: 1.15; 10 | word-break: normal; 11 | word-wrap: normal; 12 | } 13 | pre code { 14 | white-space: inherit; 15 | } 16 | p.flushright { 17 | text-align: right; 18 | } 19 | blockquote > p:last-child { 20 | text-align: right; 21 | } 22 | blockquote > p:first-child { 23 | text-align: inherit; 24 | } 25 | -------------------------------------------------------------------------------- /favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/favicon.ico -------------------------------------------------------------------------------- /figures/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/.gitkeep -------------------------------------------------------------------------------- /figures/FIXME.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/FIXME.png -------------------------------------------------------------------------------- /figures/anaconda/cloud-windspharm-ajdawson.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/anaconda/cloud-windspharm-ajdawson.png -------------------------------------------------------------------------------- /figures/anaconda/cloud-windspharm-search.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/anaconda/cloud-windspharm-search.png -------------------------------------------------------------------------------- /figures/automate/automate.drawio: -------------------------------------------------------------------------------- 1 | 7V3dc6M2EP9r/HqDwAh4vEsv7UNvpjOZaXuPCiiGBiMPyEncv77CSAYhYRO+3LGVBw8sHxLa/e2udlfKynnYfvyao138g0Q4XdlW9LFyflnZtudD9lsSDhXBDeyKsMmTqCKBmvCU/Is50eLUfRLhQrqREpLSZCcTQ5JlOKQSDeU5eZdveyGp3OoObbBCeApRqlL/SiIaV9S1Y9X033CyiXnLUPT7GYWvm5zsM97cynZejn/V5S0Sr+L3FzGKyHuD5HxfOQ85IbQ62n484LQcWTFq1XOPHVdP3c5xRvs8YMOAd4QexLcfu4/LG6yV8y2m25QdAnaIPxL6d0n+4vKzn+JKRvND41J5+pO/4PSJ1vFK9LVkDzvNSIYrymOSpvz6P5jSAxcHtKeEkUhOY7IhGUp/J2THGyxoTl7xA0lJfuy0Yx3/TlcE08p7X0hGH9E2SUtZ/BPnEcoQJ/OWABv3byhNNhk7CVnvMXvrt2pgcKRISj24nFSQfR7yu5iQcHlF+QZTQfROzGWQwWSL2Qixm3KcIpq8yS0gLryb0301B9kBZ2IHQ0XzbyjdYyGEMKX8i9nxpjzO94zdnM7e2bx0QSDe44Tipx06fvA7Q78sJBK7Q7JNQiEHWsac5cALEwzBYy4uHczk34xzij/Oc0vlAX/ABpAJ7/EhrrSAx1H6XqsAl5PiBvoFmEfxzXcmBqJ9O0gsGJZoq7dHWqO/Q9G61qDVHYvW46Osw+jQuGFHkowWjTf/URJqAQw8SfrW0GoJUPXCWpxOPespYf4QCfPuQ8QmFR93MWW/7qXsq94VRt839T10WurecRZU9956CBjtJhitJhiBhMQKtXcHRlfnefnX0OW2JStzB0ytzOHE7oJ3O7p8TndBJ2LXcReYDpnZX/BGq6jblbFJ5Wc5f8Ht5S9EeMdGHmdhgo3TIDkNIAA9vAZgzeY2DPLh25C8O9RBHeosPadnQB3shToU0oRkBnAS4Jx10ANwcDa8WRfG/dM2sLcJDFNUFCWfGngEN4JHT+eor6/hRQG7PQ0UPJ/Mj/LAEBkyIjRAhBZzpDx7CE/PxdIMU7uYChez055ip2NUKIxmxoy2rKs0pMImqmZSGafSNCYhSr/yC9skispmtOZcFrBOi95mzpwZk0DWnEC1zGLGMb1ldgcgsAG/5mz0s7PTW4ag/z8yzTPH0MS3NvAeoowRnrEBfecM2Gk75HagiZvPBft1oPBMN5cq4uNz1YXnXJDZkG1RFpk5ljzHUli61mhyXzPHcibR5FcrQblhPS4SV7IeD5ZypUTzDZiyDhut2glB15Ztna70ZC6d6g8Kctx7UNEVdYlSUNFeDGGglyH8gV4xg4+p9ZKdmFZ61teEFGdD26Bw0DRlOLds8GydwXNGwnFYZhbI0gXcqTOzg6JP02Rm702E3KuIkDezBIlPbU592aiiJDPxrjNWw21XCNsaw+HPZDgCtUK4FPEnfsrH/KIiAKtmEEyOgUlK4ItSQyYV1tSKoFlbA+7E+dMV+QeLZZTdfkX+byhP0HNaF3HUkRDjD0rIlmdfQGC4CWuogbU7AS+9QWVWJvxxHp/aOqvFMkmuWmdVRZb3hQmCfCK0vGgYxO1ZpsMQsWXyEqpK1ejbvpxuzZB08WZtEd0kAWd/fO39uYDXLatVXYIejp0h9UeomqA3CbvPa9X1kgk7OARs0hTlLteyuLo8vLtY/gYOWF4CDNe0XIPLTQvVgoaunIDxR6RKJkuuZNK4I7PlA+CQQiYz/7sAxECnPjuEYAYgqlUqOUaRia1+Ira6XjC0Ck0MZnoMQl0JClwMg1AtQYlweNxWyKCwqwxF2QNlURSaxW1DYKarQ4H2YjDrV4fyvE/SiN1G8gjnJjx2CYZyPkJTniKyBtPX1U+9D5EpLTjh8WrLlUTzkj9a7HBIjTE8g8J24Ey3Idh888J+Od6yuM/Us0uM8/0W31yNFzPbzj6BGvC8ZplGx/4nd+AYBbr5x+iKwP5yoM4/TA74InYBaINXuwvjTErXETnHkeCVHSJr1R0bUBStqbE64VeXbAzGbufVH79qslFrf/eZ2bdDF8+TQHyKxjSnMLNZYHVB2TVBfMcWWJvEXM4Cd63vNRb4HHK99vpeTTZsPgM8jfdsDPB4+DqWJrLoWEtFFk/NXzDAu5hkB2N8JQi3V3O7KoBnMr6ONc0qhdaKtxaCbWN+++BXG3Qcu5dHf0lQg46mWu/ySoQrlkA7llr2NQy83RtVtxMCxv5243eWzYb7C0O/zWJ3iLKvNPPfdkVZe19BXRBaxPinB7KaSjdu9LVgrItjLQjjfnEssi+XH5MX9hMhaupDW1a5XRfjanLyYmeHkWhmp/U/AKsWd9f/Y835/h8= -------------------------------------------------------------------------------- /figures/automate/collated.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/automate/collated.png -------------------------------------------------------------------------------- /figures/automate/concept.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/automate/concept.pdf -------------------------------------------------------------------------------- /figures/automate/concept.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/automate/concept.png -------------------------------------------------------------------------------- /figures/automate/make-dependency-graph.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/automate/make-dependency-graph.png -------------------------------------------------------------------------------- /figures/bash-basics/exercise-filesystem.drawio: -------------------------------------------------------------------------------- 1 | 7V1bc6o6FP41zpzzgAMk3B61trtz2j29ui/npYMQkVMkNmDV/vqTIIhgtN224C1OZworV7K+fCuulWADnA2n34g9GnzHLgoaquxOG6DTUFUFyjr9xySzVCKb6lziEd9NZbngwX9DWcZUOvZdFBUyxhgHsT8qCh0chsiJCzKbEDwpZuvjoNjqyPbQiuDBsYNM2tRy+U/fjQepXJflPOES+d4ga1zNUnq28+wRPA7TJhsq6CefefLQzipL80cD28WTJRE4b4AzgnE8vxpOz1DAxjcbuXm5izWpiwciKIw/UuAeD3/eGV1X7b2prd7Ty1h7fpJ0MK/m1Q7GKHsOPaAVtnt5Wf1lzHrZdrETSX4YIxLagcSUJymG3tf6NpIM+uQSgo4jOUDtS5quqJqiKwDpZl4BG4Z4lo5+JnRwgEkDtGgi8Xp/0aegf2eNZKCzq7/ZZVK+j8NYSithRUJMhnawSA5QTLsnRSPb8UOPmyVG05g+hZuMG0uXR9NiYkzsMOrTUln5EC0yTDBxi9WvFHeRg4kd+zjklE/6P0kRtdy7fIx0L/2f6MEfeqzZBTizfIql0oaTfPnoDnKoLvJBuJovIk4h0yCO2ZRrMfioF8EAND2MvQCNI0To5IvpWDUdPKRphhn9M24T//fN7PyRXLuzrtfteudX8Akp/77evN7f3N7cvV09Su1zJSZX0dsDgOq9c3mNf6Czx4n0NvrWHZhYvoPhtK9ETvvl5vvjw3+DXgtduXe6O7qwL8eob5pP9xdX2vXt7/spCqM29rDzchF0ZhairVxfjx+f7B/mZP2w9UhZQjvf4+WSncCOosJwtEajAM2R7gzs0ENSiCaBz9RYH5TXImXvob6qE7UwXCorS+WTgR+jB9oAE0+ojUmQOAzonbIotsxwKem9IkJrWBKljPcN4SGKyYxmSVMNPWXf1EKpRnqfzyd1wdCDZZ7PCtqpifEWdeccSy9Smv0TyoUcyi2ND3KpmUpvUdDDk/Nc0E4ENIGNgk9tGRVFMcHPCwMGWabQbTEDSW8TnSSSC591tcMUyMCVtaAsajib45d2AagAQq1DUxITh9y03FqdsC5/RCOKWdSIYliZEV7SiSLzdCJv0Ena4C2mszZvzSzpH6qlKiI8Jg5KSy1bz1JFGixVZJQqim3ioXilogQii1H4BGo0YaiFoRaGWhjq4zTUsMyv+u4NtX7ahtqq01ArynsG9qOW+l2TX7WlNtZaajpFQy4dzQkmcY+wyQl0Nn3XEmYm6FLOjzaRJpXNm9w8u5dmsh34HmWIjoMYpVJBht1WmjD0XTcBN0G0v3YvqYrhbcTGMxlhrd1I8GiPYzx/pi9bzVtaQbUAcADJIwlQGUeYYlkmlmViWSaWZce5LFPkPVyXWZs4t4hPhoHtLS5z8Y9HS/ifV7dpWhy4hYV60cJqKsfCwjotrKHUpe3IHvqnpGtFNj+gbN76vjplq2I5JZZTYjklllPHuZwyS6spHcJdr6YMsQNAUK6gXEG5R0q5K99g94FzeVsABOcKzhWcKzj3GDhX3UPOrc1raFP0+q/olFxJZsltaMjWjt2G2S4z4SSuwnFo7J26a/MS09XYSelatd7XtVmrqnk+4tPZlqNrsGlBK/8UtAN00ITq6mSsapeOvu1+WgsUYaVr9e7SMXlur4NBEQUPmf1iN02oaZngdyKQdZAJOtPl/J3Z8t0tIj4dRMY4XwNMqFnN8v4cA1S6ZQxYhalgFhGlwO2gyUDeVErohE3ZWJl0X4/Vlyv/l3P5fNcbqU9B9+EVuaYm8WzbwUCVCyIO0LYmPL1mwgNbogqUGQ8ataJIhFaFz0n4nITPaX99TivfSv7ETJRDq2b2BaQWlxOXcUVkVTCuYFzBuEfKuCuR1T2gXBFYFZQrKFdQ7rFSbjmwugeUu/HFBF8Zeen7YUIshxR7+dxXmtIJXNNadaZaHF1/ReiFq2regehKVI2J752athVZf1/dilanvtefZP5ifRP06kfIPSl1q6U9E1x1VxVE56qbd5ZZxBnmqRq0mpCuGLKPsqqrqkIOZvaaxc/GWE1QqqjiiANvx9XB4KmyEOsnIMoJsWpatW/l2BhiNcs1fiLEalLJDkOsvO1iB4PVqrlPU3fHfdaWCCuHW00A6kXUxh1pwhUlXFHCFSVcUYfritrtFn8+5YotLoJyBeUKyj1Syi2/I3MfKFfscRGUKyhXUO6RUq6+h5S7cY/Llx5kpSw9i/zlV8wefVhG1T5wtFHnKLyyqIxSW4DdtWP7lHQNS/F1nq5rja8rtQXYD+8U6+dofIeHWPma5oXWTyfeUA43gIJ2Kj7EWv4pl63PsK6c6KruDCsfRAcdsN/HAKvE4pJynYdYtWrOsGrsOax1Ab0dH2c96H0BlXNfpedZV7lvy+OsRmWnWelt/qOY8+z5r4+C8/8B -------------------------------------------------------------------------------- /figures/bash-basics/exercise-filesystem.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/bash-basics/exercise-filesystem.pdf -------------------------------------------------------------------------------- /figures/bash-basics/exercise-filesystem.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/bash-basics/exercise-filesystem.png -------------------------------------------------------------------------------- /figures/bash-basics/man-callouts.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/bash-basics/man-callouts.png -------------------------------------------------------------------------------- /figures/bash-basics/nano-editor.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/bash-basics/nano-editor.png -------------------------------------------------------------------------------- /figures/bash-basics/sample-filesystem.drawio: -------------------------------------------------------------------------------- 1 | 7VzbcuI4EP0aqnYfTFmWr48QwqQ2mcp9ZrMvKWEL44mxiCwC5OtX8gXjC0k2G5wMiEpVcKvVkruPTkumoQOPpstvFM0m34mHw46messOHHQ08YL8n5CsUgmAwE4lPg28TFYIroNnnAnVTDoPPByXFBkhIQtmZaFLogi7rCRDlJJFWW1MwvKoM+TjmuDaRWEu7RqF/GfgsUkmN1W1aDjBgT/JB9fylhFyH3xK5lE2ZEeD4+SVNk9RbizTjyfII4sNETzuwCNKCEvfTZdHOBT+zT2X9htuaV3fEMURe0uHKzL9eWndetroWeuN7h/nxsO9YmqpmScUznF+H2bIDfZHRV/zcS5m2feIGytBxDCNUKiI4CnAMsfGGGHF4neuYN11FRdqY8UwgWYAE0Bs2oUB4Qa2yryfC10SEtqBPd5I/dEf/C7431EncXT+7k/xNuk/JhFTMiOiS0ToFIXr5hAzPj0lniE3iPxGFYaXjN+Fl/hNtKuzZbmRURTFY94r7x/htcKCUK9svtbdwy6hiAUkauifzH+RIWpzdoWPTD/7n8QhmPpi2DU4cz3gaHzgRK/w7qSA6lpP1+t6MXVLShPGxJLrCfhow3ACuz4hfojnMaZ88THuq65LprzNsuO/5n0a3J2vjm/ombe69W9v/eNT/R6Df57On67OL84vn09vlP4xYPQ0fr6GunblnpyRH/joZqE8z77dTmyiXurRcgxit/94/v3m+tdk1MOn3qXpzYboZI7Htn1/NTw1zi7urpY4ivvEJ+7jMBysHMxHOTub39yjH/Ziu9tGtCrhkx81aaluiOK45I7ebBbiFOnuBEU+ViK8CAMRxvagvBUpXx7q9ZhoJXdpoi+XLyYBw9d8ACFe8ByTIHEa8iuw7vaEKVffSnpgTaU8TWEyxYyuuErWwdQz9s0ylG5m18V60tYMPdnk+VwRZSnGX9suOJa/yWj2v1AulJQrKVdSrqTc/aRcyyxTrmZ9PuXqDZRb8Q/2+Mkgu8ThiCyOC0E/EfAG4ZiAHx+4KGaUPKzPDLpQiryeOJPwyyQmiWQYiKkORAAFuPIRwNrCUYpfPgWoQV03BrwlOVVgL+uXTlXM750RAXY5IsBy8nPPRkyA2hQT9YWYZANeEL5qi9HsSvx1rWIiJnPq4qzX5oGlYsio5m6rYogh6mNWM5RAZO2F/4EaQyZqmahlopaJej8Ttf4Fz0bmYSdqp81EDcBrCfatmfrVlL/rTG29lKnLrCaC28hPKeMkj6jFaoWmWM/11TrcoMzU1ktM2ry2N9YxCgOf88PAxYJQuSBHbi9rmAael0CbYj45NEpMCbTNhDcT/xr9ToJGNGckvYGPogi7ghAI63CEDWiEOyMIW+7J5J5M7snknmw/92RA/YKbMqet7DoKokPKroZajrWh1bOr1WZ2zQfbfaQ9xNAhhdp0Xg+102qoQVuhvuUpOT6kWAPVMl4NNmji8N1FW5Z5yG2z3DbLbfOebpurZR6mrn/2rtmSZR6SciXlSsrdU8qtPhr+CpTbVOYhKVdSrqRcSbl7QLm1h8NfgXNlkZTkXMm5knP3lHOr29wv8Hmc1VQktZNH92w6O6QH984bPo+zG0K9u8f2TXVNh1MOp1XKyoBj77AczqqudB2WTby1Gq5m6I116zwIaLWhlkG+CqIPqJezmqqoDgdX7ZZZOsaeVFlardWBoGlAD6s8oPINHUt16ngEbaaefEK7D/av+UEV/Tjg9VC3WvRjt1YJEvN1fUihBqptvBrrxjSzu2A3FYL8Npmf+52u/hYXXd0wcsFdIlChnQsGy039wWrz6gLTgLtSgOVjNhN6JStDC7b5nQ0T6O/epXZVZ+NVNmvoXdWqtba007CbPjvfD5SacJco1dQ0Mi/t4uDLcDacrlNmrV0jGvIR9QJp9gfiu3wjnwzq3/pL6J8I6u27Zo5VW6/F85OI+L1fc69behdMtz892DZs5RRsQr0C+9TkGxcBvyx+yyxVL340Dh7/Cw== -------------------------------------------------------------------------------- /figures/bash-basics/sample-filesystem.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/bash-basics/sample-filesystem.pdf -------------------------------------------------------------------------------- /figures/bash-basics/sample-filesystem.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/bash-basics/sample-filesystem.png -------------------------------------------------------------------------------- /figures/bash-basics/the-shell.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/bash-basics/the-shell.png -------------------------------------------------------------------------------- /figures/bash-tools/pipe.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/bash-tools/pipe.pdf -------------------------------------------------------------------------------- /figures/bash-tools/pipe.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/bash-tools/pipe.png -------------------------------------------------------------------------------- /figures/bash-tools/standard-io.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/bash-tools/standard-io.pdf -------------------------------------------------------------------------------- /figures/bash-tools/standard-io.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/bash-tools/standard-io.png -------------------------------------------------------------------------------- /figures/config/jane-eyre-big-labels.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/config/jane-eyre-big-labels.png -------------------------------------------------------------------------------- /figures/config/jane-eyre-default.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/config/jane-eyre-default.png -------------------------------------------------------------------------------- /figures/errors/exceptions.drawio: -------------------------------------------------------------------------------- 1 | 7VnJbtswFPwaH1Noox0fE6dpgaLokqLLqWAsWiJKiwpFx3K/vnzctNmN6xQOEOeSiPMeKXJmHkXJo3i2rN8IXObveUrYKArSehRfjaIoQYH6C8DGAHGEDJAJmhoobIAb+ptY0PbLVjQlVSdRcs4kLbvgnBcFmcsOhoXg627agrPuXUuckQFwM8dsiH6jqcwNeu6WBfhbQrPc3TkMbGSJXbIFqhynfN2C4tejeCY4l+ZqWc8IA+4cL6bf9Y6on5gghdynw937T6vP6OuXjbj7cPfu9md0LdGZHeUes5VdsBRKqQs7ZblxPEhSq7tc5nLJFBCqy0oK/ovMOONCIQUvVOblgjLWgzCjWaGajCxghHsiJFX8Xlh4SdMUbnK5zqkkNyWewx3XykwKE3xVpAQWEKiWnasagNQ7SQg9tcqShC8JLCgKnB+tGNaNY9tcN9K6jLylqpMQWzNlftyGb3VhKf8H+qMB/ZUkpULC5yrAuCvA+VAAdEwB4l0CKHTMgK6qxEVHifHdCkr2csELebbAS8pMwagpjvGy1DTFcQJaEXZPgOtBRGfPtvXBgmK2O7/CRXVWEUEXAOmAnkelN06YRVDWPqQVPauMpBAsuNbVr0FdZeY/6m/hSLEIqN6gfMuxijSvCrmCa5gdAiKREuOh3NDnOtscNEzUDGPU8xHtCwTrdQlGvCYBSkm1TDEBHOpmu6AA1fUDgaaoOrApLIB0aQHULS4I2fKCYKvAIGKkQE2RARhopLU2U2z7sOdDnrim/BAUoM8cN0QHmwZOWrApSB+atkK2MBslWjFVoB7PWnfvi6qbXtk22PWbzRsY09SmKnlTns7FJ7Fn+iPGU22ayWDTdIJQJ8U651yd0Bqd6KmJFD2xSGjXky05FQWSJ1ZgPFCA1HNSytM5XodbztdHlWAykCDHhX4cB0S9HIrnqkO/FLYcsyfH1OF8oMMtWXBBniv/vTrYshMd9TVzOqAfLyR5tu7vsR9tOS/Fx6TffcvqHJh61JMivYDPVao1Z7iq6LwrBKmp/N66/gF0vZog27yqLX26sXGNQk3f94KG6RYh12766damIwJJB9/GehKoJfCVmJOHS19ikRH5l7xou6QPPDkcJgjDkt53p7tNSHuHj5zCq9Mux0x7VjCrtJ3a39h640x74yS9cQwLg3GU8njTSishodo93bA3X4R6HjUDNo71hD7CxMNPhadj4mhPE8cvJj7YxNNjmDg6YRPHe5o4eTHxwSYOw2O4ePjV+t9dfKghD7L/f3RxsqeLJy8u3t/F/ffF+Bgu3vYZ8VT24smeLp6+uHh/F6Oeix93ohjZ3yJa6c2vEPHrPw== -------------------------------------------------------------------------------- /figures/errors/exceptions.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/errors/exceptions.pdf -------------------------------------------------------------------------------- /figures/errors/exceptions.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/errors/exceptions.png -------------------------------------------------------------------------------- /figures/git-advanced/after-fork.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/git-advanced/after-fork.png -------------------------------------------------------------------------------- /figures/git-advanced/after-sami-pushes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/git-advanced/after-sami-pushes.png -------------------------------------------------------------------------------- /figures/git-advanced/dracula-fit.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/git-advanced/dracula-fit.png -------------------------------------------------------------------------------- /figures/git-advanced/fill-in-pull-request.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/git-advanced/fill-in-pull-request.png -------------------------------------------------------------------------------- /figures/git-advanced/fork-button.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/git-advanced/fork-button.png -------------------------------------------------------------------------------- /figures/git-advanced/new-pull-request.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/git-advanced/new-pull-request.png -------------------------------------------------------------------------------- /figures/git-advanced/open-pull-request-detail.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/git-advanced/open-pull-request-detail.png -------------------------------------------------------------------------------- /figures/git-advanced/open-pull-request.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/git-advanced/open-pull-request.png -------------------------------------------------------------------------------- /figures/git-advanced/pr-changes.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/git-advanced/pr-changes.png -------------------------------------------------------------------------------- /figures/git-advanced/pr-comment-marker.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/git-advanced/pr-comment-marker.png -------------------------------------------------------------------------------- /figures/git-advanced/pr-conflict.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/git-advanced/pr-conflict.png -------------------------------------------------------------------------------- /figures/git-advanced/pr-details.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/git-advanced/pr-details.png -------------------------------------------------------------------------------- /figures/git-advanced/pr-list.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/git-advanced/pr-list.png -------------------------------------------------------------------------------- /figures/git-advanced/pr-successful-merge.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/git-advanced/pr-successful-merge.png -------------------------------------------------------------------------------- /figures/git-advanced/pr-with-comment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/git-advanced/pr-with-comment.png -------------------------------------------------------------------------------- /figures/git-advanced/pr-with-fix.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/git-advanced/pr-with-fix.png -------------------------------------------------------------------------------- /figures/git-advanced/pr-writing-comment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/git-advanced/pr-writing-comment.png -------------------------------------------------------------------------------- /figures/git-advanced/viewing-new-pull-request.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/git-advanced/viewing-new-pull-request.png -------------------------------------------------------------------------------- /figures/git-cmdline/git-remote.drawio: -------------------------------------------------------------------------------- 1 | 7Vxtc5s4EP41nmk/OMO77Y9NnKR37bV3l7u0d19uZCxjGoyIkJO4v/4kkHgVNk4MxgnxTAsrIeB5dle7ktBAv1g9XWMQLH9Dc+gNNGX+NNCnA01T1bFF/2OSTSyhf+NY4mB3zmulghv3J+RChUvX7hyGuYoEIY+4QV5oI9+HNsnJAMboMV9tgbz8XQPgwJLgxgZeWfrNnZMll2qKkhZ8hK6zFLe2RMkKiNpcEC7BHD1mRPrlQL/ACJH4aPV0AT0GnwDm2y+bb97nO+v61z/Ce/D3+ae/vtwO48au9rkkeQcMffLspm3t1rtYbuCfQbAauepyeg9dfonyALw1B4y/K9kIBDFa+3PIGlEH+vnj0iXwJgA2K32kSkNlS7LyeHFIMLqDF8hDmEp85NNq5wvX84RooOmXFvtRec3X4q//ADGBTxlW+WteQ7SCBG9oFV6qC8a40o746WOqAOaEy5YZ7k2htIArnZM0neJKDzi0cpi/XvuL1T3efAnscPYJmNrtNRxODgxzAVFjyn4l+LMlTWE90svgqjJsxwfAVqrCqgRby6O3PZ/RA4dELx8LFoi+fBZ1636NRMEwjJzXB1pBM4KntFC0cua4hD0qb4wexu3l70HFmfu+3JaE1zLrWBcXNWZYo7xhmbpZIn8kId9ointtt105FPPghYAk3ROYiWaVrUAZRgGoMk6qIQPKbAgo/dlAbcX9xUBZeZwMq4yTzJs0hpOxzZlUOY4FWLneJnYdtAisgsgGdd1gBg29B0hcG5RKWO3o36L3UWTeJ3I+FN7Vih3s5WuUvTsURbm6GitVHqemkugVSlKpDEO90G+fScxGpg56U/7F3G02BLvAd9jZSWA8LnfXepsGZlUaWBgAv1bnLDWPkADH9Z3EWrEooAiCjL3Ed2mme64iuLHuWEKmqknYtJqyj1FDbHqIZnC0FMMAhS5B7FGPQeEV/ZtMGqTQPDqF44YoxHCFCOwEh5NJoxyOjs5hjfQyXIKAHdobz6XgYn03srOYhs+zRADsOyci5+ua0GagoCAe9VEZ+PmoI09Okzn/uJDzSziRBtyNcSLilLdMivVMUszGSJENFrwxUozOkdLVLF5X8kipilp29K2mp6osj+9Sfgrm88aT08nk4uLq6oWJU6J09bPTwpiONDuVdfuNZaeqbLTiUOnpkVA+dn6q1kj5j+KMzKIzMo/ujGS5fD2otkP/YqyGSQ+WVSOzVXCq86pueGoxkqgMwYl4bKtCPcpqIEbdJR5ab9dDyzKzHdm1HaPD+MPODLxTIl2g91ekR+/z5Gc1aIV8FEYcVeiHVD0GmllcEGDSV2bSaK47ORMQmBEIVDJlx+zBTPaaJpsq21FXTeoK/p/VjJY2EyOdlKQFMdBJAXdMyWWxUskeLOt7JE9XevDk0uSZU800mYtKakYuXTSzybQwyshj3U3L1EwZ1+GkkHkzUUZ1OX3ZzAMUIY1OE1y346zvj7NwG0zAnjznOpgwch5MHrkPJlGj09SFiLb0OYDjhR2VZh0JK49cCSuoRWmRsuTxd1IW9ywSzqJosIIzXdvCmZ4pa4IzY3/Okpjx5OkaVxuS3oYdRcK81+T1Su613tDokcP5+l1wxVIcMXN8rPBePE5nw7K5u1g0Ho+dR7/6ynCYgafiZL+m6LIUWhamNxagaf2AYLpeqysDgoIDiZHWXsElNbFHhO+iSWKlPE9MTQSGlWu7Ko3vEDNUiW9uimFVsgKy3RkqresLa2K/q3y8/DB9K+7XGMvy43bdr1UCFs4deMNPESZL5CAfeJeptOAn0zqfEQo44D8gIRu+GB2sCcrTAZ9c8p1dfmbys38yJdMn3nJ0ssmc/A6xS98b4pyLzhA7NS/HU6Nk8trerIZojW1Yo98iADtw60o9XpHBulVLMPQAcR+ylV5G+OX58OfV0Pvv+7+mNftu/rhW/FvpevNtbiA/GCIfCymMgPBG2CU+wqtoHUlc7FGtgHjIhkRYLyCrQm2JDIHnOn5cbFO6KN+5YhYEMBKTEZRsIcHADxe0UdG8n46+0O5nnr979vI0RBgW3ppaavK2mjFJj81toz+NOUxwljTgz8K4nWJvWsuHMsB2O8/GOsWCP1QVmT9sarhQah3VcU9vHadiHbPXaR3JXPixbGPfme/eNrpnG/bZq7AFQzr335Q1SD8o6/owVrBm79BwHpWs7Gw3jxqPd3pGUwxhtKIM1V/A9ZOM/SRjP8l4arNW/STjSdHVTzKWOuS5i6k3c5HProIhab2PNvN9tDbRZGOdDc08SjvpfXP7tiO2BST28pWGbNbEyIdsRotLd6XaUJ3N9iFbH7L1IdupxQB9yHZSdPUhW+dCNssq9NG6bJCt1ZCt62sWgnXYRsTGP4FvVxsKOw3pYr+uNrYCkSpDk1uBHAfj5Lt1gfH46EmS9KPpLpkcpp4RYdi41e27OcyB0uZJXiMMyf5e7eZJ0g+2+0SpT5QGfaJ0kpF3nyidFF19olTqkY+dKJX2TJSuRmg3bKveB6wbYZu9hPYdWrPDd0kgB8n7V5o9leK4sVRF2o3knrHRWB/J7d1MH8n1kVwfyfWRXB/J7e6TOxfJtbj5kbyPrt6uJBMeHWIvfb7B65vdSd8y89S3uZO+fHnK1pHXQ1IvNoZ9s9yPrCL3ZbOXcf+M760H3OWmZRlnq1/+Dw== -------------------------------------------------------------------------------- /figures/git-cmdline/git-remote.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/git-cmdline/git-remote.pdf -------------------------------------------------------------------------------- /figures/git-cmdline/git-remote.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/git-cmdline/git-remote.png -------------------------------------------------------------------------------- /figures/git-cmdline/phd-comics.gif: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/git-cmdline/phd-comics.gif -------------------------------------------------------------------------------- /figures/git-cmdline/phd-comics.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/git-cmdline/phd-comics.png -------------------------------------------------------------------------------- /figures/git-cmdline/plot-initial.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/git-cmdline/plot-initial.png -------------------------------------------------------------------------------- /figures/git-cmdline/plot-loglog.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/git-cmdline/plot-loglog.png -------------------------------------------------------------------------------- /figures/git-cmdline/repo-history.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/git-cmdline/repo-history.png -------------------------------------------------------------------------------- /figures/git-cmdline/repo-link.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/git-cmdline/repo-link.png -------------------------------------------------------------------------------- /figures/git-cmdline/staging-area.drawio: -------------------------------------------------------------------------------- 1 | 7VprV+I4GP41nLP7oZzeCx8FxXHUHRXRcb94QpuWrKGpabjUX79J74UgjMKM7iyeI+2bN7fneXJ7Q8voT5enFESTS+JB3NJVb9kyjlu6bpsO/y8MSWYwbTUzBBR5mUmrDEP0AnNj4TZDHowbjowQzFDUNLokDKHLGjZAKVk03XyCm7VGIIBrhqEL8Lr1Hnlsklt1Va0SvkAUTIqq7SJlCgrv3BBPgEcWNZNx0jL6lBCWPU2XfYgFeAUw92fJPb54sk+/XsfPYNQ7v/3rTskKG/xIlrIPFIbszUUPolNkAec0vFXJy/Lu4iIcnSpGVvQc4FkOWN5XlhQIUjILPSgK0VpGbzFBDA4j4IrUBZcMt03YFOfJMaPkCfYJJpRbQhJyt56PMC5MLd3opX/cvmO38u7PIWVwWWM17+YpJFPIaMJd8lSjoDAp3rPXRSUAs3CZ1Lg3CtGCXHRBWXSFK3/IoZXDfGvQk5Gv3D30B31/Pvr7y2V/UHahDrONmcCG8L7X8bafZ6RIUOJ0PB1xB47TskrkT4H4XhD6hMKgJfSYlTemRRpHnQ+83MxbndWUpe2D5GI4WQdkUtObTNrrTJYudSbtQxGpbyQyjkD4diJjBoKUyFUWOaSgRmJWy+ci0fhoJMomvX2QiAlfd3gqhRGJESOiqZ+aOcv5dczdnZ2a4cvUs2N9EfYtfTjo+PkKJ2NuXGUtaPGIGysoZJCGACtiI6LArmXrY9dXHN/3FdPwTUWzfKh0fNM2dFPXgd+p8ypRgpstZEIFNBj/wVvEu6IWX39m2TK5ZHmFZ0jolEsjTcOQ8SYpXA+uGPFr6ZwbxpvtpVyKRKG1KoVREMY+9y9ypkusIIVQr1lqM6MHXUIBQyRczZm2dpHzt9KcVY0XeKNp0BBC4aepZl5rDcWaNko/TeIXU7fhNGFMbBOPhGj0AZ6Y7YCQAMNZDCnfMDIOUdslU542SEzfUwe000We01k4V0AzlKEak2g2oLOb2WXSS2wr1Hvxo+foX/E8Um/BMTG/fLtanHtRHz3O/1FUfIyHxw+e/RDfHT/c4JvnM3ty/Xg1i6ORemM+nivo7jnoTuD1+TX9aiRnM6f3ZIZdqKGn+yTwGArP8fDy5WgzbLyp442zgKBp+/A/3FrrtK3GcNdMyXhXrfXxrhmHGvDmnvenK1tRWwVdzcnn95rdTz+b9rMHY8Bq4q+rxhr+lgz+Q6Hf3Y5+wOGP3glIefoD46JY9VWgLGdFqJZEqJpkYSpXq71D1dkOFT8/RuLRTbCY3qmxXa7jTNsX49IA3KcgVfy3GePFwGJPkB28NbEbqAaEurZbsKXyfZX87fJ9ZXcgI+FguwP7fw4kR92fy4HzG3PwM0CXRnFk4YUPuEq+E948dX2bsq75Q62SUvRlMYGPsEoaut3EydF/8eRgfebJ4VXuP/DkIDszf8DJ4Z3wftTJYYcDDAy9I3HTwd9IBMMm2nCJ2Pfa84NQbNvK346XuYDTl2Sjmptq5xUOkOhG+vZj0xL0Gvctr+rcksi8sFGIAUPz5i2NDP28hiuC0kB5wbPeaa8elqxmITGZURfm+SoW14oyCkqqU1dbq3+axTJAA8jWik3FUcLwdr3sMEXupBe1bTtGQzNmx9ijakTWK0gR7y+kn1lJpmm3i3N2qQC13e2o1Ud/m7Ise1VZu2mJswuSmlskHOLNXbAsW1pPJc2sxL0KVXbYKkOxzaDXzndcpiyq3g4QSwOAG26ydgisvTO8vvVS88BxodULFL27vqg5khFi7mFRQ17UGTvsIuqE0agfPuLvyeiVMLyH5lKm0wg0wCjIg88uFHH5zXHS913ErAjPz6iuxZUhnkOGXLA5aymmVH186z2digeZ2LbpsnnzU5pTqPYQCZYcFHaT4Ea9mWa3uYfaNbKzjzCwVHCb78//q4IDnvebqK1UTam2zk6z2xvExl+rn+tkC2H1oyfj5F8= -------------------------------------------------------------------------------- /figures/git-cmdline/staging-area.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/git-cmdline/staging-area.pdf -------------------------------------------------------------------------------- /figures/git-cmdline/staging-area.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/git-cmdline/staging-area.png -------------------------------------------------------------------------------- /figures/packaging/concept.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/packaging/concept.pdf -------------------------------------------------------------------------------- /figures/packaging/concept.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/packaging/concept.png -------------------------------------------------------------------------------- /figures/packaging/landing-page-original.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/packaging/landing-page-original.png -------------------------------------------------------------------------------- /figures/packaging/landing-page.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/packaging/landing-page.png -------------------------------------------------------------------------------- /figures/packaging/module-countwords.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/packaging/module-countwords.png -------------------------------------------------------------------------------- /figures/packaging/module-index.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/packaging/module-index.png -------------------------------------------------------------------------------- /figures/packaging/rse-package-py.drawio: -------------------------------------------------------------------------------- 1 | 7V1bk5s2FP41O9M+ZAckDPgxe0ky06aT6XbS5inDGtlWgxHFeHedX19hJIyQANkYsHfZhwQOGGSdc75zlXwFb1cvH2MvWn4mPgqugOG/XMG7KwBcw6T/poRtRpjYTkZYxNjPSOae8IB/IkY0GHWDfbQWbkwICRIcicQZCUM0SwSaF8fkWbxtTgLxrZG3QBLhYeYFMvVv7CfLjGobxp7+CeHFkr3Z5uN+9GY/FjHZhOx1VwDOd3/Z5ZXHH8XuXy89nzwXSPD+Ct7GhCTZ0erlFgXpzPJZyz73oeJqPuwYhYnOB4BtsYEkW/7dd8NH6Q3GFbxZJquAHpr0EL3g5B9GTo+/pcfXzoSe5t9jdy3036c8oKchCVFG+YCDgF3/FyXJlvHc2ySEkkicLMmChF7wOyERe986ickPdEsCEu9GBo3dX36Fcya9d07C5IO3wkEqcF9R7Huhx8jsTSad3BsvwIuQnszo/KB4N7Ik3u6+1PWEn37Lvwc9uXsRzrbsTJ5oNvdrsolnbCbBxGWy68ULlHDiNCMiX5BBxqCPiKwQfRO9IUaBl+AnUSQ9JtmL/L49e+kB47Ca27e/PT1svhrvFzeGEVBy9PNT+A4qRCAd2gM7ZSwUpULJLwVfVdKTTTSTH35lP83pybY459mHwEHsWdMZT0pCuKMVxLBJTJVcOo7nFRPfF9/zMT15wYY9NqJIRVFs3aD+z0ucoAd6c3r1mWK9yFRB72dkhWc5A1QaWquKczr1XNkZQyq0upIRTyhO0EutUrGrwGBauBUNz3MB7ScZaVkAeo7bp9dCzqLetBAcpYZdauEs8NbrVIIKimi2UMSpriICa1AA5gPtjfXOyHo+95bbGwZPJQymLu4sxo+UqcAO6DBvHmN6tNgdbWWaJCUU65ISHgsuE0dRGVgl8E2RE1P/9z27sMK+n75GaQBEUay0AWXE7w7NTY7LDM0dBZq7MprDE6A54JHNQR60QgVrFK3Ku67SmrNysOuhpY17rdZo01H43LZ5ap+bffQLwXSEuSBOeODLBNEuSVg2fPahkpDlo9CUO64lezzhkJEyZQ8aRZox99js2f9t0iDz5paOCFNOAeMP9Lwn849//45DnHz/fh3lgERHpnpBEacqbnyTnqbliNjkytjkKrDpFJ4mcOAx2FQDTRbICVn0b9RF/xeBTzUotDv7gmJMZz69+YDI3wbazqd7amjSN12g0iWRtTpGERl9kgN8EnN6PRE0H5gT2S3pN8gE/UQaVWkbs1Hh6lMyQlSxR5diYHF81FDhTvD0c6MiZ47HcGk8ux/etknjlYzHpEEchsvitZWF3lJ79QNt8A1XxN8EaQrw1XtwEmwrJKASyaFjlZBc4cM5Tp9AzoOcAoNn9Pt7OHzVBrkVGy3TKbHR0ksTdMfGHHoO8M/58auovp0873gc3B9QtGlt5dWJA+BYoq9oiU+oSBxIz4GWoRRx/pzsC54iAVEh0fUe5l7S7vfUYZwSXUk5OavzgivPEblHsrrhOV2zGvYVTPTn4HGtE3OHYFBfDp5zeb6eNf2CuD5LKyZ60om6W3Y5D2BC2DMoW5K3qAoHfDJTxAK/zEmaHY4QiQL06xgr1DuZtmh+TSCnfFSRApx05mLKrR8dlAn+vH9/9/n+euW/3SJBK7ExLVhOFvLcfbFOABSS01lwYsnFg0sxPKc0IFA76QPb9hJU+HkGKMkGLBuQ05UV6+egAUTW0RKHL6OuN6STXJGZhmwilIpuVQtOSw9T7mA4Q0V/DMjsx19UwARtb2qd6MvN5AZcAyW6iSrdklSVk1adR4MTCSKqa4lrMk+e6TTLVzZrBVESzzGnmQcXUo1RdhtUNUYLdOY2nHO8ehxQaKHQSdFEOx1pGWr5aOtzACiIlWXoeRzNDyr7q53jkqL1nawTGWRIGOBwBJ824AMnel30HYKPoeWo/ok8P1WvJeW3cadMe4w+a43PakE9n7VcfTgho3vyWSssxgGNLOdmMVpHqe0Yp2hJvHT34NzaVCxtWXDAoLJg9ZKbXKNkExX6lxsxnme8t1TqwjHj3ZS6tEXboEp5K21DZylvZzJCTMcQw8scGp1wA3dF6jmELSEGryIqQGPt4zjf0ijVzBSrt/qtfNhQITQ9IMqgiybrGlm1FL1tJqIlz+QSxRp58WypWl2ZGfg3t8aylZqCqZjJMQ3NRZbd2fkL6Hi/pGXTdQZcCwGGjSxtuSeaIYAyvTPqeYU5NkudqMDR0vPuuqSdvvdH0Nfzs/PL9UP/Yf3yad9VouP2vOh016F+MoP6IpEZ+OHwWy8bhMInHJNwhUJFEWeMrEQoFyMr4FpakVV3PHb73uXo7Fy27k2A9t43rj2kvlt8q4y3bALORxosY+D4Xd4eSYX+EY5G1G9A/dJmSHAKBkZ9Ry8J+2Ub4ZG5h+1bqFuJ79Ck95MbbWnSbRsUYfwd27nm4N1fzg3fHe38rDvslgTT8w3iX4vnpy8L02GDfz7QBntA47xk4wU04ButQtMeY7YY6GnW4Dtk8fl2AZ9hjt0BuprrDOuluxexVa0A4oUA7pLSdvoikUVOw4E5kMAch7Ng46vKLvSdnu8lXkVJdqzSHBsH5BuGNLVe251BgyFx61wQv+DgWRAe4uJdRgygX6EdGCrkCm1Eog19t6pJ4xlTQR4h4YC8j1i4hdC4Vmw4qWrR6A4Ueu72LkBCwXcYdFfCftwFV79Pa9guDVeO/VbEx3M8dmkcFPGVkrz5DzkN1qXh9ryKXFD1ek0/T421LyXms2WrvQmCVF29Vaoc4eM6/W8ek9UVuJW1ONqsdz1YRvLWNpA+afXeUjRi8cpKP9Z8qlfH2dXojACnq2xV4rC7PI8R+qlcbjsm9vYyMCn1xuco29QbDzuTgb4X1xxXtbfMA1f4t4/z+jEbvAtbI8k/rNmYyosoZl6YA8MY7bV0AGEJGQAYPNqbqjr4egCL81RU7azM0Iqq2Et8p6gJGhX1BIo6FdfHgqk1tKJaRj96WWHEc7t9QBnHvDasvIPj0o24ZWgvnx0WG/KBFtfUbB6pIq280B8TNofAgCFmZy2+fXYBBPIldUUUMN2DmUtP9z8hnu2ktP+Vdnj/Pw== -------------------------------------------------------------------------------- /figures/packaging/testpypi.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/packaging/testpypi.png -------------------------------------------------------------------------------- /figures/project/noble.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/project/noble.pdf -------------------------------------------------------------------------------- /figures/project/noble.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/project/noble.png -------------------------------------------------------------------------------- /figures/project/project.drawio: -------------------------------------------------------------------------------- 1 | 7VxNc6M4EP01PpoCBNg+5nPmsLlsqnZnjgrImIlALiE79v76FSCwScuz2TFo4rJySKGWAEvvvVarBUzQXb77wvF69cQSQie+m+wm6H7i+7N5JP9Xhn1jCBd+Y0h5ljQm72B4zv4hyugq6yZLSNlrKBijIlv3jTErChKLng1zzt76zZaM9u+6xikBhucYU2j9O0vEqrEGgXuwfyVZuhJt91TFC45fU842hbrdxEfL+q+pznF7KdW+XOGEvR2Z0MME3XHGRHOU7+4IrUa2HbXmvMcTtd3P5qQQHzkBtb9D7Nuuk0SOhCoyLlYsZQWmDwfrbd0/Ul3BlaWVyKk89OQh2WXiW2V2/FAVv7dVheD7b+qUuvC9bli16wahqisF5uKmglAaClaQ1vaYUdqdn7xrIS1H9T+IEHtFKrwRTJoOPfmDsbX6UaXg7JXcMcp43Xfk1n9dTQt91XbJCvGI84xWjP6L8AQXWJnVnTyJ3i2mWVrIQiz7SORVb5vxrQb1JETKVLINj1UrpRbZ75SoVrOOHlJ0hOVEjqFswgnFItv2r44V/dOu3YED8kDR4AQlPEuJC6FEZIoSvqXEhVACmaIEspS4EEoEpigRWEpcCCVCQ5RQt95iulEXTaeC46IETOnz4G2VCfK8xnUX3uQKo8+NHsgxy7O4AxwMtUSyBUXhe97obwkXZPfz8Ycj256gwu19v/h2WGJEyrQ6Wl20i4VzkAhmxsWp6jp9WnXq1ImgOuemHDZQZ8njK1XmvK9MP4rMSXNupfkppRlAaS4MSTMA0nzJCivN2muFcyc0Js5wlHnTO1JmE7famPZ/STPUxLS+IW1Go6xzLCWGp0RkKhsSAnfNSbmh4lrXOe9cdhCYdNlolHjK6vNMfUZQn8b2NILIhtgXQwpTyalolF0N6yeGp0R0YrYZnhJgHk+wwHYSrxvPzaVEfBtif0ZpzqA0PWMZkdBO4RdDCmNLcRvqXwglIlOh/gxO4cxuavRcqIEJfA5QeMKvRI4OmfgRlTe6feHyKK2OOEk3kgTEiWEdAE4OhniHTk8G7aBDHNoBpmRZXaEa2CzG9EaZ8yxJavegI0OfLif58EGo67Lqkj88zMiFgdpMgzMaAOcFwLlFE2K5pkxMMaUOtzgPImfXM4az558E2qkxsogOgOgcPg+yGAnQyGbEPmfs5C00wZOplJgH/fmPOH+Z+q4HI+3riKG8RfRfQdR8rCwI3FZqmFI6cbm1LncIl+t7uq2psbwuWozhde169dwnZDULVmNP4QWjvG9jOTECJwLXECc0WQw5Bc+mnjf14YsX1zkR6/YjxpqJg1H8to2Wz1fpXKNSU9lnTZbrK6bxnhVse62bhgihNp76DTpt05lWp59Mp+2L870HNo1tHbpAp/dygDmL92lhhdothTRC1S2EBplQYT7xz4eb+6cHJ0/gwnbVeVX9wveoPn/Z2YXxUITwYYLZc0PIiDA8nxGhfQvmc7ru0Ne4blNfHgihn6hykVMXTT0Yk1+H5wYZSe3T1WOFWCF8Ru701m65JnFGyuadX73vZjzFhbwF+Vkj68B/hRftN5eO3XebPhjcfcOdA8dxSpxLVrgVOUpZtMD+4swc9IBFSJOxjiCu7Z7NWSsqT4crEyvCG0StC64cKzLpgiP44rbFBGASRtD9jYYI3Klb4zXh2tXMMkunamJ0ym2qb9BNi/om1m9+gBG+21/PaB6vGG05E8EMvtUozOGHEJKBNCqLh88o1nVHX6pED/8C -------------------------------------------------------------------------------- /figures/provenance/release.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/provenance/release.png -------------------------------------------------------------------------------- /figures/scripting/error-message.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/scripting/error-message.png -------------------------------------------------------------------------------- /figures/scripting/jane-eyre.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/scripting/jane-eyre.png -------------------------------------------------------------------------------- /figures/ssh/ssh.drawio: -------------------------------------------------------------------------------- 1 | 7VjbcpswEP0aP7aDAJvkMfGlfUgnnWYmSR9VtAalAlEhfOnXVwKJm+3UaeLYneTFoz2skNhztLvywBsnq08CZ/EXToANXIesBt5k4LrnPlK/GlhXQOCPKiASlFQQaoAb+hsM6Bi0oATyjqPknEmadcGQpymEsoNhIfiy6zbnrLtqhiPYAG5CzDbRO0pkXKFnQ6fBPwONYrsycsyTBFtnA+QxJnzZgrzpwBsLzmU1SlZjYDp2Ni7VvNmOp/XGBKRynwmZgOtgitHs2r2dfJ1Hdw9XFx9832xOru0XA1EBMCYXMuYRTzGbNuil4EVKQL8WKavxueI8M+ADSLk2bOJCcgXFMmHmKayovG+Nv6ux8zEYGnOixeNYY22NVIr1fdtoT9N2M6+07ESC87jebS6xkBdaGApIeQoWm1HG6unEeoQM5zkNK9C46NdUUdOh2smGgXJeiBAeocA1qsYiAvmIX1BrRp014Amob1TzBDAs6aK7D2xUH9V+jTDUwGjjCToxm1xgVpiVvkHCJZS6Luf3VNRoREd0GVMJNxkuA7FUiaKrh/poaN+QJzrg5TiXgv+sD552nfNUGl0hFY5LzGiUaqJU3EFoB0XSmDMuGn71nBlOKNOKuAVBcIprDhcgJKweZ3Ez6naCZw63yW7uubGXTa5ANgHErTxx5hyIKH83USqwWaGD1OdKfb/sEVLGvR/Gzcja6DOY6zfoWFKVOy8MnFBCyoyxjf+uQnZKoE/4AclEPTKHW8gcbiHTPRSZow0yB+6ISRMFNY5k+ekV9kP0EbVo1/FtHtI+r8Emrx7awis6GLEoeFbZdZ5Xdm1preqnu2fZRd2y6/6t7L5giQz2LJHuMUuk7763UkdupfbVCXKOKhTvmELpHv5/O/tvRCZHTSfBztpP6EIvaGqqQn8V+g55aaprbduyXzeAjEc01QECsSgdbZNQvvFEegTTSr5qh1C34yfTxnvPaxBeNEccqkE4vcRgm8DTvovbv9Q6qeH1xXLiSf7/4NLussXlFVc3aH1dZxTKu9v7he+J6dwfvV46V2bzF275rPU/uDf9Aw== -------------------------------------------------------------------------------- /figures/ssh/ssh.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/ssh/ssh.pdf -------------------------------------------------------------------------------- /figures/ssh/ssh.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/ssh/ssh.png -------------------------------------------------------------------------------- /figures/style/code-review-completed-comment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/style/code-review-completed-comment.png -------------------------------------------------------------------------------- /figures/style/code-review-file-view.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/style/code-review-file-view.png -------------------------------------------------------------------------------- /figures/style/code-review-start-comment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/style/code-review-start-comment.png -------------------------------------------------------------------------------- /figures/style/code-review-writing-comment.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/style/code-review-writing-comment.png -------------------------------------------------------------------------------- /figures/teams/bug-report.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/teams/bug-report.png -------------------------------------------------------------------------------- /figures/teams/effort-impact-matrix.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/teams/effort-impact-matrix.pdf -------------------------------------------------------------------------------- /figures/teams/issue-labels.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/teams/issue-labels.png -------------------------------------------------------------------------------- /figures/teams/lifecycle.drawio: -------------------------------------------------------------------------------- 1 | 7VlNc9owEP01HNMBy0A4BgjpTNNpJ5k2yVGxha1GlhhZgN1fX8mWsI0MJCHgtsAhkVarr33vrWS7BUZRcsPhLPzKfERaTttPWmDccpyB25N/lSHNDZegnRsCjv3c1CkM9/g30kbjNsc+iiuOgjEi8Kxq9BilyBMVG+ScLatuU0aqs85ggCzDvQeJbX3AvghzK3Dbhf0zwkFoZu7qhmfovQSczameruWAafbLmyNohtL+cQh9tiyZwHULjDhjIi9FyQgRFVkTtbzfZEPratkcUfGaDgO9DJGanSNfBkJXKaPy3zDbD1I92rIWiojIYkcWUYLFY6n8pFw+dVWNCp4+6h5ZpWhb7Vm1xQJycaUQK+bLbBNMyKq/bzw8AuMYe7lRu6jZfyEhUk0jOBdMmhgXIQsYheSWsZn2iwVnL2jECOPZdkE7+61aDNjKd8qomMAIE8Xhn4j7kEJt1jN1JF5DSHBA1dLkNpEcdWhjoGGJ2Zx7OsyOJjXkAdJeIDcpAErdNG43iEVIhlE6cESgwIsqU6EmfLDyK1CXBQ18PQn0WhaQzPWg32aIWsyo0mAZYoHuZzDb0FImgSo1KiB7LFKYjTfEWcJoENEM+PDQLxAXKNkaVpO6tDLTakJaFlnAaD0sJQCj531gMFOdxXhsMQJbjG5TYgSWGK9kkAMqIT9NQXb6OxXZP5Qina2KLEh9XVh3C1SL0ki0ItBCr091uP2HAs02O07KkRinpiZjVDSpiml5r6x7Tcm6Aw5Cpc25vnN6uX5PunyZD3zfvfUWtz3xg46n4GGSXpg0UiZRvykSufbZ4GUTnObJ0OBdrWchMYLUkz7nY1qDceEe8Zx2z8n1r7hI9+1kednYids9MilO8PL2XlI4TZGib+XtCU7OOVvn7N7xUvblGYjyTWYnDoe6yGy99JbQuUPQT7M36bzl9IhcwfBZlQJVukMLjJb/LnYWUDVwbsZuTURdG7xBDXjuwcCz3yDZB2HdMVM+7ba8Mdz2wF6b1mu1YZ0Gdnx3sN/YXn1G6Bm+MyzXskl665rKDzLdp/whZW2Y9VQK1gfKN2wNlEG82vQeqNvPhm9HvfYFlWmpf7LedPtpfxhTylcJ1ybPlgTWFKXWufBuUq2TczXw3qSS1eIjY+5efMcF138A -------------------------------------------------------------------------------- /figures/teams/lifecycle.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/teams/lifecycle.pdf -------------------------------------------------------------------------------- /figures/teams/lifecycle.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/teams/lifecycle.png -------------------------------------------------------------------------------- /figures/testing/concept.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/testing/concept.pdf -------------------------------------------------------------------------------- /figures/testing/concept.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/testing/concept.png -------------------------------------------------------------------------------- /figures/testing/python-coverage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/testing/python-coverage.png -------------------------------------------------------------------------------- /figures/testing/rse-correct-concept.drawio: -------------------------------------------------------------------------------- 1 | 7V3blps2FP0aP2YWIAT4ca5NeltZcdo0fdOAbNNi5IIc2/36ChA3gW3GWAI7zayVMQIE1j57n6Ojy0zA42r3Q4TWy1+Ih4OJoXm7CXiaGIYOdI39Skr2eYlhZyWLyPd4WVkw8//FvJDfuNj4Ho5rF1JCAuqv64UuCUPs0loZiiKyrV82J0H9qWu0wI2CmYuCZukX36NLXqprWnniPfYXS/5oB/ITK5RfzAviJfLItlIEnifgMSKEZp9Wu0ccJK2Xt0t238uBs8WLRTikXW74EKz+RLG3e/5x9vBp/2usPf/+9M6w+MvRff6NsccagB+SiC7JgoQoeC5LHyKyCT2cVKuxo/KanwlZs0KdFf6FKd1zNNGGEla0pKuAn8U7n/6R3H4H+dFXXlny+WlXPdjnB6F3n+DJDkMS4qzkxQ8Cfj77HsnLH2wfXhSTTeTiI42iF+gwu8ZkhWm0Z/dFOEDU/1avH3H7WhTXFbd+JD57sqFxMkCTG0JOhdyE8iooihaY8rtKINmHymuURSm8b4DadMYDtX4CavZNKzclh1+r58rb0qOhTIRJUQrZsUY3etpSzQreCjmv9xsKNvxJE8MK2Ps+zElqmqUtWP9sSH7iXZyiec8u0OF6V55knxbJ7884pn64SPU0yqt8jfLTT4gidm7m+jhkrcjPs2+QPTW7qGGMdVPbLn2KZ2uUorBlzqVuVvyL4Yji3XEwm23PbzAdgY+5cm8rOp+L97Ii8ZYmCS54nKHcpvtTsj+93ADFse/WGKZflmFGR4ZNhyQYnKpBrBBRW6mKjghmZ0iYDUk6+kiiKAlaDe0BL9E3nzXWWNXSmNbV0jBhN7WEstTS0RuNIpd78qkXMxpQIYRJyypBjFp6go701AeNc4Akfv4Ukm2YRDIk2FCfhM1QZ6x0tUZHV1sSRh9CViV7Mnt8uN4kFXlp+DlOXHQBF9OedsMFSJNRWbhQHKGMNElyhrXaYsy9AWDVgQGwhTBaCzCmLGDM47HlbfXXFbiyXNlO+jJ7SFc2lUTHZ9bQUWK6KPSCtO8+Th4W6dKCh0MLJDRONMp1dsEvnt8EQLubmrpuW6atQccBNRx1W8An42sj29msFtYrKiIZRWnTfCCihZHxGoXnM/K30E++JsUxjStszCodCxuFnLXVMYqUR0Zw1CleY6dvzJy2becyLLZBvSLlLD6cCu/H4g8hxYsi0B01mW1rZGS22vrtShLgJxnXJRNzwdhU75oGtbR2iNUEp87tqe/4Um6dbUE3B7UFRYNV3Bb0iiWUdnHKFmqWUBrGDdoCHNIW8te8eK/1fr2OyM5fIYrTNFI6XBLiuPCx15eJBWbHDq20TKwuK13+KTc2VjdDSLt/jZPEeXLM0w/XApopgqZ1jJWkZQMdU7HrNb5L32t21NssUh3M90LFxqBg2sE1G0Pfznc/OTclyfksayBDI3P2n+fP5ziqDuGMX8ZtYYqX2Taoo6uUcR1K870uYW21z+H6iCLEXhlfkdN1TDFSsoaOlGSNxnxeRjheksC7okAW5jFQTqY8nVSBZ6oyfWQczun1FD7KZDumvouCtPrIR9c1+6NYhMChgnoTqtbBbABlYSWrh/gSEMTnIq95ypnB5m6CFLIropcNxoYZOJ7tkzUB4czkjdrYMHdVpxMzg8aGYJjlPWqnKNpdoRg0d25cYAKc2aaA71lDG9pq4y7z2I8us2yZl/zyk0zMK84kMhmfYsh2lEAmZrSObEwj8jd+JAGJyl7anMEqFKHAX4QJ/AxNFoKCh0QaE396z0+sfM9Lja1NWOsGeAFtNe166GK3SCtokVZDVugCmsbgktWaNR4f/uwJw8QA8/Tf+LAAzkksbKVQDLMsTt1SNaOrPoK+E9/ah/5F8jkCkp0nAgnD12I9spdPNucBrTbJOuSgJSFSkDm+cTZbQp99aDabxxeFjDdmHY0KmH2jpHb2WtplVECcxKJaBQ73YzvPI2qN4r4sUfItGbIJezCP1eoTdY/OJrrZYM1oyTOpjdbM42Nvw/WiCt3R36Y7F9SVnBCnB0nkBBeWfSeYi5gN7qospnmyKtniMlBv/SoSLl3tDEixMzi9mJ1ZAAxrZfYgVnbWnD5l4VFn8xp2Qw/Y7IRcJvyYZekjlCSLXLSJkySR9rr/7oIPQ8jCm4bZCD4clbEHHKZDc5u79kw7khzK6QMZwnQE4EzPcyGGML5XrCJW5EJgc8oJmVOcr4Bo5kPSbYJuOhEi7o/RtpuQ0kwIbM4RG8OyRekbc3UdmIN9Z88fyFJq5h00Kz81q5gKY7XdF0Ee2M5BFeGt/wkvjtfDOrTDE/54XH+Ni3NU6kbXHf0yKlxcNxxh3fu5SmEPrBSW2mVBb9tBtN0K9RNWOL6ZyVZXJ6f3zYb0W9/ZMmSGUbyJ0t7l1mcyebMdSagJsX7bnIO26Vy6LP9gDTQ0dlzCz01h10hZUq7Ky6FI13erwgMZSCBmIIstit6c6Rb7obZiH9E2rVqF01C97vv0VkOSBlvF5WtTcAegUezOYneym0vBnVvbSLe1HZ97tzu790E3v7Wb7n22cdPFuZqH535YJJG/Gyc/bVmx0ubk86zy5TFpOnnpLly9K+5MEGWu2BYAPdsVT4WKJLtiW5HnPaO7dsW63Nut91sdfHwawZA7c4wPU0eRkhxAlB2WfwEmI3X5h3TA838= -------------------------------------------------------------------------------- /figures/testing/travis-add-repo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/testing/travis-add-repo.png -------------------------------------------------------------------------------- /figures/testing/travis-build-pass.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/testing/travis-build-pass.png -------------------------------------------------------------------------------- /figures/testing/travis-list-repos.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/figures/testing/travis-list-repos.png -------------------------------------------------------------------------------- /glossary/glossary-html.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- Transform `\gref{TERM}{KEY}` in R Markdown files into `TERM`. 3 | -- 4 | function RawInline(el) 5 | if (el.c[1] == "tex") and (string.find(el.c[2], "\\gref{")) then 6 | -- io.stderr:write("FOUND ", el.t, " + ", el.c[1], " + ", el.c[2], "\n") 7 | el = transform(el) 8 | -- io.stderr:write("...BECOMES ", el.t, " + ", el.c[1], " + ", el.c[2], "\n") 9 | end 10 | return el 11 | end 12 | 13 | function RawBlock(el) 14 | if (el.c[1] == "latex") and (string.find(el.c[2], "\\gref{")) then 15 | -- io.stderr:write("FOUND ", el.t, " + ", el.c[1], " + ", el.c[2], "\n") 16 | el = transform(el) 17 | end 18 | return el 19 | end 20 | 21 | function transform(el) 22 | el.c[1] = "html" 23 | for term, id in string.gmatch(el.c[2], "\\gref{(.+)}{(.+)}") do 24 | el.c[2] = "" .. term .. "" 25 | end 26 | return el 27 | end 28 | -------------------------------------------------------------------------------- /glossary/glossary-pdf.lua: -------------------------------------------------------------------------------- 1 | -- 2 | -- Transform `\gref{TERM}{KEY_WITH_UNDERSCORES}` in R Markdown files into `\gref{TERM}{KEY\_WITH\_UNDERSCORES}`. 3 | -- 4 | function RawInline(el) 5 | if (el.t == "RawInline") and (el.c[1] == "tex") and (string.find(el.c[2], "\\gref{")) then 6 | -- io.stderr:write("FOUND RawInline tex ", el.c[1], " + ", el.c[2], "\n") 7 | el.c[2] = string.gsub(el.c[2], "_", "\\_") 8 | -- io.stderr:write("...BECOMES ", el.c[1], " + ", el.c[2], "\n") 9 | end 10 | return el 11 | end 12 | -------------------------------------------------------------------------------- /includes/after_body.tex: -------------------------------------------------------------------------------- 1 | \backmatter 2 | \printindex 3 | -------------------------------------------------------------------------------- /includes/before_body.tex: -------------------------------------------------------------------------------- 1 | % you may need to leave a few empty pages before the dedication page 2 | 3 | %\cleardoublepage\newpage\thispagestyle{empty}\null 4 | %\cleardoublepage\newpage\thispagestyle{empty}\null 5 | %\cleardoublepage\newpage 6 | \thispagestyle{empty} 7 | 8 | \begin{center} 9 | To David Flanders\\ 10 | who taught me so much about growing and sustaining coding communities.\\ 11 | --- Damien\\ 12 | 13 | ~\\ 14 | 15 | To the UofT Coders Group\\ 16 | who taught us much more than we taught them.\\ 17 | --- Luke and Joel\\ 18 | 19 | ~\\ 20 | 21 | To my parents Judy and John\\ 22 | who taught me to love books and everything I can learn from them.\\ 23 | --- Kate\\ 24 | 25 | ~\\ 26 | 27 | To Joshua.\\ 28 | --- Charlotte\\ 29 | 30 | ~\\ 31 | 32 | To Brent Gorda\\ 33 | without whom none of this would have happened.\\ 34 | --- Greg\\ 35 | 36 | ~\\ 37 | 38 | All royalties from this book are being donated to The Carpentries,\\ 39 | an organization that teaches foundational coding and data science skills\\ 40 | to researchers worldwide. 41 | \end{center} 42 | 43 | \setlength{\abovedisplayskip}{-5pt} 44 | \setlength{\abovedisplayshortskip}{-5pt} 45 | -------------------------------------------------------------------------------- /includes/dedication.md: -------------------------------------------------------------------------------- 1 | To David Flanders 2 | who taught me so much about growing and sustaining coding communities. 3 | --- Damien 4 |   5 | To the UofT Coders Group 6 | who taught us much more than we taught them. 7 | --- Luke and Joel 8 |   9 | To my parents Judy and John 10 | who taught me to love books and everything I can learn from them. 11 | --- Kate 12 |   13 | To Joshua. 14 | --- Charlotte 15 |   16 | To Brent Gorda 17 | without whom none of this would have happened. 18 | --- Greg 19 |   20 | All royalties from this book are being donated to The Carpentries, 21 | an organization that teaches foundational coding and data science 22 | skills 23 | to researchers worldwide. 24 | -------------------------------------------------------------------------------- /includes/header.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | -------------------------------------------------------------------------------- /includes/preamble.tex: -------------------------------------------------------------------------------- 1 | \IfFileExists{footnote.sty}{\usepackage{footnote}\makesavenoteenv{longtable}}{} 2 | \setlength{\emergencystretch}{3em} % prevent overfull lines 3 | \usepackage[bf,singlelinecheck=off]{caption} 4 | \usepackage{hyperref} 5 | \usepackage{graphicx} 6 | \usepackage{csquotes} 7 | 8 | 9 | \usepackage[framemethod=default]{mdframed} % Required for creating the definition box 10 | 11 | % temp box 12 | \newmdenv[skipabove=7pt, 13 | skipbelow=7pt, 14 | rightline=false, 15 | leftline=false, 16 | topline=false, 17 | bottomline=false, 18 | innerleftmargin=5pt, 19 | innerrightmargin=5pt, 20 | innertopmargin=0pt, 21 | leftmargin=0cm, 22 | rightmargin=0cm, 23 | linewidth=4pt, 24 | innerbottommargin=0pt]{tBox} 25 | 26 | %\usepackage[bottom]{footmisc} 27 | %\usepackage{etoolbox} 28 | \newtoggle{inshade} 29 | \togglefalse{inshade} 30 | 31 | \pretocmd{\footnote}{\iftoggle{inshade}{\stepcounter{footnote}}{\relax}}{}{} 32 | 33 | 34 | \renewenvironment{quote}{\savenotes\begin{tBox}\begin{shaded*}\toggletrue{inshade}\renewcommand{\thempfootnote}{\arabic{footnote}}}{\end{shaded*}\end{tBox}\togglefalse{inshade} 35 | \spewnotes} 36 | \let\oldhref\href 37 | \renewcommand{\href}[2]{#2\footnote{\url{#1}}} 38 | 39 | \renewcommand{\textfraction}{0.05} 40 | \renewcommand{\topfraction}{0.8} 41 | \renewcommand{\bottomfraction}{0.8} 42 | \renewcommand{\floatpagefraction}{0.75} 43 | 44 | \makeatletter 45 | 46 | \makeatletter 47 | \newenvironment{kframe}{% 48 | \medskip{} 49 | \setlength{\fboxsep}{.8em} 50 | \def\at@end@of@kframe{}% 51 | \ifinner\ifhmode% 52 | \def\at@end@of@kframe{\end{minipage}}% 53 | \begin{minipage}{\columnwidth}% 54 | \fi\fi% 55 | \def\FrameCommand##1{\hskip\@totalleftmargin \hskip-\fboxsep 56 | \colorbox{shadecolor}{##1}\hskip-\fboxsep 57 | % There is no \\@totalrightmargin, so: 58 | \hskip-\linewidth \hskip-\@totalleftmargin \hskip\columnwidth}% 59 | \MakeFramed {\advance\hsize-\width 60 | \@totalleftmargin\z@ \linewidth\hsize 61 | \@setminipage}}% 62 | {\par\unskip\endMakeFramed% 63 | \at@end@of@kframe} 64 | \makeatother 65 | 66 | \renewenvironment{Shaded}{\begin{kframe}}{\end{kframe}} 67 | 68 | \usepackage{makeidx} 69 | \makeindex 70 | 71 | \urlstyle{tt} 72 | 73 | \usepackage[hang,flushmargin]{footmisc} 74 | 75 | \usepackage{amsthm} 76 | \makeatletter 77 | \def\thm@space@setup{% 78 | \thm@preskip=8pt plus 2pt minus 4pt 79 | \thm@postskip=\thm@preskip 80 | } 81 | \makeatother 82 | 83 | % Handle glossary links specially. 84 | \newcommand{\gref}[2]{\hyperlink{#2}{\textbf{#1}}} 85 | 86 | % Allow box-drawing characters 87 | % See https://tex.stackexchange.com/questions/281368/print-box-drawing-characters-with-pdflatex 88 | % and pg 185 of http://tug.ctan.org/info/symbols/comprehensive/symbols-a4.pdf 89 | \usepackage{pmboxdraw} 90 | \usepackage{newunicodechar} 91 | \newunicodechar{│}{\textSFxi} 92 | \newunicodechar{└}{\textSFii} 93 | \newunicodechar{├}{\textSFviii} 94 | \newunicodechar{─}{\textSFx} 95 | 96 | \frontmatter 97 | -------------------------------------------------------------------------------- /keypoints/automate.md: -------------------------------------------------------------------------------- 1 | - [Make][gnu-make] is a widely used build manager. 2 | - A \gref{build manager}{build_manager} re-runs commands to update files that are out of date. 3 | - A \gref{build rule}{build_rule} has \gref{targets}{build_target}, \gref{prerequisites}{prerequisite}, and a \gref{recipe}{build_recipe}. 4 | - A target can be a file or a \gref{phony target}{phony_target} that simply triggers an action. 5 | - When a target is out of date with respect to its prerequisites, Make executes the recipe associated with its rule. 6 | - Make executes as many rules as it needs to when updating files, but always respects prerequisite order. 7 | - Make defines \gref{automatic variables}{automatic_variable} such as `$@` (target), `$^` (all prerequisites), and `$<` (first prerequisite). 8 | - \gref{Pattern rules}{pattern_rule} can use `%` as a placeholder for parts of filenames. 9 | - Makefiles can define variables using `NAME=value`. 10 | - Make also has functions such as `$(wildcard...)` and `$(patsubst...)`. 11 | - Use specially formatted comments to create self-documenting Makefiles. 12 | -------------------------------------------------------------------------------- /keypoints/bash-advanced.md: -------------------------------------------------------------------------------- 1 | - Save commands in files (usually called \gref{shell scripts}{shell_script}) for re-use. 2 | - `bash filename` runs the commands saved in a file. 3 | - `$@` refers to all of a shell script's command-line arguments. 4 | - `$1`, `$2`, etc., refer to the first command-line argument, the second command-line argument, etc. 5 | - Place variables in quotes if the values might have spaces or other special characters in them. 6 | - `find` prints a list of files with specific properties or whose names match patterns. 7 | - `$(command)` inserts a command's output in place. 8 | - `grep` selects lines in files that match patterns. 9 | - Use the `.bashrc` file in your home directory to set shell variables each time the shell runs. 10 | - Use `alias` to create shortcuts for things you type frequently. 11 | -------------------------------------------------------------------------------- /keypoints/bash-basics.md: -------------------------------------------------------------------------------- 1 | - A \gref{shell}{shell} is a program that reads commands and runs other programs. 2 | - The \gref{filesystem}{filesystem} manages information stored on disk. 3 | - Information is stored in files, which are located in directories (folders). 4 | - Directories can also store other directories, which forms a directory tree. 5 | - `pwd` prints the user's \gref{current working directory}{current_working_directory}. 6 | - `/` on its own is the \gref{root directory}{root_directory} of the whole filesystem. 7 | - `ls` prints a list of files and directories. 8 | - An \gref{absolute path}{absolute_path} specifies a location from the root of the filesystem. 9 | - A \gref{relative path}{relative_path} specifies a location in the filesystem starting from the current directory. 10 | - `cd` changes the current working directory. 11 | - `..` means the \gref{parent directory}{parent_directory}. 12 | - `.` on its own means the current directory. 13 | - `mkdir` creates a new directory. 14 | - `cp` copies a file. 15 | - `rm` removes (deletes) a file. 16 | - `mv` moves (renames) a file or directory. 17 | - `*` matches zero or more characters in a filename. 18 | - `?` matches any single character in a filename. 19 | - `wc` counts lines, words, and characters in its inputs. 20 | - `man` displays the manual page for a given command; some commands also have a `--help` option. 21 | -------------------------------------------------------------------------------- /keypoints/bash-tools.md: -------------------------------------------------------------------------------- 1 | - `cat` displays the contents of its inputs. 2 | - `head` displays the first few lines of its input. 3 | - `tail` displays the last few lines of its input. 4 | - `sort` sorts its inputs. 5 | - Use the up-arrow key to scroll up through previous commands to edit and repeat them. 6 | - Use `history` to display recent commands and `!number` to repeat a command by number. 7 | - Every process in Unix has an input channel called \gref{standard input}{stdin} 8 | and an output channel called \gref{standard output}{stdin}. 9 | - `>` redirects a command's output to a file, overwriting any existing content. 10 | - `>>` appends a command's output to a file. 11 | - `<` operator redirects input to a command. 12 | - A \gref{pipe}{pipe_shell} `|` sends the output of the command on the left to the input of the command on the right. 13 | - A `for` loop repeats commands once for every thing in a list. 14 | - Every `for` loop must have a variable to refer to the thing it is currently operating on 15 | and a \gref{body}{loop_body} containing commands to execute. 16 | - Use `$name` or `${name}` to get the value of a variable. 17 | -------------------------------------------------------------------------------- /keypoints/config.md: -------------------------------------------------------------------------------- 1 | - \gref{Overlay configuration}{overlay_configuration} specifies settings for a program in layers, 2 | each of which overrides previous layers. 3 | - Use a system-wide configuration file for general settings. 4 | - Use a user-specific configuration file for personal preferences. 5 | - Use a job-specific configuration file with settings for a particular run. 6 | - Use command-line options to change things that commonly change. 7 | - Use \gref{YAML}{yaml_glossary} or some other standard syntax to write configuration files. 8 | - Save configuration information to make your research \gref{reproducible}{reproducible_research}. 9 | -------------------------------------------------------------------------------- /keypoints/errors.md: -------------------------------------------------------------------------------- 1 | - Signal errors by \gref{raising exceptions}{raise_exception}. 2 | - Use `try`/`except` blocks to \gref{catch}{catch_exception} and handle exceptions. 3 | - Python organizes its standard exceptions in a hierarchy so that programs can catch and handle them selectively. 4 | - "Throw low, catch high," i.e., raise exceptions immediately but handle them at a higher level. 5 | - Write error messages that help users figure out what to do to fix the problem. 6 | - Store error messages in a lookup table to ensure consistency. 7 | - Use a \gref{logging framework}{logging_framework} instead of `print` statements to report program activity. 8 | - Separate logging messages into `DEBUG`, `INFO`, `WARNING`, `ERROR`, and `CRITICAL` levels. 9 | - Use `logging.basicConfig` to define basic logging parameters. 10 | -------------------------------------------------------------------------------- /keypoints/getting-started.md: -------------------------------------------------------------------------------- 1 | - Make tidiness a habit, rather than cleaning up your project files later. 2 | - Include a few standard files in all your projects, such as README, LICENSE, CONTRIBUTING, CONDUCT and CITATION. 3 | - Put runnable code in a `bin/` directory. 4 | - Put raw/original data in a `data/` directory and never modify it. 5 | - Put results in a `results/` directory. This includes cleaned-up data and figures (i.e., everything created using what's in `bin` and `data`). 6 | - Put documentation and manuscripts in a `docs/` directory. 7 | - Refer to The Carpentries [software installation guide][carpentries-install-instructions] if you're having trouble. -------------------------------------------------------------------------------- /keypoints/git-advanced.md: -------------------------------------------------------------------------------- 1 | - Use a \gref{branch-per-feature}{branch_per_feature} workflow to develop new features while leaving the master branch in working order. 2 | - `git branch` creates a new branch. 3 | - `git checkout` switches between branches. 4 | - `git merge` \gref{merges}{git_merge} changes from another branch into the current branch. 5 | - \gref{Conflicts}{git_conflict} occur when files or parts of files are changed in different ways on different branches. 6 | - Version control systems do not allow people to overwrite changes silently; 7 | instead, they highlight conflicts that need to be resolved. 8 | - \gref{Forking}{git_fork} a repository makes a copy of it on a server. 9 | - \gref{Cloning}{git_clone} a repository with `git clone` creates a local copy of a remote repository. 10 | - Create a remote called `upstream` to point to the repository a fork was derived from. 11 | - Create \gref{pull requests}{pull_request} to submit changes from your fork to the upstream repository. 12 | -------------------------------------------------------------------------------- /keypoints/git-cmdline.md: -------------------------------------------------------------------------------- 1 | - Use `git config` with the `--global` option to configure your username, 2 | email address, and other preferences once per machine. 3 | - `git init` initializes a \gref{repository}{repository}. 4 | - Git stores all repository management data in the `.git` subdirectory of the repository's root directory. 5 | - `git status` shows the status of a repository. 6 | - `git add` puts files in the repository's staging area. 7 | - `git commit` saves the staged content as a new commit in the local repository. 8 | - `git log` lists previous commits. 9 | - `git diff` shows the difference between two versions of the repository. 10 | - Synchronize your local repository with a \gref{remote repository}{remote_repository} 11 | on a \gref{forge}{forge} such as [GitHub][github]. 12 | - `git remote` manages bookmarks pointing at remote repositories. 13 | - `git push` copies changes from a local repository to a remote repository. 14 | - `git pull` copies changes from a remote repository to a local repository. 15 | - `git restore` and `git checkout` recover old versions of files. 16 | - The `.gitignore` file tells Git what files to ignore. 17 | -------------------------------------------------------------------------------- /keypoints/packaging.md: -------------------------------------------------------------------------------- 1 | - Use [`setuptools`][setuptools] to build and distribute Python packages. 2 | - Create a directory named `mypackage` containing a `setup.py` script with a subdirectory also called `mypackage` containing the package's source files. 3 | - Use \gref{semantic versioning}{semantic_versioning} for software releases. 4 | - Use a \gref{virtual environment}{virtual_environment} to test how your package installs without disrupting your main Python installation. 5 | - Use [`pip`][pip] to install Python packages. 6 | - The default repository for Python packages is [PyPI][pypi]. 7 | - Use [TestPyPI][testpypi] to test the distribution of your package. 8 | - Use a README file for package-level documentation. 9 | - Use [Sphinx][sphinx] to generate documentation for a package. 10 | - Use [Read the Docs][readthedocs] to host package documentation online. 11 | - Create a \gref{DOI}{doi} for your package using [GitHub's Zenodo integration][github-zenodo-tutorial]. 12 | - Publish details of your package in a software journal so others can cite it. 13 | -------------------------------------------------------------------------------- /keypoints/provenance.md: -------------------------------------------------------------------------------- 1 | - Publish data and code as well as papers. 2 | - Use \gref{DOIs}{doi} to identify reports, datasets, and software releases. 3 | - Use an \gref{ORCID}{orcid} to identify yourself as an author of a report, dataset, or software release. 4 | - Data should be [FAIR][go-fair]: findable, accessible, interoperable, and reusable. 5 | - Put small datasets in version control repositories; store large ones on data sharing sites. 6 | - Describe your software environment, analysis scripts, and data processing steps in \gref{reproducible}{reproducible_research} ways. 7 | - Make your analyses \gref{inspectable}{inspectability} as well as reproducible. 8 | -------------------------------------------------------------------------------- /keypoints/scripting.md: -------------------------------------------------------------------------------- 1 | - Write command-line Python programs that can be run in the \gref{Unix shell}{shell} like other command-line tools. 2 | - If the user does not specify any input files, read from \gref{standard input}{stdin}. 3 | - If the user does not specify any output files, write to \gref{standard output}{stdout}. 4 | - Place all `import` statements at the start of a module. 5 | - Use the value of `__name__` to determine if a file is being run directly or being loaded as a module. 6 | - Use [`argparse`][argparse] to handle command-line arguments in standard ways. 7 | - Use \gref{short options}{short_option} for common controls and \gref{long options}{long_option} for less common or more complicated ones. 8 | - Use \gref{docstrings}{docstring} to document functions and scripts. 9 | - Place functions that are used across multiple scripts in a separate file that those scripts can import. 10 | -------------------------------------------------------------------------------- /keypoints/teams.md: -------------------------------------------------------------------------------- 1 | - Welcome and nurture community members proactively. 2 | - Create an explicit Code of Conduct for your project modeled on the [Contributor Covenant][covenant]. 3 | - Include a license in your project so that it's clear who can do what with the material. 4 | - Create \gref{issues}{issue} for bugs, enhancement requests, and discussions. 5 | - \gref{Label issues}{issue_label} to identify their purpose. 6 | - \gref{Triage}{triage} issues regularly and group them into \gref{milestones}{milestone} to track progress. 7 | - Include contribution guidelines in your project that specify its workflow and its expectations of participants. 8 | - Make rules about \gref{governance}{governance} explicit. 9 | - Use common-sense rules to make project meetings fair and productive. 10 | - Manage conflict between participants rather than hoping it will take care of itself. 11 | -------------------------------------------------------------------------------- /keypoints/testing.md: -------------------------------------------------------------------------------- 1 | - Test software to convince people (including yourself) that software is correct enough 2 | and to make tolerances on "enough" explicit. 3 | - Add \gref{assertions}{assertion} to code so that it checks itself as it runs. 4 | - Write \gref{unit tests}{unit_test} to check individual pieces of code. 5 | - Write \gref{integration tests}{integration_test} to check that those pieces work together correctly. 6 | - Write \gref{regression tests}{regression_testing} to check if things that used to work no longer do. 7 | - A \gref{test framework}{test_framework} finds and runs tests written in a prescribed fashion and reports their results. 8 | - Test \gref{coverage}{code_coverage} is the fraction of lines of code that are executed by a set of tests. 9 | - \gref{Continuous integration}{continuous_integration} re-builds and/or re-tests software every time something changes. 10 | -------------------------------------------------------------------------------- /objectives/automate.md: -------------------------------------------------------------------------------- 1 | - Explain what a \gref{build manager}{build_manager} is and how they aid reproducible research. 2 | - Name and describe the three parts of a \gref{build rule}{build_rule}. 3 | - Write a Makefile that re-runs a multi-stage data analysis. 4 | - Explain and trace how Make chooses an order in which to execute rules. 5 | - Explain what \gref{phony targets}{phony_target} are and define a phony target. 6 | - Explain what \gref{automatic variables}{automatic_variable} are and identify three commonly used automatic variables. 7 | - Write Make rules that use automatic variables. 8 | - Explain why and how to write \gref{pattern rules}{pattern_rule} in a Makefile. 9 | - Write Make rules that use patterns. 10 | - Define variables in a Makefile explicitly and by using functions. 11 | - Make a self-documenting Makefile. 12 | -------------------------------------------------------------------------------- /objectives/bash-advanced.md: -------------------------------------------------------------------------------- 1 | - Write a \gref{shell script}{shell_script} that uses command-line arguments. 2 | - Create pipelines that include shell scripts as well as built-in commands. 3 | - Create and use variables in shell scripts with correct quoting. 4 | - Use `grep` to select lines from text files that match simple patterns. 5 | - Use `find` to find files whose names match simple patterns. 6 | - Edit the `.bashrc` file to change default shell variables. 7 | - Create aliases for commonly used commands. 8 | -------------------------------------------------------------------------------- /objectives/bash-basics.md: -------------------------------------------------------------------------------- 1 | - Explain how the shell relates to the keyboard, the screen, the operating system, and users' programs. 2 | - Explain when and why a \gref{command-line interface}{cli} should be used instead of \gref{graphical user interfaces}{gui}. 3 | - Explain the steps in the shell's \gref{read-evaluate-print loop}{repl}. 4 | - Identify the command, options, and filenames in a command-line call. 5 | - Explain the similarities and differences between files and directories. 6 | - Translate an \gref{absolute path}{absolute_path} into a \gref{relative path}{relative_path} and vice versa. 7 | - Construct absolute and relative paths that identify files and directories. 8 | - Delete, copy, and move files and directories. 9 | -------------------------------------------------------------------------------- /objectives/bash-tools.md: -------------------------------------------------------------------------------- 1 | - \gref{Redirect}{redirection} a command's output to a file. 2 | - Use redirection to process a file instead of keyboard input. 3 | - Construct \gref{pipelines}{pipe_shell} with two or more stages. 4 | - Explain Unix's "small pieces, loosely joined" philosophy. 5 | - Write a loop that applies one or more commands separately to each file in a set of files. 6 | - Trace the values taken on by a loop variable during execution of the loop. 7 | - Explain the difference between a variable's name and its value. 8 | - Demonstrate how to see recently executed commands. 9 | - Re-run recently executed commands without retyping them. -------------------------------------------------------------------------------- /objectives/config.md: -------------------------------------------------------------------------------- 1 | - Explain what \gref{overlay configuration}{overlay_configuration} is. 2 | - Describe the four levels of configuration typically used by robust software. 3 | - Create a configuration file using \gref{YAML}{yaml}. 4 | -------------------------------------------------------------------------------- /objectives/errors.md: -------------------------------------------------------------------------------- 1 | - Explain how to use exceptions to signal and handle errors in programs. 2 | - Write `try`/`except` blocks to \gref{raise}{raise_exception} and \gref{catch}{catch_exception} exceptions. 3 | - Explain what is meant by "throw low, catch high." 4 | - Describe the most common built-in exception types in Python and how they relate to each other. 5 | - Explain what makes a useful error message. 6 | - Create and use a lookup table for common error messages. 7 | - Explain the advantages of using a \gref{logging framework}{logging_framework} rather than `print` statements. 8 | - Describe the five standard logging levels and explain what each should be used for. 9 | - Create, configure, and use a simple logger. 10 | -------------------------------------------------------------------------------- /objectives/getting-started.md: -------------------------------------------------------------------------------- 1 | - Identify the few standard files that should be present in every research software project. 2 | - Explain the typical directory structure used in small and medium-sized data analysis projects. 3 | - Download the required data. 4 | - Install the required software. -------------------------------------------------------------------------------- /objectives/git-advanced.md: -------------------------------------------------------------------------------- 1 | - Explain why \gref{branches}{git_branch} are useful. 2 | - Demonstrate how to create a branch, make changes on that branch, and \gref{merge}{git_merge} those changes back into the original branch. 3 | - Explain what \gref{conflicts}{git_conflict} are and demonstrate how to resolve them. 4 | - Explain what is meant by a \gref{branch-per-feature}{branch_per_feature} workflow. 5 | - Define the terms \gref{fork}{git_fork}, \gref{clone}{git_clone}, \gref{remote}{remote_repository}, and \gref{pull request}{pull_request}. 6 | - Demonstrate how to fork a repository and submit a pull request to the original repository. 7 | -------------------------------------------------------------------------------- /objectives/git-cmdline.md: -------------------------------------------------------------------------------- 1 | - Explain the advantages and disadvantages of using \gref{Git}{git} at the command line. 2 | - Demonstrate how to configure Git on a new computer. 3 | - Create a local Git repository at the command line. 4 | - Demonstrate the modify-add-commit cycle for one or more files. 5 | - Synchronize a local repository with a \gref{remote repository}{remote_repository}. 6 | - Explain what the `HEAD` of a repository is and demonstrate how to use it in commands. 7 | - Identify and use Git commit identifiers. 8 | - Demonstrate how to compare revisions to files in a repository. 9 | - Restore old versions of files in a repository. 10 | - Explain how to use `.gitignore` to ignore files and identify files that are being ignored. 11 | -------------------------------------------------------------------------------- /objectives/packaging.md: -------------------------------------------------------------------------------- 1 | - Create a Python package using [`setuptools`][setuptools]. 2 | - Create and use a \gref{virtual environment}{virtual_environment} to manage Python package installations. 3 | - Install a Python package using [`pip`][pip]. 4 | - Distribute that package via [TestPyPI][testpypi]. 5 | - Write a README file for a Python package. 6 | - Use [Sphinx][sphinx] to create and preview documentation for a package. 7 | - Explain where and how to obtain a \gref{DOI}{doi} for a software release. 8 | - Describe some academic journals that publish software papers. 9 | -------------------------------------------------------------------------------- /objectives/provenance.md: -------------------------------------------------------------------------------- 1 | - Explain what a \gref{DOI}{doi} is and how to get one. 2 | - Explain what an \gref{ORCID}{orcid} is and get one. 3 | - Describe the [FAIR Principles][go-fair] and determine whether a dataset conforms to them. 4 | - Explain where to archive small, medium, and large datasets. 5 | - Describe good practices for archiving analysis code and determine whether a report conforms to them. 6 | - Explain the difference between \gref{reproducibility}{reproducible_research} and \gref{inspectability}{inspectability}. 7 | -------------------------------------------------------------------------------- /objectives/scripting.md: -------------------------------------------------------------------------------- 1 | - Explain the benefits of writing Python programs that can be executed at the command line. 2 | - Create a command-line Python program that respects \gref{Unix shell}{shell} conventions for reading input and writing output. 3 | - Use the [`argparse`][argparse] library to handle command-line arguments in a program. 4 | - Explain how to tell if a module is being run directly or being loaded by another program. 5 | - Write \gref{docstrings}{docstring} for programs and functions. 6 | - Explain the difference between \gref{optional arguments}{optional_argument} and \gref{positional arguments}{positional_argument}. 7 | - Create a module that contains functions used by multiple programs and import that module. 8 | -------------------------------------------------------------------------------- /objectives/teams.md: -------------------------------------------------------------------------------- 1 | - Explain how a project lead can be a good \gref{ally}{ally}. 2 | - Explain the purpose of a Code of Conduct and add one to a project. 3 | - Explain why every project should include a license and add one to a project. 4 | - Describe different kinds of licenses for software and written material. 5 | - Explain what an \gref{issue tracking system}{issue_tracking_system} does and what it should be used for. 6 | - Describe what a well-written issue should contain. 7 | - Explain how to \gref{label}{issue_label} issues to manage work. 8 | - Submit an issue to a project. 9 | - Describe common approaches to prioritizing tasks. 10 | - Describe some common-sense rules for running meetings. 11 | - Explain why every project should include contribution guidelines and add some to a project. 12 | - Explain how to handle conflict between project participants. 13 | -------------------------------------------------------------------------------- /objectives/testing.md: -------------------------------------------------------------------------------- 1 | - Explain three different goals for testing software. 2 | - Add \gref{assertions}{assertion} to a program to check that it is operating correctly. 3 | - Write and run unit tests using `pytest`. 4 | - Determine the \gref{coverage}{code_coverage} of those tests and identify untested portions of code. 5 | - Explain \gref{continuous integration}{continuous_integration} and implement it using [Travis CI][travis-ci]. 6 | - Describe and contrast \gref{test-driven development}{tdd} and checking-driven development. 7 | -------------------------------------------------------------------------------- /py-rse.Rproj: -------------------------------------------------------------------------------- 1 | Version: 1.0 2 | 3 | RestoreWorkspace: No 4 | SaveWorkspace: No 5 | AlwaysSaveHistory: No 6 | 7 | EnableCodeIndexing: Yes 8 | UseSpacesForTab: Yes 9 | NumSpacesForTab: 2 10 | Encoding: UTF-8 11 | 12 | RnwWeave: knitr 13 | LaTeX: pdfLaTeX 14 | 15 | AutoAppendNewline: Yes 16 | StripTrailingWhitespace: Yes 17 | 18 | BuildType: Makefile 19 | -------------------------------------------------------------------------------- /references.Rmd: -------------------------------------------------------------------------------- 1 | # References 2 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | pytest 2 | numpy 3 | pandas 4 | -------------------------------------------------------------------------------- /src/package-py/01/use.py: -------------------------------------------------------------------------------- 1 | from zipf import make_zipf, is_zipf 2 | 3 | generated = make_zipf(5) 4 | print('generated distribution: {}'.format(generated)) 5 | generated[-1] *= 2 6 | print('passes test with default tolerance: {}'.format(is_zipf(generated))) 7 | print('passes test with tolerance of 1.0: {}'.format(is_zipf(generated, rel=1.0))) 8 | -------------------------------------------------------------------------------- /src/package-py/01/zipf.py: -------------------------------------------------------------------------------- 1 | from pytest import approx 2 | 3 | 4 | RELATIVE_ERROR = 0.05 5 | 6 | 7 | def make_zipf(length): 8 | assert length > 0, 'Zipf distribution must have at least one element' 9 | result = [1/(1 + i) for i in range(length)] 10 | return result 11 | 12 | 13 | def is_zipf(hist, rel=RELATIVE_ERROR): 14 | assert len(hist) > 0, 'Cannot test Zipfiness without data' 15 | scaled = [h/hist[0] for h in hist] 16 | perfect = make_zipf(len(hist)) 17 | return scaled == approx(perfect, rel=rel) 18 | -------------------------------------------------------------------------------- /src/package-py/02/use.py: -------------------------------------------------------------------------------- 1 | from zipf import make_zipf, is_zipf 2 | 3 | generated = make_zipf(5) 4 | print('generated distribution: {}'.format(generated)) 5 | generated[-1] *= 2 6 | print('passes test with default tolerance: {}'.format(is_zipf(generated))) 7 | print('passes test with tolerance of 1.0: {}'.format(is_zipf(generated, rel=1.0))) 8 | -------------------------------------------------------------------------------- /src/package-py/02/zipf.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from pytest import approx 3 | 4 | 5 | USAGE = '''zipf num [num...]: are the given values Zipfy?''' 6 | RELATIVE_ERROR = 0.05 7 | 8 | 9 | def make_zipf(length): 10 | assert length > 0, 'Zipf distribution must have at least one element' 11 | result = [1/(1 + i) for i in range(length)] 12 | return result 13 | 14 | 15 | def is_zipf(hist, rel=RELATIVE_ERROR): 16 | assert len(hist) > 0, 'Cannot test Zipfiness without data' 17 | scaled = [h/hist[0] for h in hist] 18 | perfect = make_zipf(len(hist)) 19 | return scaled == approx(perfect, rel=rel) 20 | 21 | 22 | if __name__ == '__main__': 23 | if len(sys.argv) == 1: 24 | print(USAGE) 25 | else: 26 | values = [int(a) for a in sys.argv[1:]] 27 | result = is_zipf(values) 28 | print('{}: {}'.format(result, values)) 29 | sys.exit(0) 30 | -------------------------------------------------------------------------------- /src/package-py/03a/test_zipf.py: -------------------------------------------------------------------------------- 1 | from zipf import make_zipf, is_zipf 2 | 3 | def test_default_tolerance(length=5): 4 | generated = make_zipf(length) 5 | assert is_zipf(generated) 6 | 7 | def test_large_tolerance(length=5, relerror=1.0): 8 | generated = make_zipf(length) 9 | assert is_zipf(generated, rel=relerror) 10 | 11 | def test_robustness_tolerance_one(length=5): 12 | generated = make_zipf(length) 13 | generated[-1] *= 2 14 | assert is_zipf(generated, rel=1.0) -------------------------------------------------------------------------------- /src/package-py/03a/zipf.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from pytest import approx 3 | 4 | 5 | USAGE = '''zipf num [num...]: are the given values Zipfy?''' 6 | RELATIVE_ERROR = 0.05 7 | 8 | 9 | def make_zipf(length): 10 | assert length > 0, 'Zipf distribution must have at least one element' 11 | result = [1/(1 + i) for i in range(length)] 12 | return result 13 | 14 | 15 | def make_noisy_zipf(length, rel=RELATIVE_ERROR): 16 | data = make_zipf(length) 17 | minnoise = 1.0 - rel/2 18 | maxnoise = 1.0 + rel/2 19 | for i in range(length): 20 | data[i] = data[i]*random_uniform(minnoise, maxnoise) 21 | return data 22 | 23 | 24 | def is_zipf(hist, rel=RELATIVE_ERROR): 25 | assert len(hist) > 0, 'Cannot test Zipfiness without data' 26 | scaled = [h/hist[0] for h in hist] 27 | perfect = make_zipf(len(hist)) 28 | return scaled == approx(perfect, rel=rel) 29 | 30 | 31 | def is_unsorted_zipf(hist, rel=RELATIVE_ERROR): 32 | sortedhist = sorted(hist) 33 | return is_zipf(unsorted) 34 | 35 | 36 | if __name__ == '__main__': 37 | if len(sys.argv) == 1: 38 | print(USAGE) 39 | else: 40 | values = [int(a) for a in sys.argv[1:]] 41 | result = is_zipf(values) 42 | print('{}: {}'.format(result, values)) 43 | sys.exit(0) 44 | -------------------------------------------------------------------------------- /src/package-py/03b/test_zipf.py: -------------------------------------------------------------------------------- 1 | import zipfpy.zipf 2 | 3 | def test_default_tolerance(length=5): 4 | generated = zipfpy.zipf.make_zipf(length) 5 | assert zipfpy.zipf.is_zipf(generated) 6 | 7 | def test_large_tolerance(length=5, relerror=1.0): 8 | generated = zipfpy.zipf.make_zipf(length) 9 | assert zipfpy.zipf.is_zipf(generated, rel=relerror) 10 | 11 | def test_robustness_tolerance_one(length=5): 12 | generated = zipfpy.zipf.make_zipf(length) 13 | generated[-1] *= 2 14 | assert zipfpy.zipf.is_zipf(generated, rel=1.0) 15 | -------------------------------------------------------------------------------- /src/package-py/03b/zipfpy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/src/package-py/03b/zipfpy/__init__.py -------------------------------------------------------------------------------- /src/package-py/03b/zipfpy/zipf.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from pytest import approx 3 | from random import uniform 4 | 5 | 6 | USAGE = '''zipf num [num...]: are the given values Zipfy?''' 7 | RELATIVE_ERROR = 0.05 8 | 9 | 10 | def make_zipf(length): 11 | assert length > 0, 'Zipf distribution must have at least one element' 12 | result = [1/(1 + i) for i in range(length)] 13 | return result 14 | 15 | 16 | def make_noisy_zipf(length, rel=RELATIVE_ERROR): 17 | data = make_zipf(length) 18 | minnoise = 1.0 - rel/2 19 | maxnoise = 1.0 + rel/2 20 | for i in range(length): 21 | data[i] = data[i]*uniform(minnoise, maxnoise) 22 | return data 23 | 24 | 25 | def is_zipf(hist, rel=RELATIVE_ERROR): 26 | assert len(hist) > 0, 'Cannot test Zipfiness without data' 27 | scaled = [h/hist[0] for h in hist] 28 | print(scaled) 29 | perfect = make_zipf(len(hist)) 30 | return scaled == approx(perfect, rel=rel) 31 | 32 | 33 | def is_unsorted_zipf(hist, rel=RELATIVE_ERROR): 34 | sortedhist = sorted(hist) 35 | return is_zipf(sortedhist, rel) 36 | 37 | 38 | if __name__ == '__main__': 39 | if len(sys.argv) == 1: 40 | print(USAGE) 41 | else: 42 | values = [int(a) for a in sys.argv[1:]] 43 | result = is_zipf(values) 44 | print('{}: {}'.format(result, values)) 45 | sys.exit(0) 46 | -------------------------------------------------------------------------------- /src/package-py/03c/test_zipf.py: -------------------------------------------------------------------------------- 1 | import zipfpy.generate 2 | import zipfpy.check 3 | 4 | def test_default_tolerance(length=5): 5 | generated = zipfpy.generate.make_zipf(length) 6 | assert zipfpy.check.is_zipf(generated) 7 | 8 | def test_large_tolerance(length=5, relerror=1.0): 9 | generated = zipfpy.generate.make_zipf(length) 10 | assert zipfpy.check.is_zipf(generated, rel=relerror) 11 | 12 | def test_robustness_tolerance_one(length=5): 13 | generated = zipfpy.generate.make_zipf(length) 14 | generated[-1] *= 2 15 | assert zipfpy.check.is_zipf(generated, rel=1.0) 16 | -------------------------------------------------------------------------------- /src/package-py/03c/zipfpy/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/src/package-py/03c/zipfpy/__init__.py -------------------------------------------------------------------------------- /src/package-py/03c/zipfpy/check.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from pytest import approx 3 | from .generate import make_zipf 4 | 5 | 6 | USAGE = '''zipf num [num...]: are the given values Zipfy?''' 7 | RELATIVE_ERROR = 0.05 8 | 9 | 10 | def is_zipf(hist, rel=RELATIVE_ERROR): 11 | assert len(hist) > 0, 'Cannot test Zipfiness without data' 12 | scaled = [h/hist[0] for h in hist] 13 | perfect = make_zipf(len(hist)) 14 | return scaled == approx(perfect, rel=rel) 15 | 16 | 17 | def is_unsorted_zipf(hist, rel=RELATIVE_ERROR): 18 | sortedhist = sorted(hist) 19 | return is_zipf(sortedhist) 20 | 21 | 22 | if __name__ == '__main__': 23 | if len(sys.argv) == 1: 24 | print(USAGE) 25 | else: 26 | values = [int(a) for a in sys.argv[1:]] 27 | result = is_zipf(values) 28 | print('{}: {}'.format(result, values)) 29 | sys.exit(0) 30 | -------------------------------------------------------------------------------- /src/package-py/03c/zipfpy/generate.py: -------------------------------------------------------------------------------- 1 | from random import uniform 2 | 3 | RELATIVE_ERROR = 0.05 4 | 5 | def make_zipf(length): 6 | assert length > 0, 'Zipf distribution must have at least one element' 7 | result = [1/(1 + i) for i in range(length)] 8 | return result 9 | 10 | 11 | def make_noisy_zipf(length, rel=RELATIVE_ERROR): 12 | data = make_zipf(length) 13 | minnoise = 1.0 - rel/2 14 | maxnoise = 1.0 + rel/2 15 | for i in range(length): 16 | data[i] = data[i]*uniform(minnoise, maxnoise) 17 | return data -------------------------------------------------------------------------------- /src/package-py/04/test_zipf.py: -------------------------------------------------------------------------------- 1 | import zipfpy.generate 2 | import zipfpy.check 3 | 4 | def test_default_tolerance(length=5): 5 | generated = zipfpy.generate.make_zipf(length) 6 | assert zipfpy.check.is_zipf(generated) 7 | 8 | def test_large_tolerance(length=5, relerror=1.0): 9 | generated = zipfpy.generate.make_zipf(length) 10 | assert zipfpy.check.is_zipf(generated, rel=relerror) 11 | 12 | def test_robustness_tolerance_one(length=5): 13 | generated = zipfpy.generate.make_zipf(length) 14 | generated[-1] *= 2 15 | assert zipfpy.check.is_zipf(generated, rel=1.0) 16 | -------------------------------------------------------------------------------- /src/package-py/04/zipfpy/__init__.py: -------------------------------------------------------------------------------- 1 | RELATIVE_ERROR = 0.05 2 | -------------------------------------------------------------------------------- /src/package-py/04/zipfpy/check.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from pytest import approx 3 | from .generate import make_zipf 4 | from . import RELATIVE_ERROR 5 | 6 | USAGE = '''zipf num [num...]: are the given values Zipfy?''' 7 | 8 | 9 | def is_zipf(hist, rel=RELATIVE_ERROR): 10 | assert len(hist) > 0, 'Cannot test Zipfiness without data' 11 | scaled = [h/hist[0] for h in hist] 12 | perfect = make_zipf(len(hist)) 13 | return scaled == approx(perfect, rel=rel) 14 | 15 | 16 | def is_unsorted_zipf(hist, rel=RELATIVE_ERROR): 17 | sortedhist = sorted(hist) 18 | return is_zipf(sortedhist) 19 | 20 | 21 | if __name__ == '__main__': 22 | if len(sys.argv) == 1: 23 | print(USAGE) 24 | else: 25 | values = [int(a) for a in sys.argv[1:]] 26 | result = is_zipf(values) 27 | print('{}: {}'.format(result, values)) 28 | sys.exit(0) 29 | -------------------------------------------------------------------------------- /src/package-py/04/zipfpy/generate.py: -------------------------------------------------------------------------------- 1 | from random import uniform 2 | from . import RELATIVE_ERROR 3 | 4 | 5 | def make_zipf(length): 6 | assert length > 0, 'Zipf distribution must have at least one element' 7 | result = [1/(1 + i) for i in range(length)] 8 | return result 9 | 10 | 11 | def make_noisy_zipf(length, rel=RELATIVE_ERROR): 12 | data = make_zipf(length) 13 | minnoise = 1.0 - rel/2 14 | maxnoise = 1.0 + rel/2 15 | for i in range(length): 16 | data[i] = data[i]*uniform(minnoise, maxnoise) 17 | return data 18 | -------------------------------------------------------------------------------- /src/package-py/05/showpath.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | 4 | for directory in sys.path: 5 | print(directory) 6 | -------------------------------------------------------------------------------- /src/package-py/06/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name='zipfpy', 5 | version='0.1', 6 | author='Greg Wilson', 7 | packages=['zipfpy'] 8 | ) 9 | -------------------------------------------------------------------------------- /src/package-py/06/test_zipf.py: -------------------------------------------------------------------------------- 1 | import zipfpy.generate 2 | import zipfpy.check 3 | 4 | def test_default_tolerance(length=5): 5 | generated = zipfpy.generate.make_zipf(length) 6 | assert zipfpy.check.is_zipf(generated) 7 | 8 | def test_large_tolerance(length=5, relerror=1.0): 9 | generated = zipfpy.generate.make_zipf(length) 10 | assert zipfpy.check.is_zipf(generated, rel=relerror) 11 | 12 | def test_robustness_tolerance_one(length=5): 13 | generated = zipfpy.generate.make_zipf(length) 14 | generated[-1] *= 2 15 | assert zipfpy.check.is_zipf(generated, rel=1.0) 16 | -------------------------------------------------------------------------------- /src/package-py/06/zipfpy/__init__.py: -------------------------------------------------------------------------------- 1 | RELATIVE_ERROR = 0.05 2 | -------------------------------------------------------------------------------- /src/package-py/06/zipfpy/check.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from pytest import approx 3 | from .generate import make_zipf 4 | from . import RELATIVE_ERROR 5 | 6 | USAGE = '''zipf num [num...]: are the given values Zipfy?''' 7 | 8 | 9 | def is_zipf(hist, rel=RELATIVE_ERROR): 10 | assert len(hist) > 0, 'Cannot test Zipfiness without data' 11 | scaled = [h/hist[0] for h in hist] 12 | perfect = make_zipf(len(hist)) 13 | return scaled == approx(perfect, rel=rel) 14 | 15 | 16 | def is_unsorted_zipf(hist, rel=RELATIVE_ERROR): 17 | sortedhist = sorted(hist) 18 | return is_zipf(sortedhist) 19 | 20 | 21 | if __name__ == '__main__': 22 | if len(sys.argv) == 1: 23 | print(USAGE) 24 | else: 25 | values = [int(a) for a in sys.argv[1:]] 26 | result = is_zipf(values) 27 | print('{}: {}'.format(result, values)) 28 | sys.exit(0) 29 | -------------------------------------------------------------------------------- /src/package-py/06/zipfpy/generate.py: -------------------------------------------------------------------------------- 1 | from random import uniform 2 | from . import RELATIVE_ERROR 3 | 4 | 5 | def make_zipf(length): 6 | assert length > 0, 'Zipf distribution must have at least one element' 7 | result = [1/(1 + i) for i in range(length)] 8 | return result 9 | 10 | 11 | def make_noisy_zipf(length, rel=RELATIVE_ERROR): 12 | data = make_zipf(length) 13 | minnoise = 1.0 - rel/2 14 | maxnoise = 1.0 + rel/2 15 | for i in range(length): 16 | data[i] = data[i]*uniform(minnoise, maxnoise) 17 | return data 18 | -------------------------------------------------------------------------------- /src/package-py/07/requirements.txt: -------------------------------------------------------------------------------- 1 | atomicwrites==1.3.0 2 | attrs==19.1.0 3 | importlib-metadata==0.18 4 | more-itertools==7.0.0 5 | packaging==19.0 6 | pluggy==0.12.0 7 | py==1.8.0 8 | pyparsing==2.4.0 9 | pytest==4.6.3 10 | six==1.12.0 11 | wcwidth==0.1.7 12 | zipp==0.5.1 13 | -------------------------------------------------------------------------------- /src/package-py/07/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name='zipfpy', 5 | version='0.1', 6 | author='Greg Wilson', 7 | packages=['zipfpy'] 8 | ) 9 | -------------------------------------------------------------------------------- /src/package-py/07/test_zipf.py: -------------------------------------------------------------------------------- 1 | import zipfpy.generate 2 | import zipfpy.check 3 | 4 | def test_default_tolerance(length=5): 5 | generated = zipfpy.generate.make_zipf(length) 6 | assert zipfpy.check.is_zipf(generated) 7 | 8 | def test_large_tolerance(length=5, relerror=1.0): 9 | generated = zipfpy.generate.make_zipf(length) 10 | assert zipfpy.check.is_zipf(generated, rel=relerror) 11 | 12 | def test_robustness_tolerance_one(length=5): 13 | generated = zipfpy.generate.make_zipf(length) 14 | generated[-1] *= 2 15 | assert zipfpy.check.is_zipf(generated, rel=1.0) 16 | -------------------------------------------------------------------------------- /src/package-py/07/zipfpy/__init__.py: -------------------------------------------------------------------------------- 1 | RELATIVE_ERROR = 0.05 2 | -------------------------------------------------------------------------------- /src/package-py/07/zipfpy/check.py: -------------------------------------------------------------------------------- 1 | import sys 2 | from pytest import approx 3 | from .generate import make_zipf 4 | from . import RELATIVE_ERROR 5 | 6 | USAGE = '''zipf num [num...]: are the given values Zipfy?''' 7 | 8 | 9 | def is_zipf(hist, rel=RELATIVE_ERROR): 10 | assert len(hist) > 0, 'Cannot test Zipfiness without data' 11 | scaled = [h/hist[0] for h in hist] 12 | perfect = make_zipf(len(hist)) 13 | return scaled == approx(perfect, rel=rel) 14 | 15 | 16 | def is_unsorted_zipf(hist, rel=RELATIVE_ERROR): 17 | sortedhist = sorted(hist) 18 | return is_zipf(sortedhist) 19 | 20 | 21 | if __name__ == '__main__': 22 | if len(sys.argv) == 1: 23 | print(USAGE) 24 | else: 25 | values = [int(a) for a in sys.argv[1:]] 26 | result = is_zipf(values) 27 | print('{}: {}'.format(result, values)) 28 | sys.exit(0) 29 | -------------------------------------------------------------------------------- /src/package-py/07/zipfpy/generate.py: -------------------------------------------------------------------------------- 1 | from random import uniform 2 | from . import RELATIVE_ERROR 3 | 4 | 5 | def make_zipf(length): 6 | assert length > 0, 'Zipf distribution must have at least one element' 7 | result = [1/(1 + i) for i in range(length)] 8 | return result 9 | 10 | 11 | def make_noisy_zipf(length, rel=RELATIVE_ERROR): 12 | data = make_zipf(length) 13 | minnoise = 1.0 - rel/2 14 | maxnoise = 1.0 + rel/2 15 | for i in range(length): 16 | data[i] = data[i]*uniform(minnoise, maxnoise) 17 | return data 18 | -------------------------------------------------------------------------------- /src/package-py/08/bin/check_zipf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | import sys 3 | import zipfpy.check 4 | 5 | USAGE = '''zipf num [num...]: are the given values Zipfy?''' 6 | 7 | if __name__ == '__main__': 8 | if len(sys.argv) == 1: 9 | print(USAGE) 10 | else: 11 | values = [int(a) for a in sys.argv[1:]] 12 | result = zipfpy.check.is_zipf(values) 13 | print('{}: {}'.format(result, values)) 14 | sys.exit(0) 15 | -------------------------------------------------------------------------------- /src/package-py/08/requirements.txt: -------------------------------------------------------------------------------- 1 | atomicwrites==1.3.0 2 | attrs==19.1.0 3 | importlib-metadata==0.18 4 | more-itertools==7.0.0 5 | packaging==19.0 6 | pluggy==0.12.0 7 | py==1.8.0 8 | pyparsing==2.4.0 9 | pytest==4.6.3 10 | six==1.12.0 11 | wcwidth==0.1.7 12 | zipp==0.5.1 13 | -------------------------------------------------------------------------------- /src/package-py/08/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name='zipfpy', 5 | version='0.1', 6 | author='Greg Wilson', 7 | packages=['zipfpy'], 8 | scripts=['bin/check_zipf.py'] 9 | ) 10 | -------------------------------------------------------------------------------- /src/package-py/08/test_zipf.py: -------------------------------------------------------------------------------- 1 | import zipfpy.generate 2 | import zipfpy.check 3 | 4 | def test_default_tolerance(length=5): 5 | generated = zipfpy.generate.make_zipf(length) 6 | assert zipfpy.check.is_zipf(generated) 7 | 8 | def test_large_tolerance(length=5, relerror=1.0): 9 | generated = zipfpy.generate.make_zipf(length) 10 | assert zipfpy.check.is_zipf(generated, rel=relerror) 11 | 12 | def test_robustness_tolerance_one(length=5): 13 | generated = zipfpy.generate.make_zipf(length) 14 | generated[-1] *= 2 15 | assert zipfpy.check.is_zipf(generated, rel=1.0) 16 | -------------------------------------------------------------------------------- /src/package-py/08/zipfpy/__init__.py: -------------------------------------------------------------------------------- 1 | RELATIVE_ERROR = 0.05 2 | -------------------------------------------------------------------------------- /src/package-py/08/zipfpy/check.py: -------------------------------------------------------------------------------- 1 | from pytest import approx 2 | from .generate import make_zipf 3 | from . import RELATIVE_ERROR 4 | 5 | 6 | def is_zipf(hist, rel=RELATIVE_ERROR): 7 | assert len(hist) > 0, 'Cannot test Zipfiness without data' 8 | scaled = [h/hist[0] for h in hist] 9 | perfect = make_zipf(len(hist)) 10 | return scaled == approx(perfect, rel=rel) 11 | 12 | 13 | def is_unsorted_zipf(hist, rel=RELATIVE_ERROR): 14 | sortedhist = sorted(hist) 15 | return is_zipf(sortedhist) 16 | -------------------------------------------------------------------------------- /src/package-py/08/zipfpy/generate.py: -------------------------------------------------------------------------------- 1 | from random import uniform 2 | from . import RELATIVE_ERROR 3 | 4 | 5 | def make_zipf(length): 6 | assert length > 0, 'Zipf distribution must have at least one element' 7 | result = [1/(1 + i) for i in range(length)] 8 | return result 9 | 10 | 11 | def make_noisy_zipf(length, rel=RELATIVE_ERROR): 12 | data = make_zipf(length) 13 | minnoise = 1.0 - rel/2 14 | maxnoise = 1.0 + rel/2 15 | for i in range(length): 16 | data[i] = data[i]*uniform(minnoise, maxnoise) 17 | return data 18 | -------------------------------------------------------------------------------- /src/package-py/09/README.rst: -------------------------------------------------------------------------------- 1 | ====== 2 | Zipfpy 3 | ====== 4 | 5 | This python package provides routines for generating lists of counts 6 | that follow (exactly or approximately) a Zipf distribution, and for 7 | testing whether a counts distribution does or doesn't follow such 8 | a distribution. 9 | 10 | Installation 11 | ------------ 12 | 13 | To install, clone this repository, change into the repository directory 14 | and run the commands:: 15 | 16 | pip install -r requirements.txt 17 | pip install . 18 | 19 | Use 20 | --- 21 | 22 | You can use the package after installation with `import zipfpy`, with functions 23 | in the modules `check` and `generate`. 24 | 25 | You can also use the command line tool `check_zipf.py` which tests to see if 26 | a provided list of countsfollows a Zipf distribution:: 27 | 28 | $ check_zipf.py 100 50 33 25 29 | True: [100, 50, 33, 25] 30 | 31 | Authors 32 | ------- 33 | 34 | Terry Pratchett 35 | -------------------------------------------------------------------------------- /src/package-py/09/bin/check_zipf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | This program checks a list of numbers passed as arguments 4 | to see if they follow a zipf distribution. 5 | """ 6 | import sys 7 | import zipfpy.check 8 | 9 | USAGE = '''zipf num [num...]: are the given values Zipfy?''' 10 | 11 | if __name__ == '__main__': 12 | if len(sys.argv) == 1: 13 | print(USAGE) 14 | else: 15 | values = [int(a) for a in sys.argv[1:]] 16 | result = zipfpy.check.is_zipf(values) 17 | print('{}: {}'.format(result, values)) 18 | sys.exit(0) 19 | -------------------------------------------------------------------------------- /src/package-py/09/requirements.txt: -------------------------------------------------------------------------------- 1 | atomicwrites==1.3.0 2 | attrs==19.1.0 3 | importlib-metadata==0.18 4 | more-itertools==7.0.0 5 | packaging==19.0 6 | pluggy==0.12.0 7 | py==1.8.0 8 | pyparsing==2.4.0 9 | pytest==4.6.3 10 | six==1.12.0 11 | wcwidth==0.1.7 12 | zipp==0.5.1 13 | -------------------------------------------------------------------------------- /src/package-py/09/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name='zipfpy', 5 | version='0.1', 6 | author='Greg Wilson', 7 | packages=['zipfpy'], 8 | scripts=['bin/check_zipf.py'] 9 | ) 10 | -------------------------------------------------------------------------------- /src/package-py/09/test_zipf.py: -------------------------------------------------------------------------------- 1 | import zipfpy.generate 2 | import zipfpy.check 3 | 4 | def test_default_tolerance(length=5): 5 | generated = zipfpy.generate.make_zipf(length) 6 | assert zipfpy.check.is_zipf(generated) 7 | 8 | def test_large_tolerance(length=5, relerror=1.0): 9 | generated = zipfpy.generate.make_zipf(length) 10 | assert zipfpy.check.is_zipf(generated, rel=relerror) 11 | 12 | def test_robustness_tolerance_one(length=5): 13 | generated = zipfpy.generate.make_zipf(length) 14 | generated[-1] *= 2 15 | assert zipfpy.check.is_zipf(generated, rel=1.0) 16 | -------------------------------------------------------------------------------- /src/package-py/09/zipfpy/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | The zipfpy package contains tests of distributions of counts to see if 3 | they follow a Zipf distribution (https://en.wikipedia.org/wiki/Zipf%27s_law) 4 | using the routines in the check module, and routines for generating 5 | lists of counts that follow a Zipf distribution in the generate module. 6 | """ 7 | RELATIVE_ERROR = 0.05 8 | -------------------------------------------------------------------------------- /src/package-py/09/zipfpy/check.py: -------------------------------------------------------------------------------- 1 | """ 2 | The check module contains routines for testing whether a list of counts 3 | represents data that follows Zipf's law. 4 | """ 5 | from pytest import approx 6 | from .generate import make_zipf 7 | from . import RELATIVE_ERROR 8 | 9 | 10 | def is_zipf(hist, rel=RELATIVE_ERROR): 11 | """Tests if a histogram of counts follows a Zipf distribution. 12 | 13 | Given a list of counts as hist, assumed sorted in decreasing order, 14 | and a relative error tolerance (if not provided, the default value 15 | zipfpy.RELATIVE_ERROR is used), tests to see if the counts follow 16 | a Zipf distribution. 17 | 18 | Args: 19 | hist: an list or other iterable containing a list of numeric counts 20 | rel: the relative error tolerance used if provided; if not, 21 | the package default is used. 22 | 23 | Returns: 24 | True if the list of counts follows a zipf distribution within 25 | the relative tolerance. False otherwise. 26 | 27 | Raises: 28 | AssertionError: raised if an empty list is passed. 29 | """ 30 | assert len(hist) > 0, 'Cannot test Zipfiness without data' 31 | scaled = [h/hist[0] for h in hist] 32 | perfect = make_zipf(len(hist)) 33 | return scaled == approx(perfect, rel=rel) 34 | 35 | 36 | def is_unsorted_zipf(hist, rel=RELATIVE_ERROR): 37 | """Tests if a histogram of counts follows a Zipf distribution. 38 | 39 | Given a list of counts as hist, making no assumptions about sorting, 40 | and a relative error tolerance (if not provided, the default value 41 | zipfpy.RELATIVE_ERROR is used), tests to see if the counts follow 42 | a Zipf distribution. 43 | 44 | Args: 45 | hist: an list or other iterable containing a list of numeric counts 46 | rel: the relative error tolerance used if provided; if not, 47 | the package default is used. 48 | 49 | Returns: 50 | True if the list of counts follows a zipf distribution within 51 | the relative tolerance. False otherwise. 52 | 53 | Raises: 54 | AssertionError: raised if an empty list is passed. 55 | """ 56 | sortedhist = sorted(hist) 57 | return is_zipf(sortedhist) 58 | -------------------------------------------------------------------------------- /src/package-py/09/zipfpy/generate.py: -------------------------------------------------------------------------------- 1 | """ 2 | The generate module contains routines for generating ideal and noisy 3 | lists of counts that follow the Zipf distribution. 4 | """ 5 | from random import uniform 6 | from . import RELATIVE_ERROR 7 | 8 | 9 | def make_zipf(length): 10 | """Returns a list of counts that follows a Zipf distribution. 11 | 12 | Args: 13 | length: the number of counts to be generated 14 | 15 | Returns: 16 | A list of the provided length of floating point numbers corresponding 17 | exactly the zipf distribution. For example, for length=5: 18 | 19 | [1.0, 0.5, 0.3333333333333333, 0.25, 0.2] 20 | 21 | Raises: 22 | AssertionError: raised if a zero or negative length is provided 23 | """ 24 | assert length > 0, 'Zipf distribution must have at least one element' 25 | result = [1/(1 + i) for i in range(length)] 26 | return result 27 | 28 | 29 | def make_noisy_zipf(length, rel=RELATIVE_ERROR): 30 | """Returns a list of counts that approximately follows a Zipf distribution. 31 | 32 | As with make_zipf, make_noisy_zipf returns a list of floating point 33 | numbers of the desired length, but one that only approximately follows 34 | the Zipf distribution, with a level of noise controlled by the relative 35 | error parameter rel. 36 | 37 | Args: 38 | length: the number of counts to be generated 39 | rel: the relative error tolerance used if provided; if not, 40 | the package default (zipfpy.RELATIVE_ERROR) is used. 41 | 42 | Returns: 43 | A list of the provided length of floating point numbers approximately 44 | following the zipf distribution. If rel=0, gives the same results as 45 | make_zipf. 46 | 47 | For example, for length=5 and rel=0.0001, a possible result would be: 48 | [0.9999745067631108, 0.5000167385476086, 0.33332704376559663, 49 | 0.2499875253921644, 0.20000910102806616] 50 | 51 | Raises: 52 | AssertionError: raised if a zero or negative length is provided 53 | """ 54 | data = make_zipf(length) 55 | minnoise = 1.0 - rel/2 56 | maxnoise = 1.0 + rel/2 57 | for i in range(length): 58 | data[i] = data[i]*uniform(minnoise, maxnoise) 59 | return data 60 | -------------------------------------------------------------------------------- /src/package-py/10/README.rst: -------------------------------------------------------------------------------- 1 | ====== 2 | Zipfpy 3 | ====== 4 | 5 | This python package provides routines for generating lists of counts 6 | that follow (exactly or approximately) a Zipf distribution, and for 7 | testing whether a counts distribution does or doesn't follow such 8 | a distribution. 9 | 10 | Installation 11 | ------------ 12 | 13 | To install, clone this repository, change into the repository directory 14 | and run the commands:: 15 | 16 | pip install -r requirements.txt 17 | pip install . 18 | 19 | Use 20 | --- 21 | 22 | You can use the package after installation with `import zipfpy`, with functions 23 | in the modules `check` and `generate`. 24 | 25 | You can also use the command line tool `check_zipf.py` which tests to see if 26 | a provided list of countsfollows a Zipf distribution:: 27 | 28 | $ check_zipf.py 100 50 33 25 29 | True: [100, 50, 33, 25] 30 | 31 | Authors 32 | ------- 33 | 34 | Terry Pratchett 35 | -------------------------------------------------------------------------------- /src/package-py/10/bin/check_zipf.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """ 3 | This program checks a list of numbers passed as arguments 4 | to see if they follow a zipf distribution. 5 | """ 6 | import sys 7 | import zipfpy.check 8 | 9 | USAGE = '''zipf num [num...]: are the given values Zipfy?''' 10 | 11 | if __name__ == '__main__': 12 | if len(sys.argv) == 1: 13 | print(USAGE) 14 | else: 15 | values = [int(a) for a in sys.argv[1:]] 16 | result = zipfpy.check.is_zipf(values) 17 | print('{}: {}'.format(result, values)) 18 | sys.exit(0) 19 | -------------------------------------------------------------------------------- /src/package-py/10/docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line. 5 | SPHINXOPTS = 6 | SPHINXBUILD = sphinx-build 7 | SPHINXPROJ = zipfpy 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) -------------------------------------------------------------------------------- /src/package-py/10/docs/index.rst: -------------------------------------------------------------------------------- 1 | .. zipfpy documentation master file, created by 2 | sphinx-quickstart on Mon Aug 12 22:35:06 2019. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | Welcome to zipfpy's documentation! 7 | ================================== 8 | 9 | .. include:: ../README.rst 10 | 11 | .. toctree:: 12 | :maxdepth: 2 13 | :caption: Contents: 14 | 15 | zipfpy 16 | 17 | 18 | Indices and tables 19 | ================== 20 | 21 | * :ref:`genindex` 22 | * :ref:`modindex` 23 | * :ref:`search` 24 | -------------------------------------------------------------------------------- /src/package-py/10/requirements.txt: -------------------------------------------------------------------------------- 1 | atomicwrites==1.3.0 2 | attrs==19.1.0 3 | importlib-metadata==0.18 4 | more-itertools==7.0.0 5 | packaging==19.0 6 | pluggy==0.12.0 7 | py==1.8.0 8 | pyparsing==2.4.0 9 | pytest==4.6.3 10 | six==1.12.0 11 | wcwidth==0.1.7 12 | zipp==0.5.1 13 | -------------------------------------------------------------------------------- /src/package-py/10/requirements_docs.txt: -------------------------------------------------------------------------------- 1 | Sphinx==1.7.4 2 | -------------------------------------------------------------------------------- /src/package-py/10/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | setup( 4 | name='zipfpy', 5 | version='0.1', 6 | author='Greg Wilson', 7 | packages=['zipfpy'], 8 | scripts=['bin/check_zipf.py'] 9 | ) 10 | -------------------------------------------------------------------------------- /src/package-py/10/test_zipf.py: -------------------------------------------------------------------------------- 1 | import zipfpy.generate 2 | import zipfpy.check 3 | 4 | def test_default_tolerance(length=5): 5 | generated = zipfpy.generate.make_zipf(length) 6 | assert zipfpy.check.is_zipf(generated) 7 | 8 | def test_large_tolerance(length=5, relerror=1.0): 9 | generated = zipfpy.generate.make_zipf(length) 10 | assert zipfpy.check.is_zipf(generated, rel=relerror) 11 | 12 | def test_robustness_tolerance_one(length=5): 13 | generated = zipfpy.generate.make_zipf(length) 14 | generated[-1] *= 2 15 | assert zipfpy.check.is_zipf(generated, rel=1.0) 16 | -------------------------------------------------------------------------------- /src/package-py/10/zipfpy/__init__.py: -------------------------------------------------------------------------------- 1 | """ 2 | The zipfpy package contains tests of distributions of counts to see if 3 | they follow a Zipf distribution (https://en.wikipedia.org/wiki/Zipf%27s_law) 4 | using the routines in the check module, and routines for generating 5 | lists of counts that follow a Zipf distribution in the generate module. 6 | """ 7 | RELATIVE_ERROR = 0.05 8 | -------------------------------------------------------------------------------- /src/package-py/10/zipfpy/check.py: -------------------------------------------------------------------------------- 1 | """ 2 | The check module contains routines for testing whether a list of counts 3 | represents data that follows Zipf's Law. 4 | """ 5 | from pytest import approx 6 | from .generate import make_zipf 7 | from . import RELATIVE_ERROR 8 | 9 | 10 | def is_zipf(hist, rel=RELATIVE_ERROR): 11 | """Tests if a histogram of counts follows a Zipf distribution. 12 | 13 | Given a list of counts as hist, assumed sorted in decreasing order, 14 | and a relative error tolerance (if not provided, the default value 15 | zipfpy.RELATIVE_ERROR is used), tests to see if the counts follow 16 | a Zipf distribution. 17 | 18 | Args: 19 | hist: an list or other iterable containing a list of numeric counts 20 | rel: the relative error tolerance used if provided; if not, 21 | the package default is used. 22 | 23 | Returns: 24 | True if the list of counts follows a zipf distribution within 25 | the relative tolerance. False otherwise. 26 | 27 | Raises: 28 | AssertionError: raised if an empty list is passed. 29 | """ 30 | assert len(hist) > 0, 'Cannot test Zipfiness without data' 31 | scaled = [h/hist[0] for h in hist] 32 | perfect = make_zipf(len(hist)) 33 | return scaled == approx(perfect, rel=rel) 34 | 35 | 36 | def is_unsorted_zipf(hist, rel=RELATIVE_ERROR): 37 | """Tests if a histogram of counts follows a Zipf distribution. 38 | 39 | Given a list of counts as hist, making no assumptions about sorting, 40 | and a relative error tolerance (if not provided, the default value 41 | zipfpy.RELATIVE_ERROR is used), tests to see if the counts follow 42 | a Zipf distribution. 43 | 44 | Args: 45 | hist: an list or other iterable containing a list of numeric counts 46 | rel: the relative error tolerance used if provided; if not, 47 | the package default is used. 48 | 49 | Returns: 50 | True if the list of counts follows a zipf distribution within 51 | the relative tolerance. False otherwise. 52 | 53 | Raises: 54 | AssertionError: raised if an empty list is passed. 55 | """ 56 | sortedhist = sorted(hist) 57 | return is_zipf(sortedhist) 58 | -------------------------------------------------------------------------------- /src/package-py/10/zipfpy/generate.py: -------------------------------------------------------------------------------- 1 | """ 2 | The generate module contains routines for generating ideal and noisy 3 | lists of counts that follow the Zipf distribution. 4 | """ 5 | from random import uniform 6 | from . import RELATIVE_ERROR 7 | 8 | 9 | def make_zipf(length): 10 | """Returns a list of counts that follows a Zipf distribution. 11 | 12 | Args: 13 | length: the number of counts to be generated 14 | 15 | Returns: 16 | A list of the provided length of floating point numbers corresponding 17 | exactly the zipf distribution. For example, for length=5: 18 | 19 | [1.0, 0.5, 0.3333333333333333, 0.25, 0.2] 20 | 21 | Raises: 22 | AssertionError: raised if a zero or negative length is provided 23 | """ 24 | assert length > 0, 'Zipf distribution must have at least one element' 25 | result = [1/(1 + i) for i in range(length)] 26 | return result 27 | 28 | 29 | def make_noisy_zipf(length, rel=RELATIVE_ERROR): 30 | """Returns a list of counts that approximately follows a Zipf distribution. 31 | 32 | As with make_zipf, make_noisy_zipf returns a list of floating point 33 | numbers of the desired length, but one that only approximately follows 34 | the Zipf distribution, with a level of noise controlled by the relative 35 | error parameter rel. 36 | 37 | Args: 38 | length: the number of counts to be generated 39 | rel: the relative error tolerance used if provided; if not, 40 | the package default (zipfpy.RELATIVE_ERROR) is used. 41 | 42 | Returns: 43 | A list of the provided length of floating point numbers approximately 44 | following the zipf distribution. If rel=0, gives the same results as 45 | make_zipf. 46 | 47 | For example, for length=5 and rel=0.0001, a possible result would be: 48 | [0.9999745067631108, 0.5000167385476086, 0.33332704376559663, 49 | 0.2499875253921644, 0.20000910102806616] 50 | 51 | Raises: 52 | AssertionError: raised if a zero or negative length is provided 53 | """ 54 | data = make_zipf(length) 55 | minnoise = 1.0 - rel/2 56 | maxnoise = 1.0 + rel/2 57 | for i in range(length): 58 | data[i] = data[i]*uniform(minnoise, maxnoise) 59 | return data 60 | -------------------------------------------------------------------------------- /taylor-francis/abstracts-orcids.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/taylor-francis/abstracts-orcids.pdf -------------------------------------------------------------------------------- /taylor-francis/alt-text/Alternative Text (alt text).pptx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/taylor-francis/alt-text/Alternative Text (alt text).pptx -------------------------------------------------------------------------------- /taylor-francis/alt-text/Author+Guide+to+Writing+Alt+Text.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/taylor-francis/alt-text/Author+Guide+to+Writing+Alt+Text.pdf -------------------------------------------------------------------------------- /taylor-francis/alt-text/SAMPLE_AltTextFigures.doc: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/taylor-francis/alt-text/SAMPLE_AltTextFigures.doc -------------------------------------------------------------------------------- /taylor-francis/back-cover-copy.txt: -------------------------------------------------------------------------------- 1 | # Back Cover Copy 2 | 3 | (1) An introductory paragraph to interest the non-specialist, the general 4 | reader, the librarian or bookseller (about 50 words). 5 | 6 | Writing and running software is now as much a part of science as telescopes and 7 | test tubes, but most researchers are never taught how to do either well. As a 8 | result, it takes them longer to accomplish simple tasks than it should, and it 9 | is harder for them to share their work with others than it needs to be. 10 | 11 | (2) A second factual and informative paragraph which expands on the first by 12 | describing the main item or areas covered by the book (at least 100 words) 13 | 14 | This book introduces the concepts, tools, and skills that researchers need to 15 | get more done in less time and with less pain. Based on the practical 16 | experiences of its authors, who collectively have spent several decades teaching 17 | software skills to scientists, it covers everything graduate-level researchers 18 | need to automate their workflows, collaborate with colleagues, ensure that their 19 | results are trustworthy, and publish what they have built so that others can 20 | build on it. The book assumes only a basic knowledge of Python as a starting 21 | point, and shows readers how it, the Unix shell, Git, Make, and related tools 22 | can give them more time to focus on the research they actually want to do. 23 | 24 | (3) A paragraph which illustrates the main findings and originality of the 25 | research, or specifies its particular usefulness to its intended audience 26 | (about 50 words). 27 | 28 | Research Software Engineering with Python can be used as the main text in a 29 | one-semester course or for self-guided study. A running example shows how to 30 | organize a small research project step by step; over a hundred exercises give 31 | readers a chance to practice these skills themselves, while a glossary defining 32 | over two hundred terms will help readers find their way through the terminology. 33 | All of the material can be re-used under a Creative Commons license, and all 34 | royalties from sales of the book will be donated to The Carpentries, an 35 | organization that teaches foundational coding and data science skills to 36 | researchers worldwide. 37 | -------------------------------------------------------------------------------- /taylor-francis/bios.txt: -------------------------------------------------------------------------------- 1 | Dr. Damien B. Irving is post-doctoral researcher in climate science at the University of New South Wales living in Hobart, Tasmania. 2 | With a strong interest in data science education and open/reproducible research, 3 | Damien is involved in The Carpentries community as an instructor, lesson author and Regional Coordinator for Australia, 4 | is an Associate Editor with the Journal of Open Research Software, 5 | and is currently the Global Coordinator for the Research Bazaar, 6 | a worldwide festival promoting the digital literacy emerging at the center of modern research. 7 | 8 | Dr. Kate L. Hertweck is a scientist and educator who endeavors to uphold core values like 9 | diversity/equity/inclusion, accessibility of information, and learning over knowing. 10 | They currently lead training and community efforts to support biomedical researchers 11 | at Fred Hutchinson Cancer Research Center in Seattle, Washington. 12 | Kate is an instructor and trainer for the Carpentries 13 | and has also participated in that group's lesson development/maintenance and community governance. 14 | 15 | Dr. Luke Johnston is a diabetes epidemiologist working at the Steno 16 | Diabetes Center Aarhus in Denmark. He is passionate about educating 17 | researchers on modern computing tools and skills, having taught many 18 | Carpentry workshops as well as creating and instructing several 19 | intensive courses teaching computing skills and analytic 20 | reproducibility to diabetes researchers. When he isn't teaching or 21 | doing research, he is building software tools to automate common research 22 | workflows and tasks. 23 | 24 | Dr. Joel Ostblom is a post-doctoral teaching fellow 25 | in the Master's of Data Science program 26 | at the University of British Columbia in Vancouver, B.C. 27 | He has co-created or led development of several courses and workshops 28 | at the University of Toronto and the University of British Columbia. 29 | Joel cares deeply about spreading data literacy 30 | and excitement over programmatic data analysis, 31 | which is reflected in his contributions to open source projects 32 | and data science learning resources. 33 | 34 | Dr. Charlotte Wickham is a data scientist and educator, 35 | who teaches in the Statistics Department at Oregon State University, 36 | as well as operating her own consulting and training business. 37 | She loves to help people build their data super powers 38 | in the R programming language. 39 | She currently lives in Corvallis, Oregon, 40 | but originally hails from New Zealand. 41 | 42 | Dr. Greg Wilson is a programmer and educator based in Toronto, Ontario. 43 | He was the co-founder and first Executive Director of Software Carpentry, 44 | and currently leads the instructor training program at RStudio PBC. 45 | A member of the Python Software Foundation, 46 | Greg has written or edited over a dozen books 47 | and received ACM SIGSOFT's Influential Educator Award in 2020. 48 | -------------------------------------------------------------------------------- /taylor-francis/crc-author-questionnaire.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/taylor-francis/crc-author-questionnaire.docx -------------------------------------------------------------------------------- /taylor-francis/permissions-declaration-form.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/taylor-francis/permissions-declaration-form.docx -------------------------------------------------------------------------------- /taylor-francis/permissions-declaration-form.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/taylor-francis/permissions-declaration-form.pdf -------------------------------------------------------------------------------- /taylor-francis/permissions-guidelines/STM_Permissions_Guidelines_2014.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/taylor-francis/permissions-guidelines/STM_Permissions_Guidelines_2014.pdf -------------------------------------------------------------------------------- /taylor-francis/permissions-guidelines/ccc-guidelines.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/taylor-francis/permissions-guidelines/ccc-guidelines.pdf -------------------------------------------------------------------------------- /taylor-francis/permissions-guidelines/tf-figure-image-flow-chart.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/taylor-francis/permissions-guidelines/tf-figure-image-flow-chart.pdf -------------------------------------------------------------------------------- /taylor-francis/permissions-guidelines/tf-permission-flow-chart.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/taylor-francis/permissions-guidelines/tf-permission-flow-chart.pdf -------------------------------------------------------------------------------- /taylor-francis/permissions-guidelines/tf-permission-guidelines-and-faqs.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/taylor-francis/permissions-guidelines/tf-permission-guidelines-and-faqs.pdf -------------------------------------------------------------------------------- /taylor-francis/permissions-request-letter.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/taylor-francis/permissions-request-letter.docx -------------------------------------------------------------------------------- /taylor-francis/proposal-guidelines.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/taylor-francis/proposal-guidelines.pdf -------------------------------------------------------------------------------- /taylor-francis/taylor-francis-contract-2020-08-10.docx: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/taylor-francis/taylor-francis-contract-2020-08-10.docx -------------------------------------------------------------------------------- /taylor-francis/taylor-francis-contract-2020-08-10.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/taylor-francis/taylor-francis-contract-2020-08-10.pdf -------------------------------------------------------------------------------- /taylor-francis/tf-latex-manuscript-preparation-guidelines.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/taylor-francis/tf-latex-manuscript-preparation-guidelines.pdf -------------------------------------------------------------------------------- /tex-packages.txt: -------------------------------------------------------------------------------- 1 | amscls 2 | amsfonts 3 | amsmath 4 | babel 5 | babel-english 6 | biber 7 | biblatex 8 | bibtex 9 | booktabs 10 | bxbase 11 | caption 12 | cbfonts 13 | cbfonts-fd 14 | changepage 15 | cite 16 | cm 17 | colortbl 18 | csquotes 19 | dehyph 20 | dvipdfmx 21 | dvips 22 | ec 23 | enumitem 24 | environ 25 | epigrafica 26 | etex 27 | etoolbox 28 | euenc 29 | fancyhdr 30 | fancyvrb 31 | float 32 | fontspec 33 | footmisc 34 | framed 35 | fvextra 36 | geometry 37 | glyphlist 38 | graphics 39 | graphics-cfg 40 | graphics-def 41 | greek-fontenc 42 | gsftopk 43 | helvetic 44 | hyperref 45 | hyphen-base 46 | ifplatform 47 | iftex 48 | inconsolata 49 | knuth-lib 50 | kpathsea 51 | l3backend 52 | l3kernel 53 | l3packages 54 | lastpage 55 | latex 56 | latex-base-dev 57 | latex-bin 58 | latex-fonts 59 | latexconfig 60 | latexmk 61 | lineno 62 | lipsum 63 | listings 64 | lm 65 | lm-math 66 | logreq 67 | luainputenc 68 | lualibs 69 | luaotfload 70 | luatex 71 | makecell 72 | makeindex 73 | marvosym 74 | mathspec 75 | mdframed 76 | metafont 77 | metapost 78 | mfware 79 | microtype 80 | minitoc 81 | minted 82 | ms 83 | multirow 84 | natbib 85 | needspace 86 | oberdiek 87 | pdftex 88 | pgf 89 | plain 90 | psnfss 91 | pxfonts 92 | ragged2e 93 | rsfs 94 | scheme-infraonly 95 | subfigure 96 | symbol 97 | tabu 98 | tetex 99 | tex 100 | tex-ini-files 101 | tex4ht 102 | texlive.infra 103 | textcase 104 | textgreek 105 | threeparttable 106 | threeparttablex 107 | tikzsymbols 108 | times 109 | tipa 110 | titling 111 | tools 112 | trimspaces 113 | txfonts 114 | ucs 115 | ulem 116 | unicode-data 117 | unicode-math 118 | upquote 119 | url 120 | varwidth 121 | wrapfig 122 | xcolor 123 | xetex 124 | xetexconfig 125 | xkeyval 126 | xstring 127 | xunicode 128 | zapfding 129 | -------------------------------------------------------------------------------- /tugboats-3264x2448.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/tugboats-3264x2448.jpg -------------------------------------------------------------------------------- /tugboats-800x600.jpg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/tugboats-800x600.jpg -------------------------------------------------------------------------------- /unused/manning/manning-contract-2020-08-07.pdf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/unused/manning/manning-contract-2020-08-07.pdf -------------------------------------------------------------------------------- /zipf/.gitignore: -------------------------------------------------------------------------------- 1 | __pycache__ 2 | -------------------------------------------------------------------------------- /zipf/CITATION.md: -------------------------------------------------------------------------------- 1 | # Citation 2 | 3 | If you use the pyzipf package for work/research presented in a 4 | publication, we ask that you please cite: 5 | 6 | Khan A and Virtanen S, 2020. Zipf: A Python package for word 7 | count analysis. *Journal of Important Software*, 5(51), 2317, 8 | https://doi.org/10.21105/jois.02317 9 | 10 | ### BibTeX entry 11 | 12 | @article{Khan2020, 13 | title={Zipf: A Python package for word count analysis.}, 14 | author={Khan, Amira and Virtanen, Sami}, 15 | journal={Journal of Important Software}, 16 | volume={5}, 17 | number={51}, 18 | eid={2317}, 19 | year={2020}, 20 | doi={10.21105/jois.02317}, 21 | url={https://doi.org/10.21105/jois.02317}, 22 | } 23 | -------------------------------------------------------------------------------- /zipf/CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | Thank you for your interest in contributing to the Zipf's Law package! 4 | 5 | If you are new to the package and/or collaborative code development on GitHub, 6 | feel free to discuss any suggested changes via issue or email. 7 | We can then walk you through the pull request process if need be. 8 | As the project grows, 9 | we intend to develop more detailed guidelines for submitting 10 | bug reports and feature requests. 11 | 12 | We also have a code of conduct (see [`CONDUCT.md`](CONDUCT.md)). 13 | Please follow it in all your interactions with the project. 14 | -------------------------------------------------------------------------------- /zipf/KhanVirtanen2020.md: -------------------------------------------------------------------------------- 1 | The code in this repository was used in generating the results 2 | for the following paper: 3 | 4 | Khan A & Virtanen S, 2020. Zipf's Law in classic english texts. 5 | *Journal of Important Research*, 27, 134-139. 6 | 7 | The code was executed in the software environment described by 8 | `environment.yml`. It can be installed using 9 | [conda](https://docs.conda.io/en/latest/): 10 | $ conda env create -f environment.yml 11 | 12 | Figure 1 in the paper was created by running the following at 13 | the command line: 14 | $ make all 15 | -------------------------------------------------------------------------------- /zipf/LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020 Amira Khan 4 | 5 | Permission is hereby granted, free of charge, to any person 6 | obtaining a copy of this software and associated documentation 7 | files (the "Software"), to deal in the Software without 8 | restriction, including without limitation the rights to use, 9 | copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the 11 | Software is furnished to do so, subject to the following 12 | conditions: 13 | 14 | The above copyright notice and this permission notice shall be 15 | included in all copies or substantial portions of the Software. 16 | 17 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 18 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES 19 | OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 20 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT 21 | HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, 22 | WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING 23 | FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 24 | OTHER DEALINGS IN THE SOFTWARE. 25 | -------------------------------------------------------------------------------- /zipf/Makefile: -------------------------------------------------------------------------------- 1 | .PHONY: results all clean help settings 2 | 3 | COUNT=bin/countwords.py 4 | COLLATE=bin/collate.py 5 | PARAMS=bin/plotparams.yml 6 | PLOT=bin/plotcounts.py 7 | SUMMARY=bin/book_summary.sh 8 | DATA=$(wildcard data/*.txt) 9 | RESULTS=$(patsubst data/%.txt,results/%.csv,$(DATA)) 10 | 11 | ## all : regenerate all results. 12 | all : results/collated.png 13 | 14 | ## test-saveconfig : save plot configuration. 15 | test-saveconfig : 16 | python $(PLOT) --saveconfig /tmp/test-saveconfig.yml \ 17 | --plotparams $(PARAMS) 18 | 19 | ## results/collated.png : plot the collated results. 20 | results/collated.png : results/collated.csv $(PARAMS) 21 | python $(PLOT) $< --outfile $@ --plotparams $(word 2,$^) 22 | 23 | ## results/collated.csv : collate all results. 24 | results/collated.csv : $(RESULTS) $(COLLATE) 25 | @mkdir -p results 26 | python $(COLLATE) $(RESULTS) > $@ 27 | 28 | ## results/%.csv : regenerate result for any book. 29 | results/%.csv : data/%.txt $(COUNT) 30 | @bash $(SUMMARY) $< Title 31 | @bash $(SUMMARY) $< Author 32 | python $(COUNT) $< > $@ 33 | 34 | ## results : regenerate result for all books. 35 | results : ${RESULTS} 36 | 37 | ## clean : remove all generated files. 38 | clean : 39 | rm $(RESULTS) results/collated.csv results/collated.png 40 | 41 | ## settings : show variables' values. 42 | settings : 43 | @echo COUNT: $(COUNT) 44 | @echo DATA: $(DATA) 45 | @echo RESULTS: $(RESULTS) 46 | @echo COLLATE: $(COLLATE) 47 | @echo PARAMS: $(PARAMS) 48 | @echo PLOT: $(PLOT) 49 | @echo SUMMARY: $(SUMMARY) 50 | 51 | ## help : show this message. 52 | help : 53 | @grep -h -E '^##' ${MAKEFILE_LIST} | sed -e 's/## //g' \ 54 | | column -t -s ':' 55 | -------------------------------------------------------------------------------- /zipf/README.rst: -------------------------------------------------------------------------------- 1 | The ``zipf`` package tallies the occurrences of words in text files 2 | and plots each word's rank versus its frequency 3 | together with a line for the theoretical distribution for Zipf's Law. 4 | 5 | Motivation 6 | ---------- 7 | 8 | Zipf’s Law is often stated as an observational pattern seen in the 9 | relationship between the frequency and rank of words in a text: 10 | 11 | `"…the most frequent word will occur approximately twice as often 12 | as the second most frequent word, 13 | three times as often as the third most 14 | frequent word, etc."` 15 | — `wikipedia `_ 16 | 17 | Many books are available to download in plain text format 18 | from sites such as `Project Gutenberg `_, 19 | so we created this package to qualitatively explore how well different books align 20 | with the word frequencies predicted by Zipf's Law. 21 | 22 | Installation 23 | ------------ 24 | 25 | ``pip install zipf`` 26 | 27 | Usage 28 | ----- 29 | 30 | After installing this package, 31 | the following three commands will be available from the command line 32 | 33 | - ``countwords`` for counting the occurrences of words in a text. 34 | - ``collate`` for collating multiple word count files together. 35 | - ``plotcounts`` for visualizing the word counts. 36 | 37 | A typical usage scenario would include running the following from your terminal:: 38 | 39 | countwords dracula.txt > dracula.csv 40 | countwords moby_dick.txt > moby_dick.csv 41 | collate dracula.csv moby_dick.csv > collated.csv 42 | plotcounts collated.csv --outfile zipf-drac-moby.jpg 43 | 44 | Additional information on each function 45 | can be found in their docstrings and appending the ``-h`` flag, 46 | e.g. ``countwords -h``. 47 | 48 | Contributors 49 | ------------ 50 | 51 | - Amira Khan 52 | - Sami Virtanen 53 | -------------------------------------------------------------------------------- /zipf/bin/book_summary.sh: -------------------------------------------------------------------------------- 1 | # Get desired information from a Project Gutenberg eBook. 2 | # Usage: bash book_summary.sh /path/to/file.txt what_to_look_for 3 | head -n 17 $1 | tail -n 8 | grep $2 4 | -------------------------------------------------------------------------------- /zipf/bin/collate.py: -------------------------------------------------------------------------------- 1 | """ 2 | Combine multiple word count CSV-files 3 | into a single cumulative count. 4 | """ 5 | 6 | import csv 7 | import argparse 8 | from collections import Counter 9 | import logging 10 | 11 | import utilities as util 12 | 13 | 14 | def update_counts(reader, word_counts): 15 | """Update word counts with data from another reader/file.""" 16 | for word, count in csv.reader(reader): 17 | word_counts[word] += int(count) 18 | 19 | 20 | def process_file(fname, word_counts): 21 | """Read file and update word counts""" 22 | logging.debug(f'Reading in {fname}...') 23 | if fname[-4:] != '.csv': 24 | msg = util.ERRORS['not_csv_suffix'].format( 25 | fname=fname) 26 | raise OSError(msg) 27 | with open(fname, 'r') as reader: 28 | logging.debug('Computing word counts...') 29 | update_counts(reader, word_counts) 30 | 31 | 32 | def main(args): 33 | """Run the command line program.""" 34 | log_lev = logging.DEBUG if args.verbose else logging.WARNING 35 | logging.basicConfig(level=log_lev, filename=args.logfile) 36 | word_counts = Counter() 37 | logging.info('Processing files...') 38 | for fname in args.infiles: 39 | try: 40 | process_file(fname, word_counts) 41 | except FileNotFoundError: 42 | msg = f'{fname} not processed: File does not exist' 43 | logging.warning(msg) 44 | except PermissionError: 45 | msg = f'{fname} not processed: No read permission' 46 | logging.warning(msg) 47 | except Exception as error: 48 | msg = f'{fname} not processed: {error}' 49 | logging.warning(msg) 50 | util.collection_to_csv(word_counts, num=args.num) 51 | 52 | 53 | if __name__ == '__main__': 54 | parser = argparse.ArgumentParser(description=__doc__) 55 | parser.add_argument('infiles', type=str, nargs='*', 56 | help='Input file names') 57 | parser.add_argument('-n', '--num', 58 | type=int, default=None, 59 | help='Output n most frequent words') 60 | parser.add_argument('-v', '--verbose', 61 | action="store_true", default=False, 62 | help="Set logging level to DEBUG") 63 | parser.add_argument('-l', '--logfile', 64 | type=str, default='collate.log', 65 | help='Name of the log file') 66 | args = parser.parse_args() 67 | main(args) 68 | -------------------------------------------------------------------------------- /zipf/bin/countwords.py: -------------------------------------------------------------------------------- 1 | """ 2 | Count the occurrences of all words in a text 3 | and output them in CSV format. 4 | """ 5 | 6 | import argparse 7 | import string 8 | from collections import Counter 9 | 10 | import utilities as util 11 | 12 | 13 | def count_words(reader): 14 | """Count the occurrence of each word in a string.""" 15 | text = reader.read() 16 | chunks = text.split() 17 | npunc = [word.strip(string.punctuation) for word in chunks] 18 | word_list = [word.lower() for word in npunc if word] 19 | word_counts = Counter(word_list) 20 | return word_counts 21 | 22 | 23 | def main(args): 24 | """Run the command line program.""" 25 | word_counts = count_words(args.infile) 26 | util.collection_to_csv(word_counts, num=args.num) 27 | 28 | 29 | if __name__ == '__main__': 30 | parser = argparse.ArgumentParser(description=__doc__) 31 | parser.add_argument('infile', type=argparse.FileType('r'), 32 | nargs='?', default='-', 33 | help='Input file name') 34 | parser.add_argument('-n', '--num', 35 | type=int, default=None, 36 | help='Output n most frequent words') 37 | args = parser.parse_args() 38 | main(args) 39 | -------------------------------------------------------------------------------- /zipf/bin/plotparams.yml: -------------------------------------------------------------------------------- 1 | # Plot characteristics 2 | axes.labelsize: 'x-large' 3 | xtick.labelsize: 'large' 4 | ytick.labelsize: 'large' 5 | -------------------------------------------------------------------------------- /zipf/bin/script_template.py: -------------------------------------------------------------------------------- 1 | """One-line description of what the script does.""" 2 | 3 | import argparse 4 | 5 | 6 | def main(args): 7 | """Run the program.""" 8 | print('Input file:', args.infile) 9 | print('Output file:', args.outfile) 10 | 11 | 12 | if __name__ == '__main__': 13 | parser = argparse.ArgumentParser(description=__doc__) 14 | parser.add_argument('infile', type=str, 15 | help='Input file name') 16 | parser.add_argument('outfile', type=str, 17 | help='Output file name') 18 | args = parser.parse_args() 19 | main(args) 20 | -------------------------------------------------------------------------------- /zipf/bin/test_zipfs.py: -------------------------------------------------------------------------------- 1 | from collections import Counter 2 | 3 | import pytest 4 | import numpy as np 5 | 6 | import plotcounts 7 | import countwords 8 | import collate 9 | 10 | 11 | def test_alpha(): 12 | """Test the calculation of the alpha parameter. 13 | 14 | The test word counts satisfy the relationship, 15 | r = cf**(-1/alpha), where 16 | r is the rank, 17 | f the word count, and 18 | c is a constant of proportionality. 19 | 20 | To generate test word counts for an expected alpha of 21 | 1.0, a maximum word frequency of 600 is used 22 | (i.e. c = 600 and r ranges from 1 to 600) 23 | """ 24 | max_freq = 600 25 | counts = np.floor(max_freq / np.arange(1, max_freq + 1)) 26 | actual_alpha = plotcounts.get_power_law_params(counts) 27 | expected_alpha = pytest.approx(1.0, abs=0.01) 28 | assert actual_alpha == expected_alpha 29 | 30 | 31 | def test_word_count(): 32 | """Test the counting of words. 33 | 34 | The example poem is Risk, by Anais Nin. 35 | """ 36 | risk_poem_counts = {'the': 3, 'risk': 2, 'to': 2, 'and': 1, 37 | 'then': 1, 'day': 1, 'came': 1, 'when': 1, 'remain': 1, 38 | 'tight': 1, 'in': 1, 'a': 1, 'bud': 1, 'was': 1, 39 | 'more': 1, 'painful': 1, 'than': 1, 'it': 1, 'took': 1, 40 | 'blossom': 1} 41 | expected_result = Counter(risk_poem_counts) 42 | with open('test_data/risk.txt', 'r') as reader: 43 | actual_result = countwords.count_words(reader) 44 | assert actual_result == expected_result 45 | 46 | 47 | def test_integration(): 48 | """Test the full word count to alpha parameter workflow.""" 49 | with open('test_data/random_words.txt', 'r') as reader: 50 | word_counts_dict = countwords.count_words(reader) 51 | counts_array = np.array(list(word_counts_dict.values())) 52 | actual_alpha = plotcounts.get_power_law_params(counts_array) 53 | expected_alpha = pytest.approx(1.0, abs=0.01) 54 | assert actual_alpha == expected_alpha 55 | 56 | 57 | def test_regression(): 58 | """Regression test for Dracula.""" 59 | with open('data/dracula.txt', 'r') as reader: 60 | word_counts_dict = countwords.count_words(reader) 61 | counts_array = np.array(list(word_counts_dict.values())) 62 | actual_alpha = plotcounts.get_power_law_params(counts_array) 63 | expected_alpha = pytest.approx(1.087, abs=0.001) 64 | assert actual_alpha == expected_alpha 65 | 66 | 67 | def test_not_csv_error(): 68 | """Error handling test for csv check""" 69 | file_name = 'data/dracula.txt' 70 | word_counts = Counter() 71 | with pytest.raises(OSError): 72 | collate.process_file(file_name, word_counts) 73 | 74 | 75 | def test_missing_file_error(): 76 | """Error handling test for missing file""" 77 | file_name = 'fake_file.csv' 78 | word_counts = Counter() 79 | with pytest.raises(FileNotFoundError): 80 | collate.process_file(file_name, word_counts) 81 | -------------------------------------------------------------------------------- /zipf/bin/utilities.py: -------------------------------------------------------------------------------- 1 | """Collection of commonly used functions.""" 2 | 3 | import sys 4 | import csv 5 | 6 | 7 | ERRORS = { 8 | 'not_csv_suffix': '{fname}: File must end in .csv', 9 | } 10 | 11 | 12 | def collection_to_csv(collection, num=None): 13 | """ 14 | Write collection of items and counts in csv format. 15 | 16 | Parameters 17 | ---------- 18 | collection : collections.Counter 19 | Collection of items and counts 20 | num : int 21 | Limit output to N most frequent items 22 | """ 23 | collection = collection.most_common() 24 | if num is None: 25 | num = len(collection) 26 | writer = csv.writer(sys.stdout) 27 | writer.writerows(collection[0:num]) 28 | -------------------------------------------------------------------------------- /zipf/data/README.md: -------------------------------------------------------------------------------- 1 | Full text of novels downloaded from Project Gutenberg. 2 | 3 | - What is this?: text files (newline-terminated) 4 | - Source(s): Project Gutenberg, https://www.gutenberg.org/ 5 | - License: public domain 6 | - Each file contains the text of its license. 7 | - See https://www.gutenberg.org/wiki/Gutenberg:Permission_How-To for details. 8 | - Downloaded: 2018-08-15 9 | - Contact: Greg Wilson 10 | - Spatial Applicability: N/A 11 | - Temporal Applicability: N/A 12 | - Notes: 13 | - Files contain some 8-bit characters (e.g., curly quotation marks). 14 | - Files contain bibliographic information at the start. 15 | - Files contain errata, license information, and other material at the end. 16 | -------------------------------------------------------------------------------- /zipf/docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /zipf/docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # https://www.sphinx-doc.org/en/master/usage/configuration.html 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | import os 14 | import sys 15 | #sys.path.insert(0, os.path.abspath('../..')) 16 | sys.path.insert(0, os.path.abspath('../zipf')) 17 | 18 | 19 | # -- Project information ----------------------------------------------------- 20 | 21 | project = 'zipf' 22 | copyright = '2020, Amira Khan' 23 | author = 'Amira Khan' 24 | 25 | # The full version, including alpha/beta/rc tags 26 | release = '0.1' 27 | 28 | 29 | # -- General configuration --------------------------------------------------- 30 | 31 | # Add any Sphinx extension module names here, as strings. They can be 32 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 33 | # ones. 34 | extensions = ['sphinx.ext.autodoc'] 35 | 36 | # Add any paths that contain templates here, relative to this directory. 37 | templates_path = ['_templates'] 38 | 39 | # List of patterns, relative to source directory, that match files and 40 | # directories to ignore when looking for source files. 41 | # This pattern also affects html_static_path and html_extra_path. 42 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 43 | 44 | # The master toctree document. 45 | master_doc = 'index' 46 | 47 | # -- Options for HTML output ------------------------------------------------- 48 | 49 | # The theme to use for HTML and HTML Help pages. See the documentation for 50 | # a list of builtin themes. 51 | # 52 | html_theme = 'alabaster' 53 | 54 | # Add any paths that contain custom static files (such as style sheets) here, 55 | # relative to this directory. They are copied after the builtin static files, 56 | # so a file named "default.css" will overwrite the builtin "default.css". 57 | html_static_path = ['_static'] 58 | -------------------------------------------------------------------------------- /zipf/docs/index.rst: -------------------------------------------------------------------------------- 1 | Welcome to zipf's documentation! 2 | ================================ 3 | 4 | .. include:: ../README.rst 5 | 6 | .. toctree:: 7 | :maxdepth: 2 8 | :caption: Contents: 9 | 10 | 11 | 12 | Indices and tables 13 | ================== 14 | 15 | * :ref:`genindex` 16 | * :ref:`modindex` 17 | * :ref:`search` 18 | -------------------------------------------------------------------------------- /zipf/docs/source/collate.rst: -------------------------------------------------------------------------------- 1 | collate module 2 | ============== 3 | 4 | .. automodule:: collate 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /zipf/docs/source/countwords.rst: -------------------------------------------------------------------------------- 1 | countwords module 2 | ================= 3 | 4 | .. automodule:: countwords 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /zipf/docs/source/modules.rst: -------------------------------------------------------------------------------- 1 | zipf 2 | ==== 3 | 4 | .. toctree:: 5 | :maxdepth: 4 6 | 7 | collate 8 | countwords 9 | plotcounts 10 | test_zipfs 11 | utilities 12 | -------------------------------------------------------------------------------- /zipf/docs/source/plotcounts.rst: -------------------------------------------------------------------------------- 1 | plotcounts module 2 | ================= 3 | 4 | .. automodule:: plotcounts 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /zipf/docs/source/test_zipfs.rst: -------------------------------------------------------------------------------- 1 | test\_zipfs module 2 | ================== 3 | 4 | .. automodule:: test_zipfs 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /zipf/docs/source/utilities.rst: -------------------------------------------------------------------------------- 1 | utilities module 2 | ================ 3 | 4 | .. automodule:: utilities 5 | :members: 6 | :undoc-members: 7 | :show-inheritance: 8 | -------------------------------------------------------------------------------- /zipf/environment.yml: -------------------------------------------------------------------------------- 1 | FIXME 2 | -------------------------------------------------------------------------------- /zipf/requirements.txt: -------------------------------------------------------------------------------- 1 | numpy 2 | pandas 3 | matplotlib 4 | scipy 5 | pytest 6 | pyyaml 7 | -------------------------------------------------------------------------------- /zipf/requirements_docs.txt: -------------------------------------------------------------------------------- 1 | Sphinx>=1.7.4 2 | -------------------------------------------------------------------------------- /zipf/results/collated.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/zipf/results/collated.png -------------------------------------------------------------------------------- /zipf/results/dracula.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/zipf/results/dracula.png -------------------------------------------------------------------------------- /zipf/results/jane_eyre.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/merely-useful/py-rse/62217e6606842ab9752fcf8e73954d1eb4a3cf07/zipf/results/jane_eyre.png -------------------------------------------------------------------------------- /zipf/setup.py: -------------------------------------------------------------------------------- 1 | from setuptools import setup 2 | 3 | 4 | setup( 5 | name='zipf', 6 | version='0.1', 7 | author='Amira Khan', 8 | packages=['zipf'], 9 | install_requires=[ 10 | 'matplotlib', 11 | 'pandas', 12 | 'scipy', 13 | 'pyyaml', 14 | 'pytest'], 15 | entry_points={ 16 | 'console_scripts': [ 17 | 'countwords = zipf.countwords:main', 18 | 'collate = zipf.collate:main', 19 | 'plotcounts = zipf.plotcounts:main']}) 20 | -------------------------------------------------------------------------------- /zipf/test_data/risk.txt: -------------------------------------------------------------------------------- 1 | And then the day came, 2 | when the risk 3 | to remain tight 4 | in a bud 5 | was more painful 6 | than the risk 7 | it took 8 | to blossom. 9 | --------------------------------------------------------------------------------