├── .github ├── ISSUE_TEMPLATE.md └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── .mailmap ├── .update-copyright.conf ├── AUTHORS ├── CITATION ├── CONDUCT.md ├── CONTRIBUTING.md ├── LICENSE.md ├── Makefile ├── README.md ├── _config.yml ├── _episodes ├── .gitkeep ├── 00-why-hpc.md ├── 10-login-and-transfer.md ├── 20-architecture.md ├── 30-batch-system.md ├── 35-filesystem.md └── 40-environment-modules.md ├── _episodes_rmd ├── .gitkeep ├── 05-rmarkdown-example.Rmd └── data │ └── .gitkeep ├── _extras ├── .gitkeep ├── about.md ├── discuss.md └── guide.md ├── _includes ├── all_keypoints.html ├── carpentries.html ├── dc │ ├── intro.html │ ├── schedule.html │ ├── syllabus.html │ └── who.html ├── episode_break.html ├── episode_keypoints.html ├── episode_navbar.html ├── episode_overview.html ├── episode_title.html ├── javascript.html ├── lc │ ├── intro.html │ ├── schedule.html │ ├── syllabus.html │ └── who.html ├── lesson_footer.html ├── links.md ├── main_title.html ├── navbar.html ├── sc │ ├── intro.html │ ├── schedule.html │ ├── syllabus.html │ └── who.html ├── syllabus.html ├── workshop_ad.html └── workshop_footer.html ├── _layouts ├── base.html ├── break.html ├── episode.html ├── lesson.html ├── page.html ├── reference.html └── workshop.html ├── aio.md ├── assets ├── css │ ├── bootstrap-theme.css │ ├── bootstrap-theme.css.map │ ├── bootstrap.css │ ├── bootstrap.css.map │ ├── lesson.scss │ └── syntax.css ├── fonts │ ├── glyphicons-halflings-regular.eot │ ├── glyphicons-halflings-regular.svg │ ├── glyphicons-halflings-regular.ttf │ ├── glyphicons-halflings-regular.woff │ └── glyphicons-halflings-regular.woff2 ├── img │ ├── dc-icon-black.svg │ ├── lc-icon-black.png │ ├── lc-icon-black.svg │ ├── swc-icon-blue.svg │ ├── swc-logo-blue.png │ ├── swc-logo-blue.svg │ ├── swc-logo-white.png │ └── swc-logo-white.svg └── js │ ├── bootstrap.min.js │ ├── jquery.min.js │ ├── jquery.min.map │ └── lesson.js ├── bin ├── chunk-options.R ├── extract_figures.py ├── generate_md_episodes.R ├── knit_lessons.sh ├── lesson_check.py ├── lesson_initialize.py ├── markdown_ast.rb ├── repo_check.py ├── test_lesson_check.py ├── util.py └── workshop_check.py ├── code └── .gitkeep ├── data └── .gitkeep ├── design.md ├── favicon-dc.ico ├── favicon-lc.ico ├── favicon-swc.ico ├── fig ├── .gitkeep ├── episode-format-small.png ├── episode-format.eps ├── episode-format.png ├── file-mapping.odg ├── file-mapping.svg ├── forking.odg ├── forking.svg ├── repository-links.odg ├── repository-links.svg ├── rmd-plot-example-1.png └── using-github-import.png ├── files └── .gitkeep ├── img └── putty-7.jpeg ├── index.md ├── outline.md ├── reference.md ├── requirements.txt ├── setup.md ├── target-audience.md └── tools ├── check_knitr_version.R └── chunk-options.R /.github/ISSUE_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Please delete the text below before submitting your contribution. 2 | 3 | --- 4 | 5 | Thanks for contributing! If this contribution is for instructor training, please send an email to checkout@carpentries.org with a link to this contribution so we can record your progress. You’ve completed your contribution step for instructor checkout just by submitting this contribution. 6 | 7 | Please keep in mind that lesson maintainers are volunteers and it may be some time before they can respond to your contribution. Although not all contributions can be incorporated into the lesson materials, we appreciate your time and effort to improve the curriculum. If you have any questions about the lesson maintenance process or would like to volunteer your time as a contribution reviewer, please contact Kate Hertweck (k8hertweck@gmail.com). 8 | 9 | --- 10 | -------------------------------------------------------------------------------- /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | Please delete the text below before submitting your contribution. 2 | 3 | --- 4 | 5 | Thanks for contributing! If this contribution is for instructor training, please send an email to checkout@carpentries.org with a link to this contribution so we can record your progress. You’ve completed your contribution step for instructor checkout just by submitting this contribution. 6 | 7 | Please keep in mind that lesson maintainers are volunteers and it may be some time before they can respond to your contribution. Although not all contributions can be incorporated into the lesson materials, we appreciate your time and effort to improve the curriculum. If you have any questions about the lesson maintenance process or would like to volunteer your time as a contribution reviewer, please contact Kate Hertweck (k8hertweck@gmail.com). 8 | 9 | --- 10 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *~ 3 | .DS_Store 4 | .ipynb_checkpoints 5 | .sass-cache 6 | __pycache__ 7 | _site 8 | -------------------------------------------------------------------------------- /.mailmap: -------------------------------------------------------------------------------- 1 | Abigail Cabunoc Mayes 2 | Abigail Cabunoc Mayes 3 | Alexander Konovalov 4 | Amy Brown 5 | Ariel Rokem 6 | Bartosz Telenczuk 7 | Erin Becker 8 | Evan P. Williamson 9 | François Michonneau 10 | Greg Wilson 11 | Ivan Gonzalez 12 | James Allen 13 | Maxim Belkin 14 | Mike Jackson 15 | Raniere Silva 16 | Raniere Silva 17 | Rémi Emonet 18 | Rémi Emonet 19 | Timothée Poisot 20 | Valentina Staneva 21 | -------------------------------------------------------------------------------- /.update-copyright.conf: -------------------------------------------------------------------------------- 1 | [project] 2 | vcs: Git 3 | 4 | [files] 5 | authors: yes 6 | files: no 7 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | James Allen 2 | Pete Bachant 3 | Piotr Banaszkiewicz 4 | David Beitey 5 | Trevor Bekolay 6 | John Blischak 7 | Andy Boughton 8 | Amy Brown 9 | C. Titus Brown 10 | Gerard Capes 11 | Jonah Duckles 12 | Rémi Emonet 13 | Ivan Gonzalez 14 | Thomas Guignard 15 | Kate Hertweck 16 | Mike Jackson 17 | W. Trevor King 18 | Christina Koch 19 | Alexander Konovalov 20 | François Michonneau 21 | Bill Mills 22 | Aaron O'Leary 23 | Bianca Peterson 24 | Timothée Poisot 25 | Florian Rathgeber 26 | Ariel Rokem 27 | Raniere Silva 28 | Valentina Staneva 29 | Tracy Teal 30 | Greg Wilson 31 | Donny Winston 32 | Naupaka Zimmerman 33 | Bartosz Telenczuk 34 | -------------------------------------------------------------------------------- /CITATION: -------------------------------------------------------------------------------- 1 | Please cite as: 2 | 3 | Greg Wilson (ed.): "Software Carpentry: Lesson Example." Version 4 | 2016.06, June 2016, https://github.com/swcarpentry/lesson-example, 5 | 10.5281/zenodo.58153. 6 | -------------------------------------------------------------------------------- /CONDUCT.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: "Contributor Code of Conduct" 4 | --- 5 | As contributors and maintainers of this project, 6 | we pledge to respect all people who contribute through reporting issues, 7 | posting feature requests, 8 | updating documentation, 9 | submitting pull requests or patches, 10 | and other activities. 11 | 12 | We are committed to making participation in this project a harassment-free experience for everyone, 13 | regardless of level of experience, 14 | gender, 15 | gender identity and expression, 16 | sexual orientation, 17 | disability, 18 | personal appearance, 19 | body size, 20 | race, 21 | ethnicity, 22 | age, 23 | or religion. 24 | 25 | Examples of unacceptable behavior by participants include the use of sexual language or imagery, 26 | derogatory comments or personal attacks, 27 | trolling, 28 | public or private harassment, 29 | insults, 30 | or other unprofessional conduct. 31 | 32 | Project maintainers have the right and responsibility to remove, edit, or reject 33 | comments, commits, code, wiki edits, issues, and other contributions 34 | that are not aligned to this Code of Conduct. 35 | Project maintainers who do not follow the Code of Conduct may be removed from the project team. 36 | 37 | Instances of abusive, harassing, or otherwise unacceptable behavior 38 | may be reported by opening an issue or contacting one or more of the project maintainers. 39 | 40 | This Code of Conduct is adapted from 41 | the [Contributor Covenant][contrib-covenant] Version 1.0.0. 42 | 43 | {% include links.md %} 44 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | [Software Carpentry][swc-site] and [Data Carpentry][dc-site] are open source projects, 4 | and we welcome contributions of all kinds: 5 | new lessons, 6 | fixes to existing material, 7 | bug reports, 8 | and reviews of proposed changes are all welcome. 9 | 10 | ## Contributor Agreement 11 | 12 | By contributing, 13 | you agree that we may redistribute your work under [our license](LICENSE.md). 14 | In exchange, 15 | we will address your issues and/or assess your change proposal as promptly as we can, 16 | and help you become a member of our community. 17 | Everyone involved in [Software Carpentry][swc-site] and [Data Carpentry][dc-site] 18 | agrees to abide by our [code of conduct](CONDUCT.md). 19 | 20 | ## How to Contribute 21 | 22 | The easiest way to get started is to file an issue 23 | to tell us about a spelling mistake, 24 | some awkward wording, 25 | or a factual error. 26 | This is a good way to introduce yourself 27 | and to meet some of our community members. 28 | 29 | 1. If you do not have a [GitHub][github] account, 30 | you can [send us comments by email][contact]. 31 | However, 32 | we will be able to respond more quickly if you use one of the other methods described below. 33 | 34 | 2. If you have a [GitHub][github] account, 35 | or are willing to [create one][github-join], 36 | but do not know how to use Git, 37 | you can report problems or suggest improvements by [creating an issue][issues]. 38 | This allows us to assign the item to someone 39 | and to respond to it in a threaded discussion. 40 | 41 | 3. If you are comfortable with Git, 42 | and would like to add or change material, 43 | you can submit a pull request (PR). 44 | Instructions for doing this are [included below](#using-github). 45 | 46 | ## Where to Contribute 47 | 48 | 1. If you wish to change this lesson, 49 | please work in , 50 | which can be viewed at . 51 | 52 | 2. If you wish to change the example lesson, 53 | please work in , 54 | which documents the format of our lessons 55 | and can be viewed at . 56 | 57 | 3. If you wish to change the template used for workshop websites, 58 | please work in . 59 | The home page of that repository explains how to set up workshop websites, 60 | while the extra pages in 61 | provide more background on our design choices. 62 | 63 | 4. If you wish to change CSS style files, tools, 64 | or HTML boilerplate for lessons or workshops stored in `_includes` or `_layouts`, 65 | please work in . 66 | 67 | ## What to Contribute 68 | 69 | There are many ways to contribute, 70 | from writing new exercises and improving existing ones 71 | to updating or filling in the documentation 72 | and and submitting [bug reports][issues] 73 | about things that don't work, aren't clear, or are missing. 74 | If you are looking for ideas, 75 | please see [the list of issues for this repository][issues], 76 | or the issues for [Data Carpentry][dc-issues] 77 | and [Software Carpentry][swc-issues] projects. 78 | 79 | Comments on issues and reviews of pull requests are just as welcome: 80 | we are smarter together than we are on our own. 81 | Reviews from novices and newcomers are particularly valuable: 82 | it's easy for people who have been using these lessons for a while 83 | to forget how impenetrable some of this material can be, 84 | so fresh eyes are always welcome. 85 | 86 | ## What *Not* to Contribute 87 | 88 | Our lessons already contain more material than we can cover in a typical workshop, 89 | so we are usually *not* looking for more concepts or tools to add to them. 90 | As a rule, 91 | if you want to introduce a new idea, 92 | you must (a) estimate how long it will take to teach 93 | and (b) explain what you would take out to make room for it. 94 | The first encourages contributors to be honest about requirements; 95 | the second, to think hard about priorities. 96 | 97 | We are also not looking for exercises or other material that only run on one platform. 98 | Our workshops typically contain a mixture of Windows, Mac OS X, and Linux users; 99 | in order to be usable, 100 | our lessons must run equally well on all three. 101 | 102 | ## Using GitHub 103 | 104 | If you choose to contribute via GitHub, 105 | you may want to look at 106 | [How to Contribute to an Open Source Project on GitHub][how-contribute]. 107 | In brief: 108 | 109 | 1. The published copy of the lesson is in the `gh-pages` branch of the repository 110 | (so that GitHub will regenerate it automatically). 111 | Please create all branches from that, 112 | and merge the [master repository][repo]'s `gh-pages` branch into your `gh-pages` branch 113 | before starting work. 114 | Please do *not* work directly in your `gh-pages` branch, 115 | since that will make it difficult for you to work on other contributions. 116 | 117 | 2. We use [GitHub flow][github-flow] to manage changes: 118 | 1. Create a new branch in your desktop copy of this repository for each significant change. 119 | 2. Commit the change in that branch. 120 | 3. Push that branch to your fork of this repository on GitHub. 121 | 4. Submit a pull request from that branch to the [master repository][repo]. 122 | 5. If you receive feedback, 123 | make changes on your desktop and push to your branch on GitHub: 124 | the pull request will update automatically. 125 | 126 | Each lesson has two maintainers who review issues and pull requests 127 | or encourage others to do so. 128 | The maintainers are community volunteers, 129 | and have final say over what gets merged into the lesson. 130 | 131 | ## Other Resources 132 | 133 | General discussion of [Software Carpentry][swc-site] and [Data Carpentry][dc-site] 134 | happens on the [discussion mailing list][discuss-list], 135 | which everyone is welcome to join. 136 | You can also [reach us by email][contact]. 137 | 138 | [contact]: mailto:admin@software-carpentry.org 139 | [dc-issues]: https://github.com/issues?q=user%3Adatacarpentry 140 | [dc-lessons]: http://datacarpentry.org/lessons/ 141 | [dc-site]: http://datacarpentry.org/ 142 | [discuss-list]: http://lists.software-carpentry.org/listinfo/discuss 143 | [github]: http://github.com 144 | [github-flow]: https://guides.github.com/introduction/flow/ 145 | [github-join]: https://github.com/join 146 | [how-contribute]: https://egghead.io/series/how-to-contribute-to-an-open-source-project-on-github 147 | [issues]: https://github.com/swcarpentry/FIXME/issues/ 148 | [repo]: https://github.com/swcarpentry/FIXME/ 149 | [swc-issues]: https://github.com/issues?q=user%3Aswcarpentry 150 | [swc-lessons]: http://software-carpentry.org/lessons/ 151 | [swc-site]: http://software-carpentry.org/ 152 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: "Licenses" 4 | root: . 5 | --- 6 | ## Instructional Material 7 | 8 | All Software Carpentry and Data Carpentry instructional material is 9 | made available under the [Creative Commons Attribution 10 | license][cc-by-human]. The following is a human-readable summary of 11 | (and not a substitute for) the [full legal text of the CC BY 4.0 12 | license][cc-by-legal]. 13 | 14 | You are free: 15 | 16 | * to **Share**---copy and redistribute the material in any medium or format 17 | * to **Adapt**---remix, transform, and build upon the material 18 | 19 | for any purpose, even commercially. 20 | 21 | The licensor cannot revoke these freedoms as long as you follow the 22 | license terms. 23 | 24 | Under the following terms: 25 | 26 | * **Attribution**---You must give appropriate credit (mentioning that 27 | your work is derived from work that is Copyright © Software 28 | Carpentry and, where practical, linking to 29 | http://software-carpentry.org/), provide a [link to the 30 | license][cc-by-human], and indicate if changes were made. You may do 31 | so in any reasonable manner, but not in any way that suggests the 32 | licensor endorses you or your use. 33 | 34 | **No additional restrictions**---You may not apply legal terms or 35 | technological measures that legally restrict others from doing 36 | anything the license permits. With the understanding that: 37 | 38 | Notices: 39 | 40 | * You do not have to comply with the license for elements of the 41 | material in the public domain or where your use is permitted by an 42 | applicable exception or limitation. 43 | * No warranties are given. The license may not give you all of the 44 | permissions necessary for your intended use. For example, other 45 | rights such as publicity, privacy, or moral rights may limit how you 46 | use the material. 47 | 48 | ## Software 49 | 50 | Except where otherwise noted, the example programs and other software 51 | provided by Software Carpentry and Data Carpentry are made available under the 52 | [OSI][osi]-approved 53 | [MIT license][mit-license]. 54 | 55 | Permission is hereby granted, free of charge, to any person obtaining 56 | a copy of this software and associated documentation files (the 57 | "Software"), to deal in the Software without restriction, including 58 | without limitation the rights to use, copy, modify, merge, publish, 59 | distribute, sublicense, and/or sell copies of the Software, and to 60 | permit persons to whom the Software is furnished to do so, subject to 61 | the following conditions: 62 | 63 | The above copyright notice and this permission notice shall be 64 | included in all copies or substantial portions of the Software. 65 | 66 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 67 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 68 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 69 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 70 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 71 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 72 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 73 | 74 | ## Trademark 75 | 76 | "Software Carpentry" and "Data Carpentry" and their respective logos 77 | are registered trademarks of [NumFOCUS][numfocus]. 78 | 79 | {% include links.md %} 80 | 81 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ## ======================================== 2 | ## Commands for both workshop and lesson websites. 3 | 4 | # Settings 5 | MAKEFILES=Makefile $(wildcard *.mk) 6 | JEKYLL=jekyll 7 | PARSER=bin/markdown_ast.rb 8 | DST=_site 9 | 10 | # Controls 11 | .PHONY : commands clean files 12 | .NOTPARALLEL: 13 | all : commands 14 | 15 | ## commands : show all commands. 16 | commands : 17 | @grep -h -E '^##' ${MAKEFILES} | sed -e 's/## //g' 18 | 19 | ## serve : run a local server. 20 | serve : lesson-md 21 | ${JEKYLL} serve 22 | 23 | ## site : build files but do not run a server. 24 | site : lesson-md 25 | ${JEKYLL} build 26 | 27 | # repo-check : check repository settings. 28 | repo-check : 29 | @bin/repo_check.py -s . 30 | 31 | ## clean : clean up junk files. 32 | clean : 33 | @rm -rf ${DST} 34 | @rm -rf .sass-cache 35 | @rm -rf bin/__pycache__ 36 | @find . -name .DS_Store -exec rm {} \; 37 | @find . -name '*~' -exec rm {} \; 38 | @find . -name '*.pyc' -exec rm {} \; 39 | 40 | ## clean-rmd : clean intermediate R files (that need to be committed to the repo). 41 | clear-rmd : 42 | @rm -rf ${RMD_DST} 43 | @rm -rf fig/rmd-* 44 | 45 | ## ---------------------------------------- 46 | ## Commands specific to workshop websites. 47 | 48 | .PHONY : workshop-check 49 | 50 | ## workshop-check : check workshop homepage. 51 | workshop-check : 52 | @bin/workshop_check.py . 53 | 54 | ## ---------------------------------------- 55 | ## Commands specific to lesson websites. 56 | 57 | .PHONY : lesson-check lesson-md lesson-files lesson-fixme 58 | 59 | # RMarkdown files 60 | RMD_SRC = $(wildcard _episodes_rmd/??-*.Rmd) 61 | RMD_DST = $(patsubst _episodes_rmd/%.Rmd,_episodes/%.md,$(RMD_SRC)) 62 | 63 | # Lesson source files in the order they appear in the navigation menu. 64 | MARKDOWN_SRC = \ 65 | index.md \ 66 | CONDUCT.md \ 67 | setup.md \ 68 | $(sort $(wildcard _episodes/*.md)) \ 69 | reference.md \ 70 | $(sort $(wildcard _extras/*.md)) \ 71 | LICENSE.md 72 | 73 | # Generated lesson files in the order they appear in the navigation menu. 74 | HTML_DST = \ 75 | ${DST}/index.html \ 76 | ${DST}/conduct/index.html \ 77 | ${DST}/setup/index.html \ 78 | $(patsubst _episodes/%.md,${DST}/%/index.html,$(sort $(wildcard _episodes/*.md))) \ 79 | ${DST}/reference/index.html \ 80 | $(patsubst _extras/%.md,${DST}/%/index.html,$(sort $(wildcard _extras/*.md))) \ 81 | ${DST}/license/index.html 82 | 83 | ## lesson-md : convert Rmarkdown files to markdown 84 | lesson-md : ${RMD_DST} 85 | 86 | # Use of .NOTPARALLEL makes rule execute only once 87 | ${RMD_DST} : ${RMD_SRC} 88 | @bin/knit_lessons.sh ${RMD_SRC} 89 | 90 | ## lesson-check : validate lesson Markdown. 91 | lesson-check : 92 | @bin/lesson_check.py -s . -p ${PARSER} -r _includes/links.md 93 | 94 | ## lesson-check-all : validate lesson Markdown, checking line lengths and trailing whitespace. 95 | lesson-check-all : 96 | @bin/lesson_check.py -s . -p ${PARSER} -r _includes/links.md -l -w 97 | 98 | ## unittest : run unit tests on checking tools. 99 | unittest : 100 | python bin/test_lesson_check.py 101 | 102 | ## lesson-files : show expected names of generated files for debugging. 103 | lesson-files : 104 | @echo 'RMD_SRC:' ${RMD_SRC} 105 | @echo 'RMD_DST:' ${RMD_DST} 106 | @echo 'MARKDOWN_SRC:' ${MARKDOWN_SRC} 107 | @echo 'HTML_DST:' ${HTML_DST} 108 | 109 | ## lesson-fixme : show FIXME markers embedded in source files. 110 | lesson-fixme : 111 | @fgrep -i -n FIXME ${MARKDOWN_SRC} || true 112 | 113 | #------------------------------------------------------------------------------- 114 | # Include extra commands if available. 115 | #------------------------------------------------------------------------------- 116 | 117 | -include commands.mk 118 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # hpc-novice (deprecated) 2 | 3 | Development is now happening in the [hpc-carpentry][] GitHub organization. There is an 4 | introductory lesson at [hpc-intro][] and other lessons with additional topics. 5 | 6 | For announcements and updates about Carpentry style material for HPC training, please 7 | join the [mailing list][discuss-hpc]. 8 | 9 | [hpc-carpentry]: https://github.com/hpc-carpentry/ 10 | [hpc-intro]: https://github.com/hpc-carpentry/hpc-intro 11 | [discuss-hpc]: https://carpentries.topicbox.com/groups/discuss-hpc 12 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------ 2 | # Values for this lesson. 3 | #------------------------------------------------------------ 4 | 5 | # Which carpentry is this ("swc", "dc", or "lc)? 6 | carpentry: "swc" 7 | 8 | # Overall title for pages. 9 | title: "HPC Novice" 10 | 11 | # Contact email address. 12 | email: "mailto:lessons@software-carpentry.org" 13 | 14 | #------------------------------------------------------------ 15 | # Constant URL addresses across lessons 16 | #------------------------------------------------------------ 17 | # 18 | # Sites. 19 | amy_site: "https://amy.software-carpentry.org/workshops" 20 | dc_site: "http://datacarpentry.org" 21 | swc_github: "https://github.com/swcarpentry" 22 | swc_site: "https://software-carpentry.org" 23 | swc_pages: "https://swcarpentry.github.io" 24 | lc_site: "http://librarycarpentry.github.io/" 25 | template_repo: "https://github.com/swcarpentry/styles" 26 | example_repo: "https://github.com/swcarpentry/lesson-example" 27 | example_site: "https://swcarpentry.github.com/lesson-example" 28 | workshop_repo: "https://github.com/swcarpentry/workshop-template" 29 | workshop_site: "https://swcarpentry.github.io/workshop-template" 30 | training_site: "https://swcarpentry.github.io/instructor-training" 31 | 32 | # Surveys. 33 | pre_survey: "https://www.surveymonkey.com/r/swc_pre_workshop_v1?workshop_id=" 34 | post_survey: "https://www.surveymonkey.com/r/swc_post_workshop_v1?workshop_id=" 35 | 36 | #------------------------------------------------------------ 37 | # Lesson Generic settings (should not need to change). 38 | #------------------------------------------------------------ 39 | 40 | # What kind of thing is this ("workshop" or "lesson")? 41 | kind: "lesson" 42 | 43 | # Magic to make URLs resolve both locally and on GitHub. 44 | # See https://help.github.com/articles/repository-metadata-on-github-pages/. 45 | repository: / 46 | 47 | # Start time in minutes (0 to be clock-independent, 540 to show a start at 09:00 am) 48 | start_time: 0 49 | 50 | # Specify that things in the episodes collection should be output. 51 | collections: 52 | episodes: 53 | output: true 54 | permalink: /:path/index.html 55 | extras: 56 | output: true 57 | permalink: /:path/index.html 58 | 59 | # Set the default layout for things in the episodes collection. 60 | defaults: 61 | - values: 62 | root: .. 63 | - scope: 64 | path: "" 65 | type: episodes 66 | values: 67 | layout: episode 68 | 69 | #------------------------------------------------------------ 70 | # Jekyll Generic settings (should not need to change). 71 | #------------------------------------------------------------ 72 | 73 | # Files and directories that are not to be copied. 74 | exclude: 75 | - Makefile 76 | - bin 77 | 78 | # Turn on built-in syntax highlighting. 79 | highlighter: rouge 80 | -------------------------------------------------------------------------------- /_episodes/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/hpc-novice/d08fa355c3444ee8981e57f8453e936fc2f00946/_episodes/.gitkeep -------------------------------------------------------------------------------- /_episodes/00-why-hpc.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Introducing High-Performance Computing" 3 | questions: 4 | - "Why use a large-scale computing system?" 5 | - "How is using a cluster different than your laptop?" 6 | keypoints: 7 | - "A cluster or distributed system is useful when your computations require more than one computer." 8 | - "Because a cluster is distributed, it is only useful for certain types of computational problems." 9 | --- 10 | 11 | Topics 12 | 13 | When it is appropriate to use a cluster 14 | 15 | > ## How can you use a cluster? 16 | > 17 | > Talk to your neighbor about your research. How does computing 18 | > help you do your research? How could more computing help you 19 | > do more or better research? 20 | {: .callout} 21 | 22 | How is using a cluster different than your laptop? 23 | 24 | > ## Thinking ahead 25 | > How is using a large-scale computing system different 26 | > from using your laptop? Talk to your neighbor about some 27 | > differences you may already know about, and some 28 | > differences/difficulties you imagine you may run into. 29 | 30 | 31 | -------------------------------------------------------------------------------- /_episodes/10-login-and-transfer.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Logging In And Transfer Data To and From" 3 | questions: 4 | - "How do I log in to the cluster?" 5 | - "How do I transfer data back and forth to the cluser?" 6 | keypoints: 7 | - "A cluster is many small computers rather than one large computer" 8 | - "The login (head) node is the entry-point to the cluster" 9 | - "A secure shell client is a program that lets us log-in 10 | to other computers and run a command-line on them" 11 | - "" 12 | --- 13 | -------------------------------------------------------------------------------- /_episodes/20-architecture.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "HPC architecture" 3 | questions: 4 | - "To be filled with a question!" 5 | keypoints: 6 | - "To be filled with a fact/sentence." 7 | --- 8 | -------------------------------------------------------------------------------- /_episodes/30-batch-system.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Using the Batch System 101" 3 | questions: 4 | - "To be filled." 5 | keypoints: 6 | - "To be filled." 7 | --- 8 | -------------------------------------------------------------------------------- /_episodes/35-filesystem.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "The Shared File System" 3 | questions: 4 | - "To be filled." 5 | keypoints: 6 | - "to be filled." 7 | --- 8 | -------------------------------------------------------------------------------- /_episodes/40-environment-modules.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "The Environment Modules" 3 | questions: 4 | - "To be filled." 5 | keypoints: 6 | - "to be filled." 7 | --- 8 | -------------------------------------------------------------------------------- /_episodes_rmd/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/hpc-novice/d08fa355c3444ee8981e57f8453e936fc2f00946/_episodes_rmd/.gitkeep -------------------------------------------------------------------------------- /_episodes_rmd/05-rmarkdown-example.Rmd: -------------------------------------------------------------------------------- 1 | --- 2 | source: Rmd 3 | title: "Using RMarkdown" 4 | teaching: 10 5 | exercises: 2 6 | questions: 7 | - "How to write a lesson using RMarkdown?" 8 | objectives: 9 | - "Explain how to use RMarkdown with the new lesson template." 10 | - "Demonstrate how to include pieces of code, figures, and challenges." 11 | keypoints: 12 | - "It shouldn't be difficult" 13 | --- 14 | 15 | This episode demonstrates all the features that can be used when writing a 16 | lesson in [RMarkdown][r-markdown]. 17 | 18 | To generate the site, you will need to have the following packages installed: 19 | 20 | ```{r, eval=FALSE} 21 | install.packages(c("knitr", "stringr", "checkpoint")) 22 | ``` 23 | 24 | If the lesson uses additional packages, the script that converts the Rmd files 25 | into markdown, will detect them and install them for you, when you run `make 26 | serve` or `make site`. 27 | 28 | This first chunk is really important, and need to be included at the beginning of 29 | each episode written in RMarkdown. 30 | 31 | ```{r, echo=TRUE} 32 | source("../bin/chunk-options.R") 33 | ``` 34 | 35 | The rest of the lesson should be written as a normal RMarkdown file. You can 36 | include chunk for codes, just like you'd normally do. 37 | 38 | Normal output: 39 | 40 | ```{r} 41 | 1 + 1 42 | ``` 43 | 44 | Output with error message: 45 | 46 | ```{r} 47 | x[10] 48 | ``` 49 | 50 | Output generating figures: 51 | 52 | ```{r plot-example} 53 | library(ggplot2) 54 | ggplot(diamonds, aes(x = carat, y = price, color = cut)) + 55 | geom_point() 56 | ``` 57 | 58 | For the challenges and their solutions, you need to pay attention to where the 59 | `>` go and where to leave blank lines. You can include code chunks in both the 60 | instructions and solutions. For instance this: 61 | 62 | ``` 63 | > ## Challenge: Can you do it? 64 | > 65 | > What is the output of this command? 66 | > 67 | > ```{r, eval=FALSE} 68 | > paste("This", "new", "template", "looks", "good") 69 | > ``` 70 | > 71 | > > ## Solution 72 | > > 73 | > > ```{r, echo=FALSE} 74 | > > paste("This", "new", "template", "looks", "good") 75 | > > ``` 76 | > {: .solution} 77 | {: .challenge} 78 | ``` 79 | 80 | will generate this: 81 | 82 | > ## Challenge: Can you do it? 83 | > 84 | > What is the output of this command? 85 | > 86 | > ```{r, eval=FALSE} 87 | > paste("This", "new", "template", "looks", "good") 88 | > ``` 89 | > 90 | > > ## Solution 91 | > > 92 | > > ```{r, echo=FALSE} 93 | > > paste("This", "new", "template", "looks", "good") 94 | > > ``` 95 | > {: .solution} 96 | {: .challenge} 97 | 98 | {% include links.md %} 99 | -------------------------------------------------------------------------------- /_episodes_rmd/data/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/hpc-novice/d08fa355c3444ee8981e57f8453e936fc2f00946/_episodes_rmd/data/.gitkeep -------------------------------------------------------------------------------- /_extras/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/hpc-novice/d08fa355c3444ee8981e57f8453e936fc2f00946/_extras/.gitkeep -------------------------------------------------------------------------------- /_extras/about.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: About 4 | --- 5 | {% include carpentries.html %} 6 | -------------------------------------------------------------------------------- /_extras/discuss.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: Discussion 4 | --- 5 | Readers may be interested in: 6 | 7 | * PeerJ's [paper-now], which uses [Jekyll][jekyll] as a framework for writing scientific papers. 8 | * The [Morea Framework][morea], 9 | a more elaborate template for [Jekyll][jekyll]-based lessons. 10 | 11 | {% include links.md %} 12 | -------------------------------------------------------------------------------- /_extras/guide.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: "Instructor Notes" 4 | --- 5 | 6 | # Topic related 7 | 8 | Because different compute resources have different features, this 9 | lesson, by necessity, should be thought of as completely modular. 10 | We have here the basic topics that might be required, in a recommended order, 11 | but additions and deletions are encouraged, based on the needs of the 12 | audience and the compute resource that is being used. It could also be 13 | expanded or contracted to fill a certain amount of time -- the default 14 | time period should be about day, but for a complete novice audience 15 | (has **never** used the command line), it should be more like a day 16 | and a half, or two days. 17 | 18 | 19 | ## Cluster roleplay instructions (from 04-scheduler) 20 | 21 | To do this exercise, you will need about 50-100 pieces of paper or sticky notes. 22 | 23 | 1. Divide the room into groups, with specific roles. 24 | * Pick three-four people to be the "scheduler" 25 | * Have the remaining one-third of the room be "users", given several slips of 26 | paper (or post-it notes) and pens 27 | * Have the remaining two thirds of the room be "compute nodes" 28 | Make sure everyone knows what their roles are. Have the "users" 29 | go to the front of the room (or the back, wherever there's space 30 | for them to stand) and the "schedulers" stand between the users 31 | and "compute nodes" (who should remain at their seats). 32 | 33 | 2. Divide the pieces of paper / sticky notes among the "users" and have them 34 | fill out all the pages with simple math problems and their name. Tell everyone that these 35 | are the jobs that need to be done and correspond to their computing research problems. 36 | 37 | 3. Point out that we now have jobs and we have "compute nodes" (the people still sitting 38 | down) that can solve these problems. How are the jobs going to get to the nodes? 39 | The answer is the scheduling program that will take the jobs from the users and deliver 40 | them to open compute nodes. 41 | 42 | 4. Have all the "compute nodes" raise their hands. Have the users "submit" their 43 | jobs by handing them to the schedulers. Schedulers 44 | should then deliver them to "open" (hands-raised) compute nodes and collect 45 | finished problems and return them to the appropriate user. 46 | 47 | 5. Wait until most of the problems are done and then re-seat everyone. 48 | 49 | 6. Follow-up discussion: what would happen if a node couldn't solve the math problem? It 50 | might be important to indicate the *resources* that your job needs to run. Add other 51 | parallels that will be coming up in the next section of the lesson. 52 | 53 | # General 54 | 55 | The instructor notes should provide additional discussion useful to instructors, 56 | but not appropriate for inclusion in the main lessons. The following structure 57 | provides a consistent way for instructors to both prepare for a workshop and 58 | quickly find necessary information during a workshop. 59 | 60 | Please remember not to overload on details, and to keep the comments here positive! 61 | 62 | ## Lesson motivation and learning objectives 63 | 64 | These concepts should be highlighted in the main lesson material, but ideas for 65 | explaining these concepts further can be placed here. 66 | 67 | ## Lesson design 68 | 69 | Most lessons contain more material than can be taught in a single workshop. 70 | Describe a general narrative (with time estimates) for teaching either a half day 71 | or full day with this lesson material. You may also choose to include multiple 72 | options for lesson design, or what material can be skipped while teaching. 73 | This section may also include recommendations for how this lesson fits into 74 | the overall workshop. 75 | 76 | ## Technical tips and tricks 77 | 78 | Provide information on setting up your environment for learners to view your 79 | live coding (increasing text size, changing text color, etc), as well as 80 | general recommendations for working with coding tools to best suit the 81 | learning environment. 82 | 83 | ## Common problems 84 | 85 | This can include answers to common learner questions, as well as links to 86 | resources (blog posts, stack overflow answers, etc) that may solve problems that 87 | may occur during a workshop. 88 | 89 | 90 | -------------------------------------------------------------------------------- /_includes/all_keypoints.html: -------------------------------------------------------------------------------- 1 | {% comment %} 2 | Display key points of all episodes for reference. 3 | {% endcomment %} 4 |

Key Points

5 | 6 | {% for episode in site.episodes %} 7 | {% unless episode.break %} 8 | 9 | 12 | 19 | 20 | {% endunless %} 21 | {% endfor %} 22 |
10 | {{ episode.title }} 11 | 13 |
    14 | {% for keypoint in episode.keypoints %} 15 |
  • {{ keypoint|markdownify }}
    • 16 | {% endfor %} 17 |
18 |
23 | -------------------------------------------------------------------------------- /_includes/carpentries.html: -------------------------------------------------------------------------------- 1 | {% comment %} 2 | General description of Software and Data Carpentry. 3 | {% endcomment %} 4 |
5 |
6 | Software Carpentry logo 7 |
8 |
9 | Since 1998, 10 | Software Carpentry 11 | has been teaching researchers in science, engineering, medicine, and related disciplines 12 | the computing skills they need to get more done in less time and with less pain. 13 | Its volunteer instructors have run hundreds of events 14 | for thousands of learners in the past two and a half years. 15 |
16 |
17 |
18 |
19 |
20 | Data Carpentry logo 21 |
22 |
23 | Data Carpentry develops and teaches workshops on the fundamental data skills needed to conduct research. 24 | Its target audience is researchers who have little to no prior computational experience, 25 | and its lessons are domain specific, 26 | building on learners' existing knowledge to enable them to quickly apply skills learned to their own research. 27 |
28 |
29 |
30 |
31 |
32 | Library Carpentry logo 33 |
34 |
35 | Library Carpentry is made by librarians to help librarians 36 | automate repetitive, boring, error-prone tasks; 37 | create, maintain and analyse sustainable and reusable data; 38 | work effectively with IT and systems colleagues; 39 | better understand the use of software in research; 40 | and much more. 41 | Library Carpentry was the winner of the 2016 42 | British Library Labs Teaching and Learning Award. 43 |
44 |
45 | -------------------------------------------------------------------------------- /_includes/dc/intro.html: -------------------------------------------------------------------------------- 1 |

2 | Data Carpentry 3 | aims to help researchers get their work done 4 | in less time and with less pain 5 | by teaching them basic research computing skills. 6 | This hands-on workshop will cover basic concepts and tools, 7 | including program design, version control, data management, 8 | and task automation. 9 | Participants will be encouraged to help one another 10 | and to apply what they have learned to their own research problems. 11 |

12 |

13 | 14 | For more information on what we teach and why, 15 | please see our paper 16 | "Best Practices for Scientific Computing". 17 | 18 |

19 | -------------------------------------------------------------------------------- /_includes/dc/schedule.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

Day 1

4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 |
09:00 Automating tasks with the Unix shell
10:30 Coffee
12:00 Lunch break
13:00 Building programs with Python
14:30 Coffee
16:00 Wrap-up
12 |
13 |
14 |

Day 2

15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 |
09:00 Version control with Git
10:30 Coffee
12:00 Lunch break
13:00 Managing data with SQL
14:30 Coffee
16:00 Wrap-up
23 |
24 |
25 | -------------------------------------------------------------------------------- /_includes/dc/syllabus.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

The Unix Shell

4 |
    5 |
  • Files and directories
    • 6 |
  • History and tab completion
    • 7 |
  • Pipes and redirection
    • 8 |
  • Looping over files
    • 9 |
  • Creating and running shell scripts
    • 10 |
  • Finding things
    • 11 |
  • Reference...
    • 12 |
13 |
14 |
15 |

Programming in Python

16 |
    17 |
  • Using libraries
    • 18 |
  • Working with arrays
    • 19 |
  • Reading and plotting data
    • 20 |
  • Creating and using functions
    • 21 |
  • Loops and conditionals
    • 22 |
  • Defensive programming
    • 23 |
  • Using Python from the command line
    • 24 |
  • Reference...
    • 25 |
26 |
27 | 40 | 53 |
54 | 55 |
56 |
57 |

Version Control with Git

58 |
    59 |
  • Creating a repository
    • 60 |
  • Recording changes to files: add, commit, ...
    • 61 |
  • Viewing changes: status, diff, ...
    • 62 |
  • Ignoring files
    • 63 |
  • Working on the web: clone, pull, push, ...
    • 64 |
  • Resolving conflicts
    • 65 |
  • Open licenses
    • 66 |
  • Where to host work, and why
    • 67 |
  • Reference...
    • 68 |
69 |
70 | 86 |
87 |

Open Refine

88 |
    89 |
  • Introduction to OpenRefine
    • 90 |
  • Importing data
    • 91 |
  • Basic functions
    • 92 |
  • Advanced Functions
    • 93 |
  • Reference...
    • 94 |
95 |
96 |
97 | -------------------------------------------------------------------------------- /_includes/dc/who.html: -------------------------------------------------------------------------------- 1 |

2 | Who: 3 | The course is aimed at graduate students and other researchers. 4 | 5 | You don't need to have any previous knowledge of the tools 6 | that will be presented at the workshop. 7 | 8 |

9 | -------------------------------------------------------------------------------- /_includes/episode_break.html: -------------------------------------------------------------------------------- 1 | {% comment %} 2 | Display a break's timings in a box similar to a learning episode's. 3 | {% endcomment %} 4 |
5 |

Overview

6 | 7 |
8 |
9 | Break: {{ page.break }} min 10 |
11 |
12 |
13 |
14 | 15 |
16 | -------------------------------------------------------------------------------- /_includes/episode_keypoints.html: -------------------------------------------------------------------------------- 1 | {% comment %} 2 | Display key points for an episode. 3 | {% endcomment %} 4 |
5 |

Key Points

6 |
    7 | {% for keypoint in page.keypoints %} 8 |
  • {{ keypoint|markdownify }}
    • 9 | {% endfor %} 10 |
11 |
12 | -------------------------------------------------------------------------------- /_includes/episode_navbar.html: -------------------------------------------------------------------------------- 1 | {% comment %} 2 | Navigation bar for an episode. 3 | {% endcomment %} 4 |
5 |
6 |

7 | {% if page.previous.url %} 8 | previous episode 9 | {% else %} 10 | lesson home 11 | {% endif %} 12 |

13 |
14 |
15 | {% if include.episode_navbar_title %} 16 |

{{ site.title }}

17 | {% endif %} 18 |
19 |
20 |

21 | {% if page.next.url %} 22 | next episode 23 | {% else %} 24 | lesson home 25 | {% endif %} 26 |

27 |
28 |
29 | -------------------------------------------------------------------------------- /_includes/episode_overview.html: -------------------------------------------------------------------------------- 1 | {% comment %} 2 | Display an episode's timings and learning objectives. 3 | {% endcomment %} 4 |
5 |

Overview

6 | 7 |
8 |
9 | Teaching: {{ page.teaching }} min 10 |
11 | Exercises: {{ page.exercises }} min 12 |
13 |
14 | Questions 15 |
    16 | {% for question in page.questions %} 17 |
  • {{ question|markdownify }}
    • 18 | {% endfor %} 19 |
20 |
21 |
22 | 23 |
24 |
25 |
26 |
27 | Objectives 28 |
    29 | {% for objective in page.objectives %} 30 |
  • {{ objective|markdownify }}
    • 31 | {% endfor %} 32 |
33 |
34 |
35 | 36 |
37 | -------------------------------------------------------------------------------- /_includes/episode_title.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |
5 |

{{ page.title }}

6 |
7 |
8 |
9 |
10 | -------------------------------------------------------------------------------- /_includes/javascript.html: -------------------------------------------------------------------------------- 1 | {% comment %} 2 | Javascript used in lesson and workshop pages. 3 | {% endcomment %} 4 | 5 | 6 | 7 | 15 | -------------------------------------------------------------------------------- /_includes/lc/intro.html: -------------------------------------------------------------------------------- 1 |

2 | Library Carpentry 3 | is made by librarians, for librarians to help you: 4 |

5 |
    6 |
  • automate repetitive, boring, error-prone tasks
    • 7 |
  • create, maintain and analyse sustainable and reusable data
    • 8 |
  • work effectively with IT and systems colleagues
    • 9 |
  • better understand the use of software in research
    • 10 |
  • and much more...
    • 11 |
12 |

13 | 14 | Library Carpentry introduces you to the fundamentals of computing 15 | and provides you with a platform for further self-directed learning. 16 | For more information on what we teach and why, please see our paper 17 | "Library Carpentry: software skills training for library professionals". 18 | 19 |

20 | -------------------------------------------------------------------------------- /_includes/lc/schedule.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

Day 1

4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 |
09:00 Data Intro for Librarians
10:30 Coffee
12:00 Lunch break
13:00 Shell Lessons for Libraries
14:30 Coffee
16:00 Wrap-up
12 |
13 |
14 |

Day 2

15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 |
09:00 Git Intro for Librarians
10:30 Coffee
12:00 Lunch break
13:00 OpenRefine for Librarians
14:30 Coffee
16:00 Wrap-up
23 |
24 |
25 | -------------------------------------------------------------------------------- /_includes/lc/syllabus.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

Data Intro

4 |
    5 |
  • Intro to data
    • 6 |
  • Jargon busting
    • 7 |
  • Keyboard shortcuts
    • 8 |
  • Plain text formats
    • 9 |
  • Naming files
    • 10 |
  • Regular expressions
    • 11 |
  • Reference...
    • 12 |
13 |
14 |
15 |

The Unix Shell

16 |
    17 |
  • Files and directories
    • 18 |
  • History and tab completion
    • 19 |
  • Counting and sorting contents in files
    • 20 |
  • Pipes and redirection
    • 21 |
  • Mining or searching in files
    • 22 |
  • Reference...
    • 23 |
24 |
25 | 26 |
27 | 28 |
29 |
30 |

Version Control with Git

31 |
    32 |
  • Creating a repository
    • 33 |
  • Configuring git
    • 34 |
  • Recording changes to files: add, commit, ...
    • 35 |
  • Viewing state changes with status
    • 36 |
  • Working on the web: clone, pull, push, ...
    • 37 |
  • Where to host work, and why
    • 38 |
  • Reference...
    • 39 |
40 |
41 |
42 |
43 |

Open Refine

44 |
    45 |
  • Introduction to OpenRefine
    • 46 |
  • Importing data
    • 47 |
  • Basic functions
    • 48 |
  • Advanced Functions
    • 49 |
  • Reference...
    • 50 |
51 |
52 |
53 |
54 | 70 | -------------------------------------------------------------------------------- /_includes/lc/who.html: -------------------------------------------------------------------------------- 1 |

2 | Who: 3 | The course is for librarians, archivists, and other information workers. 4 | 5 | You don't need to have any previous knowledge of the tools that 6 | will be presented at the workshop. 7 | 8 |

9 | -------------------------------------------------------------------------------- /_includes/lesson_footer.html: -------------------------------------------------------------------------------- 1 | {% comment %} 2 | Footer for lesson pages. 3 | {% endcomment %} 4 | 39 | -------------------------------------------------------------------------------- /_includes/links.md: -------------------------------------------------------------------------------- 1 | [cc-by-human]: https://creativecommons.org/licenses/by/4.0/ 2 | [cc-by-legal]: https://creativecommons.org/licenses/by/4.0/legalcode 3 | [concept-maps]: http://carpentries.github.io/instructor-training/05-memory/ 4 | [email]: mailto:lessons@software-carpentry.org 5 | [contrib-covenant]: http://contributor-covenant.org/ 6 | [contributing]: {{ site.github.repository_url }}/blob/gh-pages/CONTRIBUTING.md 7 | [cran-checkpoint]: https://cran.r-project.org/web/packages/checkpoint/index.html 8 | [cran-knitr]: https://cran.r-project.org/web/packages/knitr/index.html 9 | [cran-stringr]: https://cran.r-project.org/web/packages/stringr/index.html 10 | [github-importer]: https://import.github.com/ 11 | [importer]: https://github.com/new/import 12 | [jekyll-collection]: https://jekyllrb.com/docs/collections/ 13 | [jekyll-install]: https://jekyllrb.com/docs/installation/ 14 | [jekyll-windows]: http://jekyll-windows.juthilo.com/ 15 | [jekyll]: https://jekyllrb.com/ 16 | [jupyter]: https://jupyter.org/ 17 | [mit-license]: http://opensource.org/licenses/mit-license.html 18 | [morea]: https://morea-framework.github.io/ 19 | [numfocus]: http://numfocus.org/ 20 | [osi]: http://opensource.org 21 | [pandoc]: https://pandoc.org/ 22 | [paper-now]: https://github.com/PeerJ/paper-now 23 | [python-gapminder]: https://swcarpentry.github.io/python-novice-gapminder/ 24 | [pyyaml]: https://pypi.python.org/pypi/PyYAML 25 | [r-markdown]: http://rmarkdown.rstudio.com/ 26 | [rstudio]: https://www.rstudio.com/ 27 | [ruby-install-guide]: https://www.ruby-lang.org/en/downloads/ 28 | [ruby-installer]: http://rubyinstaller.org/ 29 | [rubygems]: https://rubygems.org/pages/download/ 30 | [styles]: https://github.com/swcarpentry/styles/ 31 | [training]: http://swcarpentry.github.io/instructor-training/ 32 | [workshop-repo]: {{ site.workshop_repo }} 33 | [yaml]: http://yaml.org/ 34 | [coc]: https://software-carpentry.org/conduct/ 35 | [coc-reporting]: https://software-carpentry.org/CoC-reporting/ 36 | -------------------------------------------------------------------------------- /_includes/main_title.html: -------------------------------------------------------------------------------- 1 | {% comment %} 2 | Main title for lesson pages. 3 | {% endcomment %} 4 |

{{ site.title }}{% if page.title %}: {{ page.title }}{% endif %}

5 | -------------------------------------------------------------------------------- /_includes/navbar.html: -------------------------------------------------------------------------------- 1 | {% comment %} 2 | Lesson navigation bar. 3 | {% endcomment %} 4 | 87 | -------------------------------------------------------------------------------- /_includes/sc/intro.html: -------------------------------------------------------------------------------- 1 |

2 | Software Carpentry 3 | aims to help researchers get their work done 4 | in less time and with less pain 5 | by teaching them basic research computing skills. 6 | This hands-on workshop will cover basic concepts and tools, 7 | including program design, version control, data management, 8 | and task automation. 9 | Participants will be encouraged to help one another 10 | and to apply what they have learned to their own research problems. 11 |

12 |

13 | 14 | For more information on what we teach and why, 15 | please see our paper 16 | "Best Practices for Scientific Computing". 17 | 18 |

19 | -------------------------------------------------------------------------------- /_includes/sc/schedule.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

Day 1

4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 |
09:00 Automating tasks with the Unix shell
10:30 Coffee
12:00 Lunch break
13:00 Building programs with Python
14:30 Coffee
16:00 Wrap-up
12 |
13 |
14 |

Day 2

15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 |
09:00 Version control with Git
10:30 Coffee
12:00 Lunch break
13:00 Managing data with SQL
14:30 Coffee
16:00 Wrap-up
23 |
24 |
25 | -------------------------------------------------------------------------------- /_includes/sc/syllabus.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |

The Unix Shell

4 |
    5 |
  • Files and directories
    • 6 |
  • History and tab completion
    • 7 |
  • Pipes and redirection
    • 8 |
  • Looping over files
    • 9 |
  • Creating and running shell scripts
    • 10 |
  • Finding things
    • 11 |
  • Reference...
    • 12 |
13 |
14 |
15 |

Programming in Python

16 |
    17 |
  • Using libraries
    • 18 |
  • Working with arrays
    • 19 |
  • Reading and plotting data
    • 20 |
  • Creating and using functions
    • 21 |
  • Loops and conditionals
    • 22 |
  • Defensive programming
    • 23 |
  • Using Python from the command line
    • 24 |
  • Reference...
    • 25 |
26 |
27 | 40 | 53 |
54 | 55 |
56 |
57 |

Version Control with Git

58 |
    59 |
  • Creating a repository
    • 60 |
  • Recording changes to files: add, commit, ...
    • 61 |
  • Viewing changes: status, diff, ...
    • 62 |
  • Ignoring files
    • 63 |
  • Working on the web: clone, pull, push, ...
    • 64 |
  • Resolving conflicts
    • 65 |
  • Open licenses
    • 66 |
  • Where to host work, and why
    • 67 |
  • Reference...
    • 68 |
69 |
70 | 86 |
87 |

Open Refine

88 |
    89 |
  • Introduction to OpenRefine
    • 90 |
  • Importing data
    • 91 |
  • Basic functions
    • 92 |
  • Advanced Functions
    • 93 |
  • Reference...
    • 94 |
95 |
96 |
97 | -------------------------------------------------------------------------------- /_includes/sc/who.html: -------------------------------------------------------------------------------- 1 |

2 | Who: 3 | The course is aimed at graduate students and other researchers. 4 | 5 | You don't need to have any previous knowledge of the tools 6 | that will be presented at the workshop. 7 | 8 |

9 | -------------------------------------------------------------------------------- /_includes/syllabus.html: -------------------------------------------------------------------------------- 1 | {% comment %} 2 | Display syllabus in tabular form. 3 | Days are displayed if at least one episode has 'start = true'. 4 | {% endcomment %} 5 |
6 |

Schedule

7 | 8 | {% assign lesson_number = 0 %} 9 | {% assign day = 0 %} 10 | {% assign multiday = false %} 11 | {% for episode in site.episodes %} 12 | {% if episode.start %}{% assign multiday = true %}{% break %}{% endif %} 13 | {% endfor %} 14 | {% assign current = site.start_time %} 15 | 16 | 17 | 18 | {% if multiday %}{% endif %} 19 | 20 | 21 | 22 | 23 | {% for episode in site.episodes %} 24 | {% if episode.start %} {% comment %} Starting a new day? {% endcomment %} 25 | {% assign day = day | plus: 1 %} 26 | {% if day > 1 %} {% comment %} If about to start day 2 or later, show finishing time for previous day {% endcomment %} 27 | {% assign hours = current | divided_by: 60 %} 28 | {% assign minutes = current | modulo: 60 %} 29 | 30 | {% if multiday %}{% endif %} 31 | 32 | 33 | 34 | 35 | {% endif %} 36 | {% assign current = site.start_time %} {% comment %}Re-set start time of this episode to general daily start time {% endcomment %} 37 | {% endif %} 38 | {% assign hours = current | divided_by: 60 %} 39 | {% assign minutes = current | modulo: 60 %} 40 | 41 | {% if multiday %}{% endif %} 42 | 43 | 47 | 61 | 62 | {% assign current = current | plus: episode.teaching | plus: episode.exercises | plus: episode.break %} 63 | {% endfor %} 64 | {% assign hours = current | divided_by: 60 %} 65 | {% assign minutes = current | modulo: 60 %} 66 | 67 | {% if multiday %}{% endif %} 68 | 69 | 70 | 71 | 72 |
SetupDownload files required for the lesson
{% if hours < 10 %}0{% endif %}{{ hours }}:{% if minutes < 10 %}0{% endif %}{{ minutes }}Finish
{% if episode.start %}Day {{ day }}{% endif %}{% if hours < 10 %}0{% endif %}{{ hours }}:{% if minutes < 10 %}0{% endif %}{{ minutes }} 44 | {% assign lesson_number = lesson_number | plus: 1 %} 45 | {{ lesson_number }}. {{ episode.title }} 46 | 48 | {% if episode.break %} 49 | Break 50 | {% else %} 51 | {% if episode.questions %} 52 | {% for question in episode.questions %} 53 | {{question|markdownify|strip_html}} 54 | {% unless forloop.last %} 55 |
56 | {% endunless %} 57 | {% endfor %} 58 | {% endif %} 59 | {% endif %} 60 |
{% if hours < 10 %}0{% endif %}{{ hours }}:{% if minutes < 10 %}0{% endif %}{{ minutes }}Finish
73 | 74 |

75 | The actual schedule may vary slightly depending on the topics and exercises chosen by the instructor. 76 |

77 | 78 |
79 | -------------------------------------------------------------------------------- /_includes/workshop_ad.html: -------------------------------------------------------------------------------- 1 | {% comment %} 2 | Advertising box at the top of a workshop website home page. 3 | {% endcomment %} 4 |
5 |
6 |
7 |

{{page.venue}}

8 |
9 |
10 |

{{page.humandate}}

11 |

{% if page.humantime %}{{page.humantime}}{% endif %}

12 |
13 |
14 |

15 | Instructors: 16 | {% if page.instructor %} 17 | {{page.instructor | join: ', ' %}} 18 | {% else %} 19 | to be announced. 20 | {% endif %} 21 |

22 | {% if page.helper %} 23 |

24 | Helpers: 25 | {{page.helper | join: ', ' %}} 26 |

27 | {% endif %} 28 |
29 |
30 |
31 |
32 |
33 | -------------------------------------------------------------------------------- /_includes/workshop_footer.html: -------------------------------------------------------------------------------- 1 | {% comment %} 2 | Footer for a standard workshop. 3 | {% endcomment %} 4 | 25 | -------------------------------------------------------------------------------- /_layouts/base.html: -------------------------------------------------------------------------------- 1 | --- 2 | --- 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | {% if site.carpentry == "swc" %} 17 | 18 | {% elsif site.carpentry == "dc" %} 19 | 20 | {% elsif site.carpentry == "lc" %} 21 | 22 | {% endif %} 23 | 24 | 25 | 29 | {{ site.title }}{% if page.title %}: {{ page.title }}{% endif %} 30 | 31 | 32 |
33 | {% include navbar.html %} 34 | {{ content }} 35 | {% if site.kind == "workshop" %} 36 | {% include workshop_footer.html %} 37 | {% else %} 38 | {% include lesson_footer.html %} 39 | {% endif %} 40 |
41 | {% include javascript.html %} 42 | 43 | 44 | -------------------------------------------------------------------------------- /_layouts/break.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: base 3 | --- 4 | {% include episode_navbar.html episode_navbar_title=true %} 5 |
6 | {% include episode_title.html %} 7 | {% include episode_break.html %} 8 | {{content}} 9 |
10 | {% include episode_navbar.html episode_navbar_title=false %} 11 | -------------------------------------------------------------------------------- /_layouts/episode.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: base 3 | --- 4 | {% include episode_navbar.html episode_navbar_title=true %} 5 |
6 | {% include episode_title.html %} 7 | {% include episode_overview.html %} 8 | {{content}} 9 | {% include episode_keypoints.html %} 10 |
11 | {% include episode_navbar.html episode_navbar_title=false %} 12 | -------------------------------------------------------------------------------- /_layouts/lesson.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: base 3 | --- 4 | {% include main_title.html %} 5 | {{ content }} 6 | {% include syllabus.html %} 7 | -------------------------------------------------------------------------------- /_layouts/page.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: base 3 | --- 4 | {% include main_title.html %} 5 | {{content}} 6 | -------------------------------------------------------------------------------- /_layouts/reference.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: "Reference" 4 | --- 5 | {% include all_keypoints.html %} 6 | {{content}} 7 | -------------------------------------------------------------------------------- /_layouts/workshop.html: -------------------------------------------------------------------------------- 1 | --- 2 | --- 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | {% if page.redirect %} 21 | 22 | {% endif %} 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | {% if site.carpentry == "swc" %} 32 | 33 | {% elsif site.carpentry == "dc" %} 34 | 35 | {% elsif site.carpentry == "lc" %} 36 | 37 | {% endif %} 38 | 39 | 40 | 44 | {{ page.venue }}: {{ page.humandate }} 45 | 46 | 47 |
48 | {% include navbar.html %} 49 | {% include workshop_ad.html %} 50 | {{ content }} 51 | {% include workshop_footer.html %} 52 |
53 | {% include javascript.html %} 54 | 55 | 56 | -------------------------------------------------------------------------------- /aio.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | root: . 4 | --- 5 | 31 | {% comment %} 32 | Create anchor for each one of the episodes. 33 | {% endcomment %} 34 | {% for episode in site.episodes %} 35 |
36 | {% endfor %} 37 | -------------------------------------------------------------------------------- /assets/css/lesson.scss: -------------------------------------------------------------------------------- 1 | --- 2 | --- 3 | 4 | //---------------------------------------- 5 | // Colors. 6 | //---------------------------------------- 7 | 8 | // branding 9 | $color-brand: #2b3990 !default; 10 | 11 | // code boxes 12 | $color-error: #bd2c00 !default; 13 | $color-output: #303030 !default; 14 | $color-source: #6e5494 !default; 15 | 16 | // blockquotes 17 | $color-callout: #f4fd9c !default; 18 | $color-challenge: #eec275 !default; 19 | $color-checklist: #dfd2a0 !default; 20 | $color-discussion: #eec275 !default; 21 | $color-keypoints: #7ae78e !default; 22 | $color-objectives: #daee84 !default; 23 | $color-prereq: #9cd6dc !default; 24 | $color-solution: #ded4b9 !default; 25 | $color-testimonial: #fc8dc1 !default; 26 | 27 | //---------------------------------------- 28 | // Specialized code blocks. 29 | //---------------------------------------- 30 | 31 | @mixin cdSetup($color) { 32 | color: $color; 33 | border-left: solid 5px $color; 34 | margin-bottom: 0px; 35 | border-radius: 4px 0 0 4px; 36 | } 37 | 38 | .error { @include cdSetup($color-error); } 39 | .output { @include cdSetup($color-output); } 40 | .source { @include cdSetup($color-source); } 41 | 42 | .bash { @include cdSetup($color-source); } 43 | .make { @include cdSetup($color-source); } 44 | .matlab { @include cdSetup($color-source); } 45 | .python { @include cdSetup($color-source); } 46 | .r { @include cdSetup($color-source); } 47 | .sql { @include cdSetup($color-source); } 48 | 49 | .error pre, 50 | .output pre, 51 | .source pre, 52 | .bash pre, 53 | .make pre, 54 | .matlab pre, 55 | .python pre, 56 | .r pre, 57 | .sql pre { 58 | border-radius: 0 4px 4px 0; 59 | } 60 | 61 | //---------------------------------------- 62 | // Specialized blockquote environments for learning objectives, callouts, etc. 63 | //---------------------------------------- 64 | 65 | $codeblock-padding: 5px !default; 66 | 67 | @mixin bkSetup($color, $glyph) { 68 | 69 | $gradientcolor1: $color; 70 | $gradientcolor2: scale-color($color, $lightness: 10%); 71 | 72 | padding-left: $codeblock-padding; 73 | padding-top: 0; 74 | padding-bottom: 0; 75 | padding-right: 0; 76 | border: 1px solid; 77 | border-color: $color; 78 | padding-bottom: $codeblock-padding; 79 | 80 | h2 { 81 | padding-top: $codeblock-padding; 82 | padding-bottom: $codeblock-padding; 83 | font-size: 20px; 84 | background: linear-gradient(to bottom, $gradientcolor1, $gradientcolor2); 85 | border-color: $color; 86 | margin-top: 0px; 87 | margin-left: -$codeblock-padding; // to move back to the left margin of the enclosing blockquote 88 | } 89 | h2:before { 90 | font-family: 'Glyphicons Halflings'; 91 | content: $glyph; 92 | float: left; 93 | padding-left: $codeblock-padding; 94 | padding-right: $codeblock-padding; 95 | display: inline-block; 96 | -webkit-font-smoothing: antialiased; 97 | } 98 | 99 | } 100 | 101 | .callout{ @include bkSetup($color-callout, "\e146"); } 102 | .challenge{ @include bkSetup($color-challenge, "\270f"); } 103 | .checklist{ @include bkSetup($color-checklist, "\e067"); } 104 | .discussion{ @include bkSetup($color-discussion, "\e123"); } 105 | .keypoints{ @include bkSetup($color-keypoints, "\e101"); } 106 | .objectives{ @include bkSetup($color-objectives, "\e085"); } 107 | .prereq{ @include bkSetup($color-prereq, "\e124"); } 108 | .solution{ @include bkSetup($color-solution, "\e105"); } 109 | .testimonial{ @include bkSetup($color-testimonial, "\e143"); } 110 | 111 | //---------------------------------------- 112 | // Override Bootstrap settings. 113 | //---------------------------------------- 114 | 115 | code { 116 | padding: 2px 5px; 117 | color: #3d90d9; 118 | background-color: #e7e7e7; 119 | } 120 | 121 | img { 122 | max-width: 100%; 123 | } 124 | 125 | //---------------------------------------- 126 | // Miscellaneous. 127 | //---------------------------------------- 128 | 129 | .maintitle { 130 | text-align: center; 131 | } 132 | 133 | .footertext { 134 | text-align: center; 135 | } 136 | 137 | img.navbar-logo { 138 | height: 40px; // synchronize with height of navbar 139 | padding-top: 5px; 140 | padding-right: 10px; 141 | } 142 | 143 | div.branding { 144 | color: $color-brand; 145 | } 146 | 147 | ul, 148 | ol { 149 | padding-left: 2em; 150 | } 151 | 152 | span.fold-unfold { 153 | margin-left: 1em; 154 | opacity: 0.5; 155 | } 156 | 157 | 158 | //---------------------------------------- 159 | // keyboard key style, from StackExchange. 160 | //---------------------------------------- 161 | 162 | kbd { 163 | display: inline-block; 164 | margin: 0 .1em; 165 | padding: .1em .6em; 166 | font-family: Arial,"Helvetica Neue",Helvetica,sans-serif; 167 | font-size: 11px; 168 | line-height: 1.4; 169 | color: #242729; 170 | text-shadow: 0 1px 0 #FFF; 171 | background-color: #e1e3e5; 172 | border: 1px solid #adb3b9; 173 | border-radius: 3px; 174 | box-shadow: 0 1px 0 rgba(12,13,14,0.2), 0 0 0 2px #FFF inset; 175 | white-space: nowrap; 176 | font-style: normal; 177 | } 178 | -------------------------------------------------------------------------------- /assets/css/syntax.css: -------------------------------------------------------------------------------- 1 | .highlight .hll { background-color: #ffffcc } 2 | .highlight { background: #f8f8f8; } 3 | .highlight .c { color: #408080; font-style: italic } /* Comment */ 4 | .highlight .err { border: 1px solid #FF0000 } /* Error */ 5 | .highlight .k { color: #008000; font-weight: bold } /* Keyword */ 6 | .highlight .o { color: #666666 } /* Operator */ 7 | .highlight .ch { color: #408080; font-style: italic } /* Comment.Hashbang */ 8 | .highlight .cm { color: #408080; font-style: italic } /* Comment.Multiline */ 9 | .highlight .cp { color: #BC7A00 } /* Comment.Preproc */ 10 | .highlight .cpf { color: #408080; font-style: italic } /* Comment.PreprocFile */ 11 | .highlight .c1 { color: #408080; font-style: italic } /* Comment.Single */ 12 | .highlight .cs { color: #408080; font-style: italic } /* Comment.Special */ 13 | .highlight .gd { color: #A00000 } /* Generic.Deleted */ 14 | .highlight .ge { font-style: italic } /* Generic.Emph */ 15 | .highlight .gr { color: #FF0000 } /* Generic.Error */ 16 | .highlight .gh { color: #000080; font-weight: bold } /* Generic.Heading */ 17 | .highlight .gi { color: #00A000 } /* Generic.Inserted */ 18 | .highlight .go { color: #888888 } /* Generic.Output */ 19 | .highlight .gp { color: #000080; font-weight: bold } /* Generic.Prompt */ 20 | .highlight .gs { font-weight: bold } /* Generic.Strong */ 21 | .highlight .gu { color: #800080; font-weight: bold } /* Generic.Subheading */ 22 | .highlight .gt { color: #0044DD } /* Generic.Traceback */ 23 | .highlight .kc { color: #008000; font-weight: bold } /* Keyword.Constant */ 24 | .highlight .kd { color: #008000; font-weight: bold } /* Keyword.Declaration */ 25 | .highlight .kn { color: #008000; font-weight: bold } /* Keyword.Namespace */ 26 | .highlight .kp { color: #008000 } /* Keyword.Pseudo */ 27 | .highlight .kr { color: #008000; font-weight: bold } /* Keyword.Reserved */ 28 | .highlight .kt { color: #B00040 } /* Keyword.Type */ 29 | .highlight .m { color: #666666 } /* Literal.Number */ 30 | .highlight .s { color: #BA2121 } /* Literal.String */ 31 | .highlight .na { color: #7D9029 } /* Name.Attribute */ 32 | .highlight .nb { color: #008000 } /* Name.Builtin */ 33 | .highlight .nc { color: #0000FF; font-weight: bold } /* Name.Class */ 34 | .highlight .no { color: #880000 } /* Name.Constant */ 35 | .highlight .nd { color: #AA22FF } /* Name.Decorator */ 36 | .highlight .ni { color: #999999; font-weight: bold } /* Name.Entity */ 37 | .highlight .ne { color: #D2413A; font-weight: bold } /* Name.Exception */ 38 | .highlight .nf { color: #0000FF } /* Name.Function */ 39 | .highlight .nl { color: #A0A000 } /* Name.Label */ 40 | .highlight .nn { color: #0000FF; font-weight: bold } /* Name.Namespace */ 41 | .highlight .nt { color: #008000; font-weight: bold } /* Name.Tag */ 42 | .highlight .nv { color: #19177C } /* Name.Variable */ 43 | .highlight .ow { color: #AA22FF; font-weight: bold } /* Operator.Word */ 44 | .highlight .w { color: #bbbbbb } /* Text.Whitespace */ 45 | .highlight .mb { color: #666666 } /* Literal.Number.Bin */ 46 | .highlight .mf { color: #666666 } /* Literal.Number.Float */ 47 | .highlight .mh { color: #666666 } /* Literal.Number.Hex */ 48 | .highlight .mi { color: #666666 } /* Literal.Number.Integer */ 49 | .highlight .mo { color: #666666 } /* Literal.Number.Oct */ 50 | .highlight .sa { color: #BA2121 } /* Literal.String.Affix */ 51 | .highlight .sb { color: #BA2121 } /* Literal.String.Backtick */ 52 | .highlight .sc { color: #BA2121 } /* Literal.String.Char */ 53 | .highlight .dl { color: #BA2121 } /* Literal.String.Delimiter */ 54 | .highlight .sd { color: #BA2121; font-style: italic } /* Literal.String.Doc */ 55 | .highlight .s2 { color: #BA2121 } /* Literal.String.Double */ 56 | .highlight .se { color: #BB6622; font-weight: bold } /* Literal.String.Escape */ 57 | .highlight .sh { color: #BA2121 } /* Literal.String.Heredoc */ 58 | .highlight .si { color: #BB6688; font-weight: bold } /* Literal.String.Interpol */ 59 | .highlight .sx { color: #008000 } /* Literal.String.Other */ 60 | .highlight .sr { color: #BB6688 } /* Literal.String.Regex */ 61 | .highlight .s1 { color: #BA2121 } /* Literal.String.Single */ 62 | .highlight .ss { color: #19177C } /* Literal.String.Symbol */ 63 | .highlight .bp { color: #008000 } /* Name.Builtin.Pseudo */ 64 | .highlight .fm { color: #0000FF } /* Name.Function.Magic */ 65 | .highlight .vc { color: #19177C } /* Name.Variable.Class */ 66 | .highlight .vg { color: #19177C } /* Name.Variable.Global */ 67 | .highlight .vi { color: #19177C } /* Name.Variable.Instance */ 68 | .highlight .vm { color: #19177C } /* Name.Variable.Magic */ 69 | .highlight .il { color: #666666 } /* Literal.Number.Integer.Long */ 70 | -------------------------------------------------------------------------------- /assets/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/hpc-novice/d08fa355c3444ee8981e57f8453e936fc2f00946/assets/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /assets/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/hpc-novice/d08fa355c3444ee8981e57f8453e936fc2f00946/assets/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /assets/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/hpc-novice/d08fa355c3444ee8981e57f8453e936fc2f00946/assets/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /assets/fonts/glyphicons-halflings-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/hpc-novice/d08fa355c3444ee8981e57f8453e936fc2f00946/assets/fonts/glyphicons-halflings-regular.woff2 -------------------------------------------------------------------------------- /assets/img/dc-icon-black.svg: -------------------------------------------------------------------------------- 1 | 2 | 16 | 18 | 19 | 21 | image/svg+xml 22 | 24 | 25 | 26 | 27 | 28 | 30 | 54 | 59 | 64 | 69 | 74 | 75 | -------------------------------------------------------------------------------- /assets/img/lc-icon-black.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/hpc-novice/d08fa355c3444ee8981e57f8453e936fc2f00946/assets/img/lc-icon-black.png -------------------------------------------------------------------------------- /assets/img/lc-icon-black.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | image/svg+xml 181 | 182 | -------------------------------------------------------------------------------- /assets/img/swc-icon-blue.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /assets/img/swc-logo-blue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/hpc-novice/d08fa355c3444ee8981e57f8453e936fc2f00946/assets/img/swc-logo-blue.png -------------------------------------------------------------------------------- /assets/img/swc-logo-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/hpc-novice/d08fa355c3444ee8981e57f8453e936fc2f00946/assets/img/swc-logo-white.png -------------------------------------------------------------------------------- /assets/js/lesson.js: -------------------------------------------------------------------------------- 1 | // Make all tables striped by default. 2 | $("table").addClass("table table-striped"); 3 | 4 | 5 | // Handle foldable challenges and solutions (on click and at start). 6 | $(".solution").click(function(event) { 7 | var trigger = $(event.target).has(".fold-unfold").size() > 0 8 | || $(event.target).filter(".fold-unfold").size() > 0; 9 | if (trigger) { 10 | $(">*:not(h2)", this).toggle(400); 11 | $(">h2>span.fold-unfold", this).toggleClass("glyphicon-collapse-down glyphicon-collapse-up"); 12 | event.stopPropagation(); 13 | } 14 | }); 15 | $(".solution").each(function() { 16 | $(">*:not(h2)", this).toggle(); 17 | var h2 = $("h2:first", this); 18 | h2.append(""); 19 | }); 20 | 21 | 22 | // Handle searches. 23 | // Relies on document having 'meta' element with name 'search-domain'. 24 | function google_search() { 25 | var query = document.getElementById("google-search").value; 26 | var domain = $("meta[name=search-domain]").attr("value"); 27 | window.open("https://www.google.com/search?q=" + query + "+site:" + domain); 28 | } 29 | -------------------------------------------------------------------------------- /bin/chunk-options.R: -------------------------------------------------------------------------------- 1 | # These settings control the behavior of all chunks in the novice R materials. 2 | # For example, to generate the lessons with all the output hidden, simply change 3 | # `results` from "markup" to "hide". 4 | # For more information on available chunk options, see 5 | # http://yihui.name/knitr/options#chunk_options 6 | 7 | library("knitr") 8 | 9 | fix_fig_path <- function(pth) file.path("..", pth) 10 | 11 | 12 | ## We set the path for the figures globally below, so if we want to 13 | ## customize it for individual episodes, we can append a prefix to the 14 | ## global path. For instance, if we call knitr_fig_path("01-") in the 15 | ## first episode of the lesson, it will generate the figures in 16 | ## `fig/rmd-01-` 17 | knitr_fig_path <- function(prefix) { 18 | new_path <- paste0(opts_chunk$get("fig.path"), 19 | prefix) 20 | opts_chunk$set(fig.path = new_path) 21 | } 22 | 23 | ## We use the rmd- prefix for the figures generated by the lssons so 24 | ## they can be easily identified and deleted by `make clean-rmd`. The 25 | ## working directory when the lessons are generated is the root so the 26 | ## figures need to be saved in fig/, but when the site is generated, 27 | ## the episodes will be one level down. We fix the path using the 28 | ## `fig.process` option. 29 | 30 | opts_chunk$set(tidy = FALSE, results = "markup", comment = NA, 31 | fig.align = "center", fig.path = "fig/rmd-", 32 | fig.process = fix_fig_path) 33 | 34 | # The hooks below add html tags to the code chunks and their output so that they 35 | # are properly formatted when the site is built. 36 | 37 | hook_in <- function(x, options) { 38 | stringr::str_c("\n\n~~~\n", 39 | paste0(x, collapse="\n"), 40 | "\n~~~\n{: .language-r}\n\n") 41 | } 42 | 43 | hook_out <- function(x, options) { 44 | x <- gsub("\n$", "", x) 45 | stringr::str_c("\n\n~~~\n", 46 | paste0(x, collapse="\n"), 47 | "\n~~~\n{: .output}\n\n") 48 | } 49 | 50 | hook_error <- function(x, options) { 51 | x <- gsub("\n$", "", x) 52 | stringr::str_c("\n\n~~~\n", 53 | paste0(x, collapse="\n"), 54 | "\n~~~\n{: .error}\n\n") 55 | } 56 | 57 | knit_hooks$set(source = hook_in, output = hook_out, warning = hook_error, 58 | error = hook_error, message = hook_out) 59 | -------------------------------------------------------------------------------- /bin/extract_figures.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from __future__ import print_function 4 | import sys 5 | import os 6 | import glob 7 | from optparse import OptionParser 8 | 9 | from util import Reporter, read_markdown, IMAGE_FILE_SUFFIX 10 | 11 | def main(): 12 | """Main driver.""" 13 | 14 | args = parse_args() 15 | images = [] 16 | for filename in args.filenames: 17 | images += get_images(args.parser, filename) 18 | save(sys.stdout, images) 19 | 20 | 21 | def parse_args(): 22 | """Parse command-line arguments.""" 23 | 24 | parser = OptionParser() 25 | parser.add_option('-p', '--parser', 26 | default=None, 27 | dest='parser', 28 | help='path to Markdown parser') 29 | 30 | args, extras = parser.parse_args() 31 | require(args.parser is not None, 32 | 'Path to Markdown parser not provided') 33 | require(extras, 34 | 'No filenames specified') 35 | 36 | args.filenames = extras 37 | return args 38 | 39 | 40 | def get_filenames(source_dir): 41 | """Get all filenames to be searched for images.""" 42 | 43 | return glob.glob(os.path.join(source_dir, '*.md')) 44 | 45 | 46 | def get_images(parser, filename): 47 | """Extract all images from file.""" 48 | 49 | content = read_markdown(parser, filename) 50 | result = [] 51 | find_image_nodes(content['doc'], result) 52 | find_image_links(content['doc'], result) 53 | return result 54 | 55 | 56 | def find_image_nodes(doc, result): 57 | """Find all nested nodes representing images.""" 58 | 59 | if (doc['type'] == 'img') or \ 60 | ((doc['type'] == 'html_element') and (doc['value'] == 'img')): 61 | alt = doc['attr'].get('alt', '') 62 | result.append({'alt': alt, 'src': doc['attr']['src']}) 63 | else: 64 | for child in doc.get('children', []): 65 | find_image_nodes(child, result) 66 | 67 | 68 | def find_image_links(doc, result): 69 | """Find all links to files in the 'fig' directory.""" 70 | 71 | if ((doc['type'] == 'a') and ('attr' in doc) and ('href' in doc['attr'])) \ 72 | or \ 73 | ((doc['type'] == 'html_element') and (doc['value'] == 'a') and ('href' in doc['attr'])): 74 | path = doc['attr']['href'] 75 | if os.path.splitext(path)[1].lower() in IMAGE_FILE_SUFFIX: 76 | result.append({'alt':'', 'src': doc['attr']['href']}) 77 | else: 78 | for child in doc.get('children', []): 79 | find_image_links(child, result) 80 | 81 | 82 | def save(stream, images): 83 | """Save results as Markdown.""" 84 | 85 | text = '\n
\n'.join(['

{0}

'.format(img['alt'], img['src']) for img in images]) 86 | print(text, file=stream) 87 | 88 | 89 | def require(condition, message): 90 | """Fail if condition not met.""" 91 | 92 | if not condition: 93 | print(message, file=sys.stderr) 94 | sys.exit(1) 95 | 96 | 97 | if __name__ == '__main__': 98 | main() 99 | -------------------------------------------------------------------------------- /bin/generate_md_episodes.R: -------------------------------------------------------------------------------- 1 | generate_md_episodes <- function() { 2 | 3 | library("methods") 4 | 5 | if (require("knitr") && packageVersion("knitr") < '1.9.19') 6 | stop("knitr must be version 1.9.20 or higher") 7 | 8 | if (!require("stringr")) 9 | stop("The package stringr is required for generating the lessons.") 10 | 11 | if (require("checkpoint") && packageVersion("checkpoint") >= '0.4.0') { 12 | required_pkgs <- 13 | checkpoint:::scanForPackages(project = "_episodes_rmd", 14 | verbose=FALSE, use.knitr = TRUE)$pkgs 15 | } else { 16 | stop("The checkpoint package (>= 0.4.0) is required to build the lessons.") 17 | } 18 | 19 | missing_pkgs <- required_pkgs[!(required_pkgs %in% rownames(installed.packages()))] 20 | 21 | if (length(missing_pkgs)) { 22 | message("Installing missing required packages: ", 23 | paste(missing_pkgs, collapse=", ")) 24 | install.packages(missing_pkgs) 25 | } 26 | 27 | ## find all the Rmd files, and generate the paths for their respective outputs 28 | src_rmd <- list.files(pattern = "??-*.Rmd$", path = "_episodes_rmd", full.names = TRUE) 29 | dest_md <- file.path("_episodes", gsub("Rmd$", "md", basename(src_rmd))) 30 | 31 | ## knit the Rmd into markdown 32 | mapply(function(x, y) { 33 | knitr::knit(x, output = y) 34 | }, src_rmd, dest_md) 35 | 36 | } 37 | 38 | generate_md_episodes() 39 | -------------------------------------------------------------------------------- /bin/knit_lessons.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Only try running R to translate files if there are some files present. 4 | # The Makefile passes in the names of files. 5 | 6 | if [ $# -ne 0 ] ; then 7 | Rscript -e "source('bin/generate_md_episodes.R')" 8 | fi 9 | -------------------------------------------------------------------------------- /bin/lesson_check.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Check lesson files and their contents. 5 | """ 6 | 7 | from __future__ import print_function 8 | import sys 9 | import os 10 | import glob 11 | import json 12 | import re 13 | from optparse import OptionParser 14 | 15 | from util import Reporter, read_markdown, load_yaml, check_unwanted_files, require, IMAGE_FILE_SUFFIX 16 | 17 | __version__ = '0.3' 18 | 19 | # Where to look for source Markdown files. 20 | SOURCE_DIRS = ['', '_episodes', '_extras'] 21 | 22 | # Required files: each entry is ('path': YAML_required). 23 | # FIXME: We do not yet validate whether any files have the required 24 | # YAML headers, but should in the future. 25 | # The '%' is replaced with the source directory path for checking. 26 | # Episodes are handled specially, and extra files in '_extras' are also handled specially. 27 | # This list must include all the Markdown files listed in the 'bin/initialize' script. 28 | REQUIRED_FILES = { 29 | '%/CONDUCT.md': True, 30 | '%/CONTRIBUTING.md': False, 31 | '%/LICENSE.md': True, 32 | '%/README.md': False, 33 | '%/_extras/discuss.md': True, 34 | '%/_extras/guide.md': True, 35 | '%/index.md': True, 36 | '%/reference.md': True, 37 | '%/setup.md': True, 38 | } 39 | 40 | # Episode filename pattern. 41 | P_EPISODE_FILENAME = re.compile(r'/_episodes/(\d\d)-[-\w]+.md$') 42 | 43 | # Pattern to match lines ending with whitespace. 44 | P_TRAILING_WHITESPACE = re.compile(r'\s+$') 45 | 46 | # Pattern to match figure references in HTML. 47 | P_FIGURE_REFS = re.compile(r']+src="([^"]+)"[^>]*>') 48 | 49 | # Pattern to match internally-defined Markdown links. 50 | P_INTERNAL_LINK_REF = re.compile(r'\[([^\]]+)\]\[([^\]]+)\]') 51 | 52 | # Pattern to match reference links (to resolve internally-defined references). 53 | P_INTERNAL_LINK_DEF = re.compile(r'^\[([^\]]+)\]:\s*(.+)') 54 | 55 | # What kinds of blockquotes are allowed? 56 | KNOWN_BLOCKQUOTES = { 57 | 'callout', 58 | 'challenge', 59 | 'checklist', 60 | 'discussion', 61 | 'keypoints', 62 | 'objectives', 63 | 'prereq', 64 | 'quotation', 65 | 'solution', 66 | 'testimonial' 67 | } 68 | 69 | # What kinds of code fragments are allowed? 70 | KNOWN_CODEBLOCKS = { 71 | 'error', 72 | 'output', 73 | 'source', 74 | 'language-bash', 75 | 'html', 76 | 'language-make', 77 | 'language-matlab', 78 | 'language-python', 79 | 'language-r', 80 | 'language-shell', 81 | 'language-sql' 82 | } 83 | 84 | # What fields are required in teaching episode metadata? 85 | TEACHING_METADATA_FIELDS = { 86 | ('title', str), 87 | ('teaching', int), 88 | ('exercises', int), 89 | ('questions', list), 90 | ('objectives', list), 91 | ('keypoints', list) 92 | } 93 | 94 | # What fields are required in break episode metadata? 95 | BREAK_METADATA_FIELDS = { 96 | ('layout', str), 97 | ('title', str), 98 | ('break', int) 99 | } 100 | 101 | # How long are lines allowed to be? 102 | MAX_LINE_LEN = 100 103 | 104 | def main(): 105 | """Main driver.""" 106 | 107 | args = parse_args() 108 | args.reporter = Reporter() 109 | check_config(args.reporter, args.source_dir) 110 | args.references = read_references(args.reporter, args.reference_path) 111 | 112 | docs = read_all_markdown(args.source_dir, args.parser) 113 | check_fileset(args.source_dir, args.reporter, docs.keys()) 114 | check_unwanted_files(args.source_dir, args.reporter) 115 | for filename in docs.keys(): 116 | checker = create_checker(args, filename, docs[filename]) 117 | checker.check() 118 | 119 | args.reporter.report() 120 | 121 | 122 | def parse_args(): 123 | """Parse command-line arguments.""" 124 | 125 | parser = OptionParser() 126 | parser.add_option('-l', '--linelen', 127 | default=False, 128 | action="store_true", 129 | dest='line_lengths', 130 | help='Check line lengths') 131 | parser.add_option('-p', '--parser', 132 | default=None, 133 | dest='parser', 134 | help='path to Markdown parser') 135 | parser.add_option('-r', '--references', 136 | default=None, 137 | dest='reference_path', 138 | help='path to Markdown file of external references') 139 | parser.add_option('-s', '--source', 140 | default=os.curdir, 141 | dest='source_dir', 142 | help='source directory') 143 | parser.add_option('-w', '--whitespace', 144 | default=False, 145 | action="store_true", 146 | dest='trailing_whitespace', 147 | help='Check for trailing whitespace') 148 | 149 | args, extras = parser.parse_args() 150 | require(args.parser is not None, 151 | 'Path to Markdown parser not provided') 152 | require(not extras, 153 | 'Unexpected trailing command-line arguments "{0}"'.format(extras)) 154 | 155 | return args 156 | 157 | 158 | def check_config(reporter, source_dir): 159 | """Check configuration file.""" 160 | 161 | config_file = os.path.join(source_dir, '_config.yml') 162 | config = load_yaml(config_file) 163 | reporter.check_field(config_file, 'configuration', config, 'kind', 'lesson') 164 | reporter.check_field(config_file, 'configuration', config, 'carpentry', ('swc', 'dc', 'lc')) 165 | reporter.check_field(config_file, 'configuration', config, 'title') 166 | reporter.check_field(config_file, 'configuration', config, 'email') 167 | 168 | 169 | reporter.check({'values': {'root': '..'}} in config.get('defaults', []), 170 | 'configuration', 171 | '"root" not set to ".." in configuration') 172 | 173 | 174 | def read_references(reporter, ref_path): 175 | """Read shared file of reference links, returning dictionary of valid references 176 | {symbolic_name : URL} 177 | """ 178 | 179 | result = {} 180 | urls_seen = set() 181 | if ref_path: 182 | with open(ref_path, 'r') as reader: 183 | for (num, line) in enumerate(reader): 184 | line_num = num + 1 185 | m = P_INTERNAL_LINK_DEF.search(line) 186 | require(m, 187 | '{0}:{1} not valid reference:\n{2}'.format(ref_path, line_num, line.rstrip())) 188 | name = m.group(1) 189 | url = m.group(2) 190 | require(name, 191 | 'Empty reference at {0}:{1}'.format(ref_path, line_num)) 192 | reporter.check(name not in result, 193 | ref_path, 194 | 'Duplicate reference {0} at line {1}', 195 | name, line_num) 196 | reporter.check(url not in urls_seen, 197 | ref_path, 198 | 'Duplicate definition of URL {0} at line {1}', 199 | url, line_num) 200 | result[name] = url 201 | urls_seen.add(url) 202 | return result 203 | 204 | 205 | def read_all_markdown(source_dir, parser): 206 | """Read source files, returning 207 | {path : {'metadata':yaml, 'metadata_len':N, 'text':text, 'lines':[(i, line, len)], 'doc':doc}} 208 | """ 209 | 210 | all_dirs = [os.path.join(source_dir, d) for d in SOURCE_DIRS] 211 | all_patterns = [os.path.join(d, '*.md') for d in all_dirs] 212 | result = {} 213 | for pat in all_patterns: 214 | for filename in glob.glob(pat): 215 | data = read_markdown(parser, filename) 216 | if data: 217 | result[filename] = data 218 | return result 219 | 220 | 221 | def check_fileset(source_dir, reporter, filenames_present): 222 | """Are all required files present? Are extraneous files present?""" 223 | 224 | # Check files with predictable names. 225 | required = [p.replace('%', source_dir) for p in REQUIRED_FILES] 226 | missing = set(required) - set(filenames_present) 227 | for m in missing: 228 | reporter.add(None, 'Missing required file {0}', m) 229 | 230 | # Check episode files' names. 231 | seen = [] 232 | for filename in filenames_present: 233 | if '_episodes' not in filename: 234 | continue 235 | m = P_EPISODE_FILENAME.search(filename) 236 | if m and m.group(1): 237 | seen.append(m.group(1)) 238 | else: 239 | reporter.add(None, 'Episode {0} has badly-formatted filename', filename) 240 | 241 | # Check for duplicate episode numbers. 242 | reporter.check(len(seen) == len(set(seen)), 243 | None, 244 | 'Duplicate episode numbers {0} vs {1}', 245 | sorted(seen), sorted(set(seen))) 246 | 247 | # Check that numbers are consecutive. 248 | seen = [int(s) for s in seen] 249 | seen.sort() 250 | clean = True 251 | for i in range(len(seen) - 1): 252 | clean = clean and ((seen[i+1] - seen[i]) == 1) 253 | reporter.check(clean, 254 | None, 255 | 'Missing or non-consecutive episode numbers {0}', 256 | seen) 257 | 258 | 259 | def check_figures(source_dir, reporter): 260 | """Check that all figures are present and referenced.""" 261 | 262 | # Get references. 263 | try: 264 | all_figures_html = os.path.join(source_dir, '_includes', 'all_figures.html') 265 | with open(all_figures_html, 'r') as reader: 266 | text = reader.read() 267 | figures = P_FIGURE_REFS.findall(text) 268 | referenced = [os.path.split(f)[1] for f in figures if '/fig/' in f] 269 | except FileNotFoundError as e: 270 | reporter.add(all_figures_html, 271 | 'File not found') 272 | return 273 | 274 | # Get actual image files (ignore non-image files). 275 | fig_dir_path = os.path.join(source_dir, 'fig') 276 | actual = [f for f in os.listdir(fig_dir_path) if os.path.splitext(f)[1] in IMAGE_FILE_SUFFIX] 277 | 278 | # Report differences. 279 | unexpected = set(actual) - set(referenced) 280 | reporter.check(not unexpected, 281 | None, 282 | 'Unexpected image files: {0}', 283 | ', '.join(sorted(unexpected))) 284 | missing = set(referenced) - set(actual) 285 | reporter.check(not missing, 286 | None, 287 | 'Missing image files: {0}', 288 | ', '.join(sorted(missing))) 289 | 290 | 291 | def create_checker(args, filename, info): 292 | """Create appropriate checker for file.""" 293 | 294 | for (pat, cls) in CHECKERS: 295 | if pat.search(filename): 296 | return cls(args, filename, **info) 297 | 298 | 299 | class CheckBase(object): 300 | """Base class for checking Markdown files.""" 301 | 302 | def __init__(self, args, filename, metadata, metadata_len, text, lines, doc): 303 | """Cache arguments for checking.""" 304 | 305 | super(CheckBase, self).__init__() 306 | self.args = args 307 | self.reporter = self.args.reporter # for convenience 308 | self.filename = filename 309 | self.metadata = metadata 310 | self.metadata_len = metadata_len 311 | self.text = text 312 | self.lines = lines 313 | self.doc = doc 314 | 315 | self.layout = None 316 | 317 | 318 | def check(self): 319 | """Run tests.""" 320 | 321 | self.check_metadata() 322 | self.check_line_lengths() 323 | self.check_trailing_whitespace() 324 | self.check_blockquote_classes() 325 | self.check_codeblock_classes() 326 | self.check_defined_link_references() 327 | 328 | 329 | def check_metadata(self): 330 | """Check the YAML metadata.""" 331 | 332 | self.reporter.check(self.metadata is not None, 333 | self.filename, 334 | 'Missing metadata entirely') 335 | 336 | if self.metadata and (self.layout is not None): 337 | self.reporter.check_field(self.filename, 'metadata', self.metadata, 'layout', self.layout) 338 | 339 | 340 | def check_line_lengths(self): 341 | """Check the raw text of the lesson body.""" 342 | 343 | if self.args.line_lengths: 344 | over = [i for (i, l, n) in self.lines if (n > MAX_LINE_LEN) and (not l.startswith('!'))] 345 | self.reporter.check(not over, 346 | self.filename, 347 | 'Line(s) are too long: {0}', 348 | ', '.join([str(i) for i in over])) 349 | 350 | 351 | def check_trailing_whitespace(self): 352 | """Check for whitespace at the ends of lines.""" 353 | 354 | if self.args.trailing_whitespace: 355 | trailing = [i for (i, l, n) in self.lines if P_TRAILING_WHITESPACE.match(l)] 356 | self.reporter.check(not trailing, 357 | self.filename, 358 | 'Line(s) end with whitespace: {0}', 359 | ', '.join([str(i) for i in trailing])) 360 | 361 | 362 | def check_blockquote_classes(self): 363 | """Check that all blockquotes have known classes.""" 364 | 365 | for node in self.find_all(self.doc, {'type' : 'blockquote'}): 366 | cls = self.get_val(node, 'attr', 'class') 367 | self.reporter.check(cls in KNOWN_BLOCKQUOTES, 368 | (self.filename, self.get_loc(node)), 369 | 'Unknown or missing blockquote type {0}', 370 | cls) 371 | 372 | 373 | def check_codeblock_classes(self): 374 | """Check that all code blocks have known classes.""" 375 | 376 | for node in self.find_all(self.doc, {'type' : 'codeblock'}): 377 | cls = self.get_val(node, 'attr', 'class') 378 | self.reporter.check(cls in KNOWN_CODEBLOCKS, 379 | (self.filename, self.get_loc(node)), 380 | 'Unknown or missing code block type {0}', 381 | cls) 382 | 383 | 384 | def check_defined_link_references(self): 385 | """Check that defined links resolve in the file. 386 | 387 | Internally-defined links match the pattern [text][label]. 388 | """ 389 | 390 | result = set() 391 | for node in self.find_all(self.doc, {'type' : 'text'}): 392 | for match in P_INTERNAL_LINK_REF.findall(node['value']): 393 | text = match[0] 394 | link = match[1] 395 | if link not in self.args.references: 396 | result.add('"{0}"=>"{1}"'.format(text, link)) 397 | self.reporter.check(not result, 398 | self.filename, 399 | 'Internally-defined links may be missing definitions: {0}', 400 | ', '.join(sorted(result))) 401 | 402 | 403 | def find_all(self, node, pattern, accum=None): 404 | """Find all matches for a pattern.""" 405 | 406 | assert type(pattern) == dict, 'Patterns must be dictionaries' 407 | if accum is None: 408 | accum = [] 409 | if self.match(node, pattern): 410 | accum.append(node) 411 | for child in node.get('children', []): 412 | self.find_all(child, pattern, accum) 413 | return accum 414 | 415 | 416 | def match(self, node, pattern): 417 | """Does this node match the given pattern?""" 418 | 419 | for key in pattern: 420 | if key not in node: 421 | return False 422 | val = pattern[key] 423 | if type(val) == str: 424 | if node[key] != val: 425 | return False 426 | elif type(val) == dict: 427 | if not self.match(node[key], val): 428 | return False 429 | return True 430 | 431 | 432 | def get_val(self, node, *chain): 433 | """Get value one or more levels down.""" 434 | 435 | curr = node 436 | for selector in chain: 437 | curr = curr.get(selector, None) 438 | if curr is None: 439 | break 440 | return curr 441 | 442 | 443 | def get_loc(self, node): 444 | """Convenience method to get node's line number.""" 445 | 446 | result = self.get_val(node, 'options', 'location') 447 | if self.metadata_len is not None: 448 | result += self.metadata_len 449 | return result 450 | 451 | 452 | class CheckNonJekyll(CheckBase): 453 | """Check a file that isn't translated by Jekyll.""" 454 | 455 | def __init__(self, args, filename, metadata, metadata_len, text, lines, doc): 456 | super(CheckNonJekyll, self).__init__(args, filename, metadata, metadata_len, text, lines, doc) 457 | 458 | 459 | def check_metadata(self): 460 | self.reporter.check(self.metadata is None, 461 | self.filename, 462 | 'Unexpected metadata') 463 | 464 | 465 | class CheckIndex(CheckBase): 466 | """Check the main index page.""" 467 | 468 | def __init__(self, args, filename, metadata, metadata_len, text, lines, doc): 469 | super(CheckIndex, self).__init__(args, filename, metadata, metadata_len, text, lines, doc) 470 | self.layout = 'lesson' 471 | 472 | def check_metadata(self): 473 | super(CheckIndex, self).check_metadata() 474 | self.reporter.check(self.metadata.get('root', '') == '.', 475 | self.filename, 476 | 'Root not set to "."') 477 | 478 | 479 | class CheckEpisode(CheckBase): 480 | """Check an episode page.""" 481 | 482 | def __init__(self, args, filename, metadata, metadata_len, text, lines, doc): 483 | super(CheckEpisode, self).__init__(args, filename, metadata, metadata_len, text, lines, doc) 484 | 485 | 486 | def check(self): 487 | """Run extra tests.""" 488 | 489 | super(CheckEpisode, self).check() 490 | self.check_reference_inclusion() 491 | 492 | 493 | def check_metadata(self): 494 | super(CheckEpisode, self).check_metadata() 495 | if self.metadata: 496 | if 'layout' in self.metadata: 497 | if self.metadata['layout'] == 'break': 498 | self.check_metadata_fields(BREAK_METADATA_FIELDS) 499 | else: 500 | self.reporter.add(self.filename, 501 | 'Unknown episode layout "{0}"', 502 | self.metadata['layout']) 503 | else: 504 | self.check_metadata_fields(TEACHING_METADATA_FIELDS) 505 | 506 | 507 | def check_metadata_fields(self, expected): 508 | for (name, type_) in expected: 509 | if name not in self.metadata: 510 | self.reporter.add(self.filename, 511 | 'Missing metadata field {0}', 512 | name) 513 | elif type(self.metadata[name]) != type_: 514 | self.reporter.add(self.filename, 515 | '"{0}" has wrong type in metadata ({1} instead of {2})', 516 | name, type(self.metadata[name]), type_) 517 | 518 | 519 | def check_reference_inclusion(self): 520 | """Check that links file has been included.""" 521 | 522 | if not self.args.reference_path: 523 | return 524 | 525 | for (i, last_line, line_len) in reversed(self.lines): 526 | if last_line: 527 | break 528 | 529 | require(last_line, 530 | 'No non-empty lines in {0}'.format(self.filename)) 531 | 532 | include_filename = os.path.split(self.args.reference_path)[-1] 533 | if include_filename not in last_line: 534 | self.reporter.add(self.filename, 535 | 'episode does not include "{0}"', 536 | include_filename) 537 | 538 | 539 | class CheckReference(CheckBase): 540 | """Check the reference page.""" 541 | 542 | def __init__(self, args, filename, metadata, metadata_len, text, lines, doc): 543 | super(CheckReference, self).__init__(args, filename, metadata, metadata_len, text, lines, doc) 544 | self.layout = 'reference' 545 | 546 | 547 | class CheckGeneric(CheckBase): 548 | """Check a generic page.""" 549 | 550 | def __init__(self, args, filename, metadata, metadata_len, text, lines, doc): 551 | super(CheckGeneric, self).__init__(args, filename, metadata, metadata_len, text, lines, doc) 552 | self.layout = 'page' 553 | 554 | 555 | CHECKERS = [ 556 | (re.compile(r'CONTRIBUTING\.md'), CheckNonJekyll), 557 | (re.compile(r'README\.md'), CheckNonJekyll), 558 | (re.compile(r'index\.md'), CheckIndex), 559 | (re.compile(r'reference\.md'), CheckReference), 560 | (re.compile(r'_episodes/.*\.md'), CheckEpisode), 561 | (re.compile(r'.*\.md'), CheckGeneric) 562 | ] 563 | 564 | 565 | if __name__ == '__main__': 566 | main() 567 | -------------------------------------------------------------------------------- /bin/lesson_initialize.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """Initialize a newly-created repository.""" 4 | 5 | 6 | from __future__ import print_function 7 | import sys 8 | import os 9 | 10 | ROOT_AUTHORS = '''\ 11 | FIXME: list authors' names and email addresses. 12 | ''' 13 | 14 | ROOT_CITATION = '''\ 15 | FIXME: describe how to cite this lesson. 16 | ''' 17 | 18 | ROOT_CONTRIBUTING_MD = '''\ 19 | # Contributing 20 | 21 | [Software Carpentry][swc-site] and [Data Carpentry][dc-site] are open source projects, 22 | and we welcome contributions of all kinds: 23 | new lessons, 24 | fixes to existing material, 25 | bug reports, 26 | and reviews of proposed changes are all welcome. 27 | 28 | ## Contributor Agreement 29 | 30 | By contributing, 31 | you agree that we may redistribute your work under [our license](LICENSE.md). 32 | In exchange, 33 | we will address your issues and/or assess your change proposal as promptly as we can, 34 | and help you become a member of our community. 35 | Everyone involved in [Software Carpentry][swc-site] and [Data Carpentry][dc-site] 36 | agrees to abide by our [code of conduct](CONDUCT.md). 37 | 38 | ## How to Contribute 39 | 40 | The easiest way to get started is to file an issue 41 | to tell us about a spelling mistake, 42 | some awkward wording, 43 | or a factual error. 44 | This is a good way to introduce yourself 45 | and to meet some of our community members. 46 | 47 | 1. If you do not have a [GitHub][github] account, 48 | you can [send us comments by email][email]. 49 | 50 | However, 51 | we will be able to respond more quickly if you use one of the other methods described below. 52 | 53 | 2. If you have a [GitHub][github] account, 54 | or are willing to [create one][github-join], 55 | but do not know how to use Git, 56 | you can report problems or suggest improvements by [creating an issue][issues]. 57 | This allows us to assign the item to someone 58 | and to respond to it in a threaded discussion. 59 | 60 | 3. If you are comfortable with Git, 61 | and would like to add or change material, 62 | you can submit a pull request (PR). 63 | Instructions for doing this are [included below](#using-github). 64 | 65 | ## Where to Contribute 66 | 67 | 1. If you wish to change this lesson, 68 | please work in , 69 | which can be viewed at . 70 | 71 | 2. If you wish to change the example lesson, 72 | please work in , 73 | which documents the format of our lessons 74 | and can be viewed at . 75 | 76 | 3. If you wish to change the template used for workshop websites, 77 | please work in . 78 | The home page of that repository explains how to set up workshop websites, 79 | while the extra pages in 80 | provide more background on our design choices. 81 | 82 | 4. If you wish to change CSS style files, tools, 83 | or HTML boilerplate for lessons or workshops stored in `_includes` or `_layouts`, 84 | please work in . 85 | 86 | ## What to Contribute 87 | 88 | There are many ways to contribute, 89 | from writing new exercises and improving existing ones 90 | to updating or filling in the documentation 91 | and and submitting [bug reports][issues] 92 | about things that don't work, aren't clear, or are missing. 93 | If you are looking for ideas, 94 | please see [the list of issues for this repository][issues], 95 | or the issues for [Data Carpentry][dc-issues] 96 | and [Software Carpentry][swc-issues] projects. 97 | 98 | Comments on issues and reviews of pull requests are just as welcome: 99 | we are smarter together than we are on our own. 100 | Reviews from novices and newcomers are particularly valuable: 101 | it's easy for people who have been using these lessons for a while 102 | to forget how impenetrable some of this material can be, 103 | so fresh eyes are always welcome. 104 | 105 | ## What *Not* to Contribute 106 | 107 | Our lessons already contain more material than we can cover in a typical workshop, 108 | so we are usually *not* looking for more concepts or tools to add to them. 109 | As a rule, 110 | if you want to introduce a new idea, 111 | you must (a) estimate how long it will take to teach 112 | and (b) explain what you would take out to make room for it. 113 | The first encourages contributors to be honest about requirements; 114 | the second, to think hard about priorities. 115 | 116 | We are also not looking for exercises or other material that only run on one platform. 117 | Our workshops typically contain a mixture of Windows, Mac OS X, and Linux users; 118 | in order to be usable, 119 | our lessons must run equally well on all three. 120 | 121 | ## Using GitHub 122 | 123 | If you choose to contribute via GitHub, 124 | you may want to look at 125 | [How to Contribute to an Open Source Project on GitHub][how-contribute]. 126 | In brief: 127 | 128 | 1. The published copy of the lesson is in the `gh-pages` branch of the repository 129 | (so that GitHub will regenerate it automatically). 130 | Please create all branches from that, 131 | and merge the [master repository][repo]'s `gh-pages` branch into your `gh-pages` branch 132 | before starting work. 133 | Please do *not* work directly in your `gh-pages` branch, 134 | since that will make it difficult for you to work on other contributions. 135 | 136 | 2. We use [GitHub flow][github-flow] to manage changes: 137 | 1. Create a new branch in your desktop copy of this repository for each significant change. 138 | 2. Commit the change in that branch. 139 | 3. Push that branch to your fork of this repository on GitHub. 140 | 4. Submit a pull request from that branch to the [master repository][repo]. 141 | 5. If you receive feedback, 142 | make changes on your desktop and push to your branch on GitHub: 143 | the pull request will update automatically. 144 | 145 | Each lesson has two maintainers who review issues and pull requests 146 | or encourage others to do so. 147 | The maintainers are community volunteers, 148 | and have final say over what gets merged into the lesson. 149 | 150 | ## Other Resources 151 | 152 | General discussion of [Software Carpentry][swc-site] and [Data Carpentry][dc-site] 153 | happens on the [discussion mailing list][discuss-list], 154 | which everyone is welcome to join. 155 | You can also [reach us by email][email]. 156 | 157 | [email]: mailto:admin@software-carpentry.org 158 | [dc-issues]: https://github.com/issues?q=user%3Adatacarpentry 159 | [dc-lessons]: http://datacarpentry.org/lessons/ 160 | [dc-site]: http://datacarpentry.org/ 161 | [discuss-list]: http://lists.software-carpentry.org/listinfo/discuss 162 | [github]: http://github.com 163 | [github-flow]: https://guides.github.com/introduction/flow/ 164 | [github-join]: https://github.com/join 165 | [how-contribute]: https://egghead.io/series/how-to-contribute-to-an-open-source-project-on-github 166 | [issues]: https://github.com/swcarpentry/FIXME/issues/ 167 | [repo]: https://github.com/swcarpentry/FIXME/ 168 | [swc-issues]: https://github.com/issues?q=user%3Aswcarpentry 169 | [swc-lessons]: http://software-carpentry.org/lessons/ 170 | [swc-site]: http://software-carpentry.org/ 171 | ''' 172 | 173 | ROOT_CONFIG_YML = '''\ 174 | #------------------------------------------------------------ 175 | # Values for this lesson. 176 | #------------------------------------------------------------ 177 | 178 | # Which carpentry is this ("swc", "dc", or "lc")? 179 | carpentry: "swc" 180 | 181 | # Overall title for pages. 182 | title: "Lesson Title" 183 | 184 | # Contact. This *must* include the protocol: if it's an email 185 | # address, it must look like "mailto:lessons@software-carpentry.org", 186 | # or if it's a URL, "https://gitter.im/username/ProjectName". 187 | email: "mailto:lessons@software-carpentry.org" 188 | 189 | #------------------------------------------------------------ 190 | # Generic settings (should not need to change). 191 | #------------------------------------------------------------ 192 | 193 | # What kind of thing is this ("workshop" or "lesson")? 194 | kind: "lesson" 195 | 196 | # Magic to make URLs resolve both locally and on GitHub. 197 | # See https://help.github.com/articles/repository-metadata-on-github-pages/. 198 | repository: / 199 | 200 | # Sites. 201 | amy_site: "https://amy.software-carpentry.org/workshops" 202 | dc_site: "http://datacarpentry.org" 203 | swc_github: "https://github.com/swcarpentry" 204 | swc_site: "https://software-carpentry.org" 205 | swc_pages: "https://swcarpentry.github.io" 206 | lc_site: "http://librarycarpentry.github.io/" 207 | template_repo: "https://github.com/swcarpentry/styles" 208 | example_repo: "https://github.com/swcarpentry/lesson-example" 209 | example_site: "https://swcarpentry.github.com/lesson-example" 210 | workshop_repo: "https://github.com/swcarpentry/workshop-template" 211 | workshop_site: "https://swcarpentry.github.io/workshop-template" 212 | training_site: "https://swcarpentry.github.io/instructor-training" 213 | 214 | # Surveys. 215 | pre_survey: "https://www.surveymonkey.com/r/swc_pre_workshop_v1?workshop_id=" 216 | post_survey: "https://www.surveymonkey.com/r/swc_post_workshop_v1?workshop_id=" 217 | training_post_survey: "https://www.surveymonkey.com/r/post-instructor-training" 218 | 219 | # Start time in minutes (0 to be clock-independent, 540 to show a start at 09:00 am). 220 | start_time: 0 221 | 222 | # Specify that things in the episodes collection should be output. 223 | collections: 224 | episodes: 225 | output: true 226 | permalink: /:path/index.html 227 | extras: 228 | output: true 229 | permalink: /:path/index.html 230 | 231 | # Set the default layout for things in the episodes collection. 232 | defaults: 233 | - values: 234 | root: .. 235 | - scope: 236 | path: "" 237 | type: episodes 238 | values: 239 | layout: episode 240 | 241 | # Files and directories that are not to be copied. 242 | exclude: 243 | - Makefile 244 | - bin 245 | 246 | # Turn on built-in syntax highlighting. 247 | highlighter: rouge 248 | ''' 249 | 250 | ROOT_INDEX_MD = '''\ 251 | --- 252 | layout: lesson 253 | root: . 254 | permalink: index.html # Is the only page that don't follow the partner /:path/index.html 255 | --- 256 | FIXME: home page introduction 257 | 258 | > ## Prerequisites 259 | > 260 | > FIXME 261 | {: .prereq} 262 | ''' 263 | 264 | ROOT_REFERENCE_MD = '''\ 265 | --- 266 | layout: reference 267 | root: . 268 | --- 269 | 270 | ## Glossary 271 | 272 | FIXME 273 | ''' 274 | 275 | ROOT_SETUP_MD = '''\ 276 | --- 277 | layout: page 278 | title: Setup 279 | root: . 280 | --- 281 | FIXME 282 | ''' 283 | 284 | ROOT_AIO_MD = '''\ 285 | --- 286 | layout: page 287 | root: . 288 | --- 289 | 315 | {% comment %} 316 | Create anchor for each one of the episodes. 317 | {% endcomment %} 318 | {% for episode in site.episodes %} 319 |
320 | {% endfor %} 321 | ''' 322 | 323 | EPISODES_INTRODUCTION_MD = '''\ 324 | --- 325 | title: "Introduction" 326 | teaching: 0 327 | exercises: 0 328 | questions: 329 | - "Key question" 330 | objectives: 331 | - "First objective." 332 | keypoints: 333 | - "First key point." 334 | --- 335 | ''' 336 | 337 | EXTRAS_ABOUT_MD = '''\ 338 | --- 339 | layout: page 340 | title: About 341 | --- 342 | {% include carpentries.html %} 343 | ''' 344 | 345 | EXTRAS_DISCUSS_MD = '''\ 346 | --- 347 | layout: page 348 | title: Discussion 349 | --- 350 | FIXME 351 | ''' 352 | 353 | EXTRAS_FIGURES_MD = '''\ 354 | --- 355 | layout: page 356 | title: Figures 357 | --- 358 | 387 | {% comment %} 388 | Create anchor for each one of the episodes. 389 | {% endcomment %} 390 | {% for episode in site.episodes %} 391 |
392 | {% endfor %} 393 | ''' 394 | 395 | EXTRAS_GUIDE_MD = '''\ 396 | --- 397 | layout: page 398 | title: "Instructor Notes" 399 | --- 400 | FIXME 401 | ''' 402 | 403 | BOILERPLATE = ( 404 | ('AUTHORS', ROOT_AUTHORS), 405 | ('CITATION', ROOT_CITATION), 406 | ('CONTRIBUTING.md', ROOT_CONTRIBUTING_MD), 407 | ('_config.yml', ROOT_CONFIG_YML), 408 | ('index.md', ROOT_INDEX_MD), 409 | ('reference.md', ROOT_REFERENCE_MD), 410 | ('setup.md', ROOT_SETUP_MD), 411 | ('aio.md', ROOT_AIO_MD), 412 | ('_episodes/01-introduction.md', EPISODES_INTRODUCTION_MD), 413 | ('_extras/about.md', EXTRAS_ABOUT_MD), 414 | ('_extras/discuss.md', EXTRAS_DISCUSS_MD), 415 | ('_extras/figures.md', EXTRAS_FIGURES_MD), 416 | ('_extras/guide.md', EXTRAS_GUIDE_MD), 417 | ) 418 | 419 | 420 | def main(): 421 | """Check for collisions, then create.""" 422 | 423 | # Check. 424 | errors = False 425 | for (path, _) in BOILERPLATE: 426 | if os.path.exists(path): 427 | print('Warning: {0} already exists.'.format(path), file=sys.stderr) 428 | errors = True 429 | if errors: 430 | print('**Exiting without creating files.**', file=sys.stderr) 431 | sys.exit(1) 432 | 433 | # Create. 434 | for (path, content) in BOILERPLATE: 435 | with open(path, 'w') as writer: 436 | writer.write(content) 437 | 438 | 439 | if __name__ == '__main__': 440 | main() 441 | -------------------------------------------------------------------------------- /bin/markdown_ast.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | # Use Kramdown parser to produce AST for Markdown document. 4 | 5 | require "kramdown" 6 | require "json" 7 | 8 | markdown = STDIN.read() 9 | doc = Kramdown::Document.new(markdown) 10 | tree = doc.to_hash_a_s_t 11 | puts JSON.pretty_generate(tree) 12 | -------------------------------------------------------------------------------- /bin/repo_check.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Check repository settings. 5 | """ 6 | 7 | from __future__ import print_function 8 | import sys 9 | import os 10 | from subprocess import Popen, PIPE 11 | import re 12 | from optparse import OptionParser 13 | 14 | from util import Reporter, load_yaml, require 15 | 16 | # Import this way to produce a more useful error message. 17 | try: 18 | import requests 19 | except ImportError: 20 | print('Unable to import requests module: please install requests', file=sys.stderr) 21 | sys.exit(1) 22 | 23 | 24 | # Pattern to match Git command-line output for remotes => (user name, project name). 25 | P_GIT_REMOTE = re.compile(r'upstream\s+[^:]+:([^/]+)/([^.]+)\.git\s+\(fetch\)') 26 | 27 | # Repository URL format string. 28 | F_REPO_URL = 'https://github.com/{0}/{1}/' 29 | 30 | # Pattern to match repository URLs => (user name, project name) 31 | P_REPO_URL = re.compile(r'https?://github\.com/([^.]+)/([^/]+)/?') 32 | 33 | # API URL format string. 34 | F_API_URL = 'https://api.github.com/repos/{0}/{1}/labels' 35 | 36 | # Expected labels and colors. 37 | EXPECTED = { 38 | 'bug' : 'bd2c00', 39 | 'discussion' : 'fc8dc1', 40 | 'enhancement' : '9cd6dc', 41 | 'help-wanted' : 'f4fd9c', 42 | 'instructor-training' : '6e5494', 43 | 'newcomer-friendly' : 'eec275', 44 | 'question' : '808040', 45 | 'template-and-tools' : '2b3990', 46 | 'work-in-progress' : '7ae78e' 47 | } 48 | 49 | 50 | def main(): 51 | """ 52 | Main driver. 53 | """ 54 | 55 | args = parse_args() 56 | reporter = Reporter() 57 | repo_url = get_repo_url(args.source_dir, args.repo_url) 58 | check_labels(reporter, repo_url) 59 | reporter.report() 60 | 61 | 62 | def parse_args(): 63 | """ 64 | Parse command-line arguments. 65 | """ 66 | 67 | parser = OptionParser() 68 | parser.add_option('-r', '--repo', 69 | default=None, 70 | dest='repo_url', 71 | help='repository URL') 72 | parser.add_option('-s', '--source', 73 | default=os.curdir, 74 | dest='source_dir', 75 | help='source directory') 76 | 77 | args, extras = parser.parse_args() 78 | require(not extras, 79 | 'Unexpected trailing command-line arguments "{0}"'.format(extras)) 80 | 81 | return args 82 | 83 | 84 | def get_repo_url(source_dir, repo_url): 85 | """ 86 | Figure out which repository to query. 87 | """ 88 | 89 | # Explicitly specified. 90 | if repo_url is not None: 91 | return repo_url 92 | 93 | # Guess. 94 | cmd = 'git remote -v' 95 | p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, close_fds=True, universal_newlines=True) 96 | stdout_data, stderr_data = p.communicate() 97 | stdout_data = stdout_data.split('\n') 98 | matches = [P_GIT_REMOTE.match(line) for line in stdout_data] 99 | matches = [m for m in matches if m is not None] 100 | require(len(matches) == 1, 101 | 'Unexpected output from git remote command: "{0}"'.format(matches)) 102 | 103 | username = matches[0].group(1) 104 | require(username, 'empty username in git remote output {0}'.format(matches[0])) 105 | 106 | project_name = matches[0].group(2) 107 | require(username, 'empty project name in git remote output {0}'.format(matches[0])) 108 | 109 | url = F_REPO_URL.format(username, project_name) 110 | return url 111 | 112 | 113 | def check_labels(reporter, repo_url): 114 | """ 115 | Check labels in repository. 116 | """ 117 | 118 | actual = get_labels(repo_url) 119 | extra = set(actual.keys()) - set(EXPECTED.keys()) 120 | 121 | reporter.check(not extra, 122 | None, 123 | 'Extra label(s) in repository {0}: {1}', 124 | repo_url, ', '.join(sorted(extra))) 125 | 126 | missing = set(EXPECTED.keys()) - set(actual.keys()) 127 | reporter.check(not missing, 128 | None, 129 | 'Missing label(s) in repository {0}: {1}', 130 | repo_url, ', '.join(sorted(missing))) 131 | 132 | overlap = set(EXPECTED.keys()).intersection(set(actual.keys())) 133 | for name in sorted(overlap): 134 | reporter.check(EXPECTED[name] == actual[name], 135 | None, 136 | 'Color mis-match for label {0} in {1}: expected {2}, found {3}', 137 | name, repo_url, EXPECTED[name], actual[name]) 138 | 139 | 140 | def get_labels(repo_url): 141 | """ 142 | Get actual labels from repository. 143 | """ 144 | 145 | m = P_REPO_URL.match(repo_url) 146 | require(m, 'repository URL {0} does not match expected pattern'.format(repo_url)) 147 | 148 | username = m.group(1) 149 | require(username, 'empty username in repository URL {0}'.format(repo_url)) 150 | 151 | project_name = m.group(2) 152 | require(username, 'empty project name in repository URL {0}'.format(repo_url)) 153 | 154 | url = F_API_URL.format(username, project_name) 155 | r = requests.get(url) 156 | require(r.status_code == 200, 157 | 'Request for {0} failed with {1}'.format(url, r.status_code)) 158 | 159 | result = {} 160 | for entry in r.json(): 161 | result[entry['name']] = entry['color'] 162 | return result 163 | 164 | 165 | if __name__ == '__main__': 166 | main() 167 | -------------------------------------------------------------------------------- /bin/test_lesson_check.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import lesson_check 4 | import util 5 | 6 | class TestFileList(unittest.TestCase): 7 | def setUp(self): 8 | self.reporter = util.Reporter() ## TODO: refactor reporter class. 9 | 10 | def test_file_list_has_expected_entries(self): 11 | # For first pass, simply assume that all required files are present 12 | all_filenames = [filename.replace('%', '') 13 | for filename in lesson_check.REQUIRED_FILES] 14 | 15 | lesson_check.check_fileset('', self.reporter, all_filenames) 16 | self.assertEqual(len(self.reporter.messages), 0) 17 | 18 | if __name__ == "__main__": 19 | unittest.main() 20 | -------------------------------------------------------------------------------- /bin/util.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import sys 3 | import os 4 | import json 5 | from subprocess import Popen, PIPE 6 | 7 | # Import this way to produce a more useful error message. 8 | try: 9 | import yaml 10 | except ImportError: 11 | print('Unable to import YAML module: please install PyYAML', file=sys.stderr) 12 | sys.exit(1) 13 | 14 | 15 | # Things an image file's name can end with. 16 | IMAGE_FILE_SUFFIX = { 17 | '.gif', 18 | '.jpg', 19 | '.png', 20 | '.svg' 21 | } 22 | 23 | # Files that shouldn't be present. 24 | UNWANTED_FILES = [ 25 | '.nojekyll' 26 | ] 27 | 28 | # Marker to show that an expected value hasn't been provided. 29 | # (Can't use 'None' because that might be a legitimate value.) 30 | REPORTER_NOT_SET = [] 31 | 32 | class Reporter(object): 33 | """Collect and report errors.""" 34 | 35 | def __init__(self): 36 | """Constructor.""" 37 | 38 | super(Reporter, self).__init__() 39 | self.messages = [] 40 | 41 | 42 | def check_field(self, filename, name, values, key, expected=REPORTER_NOT_SET): 43 | """Check that a dictionary has an expected value.""" 44 | 45 | if key not in values: 46 | self.add(filename, '{0} does not contain {1}', name, key) 47 | elif expected is REPORTER_NOT_SET: 48 | pass 49 | elif type(expected) in (tuple, set, list): 50 | if values[key] not in expected: 51 | self.add(filename, '{0} {1} value {2} is not in {3}', name, key, values[key], expected) 52 | elif values[key] != expected: 53 | self.add(filename, '{0} {1} is {2} not {3}', name, key, values[key], expected) 54 | 55 | 56 | def check(self, condition, location, fmt, *args): 57 | """Append error if condition not met.""" 58 | 59 | if not condition: 60 | self.add(location, fmt, *args) 61 | 62 | 63 | def add(self, location, fmt, *args): 64 | """Append error unilaterally.""" 65 | 66 | self.messages.append((location, fmt.format(*args))) 67 | 68 | 69 | def report(self, stream=sys.stdout): 70 | """Report all messages in order.""" 71 | 72 | if not self.messages: 73 | return 74 | 75 | def pretty(item): 76 | location, message = item 77 | if isinstance(location, type(None)): 78 | return message 79 | elif isinstance(location, str): 80 | return location + ': ' + message 81 | elif isinstance(location, tuple): 82 | return '{0}:{1}: '.format(*location) + message 83 | else: 84 | assert False, 'Unknown item "{0}"'.format(item) 85 | 86 | def key(item): 87 | location, message = item 88 | if isinstance(location, type(None)): 89 | return ('', -1, message) 90 | elif isinstance(location, str): 91 | return (location, -1, message) 92 | elif isinstance(location, tuple): 93 | return (location[0], location[1], message) 94 | else: 95 | assert False, 'Unknown item "{0}"'.format(item) 96 | 97 | for m in sorted(self.messages, key=key): 98 | print(pretty(m), file=stream) 99 | 100 | 101 | def read_markdown(parser, path): 102 | """ 103 | Get YAML and AST for Markdown file, returning 104 | {'metadata':yaml, 'metadata_len':N, 'text':text, 'lines':[(i, line, len)], 'doc':doc}. 105 | """ 106 | 107 | # Split and extract YAML (if present). 108 | with open(path, 'r') as reader: 109 | body = reader.read() 110 | metadata_raw, metadata_yaml, body = split_metadata(path, body) 111 | 112 | # Split into lines. 113 | metadata_len = 0 if metadata_raw is None else metadata_raw.count('\n') 114 | lines = [(metadata_len+i+1, line, len(line)) for (i, line) in enumerate(body.split('\n'))] 115 | 116 | # Parse Markdown. 117 | cmd = 'ruby {0}'.format(parser) 118 | p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, close_fds=True, universal_newlines=True) 119 | stdout_data, stderr_data = p.communicate(body) 120 | doc = json.loads(stdout_data) 121 | 122 | return { 123 | 'metadata': metadata_yaml, 124 | 'metadata_len': metadata_len, 125 | 'text': body, 126 | 'lines': lines, 127 | 'doc': doc 128 | } 129 | 130 | 131 | def split_metadata(path, text): 132 | """ 133 | Get raw (text) metadata, metadata as YAML, and rest of body. 134 | If no metadata, return (None, None, body). 135 | """ 136 | 137 | metadata_raw = None 138 | metadata_yaml = None 139 | metadata_len = None 140 | 141 | pieces = text.split('---', 2) 142 | if len(pieces) == 3: 143 | metadata_raw = pieces[1] 144 | text = pieces[2] 145 | try: 146 | metadata_yaml = yaml.load(metadata_raw) 147 | except yaml.YAMLError as e: 148 | print('Unable to parse YAML header in {0}:\n{1}'.format(path, e), file=sys.stderr) 149 | sys.exit(1) 150 | 151 | return metadata_raw, metadata_yaml, text 152 | 153 | 154 | def load_yaml(filename): 155 | """ 156 | Wrapper around YAML loading so that 'import yaml' is only needed 157 | in one file. 158 | """ 159 | 160 | try: 161 | with open(filename, 'r') as reader: 162 | return yaml.load(reader) 163 | except (yaml.YAMLError, FileNotFoundError) as e: 164 | print('Unable to load YAML file {0}:\n{1}'.format(filename, e), file=sys.stderr) 165 | sys.exit(1) 166 | 167 | 168 | def check_unwanted_files(dir_path, reporter): 169 | """ 170 | Check that unwanted files are not present. 171 | """ 172 | 173 | for filename in UNWANTED_FILES: 174 | path = os.path.join(dir_path, filename) 175 | reporter.check(not os.path.exists(path), 176 | path, 177 | "Unwanted file found") 178 | 179 | 180 | def require(condition, message): 181 | """Fail if condition not met.""" 182 | 183 | if not condition: 184 | print(message, file=sys.stderr) 185 | sys.exit(1) 186 | -------------------------------------------------------------------------------- /bin/workshop_check.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | '''Check that a workshop's index.html metadata is valid. See the 4 | docstrings on the checking functions for a summary of the checks. 5 | ''' 6 | 7 | from __future__ import print_function 8 | import sys 9 | import os 10 | import re 11 | from datetime import date 12 | from util import Reporter, split_metadata, load_yaml, check_unwanted_files 13 | 14 | # Metadata field patterns. 15 | EMAIL_PATTERN = r'[^@]+@[^@]+\.[^@]+' 16 | HUMANTIME_PATTERN = r'((0?[1-9]|1[0-2]):[0-5]\d(am|pm)(-|to)(0?[1-9]|1[0-2]):[0-5]\d(am|pm))|((0?\d|1\d|2[0-3]):[0-5]\d(-|to)(0?\d|1\d|2[0-3]):[0-5]\d)' 17 | EVENTBRITE_PATTERN = r'\d{9,10}' 18 | URL_PATTERN = r'https?://.+' 19 | 20 | # Defaults. 21 | CARPENTRIES = ("dc", "swc") 22 | DEFAULT_CONTACT_EMAIL = 'admin@software-carpentry.org' 23 | USAGE = 'Usage: "workshop_check.py path/to/root/directory"' 24 | 25 | 26 | # Country and language codes. Note that codes mean different things: 'ar' 27 | # is 'Arabic' as a language but 'Argentina' as a country. 28 | 29 | ISO_COUNTRY = [ 30 | 'ad', 'ae', 'af', 'ag', 'ai', 'al', 'am', 'an', 'ao', 'aq', 'ar', 'as', 31 | 'at', 'au', 'aw', 'ax', 'az', 'ba', 'bb', 'bd', 'be', 'bf', 'bg', 'bh', 32 | 'bi', 'bj', 'bm', 'bn', 'bo', 'br', 'bs', 'bt', 'bv', 'bw', 'by', 'bz', 33 | 'ca', 'cc', 'cd', 'cf', 'cg', 'ch', 'ci', 'ck', 'cl', 'cm', 'cn', 'co', 34 | 'cr', 'cu', 'cv', 'cx', 'cy', 'cz', 'de', 'dj', 'dk', 'dm', 'do', 'dz', 35 | 'ec', 'ee', 'eg', 'eh', 'er', 'es', 'et', 'eu', 'fi', 'fj', 'fk', 'fm', 36 | 'fo', 'fr', 'ga', 'gb', 'gd', 'ge', 'gf', 'gg', 'gh', 'gi', 'gl', 'gm', 37 | 'gn', 'gp', 'gq', 'gr', 'gs', 'gt', 'gu', 'gw', 'gy', 'hk', 'hm', 'hn', 38 | 'hr', 'ht', 'hu', 'id', 'ie', 'il', 'im', 'in', 'io', 'iq', 'ir', 'is', 39 | 'it', 'je', 'jm', 'jo', 'jp', 'ke', 'kg', 'kh', 'ki', 'km', 'kn', 'kp', 40 | 'kr', 'kw', 'ky', 'kz', 'la', 'lb', 'lc', 'li', 'lk', 'lr', 'ls', 'lt', 41 | 'lu', 'lv', 'ly', 'ma', 'mc', 'md', 'me', 'mg', 'mh', 'mk', 'ml', 'mm', 42 | 'mn', 'mo', 'mp', 'mq', 'mr', 'ms', 'mt', 'mu', 'mv', 'mw', 'mx', 'my', 43 | 'mz', 'na', 'nc', 'ne', 'nf', 'ng', 'ni', 'nl', 'no', 'np', 'nr', 'nu', 44 | 'nz', 'om', 'pa', 'pe', 'pf', 'pg', 'ph', 'pk', 'pl', 'pm', 'pn', 'pr', 45 | 'ps', 'pt', 'pw', 'py', 'qa', 're', 'ro', 'rs', 'ru', 'rw', 'sa', 'sb', 46 | 'sc', 'sd', 'se', 'sg', 'sh', 'si', 'sj', 'sk', 'sl', 'sm', 'sn', 'so', 47 | 'sr', 'st', 'sv', 'sy', 'sz', 'tc', 'td', 'tf', 'tg', 'th', 'tj', 'tk', 48 | 'tl', 'tm', 'tn', 'to', 'tr', 'tt', 'tv', 'tw', 'tz', 'ua', 'ug', 'um', 49 | 'us', 'uy', 'uz', 'va', 'vc', 've', 'vg', 'vi', 'vn', 'vu', 'wf', 'ws', 50 | 'ye', 'yt', 'za', 'zm', 'zw' 51 | ] 52 | 53 | ISO_LANGUAGE = [ 54 | 'aa', 'ab', 'ae', 'af', 'ak', 'am', 'an', 'ar', 'as', 'av', 'ay', 'az', 55 | 'ba', 'be', 'bg', 'bh', 'bi', 'bm', 'bn', 'bo', 'br', 'bs', 'ca', 'ce', 56 | 'ch', 'co', 'cr', 'cs', 'cu', 'cv', 'cy', 'da', 'de', 'dv', 'dz', 'ee', 57 | 'el', 'en', 'eo', 'es', 'et', 'eu', 'fa', 'ff', 'fi', 'fj', 'fo', 'fr', 58 | 'fy', 'ga', 'gd', 'gl', 'gn', 'gu', 'gv', 'ha', 'he', 'hi', 'ho', 'hr', 59 | 'ht', 'hu', 'hy', 'hz', 'ia', 'id', 'ie', 'ig', 'ii', 'ik', 'io', 'is', 60 | 'it', 'iu', 'ja', 'jv', 'ka', 'kg', 'ki', 'kj', 'kk', 'kl', 'km', 'kn', 61 | 'ko', 'kr', 'ks', 'ku', 'kv', 'kw', 'ky', 'la', 'lb', 'lg', 'li', 'ln', 62 | 'lo', 'lt', 'lu', 'lv', 'mg', 'mh', 'mi', 'mk', 'ml', 'mn', 'mr', 'ms', 63 | 'mt', 'my', 'na', 'nb', 'nd', 'ne', 'ng', 'nl', 'nn', 'no', 'nr', 'nv', 64 | 'ny', 'oc', 'oj', 'om', 'or', 'os', 'pa', 'pi', 'pl', 'ps', 'pt', 'qu', 65 | 'rm', 'rn', 'ro', 'ru', 'rw', 'sa', 'sc', 'sd', 'se', 'sg', 'si', 'sk', 66 | 'sl', 'sm', 'sn', 'so', 'sq', 'sr', 'ss', 'st', 'su', 'sv', 'sw', 'ta', 67 | 'te', 'tg', 'th', 'ti', 'tk', 'tl', 'tn', 'to', 'tr', 'ts', 'tt', 'tw', 68 | 'ty', 'ug', 'uk', 'ur', 'uz', 've', 'vi', 'vo', 'wa', 'wo', 'xh', 'yi', 69 | 'yo', 'za', 'zh', 'zu' 70 | ] 71 | 72 | 73 | def look_for_fixme(func): 74 | """Decorator to fail test if text argument starts with "FIXME".""" 75 | 76 | def inner(arg): 77 | if (arg is not None) and \ 78 | isinstance(arg, str) and \ 79 | arg.lstrip().startswith('FIXME'): 80 | return False 81 | return func(arg) 82 | return inner 83 | 84 | 85 | @look_for_fixme 86 | def check_layout(layout): 87 | '''"layout" in YAML header must be "workshop".''' 88 | 89 | return layout == 'workshop' 90 | 91 | 92 | @look_for_fixme 93 | def check_carpentry(layout): 94 | '''"carpentry" in YAML header must be "dc" or "swc".''' 95 | 96 | return layout in CARPENTRIES 97 | 98 | 99 | @look_for_fixme 100 | def check_country(country): 101 | '''"country" must be a lowercase ISO-3166 two-letter code.''' 102 | 103 | return country in ISO_COUNTRY 104 | 105 | 106 | @look_for_fixme 107 | def check_language(language): 108 | '''"language" must be a lowercase ISO-639 two-letter code.''' 109 | 110 | return language in ISO_LANGUAGE 111 | 112 | 113 | @look_for_fixme 114 | def check_humandate(date): 115 | """ 116 | 'humandate' must be a human-readable date with a 3-letter month 117 | and 4-digit year. Examples include 'Feb 18-20, 2025' and 'Feb 18 118 | and 20, 2025'. It may be in languages other than English, but the 119 | month name should be kept short to aid formatting of the main 120 | Software Carpentry web site. 121 | """ 122 | 123 | if ',' not in date: 124 | return False 125 | 126 | month_dates, year = date.split(',') 127 | 128 | # The first three characters of month_dates are not empty 129 | month = month_dates[:3] 130 | if any(char == ' ' for char in month): 131 | return False 132 | 133 | # But the fourth character is empty ("February" is illegal) 134 | if month_dates[3] != ' ': 135 | return False 136 | 137 | # year contains *only* numbers 138 | try: 139 | int(year) 140 | except: 141 | return False 142 | 143 | return True 144 | 145 | 146 | @look_for_fixme 147 | def check_humantime(time): 148 | """ 149 | 'humantime' is a human-readable start and end time for the 150 | workshop, such as '09:00 - 16:00'. 151 | """ 152 | 153 | return bool(re.match(HUMANTIME_PATTERN, time.replace(' ', ''))) 154 | 155 | 156 | def check_date(this_date): 157 | """ 158 | 'startdate' and 'enddate' are machine-readable start and end dates 159 | for the workshop, and must be in YYYY-MM-DD format, e.g., 160 | '2015-07-01'. 161 | """ 162 | 163 | # YAML automatically loads valid dates as datetime.date. 164 | return isinstance(this_date, date) 165 | 166 | 167 | @look_for_fixme 168 | def check_latitude_longitude(latlng): 169 | """ 170 | 'latlng' must be a valid latitude and longitude represented as two 171 | floating-point numbers separated by a comma. 172 | """ 173 | 174 | try: 175 | lat, lng = latlng.split(',') 176 | lat = float(lat) 177 | long = float(lng) 178 | return (-90.0 <= lat <= 90.0) and (-180.0 <= long <= 180.0) 179 | except ValueError: 180 | return False 181 | 182 | 183 | def check_instructors(instructors): 184 | """ 185 | 'instructor' must be a non-empty comma-separated list of quoted 186 | names, e.g. ['First name', 'Second name', ...']. Do not use 'TBD' 187 | or other placeholders. 188 | """ 189 | 190 | # YAML automatically loads list-like strings as lists. 191 | return isinstance(instructors, list) and len(instructors) > 0 192 | 193 | 194 | def check_helpers(helpers): 195 | """ 196 | 'helper' must be a comma-separated list of quoted names, 197 | e.g. ['First name', 'Second name', ...']. The list may be empty. 198 | Do not use 'TBD' or other placeholders. 199 | """ 200 | 201 | # YAML automatically loads list-like strings as lists. 202 | return isinstance(helpers, list) and len(helpers) >= 0 203 | 204 | 205 | @look_for_fixme 206 | def check_email(email): 207 | """ 208 | <<<<<<< HEAD 209 | 'contact' must be a valid email address consisting of characters, 210 | an '@', and more characters. It should not be the default contact 211 | email address 'admin@software-carpentry.org'. 212 | """ 213 | 214 | return bool(re.match(EMAIL_PATTERN, email)) and \ 215 | (email != DEFAULT_CONTACT_EMAIL) 216 | ======= 217 | 'email' must be a comma-separated list of valid email addresses. 218 | The list may be empty. A valid email address consists of characters, 219 | an '@', and more characters. It should not contain the default contact 220 | """ 221 | 222 | # YAML automatically loads list-like strings as lists. 223 | if (isinstance(emails, list) and len(emails) >= 0): 224 | for email in emails: 225 | if ((not bool(re.match(EMAIL_PATTERN, email))) or (email == DEFAULT_CONTACT_EMAIL)): 226 | return False 227 | else: 228 | return False 229 | 230 | return True 231 | >>>>>>> 01957e22bf78c20aa3728867d103305d88826164 232 | 233 | 234 | def check_eventbrite(eventbrite): 235 | """ 236 | 'eventbrite' (the Eventbrite registration key) must be 9 or more 237 | digits. It may appear as an integer or as a string. 238 | """ 239 | 240 | if isinstance(eventbrite, int): 241 | return True 242 | else: 243 | return bool(re.match(EVENTBRITE_PATTERN, eventbrite)) 244 | 245 | 246 | @look_for_fixme 247 | <<<<<<< HEAD 248 | def check_etherpad(etherpad): 249 | """ 250 | 'etherpad' must be a valid URL. 251 | """ 252 | 253 | return bool(re.match(URL_PATTERN, etherpad)) 254 | ======= 255 | def check_collaborative_notes(collaborative_notes): 256 | """ 257 | 'collaborative_notes' must be a valid URL. 258 | """ 259 | 260 | return bool(re.match(URL_PATTERN, collaborative_notes)) 261 | >>>>>>> 01957e22bf78c20aa3728867d103305d88826164 262 | 263 | 264 | @look_for_fixme 265 | def check_pass(value): 266 | """ 267 | This test always passes (it is used for 'checking' things like the 268 | workshop address, for which no sensible validation is feasible). 269 | """ 270 | 271 | return True 272 | 273 | 274 | HANDLERS = { 275 | 'layout': (True, check_layout, 'layout isn\'t "workshop"'), 276 | 277 | 'carpentry': (True, check_carpentry, 'carpentry isn\'t in ' + 278 | ', '.join(CARPENTRIES)), 279 | 280 | 'country': (True, check_country, 281 | 'country invalid: must use lowercase two-letter ISO code ' + 282 | 'from ' + ', '.join(ISO_COUNTRY)), 283 | 284 | 'language': (False, check_language, 285 | 'language invalid: must use lowercase two-letter ISO code' + 286 | ' from ' + ', '.join(ISO_LANGUAGE)), 287 | 288 | 'humandate': (True, check_humandate, 289 | 'humandate invalid. Please use three-letter months like ' + 290 | '"Jan" and four-letter years like "2025"'), 291 | 292 | 'humantime': (True, check_humantime, 293 | 'humantime doesn\'t include numbers'), 294 | 295 | 'startdate': (True, check_date, 296 | 'startdate invalid. Must be of format year-month-day, ' + 297 | 'i.e., 2014-01-31'), 298 | 299 | 'enddate': (False, check_date, 300 | 'enddate invalid. Must be of format year-month-day, i.e.,' + 301 | ' 2014-01-31'), 302 | 303 | 'latlng': (True, check_latitude_longitude, 304 | 'latlng invalid. Check that it is two floating point ' + 305 | 'numbers, separated by a comma'), 306 | 307 | 'instructor': (True, check_instructors, 308 | 'instructor list isn\'t a valid list of format ' + 309 | '["First instructor", "Second instructor",..]'), 310 | 311 | 'helper': (True, check_helpers, 312 | 'helper list isn\'t a valid list of format ' + 313 | '["First helper", "Second helper",..]'), 314 | 315 | <<<<<<< HEAD 316 | 'contact': (True, check_email, 317 | 'contact email invalid or still set to ' + 318 | ======= 319 | 'email': (True, check_emails, 320 | 'contact email list isn\'t a valid list of format ' + 321 | '["me@example.org", "you@example.org",..] or contains incorrectly formatted email addresses or ' + 322 | >>>>>>> 01957e22bf78c20aa3728867d103305d88826164 323 | '"{0}".'.format(DEFAULT_CONTACT_EMAIL)), 324 | 325 | 'eventbrite': (False, check_eventbrite, 'Eventbrite key appears invalid'), 326 | 327 | <<<<<<< HEAD 328 | 'etherpad': (False, check_etherpad, 'Etherpad URL appears invalid'), 329 | ======= 330 | 'collaborative_notes': (False, check_collaborative_notes, 'Collaborative Notes URL appears invalid'), 331 | >>>>>>> 01957e22bf78c20aa3728867d103305d88826164 332 | 333 | 'venue': (False, check_pass, 'venue name not specified'), 334 | 335 | 'address': (False, check_pass, 'address not specified') 336 | } 337 | 338 | # REQUIRED is all required categories. 339 | REQUIRED = set([k for k in HANDLERS if HANDLERS[k][0]]) 340 | 341 | # OPTIONAL is all optional categories. 342 | OPTIONAL = set([k for k in HANDLERS if not HANDLERS[k][0]]) 343 | 344 | 345 | def check_blank_lines(reporter, raw): 346 | """ 347 | Blank lines are not allowed in category headers. 348 | """ 349 | 350 | lines = [(i, x) for (i, x) in enumerate(raw.strip().split('\n')) if not x.strip()] 351 | reporter.check(not lines, 352 | None, 353 | 'Blank line(s) in header: {0}', 354 | ', '.join(["{0}: {1}".format(i, x.rstrip()) for (i, x) in lines])) 355 | 356 | 357 | def check_categories(reporter, left, right, msg): 358 | """ 359 | Report differences (if any) between two sets of categories. 360 | """ 361 | 362 | diff = left - right 363 | reporter.check(len(diff) == 0, 364 | None, 365 | '{0}: offending entries {1}', 366 | msg, sorted(list(diff))) 367 | 368 | 369 | def check_file(reporter, path, data): 370 | """ 371 | Get header from file, call all other functions, and check file for 372 | validity. 373 | """ 374 | 375 | # Get metadata as text and as YAML. 376 | raw, header, body = split_metadata(path, data) 377 | 378 | # Do we have any blank lines in the header? 379 | check_blank_lines(reporter, raw) 380 | 381 | # Look through all header entries. If the category is in the input 382 | # file and is either required or we have actual data (as opposed to 383 | # a commented-out entry), we check it. If it *isn't* in the header 384 | # but is required, report an error. 385 | for category in HANDLERS: 386 | required, handler, message = HANDLERS[category] 387 | if category in header: 388 | if required or header[category]: 389 | reporter.check(handler(header[category]), 390 | None, 391 | '{0}\n actual value "{1}"', 392 | message, header[category]) 393 | elif required: 394 | reporter.add(None, 395 | 'Missing mandatory key "{0}"', 396 | category) 397 | 398 | # Check whether we have missing or too many categories 399 | seen_categories = set(header.keys()) 400 | check_categories(reporter, REQUIRED, seen_categories, 401 | 'Missing categories') 402 | check_categories(reporter, seen_categories, REQUIRED.union(OPTIONAL), 403 | 'Superfluous categories') 404 | 405 | 406 | def check_config(reporter, filename): 407 | """ 408 | Check YAML configuration file. 409 | """ 410 | 411 | config = load_yaml(filename) 412 | 413 | kind = config.get('kind', None) 414 | reporter.check(kind == 'workshop', 415 | filename, 416 | 'Missing or unknown kind of event: {0}', 417 | kind) 418 | 419 | carpentry = config.get('carpentry', None) 420 | reporter.check(carpentry in ('swc', 'dc'), 421 | filename, 422 | 'Missing or unknown carpentry: {0}', 423 | carpentry) 424 | 425 | 426 | def main(): 427 | '''Run as the main program.''' 428 | 429 | if len(sys.argv) != 2: 430 | print(USAGE, file=sys.stderr) 431 | sys.exit(1) 432 | 433 | root_dir = sys.argv[1] 434 | index_file = os.path.join(root_dir, 'index.html') 435 | config_file = os.path.join(root_dir, '_config.yml') 436 | 437 | reporter = Reporter() 438 | check_config(reporter, config_file) 439 | check_unwanted_files(root_dir, reporter) 440 | with open(index_file) as reader: 441 | data = reader.read() 442 | check_file(reporter, index_file, data) 443 | reporter.report() 444 | 445 | 446 | if __name__ == '__main__': 447 | main() 448 | -------------------------------------------------------------------------------- /code/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/hpc-novice/d08fa355c3444ee8981e57f8453e936fc2f00946/code/.gitkeep -------------------------------------------------------------------------------- /data/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/hpc-novice/d08fa355c3444ee8981e57f8453e936fc2f00946/data/.gitkeep -------------------------------------------------------------------------------- /design.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: lesson 3 | subtitle: Lesson Design 4 | --- 5 | 6 | ## Stage 0 - Assumptions 7 | 8 | * Audience 9 | * Graduate students (PhD students, postdocs, technicians) in numerate disciplines from cosmology to economics 10 | * Who have used the terminal or at least attended [shell-novice](https://github.com/swcarpentry/shell-novice) 11 | * Who are literate in any given programming language (python, C/C++, Fortran, ...) 12 | * Constraints 13 | * One full day 09:00-17:00 14 | * 06:30 teaching time 15 | * 1:00 for lunch 16 | * 0:30 total for two coffee breaks 17 | * Learners use native installs on their own machines (ssh session) 18 | * May use logins to local cluster 19 | * Dependence on other Carpentry modules 20 | * In particular, [shell-novice](https://github.com/swcarpentry/shell-novice) 21 | * Exercises will mostly *not* be "write this code from scratch" 22 | * Want lots of short exercises that can reliably be finished in allotted time 23 | * So use MCQs, fill-in-the-blanks, Parsons Problems, "tweak this code", etc. 24 | * Lesson materials 25 | * Notes for instructors and self-study will be written in Markdown 26 | * We've tried writing/maintaining lessons as Notebooks... 27 | * Learners will be provided with one Notebook per episode containing exercises 28 | 29 | ## Stage 1 - Desired Results 30 | 31 | ### Goals 32 | 33 | 1. Get learners to the stage decribed in the "Software" section of 34 | "[Good Enough Practices in Scientific Computing][good-enough]". 35 | * Goals 36 | 1. Make it easy for people (including your future self) to understand and (re)use your code 37 | 2. Modular, comprehensible, reusable, and testable all come together 38 | * Rules 39 | 1. Every analysis step is represented textually (complete with parameter values) 40 | 2. Every program or script has a brief explanatory comment at the start 41 | 3. Programs of all kinds (including "scripts") are broken into functions 42 | 4. No duplication 43 | 5. Functions and variables have meaningful names 44 | 6. Dependencies and requirements are explicit (e.g., a requirements.txt file) 45 | * This rule is *not* covered in this lesson 46 | 7. Commenting/uncommenting are not routinely used to control program behavior 47 | 8. Use a simple example or test data set to run to tell if it's working at all and whether it gives a known correct output for a simple known input 48 | 9. Submit code to a reputable DOI-issuing repository upon submission of paper, just like data 49 | * This rule is *not* covered in this lesson 50 | 2. Enable them to make sense of other onlines tutorials and resources 51 | 52 | ### Summative Assessment 53 | 54 | * Midpoint: plot bar chart showing average GDP per continent 55 | * Final: debug and extend a short multi-function program to handle data laid out differently 56 | 57 | ### Essential Questions 58 | 59 | How do I... 60 | 61 | * ...read, analyze, and visualize a tabular data set? 62 | * ...process multiple data sets? 63 | * ...tell if my program is working correctly? 64 | * ...fix it when it's not? 65 | * ...find and use software other people have written instead of writing my own? 66 | 67 | ### Learners Will Be Able To... 68 | 69 | * Run code interactively 70 | * Run code saved in a file 71 | * Write single-condition `if` statements 72 | * Convert between basic data types (integer, float, string) 73 | * Call built-in functions 74 | * Use `help` and online documentation 75 | * Import a library using an alias 76 | * Call something from an imported library 77 | * Read tabular data into an array or data frame 78 | * Do collective operations on arrays and data frames 79 | * Create simple plots of data in arrays and data frames 80 | * Interpret common error messages 81 | * Track down bugs by running small tests of program modules 82 | * Write non-recursive functions taking a fixed number of named parameters 83 | * Create literate programs in the Jupyter Notebook 84 | 85 | ### Learners Will Know... 86 | 87 | * That a program is a piece of lab equipment that implements an analysis 88 | * Needs to be validated/calibrated before/during use 89 | * Makes analysis reproducible, reviewable, shareable 90 | * That programs are written for people, not for computers 91 | * Meaningful variable names 92 | * Modularity for readability as well as re-use 93 | * No duplication 94 | * Document purpose and use 95 | * That there is no magic: the programs they use are no different in principle from those they build 96 | * How to assign values to variables 97 | * What integers, floats, strings, and data frames are 98 | * How to trace the execution of a `for` loop 99 | * How to create and index lists 100 | * How to trace the execution of `if`/`else` statements 101 | * The difference between defining and calling a function 102 | * What a call stack is 103 | * Where to find documentation on standard libraries 104 | * How to find out what else scientific Python offers 105 | -------------------------------------------------------------------------------- /favicon-dc.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/hpc-novice/d08fa355c3444ee8981e57f8453e936fc2f00946/favicon-dc.ico -------------------------------------------------------------------------------- /favicon-lc.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/hpc-novice/d08fa355c3444ee8981e57f8453e936fc2f00946/favicon-lc.ico -------------------------------------------------------------------------------- /favicon-swc.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/hpc-novice/d08fa355c3444ee8981e57f8453e936fc2f00946/favicon-swc.ico -------------------------------------------------------------------------------- /fig/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/hpc-novice/d08fa355c3444ee8981e57f8453e936fc2f00946/fig/.gitkeep -------------------------------------------------------------------------------- /fig/episode-format-small.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/hpc-novice/d08fa355c3444ee8981e57f8453e936fc2f00946/fig/episode-format-small.png -------------------------------------------------------------------------------- /fig/episode-format.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/hpc-novice/d08fa355c3444ee8981e57f8453e936fc2f00946/fig/episode-format.png -------------------------------------------------------------------------------- /fig/file-mapping.odg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/hpc-novice/d08fa355c3444ee8981e57f8453e936fc2f00946/fig/file-mapping.odg -------------------------------------------------------------------------------- /fig/forking.odg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/hpc-novice/d08fa355c3444ee8981e57f8453e936fc2f00946/fig/forking.odg -------------------------------------------------------------------------------- /fig/repository-links.odg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/hpc-novice/d08fa355c3444ee8981e57f8453e936fc2f00946/fig/repository-links.odg -------------------------------------------------------------------------------- /fig/rmd-plot-example-1.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/hpc-novice/d08fa355c3444ee8981e57f8453e936fc2f00946/fig/rmd-plot-example-1.png -------------------------------------------------------------------------------- /fig/using-github-import.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/hpc-novice/d08fa355c3444ee8981e57f8453e936fc2f00946/fig/using-github-import.png -------------------------------------------------------------------------------- /files/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/hpc-novice/d08fa355c3444ee8981e57f8453e936fc2f00946/files/.gitkeep -------------------------------------------------------------------------------- /img/putty-7.jpeg: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/hpc-novice/d08fa355c3444ee8981e57f8453e936fc2f00946/img/putty-7.jpeg -------------------------------------------------------------------------------- /index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: lesson 3 | root: . 4 | permalink: index.html # Is the only page that don't follow the partner /:path/index.html 5 | --- 6 | 7 | This lesson is under development. A sample prototype is being 8 | developed by @psteinb. Contact @psteinb @shwina or @christinalk 9 | for more information. 10 | -------------------------------------------------------------------------------- /outline.md: -------------------------------------------------------------------------------- 1 | # Learning Outline 2 | 3 | ## Content 4 | 5 | The workshop is divided into the following three sections: 6 | 7 | ### The Unix Shell 8 | 9 | The aim of this section is to teach learners basic unix commands in 10 | order to help them interact with a HPC infrastructure. This may entail: 11 | 12 | - filesystem navigation and manipulation 13 | - different command line and bash scripts 14 | - remote sessions with `ssh` 15 | 16 | 17 | ### Clusters and Distributed Computing Infrastructure 18 | 19 | This section is meant to convey a simplistic mental model of the cluster, 20 | and how tasks get submitted, assigned and executed on the cluster. 21 | 22 | Towards the end of the section, learners will submit a (number of) "Hello World" 23 | style batch job(s) that are aligned to the Unix Shell section above. 24 | 25 | The session concludes by introducing and practising `environment modules` based 26 | on the observation that software on HPC and for HPC systems is very diverse a 27 | nd subject to change. 28 | 29 | ### Paradigms of Parallel Computing 30 | 31 | This section will guide learners through the process of 32 | performing a large HPC simulation (code available), 33 | and a high-throughput analysis (code available) of related data. The end 34 | goal will be to compare the simulation to the data (i.e. to relate their actions on a HPC installation to the scientific method). 35 | 36 | For this, they will develop the workflow for a specific research scenario, 37 | for example: 38 | 39 | > Lola was hired by a research lab to help prepare the purchase of a multi-million dollar experiment. 40 | The experiment is known to fail at temperatures that are too low or too high. 41 | She knows that the temperature changes follow a daily pattern, 42 | and she's written some code to simulate these temperature changes. 43 | After running this code and generating the temperature predictions, 44 | she determines how closely her predictions match the actual temperature readings 45 | she has for every day in the last year. 46 | The simuation would take too long to run 47 | and generate too much data for her lab workstation, 48 | so she will use the local University's HPC facility for this work. 49 | 50 | ### Timing 51 | 52 | 1. Introduction to the Unix Shell [90 min] 53 | 54 | a. Navigating the file system (`ls`, `cd`, `pwd`, `cp`, `rm`, `mv`) [30 min] 55 | 56 | b. commands and scripts [30 min] 57 | 58 | c. taking the space shuttle `ssh` to another planet [15 min] 59 | 60 | d. transferring files to/from a remote host (`scp`, `rsync`) [15 min] 61 | 62 | 63 | 2. Clusters and Distributed Computing Infrastructure [45-60 min] 64 | 65 | a. the scheduling problem and our first "Hello World" [20 min] 66 | 67 | b. distributed file systems [15 min] 68 | 69 | c. revisiting "Hello World" using input/output [15 min] 70 | 71 | 72 | 3. Paradigms of Parallel Computing [90 min] 73 | 74 | a. calculating with many distributed systems (MPI, shared-memory parallelisation) [30 min] 75 | 76 | b. data-parallel computations (`map-reduce`) [30 min] 77 | 78 | c. what this is all for? [30 min] 79 | 80 | 81 | 82 | > All timings above should be considered lower bounds and shall be subject to change once more workshop instances of this material are performed. 83 | -------------------------------------------------------------------------------- /reference.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: reference 3 | root: . 4 | --- 5 | 6 | ## Glossary 7 | 8 | The glossary would go here, formatted as: 9 | 10 | ~~~ 11 | {:auto_ids} 12 | key word 1 13 | : explanation 1 14 | 15 | key word 2 16 | : explanation 2 17 | ~~~ 18 | {: .source} 19 | 20 | (`{:auto_ids}` is needed at the start 21 | so that Jekyll will automatically generate a unique ID for each item 22 | to allow other pages to hyperlink to specific glossary entries.) 23 | This renders as: 24 | 25 | {:auto_ids} 26 | key word 1 27 | : explanation 1 28 | 29 | key word 2 30 | : explanation 2 31 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | PyYAML 2 | update-copyright 3 | -------------------------------------------------------------------------------- /setup.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: Setup 4 | root: . 5 | --- 6 | 7 | This is a placeholder for setup instructions! 8 | -------------------------------------------------------------------------------- /target-audience.md: -------------------------------------------------------------------------------- 1 | # Target Audience 2 | 3 | HPC Carpentry is targeted at graduate students, researchers, or 4 | industry specialists who 5 | 6 | * already write and run domain-specific software on "smaller" 7 | computers, and now need to scale up/out, or 8 | 9 | * need to learn how to 10 | use large-scale resources to run some domain-specific software which can 11 | simply not be run on smaller computers 12 | 13 | and aims to cover the essential skills required to be able to get 14 | started with using such resources. 15 | 16 | This lesson is intended for **novices**. The expectation is that participants 17 | have *maybe* used a command line interface, and possibly programmed 18 | something Python or R, but have not used the command line extensively, 19 | and have never logged into a remote computer or used a job scheduler before. 20 | A very novice audience might consist of people who have never done any 21 | scripting or command line work. Therefore, this lesson covers basic skills like: 22 | 23 | * using the command line to navigate the filesystem and manipulate files 24 | * an introduction to writing a shell script and/or compiling software with modules 25 | * submitting a job using a scheduling program 26 | * best practices like good cluster citizenship and what problems are appropriate for cluster use. 27 | 28 | > Sample Learner Profile: Alice is a grad student in botany who is 29 | starting to do work in bioinformatics. She's installed and run things on her 30 | own computer, but her command line knowledge mostly consists of things she's 31 | google'd and tried, hoping they will work. She has a large set of RNAseq samples 32 | coming in and wants to be able to run them all on the campus cluster, using a 33 | common alignment program. By the end of this lesson, she should be able to log 34 | into the cluster, place her data in the appropriate location and submit a job 35 | that runs an alignment. -------------------------------------------------------------------------------- /tools/check_knitr_version.R: -------------------------------------------------------------------------------- 1 | if (require("knitr")) { 2 | if (packageVersion("knitr") < '1.9.19') { 3 | stop("knitr must be version 1.9.20 or higher") 4 | } 5 | } else stop("knitr 1.9.20 or above is needed to build the lessons.") 6 | -------------------------------------------------------------------------------- /tools/chunk-options.R: -------------------------------------------------------------------------------- 1 | # These settings control the behavior of all chunks in the novice R materials. 2 | # For example, to generate the lessons with all the output hidden, simply change 3 | # `results` from "markup" to "hide". 4 | # For more information on available chunk options, see 5 | # http://yihui.name/knitr/options#chunk_options 6 | 7 | library("knitr") 8 | opts_chunk$set(tidy = FALSE, results = "markup", comment = NA, 9 | fig.align = "center", fig.path = "fig/") 10 | 11 | # The hooks below add html tags to the code chunks and their output so that they 12 | # are properly formatted when the site is built. 13 | hook_in <- function(x, options) { 14 | stringr::str_c("\n\n~~~{.r}\n", 15 | paste0(x, collapse="\n"), 16 | "\n~~~\n\n") 17 | } 18 | 19 | hook_out <- function(x, options) { 20 | stringr::str_c("\n\n~~~{.output}\n", 21 | paste0(x, collapse="\n"), 22 | "\n~~~\n\n") 23 | } 24 | 25 | hook_error <- function(x, options) { 26 | stringr::str_c("\n\n~~~{.error}\n", 27 | paste0(x, collapse="\n"), 28 | "\n~~~\n\n") 29 | } 30 | 31 | knit_hooks$set(source = hook_in, output = hook_out, warning = hook_error, 32 | error = hook_error, message = hook_out) 33 | --------------------------------------------------------------------------------