├── code └── .gitkeep ├── data └── .gitkeep ├── fig └── .gitkeep ├── _episodes ├── .gitkeep ├── 03-advanced.md ├── 02-markdown.md └── 01-mess.md ├── _extras ├── .gitkeep ├── about.md ├── figures.md ├── discuss.md ├── guide.md └── design.md ├── files └── .gitkeep ├── _episodes_rmd ├── .gitkeep └── data │ └── .gitkeep ├── _includes ├── all_figures.html ├── github-ribbon.html ├── episode_break.html ├── main_title.html ├── episode_keypoints.html ├── javascript.html ├── workshop_footer.html ├── all_keypoints.html ├── lesson_footer.html ├── head.html ├── header.html ├── workshop_ad.html ├── episode_overview.html ├── footer.html ├── carpentries.html ├── episode_title.html ├── syllabus.html └── navbar.html ├── requirements.txt ├── .update-copyright.conf ├── reference.md ├── _layouts ├── page.html ├── reference.html ├── lesson.html ├── break.html ├── episode.html ├── base.html └── workshop.html ├── favicon-dc.ico ├── .gitignore ├── favicon-swc.ico ├── setup.md ├── assets ├── img │ ├── swc-logo-blue.png │ ├── swc-logo-white.png │ ├── swc-icon-blue.svg │ ├── dc-icon-black.svg │ └── swc-logo-blue.svg ├── fonts │ ├── glyphicons-halflings-regular.eot │ ├── glyphicons-halflings-regular.ttf │ ├── glyphicons-halflings-regular.woff │ └── glyphicons-halflings-regular.woff2 ├── js │ └── lesson.js └── css │ └── lesson.scss ├── bin ├── check_knitr_version.R ├── knit_lessons.sh ├── markdown_ast.rb ├── test_lesson_check.py ├── filters │ ├── id4glossary.py │ └── blockquote2div.py ├── generate_md_episodes.R ├── chunk-options.R ├── setup-labels ├── extract_figures.py ├── repo_check.py ├── util.py ├── validation_helpers.py ├── lesson_initialize.py ├── workshop_check.py └── lesson_check.py ├── AUTHORS ├── README.md ├── index.md ├── .mailmap ├── CONDUCT.md ├── _config.yml ├── LICENSE.md ├── Makefile └── CONTRIBUTING.md /code/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /data/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /fig/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /_episodes/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /_extras/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /files/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /_episodes_rmd/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /_episodes_rmd/data/.gitkeep: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /_includes/all_figures.html: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | PyYAML 2 | -------------------------------------------------------------------------------- /.update-copyright.conf: -------------------------------------------------------------------------------- 1 | [project] 2 | vcs: Git 3 | 4 | [files] 5 | authors: yes 6 | files: no 7 | -------------------------------------------------------------------------------- /reference.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: reference 3 | permalink: /reference/ 4 | --- 5 | FIXME: fill in 6 | -------------------------------------------------------------------------------- /_layouts/page.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: base 3 | --- 4 | {% include main_title.html %} 5 | {{content}} 6 | -------------------------------------------------------------------------------- /favicon-dc.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/modern-scientific-authoring/HEAD/favicon-dc.ico -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | *~ 3 | .DS_Store 4 | .ipynb_checkpoints 5 | .sass-cache 6 | __pycache__ 7 | _site 8 | -------------------------------------------------------------------------------- /favicon-swc.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/modern-scientific-authoring/HEAD/favicon-swc.ico -------------------------------------------------------------------------------- /setup.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: Setup 4 | permalink: /setup/ 5 | --- 6 | FIXME: setup instructions. 7 | -------------------------------------------------------------------------------- /_includes/github-ribbon.html: -------------------------------------------------------------------------------- 1 | Find us on GitHub 2 | -------------------------------------------------------------------------------- /_extras/about.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: About 4 | permalink: /about/ 5 | --- 6 | {% include carpentries.html %} 7 | -------------------------------------------------------------------------------- /_extras/figures.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: Figures 4 | permalink: /figures/ 5 | --- 6 | {% include all_figures.html %} 7 | -------------------------------------------------------------------------------- /_layouts/reference.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: "Reference" 4 | --- 5 | {% include all_keypoints.html %} 6 | {{content}} 7 | -------------------------------------------------------------------------------- /assets/img/swc-logo-blue.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/modern-scientific-authoring/HEAD/assets/img/swc-logo-blue.png -------------------------------------------------------------------------------- /assets/img/swc-logo-white.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/modern-scientific-authoring/HEAD/assets/img/swc-logo-white.png -------------------------------------------------------------------------------- /_extras/discuss.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: Discussion 4 | permalink: /discuss/ 5 | --- 6 | FIXME: links to other material. 7 | -------------------------------------------------------------------------------- /_layouts/lesson.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: base 3 | --- 4 | {% include main_title.html %} 5 | {{ content }} 6 | {% include syllabus.html %} 7 | -------------------------------------------------------------------------------- /_extras/guide.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: "Instructors' Guide" 4 | permalink: /guide/ 5 | --- 6 | FIXME: fill in instructors' guide. 7 | -------------------------------------------------------------------------------- /_layouts/break.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: base 3 | --- 4 | {% include episode_title.html %} 5 | {% include episode_break.html %} 6 | {{content}} 7 | -------------------------------------------------------------------------------- /_includes/episode_break.html: -------------------------------------------------------------------------------- 1 | {% comment %} 2 | Display information about a break. 3 | {% endcomment %} 4 |

Break: {{page.break}} min

5 | -------------------------------------------------------------------------------- /_includes/main_title.html: -------------------------------------------------------------------------------- 1 |

{{ site.title }}{% if page.title %}: {{ page.title }}{% endif %}

2 | -------------------------------------------------------------------------------- /assets/fonts/glyphicons-halflings-regular.eot: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/modern-scientific-authoring/HEAD/assets/fonts/glyphicons-halflings-regular.eot -------------------------------------------------------------------------------- /assets/fonts/glyphicons-halflings-regular.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/modern-scientific-authoring/HEAD/assets/fonts/glyphicons-halflings-regular.ttf -------------------------------------------------------------------------------- /assets/fonts/glyphicons-halflings-regular.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/modern-scientific-authoring/HEAD/assets/fonts/glyphicons-halflings-regular.woff -------------------------------------------------------------------------------- /assets/fonts/glyphicons-halflings-regular.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/swcarpentry/modern-scientific-authoring/HEAD/assets/fonts/glyphicons-halflings-regular.woff2 -------------------------------------------------------------------------------- /_layouts/episode.html: -------------------------------------------------------------------------------- 1 | --- 2 | layout: base 3 | --- 4 | {% include episode_title.html %} 5 | {% include episode_overview.html %} 6 | {{content}} 7 | {% include episode_keypoints.html %} 8 | -------------------------------------------------------------------------------- /_includes/episode_keypoints.html: -------------------------------------------------------------------------------- 1 |
2 |

Key Points

3 | 8 |
9 | -------------------------------------------------------------------------------- /bin/check_knitr_version.R: -------------------------------------------------------------------------------- 1 | if (require("knitr")) { 2 | if (packageVersion("knitr") < '1.9.19') { 3 | stop("knitr must be version 1.9.20 or higher") 4 | } 5 | } else stop("knitr 1.9.20 or above is needed to build the lessons.") 6 | -------------------------------------------------------------------------------- /bin/knit_lessons.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | # Only try running R to translate files if there are some files present. 4 | # The Makefile passes in the names of files. 5 | 6 | if [ $# -ne 0 ] ; then 7 | Rscript -e "source('bin/generate_md_episodes.R')" 8 | fi 9 | -------------------------------------------------------------------------------- /bin/markdown_ast.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | 3 | # Use Kramdown parser to produce AST for Markdown document. 4 | 5 | require "kramdown" 6 | require "json" 7 | 8 | markdown = STDIN.read() 9 | doc = Kramdown::Document.new(markdown) 10 | tree = doc.to_hash_a_s_t 11 | puts JSON.pretty_generate(tree) 12 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | James Allen 2 | Piotr Banaszkiewicz 3 | Trevor Bekolay 4 | John Blischak 5 | Andy Boughton 6 | Abigail Cabunoc Mayes 7 | Rémi Emonet 8 | Michael Hansen 9 | Mike Jackson 10 | W. Trevor King 11 | François Michonneau 12 | Bill Mills 13 | Aaron O'Leary 14 | Jon Pipitone 15 | Timothée Poisot 16 | Raniere Silva 17 | Greg Wilson 18 | -------------------------------------------------------------------------------- /_includes/javascript.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | -------------------------------------------------------------------------------- /_includes/workshop_footer.html: -------------------------------------------------------------------------------- 1 |
2 | 12 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | modern-scientific-authoring 2 | =========================== 3 | 4 | How to write, publish, and review scientific papers in the early 21st Century. 5 | Please see for a rendered version of this material 6 | and [the lesson template documentation][lesson-example] 7 | for instructions on formatting, building, and submitting material, 8 | or run `make` in this directory for a list of helpful commands. 9 | 10 | Maintainers: 11 | 12 | * [Greg Wilson][wilson_greg] 13 | 14 | [lesson-example]: https://swcarpentry.github.io/lesson-example/ 15 | [wilson_greg]: http://software-carpentry.org/team/#wilson_greg 16 | -------------------------------------------------------------------------------- /_includes/all_keypoints.html: -------------------------------------------------------------------------------- 1 | {% comment %} 2 | Display key points of all episodes for reference. 3 | {% endcomment %} 4 |

Key Points

5 | 6 | {% for episode in site.episodes %} 7 | {% unless episode.break %} 8 | 9 | 12 | 19 | 20 | {% endunless %} 21 | {% endfor %} 22 |
10 | {{ episode.title }} 11 | 13 |
    14 | {% for keypoint in episode.keypoints %} 15 |
  • {{ keypoint|markdownify }}
    • 16 | {% endfor %} 17 |
18 |
23 | -------------------------------------------------------------------------------- /bin/test_lesson_check.py: -------------------------------------------------------------------------------- 1 | import unittest 2 | 3 | import lesson_check 4 | import util 5 | 6 | class TestFileList(unittest.TestCase): 7 | def setUp(self): 8 | self.reporter = util.Reporter() ## TODO: refactor reporter class. 9 | 10 | def test_file_list_has_expected_entries(self): 11 | # For first pass, simply assume that all required files are present 12 | all_filenames = [filename.replace('%', '') 13 | for filename in lesson_check.REQUIRED_FILES] 14 | 15 | lesson_check.check_fileset('', self.reporter, all_filenames) 16 | self.assertEqual(len(self.reporter.messages), 0) 17 | 18 | if __name__ == "__main__": 19 | unittest.main() 20 | -------------------------------------------------------------------------------- /_includes/lesson_footer.html: -------------------------------------------------------------------------------- 1 |
2 | 22 | -------------------------------------------------------------------------------- /_includes/head.html: -------------------------------------------------------------------------------- 1 | 2 | Software Carpentry: {{ page.venue }} 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 13 | -------------------------------------------------------------------------------- /index.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: lesson 3 | --- 4 | 5 | The goal of this lesson is to show how straightforward writing a paper should 6 | be in the 21^st^ century. We will go through the basics of Markdown, the way 7 | to use references, tables, and figures, how to automatically generate an 8 | up-to-date PDF, and how to publish a blog or lab website. 9 | 10 | > ## Prerequisites 11 | > 12 | > Learners should have completed introductory lessons on: 13 | > 14 | > * the Unix shell 15 | > * Git (setting up a repository, committing files) 16 | > * Makefiles 17 | {: .prereq} 18 | 19 | [RStudio][rstudio] and the [Jupyter Notebook][jupyter] have Markdown support, 20 | as do the [Atom][atom] and [Sublime Text][sublime] editors. 21 | 22 | [atom]: https://atom.io/ 23 | [jupyter]: http://jupyter.org/ 24 | [rstudio]: https://www.rstudio.com/ 25 | [sublime]: https://www.sublimetext.com/ 26 | -------------------------------------------------------------------------------- /.mailmap: -------------------------------------------------------------------------------- 1 | James Allen 2 | Piotr Banaszkiewicz 3 | Trevor Bekolay 4 | John Blischak 5 | Andy Boughton 6 | Abigail Cabunoc Mayes 7 | Rémi Emonet 8 | Michael Hansen 9 | Mike Jackson 10 | W. Trevor King 11 | François Michonneau 12 | Bill Mills 13 | Aaron O'Leary 14 | Jon Pipitone 15 | Timothée Poisot 16 | Raniere Silva 17 | Greg Wilson 18 | -------------------------------------------------------------------------------- /_includes/header.html: -------------------------------------------------------------------------------- 1 | 10 |
11 |
12 |
13 | 16 |
17 |
18 |

Teaching basic lab skills
for research computing

19 |
20 |
21 |
22 | -------------------------------------------------------------------------------- /_includes/workshop_ad.html: -------------------------------------------------------------------------------- 1 |
2 |
3 |
4 |

{{page.venue}}

5 |
6 |
7 |

{{page.humandate}}

8 |

{% if page.humantime %}{{page.humantime}}{% endif %}

9 |
10 |
11 |

12 | Instructors: 13 | {% if page.instructor %} 14 | {{page.instructor | join: ', ' %}} 15 | {% else %} 16 | to be announced. 17 | {% endif %} 18 |

19 | {% if page.helper %} 20 |

21 | Helpers: 22 | {{page.helper | join: ', ' %}} 23 |

24 | {% endif %} 25 |
26 |
27 |
28 |
29 |
30 | -------------------------------------------------------------------------------- /_includes/episode_overview.html: -------------------------------------------------------------------------------- 1 | {% comment %} 2 | Display an episode's timings and learning objectives. 3 | {% endcomment %} 4 |
5 |

Overview

6 | 7 |
8 |
9 | Teaching: {{ page.teaching }} min 10 |
11 | Exercises: {{ page.exercises }} min 12 |
13 |
14 | Questions 15 |
    16 | {% for question in page.questions %} 17 |
  • {{ question|markdownify }}
    • 18 | {% endfor %} 19 |
20 |
21 |
22 | 23 |
24 |
25 |
26 |
27 | Objectives 28 |
    29 | {% for objective in page.objectives %} 30 |
  • {{ objective|markdownify }}
    • 31 | {% endfor %} 32 |
33 |
34 |
35 | 36 |
37 | -------------------------------------------------------------------------------- /bin/filters/id4glossary.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Pandoc filter to convert add ids to glossary entries. 3 | 4 | Usage: 5 | 6 | pandoc source.md --filter=id4glossary.py --output=output.html 7 | """ 8 | import pandocfilters as pf 9 | 10 | def normalize_keyword(keyword): 11 | """Normalize keyword for became id 12 | 13 | - Replace white space with '-' 14 | - Convert to lowercase""" 15 | return keyword.lower().replace(' ', '-') 16 | 17 | def keyword2html(keyword_node): 18 | """Return HTML version of keyword with id.""" 19 | keyword = pf.stringify(keyword_node) 20 | id = normalize_keyword(keyword) 21 | return [{"t": "Span", 22 | "c": [[id, [],[]], 23 | keyword_node]}] 24 | 25 | def id4glossary(key, value, format, meta): 26 | """Add id to keywords at glossary.""" 27 | if key == "DefinitionList": 28 | for definition in value: 29 | definition[0] = keyword2html(definition[0]) 30 | return {"t": key, 31 | "c": value} 32 | 33 | if __name__ == '__main__': 34 | pf.toJSONFilter(id4glossary) 35 | -------------------------------------------------------------------------------- /assets/js/lesson.js: -------------------------------------------------------------------------------- 1 | // Make all tables striped by default. 2 | $("table").addClass("table table-striped"); 3 | 4 | 5 | // Handle foldable challenges and solutions (on click and at start). 6 | $(".challenge,.discussion,.solution").click(function(event) { 7 | var trigger = $(event.target).has(".fold-unfold").size() > 0 8 | || $(event.target).filter(".fold-unfold").size() > 0; 9 | if (trigger) { 10 | $(">*:not(h2)", this).toggle(400); 11 | $(">h2>span.fold-unfold", this).toggleClass("glyphicon-collapse-down glyphicon-collapse-up"); 12 | event.stopPropagation(); 13 | } 14 | }); 15 | $(".challenge,.discussion,.solution").each(function() { 16 | $(">*:not(h2)", this).toggle(); 17 | var h2 = $("h2:first", this); 18 | h2.append(""); 19 | }); 20 | 21 | 22 | // Handle searches. 23 | // Relies on document having 'meta' element with name 'search-domain'. 24 | function google_search() { 25 | var query = document.getElementById("google-search").value; 26 | var domain = $("meta[name=search-domain]").attr("value"); 27 | window.open("https://www.google.com/search?q=" + query + "+site:" + domain); 28 | } 29 | -------------------------------------------------------------------------------- /_includes/footer.html: -------------------------------------------------------------------------------- 1 |
2 |
3 | 4 |
5 |
6 |
7 |
8 |
9 |

10 | {{ site.footer_description }} 11 | More 12 |

13 |
14 |
15 | 20 |
21 |
22 | 29 |
30 |
31 |
32 | -------------------------------------------------------------------------------- /bin/generate_md_episodes.R: -------------------------------------------------------------------------------- 1 | generate_md_episodes <- function() { 2 | 3 | if (require("knitr") && packageVersion("knitr") < '1.9.19') 4 | stop("knitr must be version 1.9.20 or higher") 5 | 6 | if (!require("stringr")) 7 | stop("The package stringr is required for generating the lessons.") 8 | 9 | if (require("checkpoint")) { 10 | required_pkgs <- 11 | checkpoint:::projectScanPackages(project = "_episodes_rmd", 12 | verbose=FALSE, use.knitr = TRUE)$pkgs 13 | } else { 14 | stop("The checkpoint package is required to build the lessons.") 15 | } 16 | 17 | missing_pkgs <- required_pkgs[!(required_pkgs %in% rownames(installed.packages()))] 18 | 19 | if (length(missing_pkgs)) { 20 | message("Installing missing required packages: ", 21 | paste(missing_pkgs, collapse=", ")) 22 | install.packages(missing_pkgs) 23 | } 24 | 25 | ## find all the Rmd files, and generates the paths for their respective outputs 26 | src_rmd <- list.files(pattern = "??-*.Rmd$", path = "_episodes_rmd", full.names = TRUE) 27 | dest_md <- file.path("_episodes", gsub("Rmd$", "md", basename(src_rmd))) 28 | 29 | ## knit the Rmd into markdown 30 | mapply(function(x, y) { 31 | knitr::knit(x, output = y) 32 | }, src_rmd, dest_md) 33 | 34 | } 35 | 36 | generate_md_episodes() 37 | -------------------------------------------------------------------------------- /_includes/carpentries.html: -------------------------------------------------------------------------------- 1 |
2 |
3 | Software Carpentry logo 4 |
5 |
6 | Since 1998, 7 | Software Carpentry 8 | has been teaching researchers in science, engineering, medicine, and related disciplines 9 | the computing skills they need to get more done in less time and with less pain. 10 | Its volunteer instructors have run hundreds of events 11 | for thousands of learners in the past two and a half years. 12 |
13 |
14 |
15 |
16 |
17 | Data Carpentry logo 18 |
19 |
20 | Data Carpentry develops and teaches workshops on the fundamental data skills needed to conduct research. 21 | Its target audience is researchers who have little to no prior computational experience, 22 | and its lessons are domain specific, 23 | building on learners' existing knowledge to enable them to quickly apply skills learned to their own research. 24 |
25 |
26 | 27 | 28 | -------------------------------------------------------------------------------- /CONDUCT.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: "Contributor Code of Conduct" 4 | permalink: /conduct/ 5 | --- 6 | As contributors and maintainers of this project, 7 | we pledge to respect all people who contribute through reporting issues, 8 | posting feature requests, 9 | updating documentation, 10 | submitting pull requests or patches, 11 | and other activities. 12 | 13 | We are committed to making participation in this project a harassment-free experience for everyone, 14 | regardless of level of experience, 15 | gender, 16 | gender identity and expression, 17 | sexual orientation, 18 | disability, 19 | personal appearance, 20 | body size, 21 | race, 22 | ethnicity, 23 | age, 24 | or religion. 25 | 26 | Examples of unacceptable behavior by participants include the use of sexual language or imagery, 27 | derogatory comments or personal attacks, 28 | trolling, 29 | public or private harassment, 30 | insults, 31 | or other unprofessional conduct. 32 | 33 | Project maintainers have the right and responsibility to remove, edit, or reject 34 | comments, commits, code, wiki edits, issues, and other contributions 35 | that are not aligned to this Code of Conduct. 36 | Project maintainers who do not follow the Code of Conduct may be removed from the project team. 37 | 38 | Instances of abusive, harassing, or otherwise unacceptable behavior 39 | may be reported by opening an issue or contacting one or more of the project maintainers. 40 | 41 | This Code of Conduct is adapted from 42 | the [Contributor Covenant][contrib-covenant] Version 1.0.0. 43 | 44 | [contrib-covenant]: http://contributor-covenant.org/ 45 | -------------------------------------------------------------------------------- /_includes/episode_title.html: -------------------------------------------------------------------------------- 1 | {% comment %} 2 | Find previous and next episodes (if any). 3 | {% endcomment %} 4 | {% for episode in site.episodes %} 5 | {% if episode.url == page.url %} 6 | {% unless forloop.first %} 7 | {% assign prev_episode = prev %} 8 | {% endunless %} 9 | {% unless forloop.last %} 10 | {% assign next_episode = site.episodes[forloop.index] %} 11 | {% endunless %} 12 | {% endif %} 13 | {% assign prev = episode %} 14 | {% endfor %} 15 | 16 | {% comment %} 17 | Display title and prev/next links. 18 | {% endcomment %} 19 |
20 |
21 |

22 | {% if prev_episode %} 23 | 24 | {% elsif site.github.url %} 25 | 26 | {% else %} 27 | 28 | {% endif %} 29 |

30 |
31 |
32 |

{{ site.title }}

33 |

{{ page.title }}

34 |
35 |
36 |

37 | {% if next_episode %} 38 | 39 | {% elsif site.github.url %} 40 | 41 | {% else %} 42 | 43 | {% endif %} 44 |

45 |
46 |
47 | -------------------------------------------------------------------------------- /_layouts/base.html: -------------------------------------------------------------------------------- 1 | --- 2 | --- 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | {% if site.carpentry == "swc" %} 15 | 16 | {% endif %} 17 | {% if site.carpentry == "dc" %} 18 | 19 | {% endif %} 20 | 21 | 22 | 26 | {{ site.title }}{% if page.title %}: {{ page.title }}{% endif %} 27 | 28 | 29 |
30 | {% include navbar.html %} 31 | {{ content }} 32 | {% if site.kind == "workshop" %} 33 | {% include workshop_footer.html %} 34 | {% else %} 35 | {% include lesson_footer.html %} 36 | {% endif %} 37 |
38 | {% include javascript.html %} 39 | 40 | 41 | -------------------------------------------------------------------------------- /_config.yml: -------------------------------------------------------------------------------- 1 | #------------------------------------------------------------ 2 | # Values for this lesson. 3 | #------------------------------------------------------------ 4 | 5 | # Which carpentry is this ("swc" or "dc")? 6 | carpentry: "swc" 7 | 8 | # Overall title for pages. 9 | title: "Modern Scientific Authoring" 10 | 11 | # Contact email address. 12 | email: lessons@software-carpentry.org 13 | 14 | #------------------------------------------------------------ 15 | # Generic settings (should not need to change). 16 | #------------------------------------------------------------ 17 | 18 | # What kind of thing is this ("workshop" or "lesson")? 19 | kind: "lesson" 20 | 21 | # Magic to make URLs resolve both locally and on GitHub. 22 | # See https://help.github.com/articles/repository-metadata-on-github-pages/. 23 | repository: / 24 | 25 | # Sites. 26 | amy_site: "https://amy.software-carpentry.org/workshops" 27 | dc_site: "http://datacarpentry.org" 28 | swc_github: "https://github.com/swcarpentry" 29 | swc_site: "https://software-carpentry.org" 30 | swc_pages: "https://swcarpentry.github.io" 31 | template_repo: "https://github.com/swcarpentry/styles" 32 | example_repo: "https://github.com/swcarpentry/lesson-example" 33 | example_site: "https://swcarpentry.github.com/lesson-example" 34 | workshop_repo: "https://github.com/swcarpentry/workshop-template" 35 | workshop_site: "https://swcarpentry.github.io/workshop-template" 36 | training_site: "https://swcarpentry.github.io/instructor-training" 37 | 38 | # Surveys. 39 | pre_survey: "https://www.surveymonkey.com/r/swc_pre_workshop_v1?workshop_id=" 40 | post_survey: "https://www.surveymonkey.com/r/swc_post_workshop_v1?workshop_id=" 41 | 42 | # Start time in minutes (0 to be clock-independent, 540 to show a start at 09:00 am) 43 | start_time: 0 44 | 45 | # Specify that things in the episodes collection should be output. 46 | collections: 47 | episodes: 48 | output: true 49 | permalink: /:path/ 50 | extras: 51 | output: true 52 | 53 | # Set the default layout for things in the episodes collection. 54 | defaults: 55 | - scope: 56 | path: "" 57 | type: episodes 58 | values: 59 | layout: episode 60 | 61 | # Files and directories that are not to be copied. 62 | exclude: 63 | - Makefile 64 | - bin 65 | 66 | # Turn off built-in syntax highlighting. 67 | highlighter: false 68 | -------------------------------------------------------------------------------- /bin/chunk-options.R: -------------------------------------------------------------------------------- 1 | # These settings control the behavior of all chunks in the novice R materials. 2 | # For example, to generate the lessons with all the output hidden, simply change 3 | # `results` from "markup" to "hide". 4 | # For more information on available chunk options, see 5 | # http://yihui.name/knitr/options#chunk_options 6 | 7 | library("knitr") 8 | 9 | fix_fig_path <- function(pth) file.path("..", pth) 10 | 11 | 12 | ## We set the path for the figures globally below, so if we want to 13 | ## customize it for individual episodes, we can append a prefix to the 14 | ## global path. For instance, if we call knitr_fig_path("01-") in the 15 | ## first episode of the lesson, it will generate the figures in 16 | ## `fig/rmd-01-` 17 | knitr_fig_path <- function(prefix) { 18 | new_path <- paste0(opts_chunk$get("fig.path"), 19 | prefix) 20 | opts_chunk$set(fig.path = new_path) 21 | } 22 | 23 | ## We use the rmd- prefix for the figures generated by the lssons so 24 | ## they can be easily identified and deleted by `make clean-rmd`. The 25 | ## working directory when the lessons are generated is the root so the 26 | ## figures need to be saved in fig/, but when the site is generated, 27 | ## the episodes will be one level down. We fix the path using the 28 | ## `fig.process` option. 29 | 30 | opts_chunk$set(tidy = FALSE, results = "markup", comment = NA, 31 | fig.align = "center", fig.path = "fig/rmd-", 32 | fig.process = fix_fig_path) 33 | 34 | # The hooks below add html tags to the code chunks and their output so that they 35 | # are properly formatted when the site is built. 36 | 37 | hook_in <- function(x, options) { 38 | stringr::str_c("\n\n~~~\n", 39 | paste0(x, collapse="\n"), 40 | "\n~~~\n{: .r}\n\n") 41 | } 42 | 43 | hook_out <- function(x, options) { 44 | x <- gsub("\n$", "", x) 45 | stringr::str_c("\n\n~~~\n", 46 | paste0(x, collapse="\n"), 47 | "\n~~~\n{: .output}\n\n") 48 | } 49 | 50 | hook_error <- function(x, options) { 51 | x <- gsub("\n$", "", x) 52 | stringr::str_c("\n\n~~~\n", 53 | paste0(x, collapse="\n"), 54 | "\n~~~\n{: .error}\n\n") 55 | } 56 | 57 | knit_hooks$set(source = hook_in, output = hook_out, warning = hook_error, 58 | error = hook_error, message = hook_out) 59 | -------------------------------------------------------------------------------- /_layouts/workshop.html: -------------------------------------------------------------------------------- 1 | --- 2 | --- 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | {% if page.redirect %} 21 | 22 | {% endif %} 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | {% if site.carpentry == "swc" %} 32 | 33 | {% endif %} 34 | {% if site.carpentry == "dc" %} 35 | 36 | {% endif %} 37 | 38 | 39 | 43 | {{ page.venue }}: {{ page.humandate }} 44 | 45 | 46 |
47 | {% include navbar.html %} 48 | {% include workshop_ad.html %} 49 | {{ content }} 50 | {% include workshop_footer.html %} 51 |
52 | {% include javascript.html %} 53 | 54 | 55 | -------------------------------------------------------------------------------- /_includes/syllabus.html: -------------------------------------------------------------------------------- 1 | {% comment %} 2 | Display syllabus in tabular form. 3 | Days are displayed if at least one episode has 'start = true'. 4 | {% endcomment %} 5 |
6 |

Schedule

7 | 8 | {% assign day = 0 %} 9 | {% assign multiday = false %} 10 | {% for episode in site.episodes %} 11 | {% if episode.start %}{% assign multiday = true %}{% break %}{% endif %} 12 | {% endfor %} 13 | {% assign current = site.start_time %} 14 | 15 | 16 | {% for episode in site.episodes %} 17 | {% if episode.start %} {% comment %} Starting a new day? {% endcomment %} 18 | {% assign day = day | plus: 1 %} 19 | {% if day > 1 %} {% comment %} If about to start day 2 or later, show finishing time for previous day {% endcomment %} 20 | {% assign hours = current | divided_by: 60 %} 21 | {% assign minutes = current | modulo: 60 %} 22 | 23 | {% if multiday %}{% endif %} 24 | 25 | 26 | 27 | 28 | {% endif %} 29 | {% assign current = site.start_time %} {% comment %}Re-set start time of this episode to general daily start time {% endcomment %} 30 | {% endif %} 31 | {% assign hours = current | divided_by: 60 %} 32 | {% assign minutes = current | modulo: 60 %} 33 | 34 | {% if multiday %}{% endif %} 35 | 36 | 39 | 53 | 54 | {% assign current = current | plus: episode.teaching | plus: episode.exercises | plus: episode.break %} 55 | {% endfor %} 56 | {% assign hours = current | divided_by: 60 %} 57 | {% assign minutes = current | modulo: 60 %} 58 | 59 | {% if multiday %}{% endif %} 60 | 61 | 62 | 63 | 64 |
{% if hours < 10 %}0{% endif %}{{ hours }}:{% if minutes < 10 %}0{% endif %}{{ minutes }}Finish
{% if episode.start %}Day {{ day }}{% endif %}{% if hours < 10 %}0{% endif %}{{ hours }}:{% if minutes < 10 %}0{% endif %}{{ minutes }} 37 | {{ episode.title }} 38 | 40 | {% if episode.break %} 41 | Break 42 | {% else %} 43 | {% if episode.questions %} 44 | {% for question in episode.questions %} 45 | {{question|markdownify|strip_html}} 46 | {% unless forloop.last %} 47 |
48 | {% endunless %} 49 | {% endfor %} 50 | {% endif %} 51 | {% endif %} 52 |
{% if hours < 10 %}0{% endif %}{{ hours }}:{% if minutes < 10 %}0{% endif %}{{ minutes }}Finish
65 |
66 | -------------------------------------------------------------------------------- /bin/setup-labels: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | ## This script sets up labels for issues in your GitHub repository. 4 | ## 5 | ## Syntax: 6 | ## 7 | ## $ tools/setup-label OWNER REPO 8 | ## 9 | ## Parameters: 10 | ## 11 | ## - OWNER: GitHub username of the owner of the repository 12 | ## - REPO: the name of the repository 13 | ## 14 | ## Example: 15 | ## 16 | ## For set up the labels at https://github.com/wking/swc-modular-shell use 17 | ## 18 | ## $ tools/setup-label wking swc-modular-shell 19 | 20 | CURL_FLAGS="--silent --fail" 21 | 22 | if test $# -lt 2 23 | then 24 | echo "Missing parameters." 25 | echo 26 | grep '^##' tools/setup-labels | sed 's/## //' | sed 's/##//' 27 | exit 1 28 | fi 29 | 30 | OWNER=$1 31 | REPO=$2 32 | GITHUB_URL=https://github.com/${OWNER}/${REPO} 33 | LABELS=(bug build defer discussion documentation enhancement newcomer-friendly upstream work-in-progress) 34 | COLORS=(FF0000 551033 66FF00 0000FF D4318C E0115F FCE883 83F52C 545AA7) 35 | LABELS_TO_DELETE=(duplicate filed-by-newcomer getting-started help%20wanted help-wanted invalid left-as-was suitable-for-newcomer question wontfix) 36 | 37 | # Test if repository exists 38 | curl -s --head ${GITHUB_URL} | head -n 1 | grep -q "HTTP/1.[01] [23].." 39 | if test $? -ne 0 40 | then 41 | echo "ERROR: this repository doesn't exist" 42 | exit $? 43 | fi 44 | 45 | echo "Before setup the labels for ${GITHUB_URL}" 46 | echo "you must provide some informations." 47 | echo "Your GitHub username:" 48 | read USERNAME 49 | echo "Your GitHub password:" 50 | read -s PASSWORD 51 | 52 | # Delete labels 53 | for INDEX in $(seq 0 $((${#LABELS_TO_DELETE[*]} - 1))) 54 | do 55 | # Try to delete label 56 | curl ${CURL_FLAGS} -X DELETE \ 57 | -u ${USERNAME}:${PASSWORD} \ 58 | "https://api.github.com/repos/${OWNER}/${REPO}/labels/${LABELS_TO_DELETE[${INDEX}]}" > /dev/null 59 | done 60 | # Create labels 61 | for INDEX in $(seq 0 $((${#LABELS[*]} - 1))) 62 | do 63 | # Try create new label 64 | curl ${CURL_FLAGS} -X POST \ 65 | -u ${USERNAME}:${PASSWORD} \ 66 | -d "{\"name\":\"${LABELS[${INDEX}]}\",\"color\":\"${COLORS[${INDEX}]}\"}" \ 67 | "https://api.github.com/repos/${OWNER}/${REPO}/labels" > /dev/null 68 | if test $? -ne 0 69 | then 70 | # Try to fix label color 71 | curl ${CURL_FLAGS} -X PATCH \ 72 | -u ${USERNAME}:${PASSWORD} \ 73 | -d "{\"name\":\"${LABELS[${INDEX}]}\",\"color\":\"${COLORS[${INDEX}]}\"}" \ 74 | "https://api.github.com/repos/${OWNER}/${REPO}/labels/${LABELS[${INDEX}]}" > /dev/null 75 | if test $? -ne 0 76 | then 77 | echo "Failed when trying to create and update the label ${LABELS[${INDEX}]}." 78 | echo "Please check at ${GITHUB_URL}/labels" 79 | echo "" 80 | echo "If you find a bug report it at" 81 | echo "https://github.com/swcarpentry/lesson-template/." 82 | fi 83 | fi 84 | done 85 | -------------------------------------------------------------------------------- /bin/extract_figures.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | from __future__ import print_function 4 | import sys 5 | import os 6 | import glob 7 | from optparse import OptionParser 8 | 9 | from util import Reporter, read_markdown, IMAGE_FILE_SUFFIX 10 | 11 | def main(): 12 | """Main driver.""" 13 | 14 | args = parse_args() 15 | images = [] 16 | for filename in args.filenames: 17 | images += get_images(args.parser, filename) 18 | save(sys.stdout, images) 19 | 20 | 21 | def parse_args(): 22 | """Parse command-line arguments.""" 23 | 24 | parser = OptionParser() 25 | parser.add_option('-p', '--parser', 26 | default=None, 27 | dest='parser', 28 | help='path to Markdown parser') 29 | 30 | args, extras = parser.parse_args() 31 | require(args.parser is not None, 32 | 'Path to Markdown parser not provided') 33 | require(extras, 34 | 'No filenames specified') 35 | 36 | args.filenames = extras 37 | return args 38 | 39 | 40 | def get_filenames(source_dir): 41 | """Get all filenames to be searched for images.""" 42 | 43 | return glob.glob(os.path.join(source_dir, '*.md')) 44 | 45 | 46 | def get_images(parser, filename): 47 | """Extract all images from file.""" 48 | 49 | content = read_markdown(parser, filename) 50 | result = [] 51 | find_image_nodes(content['doc'], result) 52 | find_image_links(content['doc'], result) 53 | return result 54 | 55 | 56 | def find_image_nodes(doc, result): 57 | """Find all nested nodes representing images.""" 58 | 59 | if (doc['type'] == 'img') or \ 60 | ((doc['type'] == 'html_element') and (doc['value'] == 'img')): 61 | alt = doc['attr'].get('alt', '') 62 | result.append({'alt': alt, 'src': doc['attr']['src']}) 63 | else: 64 | for child in doc.get('children', []): 65 | find_image_nodes(child, result) 66 | 67 | 68 | def find_image_links(doc, result): 69 | """Find all links to files in the 'fig' directory.""" 70 | 71 | if ((doc['type'] == 'a') and ('attr' in doc) and ('href' in doc['attr'])) \ 72 | or \ 73 | ((doc['type'] == 'html_element') and (doc['value'] == 'a') and ('href' in doc['attr'])): 74 | path = doc['attr']['href'] 75 | if os.path.splitext(path)[1].lower() in IMAGE_FILE_SUFFIX: 76 | result.append({'alt':'', 'src': doc['attr']['href']}) 77 | else: 78 | for child in doc.get('children', []): 79 | find_image_links(child, result) 80 | 81 | 82 | def save(stream, images): 83 | """Save results as Markdown.""" 84 | 85 | text = '\n
\n'.join(['

{0}

'.format(img['alt'], img['src']) for img in images]) 86 | print(text, file=stream) 87 | 88 | 89 | def require(condition, message): 90 | """Fail if condition not met.""" 91 | 92 | if not condition: 93 | print(message, file=sys.stderr) 94 | sys.exit(1) 95 | 96 | 97 | if __name__ == '__main__': 98 | main() 99 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: "Licenses" 4 | permalink: /license/ 5 | --- 6 | ## Instructional Material 7 | 8 | All Software Carpentry and Data Carpentry instructional material is 9 | made available under the [Creative Commons Attribution 10 | license][cc-by-human]. The following is a human-readable summary of 11 | (and not a substitute for) the [full legal text of the CC BY 4.0 12 | license][cc-by-legal]. 13 | 14 | You are free: 15 | 16 | * to **Share**---copy and redistribute the material in any medium or format 17 | * to **Adapt**---remix, transform, and build upon the material 18 | 19 | for any purpose, even commercially. 20 | 21 | The licensor cannot revoke these freedoms as long as you follow the 22 | license terms. 23 | 24 | Under the following terms: 25 | 26 | * **Attribution**---You must give appropriate credit (mentioning that 27 | your work is derived from work that is Copyright © Software 28 | Carpentry and, where practical, linking to 29 | http://software-carpentry.org/), provide a [link to the 30 | license][cc-by-human], and indicate if changes were made. You may do 31 | so in any reasonable manner, but not in any way that suggests the 32 | licensor endorses you or your use. 33 | 34 | **No additional restrictions**---You may not apply legal terms or 35 | technological measures that legally restrict others from doing 36 | anything the license permits. With the understanding that: 37 | 38 | Notices: 39 | 40 | * You do not have to comply with the license for elements of the 41 | material in the public domain or where your use is permitted by an 42 | applicable exception or limitation. 43 | * No warranties are given. The license may not give you all of the 44 | permissions necessary for your intended use. For example, other 45 | rights such as publicity, privacy, or moral rights may limit how you 46 | use the material. 47 | 48 | ## Software 49 | 50 | Except where otherwise noted, the example programs and other software 51 | provided by Software Carpentry and Data Carpentry are made available under the 52 | [OSI][osi]-approved 53 | [MIT license][mit-license]. 54 | 55 | Permission is hereby granted, free of charge, to any person obtaining 56 | a copy of this software and associated documentation files (the 57 | "Software"), to deal in the Software without restriction, including 58 | without limitation the rights to use, copy, modify, merge, publish, 59 | distribute, sublicense, and/or sell copies of the Software, and to 60 | permit persons to whom the Software is furnished to do so, subject to 61 | the following conditions: 62 | 63 | The above copyright notice and this permission notice shall be 64 | included in all copies or substantial portions of the Software. 65 | 66 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, 67 | EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF 68 | MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND 69 | NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE 70 | LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION 71 | OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION 72 | WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. 73 | 74 | ## Trademark 75 | 76 | "Software Carpentry" an "Data Carpentry" and their respective logos 77 | are registered trademarks of [NumFOCUS][numfocus]. 78 | 79 | [cc-by-human]: https://creativecommons.org/licenses/by/4.0/ 80 | [cc-by-legal]: https://creativecommons.org/licenses/by/4.0/legalcode 81 | [mit-license]: http://opensource.org/licenses/mit-license.html 82 | [numfocus]: http://numfocus.org/ 83 | [osi]: http://opensource.org 84 | -------------------------------------------------------------------------------- /Makefile: -------------------------------------------------------------------------------- 1 | ## ======================================== 2 | ## Commands for both workshop and lesson websites. 3 | 4 | # Settings 5 | MAKEFILES=Makefile $(wildcard *.mk) 6 | JEKYLL=jekyll 7 | PARSER=bin/markdown_ast.rb 8 | DST=_site 9 | 10 | # Controls 11 | .PHONY : commands clean files 12 | all : commands 13 | 14 | ## commands : show all commands. 15 | commands : 16 | @grep -h -E '^##' ${MAKEFILES} | sed -e 's/## //g' 17 | 18 | ## serve : run a local server. 19 | serve : lesson-rmd 20 | ${JEKYLL} serve 21 | 22 | ## site : build files but do not run a server. 23 | site : lesson-rmd 24 | ${JEKYLL} build 25 | 26 | # repo-check : check repository settings. 27 | repo-check : 28 | @bin/repo_check.py -s . 29 | 30 | ## clean : clean up junk files. 31 | clean : 32 | @rm -rf ${DST} 33 | @rm -rf .sass-cache 34 | @rm -rf bin/__pycache__ 35 | @find . -name .DS_Store -exec rm {} \; 36 | @find . -name '*~' -exec rm {} \; 37 | @find . -name '*.pyc' -exec rm {} \; 38 | 39 | ## clean-rmd : clean intermediate R files (that need to be committed to the repo). 40 | clear-rmd : 41 | @rm -rf ${RMD_DST} 42 | @rm -rf fig/rmd-* 43 | 44 | ## ---------------------------------------- 45 | ## Commands specific to workshop websites. 46 | 47 | .PHONY : workshop-check 48 | 49 | ## workshop-check : check workshop homepage. 50 | workshop-check : 51 | @bin/workshop_check.py . 52 | 53 | ## ---------------------------------------- 54 | ## Commands specific to lesson websites. 55 | 56 | .PHONY : lesson-check lesson-rmd lesson-files lesson-fixme 57 | 58 | # RMarkdown files 59 | RMD_SRC = $(wildcard _episodes_rmd/??-*.Rmd) 60 | RMD_DST = $(patsubst _episodes_rmd/%.Rmd,_episodes/%.md,$(RMD_SRC)) 61 | 62 | # Lesson source files in the order they appear in the navigation menu. 63 | MARKDOWN_SRC = \ 64 | index.md \ 65 | CONDUCT.md \ 66 | setup.md \ 67 | $(wildcard _episodes/*.md) \ 68 | reference.md \ 69 | $(wildcard _extras/*.md) \ 70 | LICENSE.md 71 | 72 | # Generated lesson files in the order they appear in the navigation menu. 73 | HTML_DST = \ 74 | ${DST}/index.html \ 75 | ${DST}/conduct/index.html \ 76 | ${DST}/setup/index.html \ 77 | $(patsubst _episodes/%.md,${DST}/%/index.html,$(wildcard _episodes/*.md)) \ 78 | ${DST}/reference/index.html \ 79 | $(patsubst _extras/%.md,${DST}/%/index.html,$(wildcard _extras/*.md)) \ 80 | ${DST}/license/index.html 81 | 82 | ## lesson-rmd : convert Rmarkdown files to markdown 83 | lesson-rmd: $(RMD_SRC) 84 | @bin/knit_lessons.sh $(RMD_SRC) 85 | 86 | ## lesson-check : validate lesson Markdown. 87 | lesson-check : 88 | @bin/lesson_check.py -s . -p ${PARSER} 89 | 90 | ## lesson-check-all : validate lesson Markdown, checking line lengths and trailing whitespace. 91 | lesson-check-all : 92 | @bin/lesson_check.py -s . -p ${PARSER} -l -w 93 | 94 | ## lesson-figures : re-generate inclusion displaying all figures. 95 | lesson-figures : 96 | @bin/extract_figures.py -p ${PARSER} ${MARKDOWN_SRC} > _includes/all_figures.html 97 | 98 | ## unittest : run unit tests on checking tools. 99 | unittest : 100 | python bin/test_lesson_check.py 101 | 102 | ## lesson-files : show expected names of generated files for debugging. 103 | lesson-files : 104 | @echo 'RMD_SRC:' ${RMD_SRC} 105 | @echo 'RMD_DST:' ${RMD_DST} 106 | @echo 'MARKDOWN_SRC:' ${MARKDOWN_SRC} 107 | @echo 'HTML_DST:' ${HTML_DST} 108 | 109 | ## lesson-fixme : show FIXME markers embedded in source files. 110 | lesson-fixme : 111 | @fgrep -i -n FIXME ${MARKDOWN_SRC} || true 112 | 113 | #------------------------------------------------------------------------------- 114 | # Include extra commands if available. 115 | #------------------------------------------------------------------------------- 116 | 117 | -include commands.mk 118 | -------------------------------------------------------------------------------- /_includes/navbar.html: -------------------------------------------------------------------------------- 1 | 77 | -------------------------------------------------------------------------------- /assets/css/lesson.scss: -------------------------------------------------------------------------------- 1 | --- 2 | --- 3 | 4 | //---------------------------------------- 5 | // Colors. 6 | //---------------------------------------- 7 | 8 | // branding 9 | $color-brand: #2b3990 !default; 10 | 11 | // code boxes 12 | $color-error: #bd2c00 !default; 13 | $color-output: #303030 !default; 14 | $color-source: #6e5494 !default; 15 | 16 | // blockquotes 17 | $color-callout: #f4fd9c !default; 18 | $color-challenge: #eec275 !default; 19 | $color-checklist: #dfd2a0 !default; 20 | $color-discussion: #eec275 !default; 21 | $color-keypoints: #7ae78e !default; 22 | $color-objectives: #daee84 !default; 23 | $color-prereq: #9cd6dc !default; 24 | $color-solution: #ded4b9 !default; 25 | $color-testimonial: #fc8dc1 !default; 26 | 27 | //---------------------------------------- 28 | // Specialized code blocks. 29 | //---------------------------------------- 30 | 31 | @mixin cdSetup($color) { 32 | color: $color; 33 | border-left: solid 5px $color; 34 | margin-bottom: 0px; 35 | } 36 | 37 | .error { @include cdSetup($color-error); } 38 | .output { @include cdSetup($color-output); } 39 | .source { @include cdSetup($color-source); } 40 | 41 | .bash { @include cdSetup($color-source); } 42 | .make { @include cdSetup($color-source); } 43 | .matlab { @include cdSetup($color-source); } 44 | .python { @include cdSetup($color-source); } 45 | .r { @include cdSetup($color-source); } 46 | .sql { @include cdSetup($color-source); } 47 | 48 | //---------------------------------------- 49 | // Specialized blockquote environments for learning objectives, callouts, etc. 50 | //---------------------------------------- 51 | 52 | $codeblock-padding: 5px !default; 53 | 54 | @mixin bkSetup($color, $glyph) { 55 | 56 | $gradientcolor1: $color; 57 | $gradientcolor2: scale-color($color, $lightness: 10%); 58 | 59 | padding-left: $codeblock-padding; 60 | padding-top: 0; 61 | padding-bottom: 0; 62 | padding-right: 0; 63 | border: 1px solid; 64 | border-color: $color; 65 | padding-bottom: $codeblock-padding; 66 | 67 | h2 { 68 | padding-top: $codeblock-padding; 69 | padding-bottom: $codeblock-padding; 70 | font-size: 20px; 71 | background: linear-gradient(to bottom, $gradientcolor1, $gradientcolor2); 72 | border-color: $color; 73 | margin-top: 0px; 74 | margin-left: -$codeblock-padding; // to move back to the left margin of the enclosing blockquote 75 | } 76 | h2:before { 77 | font-family: 'Glyphicons Halflings'; 78 | content: $glyph; 79 | float: left; 80 | padding-left: $codeblock-padding; 81 | padding-right: $codeblock-padding; 82 | display: inline-block; 83 | -webkit-font-smoothing: antialiased; 84 | } 85 | 86 | } 87 | 88 | .callout{ @include bkSetup($color-callout, "\e146"); } 89 | .challenge{ @include bkSetup($color-challenge, "\270f"); } 90 | .checklist{ @include bkSetup($color-checklist, "\e067"); } 91 | .discussion{ @include bkSetup($color-discussion, "\e123"); } 92 | .keypoints{ @include bkSetup($color-keypoints, "\e101"); } 93 | .objectives{ @include bkSetup($color-objectives, "\e085"); } 94 | .prereq{ @include bkSetup($color-prereq, "\e124"); } 95 | .solution{ @include bkSetup($color-solution, "\e105"); } 96 | .testimonial{ @include bkSetup($color-testimonial, "\e143"); } 97 | 98 | //---------------------------------------- 99 | // Override Bootstrap settings. 100 | //---------------------------------------- 101 | 102 | code { 103 | padding: 0 0; 104 | color: inherit; 105 | background-color: inherit; 106 | } 107 | 108 | img { 109 | max-width: 100%; 110 | } 111 | 112 | //---------------------------------------- 113 | // Miscellaneous. 114 | //---------------------------------------- 115 | 116 | .maintitle { 117 | text-align: center; 118 | } 119 | 120 | .footertext { 121 | text-align: center; 122 | } 123 | 124 | img.navbar-logo { 125 | height: 40px; // synchronize with height of navbar 126 | padding-top: 5px; 127 | padding-right: 10px; 128 | } 129 | 130 | div.branding { 131 | color: $color-brand; 132 | } 133 | 134 | ul, 135 | ol { 136 | padding-left: 1em; 137 | } 138 | 139 | span.fold-unfold { 140 | margin-left: 1em; 141 | opacity: 0.5; 142 | } 143 | -------------------------------------------------------------------------------- /_episodes/03-advanced.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Advanced Usage" 3 | teaching: 10 4 | exercises: 10 5 | questions: 6 | - "FIXME?" 7 | objectives: 8 | - "FIXME." 9 | keypoints: 10 | - "FIXME." 11 | --- 12 | 13 | Academic papers have more information than text. In this section, we will cover 14 | how to add references, tables, figures, and equations. One important thing to 15 | keep in mind, is that (at least when using `pandoc`, which is the most common 16 | program) `LaTeX` commands will be understood when converting to other formats. 17 | This greatly simplifies the use of equations, for example. 18 | 19 | # Equations 20 | 21 | Equations can be written using the `LaTeX` syntax. For example, the block below 22 | is valid `markdown`, 23 | 24 | ~~~ 25 | The equation for a polynomial function is $y(x) = ax^2 + bx +c$. 26 | ~~~ 27 | {: .source} 28 | 29 | and so is this one: 30 | 31 | ~~~ 32 | The sum of a vector of numbers ($\mathbf{v}$) is noted 33 | 34 | \begin{equation} 35 | \sum_{x=1}^n\mathbf{v}_i 36 | \end{equation} 37 | ~~~ 38 | {: .source} 39 | 40 | # Tables 41 | 42 | One of the issues with markdown is that tables tend to be poorly supported 43 | (although it is possible to use raw `LaTeX`). Nevertheless, there is a way to 44 | write tables that are relatively simple. The table below 45 | 46 | | Lesson | Maintainer | Pre-requisites | 47 | |:---------|:-----------|----------------------:| 48 | | markdown | Tim | shell, git, makefiles | 49 | 50 | is noted as 51 | 52 | ~~~ 53 | | Lesson | Maintainer | Pre-requisites | 54 | |:---------|:-----------|----------------------:| 55 | | markdown | Tim | shell, git, makefiles | 56 | ~~~ 57 | {: .source} 58 | 59 | There are a few elements to tables. The first line is the *headers*. The second 60 | line gives the *alignment*. The subsequent lines are the *content* of the table. 61 | 62 | Columns are separated by a pipe (`|`). The pipes do not need to be vertically 63 | aligned (but it helps a great deal when *reading* the raw documents -- there are 64 | plugins to take care of this in most editors). 65 | 66 | By default, columns are *left-aligned*. To specify the alignment, one needs 67 | using `:` in the second line, in the following ways: 68 | 69 | | Left-aligned | Centered | Right aligned | Default (left) | 70 | |:-------------|:--------:|--------------:|:---------------| 71 | | `:---` | `:--:` | `---:` | `----` | 72 | 73 | The code for this table is 74 | 75 | ~~~ 76 | | Left-aligned | Centered | Right aligned | Default (left) | 77 | |:-------------|:--------:|--------------:|:---------------| 78 | | `:---` | `:--:` | `---:` | `----` | 79 | ~~~ 80 | {: .source} 81 | 82 | # Figures 83 | 84 | Figures are well supported by markdown. Their notation closely follows the one 85 | for links, only preceded by an exclamation mark (`!`). 86 | 87 | For example, 88 | 89 | ![Software carpentry logo \label{f:swc}]({{ site.github.url }}/assets/img/swc-logo-blue.svg) 90 | 91 | is noted: 92 | 93 | ~~~ 94 | ![Software carpentry logo \label{f:swc}]({{ site.github.url }}/assets/img/swc-logo-blue.svg) 95 | ~~~ 96 | {: .source} 97 | 98 | or alternatively, 99 | 100 | ~~~ 101 | ![Software carpentry logo \label{f:swc}][swc] 102 | 103 | [swc]: {{ site.github.url }}/assets/img/swc-logo-blue.svg 104 | ~~~ 105 | {: .source} 106 | 107 | Note the presence of `\label{f:swc}`, which is a `LaTeX` command. This allows to 108 | refer to the figure in the text, using `\autoref{f:swc}`. The `autoref` package 109 | for `LaTeX` is an incredibly useful one, that knows the type of object being 110 | referred to, and will write out `Fig. 1`, `Tab. 2`, `Eqn. 3`, or whatever is 111 | needed, without human intervention. 112 | 113 | # References 114 | 115 | The last requirement for an academic paper is references. Markdown, through 116 | `pandoc` and its extension `pandoc-citeproc`, handles these very graciously. The 117 | `pandoc` bibliography module is able to read citations from a variety of 118 | formats. Originally designed for CSL JSON and CSL YAML, it can acomodate, for 119 | example, bibtex and RIS. 120 | 121 | The way to note a reference is `@CitationKey`. For example, if your (bibtex) 122 | library contains the following reference: 123 | 124 | ~~~ 125 | @ARTICLE{thom99, 126 | title = {The raw material for coevolution}, 127 | journal = {Oikos}, 128 | author = {Thompson, John N}, 129 | number = {1}, 130 | volume = {84}, 131 | year = {1999}, 132 | pages = {5--16}, 133 | } 134 | ~~~ 135 | {: .source} 136 | 137 | you can refer to this article by `@thom99` in the text. All modern reference 138 | management software can export to one of the formats supported by `pandoc`, and 139 | most of them will let you customize the way the citation key looks. 140 | 141 | References can be combined (`[@John2012; @Jack2014]`), written *inline* 142 | (`@Doe2013` will result in `Doe (2013)` in the text if a "author-year" style is 143 | in use), or in parentheses (`[@Doe2013]` will yield `(Doe, 2013)`). It is also 144 | possible to add text to them: `[see @Billy2015 for a review]` will result in 145 | `(see Billy et al., 2015 for a review)`. 146 | 147 | The bibliography is automatically inserted at the end of the document, and, as 148 | we'll see in the next section, there are thousands of way to format it to match 149 | the journal requirements. 150 | -------------------------------------------------------------------------------- /bin/repo_check.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Check repository settings. 5 | """ 6 | 7 | from __future__ import print_function 8 | import sys 9 | import os 10 | from subprocess import Popen, PIPE 11 | import re 12 | from optparse import OptionParser 13 | 14 | from util import Reporter, load_yaml, require 15 | 16 | # Import this way to produce a more useful error message. 17 | try: 18 | import requests 19 | except ImportError: 20 | print('Unable to import requests module: please install requests', file=sys.stderr) 21 | sys.exit(1) 22 | 23 | 24 | # Pattern to match Git command-line output for remotes => (user name, project name). 25 | P_GIT_REMOTE = re.compile(r'upstream\s+[^:]+:([^/]+)/([^.]+)\.git\s+\(fetch\)') 26 | 27 | # Repository URL format string. 28 | F_REPO_URL = 'https://github.com/{0}/{1}/' 29 | 30 | # Pattern to match repository URLs => (user name, project name) 31 | P_REPO_URL = re.compile(r'https?://github\.com/([^.]+)/([^/]+)/?') 32 | 33 | # API URL format string. 34 | F_API_URL = 'https://api.github.com/repos/{0}/{1}/labels' 35 | 36 | # Expected labels and colors. 37 | EXPECTED = { 38 | 'bug' : 'bd2c00', 39 | 'discussion' : 'fc8dc1', 40 | 'enhancement' : '9cd6dc', 41 | 'help-wanted' : 'f4fd9c', 42 | 'instructor-training' : '6e5494', 43 | 'newcomer-friendly' : 'eec275', 44 | 'question' : '808040', 45 | 'template-and-tools' : '2b3990', 46 | 'work-in-progress' : '7ae78e' 47 | } 48 | 49 | 50 | def main(): 51 | """ 52 | Main driver. 53 | """ 54 | 55 | args = parse_args() 56 | reporter = Reporter() 57 | repo_url = get_repo_url(args.source_dir, args.repo_url) 58 | check_labels(reporter, repo_url) 59 | reporter.report() 60 | 61 | 62 | def parse_args(): 63 | """ 64 | Parse command-line arguments. 65 | """ 66 | 67 | parser = OptionParser() 68 | parser.add_option('-r', '--repo', 69 | default=None, 70 | dest='repo_url', 71 | help='repository URL') 72 | parser.add_option('-s', '--source', 73 | default=os.curdir, 74 | dest='source_dir', 75 | help='source directory') 76 | 77 | args, extras = parser.parse_args() 78 | require(not extras, 79 | 'Unexpected trailing command-line arguments "{0}"'.format(extras)) 80 | 81 | return args 82 | 83 | 84 | def get_repo_url(source_dir, repo_url): 85 | """ 86 | Figure out which repository to query. 87 | """ 88 | 89 | # Explicitly specified. 90 | if repo_url is not None: 91 | return repo_url 92 | 93 | # Guess. 94 | cmd = 'git remote -v' 95 | p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, close_fds=True, universal_newlines=True) 96 | stdout_data, stderr_data = p.communicate() 97 | stdout_data = stdout_data.split('\n') 98 | matches = [P_GIT_REMOTE.match(line) for line in stdout_data] 99 | matches = [m for m in matches if m is not None] 100 | require(len(matches) == 1, 101 | 'Unexpected output from git remote command: "{0}"'.format(matches)) 102 | 103 | username = matches[0].group(1) 104 | require(username, 'empty username in git remote output {0}'.format(matches[0])) 105 | 106 | project_name = matches[0].group(2) 107 | require(username, 'empty project name in git remote output {0}'.format(matches[0])) 108 | 109 | url = F_REPO_URL.format(username, project_name) 110 | return url 111 | 112 | 113 | def check_labels(reporter, repo_url): 114 | """ 115 | Check labels in repository. 116 | """ 117 | 118 | actual = get_labels(repo_url) 119 | extra = set(actual.keys()) - set(EXPECTED.keys()) 120 | 121 | reporter.check(not extra, 122 | None, 123 | 'Extra label(s) in repository {0}: {1}', 124 | repo_url, ', '.join(sorted(extra))) 125 | 126 | missing = set(EXPECTED.keys()) - set(actual.keys()) 127 | reporter.check(not missing, 128 | None, 129 | 'Missing label(s) in repository {0}: {1}', 130 | repo_url, ', '.join(sorted(missing))) 131 | 132 | overlap = set(EXPECTED.keys()).intersection(set(actual.keys())) 133 | for name in sorted(overlap): 134 | reporter.check(EXPECTED[name] == actual[name], 135 | None, 136 | 'Color mis-match for label {0} in {1}: expected {2}, found {3}', 137 | name, repo_url, EXPECTED[name], actual[name]) 138 | 139 | 140 | def get_labels(repo_url): 141 | """ 142 | Get actual labels from repository. 143 | """ 144 | 145 | m = P_REPO_URL.match(repo_url) 146 | require(m, 'repository URL {0} does not match expected pattern'.format(repo_url)) 147 | 148 | username = m.group(1) 149 | require(username, 'empty username in repository URL {0}'.format(repo_url)) 150 | 151 | project_name = m.group(2) 152 | require(username, 'empty project name in repository URL {0}'.format(repo_url)) 153 | 154 | url = F_API_URL.format(username, project_name) 155 | r = requests.get(url) 156 | require(r.status_code == 200, 157 | 'Request for {0} failed with {1}'.format(url, r.status_code)) 158 | 159 | result = {} 160 | for entry in r.json(): 161 | result[entry['name']] = entry['color'] 162 | return result 163 | 164 | 165 | if __name__ == '__main__': 166 | main() 167 | -------------------------------------------------------------------------------- /bin/filters/blockquote2div.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | """Pandoc filter to convert Blockquotes with attributes into Div 3 | with attributes. 4 | 5 | Usage: 6 | 7 | pandoc source.md --filter=blockquote2div.py --output=output.html 8 | 9 | A blockquote will be converted if 10 | 11 | 1. it begins with a header 12 | 2. that either 13 | 1. matches "Prerequisites", "Objectives", "Callout" or "Challenge" OR 14 | 2. has attributes containing a single class matching 15 | one of ['prereq', 'objectives', 'callout', 'challenge'] 16 | 17 | For example, this is a valid blockquote: 18 | 19 | > ## Callout time! {.callout} 20 | > Let's do something 21 | 22 | and it will be converted into this markdown: 23 | 24 |
25 | ## Callout time! 26 | Let's do something. 27 |
28 | 29 | This is also a valid blockquote: 30 | 31 | > ## Prerequisites 32 | > Breakfast! 33 | 34 | and it will be converted into this markdown: 35 | 36 |
37 | ## Prerequisites 38 | Breakfast! 39 |
40 | 41 | 42 | For debugging purposes you may find it useful to test the filter 43 | like this: 44 | 45 | pandoc source.md --to json | python blockquote2div.py | pandoc --from json 46 | """ 47 | import pandocfilters as pf 48 | 49 | # These are classes that, if set on the title of a blockquote, will 50 | # trigger the blockquote to be converted to a div. 51 | SPECIAL_CLASSES = { 52 | "callout": ("panel-info", "glyphicon-pushpin"), 53 | "challenge": ("panel-success", "glyphicon-pencil"), 54 | "prereq": ("panel-warning", "glyphicon-education"), 55 | "getready": ("panel-warning", "glyphicon-check"), 56 | "objectives": ("panel-warning", "glyphicon-certificate"), 57 | } 58 | 59 | 60 | def find_header(blockquote): 61 | """Find attributes in a blockquote if they are defined on a 62 | header that is the first thing in the block quote. 63 | 64 | Returns the attributes, a list [id, classes, kvs] 65 | where id = str, classes = list, kvs = list of key, value pairs 66 | """ 67 | if blockquote[0]['t'] == 'Header': 68 | level, attr, inline = blockquote[0]['c'] 69 | return level, attr, inline 70 | 71 | 72 | def blockquote2div(key, value, format, meta): 73 | """Convert a blockquote into a div if it begins with a header 74 | that has attributes containing a single class that is in the 75 | allowed classes. 76 | 77 | This function can be passed directly to toJSONFilter 78 | from pandocfilters. 79 | """ 80 | if key == 'BlockQuote': 81 | blockquote = value 82 | 83 | header = find_header(blockquote) 84 | if not header: 85 | return 86 | else: 87 | level, attr, inlines = header 88 | 89 | id, classes, kvs = attr 90 | 91 | if len(classes) == 1 and classes[0] in SPECIAL_CLASSES: 92 | panel_kind, glyphicon_kind = SPECIAL_CLASSES[classes[0]] 93 | 94 | h_level, h_attr, h_inlines = blockquote[0]['c'] 95 | 96 | # insert an icon as the first sub-item of the header 97 | span = pf.Span(["", ["glyphicon", glyphicon_kind], []], []) 98 | h_inlines.insert(0, span) 99 | 100 | # only the header goes into panel-heading 101 | header = pf.Header(h_level, [h_attr[0], [], []], h_inlines) 102 | panel_header = pf.Div(("", ["panel-heading"], []), [header]) 103 | 104 | # the rest of the blockquote goes into panel-body 105 | panel_body = pf.Div(("", ["panel-body"], []), blockquote[1:]) 106 | 107 | # apply Bootstrap panel classes to the div 108 | classes.append("panel") 109 | classes.append(panel_kind) 110 | 111 | # a blockquote is just a list of blocks, so it can be 112 | # passed directly to Div, which expects Div(attr, blocks) 113 | if classes[0] == "callout": 114 | return [{"t": "RawBlock", "c": [ "html", "" ]}] 118 | else: 119 | return [{"t": "RawBlock", "c": [ "html", "
".format(' '.join(classes)) ]}, 120 | panel_header, 121 | panel_body, 122 | {"t": "RawBlock", "c": [ "html", "
" ]}] 123 | 124 | 125 | if __name__ == '__main__': 126 | # pandocfilters.toJSONFilter is a convenience method that 127 | # makes a command line json filter from a given function. 128 | # JSON emitted from pandoc is read from stdin. The JSON tree is 129 | # walked, with the function being applied to each element in the 130 | # tree. 131 | # 132 | # The function passed to to JSONFilter must accept (key, value, 133 | # format, metadata) as arguments: 134 | # 135 | # key - element type (e.g. 'Str', 'Header') 136 | # value - element contents 137 | # format - destination format 138 | # metadata - document metadata 139 | # 140 | # The function return values determine what happens to the 141 | # element: 142 | # returns None: the element is unmodified; 143 | # returns []: delete the element 144 | # otherwise: replace the element with the return value 145 | # 146 | # The JSON is then output to stdout, where it can be consumed by 147 | # pandoc. 148 | pf.toJSONFilter(blockquote2div) 149 | -------------------------------------------------------------------------------- /assets/img/swc-icon-blue.svg: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | 61 | 62 | 63 | 64 | 65 | 66 | 67 | 68 | 69 | 70 | 71 | -------------------------------------------------------------------------------- /assets/img/dc-icon-black.svg: -------------------------------------------------------------------------------- 1 | 2 | 16 | 18 | 19 | 21 | image/svg+xml 22 | 24 | 25 | 26 | 27 | 28 | 30 | 54 | 59 | 64 | 69 | 74 | 75 | -------------------------------------------------------------------------------- /_episodes/02-markdown.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "Basic Markdown" 3 | teaching: 10 4 | exercises: 10 5 | questions: 6 | - "FIXME?" 7 | objectives: 8 | - "FIXME." 9 | keypoints: 10 | - "FIXME." 11 | --- 12 | 13 | Markdown is a *markup* format. As opposed to software like *Word*, *OpenOffice*, 14 | or *Papers*, what you see on the screen is the raw text, and some informations 15 | about formatting. The formatting itself is done later, by an additional software 16 | (see **part 4**). 17 | 18 | This part of the lesson will give an overview of the core markdown syntax. 19 | Before we begin, it is worth keeping in mind that there are a few different 20 | markdowns around. [CommonMark][cm] is built to be a standard, and [GFM][gfm] 21 | (developped by *GitHub*) is so widely used that it could well be a standard 22 | already. This lesson will present the syntax that is *common* to both (a very 23 | large part of it is). Another interesting, yet immature, project is [Scholarly 24 | Markdown][scm], which is intended for academic authoring. 25 | 26 | ## Metadata 27 | 28 | Markdown allows authors to indicate metadata in the document, in the form a 29 | `YAML` header. `YAML` stands from Yet Another Markup Language, but this is 30 | hardly important. A `YAML` header could look like 31 | 32 | ~~~ 33 | --- 34 | title: "Modern scientific authoring using Markdown and `pandoc`" 35 | shorttitle: Modern scientific authoring 36 | author: Timothée Poisot 37 | date: May 9, 2015 38 | --- 39 | ~~~ 40 | {: .source} 41 | 42 | These elements will be used by the *template*. 43 | 44 | ## Basic syntax 45 | 46 | ### Headers 47 | 48 | Levels of sub-division in your text can be indicated by writing a single line, 49 | with between one and six octothorpes. For example, the following document will 50 | have two first-level headers (`Introduction` and `Methods`), and a second-level 51 | header nested under `Methods`: `Model of population dynamics`. 52 | 53 | ~~~ 54 | # Introduction 55 | # Methods 56 | ## Model of population dynamics 57 | ~~~ 58 | {: .source} 59 | 60 | ### Text style 61 | 62 | Markdown easily allows to specify *italics*, **bold**, and ***bold italics*** 63 | (although not all "flavors" of markdown agree on the last point). These styles 64 | can be applied using either `*` or `_`, so that the following commands are all 65 | equivalent: 66 | 67 | ~~~ 68 | *italics* and _italics_ 69 | **bold** and __bold__ 70 | ***bold it.*** and ___bold it.___ 71 | ~~~ 72 | {: .source} 73 | 74 | ### Code 75 | 76 | Code can be written either *inline*, by wrapping the text in backticks, 77 | 78 | ~~~ 79 | The program can be compiled using `make`. 80 | ~~~ 81 | {: .source} 82 | 83 | or with codeblocks by using a line with three backticks or three tildes (`~`) 84 | to delimitate the code block: 85 | 86 | ~~~ 87 | ``` 88 | this is 89 | a 90 | code block 91 | ``` 92 | ~~~ 93 | {: .source} 94 | 95 | On the first line of the codeblock, it is possible to specify the *language*: 96 | 97 | ~~~ 98 | We can do `python`: 99 | 100 | ``` python 101 | for i in xrange(5): 102 | print "This is line " + str(i) + " of this useless loop.\n" 103 | ``` 104 | 105 | Looking good! 106 | ~~~ 107 | {: .source} 108 | 109 | Code can also be written by using tabulations: 110 | 111 | ~~~ 112 | This is text 113 | 114 | and this 115 | is code 116 | 117 | ~~~ 118 | {: .source} 119 | 120 | ### Links 121 | 122 | There are two ways to write hyperlinks. The first is to write them *inline*, 123 | using the `[text](http://link.tld)` syntax. The second is to use named markers, 124 | for example: 125 | 126 | ~~~ 127 | This is [a link], and this is another [link][link2]. 128 | 129 | [a link]: http://link.1 130 | [link2]: http://link.2 131 | ~~~ 132 | {: .source} 133 | 134 | Note that this syntax is `[text][marker]`, followed later in the document by 135 | `[marker]: http://link`. This being said, if there is no `[marker]`, then 136 | `[text]: link` *will* work. 137 | 138 | ## Compilation 139 | 140 | So far, our manuscript is a raw markdown file (extension `.md`, `.mkd`, 141 | `.markdown`, or `.pandoc`, because who needs standards anyways?). We need to 142 | convert it into something else, usually a PDF, or a document that can be viewed 143 | in a text processor. 144 | 145 | ### Compiling with `pandoc` 146 | 147 | The `pandoc` program is the tool of choice to do this (although there are 148 | web-specific tools, like `jekyll`). Like most command-line tools, `pandoc` takes 149 | a series of files as inputs, and some (optional) *flags*. The basic way to 150 | invoke `pandoc` is: 151 | 152 | ~~~ 153 | pandoc input.ext -o output.ext 154 | ~~~ 155 | {: .source} 156 | 157 | #### Basic Syntax 158 | 159 | The *magic* behind `pandoc` is that the input file can be (approximately), and 160 | so can the output file. In our case, the input file will be markdown. To create 161 | a PDF, the command is: 162 | 163 | ~~~ 164 | pandoc manuscript.md -o manuscript.pdf 165 | ~~~ 166 | {: .source} 167 | 168 | and for a Word document, 169 | 170 | ~~~ 171 | pandoc manuscript.md -o manuscript.doc 172 | ~~~ 173 | {: .source} 174 | 175 | Note that `docx`, and `otf`, would have given a new Word document, and a 176 | LibreOffice text, respecitvely. Try with `txt`, `rtf`, and `html` to see what 177 | happens. 178 | 179 | ### Templates 180 | 181 | How does `pandoc` knows where to put things in the final document, you ask? 182 | There are a variety of *templates*, which are files in which `pandoc` will look 183 | to see where every element should go. You can have a look at them on `pandoc`'s 184 | website, which is a great way to copy and modify them. There are also a number 185 | of re-usable templates one quick google away. 186 | 187 | ### Flags 188 | 189 | Flags are a way to pass additional arguments to `pandoc`. There are a large 190 | number of them (see `man pandoc` in your shell session, or the really good 191 | online documentation). We will focus on the two that are related to the 192 | bibliography. 193 | 194 | [cm]: http://commonmark.org/ 195 | [gfm]: https://help.github.com/articles/github-flavored-markdown/ 196 | [scm]: http://scholarlymarkdown.com/ 197 | -------------------------------------------------------------------------------- /bin/util.py: -------------------------------------------------------------------------------- 1 | from __future__ import print_function 2 | import sys 3 | import os 4 | import json 5 | from subprocess import Popen, PIPE 6 | 7 | # Import this way to produce a more useful error message. 8 | try: 9 | import yaml 10 | except ImportError: 11 | print('Unable to import YAML module: please install PyYAML', file=sys.stderr) 12 | sys.exit(1) 13 | 14 | 15 | # Things an image file's name can end with. 16 | IMAGE_FILE_SUFFIX = { 17 | '.gif', 18 | '.jpg', 19 | '.png', 20 | '.svg' 21 | } 22 | 23 | # Files that shouldn't be present. 24 | UNWANTED_FILES = [ 25 | '.nojekyll' 26 | ] 27 | 28 | # Marker to show that an expected value hasn't been provided. 29 | # (Can't use 'None' because that might be a legitimate value.) 30 | REPORTER_NOT_SET = [] 31 | 32 | class Reporter(object): 33 | """Collect and report errors.""" 34 | 35 | def __init__(self): 36 | """Constructor.""" 37 | 38 | super(Reporter, self).__init__() 39 | self.messages = [] 40 | 41 | 42 | def check_field(self, filename, name, values, key, expected=REPORTER_NOT_SET): 43 | """Check that a dictionary has an expected value.""" 44 | 45 | if key not in values: 46 | self.add(filename, '{0} does not contain {1}', name, key) 47 | elif expected is REPORTER_NOT_SET: 48 | pass 49 | elif type(expected) in (tuple, set, list): 50 | if values[key] not in expected: 51 | self.add(filename, '{0} {1} value {2} is not in {3}', name, key, values[key], expected) 52 | elif values[key] != expected: 53 | self.add(filename, '{0} {1} is {2} not {3}', name, key, values[key], expected) 54 | 55 | 56 | def check(self, condition, location, fmt, *args): 57 | """Append error if condition not met.""" 58 | 59 | if not condition: 60 | self.add(location, fmt, *args) 61 | 62 | 63 | def add(self, location, fmt, *args): 64 | """Append error unilaterally.""" 65 | 66 | self.messages.append((location, fmt.format(*args))) 67 | 68 | 69 | def report(self, stream=sys.stdout): 70 | """Report all messages in order.""" 71 | 72 | if not self.messages: 73 | return 74 | 75 | def pretty(item): 76 | location, message = item 77 | if isinstance(location, type(None)): 78 | return message 79 | elif isinstance(location, str): 80 | return location + ': ' + message 81 | elif isinstance(location, tuple): 82 | return '{0}:{1}: '.format(*location) + message 83 | else: 84 | assert False, 'Unknown item "{0}"'.format(item) 85 | 86 | def key(item): 87 | location, message = item 88 | if isinstance(location, type(None)): 89 | return ('', -1, message) 90 | elif isinstance(location, str): 91 | return (location, -1, message) 92 | elif isinstance(location, tuple): 93 | return (location[0], location[1], message) 94 | else: 95 | assert False, 'Unknown item "{0}"'.format(item) 96 | 97 | for m in sorted(self.messages, key=key): 98 | print(pretty(m), file=stream) 99 | 100 | 101 | def read_markdown(parser, path): 102 | """ 103 | Get YAML and AST for Markdown file, returning 104 | {'metadata':yaml, 'metadata_len':N, 'text':text, 'lines':[(i, line, len)], 'doc':doc}. 105 | """ 106 | 107 | # Split and extract YAML (if present). 108 | with open(path, 'r') as reader: 109 | body = reader.read() 110 | metadata_raw, metadata_yaml, body = split_metadata(path, body) 111 | 112 | # Split into lines. 113 | metadata_len = 0 if metadata_raw is None else metadata_raw.count('\n') 114 | lines = [(metadata_len+i+1, line, len(line)) for (i, line) in enumerate(body.split('\n'))] 115 | 116 | # Parse Markdown. 117 | cmd = 'ruby {0}'.format(parser) 118 | p = Popen(cmd, shell=True, stdin=PIPE, stdout=PIPE, close_fds=True, universal_newlines=True) 119 | stdout_data, stderr_data = p.communicate(body) 120 | doc = json.loads(stdout_data) 121 | 122 | return { 123 | 'metadata': metadata_yaml, 124 | 'metadata_len': metadata_len, 125 | 'text': body, 126 | 'lines': lines, 127 | 'doc': doc 128 | } 129 | 130 | 131 | def split_metadata(path, text): 132 | """ 133 | Get raw (text) metadata, metadata as YAML, and rest of body. 134 | If no metadata, return (None, None, body). 135 | """ 136 | 137 | metadata_raw = None 138 | metadata_yaml = None 139 | metadata_len = None 140 | 141 | pieces = text.split('---', 2) 142 | if len(pieces) == 3: 143 | metadata_raw = pieces[1] 144 | text = pieces[2] 145 | try: 146 | metadata_yaml = yaml.load(metadata_raw) 147 | except yaml.YAMLError as e: 148 | print('Unable to parse YAML header in {0}:\n{1}'.format(path, e), file=sys.stderr) 149 | sys.exit(1) 150 | 151 | return metadata_raw, metadata_yaml, text 152 | 153 | 154 | def load_yaml(filename): 155 | """ 156 | Wrapper around YAML loading so that 'import yaml' is only needed 157 | in one file. 158 | """ 159 | 160 | try: 161 | with open(filename, 'r') as reader: 162 | return yaml.load(reader) 163 | except (yaml.YAMLError, FileNotFoundError) as e: 164 | print('Unable to load YAML file {0}:\n{1}'.format(filename, e), file=sys.stderr) 165 | sys.exit(1) 166 | 167 | 168 | def check_unwanted_files(dir_path, reporter): 169 | """ 170 | Check that unwanted files are not present. 171 | """ 172 | 173 | for filename in UNWANTED_FILES: 174 | path = os.path.join(dir_path, filename) 175 | reporter.check(not os.path.exists(path), 176 | path, 177 | "Unwanted file found") 178 | 179 | 180 | def require(condition, message): 181 | """Fail if condition not met.""" 182 | 183 | if not condition: 184 | print(message, file=sys.stderr) 185 | sys.exit(1) 186 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | [Software Carpentry][swc-site] and [Data Carpentry][dc-site] are open source projects, 4 | and we welcome contributions of all kinds: 5 | new lessons, 6 | fixes to existing material, 7 | bug reports, 8 | and reviews of proposed changes are all welcome. 9 | 10 | ## Contributor Agreement 11 | 12 | By contributing, 13 | you agree that we may redistribute your work under [our license](LICENSE.md). 14 | In exchange, 15 | we will address your issues and/or assess your change proposal as promptly as we can, 16 | and help you become a member of our community. 17 | Everyone involved in [Software Carpentry][swc-site] and [Data Carpentry][dc-site] 18 | agrees to abide by our [code of conduct](CONDUCT.md). 19 | 20 | ## How to Contribute 21 | 22 | The easiest way to get started is to file an issue 23 | to tell us about a spelling mistake, 24 | some awkward wording, 25 | or a factual error. 26 | This is a good way to introduce yourself 27 | and to meet some of our community members. 28 | 29 | 1. If you do not have a [GitHub][github] account, 30 | you can [send us comments by email][contact]. 31 | However, 32 | we will be able to respond more quickly if you use one of the other methods described below. 33 | 34 | 2. If you have a [GitHub][github] account, 35 | or are willing to [create one][github-join], 36 | but do not know how to use Git, 37 | you can report problems or suggest improvements by [creating an issue][issues]. 38 | This allows us to assign the item to someone 39 | and to respond to it in a threaded discussion. 40 | 41 | 3. If you are comfortable with Git, 42 | and would like to add or change material, 43 | you can submit a pull request (PR). 44 | Instructions for doing this are [included below](#using-github). 45 | 46 | ## Where to Contribute 47 | 48 | 1. If you wish to change this lesson, 49 | please work in , 50 | which can be viewed at . 51 | 52 | 2. If you wish to change the example lesson, 53 | please work in , 54 | which documents the format of our lessons 55 | and can be viewed at . 56 | 57 | 3. If you wish to change the template used for workshop websites, 58 | please work in . 59 | The home page of that repository explains how to set up workshop websites, 60 | while the extra pages in 61 | provide more background on our design choices. 62 | 63 | 4. If you wish to change CSS style files, tools, 64 | or HTML boilerplate for lessons or workshops stored in `_includes` or `_layouts`, 65 | please work in . 66 | 67 | ## What to Contribute 68 | 69 | There are many ways to contribute, 70 | from writing new exercises and improving existing ones 71 | to updating or filling in the documentation 72 | and and submitting [bug reports][issues] 73 | about things that don't work, aren't clear, or are missing. 74 | If you are looking for ideas, 75 | please see [the list of issues for this repository][issues], 76 | or the issues for [Data Carpentry][dc-issues] 77 | and [Software Carpentry][swc-issues] projects. 78 | 79 | Comments on issues and reviews of pull requests are just as welcome: 80 | we are smarter together than we are on our own. 81 | Reviews from novices and newcomers are particularly valuable: 82 | it's easy for people who have been using these lessons for a while 83 | to forget how impenetrable some of this material can be, 84 | so fresh eyes are always welcome. 85 | 86 | ## What *Not* to Contribute 87 | 88 | Our lessons already contain more material than we can cover in a typical workshop, 89 | so we are usually *not* looking for more concepts or tools to add to them. 90 | As a rule, 91 | if you want to introduce a new idea, 92 | you must (a) estimate how long it will take to teach 93 | and (b) explain what you would take out to make room for it. 94 | The first encourages contributors to be honest about requirements; 95 | the second, to think hard about priorities. 96 | 97 | We are also not looking for exercises or other material that only run on one platform. 98 | Our workshops typically contain a mixture of Windows, Mac OS X, and Linux users; 99 | in order to be usable, 100 | our lessons must run equally well on all three. 101 | 102 | ## Using GitHub 103 | 104 | If you choose to contribute via GitHub, 105 | you may want to look at 106 | [How to Contribute to an Open Source Project on GitHub][how-contribute]. 107 | In brief: 108 | 109 | 1. The published copy of the lesson is in the `gh-pages` branch of the repository 110 | (so that GitHub will regenerate it automatically). 111 | Please create all branches from that, 112 | and merge the [master repository][repo]'s `gh-pages` branch into your `gh-pages` branch 113 | before starting work. 114 | Please do *not* work directly in your `gh-pages` branch, 115 | since that will make it difficult for you to work on other contributions. 116 | 117 | 2. We use [GitHub flow][github-flow] to manage changes: 118 | 1. Create a new branch in your desktop copy of this repository for each significant change. 119 | 2. Commit the change in that branch. 120 | 3. Push that branch to your fork of this repository on GitHub. 121 | 4. Submit a pull request from that branch to the [master repository][repo]. 122 | 5. If you receive feedback, 123 | make changes on your desktop and push to your branch on GitHub: 124 | the pull request will update automatically. 125 | 126 | Each lesson has two maintainers who review issues and pull requests 127 | or encourage others to do so. 128 | The maintainers are community volunteers, 129 | and have final say over what gets merged into the lesson. 130 | 131 | ## Other Resources 132 | 133 | General discussion of [Software Carpentry][swc-site] and [Data Carpentry][dc-site] 134 | happens on the [discussion mailing list][discuss-list], 135 | which everyone is welcome to join. 136 | You can also [reach us by email][contact]. 137 | 138 | [contact]: mailto:admin@software-carpentry.org 139 | [dc-issues]: https://github.com/issues?q=user%3Adatacarpentry 140 | [dc-lessons]: http://datacarpentry.org/lessons/ 141 | [dc-site]: http://datacarpentry.org/ 142 | [discuss-list]: http://lists.software-carpentry.org/listinfo/discuss 143 | [example-site]: https://swcarpentry.github.io/lesson-example/ 144 | [github]: http://github.com 145 | [github-flow]: https://guides.github.com/introduction/flow/ 146 | [github-join]: https://github.com/join 147 | [how-contribute]: https://egghead.io/series/how-to-contribute-to-an-open-source-project-on-github 148 | [issues]: https://github.com/swcarpentry/modern-scientific-authoring/issues/ 149 | [repo]: https://github.com/swcarpentry/modern-scientific-authoring/ 150 | [swc-issues]: https://github.com/issues?q=user%3Aswcarpentry 151 | [swc-lessons]: http://software-carpentry.org/lessons/ 152 | [swc-site]: http://software-carpentry.org/ 153 | -------------------------------------------------------------------------------- /_extras/design.md: -------------------------------------------------------------------------------- 1 | --- 2 | layout: page 3 | title: "Lesson Design" 4 | permalink: /design/ 5 | --- 6 | 7 | ## Process Used 8 | 9 | This lesson was developed using a slimmed-down variant of the "Understanding by Design" process. 10 | The main sections are: 11 | 12 | 1. Assumptions about audience, time, etc. 13 | (The current draft also includes some conclusions and decisions in this 14 | section - that should be refactored.) 15 | 16 | 2. Desired results: 17 | overall goals, summative assessments at half-day granularity, what learners 18 | will be able to do, what learners will know. 19 | 20 | 3. Learning plan: 21 | each episode has a heading that summarizes what will be covered, 22 | then estimates time that will be spent on teaching and on exercises, 23 | while the exercises are given as bullet points. 24 | 25 | ## Stage 1: Assumptions 26 | 27 | * Audience 28 | * Graduate students or equivalent in all fields from astronomy to the digital humanities 29 | * Who are writing, publishing, and managing papers, theses, grant proposals, lessons, etc. 30 | * Prerequisites 31 | * Git and GitHub (for publishing on GitHub Pages) 32 | * Unix shell (for Pandoc and LaTeX) 33 | * Requirements 34 | * Half day: 3.0 hours of instruction + exercises 35 | * Learners use their own machines 36 | 37 | As stated in "[Good Enough Practices in Scientific Computing][good-enough]": 38 | 39 | > 1. Make text accessible to yourself and others now and in the future 40 | > 2. Reduce chances of work being lost or people overwriting each other's work. 41 | > 3. Make it easy to track and combine contributions from multiple collaborators. 42 | > 4. Avoid duplication and manual entry of information, 43 | > particularly in constructing bibliographies, tables of contents, and lists 44 | > 5. Make it easy to regenerate the final shared form (e.g., the PDF), 45 | > and to tell if the PDF in hand is up to date. 46 | > 6. Make it easy to share the final version with collaborators and submit it to journals 47 | > 48 | > We recommend against traditional desktop tools like LibreOffice and Microsoft Word 49 | > because they make collaboration difficult: 50 | > six people mailing each other copies of a paper with "Track Changes" enabled 51 | > is a sure way to lose work (and time). 52 | > The nerd alternative is: 53 | > 54 | > 1. Manuscripts are written in a plain text format such as LaTeX or Markdown that plays nicely with version control 55 | > 2. Tools needed to compile manuscripts are managed just like tools used to do simulation or analysis 56 | > 57 | > But as Stephen Turner said, 58 | > "...try to explain the notion of compiling a document to an overworked physician you collaborate with. 59 | > Oh, but before that, you have to explain the difference between plain text and word processing. 60 | > And text editors. 61 | > And Markdown/LaTeX compilers. 62 | > And BiBTeX. 63 | > And Git. 64 | > And GitHub. Etc. 65 | > Meanwhile he/she is getting paged from the OR..." 66 | {: .quotation} 67 | 68 | We therefore *also* support an alternative to text + version control: 69 | 70 | 1. Manuscripts are written using Google Docs or some other online tools with rich formatting and change tracking 71 | 2. A short text file is added to the `doc` directory with metadata about each online manuscript 72 | * Just as the `data` directory might contain links rather than actual data 73 | 3. The manuscript is downloaded and saved in `doc` in an editable form (e.g., `.docx` or `.odt`) after major changes 74 | 75 | The justification is: 76 | 77 | * If the document lives online (Google Docs or Authorea), 78 | then everyone's changes are in one place, 79 | and hence don't need to be merged manually. 80 | * If the document lives in a version control system (GitHub), 81 | it provides good support for finding and merging differences resulting from concurrent changes. 82 | 83 | ## Stage 2: Desired Results 84 | 85 | ### Questions 86 | 87 | * How do I make my work more accessible to others in my field? 88 | * How do I collaborate with colleagues while writing? 89 | * How do publishers and researchers keep track of who published what? 90 | * How can open access publication and open review be implemented? 91 | * What are their pros and cons? 92 | 93 | ### Skills 94 | 95 | I can... 96 | 97 | * ...write and review research-related documents in Google Docs 98 | * ...write research-related documents in Markdown 99 | * ...review documents on GitHub 100 | * ...turn Markdown into PDF or HTML using Pandoc 101 | * ...publish Markdown using GitHub Pages and Jekyll 102 | * ...comment on a web page using hypothes.is 103 | * ...add my ORCID to electronic manuscripts 104 | * ...get a DOI for a publication from Zenodo 105 | 106 | ### Knowledge 107 | 108 | I know... 109 | 110 | * ...the difference between WYSIWYG formats and markup 111 | * ...the pros and cons of each 112 | * ...how LaTeX, Pandoc, and Authorea work 113 | * ...how and why to use [hypothes.is][hypothesis] 114 | * ...what an ORCID is (and how to get and use one) 115 | * ...why data and programs should have DOIs (and how to get and use them) 116 | * ...the pros and cons of open access publication and open review 117 | 118 | ## Stage 3: Assessment 119 | 120 | ### Summative Assessment 121 | 122 | 1. Convert a Markdown document to a Google Doc to share with collaborators. 123 | 2. Publish the same document as a single-page website with GitHub Pages. 124 | 125 | ### Introduction (9:00) 126 | 127 | * Teaching: 15 min 128 | * Evolution of WYSIWYG and markup 129 | * Pros and cons of each: human-readable vs. machine-readable 130 | * Challenges: 10 min 131 | * Try to interpret the source of an RTF document 132 | 133 | ### Basic Markdown (09:25) 134 | 135 | * Teaching: 10 min 136 | * Basic markup 137 | * Note: use in-browser editor, not compilation 138 | * Challenges: 5 min 139 | * Replicate a simple web page 140 | 141 | ### Translating Markdown (09:40) 142 | 143 | * Teaching: 15 min (includes setup issues) 144 | * Using Pandoc on the command line to create HTML and PDF 145 | * Challenges: 10 min 146 | * Translate simple page into other formats (including Word) 147 | 148 | ### GitHub Pages (10:05) 149 | 150 | * Teaching: 20 min 151 | * The `gh-pages` branch 152 | * Layouts 153 | * `_config.yml` 154 | * Empty YAML headers 155 | * Where to view generated site 156 | * Challenges: 10 min 157 | * Reproduce what the instructor has been doing 158 | 159 | ### Coffee (10:35): 10 min 160 | 161 | ### Reviewing Material on GitHub (10:45) 162 | 163 | * Teaching: 10 min 164 | * Line comments 165 | * Issues 166 | * Do *not* introduce pull requests 167 | * Challenges: 10 min 168 | * Leave comments on instructor's pages 169 | 170 | ### Open Publication and Review (11:05) 171 | 172 | * Teaching: 5 min 173 | * Ask leading questions 174 | * Challenges: 15 min 175 | * Guided discussion of pros and cons 176 | 177 | ### Tracking Work (11:25) 178 | 179 | * Teaching: 15 min 180 | * DOIs 181 | * ORCIDs 182 | * Challenges: 10 min 183 | * Get a DOI from Zenodo 184 | * Get an ORCID and add it to a page 185 | 186 | ### Working with Others (11:55) 187 | 188 | * Teaching: 10 min 189 | * Markdown to Word via Pandoc 190 | * Uploading to Google Docs 191 | * Commenting on Google Docs 192 | * Challenges: 5 min 193 | * Replicate instructor's work 194 | 195 | ### Wrap-Up (12:10) 196 | 197 | * Teaching: 10 min 198 | * Quick summary with pointers 199 | * Challenges: 10 min 200 | * Give feedback 201 | 202 | ### Finish (12:30) 203 | 204 | [good-enough]: https://github.com/swcarpentry/good-enough-practices-in-scientific-computing 205 | [hypothesis]: hypothes.is 206 | -------------------------------------------------------------------------------- /bin/validation_helpers.py: -------------------------------------------------------------------------------- 1 | #! /usr/bin/env python 2 | 3 | import json 4 | import logging 5 | import re 6 | import sys 7 | 8 | try: # Hack to make codebase compatible with python 2 and 3 9 | basestring 10 | except NameError: 11 | basestring = str 12 | 13 | 14 | # Common validation functions 15 | def is_list(text): 16 | """Validate whether the provided string can be converted to python list""" 17 | text = text.strip() 18 | try: 19 | text_as_list = json.loads(text) 20 | except ValueError: 21 | logging.debug("Could not convert string to python object: {0}".format(text)) 22 | return False 23 | 24 | return isinstance(text_as_list, list) 25 | 26 | 27 | def is_str(text): 28 | """Validate whether the input is a non-blank python string""" 29 | return isinstance(text, basestring) and len(text) > 0 30 | 31 | 32 | def is_numeric(text): 33 | """Validate whether the string represents a number (including unicode)""" 34 | try: 35 | float(text) 36 | return True 37 | except ValueError: 38 | return False 39 | 40 | 41 | #### Text cleanup functions, pre-validation 42 | def strip_attrs(s): 43 | """Strip attributes of the form {.name} from a markdown title string""" 44 | return re.sub(r"\s\{\..*?\}", "", s) 45 | 46 | 47 | def get_css_class(s): 48 | """Return any and all CSS classes (when a line is suffixed by {.classname}) 49 | Returns empty list when """ 50 | return re.findall("\{\.(.*?)\}", s) 51 | 52 | 53 | ### Helper objects 54 | class CommonMarkHelper(object): 55 | """Basic helper functions for working with the internal abstract syntax 56 | tree produced by CommonMark parser""" 57 | def __init__(self, ast): 58 | self.data = ast 59 | self.children = self.data.children 60 | 61 | def get_doc_header_title(self): 62 | """Helper method for SWC templates: get the document title from 63 | the YAML headers""" 64 | doc_headers = self.data.children[1] # Throw index error if none found 65 | 66 | for s in doc_headers.strings: 67 | label, contents = s.split(":", 1) 68 | if label.lower() == "title": 69 | return contents.strip() 70 | 71 | # If title not found, return an empty string for display purposes 72 | return '' 73 | 74 | def get_doc_header_subtitle(self): 75 | """Helper method for SWC templates: get the document title from 76 | the YAML headers""" 77 | doc_headers = self.data.children[1] # Throw index error if none found 78 | 79 | for s in doc_headers.strings: 80 | label, contents = s.split(":", 1) 81 | if label.lower() == "subtitle": 82 | return contents.strip() 83 | 84 | # If title not found, return an empty string for display purposes 85 | return '' 86 | 87 | def get_block_titled(self, title, heading_level=2, ast_node=None): 88 | """Examine children. Return all children of the given node that: 89 | a) are blockquoted elements, and 90 | b) contain a heading with the specified text, at the specified level. 91 | For example, this can be used to find the "Prerequisites" section 92 | in index.md 93 | 94 | Returns empty list if no appropriate node is found""" 95 | 96 | # TODO: Deprecate in favor of callout validator 97 | if ast_node is None: 98 | ast_node = self.data 99 | return [n for n in ast_node.children 100 | if self.is_block(n) and 101 | self.has_section_heading( 102 | title, 103 | ast_node=n, 104 | heading_level=heading_level, 105 | show_msg=False)] 106 | 107 | # Helpers to fetch specific document sections 108 | def get_section_headings(self, ast_node=None): 109 | """Returns a list of ast nodes that are headings""" 110 | if ast_node is None: 111 | ast_node = self.data 112 | return [n for n in ast_node.children if self.is_heading(n)] 113 | 114 | def get_callouts(self, ast_node=None): 115 | if ast_node is None: 116 | ast_node = self.data 117 | return [n for n in ast_node.children if self.is_callout(n)] 118 | 119 | def find_external_links(self, ast_node=None, parent_crit=None): 120 | """Recursive function that locates all references to external content 121 | under specified node. (links or images)""" 122 | ast_node = ast_node or self.data 123 | if parent_crit is None: 124 | # User can optionally provide a function to filter link list 125 | # based on where link appears. (eg, only links in headings) 126 | # If no filter is provided, accept all links in that node. 127 | parent_crit = lambda n: True 128 | 129 | # Link can be node itself, or hiding in inline content 130 | links = [n for n in ast_node.inline_content 131 | if self.is_external(n) and parent_crit(ast_node)] 132 | 133 | if self.is_external(ast_node): 134 | links.append(ast_node) 135 | 136 | # Also look for links in sub-nodes 137 | for n in ast_node.children: 138 | links.extend(self.find_external_links(n, 139 | parent_crit=parent_crit)) 140 | 141 | return links 142 | 143 | # Helpers to get information from a specific node type 144 | def get_link_info(self, link_node): 145 | """Given a link node, return the link title and destination""" 146 | if not self.is_external(link_node): 147 | raise TypeError("Cannot apply this method to something that is not a link") 148 | 149 | dest = link_node.destination 150 | try: 151 | link_text = link_node.label[0].c 152 | except: 153 | link_text = None 154 | 155 | return dest, link_text 156 | 157 | def get_heading_info(self, heading_node): 158 | """Get heading text and list of all css styles applied""" 159 | heading = heading_node.strings[0] 160 | text = strip_attrs(heading) 161 | css = get_css_class(heading) 162 | return text, css 163 | 164 | # Functions to query type or content of nodes 165 | def has_section_heading(self, section_title, ast_node=None, 166 | heading_level=2, limit=sys.maxsize, show_msg=True): 167 | """Does the section contain (<= x copies of) specified heading text? 168 | Will strip off any CSS attributes when looking for the section title""" 169 | if ast_node is None: 170 | ast_node = self.data 171 | 172 | num_nodes = len([n for n in self.get_section_headings(ast_node) 173 | if (strip_attrs(n.strings[0]) == section_title) 174 | and (n.level == heading_level)]) 175 | 176 | # Suppress error msg if used as a helper method 177 | if show_msg and num_nodes == 0: 178 | logging.error("Document does not contain the specified " 179 | "heading: {0}".format(section_title)) 180 | elif show_msg and num_nodes > limit: 181 | logging.error("Document must not contain more than {0} copies of" 182 | " the heading {1}".format(limit, section_title or 0)) 183 | elif show_msg: 184 | logging.info("Verified that document contains the specified" 185 | " heading: {0}".format(section_title)) 186 | return (0 < num_nodes <= limit) 187 | 188 | def has_number_children(self, ast_node, 189 | exact=None, minc=0, maxc=sys.maxsize): 190 | """Does the specified node (such as a bulleted list) have the expected 191 | number of children?""" 192 | 193 | if exact: # If specified, must have exactly this number of children 194 | minc = maxc = exact 195 | 196 | return (minc <= len(ast_node.children) <= maxc) 197 | 198 | # Helpers, in case the evolving CommonMark spec changes the names of nodes 199 | def is_hr(self, ast_node): 200 | """Is the node a horizontal rule (hr)?""" 201 | return ast_node.t == 'HorizontalRule' 202 | 203 | def is_heading(self, ast_node, heading_level=None): 204 | """Is the node a heading/ title?""" 205 | has_tag = ast_node.t == "ATXHeader" 206 | 207 | if heading_level is None: 208 | has_level = True 209 | else: 210 | has_level = (ast_node.level == heading_level) 211 | return has_tag and has_level 212 | 213 | def is_paragraph(self, ast_node): 214 | """Is the node a paragraph?""" 215 | return ast_node.t == "Paragraph" 216 | 217 | def is_list(self, ast_node): 218 | """Is the node a list? (ordered or unordered)""" 219 | return ast_node.t == "List" 220 | 221 | def is_link(self, ast_node): 222 | """Is the node a link?""" 223 | return ast_node.t == "Link" 224 | 225 | def is_external(self, ast_node): 226 | """Does the node reference content outside the file? (image or link)""" 227 | return ast_node.t in ("Link", "Image") 228 | 229 | def is_block(self, ast_node): 230 | """Is the node a BlockQuoted element?""" 231 | return ast_node.t == "BlockQuote" 232 | 233 | def is_callout(self, ast_node): 234 | """Composite element: "callout" elements in SWC templates are 235 | blockquotes whose first child element is a heading""" 236 | if len(ast_node.children) > 0 and \ 237 | self.is_heading(ast_node.children[0]): 238 | has_heading = True 239 | else: 240 | has_heading = False 241 | 242 | return self.is_block(ast_node) and has_heading 243 | -------------------------------------------------------------------------------- /bin/lesson_initialize.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """Initialize a newly-created repository.""" 4 | 5 | 6 | from __future__ import print_function 7 | import sys 8 | import os 9 | 10 | ROOT_AUTHORS = '''\ 11 | FIXME: list authors' names and email addresses. 12 | ''' 13 | 14 | ROOT_CITATION = '''\ 15 | FIXME: describe how to cite this lesson. 16 | ''' 17 | 18 | ROOT_CONTRIBUTING_MD = '''\ 19 | # Contributing 20 | 21 | [Software Carpentry][swc-site] and [Data Carpentry][dc-site] are open source projects, 22 | and we welcome contributions of all kinds: 23 | new lessons, 24 | fixes to existing material, 25 | bug reports, 26 | and reviews of proposed changes are all welcome. 27 | 28 | ## Contributor Agreement 29 | 30 | By contributing, 31 | you agree that we may redistribute your work under [our license](LICENSE.md). 32 | In exchange, 33 | we will address your issues and/or assess your change proposal as promptly as we can, 34 | and help you become a member of our community. 35 | Everyone involved in [Software Carpentry][swc-site] and [Data Carpentry][dc-site] 36 | agrees to abide by our [code of conduct](CONDUCT.md). 37 | 38 | ## How to Contribute 39 | 40 | The easiest way to get started is to file an issue 41 | to tell us about a spelling mistake, 42 | some awkward wording, 43 | or a factual error. 44 | This is a good way to introduce yourself 45 | and to meet some of our community members. 46 | 47 | 1. If you do not have a [GitHub][github] account, 48 | you can [send us comments by email][contact]. 49 | However, 50 | we will be able to respond more quickly if you use one of the other methods described below. 51 | 52 | 2. If you have a [GitHub][github] account, 53 | or are willing to [create one][github-join], 54 | but do not know how to use Git, 55 | you can report problems or suggest improvements by [creating an issue][issues]. 56 | This allows us to assign the item to someone 57 | and to respond to it in a threaded discussion. 58 | 59 | 3. If you are comfortable with Git, 60 | and would like to add or change material, 61 | you can submit a pull request (PR). 62 | Instructions for doing this are [included below](#using-github). 63 | 64 | ## Where to Contribute 65 | 66 | 1. If you wish to change this lesson, 67 | please work in , 68 | which can be viewed at . 69 | 70 | 2. If you wish to change the example lesson, 71 | please work in , 72 | which documents the format of our lessons 73 | and can be viewed at . 74 | 75 | 3. If you wish to change the template used for workshop websites, 76 | please work in . 77 | The home page of that repository explains how to set up workshop websites, 78 | while the extra pages in 79 | provide more background on our design choices. 80 | 81 | 4. If you wish to change CSS style files, tools, 82 | or HTML boilerplate for lessons or workshops stored in `_includes` or `_layouts`, 83 | please work in . 84 | 85 | ## What to Contribute 86 | 87 | There are many ways to contribute, 88 | from writing new exercises and improving existing ones 89 | to updating or filling in the documentation 90 | and and submitting [bug reports][issues] 91 | about things that don't work, aren't clear, or are missing. 92 | If you are looking for ideas, 93 | please see [the list of issues for this repository][issues], 94 | or the issues for [Data Carpentry][dc-issues] 95 | and [Software Carpentry][swc-issues] projects. 96 | 97 | Comments on issues and reviews of pull requests are just as welcome: 98 | we are smarter together than we are on our own. 99 | Reviews from novices and newcomers are particularly valuable: 100 | it's easy for people who have been using these lessons for a while 101 | to forget how impenetrable some of this material can be, 102 | so fresh eyes are always welcome. 103 | 104 | ## What *Not* to Contribute 105 | 106 | Our lessons already contain more material than we can cover in a typical workshop, 107 | so we are usually *not* looking for more concepts or tools to add to them. 108 | As a rule, 109 | if you want to introduce a new idea, 110 | you must (a) estimate how long it will take to teach 111 | and (b) explain what you would take out to make room for it. 112 | The first encourages contributors to be honest about requirements; 113 | the second, to think hard about priorities. 114 | 115 | We are also not looking for exercises or other material that only run on one platform. 116 | Our workshops typically contain a mixture of Windows, Mac OS X, and Linux users; 117 | in order to be usable, 118 | our lessons must run equally well on all three. 119 | 120 | ## Using GitHub 121 | 122 | If you choose to contribute via GitHub, 123 | you may want to look at 124 | [How to Contribute to an Open Source Project on GitHub][how-contribute]. 125 | In brief: 126 | 127 | 1. The published copy of the lesson is in the `gh-pages` branch of the repository 128 | (so that GitHub will regenerate it automatically). 129 | Please create all branches from that, 130 | and merge the [master repository][repo]'s `gh-pages` branch into your `gh-pages` branch 131 | before starting work. 132 | Please do *not* work directly in your `gh-pages` branch, 133 | since that will make it difficult for you to work on other contributions. 134 | 135 | 2. We use [GitHub flow][github-flow] to manage changes: 136 | 1. Create a new branch in your desktop copy of this repository for each significant change. 137 | 2. Commit the change in that branch. 138 | 3. Push that branch to your fork of this repository on GitHub. 139 | 4. Submit a pull request from that branch to the [master repository][repo]. 140 | 5. If you receive feedback, 141 | make changes on your desktop and push to your branch on GitHub: 142 | the pull request will update automatically. 143 | 144 | Each lesson has two maintainers who review issues and pull requests 145 | or encourage others to do so. 146 | The maintainers are community volunteers, 147 | and have final say over what gets merged into the lesson. 148 | 149 | ## Other Resources 150 | 151 | General discussion of [Software Carpentry][swc-site] and [Data Carpentry][dc-site] 152 | happens on the [discussion mailing list][discuss-list], 153 | which everyone is welcome to join. 154 | You can also [reach us by email][contact]. 155 | 156 | [contact]: mailto:admin@software-carpentry.org 157 | [dc-issues]: https://github.com/issues?q=user%3Adatacarpentry 158 | [dc-lessons]: http://datacarpentry.org/lessons/ 159 | [dc-site]: http://datacarpentry.org/ 160 | [discuss-list]: http://lists.software-carpentry.org/listinfo/discuss 161 | [example-site]: https://swcarpentry.github.io/lesson-example/ 162 | [github]: http://github.com 163 | [github-flow]: https://guides.github.com/introduction/flow/ 164 | [github-join]: https://github.com/join 165 | [how-contribute]: https://egghead.io/series/how-to-contribute-to-an-open-source-project-on-github 166 | [issues]: https://github.com/swcarpentry/FIXME/issues/ 167 | [repo]: https://github.com/swcarpentry/FIXME/ 168 | [swc-issues]: https://github.com/issues?q=user%3Aswcarpentry 169 | [swc-lessons]: http://software-carpentry.org/lessons/ 170 | [swc-site]: http://software-carpentry.org/ 171 | ''' 172 | 173 | ROOT_CONFIG_YML = '''\ 174 | #------------------------------------------------------------ 175 | # Values for this lesson. 176 | #------------------------------------------------------------ 177 | 178 | # Which carpentry is this ("swc" or "dc")? 179 | carpentry: "swc" 180 | 181 | # Overall title for pages. 182 | title: "Lesson Title" 183 | 184 | # Contact email address. 185 | email: lessons@software-carpentry.org 186 | 187 | #------------------------------------------------------------ 188 | # Generic settings (should not need to change). 189 | #------------------------------------------------------------ 190 | 191 | # What kind of thing is this ("workshop" or "lesson")? 192 | kind: "lesson" 193 | 194 | # Magic to make URLs resolve both locally and on GitHub. 195 | # See https://help.github.com/articles/repository-metadata-on-github-pages/. 196 | repository: / 197 | 198 | # Sites. 199 | amy_site: "https://amy.software-carpentry.org/workshops" 200 | dc_site: "http://datacarpentry.org" 201 | swc_github: "https://github.com/swcarpentry" 202 | swc_site: "https://software-carpentry.org" 203 | swc_pages: "https://swcarpentry.github.io" 204 | template_repo: "https://github.com/swcarpentry/styles" 205 | example_repo: "https://github.com/swcarpentry/lesson-example" 206 | example_site: "https://swcarpentry.github.com/lesson-example" 207 | workshop_repo: "https://github.com/swcarpentry/workshop-template" 208 | workshop_site: "https://swcarpentry.github.io/workshop-template" 209 | training_site: "https://swcarpentry.github.io/instructor-training" 210 | 211 | # Surveys. 212 | pre_survey: "https://www.surveymonkey.com/r/swc_pre_workshop_v1?workshop_id=" 213 | post_survey: "https://www.surveymonkey.com/r/swc_post_workshop_v1?workshop_id=" 214 | 215 | # Start time in minutes (0 to be clock-independent, 540 to show a start at 09:00 am). 216 | start_time: 0 217 | 218 | # Specify that things in the episodes collection should be output. 219 | collections: 220 | episodes: 221 | output: true 222 | permalink: /:path/ 223 | extras: 224 | output: true 225 | 226 | # Set the default layout for things in the episodes collection. 227 | defaults: 228 | - scope: 229 | path: "" 230 | type: episodes 231 | values: 232 | layout: episode 233 | 234 | # Files and directories that are not to be copied. 235 | exclude: 236 | - Makefile 237 | - bin 238 | 239 | # Turn off built-in syntax highlighting. 240 | highlighter: false 241 | ''' 242 | 243 | ROOT_INDEX_MD = '''\ 244 | --- 245 | layout: lesson 246 | --- 247 | FIXME: home page introduction 248 | 249 | > ## Prerequisites 250 | > 251 | > FIXME 252 | {: .prereq} 253 | ''' 254 | 255 | ROOT_REFERENCE_MD = '''\ 256 | --- 257 | layout: reference 258 | permalink: /reference/ 259 | --- 260 | 261 | ## Glossary 262 | 263 | FIXME 264 | ''' 265 | 266 | ROOT_SETUP_MD = '''\ 267 | --- 268 | layout: page 269 | title: Setup 270 | permalink: /setup/ 271 | --- 272 | FIXME 273 | ''' 274 | 275 | EPISODES_INTRODUCTION_MD = '''\ 276 | --- 277 | title: "Introduction" 278 | teaching: 0 279 | exercises: 0 280 | questions: 281 | - "Key question" 282 | objectives: 283 | - "First objective." 284 | keypoints: 285 | - "First key point." 286 | --- 287 | ''' 288 | 289 | EXTRAS_ABOUT_MD = '''\ 290 | --- 291 | layout: page 292 | title: About 293 | permalink: /about/ 294 | --- 295 | {% include carpentries.html %} 296 | ''' 297 | 298 | EXTRAS_DISCUSS_MD = '''\ 299 | --- 300 | layout: page 301 | title: Discussion 302 | permalink: /discuss/ 303 | --- 304 | FIXME 305 | ''' 306 | 307 | EXTRAS_FIGURES_MD = '''\ 308 | --- 309 | layout: page 310 | title: Figures 311 | permalink: /figures/ 312 | --- 313 | {% include all_figures.html %} 314 | ''' 315 | 316 | EXTRAS_GUIDE_MD = '''\ 317 | --- 318 | layout: page 319 | title: "Instructors' Guide" 320 | permalink: /guide/ 321 | --- 322 | FIXME 323 | ''' 324 | 325 | INCLUDES_ALL_FIGURES_HTML = '''\ 326 | 327 | ''' 328 | 329 | BOILERPLATE = ( 330 | ('AUTHORS', ROOT_AUTHORS), 331 | ('CITATION', ROOT_CITATION), 332 | ('CONTRIBUTING.md', ROOT_CONTRIBUTING_MD), 333 | ('_config.yml', ROOT_CONFIG_YML), 334 | ('index.md', ROOT_INDEX_MD), 335 | ('reference.md', ROOT_REFERENCE_MD), 336 | ('setup.md', ROOT_SETUP_MD), 337 | ('_episodes/01-introduction.md', EPISODES_INTRODUCTION_MD), 338 | ('_extras/about.md', EXTRAS_ABOUT_MD), 339 | ('_extras/discuss.md', EXTRAS_DISCUSS_MD), 340 | ('_extras/figures.md', EXTRAS_FIGURES_MD), 341 | ('_extras/guide.md', EXTRAS_GUIDE_MD), 342 | ('_includes/all_figures.html', INCLUDES_ALL_FIGURES_HTML) 343 | ) 344 | 345 | 346 | def main(): 347 | """Check for collisions, then create.""" 348 | 349 | # Check. 350 | errors = False 351 | for (path, _) in BOILERPLATE: 352 | if os.path.exists(path): 353 | print('Warning: {0} already exists.'.format(path), file=sys.stderr) 354 | errors = True 355 | if errors: 356 | print('**Exiting without creating files.**', file=sys.stderr) 357 | sys.exit(1) 358 | 359 | # Create. 360 | for (path, content) in BOILERPLATE: 361 | with open(path, 'w') as writer: 362 | writer.write(content) 363 | 364 | 365 | if __name__ == '__main__': 366 | main() 367 | -------------------------------------------------------------------------------- /_episodes/01-mess.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: "The Mess We're In" 3 | teaching: 10 4 | exercises: 10 5 | questions: 6 | - "FIXME?" 7 | objectives: 8 | - "FIXME." 9 | keypoints: 10 | - "FIXME." 11 | --- 12 | 13 | In the beginning was the finger, and paint, and the wall of the cave. 14 | Whoever painted the first pictures this way could create whatever they wanted---the medium 15 | allowed them to create anything they could imagine. 16 | So did pens and paper: 17 | pictures and text could be arranged on the page 18 | however the author wanted. 19 | 20 | The first printing presses didn't change this. 21 | They made impressions from woodblock carvings, 22 | which still allowed authors to put whatever they wanted wherever they wanted it. 23 | But then, 24 | around 1370, 25 | craftsmen in Korea invented movable type. 26 | It spread like wildfire after Gutenberg introduced it to Europe in the 1440s, 27 | and humanity's long fall from written grace began. 28 | 29 | While movable type allowed printers to set pages 30 | many times more quickly than carvers could produce woodblocks, 31 | the cost was flexibility: 32 | where scribes could draw anywhere on the page, 33 | typesetters had to put letters of uniform size in rows. 34 | And while diagrams were still possible, 35 | the lowered cost of words made them relatively many times more expensive than they had been. 36 | 37 | The typewriter (invented in the 1860s) put "printing" in millions of middle-class hands. 38 | Mechanical, electrical, and then electronic computers all re-used typewriter technology 39 | to print their output. 40 | When the first pen plotters appeared in the 1950s, 41 | they were too slow and too expensive to displace line printers. 42 | What's more, 43 | the two technologies didn't work well together: 44 | it's possible to draw pictures using ASCII art, 45 | or to write letters with a pen plotter, 46 | but neither is particularly attractive. 47 | 48 | One sign of this gap between tools meant for words and tools meant for pictures 49 | was the development of separate languages for controlling them. 50 | Plotters were typically controlled by *drawing languages* that had commands to say, 51 | "Pen up, 52 | move to this (x,y) location, 53 | pen down, 54 | and move again." 55 | 56 | ~~~ 57 | PU; 58 | PA200,150; 59 | PD; 60 | PA250,250; 61 | ~~~ 62 | {: .source} 63 | 64 | *Typesetting languages* for line printers, 65 | on the other hand, 66 | let authors tell the computer to lay out a phrase as a second-level heading 67 | or set certain words in italics, 68 | but it was then the computer's job to determine 69 | where things would go and what they would look like: 70 | 71 | ~~~ 72 | .t2 Section Heading 73 | 74 | Empty lines separate 75 | .it paragraphs 76 | and lines starting with '.' are commands. 77 | ~~~ 78 | {: .source} 79 | 80 | A third kind of language emerged in this period as well, 81 | one meant to describe the *content* of a document rather than its *appearance*. 82 | Doctors and lawyers wanted to be able to search patient histories and precedents, 83 | but the computers of the time weren't powerful enough to handle natural language. 84 | Instead, 85 | companies like IBM developed *markup languages* 86 | so that people could make the meaning, or *semantics*, of their documents explicit: 87 | 88 | ~~~ 89 | Derstmann still questions the importance of methane release 90 | in the Fukuyama disaster. 91 | ~~~ 92 | {: .source} 93 | 94 | These worlds collided after the invention of the laser printer in the 1970s, 95 | and that tension was only magnified by high-resolution computer screens in the 1980s 96 | and the World Wide Web in the 1990s. 97 | On the one hand, 98 | most people simply want to write---to put these words here and those words there, 99 | or make some of them green and others italic. 100 | WYSIWIG (what you see is what you get) editors like MacWrite and Microsoft Word fill this need, 101 | but documents produced this way have two shortcomings: 102 | 103 | 1. They are *rigid*. 104 | If someone lays things out manually, 105 | then changes the size of the page, 106 | their hard work must be re-done. 107 | 108 | 2. They are *opaque*. 109 | Telling the computer to display something in italics 110 | doesn't tell it whether that phrase is a book title, 111 | in a foreign language, 112 | or defining a new term. 113 | 114 | Typesetting and markup languages address both problems. 115 | Instead of saying what things should look like and where they should go on the page, 116 | authors are supposed to tell the computer what kinds of things they are, 117 | e.g., a title or a new term. 118 | The computer is then supposed to decide what it should look like and where it should go. 119 | Separating semantics and appearance in this way also allows people to switch styles easily and consistently 120 | by telling the computer, 121 | "Typeset all second-level headings in 16-point Garamond, left-aligned." 122 | 123 | But this approach also has shortcomings: 124 | 125 | 1. Computers don't always lay out text the way human beings would 126 | because they don't understand it. 127 | People have therefore always insisted on being able to override the computer's choices, 128 | even though it re-introduces rigidity. 129 | 130 | 2. Specifying their documents' semantics seems alien to most people, 131 | and much more work than just enlarging the title a few times. 132 | 133 | 3. Interpreting what the user typed in and figuring out what to display takes the computer time. 134 | Figuring out why the document doesn't look like it ought to takes the person even more time; 135 | it's exactly like debugging a program, 136 | and debugging is frustrating. 137 | 138 | No-one has invented something that avoids all of these problems, 139 | so today's researchers have a confusing variety of choices when it comes to writing: 140 | 141 | 1. A desktop WYSIWYG tool like Microsoft Word or LibreOffice (both of which work with the `.docx` format). 142 | This is by far the easiest way to create simple things like letters, 143 | but it is rigid and opaque, 144 | has poor support for laying out equations, 145 | and doesn't work well with version control systems (something we discuss below). 146 | 147 | 2. A web-based WYSIWYG tool like Google Docs. 148 | This has the immediacy of Word or LibreOffice, 149 | and makes collaboration easier 150 | (since everyone shares one copy of the document). 151 | It is still rigid and opaque, 152 | though, 153 | and a growing number of people are uncomfortable with putting all their eggs 154 | in an unaccountable private company's basket. 155 | 156 | 3. [LaTeX][latex] on the desktop. 157 | This powerful typesetting language has excellent support for equations and bibliographies. 158 | It also works well with version control, 159 | since documents are written as plain text. 160 | However, 161 | it is by far the most complex to learn, 162 | and getting things laid out exactly as desired can take many painful hours. 163 | 164 | 4. Web-based tools like [Authorea][authorea] and [Overleaf][overleaf] 165 | that offer users a WYSIWYG editing interface 166 | but store documents as LaTeX 167 | and re-display them in real time as changes are typed in. 168 | 169 | 5. HTML. 170 | The native language of the web is much (much) simpler than LaTeX, 171 | but also does much less: 172 | even simple things like footnotes, bibliographic references, and numbered sections aren't directly supported. 173 | It can also be quite verbose, 174 | and CSS 175 | (the language used to tell browsers how to display HTML) 176 | is famously quirky. 177 | 178 | 6. [Markdown][markdown] was created as a simple alternative to HTML. 179 | It uses the conventions of plain-text email: 180 | blank lines separate paragraphs, 181 | putting something in `*asterisks*` makes it italic, 182 | and so on. 183 | It does less than HTML, 184 | but requires less typing. 185 | Unfortunately, 186 | though, 187 | almost every implementation adds its own features, 188 | so "standard Markdown" is an oxymoron. 189 | 190 | And if that wasn't confusing enough: 191 | HTML and Markdown do not support equations directly, 192 | but packages exist to allow authors to embed LaTeX-style equations in documents of either kind. 193 | The [Jupyter Notebook][jupyter] relies on one such package, 194 | which allows users to put equations and other things in Markdown cells 195 | to be rendered in the browser. 196 | 197 | One final consideration is that 198 | it is relatively straightforward to integrate desktop text-based systems like LaTeX 199 | and other tools that manage computation 200 | to support reproducible research. 201 | It's much more complicated, at least right now, 202 | to integrate a typical geophysics or bioinformatics pipeline 203 | with a Google Doc or LibreOffice 204 | so that figures are automatically updated when data changes. 205 | 206 | > ## More Heat than Light 207 | > 208 | > The division between WYSIWYG and typesetting/markup 209 | > has more to do with tools than with actual formats. 210 | > A `.docx` file actually contains a mix of typesetting commands and text, 211 | > just like a LaTeX, HTML, or Markdown file. 212 | > The difference is that the commands in the latter are stored as human-readable text, 213 | > which means that the standard Unix command-line utilities can process them 214 | > (though as [this comment on Stack Overflow][html-regexp] indicates, 215 | > there are limits to how much they can actually do). 216 | > In contrast, 217 | > the formatting instructions embedded in Microsoft Word and LibreOffice 218 | > are created by and for specific special-purpose programs, 219 | > so plain-text tools like `grep` can't handle them. 220 | > 221 | > The same is true of Google Docs: 222 | > formatting instructions are embedded in the document, 223 | > then executed by Javascript running in the user's browser 224 | > to create the rendered page that the user interacts with. 225 | > Authorea and Overleaf do the same thing, 226 | > except their storage format is LaTeX. 227 | > 228 | > Hard-core programmers may sneer at WYSIWYG tools and their non-textual formats, 229 | > but their feet are made of clay. 230 | > Microsoft Word has been around for three decades; 231 | > its document format has changed several times in those years, 232 | > but there has still been plenty of time for command-line aficionados 233 | > to adapt their favored tools to handle it. 234 | > That hasn't happened, though, 235 | > which means that most version control systems still can't handle 236 | > the most widely-used documents formats in the world: 237 | > when confronted with two different version of a Microsoft Word file, 238 | > all Git and its kin can say is, "Difference detected." 239 | > The net effect is that 240 | > anyone who wants to adopt version control 241 | > has to abandon the tools that they and their colleagues have used productively for years 242 | > in the hope of greater productivity at some future date. 243 | {: .callout} 244 | 245 | The discussion above has assumed that authors are creating letters and papers, 246 | but researchers also frequently need to create posters and slides to present their work. 247 | PowerPoint is the undisputed queen of presentation tools; 248 | while many people have [critiqued it][tufte-powerpoint], 249 | blaming PowerPoint for bad presentations is like blaming fountain pens for bad poetry. 250 | PowerPoint and its imitators make it easy for people to use their computer's screen as if it was a whiteboard. 251 | Yes, 252 | they can create mind-numbing pages of bullet-point lists if they choose, 253 | but they can also freely *and easily* mix images, diagrams, and text. 254 | LaTeX and HTML can do this, 255 | but neither makes it easy. 256 | In fact, 257 | it's so hard in both that most people don't bother. 258 | Even when they do, 259 | the graphical elements are external foreign inserts 260 | rather than integral parts of the document. 261 | 262 | All this leaves us in an uncomfortable situation. 263 | On the one hand, 264 | papers and presentations are integral parts of research projects, 265 | and should be tracked and shared just like code and data. 266 | On the other hand, 267 | as [Stephen Turner said][turner-comment-docs]: 268 | 269 | > ...try to explain the notion of compiling a document to an overworked physician you collaborate with. 270 | > Oh, but before that, you have to explain the difference between plain text and word processing. 271 | > And text editors. 272 | > And Markdown/LaTeX compilers. 273 | > And BiBTeX. 274 | > And Git. 275 | > And GitHub. Etc. 276 | > Meanwhile he/she is getting paged from the OR... 277 | > 278 | > ...as much as we want to convince ourselves otherwise, 279 | > when you have to collaborate with those outside the scientific computing bubble, 280 | > the barrier to collaborating on papers in this framework is simply too high to overcome. 281 | > Good intentions aside, 282 | > it always comes down to "just give me a Word document with tracked changes," or similar. 283 | {: .quotation} 284 | 285 | For the foreseeable future, 286 | many researchers will therefore continue to use WYSIWYG editors 287 | (and their associated formats) 288 | rather than switch to pure-text typesetting tools. 289 | Hybrid systems like [Authorea][authorea] and [Overleaf][overleaf] may turn this cliff into a ramp, 290 | and programmers might finally have the decency to pay attention to 291 | the document formats that the other 99% of the human race prefers, 292 | but this will be the task of years. 293 | 294 | Since most researchers are already familiar with desktop WYSIWYG systems like Microsoft Word 295 | and cloud-based alternatives like Google Docs, 296 | this lesson will cover two pure-text alternatives: 297 | Markdown for websites and blogs, 298 | and LaTeX for manuscripts. 299 | We recommend Markdown for the web because it does everything most people want HTML to do, 300 | without as much typing. 301 | We recommend *against* it for manuscripts (at least for now) because: 302 | 303 | * Most journals don't accept it as a submission format. 304 | * The odds are against senior collaborators being willing to adopt it. 305 | (Of course, 306 | the odds are also against them being willing to switch to LaTeX if they're not already using it...) 307 | * It doesn't yet do many of the things researchers want (like bibliographic citations). 308 | 309 | LaTeX, on the other hand: 310 | 311 | * compiles to PDF and other standard formats, 312 | * does a pretty good job of laying out figures and tables, 313 | * plays nicely with version control, 314 | * is compatible with lots of bibliograpy management software, and 315 | * is accepted by many journals (though this varies widely from discipline to discpline). 316 | 317 | [authorea]: https://www.authorea.com/ 318 | [html-regexp]: http://stackoverflow.com/a/1732454/1403470 319 | [jupyter]: http://jupyter.org/ 320 | [latex]: http://www.latex-project.org/ 321 | [markdown]: http://daringfireball.net/projects/markdown/ 322 | [overleaf]: https://www.overleaf.com/ 323 | [tufte-powerpoint]: http://www.edwardtufte.com/tufte/powerpoint 324 | [turner-comment-docs]: https://github.com/swcarpentry/good-enough-practices-in-scientific-computing/issues/2#issue-116784345 325 | -------------------------------------------------------------------------------- /bin/workshop_check.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | '''Check that a workshop's index.html metadata is valid. See the 4 | docstrings on the checking functions for a summary of the checks. 5 | ''' 6 | 7 | from __future__ import print_function 8 | import sys 9 | import os 10 | import re 11 | from datetime import date 12 | from util import Reporter, split_metadata, load_yaml, check_unwanted_files 13 | 14 | # Metadata field patterns. 15 | EMAIL_PATTERN = r'[^@]+@[^@]+\.[^@]+' 16 | HUMANTIME_PATTERN = r'((0?[1-9]|1[0-2]):[0-5]\d(am|pm)(-|to)(0?[1-9]|1[0-2]):[0-5]\d(am|pm))|((0?\d|1\d|2[0-3]):[0-5]\d(-|to)(0?\d|1\d|2[0-3]):[0-5]\d)' 17 | EVENTBRITE_PATTERN = r'\d{9,10}' 18 | URL_PATTERN = r'https?://.+' 19 | 20 | # Defaults. 21 | CARPENTRIES = ("dc", "swc") 22 | DEFAULT_CONTACT_EMAIL = 'admin@software-carpentry.org' 23 | 24 | USAGE = 'Usage: "check-workshop path/to/root/directory"' 25 | 26 | # Country and language codes. Note that codes mean different things: 'ar' 27 | # is 'Arabic' as a language but 'Argentina' as a country. 28 | 29 | ISO_COUNTRY = [ 30 | 'ad', 'ae', 'af', 'ag', 'ai', 'al', 'am', 'an', 'ao', 'aq', 'ar', 'as', 31 | 'at', 'au', 'aw', 'ax', 'az', 'ba', 'bb', 'bd', 'be', 'bf', 'bg', 'bh', 32 | 'bi', 'bj', 'bm', 'bn', 'bo', 'br', 'bs', 'bt', 'bv', 'bw', 'by', 'bz', 33 | 'ca', 'cc', 'cd', 'cf', 'cg', 'ch', 'ci', 'ck', 'cl', 'cm', 'cn', 'co', 34 | 'cr', 'cu', 'cv', 'cx', 'cy', 'cz', 'de', 'dj', 'dk', 'dm', 'do', 'dz', 35 | 'ec', 'ee', 'eg', 'eh', 'er', 'es', 'et', 'eu', 'fi', 'fj', 'fk', 'fm', 36 | 'fo', 'fr', 'ga', 'gb', 'gd', 'ge', 'gf', 'gg', 'gh', 'gi', 'gl', 'gm', 37 | 'gn', 'gp', 'gq', 'gr', 'gs', 'gt', 'gu', 'gw', 'gy', 'hk', 'hm', 'hn', 38 | 'hr', 'ht', 'hu', 'id', 'ie', 'il', 'im', 'in', 'io', 'iq', 'ir', 'is', 39 | 'it', 'je', 'jm', 'jo', 'jp', 'ke', 'kg', 'kh', 'ki', 'km', 'kn', 'kp', 40 | 'kr', 'kw', 'ky', 'kz', 'la', 'lb', 'lc', 'li', 'lk', 'lr', 'ls', 'lt', 41 | 'lu', 'lv', 'ly', 'ma', 'mc', 'md', 'me', 'mg', 'mh', 'mk', 'ml', 'mm', 42 | 'mn', 'mo', 'mp', 'mq', 'mr', 'ms', 'mt', 'mu', 'mv', 'mw', 'mx', 'my', 43 | 'mz', 'na', 'nc', 'ne', 'nf', 'ng', 'ni', 'nl', 'no', 'np', 'nr', 'nu', 44 | 'nz', 'om', 'pa', 'pe', 'pf', 'pg', 'ph', 'pk', 'pl', 'pm', 'pn', 'pr', 45 | 'ps', 'pt', 'pw', 'py', 'qa', 're', 'ro', 'rs', 'ru', 'rw', 'sa', 'sb', 46 | 'sc', 'sd', 'se', 'sg', 'sh', 'si', 'sj', 'sk', 'sl', 'sm', 'sn', 'so', 47 | 'sr', 'st', 'sv', 'sy', 'sz', 'tc', 'td', 'tf', 'tg', 'th', 'tj', 'tk', 48 | 'tl', 'tm', 'tn', 'to', 'tr', 'tt', 'tv', 'tw', 'tz', 'ua', 'ug', 'um', 49 | 'us', 'uy', 'uz', 'va', 'vc', 've', 'vg', 'vi', 'vn', 'vu', 'wf', 'ws', 50 | 'ye', 'yt', 'za', 'zm', 'zw' 51 | ] 52 | 53 | ISO_LANGUAGE = [ 54 | 'aa', 'ab', 'ae', 'af', 'ak', 'am', 'an', 'ar', 'as', 'av', 'ay', 'az', 55 | 'ba', 'be', 'bg', 'bh', 'bi', 'bm', 'bn', 'bo', 'br', 'bs', 'ca', 'ce', 56 | 'ch', 'co', 'cr', 'cs', 'cu', 'cv', 'cy', 'da', 'de', 'dv', 'dz', 'ee', 57 | 'el', 'en', 'eo', 'es', 'et', 'eu', 'fa', 'ff', 'fi', 'fj', 'fo', 'fr', 58 | 'fy', 'ga', 'gd', 'gl', 'gn', 'gu', 'gv', 'ha', 'he', 'hi', 'ho', 'hr', 59 | 'ht', 'hu', 'hy', 'hz', 'ia', 'id', 'ie', 'ig', 'ii', 'ik', 'io', 'is', 60 | 'it', 'iu', 'ja', 'jv', 'ka', 'kg', 'ki', 'kj', 'kk', 'kl', 'km', 'kn', 61 | 'ko', 'kr', 'ks', 'ku', 'kv', 'kw', 'ky', 'la', 'lb', 'lg', 'li', 'ln', 62 | 'lo', 'lt', 'lu', 'lv', 'mg', 'mh', 'mi', 'mk', 'ml', 'mn', 'mr', 'ms', 63 | 'mt', 'my', 'na', 'nb', 'nd', 'ne', 'ng', 'nl', 'nn', 'no', 'nr', 'nv', 64 | 'ny', 'oc', 'oj', 'om', 'or', 'os', 'pa', 'pi', 'pl', 'ps', 'pt', 'qu', 65 | 'rm', 'rn', 'ro', 'ru', 'rw', 'sa', 'sc', 'sd', 'se', 'sg', 'si', 'sk', 66 | 'sl', 'sm', 'sn', 'so', 'sq', 'sr', 'ss', 'st', 'su', 'sv', 'sw', 'ta', 67 | 'te', 'tg', 'th', 'ti', 'tk', 'tl', 'tn', 'to', 'tr', 'ts', 'tt', 'tw', 68 | 'ty', 'ug', 'uk', 'ur', 'uz', 've', 'vi', 'vo', 'wa', 'wo', 'xh', 'yi', 69 | 'yo', 'za', 'zh', 'zu' 70 | ] 71 | 72 | 73 | def look_for_fixme(func): 74 | """Decorator to fail test if text argument starts with "FIXME".""" 75 | 76 | def inner(arg): 77 | if (arg is not None) and \ 78 | isinstance(arg, str) and \ 79 | arg.lstrip().startswith('FIXME'): 80 | return False 81 | return func(arg) 82 | return inner 83 | 84 | 85 | @look_for_fixme 86 | def check_layout(layout): 87 | '''"layout" in YAML header must be "workshop".''' 88 | 89 | return layout == 'workshop' 90 | 91 | 92 | @look_for_fixme 93 | def check_carpentry(layout): 94 | '''"carpentry" in YAML header must be "dc" or "swc".''' 95 | 96 | return layout in CARPENTRIES 97 | 98 | 99 | @look_for_fixme 100 | def check_country(country): 101 | '''"country" must be a lowercase ISO-3166 two-letter code.''' 102 | 103 | return country in ISO_COUNTRY 104 | 105 | 106 | @look_for_fixme 107 | def check_language(language): 108 | '''"language" must be a lowercase ISO-639 two-letter code.''' 109 | 110 | return language in ISO_LANGUAGE 111 | 112 | 113 | @look_for_fixme 114 | def check_humandate(date): 115 | """ 116 | 'humandate' must be a human-readable date with a 3-letter month 117 | and 4-digit year. Examples include 'Feb 18-20, 2025' and 'Feb 18 118 | and 20, 2025'. It may be in languages other than English, but the 119 | month name should be kept short to aid formatting of the main 120 | Software Carpentry web site. 121 | """ 122 | 123 | if ',' not in date: 124 | return False 125 | 126 | month_dates, year = date.split(',') 127 | 128 | # The first three characters of month_dates are not empty 129 | month = month_dates[:3] 130 | if any(char == ' ' for char in month): 131 | return False 132 | 133 | # But the fourth character is empty ("February" is illegal) 134 | if month_dates[3] != ' ': 135 | return False 136 | 137 | # year contains *only* numbers 138 | try: 139 | int(year) 140 | except: 141 | return False 142 | 143 | return True 144 | 145 | 146 | @look_for_fixme 147 | def check_humantime(time): 148 | """ 149 | 'humantime' is a human-readable start and end time for the 150 | workshop, such as '09:00 - 16:00'. 151 | """ 152 | 153 | return bool(re.match(HUMANTIME_PATTERN, time.replace(' ', ''))) 154 | 155 | 156 | def check_date(this_date): 157 | """ 158 | 'startdate' and 'enddate' are machine-readable start and end dates 159 | for the workshop, and must be in YYYY-MM-DD format, e.g., 160 | '2015-07-01'. 161 | """ 162 | 163 | # YAML automatically loads valid dates as datetime.date. 164 | return isinstance(this_date, date) 165 | 166 | 167 | @look_for_fixme 168 | def check_latitude_longitude(latlng): 169 | """ 170 | 'latlng' must be a valid latitude and longitude represented as two 171 | floating-point numbers separated by a comma. 172 | """ 173 | 174 | try: 175 | lat, lng = latlng.split(',') 176 | lat = float(lat) 177 | long = float(lng) 178 | return (-90.0 <= lat <= 90.0) and (-180.0 <= long <= 180.0) 179 | except ValueError: 180 | return False 181 | 182 | 183 | def check_instructors(instructors): 184 | """ 185 | 'instructor' must be a non-empty comma-separated list of quoted 186 | names, e.g. ['First name', 'Second name', ...']. Do not use 'TBD' 187 | or other placeholders. 188 | """ 189 | 190 | # YAML automatically loads list-like strings as lists. 191 | return isinstance(instructors, list) and len(instructors) > 0 192 | 193 | 194 | def check_helpers(helpers): 195 | """ 196 | 'helper' must be a comma-separated list of quoted names, 197 | e.g. ['First name', 'Second name', ...']. The list may be empty. 198 | Do not use 'TBD' or other placeholders. 199 | """ 200 | 201 | # YAML automatically loads list-like strings as lists. 202 | return isinstance(helpers, list) and len(helpers) >= 0 203 | 204 | 205 | @look_for_fixme 206 | def check_email(email): 207 | """ 208 | 'contact' must be a valid email address consisting of characters, 209 | an '@', and more characters. It should not be the default contact 210 | email address 'admin@software-carpentry.org'. 211 | """ 212 | 213 | return bool(re.match(EMAIL_PATTERN, email)) and \ 214 | (email != DEFAULT_CONTACT_EMAIL) 215 | 216 | 217 | def check_eventbrite(eventbrite): 218 | """ 219 | 'eventbrite' (the Eventbrite registration key) must be 9 or more 220 | digits. It may appear as an integer or as a string. 221 | """ 222 | 223 | if isinstance(eventbrite, int): 224 | return True 225 | else: 226 | return bool(re.match(EVENTBRITE_PATTERN, eventbrite)) 227 | 228 | 229 | @look_for_fixme 230 | def check_etherpad(etherpad): 231 | """ 232 | 'etherpad' must be a valid URL. 233 | """ 234 | 235 | return bool(re.match(URL_PATTERN, etherpad)) 236 | 237 | 238 | @look_for_fixme 239 | def check_pass(value): 240 | """ 241 | This test always passes (it is used for 'checking' things like the 242 | workshop address, for which no sensible validation is feasible). 243 | """ 244 | 245 | return True 246 | 247 | 248 | HANDLERS = { 249 | 'layout': (True, check_layout, 'layout isn\'t "workshop"'), 250 | 251 | 'carpentry': (True, check_carpentry, 'carpentry isn\'t in ' + 252 | ', '.join(CARPENTRIES)), 253 | 254 | 'country': (True, check_country, 255 | 'country invalid: must use lowercase two-letter ISO code ' + 256 | 'from ' + ', '.join(ISO_COUNTRY)), 257 | 258 | 'language': (False, check_language, 259 | 'language invalid: must use lowercase two-letter ISO code' + 260 | ' from ' + ', '.join(ISO_LANGUAGE)), 261 | 262 | 'humandate': (True, check_humandate, 263 | 'humandate invalid. Please use three-letter months like ' + 264 | '"Jan" and four-letter years like "2025"'), 265 | 266 | 'humantime': (True, check_humantime, 267 | 'humantime doesn\'t include numbers'), 268 | 269 | 'startdate': (True, check_date, 270 | 'startdate invalid. Must be of format year-month-day, ' + 271 | 'i.e., 2014-01-31'), 272 | 273 | 'enddate': (False, check_date, 274 | 'enddate invalid. Must be of format year-month-day, i.e.,' + 275 | ' 2014-01-31'), 276 | 277 | 'latlng': (True, check_latitude_longitude, 278 | 'latlng invalid. Check that it is two floating point ' + 279 | 'numbers, separated by a comma'), 280 | 281 | 'instructor': (True, check_instructors, 282 | 'instructor list isn\'t a valid list of format ' + 283 | '["First instructor", "Second instructor",..]'), 284 | 285 | 'helper': (True, check_helpers, 286 | 'helper list isn\'t a valid list of format ' + 287 | '["First helper", "Second helper",..]'), 288 | 289 | 'contact': (True, check_email, 290 | 'contact email invalid or still set to ' + 291 | '"{0}".'.format(DEFAULT_CONTACT_EMAIL)), 292 | 293 | 'eventbrite': (False, check_eventbrite, 'Eventbrite key appears invalid'), 294 | 295 | 'etherpad': (False, check_etherpad, 'Etherpad URL appears invalid'), 296 | 297 | 'venue': (False, check_pass, 'venue name not specified'), 298 | 299 | 'address': (False, check_pass, 'address not specified') 300 | } 301 | 302 | # REQUIRED is all required categories. 303 | REQUIRED = set([k for k in HANDLERS if HANDLERS[k][0]]) 304 | 305 | # OPTIONAL is all optional categories. 306 | OPTIONAL = set([k for k in HANDLERS if not HANDLERS[k][0]]) 307 | 308 | 309 | def check_blank_lines(reporter, raw): 310 | """ 311 | Blank lines are not allowed in category headers. 312 | """ 313 | 314 | lines = [(i, x) for (i, x) in enumerate(raw.strip().split('\n')) if not x.strip()] 315 | reporter.check(not lines, 316 | None, 317 | 'Blank line(s) in header: {0}', 318 | ', '.join(["{0}: {1}".format(i, x.rstrip()) for (i, x) in lines])) 319 | 320 | 321 | def check_categories(reporter, left, right, msg): 322 | """ 323 | Report differences (if any) between two sets of categories. 324 | """ 325 | 326 | diff = left - right 327 | reporter.check(len(diff) == 0, 328 | None, 329 | '{0}: offending entries {1}', 330 | msg, sorted(list(diff))) 331 | 332 | 333 | def check_file(reporter, path, data): 334 | """ 335 | Get header from file, call all other functions, and check file for 336 | validity. 337 | """ 338 | 339 | # Get metadata as text and as YAML. 340 | raw, header, body = split_metadata(path, data) 341 | 342 | # Do we have any blank lines in the header? 343 | check_blank_lines(reporter, raw) 344 | 345 | # Look through all header entries. If the category is in the input 346 | # file and is either required or we have actual data (as opposed to 347 | # a commented-out entry), we check it. If it *isn't* in the header 348 | # but is required, report an error. 349 | for category in HANDLERS: 350 | required, handler, message = HANDLERS[category] 351 | if category in header: 352 | if required or header[category]: 353 | reporter.check(handler(header[category]), 354 | None, 355 | '{0}\n actual value "{1}"', 356 | message, header[category]) 357 | elif required: 358 | reporter.add(None, 359 | 'Missing mandatory key "{0}"', 360 | category) 361 | 362 | # Check whether we have missing or too many categories 363 | seen_categories = set(header.keys()) 364 | check_categories(reporter, REQUIRED, seen_categories, 365 | 'Missing categories') 366 | check_categories(reporter, seen_categories, REQUIRED.union(OPTIONAL), 367 | 'Superfluous categories') 368 | 369 | 370 | def check_config(reporter, filename): 371 | """ 372 | Check YAML configuration file. 373 | """ 374 | 375 | config = load_yaml(filename) 376 | 377 | kind = config.get('kind', None) 378 | reporter.check(kind == 'workshop', 379 | filename, 380 | 'Missing or unknown kind of event: {0}', 381 | kind) 382 | 383 | carpentry = config.get('carpentry', None) 384 | reporter.check(carpentry in ('swc', 'dc'), 385 | filename, 386 | 'Missing or unknown carpentry: {0}', 387 | carpentry) 388 | 389 | 390 | def main(): 391 | '''Run as the main program.''' 392 | 393 | if len(sys.argv) != 2: 394 | print(USAGE, file=sys.stderr) 395 | sys.exit(1) 396 | 397 | root_dir = sys.argv[1] 398 | index_file = os.path.join(root_dir, 'index.html') 399 | config_file = os.path.join(root_dir, '_config.yml') 400 | 401 | reporter = Reporter() 402 | check_config(reporter, config_file) 403 | check_unwanted_files(root_dir, reporter) 404 | with open(index_file) as reader: 405 | data = reader.read() 406 | check_file(reporter, index_file, data) 407 | reporter.report() 408 | 409 | 410 | if __name__ == '__main__': 411 | main() 412 | -------------------------------------------------------------------------------- /bin/lesson_check.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | """ 4 | Check lesson files and their contents. 5 | """ 6 | 7 | from __future__ import print_function 8 | import sys 9 | import os 10 | import glob 11 | import json 12 | import re 13 | from optparse import OptionParser 14 | 15 | from util import Reporter, read_markdown, load_yaml, check_unwanted_files, require, IMAGE_FILE_SUFFIX 16 | 17 | __version__ = '0.2' 18 | 19 | # Where to look for source Markdown files. 20 | SOURCE_DIRS = ['', '_episodes', '_extras'] 21 | 22 | # Required files: each entry is ('path': YAML_required). 23 | # FIXME: We do not yet validate whether any files have the required 24 | # YAML headers, but should in the future. 25 | # The '%' is replaced with the source directory path for checking. 26 | # Episodes are handled specially, and extra files in '_extras' are also handled specially. 27 | # This list must include all the Markdown files listed in the 'bin/initialize' script. 28 | REQUIRED_FILES = { 29 | '%/CONDUCT.md': True, 30 | '%/CONTRIBUTING.md': False, 31 | '%/LICENSE.md': True, 32 | '%/README.md': False, 33 | '%/_extras/discuss.md': True, 34 | '%/_extras/figures.md': True, 35 | '%/_extras/guide.md': True, 36 | '%/index.md': True, 37 | '%/reference.md': True, 38 | '%/setup.md': True, 39 | } 40 | 41 | # Episode filename pattern. 42 | P_EPISODE_FILENAME = re.compile(r'/_episodes/(\d\d)-[-\w]+.md$') 43 | 44 | # Pattern to match lines ending with whitespace. 45 | P_TRAILING_WHITESPACE = re.compile(r'\s+$') 46 | 47 | # Pattern to match figure references in HTML. 48 | P_FIGURE_REFS = re.compile(r']+src="([^"]+)"[^>]*>') 49 | 50 | # Pattern to match internally-defined Markdown links. 51 | P_INTERNALLY_DEFINED_LINK = re.compile(r'\[[^\]]+\]\[[^\]]+\]') 52 | 53 | # What kinds of blockquotes are allowed? 54 | KNOWN_BLOCKQUOTES = { 55 | 'callout', 56 | 'challenge', 57 | 'checklist', 58 | 'discussion', 59 | 'keypoints', 60 | 'objectives', 61 | 'prereq', 62 | 'quotation', 63 | 'solution', 64 | 'testimonial' 65 | } 66 | 67 | # What kinds of code fragments are allowed? 68 | KNOWN_CODEBLOCKS = { 69 | 'error', 70 | 'output', 71 | 'source', 72 | 'bash', 73 | 'make', 74 | 'matlab', 75 | 'python', 76 | 'r', 77 | 'sql' 78 | } 79 | 80 | # What fields are required in teaching episode metadata? 81 | TEACHING_METADATA_FIELDS = { 82 | ('title', str), 83 | ('teaching', int), 84 | ('exercises', int), 85 | ('questions', list), 86 | ('objectives', list), 87 | ('keypoints', list) 88 | } 89 | 90 | # What fields are required in break episode metadata? 91 | BREAK_METADATA_FIELDS = { 92 | ('layout', str), 93 | ('title', str), 94 | ('break', int) 95 | } 96 | 97 | # How long are lines allowed to be? 98 | MAX_LINE_LEN = 100 99 | 100 | def main(): 101 | """Main driver.""" 102 | 103 | args = parse_args() 104 | args.reporter = Reporter() 105 | check_config(args.reporter, args.source_dir) 106 | docs = read_all_markdown(args.source_dir, args.parser) 107 | check_fileset(args.source_dir, args.reporter, docs.keys()) 108 | check_unwanted_files(args.source_dir, args.reporter) 109 | for filename in docs.keys(): 110 | checker = create_checker(args, filename, docs[filename]) 111 | checker.check() 112 | check_figures(args.source_dir, args.reporter) 113 | args.reporter.report() 114 | 115 | 116 | def parse_args(): 117 | """Parse command-line arguments.""" 118 | 119 | parser = OptionParser() 120 | parser.add_option('-l', '--linelen', 121 | default=False, 122 | action="store_true", 123 | dest='line_lengths', 124 | help='Check line lengths') 125 | parser.add_option('-p', '--parser', 126 | default=None, 127 | dest='parser', 128 | help='path to Markdown parser') 129 | parser.add_option('-s', '--source', 130 | default=os.curdir, 131 | dest='source_dir', 132 | help='source directory') 133 | parser.add_option('-w', '--whitespace', 134 | default=False, 135 | action="store_true", 136 | dest='trailing_whitespace', 137 | help='Check for trailing whitespace') 138 | 139 | args, extras = parser.parse_args() 140 | require(args.parser is not None, 141 | 'Path to Markdown parser not provided') 142 | require(not extras, 143 | 'Unexpected trailing command-line arguments "{0}"'.format(extras)) 144 | 145 | return args 146 | 147 | 148 | def check_config(reporter, source_dir): 149 | """Check configuration file.""" 150 | 151 | config_file = os.path.join(source_dir, '_config.yml') 152 | config = load_yaml(config_file) 153 | reporter.check_field(config_file, 'configuration', config, 'kind', 'lesson') 154 | reporter.check_field(config_file, 'configuration', config, 'carpentry', ('swc', 'dc')) 155 | reporter.check_field(config_file, 'configuration', config, 'title') 156 | reporter.check_field(config_file, 'configuration', config, 'email') 157 | 158 | 159 | def read_all_markdown(source_dir, parser): 160 | """Read source files, returning 161 | {path : {'metadata':yaml, 'metadata_len':N, 'text':text, 'lines':[(i, line, len)], 'doc':doc}} 162 | """ 163 | 164 | all_dirs = [os.path.join(source_dir, d) for d in SOURCE_DIRS] 165 | all_patterns = [os.path.join(d, '*.md') for d in all_dirs] 166 | result = {} 167 | for pat in all_patterns: 168 | for filename in glob.glob(pat): 169 | data = read_markdown(parser, filename) 170 | if data: 171 | result[filename] = data 172 | return result 173 | 174 | 175 | def check_fileset(source_dir, reporter, filenames_present): 176 | """Are all required files present? Are extraneous files present?""" 177 | 178 | # Check files with predictable names. 179 | required = [p.replace('%', source_dir) for p in REQUIRED_FILES] 180 | missing = set(required) - set(filenames_present) 181 | for m in missing: 182 | reporter.add(None, 'Missing required file {0}', m) 183 | 184 | # Check episode files' names. 185 | seen = [] 186 | for filename in filenames_present: 187 | if '_episodes' not in filename: 188 | continue 189 | m = P_EPISODE_FILENAME.search(filename) 190 | if m and m.group(1): 191 | seen.append(m.group(1)) 192 | else: 193 | reporter.add(None, 'Episode {0} has badly-formatted filename', filename) 194 | 195 | # Check for duplicate episode numbers. 196 | reporter.check(len(seen) == len(set(seen)), 197 | None, 198 | 'Duplicate episode numbers {0} vs {1}', 199 | sorted(seen), sorted(set(seen))) 200 | 201 | # Check that numbers are consecutive. 202 | seen = [int(s) for s in seen] 203 | seen.sort() 204 | clean = True 205 | for i in range(len(seen) - 1): 206 | clean = clean and ((seen[i+1] - seen[i]) == 1) 207 | reporter.check(clean, 208 | None, 209 | 'Missing or non-consecutive episode numbers {0}', 210 | seen) 211 | 212 | 213 | def check_figures(source_dir, reporter): 214 | """Check that all figures are present and referenced.""" 215 | 216 | # Get references. 217 | try: 218 | all_figures_html = os.path.join(source_dir, '_includes', 'all_figures.html') 219 | with open(all_figures_html, 'r') as reader: 220 | text = reader.read() 221 | figures = P_FIGURE_REFS.findall(text) 222 | referenced = [os.path.split(f)[1] for f in figures if '/fig/' in f] 223 | except FileNotFoundError as e: 224 | reporter.add(all_figures_html, 225 | 'File not found') 226 | return 227 | 228 | # Get actual image files (ignore non-image files). 229 | fig_dir_path = os.path.join(source_dir, 'fig') 230 | actual = [f for f in os.listdir(fig_dir_path) if os.path.splitext(f)[1] in IMAGE_FILE_SUFFIX] 231 | 232 | # Report differences. 233 | unexpected = set(actual) - set(referenced) 234 | reporter.check(not unexpected, 235 | None, 236 | 'Unexpected image files: {0}', 237 | ', '.join(sorted(unexpected))) 238 | missing = set(referenced) - set(actual) 239 | reporter.check(not missing, 240 | None, 241 | 'Missing image files: {0}', 242 | ', '.join(sorted(missing))) 243 | 244 | 245 | def create_checker(args, filename, info): 246 | """Create appropriate checker for file.""" 247 | 248 | for (pat, cls) in CHECKERS: 249 | if pat.search(filename): 250 | return cls(args, filename, **info) 251 | 252 | 253 | class CheckBase(object): 254 | """Base class for checking Markdown files.""" 255 | 256 | def __init__(self, args, filename, metadata, metadata_len, text, lines, doc): 257 | """Cache arguments for checking.""" 258 | 259 | super(CheckBase, self).__init__() 260 | self.args = args 261 | self.reporter = self.args.reporter # for convenience 262 | self.filename = filename 263 | self.metadata = metadata 264 | self.metadata_len = metadata_len 265 | self.text = text 266 | self.lines = lines 267 | self.doc = doc 268 | 269 | self.layout = None 270 | 271 | 272 | def check(self): 273 | """Run tests on metadata.""" 274 | 275 | self.check_metadata() 276 | self.check_line_lengths() 277 | self.check_trailing_whitespace() 278 | self.check_blockquote_classes() 279 | self.check_codeblock_classes() 280 | self.check_defined_link_references() 281 | 282 | 283 | def check_metadata(self): 284 | """Check the YAML metadata.""" 285 | 286 | self.reporter.check(self.metadata is not None, 287 | self.filename, 288 | 'Missing metadata entirely') 289 | 290 | if self.metadata and (self.layout is not None): 291 | self.reporter.check_field(self.filename, 'metadata', self.metadata, 'layout', self.layout) 292 | 293 | 294 | def check_line_lengths(self): 295 | """Check the raw text of the lesson body.""" 296 | 297 | if self.args.line_lengths: 298 | over = [i for (i, l, n) in self.lines if (n > MAX_LINE_LEN) and (not l.startswith('!'))] 299 | self.reporter.check(not over, 300 | self.filename, 301 | 'Line(s) are too long: {0}', 302 | ', '.join([str(i) for i in over])) 303 | 304 | 305 | def check_trailing_whitespace(self): 306 | """Check for whitespace at the ends of lines.""" 307 | 308 | if self.args.trailing_whitespace: 309 | trailing = [i for (i, l, n) in self.lines if P_TRAILING_WHITESPACE.match(l)] 310 | self.reporter.check(not trailing, 311 | self.filename, 312 | 'Line(s) end with whitespace: {0}', 313 | ', '.join([str(i) for i in trailing])) 314 | 315 | 316 | def check_blockquote_classes(self): 317 | """Check that all blockquotes have known classes.""" 318 | 319 | for node in self.find_all(self.doc, {'type' : 'blockquote'}): 320 | cls = self.get_val(node, 'attr', 'class') 321 | self.reporter.check(cls in KNOWN_BLOCKQUOTES, 322 | (self.filename, self.get_loc(node)), 323 | 'Unknown or missing blockquote type {0}', 324 | cls) 325 | 326 | 327 | def check_codeblock_classes(self): 328 | """Check that all code blocks have known classes.""" 329 | 330 | for node in self.find_all(self.doc, {'type' : 'codeblock'}): 331 | cls = self.get_val(node, 'attr', 'class') 332 | self.reporter.check(cls in KNOWN_CODEBLOCKS, 333 | (self.filename, self.get_loc(node)), 334 | 'Unknown or missing code block type {0}', 335 | cls) 336 | 337 | 338 | def check_defined_link_references(self): 339 | """Check that defined links resolve in the file. 340 | 341 | Internally-defined links match the pattern [text][label]. If 342 | the label contains '{{...}}', it is hopefully a references to 343 | a configuration value - we should check that, but don't right 344 | now. 345 | """ 346 | 347 | result = set() 348 | for node in self.find_all(self.doc, {'type' : 'text'}): 349 | for match in P_INTERNALLY_DEFINED_LINK.findall(node['value']): 350 | if '{{' not in match: 351 | result.add(match) 352 | self.reporter.check(not result, 353 | self.filename, 354 | 'Internally-defined links may be missing definitions: {0}', 355 | ', '.join(sorted(result))) 356 | 357 | 358 | def find_all(self, node, pattern, accum=None): 359 | """Find all matches for a pattern.""" 360 | 361 | assert type(pattern) == dict, 'Patterns must be dictionaries' 362 | if accum is None: 363 | accum = [] 364 | if self.match(node, pattern): 365 | accum.append(node) 366 | for child in node.get('children', []): 367 | self.find_all(child, pattern, accum) 368 | return accum 369 | 370 | 371 | def match(self, node, pattern): 372 | """Does this node match the given pattern?""" 373 | 374 | for key in pattern: 375 | if key not in node: 376 | return False 377 | val = pattern[key] 378 | if type(val) == str: 379 | if node[key] != val: 380 | return False 381 | elif type(val) == dict: 382 | if not self.match(node[key], val): 383 | return False 384 | return True 385 | 386 | 387 | def get_val(self, node, *chain): 388 | """Get value one or more levels down.""" 389 | 390 | curr = node 391 | for selector in chain: 392 | curr = curr.get(selector, None) 393 | if curr is None: 394 | break 395 | return curr 396 | 397 | 398 | def get_loc(self, node): 399 | """Convenience method to get node's line number.""" 400 | 401 | result = self.get_val(node, 'options', 'location') 402 | if self.metadata_len is not None: 403 | result += self.metadata_len 404 | return result 405 | 406 | 407 | class CheckNonJekyll(CheckBase): 408 | """Check a file that isn't translated by Jekyll.""" 409 | 410 | def __init__(self, args, filename, metadata, metadata_len, text, lines, doc): 411 | super(CheckNonJekyll, self).__init__(args, filename, metadata, metadata_len, text, lines, doc) 412 | 413 | 414 | def check_metadata(self): 415 | self.reporter.check(self.metadata is None, 416 | self.filename, 417 | 'Unexpected metadata') 418 | 419 | 420 | class CheckIndex(CheckBase): 421 | """Check the main index page.""" 422 | 423 | def __init__(self, args, filename, metadata, metadata_len, text, lines, doc): 424 | super(CheckIndex, self).__init__(args, filename, metadata, metadata_len, text, lines, doc) 425 | self.layout = 'lesson' 426 | 427 | 428 | class CheckEpisode(CheckBase): 429 | """Check an episode page.""" 430 | 431 | def __init__(self, args, filename, metadata, metadata_len, text, lines, doc): 432 | super(CheckEpisode, self).__init__(args, filename, metadata, metadata_len, text, lines, doc) 433 | 434 | def check_metadata(self): 435 | super(CheckEpisode, self).check_metadata() 436 | if self.metadata: 437 | if 'layout' in self.metadata: 438 | if self.metadata['layout'] == 'break': 439 | self.check_metadata_fields(BREAK_METADATA_FIELDS) 440 | else: 441 | self.reporter.add(self.filename, 442 | 'Unknown episode layout "{0}"', 443 | self.metadata['layout']) 444 | else: 445 | self.check_metadata_fields(TEACHING_METADATA_FIELDS) 446 | 447 | 448 | def check_metadata_fields(self, expected): 449 | for (name, type_) in expected: 450 | if name not in self.metadata: 451 | self.reporter.add(self.filename, 452 | 'Missing metadata field {0}', 453 | name) 454 | elif type(self.metadata[name]) != type_: 455 | self.reporter.add(self.filename, 456 | '"{0}" has wrong type in metadata ({1} instead of {2})', 457 | name, type(self.metadata[name]), type_) 458 | 459 | 460 | class CheckReference(CheckBase): 461 | """Check the reference page.""" 462 | 463 | def __init__(self, args, filename, metadata, metadata_len, text, lines, doc): 464 | super(CheckReference, self).__init__(args, filename, metadata, metadata_len, text, lines, doc) 465 | self.layout = 'reference' 466 | 467 | 468 | class CheckGeneric(CheckBase): 469 | """Check a generic page.""" 470 | 471 | def __init__(self, args, filename, metadata, metadata_len, text, lines, doc): 472 | super(CheckGeneric, self).__init__(args, filename, metadata, metadata_len, text, lines, doc) 473 | self.layout = 'page' 474 | 475 | 476 | CHECKERS = [ 477 | (re.compile(r'CONTRIBUTING\.md'), CheckNonJekyll), 478 | (re.compile(r'README\.md'), CheckNonJekyll), 479 | (re.compile(r'index\.md'), CheckIndex), 480 | (re.compile(r'reference\.md'), CheckReference), 481 | (re.compile(r'_episodes/.*\.md'), CheckEpisode), 482 | (re.compile(r'.*\.md'), CheckGeneric) 483 | ] 484 | 485 | 486 | if __name__ == '__main__': 487 | main() 488 | -------------------------------------------------------------------------------- /assets/img/swc-logo-blue.svg: -------------------------------------------------------------------------------- 1 | 2 | image/svg+xml --------------------------------------------------------------------------------