├── .github └── workflows │ ├── create-pdfs-from-markdown.yml │ ├── linkchecker.yml │ └── markdownlint.yml ├── .mdl.rb ├── .mdlrc ├── 00_organization ├── README.md ├── challenge_intro_slides.md ├── course_intro_slides.md ├── organizational_remarks_week8_slides.md ├── rse_basics_slides.md ├── topic_overview_demo.md ├── wt2122 │ ├── organizational_remarks_week3_slides_old.md │ ├── organizational_remarks_week7_slides_old.md │ └── timetable.md └── wt2223 │ ├── organizational_remarks_week10_slides_old.md │ ├── organizational_remarks_week7_slides_old.md │ └── timetable.md ├── 01_version_control ├── README.md ├── figs │ ├── history_branch │ │ ├── fig.png │ │ └── fig.tex │ ├── history_fastforward │ │ ├── fig.png │ │ └── fig.tex │ ├── history_linear │ │ ├── fig.png │ │ └── fig.tex │ ├── history_merge │ │ ├── fig.png │ │ └── fig.tex │ └── history_rebase │ │ ├── fig.png │ │ └── fig.tex ├── git_basics_demo.md ├── git_exercise.md ├── git_quiz.md ├── intro_slides.md ├── merge_rebase_slides.md ├── my_favorite_neat_little_git_trick_demo.md ├── standards_slides.md └── workflow_slides.md ├── 02_virtualization_and_containers ├── README.md ├── containers_slides.md ├── docker_demo.md ├── docker_slides.md ├── figs │ ├── container-sketch.png │ ├── virtualmachine-sketch.png │ └── virtualmachines-containers.svg ├── intro_slides.md ├── singularity_demo.md ├── singularity_slides.md ├── vagrant_demo.md ├── vagrant_slides.md ├── virtualbox_demo.md ├── virtualbox_slides.md ├── virtualmachines_containers_exercise.md └── virtualmachines_slides.md ├── 03_building_and_packaging ├── README.md ├── cmake_demo.md ├── cmake_exercise.md ├── cmake_more_demo.md ├── cmake_slides.md ├── cpack_demo.md ├── cpack_exercise.md ├── cpack_slides.md ├── examples │ ├── cmake │ │ ├── CMakeLists.txt │ │ ├── main.cpp │ │ └── sse │ │ │ ├── sse.cpp │ │ │ └── sse.hpp │ ├── cpack │ │ ├── CMakeLists.txt │ │ ├── Dockerfile │ │ ├── cmake │ │ │ └── CPackConfig.cmake │ │ ├── main.cpp │ │ └── sse │ │ │ ├── sse.cpp │ │ │ └── sse.hpp │ ├── environment-variables │ │ ├── Dockerfile │ │ └── print_my_environment_variable.sh │ ├── make │ │ ├── Makefile │ │ ├── main.cpp │ │ └── sse │ │ │ ├── sse.cpp │ │ │ └── sse.hpp │ └── shared-and-static-c++ │ │ └── main.cpp ├── figs │ ├── filesystem_hierarchy_structure │ │ ├── fig.png │ │ └── fig.tex │ ├── filesystem_paths_python │ │ ├── fig-user-level.png │ │ ├── fig-user-level.tex │ │ ├── fig.png │ │ └── fig.tex │ └── spack │ │ └── fenics-dependencies.png ├── intro_slides.md ├── linux_fundamentals_demo.md ├── linux_fundamentals_slides.md ├── make_demo.md ├── make_slides.md ├── pip_demo.md ├── pypi_exercise.md ├── pypi_slides.md ├── spack_demo.md ├── spack_exercise.md └── spack_slides.md ├── 04_documentation ├── README.md ├── examples │ ├── markdown-example.md │ ├── pandoc │ │ └── example.md │ ├── reStructuredText-example.rst │ ├── sphinx │ │ ├── README.rst │ │ ├── docs │ │ │ └── tutorials │ │ │ │ ├── overview.rst │ │ │ │ ├── tutorial_case1.rst │ │ │ │ └── tutorial_case2.rst │ │ ├── setup.cfg │ │ ├── setup.py │ │ └── src │ │ │ └── solver │ │ │ ├── __init__.py │ │ │ └── diffusionsolver.py │ └── technical_writing │ │ └── diffusion2d.py ├── figs │ ├── pandoc-latex-workflow │ │ ├── fig.png │ │ └── fig.tex │ └── pandoc │ │ ├── fig.png │ │ └── fig.tex ├── markup_languages_slides.md ├── pandoc_demo.md ├── pandoc_slides.md ├── technical_writing_demo.md ├── technical_writing_slides.md ├── website_generators_demo.md └── website_generators_slides.md ├── 05_testing_and_ci ├── README.md ├── automation_exercise.md ├── automation_slides.md ├── boost_testing_exercise.md ├── boost_testing_intro_demo.md ├── boost_testing_intro_slides.md ├── boost_testing_precice_demo.md ├── boost_testing_sidemade_demo.md ├── examples │ ├── automation │ │ ├── Dockerfile │ │ ├── Vagrantfile │ │ └── bootstrap.sh │ ├── boost_test │ │ └── helloWorldTest.cpp │ └── python_testing │ │ ├── mean_data.csv │ │ ├── operations.py │ │ ├── test_operations.py │ │ ├── test_operations_unittests.py │ │ └── tox.toml ├── figs │ └── automation_exercise │ │ └── gitlab_coverage.png ├── github_actions_demo.md ├── github_actions_slides.md ├── gitlab_ci_demo.md ├── gitlab_ci_slides.md ├── intro_slides.md ├── python_testing_demo.md ├── python_testing_exercise.md └── python_testing_slides.md ├── 06_miscellaneous ├── README.md ├── doi_zenodo_darus_slides.md ├── floss_licenses_slides.md ├── master_thesis_suggestions_slides.md ├── repository_layouts_slides.md ├── tools_alex_demo.md ├── tools_alex_slides.md └── versioning_slides.md ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── docs ├── pull_request_template.md └── styleguide.md ├── scripts └── create-pdf-from-markdown.sh └── timetable.md /.github/workflows/create-pdfs-from-markdown.yml: -------------------------------------------------------------------------------- 1 | name: PDF creation 2 | 3 | on: 4 | push: 5 | branches: 6 | - "*" 7 | pull_request: 8 | branches: 9 | - "*" 10 | workflow_dispatch: 11 | 12 | jobs: 13 | create_pdfs: 14 | runs-on: ubuntu-22.04 15 | container: pandoc/latex:2 16 | steps: 17 | - name: Install bash 18 | run: apk add bash 19 | - name: Checkout repository 20 | uses: actions/checkout@v2 21 | - name: Compile PDFs (slides) 22 | run: | 23 | ./scripts/create-pdf-from-markdown.sh slides 24 | - name: Upload slides 25 | uses: actions/upload-artifact@master 26 | with: 27 | name: slides 28 | path: slides 29 | retention-days: 7 30 | - name: Compile PDFs (quizzes) 31 | run: | 32 | ./scripts/create-pdf-from-markdown.sh quiz 33 | - name: Compile PDFs (exercises) 34 | run: | 35 | ./scripts/create-pdf-from-markdown.sh exercise 36 | - name: Compile PDFs (demos) 37 | run: | 38 | ./scripts/create-pdf-from-markdown.sh demo 39 | - name: Upload texts 40 | uses: actions/upload-artifact@master 41 | with: 42 | name: texts 43 | path: texts 44 | retention-days: 7 45 | -------------------------------------------------------------------------------- /.github/workflows/linkchecker.yml: -------------------------------------------------------------------------------- 1 | name: Link checker 2 | 3 | on: 4 | push: 5 | branches: [ '*' ] 6 | pull_request: 7 | branches: [ '*' ] 8 | schedule: 9 | - cron: "0 0 1 * 0" 10 | 11 | jobs: 12 | build: 13 | 14 | runs-on: ubuntu-latest 15 | 16 | steps: 17 | - uses: actions/checkout@v2 18 | - name: Set up Ruby 2.6 19 | uses: actions/setup-ruby@v1 20 | with: 21 | ruby-version: 2.6.x 22 | - name: Check links 23 | #FIXME Need to allow duplicates (same links on the same page) 24 | run: | 25 | gem install awesome_bot 26 | awesome_bot --white-list https://gitlab-sim.informatik.uni-stuttgart.de/ --allow-ssl --allow-redirect --allow-dupe $(find . -iname "*.md" -type f) 27 | -------------------------------------------------------------------------------- /.github/workflows/markdownlint.yml: -------------------------------------------------------------------------------- 1 | name: markdownlint 2 | on: [pull_request, push] 3 | jobs: 4 | mdl: 5 | runs-on: ubuntu-latest 6 | steps: 7 | - name: Check out code 8 | uses: actions/checkout@v2 9 | - name: Run mdl 10 | uses: actionshub/markdownlint@main 11 | -------------------------------------------------------------------------------- /.mdl.rb: -------------------------------------------------------------------------------- 1 | # Enable all rules by default 2 | all 3 | 4 | ###################################################### 5 | # Deactivated/excluded rules 6 | ###################################################### 7 | 8 | # Allow first header to not be a top level header 9 | # since top-level is set by metadata 10 | exclude_rule 'MD002' 11 | # Allow hard tabs such that they can be used in code blocks 12 | # as needed for Makefiles 13 | exclude_rule 'MD010' 14 | # First line in file does not have to be a top level header 15 | # since top-level is set by metadata 16 | exclude_rule 'MD041' 17 | # Allow multiple consecutive blank lines 18 | exclude_rule 'MD012' 19 | # Allow raw HTML to include videos, for example. 20 | exclude_rule 'MD033' 21 | # Bare URL used rule seems buggy (false positives), therefore excluded 22 | exclude_rule 'MD034' 23 | # Allow code blocks without a language specified 24 | exclude_rule 'MD040' 25 | 26 | ###################################################### 27 | # Adapted rules 28 | ###################################################### 29 | 30 | # Enforce dashes as symvol in lists 31 | rule 'MD004', :style => :dash 32 | # Enforce indent with four space characters. This helps with 33 | # mixing ordered and unordered lists 34 | rule 'MD007', :indent => 4 35 | # Extend line length for text. 36 | # This will complain for overly wide tables and code blocks. 37 | rule 'MD013', :line_length => 99999 38 | 39 | # Nested lists should be indented with four spaces. 40 | # Modification: Question marks should be allowed 41 | rule 'MD026', :punctuation => '.,;:!' 42 | 43 | # Ordered lists must have prefix that increases in order 44 | rule 'MD029', :style => :ordered 45 | -------------------------------------------------------------------------------- /.mdlrc: -------------------------------------------------------------------------------- 1 | # Load style file (ruby script) 2 | style '.mdl.rb' -------------------------------------------------------------------------------- /00_organization/README.md: -------------------------------------------------------------------------------- 1 | # Organization and Introduction to Research Software Engineering 2 | 3 | Learning goals: 4 | 5 | - Lecture, lab organization, and the challenge 6 | - What is research software engineering and what is special about it? 7 | - Why do we need research software engineering? 8 | 9 | | Duration | Content | 10 | | --- | --- | 11 | | 20 minutes | [`challenge_intro_slides.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/00_organization/challenge_intro_slides.md) | 12 | | 30 minutes | [`course_intro_slides.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/00_organization/course_intro_slides.md) | 13 | | 5 minutes | [`organizational_remarks_week8_slides.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/00_organization/organizational_remarks_week8_slides) | 14 | | 15 minutes | [`rse_basics.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/00_organization/rse_basics_slides.md) | 15 | | 10 minutes | [`topic_overview_demo.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/00_organization/topic_overview_demo.md) | 16 | 17 | Folders such as `wt2122` contain organizational content specific to previous lectures. 18 | -------------------------------------------------------------------------------- /00_organization/organizational_remarks_week8_slides.md: -------------------------------------------------------------------------------- 1 | --- 2 | type: slide 3 | slideOptions: 4 | transition: slide 5 | width: 1400 6 | height: 900 7 | margin: 0.1 8 | data-background-color: green 9 | --- 10 | 11 | 35 | 36 | # Organizational Things Week 8 37 | 38 | --- 39 | 40 | ## Time Table 41 | 42 | 47 | 48 | | Date | Type | Chapter | Topic | Lecturer | 49 | | ---- | ---- | ------- |------ | -------- | 50 | | 13.11. |Lecture | 3 | Intro packaging, Python packaging | Ishaan | 51 | | 13.11. |Lab | 3 | Python packaging | Ishaan | 52 | | 20.11. |Lecture | 3 | Linux fundamentals, Make, CMake | Benjamin | 53 | | 20.11. |Lab | 3 | CMake and Docker | Benjamin | 54 | | 27.11. |Lecture | 3 | Spack | Ishaan | 55 | | 27.11. |Lab | 3 | Spack | Ishaan | 56 | | **04.12.** |**Lecture** | **3** | **CPack and more CMake** | **Benjamin** | 57 | | 04.12. |Lab | 3 | CPack | Benjamin | 58 | | 11.12. |Lecture | 4 | Technical writing | Gerasimos | 59 | | 11.12. |Presentations | C | 2nd student presentations | students | 60 | 61 | --- 62 | 63 | ## Challenge First Report: MR Feedback 64 | 65 | - Try to write good MR titles and descriptions 66 | - Resolve threads when comments are answered, done, or dropped 67 | - If changes requested: try to fix in same MR 68 | - Approval by reviewer: ready to merge 69 | - All MRs should get merged 70 | 71 | --- 72 | 73 | ## Challenge Contribution 74 | 75 | - Please open an issue in the [challenge repo](https://gitlab-sim.informatik.uni-stuttgart.de/simulation-software-engineering-wite2425/challenge) explaining what you want to work on in the contribution. We will then use these issues to further discuss the progress of the contribution. Provide links to issues, pull requests etc. 76 | - Deadline (for opening the issue): Dec 16, 2024, 23:59 77 | - If you open earlier, you get feedback earlier 78 | - Latest feedback from teachers: Jan 8, 2025, 09:45 79 | - Title of issue: "Contribution to SOFTWARE by USERNAME" 80 | - Get in touch with maintainers early (through official channels) 81 | - Discuss before you start developing 82 | -------------------------------------------------------------------------------- /00_organization/rse_basics_slides.md: -------------------------------------------------------------------------------- 1 | --- 2 | type: slide 3 | slideOptions: 4 | transition: slide 5 | width: 1400 6 | height: 900 7 | margin: 0.1 8 | --- 9 | 10 | 34 | 35 | # Introduction to Research Software Engineering 36 | 37 | --- 38 | 39 | ## Starting Points 40 | 41 | - [DORA declaration](https://sfdora.org/) in 2012: reshape how research impact should be assessed underlining importance of software 42 | 43 | > For the purposes of research assessment, consider the value and impact of all research outputs (including datasets and software) in addition to research publications ... 44 | 45 | - [UK survey in 2014](https://zenodo.org/record/1183562): 7 out of 10 researchers could not conduct research without software. 46 | - DFG funding calls on research software in 2016 & 2019 & [2022](https://www.dfg.de/en/news/news-topics/announcements-proposals/2022/info-wissenschaft-22-85) 47 | - [Nationale Forschungsdaten Infrastruktur, NFDI](https://www.nfdi.de/?lang=en) since 2020 48 | - Lack of careers for software developers in academia 49 | - Lack of reproducibility of research that uses software (*"works for me on my machine today"* vs. *"works for everyone everywhere always"*) 50 | 51 | --- 52 | 53 | ## RSE Movement 54 | 55 | ... academic software developers needed a name: 56 | **Research Software Engineers** 57 | 58 | [UK Society of RSE](https://society-rse.org/): 59 | 60 | > A Research Software Engineer (RSE) combines professional software engineering expertise with an intimate understanding of research. 61 | 62 | - *"Movement"* started in the UK, first UK RSE conference in 2016 63 | - First conferences in Germany and the Netherlands in 2019 64 | - [First de-RSE position paper](https://f1000research.com/articles/9-295/v2) in 2020, [several working groups](https://de-rse.org/en/working_groups.html) 65 | - [Learning and teaching RSE](https://de-rse.org/learn-and-teach/), [foundational competencies](https://de-rse.org/blog/2024/10/08/identifying-the-foundational-competencies-of-an-RSE-en.html) 66 | - [Why be an RSE?](https://researchit.blogs.bristol.ac.uk/2021/10/14/international-rse-day-why-be-an-rse/) Interesting and novel projects, technical freedom, RSEs come from varied backgrounds, development for social good 67 | 68 | --- 69 | 70 | ## How is Simulation Software Engineering different from RSE? 71 | 72 | > Why is the name of the course SSE and not RSE? 73 | 74 | - Difference is not so big actually. 75 | - Simulation research depends on software by definition (even less a *"by-product"* than for research in Chemistry, Biology, ...). 76 | - Already prototypes are so complicated today that one (wo)man hero PhD projects are no longer feasible. --> Learn how to work with / contribute to community software. 77 | - We focus on simulation software in the challenge. 78 | 79 | --- 80 | 81 | ## Do I need this in Industry? 82 | 83 | - Yes. Research also happens in industry. 84 | - All things we learn (Git, packaging, CI/CD, virtualization, documentation, ...) is also highly relevant for non-research software. 85 | - Companies use (more and more) the same workflows and tools. 86 | - It is not just about coding. It is about collaborative work. 87 | - Open-source development excellent door opener for industry. 88 | - Some companies use open-source software and need to make contributions. 89 | - Some companies develop their software as open-source software. 90 | -------------------------------------------------------------------------------- /00_organization/topic_overview_demo.md: -------------------------------------------------------------------------------- 1 | # Topic Overview Demo 2 | 3 | - We will use [preCICE](https://precice.org/) as an example for many things in this course. 4 | - Briefly sprint through all 6 chapters of the course and preview what we will learn. 5 | - preCICE is a coupling library for partitioned multi-physics simulations, developed at US and TUM. 6 | - Chapter 1. Version Control 7 | - [preCICE organization on GitHub](https://github.com/precice), multi-repo approach 8 | - at the core is a [library called precice](https://github.com/precice/precice) 9 | - forking workflow with protected main branch, develop is default, open a PR, explain source and target 10 | - linear history, show commits 11 | - Chapter 2. Virtualization and Containers 12 | - being a coupling library, many software components are involved in actual simulations --> virtualization is an important topic to handle all these components 13 | - [preCICE Docker images](https://hub.docker.com/u/precice) for testing only (preCICE and downstream repositories), not meant for users 14 | - [virtual machine](https://precice.org/installation-vm.html) for demo (sandbox) mode with many components, scripted using Vagrant 15 | - Chapter 3. Building and Packaging 16 | - [installation overview](https://precice.org/installation-overview.html), Debian package, spack, Python bindings on PyPI 17 | - package creation and [building from source](https://precice.org/installation-source-configuration.html) via CMake 18 | - Chapter 4. Documentation 19 | - written in markdown in distributed places (click edit on some page), rendered via jekyll, hosted via GitHub Pages 20 | - distinguish user from dev docs, tutorials, configuration references 21 | - [brief styleguide](https://precice.org/docs-meta-content-guidelines.html) 22 | - [changelog](https://github.com/precice/precice/blob/develop/CHANGELOG.md) 23 | - Chapter 5. Testing and Continuous Integration 24 | - [unit and integration](https://precice.org/dev-docs-dev-testing.html) tests via Boost.Test 25 | - [tests and more automation](https://github.com/precice/precice/actions) run on Github Actions 26 | - many Python components use unittest 27 | - Chapter 6. Miscellaneous 28 | - LGPL3 license 29 | - semantic versioning scheme 30 | - [distribution published on DaRUS](https://darus.uni-stuttgart.de/dataset.xhtml?persistentId=doi:10.18419/darus-4167) 31 | -------------------------------------------------------------------------------- /00_organization/wt2122/organizational_remarks_week3_slides_old.md: -------------------------------------------------------------------------------- 1 | --- 2 | type: slide 3 | slideOptions: 4 | transition: slide 5 | width: 1400 6 | height: 900 7 | margin: 0.1 8 | data-background-color: green 9 | --- 10 | 11 | 35 | 36 | # Organizational Things 37 | 38 | --- 39 | 40 | ## Misc 41 | 42 | - Please add presentations from last week to [*"My neat little Git trick"*](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/01_version_control/my_favorite_neat_little_git_trick_demo.md) 43 | - [GitHub Universe](https://www.githubuniverse.com/2021) was last week. 44 | - Amazing new feature: `ctrl` + `k` and never use mouse again 45 | 46 | --- 47 | 48 | ## Git Cheat Sheet 49 | 50 | - Observations: 51 | - Pay more attention to descriptions of issues and PRs: what, why, how, ... 52 | - As reviewer, don't just accept, but think about how to increase quality 53 | - You don't seem to like perfect linear histories :grin: ... in many cases just use `rebase & merge` or `squash & merge` button on GitHub 54 | - **Deadline**: next Monday, Nov 8, 9am 55 | 56 | --- 57 | 58 | ## Challenge 59 | 60 | - Have you seen **matching** on [GitLab](https://gitlab-sim.informatik.uni-stuttgart.de/simulation-software-engineering/challenge/-/issues/1)? 61 | - Information on **first presentations** also on [GitLab](https://gitlab-sim.informatik.uni-stuttgart.de/simulation-software-engineering/challenge/-/issues/2) 62 | - Half of students present first step, the other half second step (everybody third step) 63 | - Still, everybody needs to hand in a report each time 64 | - Please let us know (as early as possible) if you drop out such that we can find another presenter 65 | -------------------------------------------------------------------------------- /00_organization/wt2122/organizational_remarks_week7_slides_old.md: -------------------------------------------------------------------------------- 1 | --- 2 | type: slide 3 | slideOptions: 4 | transition: slide 5 | width: 1400 6 | height: 900 7 | margin: 0.1 8 | data-background-color: green 9 | --- 10 | 11 | 35 | 36 | # Organizational Things Week 7 37 | 38 | --- 39 | 40 | ## Poll on First 5 Weeks (1/2) 41 | 42 | - Thanks for participating! 43 | - Many wrote that they liked the concept. Made us very happy. :blush: 44 | - **Multiple sources of content (website, GitLab, GitHub)** 45 | - We are aware of the problem and are thinking about solutions. 46 | - Reason 1: you should learn GitLab + GitHub 47 | - Reason 2: private information (GitLab) vs. public information (GitHub) 48 | - We need privacy for some things. We want public domain for other things. 49 | - In general: lecture on GitHub, challenge on GitLab 50 | 51 | --- 52 | 53 | ## Poll on First 5 Weeks (2/2) 54 | 55 | - **Exercises too much workload** 56 | - How much time do you still need at home to finish? 57 | - You are not expected to finish within the exercise session (50% done is good). 58 | - Ask for help! Also through draft PRs. 59 | - **Git exercise and VM/containers lecture need improvement** -> YES 60 | - Demo challenge presentation by lecturers? 61 | - We are thinking about it. 62 | - What do you hope to learn exactly? 63 | 64 | --- 65 | 66 | ## Challenge 67 | 68 | - Information on **second step** on [GitLab](https://gitlab-sim.informatik.uni-stuttgart.de/simulation-software-engineering/challenge/-/issues/3) 69 | - Now, the *"other half"* presents. 70 | - Still, everybody needs to hand in a report (deadline: Dec 16, 2021, 15:45). 71 | - Please open an issue in the [challenge repo](https://gitlab-sim.informatik.uni-stuttgart.de/simulation-software-engineering/challenge/) explaining what you want to work on for step 3. We will then use these issues to further discuss the progress of the contribution. Provide links to issues, pull requests etc. 72 | - Deadline (for opening the issue): Jan 10, 2022, 23:59 73 | - Title of issue: "Contribution to SOFTWARE by USERNAME" 74 | -------------------------------------------------------------------------------- /00_organization/wt2223/organizational_remarks_week10_slides_old.md: -------------------------------------------------------------------------------- 1 | --- 2 | type: slide 3 | slideOptions: 4 | transition: slide 5 | width: 1400 6 | height: 900 7 | margin: 0.1 8 | data-background-color: green 9 | --- 10 | 11 | 35 | 36 | # Organizational Things Week 10 37 | 38 | --- 39 | 40 | ## Overview of Upcoming Weeks 41 | 42 | We had to move things around a bit. 43 | 44 | 49 | 50 | | Date | Type | Chapter | Topic | Lecturer | 51 | | ---- | ---- | ------- |------ | -------- | 52 | | 12.01. |Lecture | 4 | Documentation Tools | Benjamin | 53 | | 12.01. |Lecture | 6 | Licenses, Versioning, Repo Layouts, ... | Benjamin | 54 | | 19.01. |Lecture | 5 | Testing in Python | Ishaan | 55 | | 19.01. |Lab | 5 | Testing in Python | Ishaan | 56 | | 26.01. |Lecture | 5 | Automation, CI/CD | Benjamin | 57 | | 26.01. |Lab | 5 | Automation, CI/CD | Benjamin | 58 | | 02.02. |Lecture | 5 | Boost.Test | Benjamin | 59 | | 02.02. |Lab | 5 | Boost.Test | Benjamin | 60 | | 09.02. |Presentations | C | **final student presentations** | students | 61 | | 09.02. |Presentations | C | **final student presentations** | students | 62 | 63 | --- 64 | 65 | ## Challenge Step 3: Suggestions 66 | 67 | - Heterogeneous field: from *"already did good contributions"* to *"not really started"* 68 | - If we think you are doing good, we will not interfere much, but you can of course always ask for input. 69 | - A few have not yet opened an issue in the challenge repo (deadline was Dec 18). Please get in touch with your adviser asap. 70 | - In general, please use the issue to keep us posted about any updates. 71 | 72 | --- 73 | 74 | ## Challenge Step 3: Report and Presentations 75 | 76 | - Again report + presentation 77 | - Presentations and deadline for report on Feb 9 78 | - Issue with detailed instructions will follow soon 79 | - A bit more and a bit longer presentations 80 | - We will assign presenters: 81 | - Everybody should give at least one presentation. 82 | - We will pick nice contributions for additional presentations. 83 | -------------------------------------------------------------------------------- /00_organization/wt2223/organizational_remarks_week7_slides_old.md: -------------------------------------------------------------------------------- 1 | --- 2 | type: slide 3 | slideOptions: 4 | transition: slide 5 | width: 1400 6 | height: 900 7 | margin: 0.1 8 | data-background-color: green 9 | --- 10 | 11 | 35 | 36 | # Organizational Things Week 7 37 | 38 | --- 39 | 40 | ## Challenge Step 1: MR Feedback 41 | 42 | - Try to write good MR titles and descriptions 43 | - Resolve threads when comments are answered, done, or dropped 44 | - If changes requested: try to fix in same MR 45 | - Approval by reviewer: ready to merge 46 | 47 | --- 48 | 49 | ## Challenge Step 1+2: Content 50 | 51 | - Steps 1 and 2 should lead you towards the contribution (step 3) 52 | - Step 1: understand software from user perspective 53 | - Step 2: understand software from dev perspective 54 | - Reports are reports on what you did, not the actual work 55 | 56 | --- 57 | 58 | ## Challenge Step 3 59 | 60 | - Please open an issue in the [challenge repo](https://gitlab-sim.informatik.uni-stuttgart.de/simulation-software-engineering-wite2223/challenge) explaining what you want to work on for step 3. We will then use these issues to further discuss the progress of the contribution. Provide links to issues, pull requests etc. 61 | - Deadline (for opening the issue): Dec 18, 2022, 23:59 62 | - If you open earlier, you get feedback earlier 63 | - Latest feedback from teachers: Jan 12, 2023, 09:45 64 | - Title of issue: "Contribution to SOFTWARE by USERNAME" 65 | - Get in touch with maintainers early (through official channels) 66 | - Discuss before you start developing 67 | -------------------------------------------------------------------------------- /01_version_control/README.md: -------------------------------------------------------------------------------- 1 | # Git – The Standard Version Control System 2 | 3 | Learning goals: 4 | 5 | - Refresh and organize students' existing knowledge on Git (and learn how to learn more). 6 | - Explain difference between merge and rebase and when to use what. 7 | - Use Git workflows to organize research software development in a team. 8 | - Know about a few useful GitHub/GitLab standards and a few helpful tools. 9 | 10 | | Duration | Content | 11 | | --- | --- | 12 | | 90 minutes | [`git_exercise.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/01_version_control/git_exercise.md) | 13 | | 30 minutes | [`git_quiz.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/01_version_control/git_quiz.md) | 14 | | 50 minutes | [`git_basics_demo.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/01_version_control/git_basics_demo.md) | 15 | | 30-45 minutes| [`my_favorite_neat_little_git_trick_demo.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/01_version_control/my_favorite_neat_little_git_trick_demo.md) | 16 | | 20 minutes | [`intro_slides.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/01_version_control/intro_slides.md) | 17 | | 25 minutes | [`merge_rebase_slides.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/01_version_control/merge_rebase_slides.md) | 18 | | 10 minutes | [`standards_slides.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/01_version_control/standards_slides.md) | 19 | | 20 minutes | [`workflow_slides.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/01_version_control/workflow_slides.md) | 20 | -------------------------------------------------------------------------------- /01_version_control/figs/history_branch/fig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Simulation-Software-Engineering/Lecture-Material/d58a42f0ddd51ed992519d7fc067d7866c8bb3fa/01_version_control/figs/history_branch/fig.png -------------------------------------------------------------------------------- /01_version_control/figs/history_branch/fig.tex: -------------------------------------------------------------------------------- 1 | % build with: pdflatex --shell-escape fig.tex 2 | \documentclass[tikz, convert={density=800}]{standalone} 3 | 4 | \usepackage{tikz} 5 | \definecolor{myOrange}{HTML}{ffa500} 6 | \definecolor{myBlue}{HTML}{42affa} 7 | 8 | \begin{document} 9 | \begin{tikzpicture}[>=latex, thick, shorten >=2pt, shorten <=2pt, ->, myBlue] 10 | \tikzstyle{C}=[circle,fill=myBlue,text=white,minimum size=30pt,inner sep=2pt, font=\LARGE] 11 | \tikzstyle{M}=[circle,fill=myOrange,text=white,minimum size=30pt,inner sep=2pt, font=\LARGE] 12 | \tikzstyle{B}=[text=myBlue, font=\LARGE] 13 | 14 | \node[C] at (0,3) (A) {\texttt{A}}; 15 | \node[C] at (3,3) (B) {\texttt{B}}; 16 | \node[C] at (6,3) (C) {\texttt{C}}; 17 | 18 | \node[C] at (3,1) (D) {\texttt{D}}; 19 | \node[C] at (6,1) (E) {\texttt{E}}; 20 | 21 | \draw [<-] (A) to (B); 22 | \draw [<-] (B) to (C); 23 | \draw [<-] (A) to (D); 24 | \draw [<-] (D) to (E); 25 | 26 | \node[B] at (-3,3) {\texttt{main}}; 27 | \node[B] at (-3,1) {\texttt{feature}}; 28 | 29 | \end{tikzpicture} 30 | \end{document} 31 | -------------------------------------------------------------------------------- /01_version_control/figs/history_fastforward/fig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Simulation-Software-Engineering/Lecture-Material/d58a42f0ddd51ed992519d7fc067d7866c8bb3fa/01_version_control/figs/history_fastforward/fig.png -------------------------------------------------------------------------------- /01_version_control/figs/history_fastforward/fig.tex: -------------------------------------------------------------------------------- 1 | % build with: pdflatex --shell-escape fig.tex 2 | \documentclass[tikz, convert={density=800}]{standalone} 3 | 4 | \usepackage{tikz} 5 | \definecolor{myOrange}{HTML}{ffa500} 6 | \definecolor{myBlue}{HTML}{42affa} 7 | 8 | \begin{document} 9 | \begin{tikzpicture}[>=latex, thick, shorten >=2pt, shorten <=2pt, ->, myBlue] 10 | \tikzstyle{C}=[circle,fill=myBlue,text=white,minimum size=30pt,inner sep=2pt, font=\LARGE] 11 | \tikzstyle{M}=[circle,fill=myOrange,text=white,minimum size=30pt,inner sep=2pt, font=\LARGE] 12 | \tikzstyle{B}=[text=myBlue, font=\LARGE] 13 | 14 | \node[C] at (0,3) (A) {\texttt{A}}; 15 | 16 | \node[C] at (3,1) (D) {\texttt{D}}; 17 | \node[C] at (6,1) (E) {\texttt{E}}; 18 | 19 | \draw [<-] (A) to (D); 20 | \draw [<-] (D) to (E); 21 | 22 | \node[B] at (-3,3) {\texttt{main}}; 23 | \node[B] at (-3,1) {\texttt{feature}}; 24 | 25 | \end{tikzpicture} 26 | \end{document} 27 | -------------------------------------------------------------------------------- /01_version_control/figs/history_linear/fig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Simulation-Software-Engineering/Lecture-Material/d58a42f0ddd51ed992519d7fc067d7866c8bb3fa/01_version_control/figs/history_linear/fig.png -------------------------------------------------------------------------------- /01_version_control/figs/history_linear/fig.tex: -------------------------------------------------------------------------------- 1 | % build with: pdflatex --shell-escape fig.tex 2 | \documentclass[tikz, convert={density=800}]{standalone} 3 | 4 | \usepackage{tikz} 5 | \definecolor{myBlue}{HTML}{42affa} 6 | 7 | \begin{document} 8 | \begin{tikzpicture}[>=latex, thick, shorten >=2pt, shorten <=2pt, ->, myBlue] 9 | \tikzstyle{C}=[circle,fill=myBlue,text=white,minimum size=30pt,inner sep=2pt, font=\LARGE] 10 | \tikzstyle{B}=[text=myBlue, font=\LARGE] 11 | 12 | \node[C] at (0,1) (A) {\texttt{A}}; 13 | \node[C] at (3,1) (B) {\texttt{B}}; 14 | \node[C] at (6,1) (C) {\texttt{C}}; 15 | 16 | \draw [<-] (A) to (B); 17 | \draw [<-] (B) to (C); 18 | 19 | \node[B] at (-3,1) {\texttt{main}}; 20 | 21 | \end{tikzpicture} 22 | \end{document} 23 | -------------------------------------------------------------------------------- /01_version_control/figs/history_merge/fig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Simulation-Software-Engineering/Lecture-Material/d58a42f0ddd51ed992519d7fc067d7866c8bb3fa/01_version_control/figs/history_merge/fig.png -------------------------------------------------------------------------------- /01_version_control/figs/history_merge/fig.tex: -------------------------------------------------------------------------------- 1 | % build with: pdflatex --shell-escape fig.tex 2 | \documentclass[tikz, convert={density=800}]{standalone} 3 | 4 | \usepackage{tikz} 5 | \definecolor{myOrange}{HTML}{ffa500} 6 | \definecolor{myBlue}{HTML}{42affa} 7 | 8 | \begin{document} 9 | \begin{tikzpicture}[>=latex, thick, shorten >=2pt, shorten <=2pt, ->, myBlue] 10 | \tikzstyle{C}=[circle,fill=myBlue,text=white,minimum size=30pt,inner sep=2pt, font=\LARGE] 11 | \tikzstyle{M}=[circle,fill=myOrange,text=white,minimum size=30pt,inner sep=2pt, font=\LARGE] 12 | \tikzstyle{B}=[text=myBlue, font=\LARGE] 13 | 14 | \node[C] at (0,3) (A) {\texttt{A}}; 15 | \node[C] at (3,3) (B) {\texttt{B}}; 16 | \node[C] at (6,3) (C) {\texttt{C}}; 17 | 18 | \node[C] at (3,1) (D) {\texttt{D}}; 19 | \node[C] at (6,1) (E) {\texttt{E}}; 20 | 21 | \node[M] at (9,3) (M) {\texttt{M}}; 22 | 23 | \draw [<-] (A) to (B); 24 | \draw [<-] (B) to (C); 25 | \draw [<-] (A) to (D); 26 | \draw [<-] (D) to (E); 27 | \draw [<-, myOrange] (C) to (M); 28 | \draw [<-, myOrange] (E) to (M); 29 | 30 | \node[B] at (-3,3) {\texttt{main}}; 31 | \node[B] at (-3,1) {\texttt{feature}}; 32 | 33 | \end{tikzpicture} 34 | \end{document} 35 | -------------------------------------------------------------------------------- /01_version_control/figs/history_rebase/fig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Simulation-Software-Engineering/Lecture-Material/d58a42f0ddd51ed992519d7fc067d7866c8bb3fa/01_version_control/figs/history_rebase/fig.png -------------------------------------------------------------------------------- /01_version_control/figs/history_rebase/fig.tex: -------------------------------------------------------------------------------- 1 | % build with: pdflatex --shell-escape fig.tex 2 | \documentclass[tikz, convert={density=800}]{standalone} 3 | 4 | \usepackage{tikz} 5 | \definecolor{myOrange}{HTML}{ffa500} 6 | \definecolor{myBlue}{HTML}{42affa} 7 | 8 | \begin{document} 9 | \begin{tikzpicture}[>=latex, thick, shorten >=2pt, shorten <=2pt, ->, myBlue] 10 | \tikzstyle{C}=[circle,fill=myBlue,text=white,minimum size=30pt,inner sep=2pt, font=\LARGE] 11 | \tikzstyle{M}=[circle,fill=myOrange,text=white,minimum size=30pt,inner sep=2pt, font=\LARGE] 12 | \tikzstyle{B}=[text=myBlue, font=\LARGE] 13 | 14 | \node[C] at (0,3) (A) {\texttt{A}}; 15 | \node[C] at (3,3) (B) {\texttt{B}}; 16 | \node[C] at (6,3) (C) {\texttt{C}}; 17 | 18 | \node[C] at (9,1) (D) {\texttt{D'}}; 19 | \node[C] at (12,1) (E) {\texttt{E'}}; 20 | 21 | \draw [<-] (A) to (B); 22 | \draw [<-] (B) to (C); 23 | \draw [<-] (C) to (D); 24 | \draw [<-] (D) to (E); 25 | 26 | \node[B] at (-3,3) {\texttt{main}}; 27 | \node[B] at (-3,1) {\texttt{feature}}; 28 | 29 | \end{tikzpicture} 30 | \end{document} 31 | -------------------------------------------------------------------------------- /01_version_control/git_exercise.md: -------------------------------------------------------------------------------- 1 | # Exercise: Git Workflows 2 | 3 | Deadline: **Wednesday, October 30, 2024, 9:00** 4 | 5 | ## Preparation 6 | 7 | - Work on the exercises in groups of two. One of you creates a new organization on GitHub (public, free plan) and adds their partner. 8 | - Import the [exercise repository](https://github.com/Simulation-Software-Engineering/git-exercise) and name it `git-exercise` again. Make it public to get access to all features we need. We refer to this repository as the *upstream* repository. 9 | - Add a branch protection rule for the `main` branch of the repository: 10 | - Require a pull request before merging 11 | - Require linear history 12 | - Finally, everybody forks the repository into their own namespace. 13 | 14 | ## Idea of the Exercise 15 | 16 | - There are three separate exercises. Each one consists of an individual Python file `exerciseX.py`. The exercises build upon each other, work from one to three. Each exercise is described in the comments at the top of the respective Python file. 17 | - Each exercise has two tasks: A and B. Distribute the two tasks and **work on them in parallel**. 18 | - For each task, create a new branch with an appropriate name (e.g., `ex-1-task-A`) and use appropriate commit messages (not `Solve task A`, but `Fix missing copy in Vector construction` or similar). 19 | - When ready, push the new branch via `git push -u ` to your own remote and open a pull request to the `main` branch upstream (i.e. use the feature branch workflow, but with feature branches on forks). 20 | - Assign your partner as reviewer. Review, fix problems, and eventually merge. For the best learning experience, use a different order of steps in every exercise. 21 | - **Goal of the exercises**: Create a linear history with readable commit messages. 22 | 23 | ## Pytest and Python 24 | 25 | - In each task, you typically need to make one of already existing tests pass. We use `pytest` here (you will learn more about `pytest` after Christmas): 26 | - Install from [PyPI](https://pypi.org/project/pytest/): `pip install --user pytest`. 27 | - Run all tests of a file with `python3 -m pytest exerciseX.py`. 28 | - Run a specific test with `python3 -m pytest exerciseX.py -k TESTNAME`. 29 | - Ask if you are stuck with Python issues (quick search on Google often helps as well). This exercise is about Git, not Python. 30 | 31 | ## Submission 32 | 33 | Please open an issue (one per group) in the [exercise repository](https://github.com/Simulation-Software-Engineering/git-exercise) with the title `Submission ` and post a link to your upstream repository. 34 | 35 | ## Credits 36 | 37 | These exercises are taken from the [RSE 102 block course](https://github.com/RSE-102), created by [Dennis Gläser](https://github.com/dglaeser). 38 | -------------------------------------------------------------------------------- /01_version_control/git_quiz.md: -------------------------------------------------------------------------------- 1 | # Git Quiz 2 | 3 | Questions are taken from [py-rse](https://third-bit.com/py-rse/index.html), [learn.co](https://learn.co/lessons/git-github-learn-quiz), [w3schools](https://www.w3schools.com/quiztest/quiztest.asp?qtest=GIT), [this gist from @aspencer8111](https://gist.github.com/aspencer8111/17a80fb0a2be7b4718237fe8caa6e09c), and [codercrunch](https://www.codercrunch.com/quiz/take/1650218502/git-branching). 4 | 5 | ## Add & Commit I 6 | 7 | Which command(s) below would save changes to myfile.txt to a local (already existing) Git repository? (multiple choice) 8 | 9 | - `git commit -m "Add recent changes"` 10 | - `git init myfile.txt` && `git commit -m "Add recent changes"` 11 | - `git add myfile.txt` && `git commit -m "Add recent changes"` 12 | - `git commit myfile.txt -m "Add recent changes"` 13 | 14 | ## Add & Commit II 15 | 16 | Assume you made the following changes. What is the output of the last command in the sequence below? 17 | 18 | ```bash 19 | touch motivation.txt 20 | echo "Sharing information about myself." > motivation.txt 21 | git add motivation.txt 22 | echo "Documenting major milestones." > motivation.txt 23 | git commit -m "Motivate project" 24 | git restore motivation.txt 25 | cat motivation.txt 26 | ``` 27 | 28 | - `Sharing information about myself.` 29 | - `Documenting major milestones.` 30 | - The following: 31 | 32 | ``` 33 | Sharing information about myself. 34 | Documenting major milestones. 35 | ``` 36 | 37 | - An error message because we have changed motivation.txt without committing first. 38 | 39 | ## Create Branch I 40 | 41 | What is the command to create a new branch named `add-feature-x`? 42 | 43 | - `git add branch add-feature-x` 44 | - `git branch new add-feature-x` 45 | - `git branch add add-feature-x` 46 | - `git branch add-feature-x` 47 | 48 | ## Create Branch II 49 | 50 | What is the difference between `git branch add-feature-x` and `git checkout -b add-feature-x`? 51 | 52 | - They do the same thing. 53 | - The second command creates a copy of a remote branch while the first command branches from the current local branch. 54 | - They both create a new branch "add-feature-x". However, the first command stays at the current branch while the second directly changes to the new branch. 55 | - The first command creates a new branch. The second command changes to "add-feature-x" if the branch already exists. 56 | 57 | ## Merge Branches 58 | 59 | How can you merge `add-feature-x` into `main`? 60 | 61 | - `git merge add-feature-x main` 62 | - `git merge main add-feature-x` 63 | - `git checkout main` && `git merge add-feature-x` 64 | - `git checkout add-feature-x` && `git merge main` 65 | 66 | ## Delete Branches 67 | 68 | What is the command to delete the branch "add-feature-x"? 69 | 70 | - `git delete add-feature-x` 71 | - `git rm add-feature-x` 72 | - `git delete branch add-feature-x` 73 | - `git branch -d add-feature-x` 74 | 75 | ## Add Remote 76 | 77 | What is the command to add the remote repository `https://ab.c/d/e.git` as "origin"? 78 | 79 | - `git remote add origin https://ab.c/d/e.git` 80 | - `git origin=https://ab.c/d/e.git` 81 | - `git add origin https://ab.c/d/e.git` 82 | - `git remote https://ab.c/d/e.git` 83 | 84 | ## Pull 85 | 86 | `git pull` is a combination of: 87 | 88 | - `git fetch` and `git merge` 89 | - `git fetch` and `git rebase` 90 | - `git fetch` and `git update` 91 | - `git fetch` and `git checkout` 92 | 93 | ## Undo I 94 | 95 | To unstage a currently staged file called `octocat.txt` without losing the changes, the git command is? (multiple choice) 96 | 97 | - `git reset --soft octocat.txt` 98 | - `git reset octocat.txt` 99 | - `git checkout HEAD octocat.txt` 100 | - `git restore --staged octocat.txt` 101 | 102 | ## Undo II 103 | 104 | You made changes to a shell script called `data_cruncher.sh`, which broke it. Which of the git commands recovers the last committed version of the script? (multiple choice) 105 | 106 | - `git checkout HEAD` 107 | - `git checkout HEAD data_cruncher.sh` 108 | - `git checkout HEAD~1 data_cruncher.sh` 109 | - `git checkout data_cruncher.sh` 110 | - `git restore data_cruncher.sh` 111 | - `git restore HEAD` 112 | 113 | ## Undo III 114 | 115 | How can you create a commit that undoes the changes of some other commit? 116 | 117 | - `git undo` 118 | - `git reset` 119 | - `git checkout` 120 | - `git revert` 121 | 122 | ## Clean 123 | 124 | What does `git clean` do? 125 | 126 | - Remove all untracked files except ignored ones 127 | - Remove all ignored files 128 | - Unstage everything 129 | - Delete all commits that are not pushed yet 130 | 131 | ## Detached HEAD 132 | 133 | What can cause you to enter a "detached HEAD" state? 134 | 135 | - `git clean` 136 | - Checking out a commit that doesn't have a branch pointing to it 137 | - Cloning a bare repository 138 | - Finishing a merge 139 | 140 | ## Delete Remote Branch 141 | 142 | How can you delete a branch in a remote repository? 143 | 144 | - `git push delete branchName` 145 | - `git push origin --delete branchName` 146 | - `git rm --push branchName` 147 | - You can only do this through the web interface of the remote (e.g. GitHub) 148 | -------------------------------------------------------------------------------- /01_version_control/intro_slides.md: -------------------------------------------------------------------------------- 1 | --- 2 | type: slide 3 | slideOptions: 4 | transition: slide 5 | width: 1400 6 | height: 900 7 | margin: 0.1 8 | --- 9 | 10 | 25 | 26 | # Introduction to version control 27 | 28 | --- 29 | 30 | ## Learning goals of chapter 31 | 32 | - Refresh and organize students' existing knowledge on Git (learn how to learn more). 33 | - Students can explain difference between merge and rebase and when to use what. 34 | - How to use Git workflows to organize research software development in a team. 35 | - Get to know a few useful GitHub/GitLab standards and a few helpful tools. 36 | 37 | --- 38 | 39 | ## Why do we need version control? 40 | 41 | Version control ... 42 | 43 | - tracks changes to files and helps people share those changes with each other. 44 | - Could also be done via email / Google Docs / ..., but not as accurately and efficiently 45 | - was originally developed for software development, but today cornerstone of *reproducible research* 46 | 47 | > "If you can't git diff a file format, it's broken." 48 | 49 | --- 50 | 51 | ## How does version control work? 52 | 53 | - *master* (or *main*) copy of code in repository, can't edit directly 54 | - Instead: check out a working copy of code, edit, commit changes back 55 | - Repository records complete revision history 56 | - You can go back in time 57 | - It's clear who did what when 58 | 59 | --- 60 | 61 | ## The alternative: A story told in file names 62 | 63 | 64 | 65 | [http://phdcomics.com/comics/archive/phd052810s.gif](http://phdcomics.com/comics/archive/phd052810s.gif) 66 | 67 | --- 68 | 69 | ## A very short history of version control I 70 | 71 | The old centralized variants: 72 | 73 | - 1982: RCS (Revision Control System), operates on single files 74 | - 1986 (release in 1990): CVS (Concurrent Versions System), front end of RCS, operates on whole projects 75 | - 1994: VSS (Microsoft Visual SourceSafe) 76 | - 2000: SVN (Apache Subversion), mostly compatible successor of CVS, *still used today* 77 | 78 | --- 79 | 80 | ## A very short history of version control II 81 | 82 | Distributed version control: 83 | 84 | - Besides remote master version, also local copy of repository 85 | - More memory required, but much better performance 86 | - For a long time: highly fragmented market 87 | - 2000: BitKeeper (originally proprietary software) 88 | - 2005: Mercurial 89 | - 2005: Git 90 | - A few more 91 | 92 | Learn more: [Podcast All Things Git: History of VC](https://www.allthingsgit.com/episodes/the_history_of_vc_with_eric_sink.html) 93 | 94 | --- 95 | 96 | ## The only standard today: Git 97 | 98 | No longer a fragmented market, there is nearly only Git today: 99 | 100 | - [Stackoverflow developer survey 2021](https://insights.stackoverflow.com/survey/2021#technology-most-popular-technologies): 101 | > "Over 90% of respondents use Git, suggesting that it is a fundamental tool to being a developer." 102 | - All software project candidates for *contribution challenge* use Git 103 | - Is this good or bad? 104 | 105 | --- 106 | 107 | ## More facts on Git 108 | 109 | Git itself is open-source: GPL license 110 | 111 | - source on [GitHub](https://github.com/git/git), contributions are a bit more complicated than a simple PR 112 | - written mainly in C 113 | - started by Linus Torvalds, core maintainer since later 2005: Junio Hamano 114 | -------------------------------------------------------------------------------- /01_version_control/standards_slides.md: -------------------------------------------------------------------------------- 1 | --- 2 | type: slide 3 | slideOptions: 4 | transition: slide 5 | width: 1400 6 | height: 900 7 | margin: 0.1 8 | --- 9 | 10 | 34 | 35 | # GitHub / GitLab Standards 36 | 37 | --- 38 | 39 | ## What do we mean with Standards? 40 | 41 | - GitHub uses standards or conventions. 42 | - Certain files or names trigger certain behavior automatically. 43 | - Many are supported by most forges. 44 | - **This is good.** 45 | - Everybody should know them. 46 | 47 | --- 48 | 49 | ## Special Files 50 | 51 | Certain files lead to special formatting (normally directly at root of repo): 52 | 53 | - `README.md` 54 | - ... contains meta information / overview / first steps of software. 55 | - ... gets rendered on landing page (and in every folder). 56 | - `LICENSE` 57 | - ... contains software license. 58 | - ... gets rendered on right sidebar, when clicking on license, and on repo preview. 59 | - `CONTRIBUTING.md` 60 | - ... contains guidelines for contributing. 61 | - First-time contributors see banner. 62 | - `CODE_OF_CONDUCT.md` 63 | - ... contains code of conduct. 64 | - ... gets rendered on right sidebar. 65 | 66 | --- 67 | 68 | ## Issues and PRs 69 | 70 | - Templates for description in `.github` folder 71 | - `closes #34` (or several other keywords: `fixes`, `resolves`) in commit message or PR description will close issue 34 when merged. 72 | - `help wanted` label gets rendered on repo preview (e.g. *"3 issues need help"*). 73 | -------------------------------------------------------------------------------- /01_version_control/workflow_slides.md: -------------------------------------------------------------------------------- 1 | --- 2 | type: slide 3 | slideOptions: 4 | transition: slide 5 | width: 1400 6 | height: 900 7 | margin: 0.1 8 | --- 9 | 10 | 29 | 30 | # Working in Teams / Git Workflows 31 | 32 | --- 33 | 34 | ## Why Workflows? 35 | 36 | - Git offers a lot of flexibility in managing changes. 37 | - When working in a team, some agreements need to be made however (especially on how to work with branches). 38 | 39 | --- 40 | 41 | ## Which Workflow? 42 | 43 | - There are standard solutions. 44 | - It depends on the size of the team. 45 | - Workflow should enhance effectiveness of team, not be a burden that limits productivity. 46 | 47 | --- 48 | 49 | ## Centralized Workflow 50 | 51 | - Only one branch: the `main` branch 52 | - Keep your changes in local commits till some feature is ready 53 | - If ready, directly push to `main`; no PRs, no reviews 54 | - Conflicts: fix locally (push not allowed anyway), use `git pull --rebase` 55 | - **Good for**: small teams, small projects, projects that are anyway reviewed over and over again 56 | - Example: LaTeX papers 57 | - Put each section in separate file 58 | - Put each sentence in separate line 59 | 60 | --- 61 | 62 | ## Feature Branch Workflow 63 | 64 | - Each feature (or bugfix) in separate branch 65 | - Push feature branch to remote, use descriptive name 66 | - e.g. issue number in name if each branch closes one issue 67 | - `main` should never contain broken code 68 | - Protect direct push to `main` 69 | - PR (or MR) with review to merge from feature branch to `main` 70 | - Rebase feature branch on `main` if necessary 71 | - Delete remote branch once merged and no longer needed (one click on GitHub after merge) 72 | - **Good for**: small teams, small projects, prototyping, websites (continuous deployment), documentation 73 | - Aka. [trunk-based development](https://www.atlassian.com/continuous-delivery/continuous-integration/trunk-based-development) or [GitHub flow](https://guides.github.com/introduction/flow/) 74 | 75 | --- 76 | 77 | ## Gitflow 78 | 79 | - [Visualization by Vincent Driessen](https://nvie.com/img/git-model@2x.png), from [original blog post in 2010](https://nvie.com/posts/a-successful-git-branching-model/) 80 | - `main` and `develop` 81 | - `main` contains releases as tags 82 | - `develop` contains latest features 83 | - Feature branches created of `develop`, PRs back to `develop` 84 | - Protect `main` and (possibly) `develop` from direct pushes 85 | - Dedicated release branches (e.g., `v1.0`) created of `develop` 86 | - Tested, fixed, merged to `main` 87 | - Afterwards, tagged, merged back to `develop` 88 | - Hotfix branches directly of and to `main` 89 | - **Good for**: software with users, larger teams 90 | - There is a tool `git-flow`, a wrapper around `git`, e.g. `git flow init` ... but not really necessary IMHO 91 | 92 | --- 93 | 94 | ## Forking Workflow 95 | 96 | - Gitflow + feature branches on other forks 97 | - More control over access rights, distinguish between maintainers and external contributors 98 | - Should maintainers also use branches on their forks? 99 | - Makes overview of branches easier 100 | - Distinguishes between prototype branches (on fork, no PR), serious enhancements (on fork with PR), joint enhancements (on upstream) 101 | - **Good for**: open-source projects with external contributions (used more or less in preCICE) 102 | 103 | --- 104 | 105 | ## Do Small PRs 106 | 107 | - For all workflows, it is better to do small PRs 108 | - Easier to review 109 | - Faster to merge --> fewer conflicts 110 | - Easier to squash 111 | 112 | --- 113 | 114 | ## Quick reads 115 | 116 | - [Atlassian docs on workflows](https://www.atlassian.com/git/tutorials/comparing-workflows) 117 | - [Original gitflow blog post](https://nvie.com/posts/a-successful-git-branching-model/) 118 | - [Trunk-based development](https://www.atlassian.com/continuous-delivery/continuous-integration/trunk-based-development) 119 | - [GitHub flow](https://guides.github.com/introduction/flow/) 120 | - [How to keep pull requests manageable](https://gist.github.com/sktse/569cb192ce1518f83db58567591e3205) 121 | -------------------------------------------------------------------------------- /02_virtualization_and_containers/README.md: -------------------------------------------------------------------------------- 1 | # Virtualization and Containers 2 | 3 | Learning goals: 4 | 5 | - Name differences between virtualization and containers and name use cases for each. 6 | - Create and modify virtual machines with VirtualBox and generate them with Vagrant. 7 | - Create and manage Docker containers. 8 | - Name containerization technologies beyond Docker and name their main differences. 9 | 10 | | Duration | Content | 11 | | --- | --- | 12 | | 10 minutes | [`containers_slides.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/02_virtualization_and_containers/containers_slides.md), [`containers_demo.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/02_virtualization_and_containers/containers_demo.md) | 13 | | 65 minutes | [`docker_slides.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/02_virtualization_and_containers/docker_slides.md), [`docker_demo.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/02_virtualization_and_containers/docker_demo.md) | 14 | | 5 minutes | [`intro_slides.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/02_virtualization_and_containers/intro_slides.md) | 15 | | 20 minutes | [`singularity_slides.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/02_virtualization_and_containers/singularity_slides.md), [`singularity_demo.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/02_virtualization_and_containers/singularity_demo.md) | 16 | | 30 minutes | [`vagrant_slides.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/02_virtualization_and_containers/vagrant_slides.md), [`vagrant_demo.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/02_virtualization_and_containers/vagrant_demo.md) | 17 | | 40 minutes | [`virtualbox_slides.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/02_virtualization_and_containers/virtualbox_slides.md), [`virtualbox_demo.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/02_virtualization_and_containers/virtualbox_demo.md) | 18 | | 5 minutes | [`virtualmachines_slides.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/02_virtualization_and_containers/virtualmachines_slides.md) | 19 | | 90 minutes | [`virtualmachines_containers_exercise.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/02_virtualization_and_containers/virtualmachines_containers_exercise.md) | 20 | -------------------------------------------------------------------------------- /02_virtualization_and_containers/containers_slides.md: -------------------------------------------------------------------------------- 1 | --- 2 | type: slide 3 | slideOptions: 4 | transition: slide 5 | width: 1400 6 | height: 900 7 | margin: 0.1 8 | --- 9 | 10 | 25 | 26 | # Containers 27 | 28 | --- 29 | 30 | ## What is a Container? 31 | 32 | 33 | 34 | - Container operates in "fenced off" part of the operating system (`namespaces`) 35 | - Lower overhead than virtual machines 36 | - Runs on kernel (and libraries) of the host OS 37 | - Cheap to start and stop a container 38 | - Available features depend on Host (Linux, Windows) 39 | - Container can be isolated. 40 | 41 | --- 42 | 43 | ## Common Use-Cases 44 | 45 | - Microservices 46 | - Reproducible environments for developing and testing (DevOps) 47 | - Container hype strongly driven by [Docker](https://www.docker.com/). 48 | - More and more in science 49 | - High-performance computing, "Bring Your Own Environment" 50 | - Reproducible research 51 | 52 | --- 53 | 54 | ## Container Solutions 55 | 56 | - Plenty of different container formats 57 | - [lxc/lxd](https://linuxcontainers.org/), [Docker](https://www.docker.com/), [Apptainer](https://apptainer.org/), [podman](https://podman.io/), [Sarus](https://user.cscs.ch/tools/containers/sarus/)... 58 | - Different solutions with different strengths due to different use cases 59 | - Working on the (Super-)Userspace 60 | - Direct access to hardware vs. encapsulation 61 | - Generic or with integration in software ecosystem (e.g. job schedulers) 62 | - Effort to establish certain standards 63 | - [Open Container Initiative (OCI)](https://opencontainers.org/) 64 | 65 | --- 66 | 67 | ## Security 68 | 69 | - Containers are isolated 70 | 71 | > Applications are safer in containers and Docker provides the strongest default isolation capabilities in the industry 72 | 73 | - Read the manual 74 | - Are third-part containers trust-worthy? 75 | 76 | --- 77 | 78 | ## Summary 79 | 80 | - Shares many similarities with VMs, but 81 | - Lightweight alternative to VMs 82 | - Stricter limitations than VMs 83 | - Often different use cases and working together 84 | - Many different containers solutions 85 | - Standardization effort 86 | - Choose right solution for your use case 87 | 88 | --- 89 | 90 | ## Further Reading 91 | 92 | - [Docker](https://www.docker.com/) 93 | - [Docker Hub](https://hub.docker.com/) 94 | - [Apptainer](https://apptainer.org/) 95 | - [Sarus](https://user.cscs.ch/tools/containers/sarus/) 96 | - [lxc/lxd](https://linuxcontainers.org/) 97 | - [podman](https://podman.io/) 98 | - [Open Container Initiative (OCI)](https://opencontainers.org/) 99 | - [Singularity: Scientific containers for mobility of compute](https://doi.org/10.1371/journal.pone.0177459) 100 | - [Malicious Docker Hub Container Images Used for Cryptocurrency Mining](https://www.trendmicro.com/vinfo/fr/security/news/virtualization-and-cloud/malicious-docker-hub-container-images-cryptocurrency-mining) 101 | -------------------------------------------------------------------------------- /02_virtualization_and_containers/docker_demo.md: -------------------------------------------------------------------------------- 1 | # Docker Demo 2 | 3 | - Show containers on [DockerHub](https://hub.docker.com/) 4 | 5 | ## Some Management Commands 6 | 7 | - Show running containers `docker container ls` 8 | - Show all containers `docker container ls -a` 9 | - Show images `docker images` 10 | - Potentially remove some image/container `docker image rm NAME` or `docker container rm NAME/ID` 11 | 12 | ## Tutorial Case 13 | 14 | - From tutorial `docker run -i -t ubuntu /bin/bash` 15 | - This pulls the latest `ubuntu` image `docker pull ubuntu` 16 | - Creates container `docker container create` 17 | - Creates read-write filesystem (last layer) 18 | - Creates network interface 19 | - Starts container and runs `/bin/bash` 20 | - `-i` means interactive 21 | - `-t` allocates pseudo-tty 22 | - Note that the container will still be there `docker container list -a` vs. `docker container list -a` 23 | - We can make sure that the container is removed after exiting by the `--rm` options, i.e., `docker run --rm -i -t ubuntu /bin/bash` 24 | 25 | - When container is running, we see it when calling `docker ps` 26 | - Start container (with name `tutoral`) `docker run --rm -i -t --name tutorial ubuntu /bin/bash` 27 | - Leave it `CTRL-P-Q` (all keys pressed at the same time) 28 | - Show container running `docker ps` 29 | - Reattach to container `docker container attach tutorial` 30 | - After quitting again, show `docker ps -a` 31 | 32 | ## Restarting a stopped container 33 | 34 | ```shell 35 | docker ps -a 36 | docker container start tutorial 37 | docker container attach tutorial 38 | ``` 39 | 40 | ## Files in containers 41 | 42 | - We can change files inside the container. 43 | - `docker run -i -t ubuntu /bin/bash` 44 | - `touch asdf` 45 | - leave container 46 | - enter container `docker run -i -t ubuntu /bin/bash` 47 | - File is not present because we implicitly created a new container based on the same image. 48 | 49 | ## Bind Mount 50 | 51 | - `docker run -i -t --mount type=bind,source="$(pwd)",target=/mnt/share ubuntu` 52 | - Create detached container and bind mount 53 | - Mounts current directory on Host to `/mnt/share`. 54 | - Bind mount your source code for development for example 55 | - I do not need `/bin/bash` because that is the default command for the `ubuntu` image. 56 | 57 | ## Demo: Building own example 58 | 59 | - `cd dockerfile-example` 60 | - Contains Dockerfile 61 | 62 | ```Dockerfile 63 | FROM ubuntu:24.04 64 | 65 | RUN apt-get update -y && apt install -y neofetch 66 | WORKDIR /app 67 | COPY testfile . 68 | CMD ["neofetch"] 69 | ``` 70 | 71 | - `docker buildx build --tag testimage .` 72 | - `docker run -i -t testimage /bin/bash` 73 | - `docker run testimage` will run container and `CMD` will be executed 74 | - Create file `touch testfile`, if not present. 75 | - When going into the container we are in the directory `/app` and the file `testfile` is present. 76 | - Copy files with `docker cp`. `touch file-to-copy` 77 | - `docker cp file-to-copy CONTAINERNAME:/app` 78 | - `docker cp CONTAINERNAME:/app file-to-copy` 79 | - This will preserve user and group id 80 | - `docker run -i -t -v $(pwd):/app testimage /bin/bash` starts container, creates volume `/app` and sets working directory to /app 81 | 82 | -------------------------------------------------------------------------------- /02_virtualization_and_containers/figs/container-sketch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Simulation-Software-Engineering/Lecture-Material/d58a42f0ddd51ed992519d7fc067d7866c8bb3fa/02_virtualization_and_containers/figs/container-sketch.png -------------------------------------------------------------------------------- /02_virtualization_and_containers/figs/virtualmachine-sketch.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Simulation-Software-Engineering/Lecture-Material/d58a42f0ddd51ed992519d7fc067d7866c8bb3fa/02_virtualization_and_containers/figs/virtualmachine-sketch.png -------------------------------------------------------------------------------- /02_virtualization_and_containers/intro_slides.md: -------------------------------------------------------------------------------- 1 | --- 2 | type: slide 3 | slideOptions: 4 | transition: slide 5 | width: 1400 6 | height: 900 7 | margin: 0.1 8 | --- 9 | 10 | 25 | 26 | # Virtualization and Containers 27 | 28 | --- 29 | 30 | ## Learning Goals 31 | 32 | - Name differences between virtualization and containers and name use cases for each. 33 | - Create and modify virtual machines with VirtualBox and generate them with Vagrant. 34 | - Create and manage Docker containers. 35 | - Name containerization technologies beyond Docker and name their main differences. 36 | 37 | --- 38 | 39 | ## Virtualization and Containers for SSE 40 | 41 | - Developing, testing and debugging software 42 | - Abstraction layer for different OS, configurations etc. 43 | - Moving/Sharing software 44 | - Avoid compilation and/or setup complexity, OS dependency 45 | - Manage complex workflows, ideally under version control 46 | - Reproducible research 47 | - Make sure others are able to run your software (also in future). 48 | - Make sure others get the same results. 49 | 50 | --- 51 | 52 | ## Contents of this Chapter 53 | 54 | - Virtual Machines (VMs) 55 | - VirtualBox: Popular hypervisor for running VMs 56 | - Vagrant: Popular management system for VMs (and more) 57 | - Containers 58 | - Docker: Popular container framework (and more) 59 | - Singularity: Popular scientific container framework 60 | -------------------------------------------------------------------------------- /02_virtualization_and_containers/singularity_demo.md: -------------------------------------------------------------------------------- 1 | # Singularity Demo 2 | 3 | ## Demo: Run prebuilt containers 4 | 5 | - `apptainer pull docker://precice/precice` 6 | - Obtain existing Docker image, convert it, and store it as `precice_latest.sif` 7 | - `apptainer shell precice_latest.sif` 8 | - Run shell in container 9 | - `whoami` will show same user as I am on the computer 10 | - `apptainer exec precice_latest.sif precice-tools --version` 11 | - Run command in container 12 | - Show the mounted filesystems and hardware access. 13 | - Show that we cannot run things as root. 14 | - Note that images are executable by default: `./precice-latest.sif` 15 | 16 | ## Demo: Build own containers 17 | 18 | - Recipe `lolcow.def` 19 | 20 | ```Singularity 21 | BootStrap: docker 22 | From: ubuntu:22.04 23 | 24 | %post 25 | apt-get -y update 26 | apt-get -y install cowsay lolcat 27 | 28 | %environment 29 | export LC_ALL=C 30 | export PATH=/usr/games:$PATH 31 | 32 | %runscript 33 | date | cowsay | lolcat 34 | ``` 35 | 36 | - Show file `lolcow.def` content 37 | - `apptainer build testimage lolcow.def` 38 | - Creates image which is identical to the prebuilt image 39 | -------------------------------------------------------------------------------- /02_virtualization_and_containers/vagrant_demo.md: -------------------------------------------------------------------------------- 1 | # Vagrant Demo 2 | 3 | ## Demo: Create VM based on a base Ubuntu box 4 | 5 | - Create a working directory and enter it. 6 | - `vagrant init bento/ubuntu-24.04` creates a base `Vagrantfile`. 7 | - `vagrant up` Creates `Vagrantfile`. This file can be put in version control. 8 | - Download and set up VM. 9 | - Will also start VM 10 | - Open VirtualBox to see a VM running. We can open the box there, but are prompted for login username and password. 11 | - Note: The actual "box" is put under your home directory 12 | - `vagrant ssh` Connect to running VM 13 | - `vagrant destroy` Will destroy (delete) current VM 14 | - Alternatives: 15 | - `vagrant suspend` Suspends VM 16 | - `vagrant halt` Shuts VM down, keeping changes in the VM 17 | - `vagrant status` Shows status of currently running VMs 18 | - Exchanging files 19 | - Check output `default: Mounting shared folders...` 20 | - Default: Directory containing `Vagrantfile` is mounted as `/vagrant` 21 | - Can configure more drives 22 | - Show example by creating file from Vagrant session 23 | 24 | ## Demo: Start VM from own box 25 | 26 | - Open `Vagrantfile` and refer to name of VM 27 | 28 | ```ruby 29 | Vagrant.configure("2") do |config| 30 | config.vm.box = "bento/ubuntu-24.04" 31 | config.vm.provision :shell, path: "bootstrap.sh" 32 | 33 | config.vm.provider "virtualbox" do |vb| 34 | vb.name = "sse-first-step" 35 | end 36 | end 37 | ``` 38 | 39 | - Open `bootstrap.sh` to show that it will do 40 | 41 | ```bash 42 | !/usr/bin/env bash 43 | 44 | apt-get update 45 | apt-get install -y neofetch vim 46 | echo "export TEST_ENV_VAR=\"Hello SSE\"" >> .bashrc 47 | ``` 48 | 49 | - Installs software and sets `TEST_ENV_VAR` 50 | - `vagrant up`: Create VM 51 | - Image should be prebuilt 52 | - `vagrant ssh` 53 | - Call `neofetch` 54 | - `echo $TEST_ENV_VAR` to show that variable is actually set. 55 | - `vagrant package --base "sse-first-step" --output sse-first-step.box`: Export VM 56 | - `sse-first-steps.box` can be uploaded to [Vagrant Cloud](https://app.vagrantup.com) 57 | - Go to directory `using-own-box/` 58 | - Show `Vagrantfile` that is has my uploaded image as base image 59 | - Skip building box `vagrant up` as it takes too long, should be prebuilt 60 | - After `vagrant box add --name `, `vagrant box list` should show my box in overview. 61 | 62 | ## Demo: preCICE VM 63 | 64 | - Show repository: 65 | - [https://github.com/precice/vm](https://github.com/precice/vm) 66 | - Run [Premade box](https://app.vagrantup.com/precice/boxes/precice-vm) 67 | - Create a dedicated folder `mkdir precice` 68 | - Create box with `vagrant init` (Done before lecture as it takes to long) 69 | - Start box `vagrant up` 70 | - Comes with a GUI 71 | - Preconfigured 72 | - Destroy in the end `vagrant destroy` since it is large (?) 73 | - If time allows, show preCICE example 74 | - [Tutorial](https://precice.org/installation-vm.html#how-to-use-this) 75 | - [Quickstart](https://precice.org/quickstart.html) 76 | - Open two terminals 77 | - Go to `/home/vagrant/Desktop/tutorials/quickstart` 78 | - In one terminal go to `cpp-solid` and build solver `cmake . && make` 79 | - Afterwards start simulation with `./run.sh` 80 | - In other terminal go to `fluid-openfoam` 81 | - Afterwards start simulation with `./run.sh` 82 | - Watch simulation running 83 | -------------------------------------------------------------------------------- /02_virtualization_and_containers/virtualbox_demo.md: -------------------------------------------------------------------------------- 1 | # VirtualBox demo 2 | 3 | - Discuss configuration: 4 | - Virtual machine will reserve cores/threads of your CPU and main memory. 5 | - Maybe go through installation process to show that it is the same as installing an OS on your normal computer. 6 | - Create a new virtual machine and boot it up (nothing will happen). 7 | - Boot up some configured virtual machine (Ubuntu). 8 | - Creating a new virtual machine 9 | - Click on `new`. 10 | - Use `expert mode` to set disk location, size and memory. Note, that one can change that also later on. 11 | - After clicking on `Create` a menu to create the `Virtual Hard Disk` opens. 12 | - One can choose between fixed size and dynamically allocated (i.e. the drive grows). I personally recommend avoiding dynamically allocated without upper limit. 13 | - Creates empty machine. Will not do much as we do not have any OS installed. VirtualBox will prompt to mount an image (iso). 14 | - Show system settings of VM. 15 | - Storage -> Controller (Mount drive here) and boot again. -> Installation will start up. 16 | - Normally should unmount image after installation. 17 | - Install "VirtualBox Guest Additions". Might have to download the image. 18 | - Devices `Insert Guest Additions CD image" 19 | 20 | ```bash 21 | sudo mkdir -p /media/cdrom 22 | sudo mount /dev/cdrom /media/cdrom 23 | cd /media/cdrom 24 | sudo apt-get install -y perl dkms build-essential linux-headers-generic linux-headers-$(uname -r) 25 | sudo su 26 | ./VBoxLinuxAdditions.run 27 | ``` 28 | 29 | - Will enable clipboard sharing etc 30 | - Alternative on Ubuntu 31 | 32 | ```bash 33 | sudo apt install virtualbox-guest-dkms virtualbox-guest-x11 virtualbox-guest-utils 34 | ``` 35 | 36 | - The recommended way to install VirtualBox Guest Additions is via the VirtualBox GUI. 37 | - VM will capture mouse pointer. Use `Right-CTRL` to "free" pointer again. 38 | - OS on VM may have English keyboard layout by default which may need to be changed if Host machine as a different layout (for e.g. German). 39 | - Create snapshots on image overview (Burger symbol on the right) 40 | - Load snapshots for different configuration stages 41 | - Configure network for ssh 42 | - Install `openssh-server` in the VM. 43 | - Shutdown VM. 44 | - In VirtualBox window -> File -> Host Network Manager -> Verify that network device is configured `vboxnet0` 45 | - In virtual machine's settings -> Network -> Adapter 2 -> Enable and set Host only adapter `vboxnet0` 46 | - Boot VM. 47 | - Verify additional network device `enp0s8` and check for ip address. In my case the IP is `192.168.56.101` (seems to be standard). 48 | - On host machine `ssh VM-USERNAME@192.168.56.101`. 49 | - In case of `Too many authentication failures`, see https://serverfault.com/a/540613 50 | - Show/install extensions to allow for 51 | - ... some 3D rendering 52 | - ... shared clipboard 53 | - In general: *Better integration* into host system 54 | - Create a shared drive to exchange data instead of copy and pasting file content. 55 | - In case one cannot access it `sudo usermod -aG vboxsf $(whoami)` 56 | - You need to logout and in again afterwards for usergroups to be recognized. 57 | - I share `~/vm_shared/` 58 | - Add some file there and check that it appears Host and vice versa. 59 | -------------------------------------------------------------------------------- /02_virtualization_and_containers/virtualbox_slides.md: -------------------------------------------------------------------------------- 1 | --- 2 | type: slide 3 | slideOptions: 4 | transition: slide 5 | width: 1400 6 | height: 900 7 | margin: 0.1 8 | slideNumber: true 9 | --- 10 | 11 | 26 | 27 | # VirtualBox 28 | 29 | 30 | 31 | --- 32 | 33 | ## Introduction 34 | 35 | - Hosted hypervisor created by Innotek GmbH (Weinstadt, Germany) 36 | - Obtained by Sun Microsystems in 2008. Since 2010, owned by Oracle 37 | - Open source (GPLv3) 38 | 39 | --- 40 | 41 | ## Requirements 42 | 43 | - Linux, macOS, Windows 44 | - Reasonably new hardware with virtualization support 45 | - Check virtualization settings in your BIOS/UEFI 46 | - Sufficient space on hard drive 47 | - VM uses a virtual hard drive, i.e., a file on your drive 48 | - Some video memory 49 | - Exact requirements depend on VM: CPU cores, RAM, ... 50 | 51 | --- 52 | 53 | ## Virtual Hard Drive Formats 54 | 55 | - Virtual hard drive as a file 56 | - Multiple formats, for example: 57 | - **VDI**: "Virtual Disk Image", VirtualBox' native format 58 | - **VHD**: Format used by Microsoft 59 | - **VMDK**: VMWare's virtual disk format 60 | - Support for dynamic allocation 61 | 62 | --- 63 | 64 | ## Content of a VirtualBox Image Directory 65 | 66 | - `VM.vdi`: The virtual hard drive containing the Guest OS 67 | - `VM.vbox`: XML containing metadata and configuration information (RAM, network devices...) 68 | - `VM.vbox-prev`: Backup of previous settings 69 | - `Logs/`: Directory containing log files 70 | - `Snapshots/`: Snapshots of image 71 | 72 | --- 73 | 74 | ## Guest Additions 75 | 76 | - Software and drivers to improves Guest's performance 77 | - Better video support, shared clipboard, ... 78 | - Might need additional packages. On Ubuntu 24.04 (guest): 79 | 80 | ```bash 81 | sudo apt install virtualbox-guest-utils virtualbox-guest-x11 82 | ``` 83 | 84 | --- 85 | 86 | ## Demo 87 | 88 | Details available in [`virtualbox_demo.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/02_virtualization_and_containers/virtualbox_demo.md) 89 | 90 | --- 91 | 92 | ## Summary and Outlook 93 | 94 | - VirtualBox is an open-source system for virtual machines 95 | - Simple to set up and run 96 | - Manual setup, but command-line interface exists 97 | - Next step: Infrastructure as code 98 | - Consistency of the environment? 99 | - How to share a VM image with others? 100 | 101 | > "If you can't git diff a file format, it's broken." 102 | 103 | --- 104 | 105 | ## Further Reading 106 | 107 | - [VirtualBox Manual](https://www.virtualbox.org/manual/UserManual.html) 108 | - [VirtualBox Manual: 4. Guest Additions](https://www.virtualbox.org/manual/ch04.html) 109 | - [Overview of different disk formats](https://www.parallels.com/blogs/ras/vdi-vs-vhd-vs-vmdk/) 110 | -------------------------------------------------------------------------------- /02_virtualization_and_containers/virtualmachines_slides.md: -------------------------------------------------------------------------------- 1 | --- 2 | type: slide 3 | slideOptions: 4 | transition: slide 5 | width: 1400 6 | height: 900 7 | margin: 0.1 8 | --- 9 | 10 | 25 | 26 | # Virtual Machines 27 | 28 | --- 29 | 30 | ## What is a Virtual Machine? 31 | 32 | 33 | 34 | - A computer inside your computer, with its own OS and resources 35 | - Virtual Machines are portable and flexible 36 | 37 | --- 38 | 39 | ## Common Terms 40 | 41 | - Host operating system 42 | - The OS the hypervisor is installed on 43 | - Guest operating system 44 | - The OS running inside the virtual machine 45 | - Virtual machine (VM) 46 | - Environment the guest is running in 47 | 48 | --- 49 | 50 | ## Types of Hypervisors 51 | 52 | - **Type 1** 53 | - Runs directly on bare-metal hardware, like the host OS 54 | - Examples: [Microsoft Hyper-V](https://docs.microsoft.com/en-us/virtualization/hyper-v-on-windows/about/), [VMware ESXi](https://www.vmware.com/products/esxi-and-esx.html), [Xen](https://xenproject.org/), [KVM](https://en.wikipedia.org/wiki/Kernel-based_Virtual_Machine), ... 55 | - **Type 2** 56 | - Negotiates resources shared with a host OS 57 | - Examples: [VirtualBox](https://www.virtualbox.org/), [VMWare Workstation Player](https://www.vmware.com/products/workstation-player.html), [Parallels](https://www.parallels.com/eu/products/desktop/)... 58 | 59 | --- 60 | 61 | ## (Some) Features of Virtual Machines 62 | 63 | - Exclusive access to some of your resources 64 | - Behaves like a native installation 65 | - Isolation from host operating system 66 | - Popular for 67 | - Safety critical tasks 68 | - Development and testing 69 | - Wherever one wants a portable solution 70 | 71 | --- 72 | 73 | ## Why is Virtualization Useful? 74 | 75 | - Running multiple operating systems simultaneously 76 | - Test/develop/debug software for other OS 77 | - Easier software installations and testing 78 | - Preconfigured VMs for teaching (see [preCICE Demo VM](https://precice.org/installation-vm.html)) 79 | - Testing and disaster recovery 80 | - Create snapshots before testing, copy VMs etc. 81 | - Separate services from each other for security (see [Qubes OS](https://www.qubes-os.org/intro/)) 82 | - Reproducibility 83 | 84 | --- 85 | 86 | ## Summary 87 | 88 | - Isolated environment that emulates a computer 89 | - Different types of Hypervisors (bare-metal or running on OS) 90 | - Run additional OSes on your machine 91 | - VMs are portable, reproducible, easy to manage, and provide isolation 92 | 93 | --- 94 | 95 | ### Further Reading 96 | 97 | - [VirtualBox Manual: 1. First Steps](https://www.virtualbox.org/manual/ch01.html) 98 | - [VirtualBox Manual: 13. Security Guide](https://www.virtualbox.org/manual/ch13.html) 99 | - ["How To Make Package Managers Cry"](https://archive.fosdem.org/2018/schedule/event/how_to_make_package_managers_cry/) 100 | (Kenneth Hoste, FOSDEM 2018) 101 | -------------------------------------------------------------------------------- /03_building_and_packaging/README.md: -------------------------------------------------------------------------------- 1 | # Building and Packaging 2 | 3 | Learning goals: 4 | 5 | - Explain why software is packaged. 6 | - Create Python packages, publish on PyPI, and install with pip. 7 | - Understand the difference between static and dynamic libraries and common ways of installation on Linux. 8 | - Build C++ software and handle dependencies with Make and CMake. 9 | - Package C++ software with CPack and create Debian packages. 10 | - Create Spack packages, e.g., for high-performance computing systems with restricted access rights. 11 | 12 | | Duration | Content | 13 | | --- | --- | 14 | | 60 minutes | [cmake_slides.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/03_building_and_packaging/cmake_slides.md), [cmake_demo.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/03_building_and_packaging/cmake_demo.md) | 15 | | 90 minutes | [cmake_exercise.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/03_building_and_packaging/cmake_exercise.md) | 16 | | 20 minutes | [cmake_more_demo.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/03_building_and_packaging/cmake_more_demo.md) | 17 | | 90 minutes | [cpack_exercise.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/03_building_and_packaging/cpack_exercise.md) | 18 | | 70 minutes | [cpack_slides.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/03_building_and_packaging/cpack_slides.md), [cpack_demo.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/03_building_and_packaging/cpack_demo.md) | 19 | | 15 minutes | [intro_slides.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/03_building_and_packaging/intro_slides.md) | 20 | | 25 minutes | [linux_fundamentals_slides.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/03_building_and_packaging/linux_fundamentals_slides.md), [linux_fundamentals_demo.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/03_building_and_packaging/linux_fundamentals_demo.md) | 21 | | 15 minutes | [make_slides.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/03_building_and_packaging/make_slides.md), [make_demo.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/03_building_and_packaging/make_demo.md) | 22 | | 30 minutes | [pip_demo.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/03_building_and_packaging/pip_demo.md) | 23 | | 90 minutes | [pypi_exercise.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/03_building_and_packaging/pypi_exercise.md) | 24 | | 45 minutes | [pypi_slides.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/03_building_and_packaging/pypi_slides.md) | 25 | | 45 minutes | [spack_slides.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/03_building_and_packaging/spack_slides.md), [spack_demo.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/03_building_and_packaging/spack_demo.md) | 26 | | 45 minutes | [spack_exercise.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/03_building_and_packaging/spack_exercise.md) | 27 | -------------------------------------------------------------------------------- /03_building_and_packaging/cmake_exercise.md: -------------------------------------------------------------------------------- 1 | # Let's Fight With CMake, Docker, and Some Dependencies 2 | 3 | In this exercise, we need to fight. Not everything always works smoothly. This is the only way to really learn how to juggle with dependencies using CMake. We build a program that uses common, representative dependencies as often used in simulation software. We are not so much interested in the program itself, but in what steps are necessary to build the program. 4 | 5 | To get an independent and reproducible environment as common ground, we use and, thus repeat, Docker. 6 | 7 | Deadline: **Wednesday, November 27th, 2024, 9:00** 8 | 9 | ## Overview 10 | 11 | - The goal of the exercise is to open a pull request from a fork of [the CMake exercise repository](https://github.com/Simulation-Software-Engineering/cmake-exercise-wt2425). Please name your pull request `Add building and container recipes` and assign yourself. 12 | - Your pull request should add a `Dockerfile` and a `CMakeLists.txt`, besides some minor changes in `main.cpp`, such as commenting in some code parts. 13 | - From your pull request, it should be possible to create an executable container. After running the container, it should be possible to `cd` into `cmake-exercise` and then run the `build_and_run.sh` script. 14 | - Use as many of the currently commented-out additional files, which induce additional dependencies. 15 | 16 | ## First Steps 17 | 18 | 1. Fork and clone the repository, have a look at the `main.cpp` and the `README.md`. 19 | 2. Build `main.cpp` manually (e.g. `g++ main.cpp -o main`) and run the executable (`./main`). 20 | 3. Build a Docker image, run a container (in interactive mode), and repeat steps 1 and 2 within the container. 21 | 4. Look into and use the `build_and_run.sh` script. 22 | 23 | ## Repository Structure 24 | 25 | The bare `main.cpp` uses several additions, which are located in the following subdirectories, each containing a `cpp` and a `hpp` file with the same name. 26 | 27 | - `flatset` adds a function to create and modify a flat set using [Boost Container](https://www.boost.org/doc/libs/1_86_0/doc/html/container.html) and outputs the set. This example is adapted from a [cppsecrets blog post](http://cppsecrets.com/article.php?id=2834). 28 | - `filesystem` adds a function to inspect and output the current directory using [Boost Filesystem](https://www.boost.org/doc/libs/1_86_0/libs/filesystem/doc/index.htm). This example is adapted from [tutorial 4](https://www.boost.org/doc/libs/1_86_0/libs/filesystem/example/tut4.cpp) of Boost Filesystem. 29 | - `fem` defines a class to solve the Poisson problem with the finite-element method (FEM) using [deal.II](https://www.dealii.org/). Output is written to a file `solution.vtk`, which can be visualized with, for example, [Paraview](https://www.paraview.org/). This example is adapted from the [tutorial step 3](https://dealii.org/current/doxygen/deal.II/step_3.html) of deal.II. 30 | - `yamlParser` adds a function to parse a simple yaml file using [yaml-cpp](https://github.com/jbeder/yaml-cpp) and to output the value of the key `version`. The folder also contains an example file `config.yml`. 31 | 32 | ## Docker Setup 33 | 34 | The code and all dependencies should run in a Docker container based on the `ubuntu:24.04` image. As by now, you already know how to set up a basic Docker container, we do no longer provide detailed instructions. We recommend building the Dockerfile incrementally. Start with a rather empty one and install dependencies manually in the interactive mode. Take notes of the commands you use, so you can integrate them into the Dockerfile afterwards and rebuild your image. 35 | 36 | Some standard packages available on Aptitude might come handy: 37 | 38 | - `build-essential` 39 | - `cmake` 40 | - `unzip` 41 | - `wget` 42 | - `vim` 43 | 44 | ## Dependencies 45 | 46 | Add dependencies one by one: Comment in the parts of `main.cpp` that are connected to a specific dependency. Then, install the dependency and extend the `CMakeLists.txt`. Verify that you can build and run the executable. If everything works, go on and include the next dependency. 47 | 48 | - Maybe start with the boost dependencies. Boost Container is a header-only dependency, Boost Filesystem needs to be linked. Both are available in `libboost-all-dev`. There is a CMake module to [find boost libraries](https://cmake.org/cmake/help/latest/module/FindBoost.html). 49 | - deal.II is available in `libdeal.ii-dev`. deal.II uses some specific [CMake macros](https://www.dealii.org/current/users/cmake_user.html). 50 | - yaml-cpp is an optional bonus task. For some arbitrary reason, we are not happy with the latest release of the software (which would be available through Aptitude), but we want to use exactly version `v0.6.3` [directly from GitHub](https://github.com/jbeder/yaml-cpp/releases/tag/yaml-cpp-0.6.3). Get the sources with [wget](https://linuxize.com/post/wget-command-examples/) and build and install (`make install`) it yourself. Do not forget to add the necessary instructions to the Dockerfile. If libraries in `/usr/local/lib` are not found by CMake, please add the path to the environment variable `LD_LIBRARY_PATH`. 51 | -------------------------------------------------------------------------------- /03_building_and_packaging/cmake_more_demo.md: -------------------------------------------------------------------------------- 1 | # Notes for More CMake Demo 2 | 3 | ## Walk Through preCICE `CMakeLists.txt` 4 | 5 | - [`CMakeLists.txt` on GitHub](https://github.com/precice/precice/blob/develop/CMakeLists.txt) 6 | - First look around: 7 | - `CMakeLists.txt`: lengthy 8 | - `tree cmake` 9 | - If software should be usable (for everybody everywhere), building and packaging is a project by itself 10 | - In `modules`: some `FindX.cmake`, partially third-party, partially developed by preCICE devs 11 | - In `CMakeLists.txt`: `sources.cmake` included (around line 510) 12 | - No glob, but generated externally (some python script) 13 | - Look at `src/sources.cmake` 14 | - Look at `CPackConfig.cmake` 15 | - `cmake ..` and read through output 16 | 17 | ## CMake Curses Interface 18 | 19 | - There are many tools around CMake, `ccmake` is also developed by KitWare 20 | - Separate package on Ubuntu: `sudo apt-get install cmake-curses-gui` 21 | - Delete previous build folder and start from scratch 22 | - `ccmake ..` 23 | - `[c]` if not yet configured before 24 | - Change variable with `[enter]`, e.g. `CMAKE_BUILD_TYPE` 25 | - Look at short description of options at the bottom 26 | - `[t]` to show also CMake options, helpful to find out which variables exist 27 | - `[c]` to configure again (because we changed something) 28 | - `[g]` to generate `Makefile` and exit 29 | -------------------------------------------------------------------------------- /03_building_and_packaging/cpack_exercise.md: -------------------------------------------------------------------------------- 1 | # Packaging with CPack 2 | 3 | Let us try to package the code from the CMake exercise with CPack, such that we can give the (binary) software to somebody else. 4 | 5 | Deadline: **December 11, 2024, 9:00** 6 | 7 | ## Overview 8 | 9 | - The goal of the exercise is to open a pull request from a fork of [the CPack exercise repository](https://github.com/Simulation-Software-Engineering/cpack-exercise-wt2425). Please name your pull request `Add installation and packaging targets`. In the pull request description, please explain what we need to do to test your code. If you work on any of the optional tasks below, please document in the description as well. 10 | - The current state of the code is basically a solution of the CMake exercise from last week. For demonstration purpose the code is now, however, artificially split into a library `cpackexamplelib` and an executable `cpackexample`. 11 | - Your task is to develop a CMake/CPack configuration that allows generating a `.tar.gz` and a Debian `.deb` package of the code. To this end, follow the same four steps as in the lecture (details below). 12 | 13 | ## Getting Started 14 | 15 | 1. Fork and clone the repository. 16 | 2. Build the Docker image: `docker build -t IMAGENAME .` (might take a few minutes, continue reading already) 17 | 3. Take a look at the `CMakeLists.txt` file. It should look familiar. 18 | 4. Once the Docker image is ready, run it and mount the current directory: 19 | - `docker run --rm -it --mount type=bind,source="$(pwd)",target=/mnt/cpack-exercise IMAGENAME` 20 | 5. In the Docker container: build the code and run `cpackexample`. 21 | 22 | ## Tasks 23 | 24 | The same four steps as in the lecture: 25 | 26 | ### 1. Add Install Target to CMake Configuration 27 | 28 | - The executable `cpackexample` should be installed in a `/bin/` directory, the library `libcpackexamplelib.a` or `libcpackexamplelib.so` should be installed in a `/lib/` directory, and all header files (`fem.hpp`, `filesystem.hpp`, `flatset.hpp`, `yamlParser.hpp`) should be installed in a `/include/cpackexamplelib` directory. 29 | - Test whether `make install` works as expected. 30 | 31 | ### 2. Add CPack Configuration 32 | 33 | - Write a separate CMake module `cmake/CPackConfig.cmake` for the packaging process and include it in the `CMakeLists.txt` file. The created package should contain sufficient information about the package, at least: maintainer, contact, project description, vendor, and homepage (e.g. your fork on GitHub). Feel free to set more [additional options](https://cmake.org/cmake/help/latest/module/CPack.html). 34 | - `make package` should (only) create a `tar.gz` and a `deb` package. 35 | - Inspect that both packages contain the correct content: 36 | - To inspect the contents of a `.tar.gz.` file, you can unpack it using the tool `tar`: 37 | 38 | ```bash 39 | tar -xzf TARGZFILE 40 | ``` 41 | 42 | - To inspect the contents of a `.deb` file, you can unpack it using the tool `dpkg-deb`: 43 | 44 | ```bash 45 | dpkg-deb -R DEBFILE DIRECTORY_FOR_UNPACKED_DEBFILE 46 | ``` 47 | 48 | ### 3. Create Debian Package 49 | 50 | - Extend the configuration for the generation of Debian packages. Make sure that the package file name is generated according to the Debian package naming scheme. 51 | - Make sure that you can install the Debian package (`apt install ./DEBFILENAME`) and that you can run the executable `cpackexample`. The executable should now be located in `/usr/bin`. 52 | - Optional: Inspect the content of the Debian package again. Do the dependencies look correct? Modify the Dockerfile such that `libyaml-cpp` also properly appears as dependency. 53 | 54 | ### 4. Check Debian Package 55 | 56 | - Inspect your Debian package with [lintian](https://manpages.ubuntu.com/manpages/trusty/man1/lintian.1.html): `lintian ./DEBFILENAME`. Check and save the output (report in pull request). 57 | - Make sure that your compiled code [gets stripped](https://cmake.org/cmake/help/latest/module/CPack.html#variable:CPACK_STRIP_FILES). 58 | - Optional: Create your package once with stripped and once with unstripped files. This should show a difference in file size, which you can check, for example, with `du -h FILENAME`. 59 | - Optional: Fix more errors and warnings (not necessarily all). Add the initial and the final output of `lintian` to the pull request such that we can see what errors or warnings disappeared. Briefly describe in the pull request what you did to get rid of errors and warnings. 60 | 61 | ## Optional Bonus Task 62 | 63 | Let us completely automatize the package creation using the Docker image. Simply running a container via `docker run` should automatically build and save the created packages at a predefined location. No further user interaction should be necessary. Once the container exits, the `tar.gz` and `deb` packages should be present on the host system. However, no `build` directory or other temporary files should be present on the host system, i.e., make sure the package is not created in the mounted drive. Please package `cpackexamplelib` as **shared** library when creating the packages. 64 | 65 | Extend the Dockerfile accordingly. Please state the actual command that has to be used to run the container in the pull request. 66 | -------------------------------------------------------------------------------- /03_building_and_packaging/examples/cmake/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION "3.12") 2 | 3 | project("HelloWorld") 4 | 5 | add_executable(helloworld main.cpp) 6 | -------------------------------------------------------------------------------- /03_building_and_packaging/examples/cmake/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | int main(int argc, char *argv[]) 4 | { 5 | std::cout << "Hello World!" << std::endl; 6 | return 0; 7 | } 8 | -------------------------------------------------------------------------------- /03_building_and_packaging/examples/cmake/sse/sse.cpp: -------------------------------------------------------------------------------- 1 | #include "sse.hpp" 2 | #include 3 | 4 | void sse() 5 | { 6 | std::cout << "SSE is great!" << std::endl; 7 | } 8 | -------------------------------------------------------------------------------- /03_building_and_packaging/examples/cmake/sse/sse.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | void sse(); 4 | -------------------------------------------------------------------------------- /03_building_and_packaging/examples/cpack/CMakeLists.txt: -------------------------------------------------------------------------------- 1 | cmake_minimum_required(VERSION "3.12") 2 | 3 | project("HelloWorld" VERSION 0.1.0) 4 | 5 | add_library(sse sse/sse.cpp) 6 | set_target_properties(sse PROPERTIES PUBLIC_HEADER sse/sse.hpp) 7 | 8 | # Set up library includes 9 | target_include_directories(sse 10 | PRIVATE 11 | ${CMAKE_CURRENT_SOURCE_DIR} 12 | PUBLIC 13 | $ 14 | $ 15 | ) 16 | 17 | add_executable(helloworld main.cpp) 18 | target_link_libraries(helloworld PRIVATE sse) 19 | 20 | # Create install targets 21 | include(GNUInstallDirs) 22 | install(TARGETS helloworld sse 23 | LIBRARY DESTINATION ${CMAKE_INSTALL_LIBDIR} 24 | ARCHIVE DESTINATION ${CMAKE_INSTALL_LIBDIR} 25 | RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR} 26 | PUBLIC_HEADER DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/helloworld 27 | INCLUDES DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/helloworld 28 | ) 29 | -------------------------------------------------------------------------------- /03_building_and_packaging/examples/cpack/Dockerfile: -------------------------------------------------------------------------------- 1 | From ubuntu:24.04 2 | 3 | RUN apt-get -qq update && \ 4 | apt-get -qq -y install \ 5 | build-essential \ 6 | cmake \ 7 | g++ \ 8 | vim \ 9 | tree \ 10 | lintian 11 | 12 | # This is some strang Docker problem. Normally, you don't need to add /usr/local/lib to LD_LIBRARY_PATH 13 | ENV LD_LIBRARY_PATH $LD_LIBRARY_PATH:/usr/local/lib/ 14 | 15 | ADD . /helloworld 16 | CMD ["/bin/bash"] 17 | -------------------------------------------------------------------------------- /03_building_and_packaging/examples/cpack/cmake/CPackConfig.cmake: -------------------------------------------------------------------------------- 1 | set(CPACK_PACKAGE_NAME ${PROJECT_NAME}) 2 | 3 | set(CPACK_PACKAGE_DESCRIPTION_SUMMARY "SSE hello world example project" 4 | CACHE STRING "Extended summary.") 5 | 6 | set(CPACK_PACKAGE_VENDOR "SSE Lecturers") 7 | set(CPACK_PACKAGE_CONTACT "firstname.lastname@example.com") 8 | set(CPACK_PACKAGE_MAINTAINERS "SSE lecturers ${CPACK_PACKAGE_CONTACT}") 9 | set(CPACK_PACKAGE_HOMEPAGE_URL "https://simulation-software-engineering.github.io") 10 | 11 | include(CPack) 12 | -------------------------------------------------------------------------------- /03_building_and_packaging/examples/cpack/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "sse/sse.hpp" 4 | 5 | int main(int argc, char *argv[]) 6 | { 7 | std::cout << "Hello World!" << std::endl; 8 | sse(); 9 | return 0; 10 | } 11 | -------------------------------------------------------------------------------- /03_building_and_packaging/examples/cpack/sse/sse.cpp: -------------------------------------------------------------------------------- 1 | #include "sse.hpp" 2 | #include 3 | 4 | void sse() 5 | { 6 | std::cout << "SSE is great!" << std::endl; 7 | } 8 | -------------------------------------------------------------------------------- /03_building_and_packaging/examples/cpack/sse/sse.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | void sse(); 4 | -------------------------------------------------------------------------------- /03_building_and_packaging/examples/environment-variables/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:24.04 2 | 3 | ADD print_my_environment_variable.sh . 4 | CMD ["/bin/bash"] 5 | -------------------------------------------------------------------------------- /03_building_and_packaging/examples/environment-variables/print_my_environment_variable.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | if [[ -z "${MY_ENV_VARIABLE}" ]]; then 4 | echo "The environment variable MY_ENV_VARIABLE is empty (zero length)" 5 | else 6 | echo "The environment variable MY_ENV_VARIABLE is set to" 7 | echo " ${MY_ENV_VARIABLE}" 8 | fi -------------------------------------------------------------------------------- /03_building_and_packaging/examples/make/Makefile: -------------------------------------------------------------------------------- 1 | helloworld : main.cpp 2 | $(CXX) -o helloworld main.cpp 3 | -------------------------------------------------------------------------------- /03_building_and_packaging/examples/make/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | int main() 4 | { 5 | std::cout << "Hello World!" << std::endl; 6 | return 0; 7 | } 8 | -------------------------------------------------------------------------------- /03_building_and_packaging/examples/make/sse/sse.cpp: -------------------------------------------------------------------------------- 1 | #include "sse.hpp" 2 | #include 3 | 4 | void sse() 5 | { 6 | std::cout << "SSE is great!" << std::endl; 7 | } 8 | -------------------------------------------------------------------------------- /03_building_and_packaging/examples/make/sse/sse.hpp: -------------------------------------------------------------------------------- 1 | #pragma once 2 | 3 | void sse(); 4 | -------------------------------------------------------------------------------- /03_building_and_packaging/examples/shared-and-static-c++/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | int main() 4 | { 5 | std::cout << "Hello, world!" << std::endl; 6 | return 0; 7 | } 8 | -------------------------------------------------------------------------------- /03_building_and_packaging/figs/filesystem_hierarchy_structure/fig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Simulation-Software-Engineering/Lecture-Material/d58a42f0ddd51ed992519d7fc067d7866c8bb3fa/03_building_and_packaging/figs/filesystem_hierarchy_structure/fig.png -------------------------------------------------------------------------------- /03_building_and_packaging/figs/filesystem_hierarchy_structure/fig.tex: -------------------------------------------------------------------------------- 1 | % build with: pdflatex --shell-escape fig.tex 2 | \documentclass[tikz, convert={density=800}]{standalone} 3 | 4 | \usepackage{tikz} 5 | \definecolor{myOrange}{HTML}{ffa500} 6 | \definecolor{myBlue}{HTML}{42affa} 7 | 8 | \begin{document} 9 | \begin{tikzpicture}[>=latex, thick, shorten >=2pt, shorten <=2pt, ->, myBlue] 10 | 11 | % \tikzstyle{C}=[circle,fill=myBlue,text=white,minimum size=30pt,inner sep=2pt, font=\LARGE] 12 | \tikzstyle{C}=[rectangle,fill=myBlue,text=white,minimum size=30pt,inner sep=2pt, font=\LARGE] 13 | \tikzstyle{M}=[circle,fill=myOrange,text=white,minimum size=30pt,inner sep=2pt, font=\LARGE] 14 | \tikzstyle{B}=[text=myBlue, font=\LARGE] 15 | 16 | \node[C] at (0,3) (bin) {\texttt{/bin}}; 17 | \node[C] at (2,3) (sbin) {\texttt{/sbin}}; 18 | \node[C] at (4,3) (usr) {\texttt{/usr}}; 19 | \node[C] at (6,3) (lib) {\texttt{/lib}}; 20 | \node[C] at (8,3) (dots) {\texttt{/home}}; 21 | \node[C] at (10,3) (dots) {\texttt{\ldots}}; 22 | % \node[C] at (6,3) (var) {\texttt{/var}}; 23 | % \node[C] at (3,3) (B) {\texttt{B}}; 24 | % \node[C] at (6,3) (C) {\texttt{C}}; 25 | % 26 | \node[C] at (0,1) (usrbin) {\texttt{/bin}}; 27 | \node[C] at (2,1) (usrlocal) {\texttt{/local}}; 28 | \node[C] at (4,1) (usrlib) {\texttt{/lib}}; 29 | \node[C] at (6.5,1) (usrinclude) {\texttt{/include}}; 30 | \node[C] at (9,1) (usrdots) {\texttt{\ldots}}; 31 | 32 | \node[C] at (0,-1) (usrlocalbin) {\texttt{/bin}}; 33 | \node[C] at (2,-1) (usrlocallib) {\texttt{/lib}}; 34 | \node[C] at (4.5,-1) (usrlocalinclude) {\texttt{/include}}; 35 | \node[C] at (7,-1) (usrlocaldots) {\texttt{\ldots}}; 36 | 37 | 38 | % \node[C] at (6,1) (E) {\texttt{E}}; 39 | % 40 | % \node[M] at (9,3) (M) {\texttt{M}}; 41 | 42 | % \draw [<-] (usr) to (usrbin); 43 | % \draw [<-] (usr) to (usrlocal); 44 | % \draw [<-] (usr) to (usrinclude); 45 | % \draw [<-] (usr) to (usrlib); 46 | % \draw [<-] (usr) to (usrdots); 47 | % 48 | % 49 | % \draw [<-] (usrlocal) to (usrlocalbin); 50 | % \draw [<-] (usrlocal) to (usrlocallib); 51 | % \draw [<-] (usrlocal) to (usrlocalinclude); 52 | % \draw [<-] (usrlocal) to (usrlocaldots); 53 | 54 | % \draw [<-] (A) to (B); 55 | % \draw [<-] (B) to (C); 56 | % \draw [<-] (A) to (D); 57 | % \draw [<-] (D) to (E); 58 | % \draw [<-, myOrange] (C) to (M); 59 | % \draw [<-, myOrange] (E) to (M); 60 | 61 | \node[B] at (-3,3) (rootlevel) {\texttt{/}}; 62 | \node[B] at (-3,1) (usrlevel) {\texttt{/usr}}; 63 | \node[B] at (-3,-1) (usrlocallevel) {\texttt{/usr/local}}; 64 | 65 | \draw [] (usr.south) to (usrlevel.north); 66 | \draw [] (usrlocal) to (usrlocallevel); 67 | 68 | \end{tikzpicture} 69 | \end{document} 70 | -------------------------------------------------------------------------------- /03_building_and_packaging/figs/filesystem_paths_python/fig-user-level.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Simulation-Software-Engineering/Lecture-Material/d58a42f0ddd51ed992519d7fc067d7866c8bb3fa/03_building_and_packaging/figs/filesystem_paths_python/fig-user-level.png -------------------------------------------------------------------------------- /03_building_and_packaging/figs/filesystem_paths_python/fig-user-level.tex: -------------------------------------------------------------------------------- 1 | % build with: pdflatex --shell-escape fig.tex 2 | \documentclass[tikz, convert={density=800}]{standalone} 3 | 4 | \usepackage{tikz} 5 | \definecolor{myOrange}{HTML}{ffa500} 6 | \definecolor{myBlue}{HTML}{42affa} 7 | 8 | \begin{document} 9 | \begin{tikzpicture}[>=latex, thick, shorten >=2pt, shorten <=2pt, ->, myBlue] 10 | 11 | \tikzstyle{C}=[rectangle,fill=myBlue,text=white,minimum size=30pt,inner sep=2pt, font=\LARGE] 12 | \tikzstyle{M}=[circle,fill=myOrange,text=white,minimum size=30pt,inner sep=2pt, font=\LARGE] 13 | \tikzstyle{B}=[text=myBlue, font=\LARGE] 14 | 15 | \node[C] at (1,-1) (usrlocalbin) {\texttt{/bin}}; 16 | \node[C] at (4.5,-1) (usrlocallib) {\texttt{/lib/pythonX.Y}}; 17 | 18 | \node[B] at (-3,-1) (usrlocallevel) {\texttt{\$\{HOME\}/.local}}; 19 | \end{tikzpicture} 20 | \end{document} 21 | -------------------------------------------------------------------------------- /03_building_and_packaging/figs/filesystem_paths_python/fig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Simulation-Software-Engineering/Lecture-Material/d58a42f0ddd51ed992519d7fc067d7866c8bb3fa/03_building_and_packaging/figs/filesystem_paths_python/fig.png -------------------------------------------------------------------------------- /03_building_and_packaging/figs/filesystem_paths_python/fig.tex: -------------------------------------------------------------------------------- 1 | % build with: pdflatex --shell-escape fig.tex 2 | \documentclass[tikz, convert={density=800}]{standalone} 3 | 4 | \usepackage{tikz} 5 | \definecolor{myOrange}{HTML}{ffa500} 6 | \definecolor{myBlue}{HTML}{42affa} 7 | 8 | \begin{document} 9 | \begin{tikzpicture}[>=latex, thick, shorten >=2pt, shorten <=2pt, ->, myBlue] 10 | 11 | \tikzstyle{C}=[rectangle,fill=myBlue,text=white,minimum size=30pt,inner sep=2pt, font=\LARGE] 12 | \tikzstyle{M}=[circle,fill=myOrange,text=white,minimum size=30pt,inner sep=2pt, font=\LARGE] 13 | \tikzstyle{B}=[text=myBlue, font=\LARGE] 14 | 15 | \node[C] at (1,1) (usrbin) {\texttt{/bin}}; 16 | \node[C] at (4.5,1) (usrlocal) {\texttt{/lib/pythonX.Y}}; 17 | \node[C] at (8.0,1) (usrlocal) {\texttt{\ldots}}; 18 | 19 | \node[C] at (1,-0.5) (usrlocalbin) {\texttt{/bin}}; 20 | \node[C] at (4.5,-0.5) (usrlocallib) {\texttt{/lib/pythonX.Y}}; 21 | \node[C] at (8.0,-0.5) (usrlocal) {\texttt{\ldots}}; 22 | 23 | \node[B] at (-3,1) (usrlevel) {\texttt{/usr}}; 24 | \node[B] at (-3,-0.5) (usrlocallevel) {\texttt{\$\{HOME\}/.local}}; 25 | 26 | \end{tikzpicture} 27 | \end{document} 28 | -------------------------------------------------------------------------------- /03_building_and_packaging/figs/spack/fenics-dependencies.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Simulation-Software-Engineering/Lecture-Material/d58a42f0ddd51ed992519d7fc067d7866c8bb3fa/03_building_and_packaging/figs/spack/fenics-dependencies.png -------------------------------------------------------------------------------- /03_building_and_packaging/intro_slides.md: -------------------------------------------------------------------------------- 1 | --- 2 | type: slide 3 | slideOptions: 4 | transition: slide 5 | width: 1400 6 | height: 900 7 | margin: 0.1 8 | --- 9 | 10 | 25 | 26 | # Introduction to Packaging 27 | 28 | --- 29 | 30 | ## Learning goals of chapter 31 | 32 | - Explain why software is packaged. 33 | - Create Python packages, publish on PyPI, and install with pip. 34 | - Understand the difference between static and dynamic libraries and common ways of installation on Linux. 35 | - Build C++ software and handle dependencies with Make and CMake. 36 | - Package C++ software with CPack and create Debian packages. 37 | - Create Spack packages, e.g., for high-performance computing systems with restricted access rights. 38 | 39 | --- 40 | 41 | ## What is packaging? 42 | 43 | - Bare code is often hard to understand for everyone except the developer(s). 44 | - Packaging is a workflow to convert a code into a standardized distributable software. 45 | - A code can be standardized in various ways. Some examples are 46 | - creating a compact form by following a standardization. 47 | - providing an installation recipe, for example, using CMake / make. 48 | - bundling code into an app or software with some UI. 49 | - We discuss **creating a compact form by following a standardization**. 50 | 51 | --- 52 | 53 | ## Why should we package code? 1/2 54 | 55 | - A bare code with many files typically has difficulties like 56 | - multiple dependencies and requirements of specific versions of dependencies. 57 | - intricate compilation / installation steps which are hard to get right. 58 | - missing or limited starting information / documentation, which means a high entry barrier. 59 | 60 | --- 61 | 62 | ## Why should we package code? 2/2 63 | 64 | - Create a package to 65 | - benefit from a package index or package manager which is familiar for a broad audience. 66 | - benefit from automated handling of dependencies of package managers. 67 | - have ease of distribution and maintenance due to standardization. 68 | - increase overall usability and sustainability of your code. 69 | 70 | --- 71 | 72 | ## How to package code? 73 | 74 | - First step is finding the right standard for your code. 75 | - There are several options: 76 | - One of the many Linux package managers: apt, dpkg, yum, RPM and many more ... 77 | - [CMake](https://cmake.org/) : building / installation / packaging tool mostly for C, C++ projects 78 | - [Spack](https://spack.io/) : a package management tool mostly for supercomputing centers 79 | - [Conan](https://conan.io/) : open-source package manager for C and C++ development 80 | - [PyPI](https://pypi.org/) and [pip](https://pypi.org/project/pip/) 81 | - [Conda](https://docs.conda.io/en/latest/) 82 | 83 | --- 84 | 85 | ## Why do we look at packaging a Python code? 86 | 87 | - Python is easy to understand and widely used in research software. 88 | - A well established packaging workflow already exists in the Python community. 89 | - Various examples of packaged codes already exist: [NumPy](https://pypi.org/project/numpy/), [SciPy](https://pypi.org/project/scipy/), [PyTorch](https://pypi.org/project/torch/) and more ... 90 | 91 | --- 92 | 93 | ## Key takeaways 94 | 95 | - Packaging or creating build recipe of a code is a standardized process. 96 | - Many options in packaging / building tools. 97 | - Most of these tools / methods are customized for use cases. 98 | - In this lecture we will concentrate on packaging of Python code. 99 | -------------------------------------------------------------------------------- /03_building_and_packaging/linux_fundamentals_demo.md: -------------------------------------------------------------------------------- 1 | # Notes for Some Fundamentals of Linux Systems Demo 2 | 3 | ## Static and Shared Linking 4 | 5 | Example code is in [`building-and-packaging/material/examples/shared-and-static-c++`](https://github.com/Simulation-Software-Engineering/Lecture-Material/tree/main/03_building_and_packaging/examples/shared-and-static-c++). 6 | 7 | - This example is based on the C++ standard library to emphasize the difference between static and shared libraries. 8 | - Show `main.cpp`, which uses standard lib 9 | - Compile code: 10 | - Shared: `g++ main.cpp -o main-shared` 11 | - Static: `g++ -static main.cpp -o main-static` 12 | - Run codes to show that they produce the same output: 13 | - `./main-static` 14 | - `./main-shared` 15 | - Run `ldd` on `main-shared` and `main-static`. 16 | - There will several libraries linked to `main-shared`. 17 | - When using `ldd` on `main-static`, we get the message `not a dynamic executable`. 18 | - Easier to grasp: `libtree -v main-shared`, [libtree](https://github.com/haampie/libtree) 19 | - Run `ls -lah` to show different executable sizes: 20 | 21 | ```bash 22 | $ ls -lah 23 | total 2.4M 24 | 17K main-shared 25 | 2.3M main-static 26 | ``` 27 | 28 | - `main-static` could be copied over to other systems, containers etc. where the standard library is not preinstalled. 29 | 30 | ## Filesystem Hierarchy Standard (FHS) 31 | 32 | - `/`: primary root 33 | - `/home`: contains user's home directories 34 | - `/bin`: executables installed by package manager 35 | - `/sbin`: executables installed by package manager, executed by the super user 36 | - `/lib`: libraries installed by package manager 37 | - `/usr/`: historic reasons, often today simply symbolic links 38 | - `/usr/local`: third level hierarchy level containing packages installed manually 39 | 40 | ## Environment Variables 41 | 42 | Example code is in [`building-and-packaging/material/examples/environment-variables`](https://github.com/Simulation-Software-Engineering/Lecture-Material/tree/main/03_building_and_packaging/examples/environment-variables). 43 | 44 | - If necessary build Docker image: `docker build -t "demo_variables" .` 45 | - Start docker container to have fresh system: `docker run -it demo_variables` 46 | - Show file structure: 47 | - `ls` 48 | - `ls usr` 49 | - `ls usr/local` 50 | - `echo $PATH` 51 | - Show all environment variables: `env` 52 | - Show script `print_my_environment_variable.sh` 53 | - Use script: 54 | - `./print_my_environment_variable.sh` -> empty 55 | - `MY_ENV_VARIABLE="Hi students" ./print_my_environment_variable.sh` -> good 56 | - `echo $MY_ENV_VARIABLE` -> empty (dereference variable) 57 | - `./print_my_environment_variable.sh` -> empty 58 | - `MY_ENV_VARIABLE="Hi students"` and `echo $MY_ENV_VARIABLE` -> good 59 | - `./print_my_environment_variable.sh` -> empty 60 | - `export MY_ENV_VARIABLE="Hi students"` (share with child process) 61 | - `./print_my_environment_variable.sh` -> good 62 | -------------------------------------------------------------------------------- /03_building_and_packaging/make_demo.md: -------------------------------------------------------------------------------- 1 | # Notes for Make Demo 2 | 3 | Example code is in [`building-and-packaging/material/examples/make`](https://github.com/Simulation-Software-Engineering/Lecture-Material/tree/main/03_building_and_packaging/examples/make). 4 | 5 | ## "Hello World" Starting Point 6 | 7 | Show `main.cpp` and build by hand `g++ -o helloworld main.cpp` 8 | 9 | ## Single Rule Example 10 | 11 | - Remove `helloworld`. 12 | - Show `Makefile`. 13 | - Explain that `helloworld` depends on `main.cpp` and rule to update. 14 | - Three cases: 15 | - no `helloworld`: run update 16 | - `helloworld` older than `main.cpp`: run update 17 | - `helloworld` newer than `main.cpp`: do nothing 18 | 19 | - Run make twice. 20 | - `ls -la` to show timestamp 21 | - Requires actual tabs, spaces not allowed. Show error message. 22 | 23 | > Makefile:2: *** missing separator (did you mean TAB instead of 8 spaces?). Stop. 24 | 25 | ## Multiple Rules Example and Phony Targets 26 | 27 | - Show `sse.hpp` and `sse.cpp` in subfolder `sse`. 28 | - Use it. 29 | 30 | `main.cpp`: 31 | 32 | ```diff 33 | #include 34 | + #include "sse/sse.hpp" 35 | 36 | int main() 37 | { 38 | std::cout << "Hello World!" << std::endl; 39 | + sse(); 40 | return 0; 41 | } 42 | ``` 43 | 44 | `Makefile`: 45 | 46 | ```makefile 47 | sse.o : sse/sse.hpp sse/sse.cpp 48 | $(CXX) -c sse/sse.cpp 49 | 50 | helloworld : main.cpp sse.o 51 | $(CXX) -o helloworld sse.o main.cpp 52 | ``` 53 | 54 | - Run `make`, only builds `sse.o` 55 | - By default, first target is built. 56 | - `make helloworld` to build specific target 57 | 58 | ```diff 59 | + all : helloworld sse.o 60 | ``` 61 | 62 | - Why does Make not just directly build all targets? We could want to do different things with different targets. 63 | - `all` typically comes first. 64 | - Add `clean` target, has no dependency. 65 | 66 | ```diff 67 | + clean : 68 | + rm -f helloworld sse.o 69 | ``` 70 | 71 | - Run `make clean` 72 | - `mkdir clean` and `make clean` confuses Make. 73 | - phony target (a helper target, doesn't correspond to a file) 74 | 75 | ``` 76 | + .PHONY : all clean 77 | ``` 78 | 79 | - `all` and `clean` are standard targets. All makefiles should have them. 80 | - Last remark: `make -j` builds in parallel. 81 | -------------------------------------------------------------------------------- /03_building_and_packaging/make_slides.md: -------------------------------------------------------------------------------- 1 | --- 2 | type: slide 3 | slideOptions: 4 | transition: slide 5 | width: 1400 6 | height: 900 7 | margin: 0.1 8 | --- 9 | 10 | 29 | 30 | # Introduction to Make 31 | 32 | --- 33 | 34 | ## Learning Goals of This Unit 35 | 36 | - Explain the basic functionality of makefiles (timestamps, dependencies, update rules). 37 | - Read simple makefiles and know where to look for additional material for complex makefiles. 38 | - Write simple makefiles for small projects. 39 | 40 | --- 41 | 42 | ## Demo Part 1 43 | 44 | Introduce `Hello-World` example 45 | 46 | --- 47 | 48 | ## What is Make? 49 | 50 | - A build system 51 | - The / a go-to solution for small (research) projects (e.g., latex document, processing data, ...), though also used in big projects ([Linux kernel](https://github.com/torvalds/linux)) 52 | - A building block for CMake 53 | - Nice non-expert introduction in [py-RSE book, chapter 9](https://third-bit.com/py-rse/automate.html) 54 | - [GNU Make](https://www.gnu.org/software/make/): standard implementation of Make for Linux and macOS 55 | 56 | --- 57 | 58 | ## How Does it Work? 59 | 60 | - When you create / change a file, the OS updates timestamp of file. 61 | - Make compares timestamps to see which files are older / newer than others. 62 | - In `Makefile`: 63 | - Rules which file(s) depend on which other file(s) 64 | - Rules how to update out-of-date files 65 | - When you run `make`, Make updates out-of-date files including transitive dependencies in correct order. 66 | 67 | --- 68 | 69 | ## Demo Part 2 70 | 71 | - Single rule makefile 72 | - Multiple rules 73 | - Phony targets 74 | 75 | --- 76 | 77 | ## Advanced Make 78 | 79 | - There is more: 80 | - Variables 81 | - Rules 82 | - Wildcards 83 | - ... but becomes quickly [very hard to read](https://www.gnu.org/software/make/manual/html_node/Automatic-Variables.html). 84 | - Not covered because CMake does this for us. 85 | - But nicely documented in [py-RSE chapter 9](https://third-bit.com/py-rse/automate.html). 86 | 87 | --- 88 | 89 | ## Summary 90 | 91 | - Make is an important build system by itself and an important building block for CMake. 92 | - Make works by checking timestamps and updating outdated files if necessary. 93 | - Each file can be a target. A target has dependencies and an update rule. 94 | - `make target` updates "target", just `make` updates first target. 95 | - Phony targets are empty (helper) targets, not files. 96 | - Always define the standard phony targets `all` and `clean`. 97 | 98 | --- 99 | 100 | ## Further Reading 101 | 102 | - [GNU Make manual](https://www.gnu.org/software/make/manual/html_node/index.html) 103 | -------------------------------------------------------------------------------- /03_building_and_packaging/pip_demo.md: -------------------------------------------------------------------------------- 1 | # `pip`: Python Packaging Tool Demo Notes 2 | 3 | **Note**: It is recommended run everything in a Ubuntu Docker container or a virtual environment. Launch a container using `docker run --rm -it ubuntu:jammy`. A virtual environment can be created as follows 4 | 5 | ```bash 6 | python -m venv env-name 7 | source env-name/bin/activate 8 | ``` 9 | 10 | ## 0. Getting `pip` 11 | 12 | A clean Docker container may not have Python installed. Install Python using `apt`: `apt install python3`. If Python has been installed using a package manager like `apt`, it will not come with `pip`. `pip`can be installed in several ways 13 | 14 | ```bash 15 | python -m ensurepip --upgrade 16 | ``` 17 | 18 | or 19 | 20 | ```bash 21 | apt install python3-pip 22 | ``` 23 | 24 | - If installed with Python, the Python version is used to determine which `pip` is installed. So Python 3.8 will install a `pip` by the name of `pip3.8` which will be compatible with Python 3.8. 25 | - In general `pip3` works with Python v3.x. 26 | - **Note**: Different `pip` names (`pip3`) are not to be confused with the version of the `pip` package, which can be seen with `pip --version`. 27 | 28 | ## 1. Installing packages with `pip` 29 | 30 | Let us install the finite element library [Nutils](https://nutils.org/). 31 | 32 | ```bash 33 | pip install nutils 34 | ``` 35 | 36 | Install a specific version of a package 37 | 38 | ```bash 39 | pip install nutils==7.0.0 40 | ``` 41 | 42 | Uninstalling a package 43 | 44 | ```bash 45 | pip uninstall nutils 46 | ``` 47 | 48 | - By default `pip` tries to install packages in the system directory, for example `/usr/local/lib/python`. This is problematic as it has no user isolation and it requires root privileges. 49 | - With the `--user` flag, packages are installed in the `home` directory, typically at `/home/user/.local/lib/python/site-packages/`. This is important when one is not working with virtual environments. 50 | 51 | Installing a package from a folder 52 | 53 | ```bash 54 | git clone https://github.com/evalf/nutils.git 55 | cd nutils 56 | pip install --user . 57 | ``` 58 | 59 | ## 2. Getting information of currently installed packages 60 | 61 | ```bash 62 | pip list 63 | ``` 64 | 65 | or for a specific package 66 | 67 | ```bash 68 | pip show nutils 69 | ``` 70 | 71 | ## 3. Running `pip` as a Python module 72 | 73 | ```bash 74 | python -m pip install package-name 75 | ``` 76 | 77 | ## 4. How to read a PEP 78 | 79 | - Have a look at [PEP 8](https://peps.python.org/pep-0008/) 80 | 81 | ## 5. Understanding a PyPI package webpage 82 | 83 | - Having a look at [fenicsprecice](https://pypi.org/project/fenicsprecice/) 84 | -------------------------------------------------------------------------------- /04_documentation/README.md: -------------------------------------------------------------------------------- 1 | # Documentation 2 | 3 | Learning goals: 4 | 5 | - Be aware that documentation is crucial in RSE. 6 | - Understand that there is a difference between documentation and *good* documentation. 7 | - Know the purpose and basic structure of several standard documentation building blocks (README, commit message, changelog, ...). 8 | - Read and write common lightweight markup languages and generate PDF files from them. 9 | - Know about the most important website generators and hosting options for research software documentation. 10 | 11 | | Duration | Content | 12 | | --- | --- | 13 | | 25 minutes | [`markup_languages_slides.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/04_documentation/markup_languages_slides.md)| 14 | | 20 minutes | [`pandoc_slides.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/04_documentation/pandoc_slides.md), [`pandoc_demo.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/04_documentation/pandoc_demo.md) | 15 | | 90 minutes | [`technical_writing_slides.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/04_documentation/technical_writing_slides.md), [`technical_writing_demo.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/04_documentation/technical_writing_demo.md) | 16 | | 35 minutes | [`website_generators_slides.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/04_documentation/website_generators_slides.md), [`website_generators_demo.md`](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/04_documentation/website_generators_demo.md) | 17 | -------------------------------------------------------------------------------- /04_documentation/examples/markdown-example.md: -------------------------------------------------------------------------------- 1 | # This is the top level header 2 | 3 | If you follow some (?) standard strictly, the top level header should be the very first line in a markdown file. 4 | 5 | ## You can write text 6 | 7 | Hello there. 8 | *This text is emphasized (italics)* 9 | **This text is heavily emphasized (bold)** 10 | 11 | We can also have nice lists 12 | 13 | - Item 1 14 | - Item 2 15 | 16 | which can be numbered as well 17 | 18 | 1. Item 1 19 | 2. Item 2 20 | 1. Item 2a 21 | 2. Item 2b 22 | 23 | As one can see we can also have sublists. 24 | 25 | Code uses backticks. This can be code/variable names in text that `appear` in a monospace font. Code blocks start and end with triple backticks and often take a specifier of the presented language. 26 | 27 | ```c++ 28 | int main() 29 | { 30 | return 0; 31 | } 32 | ``` 33 | 34 | One could also define code blocks with by indentation (usually 4 spaces), but it is better to not mix styles. 35 | 36 | > Quotations can be done by angle brackets 37 | 38 | We can also have hyperlinks to other websites. We could link to the [course website](https://simulation-software-engineering.github.io/homepage/) within the text, but separate [the link][course-website] in the text from the target definition. You find the target definition at the bottom of the Markdown file. 39 | 40 | ## This is a subheader 41 | 42 | We can have nice tables 43 | 44 | | Column A | Column B | 45 | | -------- | -------- | 46 | | Value A | Value B | 47 | 48 | [course-website]: https://simulation-software-engineering.github.io/homepage/ 49 | 50 | -------------------------------------------------------------------------------- /04_documentation/examples/pandoc/example.md: -------------------------------------------------------------------------------- 1 | --- 2 | title: My awesome title 3 | author: Firstname lastname 4 | date: 2025-01-08 5 | --- 6 | 7 | # Introduction 8 | 9 | This is text, **bold text**, *italic text*, and `inline code`. 10 | 11 | - Item one 12 | - Two 13 | - Three 14 | 15 | A link to [SSE](https://simulation-software-engineering.github.io). 16 | 17 | ```cpp 18 | #include 19 | 20 | int main(int argc, char *argv[]) 21 | { 22 | std::cout << "Hello World!" << std::endl; 23 | return 0; 24 | } 25 | ``` 26 | 27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /04_documentation/examples/reStructuredText-example.rst: -------------------------------------------------------------------------------- 1 | ================== 2 | Top-level headline 3 | ================== 4 | 5 | Overlining is optional. One can use other characters for the headline marking such as ``-`` or ``*``. There is no strict hierarchy of these characters defined to mark sections, subsections etc. The actual hierarchy is determined by the order of headings. However, the Python style guide proposes ``#`` > ``*`` > ``=`` and so on with each character referring to a certain type of structure (part, chapter, section...) 6 | 7 | More information can be found in the `Python documentation style guide `_ 8 | 9 | ------------------------------- 10 | This is the second-level header 11 | ------------------------------- 12 | 13 | Hello there. 14 | 15 | *This text is emphasized (italics)* 16 | 17 | **This text is heavily emphasized (bold)** 18 | 19 | We can also have nice lists 20 | 21 | * Item 1 22 | * Item 2 23 | 24 | which can be numbered as well 25 | 26 | 1. Item 1 27 | 2. Item 2 28 | 29 | 1. Item 2a 30 | 2. Item 2b 31 | 32 | As one can see we can also have sublists. In contrast to markdown there has to be a line break before the sublist starts. 33 | 34 | Code uses backticks. This can be code/variable names in text that ``appear`` in a monospace font. Code blocks with the ``.. code-block::`` directive and may take a language specified. Note that the code is indented with three (3) spaces. 35 | 36 | .. code-block:: c++ 37 | 38 | int main() 39 | { 40 | return 0; 41 | } 42 | 43 | .. 44 | Quotations work by indenting them more than the surrounding. We use an empty comment ``..`` in order to separate the code block from this quote. 45 | 46 | We can also have hyperlinks to other websites. We could link to the `course website`_, but separate the link and target definition. 47 | 48 | .. _course website: https://simulation-software-engineering.github.io/homepage 49 | 50 | ------ 51 | Tables 52 | ------ 53 | 54 | There is two ways to create tables. The more powerful way uses grid tables: 55 | 56 | +----------+----------+ 57 | | Column A | Column B | 58 | +==========+==========+ 59 | | Value A | Value B | 60 | +----------+----------+ 61 | 62 | Tables the simple way 63 | 64 | ========== ========== 65 | Column A Column B 66 | ========== ========== 67 | Value A Value B 68 | Value C Value D 69 | ========== ========== 70 | 71 | -------------------------------------------------------------------------------- /04_documentation/examples/sphinx/README.rst: -------------------------------------------------------------------------------- 1 | Python solver package 2 | ===================== 3 | 4 | A Python package to solve the diffusion equation in 2 space dimensions. 5 | 6 | How to use 7 | ---------- 8 | 9 | Move into the ``src/solver`` directory and the execute the solver there, e.g., 10 | 11 | .. code-block:: bash 12 | 13 | cd src/solver 14 | python diffusionsolver.py 15 | 16 | License 17 | ------- 18 | 19 | I should have added a license here. 20 | 21 | -------------------------------------------------------------------------------- /04_documentation/examples/sphinx/docs/tutorials/overview.rst: -------------------------------------------------------------------------------- 1 | Tutorial overview 2 | ================= 3 | 4 | .. toctree:: 5 | :maxdepth: 2 6 | :caption: tutorials: 7 | 8 | tutorial_case1.rst 9 | tutorial_case2.rst 10 | -------------------------------------------------------------------------------- /04_documentation/examples/sphinx/docs/tutorials/tutorial_case1.rst: -------------------------------------------------------------------------------- 1 | Tutorial 1 2 | ========== 3 | 4 | Tutorial description goes here. 5 | 6 | Step 1 7 | ------ 8 | 9 | Description of step 1. 10 | 11 | Step 2 12 | ------ 13 | 14 | Description of step 2. -------------------------------------------------------------------------------- /04_documentation/examples/sphinx/docs/tutorials/tutorial_case2.rst: -------------------------------------------------------------------------------- 1 | Tutorial 2 2 | ========== 3 | 4 | Tutorial description goes here. 5 | 6 | Step 1 7 | ------ 8 | 9 | Description of step 1. 10 | 11 | Step 2 12 | ------ 13 | 14 | Description of step 2. -------------------------------------------------------------------------------- /04_documentation/examples/sphinx/setup.cfg: -------------------------------------------------------------------------------- 1 | [metadata] 2 | name = example-SSE-package 3 | version = 0.0.1 4 | author = Benjamin Uekermann 5 | author_email = benjamin.uekermann@ipvs.uni-stuttgart.de 6 | description = A package that solves some equation as part of an exercise of a lecture. 7 | long_description = file: README.rst 8 | long_description_content_type = text 9 | url = https://github.com/Simulation-Software-Engineering/test-exercise-packaging 10 | project_urls = 11 | Bug Tracker = https://github.com/Simulation-Software-Engineering/test-exercise-packaging/issues 12 | classifiers = 13 | Programming Language :: Python :: 3 14 | License :: OSI Approved :: NONE 15 | Operating System :: OS Independent 16 | 17 | [options] 18 | package_dir = solver 19 | packages = find: 20 | python_requires = >=3.6 21 | 22 | [options.packages.find] 23 | where = solver 24 | -------------------------------------------------------------------------------- /04_documentation/examples/sphinx/setup.py: -------------------------------------------------------------------------------- 1 | import setuptools 2 | 3 | with open("README.md", "r", encoding="utf-8") as fh: 4 | long_description = fh.read() 5 | 6 | setuptools.setup( 7 | name="uekermbn-SSE-package", 8 | version="0.0.1", 9 | author="Benjamin Uekermann", 10 | author_email="benjain.uekermann@ipvs.uni-stuttgart.de", 11 | description="A small description", 12 | long_description=long_description, 13 | long_description_content_type="text/markdown", 14 | url="https://github.com/Simulation-Software-Engineering/test-exercise-packaging", 15 | project_urls={ 16 | "Bug Tracker": "https://github.com/Simulation-Software-Engineering/test-exercise-packaging/issues", 17 | }, 18 | classifiers=[ 19 | "Programming Language :: Python :: 3", 20 | "License :: OSI Approved", 21 | "Operating System :: OS Independent", 22 | ], 23 | package_dir={"": "src"}, 24 | packages=setuptools.find_packages(where="src"), 25 | python_requires=">=3.6", 26 | ) 27 | -------------------------------------------------------------------------------- /04_documentation/examples/sphinx/src/solver/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Simulation-Software-Engineering/Lecture-Material/d58a42f0ddd51ed992519d7fc067d7866c8bb3fa/04_documentation/examples/sphinx/src/solver/__init__.py -------------------------------------------------------------------------------- /04_documentation/examples/technical_writing/diffusion2d.py: -------------------------------------------------------------------------------- 1 | """ 2 | Example acquired from https://scipython.com/book/chapter-7-matplotlib/examples/the-two-dimensional-diffusion-equation/ 3 | """ 4 | 5 | import numpy as np 6 | import matplotlib.pyplot as plt 7 | 8 | 9 | class SolveDiffusion2D: 10 | 11 | def __init__(self): 12 | 13 | # plate size, mm 14 | self.w = None 15 | self.h = None 16 | 17 | # intervals in x-, y- directions, mm 18 | self.dx = None 19 | self.dy = None 20 | 21 | # Number of discrete mesh points in X and Y directions 22 | self.nx = None 23 | self.ny = None 24 | 25 | # Thermal diffusivity of steel, mm^2/s 26 | self.D = None 27 | 28 | # Initial cold temperature of square domain 29 | self.T_cold = None 30 | 31 | # Initial hot temperature of circular disc at the center 32 | self.T_hot = None 33 | 34 | # Timestep 35 | self.dt = None 36 | 37 | def initialize_domain(self, w=10., h=10., dx=0.1, dy=0.1): 38 | 39 | self.w = w 40 | self.h = h 41 | self.dx = dx 42 | self.dy = dy 43 | self.nx = int(w / dx) 44 | self.ny = int(h / dy) 45 | 46 | def initialize_physical_parameters(self, d=4., T_cold=300, T_hot=700): 47 | 48 | self.D = d 49 | self.T_cold = T_cold 50 | self.T_hot = T_hot 51 | 52 | # Computing a stable time step 53 | dx2, dy2 = self.dx * self.dx, self.dy * self.dy 54 | self.dt = dx2 * dy2 / (2 * self.D * (dx2 + dy2)) 55 | 56 | print("dt = {}".format(self.dt)) 57 | 58 | def set_initial_condition(self): 59 | u = self.T_cold * np.ones((self.nx, self.ny)) 60 | 61 | # Initial conditions - circle of radius r centred at (cx,cy) (mm) 62 | r, cx, cy = 2, 5, 5 63 | r2 = r ** 2 64 | for i in range(self.nx): 65 | for j in range(self.ny): 66 | p2 = (i * self.dx - cx) ** 2 + (j * self.dy - cy) ** 2 67 | if p2 < r2: 68 | u[i, j] = self.T_hot 69 | 70 | return u.copy() 71 | 72 | def do_timestep(self, u_nm1): 73 | u = u_nm1.copy() 74 | 75 | dx2 = self.dx * self.dx 76 | dy2 = self.dy * self.dy 77 | 78 | # Propagate with forward-difference in time, central-difference in space 79 | u[1:-1, 1:-1] = u_nm1[1:-1, 1:-1] + self.D * self.dt * ( 80 | (u_nm1[2:, 1:-1] - 2 * u_nm1[1:-1, 1:-1] + u_nm1[:-2, 1:-1]) / dx2 81 | + (u_nm1[1:-1, 2:] - 2 * u_nm1[1:-1, 1:-1] + u_nm1[1:-1, :-2]) / dy2) 82 | 83 | return u.copy() 84 | 85 | def create_figure(self, fig, u, n, fignum): 86 | fignum += 1 87 | ax = fig.add_subplot(220 + fignum) 88 | im = ax.imshow(u.copy(), cmap=plt.get_cmap('hot'), vmin=self.T_cold, vmax=self.T_hot) 89 | ax.set_axis_off() 90 | ax.set_title('{:.1f} ms'.format(n * self.dt * 1000)) 91 | 92 | return fignum, im 93 | 94 | 95 | def output_figure(fig, im): 96 | fig.subplots_adjust(right=0.85) 97 | cbar_ax = fig.add_axes([0.9, 0.15, 0.03, 0.7]) 98 | cbar_ax.set_xlabel('$T$ / K', labelpad=20) 99 | fig.colorbar(im, cax=cbar_ax) 100 | plt.show() 101 | 102 | 103 | def main(): 104 | DiffusionSolver = SolveDiffusion2D() 105 | 106 | DiffusionSolver.initialize_domain() 107 | 108 | DiffusionSolver.initialize_physical_parameters() 109 | 110 | u0 = DiffusionSolver.set_initial_condition() 111 | 112 | # Number of timesteps 113 | nsteps = 101 114 | 115 | # Output 4 figures at these timesteps 116 | n_output = [0, 10, 50, 100] 117 | 118 | fig_counter = 0 119 | fig = plt.figure() 120 | 121 | im = None 122 | 123 | # Time loop 124 | for n in range(nsteps): 125 | u = DiffusionSolver.do_timestep(u0) 126 | 127 | # Create figure 128 | if n in n_output: 129 | fig_counter, im = DiffusionSolver.create_figure(fig, u, n, fig_counter) 130 | 131 | u0 = u.copy() 132 | 133 | # Plot output figures 134 | output_figure(fig, im) 135 | 136 | 137 | if __name__ == "__main__": 138 | main() 139 | -------------------------------------------------------------------------------- /04_documentation/figs/pandoc-latex-workflow/fig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Simulation-Software-Engineering/Lecture-Material/d58a42f0ddd51ed992519d7fc067d7866c8bb3fa/04_documentation/figs/pandoc-latex-workflow/fig.png -------------------------------------------------------------------------------- /04_documentation/figs/pandoc-latex-workflow/fig.tex: -------------------------------------------------------------------------------- 1 | % build with: pdflatex --shell-escape fig.tex 2 | \documentclass[tikz, convert={density=800}]{standalone} 3 | 4 | \usepackage{tikz} 5 | \definecolor{myOrange}{HTML}{ffa500} 6 | \definecolor{myBlue}{HTML}{42affa} 7 | 8 | \begin{document} 9 | \begin{tikzpicture}[>=latex, thick, shorten >=2pt, shorten <=2pt, ->, myBlue] 10 | 11 | % \tikzstyle{C}=[circle,fill=myBlue,text=white,minimum size=30pt,inner sep=2pt, font=\LARGE] 12 | \tikzstyle{C}=[rectangle,fill=myBlue,text=white,minimum size=30pt,inner sep=2pt, font=\LARGE] 13 | \tikzstyle{M}=[circle,fill=myOrange,text=white,minimum size=30pt,inner sep=2pt, font=\LARGE] 14 | \tikzstyle{B}=[text=myBlue, font=\LARGE] 15 | 16 | \node[C] at (-2,3) (markdown) {\texttt{markdown}}; 17 | \node[C] at (-2,1.5) (rst) {\texttt{rst}}; 18 | \node[C] at (-2,-0.0) (gfm) {\texttt{gfm}}; 19 | \node[B] at (-2,-1.) (dots) {\texttt{\vdots}}; 20 | 21 | 22 | \node[C] at (2,0.75) (pandoc) {\texttt{pandoc}}; 23 | \node[C] at (5,0.75) (latex) {\texttt{\LaTeX}}; 24 | 25 | \node[C] at (8,0.75) (pdf) {\texttt{PDF}}; 26 | 27 | \draw [] (markdown.east) to (pandoc.west); 28 | \draw [] (rst.east) to (pandoc.west); 29 | \draw [] (gfm.east) to (pandoc.west); 30 | 31 | \draw [] (pandoc.east) to (latex.west); 32 | \draw [] (latex.east) to (pdf.west); 33 | 34 | % \draw [] (pandoc.east) to (markdownR.west); 35 | % \draw [] (pandoc.east) to (rstR.west); 36 | % \draw [] (pandoc.east) to (gfmR.west); 37 | % \draw [] (pandoc.east) to (manR.west); 38 | % \draw [] (pandoc.east) to (docxR.west); 39 | 40 | 41 | \end{tikzpicture} 42 | \end{document} 43 | -------------------------------------------------------------------------------- /04_documentation/figs/pandoc/fig.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Simulation-Software-Engineering/Lecture-Material/d58a42f0ddd51ed992519d7fc067d7866c8bb3fa/04_documentation/figs/pandoc/fig.png -------------------------------------------------------------------------------- /04_documentation/figs/pandoc/fig.tex: -------------------------------------------------------------------------------- 1 | % build with: pdflatex --shell-escape fig.tex 2 | \documentclass[tikz, convert={density=800}]{standalone} 3 | 4 | \usepackage{tikz} 5 | \definecolor{myOrange}{HTML}{ffa500} 6 | \definecolor{myBlue}{HTML}{42affa} 7 | 8 | \begin{document} 9 | \begin{tikzpicture}[>=latex, thick, shorten >=2pt, shorten <=2pt, ->, myBlue] 10 | 11 | % \tikzstyle{C}=[circle,fill=myBlue,text=white,minimum size=30pt,inner sep=2pt, font=\LARGE] 12 | \tikzstyle{C}=[rectangle,fill=myBlue,text=white,minimum size=30pt,inner sep=2pt, font=\LARGE] 13 | \tikzstyle{M}=[circle,fill=myOrange,text=white,minimum size=30pt,inner sep=2pt, font=\LARGE] 14 | \tikzstyle{B}=[text=myBlue, font=\LARGE] 15 | 16 | % \node[C] at (0,3) (bin) {\texttt{/bin}}; 17 | % \node[C] at (2,3) (sbin) {\texttt{/sbin}}; 18 | % \node[C] at (4,3) (usr) {\texttt{/usr}}; 19 | % \node[C] at (6,3) (lib) {\texttt{/lib}}; 20 | % \node[C] at (8,3) (dots) {\texttt{/home}}; 21 | % \node[C] at (10,3) (dots) {\texttt{\ldots}}; 22 | 23 | % \node[C] at (6,1) (E) {\texttt{E}}; 24 | % 25 | % \node[M] at (9,3) (M) {\texttt{M}}; 26 | 27 | % \draw [<-] (usr) to (usrbin); 28 | % \draw [<-] (usr) to (usrlocal); 29 | % \draw [<-] (usr) to (usrinclude); 30 | % \draw [<-] (usr) to (usrlib); 31 | % \draw [<-] (usr) to (usrdots); 32 | % 33 | % 34 | % \draw [<-] (usrlocal) to (usrlocalbin); 35 | % \draw [<-] (usrlocal) to (usrlocallib); 36 | % \draw [<-] (usrlocal) to (usrlocalinclude); 37 | % \draw [<-] (usrlocal) to (usrlocaldots); 38 | 39 | % \draw [<-] (A) to (B); 40 | % \draw [<-] (B) to (C); 41 | % \draw [<-] (A) to (D); 42 | % \draw [<-] (D) to (E); 43 | % \draw [<-, myOrange] (C) to (M); 44 | % \draw [<-, myOrange] (E) to (M); 45 | 46 | \node[C] at (-2,3) (markdown) {\texttt{markdown}}; 47 | \node[C] at (-2,1.5) (rst) {\texttt{rst}}; 48 | \node[C] at (-2,-0.0) (gfm) {\texttt{gfm}}; 49 | \node[B] at (-2,-1.5) (dots) {\texttt{\vdots}}; 50 | \node[C] at (-2,-3.0) (docx) {\texttt{docx}}; 51 | 52 | 53 | \node[C] at (2,-0) (pandoc) {\texttt{pandoc}}; 54 | 55 | % \draw [] (usr.south) to (usrlevel.north); 56 | % \draw [] (usrlocal) to (usrlocallevel); 57 | 58 | \draw [] (markdown.east) to (pandoc.west); 59 | \draw [] (rst.east) to (pandoc.west); 60 | \draw [] (gfm.east) to (pandoc.west); 61 | \draw [] (docx.east) to (pandoc.west); 62 | 63 | \node[C] at (6,3) (markdownR) {\texttt{markdown}}; 64 | \node[C] at (6,1.5) (rstR) {\texttt{rst}}; 65 | \node[C] at (6,-0.) (gfmR) {\texttt{gfm}}; 66 | \node[C] at (6,-1.5) (manR) {\texttt{man}}; 67 | \node[B] at (6,-2.5) (dotsR) {\texttt{\vdots}}; 68 | \node[C] at (6,-3.5) (docxR) {\texttt{docx}}; 69 | 70 | \draw [] (pandoc.east) to (markdownR.west); 71 | \draw [] (pandoc.east) to (rstR.west); 72 | \draw [] (pandoc.east) to (gfmR.west); 73 | \draw [] (pandoc.east) to (manR.west); 74 | \draw [] (pandoc.east) to (docxR.west); 75 | 76 | 77 | \end{tikzpicture} 78 | \end{document} 79 | -------------------------------------------------------------------------------- /04_documentation/pandoc_demo.md: -------------------------------------------------------------------------------- 1 | # Pandoc Demo Notes 2 | 3 | ## Basic Usage 4 | 5 | - Markdown example in `./examples/pandoc/example.md` 6 | - Convert to `rst`: `pandoc example.md -o example.rst` 7 | - Meta data not used 8 | - Convert to `pdf`: `pandoc example.md -o example.pdf` 9 | - Meta data used 10 | - Convert to `docx`: `pandoc example.md -o example.docx` 11 | - Convert to `tex`: `pandoc example.md -o example.tex` 12 | - No standalone file (cannot be further compiled into pdf just like that) 13 | - Not readable 14 | - Standalone `tex`: `pandoc example.md -s -o example.tex` 15 | - [Try pandoc online](https://pandoc.org/try/) 16 | - Not all file formats available 17 | - Choose example from drop down menu, e.g. "Markdown to reStructureText" 18 | - Command in right top corner 19 | 20 | ## Default Templates 21 | 22 | - Print `latex` template: `pandoc -D latex >> template.tex` 23 | - Very long 24 | - There is no `pdf` template, always goes through `latex` 25 | - Convert to `pdf`: `pandoc example.md -o example.pdf` 26 | - Needs no `-s` option, since `pdf` is always standalone 27 | - Point out that link is not colored 28 | - Color link: `pandoc example.md -V colorlinks -o example.pdf` 29 | - Binary option, no value 30 | - Find `colorlinks` in `template.tex` 31 | - [All variables for LaTeX](https://pandoc.org/MANUAL.html#variables-for-latex) 32 | 33 | ## Custom Templates 34 | 35 | - Show that I have `eisvogel` installed: `ls ~/.pandoc/templates` 36 | - Use template: `pandoc example.md --template eisvogel -o example.pdf` 37 | - Even nicer code: use `--listings` option 38 | - Uses latex listings package 39 | - Has nothing to do with `eisvogel` template 40 | - Extensive number of [options](https://pandoc.org/MANUAL.html#options) 41 | -------------------------------------------------------------------------------- /04_documentation/pandoc_slides.md: -------------------------------------------------------------------------------- 1 | --- 2 | type: slide 3 | slideOptions: 4 | transition: slide 5 | width: 1400 6 | height: 900 7 | margin: 0.1 8 | --- 9 | 10 | 34 | 35 | # Document Conversion with Pandoc 36 | 37 | --- 38 | 39 | ## Learning Goals 40 | 41 | - Convert markup files (Markdown, reStructuredText, ...) into other formats (Markdown, reStructuredText, PDF, docx, ...) with pandoc. 42 | - Explain how markup languages, version control, and conversion pipelines play together. 43 | 44 | --- 45 | 46 | ## Pandoc 47 | 48 | > - A Universal Document Converter 49 | > - If you need to convert files from one markup format into another, pandoc is your swiss-army knife. 50 | 51 | From [pandoc website](https://pandoc.org/) 52 | 53 | - Written in Haskell 54 | - Converts **>=38** file formats into **>=60** file formats 55 | - Includes dialects (e.g. different markdown flavors) 56 | 57 | --- 58 | 59 | ## Conversion Overview 60 | 61 | 62 | 63 | [Full diagram](https://pandoc.org/diagram.jpg) on homepage 64 | 65 | - Idea: Write documentation/text in lightweight, easy to read/write, git-diffable language, then convert to any (non-git-diffable) format. 66 | 67 | --- 68 | 69 | ## Basic Usage 70 | 71 | 72 | ```bash 73 | pandoc INPUTFILE -o OUTPUTFILE 74 | ``` 75 | 76 | - Deduces conversion type from `OUTPUTFILE` file extension (by default) 77 | - If explicit formats required (e.g. specific md flavors): 78 | 79 | ```bash 80 | --from/-f FORMAT --to/-t FORMAT 81 | ``` 82 | 83 | --- 84 | 85 | ## Demo: Basic Usage 86 | 87 | --- 88 | 89 | ## Pandoc + LaTeX = Awesome PDFs 90 | 91 | 92 | 93 | - Specify LaTeX compiler: `--pdf-engine=xelatex/lualatex/pdflatex` 94 | 95 | --- 96 | 97 | ## Standalone and Default Templates 98 | 99 | - Create [standalone](https://pandoc.org/MANUAL.html#option--standalone) file 100 | 101 | ```bash 102 | -s/--standalone 103 | ``` 104 | 105 | - This uses a [default template](https://pandoc.org/MANUAL.html#templates) for specific target format 106 | - To define, e.g., footer and header 107 | - Print template with, e.g. 108 | 109 | ```bash 110 | pandoc -D latex 111 | ``` 112 | 113 | - Set options defined in template 114 | 115 | ```bash 116 | -V OPTION=VALUE 117 | ``` 118 | 119 | --- 120 | 121 | ## YAML Metadata Blocks 122 | 123 | - Template may understand [`YAML` metadata blocks](https://pandoc.org/MANUAL.html#extension-yaml_metadata_block) 124 | 125 | ```yaml 126 | --- 127 | title: My awesome title 128 | author: Firstname lastname 129 | ... 130 | --- 131 | ``` 132 | 133 | May also be specified in separate `yaml` file. 134 | 135 | - Available parameters depend on template 136 | 137 | --- 138 | 139 | ## Demo: Default Templates 140 | 141 | --- 142 | 143 | ## Custom Templates 144 | 145 | - Install additional templates in 146 | 147 | ```bash 148 | ${HOME}/.local/share/pandoc/templates/ 149 | ``` 150 | 151 | or 152 | 153 | ```bash 154 | ${HOME}/.pandoc/templates/ 155 | ``` 156 | 157 | - Specify template on command line, e.g. [Eisvogel template](https://github.com/Wandmalfarbe/pandoc-latex-template) 158 | 159 | ```bash 160 | pandoc --template eisvogel 161 | ``` 162 | 163 | --- 164 | 165 | ## Demo: Custom Templates 166 | 167 | --- 168 | 169 | ## SSE Lecture Material as Example 170 | 171 | - [GitHub Action](https://github.com/Simulation-Software-Engineering/Lecture-Material/tree/main/.github/workflows) and [script](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/scripts/create-pdf-from-markdown.sh) 172 | - Current lecture slides command: 173 | 174 | ```text 175 | pandoc --pdf-engine=xelatex -t beamer -V aspectratio=169 -V \ 176 | linkcolor:blue -V fontsize=12pt --listings -s \ 177 | --output=OUTPUTFILENAME INPUTFILENAME 178 | ``` 179 | 180 | - Current lecture notes command: 181 | 182 | ```text 183 | pandoc --pdf-engine=xelatex -V geometry:a4paper \ 184 | -V geometry:left=2.5cm -V geometry:right=2.5cm \ 185 | -V geometry:bottom=2.5cm -V geometry:top=2.5cm \ 186 | -V colorlinks:true -V linkcolor:blue -V fontsize=10pt \ 187 | --listings -s --output=OUTPUTFILENAME INPUTFILENAME 188 | ``` 189 | -------------------------------------------------------------------------------- /04_documentation/technical_writing_demo.md: -------------------------------------------------------------------------------- 1 | # Notes for Technical Writing Demo 2 | 3 | Example code is in [`04_documentation/examples/technical_writing`](https://github.com/Simulation-Software-Engineering/Lecture-Material/tree/main/04_documentation/examples/technical_writing). 4 | 5 | - Look at Python code. Same as in packaging exercise, but now refactored into different functions. Run code. 6 | - We are using [numpydoc](https://numpydoc.readthedocs.io/en/latest/) as syntax (there is also Google style). 7 | 8 | ## Class Documentation 9 | 10 | - Add short description: 11 | 12 | ```diff 13 | + """2D diffusion equation solved with finite differences 14 | + """ 15 | ``` 16 | 17 | - Add long description, example usage: 18 | 19 | ```diff 20 | + See `main()` for example usage 21 | ``` 22 | 23 | ## Method Documentation 24 | 25 | - Constructor: 26 | 27 | ```diff 28 | + """Constructs an uninitialized 2D diffusion solver 29 | + 30 | + After construction, initialize domain with `initialize_domain` and 31 | + initialize the physical parameters with `initialize_physical_parameters`. 32 | + """ 33 | ``` 34 | 35 | - `initialize_domain`: 36 | 37 | ```diff 38 | + """Initializes domain and mesh of the domain 39 | + 40 | + - Sets width and height of the domain. 41 | + - Also sets mesh width in x and in y direction. 42 | + - Using these four values, the method computes the number of mesh elements in x and in y direction. 43 | """ 44 | ``` 45 | 46 | - `initialize_physical_parameters`: 47 | 48 | ```diff 49 | + """Initializes physical parameters and computes time step size 50 | + 51 | + You need to call `initialize_domain` before this method. 52 | + """ 53 | ``` 54 | 55 | - Ideally, we also add a link to source of time step size computation here. 56 | 57 | ## Parameters Documentation 58 | 59 | - Add to `initialize_domain`: 60 | 61 | ```diff 62 | + Parameters 63 | + ---------- 64 | + w : float, default: 10.0 65 | + width of the domain 66 | + h : float, default: 10.0 67 | + height of the domain 68 | + dx : float, default: 0.1 69 | + mesh width in x direction 70 | + dy : float, default: 0.1 71 | + mesh width in y direction 72 | ``` 73 | -------------------------------------------------------------------------------- /04_documentation/website_generators_demo.md: -------------------------------------------------------------------------------- 1 | # Notes for Documentation Website Generators Demo 2 | 3 | Example code is in [`04_documentation/examples/sphinx`](https://github.com/Simulation-Software-Engineering/Lecture-Material/tree/main/04_documentation/examples/sphinx). 4 | 5 | - Look at Python code. Same as in packaging exercise, but now refactored into different functions. Run code. 6 | 7 | ## Docstrings 8 | 9 | - Look into `src/solver/diffusionsolver.py` 10 | - We are using [numpydoc](https://numpydoc.readthedocs.io/en/latest/) as syntax (there is also Google style and Sphinx standard). 11 | - Explain notation of docstring 12 | - Short and long description 13 | - Run `python` to start interpreter 14 | - `import diffusionsolver` 15 | - `print(diffusionsolver.__doc__)`, but actually private, not the way how to work with docstrings 16 | - `help(diffusionsolver)` 17 | - `help(diffusionsolver.SolveDiffusion2D)` 18 | 19 | ## Sphinx Setup 20 | 21 | - `cd docs`; there is already a `tutorials` subfolder; will explain later 22 | - `sphinx-quickstart` 23 | - You only have to do this once 24 | - Separate source and build directories (y/n) [n]: n 25 | - ... of the documentation obviously. Both will go into `docs`. 26 | - Project name: Diffusion Solver 27 | - Author name(s): Benjamin Uekermann 28 | - Project release []: 0.1.0 29 | - Can distinguish releases 30 | - Project language [en]: en 31 | - Inspect structure: `tree .` 32 | - `conf.py`: configure documentation, contains most information 33 | - `index.rst`: main page 34 | - `Makefile` and `make.bat`: to build the website 35 | - more private folders for Sphinx, all empty still 36 | - Build website 37 | - `make` gives all target options 38 | - `make html` 39 | - `tree .`, `_build` is now filled 40 | - Open and inspect `_build/html/index.html` 41 | - Add tutorials 42 | - Look at files in `tutorials` subfolder 43 | - Table of contents in `overview.rst` 44 | - Add in `index.rst` after `toctree` with indentation of 3 spaces: 45 | 46 | ```diff 47 | + tutorials/overview 48 | ``` 49 | 50 | - Rebuild and inspect changes: `make html` 51 | - Add `README.rst`: 52 | 53 | ```diff 54 | +.. include:: ../README.rst 55 | ``` 56 | 57 | - Look at `conf.py` and change theme to `classic` 58 | 59 | ## Code Documentation 60 | 61 | - Edit `conf.py` 62 | - Add `'sphinx.ext.autodoc', 'numpydoc'` to extensions 63 | - Include source files, at the top add: 64 | 65 | ```python 66 | import os 67 | import sys 68 | sys.path.insert(0, os.path.abspath('../src')) 69 | ``` 70 | 71 | - Create autodoc template files: 72 | 73 | ```bash 74 | sphinx-apidoc -o source ../src/ 75 | ``` 76 | 77 | - Inspect: The files do not contain the actual documentation, but rather a template. The actual documentation is generated once `sphinx` is invoked. 78 | - `make html` and check `Index` 79 | -------------------------------------------------------------------------------- /05_testing_and_ci/README.md: -------------------------------------------------------------------------------- 1 | # Testing and CI 2 | 3 | Learning goals: 4 | 5 | - Justify the effort of developing testing infrastructure for simulation software. 6 | - Discern the concepts of unit testing, integration testing, and regression testing with the perspective of simulation software. 7 | - Work with the Python testing frameworks `pytest` and `unittest`. 8 | - Write simple tests for C++ toy codes with `Boost.Test`. 9 | - Know about general concepts of test frameworks such as fixtures and decorators. 10 | - Name and explain common workflows to automate in RSE. 11 | - Write basic automation scripts for GitHub Actions. 12 | - Read basic automation scripts for GitLab CI/CD and explain the necessary steps to host GitLab Runners yourself. 13 | - Use CTest to call tests from CMake. 14 | 15 | | Duration | Content | 16 | | --- | --- | 17 | | 15 minutes | [automation_slides.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/05_testing_and_ci/automation_slides.md) | 18 | | 90 minutes | [automation_exercise.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/05_testing_and_ci/automation_exercise.md) | 19 | | 90 minutes | [boost_testing_exercise.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/05_testing_and_ci/boost_testing_exercise.md) | 20 | | 20 minutes | [boost_testing_intro_slides.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/05_testing_and_ci/boost_testing_intro_slides.md), [boost_testing_intro_demo.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/05_testing_and_ci/boost_testing_intro_demo.md) | 21 | | 10 minutes | [boost_testing_precice_demo.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/05_testing_and_ci/boost_testing_precice_demo.md) | 22 | | 60 minutes | [boost_testing_sidemade_demo.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/05_testing_and_ci/boost_testing_sidemade_demo.md) | 23 | | 30 minutes | [intro_slides.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/05_testing_and_ci/intro_slides.md) | 24 | | 60 minutes | [python_testing_slides.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/05_testing_and_ci/python_testing_slides.md), [python_testing_demo.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/05_testing_and_ci/python_testing_demo.md) | 25 | | 45 minutes | [github_actions_slides.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/05_testing_and_ci/github_actions_slides.md), [github_actions_demo.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/05_testing_and_ci/github_actions_demo.md) | 26 | | 30 minutes | [gitlab_ci_slides.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/05_testing_and_ci/gitlab_ci_slides.md), [gitlab_ci_demo.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/05_testing_and_ci/gitlab_ci_demo.md) | 27 | | 90 minutes | [python_testing_exercise.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/05_testing_and_ci/python_testing_exercise.md) | 28 | -------------------------------------------------------------------------------- /05_testing_and_ci/automation_exercise.md: -------------------------------------------------------------------------------- 1 | # Automating Workflows with GitHub Actions 2 | 3 | In this exercise, we create automated workflows and pipelines based on GitHub Actions. We are again working with our beloved diffusion Python code. 4 | 5 | Deadline: **Wednesday, January 29th, 2025, 09:00** 6 | 7 | ## Preparations 8 | 9 | Import the [automation exercise repository](https://github.com/Simulation-Software-Engineering/automation-exercise-wt2425) in your own account/namespace on GitHub and name it `automation-exercise`. It is also easier to add badges (see below) to the `README.md` if the repository is under your control. 10 | 11 | **Note**: We cannot work with forks here because GitHub Actions may not work in pull requests without explicit approval of the owner of the target repository 12 | 13 | ## Task descriptions 14 | 15 | The CI workflow should be triggered by `push` events and should have three jobs: 16 | 17 | 1. Name: `style_check`. Check the code style of the Python files in the repository using [`black`](https://github.com/psf/black). This check should fail if `black` finds any file to reformat. 18 | 2. Name: `test`. Run tests while also collecting coverage information via `pytest` and the `coverage` package. The coverage information (stored in the file `.coverage`) has to be stored and handed over to the next step. 19 | - The intermediate coverage information (`.coverage`) should be kept for one day. This is the minimum amount of time an artifact must be stored. 20 | - Adding a `.gitignore` file helps to not accidentally commit the temporary coverage files. 21 | 3. Name: `coverage_report`. Create a coverage report (`coverage report`) in the terminal and afterwards also convert the coverage information into XML format (`coverage xml`). For this you have to reuse the coverage information (i.e. `.coverage`) from the previous step. 22 | - Running `coverage report` allows us to inspect the coverage directly in the workflow's output. Saving the XML file allows us to analyze the coverage in more detail with other tools if needed. 23 | - The resulting file `coverage.xml` should be kept for 14 days. 24 | 25 | Once the workflow runs successfully, add a [GitHub workflow status badge](https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/adding-a-workflow-status-badge) for your workflow to the `README.md`. Label this badge with "SSE CI/CD". 26 | 27 | ## Submission 28 | 29 | - Submit your solution via an issue in the *upstream* [automation exercise repository](https://github.com/Simulation-Software-Engineering/automation-exercise-wt2425). The issue should be named `[USERNAME] Automation exercise`, e.g., `[uekermbn] Automation exercise`. Please use your GitLab username here. Add a link to your GitHub repository in the issue. If necessary, add further explanations in the issue description. 30 | 31 | ## Bonus: GitLab CI/CD 32 | 33 | Realize the same workflows in GitLab CI/CD :) 34 | 35 | - Create a new project under your namespace and push your existing automation repository to this remote. 36 | - Create a VM on [bwCloud](https://www.bw-cloud.org/) and create a GitLab Runner with a Docker executor there. Register the runner in your repository. 37 | - Add a link to this repository in the submission issue above and, if necessary, explain what you did. Be sure that your repository is visible to us (either public or add `gchourdakis`). 38 | -------------------------------------------------------------------------------- /05_testing_and_ci/automation_slides.md: -------------------------------------------------------------------------------- 1 | --- 2 | type: slide 3 | slideOptions: 4 | transition: slide 5 | width: 1400 6 | height: 900 7 | margin: 0.1 8 | --- 9 | 10 | 34 | 35 | # Workflow Automation 36 | 37 | --- 38 | 39 | ## Learning Goals 40 | 41 | - Name and explain common workflows to automate in RSE. 42 | - Explain the differences between the various continuous methodologies. 43 | - Explain why automation is crucial in RSE. 44 | - Write basic automation scripts for GitHub Actions. 45 | - Read basic automation scripts for GitLab CI/CD. 46 | - Name and roughly explain the necessary steps to host GitLab Runners yourself. 47 | 48 | --- 49 | 50 | ## Why Automation? 51 | 52 | - Automatize tasks 53 | - Run tests frequently, give feedback early etc. 54 | - Ensure reproducible test environments 55 | - Cannot forget automatized tasks 56 | - Less burden to developer (and their workstation) 57 | - Avoid manual errors 58 | - Process often integrated in development workflow 59 | - Example: Support by Git hooks or Git forges 60 | 61 | --- 62 | 63 | ## Typical Automation Tasks in RSE 64 | 65 | - Check code formatting and quality 66 | - Compile and test code for different platforms 67 | - Periodically run tasks 68 | - Big tests, nightly builds... 69 | - Build documentation and deploy it 70 | - Generate coverage reports and visualization 71 | - Build, package, and upload releases 72 | 73 | --- 74 | 75 | ## Continuous Methodologies (1/2) 76 | 77 | - **Continuous Integration** (CI) 78 | - Continuously integrate changes into "main" branch 79 | - Avoids "merge hell" 80 | - Relies on testing and checking code continuously 81 | - Should be automatized 82 | 83 | --- 84 | 85 | ## Continuous Methodologies (2/2) 86 | 87 | - **Continuous Delivery** (CD) 88 | - Software is in a state that allows new release at any time 89 | - Software package is built 90 | - Actual release triggered manually 91 | - **Continuous Deployment** (CD) 92 | - Software is in a state that allows new release at any time 93 | - Software package is built 94 | - Actual release triggered automatically (continuously) 95 | 96 | --- 97 | 98 | ## Automation Services/Software 99 | 100 | - [GitHub Actions](https://github.com/features/actions) 101 | - [GitLab CI/CD](https://docs.gitlab.com/ee/ci/) 102 | - [Circle CI](https://circleci.com/) 103 | - [Travis CI](https://www.travis-ci.com/) 104 | - [Jenkins](https://www.jenkins.io/) 105 | - ... 106 | 107 | --- 108 | 109 | ## Further Reading 110 | 111 | - [GitHub Actions documentation](https://docs.github.com/en/actions) 112 | - [GitLab CI/CD documentation](https://docs.gitlab.com/ee/ci/) 113 | - [What is Continuous Integration?](https://www.atlassian.com/continuous-delivery/continuous-integration) 114 | -------------------------------------------------------------------------------- /05_testing_and_ci/boost_testing_exercise.md: -------------------------------------------------------------------------------- 1 | # Boost.Test and CTest in Action: SideMade Exercise 2 | 3 | In this exercise, you extend and automate the unit tests of the [SideMade code](https://github.com/Simulation-Software-Engineering/testing-boost-exercise-wt2425), on which we worked during the lecture. 4 | 5 | Deadline: **Thursday, February 5, 2025, 09:00** 6 | 7 | ## Preparation and Submission 8 | 9 | - Import the [testing boost exercise repository](https://github.com/Simulation-Software-Engineering/testing-boost-exercise-wt2425) in your own account/namespace on GitHub and name it `testing-boost-exercise` again. **Note**: We cannot work with forks here because GitHub Actions may not work in pull requests without explicit approval of the owner of the target repository 10 | - Create a new branch `extend-tests` from `main` and work in this branch. 11 | - To submit, open a PR from `extend-tests` to `main` in your own repository. Then paste a link to this PR in a new issue in the original repository. Use `[GITLAB-USERNAME] Boost test exercise` as title of the issue. 12 | 13 | ## Task Descriptions 14 | 15 | ### (1) IO Testing 16 | 17 | - Following similar style and structure as in the lecture, implement a simple unit test for the function `matrixIO::openData`. 18 | 19 | ### (2) Automation 20 | 21 | - Similar to last week, add a GitHub Action workflow with three jobs: 22 | - Style: Check whether all `cpp` and `hpp` files (in `src` and `tests` are formatted correctly using `clang-format` (see `README.md`). 23 | - Build: Check whether the code builds successfully. 24 | - Test: Check whether all tests run successfully. 25 | - Add a corresponding [GitHub workflow status badge](https://docs.github.com/en/actions/monitoring-and-troubleshooting-workflows/adding-a-workflow-status-badge) to the `README.md`. 26 | 27 | ## Optional Tasks 28 | 29 | - Extend the automation with a [build matrix](https://docs.github.com/en/actions/using-jobs/using-a-build-matrix-for-your-jobs). Test whether your code builds in Debug and in Release mode, and with the gcc and the clang compiler. Make use of CMake variables to modify these parameters. 30 | - Implement more tests. 31 | 32 | ## Hints and Remarks 33 | 34 | - When importing a project on GitHub, it could be that by default actions are disabled. You can enable them via `Settings -> Actions -> General -> Allow all actions`. 35 | - Be careful: the style job should not format the code, but rather report whether the code was formatted correctly or not. There are different ways how to do this. You could use the option `--dry-run`, but then you still need to interpret warnings as errors with `-Werror`. Or you could format inplace (`-i`) and use `git diff`. 36 | - Try to use the build from the build job for the tests in the test job by uploading and downloading the build as an artifact. Previously, there was an issue with file permissions, where archiving was [a known workaround](https://github.com/actions/upload-artifact/tree/v3.2.1?tab=readme-ov-file#maintaining-file-permissions-and-case-sensitive-files). This should no longer be a problem. 37 | -------------------------------------------------------------------------------- /05_testing_and_ci/boost_testing_intro_demo.md: -------------------------------------------------------------------------------- 1 | # Introduction to Boost.Test – Hello World Demo 2 | 3 | Code is in `05_testing_and_ci/examples/boost_test` 4 | 5 | - Explain code 6 | - Uses UTF in header-only mode (can also be used as library, more later) 7 | - Give name to `BOOST_TEST_MODULE` 8 | - UTF defines `main` function for us (could also be done manually) 9 | - Single test case, which is added automatically with `AUTO` (could also be done manually) 10 | - Build: `g++ -o test helloWorldTest.cpp` 11 | - Run: `./test` 12 | - Inspect: `./test --help` and `./test --list_content` (`*` means that a testcase is active) 13 | -------------------------------------------------------------------------------- /05_testing_and_ci/boost_testing_precice_demo.md: -------------------------------------------------------------------------------- 1 | # Boost.Test in the Real World: preCICE Demo 2 | 3 | Look around preCICE in the terminal + text editor. 4 | 5 | ## First Look 6 | 7 | - `cd src && ls`: preCICE is organized in several namespaces. 8 | - `cd math && ls`: Each namespace has a `tests` folder 9 | - Look at `math/tests/DifferenceTest.cpp` 10 | - Imports `testing/Testing.hpp`, there we handle UTF imports 11 | - Test suite for `math` namespace and test suite per file (here `math/differences.hpp`) 12 | - `BOOST_CHECK` actually not recommended to use. Is used internally by `BOOST_TEST`. 13 | - Powerful macro `PRECICE_TEST()` to setup test context (resources, singletons, not data) 14 | 15 | ## Unit vs. Integration Tests 16 | 17 | - Clear separation in preCICE: integration tests only directly use API of preCICE. 18 | - They are located in `tests` folder. 19 | - Look at `tests/serial/initialize-data/Explicit.cpp`: 20 | - Explain `PRECICE_TEST_SETUP` and how it is used: test is run on two MPI ranks living in seperate MPI communicators. 21 | - Information can be accessed via `context`. 22 | - More information: [blog post on bssw.io on multiphysics testing](https://bssw.io/blog_posts/overcoming-complexity-in-testing-multiphysics-coupling-software) 23 | 24 | ## White-Box Testing 25 | 26 | - Not a UTF feature, but an independent C++ trick 27 | - Explain black-box testing: 28 | - A class has a public API and we only use this API for testing. 29 | - Makes a lot of sense 30 | - Sometimes, we want white-box testing: 31 | - Why? Access and/or check private members 32 | - Could `friend` the test, but this quickly gets out of hand 33 | - Example: class `src/time/Waveform.hpp` 34 | - Has public and private members. We want to check the private members in tests. 35 | - Does not `friend` every test, but only `WaveformFixture` 36 | - `src/testing/WaveformFixture.hpp` has functions to access private members 37 | - This fixture is used in many tests in `src/time/tests/WaveformTests` (but not handed over to test like normal UTF fixtures) 38 | 39 | ## Test Matrices 40 | 41 | - Look at `tests/serial/mapping-nearest-projection/QuadMappingDiagonalNearestProjectionEdgesTallKite.cpp`: 42 | - Define test matrix with data sets: `boost::unit_test::data::make` 43 | 44 | ## Boost Test Context 45 | 46 | - Look at `src/mapping/tests/NearestProjectionMappingTest.cpp`: 47 | - `BOOST_TEST_CONTEXT` outputs context information on failure 48 | 49 | ## CMake 50 | 51 | - Look at `cmake/CTestConfig.cmake`: Complicated, but more things tested than only UTF (search for `add_test`) 52 | 53 | ## CTest in Parallel 54 | 55 | - Build preCICE and run tests via `ctest -j 16`, runs tests in parallel, automatic feature of CTest 56 | -------------------------------------------------------------------------------- /05_testing_and_ci/examples/automation/Dockerfile: -------------------------------------------------------------------------------- 1 | # Use Alpine Linux to get images of minimal size. Alpine is 2 | # optimized for containerized applications and normally creates 3 | # container much smaller than other base images/Linux distributions. 4 | FROM alpine:3.15.0 5 | 6 | # Install Python and make sure that we can call it 7 | # as `python` instead of `python3` 8 | RUN apk add --no-cache python3 sudo \ 9 | && ln -sf python3 /usr/bin/python 10 | 11 | # pip should be installed and used as a user. Therefore we 12 | # create a new user "testuser" and change to this user. 13 | ARG USER_NAME=testuser 14 | RUN addgroup -g 1000 $USER_NAME \ 15 | && adduser -u 1000 -G $USER_NAME -D $USER_NAME 16 | RUN echo "$USER_NAME" ALL=\(root\) NOPASSWD:ALL > "/etc/sudoers.d/$USER_NAME" && \ 17 | chmod 0440 "/etc/sudoers.d/$USER_NAME" 18 | USER $USER_NAME 19 | 20 | # Install pip, make sure it can be called as `pip` instead of 21 | # `pip3``. The installation location of `pip` has to be added 22 | # to the path. Afterwards, we can install missing tools via 23 | # `pip`. 24 | ENV PATH=/home/$USER_NAME/.local/bin:${PATH} 25 | RUN python -m ensurepip --upgrade \ 26 | && ln -sf /home/$USER_NAME/.local/bin/pip3 /home/$USER_NAME/.local/bin/pip \ 27 | && pip install --upgrade pip 28 | RUN pip install --user black 29 | 30 | CMD ["/bin/sh"] 31 | -------------------------------------------------------------------------------- /05_testing_and_ci/examples/automation/Vagrantfile: -------------------------------------------------------------------------------- 1 | # -*- mode: ruby -*- 2 | # vi: set ft=ruby : 3 | 4 | # All Vagrant configuration is done below. The "2" in Vagrant.configure 5 | # configures the configuration version (we support older styles for 6 | # backwards compatibility). Please don't change it unless you know what 7 | # you're doing. 8 | Vagrant.configure("2") do |config| 9 | config.vm.box = "ubuntu/focal64" 10 | 11 | # Run basic installation 12 | config.vm.provision :shell, path: "bootstrap.sh" 13 | 14 | config.vm.provider "virtualbox" do |vb| 15 | vb.name = "sse-automation" 16 | vb.memory = 2048 17 | vb.cpus = 2 18 | end 19 | end 20 | -------------------------------------------------------------------------------- /05_testing_and_ci/examples/automation/bootstrap.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | 4 | apt-get update 5 | apt-get install -y vim tmux fonts-powerline git 6 | 7 | # Get config 8 | git clone https://github.com/ajaust/dotfiles.git "${HOME}/dotfiles" 9 | pushd ${HOME}/dotfiles && ./setup_dotfiles.sh && popd 10 | 11 | # Docker 12 | apt-get install -y ca-certificates curl gnupg lsb-release 13 | # Add key 14 | curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg 15 | # Set up stable repository 16 | echo \ 17 | "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu \ 18 | $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null 19 | 20 | # Install Docker engine 21 | apt-get update 22 | apt-get install -y docker-ce docker-ce-cli containerd.io 23 | 24 | # Add user to docker group for sudoless Docker 25 | # Note: The change in usergroup is only picked up after 26 | # the VM has been stopped and restarted, i.e., run 27 | # 28 | # vagrant halt 29 | # vagrant up 30 | # 31 | # after the setup finished. The `docker` commands below 32 | # work since we have super user rights during the 33 | # bootstrapping. 34 | usermod -aG docker vagrant 35 | 36 | # Test installation 37 | docker run hello-world 38 | 39 | # Install act 40 | curl https://raw.githubusercontent.com/nektos/act/master/install.sh | sudo bash 41 | 42 | # Install GitLab runner 43 | docker run -d --name gitlab-runner --restart always \ 44 | -v /srv/gitlab-runner/config:/etc/gitlab-runner \ 45 | -v /var/run/docker.sock:/var/run/docker.sock \ 46 | gitlab/gitlab-runner:latest 47 | -------------------------------------------------------------------------------- /05_testing_and_ci/examples/boost_test/helloWorldTest.cpp: -------------------------------------------------------------------------------- 1 | #define BOOST_TEST_MODULE Hello World Test 2 | #include 3 | 4 | BOOST_AUTO_TEST_CASE(SomeTest) 5 | { 6 | int i = 1; 7 | BOOST_TEST(i == 2); 8 | } 9 | -------------------------------------------------------------------------------- /05_testing_and_ci/examples/python_testing/mean_data.csv: -------------------------------------------------------------------------------- 1 | 14,38,57,88,1,18,198 2 | 59.14 3 | -------------------------------------------------------------------------------- /05_testing_and_ci/examples/python_testing/operations.py: -------------------------------------------------------------------------------- 1 | """ 2 | A set of mathematical operations. 3 | """ 4 | 5 | 6 | def find_max(data): 7 | """ 8 | Find maximum of all elements of a given list 9 | 10 | Parameters 11 | ---------- 12 | data : list 13 | List of data. Elements are numbers 14 | 15 | Returns 16 | ------- 17 | find_max : float 18 | Maximum of list 19 | """ 20 | # Check that the input list has numbers 21 | for n in data: 22 | assert type(n) == int or type(n) == float 23 | 24 | max_num = data[0] # Assume the first number is the maximum 25 | for n in data: 26 | if n > max_num: 27 | max_num = n 28 | 29 | return max_num 30 | 31 | 32 | def find_mean(data): 33 | """ 34 | Find mean of all elements of a given list 35 | 36 | Parameters 37 | ---------- 38 | data : list 39 | List of data. Elements are numbers 40 | 41 | Returns 42 | ------- 43 | float : float 44 | Mean of list 45 | """ 46 | # Check that the input list has numbers 47 | for n in data: 48 | assert type(n) == int or type(n) == float 49 | 50 | return sum(data) / len(data) 51 | 52 | 53 | def main(): 54 | data = [5, 3, 14, 27, 4, 9] 55 | 56 | maximum = find_max(data) 57 | print("Maximum of {} is {}".format(data, maximum)) 58 | 59 | mean = find_mean(data) 60 | print("Average of {} is {}".format(data, mean)) 61 | 62 | 63 | if __name__ == "__main__": 64 | main() 65 | -------------------------------------------------------------------------------- /05_testing_and_ci/examples/python_testing/test_operations.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for mathematical operations functions. 3 | """ 4 | 5 | from operations import find_max, find_mean 6 | import pytest 7 | import csv 8 | 9 | 10 | # Unit test 11 | def test_find_max(): 12 | """ 13 | Test operations.find_max 14 | """ 15 | # Fixture 16 | data = [43, 32, 167, 18, 1, 209] 17 | 18 | # Expected result 19 | expected_max = 209 20 | 21 | # Actual result 22 | actual_max = find_max(data) 23 | 24 | # Test 25 | assert actual_max == expected_max 26 | 27 | 28 | # Unit test 29 | def test_find_mean(): 30 | """ 31 | Test operations.find_mean 32 | """ 33 | # Fixture 34 | data = [43, 32, 167, 18, 1, 209] 35 | 36 | # Expected result 37 | expected_mean = 78.33 38 | # expected_mean = pytest.approx(78.33, abs=0.01) 39 | 40 | # Actual result 41 | actual_mean = find_mean(data) 42 | 43 | # Test 44 | assert actual_mean == expected_mean 45 | 46 | 47 | # Integration test 48 | def test_mean_of_max(): 49 | """ 50 | Test operations.find_max and operations.find_mean 51 | """ 52 | # Fixture 53 | data1 = [43, 32, 167, 18, 1, 209] 54 | data2 = [3, 13, 33, 23, 498] 55 | 56 | # Expected result 57 | expected_mean_of_max = 353.5 58 | 59 | maximum1 = find_max(data1) 60 | maximum2 = find_max(data2) 61 | 62 | # Actual result 63 | actual_mean_of_max = find_mean([maximum1, maximum2]) 64 | 65 | # Test 66 | assert actual_mean_of_max == expected_mean_of_max 67 | 68 | 69 | # Regression test 70 | def test_regression_mean(): 71 | """ 72 | Test operations.find_mean on a previously generated dataset 73 | """ 74 | with open("mean_data.csv") as f: 75 | rows = csv.reader(f, quoting=csv.QUOTE_NONNUMERIC) 76 | # Fixture 77 | data = next(rows) 78 | 79 | # Expected result 80 | reference_mean = next(rows) 81 | 82 | # Actual result 83 | actual_mean = find_mean(data) 84 | 85 | expected_mean = pytest.approx(reference_mean[0], abs=0.01) 86 | 87 | # Test 88 | assert actual_mean == expected_mean 89 | -------------------------------------------------------------------------------- /05_testing_and_ci/examples/python_testing/test_operations_unittests.py: -------------------------------------------------------------------------------- 1 | """ 2 | Tests for mathematical operations functions. 3 | """ 4 | from operations import find_max, find_mean 5 | import unittest 6 | from unittest import TestCase 7 | import csv 8 | 9 | 10 | class TestOperations(TestCase): 11 | """ 12 | Test suite for mathematical operations functions. 13 | """ 14 | def setUp(self): 15 | # Fixture 16 | self.data1 = [43, 32, 167, 18, 1, 209] 17 | self.data2 = [3, 13, 33, 23, 498] 18 | 19 | # Unit test 20 | def test_find_max(self): 21 | """ 22 | Test operations.find_max 23 | """ 24 | # Expected result 25 | expected_max = 209 26 | 27 | # Actual result 28 | actual_max = find_max(self.data1) 29 | 30 | # Test 31 | self.assertEqual(actual_max, expected_max) 32 | 33 | # Unit test 34 | def test_find_mean(self): 35 | """ 36 | Test operations.find_mean 37 | """ 38 | # Expected result 39 | expected_mean = 78.33 40 | 41 | # Actual result 42 | actual_mean = find_mean(self.data1) 43 | 44 | # Test 45 | self.assertAlmostEqual(actual_mean, expected_mean, 2) 46 | 47 | # Integration test 48 | def test_mean_of_max(self): 49 | """ 50 | Test operations.find_max and operations.find_mean 51 | """ 52 | # Expected result 53 | expected_mean_of_max = 353.5 54 | 55 | maximum1 = find_max(self.data1) 56 | maximum2 = find_max(self.data2) 57 | 58 | # Actual result 59 | actual_mean_of_max = find_mean([maximum1, maximum2]) 60 | 61 | # Test 62 | self.assertEqual(actual_mean_of_max, expected_mean_of_max) 63 | 64 | # Regression test 65 | def test_regression_mean(self): 66 | """ 67 | Test operations.find_mean on a previously generated dataset 68 | """ 69 | with open("mean_data.csv") as f: 70 | rows = csv.reader(f, quoting=csv.QUOTE_NONNUMERIC) 71 | # Fixture 72 | data = next(rows) 73 | 74 | # Expected result 75 | reference_mean = next(rows) 76 | 77 | # Actual result 78 | actual_mean = find_mean(data) 79 | 80 | # Test 81 | self.assertAlmostEqual(actual_mean, reference_mean[0], 2) 82 | 83 | if __name__ == "__main__": 84 | # Run the tests 85 | unittest.main() 86 | -------------------------------------------------------------------------------- /05_testing_and_ci/examples/python_testing/tox.toml: -------------------------------------------------------------------------------- 1 | requires = ["tox>=4"] 2 | env_list = ["testing"] 3 | 4 | [env.testing] 5 | description = "Run pytest" 6 | deps = ["pytest>=8"] 7 | commands = [["pytest"]] 8 | -------------------------------------------------------------------------------- /05_testing_and_ci/figs/automation_exercise/gitlab_coverage.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Simulation-Software-Engineering/Lecture-Material/d58a42f0ddd51ed992519d7fc067d7866c8bb3fa/05_testing_and_ci/figs/automation_exercise/gitlab_coverage.png -------------------------------------------------------------------------------- /05_testing_and_ci/github_actions_demo.md: -------------------------------------------------------------------------------- 1 | # Demo: Automation with GitHub Actions 2 | 3 | ## 1. Setting up a Test Job 4 | 5 | - Clone [automation lecture repository](https://github.com/Simulation-Software-Engineering/automation-lecture-wt2425) and run code and tests 6 | - Set up workflow file 7 | 8 | ```bash 9 | mkdir -p .github/workflows 10 | cd .github/workflows 11 | vi testing.yml 12 | ``` 13 | 14 | - In the first go, we only want to run the `unittest` tests. 15 | - Edit `testing.yml` to have following content 16 | 17 | ```yaml 18 | name: Testing workflow 19 | 20 | on: [push, pull_request] 21 | 22 | jobs: 23 | test: 24 | runs-on: ubuntu-latest 25 | steps: 26 | - uses: actions/checkout@v2 27 | - uses: actions/setup-python@v2 28 | with: 29 | python-version: '3.8' 30 | - name: "Run unittest" 31 | run: python -m unittest 32 | ``` 33 | 34 | - `runs-on` does **not** refer to a Docker container, but to a runner tag. 35 | - This specific Python version as it is the current version on my laptop (and let's say I want to reproduce this environment) 36 | - Add, commit, push 37 | - After the push, inspect "Action" panel on GitHub repository 38 | - GitHub will schedule a run (yellow dot) 39 | - Hooray. We have set up our first action. 40 | - Failing test example: 41 | - Edit settings on GitHub that one can only merge if all tests pass: 42 | - Settings -> Branches -> Branch protection rule 43 | - Choose `main` branch 44 | - Enable "Require status checks to pass before merging". Optionally enable "Require branches to be up to date before merging" 45 | - Choose status checks that need to pass: `test` 46 | - Click on "Create" at bottom of page. 47 | - Create a new branch `break-code`. 48 | - Edit `operations.py`, break the code, commit it and push it to the branch. Afterwards open a new PR and inspect the failing test. We are also not able to merge the changes as the "Merge" button should be inactive. 49 | 50 | ## 2. Extend Action to Have Several Dependent Jobs 51 | 52 | - Briefly explain what `black` is: a compact, easy-to-use formatting tool. 53 | - Run `black` locally on repository and explain what it does. 54 | 55 | ```bash 56 | black --check . 57 | ``` 58 | 59 | - Add an empty line somewhere and run again. 60 | - Run without `--check` and `git status`. 61 | 62 | - Adding additional jobs by editing on GitHub. The workflow should have the following content: 63 | 64 | ```yaml 65 | name: Testing workflow 66 | 67 | on: [push, pull_request] 68 | 69 | jobs: 70 | style: 71 | runs-on: ubuntu-latest 72 | steps: 73 | - uses: actions/checkout@v2 74 | - uses: actions/setup-python@v2 75 | with: 76 | python-version: '3.8' 77 | - name: "Install style checker" 78 | run: pip install black 79 | - name: "Run style check" 80 | run: black --check . 81 | build: 82 | needs: style 83 | runs-on: ubuntu-latest 84 | env: 85 | PROJECT_NAME: "Automation Lecture" 86 | steps: 87 | - name: "Run build phase" 88 | run: echo "Building project $PROJECT_NAME" 89 | test: 90 | needs: build 91 | runs-on: ubuntu-latest 92 | steps: 93 | - uses: actions/checkout@v2 94 | - uses: actions/setup-python@v2 95 | with: 96 | python-version: '3.8' 97 | - name: "Run unittest" 98 | run: python -m unittest 99 | ``` 100 | 101 | - We need to run `actions/checkout@v2` in each job 102 | - We could share the repository between jobs via artifacts, but that is uncommon. 103 | - We need to run `actions/setup-python@v2` since jobs do not share the environment. 104 | - We specify dependencies by `needs` such that the steps run after each other. 105 | - We do not have a real build step since it is Python. However, this might be interesting for compiled code. 106 | 107 | ## 3. Other Workflows 108 | 109 | - Show workflows of [preCICE](https://github.com/precice/precice) 110 | - Show `Actions` tab 111 | - `Build and test` job, click on a run 112 | - Jobs created through test matrix 113 | - Click on a job, click on a few steps 114 | - Show `workflows` folder, click on `Build and Test` 115 | - Only one job. `build`, `test`, ... are modeled as steps 116 | 117 | ## 4. act Demo 118 | 119 | - `act` is for quick checks while developing workflows, not for developing the code 120 | - Check available jobs 121 | 122 | ```bash 123 | act -l 124 | ``` 125 | 126 | - Run jobs for `push` event (default event) 127 | 128 | ```bash 129 | act 130 | ``` 131 | 132 | - Run a specific job 133 | 134 | ```bash 135 | act -j test 136 | ``` 137 | -------------------------------------------------------------------------------- /05_testing_and_ci/gitlab_ci_demo.md: -------------------------------------------------------------------------------- 1 | # Demo: GitLab Runner 2 | 3 | Test code in [automation lecture repository](https://gitlab-sim.informatik.uni-stuttgart.de/simulation-software-engineering-wite2425/lecture-automation) on our GitLab instance 4 | 5 | ## Inspect Repository 6 | 7 | - Same code as for GitHub Actions, but now `gitlab-ci.yml` instead of `.github/workflows/...` 8 | - Look at `gitlab-ci.yml`: 9 | - Does the same things as GitHub actions 10 | - Runs on a Docker image `ajaust/automation-lecture` 11 | - Self-hosted GitLab instance: where should the CI run? 12 | - Click on CI symbol of latest commit: pipeline is stuck 13 | - Click on job: "doesn't have any runners" 14 | - Edit in pipeline editor -> Visualize 15 | - Settings -> CI/CD -> Runners -> Specific runners 16 | - URL and Token; we will need this in a minute 17 | 18 | ## Inspect bwCloud 19 | 20 | - bwCloud: many services academia in BW can use; e.g. VMs 21 | - Go to dashboard: https://portal.bw-cloud.org/ and login with Uni Stuttgart account 22 | - I have already set up a VM. What I did: 23 | - Add public SSH key 24 | - Instances -> Launch instance 25 | - Ubuntu 22.04 26 | - Flavor: m1.small 27 | - `sudo apt update && sudo apt -y upgrade` 28 | - `sudo apt install -y docker.io` 29 | - VM is up and running, connect to it: `ssh ubuntu@` 30 | 31 | ## Setup GitLab Runner 32 | 33 | - Install GitLab Runner via Docker: 34 | 35 | ```bash 36 | sudo docker run -d --name gitlab-runner --restart always \ 37 | -v /srv/gitlab-runner/config:/etc/gitlab-runner \ 38 | -v /var/run/docker.sock:/var/run/docker.sock \ 39 | gitlab/gitlab-runner:latest 40 | ``` 41 | 42 | - `docker run -d --name gitlab-runner --restart always` runs the container in the background (`-d` means detached) names it `gitlab-runner` and makes sure that it always runs. The container is automatically restarted once it stops/crashes. If you want to stop the container, you have to stop it manually (`docker container stop`). 43 | - `-v /srv/gitlab-runner/config:/etc/gitlab-runner` mounts the directory `/srv/gitlab-runner/config` into the container. 44 | - `-v /var/run/docker.sock:/var/run/docker.sock` mounts important Docker files into the container such that the container can start other containers (for pipelines). 45 | - `gitlab/gitlab-runner:latest` is the GitLab Runner image used from Docker Hub. 46 | - Summary: We start a manager Docker container (called runner) that will handle the jobs 47 | 48 | - More information: 49 | - [installation instructions](https://docs.gitlab.com/runner/install/) 50 | - And more specific [Run GitLab Runner in a container](https://docs.gitlab.com/runner/install/docker.html) 51 | 52 | ## Register Runner 53 | 54 | - `sudo docker run --rm -it -v /srv/gitlab-runner/config:/etc/gitlab-runner gitlab/gitlab-runner register` 55 | - URL: `https://gitlab-sim.informatik.uni-stuttgart.de/` 56 | - Token: see above 57 | - Description: `SSE Automation Demo Runner` 58 | - No tags, no maintenance note 59 | - Executor: `docker` 60 | - Default Docker image: `alpine:latest` (used for pipelines that do not specify any Docker image themselves, can be overwritten in configuration of pipeline) 61 | - `sudo vi /srv/gitlab-runner/config/config.toml` 62 | - Verify that there is now a runner in the repo settings 63 | - Verify that pipeline now ran 64 | 65 | - More information: 66 | - [Executors and their abilities](https://docs.gitlab.com/runner/executors/) 67 | - [Registering runners](https://docs.gitlab.com/runner/register/index.html#docker). 68 | -------------------------------------------------------------------------------- /05_testing_and_ci/python_testing_slides.md: -------------------------------------------------------------------------------- 1 | --- 2 | type: slide 3 | slideOptions: 4 | transition: slide 5 | width: 1400 6 | height: 900 7 | margin: 0.1 8 | --- 9 | 10 | 25 | 26 | # Testing a Python Code 27 | 28 | --- 29 | 30 | ## pytest 31 | 32 | - Library to write and manage tests. 33 | - Command-line tool also called `pytest`. 34 | - Install using pip: `pip install -U pytest`. 35 | - All tests need to be in files named `test_*.py`. 36 | - Each test function needs to be named as `test_*`. 37 | - pytest gives a detailed description of assertion checks. 38 | 39 | --- 40 | 41 | ## pytest Demo 42 | 43 | --- 44 | 45 | ## unittest 46 | 47 | - Python framework specifically designed to run, monitor and automate unit tests. 48 | - Many features like test automation, sharing of setup and shutdown of tests, etc. 49 | - Use the base class `unittest.TestCase` to create a test suite. 50 | - Command-line interface: `python -m unittest test_module1 test_module2 ...`. 51 | 52 | --- 53 | 54 | ## unittest Demo 55 | 56 | --- 57 | 58 | ## coverage 59 | 60 | - Python library to check code coverage. Installation: `pip install coverage`. 61 | - Testing frameworks can be run via coverage to generate code coverage data while tests run. 62 | - Code coverage information can be viewed on the terminal using: `coverage report -m`. 63 | 64 | --- 65 | 66 | ## coverage Demo 67 | 68 | --- 69 | 70 | ## tox 71 | 72 | - Environment orchestrator to setup and execute various tools for a project. 73 | - Creates virtual environments for each process. 74 | - Processes include testing, linting, building, documentation generation, and more. 75 | - Configuration via `tox.toml` or `tox.ini` file. 76 | 77 | --- 78 | 79 | ## tox Demo 80 | 81 | --- 82 | 83 | ## Other Testing Frameworks 84 | 85 | - [nose](https://pypi.org/project/nose2/) is an extension to `unittest` with added plugins. 86 | - [testify](https://pypi.org/project/testify/) based on unittest and nose with additional features. 87 | - [robotframework](https://pypi.org/project/robotframework/) is a generic automation framework. 88 | 89 | --- 90 | 91 | ## Further Reading 92 | 93 | - [pytest documentation](https://docs.pytest.org/en/6.2.x/) 94 | - [unittest documentation](https://docs.python.org/3/library/unittest.html) 95 | - [tox user guide](https://tox.wiki/en/4.23.2/user_guide.html#user-guide) 96 | -------------------------------------------------------------------------------- /06_miscellaneous/README.md: -------------------------------------------------------------------------------- 1 | # Miscellaneous 2 | 3 | Learning goals: 4 | 5 | - Know the basics about several *other* important things concerning research software engineering: FAIRness of research data and research software, FLOSS licenses, versioning schemes, repository layout standards, and more. 6 | 7 | | Duration | Content | 8 | | --- | --- | 9 | | 20 minutes | [doi_zenodo_darus_slides.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/06_miscellaneous/doi_zenodo_darus_slides.md) | 10 | | 20 minutes | [floss_licenses_slides.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/06_miscellaneous/floss_licenses_slides.md) | 11 | | 20 minutes | [versioning_slides.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/06_miscellaneous/versioning_slides.md) | 12 | | 20 minutes | [repository_layouts_slides.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/06_miscellaneous/repository_layouts_slides.md) | 13 | | 20 minutes | [tools_alex_slides.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/06_miscellaneous/tools_alex_slides.md), [tools_alex_demo.md](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/06_miscellaneous/tools_alex_demo.md) | 14 | 15 | -------------------------------------------------------------------------------- /06_miscellaneous/doi_zenodo_darus_slides.md: -------------------------------------------------------------------------------- 1 | --- 2 | type: slide 3 | slideOptions: 4 | transition: slide 5 | width: 1400 6 | height: 900 7 | margin: 0.1 8 | --- 9 | 10 | 34 | 35 | # DOI, Zenodo, and Dataverses 36 | 37 | --- 38 | 39 | ## FAIR Research Data 40 | 41 | Research data should be ... 42 | 43 | - **F**indable (metadata, easy to find for humans and machines, ...) 44 | - **A**ccessible (once found, how can one access data, ... open data ...) 45 | - **I**nteroperable (with applications for analysis/storage/processing ...) 46 | - **R**eusable (documentation ...) 47 | 48 | - Principles [published in 2016](https://doi.org/10.1038%2FSDATA.2016.18) by [GO FAIR](https://www.go-fair.org/) 49 | 50 | > A bottom-up, stakeholder-driven and self-governed initiative that aims to implement the FAIR data principles 51 | 52 | - Often research software regarded as data, but there is also [FAIR4RS](https://doi.org/10.15497/RDA00068) 53 | 54 | --- 55 | 56 | ## Challenges 57 | 58 | How to reliably find, access, and reuse research output? 59 | 60 | - Links may change 61 | - People might change affiliation 62 | - Services might stop operating 63 | - Research data might be large 64 | - Closed-source software, data formats, ... 65 | - ... 66 | 67 | --- 68 | 69 | ## Digital Object Identifier (DOI) System 70 | 71 | 72 | 73 | - Organized by [International DOI Foundation (IDF)](https://www.doi.org/) 74 | - Standardized by ISO 75 | - In use since 1998, launched in 2000 76 | - Can refer to resources, machines, data, licenses, books, ... (digital and physical things) 77 | - Used in different fields: Academia, EU, ... 78 | - Different registration agencies 79 | - [Crossref](https://www.crossref.org/): Scholarly data (articles, book chapters etc.) 80 | - [DataCite](https://datacite.org/): Research datasets 81 | 82 | --- 83 | 84 | ## DOI Properties 85 | 86 | - Has a `prefix/suffix` system 87 | - Prefix usually `10.NUMBER` 88 | - `10.` refers to DOI namespace 89 | - Suffix almost arbitrary string 90 | - DOI Examples: 91 | 92 | ```text 93 | 10.1000/182 (DOI handbook) 94 | 10.5281/zenodo.5152939 (DuMuX 3.4.0) 95 | 10.18419/darus-1778 (DaRUS dataset) 96 | ``` 97 | 98 | - Proxy (resolver) example: 99 | 100 | ```text 101 | https://www.doi.org/10.1000/182 102 | https://www.doi.org/10.18419/darus-1778 103 | ``` 104 | 105 | --- 106 | 107 | ## Zenodo 108 | 109 | > Zenodo, a CERN service, is an open dependable home for the long-tail of science, enabling researchers to share and preserve any research outputs in any size and format ... 110 | 111 | - Storage option for 112 | - Papers/preprints 113 | - Datasets 114 | - Software 115 | - Store data under a license 116 | - Assigns DOI to data 117 | - [Zenodo](https://zenodo.org/)'s code is [open source](https://github.com/zenodo/zenodo) 118 | - Example: [DuMuX v3.5.0](https://doi.org/10.5281/zenodo.2479594) 119 | 120 | --- 121 | 122 | ## DaRUS 123 | 124 | University of Stuttgart's [Dataverse](https://dataverse.org/) instance 125 | 126 | - Accessible at 127 | - Extensive metadata description 128 | - Store data under a license 129 | - Assigns DOI to data 130 | - Indexed on [B2FIND](http://b2find.eudat.eu/group/darus), [OpenAIRE](https://explore.openaire.eu/) and [Google Dataset Search](https://datasetsearch.research.google.com) 131 | - Demo instance [DemoDaRUS](https://demodarus.izus.uni-stuttgart.de/) 132 | - Example: [Jaust et al. SFB1313 D02 data set](https://doi.org/10.18419/darus-1778) 133 | 134 | --- 135 | 136 | ## Further Reading 137 | 138 | - [Digital Object Identifier System (DOI)](https://www.doi.org/) 139 | - [DataCite: DOI basics](https://support.datacite.org/docs/doi-basics) 140 | - [Competence Center for Research Data Management](https://www.izus.uni-stuttgart.de/en/fokus/) 141 | -------------------------------------------------------------------------------- /06_miscellaneous/master_thesis_suggestions_slides.md: -------------------------------------------------------------------------------- 1 | --- 2 | type: slide 3 | slideOptions: 4 | transition: slide 5 | width: 1400 6 | height: 900 7 | margin: 0.1 8 | --- 9 | 10 | 34 | 35 | # Master Thesis Suggestions 36 | 37 | --- 38 | 39 | ## General Remark 40 | 41 | - There are rarely explicit topic suggestions online 42 | - Thus, ask and discuss 43 | - Applies to all *"IPVS sim groups"*: SGS (Schulte, Jaust), SC (Pflüger), and US3 (Uekermann) 44 | 45 | --- 46 | 47 | ## Classical Types of Topics 48 | 49 | Various different types of topics are possible: 50 | 51 | - Focus on **numerical methods** 52 | - e.g. new data mapping method in preCICE 53 | - Focus on **applications** 54 | - e.g. applying preCICE in new application field 55 | - Focus on **HPC** 56 | - e.g. load balancing for macro-micro-coupled problems 57 | 58 | All such topics require *"SSE skills"*, but also some simulation background (SimTech or lectures from SGS/SC/US3). 59 | 60 | --- 61 | 62 | ## Pure SSE Topics 63 | 64 | ... are also possible and do not require much simulation background. 65 | 66 | Examples: 67 | 68 | - New approaches to multi-component (multi-software) tests 69 | - Mock testing for middle layers (e.g. preCICE adapters) 70 | 71 | --- 72 | 73 | ## Continuing with your Challenge Software? 74 | 75 | There are always interesting topics concerning preCICE adapters: 76 | 77 | - DuMuX 78 | - FEniCS 79 | - OpenFOAM 80 | - ... 81 | 82 | There are also ongoing collaborations on how to integrate preCICE with other tools: 83 | 84 | - pyiron 85 | - pyMOR 86 | 87 | --- 88 | 89 | ## More 90 | 91 | - If you have a good idea for a contribution to the SSE lecture, this could also be a topic. 92 | -------------------------------------------------------------------------------- /06_miscellaneous/repository_layouts_slides.md: -------------------------------------------------------------------------------- 1 | --- 2 | type: slide 3 | slideOptions: 4 | transition: slide 5 | width: 1400 6 | height: 900 7 | margin: 0.1 8 | --- 9 | 10 | 34 | 35 | # Repository Layouts 36 | 37 | --- 38 | 39 | ## Introduction 40 | 41 | ```text 42 | - README 43 | - LICENSE 44 | - CMakeLists.txt 45 | - cmake/ 46 | - docs/ 47 | - src/ 48 | - tests/ 49 | ... 50 | ``` 51 | 52 | - Structure of repository important 53 | - Makes navigation easier (newcomers and maintainers) 54 | - Clear where to put new files (maintainers) 55 | - There are standards 56 | 57 | --- 58 | 59 | ## Some Layout Standards 60 | 61 | - C and C++ 62 | - [Pitchfork](https://github.com/vector-of-bool/pitchfork) 63 | - Python: 64 | - [The Hitchiker's Guide to Python](https://docs.python-guide.org/writing/structure/) 65 | - Layout also depends on frameworks used 66 | - There are more 67 | 68 | --- 69 | 70 | ## Pitchfork (Layout) 71 | 72 | - Originated from [Reddit](https://www.reddit.com/r/cpp/comments/996q8o/prepare_thy_pitchforks_a_de_facto_standard/): "Prepare thy Pitchforks: A (de facto) Standard Project Layout" 73 | - [GitHub repo](https://github.com/vector-of-bool/pitchfork) includes conventions and tools 74 | 75 | ```text 76 | README 77 | LICENSE 78 | build/ 79 | src/ 80 | include/ (optional) 81 | tests/ 82 | examples/ 83 | external/ 84 | extras/ 85 | data/ 86 | tools/ 87 | docs/ 88 | libs/ 89 | ``` 90 | 91 | - Adding more is not forbidden (`CMakeLists.txt`, `cmake/`, hidden configuration files, ...) 92 | 93 | --- 94 | 95 | ## The Hitchhiker's Guide to Python 96 | 97 | - Based on [K. Reitz's recommendation](https://kennethreitz.org/essays/2013/01/27/repository-structure-and-python) 98 | 99 | ```text 100 | README.rst 101 | LICENSE 102 | setup.py 103 | requirements.txt 104 | sample/__init__.py 105 | sample/core.py 106 | sample/helpers.py 107 | docs/conf.py 108 | docs/index.rst 109 | tests/test_basic.py 110 | tests/test_advanced.py 111 | ``` 112 | 113 | --- 114 | 115 | ## Remarks/Tips 116 | 117 | - Stick to standards as long as meaningful, do not reinvent the wheel 118 | - Use short, but meaningful names 119 | - As few files in root of repository as possible 120 | - Projects / communities might define own structure -> Then, follow them 121 | - Example: [DUNE/DuMuX](https://tu-dresden.de/mn/math/numerik/sander/ressourcen/dateien/sander-getting-started-with-dune-2-7.pdf?lang=en) 122 | -------------------------------------------------------------------------------- /06_miscellaneous/tools_alex_demo.md: -------------------------------------------------------------------------------- 1 | # Demo: Tools (Alex) 2 | 3 | 4 | ## VS Code 5 | 6 | [SSH support](https://code.visualstudio.com/docs/remote/ssh) 7 | 8 | - Start VS Code 9 | - CTRL-P -> Connect to host -> neon 10 | - Check terminal output 11 | - On first connection some software will be installed on the remote 12 | - Open new terminal. Show that hostname is different. 13 | - Can also be seen at bottom left. Should say `SSH: neon` 14 | - I can install tools on the remote now 15 | - One could connect to a VM, container etc. if one wants to. 16 | 17 | ## Dotfiles 18 | 19 | - Show 20 | - Configuration files for a variety of software 21 | - Config of command prompt 22 | 23 | ```text 24 | export PS1="\[\033[0;32m\]\u\[\033[0;36m\]@\h:\w\[\033[0;32m\]\$(__git_ps1)\n└─(\[\033[1;32m\]\t , \$(ls -1 | wc -l | sed 's: ::g') files, \$(ls -sh | head -n1 | sed 's/total //')b\[\033[1;37m\]\[\ 033[0;32m\])\342\224\200>\[\033[0m\] " 25 | ``` 26 | 27 | In a normal directory in gives: 28 | 29 | ```text 30 | jaustar@lapsgs24:~ 31 | └─(12:43:29, 58 files, 336Kb)─> 32 | ``` 33 | 34 | In a git repository: 35 | 36 | ```text 37 | jaustar@lapsgs24:~/dotfiles (master) 38 | └─(13:05:11, 26 files, 104Kb)─> 39 | ``` 40 | 41 | ## tmux 42 | 43 | - Connect to `helium` 44 | - Open `tmux` window 45 | - Split window vertically and horizontally (`CTRL |` and `CTRL -`) 46 | - Leave tmux (`CTRL A` then `D`) 47 | - Disconnect from `helium` 48 | - Reconnect to `helium` 49 | - Check tmux sessions `tmux ls` 50 | - Reconnect to existing tmux session `tmux a -t TARGET` 51 | 52 | -------------------------------------------------------------------------------- /06_miscellaneous/tools_alex_slides.md: -------------------------------------------------------------------------------- 1 | --- 2 | type: slide 3 | slideOptions: 4 | transition: slide 5 | width: 1400 6 | height: 900 7 | margin: 0.1 8 | --- 9 | 10 | 34 | 35 | # Tools (Alex) 36 | 37 | --- 38 | 39 | ## Introduction 40 | 41 | - Some overview over tools I use commonly 42 | - My opinion/experience 43 | 44 | --- 45 | 46 | ## Editor and IDEs 47 | 48 | - Current choices 49 | - [VS Code](https://code.visualstudio.com/) 50 | - Plenty of addons, powerful feature, 51 | - Based on Electron (bad?) 52 | - [VIM](https://www.vim.org/) 53 | - Plenty of addons, small footprint, special input method 54 | - [Sublime Text](https://www.sublimetext.com/) 55 | - Plenty of addons 56 | 57 | - Earlier choices: Gedit, NEdit, Emacs, QtCreator... 58 | - Switching editors may impact productivity 59 | - Knowing basics of one CLI-capable editor useful 60 | - nano, ed, vi, vim, emacs... 61 | 62 | --- 63 | 64 | ## Dotfiles 65 | 66 | - I (try to) organize my dotfiles 67 | - [Git repository](https://github.com/ajaust/dotfiles) 68 | - Easy to deploy via Git 69 | - Should be portable 70 | - Configuration files not applicable to a system are usually ignored 71 | - **Warning**: Do not share secrets by accident 72 | - Check out other people's dotfiles 73 | 74 | --- 75 | 76 | ## Tools 77 | 78 | - `find`, `grep`, `man` etc.: Basic Unix/Linux commands 79 | - [ripgrep](https://github.com/BurntSushi/ripgrep): Fast grep respecting `.gitignore` tailored to search through (typical) code/repositories 80 | - [lmod](https://lmod.readthedocs.io/en/latest/): Module system 81 | - [Bash](https://www.gnu.org/software/bash/) 82 | - [spack](https://spack.io/) 83 | - Docker 84 | - Vagrant 85 | 86 | --- 87 | 88 | ## Terminal 89 | 90 | - [fzf](https://github.com/junegunn/fzf): A command line fuzzy finder 91 | - [Fuzzy completion](https://github.com/junegunn/fzf#fuzzy-completion-for-bash-and-zsh) 92 | - Bash prompt (`PS1`) 93 | - Show directory 94 | - Show branch if in a Git repository 95 | - Terminal emulator with UTF-8 support 96 | - Tiling window manager (i3wm, swaywm) 97 | 98 | --- 99 | 100 | ## Working remote 101 | 102 | - SSH 103 | - Note that VS Code has SSH support 104 | - tmux 105 | - Open several terminals (splitting) 106 | - Reconnect to terminals 107 | - Reconfigured bindings immediately 108 | - scp: Copy data over network 109 | - rsync: Copy data, but more powerful 110 | 111 | --- 112 | 113 | ## SSH 114 | 115 | Configure connections in `.ssh/config` 116 | 117 | - Configure target with name 118 | 119 | ```text 120 | Host WorkPC 121 | HostName 123.123.123.123 122 | User USERNAME 123 | IdentityFile ~/.ssh/some-private-key 124 | ``` 125 | 126 | - Connect via login server 127 | 128 | ```text 129 | Host WorkPC 130 | ProxyCommand ssh -q LoginServerName nc -q0 %h 22 131 | User USERNAME 132 | IdentityFile ~/.ssh/some-private-key 133 | IdentityFile ~/.ssh/another-private-key 134 | ``` 135 | 136 | Connection to `WorkPC` will go via `LoginServerName` 137 | 138 | --- 139 | 140 | 141 | ## C++ 142 | 143 | - [clang-format](https://clang.llvm.org/docs/ClangFormat.html): Code formatting 144 | - [valgrind](https://valgrind.org/): Dynamic analysis 145 | - Memory leak check, memory consumption... 146 | - CMake 147 | - gdb 148 | 149 | --- 150 | 151 | ## Python 152 | 153 | - [black](https://pypi.org/project/black/) 154 | 155 | --- 156 | 157 | ## Documentation 158 | 159 | - Markdown or reStructuredText. 160 | - Whatever fits better. 161 | - mkdocs 162 | - pandoc 163 | - LaTeX 164 | - Doxygen (C++) or docstrings (Python) 165 | 166 | --- 167 | 168 | ## Remarks/Tips 169 | 170 | - Customization vs. portability 171 | - Get acquainted with your tools 172 | 173 | --- 174 | 175 | ## Further reading 176 | 177 | - [Bash/Prompt customization](https://wiki.archlinux.org/title/Bash/Prompt_customization) 178 | -------------------------------------------------------------------------------- /06_miscellaneous/versioning_slides.md: -------------------------------------------------------------------------------- 1 | --- 2 | type: slide 3 | slideOptions: 4 | transition: slide 5 | width: 1400 6 | height: 900 7 | margin: 0.1 8 | --- 9 | 10 | 34 | 35 | # Versioning 36 | 37 | --- 38 | 39 | ## Introduction 40 | 41 | - Software version may indicate 42 | - Currentness/"Freshness" 43 | - (In)compatibility 44 | - Significance of changes 45 | - Easy-to-understand label that something changed 46 | - Important for packaging and reproducibility 47 | 48 | --- 49 | 50 | ## Versioning Examples 51 | 52 | - Semantic versioning 53 | - [Linux Kernel](https://kernel.org/): 5.16.5 (Release), 5.17-rc2 (Preview) 54 | - [PEP 440](https://www.python.org/dev/peps/pep-0440/) 55 | - [pyMOR](https://pymor.org/): 2022.2.0 56 | - Calendar versioning 57 | - [Ubuntu](ubuntu.com/): 22.10 (Latest), 22.04 (LTS) 58 | - [OpenFOAM](https://www.openfoam.com/current-release) (`.com`): v2112 59 | - Others 60 | - [TeX](https://tug.org/): 3.141592653 (Latest), 3.14159265 (Ubuntu 20.04) 61 | - [OpenFOAM](https://openfoam.org/release/) (`.org`): 10 (Latest), but "always releasable" (date patches, e.g. `20221128`) 62 | 63 | --- 64 | 65 | ## Versioning Standards 66 | 67 | - [Semantic Versioning](https://semver.org/) 68 | - [Calendar Versioning](https://calver.org/) 69 | - [PEP 440](https://www.python.org/dev/peps/pep-0440/) 70 | 71 | --- 72 | 73 | ## Semantic Versioning 74 | 75 | ```text 76 | MAJOR.MINOR.PATCH-+ 77 | ``` 78 | 79 | - `MAJOR` version when you make incompatible API changes, 80 | - `MINOR` version when you add functionality in a backwards compatible manner, and 81 | - `PATCH` version when you make backwards compatible bug fixes (no new functionality). 82 | 83 | Homepage: [semver.org](https://semver.org) 84 | 85 | --- 86 | 87 | ## Semantic Versioning Examples 88 | 89 | ```text 90 | MAJOR.MINOR.PATCH-+ 91 | ``` 92 | 93 | - `MAJOR`, `MINOR` and `PATCH` are numeric (mandatory) 94 | 95 | ```text 96 | 0.0.1, 2.3.6, 5.0.1 97 | ``` 98 | 99 | - Major `0`: often work-in-progress, any breaking changes possible 100 | 101 | - `` dot-separated pre-release identifiers (optional) 102 | 103 | ```text 104 | 1.0.0-alpha, 1.0.0-alpha.1, 1.0.0-0.3.7, 1.0.0-x.7.z.92 105 | ``` 106 | 107 | - `` dot-separated build identifiers (optional) 108 | 109 | ```text 110 | 1.0.0-alpha+001, 1.0.0+20130313144700, 1.0.0-beta+exp.sha.5114f85 111 | ``` 112 | 113 | - Implies ordering 114 | 115 | ```text 116 | 1.0.0-alpha < 1.0.0 < 2.0.0 < 2.1.0 < 2.1.1 117 | ``` 118 | 119 | --- 120 | 121 | ## Library Example 122 | 123 | In my `/usr/local/lib/`: 124 | 125 | ```text 126 | libprecice.so -> libprecice.so.2 127 | libprecice.so.2 -> libprecice.so.2.5.0 128 | libprecice.so.2.2.0 129 | libprecice.so.2.5.0 130 | ``` 131 | 132 | - Two installed versions: `libprecice.so.2.2.0` and `libprecice.so.2.5.0` 133 | - Symbolic links `libprecice.so` to a (often the newest) version of a library 134 | - But we can also request a specific version, e.g. `libprecice.so.2.2.0` 135 | 136 | --- 137 | 138 | ## General Tips/Remarks 139 | 140 | - Use a well-known versioning scheme 141 | - Additionally if necessary: Define meaning of version jumps 142 | - Example DuMuX: backwards compatibility for at least two minor versions 143 | - Do **not** modify released code 144 | 145 | --- 146 | 147 | ## Further Reading 148 | 149 | - [Semantic versioning homepage](https://semver.org/) 150 | - [PEP 440: Version Identification and Dependency Specification](https://www.python.org/dev/peps/pep-0440/) 151 | - [Wikipedia: Software versioning](https://en.wikipedia.org/wiki/Software_versioning) 152 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contribution Guide 2 | 3 | We appreciate all kinds of contributions to this project. This document gives you some information on how to contribute to the project. 4 | 5 | ## How Can I Contribute? 6 | 7 | - We suggest to follow the common workflow based on issues and pull requests. In the issue, you should describe the problem. Based on this discussion, one can create a pull request that should solve the issue. 8 | - A good issue describes what is wrong and why it is wrong. Ideally, one has already a suggestion on how to fix the problem, but the suggestions is not mandatory. 9 | - A good pull request should refer to the issue and thus to the problem that it should solve. Explain what the pull request does and how it addresses the problem/issue. If you still work on a new feature or a fix, you can open a pull request and mark is as draft. Once you consider the work done, please mark it as ready for review and request a reviewer. We will give feedback and eventually merge the pull request. 10 | - We enforce consistency of the formatting via automatic style checking. When marking a pull request as ready for review, please make sure that all checks pass successfully. 11 | - For *small changes*, such as typos, broken links etc., please directly create a pull request. 12 | - For larger changes, such as the description of new tools, a new chapter etc., please open an issue first. 13 | 14 | ## Style 15 | 16 | Please follow the guidelines in `docs/styleguide.md`. 17 | 18 | ## Licensing of Contributed Material 19 | 20 | By contributing to this project you agree that your contributions will be covered by the same license of as the project. More information about the license can be found in the file `LICENSE`. 21 | 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Simulation Software Engineering Lecture Material 2 | 3 | ![markdownlint](https://github.com/Simulation-Software-Engineering/lecture-materials/actions/workflows/markdownlint.yml/badge.svg) 4 | [![PDFs](https://github.com/Simulation-Software-Engineering/lecture-materials/actions/workflows/create-pdfs-from-markdown.yml/badge.svg)](https://github.com/Simulation-Software-Engineering/Lecture-Material/actions/workflows/create-pdfs-from-markdown.yml) 5 | [![CC BY 4.0][cc-by-shield]][cc-by] 6 | 7 | Material of the Simulation Software Engineering lecture. There are different ways how to get an overview: 8 | 9 | - Look at `timetable.md` of the [current course](https://github.com/Simulation-Software-Engineering/Lecture-Material/blob/main/timetable.md) or [a previous course](https://github.com/Simulation-Software-Engineering/Lecture-Material/tree/main/00_organization/wt2223/timetable.md), or 10 | - Look at the `README.md` files of each chapter / folder. 11 | 12 | Please note that we update the material over the course of each semester. 13 | 14 | ## List of Chapters 15 | 16 | 1. [Version Control](https://github.com/Simulation-Software-Engineering/Lecture-Material/tree/main/01_version_control) 17 | 2. [Virtualization and Containers](https://github.com/Simulation-Software-Engineering/Lecture-Material/tree/main/02_virtualization_and_containers) 18 | 3. [Building and Packaging](https://github.com/Simulation-Software-Engineering/Lecture-Material/tree/main/03_building_and_packaging) 19 | 4. [Documentation](https://github.com/Simulation-Software-Engineering/Lecture-Material/tree/main/04_documentation) 20 | 5. [Testing and CI](https://github.com/Simulation-Software-Engineering/Lecture-Material/tree/main/05_testing_and_ci) 21 | 6. [Miscellaneous](https://github.com/Simulation-Software-Engineering/Lecture-Material/tree/main/06_miscellaneous) 22 | 23 | ## Linting 24 | 25 | The markdown files can be checked using [markdownlint](https://github.com/markdownlint/markdownlint/). Once the linter is installed, run it locally from the root of this repository using 26 | 27 | ``` 28 | mdl . 29 | ``` 30 | 31 | It will automatically read the markdownlint configuration of this repository. The linter is configured in the files `.mdl.rb` and `.mdlrc`. The majority of the configuration is done in `.mdl.rb`. 32 | 33 | ## Third-Party Content 34 | 35 | In several parts of the material, we use content from 36 | 37 | > Irving, Hertweck, Johnston, Ostblom, Wickham, and Wilson: [Research Software Engineering with Python](https://third-bit.com/py-rse), 2021, 38 | 39 | a book, which we also recommend to recap Git/Bash/Python basics. 40 | 41 | ## License 42 | 43 | This work is licensed under a 44 | [Creative Commons Attribution 4.0 International License][cc-by]. 45 | 46 | [![CC BY 4.0][cc-by-image]][cc-by] 47 | 48 | [cc-by]: http://creativecommons.org/licenses/by/4.0/ 49 | [cc-by-image]: https://i.creativecommons.org/l/by/4.0/88x31.png 50 | [cc-by-shield]: https://img.shields.io/badge/License-CC%20BY%204.0-blue.svg 51 | -------------------------------------------------------------------------------- /docs/pull_request_template.md: -------------------------------------------------------------------------------- 1 | ## Description 2 | 3 | 4 | 5 | ## Checklist 6 | 7 | 8 | 9 | - [ ] I made sure that the markdown files are formatted properly. 10 | - [ ] I made sure that all hyperlinks are working. 11 | 12 | 13 | 14 | - [ ] I added the new content to the `README.md` inside the topics subdirectory, e.g., `01_version_control/README.md`. 15 | - [ ] I added the new content to the `timetable.md` in the root of this repository. 16 | -------------------------------------------------------------------------------- /docs/styleguide.md: -------------------------------------------------------------------------------- 1 | # Style Guide for SSE Lecture Material 2 | 3 | ## Names of Folders and Files 4 | 5 | - Use `snail_case`. Only exception are standard files such as `CONTRIBUTING.md`. 6 | - Slides end with `_slides.md`, demo notes with `_demo.md`, and exercise sheets with `_exercise.md`. 7 | 8 | ## Headings 9 | 10 | - Use "Title Case for Headings". 11 | - Each document has exactly one h1 heading. 12 | 13 | ## Listings 14 | 15 | The two important questions are: 16 | 17 | - Should an item start with a capital or a lower case letter? 18 | - How should an item end? 19 | 20 | ### Full Sentences 21 | 22 | - ... start with a capital letter. 23 | - ... end with a period, a questions mark, or an exclamation mark. Use exclamation marks with care (do not shout at the student). 24 | 25 | Example: 26 | 27 | > - Running predefined workflows is simple. 28 | 29 | ### Items That are not Full Sentences 30 | 31 | - ... also start with a capital letter. 32 | - ... do **not** end with a period, but can end with a question mark, or an exclamation mark. Use exclamation marks with care (do not shout at the student). 33 | 34 | Example: 35 | 36 | > - Ensuring reproducible test environments 37 | 38 | ### Half Sentences 39 | 40 | What if the sentence starts in a heading and continues in the item? 41 | 42 | - Start the item with `...`, a space, and then a lower case letter 43 | - End with a period, a question mark, or an exclamation mark 44 | 45 | Example: 46 | 47 | > Git 48 | > - ... is a version control system. 49 | > - ... is very important. 50 | 51 | ### More Rules 52 | 53 | - After a colon: 54 | - Start a full sentence with a capital letter. 55 | - Otherwise, start with a lower-case letter. 56 | - Use [Oxford Comma](https://en.wikipedia.org/wiki/Serial_comma) in a series of three or more terms. 57 | 58 | ## Links 59 | 60 | - Do not use [click here](https://www.smashingmagazine.com/2012/06/links-should-never-say-click-here/) links. 61 | 62 | ## Slides 63 | 64 | - Each slide has exactly one heading, which is a h2 heading. 65 | - If headings are repeated on consecutive slides, use the following construction: 66 | 67 | > Some Title Case Heading 1/2 68 | > Some Title Case Heading 2/2 69 | 70 | - If a sentence needs to be highlighted, add **Note** before it, for example: 71 | 72 | > **Note**: This particular concept is only applicable for specific compilers. 73 | 74 | ## Learning Goals 75 | 76 | ### Style 77 | 78 | - Start with a verb. Think of an implicit "After listening to all lectures and doing all exercises of this unit, students are able to ...". 79 | - Be specific. 80 | 81 | > - Describe the tools available within the preCICE ecosystem and their usage (performance analysis, testing of data mapping, . . . ). 82 | > - Navigate the preCICE documentation and all community channels. 83 | > - Choose and apply common software engineering concepts for a preCICE adapter for a given 84 | solver. 85 | 86 | ### Location 87 | 88 | - In the `README.md` of each chapter 89 | - On the [course content page](https://simulation-software-engineering.github.io/homepage/course-content/) of the website 90 | - At the beginning of intro slides set of each chapter 91 | - Do not use separate learning goals for a specific lecture session. However, listing parts of or all learning goals again is possible. 92 | -------------------------------------------------------------------------------- /scripts/create-pdf-from-markdown.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | PANDOC_SLIDE_OPTIONS=("--pdf-engine=xelatex" "-t beamer" "-V aspectratio=169" "-V linkcolor:blue" "-V fontsize=12pt" "--listings" "-s") 4 | PANDOC_TEXT_OPTIONS=("--pdf-engine=xelatex" "-V geometry:a4paper" "-V geometry:left=2.5cm" "-V geometry:right=2.5cm" "-V geometry:bottom=2.5cm" "-V geometry:top=2.5cm" "-V colorlinks:true" "-V linkcolor:blue" "-V fontsize=10pt" "--listings" "-s") 5 | 6 | if [[ $# == 1 ]]; 7 | then 8 | TARGET_FILES="$1" 9 | else 10 | echo "Error: Expect one parameter passed to script!" 11 | echo " Valid arguments are: slides, quiz, demo, or exercise" 12 | exit 1 13 | fi 14 | 15 | if [[ "$TARGET_FILES" == "slides" ]] || [[ "$TARGET_FILES" == "quiz" ]] || [[ "$TARGET_FILES" == "demo" ]] || [[ "$TARGET_FILES" == "exercise" ]]; 16 | then 17 | file_list=$(find . -type f -iname "*_${TARGET_FILES}.md") 18 | else 19 | echo "Error: Argument not supported." 20 | echo " Supplied argument: ${TARGET_FILES}" 21 | echo " Valid arguments are: slides, quiz, demo, or exercise" 22 | exit 1 23 | fi 24 | 25 | if [[ "$TARGET_FILES" == "slides" ]]; 26 | then 27 | output_options="${PANDOC_SLIDE_OPTIONS[@]}" 28 | target_base="slides" 29 | else 30 | output_options="${PANDOC_TEXT_OPTIONS[@]}" 31 | target_base="texts" 32 | fi 33 | 34 | echo "In directory ${PWD}" 35 | mkdir -p ${target_base} 36 | for f in ${file_list}; 37 | do 38 | echo $f 39 | output_dir=$(echo $f | cut -d "/" -f 2) 40 | target_dir=${target_base}/${output_dir} 41 | mkdir -p ${target_dir} 42 | echo "Output directory: ${target_dir}" 43 | 44 | output_filename=$(echo $f | rev| cut -d "/" -f 1 | rev | sed "s/md/pdf/") 45 | echo "Output file name: ${output_filename}" 46 | 47 | pandoc ${output_options} --output=${target_dir}/${output_filename} ${f} 48 | 49 | if test -f "${target_dir}/${output_filename}"; then 50 | echo "${target_dir}/${output_filename} successfully created." 51 | else 52 | echo "Creation of ${target_dir}/${output_filename} failed." 53 | echo " Aborting..." 54 | exit 1 55 | fi 56 | done 57 | --------------------------------------------------------------------------------