├── .editorconfig ├── .githooks └── pre-commit ├── .github ├── ISSUE_TEMPLATE │ ├── apply_join_team.md │ ├── bug_report.md │ └── feature_request.md ├── actions │ └── gcloud-login │ │ └── action.yml └── workflows │ ├── ci-cd-build-packages-1.yml │ ├── ci-cd-build-packages-2.yml │ ├── ci-cd-build-packages-3.yml │ ├── ci-cd-build-packages-4.yml │ ├── ci-cd-build-packages.yml.erb │ ├── ci-cd-main.yml │ ├── ci-cd-main.yml.erb │ ├── ci-cd-prepare.yml │ ├── ci-cd-prepare.yml.erb │ ├── ci-cd-publish-test-production.yml │ ├── ci-cd-publish-test-production.yml.erb │ ├── ci-cd-publish-test-test.yml │ └── ci-cd-publish-test-test.yml.erb ├── .gitignore ├── CODE-OF-CONDUCT.md ├── CONTRIBUTING.md ├── Gemfile ├── Gemfile.lock ├── LICENSE.md ├── README.md ├── RELEASE-CHECKLIST.md ├── build-common-deb ├── build-common-rpm ├── build-environment-image ├── build-jemalloc ├── build-rbenv-deb ├── build-rbenv-rpm ├── build-ruby ├── build-ruby-deb ├── build-ruby-rpm ├── config.yml ├── container-entrypoints ├── build-common-deb ├── build-common-rpm ├── build-jemalloc ├── build-rbenv-deb ├── build-rbenv-rpm ├── build-ruby ├── build-ruby-deb ├── build-ruby-rpm ├── test-debs ├── test-debs-prepare ├── test-rpms └── test-rpms-prepare ├── dev-handbook ├── README.md ├── add-new-distro.md ├── add-new-ruby-version.md ├── apt-yum-repo-infra.drawio.svg ├── apt-yum-repo.md ├── build-environments.md ├── build-steps.md ├── build-steps.png ├── build-workflow-management.md ├── building-packages-locally.md ├── ci-cd-caching.md ├── ci-cd-resumption.md ├── ci-cd-split-multiple-workflows.md ├── ci-run-number.png ├── dev-environment-setup.md ├── fixing-bugs.md ├── github-actions-checks.png ├── members.md ├── mentorship.md ├── minimal-dependencies-principle.md ├── modifying-and-debugging-tests.md ├── offboarding.md ├── onboarding.md ├── package-organization.md ├── responsibilities-expectations.md ├── source-organization.md ├── speeding-up-ci-feedback.md ├── testing-packages-locally.md ├── troubleshooting-corrupt-ci-cd-artifacts.md └── way-of-working.md ├── environments ├── centos-8 │ ├── Dockerfile │ └── image_tag ├── debian-10 │ ├── Dockerfile │ └── image_tag ├── debian-11 │ ├── Dockerfile │ └── image_tag ├── debian-12 │ ├── Dockerfile │ └── image_tag ├── el-9 │ ├── Dockerfile │ └── image_tag ├── ubuntu-20.04 │ ├── Dockerfile │ └── image_tag ├── ubuntu-22.04 │ ├── Dockerfile │ └── image_tag ├── ubuntu-24.04 │ ├── Dockerfile │ └── image_tag └── utility │ ├── Dockerfile │ ├── Gemfile │ ├── Gemfile.lock │ └── image_tag ├── fullstaq-ruby.asc ├── internal-scripts ├── autodetect-shlib-dependencies ├── ci-cd │ ├── build-common-deb │ │ └── build-package.sh │ ├── build-common-rpm │ │ └── build-package.sh │ ├── build-docker-images │ │ ├── build.sh │ │ └── dump-image.sh │ ├── build-jemalloc-binaries │ │ ├── build.sh │ │ └── download-source.sh │ ├── build-rbenv-deb │ │ └── build-package.sh │ ├── build-rbenv-rpm │ │ └── build-package.sh │ ├── build-ruby-packages │ │ ├── build-binaries.sh │ │ └── build-package.sh │ ├── check-version-numbers-need-changing │ │ ├── check-common-deb-version-revision.sh │ │ ├── check-common-rpm-version-revision.sh │ │ ├── check-minor-ruby-package-revisions.sh │ │ ├── check-rbenv-package-revision.sh │ │ ├── check-rbenv-version.sh │ │ ├── check-ruby-package-revisions.sh │ │ ├── determine-latest-release-tag.sh │ │ ├── determine-unbumped-minor-ruby-package-versions.rb │ │ ├── determine-unbumped-ruby-package-versions.rb │ │ └── extract-rbenv-source.sh │ ├── check-workflow-uptodate │ │ └── check.sh │ ├── create-git-tag │ │ └── determine-next-epic-version.sh │ ├── determine-necessary-jobs │ │ ├── determine-necessary-jobs.rb │ │ └── list-artifacts.sh │ ├── download-artifact.sh │ ├── download-artifacts.sh │ ├── download-rbenv-source │ │ ├── download.sh │ │ └── prepare.sh │ ├── download-ruby-sources │ │ └── download.sh │ ├── load-docker-image.sh │ ├── publish │ │ ├── clean-disk-space.sh │ │ ├── install-aptly.sh │ │ ├── publish-debs.rb │ │ ├── publish-rpms.rb │ │ └── restart-web-server.rb │ ├── test-packages │ │ └── run-tests.sh │ └── upload-artifact.sh ├── generate-ci-cd-yaml.rb ├── test-debs └── test-rpms ├── lib ├── ci_workflow_support.rb ├── gcloud_storage_lock.rb ├── general_support.rb ├── library.sh ├── publishing_support.rb └── shell_scripting_support.rb ├── push-environment-images ├── resources ├── jemalloc_cxx_fix.patch ├── ruby_31_jemalloc.patch ├── ruby_31_malloctrim.patch ├── ruby_32_jemalloc.patch ├── ruby_32_malloctrim.patch ├── ruby_33_jemalloc.patch └── test-env │ ├── Gemfile │ ├── Gemfile.lock │ └── jemalloc-cxx.cpp ├── test-debs └── test-rpms /.editorconfig: -------------------------------------------------------------------------------- 1 | # http://EditorConfig.org 2 | 3 | root = true 4 | 5 | [*] 6 | end_of_line = lf 7 | charset = utf-8 8 | trim_trailing_whitespace = true 9 | 10 | [Dockerfile] 11 | indent_style = space 12 | indent_size = 4 13 | 14 | [*.sh] 15 | indent_style = space 16 | indent_size = 4 17 | 18 | [*.rb] 19 | indent_style = space 20 | indent_size = 2 21 | 22 | [*.yml] 23 | indent_style = space 24 | indent_size = 2 25 | 26 | [*.patch] 27 | trim_trailing_whitespace = false 28 | insert_final_newline = false 29 | 30 | [container-entrypoints/*] 31 | indent_style = space 32 | indent_size = 4 33 | 34 | [{build-everything,build-jemalloc,build-ruby,build-ruby-deb,build-rbenv-deb,build-ruby-rpm,build-rbenv-rpm,test-debs,test-rpms,upload-debs,upload-rpms}] 35 | indent_style = space 36 | indent_size = 4 37 | 38 | [internal-scripts/autodetect-shlib-dependencies] 39 | indent_style = space 40 | indent_size = 2 41 | 42 | [internal-scripts/test-debs] 43 | indent_style = space 44 | indent_size = 4 45 | 46 | [internal-scripts/test-rpms] 47 | indent_style = space 48 | indent_size = 4 49 | 50 | [internal-scripts/upload-deb] 51 | indent_style = space 52 | indent_size = 4 53 | 54 | [internal-scripts/upload-rpm] 55 | indent_style = space 56 | indent_size = 4 57 | 58 | [aptly.conf] 59 | indent_style = space 60 | indent_size = 2 61 | -------------------------------------------------------------------------------- /.githooks/pre-commit: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | set -e 3 | exec 2>&1 4 | ./internal-scripts/generate-ci-cd-yaml.rb 5 | git add --update .github/workflows 6 | git add .github/workflows/ci-cd-*.yml 7 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/apply_join_team.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Join team 3 | about: Apply to join the team 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | --- 8 | 9 | Hi, I'd like to apply to join the Fullstaq Ruby Server Edition team. 10 | 11 | About me: 12 | 13 | 14 | ## Responsibilities 15 | 16 | I'm interested in assuming the following responsibilities: 17 | 18 | - [ ] Investigating and fixing urgent issues and down time. 19 | - [ ] Ensuring that packages are kept up-to-date with upstream versions. 20 | - [ ] Ensuring that new distribution versions are supported. 21 | - [ ] Reviewing Server Edition pull requests. 22 | - [ ] Ensuring that community interactions follow our Code of Conduct. 23 | - [ ] Providing mentorship to contributors and fellow team members. 24 | 25 | (Note: this merely captures what you want initially. It's fine if your interests change over time.) 26 | 27 | ## Establishing trustworthiness 28 | 29 | 34 | 35 | ## Contact 36 | 37 | 38 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug report 3 | about: Create a report to help us improve 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Describe the bug** 11 | A clear and concise description of what the bug is. 12 | 13 | **To Reproduce** 14 | Linux distribution: [...] 15 | Package name: [...] 16 | 17 | Steps to reproduce the behavior: 18 | 1. Run command '....' 19 | 2. See error: 20 | 21 | ~~~ 22 | 23 | ~~~ 24 | 25 | **Expected behavior** 26 | A clear and concise description of what you expected to happen. 27 | 28 | **Additional logs** 29 | If applicable, add additional logs to help explain your problem. 30 | 31 | ~~~ 32 | 33 | ~~~ 34 | 35 | **Additional context** 36 | Add any other context about the problem here. 37 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: enhancement 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you want to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or logs about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/actions/gcloud-login/action.yml: -------------------------------------------------------------------------------- 1 | name: Login to Google Cloud 2 | description: Activate a service account 3 | inputs: 4 | private_key: 5 | description: The service account private key, in JSON format, base64-encoded 6 | required: true 7 | write_keyfile_to: 8 | description: Write the keyfile to the given path 9 | required: false 10 | runs: 11 | using: composite 12 | steps: 13 | - run: gcloud auth activate-service-account --key-file <(base64 -d <<< "${{ inputs.private_key }}") 14 | shell: bash 15 | - run: base64 -d <<< "$PRIVATE_KEY" > "$OUTPUT" 16 | shell: bash 17 | if: inputs.write_keyfile_to != '' 18 | env: 19 | PRIVATE_KEY: ${{ inputs.private_key }} 20 | OUTPUT: ${{ inputs.write_keyfile_to }} 21 | -------------------------------------------------------------------------------- /.github/workflows/ci-cd-main.yml.erb: -------------------------------------------------------------------------------- 1 | <%= editing_warning_comment('ci-cd-main') %> 2 | 3 | name: 'CI/CD: main' 4 | 5 | on: 6 | workflow_dispatch: 7 | inputs: 8 | ci_artifacts_run_number: 9 | required: false 10 | description: Run number for CI artifacts 11 | push: 12 | paths-ignore: 13 | - '**.md' 14 | - 'dev-handbook/**' 15 | 16 | env: 17 | ## Set the following variable to a specific number to make the 18 | ## Google Cloud artifact upload/download actions treat as 19 | ## if we're running the given CI run number. Useful for 20 | ## speeding up development of the CI itself, in order to 21 | ## avoid rebuilding. 22 | CI_ARTIFACTS_RUN_NUMBER: ${{ github.event.inputs.ci_artifacts_run_number || github.run_number }} 23 | 24 | jobs: 25 | # Determines which jobs should be run, or (in case this is a re-run) 26 | # which jobs can be skipped this time because the last run succeeded. 27 | # We determine this by checking whether the artifacts produced by jobs 28 | # exist in this run. 29 | determine_necessary_jobs: 30 | name: Determine necessary jobs 31 | runs-on: ubuntu-24.04 32 | environment: test 33 | permissions: 34 | id-token: write 35 | packages: read 36 | outputs: 37 | ci_artifacts_run_number: ${{ steps.get_ci_artifacts_run_number.outputs.number }} 38 | necessary_jobs: ${{ steps.check.outputs.necessary_jobs }} 39 | steps: 40 | - name: Workaround for detecting new workflows in branches 41 | run: | 42 | echo 'New workflow detected. Please delete the fix/cicd-new-workflows branch now.' 43 | exit 1 44 | if: github.event_name == 'push' && github.ref == 'refs/heads/fix/cicd-new-workflows' 45 | 46 | - uses: actions/checkout@v4 47 | - uses: google-github-actions/auth@v2 48 | with: 49 | project_id: ${{ vars.GCLOUD_PROJECT_ID }} 50 | workload_identity_provider: projects/${{ vars.GCLOUD_PROJECT_NUM }}/locations/global/workloadIdentityPools/github-ci-test/providers/github-ci-test 51 | - name: Set up Cloud SDK 52 | uses: google-github-actions/setup-gcloud@v2 53 | with: 54 | version: '>= 363.0.0' 55 | 56 | - name: Take note of CI artifacts run number 57 | id: get_ci_artifacts_run_number 58 | run: echo "number=$CI_ARTIFACTS_RUN_NUMBER" >> "$GITHUB_OUTPUT" 59 | 60 | - name: List artifacts built in previous try of same CI run 61 | run: ./internal-scripts/ci-cd/determine-necessary-jobs/list-artifacts.sh 62 | env: 63 | CI_ARTIFACTS_BUCKET: ${{ vars.CI_ARTIFACTS_BUCKET }} 64 | 65 | - name: Determine necessary jobs 66 | id: check 67 | run: ./internal-scripts/ci-cd/determine-necessary-jobs/determine-necessary-jobs.rb 68 | env: 69 | GITHUB_ACTOR: ${{ github.actor }} 70 | GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} 71 | 72 | 73 | check_workflow_uptodate: 74 | name: Check whether workflow is up-to-date 75 | runs-on: ubuntu-24.04 76 | steps: 77 | - uses: actions/checkout@v4 78 | - name: Check 79 | run: ./internal-scripts/ci-cd/check-workflow-uptodate/check.sh 80 | 81 | 82 | prepare: 83 | name: Prepare 84 | needs: 85 | - determine_necessary_jobs 86 | - check_workflow_uptodate 87 | permissions: 88 | id-token: write 89 | packages: read 90 | secrets: inherit 91 | uses: ./.github/workflows/ci-cd-prepare.yml 92 | with: 93 | ci_artifacts_run_number: ${{ needs.determine_necessary_jobs.outputs.ci_artifacts_run_number }} 94 | necessary_jobs: ${{ needs.determine_necessary_jobs.outputs.necessary_jobs }} 95 | 96 | 97 | <%- distribution_buckets.each_with_index do |distributions, i| %> 98 | <%- unindent(2) do %> 99 | build_packages_<%= i + 1 %>: 100 | name: Build [<%= i + 1 %>] 101 | needs: 102 | - determine_necessary_jobs 103 | - prepare 104 | permissions: 105 | id-token: write 106 | packages: read 107 | secrets: inherit 108 | uses: ./.github/workflows/ci-cd-build-packages-<%= i + 1 %>.yml 109 | with: 110 | ci_artifacts_run_number: ${{ needs.determine_necessary_jobs.outputs.ci_artifacts_run_number }} 111 | necessary_jobs: ${{ needs.determine_necessary_jobs.outputs.necessary_jobs }} 112 | # # Run even if a transitively dependent job has been skipped 113 | # if: | 114 | # needs.prepare.result == 'success' 115 | # && !failure() && !cancelled() 116 | <%- end -%> 117 | <%- end %> 118 | 119 | 120 | publish_test: 121 | name: Publish & test against test repos 122 | needs: 123 | - determine_necessary_jobs 124 | - prepare 125 | <%- distribution_buckets.size.times do |i| %> 126 | - build_packages_<%= i + 1 %> 127 | <%- end %> 128 | permissions: 129 | id-token: write 130 | packages: read 131 | secrets: inherit 132 | uses: ./.github/workflows/ci-cd-publish-test-test.yml 133 | with: 134 | ci_artifacts_run_number: ${{ needs.determine_necessary_jobs.outputs.ci_artifacts_run_number }} 135 | necessary_jobs: ${{ needs.determine_necessary_jobs.outputs.necessary_jobs }} 136 | 137 | 138 | publish_production: 139 | name: Publish & test against production repos 140 | needs: 141 | - determine_necessary_jobs 142 | - prepare 143 | - publish_test 144 | permissions: 145 | id-token: write 146 | packages: read 147 | contents: write 148 | secrets: inherit 149 | uses: ./.github/workflows/ci-cd-publish-test-production.yml 150 | with: 151 | ci_artifacts_run_number: ${{ needs.determine_necessary_jobs.outputs.ci_artifacts_run_number }} 152 | necessary_jobs: ${{ needs.determine_necessary_jobs.outputs.necessary_jobs }} 153 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .DS_Store 2 | .bundle 3 | *.tar.gz 4 | *.tar.bz2 5 | *.deb 6 | *.rpm 7 | /config-dev.yml 8 | /cache 9 | /output 10 | /logs 11 | /my-* 12 | -------------------------------------------------------------------------------- /CODE-OF-CONDUCT.md: -------------------------------------------------------------------------------- 1 | # Contributor Covenant Code of Conduct 2 | 3 | ## Our Pledge 4 | 5 | We as members, contributors, and leaders pledge to make participation in our 6 | community a harassment-free experience for everyone, regardless of age, body 7 | size, visible or invisible disability, ethnicity, sex characteristics, gender 8 | identity and expression, level of experience, education, socio-economic status, 9 | nationality, personal appearance, race, religion, or sexual identity 10 | and orientation. 11 | 12 | We pledge to act and interact in ways that contribute to an open, welcoming, 13 | diverse, inclusive, and healthy community. 14 | 15 | ## Our Standards 16 | 17 | Examples of behavior that contributes to a positive environment for our 18 | community include: 19 | 20 | * Demonstrating empathy and kindness toward other people 21 | * Being respectful of differing opinions, viewpoints, and experiences 22 | * Giving and gracefully accepting constructive feedback 23 | * Accepting responsibility and apologizing to those affected by our mistakes, 24 | and learning from the experience 25 | * Focusing on what is best not just for us as individuals, but for the 26 | overall community 27 | 28 | Examples of unacceptable behavior include: 29 | 30 | * The use of sexualized language or imagery, and sexual attention or 31 | advances of any kind 32 | * Trolling, insulting or derogatory comments, and personal or political attacks 33 | * Public or private harassment 34 | * Publishing others' private information, such as a physical or email 35 | address, without their explicit permission 36 | * Other conduct which could reasonably be considered inappropriate in a 37 | professional setting 38 | 39 | ## Enforcement Responsibilities 40 | 41 | Community leaders are responsible for clarifying and enforcing our standards of 42 | acceptable behavior and will take appropriate and fair corrective action in 43 | response to any behavior that they deem inappropriate, threatening, offensive, 44 | or harmful. 45 | 46 | Community leaders have the right and responsibility to remove, edit, or reject 47 | comments, commits, code, wiki edits, issues, and other contributions that are 48 | not aligned to this Code of Conduct, and will communicate reasons for moderation 49 | decisions when appropriate. 50 | 51 | ## Scope 52 | 53 | This Code of Conduct applies within all community spaces, and also applies when 54 | an individual is officially representing the community in public spaces. 55 | Examples of representing our community include using an official e-mail address, 56 | posting via an official social media account, or acting as an appointed 57 | representative at an online or offline event. 58 | 59 | ## Enforcement 60 | 61 | Instances of abusive, harassing, or otherwise unacceptable behavior may be 62 | reported the [project team members](dev-handbook/members.md), or by contacting the project team at open-source@fullstaq.com ([@fullstaq-labs/open-source](https://github.com/orgs/fullstaq-labs/teams/open-source/members)). 63 | 64 | All complaints will be reviewed and investigated promptly and fairly. 65 | 66 | All project team members are obligated to respect the privacy and security of the 67 | reporter of any incident. 68 | 69 | ## Enforcement Guidelines 70 | 71 | Community leaders will follow these Community Impact Guidelines in determining 72 | the consequences for any action they deem in violation of this Code of Conduct: 73 | 74 | ### 1. Correction 75 | 76 | **Community Impact**: Use of inappropriate language or other behavior deemed 77 | unprofessional or unwelcome in the community. 78 | 79 | **Consequence**: A private, written warning from community leaders, providing 80 | clarity around the nature of the violation and an explanation of why the 81 | behavior was inappropriate. A public apology may be requested. 82 | 83 | ### 2. Warning 84 | 85 | **Community Impact**: A violation through a single incident or series 86 | of actions. 87 | 88 | **Consequence**: A warning with consequences for continued behavior. No 89 | interaction with the people involved, including unsolicited interaction with 90 | those enforcing the Code of Conduct, for a specified period of time. This 91 | includes avoiding interactions in community spaces as well as external channels 92 | like social media. Violating these terms may lead to a temporary or 93 | permanent ban. 94 | 95 | ### 3. Temporary Ban 96 | 97 | **Community Impact**: A serious violation of community standards, including 98 | sustained inappropriate behavior. 99 | 100 | **Consequence**: A temporary ban from any sort of interaction or public 101 | communication with the community for a specified period of time. No public or 102 | private interaction with the people involved, including unsolicited interaction 103 | with those enforcing the Code of Conduct, is allowed during this period. 104 | Violating these terms may lead to a permanent ban. 105 | 106 | ### 4. Permanent Ban 107 | 108 | **Community Impact**: Demonstrating a pattern of violation of community 109 | standards, including sustained inappropriate behavior, harassment of an 110 | individual, or aggression toward or disparagement of classes of individuals. 111 | 112 | **Consequence**: A permanent ban from any sort of public interaction within 113 | the community. 114 | 115 | ## Attribution 116 | 117 | This Code of Conduct is adapted from the [Contributor Covenant][homepage], 118 | version 2.0, available at 119 | https://www.contributor-covenant.org/version/2/0/code_of_conduct.html. 120 | 121 | Community Impact Guidelines were inspired by [Mozilla's code of conduct 122 | enforcement ladder](https://github.com/mozilla/diversity). 123 | 124 | [homepage]: https://www.contributor-covenant.org 125 | 126 | For answers to common questions about this code of conduct, see the FAQ at 127 | https://www.contributor-covenant.org/faq. Translations are available at 128 | https://www.contributor-covenant.org/translations. -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contribution guide 2 | 3 | > You are reading the contribution guide for the **Server Edition**. Interested in contributing to other parts of Fullstaq Ruby? Check the [Fullstaq Ruby Umbrella contribution guide](https://github.com/fullstaq-ruby/umbrella/blob/main/CONTRIBUTING.md). 4 | 5 | Thanks for considering to contribute! 😀 Your help is essential to [keeping Fullstaq Ruby great](https://www.joyfulbikeshedding.com/blog/2020-05-15-why-fullstaq-ruby.html). We welcome all contributions, no matter who you are, and no matter whether it's big or small (see also our [Code of Conduct](CODE_OF_CONDUCT.md)). With this guide, we aim to make contributing as clear and easy as possible. 6 | 7 | ## What counts as a contribution? 8 | 9 | Anything that helps improving the project, whether directly (through a pull request) or indirectly (by engaging with us) counts as a contribution. Here's a non-exhaustive list: 10 | 11 | * Reporting an issue. 12 | * Triaging issues: determining whether an issue report is clear enough, whether the issue still persists, and whether it is reproducible. 13 | * Updating documentation. 14 | * Proposing an improvement. 15 | * Sending a pull request. 16 | * Reviewing someone else's pull request. 17 | 18 | ## Not sure how to get started? 19 | 20 | Have a look at our [issue tracker](https://github.com/fullstaq-ruby/server-edition/issues). Issues with the following labels are good starting points: 21 | 22 | * "good first issue" if you're looking for something easy. 23 | * "help wanted" if you're in for a challenge, or if you want to help with a high-impact issue. 24 | 25 | ## Development handbook 26 | 27 | To learn how the Fullstaq Ruby Server Edition codebase works and how to develop it, please read our [development handbook](dev-handbook/README.md). 28 | 29 | ## Stuck? Need help? 30 | 31 | Please post something on our [discussion forum](https://github.com/fullstaq-ruby/server-edition/discussions). 32 | 33 | ## Joining the team 34 | 35 | Anyone who wishes to contribute regularly to Fullstaq Ruby Server Edition, or who wishes to assume responsibility for the health and progress of this project, is welcome to join the team! 36 | 37 | To learn more about what it means to be a team member, see [Responsibilities & expectations](dev-handbook/responsibilities-expectations.md). 38 | 39 | ### Trust 40 | 41 | Because joining the team means gaining access to protected resources, trust is essential. We judge trustworthiness through the following manners: 42 | 43 | * Having an established relationship with either the Fullstaq Ruby project, or the wider Ruby community. The longer the better. 44 | * A contractual relationship (such as employment) with [Fullstaq B.V.](https://fullstaq.com/). Contractual relationships carry legal weight and provide greater likelihood of a stable trust relationship; at a minimum they establish strong legal accountability. 45 | 46 | ### Apply 47 | 48 | If you wish to join, please [apply by submitting an issue](https://github.com/fullstaq-ruby/server-edition/issues/new?template=apply_join_team.md). 49 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | ruby '>= 3.2' 4 | 5 | gem 'distributed-lock-google-cloud-storage' 6 | -------------------------------------------------------------------------------- /Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | addressable (2.8.4) 5 | public_suffix (>= 2.0.2, < 6.0) 6 | declarative (0.0.20) 7 | digest-crc (0.6.4) 8 | rake (>= 12.0.0, < 14.0.0) 9 | distributed-lock-google-cloud-storage (1.1.0) 10 | google-cloud-storage (~> 1.32) 11 | faraday (2.7.9) 12 | faraday-net_http (>= 2.0, < 3.1) 13 | ruby2_keywords (>= 0.0.4) 14 | faraday-net_http (3.0.2) 15 | google-apis-core (0.11.0) 16 | addressable (~> 2.5, >= 2.5.1) 17 | googleauth (>= 0.16.2, < 2.a) 18 | httpclient (>= 2.8.1, < 3.a) 19 | mini_mime (~> 1.0) 20 | representable (~> 3.0) 21 | retriable (>= 2.0, < 4.a) 22 | rexml 23 | webrick 24 | google-apis-iamcredentials_v1 (0.17.0) 25 | google-apis-core (>= 0.11.0, < 2.a) 26 | google-apis-storage_v1 (0.19.0) 27 | google-apis-core (>= 0.9.0, < 2.a) 28 | google-cloud-core (1.6.0) 29 | google-cloud-env (~> 1.0) 30 | google-cloud-errors (~> 1.0) 31 | google-cloud-env (1.6.0) 32 | faraday (>= 0.17.3, < 3.0) 33 | google-cloud-errors (1.3.1) 34 | google-cloud-storage (1.44.0) 35 | addressable (~> 2.8) 36 | digest-crc (~> 0.4) 37 | google-apis-iamcredentials_v1 (~> 0.1) 38 | google-apis-storage_v1 (~> 0.19.0) 39 | google-cloud-core (~> 1.6) 40 | googleauth (>= 0.16.2, < 2.a) 41 | mini_mime (~> 1.0) 42 | googleauth (1.6.0) 43 | faraday (>= 0.17.3, < 3.a) 44 | jwt (>= 1.4, < 3.0) 45 | memoist (~> 0.16) 46 | multi_json (~> 1.11) 47 | os (>= 0.9, < 2.0) 48 | signet (>= 0.16, < 2.a) 49 | httpclient (2.8.3) 50 | jwt (2.7.1) 51 | memoist (0.16.2) 52 | mini_mime (1.1.2) 53 | multi_json (1.15.0) 54 | os (1.1.4) 55 | public_suffix (5.0.1) 56 | rake (13.0.6) 57 | representable (3.2.0) 58 | declarative (< 0.1.0) 59 | trailblazer-option (>= 0.1.1, < 0.2.0) 60 | uber (< 0.2.0) 61 | retriable (3.1.2) 62 | rexml (3.2.5) 63 | ruby2_keywords (0.0.5) 64 | signet (0.17.0) 65 | addressable (~> 2.8) 66 | faraday (>= 0.17.5, < 3.a) 67 | jwt (>= 1.5, < 3.0) 68 | multi_json (~> 1.10) 69 | trailblazer-option (0.1.2) 70 | uber (0.1.0) 71 | webrick (1.8.1) 72 | 73 | PLATFORMS 74 | arm64-darwin-21 75 | ruby 76 | x86_64-darwin-21 77 | x86_64-linux 78 | 79 | DEPENDENCIES 80 | distributed-lock-google-cloud-storage 81 | 82 | RUBY VERSION 83 | ruby 3.2.1p31 84 | 85 | BUNDLED WITH 86 | 2.4.15 87 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2019 Hongli Lai. 4 | Copyright (c) 2019 Fullstaq B.V. 5 | 6 | Permission is hereby granted, free of charge, to any person obtaining a copy 7 | of this software and associated documentation files (the "Software"), to deal 8 | in the Software without restriction, including without limitation the rights 9 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 10 | copies of the Software, and to permit persons to whom the Software is 11 | furnished to do so, subject to the following conditions: 12 | 13 | The above copyright notice and this permission notice shall be included in all 14 | copies or substantial portions of the Software. 15 | 16 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 17 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 18 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 19 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 20 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 21 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 22 | SOFTWARE. 23 | -------------------------------------------------------------------------------- /RELEASE-CHECKLIST.md: -------------------------------------------------------------------------------- 1 | # Release checklist 2 | 3 | ## Preparation 4 | 5 | Find out what the previous release's Git tag is (e.g. `epic-1`) and set an environment variable: 6 | 7 | MERGE_BASE=$(git merge-base origin/main HEAD) 8 | PREV_RELEASE_TAG=$(git describe $MERGE_BASE --tags --abbrev=0) 9 | 10 | ## Step 1: check whether the Rbenv package revision needs to be changed 11 | 12 | Check whether config.yml's `rbenv.ref` has changed since the previous release: 13 | 14 | ~~~bash 15 | git archive $PREV_RELEASE_TAG config.yml | tar -xO | ruby -ryaml -e 'puts YAML.load(STDIN)["rbenv"]["ref"]' 16 | cat config.yml | ruby -ryaml -e 'puts YAML.load(STDIN)["rbenv"]["ref"]' 17 | ~~~ 18 | 19 | * If true, then check whether the new Rbenv commit ref changed its version number compared to the previous Fullstaq Ruby release. 20 | 21 | - If true, then reset `rbenv.package_revision` to 0. 22 | - Otherwise, bump `rbenv.package_revision` if not already done. 23 | 24 | * Otherwise, check whether any of these files have changed in such a way that they would change the Rbenv package contents or metadata: 25 | 26 | git diff $PREV_RELEASE_TAG..HEAD \ 27 | container-entrypoints/build-rbenv-deb \ 28 | container-entrypoints/build-rbenv-rpm 29 | 30 | If true, then bump `rbenv.package_revision` if not already done. 31 | 32 | ## Step 2: check whether the fullstaq-ruby-common package version or revision needs to be changed 33 | 34 | Check whether any of these files have changed in such a way that they would change the fullstaq-ruby-common package's contents or metadata: 35 | 36 | git diff $PREV_RELEASE_TAG..HEAD \ 37 | container-entrypoints/build-common-deb \ 38 | container-entrypoints/build-common-rpm 39 | 40 | If true, then bump `common.(deb|rpm).(version|package_revision)` as appropriate (unless already done). 41 | 42 | ## Step 3: check whether any Ruby package revisions need to be changed 43 | 44 | Check whether any of these files have changed in such a way that they would change the package contents or metadata: 45 | 46 | git diff $PREV_RELEASE_TAG..HEAD \ 47 | container-entrypoints/build-jemalloc \ 48 | container-entrypoints/build-ruby \ 49 | container-entrypoints/build-ruby-deb \ 50 | container-entrypoints/build-ruby-rpm 51 | 52 | If true, then for all Ruby versions that aren't newly introduced in the next release, bump their package revision number (unless already done). 53 | 54 | ## Step 4: check whether any minor Ruby package revisions need to be changed 55 | 56 | Inside config.yml, for each entry in `ruby.minor_version_packages`, check whether the `full_version` has changed since the previous release (ignore newly introduced entries): 57 | 58 | ~~~bash 59 | git archive $PREV_RELEASE_TAG config.yml | tar -xO | ruby -ryaml -e 'puts YAML.load(STDIN)["ruby"]["minor_version_packages"]' 60 | cat config.yml | ruby -ryaml -e 'puts YAML.load(STDIN)["ruby"]["minor_version_packages"]' 61 | ~~~ 62 | 63 | If true, then for each changed entry, bump the corresponding `package_revision`. 64 | -------------------------------------------------------------------------------- /build-common-deb: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | SELFDIR=$(cd "$SELFDIR" && pwd) 6 | # shellcheck source=lib/library.sh 7 | source "$SELFDIR/lib/library.sh" 8 | 9 | OUTPUT_PATH= 10 | VERSION=1.0 11 | REVISION=0 12 | 13 | function usage() 14 | { 15 | echo "Usage: ./build-common-deb " 16 | echo "Build the fullstaq-ruby-common Debian package." 17 | echo 18 | echo "Required options:" 19 | echo " -o PATH Path to output package file" 20 | echo 21 | echo "Optional options:" 22 | echo " -v VERSION Package version number (default: $VERSION)" 23 | echo " -r REVISION Package revision number (default: $REVISION)" 24 | echo " -h Show usage" 25 | } 26 | 27 | function parse_options() 28 | { 29 | local OPTIND=1 30 | local ORIG_ARGV 31 | local opt 32 | while getopts "o:v:r:h" opt; do 33 | case "$opt" in 34 | o) 35 | OUTPUT_PATH=$(absolute_path "$OPTARG") 36 | ;; 37 | v) 38 | VERSION="$OPTARG" 39 | ;; 40 | r) 41 | REVISION="$OPTARG" 42 | ;; 43 | h) 44 | usage 45 | exit 46 | ;; 47 | *) 48 | return 1 49 | ;; 50 | esac 51 | done 52 | 53 | (( OPTIND -= 1 )) || true 54 | shift $OPTIND || true 55 | ORIG_ARGV=("$@") 56 | 57 | if [[ "$OUTPUT_PATH" = "" ]]; then 58 | echo 'ERROR: please specify an output package path with -o.' >&2 59 | exit 1 60 | fi 61 | } 62 | 63 | parse_options "$@" 64 | 65 | if tty -s; then 66 | TTY_ARGS=(-t -i) 67 | else 68 | TTY_ARGS=() 69 | fi 70 | 71 | IMAGE_VERSION=$(read_single_value_file "$SELFDIR/environments/utility/image_tag") 72 | OUTPUT_TEMP_PATH=$(mktemp "$OUTPUT_PATH.XXXXXX") 73 | create_file_if_missing "$OUTPUT_TEMP_PATH" 74 | 75 | echo '--- Entering Docker container ---' 76 | 77 | function _cleanup() { 78 | echo 79 | echo '--- Exited Docker container ---' 80 | run rm -f "$OUTPUT_TEMP_PATH" 81 | } 82 | 83 | verbose_run docker run --rm --init "${TTY_ARGS[@]}" \ 84 | -v "$SELFDIR:/system:ro" \ 85 | -v "$OUTPUT_TEMP_PATH:/output/common.deb" \ 86 | -e "VERSION=$VERSION" \ 87 | -e "REVISION=$REVISION" \ 88 | --user "$(id -u):$(id -g)" \ 89 | "ghcr.io/fullstaq-ruby/server-edition-ci-images:utility-v$IMAGE_VERSION" \ 90 | /system/container-entrypoints/build-common-deb 91 | 92 | function _cleanup() { 93 | echo '--- Exited Docker container ---' 94 | run mv "$OUTPUT_TEMP_PATH" "$OUTPUT_PATH" 95 | } 96 | -------------------------------------------------------------------------------- /build-common-rpm: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | SELFDIR=$(cd "$SELFDIR" && pwd) 6 | # shellcheck source=lib/library.sh 7 | source "$SELFDIR/lib/library.sh" 8 | 9 | OUTPUT_PATH= 10 | VERSION=1.0 11 | REVISION=0 12 | 13 | function usage() 14 | { 15 | echo "Usage: ./build-rbenv-rpm " 16 | echo "Build the fullstaq-ruby-common RPM package." 17 | echo 18 | echo "Required options:" 19 | echo " -o PATH Path to output package file" 20 | echo 21 | echo "Optional options:" 22 | echo " -v VERSION Package version number (default: $VERSION)" 23 | echo " -r REVISION Package revision number (default: $REVISION)" 24 | echo " -h Show usage" 25 | } 26 | 27 | function parse_options() 28 | { 29 | local OPTIND=1 30 | local ORIG_ARGV 31 | local opt 32 | while getopts "o:v:r:h" opt; do 33 | case "$opt" in 34 | o) 35 | OUTPUT_PATH=$(absolute_path "$OPTARG") 36 | ;; 37 | v) 38 | VERSION="$OPTARG" 39 | ;; 40 | r) 41 | REVISION="$OPTARG" 42 | ;; 43 | h) 44 | usage 45 | exit 46 | ;; 47 | *) 48 | return 1 49 | ;; 50 | esac 51 | done 52 | 53 | (( OPTIND -= 1 )) || true 54 | shift $OPTIND || true 55 | ORIG_ARGV=("$@") 56 | 57 | if [[ "$OUTPUT_PATH" = "" ]]; then 58 | echo 'ERROR: please specify an output package path with -o.' >&2 59 | exit 1 60 | fi 61 | } 62 | 63 | parse_options "$@" 64 | 65 | if tty -s; then 66 | TTY_ARGS=(-t -i) 67 | else 68 | TTY_ARGS=() 69 | fi 70 | 71 | IMAGE_VERSION=$(read_single_value_file "$SELFDIR/environments/utility/image_tag") 72 | OUTPUT_TEMP_PATH=$(mktemp "$OUTPUT_PATH.XXXXXX") 73 | create_file_if_missing "$OUTPUT_TEMP_PATH" 74 | 75 | echo '--- Entering Docker container ---' 76 | 77 | function _cleanup() { 78 | echo 79 | echo '--- Exited Docker container ---' 80 | run rm -f "$OUTPUT_TEMP_PATH" 81 | } 82 | 83 | verbose_run docker run --rm --init "${TTY_ARGS[@]}" \ 84 | -v "$SELFDIR:/system:ro" \ 85 | -v "$OUTPUT_TEMP_PATH:/output/common.rpm" \ 86 | -e "VERSION=$VERSION" \ 87 | -e "REVISION=$REVISION" \ 88 | --user "$(id -u):$(id -g)" \ 89 | "ghcr.io/fullstaq-ruby/server-edition-ci-images:utility-v$IMAGE_VERSION" \ 90 | /system/container-entrypoints/build-common-rpm 91 | 92 | function _cleanup() { 93 | echo '--- Exited Docker container ---' 94 | run mv "$OUTPUT_TEMP_PATH" "$OUTPUT_PATH" 95 | } 96 | -------------------------------------------------------------------------------- /build-environment-image: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | SELFDIR=$(cd "$SELFDIR" && pwd) 6 | # shellcheck source=lib/library.sh 7 | source "$SELFDIR/lib/library.sh" 8 | 9 | function usage() 10 | { 11 | echo "Usage: ./build-environment [OPTIONS] " 12 | echo "Build a build environment Docker image. A build environment is an image based on" 13 | echo "a certain Linux distro, and is used to compile Ruby for that distro." 14 | echo 15 | echo " is the name of a subdirectory in environments/, such as 'ubuntu-22.04'." 16 | echo 17 | echo "This script will produce a Docker image called" 18 | echo "'ghcr.io/fullstaq-ruby/server-edition-ci-images:-v'." 19 | echo 20 | echo "Optional options:" 21 | echo " -h Show usage" 22 | } 23 | 24 | function parse_options() 25 | { 26 | local OPTIND=1 27 | local ORIG_ARGV 28 | local opt 29 | while getopts "f:h" opt; do 30 | case "$opt" in 31 | f) 32 | FORMAT="$OPTARG" 33 | ;; 34 | h) 35 | usage 36 | exit 37 | ;; 38 | *) 39 | return 1 40 | ;; 41 | esac 42 | done 43 | 44 | (( OPTIND -= 1 )) || true 45 | shift $OPTIND || true 46 | ORIG_ARGV=("$@") 47 | 48 | if [[ $# -ne 1 ]]; then 49 | usage 50 | exit 1 51 | fi 52 | } 53 | 54 | parse_options "$@" 55 | 56 | ENV="$1" 57 | IMAGE_NAME=ghcr.io/fullstaq-ruby/server-edition-ci-images 58 | IMAGE_VERSION=$(read_single_value_file "$SELFDIR/environments/$ENV/image_tag") 59 | 60 | if [[ ! -e "$SELFDIR/environments/$ENV" ]]; then 61 | echo "ERROR: $SELFDIR/environments/$ENV does not exist." >&2 62 | exit 1 63 | fi 64 | 65 | run docker build --pull -t "$IMAGE_NAME:$ENV-v$IMAGE_VERSION" "$SELFDIR/environments/$ENV" 66 | -------------------------------------------------------------------------------- /build-jemalloc: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | SELFDIR=$(cd "$SELFDIR" && pwd) 6 | # shellcheck source=lib/library.sh 7 | source "$SELFDIR/lib/library.sh" 8 | 9 | ENVIRONMENT_NAME= 10 | SOURCE_PATH= 11 | OUTPUT_PATH= 12 | BUILD_CONCURRENCY=1 13 | CACHE_CONNECTION_STRING_FILE= 14 | CACHE_CONTAINER= 15 | CACHE_KEY_PREFIX= 16 | 17 | function usage() 18 | { 19 | echo "Usage: ./build-jemalloc " 20 | echo "Build a Jemalloc binary tarball from its source tarball." 21 | echo 22 | echo "Required options:" 23 | echo " -n NAME Name of environment to build in (one of: $(list_environment_names "$SELFDIR/environments"))" 24 | echo " -s PATH Path to Jemalloc source tarball" 25 | echo " -o PATH Path to output tarball" 26 | echo 27 | echo "Optional options:" 28 | echo " -j NUM Build concurrency (default: $BUILD_CONCURRENCY)" 29 | echo " -c PATH Cache to Azure Blob Storage, use connection string in given file" 30 | echo " -r CONTAINER Use given Azure Blob Storage container" 31 | echo " -d NAME Use given Azure Blob Storage key prefix" 32 | echo " -h Show usage" 33 | } 34 | 35 | function parse_options() 36 | { 37 | local OPTIND=1 38 | local opt 39 | while getopts "n:s:o:j:c:r:d:h" opt; do 40 | case "$opt" in 41 | n) 42 | ENVIRONMENT_NAME="$OPTARG" 43 | ;; 44 | s) 45 | SOURCE_PATH=$(absolute_path "$OPTARG") 46 | ;; 47 | o) 48 | OUTPUT_PATH=$(absolute_path "$OPTARG") 49 | ;; 50 | j) 51 | BUILD_CONCURRENCY="$OPTARG" 52 | ;; 53 | c) 54 | CACHE_CONNECTION_STRING_FILE=$(absolute_path "$OPTARG") 55 | ;; 56 | r) 57 | CACHE_CONTAINER="$OPTARG" 58 | ;; 59 | d) 60 | CACHE_KEY_PREFIX="$OPTARG" 61 | ;; 62 | h) 63 | usage 64 | exit 65 | ;; 66 | *) 67 | return 1 68 | ;; 69 | esac 70 | done 71 | 72 | (( OPTIND -= 1 )) || true 73 | shift $OPTIND || true 74 | 75 | if [[ "$ENVIRONMENT_NAME" = "" ]]; then 76 | echo 'ERROR: please specify an environment name with -n.' >&2 77 | exit 1 78 | fi 79 | if [[ "$SOURCE_PATH" = "" ]]; then 80 | echo 'ERROR: please specify a Jemalloc source tarball path with -s.' >&2 81 | exit 1 82 | fi 83 | if [[ ! -e "$SOURCE_PATH" ]]; then 84 | echo "ERROR: $SOURCE_PATH does not exist." >&2 85 | exit 1 86 | fi 87 | if [[ "$OUTPUT_PATH" = "" ]]; then 88 | echo 'ERROR: please specify an output tarball path with -o.' >&2 89 | exit 1 90 | fi 91 | if [[ -n "$CACHE_CONNECTION_STRING_FILE" && -z "$CACHE_CONTAINER" ]]; then 92 | echo "ERROR: if a cache connection string file is given, then an Azure Blob Storage container name must also be given." >&2 93 | exit 1 94 | fi 95 | } 96 | 97 | parse_options "$@" 98 | 99 | if tty -s; then 100 | TTY_ARGS=(-t -i) 101 | else 102 | TTY_ARGS=() 103 | fi 104 | if [[ -n "$CACHE_CONNECTION_STRING_FILE" ]]; then 105 | CACHE_PATH_MOUNT_ARGS=(-v "$CACHE_CONNECTION_STRING_FILE:/azure-connection-string.txt:ro") 106 | else 107 | CACHE_PATH_MOUNT_ARGS=() 108 | fi 109 | 110 | IMAGE_VERSION=$(read_single_value_file "$SELFDIR/environments/$ENVIRONMENT_NAME/image_tag") 111 | OUTPUT_TEMP_PATH=$(mktemp "$OUTPUT_PATH.XXXXXX") 112 | create_file_if_missing "$OUTPUT_TEMP_PATH" 113 | 114 | echo '--- Entering Docker container ---' 115 | 116 | function _cleanup() { 117 | echo 118 | echo '--- Exited Docker container ---' 119 | run rm -f "$OUTPUT_TEMP_PATH" 120 | } 121 | 122 | verbose_run docker run --rm --init "${TTY_ARGS[@]}" \ 123 | -v "$SELFDIR:/system:ro" \ 124 | -v "$SOURCE_PATH:/input/jemalloc-src.tar.bz2:ro" \ 125 | -v "$OUTPUT_TEMP_PATH:/output/jemalloc-bin.tar.gz" \ 126 | "${CACHE_PATH_MOUNT_ARGS[@]}" \ 127 | -e "ENVIRONMENT_NAME=$ENVIRONMENT_NAME" \ 128 | -e "BUILD_CONCURRENCY=$BUILD_CONCURRENCY" \ 129 | -e "CACHE_CONTAINER=$CACHE_CONTAINER" \ 130 | -e "CACHE_KEY_PREFIX=$CACHE_KEY_PREFIX" \ 131 | --user "$(id -u):$(id -g)" \ 132 | "ghcr.io/fullstaq-ruby/server-edition-ci-images:$ENVIRONMENT_NAME-v$IMAGE_VERSION" \ 133 | /system/container-entrypoints/build-jemalloc 134 | 135 | function _cleanup() { 136 | echo '--- Exited Docker container ---' 137 | run mv "$OUTPUT_TEMP_PATH" "$OUTPUT_PATH" 138 | } 139 | -------------------------------------------------------------------------------- /build-rbenv-deb: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | SELFDIR=$(cd "$SELFDIR" && pwd) 6 | # shellcheck source=lib/library.sh 7 | source "$SELFDIR/lib/library.sh" 8 | 9 | SOURCE_PATH= 10 | OUTPUT_PATH= 11 | REVISION=0 12 | 13 | function usage() 14 | { 15 | echo "Usage: ./build-rbenv-deb " 16 | echo "Build an Rbenv Debian package." 17 | echo 18 | echo "To find out the right value for -n:" 19 | echo " bin/rbenv --version | awk '{ print \$2 }' | sed -E 's/(.+)-.*/\1/'" 20 | echo 21 | echo "Required options:" 22 | echo " -s PATH Path to Rbenv source tree" 23 | echo " -o PATH Path to output package file" 24 | echo " -n VERSION Rbenv version number without commit hash suffix" 25 | echo 26 | echo "Optional options:" 27 | echo " -r REVISION Package revision number (default: $REVISION)" 28 | echo " -h Show usage" 29 | } 30 | 31 | function parse_options() 32 | { 33 | local OPTIND=1 34 | local ORIG_ARGV 35 | local opt 36 | while getopts "s:o:n:r:h" opt; do 37 | case "$opt" in 38 | s) 39 | SOURCE_PATH=$(absolute_path "$OPTARG") 40 | ;; 41 | o) 42 | OUTPUT_PATH=$(absolute_path "$OPTARG") 43 | ;; 44 | n) 45 | VERSION="$OPTARG" 46 | ;; 47 | r) 48 | REVISION="$OPTARG" 49 | ;; 50 | h) 51 | usage 52 | exit 53 | ;; 54 | *) 55 | return 1 56 | ;; 57 | esac 58 | done 59 | 60 | (( OPTIND -= 1 )) || true 61 | shift $OPTIND || true 62 | ORIG_ARGV=("$@") 63 | 64 | if [[ "$SOURCE_PATH" = "" ]]; then 65 | echo 'ERROR: please specify an Rbenv source path with -s.' >&2 66 | exit 1 67 | fi 68 | if [[ ! -e "$SOURCE_PATH" ]]; then 69 | echo "ERROR: $SOURCE_PATH does not exist." >&2 70 | exit 1 71 | fi 72 | if [[ "$OUTPUT_PATH" = "" ]]; then 73 | echo 'ERROR: please specify an output package path with -o.' >&2 74 | exit 1 75 | fi 76 | if [[ "$VERSION" = "" ]]; then 77 | echo 'ERROR: please specify an Rbenv version with -n.' >&2 78 | exit 1 79 | fi 80 | } 81 | 82 | parse_options "$@" 83 | 84 | if tty -s; then 85 | TTY_ARGS=(-t -i) 86 | else 87 | TTY_ARGS=() 88 | fi 89 | 90 | IMAGE_VERSION=$(read_single_value_file "$SELFDIR/environments/utility/image_tag") 91 | OUTPUT_TEMP_PATH=$(mktemp "$OUTPUT_PATH.XXXXXX") 92 | create_file_if_missing "$OUTPUT_TEMP_PATH" 93 | 94 | echo '--- Entering Docker container ---' 95 | 96 | function _cleanup() { 97 | echo 98 | echo '--- Exited Docker container ---' 99 | run rm -f "$OUTPUT_TEMP_PATH" 100 | } 101 | 102 | verbose_run docker run --rm --init "${TTY_ARGS[@]}" \ 103 | -v "$SELFDIR:/system:ro" \ 104 | -v "$SOURCE_PATH:/input/rbenv:ro" \ 105 | -v "$OUTPUT_TEMP_PATH:/output/rbenv.deb" \ 106 | -e "VERSION=$VERSION" \ 107 | -e "REVISION=$REVISION" \ 108 | --user "$(id -u):$(id -g)" \ 109 | "ghcr.io/fullstaq-ruby/server-edition-ci-images:utility-v$IMAGE_VERSION" \ 110 | /system/container-entrypoints/build-rbenv-deb 111 | 112 | function _cleanup() { 113 | echo '--- Exited Docker container ---' 114 | run mv "$OUTPUT_TEMP_PATH" "$OUTPUT_PATH" 115 | } 116 | -------------------------------------------------------------------------------- /build-rbenv-rpm: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | SELFDIR=$(cd "$SELFDIR" && pwd) 6 | # shellcheck source=lib/library.sh 7 | source "$SELFDIR/lib/library.sh" 8 | 9 | SOURCE_PATH= 10 | OUTPUT_PATH= 11 | REVISION=0 12 | 13 | function usage() 14 | { 15 | echo "Usage: ./build-rbenv-rpm " 16 | echo "Build an Rbenv RPM package." 17 | echo 18 | echo "To find out the right value for -n:" 19 | echo " bin/rbenv --version | awk '{ print \$2 }' | sed -E 's/(.+)-.*/\1/'" 20 | echo 21 | echo "Required options:" 22 | echo " -s PATH Path to Rbenv source tree" 23 | echo " -o PATH Path to output package file" 24 | echo " -n VERSION Rbenv version number without commit hash suffix" 25 | echo 26 | echo "Optional options:" 27 | echo " -r REVISION Package revision number (default: $REVISION)" 28 | echo " -h Show usage" 29 | } 30 | 31 | function parse_options() 32 | { 33 | local OPTIND=1 34 | local ORIG_ARGV 35 | local opt 36 | while getopts "s:o:n:r:h" opt; do 37 | case "$opt" in 38 | s) 39 | SOURCE_PATH=$(absolute_path "$OPTARG") 40 | ;; 41 | o) 42 | OUTPUT_PATH=$(absolute_path "$OPTARG") 43 | ;; 44 | n) 45 | VERSION="$OPTARG" 46 | ;; 47 | r) 48 | REVISION="$OPTARG" 49 | ;; 50 | h) 51 | usage 52 | exit 53 | ;; 54 | *) 55 | return 1 56 | ;; 57 | esac 58 | done 59 | 60 | (( OPTIND -= 1 )) || true 61 | shift $OPTIND || true 62 | ORIG_ARGV=("$@") 63 | 64 | if [[ "$SOURCE_PATH" = "" ]]; then 65 | echo 'ERROR: please specify an Rbenv source path with -s.' >&2 66 | exit 1 67 | fi 68 | if [[ ! -e "$SOURCE_PATH" ]]; then 69 | echo "ERROR: $SOURCE_PATH does not exist." >&2 70 | exit 1 71 | fi 72 | if [[ "$OUTPUT_PATH" = "" ]]; then 73 | echo 'ERROR: please specify an output package path with -o.' >&2 74 | exit 1 75 | fi 76 | if [[ "$VERSION" = "" ]]; then 77 | echo 'ERROR: please specify an Rbenv version with -n.' >&2 78 | exit 1 79 | fi 80 | } 81 | 82 | parse_options "$@" 83 | 84 | if tty -s; then 85 | TTY_ARGS=(-t -i) 86 | else 87 | TTY_ARGS=() 88 | fi 89 | 90 | IMAGE_VERSION=$(read_single_value_file "$SELFDIR/environments/utility/image_tag") 91 | OUTPUT_TEMP_PATH=$(mktemp "$OUTPUT_PATH.XXXXXX") 92 | create_file_if_missing "$OUTPUT_TEMP_PATH" 93 | 94 | echo '--- Entering Docker container ---' 95 | 96 | function _cleanup() { 97 | echo 98 | echo '--- Exited Docker container ---' 99 | run rm -f "$OUTPUT_TEMP_PATH" 100 | } 101 | 102 | verbose_run docker run --rm --init "${TTY_ARGS[@]}" \ 103 | -v "$SELFDIR:/system:ro" \ 104 | -v "$SOURCE_PATH:/input/rbenv:ro" \ 105 | -v "$OUTPUT_TEMP_PATH:/output/rbenv.rpm" \ 106 | -e "VERSION=$VERSION" \ 107 | -e "REVISION=$REVISION" \ 108 | --user "$(id -u):$(id -g)" \ 109 | "ghcr.io/fullstaq-ruby/server-edition-ci-images:utility-v$IMAGE_VERSION" \ 110 | /system/container-entrypoints/build-rbenv-rpm 111 | 112 | function _cleanup() { 113 | echo '--- Exited Docker container ---' 114 | run mv "$OUTPUT_TEMP_PATH" "$OUTPUT_PATH" 115 | } 116 | -------------------------------------------------------------------------------- /build-ruby-deb: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | SELFDIR=$(cd "$SELFDIR" && pwd) 6 | # shellcheck source=lib/library.sh 7 | source "$SELFDIR/lib/library.sh" 8 | 9 | BINARY_TARBALL_PATH= 10 | OUTPUT_PATH= 11 | REVISION=0 12 | 13 | function usage() 14 | { 15 | echo "Usage: ./build-ruby-deb " 16 | echo "Build a Ruby Debian package from its binary tarball (which is produced by ./build-ruby)." 17 | echo 18 | echo "Required options:" 19 | echo " -b PATH Path to Ruby binary tarball (as built by ./build-ruby)" 20 | echo " -o PATH Path to output package file" 21 | echo 22 | echo "Optional options:" 23 | echo " -r REVISION Package revision number (default: $REVISION)" 24 | echo " -h Show usage" 25 | } 26 | 27 | function parse_options() 28 | { 29 | local OPTIND=1 30 | local ORIG_ARGV 31 | local opt 32 | while getopts "b:o:r:h" opt; do 33 | case "$opt" in 34 | b) 35 | BINARY_TARBALL_PATH=$(absolute_path "$OPTARG") 36 | ;; 37 | o) 38 | OUTPUT_PATH=$(absolute_path "$OPTARG") 39 | ;; 40 | r) 41 | REVISION="$OPTARG" 42 | ;; 43 | h) 44 | usage 45 | exit 46 | ;; 47 | *) 48 | return 1 49 | ;; 50 | esac 51 | done 52 | 53 | (( OPTIND -= 1 )) || true 54 | shift $OPTIND || true 55 | ORIG_ARGV=("$@") 56 | 57 | if [[ "$BINARY_TARBALL_PATH" = "" ]]; then 58 | echo 'ERROR: please specify a Ruby binary tarball path with -b.' >&2 59 | exit 1 60 | fi 61 | if [[ ! -e "$BINARY_TARBALL_PATH" ]]; then 62 | echo "ERROR: $BINARY_TARBALL_PATH does not exist." >&2 63 | exit 1 64 | fi 65 | if [[ "$OUTPUT_PATH" = "" ]]; then 66 | echo 'ERROR: please specify an output package path with -o.' >&2 67 | exit 1 68 | fi 69 | } 70 | 71 | parse_options "$@" 72 | 73 | if tty -s; then 74 | TTY_ARGS=(-t -i) 75 | else 76 | TTY_ARGS=() 77 | fi 78 | 79 | IMAGE_VERSION=$(read_single_value_file "$SELFDIR/environments/utility/image_tag") 80 | OUTPUT_TEMP_PATH=$(mktemp "$OUTPUT_PATH.XXXXXX") 81 | create_file_if_missing "$OUTPUT_TEMP_PATH" 82 | 83 | echo '--- Entering Docker container ---' 84 | 85 | function _cleanup() { 86 | echo 87 | echo '--- Exited Docker container ---' 88 | run rm -f "$OUTPUT_TEMP_PATH" 89 | } 90 | 91 | verbose_run docker run --rm --init "${TTY_ARGS[@]}" \ 92 | -v "$SELFDIR:/system:ro" \ 93 | -v "$BINARY_TARBALL_PATH:/input/ruby-bin.tar.gz:ro" \ 94 | -v "$OUTPUT_TEMP_PATH:/output/ruby.deb" \ 95 | -e "REVISION=$REVISION" \ 96 | --user "$(id -u):$(id -g)" \ 97 | "ghcr.io/fullstaq-ruby/server-edition-ci-images:utility-v$IMAGE_VERSION" \ 98 | /system/container-entrypoints/build-ruby-deb 99 | 100 | function _cleanup() { 101 | echo '--- Exited Docker container ---' 102 | run mv "$OUTPUT_TEMP_PATH" "$OUTPUT_PATH" 103 | } 104 | -------------------------------------------------------------------------------- /build-ruby-rpm: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | SELFDIR=$(cd "$SELFDIR" && pwd) 6 | # shellcheck source=lib/library.sh 7 | source "$SELFDIR/lib/library.sh" 8 | 9 | BINARY_TARBALL_PATH= 10 | OUTPUT_PATH= 11 | REVISION=0 12 | 13 | function usage() 14 | { 15 | echo "Usage: ./build-ruby-rpm " 16 | echo "Build a Ruby RPM package from its binary tarball (which is produced by ./build-ruby)." 17 | echo 18 | echo "Required options:" 19 | echo " -b PATH Path to Ruby binary tarball (as built by ./build-ruby)" 20 | echo " -o PATH Path to output package file" 21 | echo 22 | echo "Optional options:" 23 | echo " -r REVISION Package revision number (default: $REVISION)" 24 | echo " -h Show usage" 25 | } 26 | 27 | function parse_options() 28 | { 29 | local OPTIND=1 30 | local ORIG_ARGV 31 | local opt 32 | while getopts "b:o:r:h" opt; do 33 | case "$opt" in 34 | b) 35 | BINARY_TARBALL_PATH=$(absolute_path "$OPTARG") 36 | ;; 37 | o) 38 | OUTPUT_PATH=$(absolute_path "$OPTARG") 39 | ;; 40 | r) 41 | REVISION="$OPTARG" 42 | ;; 43 | h) 44 | usage 45 | exit 46 | ;; 47 | *) 48 | return 1 49 | ;; 50 | esac 51 | done 52 | 53 | (( OPTIND -= 1 )) || true 54 | shift $OPTIND || true 55 | ORIG_ARGV=("$@") 56 | 57 | if [[ "$BINARY_TARBALL_PATH" = "" ]]; then 58 | echo 'ERROR: please specify a Ruby binary tarball path with -b.' >&2 59 | exit 1 60 | fi 61 | if [[ ! -e "$BINARY_TARBALL_PATH" ]]; then 62 | echo "ERROR: $BINARY_TARBALL_PATH does not exist." >&2 63 | exit 1 64 | fi 65 | if [[ "$OUTPUT_PATH" = "" ]]; then 66 | echo 'ERROR: please specify an output package path with -o.' >&2 67 | exit 1 68 | fi 69 | } 70 | 71 | parse_options "$@" 72 | 73 | if tty -s; then 74 | TTY_ARGS=(-t -i) 75 | else 76 | TTY_ARGS=() 77 | fi 78 | 79 | IMAGE_VERSION=$(read_single_value_file "$SELFDIR/environments/utility/image_tag") 80 | OUTPUT_TEMP_PATH=$(mktemp "$OUTPUT_PATH.XXXXXX") 81 | create_file_if_missing "$OUTPUT_TEMP_PATH" 82 | 83 | echo '--- Entering Docker container ---' 84 | 85 | function _cleanup() { 86 | echo 87 | echo '--- Exited Docker container ---' 88 | run rm -f "$OUTPUT_TEMP_PATH" 89 | } 90 | 91 | verbose_run docker run --rm --init "${TTY_ARGS[@]}" \ 92 | -v "$SELFDIR:/system:ro" \ 93 | -v "$BINARY_TARBALL_PATH:/input/ruby-bin.tar.gz:ro" \ 94 | -v "$OUTPUT_TEMP_PATH:/output/ruby.rpm" \ 95 | -e "REVISION=$REVISION" \ 96 | --user "$(id -u):$(id -g)" \ 97 | "ghcr.io/fullstaq-ruby/server-edition-ci-images:utility-v$IMAGE_VERSION" \ 98 | /system/container-entrypoints/build-ruby-rpm 99 | 100 | function _cleanup() { 101 | echo '--- Exited Docker container ---' 102 | run mv "$OUTPUT_TEMP_PATH" "$OUTPUT_PATH" 103 | } 104 | -------------------------------------------------------------------------------- /config.yml: -------------------------------------------------------------------------------- 1 | ## This is the configuration file for the CI/CD system, and dictates 2 | ## which Ruby versions, variants and distributions we build. 3 | 4 | ## (Required) 5 | ## Ruby package settings. 6 | ruby: 7 | ## (Optional) 8 | ## Which minor Ruby version packages to build, and what their latest tiny 9 | ## version is. 10 | ## 11 | ## NOTE: if you change an existing `full_version` value, 12 | ## then be sure to bump the corresponding `package_revision`! 13 | minor_version_packages: 14 | - minor_version: '3.4' 15 | full_version: '3.4.4' 16 | package_revision: '3' 17 | - minor_version: '3.3' 18 | full_version: '3.3.8' 19 | package_revision: '8' 20 | - minor_version: '3.2' 21 | full_version: '3.2.8' 22 | package_revision: '11' 23 | - minor_version: '3.1' 24 | full_version: '3.1.7' 25 | package_revision: '16' 26 | 27 | ## (Optional) 28 | ## Which tiny Ruby version packages to build. 29 | tiny_version_packages: 30 | - full_version: '3.4.4' 31 | package_revision: '0' 32 | - full_version: '3.3.8' 33 | package_revision: '0' 34 | - full_version: '3.2.8' 35 | package_revision: '0' 36 | - full_version: '3.1.7' 37 | package_revision: '0' 38 | 39 | ## (Required) 40 | ## Which Rbenv version to package. 41 | rbenv: 42 | ## (Required: specify either 'repo' and 'ref', or just 'path') 43 | 44 | ## Package Rbenv from the following Git repo at the given commit hash or 45 | ## tag. 46 | ## 47 | ## NOTE: if you change one of these, but the Rbenv version number hasn't 48 | ## changed since the last fullstaq-rbenv package release, then be sure to 49 | ## bump `package_revision`! 50 | repo: https://github.com/fullstaq-ruby/rbenv.git 51 | ref: fbaa15993171bf 52 | 53 | ## Or package Rbenv from the following local directory. 54 | ## Be sure to comment out 'repo' and 'ref' if you use this. 55 | # path: /path-to-your-local-fork 56 | 57 | ## (Required) 58 | ## Specify the rbenv version number without commit hash 59 | ## suffix. Find it out with: 60 | ## 61 | ## bin/rbenv --version | awk '{ print $2 }' | sed -E 's/(.+)-.*/\1/' 62 | ## 63 | ## NOTE: if you change `version`, be sure to reset 64 | ## `package_revision` to 0. 65 | version: 1.1.2-16 66 | 67 | ## (Required) 68 | ## Package revision number. 69 | package_revision: 1 70 | 71 | ## (Required) 72 | ## fullstaq-ruby-common package parameters. 73 | ## 74 | ## NOTE: if you change `version`, be sure to reset 75 | ## `package_revision` to 0. 76 | common: 77 | rpm: 78 | version: '1.0' 79 | package_revision: '1' 80 | deb: 81 | version: '1.0' 82 | package_revision: '1' 83 | 84 | ## (Required) 85 | ## Which variants to build. 86 | variants: 87 | normal: true 88 | jemalloc: true 89 | malloctrim: true 90 | 91 | ## (Required if variants.jemalloc is true) 92 | ## Version of Jemalloc to compile against. 93 | ## 94 | ## NOTE: If you change this, then be sure to bump the Ruby 95 | ## `package_revision` numbers too! 96 | jemalloc_version: '3.6.0' 97 | 98 | ## (Required) 99 | ## Specify which distributions to build packages for. 100 | ## When set to 'all', will build for all supported distributions. 101 | distributions: all 102 | ## You can also set it to a list of distribution names. 103 | ## The names must match the folder names found in the environments/ 104 | ## directory in this project. 105 | # distributions: 106 | # - centos-8 107 | # - el-9 108 | # - debian-10 109 | # - debian-11 110 | # - debian-12 111 | # - ubuntu-20.04 112 | # - ubuntu-22.04 113 | # - ubuntu-24.04 114 | 115 | ## (Optional) 116 | ## Excludes certain Ruby minor versions from certain distributions. 117 | ## Useful if certain older Ruby versions don't compile on newer distributions or viceversa. 118 | distribution_exclusions: 119 | - ruby_minor_version: '3.1' 120 | distros: 121 | - debian-12 122 | - centos-8 123 | -------------------------------------------------------------------------------- /container-entrypoints/build-common-deb: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | # If you make a change that changes the DEB's contents or metadata, 5 | # then be sure to bump `common.deb.version` or 6 | # `common.deb.package_revision` in config.yml. 7 | 8 | SELFDIR=$(dirname "$0") 9 | SELFDIR=$(cd "$SELFDIR" && pwd) 10 | # shellcheck source=lib/library.sh 11 | source "$SELFDIR/../lib/library.sh" 12 | 13 | WORK_DIR=/home/utility/work 14 | OUTPUT_PATH=/output/common.deb 15 | 16 | require_container_mount "$OUTPUT_PATH" 17 | require_container_envvar VERSION 18 | require_container_envvar REVISION 19 | 20 | 21 | header "Building package..." 22 | run mkdir "$WORK_DIR" 23 | echo "+ cd $WORK_DIR" 24 | cd "$WORK_DIR" 25 | echo "+ export BUNDLE_GEMFILE=/home/utility/Gemfile" 26 | export BUNDLE_GEMFILE=/home/utility/Gemfile 27 | 28 | set -x 29 | bundle exec fpm -s dir -t deb -f --log info \ 30 | --name "fullstaq-ruby-common" \ 31 | --version "$VERSION" \ 32 | --iteration "$REVISION" \ 33 | --architecture all \ 34 | --license MIT \ 35 | --vendor Fullstaq \ 36 | --maintainer "Fullstaq " \ 37 | --description "Fullstaq Ruby common dependencies" \ 38 | --url "https://github.com/fullstaq-ruby/server-edition" \ 39 | --depends "fullstaq-rbenv" \ 40 | . 41 | set +x 42 | run cp *.deb "$OUTPUT_PATH" 43 | -------------------------------------------------------------------------------- /container-entrypoints/build-common-rpm: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | # If you make a change that changes the RPM's contents or metadata, 5 | # then be sure to bump `common.rpm.version` or 6 | # `common.rpm.package_revision` in config.yml. 7 | 8 | SELFDIR=$(dirname "$0") 9 | SELFDIR=$(cd "$SELFDIR" && pwd) 10 | # shellcheck source=lib/library.sh 11 | source "$SELFDIR/../lib/library.sh" 12 | 13 | WORK_DIR=/home/utility/work 14 | OUTPUT_PATH=/output/common.rpm 15 | 16 | require_container_mount "$OUTPUT_PATH" 17 | require_container_envvar VERSION 18 | require_container_envvar REVISION 19 | 20 | 21 | header "Building package..." 22 | run mkdir "$WORK_DIR" 23 | echo "+ cd $WORK_DIR" 24 | cd "$WORK_DIR" 25 | echo "+ export BUNDLE_GEMFILE=/home/utility/Gemfile" 26 | export BUNDLE_GEMFILE=/home/utility/Gemfile 27 | 28 | set -x 29 | bundle exec fpm -s dir -t rpm -f --log info \ 30 | --name "fullstaq-ruby-common" \ 31 | --version "$VERSION" \ 32 | --iteration "$REVISION" \ 33 | --architecture all \ 34 | --license MIT \ 35 | --vendor Fullstaq \ 36 | --maintainer "Fullstaq " \ 37 | --description "Fullstaq Ruby common dependencies" \ 38 | --url "https://github.com/fullstaq-ruby/server-edition" \ 39 | --depends "fullstaq-rbenv" \ 40 | . 41 | set +x 42 | run cp *.rpm "$OUTPUT_PATH" 43 | -------------------------------------------------------------------------------- /container-entrypoints/build-jemalloc: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | SELFDIR=$(cd "$SELFDIR" && pwd) 6 | # shellcheck source=lib/library.sh 7 | source "$SELFDIR/../lib/library.sh" 8 | 9 | SRC_CONTAINER_DIR=/home/builder/jemalloc-src 10 | INSTALL_PREFIX=/home/builder/jemalloc-inst 11 | INPUT_PATH=/input/jemalloc-src.tar.bz2 12 | OUTPUT_PATH=/output/jemalloc-bin.tar.gz 13 | 14 | require_container_mount "$INPUT_PATH" 15 | require_container_mount "$OUTPUT_PATH" 16 | require_container_envvar ENVIRONMENT_NAME 17 | 18 | BUILD_CONCURRENCY="${BUILD_CONCURRENCY:-1}" 19 | 20 | 21 | header "Setting up..." 22 | if [[ -n "$CACHE_CONTAINER" ]]; then 23 | echo "+ Caching enabled, will cache to Azure Blob Storage container $CACHE_CONTAINER." 24 | require_container_mount /azure-connection-string.txt 25 | SCCACHE_AZURE_CONNECTION_STRING=$(cat /azure-connection-string.txt) 26 | export SCCACHE_AZURE_CONNECTION_STRING 27 | export SCCACHE_AZURE_BLOB_CONTAINER="$CACHE_CONTAINER" 28 | export SCCACHE_AZURE_KEY_PREFIX="$CACHE_KEY_PREFIX" 29 | export SCCACHE_ERROR_LOG=/proc/$$/fd/2 30 | SCCACHE_START_SERVER=1 sccache 31 | export PATH="/usr/local/lib/sccache:$PATH" 32 | echo "+ Activating ccache compilers in /usr/local/lib/sccache." 33 | else 34 | echo "+ Caching disabled, not using sccache." 35 | fi 36 | echo 37 | 38 | 39 | header "Extracting Jemalloc tarball..." 40 | run mkdir "$SRC_CONTAINER_DIR" 41 | run tar -xjf "$INPUT_PATH" -C "$SRC_CONTAINER_DIR" 42 | 43 | # shellcheck disable=SC2012 44 | subdir="$(ls -1 "$SRC_CONTAINER_DIR" | head -n 1)" 45 | echo "+ cd $SRC_CONTAINER_DIR/$subdir" 46 | cd "$SRC_CONTAINER_DIR/$subdir" 47 | echo 48 | 49 | 50 | header "Compiling..." 51 | 52 | # https://github.com/fullstaq-ruby/server-edition/issues/34 53 | # https://github.com/fullstaq-ruby/server-edition/pull/32#issuecomment-554764548 54 | echo "+ patch -p1 < /system/resources/jemalloc_cxx_fix.patch" 55 | patch -t -p1 < /system/resources/jemalloc_cxx_fix.patch 56 | 57 | run ./configure --prefix="$INSTALL_PREFIX" --enable-shared --disable-static 58 | run make "-j$BUILD_CONCURRENCY" 59 | run make install 60 | run strip --strip-debug "$INSTALL_PREFIX/lib/libjemalloc.so" 61 | run rm -rf "$INSTALL_PREFIX/lib/pkgconfig" 62 | 63 | echo 64 | 65 | 66 | header "Packaging up..." 67 | echo "+ echo $ENVIRONMENT_NAME > $INSTALL_PREFIX/ENVIRONMENT" 68 | echo "$ENVIRONMENT_NAME" > "$INSTALL_PREFIX/ENVIRONMENT" 69 | run tar -czf "$OUTPUT_PATH" -C "$INSTALL_PREFIX" ENVIRONMENT include lib 70 | if [[ -n "$CACHE_CONTAINER" ]]; then 71 | run sccache --stop-server 72 | fi 73 | -------------------------------------------------------------------------------- /container-entrypoints/build-rbenv-deb: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | # If you make a change that changes the DEB's contents or metadata, 5 | # then be sure to bump `rbenv.package_revision` in config.yml. 6 | 7 | SELFDIR=$(dirname "$0") 8 | SELFDIR=$(cd "$SELFDIR" && pwd) 9 | # shellcheck source=lib/library.sh 10 | source "$SELFDIR/../lib/library.sh" 11 | 12 | WORK_DIR=/home/utility/work 13 | INPUT_PATH=/input/rbenv 14 | OUTPUT_PATH=/output/rbenv.deb 15 | 16 | require_container_mount "$INPUT_PATH" 17 | require_container_mount "$OUTPUT_PATH" 18 | require_container_envvar VERSION 19 | require_container_envvar REVISION 20 | 21 | 22 | header "Preparing input..." 23 | run mkdir "$WORK_DIR" 24 | echo "+ cd $WORK_DIR" 25 | cd "$WORK_DIR" 26 | run cp -dpR "$INPUT_PATH"/{bin,completions,libexec,rbenv.d} ./ 27 | run mkdir system 28 | run mkdir system/bin 29 | run ln -s /usr/lib/rbenv/libexec/rbenv system/bin/rbenv 30 | echo 31 | 32 | header "Building package..." 33 | echo "+ cd $WORK_DIR" 34 | cd "$WORK_DIR" 35 | echo "+ export BUNDLE_GEMFILE=/home/utility/Gemfile" 36 | export BUNDLE_GEMFILE=/home/utility/Gemfile 37 | 38 | set -x 39 | bundle exec fpm -s dir -t deb -f --log info \ 40 | --name "fullstaq-rbenv" \ 41 | --version "$VERSION" \ 42 | --iteration "$REVISION" \ 43 | --architecture all \ 44 | --license MIT \ 45 | --vendor Fullstaq \ 46 | --maintainer "Fullstaq " \ 47 | --description "Fullstaq Rbenv $VERSION" \ 48 | --url "https://github.com/fullstaq-labs/fullstaq-rbenv" \ 49 | --provides rbenv \ 50 | system/bin=/usr/ \ 51 | bin=/usr/lib/rbenv/ \ 52 | libexec=/usr/lib/rbenv/ \ 53 | rbenv.d=/usr/lib/rbenv/ \ 54 | completions=/usr/lib/rbenv/ 55 | set +x 56 | run cp *.deb "$OUTPUT_PATH" 57 | -------------------------------------------------------------------------------- /container-entrypoints/build-rbenv-rpm: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | # If you make a change that changes the RPM's contents or metadata, 5 | # then be sure to bump `rbenv.package_revision` in config.yml. 6 | 7 | SELFDIR=$(dirname "$0") 8 | SELFDIR=$(cd "$SELFDIR" && pwd) 9 | # shellcheck source=lib/library.sh 10 | source "$SELFDIR/../lib/library.sh" 11 | 12 | WORK_DIR=/home/utility/work 13 | INPUT_PATH=/input/rbenv 14 | OUTPUT_PATH=/output/rbenv.rpm 15 | 16 | require_container_mount "$INPUT_PATH" 17 | require_container_mount "$OUTPUT_PATH" 18 | require_container_envvar VERSION 19 | require_container_envvar REVISION 20 | 21 | 22 | header "Preparing input..." 23 | run mkdir "$WORK_DIR" 24 | echo "+ cd $WORK_DIR" 25 | cd "$WORK_DIR" 26 | run cp -dpR "$INPUT_PATH"/{bin,completions,libexec,rbenv.d} ./ 27 | run mkdir system 28 | run mkdir system/bin 29 | run ln -s /usr/lib/rbenv/libexec/rbenv system/bin/rbenv 30 | echo 31 | 32 | header "Building package..." 33 | echo "+ cd $WORK_DIR" 34 | cd "$WORK_DIR" 35 | echo "+ export BUNDLE_GEMFILE=/home/utility/Gemfile" 36 | export BUNDLE_GEMFILE=/home/utility/Gemfile 37 | 38 | set -x 39 | bundle exec fpm -s dir -t rpm -f --log info \ 40 | --name "fullstaq-rbenv" \ 41 | --version "$VERSION" \ 42 | --iteration "$REVISION" \ 43 | --architecture all \ 44 | --license MIT \ 45 | --vendor Fullstaq \ 46 | --maintainer "Fullstaq " \ 47 | --description "Fullstaq Rbenv $VERSION" \ 48 | --url "https://github.com/fullstaq-labs/fullstaq-rbenv" \ 49 | --provides rbenv \ 50 | system/bin=/usr/ \ 51 | bin=/usr/lib/rbenv/ \ 52 | libexec=/usr/lib/rbenv/ \ 53 | rbenv.d=/usr/lib/rbenv/ \ 54 | completions=/usr/lib/rbenv/ 55 | set +x 56 | run cp *.rpm "$OUTPUT_PATH" 57 | -------------------------------------------------------------------------------- /container-entrypoints/build-ruby-deb: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | SELFDIR=$(cd "$SELFDIR" && pwd) 6 | # shellcheck source=lib/library.sh 7 | source "$SELFDIR/../lib/library.sh" 8 | 9 | WORK_DIR=/home/utility/work 10 | INPUT_PATH=/input/ruby-bin.tar.gz 11 | OUTPUT_PATH=/output/ruby.deb 12 | 13 | require_container_mount "$INPUT_PATH" 14 | require_container_mount "$OUTPUT_PATH" 15 | require_container_envvar REVISION 16 | 17 | 18 | header "Extracting binaries..." 19 | run mkdir "$WORK_DIR" 20 | run tar -xzf "$INPUT_PATH" -C "$WORK_DIR" 21 | echo "+ cd $WORK_DIR" 22 | cd "$WORK_DIR" 23 | ENVIRONMENT=$(cat ENVIRONMENT) 24 | echo "+ Detected environment in which binaries were built: $ENVIRONMENT" 25 | echo 26 | 27 | header "Building package..." 28 | echo "+ export BUNDLE_GEMFILE=/home/utility/Gemfile" 29 | export BUNDLE_GEMFILE=/home/utility/Gemfile 30 | DEPS=$(cat shlib-deps.txt) 31 | PACKAGE_VERSION_WITH_SUFFIX=$(ls usr/lib/fullstaq-ruby/versions) 32 | PACKAGE_VERSION_WITHOUT_SUFFIX=$(cat PACKAGE_VERSION) 33 | VARIANT=$(cat VARIANT) 34 | echo "+ Package version (with variant suffix) detected: $PACKAGE_VERSION_WITH_SUFFIX" 35 | echo "+ Package version (without variant suffix) detected: $PACKAGE_VERSION_WITHOUT_SUFFIX" 36 | echo "+ Variant detected: $VARIANT" 37 | echo 38 | 39 | set -x 40 | # We embed metadata (variant and distro name) in the description field. 41 | # This metadata is used by scripts such as internal-scripts/upload-deb. 42 | bundle exec fpm -s dir -t deb -f --log info -C "$WORK_DIR" \ 43 | --name "fullstaq-ruby-$PACKAGE_VERSION_WITH_SUFFIX" \ 44 | --version "$REVISION" \ 45 | --iteration "$ENVIRONMENT" \ 46 | --license 'BSD 2-clause' \ 47 | --vendor Fullstaq \ 48 | --maintainer "Fullstaq " \ 49 | --description "Fullstaq Ruby $PACKAGE_VERSION_WITH_SUFFIX 50 | Package version with variant suffix: $PACKAGE_VERSION_WITH_SUFFIX 51 | Package version without variant suffix: $PACKAGE_VERSION_WITHOUT_SUFFIX 52 | Variant: $VARIANT 53 | Distribution: $ENVIRONMENT" \ 54 | --url "https://github.com/fullstaq-ruby/server-edition" \ 55 | --depends "$DEPS" \ 56 | --provides fullstaq-ruby \ 57 | --deb-recommends fullstaq-ruby-common \ 58 | --deb-dist "$ENVIRONMENT" \ 59 | usr 60 | set +x 61 | run cp *.deb "$OUTPUT_PATH" 62 | -------------------------------------------------------------------------------- /container-entrypoints/build-ruby-rpm: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | SELFDIR=$(cd "$SELFDIR" && pwd) 6 | # shellcheck source=lib/library.sh 7 | source "$SELFDIR/../lib/library.sh" 8 | 9 | WORK_DIR=/home/utility/work 10 | INPUT_PATH=/input/ruby-bin.tar.gz 11 | OUTPUT_PATH=/output/ruby.rpm 12 | 13 | require_container_mount "$INPUT_PATH" 14 | require_container_mount "$OUTPUT_PATH" 15 | require_container_envvar REVISION 16 | 17 | 18 | header "Extracting binaries..." 19 | run mkdir "$WORK_DIR" 20 | run tar -xzf "$INPUT_PATH" -C "$WORK_DIR" 21 | echo "+ cd $WORK_DIR" 22 | cd "$WORK_DIR" 23 | ENVIRONMENT=$(cat ENVIRONMENT) 24 | echo "+ Detected environment in which binaries were built: $ENVIRONMENT" 25 | echo 26 | 27 | 28 | header "Autodetecting dependencies..." 29 | 30 | DEP_BLACKLIST='libruby|libjemalloc|ld-linux' 31 | 32 | function filter_real_executables() 33 | { 34 | while read -r F; do 35 | if file "$F" | grep -q ' ELF '; then 36 | echo "$F" 37 | fi 38 | done 39 | } 40 | 41 | set -o pipefail 42 | echo "+ find . -name '*.so' -print0 | xargs -0 objdump -p > /tmp/binaries.txt" 43 | find . -name '*.so' -print0 | xargs -0 objdump -p > /tmp/binaries.txt 44 | echo "+ find . -executable -type f | filter_real_executables | xargs -0 objdump -p >> /tmp/binaries.txt" 45 | find . -executable -type f | filter_real_executables | xargs objdump -p >> /tmp/binaries.txt 46 | echo "+ grep -vE 'libruby|libjemalloc' < /tmp/binaries.txt | grep NEEDED | sort -u | awk '{ print \$2 }" 47 | grep -vE "$DEP_BLACKLIST" < /tmp/binaries.txt | grep NEEDED | sort -u | awk '{ print $2 "()(64bit)" }' > /tmp/deps.txt 48 | echo "+ Reading /tmp/deps.txt" 49 | DEPS=$(cat /tmp/deps.txt | xargs echo) 50 | 51 | 52 | header "Building package..." 53 | echo "+ export BUNDLE_GEMFILE=/home/utility/Gemfile" 54 | export BUNDLE_GEMFILE=/home/utility/Gemfile 55 | PACKAGE_VERSION_WITH_SUFFIX=$(ls usr/lib/fullstaq-ruby/versions) 56 | PACKAGE_VERSION_WITHOUT_SUFFIX=$(cat PACKAGE_VERSION) 57 | VARIANT=$(cat VARIANT) 58 | echo "+ Package version (with variant suffix) detected: $PACKAGE_VERSION_WITH_SUFFIX" 59 | echo "+ Package version (without variant suffix) detected: $PACKAGE_VERSION_WITHOUT_SUFFIX" 60 | echo "+ Variant detected: $VARIANT" 61 | 62 | set -x 63 | ITERATION=$(sed 's/-//g' <<<"$ENVIRONMENT") 64 | # We embed metadata (variant and distro name) in the description field. 65 | # This metadata is used by scripts such as internal-scripts/upload-rpm. 66 | bundle exec fpm -s dir -t rpm -f --log info -C "$WORK_DIR" \ 67 | --name "fullstaq-ruby-$PACKAGE_VERSION_WITH_SUFFIX" \ 68 | --version "rev$REVISION" \ 69 | --iteration "$ITERATION" \ 70 | --license 'BSD 2-clause' \ 71 | --vendor Fullstaq \ 72 | --maintainer "Fullstaq " \ 73 | --rpm-summary "Fullstaq Ruby $PACKAGE_VERSION_WITH_SUFFIX" \ 74 | --description "Fullstaq Ruby $PACKAGE_VERSION_WITH_SUFFIX 75 | Package version with variant suffix: $PACKAGE_VERSION_WITH_SUFFIX 76 | Package version without variant suffix: $PACKAGE_VERSION_WITHOUT_SUFFIX 77 | Variant: $VARIANT 78 | Distribution: $ENVIRONMENT" \ 79 | --url "https://github.com/fullstaq-ruby/server-edition" \ 80 | --provides fullstaq-ruby \ 81 | --depends "$DEPS" \ 82 | --rpm-tag '%define _build_id_links none' \ 83 | --rpm-tag '%undefine _missing_build_ids_terminate_build' \ 84 | usr 85 | set +x 86 | run cp *.rpm "$OUTPUT_PATH" 87 | -------------------------------------------------------------------------------- /container-entrypoints/test-debs: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | SELFDIR=$(cd "$SELFDIR" && pwd) 6 | # shellcheck source=lib/library.sh 7 | source "$SELFDIR/../lib/library.sh" 8 | 9 | export REPO_PATH=/input/repo 10 | 11 | require_container_mount "$REPO_PATH" 12 | require_container_envvar EXPECTED_VARIANT 13 | if [[ "$SERVER" != "" ]]; then 14 | require_container_envvar APT_DISTRO_NAME 15 | require_container_envvar RUBY_PACKAGE_VERSION 16 | fi 17 | require_container_envvar DEBUG_AFTER_TESTS 18 | 19 | if /system/internal-scripts/test-debs; then 20 | echo 21 | echo '-------------------' 22 | if $DEBUG_AFTER_TESTS; then 23 | if tty -s; then 24 | echo '*** NOTICE: Test run succeeded. Opening a shell session for debugging.' 25 | echo ' TIP: run this to activate Rbenv: eval "$(rbenv init -)"' 26 | bash -l 27 | else 28 | echo '*** ERROR: Cannot open a debug shell session because there is no TTY.' 29 | fi 30 | fi 31 | exit 0 32 | else 33 | echo 34 | echo '-------------------' 35 | if $DEBUG_AFTER_TESTS; then 36 | if tty -s; then 37 | echo '*** ERROR: Test run failed. Opening a shell session for debugging.' 38 | echo ' TIP: run this to activate Rbenv: eval "$(rbenv init -)"' 39 | bash -l 40 | else 41 | echo '*** ERROR: Test run failed. Cannot open a debug shell session either because there is no TTY.' 42 | fi 43 | else 44 | echo '*** ERROR: Test run failed. If you passed -D then a debug shell session would have been opened.' 45 | fi 46 | exit 1 47 | fi 48 | -------------------------------------------------------------------------------- /container-entrypoints/test-debs-prepare: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | SELFDIR=$(cd "$SELFDIR" && pwd) 6 | # shellcheck source=lib/library.sh 7 | source "$SELFDIR/../lib/library.sh" 8 | 9 | INPUT_PATH=/input 10 | OUTPUT_PATH=/output 11 | 12 | require_container_mount "$INPUT_PATH" 13 | require_container_mount "$OUTPUT_PATH" 14 | 15 | 16 | run aptly repo create test 17 | run aptly repo add test /input/*.deb 18 | run aptly publish repo -skip-signing -distribution=test test 19 | run cp -dpR ~/.aptly/public/* "$OUTPUT_PATH/" 20 | -------------------------------------------------------------------------------- /container-entrypoints/test-rpms: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | SELFDIR=$(cd "$SELFDIR" && pwd) 6 | # shellcheck source=lib/library.sh 7 | source "$SELFDIR/../lib/library.sh" 8 | 9 | export REPO_PATH=/input/repo 10 | 11 | require_container_mount "$REPO_PATH" 12 | require_container_envvar EXPECTED_VARIANT 13 | if [[ "$SERVER" != "" ]]; then 14 | require_container_envvar RUBY_PACKAGE_VERSION 15 | fi 16 | require_container_envvar DEBUG_AFTER_TESTS 17 | 18 | if /system/internal-scripts/test-rpms; then 19 | echo 20 | echo '-------------------' 21 | if $DEBUG_AFTER_TESTS; then 22 | if tty -s; then 23 | echo '*** NOTICE: Test run succeeded. Opening a shell session for debugging.' 24 | echo ' TIP: run this to activate Rbenv: eval "$(rbenv init -)"' 25 | bash -l 26 | else 27 | echo '*** ERROR: Cannot open a debug shell session because there is no TTY.' 28 | fi 29 | fi 30 | exit 0 31 | else 32 | echo 33 | echo '-------------------' 34 | if $DEBUG_AFTER_TESTS; then 35 | if tty -s; then 36 | echo '*** ERROR: Test run failed. Opening a shell session for debugging.' 37 | echo ' TIP: run this to activate Rbenv: eval "$(rbenv init -)"' 38 | bash -l 39 | else 40 | echo '*** ERROR: Test run failed. Cannot open a debug shell session either because there is no TTY.' 41 | fi 42 | else 43 | echo '*** ERROR: Test run failed. If you passed -D then a debug shell session would have been opened.' 44 | fi 45 | exit 1 46 | fi 47 | -------------------------------------------------------------------------------- /container-entrypoints/test-rpms-prepare: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | SELFDIR=$(cd "$SELFDIR" && pwd) 6 | # shellcheck source=lib/library.sh 7 | source "$SELFDIR/../lib/library.sh" 8 | 9 | INPUT_PATH=/input 10 | OUTPUT_PATH=/output 11 | 12 | require_container_mount "$INPUT_PATH" 13 | require_container_mount "$OUTPUT_PATH" 14 | 15 | 16 | run cp /input/* /output/ 17 | run createrepo /output 18 | -------------------------------------------------------------------------------- /dev-handbook/README.md: -------------------------------------------------------------------------------- 1 | # Development handbook 2 | 3 | ## Concepts 4 | 5 | ### Core concepts 6 | 7 | * [The "minimal dependencies" principle & the use of Docker](minimal-dependencies-principle.md) — on the fact that we don't require many dependencies to be installed on the local machine, and that instead we use dependencies through Docker. 8 | * [Package organization](package-organization.md) — which package types exist, what they are for, and how they relate to each other. 9 | * [Build steps](build-steps.md) — what steps are involved in order to build packages, and how they relate to each other. 10 | * [Build workflow management](build-workflow-management.md) — how we manage the fact that we have to build packages for multiple distributions, variants and Ruby versions. 11 | * [Build environments](build-environments.md) — how we utilize Docker images to build packages for different distributions and for running tasks. 12 | * [Source organization](source-organization.md) — how the Git repository is organized, and which files serve what purpose. 13 | 14 | ### Advanced concepts 15 | 16 | * [CI/CD system resumption support](ci-cd-resumption.md) 17 | * [CI/CD system and splitting into multiple workflows](ci-cd-split-multiple-workflows.md) 18 | * [CI/CD caching](ci-cd-caching.md) 19 | * [APT and YUM repository setup](apt-yum-repo.md) 20 | 21 | ## Tutorials & tasks 22 | 23 | ### General 24 | 25 | * [Development environment set up](dev-environment-setup.md) 26 | * [Building packages locally](building-packages-locally.md) 27 | * [Fixing bugs](fixing-bugs.md) 28 | 29 | ### Testing 30 | 31 | * [Testing packages locally](testing-packages-locally.md) 32 | * [Adding, modifying & debugging tests](#modifying-and-debugging-tests.md) 33 | 34 | ### CI/CD 35 | 36 | * [Troubleshooting corrupt CI/CD artifacts](troubleshooting-corrupt-ci-cd-artifacts.md) 37 | * [Speeding up CI feedback](speeding-up-ci-feedback.md) 38 | 39 | ### Routine maintenance 40 | 41 | * [Adding support for a new distribution](add-new-distro.md) 42 | * [Adding support for a new Ruby version](add-new-ruby-version.md) 43 | 44 | ## Organizational (for team members) 45 | 46 | * [Way of working](way-of-working.md) 47 | * [Responsibilities & expectations](responsibilities-expectations.md) 48 | * [Members](members.md) 49 | * [Mentorship](mentorship.md) 50 | * [Onboarding](onboarding.md) 51 | * [Offboarding](offboarding.md) 52 | 53 | ## References & recommended reading 54 | 55 | * [How Debian packaging works](https://www.joyfulbikeshedding.com/blog/2020-08-03-how-debian-packaging-works.html) 56 | * [Inspecting & extracting RPM packages](https://blog.packagecloud.io/eng/2015/10/13/inspect-extract-contents-rpm-packages/) 57 | * [FPM manual](http://fpm.readthedocs.io/en/latest/) 58 | * [Advanced Bash-Scripting Guide](https://tldp.org/LDP/abs/html/) 59 | -------------------------------------------------------------------------------- /dev-handbook/add-new-distro.md: -------------------------------------------------------------------------------- 1 | # Adding support for a new distribution 2 | 3 | All packages are built within [build environments](build-environments.md): Docker containers that are based on the distribution that we want to package for, and that contains all the tooling (such as compilers) that we need. So adding support for a new distribution involves creating a new build environment for that distribution. 4 | 5 | Note that older Ruby versions may fail to compile on newer distributions. See [Excluding Ruby versions from certain distributions](#excluding-ruby-versions-from-certain-distributions) to learn how to deal with that. 6 | 7 | ## Before you begin 8 | 9 | Follow the [Development environment setup](dev-environment-setup.md) instructions. In particular, be sure to setup the Git hooks. 10 | 11 | ## Step 1: Creating the build environment 12 | 13 | Create a new directory `environments/-` (for example `environments/centos-16`). This directory should contain: 14 | 15 | * A `Dockerfile`. 16 | 17 | - Ensure its `FROM` is set to an appropriate image that corresponds to the distribution and version you want to support. For example `FROM centos:16`. 18 | - Ensure it has a user account named `builder`, and that the Dockerfile's `USER` directive is set to that account. 19 | - Ensure a C and C++ compiler toolchain is installed. 20 | - Ensure [sccache](https://github.com/mozilla/sccache) is installed and configured. 21 | - Ensure development headers for OpenSSL, zlib, FFI, readline, ncurses and GDBM are installed. 22 | 23 | * An `image_tag`. This file is used for [versioning](build-environments.md#versioning). Since this is a new file, set its contents to `1`. 24 | 25 | When done, build this build environment locally by running: 26 | 27 | ~~~bash 28 | ./build-environment-image ${DISTRO_NAME}-${DISTRO_VERSION} 29 | ~~~ 30 | 31 | For example: 32 | 33 | ~~~bash 34 | ./build-environment-image debian-13 35 | ~~~ 36 | 37 | ## Step 2: Updating config.yml 38 | 39 | In `config.yml`, scroll to the `distributions` section, and add a comment indicating that we support this distribution. This comment doesn't do anything, and is only there for completeness' sake. 40 | 41 | ## Step 3: Testing the build environment 42 | 43 | Test this build environment by: 44 | 45 | 1. [Building packages with it](building-packages-locally.md). 46 | 2. [Testing the built packages](testing-packages-locally.md). 47 | 48 | ## Step 4: Update docs 49 | 50 | In README.md under "Installation", add instructions for this new distribution. 51 | 52 | ## Step 5: Commit and push your changes 53 | 54 | Commit and push your changes to the Git repo. 55 | 56 | ## Excluding Ruby versions from certain distributions 57 | 58 | If an older Ruby version fails to compile on a newer distribution (for example, Ruby < 3.1 is not compatible with OpenSSL v3, which is shipped by Ubuntu 22.04), then you should add an _exclusion_ for this particular pair Ruby version and distribution. 59 | 60 | In `config.yml`, scroll to the `distribution_exclusions` section and add an entry (or extend an existing one), like this: 61 | 62 | ~~~yaml 63 | distribution_exclusions: 64 | - ruby_minor_version: '3.0' 65 | distros: 66 | - ubuntu-22.04 67 | ~~~ 68 | 69 | With exclusions in place, the CI/CD system will skip building the specified Ruby versions from the associated distributions. 70 | -------------------------------------------------------------------------------- /dev-handbook/add-new-ruby-version.md: -------------------------------------------------------------------------------- 1 | # Adding support for a new Ruby version 2 | 3 | To add support for a new Ruby version, open `config.yml` and add it to the `ruby` section. Make sure you update both `minor_version_packages` and `tiny_version_packages`. 4 | 5 | Under `minor_version_packages`, if you're modifying an existing entry (as opposed to adding a new one), then don't forget to bump the `package_revision`. 6 | -------------------------------------------------------------------------------- /dev-handbook/build-environments.md: -------------------------------------------------------------------------------- 1 | # Build environments 2 | 3 | ## Distribution compilation environments 4 | 5 | In order to build a package for a specific distribution, we must compile source code using that distribution. We use Docker for this purpose. For each distribution that we support, we maintain a Docker image in the directory `environments//`. 6 | 7 | These images are used by the `build-ruby` and `build-jemalloc` scripts. These images contain a compiler toolchain, neccessary libraries, and whatever else is needed in order to compile Ruby and Jemalloc for that distribution. 8 | 9 | ## Utility environment 10 | 11 | A special Docker image is `environments/utility/`, which is used by scripts that perform work that's not related to any specific distribution. These scripts may require certain tools, but because we follow [the "minimal dependencies" principle](minimal-dependencies-principle.md) we don't want to require users to install such tools on their local machines. So we install these tools in the `utility` image, and perform the bulk of the work in the context of such a container. 12 | 13 | For example: `build-common-deb` builds the fullstaq-ruby-common Debian package (see [Package organization](package-organization.md)). This scripts requires `dpkg` and FPM to be installed. The `utility` image has both of these installed. 14 | 15 | ## Environment do not contain scripts 16 | 17 | None of the build environment images contain scripts. They only contain dependencies and tools required by scripts. We run environments by: 18 | 19 | 1. Creating a container from that build environment's image. 20 | 2. Having that container mount the Fullstaq Ruby's source root. 21 | 3. Telling the container to run a script inside our source tree. 22 | 23 | ## Mounts 24 | 25 | Scripts always run build environment containers in such a way, that inside the container, `/system` is mounted read-only to the source root. 26 | 27 | ## Versioning 28 | 29 | All build environment Docker images are versioned. This is why each `environments/*/` directory has a file `image_tag`. This tag must be bumped every time we make a change to an image. All scripts, which delegate their work to a build environment, read that build environment's `image_tag` file, and runs the Docker container based on an image with that tag. 30 | -------------------------------------------------------------------------------- /dev-handbook/build-steps.md: -------------------------------------------------------------------------------- 1 | # Build steps 2 | 3 | The process of building all Fullstaq Ruby packages is separated in multiple smaller steps. The following diagram describes what all the steps are: 4 | 5 | ![](build-steps.png) 6 | 7 | The black arrows denote dependencies. A dependency exists when the output of a step is used as input by the dependent step. 8 | 9 | Each step in the diagram specifies whether the Git repo contains a shell script for executing that step. 10 | 11 | Some steps should be executed multiple times: 12 | 13 | * "Build Ruby binaries" should be executed once for every distribution, variant and Ruby version we want to build for. 14 | * "Build Jemalloc binaries" should be executed once for every distribution we want to build for. 15 | * "Build Ruby DEB|RPM" should be executed once for every distribution, variant and Ruby version we want to build for. 16 | -------------------------------------------------------------------------------- /dev-handbook/build-steps.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fullstaq-ruby/server-edition/683685feb308f3c823501e566edea3c8ac43bc5c/dev-handbook/build-steps.png -------------------------------------------------------------------------------- /dev-handbook/build-workflow-management.md: -------------------------------------------------------------------------------- 1 | # Build workflow management 2 | 3 | ## What is build workflow management? 4 | 5 | Building packages involves [multiple steps](build-steps.md), some of which have dependencies, and some of which must be run multiple times (e.g. once per distribution, variant, Ruby version). The entire _workflow_ of building all packages that we intend to build, means: 6 | 7 | * Running the steps in the right order. 8 | * Ensuring that each step's output is properly passed to its depending steps as input. 9 | * Running a step the proper number of times, each time against the proper distribution, variant and Ruby version. 10 | 11 | The parameters for the workflow are defined in `config.yml`. This file defines: 12 | 13 | * Which Ruby versions we want to build packages for. 14 | * Which variants we want to build packages for. 15 | * Which distributions we want to build packages for. 16 | 17 | ## Why build workflow management is needed 18 | 19 | Workflow management is needed because, given the number of Ruby versions, variants and distributions we want to package for, it's very easy to reach a huge combinatorial explosion. 20 | 21 | For example, suppose that want to support Ruby 2.6.6 and 2.7.1 only, on Debian 9 and 10 only. We'll have to run the "Build Ruby" step at least `num_ruby_versions * num_distributions * num_variants = 2 * 2 * 3 = 12` times: 22 | 23 | * Ruby 2.6.6 + Debian 9 + normal variant 24 | * Ruby 2.6.6 + Debian 9 + malloctrim variant 25 | * Ruby 2.6.6 + Debian 9 + jemalloc variant 26 | * Ruby 2.6.6 + Debian 10 + normal variant 27 | * Ruby 2.6.6 + Debian 10 + malloctrim variant 28 | * Ruby 2.6.6 + Debian 10 + jemalloc variant 29 | * Ruby 2.7.1 + Debian 9 + normal variant 30 | * Ruby 2.7.1 + Debian 9 + malloctrim variant 31 | * Ruby 2.7.1 + Debian 9 + jemalloc variant 32 | * Ruby 2.7.1 + Debian 10 + normal variant 33 | * Ruby 2.7.1 + Debian 10 + malloctrim variant 34 | * Ruby 2.7.1 + Debian 10 + jemalloc variant 35 | 36 | This is already a pretty big list. It's pretty cumbersome to manually run the "Build Ruby" step so many times, each time with the proper parameters, and hoping that we didn't make a mistake. 37 | 38 | It gets worse very quikcly. Let's say the next day we want to support Ruby 2.7.2 and CentOS 8 too. That results in `num_ruby_versions * num_distributions * num_variants = 3 * 3 * 3 = 27` combinations. Just by adding 1 Ruby version and 1 distribution to support, we've more than doubled the amount of combinations. 39 | 40 | The more "stuff" we support, the more quickly the number rises! Clearly, some sort of automation is needed, which takes care of not only calculating what all the different combinations are, but also running the build steps with the proper parameters. 41 | 42 | ## Implementation 43 | 44 | In our codebase, workflow management is separated from step execution. This means that the build scripts for the individual steps (e.g. `build-ruby`) know nothing about config.yml. These scripts accept parameters, and it is up to a higher workflow management system to pass the right parameters. 45 | 46 | The workflow management system is implemented in Github Workflow, in `.github/workflows/ci-cd-*.yml.erb`. These are ERB templates that, given the parameters in `config.yml`, outputs a bunch of Github Workflow YAML files that roughly implements the build pipeline described in [Build steps](build-steps.md): 47 | 48 | ![](build-steps.png) 49 | 50 | The ERB templates are run by `./internal-scripts/generate-ci-cd.rb`. It is recommended that you [setup a Git hook](dev-environment-setup.md), which runs that script automatically before every commit, so that the Github workflow file gets properly updated whenever you either modify the ERB template, or config.yml. 51 | -------------------------------------------------------------------------------- /dev-handbook/ci-cd-caching.md: -------------------------------------------------------------------------------- 1 | # CI/CD caching 2 | 3 | The CI/CD system caches C/C++ compiler invocations using [sccache](https://github.com/mozilla/sccache). Sccache is a C/C++/Rust compiler cacher, similar to [ccache](https://ccache.dev/), but caches to cloud storage instead of the local filesystem. 4 | 5 | We used to use ccache, in combination with Github's [cache action](https://github.com/actions/cache) to cache the ccache directory. We had a unique ccache directory on a per-distribution, per-Ruby version and per-variant basis. But each ccache directory can be huge (hundreds of MBs or several GBs). This gave rise to several problems: 6 | 7 | * Because [we build so many different packages](build-workflow-management.md), we easily exceed the Github Actions cache storage limit of 10 GB. 8 | * Downloading/uploading the entire ccache directory takes a significant amount of time. 9 | * Ccache may not need to access all files in the ccache directory, so downloading the entire ccache directory can be overkill. 10 | * Variants have many similarities, and so we should be able to achieve a reasonable cache hit rate if compilations of different variants (given the same distribution and Ruby version) share the same cache. With the ccache approach, cache sharing between different CI jobs is very difficult. 11 | * CI jobs for Git branches can't warm up the cache for the CI job that will run when the branch is merged to main. 12 | 13 | Sccache solves all of the above problems. We cache Azure Blob Storage. All CI jobs access the same Azure Blob Storage container, but we divide that container into multiple separate logical caches by using prefixes (directory names). We assign a unique prefix only on a per-distribution basis. This means that all CI jobs targeting the same distribution can share the same cache, regardless of the Ruby version or variant being compiled, and regardless of the branch. 14 | 15 | > Trivia: we [contributed the ability to specify a prefix within Azure Blob Storage](https://github.com/mozilla/sccache/pull/1109) so that we can use sccache in Fullstaq Ruby. 16 | 17 | ## Why Azure Blob Storage? 18 | 19 | The choice of Azure Blob Storage may seem odd given the fact that [the rest of our infrastructure is hosted on Google Cloud](https://github.com/fullstaq-ruby/infra/blob/main/docs/infrastructure-overview.md). The reason is performance. Github-hosted Actions runners are hosted on Azure. [Research has shown](https://github.com/fullstaq-ruby/server-edition/issues/86#issuecomment-1032643774) that latency between Azure and Google Cloud Storage is pretty abysmal: caching to Google Cloud Storage actually makes things **slower**, even if the runner and the Google Cloud Storage bucket are located near each other. In contrast, latency between runners and Azure Blob Storage is very good, even if runners are located on two opposite sides of the US continent. 20 | 21 | ## Expiration 22 | 23 | We expire cache entries based on last access time. Using [lifecycle rules](https://docs.microsoft.com/en-us/azure/storage/blobs/lifecycle-management-overview#rule-actions), we automatically delete objects that haven't been accessed in 90 days. 24 | 25 | Ideally we want LRU expiration based by total cache size, like ccache does. But this approach is not supported by sccache when using Azure Blob Storage (likely because calculating the total cache size is expensive), so last-access-time-based expiration is the next best thing. 26 | 27 | ## Performance impact 28 | 29 | The use of sccache impacts performance as follows (compared to not using sccache): 30 | 31 | * Cold compilations are ~28% slower. 32 | * Hot compilations are ~56% faster. 33 | -------------------------------------------------------------------------------- /dev-handbook/ci-cd-resumption.md: -------------------------------------------------------------------------------- 1 | # CI/CD system resumption support 2 | 3 | Our Github Actions-based CI/CD system performs [a lot of work](build-workflow-management.md), and takes \~45 minutes per run. Sometimes it could fail due to a random error, such as a network issue or a CI job runner that gets stuck. When that happens, we need to re-run the CI run. Unfortunately, Github Actions only supports re-running a CI job from scratch, instead of only failed jobs. Re-running from scratch wastes a lot of time and resources, and a re-run could also fail. 4 | 5 | In order to aleviate this problem, we implement the ability to re-run only failed jobs, ourselves. We call this _resumption support_: the ability for the CI to resume where it left off last time. 6 | 7 | ## How resumption support works 8 | 9 | Resumption support works by checking, for each CI job, whether the artifact that that job should produce, already exists. If so, then that job can be skipped. 10 | 11 | When you re-run a CI run, Github Actions wipes all previous state (including artifacts). Therefore, we store artifacts primarily in a Google Cloud Storage bucket ([fullstaq-ruby-server-edition-ci-artifacts](https://storage.googleapis.com/fullstaq-ruby-server-edition-ci-artifacts), part of the [infrastructure](https://github.com/fullstaq-labs/fullstaq-ruby-infra)), which isn't wiped before a re-run. 12 | 13 | Here's an example artifact URL: 14 | 15 | ~~~ 16 | gs://fullstaq-ruby-server-edition-ci-artifacts/249/rbenv-deb.tar.zst 17 | ~~~ 18 | 19 | Artifacts are stored on a per-CI-run basis. Thus, they always contains the CI run's number. Note that the CI run number does not change even for re-runs. 20 | 21 | At the beginning of a CI run, a job named `determine_necessary_jobs` checks which artifacts exist in the Google Cloud Storage bucket, and determines based on that information which other jobs should be run. Here's an example step in that job: 22 | 23 | ~~~ 24 | ##### Determine whether Rbenv DEB needs to be built ##### 25 | --> Run ./.github/actions/check-artifact-exists 26 | Checking gs://fullstaq-ruby-server-edition-ci-artifacts/249/rbenv-deb.tar.zst 27 | Artifact exists 28 | ~~~ 29 | 30 | The `determine_necessary_jobs` job [outputs a variable](https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#jobsjobs_idoutputs) that indicate which other jobs should be run: 31 | 32 | ~~~yaml 33 | determine_necessary_jobs: 34 | name: Determine necessary jobs 35 | runs-on: ubuntu-20.04 36 | outputs: 37 | necessary_jobs: ${{ steps.check.outputs.necessary_jobs }} 38 | ... 39 | ~~~ 40 | 41 | The value of this variable is set by `internal-scripts/ci-cd/determine-necessary-jobs/determine-necessary-jobs.rb`. This script sets the variable to a string in the following format: 42 | 43 | ~~~ 44 | ;Download Ruby source 2.7.1;Download Rbenv source;...;;...; 45 | ~~~ 46 | 47 | Then, other jobs use an `if` statement which performs a substring match, in order to check whether that particular job should be run: 48 | 49 | ~~~yaml 50 | download_ruby_source_<%= slug(ruby_version) %>: 51 | name: Download Ruby source [<%= ruby_version %>] 52 | runs-on: ubuntu-20.04 53 | needs: 54 | - determine_necessary_jobs 55 | if: contains(needs.determine_necessary_jobs.outputs.necessary_jobs, ';Download Ruby source <%= ruby_version %>;') 56 | ~~~ 57 | -------------------------------------------------------------------------------- /dev-handbook/ci-run-number.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fullstaq-ruby/server-edition/683685feb308f3c823501e566edea3c8ac43bc5c/dev-handbook/ci-run-number.png -------------------------------------------------------------------------------- /dev-handbook/dev-environment-setup.md: -------------------------------------------------------------------------------- 1 | # Development environment set up 2 | 3 | ## Host OS & dependencies 4 | 5 | You can develop Fullstaq Ruby on Linux, macOS, or Windows with WSL2. You just need the following things installed: 6 | 7 | * Bash 8 | * Docker 9 | * Ruby (any version >= 3.2) 10 | 11 | The small number of dependencies is [intentional](minimal-dependencies-principle.md). 12 | 13 | ## Git hooks 14 | 15 | You should also setup our Git hooks, so that on every commit, the Github Workflow file is properly updated from any changes you may make from either its ERB template, or the workflow configuration file. For more info about this, see [Build workflow management](build-workflow-management.md). 16 | 17 | To setup the Git hooks, run: 18 | 19 | ~~~bash 20 | git config core.hooksPath .githooks 21 | ~~~ 22 | -------------------------------------------------------------------------------- /dev-handbook/github-actions-checks.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fullstaq-ruby/server-edition/683685feb308f3c823501e566edea3c8ac43bc5c/dev-handbook/github-actions-checks.png -------------------------------------------------------------------------------- /dev-handbook/members.md: -------------------------------------------------------------------------------- 1 | # Members 2 | 3 | * Hongli Lai [(he/him)](https://www.mypronouns.org/he-him) - [@FooBarWidget](https://github.com/FooBarWidget) -- honglilai@gmail.com 4 | * Daniel Paulus [(he/him)](https://www.mypronouns.org/he-him) - [@PaulusTM](https://github.com/PaulusTM) -- d.paulus@gmail.com 5 | -------------------------------------------------------------------------------- /dev-handbook/mentorship.md: -------------------------------------------------------------------------------- 1 | # Mentorship 2 | 3 | Some [team members](members.md) provide mentorship, for both people seeking to contribute, as well as to other team members. This happens on request. 4 | 5 | To request mentorship, please [post to the discussion forum](https://github.com/fullstaq-ruby/server-edition/discussions) and describe what you need. 6 | -------------------------------------------------------------------------------- /dev-handbook/minimal-dependencies-principle.md: -------------------------------------------------------------------------------- 1 | # The "minimal dependencies" principle & the use of Docker 2 | 3 | Our source code follows the following principles: 4 | 5 | * Users must be able to build packages for any distribution, regardless of which OS or distribution they're running on. 6 | * All scripts must be usable without requiring the local machine to have anything other than Bash, Ruby and Docker installed. 7 | * Do not require the user to run `gem install` or `bundle install`. 8 | 9 | This means that we do most of our work in Docker containers, so that we can have controlled [build environments](build-environments.md). All other than Bash and Ruby are to be used through Docker containers. 10 | 11 | This is why most of the scripts in the root directory are simple wrappers that parse arguments, and then hand off the bulk of the work to a different script, which is run inside a Docker container. For this purpose, we maintain [multiple Docker images](build-environments.md). 12 | 13 | Let's look at `build-ruby-deb` for example. This script takes as input a tarball with compiled Ruby binaries, and produces as output a .deb file. [We use FPM](https://www.joyfulbikeshedding.com/blog/2020-08-03-how-debian-packaging-works.html) — which is a Ruby gem — to produce such a .deb file. Instead of telling the user to `gem install fpm`, the `build-ruby-deb` script invokes the `container-entrypoints/build-ruby-deb` script inside the `environments/utility` Docker container. That container has FPM installed. 14 | -------------------------------------------------------------------------------- /dev-handbook/modifying-and-debugging-tests.md: -------------------------------------------------------------------------------- 1 | # Adding, modifying & debugging tests 2 | 3 | The [test suite](testing-packages-locally.md) consists of the following key scripts. This guide tells you: 4 | 5 | * In which order the scripts are run, which script serves what purpose and in what environment they run. This allows you to know which files you need to modify in order to modify or extend the test suite. 6 | * How to develop tests in a fast cycle. 7 | * How to debug the test suite. 8 | 9 | **Table of contents** 10 | 11 | * [Test suite phases](#test-suite-phases) 12 | - [Phase 1: test-{debs,rpms}](#phase-1-test-debsrpms) 13 | - [Phase 2: container-entrypoints/test-{debs,rpms}-prepare](#phase-2-container-entrypointstest-debsrpms-prepare) 14 | - [Phase 3: container-entrypoints/test-{debs,rpms}](#phase-3-container-entrypointstest-debsrpms) 15 | - [Phase 4: internal-scripts/test-{debs,rpms}](#phase-4-internal-scriptstest-debsrpms) 16 | * [Debugging](#debugging) 17 | * [Speeding up the test suite development cycle](#speeding-up-the-test-suite-development-cycle) 18 | 19 | ## Test suite phases 20 | 21 | ### Phase 1: test-{debs,rpms} 22 | 23 | These are simply wrappers that parse arguments, and then delegate most of their work to Docker containers, which then run the other scripts mention in this section. 24 | 25 | You only need to modify these scripts when you need to change CLI parameters. 26 | 27 | ### Phase 2: container-entrypoints/test-{debs,rpms}-prepare 28 | 29 | The test script in phase 1, first launches a [utility](build-environments.md) Docker container, in which it runs the corresponding "prepare" script. 30 | 31 | This phase 2 script sets up an APT or YUM repo. So that the script in phase 3 can install the package from a local APT/YUM repo. 32 | 33 | You usually don't need to modify this phase 2 script. 34 | 35 | ### Phase 3: container-entrypoints/test-{debs,rpms} 36 | 37 | The test script in phase 1 now launches a Docker container (whose image corresponds to the `-i` parameter). Inside the container, it calls the corresponding container-entrypoint script. 38 | 39 | This phase 3 script's only responsibility is to do the following, inside said container, in the given order: 40 | 41 | 1. Run the script in phase 4. 42 | 2. Open a shell, if `-D` was passed to `./test-{debs,rpms}`. See section [Debugging](#debugging). 43 | 44 | You usually don't need to modify this phase 3 script. 45 | 46 | ### Phase 4: internal-scripts/test-{debs,rpms} 47 | 48 | **This is where the main test suite code is located!** This is probably the file you want to modify. 49 | 50 | Notes: 51 | 52 | * This script is run as root. But most test commands should be run as the `utility` user. You can use `sudo -u utility -H` for that purpose. 53 | * You should use the `run` function in order to run commands. This function logs the command to be run, then runs it. This way you don't need to perform a separate "echo" in order to tell the user which command is being run. 54 | 55 | ## Debugging 56 | 57 | If you want to debug the test suite (regardless of whether it fails), then you can pass the `-D` option to the `./test-{debs,rpms}` script in the source root. This way, when the test finishes (regardless of success or failure) it will spawn a Bash shell inside the phase 3 script's Docker container, so that you can poke around and inspect things. 58 | 59 | The shell is launched as the `utility` user, which most test commands are run as. If you require root privileges, then use `sudo`, which does not require a password. 60 | 61 | Inside the container, the Fullstaq Ruby source tree is mounted under /system. 62 | 63 | ## Speeding up the test suite development cycle 64 | 65 | Running the test suite takes a while. Most of the time is spent on the setup phase, where the test suite installs prerequisite tools inside the container. In order to speed up your development cycle, we recommend the following methodology, which bypasses the installation phase: 66 | 67 | 1. Launch a shell inside the test container (see section "Debugging"). 68 | 2. Modify `internal-scripts/test-{debs,rpms}` as you see fit. 69 | 3. Test whether the modifications in step 2 works, by running them directly in the shell launched by step 1. 70 | 71 | Caveats to consider: 72 | 73 | - There is no `run` function in the shell, so omit that when running things in the shell. 74 | - `internal-scripts/test-{debs,rpms}` runs as root, whereas the shell is launched as the `utility` user. If you need root privileges, use sudo. 75 | - Environment variables set by the phase 4 script, are not available in the shell. If you need them, set them yourself, based on how the phase 4 script does it. 76 | 77 | 4. Repeat 2 and 3 until satisfied. 78 | 5. Exit the shell. [Run the entire test suite again](testing-packages-locally.md), to check whether the modifications work. 79 | -------------------------------------------------------------------------------- /dev-handbook/offboarding.md: -------------------------------------------------------------------------------- 1 | # Offboarding 2 | 3 | Offboarding is to be done by someone with the "Admin" role. 4 | 5 | - [ ] Remove member from the [Github repo's members list](https://github.com/fullstaq-ruby/server-edition/settings/access). 6 | - [ ] In the [Members](members.md) document, move member to the "Alumni" section. 7 | -------------------------------------------------------------------------------- /dev-handbook/onboarding.md: -------------------------------------------------------------------------------- 1 | # Onboarding 2 | 3 | - [ ] Add member to the [Github repo's members list](https://github.com/fullstaq-ruby/server-edition/settings/access). 4 | - Assign "Maintain" role. 5 | - [ ] Add member to the [Members](members.md) document. 6 | -------------------------------------------------------------------------------- /dev-handbook/package-organization.md: -------------------------------------------------------------------------------- 1 | # Package organization 2 | 3 | This page explains which package types exist, how they relate to each other, and which script generates them. 4 | 5 | ## fullstaq-ruby-[VERSION AND VARIANT] 6 | 7 | ### Purpose 8 | 9 | These packages contain a specific Ruby version. Each package is for a specific distribution, architecture and variant. 10 | 11 | It places files primarily in `/usr/lib/fullstaq-ruby/versions/`. 12 | 13 | For the purpose of allowing Rbenv to find this Ruby version, the package also installs a symlink `/usr/lib/rbenv/versions/`, which points to the aforementioned directory. 14 | 15 | ### Relationship with other packages 16 | 17 | The fullstaq-ruby-XXX package can be installed independently and has no hard dependencies. 18 | 19 | The Debian package recommends fullstaq-ruby-common. 20 | 21 | The RPM package does not specify fullstaq-ruby-common as a recommended dependency, even though the latter is in fact recommended. The main reason for this is because it's not clear whether RPM supports recommended dependencies, and even if it does, whether FPM supports specifying them. So instead, we tell users through documentation to manually install fullstaq-ruby-common prior to installing fullstaq-ruby-XXX. 22 | 23 | ### Example package filenames 24 | 25 | * `fullstaq-ruby-2.7-jemalloc_0-debian-10_amd64.deb` 26 | * `fullstaq-ruby-2.7-jemalloc-rev2-centos8.x86_64.rpm` 27 | 28 | ### Contents 29 | 30 | * `/usr/lib/fullstaq-ruby/versions/` 31 | * `/usr/lib/rbenv/versions/` (symlink to previous entry) 32 | 33 | ### Generated by 34 | 35 | * `container-entrypoints/build-ruby-deb` 36 | * `container-entrypoints/build-ruby-rpm` 37 | 38 | ## fullstaq-ruby-common 39 | 40 | ### Purpose 41 | 42 | This is a dummy package which contains no files, and only serves as a "glue" that specifies what all the other recommended dependencies are. 43 | 44 | Currently the only recommended dependency is fullstaq-rbenv. If we ever introduce more recommended depdendencies, then we don't have to tell users to manually install them: they'll get the new recommended dependencies automatically through fullstaq-ruby-common. 45 | 46 | This package has a hard dependency on fullstaq-rbenv. 47 | 48 | Since this package contains no files, and does not depend on any other OS packages, it's distribution- and architecture-independent. 49 | 50 | All fullstaq-ruby-XXX Debian packages recommend fullstaq-ruby-common. 51 | 52 | ### Example package filenames 53 | 54 | * `fullstaq-ruby-common_1.0-0_all.deb` 55 | * `fullstaq-ruby-common-1.0-0.noarch.rpm` 56 | 57 | ### Generated by 58 | 59 | * `container-entrypoints/build-common-deb` 60 | * `container-entrypoints/build-common-rpm` 61 | 62 | ## fullstaq-rbenv 63 | 64 | ### Purpose 65 | 66 | This packages our [Rbenv fork with support for system-wide Rubies](https://github.com/fullstaq-labs/fullstaq-rbenv). 67 | 68 | Since Rbenv only consists of shell scripts, and does not depend on any other OS packages which aren't already preinstalled anywhere, it's distribution- and architecture-independent. 69 | 70 | ### Example package filenames 71 | 72 | * `fullstaq-rbenv_1.1.2-16-0_all.deb` 73 | * `fullstaq-rbenv-1.1.2-16-0.noarch.rpm` 74 | 75 | ### Generated by 76 | 77 | * `container-entrypoints/build-rbenv-deb` 78 | * `container-entrypoints/build-rbenv-rpm` 79 | -------------------------------------------------------------------------------- /dev-handbook/responsibilities-expectations.md: -------------------------------------------------------------------------------- 1 | # Responsibilities & expectations 2 | 3 | Joining the team means assuming one or more of the following responsibilities: 4 | 5 | * Investigating and fixing urgent issues and down time. 6 | * Ensuring that packages are kept up-to-date with upstream versions. 7 | * Ensuring that new distribution versions are supported. 8 | * Reviewing Server Edition pull requests. 9 | * Ensuring that community interactions follow our [Code of Conduct](../CODE_OF_CONDUCT.md). 10 | * Providing [mentorship](mentorship.md) to contributors and fellow team members. 11 | 12 | 13 | 14 | You do not need to be an expert on all the areas that we deal with. It's fine if your skills on one or more areas is entry-level, as long as you are curious and want to learn. That's why team members mentor each other. 15 | 16 | You don't need to assume all responsibilities at once, nor do you need to be available at all times. You are free to decide how much time you are willing to invest as a team member, and you are free to leave whenever you want. 17 | 18 | Our goal is to build a resilient team by having many members cover for each other, even if they individually only have limited time resources. 19 | 20 | But at the very least, we expect that you are responsive for contact, and that you are at least occasionally available to help. 21 | -------------------------------------------------------------------------------- /dev-handbook/source-organization.md: -------------------------------------------------------------------------------- 1 | # Source organization 2 | 3 | Build and release scripts: 4 | 5 | * `build-*` — most of these scripts are for executing a specific [build step](build-steps.md). 6 | * `test-*` — [tests a Ruby package](testing-packages-locally.md). 7 | * `upload-*` — publishes built packages to repositories. 8 | 9 | [Build workflow management](build-workflow-management.md) scripts and files: 10 | 11 | * `config.yml` — specifies which distributions, variants, Ruby versions and [fullstaq-rbenv](https://github.com/fullstaq-labs/fullstaq-rbenv) version we want to build for. 12 | 13 | [Build environments](build-environments.md) related: 14 | 15 | * `build-environment-image` — builds a specific build environment's Docker image. 16 | * `push-environment-images` — Publishes all build environment Docker images to the Docker registry. 17 | * `environments/` — Docker images for all build environments. 18 | 19 | Other: 20 | 21 | * `fullstaq-ruby.asc` — the GPG public key that can be used to verify the APT and YUM repositories. This file is referred to by the YUM .repo file mentioned in the installation documentation. 22 | 23 | * `container-entrypoints/` — because of [the "minimal dependencies" principle](minimal-dependencies-principle.md), most of the scripts in the root directory delegate work to a Docker container, in which it runs a corresponding script in this subdirectory. 24 | 25 | Scripts in this subdirectory may require further libraries, or call further scripts. Those required libraries and scripts are not located within this subdirectory. 26 | 27 | * `internal-scripts/` — scripts that are only invoked by other scripts in this repo, and are not supposed to be invoked by users. 28 | 29 | * `lib/` — libraries used by other scripts in this repo. 30 | 31 | * `resources/` — miscellaneous resources used by other scripts/code in this repo. 32 | 33 | * `resources/test-env` — files used by the [package testing script](testing-packages-locally.md). 34 | 35 | ## Root directory may only contain "public" scripts 36 | 37 | The root directory may only contain scripts that are actually supposed to be invoked by users. If a script is considered internal, i.e. only invoked by other scripts, then put them in `internal-scripts/`. 38 | -------------------------------------------------------------------------------- /dev-handbook/speeding-up-ci-feedback.md: -------------------------------------------------------------------------------- 1 | # Speeding up CI feedback 2 | 3 | Our Github Actions-based CI/CD system performs [a lot of work](build-workflow-management.md), and takes \~45 minutes per run. During development, you may want to get faster CI feedback. This can be achieved through two tricks: 4 | 5 | 1. Temporarily reducing the number of combinations. 6 | 2. Temporarily reusing a previous CI run's artifacts. 7 | 8 | The emphasis is on *temporary*: these tricks may only be enabled during development, and must be disabled before submitting a pull request or merging to main. 9 | 10 | ## Reducing the number of combinations 11 | 12 | Edit config.yml and get rid of all the Ruby versions, variants and distributions that you don't want the CI to test. For example, when upgrading Ruby 2.7.1 to Ruby 2.7.2, change this... 13 | 14 | ~~~yaml 15 | ruby: 16 | minor_version_packages: 17 | - minor_version: 2.7 18 | full_version: 2.7.1 19 | package_revision: 2 20 | - minor_version: 2.6 21 | full_version: 2.6.6 22 | package_revision: 6 23 | - minor_version: 2.5 24 | full_version: 2.5.8 25 | package_revision: 5 26 | 27 | tiny_version_packages: 28 | - full_version: 2.7.1 29 | package_revision: 1 30 | - full_version: 2.6.6 31 | package_revision: 2 32 | - full_version: 2.5.8 33 | package_revision: 2 34 | ~~~ 35 | 36 | ...to this: 37 | 38 | ~~~yaml 39 | ruby: 40 | minor_version_packages: 41 | - minor_version: 2.7 42 | full_version: 2.7.2 43 | package_revision: 3 44 | 45 | tiny_version_packages: 46 | - full_version: 2.7.2 47 | package_revision: 1 48 | ~~~ 49 | 50 | And: 51 | 52 | ~~~ 53 | # Test only jemalloc variant for now 54 | variants: 55 | normal: false 56 | jemalloc: true 57 | malloctrim: false 58 | 59 | # Test a random DEB and RPM distribution 60 | distributions: 61 | - debian-10 62 | - centos-8 63 | ~~~ 64 | 65 | ## Reusing a previous CI run's artifacts 66 | 67 | You can tell the CI to behave as if it's running as a previous CI run. Because of [CI/CD system resumption support](ci-cd-resumption.md), the CI will skip the jobs for which the associated artifacts have already been built. 68 | 69 | You can find the CI run number in the Github Actions CI run listing, right below the commit message: 70 | 71 | ![](ci-run-number.png) 72 | 73 | Open .gitub/workflows/ci-cd.yml.erb. Go to the `env` section, and set `CI_ARTIFACTS_RUN_NUMBER` to the CI number for which the artifacts you want to reuse. For example: 74 | 75 | ~~~yaml 76 | env: 77 | ... 78 | CI_ARTIFACTS_RUN_NUMBER: 290 79 | ~~~ 80 | 81 | ## Don't commit these changes 82 | 83 | The tricks described in this document are supposed to be temporary. Don't commit them into your pull request, or into the main branch. 84 | -------------------------------------------------------------------------------- /dev-handbook/testing-packages-locally.md: -------------------------------------------------------------------------------- 1 | # Testing packages locally & debugging test failures 2 | 3 | We have an automated test suite for testing whether packages work. This guide shows how to run the test suite locally, how to modify the test suite, and how to debug test failures. You can test either packages that you [built yourself](building-packages-locally.md), or packages that were built by the CI system. 4 | 5 | Because we follow [the "minimal dependencies" principle](minimal-dependencies-principle.md), you can test packages for any distribution, regardless of which OS or distribution you're running on. 6 | 7 | **Table of contents** 8 | 9 | * [What the test suite does](#what-the-test-suite-does) 10 | * [Prerequisites](#prerequisites) 11 | * [Preparing the packages](#preparing-the-packages) 12 | * [Running the test script](#running-the-test-script) 13 | - [Testing Debian packages](#testing-debian-packages) 14 | - [Testing an RPM package](#testing-an-rpm-package) 15 | * [See also](#see-also) 16 | 17 | ## What the test suite does 18 | 19 | A test suite performs roughly the following work: 20 | 21 | * Test whether packages are installable on a clean system. 22 | * Test whether key files (like the `ruby` binary) are in the right places. 23 | * Test whether Ruby actually works. 24 | * Test whether installing gems (including native extensions) works. 25 | 26 | A test is supposed to be run inside a Docker container — that corresponds to the distribution that the packages were built for — that has nothing preinstalled. The test suite will take care of installing necessary tools, such as the compiler toolchain. 27 | 28 | The test suite sets up a user account called `utility`. Most test commands are run as that user. 29 | 30 | Inside the Docker container, the source tree is mounted under /system. 31 | 32 | ## Prerequisites 33 | 34 | Make sure you've [set up the development environment](dev-environment-setup.md) and that you've cloned the Fullstaq Ruby repository: 35 | 36 | ~~~bash 37 | git clone https://github.com/fullstaq-ruby/server-edition.git 38 | cd fullstaq-ruby-server-edition 39 | ~~~ 40 | 41 | ## Preparing the packages 42 | 43 | The first step is to either [build](building-packages-locally.md), or download the package files that you want to test. You need three [package types](package-organization.md): 44 | 45 | * `fullstaq-ruby-XXX` 46 | * `fullstaq-ruby-common` 47 | * `fullstaq-rbenv` 48 | 49 | In case you want to test packages built by the CI system: you can download them from the corresponding Github Actions CI run, under "Artifacts". 50 | 51 | ## Running the test script 52 | 53 | ### Testing Debian packages 54 | 55 | Run the `./test-debs` script to test Debian packages. Example invocation: 56 | 57 | ~~~bash 58 | ./test-debs \ 59 | -i debian:10 \ 60 | -v jemalloc \ 61 | -r fullstaq-ruby-2.7-jemalloc_0-debian-10_amd64.deb \ 62 | -b fullstaq-rbenv_1.1.2-16-0_all.deb \ 63 | -c fullstaq-ruby-common_1.0-0_all.deb 64 | ~~~ 65 | 66 | Here's what the parameters mean: 67 | 68 | * `-i` — a Docker image to run the tests in. This should be a Linux distribution image that corresponds to the distribution that the package was built for. For example `debian:VERSION` or `ubuntu:VERSION`. 69 | * `-v` — The variant that the `fullstaq-ruby-XXX` package was built for. One of `normal`, `jemalloc`, `malloctrim`. 70 | * `-r` — path to the Ruby package. 71 | * `-b` — path to the Rbenv package. 72 | * `-c` — path to the common package. 73 | 74 | ### Testing an RPM package 75 | 76 | Run the `./test-rpms` script to test RPM packages. Example invocation: 77 | 78 | ~~~bash 79 | ./test-rpms \ 80 | -i rockylinux:8 \ 81 | -v jemalloc \ 82 | -r fullstaq-ruby-2.7-jemalloc-rev2-centos8.x86_64.rpm \ 83 | -b fullstaq-rbenv-1.1.2-16-0.noarch.rpm \ 84 | -c fullstaq-ruby-common-1.0-0.noarch.rpm 85 | ~~~ 86 | 87 | Here's what the parameters mean: 88 | 89 | * `-i` — a Docker image to run the tests in. This should be a Linux distribution image that corresponds to the distribution that the package was built for. For example `centos:VERSION`. 90 | * `-v` — The variant that the `fullstaq-ruby-XXX` package was built for. One of `normal`, `jemalloc`, `malloctrim`. 91 | * `-r` — path to the Ruby package. 92 | * `-b` — path to the Rbenv package. 93 | * `-c` — path to the common package. 94 | 95 | ## See also 96 | 97 | [Adding, modifying & debugging tests](modifying-and-debugging-tests.md) 98 | -------------------------------------------------------------------------------- /dev-handbook/troubleshooting-corrupt-ci-cd-artifacts.md: -------------------------------------------------------------------------------- 1 | # Troubleshooting corrupt CI/CD artifacts 2 | 3 | Artifacts produced by the CI could become corrupted due to an external problem, for temporary network data corruption. When that happens, re-running the CI run will not mitigate the problem. That's because our CI implements [resumption support](ci-cd-resumption.md), which means that the CI run won't regenerate artifacts. Instead, any corrupted artifacts that are already stored in Google Cloud Storage, will remain there. 4 | 5 | When this problem occurs, you should file a support ticket with the [infrastructure team](https://github.com/fullstaq-labs/fullstaq-ruby-infra), telling them to clear the artifacts in Google Cloud Storage. 6 | 7 | 1. Go to the [infrastructure issue tracker](https://github.com/fullstaq-labs/fullstaq-ruby-infra/new/choose). 8 | 2. Create a "Clear CI artifacts" ticket and fill in the template. 9 | -------------------------------------------------------------------------------- /dev-handbook/way-of-working.md: -------------------------------------------------------------------------------- 1 | # Way of working 2 | 3 | ## Development process 4 | 5 | * Never push to the `main` branch directly, except for trivial documentation updates. 6 | 7 | - Instead, propose most changes through pull requests. 8 | 9 | * The person who merges a pull request is responsible for testing it after merge, and for providing feedback to the submitter if something is wrong. 10 | 11 | * Write [joyous commit messages](https://medium.com/@joshuatauberer/write-joyous-git-commit-messages-2f98891114c4). 12 | 13 | * Strive to respond to pull requests in 48h, in order to encourage contributions. 14 | 15 | ## Communication 16 | 17 | We communicate publicly as much as possible. All such public communication happens through [our issue tracker](https://github.com/fullstaq-ruby/server-edition/issues) and our [discussion forum](https://github.com/fullstaq-ruby/server-edition/discussions), so be sure to subscribe for notifications. 18 | -------------------------------------------------------------------------------- /environments/centos-8/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM rockylinux:8 2 | 3 | # Used to link container image to the repo: 4 | # https://docs.github.com/en/free-pro-team@latest/packages/managing-container-images-with-github-container-registry/connecting-a-repository-to-a-container-image#connecting-a-repository-to-a-container-image-on-the-command-line 5 | LABEL org.opencontainers.image.source https://github.com/fullstaq-ruby/server-edition 6 | 7 | # If you make a change and you want to force users to re-pull the image 8 | # (e.g. when your change adds a feature that our scripts rely on, or is 9 | # breaking), then bump the version number in the `image_tag` file. 10 | 11 | RUN set -x && \ 12 | dnf install -y dnf-plugins-core epel-release && \ 13 | dnf install -y --enablerepo epel --enablerepo powertools \ 14 | findutils gcc gcc-c++ make patch bzip2 curl autoconf automake \ 15 | openssl-devel libyaml-devel libffi-devel readline-devel zlib-devel \ 16 | gdbm-devel ncurses-devel libtool \ 17 | rust-toolset && \ 18 | dnf clean all && \ 19 | rm -rf /tmp/* /var/tmp/* 20 | 21 | # Ruby checks for pkg-config usability by running `pkg-config --print-errors --version`. 22 | # EL8 ships pkgconf 1.7.3 which doesn't support this combination of flags. 23 | # So we upgrade it. 24 | RUN curl -fsSLo pkgconf.tar.gz https://github.com/pkgconf/pkgconf/archive/refs/tags/pkgconf-1.7.3.tar.gz && \ 25 | tar xzf pkgconf.tar.gz && \ 26 | cd pkgconf-* && \ 27 | ./autogen.sh && \ 28 | ./configure --prefix=/usr && \ 29 | make -j$(nproc) && \ 30 | make install-strip && \ 31 | pkg-config --print-errors --version && \ 32 | cd .. && \ 33 | rm -rf pkgconf-* 34 | 35 | RUN curl -fsSLo sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.3.1/sccache-v0.3.1-x86_64-unknown-linux-musl.tar.gz && \ 36 | tar xzf sccache.tar.gz && \ 37 | mv sccache-*/sccache /usr/local/bin/ && \ 38 | chmod +x /usr/local/bin/sccache && \ 39 | chown root: /usr/local/bin/sccache && \ 40 | rm -rf sccache-* && \ 41 | mkdir /usr/local/lib/sccache && \ 42 | echo -e '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/cc "$@"' > /usr/local/lib/sccache/cc && \ 43 | echo -e '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/c++ "$@"' > /usr/local/lib/sccache/c++ && \ 44 | echo -e '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/gcc "$@"' > /usr/local/lib/sccache/gcc && \ 45 | echo -e '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/g++ "$@"' > /usr/local/lib/sccache/g++ && \ 46 | chmod +x /usr/local/lib/sccache/* && \ 47 | \ 48 | curl -fsSLo /sbin/matchhostfsowner.gz https://github.com/FooBarWidget/matchhostfsowner/releases/download/v0.9.8/matchhostfsowner-0.9.8-x86_64-linux.gz && \ 49 | gunzip /sbin/matchhostfsowner.gz && \ 50 | chmod +x,+s /sbin/matchhostfsowner && \ 51 | mkdir /etc/matchhostfsowner && \ 52 | echo 'app_account: builder' > /etc/matchhostfsowner/config.yml && \ 53 | \ 54 | groupadd --gid 9999 builder && \ 55 | adduser --uid 9999 --gid 9999 --password '#' builder && \ 56 | rm -rf /tmp/* /var/tmp/* 57 | 58 | USER builder 59 | ENTRYPOINT ["/sbin/matchhostfsowner"] 60 | -------------------------------------------------------------------------------- /environments/centos-8/image_tag: -------------------------------------------------------------------------------- 1 | # Bump this version whenever you've made a change and you want 2 | # to force users to re-pull the image (e.g. when your change adds 3 | # a feature that our scripts rely on, or is breaking). 4 | 3 5 | -------------------------------------------------------------------------------- /environments/debian-10/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:10 2 | 3 | # Used to link container image to the repo: 4 | # https://docs.github.com/en/free-pro-team@latest/packages/managing-container-images-with-github-container-registry/connecting-a-repository-to-a-container-image#connecting-a-repository-to-a-container-image-on-the-command-line 5 | LABEL org.opencontainers.image.source https://github.com/fullstaq-ruby/server-edition 6 | 7 | # If you make a change and you want to force users to re-pull the image 8 | # (e.g. when your change adds a feature that our scripts rely on, or is 9 | # breaking), then bump the version number in the `image_tag` file. 10 | 11 | RUN set -x && \ 12 | apt update && \ 13 | apt install -y autoconf bison bzip2 build-essential \ 14 | dpkg-dev curl ca-certificates pkg-config \ 15 | libssl-dev libyaml-dev libreadline-dev zlib1g-dev \ 16 | libncurses5-dev libffi-dev libgdbm6 libgdbm-dev && \ 17 | apt clean && \ 18 | rm -rf /tmp/* /var/tmp/* /var/lib/apt/lists/* 19 | 20 | RUN curl -fsSLo sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.3.1/sccache-v0.3.1-x86_64-unknown-linux-musl.tar.gz && \ 21 | tar xzf sccache.tar.gz && \ 22 | mv sccache-*/sccache /usr/local/bin/ && \ 23 | chmod +x /usr/local/bin/sccache && \ 24 | chown root: /usr/local/bin/sccache && \ 25 | rm -rf sccache-* && \ 26 | mkdir /usr/local/lib/sccache && \ 27 | echo '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/cc "$@"' > /usr/local/lib/sccache/cc && \ 28 | echo '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/c++ "$@"' > /usr/local/lib/sccache/c++ && \ 29 | echo '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/gcc "$@"' > /usr/local/lib/sccache/gcc && \ 30 | echo '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/g++ "$@"' > /usr/local/lib/sccache/g++ && \ 31 | chmod +x /usr/local/lib/sccache/* && \ 32 | \ 33 | curl -fsSLo /sbin/matchhostfsowner.gz https://github.com/FooBarWidget/matchhostfsowner/releases/download/v0.9.8/matchhostfsowner-0.9.8-x86_64-linux.gz && \ 34 | gunzip /sbin/matchhostfsowner.gz && \ 35 | chmod +x,+s /sbin/matchhostfsowner && \ 36 | mkdir /etc/matchhostfsowner && \ 37 | echo 'app_account: builder' > /etc/matchhostfsowner/config.yml && \ 38 | \ 39 | addgroup --gid 9999 builder && \ 40 | adduser --uid 9999 --gid 9999 --disabled-password --gecos Builder builder && \ 41 | usermod -L builder && \ 42 | \ 43 | curl -o /tmp/rustup-init.sh --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs && \ 44 | su builder -c 'bash /tmp/rustup-init.sh -y --no-modify-path --profile minimal --default-toolchain 1.61' && \ 45 | \ 46 | rm -rf /tmp/* /var/tmp/* 47 | 48 | USER builder 49 | ENV PATH /home/builder/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin 50 | ENTRYPOINT ["/sbin/matchhostfsowner"] 51 | -------------------------------------------------------------------------------- /environments/debian-10/image_tag: -------------------------------------------------------------------------------- 1 | # Bump this version whenever you've made a change and you want 2 | # to force users to re-pull the image (e.g. when your change adds 3 | # a feature that our scripts rely on, or is breaking). 4 | 3 5 | -------------------------------------------------------------------------------- /environments/debian-11/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:11 2 | 3 | # Used to link container image to the repo: 4 | # https://docs.github.com/en/free-pro-team@latest/packages/managing-container-images-with-github-container-registry/connecting-a-repository-to-a-container-image#connecting-a-repository-to-a-container-image-on-the-command-line 5 | LABEL org.opencontainers.image.source https://github.com/fullstaq-ruby/server-edition 6 | 7 | # If you make a change and you want to force users to re-pull the image 8 | # (e.g. when your change adds a feature that our scripts rely on, or is 9 | # breaking), then bump the version number in the `image_tag` file. 10 | 11 | RUN set -x && \ 12 | apt update && \ 13 | apt install -y autoconf bison bzip2 build-essential \ 14 | dpkg-dev curl ca-certificates pkg-config \ 15 | libssl-dev libyaml-dev libreadline-dev zlib1g-dev \ 16 | libncurses5-dev libffi-dev libgdbm6 libgdbm-dev && \ 17 | apt clean && \ 18 | rm -rf /tmp/* /var/tmp/* /var/lib/apt/lists/* 19 | 20 | RUN curl -fsSLo sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.3.1/sccache-v0.3.1-x86_64-unknown-linux-musl.tar.gz && \ 21 | tar xzf sccache.tar.gz && \ 22 | mv sccache-*/sccache /usr/local/bin/ && \ 23 | chmod +x /usr/local/bin/sccache && \ 24 | chown root: /usr/local/bin/sccache && \ 25 | rm -rf sccache-* && \ 26 | mkdir /usr/local/lib/sccache && \ 27 | echo '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/cc "$@"' > /usr/local/lib/sccache/cc && \ 28 | echo '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/c++ "$@"' > /usr/local/lib/sccache/c++ && \ 29 | echo '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/gcc "$@"' > /usr/local/lib/sccache/gcc && \ 30 | echo '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/g++ "$@"' > /usr/local/lib/sccache/g++ && \ 31 | chmod +x /usr/local/lib/sccache/* && \ 32 | \ 33 | curl -fsSLo /sbin/matchhostfsowner.gz https://github.com/FooBarWidget/matchhostfsowner/releases/download/v0.9.8/matchhostfsowner-0.9.8-x86_64-linux.gz && \ 34 | gunzip /sbin/matchhostfsowner.gz && \ 35 | chmod +x,+s /sbin/matchhostfsowner && \ 36 | mkdir /etc/matchhostfsowner && \ 37 | echo 'app_account: builder' > /etc/matchhostfsowner/config.yml && \ 38 | \ 39 | addgroup --gid 9999 builder && \ 40 | adduser --uid 9999 --gid 9999 --disabled-password --gecos Builder builder && \ 41 | usermod -L builder && \ 42 | \ 43 | curl -o /tmp/rustup-init.sh --proto '=https' --tlsv1.2 -fsSL https://sh.rustup.rs && \ 44 | su builder -c 'bash /tmp/rustup-init.sh -y --no-modify-path --profile minimal --default-toolchain 1.61' && \ 45 | \ 46 | rm -rf /tmp/* /var/tmp/* 47 | 48 | USER builder 49 | ENV PATH /home/builder/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin 50 | ENTRYPOINT ["/sbin/matchhostfsowner"] 51 | -------------------------------------------------------------------------------- /environments/debian-11/image_tag: -------------------------------------------------------------------------------- 1 | # Bump this version whenever you've made a change and you want 2 | # to force users to re-pull the image (e.g. when your change adds 3 | # a feature that our scripts rely on, or is breaking). 4 | 3 5 | -------------------------------------------------------------------------------- /environments/debian-12/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM debian:12 2 | 3 | # Used to link container image to the repo: 4 | # https://docs.github.com/en/free-pro-team@latest/packages/managing-container-images-with-github-container-registry/connecting-a-repository-to-a-container-image#connecting-a-repository-to-a-container-image-on-the-command-line 5 | LABEL org.opencontainers.image.source https://github.com/fullstaq-ruby/server-edition 6 | 7 | # If you make a change and you want to force users to re-pull the image 8 | # (e.g. when your change adds a feature that our scripts rely on, or is 9 | # breaking), then bump the version number in the `image_tag` file. 10 | 11 | RUN set -x && \ 12 | apt update && \ 13 | apt install -y autoconf bison bzip2 build-essential \ 14 | dpkg-dev curl ca-certificates pkg-config rustc \ 15 | libssl-dev libyaml-dev libreadline-dev zlib1g-dev \ 16 | libncurses5-dev libffi-dev libgdbm6 libgdbm-dev && \ 17 | apt clean && \ 18 | rm -rf /tmp/* /var/tmp/* /var/lib/apt/lists/* 19 | 20 | RUN curl -fsSLo sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.3.1/sccache-v0.3.1-x86_64-unknown-linux-musl.tar.gz && \ 21 | tar xzf sccache.tar.gz && \ 22 | mv sccache-*/sccache /usr/local/bin/ && \ 23 | chmod +x /usr/local/bin/sccache && \ 24 | chown root: /usr/local/bin/sccache && \ 25 | rm -rf sccache-* && \ 26 | mkdir /usr/local/lib/sccache && \ 27 | echo '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/cc "$@"' > /usr/local/lib/sccache/cc && \ 28 | echo '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/c++ "$@"' > /usr/local/lib/sccache/c++ && \ 29 | echo '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/gcc "$@"' > /usr/local/lib/sccache/gcc && \ 30 | echo '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/g++ "$@"' > /usr/local/lib/sccache/g++ && \ 31 | chmod +x /usr/local/lib/sccache/* && \ 32 | \ 33 | curl -fsSLo /sbin/matchhostfsowner.gz https://github.com/FooBarWidget/matchhostfsowner/releases/download/v0.9.8/matchhostfsowner-0.9.8-x86_64-linux.gz && \ 34 | gunzip /sbin/matchhostfsowner.gz && \ 35 | chmod +x,+s /sbin/matchhostfsowner && \ 36 | mkdir /etc/matchhostfsowner && \ 37 | echo 'app_account: builder' > /etc/matchhostfsowner/config.yml && \ 38 | \ 39 | addgroup --gid 9999 builder && \ 40 | adduser --uid 9999 --gid 9999 --disabled-password --gecos Builder builder && \ 41 | usermod -L builder && \ 42 | \ 43 | rm -rf /tmp/* /var/tmp/* 44 | 45 | USER builder 46 | ENV PATH /home/builder/.cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin 47 | ENTRYPOINT ["/sbin/matchhostfsowner"] 48 | -------------------------------------------------------------------------------- /environments/debian-12/image_tag: -------------------------------------------------------------------------------- 1 | # Bump this version whenever you've made a change and you want 2 | # to force users to re-pull the image (e.g. when your change adds 3 | # a feature that our scripts rely on, or is breaking). 4 | 2 5 | -------------------------------------------------------------------------------- /environments/el-9/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM rockylinux:9 2 | 3 | # Used to link container image to the repo: 4 | # https://docs.github.com/en/free-pro-team@latest/packages/managing-container-images-with-github-container-registry/connecting-a-repository-to-a-container-image#connecting-a-repository-to-a-container-image-on-the-command-line 5 | LABEL org.opencontainers.image.source https://github.com/fullstaq-ruby/server-edition 6 | 7 | # If you make a change and you want to force users to re-pull the image 8 | # (e.g. when your change adds a feature that our scripts rely on, or is 9 | # breaking), then bump the version number in the `image_tag` file. 10 | 11 | RUN set -x && \ 12 | dnf install -y dnf-plugins-core epel-release && \ 13 | dnf install -y --enablerepo epel --enablerepo devel --allowerasing \ 14 | findutils gcc gcc-c++ make patch bzip2 curl autoconf automake \ 15 | openssl-devel libyaml-devel libffi-devel readline-devel zlib-devel \ 16 | gdbm-devel ncurses-devel \ 17 | rust-toolset && \ 18 | dnf clean all && \ 19 | rm -rf /tmp/* /var/tmp/* 20 | 21 | RUN curl -fsSLo sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.3.1/sccache-v0.3.1-x86_64-unknown-linux-musl.tar.gz && \ 22 | tar xzf sccache.tar.gz && \ 23 | mv sccache-*/sccache /usr/local/bin/ && \ 24 | chmod +x /usr/local/bin/sccache && \ 25 | chown root: /usr/local/bin/sccache && \ 26 | rm -rf sccache-* && \ 27 | mkdir /usr/local/lib/sccache && \ 28 | echo -e '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/cc "$@"' > /usr/local/lib/sccache/cc && \ 29 | echo -e '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/c++ "$@"' > /usr/local/lib/sccache/c++ && \ 30 | echo -e '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/gcc "$@"' > /usr/local/lib/sccache/gcc && \ 31 | echo -e '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/g++ "$@"' > /usr/local/lib/sccache/g++ && \ 32 | chmod +x /usr/local/lib/sccache/* && \ 33 | \ 34 | curl -fsSLo /sbin/matchhostfsowner.gz https://github.com/FooBarWidget/matchhostfsowner/releases/download/v0.9.8/matchhostfsowner-0.9.8-x86_64-linux.gz && \ 35 | gunzip /sbin/matchhostfsowner.gz && \ 36 | chmod +x,+s /sbin/matchhostfsowner && \ 37 | mkdir /etc/matchhostfsowner && \ 38 | echo 'app_account: builder' > /etc/matchhostfsowner/config.yml && \ 39 | \ 40 | groupadd --gid 9999 builder && \ 41 | adduser --uid 9999 --gid 9999 --password '#' builder && \ 42 | rm -rf /tmp/* /var/tmp/* 43 | 44 | USER builder 45 | ENTRYPOINT ["/sbin/matchhostfsowner"] 46 | -------------------------------------------------------------------------------- /environments/el-9/image_tag: -------------------------------------------------------------------------------- 1 | # Bump this version whenever you've made a change and you want 2 | # to force users to re-pull the image (e.g. when your change adds 3 | # a feature that our scripts rely on, or is breaking). 4 | 2 5 | -------------------------------------------------------------------------------- /environments/ubuntu-20.04/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:20.04 2 | 3 | # Used to link container image to the repo: 4 | # https://docs.github.com/en/free-pro-team@latest/packages/managing-container-images-with-github-container-registry/connecting-a-repository-to-a-container-image#connecting-a-repository-to-a-container-image-on-the-command-line 5 | LABEL org.opencontainers.image.source https://github.com/fullstaq-ruby/server-edition 6 | 7 | # If you make a change and you want to force users to re-pull the image 8 | # (e.g. when your change adds a feature that our scripts rely on, or is 9 | # breaking), then bump the version number in the `image_tag` file. 10 | 11 | RUN set -x && \ 12 | export DEBIAN_FRONTEND=noninteractive && \ 13 | apt update && \ 14 | apt install -y autoconf bison bzip2 build-essential \ 15 | dpkg-dev curl ca-certificates pkg-config rustc \ 16 | libssl-dev libyaml-dev libreadline-dev zlib1g-dev \ 17 | libncurses5-dev libffi-dev libgdbm6 libgdbm-dev && \ 18 | apt clean && \ 19 | rm -rf /tmp/* /var/tmp/* /var/lib/apt/lists/* 20 | 21 | RUN curl -fsSLo sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.3.1/sccache-v0.3.1-x86_64-unknown-linux-musl.tar.gz && \ 22 | tar xzf sccache.tar.gz && \ 23 | mv sccache-*/sccache /usr/local/bin/ && \ 24 | chmod +x /usr/local/bin/sccache && \ 25 | chown root: /usr/local/bin/sccache && \ 26 | rm -rf sccache-* && \ 27 | mkdir /usr/local/lib/sccache && \ 28 | echo '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/cc "$@"' > /usr/local/lib/sccache/cc && \ 29 | echo '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/c++ "$@"' > /usr/local/lib/sccache/c++ && \ 30 | echo '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/gcc "$@"' > /usr/local/lib/sccache/gcc && \ 31 | echo '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/g++ "$@"' > /usr/local/lib/sccache/g++ && \ 32 | chmod +x /usr/local/lib/sccache/* && \ 33 | \ 34 | curl -fsSLo /sbin/matchhostfsowner.gz https://github.com/FooBarWidget/matchhostfsowner/releases/download/v0.9.8/matchhostfsowner-0.9.8-x86_64-linux.gz && \ 35 | gunzip /sbin/matchhostfsowner.gz && \ 36 | chmod +x,+s /sbin/matchhostfsowner && \ 37 | mkdir /etc/matchhostfsowner && \ 38 | echo 'app_account: builder' > /etc/matchhostfsowner/config.yml && \ 39 | \ 40 | addgroup --gid 9999 builder && \ 41 | adduser --uid 9999 --gid 9999 --disabled-password --gecos Builder builder && \ 42 | usermod -L builder 43 | 44 | USER builder 45 | ENTRYPOINT ["/sbin/matchhostfsowner"] 46 | -------------------------------------------------------------------------------- /environments/ubuntu-20.04/image_tag: -------------------------------------------------------------------------------- 1 | # Bump this version whenever you've made a change and you want 2 | # to force users to re-pull the image (e.g. when your change adds 3 | # a feature that our scripts rely on, or is breaking). 4 | 2 5 | -------------------------------------------------------------------------------- /environments/ubuntu-22.04/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:22.04 2 | 3 | # Used to link container image to the repo: 4 | # https://docs.github.com/en/free-pro-team@latest/packages/managing-container-images-with-github-container-registry/connecting-a-repository-to-a-container-image#connecting-a-repository-to-a-container-image-on-the-command-line 5 | LABEL org.opencontainers.image.source https://github.com/fullstaq-ruby/server-edition 6 | 7 | # If you make a change and you want to force users to re-pull the image 8 | # (e.g. when your change adds a feature that our scripts rely on, or is 9 | # breaking), then bump the version number in the `image_tag` file. 10 | 11 | RUN set -x && \ 12 | apt update && \ 13 | apt install -y autoconf bison bzip2 build-essential \ 14 | dpkg-dev curl ca-certificates pkg-config rustc \ 15 | libssl-dev libyaml-dev libreadline-dev zlib1g-dev \ 16 | libncurses5-dev libffi-dev libgdbm6 libgdbm-dev && \ 17 | apt clean && \ 18 | rm -rf /tmp/* /var/tmp/* /var/lib/apt/lists/* 19 | 20 | RUN curl -fsSLo sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.3.1/sccache-v0.3.1-x86_64-unknown-linux-musl.tar.gz && \ 21 | tar xzf sccache.tar.gz && \ 22 | mv sccache-*/sccache /usr/local/bin/ && \ 23 | chmod +x /usr/local/bin/sccache && \ 24 | chown root: /usr/local/bin/sccache && \ 25 | rm -rf sccache-* && \ 26 | mkdir /usr/local/lib/sccache && \ 27 | echo '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/cc "$@"' > /usr/local/lib/sccache/cc && \ 28 | echo '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/c++ "$@"' > /usr/local/lib/sccache/c++ && \ 29 | echo '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/gcc "$@"' > /usr/local/lib/sccache/gcc && \ 30 | echo '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/g++ "$@"' > /usr/local/lib/sccache/g++ && \ 31 | chmod +x /usr/local/lib/sccache/* && \ 32 | \ 33 | curl -fsSLo /sbin/matchhostfsowner.gz https://github.com/FooBarWidget/matchhostfsowner/releases/download/v0.9.8/matchhostfsowner-0.9.8-x86_64-linux.gz && \ 34 | gunzip /sbin/matchhostfsowner.gz && \ 35 | chmod +x,+s /sbin/matchhostfsowner && \ 36 | mkdir /etc/matchhostfsowner && \ 37 | echo 'app_account: builder' > /etc/matchhostfsowner/config.yml && \ 38 | \ 39 | addgroup --gid 9999 builder && \ 40 | adduser --uid 9999 --gid 9999 --disabled-password --gecos Builder builder && \ 41 | usermod -L builder 42 | 43 | USER builder 44 | ENTRYPOINT ["/sbin/matchhostfsowner"] 45 | -------------------------------------------------------------------------------- /environments/ubuntu-22.04/image_tag: -------------------------------------------------------------------------------- 1 | # Bump this version whenever you've made a change and you want 2 | # to force users to re-pull the image (e.g. when your change adds 3 | # a feature that our scripts rely on, or is breaking). 4 | 3 5 | -------------------------------------------------------------------------------- /environments/ubuntu-24.04/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:24.04 2 | 3 | # Used to link container image to the repo: 4 | # https://docs.github.com/en/free-pro-team@latest/packages/managing-container-images-with-github-container-registry/connecting-a-repository-to-a-container-image#connecting-a-repository-to-a-container-image-on-the-command-line 5 | LABEL org.opencontainers.image.source https://github.com/fullstaq-ruby/server-edition 6 | 7 | # If you make a change and you want to force users to re-pull the image 8 | # (e.g. when your change adds a feature that our scripts rely on, or is 9 | # breaking), then bump the version number in the `image_tag` file. 10 | 11 | RUN set -x && \ 12 | apt update && \ 13 | apt install -y autoconf bison bzip2 build-essential \ 14 | dpkg-dev curl ca-certificates pkg-config rustc \ 15 | libssl-dev libyaml-dev libreadline-dev zlib1g-dev \ 16 | libncurses5-dev libffi-dev libgdbm6 libgdbm-dev && \ 17 | apt clean && \ 18 | rm -rf /tmp/* /var/tmp/* /var/lib/apt/lists/* 19 | 20 | RUN curl -fsSLo sccache.tar.gz https://github.com/mozilla/sccache/releases/download/v0.3.1/sccache-v0.3.1-x86_64-unknown-linux-musl.tar.gz && \ 21 | tar xzf sccache.tar.gz && \ 22 | mv sccache-*/sccache /usr/local/bin/ && \ 23 | chmod +x /usr/local/bin/sccache && \ 24 | chown root: /usr/local/bin/sccache && \ 25 | rm -rf sccache-* && \ 26 | mkdir /usr/local/lib/sccache && \ 27 | echo '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/cc "$@"' > /usr/local/lib/sccache/cc && \ 28 | echo '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/c++ "$@"' > /usr/local/lib/sccache/c++ && \ 29 | echo '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/gcc "$@"' > /usr/local/lib/sccache/gcc && \ 30 | echo '#!/bin/sh\nexec /usr/local/bin/sccache /usr/bin/g++ "$@"' > /usr/local/lib/sccache/g++ && \ 31 | chmod +x /usr/local/lib/sccache/* && \ 32 | \ 33 | curl -fsSLo /sbin/matchhostfsowner.gz https://github.com/FooBarWidget/matchhostfsowner/releases/download/v0.9.8/matchhostfsowner-0.9.8-x86_64-linux.gz && \ 34 | gunzip /sbin/matchhostfsowner.gz && \ 35 | chmod +x,+s /sbin/matchhostfsowner && \ 36 | mkdir /etc/matchhostfsowner && \ 37 | echo 'app_account: builder' > /etc/matchhostfsowner/config.yml && \ 38 | \ 39 | addgroup --gid 9999 builder && \ 40 | adduser --uid 9999 --gid 9999 --disabled-password --gecos Builder builder && \ 41 | usermod -L builder 42 | 43 | USER builder 44 | ENTRYPOINT ["/sbin/matchhostfsowner"] 45 | -------------------------------------------------------------------------------- /environments/ubuntu-24.04/image_tag: -------------------------------------------------------------------------------- 1 | # Bump this version whenever you've made a change and you want 2 | # to force users to re-pull the image (e.g. when your change adds 3 | # a feature that our scripts rely on, or is breaking). 4 | 1 5 | -------------------------------------------------------------------------------- /environments/utility/Dockerfile: -------------------------------------------------------------------------------- 1 | FROM ubuntu:22.04 2 | 3 | # Used to link container image to the repo: 4 | # https://docs.github.com/en/free-pro-team@latest/packages/managing-container-images-with-github-container-registry/connecting-a-repository-to-a-container-image#connecting-a-repository-to-a-container-image-on-the-command-line 5 | LABEL org.opencontainers.image.source https://github.com/fullstaq-ruby/server-edition 6 | 7 | # If you make a change and you want to force users to re-pull the image 8 | # (e.g. when your change adds a feature that our scripts rely on, or is 9 | # breaking), then bump the version number in the `image_tag` file. 10 | 11 | COPY Gemfile Gemfile.lock /utility_build/ 12 | 13 | RUN set -x && \ 14 | apt update && \ 15 | apt install -y wget ca-certificates binutils build-essential \ 16 | curl ca-certificates rpm file ruby ruby-dev rubygems sudo \ 17 | aptly createrepo-c parallel && \ 18 | apt clean && \ 19 | rm -rf /tmp/* /var/tmp/* /var/lib/apt/lists/* 20 | 21 | RUN set -x && \ 22 | curl -fsSLo /sbin/matchhostfsowner.gz https://github.com/FooBarWidget/matchhostfsowner/releases/download/v0.9.8/matchhostfsowner-0.9.8-x86_64-linux.gz && \ 23 | gunzip /sbin/matchhostfsowner.gz && \ 24 | chmod +x,+s /sbin/matchhostfsowner && \ 25 | mkdir /etc/matchhostfsowner && \ 26 | echo 'app_account: utility' > /etc/matchhostfsowner/config.yml && \ 27 | addgroup --gid 9999 utility && \ 28 | adduser --uid 9999 --gid 9999 --disabled-password --gecos Utility utility && \ 29 | usermod -L utility 30 | 31 | # Bundler version must match the one in Gemfile.lock. 32 | RUN set -x && \ 33 | gem install bundler --no-document -v "$(grep -A 1 'BUNDLED WITH' /utility_build/Gemfile.lock | tail -n 1)" && \ 34 | mkdir /bundle && \ 35 | chown utility: /bundle && \ 36 | cp /utility_build/* /home/utility/ && \ 37 | cd /home/utility && \ 38 | sudo -H -u utility bundle config set --local path /bundle && \ 39 | sudo -H -u utility bundle install -j4 && \ 40 | rm -rf /utility_build /tmp/* /var/tmp/* 41 | 42 | USER utility 43 | ENTRYPOINT ["/sbin/matchhostfsowner"] 44 | -------------------------------------------------------------------------------- /environments/utility/Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | # Must match Ruby version installed in the utility image. 4 | ruby '>= 3.0' 5 | 6 | gem 'fpm' 7 | -------------------------------------------------------------------------------- /environments/utility/Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | arr-pm (0.0.12) 5 | backports (3.24.1) 6 | cabin (0.9.0) 7 | clamp (1.0.1) 8 | dotenv (2.8.1) 9 | fpm (1.15.1) 10 | arr-pm (~> 0.0.11) 11 | backports (>= 2.6.2) 12 | cabin (>= 0.6.0) 13 | clamp (~> 1.0.0) 14 | pleaserun (~> 0.0.29) 15 | rexml 16 | stud 17 | insist (1.0.0) 18 | mustache (0.99.8) 19 | pleaserun (0.0.32) 20 | cabin (> 0) 21 | clamp 22 | dotenv 23 | insist 24 | mustache (= 0.99.8) 25 | stud 26 | rexml (3.2.5) 27 | stud (0.0.23) 28 | 29 | PLATFORMS 30 | arm-darwin-21 31 | ruby 32 | x86_64-darwin-21 33 | x86_64-linux 34 | 35 | DEPENDENCIES 36 | fpm 37 | 38 | RUBY VERSION 39 | ruby 3.2.1p31 40 | 41 | BUNDLED WITH 42 | 2.4.15 43 | -------------------------------------------------------------------------------- /environments/utility/image_tag: -------------------------------------------------------------------------------- 1 | # Bump this version whenever you've made a change and you want 2 | # to force users to re-pull the image (e.g. when your change adds 3 | # a feature that our scripts rely on, or is breaking). 4 | 2 5 | -------------------------------------------------------------------------------- /fullstaq-ruby.asc: -------------------------------------------------------------------------------- 1 | -----BEGIN PGP PUBLIC KEY BLOCK----- 2 | Comment: GPGTools - http://gpgtools.org 3 | 4 | mQINBF0fG6IBEADHxVNQVq1fUBkyGXaNDXbmeXUJS/H2Pt8YCDrLmyydkbj3Q9zQ 5 | yeRA9rRK0xhmGOcjUzffbai8coHbIgbLrKJwIDa1GJp9ZJ0Yn11/bx9HyFcxMv6q 6 | 1WLqqXimRMzOCVd3aHJ+O92E8MvB/v+jM1PRaZq1GsOMYq6YL07yn/VuLdWkXxhD 7 | wlhpuDfg7rU13WBQXKFjJvLbAyXDPznH+2MSdiLbPA/GzR/m0q4eFDg46UXR/bM4 8 | IHtilENsANo9/OFr1rbXNgBIVKgX1WQt/FIkTGgGHsp/CxH+2ntYIOA8Zx9x4cQg 9 | 14nbExod9ethw20Qfrsc8o3wQppbRFe+8zjmsw5/c6EkvtandUVvGZYvR3sG9idQ 10 | qDN80J6G4Koiy5VRF7P6EQyNMEjTqRIxOMVcQTNvFkdQFBHJmaWIVUzisuLhOCGC 11 | pnWZB4M+EiTUmdJuL5sOiBq7nl+xPJwFX6Y7gUa/hwwZQDpzGRBVfbhE/BSJ/p42 12 | OQPrfBAE8CVCa+QgPi/RhZj82JekFjzDMRr6UVeSGWmi3+qoljrP25FUvOo4kSYh 13 | gFYljL4sRa/NNv/RZ/KiKjNJSDvvQF+MzA0UGN+/RUGVL+ngWydWgkLy472tEw9o 14 | aaaC2mJF/NlhZlVevuJhLyiy2kc1/r1LhGwa/8FSMedYfuXCgDOivZE9RQARAQAB 15 | tCFGdWxsc3RhcSBSdWJ5IDxpbmZvQGZ1bGxzdGFxLmNvbT6JAk4EEwEKADgWIQQ5 16 | T4g+DENWlFD9+5Kprxx8LtZcwAUCXR8bogIbAwULCQgHAwUVCgkICwUWAgMBAAIe 17 | AQIXgAAKCRCprxx8LtZcwODgD/4qO4QLXOEUDyG89FWsChcelsj8BK8ecneOvH9h 18 | iIVN1RN2uN/UNT/y58+CH+5osZ+0ypNHYwhwHCgM62/kfZzVIDtdt5vqoSd0GFhk 19 | X/2o+dcM6ozMYGXzrMdRcbHPO4s+ZP4HrpVXXOaVYcpUEWntoBjPWWv3yFtGTswR 20 | HmuVByZOGgH3G2OG2MCAmmj6KGVKMNl7a9py0WJLFFoqzLYshFFT3Vba46YzXcL1 21 | r0wfuYjZ/3y93/+FZSaBEfeX6Ztypy4s9XX3tLm3M0kLQs/HdwrdugOU4A28N1E3 22 | 6H73rRbpcxR1M874E9xYKo3uF9Cx+p8FVjbfU97x5Uiae0QaXbrqeFeMuJKiKikp 23 | gOJOIY/E4HOyzkUP4rGiX35T8dAgyZW1FviQGN6G5tawxQYE6p5Pfgz850A81oOY 24 | CTLk6FeW0BBRZKqss5BTWZGUf3ytTEKTDd2LdUezVQUboOSx+lXTQNWZgvH69apl 25 | ovWIxaoe1FKpA4/hfhSTvkzxQy77dLMUu18a7LanCE5GscVZ+SGdexysqIojmY3W 26 | SVzzSGlE6Gde09I6BE36pH8zK60cQ5mGLNuDj15PWmEIGMGK0VtSvLPtGGnSb68L 27 | KuQld66CHXfWbFdDmQSDT180S2HB2Pxh5zZtvwapak3tmAOAVuZNONrJnfL4SJfn 28 | /b5uxbkCDQRdHxuiARAAwkxyYecgaA2CwxpX+N9sKD5Xw8XLwizIu9Diamm2VLvK 29 | j+Ru7EBIm1yOBQ2Y2LOYLsxyVKIUH7IaICMPBF1UZnaMdtpFAYTuKAu1M+Ps8lgx 30 | AeUvhG1K1dPScikllKtdSSS7FQPmoq5jSUDxDAzI88yfAsFJZ0jV9x/xgVKiL0Z8 31 | 0eqO6DVjPqLbmIGmABzHeKrBZ1tmSv0FkRnPzjqysIdFb9wyPjEoNTmN5x3CUopX 32 | j3uuWIPQedwJNembW293TgQpOQmKIiym/l8wh7e72OQ3agN42nSARhJCgvsgKw0T 33 | 1OJemaLc0NTT+hfw55ZVi/QVc1XDRSiOtTZx8LFMukIXKbI1dAkuo0fjvCTf/7Vt 34 | d6010KGd2bfeTFM7UiEpPwq/F6Ta3JHChku3qrXmM9MYMEorzOfGT/3LGBTSyuV1 35 | wUmTMchWnopsfBAiv4NenAOsIAziWw3e3Z4Sg5AbTDqFsMsNxk2JDGpdXVZbFzut 36 | qyO8TPSou4oPXI6SbKDtQUoCjIPS/jxmzJRTxfLre5BYunlqHmQ0ncDqLuEHtxAr 37 | aop1EZdR29DJ8Odlc9IvRFR+KvCOr0x9ZEbbi4IRnQwu/4+vYvYNMQA5gFBiVn7t 38 | LDAC2VpXm8zVbxOcVXQ4Gh2KNAOl+SDrQQpF3nsolA73kMC4fMmo6VqSPTw+//cA 39 | EQEAAYkCNgQYAQoAIBYhBDlPiD4MQ1aUUP37kqmvHHwu1lzABQJdHxuiAhsMAAoJ 40 | EKmvHHwu1lzAzGwQAJIPytoRtbma25kIkB/f4B+VQdTXQS6ogifScvwL/kaUUSXa 41 | GieK83hHtZD7nKO/97CBPVXOmy6XtrNxNPVPw8L3Ce3Qtx6+2OGKliXR8ssr+wAr 42 | Jjx+Zz+fBMoenTBMBxpFrkNXb8m9YYkHoMAhMVLunb3u35mItq3aADfYsLa+9fdw 43 | ds40ncqN9I3HPrkCu4y/gaoLuDCaR6nfjQkbGe3g1b3TCQh2tEB00wxtWG75qjB/ 44 | yCJc/sKf5bobLrgMBTqXxy63lqYQUnjk2WDzjMwsL4PmqTGXN/Vb9I3QL24iDb2K 45 | aLwYpHMU+9MCAqnCeq11mH9Xi4ftJ+CtoPuft9HRG9gCOwZHoNrunqzxnziS/VkA 46 | iOUDsI3OaBnCUr4rY631MNBI7W1JhxtfJ5rynG6rVRG4mlpXOHKob0MA58YmuQEB 47 | rQCvfBJAs+Lh7Yg4FeYALNbx4WG3Wgg91GpHiv/pBEpJlzr9CsoDycwsd2cB4pCS 48 | vB4U6UpNRMMXIR8+dVulZkRiLd4sKSNBT56ENvzKYlI/XX0yroSluRePwZh+sK8b 49 | hkIbtTMOoR/YLyTvAjSEhGncclH3bq66QKQfl3z8X7dXU5Df1w3wxdgS32KAKH/3 50 | qRfu/r1MfYpa/A5F/5l6RU2xeZx9FDFpq2BcxcLZ9+ZEwGrhGK8eCFXR6l23 51 | =xJpH 52 | -----END PGP PUBLIC KEY BLOCK----- 53 | -------------------------------------------------------------------------------- /internal-scripts/autodetect-shlib-dependencies: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # encoding: utf-8 3 | # 4 | # Usage: ./scripts/autodetect-shlib-dependencies 5 | # 6 | # This script autodetects shared library dependencies of all 7 | # binaries and libraries in a directively (recursively), and 8 | # outputs a list of package dependency names, like so: 9 | # 10 | # libc6 (>= 2.25), zlib1g (>= 1:1.1.4) 11 | # 12 | # This script aids the building of Debian packages via FPM, 13 | # so that we don't have to manually specify each dependency 14 | # because that is error-prone. 15 | # 16 | # The official Debian packaging tools already have this feature 17 | # via dh_shlibdeps, but FPM can't automatically make use of that, 18 | # so we do it ourselves. Under the hood, dh_shlibdeps makes use 19 | # of dpkg-shlibdeps, which is invoked like so: 20 | # 21 | # dpkg-shlibdeps -O 22 | # 23 | # However, dpkg-shlibs requires that the current working directory 24 | # contains these files: 25 | # 26 | # debian/control 27 | # debian//DEBIAN/shlibs 28 | # 29 | # `control` can be any valid Debian package control file. The contents 30 | # don't matter. 31 | # 32 | # `shlibs` must contain a list of entries corresponding to input libraries 33 | # that contain a SONAME attribute. Each entry must specify, in a space-delimited 34 | # format: 35 | # 36 | # - The SONAME without the .so extension and version number. 37 | # - The version number itself. 38 | # - A package name (which can be anything). 39 | # 40 | # Example: suppose you have libruby.so.2.6.1 and libjemalloc.so.2. Then the 41 | # `shlibs` file must contain: 42 | # 43 | # libruby 2.6.1 foo 44 | # libjemalloc 2 foo 45 | 46 | require 'fileutils' 47 | require 'tempfile' 48 | 49 | DUMMY_PACKAGE_NAME = 'foo' 50 | 51 | def capture!(*command) 52 | data = IO.popen(command + [in: :in], 'r:utf-8') do |io| 53 | io.read 54 | end 55 | if $?.exitstatus != 0 56 | abort "ERROR: command failed: #{command.join(' ')}" 57 | else 58 | data 59 | end 60 | end 61 | 62 | def list_executable_and_library_files 63 | executables = capture!('find', '.', '-executable', '-type', 'f').split("\n").sort 64 | libs = capture!('find', '.', '-name', '*.so').split("\n").sort 65 | [executables, libs] 66 | end 67 | 68 | def create_dummy_debian_dir 69 | begin 70 | FileUtils.mkdir_p('debian/root/DEBIAN') 71 | File.open('debian/control', 'w:utf-8') do |f| 72 | f.write( 73 | "Source: $DUMMY_PACKAGE_NAME\n" \ 74 | "Section: devel\n" \ 75 | "Priority: optional\n" \ 76 | "Maintainer: John Doe \n" \ 77 | "\n" \ 78 | "Package: #{DUMMY_PACKAGE_NAME}\n" \ 79 | "Architecture: any\n" \ 80 | "Description: #{DUMMY_PACKAGE_NAME}\n" 81 | ) 82 | end 83 | yield 84 | ensure 85 | FileUtils.remove_entry_secure('debian') 86 | end 87 | end 88 | 89 | def create_debian_shlibs_file(libs) 90 | File.open('debian/root/DEBIAN/shlibs', 'w:utf-8') do |f| 91 | libs.each do |lib| 92 | soname_lines = capture!('objdump', '-p', lib).split("\n").grep(/SONAME/) 93 | next if soname_lines.empty? 94 | 95 | soname = soname_lines[0].strip.split(/ +/)[1] 96 | next if soname.empty? 97 | 98 | sobarename = soname.sub(/\.so\..*/, '') 99 | soversion = soname.sub(/.*\.so\./, '') 100 | 101 | if !soversion.empty? 102 | f.puts "#{sobarename} #{soversion} #{DUMMY_PACKAGE_NAME}" 103 | end 104 | end 105 | end 106 | end 107 | 108 | def run_dpkg_shlibdeps(executables, libs) 109 | # dpkg-shlibdeps outputs something like: 110 | # shlibs:Depends=foo, libc6 (>= 2.25), zlib1g (>= 1:1.1.4) 111 | # 112 | # There may be multiple lines like these. 113 | 114 | all_files = [executables, libs].sort.uniq 115 | command = ['dpkg-shlibdeps', '-O', all_files].flatten 116 | 117 | Tempfile.open('dpkg-shlibdeps-stderr', encoding: 'utf-8') do |tmpfile| 118 | result = [] 119 | 120 | IO.popen(command + [in: :in, err: tmpfile], 'r:utf-8') do |io| 121 | io.each_line do |line| 122 | result.concat(line.strip.sub(/.*?=/, '').split(/ *, */)) 123 | end 124 | end 125 | 126 | result.delete(DUMMY_PACKAGE_NAME) 127 | 128 | # Print dpkg-shlibdeps stderr except for some expected and 129 | # uninteresting messages. 130 | tmpfile.rewind 131 | stderr_lines = tmpfile.read.split("\n") 132 | stderr_lines.reject! do |line| 133 | line =~ /binaries to analyze should already be installed in their package's directory/ 134 | end 135 | STDERR.puts stderr_lines 136 | puts result.join(', ') 137 | 138 | if $?.exitstatus != 0 139 | abort "ERROR: command failed: dpkg-shlibdeps -O " 140 | end 141 | end 142 | end 143 | 144 | def main 145 | Dir.chdir(ARGV[0]) if ARGV[0] 146 | executables, libs = list_executable_and_library_files 147 | create_dummy_debian_dir do 148 | create_debian_shlibs_file(libs) 149 | run_dpkg_shlibdeps(executables, libs) 150 | end 151 | end 152 | 153 | main 154 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/build-common-deb/build-package.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | ROOTDIR=$(cd "$SELFDIR/../../.." && pwd) 6 | # shellcheck source=../../../lib/library.sh 7 | source "$ROOTDIR/lib/library.sh" 8 | 9 | require_envvar PACKAGE_BASENAME 10 | require_envvar VERSION 11 | require_envvar REVISION 12 | 13 | 14 | set -x 15 | mkdir output 16 | exec "$ROOTDIR/build-common-deb" \ 17 | -o "output/$PACKAGE_BASENAME" \ 18 | -v "$VERSION" \ 19 | -r "$REVISION" 20 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/build-common-rpm/build-package.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | ROOTDIR=$(cd "$SELFDIR/../../.." && pwd) 6 | # shellcheck source=../../../lib/library.sh 7 | source "$ROOTDIR/lib/library.sh" 8 | 9 | require_envvar PACKAGE_BASENAME 10 | require_envvar VERSION 11 | require_envvar REVISION 12 | 13 | 14 | set -x 15 | mkdir output 16 | exec "$ROOTDIR/build-common-rpm" \ 17 | -o "output/$PACKAGE_BASENAME" \ 18 | -v "$VERSION" \ 19 | -r "$REVISION" 20 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/build-docker-images/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | ROOTDIR=$(cd "$SELFDIR/../../.." && pwd) 6 | # shellcheck source=../../../lib/library.sh 7 | source "$ROOTDIR/lib/library.sh" 8 | 9 | 10 | require_envvar IMAGE_NAME 11 | require_envvar IMAGE_TAG 12 | require_envvar SOURCE_DIR 13 | 14 | set -x 15 | exec docker build --pull -t "$IMAGE_NAME:$IMAGE_TAG" "$SOURCE_DIR" 16 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/build-docker-images/dump-image.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | set -o pipefail 4 | 5 | SELFDIR=$(dirname "$0") 6 | ROOTDIR=$(cd "$SELFDIR/../../.." && pwd) 7 | # shellcheck source=../../../lib/library.sh 8 | source "$ROOTDIR/lib/library.sh" 9 | 10 | 11 | require_envvar IMAGE_NAME 12 | require_envvar IMAGE_TAG 13 | 14 | set -x 15 | mkdir output 16 | docker save "$IMAGE_NAME:$IMAGE_TAG" | zstd -o output/image.tar.zst 17 | set +x 18 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/build-jemalloc-binaries/build.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | ROOTDIR=$(cd "$SELFDIR/../../.." && pwd) 6 | # shellcheck source=../../../lib/library.sh 7 | source "$ROOTDIR/lib/library.sh" 8 | 9 | require_envvar ENVIRONMENT_NAME 10 | require_envvar CACHE_CONTAINER 11 | require_envvar CACHE_KEY_PREFIX 12 | 13 | 14 | set -x 15 | mkdir output 16 | exec "$ROOTDIR/build-jemalloc" \ 17 | -n "$ENVIRONMENT_NAME" \ 18 | -s "$(pwd)/cache/jemalloc-src.tar.bz2" \ 19 | -o "$(pwd)/output/jemalloc-bin.tar.gz" \ 20 | -j 2 \ 21 | -c azure-connection-string.txt \ 22 | -r "$CACHE_CONTAINER" \ 23 | -d "$CACHE_KEY_PREFIX" 24 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/build-jemalloc-binaries/download-source.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | ROOTDIR=$(cd "$SELFDIR/../../.." && pwd) 6 | # shellcheck source=../../../lib/library.sh 7 | source "$ROOTDIR/lib/library.sh" 8 | 9 | require_envvar JEMALLOC_VERSION 10 | 11 | 12 | if [[ -e cache/jemalloc-src.tar.bz2 ]]; then 13 | echo "Source fetched from cache, no need to download." 14 | else 15 | run wget --output-document=cache/jemalloc-src.tar.bz2 "https://github.com/jemalloc/jemalloc/releases/download/$JEMALLOC_VERSION/jemalloc-$JEMALLOC_VERSION.tar.bz2" 16 | fi 17 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/build-rbenv-deb/build-package.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | ROOTDIR=$(cd "$SELFDIR/../../.." && pwd) 6 | # shellcheck source=../../../lib/library.sh 7 | source "$ROOTDIR/lib/library.sh" 8 | 9 | require_envvar PACKAGE_BASENAME 10 | require_envvar VERSION 11 | require_envvar REVISION 12 | 13 | 14 | set -x 15 | mkdir rbenv 16 | tar -C rbenv -xzf rbenv-src.tar.gz 17 | mkdir output 18 | exec "$ROOTDIR/build-rbenv-deb" \ 19 | -s rbenv \ 20 | -o "output/$PACKAGE_BASENAME" \ 21 | -n "$VERSION" \ 22 | -r "$REVISION" 23 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/build-rbenv-rpm/build-package.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | ROOTDIR=$(cd "$SELFDIR/../../.." && pwd) 6 | # shellcheck source=../../../lib/library.sh 7 | source "$ROOTDIR/lib/library.sh" 8 | 9 | require_envvar PACKAGE_BASENAME 10 | require_envvar VERSION 11 | require_envvar REVISION 12 | 13 | 14 | set -x 15 | 16 | mkdir rbenv 17 | tar -C rbenv -xzf rbenv-src.tar.gz 18 | mkdir output 19 | exec ./build-rbenv-rpm \ 20 | -s rbenv \ 21 | -o "output/$PACKAGE_BASENAME" \ 22 | -n "$VERSION" \ 23 | -r "$REVISION" 24 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/build-ruby-packages/build-binaries.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | ROOTDIR=$(cd "$SELFDIR/../../.." && pwd) 6 | # shellcheck source=../../../lib/library.sh 7 | source "$ROOTDIR/lib/library.sh" 8 | 9 | require_envvar ENVIRONMENT_NAME 10 | require_envvar VARIANT_NAME 11 | require_envvar RUBY_PACKAGE_VERSION_ID 12 | require_envvar CACHE_CONTAINER 13 | require_envvar CACHE_KEY_PREFIX 14 | 15 | 16 | if [[ "$VARIANT_NAME" = jemalloc ]]; then 17 | VARIANT_ARGS=(-m "$(pwd)/jemalloc-bin.tar.gz") 18 | elif [[ "$VARIANT_NAME" = malloctrim ]]; then 19 | VARIANT_ARGS=(-t) 20 | else 21 | VARIANT_ARGS=() 22 | fi 23 | 24 | set -x 25 | exec "$ROOTDIR/build-ruby" \ 26 | -n "$ENVIRONMENT_NAME" \ 27 | -s "$(pwd)/ruby-src.tar.gz" \ 28 | -v "$RUBY_PACKAGE_VERSION_ID" \ 29 | -o "$(pwd)/ruby-bin-$VARIANT_NAME.tar.gz" \ 30 | "${VARIANT_ARGS[@]}" \ 31 | -j 2 \ 32 | -c azure-connection-string.txt \ 33 | -r "$CACHE_CONTAINER" \ 34 | -d "$CACHE_KEY_PREFIX" 35 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/build-ruby-packages/build-package.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | ROOTDIR=$(cd "$SELFDIR/../../.." && pwd) 6 | # shellcheck source=../../../lib/library.sh 7 | source "$ROOTDIR/lib/library.sh" 8 | 9 | require_envvar DISTRIBUTION_NAME 10 | require_envvar VARIANT_NAME 11 | require_envvar PACKAGE_FORMAT 12 | require_envvar RUBY_PACKAGE_VERSION_ID 13 | require_envvar RUBY_PACKAGE_REVISION 14 | # Optional envvar: VARIANT_PACKAGE_SUFFIX 15 | 16 | 17 | mkdir "output-$VARIANT_NAME" 18 | 19 | if [[ "$PACKAGE_FORMAT" = DEB ]]; then 20 | PACKAGE_BASENAME=fullstaq-ruby-${RUBY_PACKAGE_VERSION_ID}${VARIANT_PACKAGE_SUFFIX}_${RUBY_PACKAGE_REVISION}-${DISTRIBUTION_NAME}_amd64.deb 21 | set -x 22 | exec "$ROOTDIR/build-ruby-deb" \ 23 | -b "ruby-bin-$VARIANT_NAME.tar.gz" \ 24 | -o "output-$VARIANT_NAME/$PACKAGE_BASENAME" \ 25 | -r "$RUBY_PACKAGE_REVISION" 26 | else 27 | # shellcheck disable=SC2001 28 | DISTRO_SUFFIX=$(sed 's/-//g' <<<"$DISTRIBUTION_NAME") 29 | PACKAGE_BASENAME=fullstaq-ruby-${RUBY_PACKAGE_VERSION_ID}${VARIANT_PACKAGE_SUFFIX}-rev${RUBY_PACKAGE_REVISION}-${DISTRO_SUFFIX}.x86_64.rpm 30 | set -x 31 | exec "$ROOTDIR/build-ruby-rpm" \ 32 | -b "ruby-bin-$VARIANT_NAME.tar.gz" \ 33 | -o "output-$VARIANT_NAME/$PACKAGE_BASENAME" \ 34 | -r "$RUBY_PACKAGE_REVISION" 35 | fi 36 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/check-version-numbers-need-changing/check-minor-ruby-package-revisions.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Checks whether any minor Ruby package revisions need to be changed, 3 | # as a result of packaging a different tiny Ruby version. 4 | # 5 | # For example, if in the last release we had this config... 6 | # 7 | # minor_version_packages: 8 | # - minor_version: 2.7 9 | # full_version: 2.7.0 10 | # package_revision: 0 11 | # 12 | # ...and now we have this... 13 | # 14 | # minor_version_packages: 15 | # - minor_version: 2.7 16 | # full_version: 2.7.1 # <---- !!!! 17 | # package_revision: 0 18 | # 19 | # ...then this script will complain that 'package_revision' also needs 20 | # to be bumped. 21 | set -e 22 | set -o pipefail 23 | 24 | SELFDIR=$(dirname "$0") 25 | ROOTDIR=$(cd "$SELFDIR/../../.." && pwd) 26 | # shellcheck source=../../../lib/library.sh 27 | source "$ROOTDIR/lib/library.sh" 28 | 29 | require_envvar LATEST_RELEASE_TAG 30 | 31 | # The following optional variables are for testing purposes. 32 | HEAD_SHA=${HEAD_SHA:-$(git rev-parse HEAD)} 33 | MOCK_APPROVAL_STATUS=${MOCK_APPROVAL_STATUS:-not set} # may be set to true or false 34 | 35 | 36 | HEAD_SHA_SHORT=${HEAD_SHA:0:8} 37 | 38 | 39 | git archive "$LATEST_RELEASE_TAG" config.yml | tar -xO > config-latest-release.yml 40 | # Find all minor Ruby versions for which all of the following is true: 41 | # 42 | # - It's packaged by both the previous Fullstaq Ruby release 43 | # as well as the current one. 44 | # - Its `full_version` has changed compared to the previous release. 45 | # - The package revision has not been bumped. 46 | # 47 | # shellcheck disable=SC2207 48 | IFS=$'\n' UNBUMPED_MINOR_RUBY_PACKAGE_VERSIONS=($("$SELFDIR"/determine-unbumped-minor-ruby-package-versions.rb \ 49 | "$ROOTDIR/config.yml" config-latest-release.yml)) 50 | 51 | 52 | if [[ ${#UNBUMPED_MINOR_RUBY_PACKAGE_VERSIONS[@]} -eq 0 ]]; then 53 | echo "All relevant Ruby minor package revisions have already been bumped compared to $LATEST_RELEASE_TAG." 54 | else 55 | echo "In config.yml, please bump the following ${BOLD}ruby.minor_version_packages.package_revision${RESET}s:" 56 | echo 57 | for MINOR_RUBY_PACKAGE_VERSION in "${UNBUMPED_MINOR_RUBY_PACKAGE_VERSIONS[@]}"; do 58 | echo " * $MINOR_RUBY_PACKAGE_VERSION" 59 | done 60 | fi 61 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/check-version-numbers-need-changing/check-rbenv-version.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Checks whether the Rbenv version specified in 3 | # config.yml matches the actual Rbenv version. 4 | set -e 5 | 6 | EXPECTED_RBENV_VERSION=$(./rbenv/bin/rbenv --version | awk '{ print $2 }' | sed -E 's/(.+)-.*/\1/') 7 | ACTUAL_RBENV_VERSION=$(ruby -ryaml -e 'puts YAML.load_file("config.yml")["rbenv"]["version"]') 8 | 9 | if [[ "$EXPECTED_RBENV_VERSION" = "$ACTUAL_RBENV_VERSION" ]]; then 10 | echo 'All good!' 11 | else 12 | echo 'ERROR: the Rbenv version in config.yml is wrong.' 13 | echo "Expected: $EXPECTED_RBENV_VERSION" 14 | echo " Actual: $ACTUAL_RBENV_VERSION" 15 | echo 16 | echo "Please open config.yml and edit rbenv.version" 17 | exit 1 18 | fi 19 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/check-version-numbers-need-changing/check-ruby-package-revisions.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Checks whether any Ruby package revisions need to be changed, 3 | # as a result of changed package metadata or contents. 4 | # 5 | # For example, if we've modified container-entrypoints/build-ruby to 6 | # add another file to the Ruby package, then this script will complain 7 | # that all 'ruby.*.package_revision' fields in config.yml need to be 8 | # bumped. 9 | set -e 10 | set -o pipefail 11 | 12 | SELFDIR=$(dirname "$0") 13 | ROOTDIR=$(cd "$SELFDIR/../../.." && pwd) 14 | # shellcheck source=../../../lib/library.sh 15 | source "$ROOTDIR/lib/library.sh" 16 | 17 | require_envvar LATEST_RELEASE_TAG 18 | 19 | # The following optional variables are for testing purposes. 20 | HEAD_SHA=${HEAD_SHA:-$(git rev-parse HEAD)} 21 | MOCK_APPROVAL_STATUS=${MOCK_APPROVAL_STATUS:-not set} # may be set to true or false 22 | # Optional: MOCK_UNBUMPED_RUBY_PACKAGE_VERSIONS 23 | 24 | 25 | HEAD_SHA_SHORT=${HEAD_SHA:0:8} 26 | 27 | # Check whether any of the following files have changed 28 | # in such a way that they would change the Ruby packages' 29 | # contents or metadata. 30 | 31 | REVIEW_GLOB='container-entrypoints/build-{jemalloc,ruby,ruby-deb,ruby-rpm}' 32 | # shellcheck disable=SC2207,SC2012 33 | REVIEW_FILES=($(ls container-entrypoints/build-{jemalloc,ruby,ruby-deb,ruby-rpm} | sort)) 34 | 35 | 36 | DIFF=$(git diff "$LATEST_RELEASE_TAG" "${REVIEW_FILES[@]}") 37 | if [[ -z "$DIFF" ]]; then 38 | echo " * Relevant scripts did not change since $LATEST_RELEASE_TAG." 39 | echo " Scripts checked: $REVIEW_GLOB" 40 | exit 41 | fi 42 | 43 | # A relevant script changed. So Ruby package revision numbers 44 | # may need bumping. 45 | 46 | echo " * Relevant scripts have changed compared to $LATEST_RELEASE_TAG" 47 | echo " Scripts checked: $REVIEW_GLOB" 48 | echo 49 | 50 | 51 | # Find all Ruby versions for which all of the following is true: 52 | # 53 | # - It's packaged by both the previous Fullstaq Ruby release 54 | # as well as the current one. 55 | # - The package revision has not been bumped. 56 | if [[ -z "$MOCK_UNBUMPED_RUBY_PACKAGE_VERSIONS" ]]; then 57 | git archive "$LATEST_RELEASE_TAG" config.yml | tar -xO > config-latest-release.yml 58 | # shellcheck disable=SC2207 59 | IFS=$'\n' UNBUMPED_RUBY_PACKAGE_VERSIONS=($("$SELFDIR"/determine-unbumped-ruby-package-versions.rb \ 60 | "$ROOTDIR/config.yml" config-latest-release.yml)) 61 | else 62 | # shellcheck disable=SC2206 63 | UNBUMPED_RUBY_PACKAGE_VERSIONS=($MOCK_UNBUMPED_RUBY_PACKAGE_VERSIONS) 64 | fi 65 | 66 | 67 | if [[ ${#UNBUMPED_RUBY_PACKAGE_VERSIONS[@]} -eq 0 ]]; then 68 | echo " * All relevant Ruby package revisions have already been bumped compared to $LATEST_RELEASE_TAG." 69 | exit 70 | fi 71 | 72 | 73 | echo " * Some Ruby package revisions have not been bumped compared to $LATEST_RELEASE_TAG:" 74 | echo 75 | for RUBY_PACKAGE_VERSION in "${UNBUMPED_RUBY_PACKAGE_VERSIONS[@]}"; do 76 | echo " - $RUBY_PACKAGE_VERSION" 77 | done 78 | echo 79 | echo "------------------" 80 | echo 81 | echo "Checking whether manual approval is given..." 82 | 83 | APPROVAL_DATA=$( 84 | echo "project=fullstaq-ruby-server-edition" && 85 | echo "component=ruby-package-revisions" && 86 | echo "base=$LATEST_RELEASE_TAG" && 87 | sha256sum "${REVIEW_FILES[@]}" 88 | ) 89 | APPROVAL_CHECKSUM=$(md5sum <<<"$APPROVAL_DATA" | awk '{ print $1 }') 90 | 91 | if [[ "$MOCK_APPROVAL_STATUS" = true ]]; then 92 | echo "$APPROVAL_CHECKSUM" > approvals.txt 93 | elif [[ "$MOCK_APPROVAL_STATUS" = false ]]; then 94 | echo -n > approvals.txt 95 | else 96 | curl -fsSLO https://raw.githubusercontent.com/fullstaq-labs/fullstaq-ruby-ci-approvals/main/approvals.txt 97 | fi 98 | 99 | if grep -q "^${APPROVAL_CHECKSUM}$" approvals.txt; then 100 | echo "Manual approval detected." 101 | else 102 | echo "No manual approval detected." 103 | echo 104 | echo "${BOLD}${YELLOW}*** MANUAL REVIEW AND ACTION REQUIRED ***${RESET}" 105 | echo 106 | echo "$REVIEW_GLOB has changed." 107 | echo "${BOLD}Please review${RESET} the changes in these files between $LATEST_RELEASE_TAG and $HEAD_SHA_SHORT:" 108 | echo 109 | echo " ${CYAN}git diff $LATEST_RELEASE_TAG..$HEAD_SHA_SHORT $REVIEW_GLOB${RESET}" 110 | echo 111 | echo "${BOLD}${YELLOW}## How to review?${RESET}" 112 | echo 113 | echo "Check whether the code would ${BOLD}change any Ruby package contents or metadata${RESET}." 114 | echo 115 | echo "${BOLD}${YELLOW}## How to take action?${RESET}" 116 | echo 117 | echo " * If the package contents or metadata will change, then open config.yml and" 118 | echo " bump the package_revision for the following Ruby packages:" 119 | echo 120 | for RUBY_PACKAGE_VERSION in "${UNBUMPED_RUBY_PACKAGE_VERSIONS[@]}"; do 121 | echo " - $RUBY_PACKAGE_VERSION" 122 | done 123 | echo 124 | echo " ${BOLD}-- OR --${RESET}" 125 | echo 126 | echo " * If the package contents or metadata will NOT change, then manually approve" 127 | echo " by adding this line..." 128 | echo 129 | echo " $APPROVAL_CHECKSUM" 130 | echo 131 | echo " ...to github.com/fullstaq-labs/fullstaq-ruby-ci-approvals," 132 | echo " file approvals.txt:" 133 | echo 134 | echo " https://github.com/fullstaq-labs/fullstaq-ruby-ci-approvals/edit/main/approvals.txt" 135 | echo 136 | echo " You can also use this command:" 137 | echo 138 | echo " git clone --depth=1 git@github.com:fullstaq-labs/fullstaq-ruby-ci-approvals.git &&" 139 | echo " cd fullstaq-ruby-ci-approvals &&" 140 | echo " echo $APPROVAL_CHECKSUM >> approvals.txt &&" 141 | echo " git commit -a -m 'Approve fullstaq-ruby-server-edition ruby-package-revision $HEAD_SHA_SHORT' &&" 142 | echo " git push" 143 | exit 1 144 | fi 145 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/check-version-numbers-need-changing/determine-latest-release-tag.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | ROOTDIR=$(cd "$SELFDIR/../../.." && pwd) 6 | # shellcheck source=../../../lib/library.sh 7 | source "$ROOTDIR/lib/library.sh" 8 | 9 | require_envvar GITHUB_ENV 10 | 11 | 12 | run git fetch 13 | 14 | echo "+ Calculating..." 15 | 16 | MERGE_BASE=$(git merge-base origin/main HEAD) 17 | LATEST_RELEASE_TAG=$(git describe "$MERGE_BASE" --tags --abbrev=0 --match='epic-*') 18 | 19 | echo "LATEST_RELEASE_TAG=$LATEST_RELEASE_TAG" >> "$GITHUB_ENV" 20 | echo "Latest release tag: $LATEST_RELEASE_TAG" 21 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/check-version-numbers-need-changing/determine-unbumped-minor-ruby-package-versions.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # Print all minor Ruby versions for which all of the following is true: 3 | # 4 | # - It's packaged by both the previous Fullstaq Ruby release 5 | # as well as the current one. 6 | # - Its `full_version` has changed compared to the previous release. 7 | # - The package revision has not been bumped. 8 | require 'yaml' 9 | 10 | CURRENT_CONFIG_PATH = ARGV[0] 11 | LATEST_RELEASE_CONFIG_PATH = ARGV[1] 12 | 13 | def load_current_config 14 | File.open(CURRENT_CONFIG_PATH, 'r:utf-8') do |f| 15 | YAML.safe_load(f.read, filename: CURRENT_CONFIG_PATH) 16 | end 17 | end 18 | 19 | def load_latest_release_config 20 | File.open(LATEST_RELEASE_CONFIG_PATH, 'r:utf-8') do |f| 21 | YAML.safe_load(f.read, filename: LATEST_RELEASE_CONFIG_PATH) 22 | end 23 | end 24 | 25 | def find_minor_version_package(config, minor_version_package) 26 | config['ruby']['minor_version_packages'].find do |candidate| 27 | candidate['minor_version'] == minor_version_package['minor_version'] 28 | end 29 | end 30 | 31 | def package_revision_needs_bumping?(current, latest) 32 | current['full_version'] != latest['full_version'] && \ 33 | current['package_revision'] == latest['package_revision'] 34 | end 35 | 36 | def main 37 | current_config = load_current_config 38 | latest_release_config = load_latest_release_config 39 | 40 | current_config['ruby']['minor_version_packages'].each do |current_minor_version_package| 41 | latest_release_minor_version_package = find_minor_version_package( 42 | latest_release_config, current_minor_version_package) 43 | next if latest_release_minor_version_package.nil? 44 | 45 | if package_revision_needs_bumping?(current_minor_version_package, latest_release_minor_version_package) 46 | puts "minor_version == #{current_minor_version_package['minor_version']}" 47 | end 48 | end 49 | end 50 | 51 | main 52 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/check-version-numbers-need-changing/determine-unbumped-ruby-package-versions.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # Print all Ruby versions for which all of the following is true: 3 | # 4 | # - It's packaged by both the previous Fullstaq Ruby release 5 | # as well as the current one. 6 | # - The package revision has not been bumped. 7 | require 'yaml' 8 | 9 | CURRENT_CONFIG_PATH = ARGV[0] 10 | LATEST_RELEASE_CONFIG_PATH = ARGV[1] 11 | 12 | def load_current_config 13 | File.open(CURRENT_CONFIG_PATH, 'r:utf-8') do |f| 14 | YAML.safe_load(f.read, filename: CURRENT_CONFIG_PATH) 15 | end 16 | end 17 | 18 | def load_latest_release_config 19 | File.open(LATEST_RELEASE_CONFIG_PATH, 'r:utf-8') do |f| 20 | YAML.safe_load(f.read, filename: LATEST_RELEASE_CONFIG_PATH) 21 | end 22 | end 23 | 24 | def find_minor_version_package(config, minor_version_package) 25 | config['ruby']['minor_version_packages'].find do |candidate| 26 | candidate['minor_version'] == minor_version_package['minor_version'] && 27 | candidate['full_version'] == minor_version_package['full_version'] 28 | end 29 | end 30 | 31 | def find_tiny_version_package(config, tiny_version_package) 32 | config['ruby']['tiny_version_packages'].find do |candidate| 33 | candidate['full_version'] == tiny_version_package['full_version'] 34 | end 35 | end 36 | 37 | def main 38 | current_config = load_current_config 39 | latest_release_config = load_latest_release_config 40 | 41 | current_config['ruby']['minor_version_packages'].each do |current_minor_version_package| 42 | latest_release_minor_version_package = find_minor_version_package( 43 | latest_release_config, current_minor_version_package) 44 | next if latest_release_minor_version_package.nil? 45 | 46 | if current_minor_version_package['package_revision'] \ 47 | == latest_release_minor_version_package['package_revision'] 48 | 49 | puts "minor version package #{current_minor_version_package['minor_version']}" 50 | end 51 | end 52 | 53 | current_config['ruby']['tiny_version_packages'].each do |current_tiny_version_package| 54 | latest_release_tiny_version_package = find_tiny_version_package( 55 | latest_release_config, current_tiny_version_package) 56 | next if latest_release_tiny_version_package.nil? 57 | 58 | if current_tiny_version_package['package_revision'] \ 59 | == latest_release_tiny_version_package['package_revision'] 60 | 61 | puts "tiny version package #{current_tiny_version_package['full_version']}" 62 | end 63 | end 64 | end 65 | 66 | main 67 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/check-version-numbers-need-changing/extract-rbenv-source.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # Extracts the Rbenv source tarball, which we downloaded 3 | # as an artifact. 4 | set -e 5 | 6 | mkdir rbenv 7 | tar -C rbenv -xzf rbenv-src.tar.gz 8 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/check-workflow-uptodate/check.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | set -o pipefail 4 | 5 | SELFDIR=$(dirname "$0") 6 | ROOTDIR=$(cd "$SELFDIR/../../.." && pwd) 7 | 8 | 9 | "$ROOTDIR"/internal-scripts/generate-ci-cd-yaml.rb 10 | 11 | if [[ -z "$(git status --porcelain --untracked-files=no .github/workflows)" ]]; then 12 | echo 'All workflow files up-to-date!' 13 | else 14 | echo 'ERROR: one or more workflow files are not up-to-date!' 15 | git status --porcelain --untracked-files=no .github/workflows 16 | 17 | echo 18 | echo 'Please run this script, then commit and push:' 19 | echo 20 | echo ' ./internal-scripts/generate-ci-cd-yaml.rb' 21 | echo 22 | echo 'TIP: run this on your development machine to ensure generate-ci-cd-yaml.rb is run automatically as a Git pre-commit hook:' 23 | echo 24 | echo ' git config core.hooksPath .githooks' 25 | echo 26 | echo "Here's the difference:" 27 | git diff --color .github/workflows 28 | exit 1 29 | fi 30 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/create-git-tag/determine-next-epic-version.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | ROOTDIR=$(cd "$SELFDIR/../../.." && pwd) 6 | # shellcheck source=../../../lib/library.sh 7 | source "$ROOTDIR/lib/library.sh" 8 | 9 | require_envvar GITHUB_ENV 10 | require_envvar LATEST_RELEASE_TAG 11 | 12 | 13 | # shellcheck disable=SC2001 14 | LATEST_RELEASE_VERSION=$(sed 's/^epic-//' <<<"$LATEST_RELEASE_TAG") 15 | 16 | # shellcheck disable=SC2206 17 | IFS=. VERSION_COMPONENTS=($LATEST_RELEASE_VERSION) 18 | 19 | TO_BUMP=minor 20 | # shellcheck disable=SC2207 21 | IFS=$'\n' COMMIT_HASHES=($(git rev-list "$LATEST_RELEASE_TAG"..HEAD)) 22 | 23 | for COMMIT_HASH in "${COMMIT_HASHES[@]}"; do 24 | echo "Analyzing $COMMIT_HASH..." 25 | COMMIT_MESSAGE=$(git log --format=%B -n 1 "$COMMIT_HASH") 26 | if [[ "$COMMIT_MESSAGE" =~ \[major\] ]]; then 27 | TO_BUMP=major 28 | echo " => Will bump major version" 29 | elif [[ "$COMMIT_MESSAGE" =~ \[minor\] ]]; then 30 | if [[ "$TO_BUMP" != major ]]; then 31 | TO_BUMP=minor 32 | echo " => Will bump minor version" 33 | fi 34 | fi 35 | done 36 | 37 | if [[ "$TO_BUMP" = major ]]; then 38 | (( VERSION_COMPONENTS[0]++ )) || true 39 | VERSION_COMPONENTS[1]=0 40 | else 41 | (( VERSION_COMPONENTS[1]++ )) || true 42 | fi 43 | 44 | echo 45 | echo "NEXT_RELEASE_VERSION=${VERSION_COMPONENTS[0]}.${VERSION_COMPONENTS[1]}" >> "$GITHUB_ENV" 46 | echo "Next epic version: ${VERSION_COMPONENTS[0]}.${VERSION_COMPONENTS[1]}" 47 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/determine-necessary-jobs/list-artifacts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | set -o pipefail 4 | 5 | SELFDIR=$(dirname "$0") 6 | ROOTDIR=$(cd "$SELFDIR/../../.." && pwd) 7 | # shellcheck source=../../../lib/library.sh 8 | source "$ROOTDIR/lib/library.sh" 9 | 10 | 11 | require_envvar CI_ARTIFACTS_BUCKET 12 | require_envvar CI_ARTIFACTS_RUN_NUMBER 13 | 14 | URL_PREFIX="gs://$CI_ARTIFACTS_BUCKET/$CI_ARTIFACTS_RUN_NUMBER" 15 | echo "--> Checking $URL_PREFIX/*.tar.zst" 16 | 17 | if ENTRIES=$(gsutil ls "$URL_PREFIX/*.tar.zst" 2>stderr.txt); then 18 | ENTRIES=$(gsutil ls "$URL_PREFIX/*.tar.zst" | sed 's|.*/||; s|\.tar\.zst||') 19 | echo "$ENTRIES" 20 | echo "$ENTRIES" > artifacts.txt 21 | elif grep -q "matched no objects" stderr.txt; then 22 | echo "(no entries)" 23 | echo -n > artifacts.txt 24 | else 25 | cat stderr.txt >&2 26 | exit 1 27 | fi 28 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/download-artifact.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | set -o pipefail 4 | 5 | SELFDIR=$(dirname "$0") 6 | ROOTDIR=$(cd "$SELFDIR/../.." && pwd) 7 | # shellcheck source=../../lib/library.sh 8 | source "$ROOTDIR/lib/library.sh" 9 | 10 | 11 | require_envvar GITHUB_RUN_NUMBER 12 | require_envvar CI_ARTIFACTS_BUCKET 13 | require_envvar ARTIFACT_NAME 14 | require_envvar ARTIFACT_PATH 15 | CI_ARTIFACTS_RUN_NUMBER=${CI_ARTIFACTS_RUN_NUMBER:-$GITHUB_RUN_NUMBER} 16 | CLEAR=${CLEAR:-false} 17 | 18 | URL="gs://$CI_ARTIFACTS_BUCKET/$CI_ARTIFACTS_RUN_NUMBER/$ARTIFACT_NAME.tar.zst" 19 | if [[ "$CLEAR" = true ]]; then 20 | echo "--> Clearing destination directory" 21 | rm -rf "$ARTIFACT_PATH" 22 | fi 23 | echo "--> Will download from $URL" 24 | mkdir -p "$ARTIFACT_PATH" 25 | gsutil cp "$URL" - | zstd -d | tar -C "$ARTIFACT_PATH" -xf - 26 | 27 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/download-artifacts.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | set -o pipefail 4 | 5 | SELFDIR=$(dirname "$0") 6 | ROOTDIR=$(cd "$SELFDIR/../.." && pwd) 7 | # shellcheck source=../../lib/library.sh 8 | source "$ROOTDIR/lib/library.sh" 9 | 10 | 11 | require_envvar GITHUB_RUN_NUMBER 12 | require_envvar CI_ARTIFACTS_BUCKET 13 | require_envvar ARTIFACT_NAMES 14 | require_envvar ARTIFACT_PATH 15 | CI_ARTIFACTS_RUN_NUMBER=${CI_ARTIFACTS_RUN_NUMBER:-$GITHUB_RUN_NUMBER} 16 | CLEAR=${CLEAR:-false} 17 | TMPDIR=${TMPDIR:-/tmp} 18 | 19 | function cleanup() 20 | { 21 | if [[ -n "$WORK_DIR" ]]; then 22 | rm -rf "$WORK_DIR" 23 | fi 24 | } 25 | 26 | if [[ "$CLEAR" = true ]]; then 27 | echo "--> Clearing destination directory" 28 | rm -rf "$ARTIFACT_PATH" 29 | fi 30 | 31 | URLS=() 32 | # shellcheck disable=SC2153 33 | for ARTIFACT_NAME in $ARTIFACT_NAMES; do 34 | URL="gs://$CI_ARTIFACTS_BUCKET/$CI_ARTIFACTS_RUN_NUMBER/$ARTIFACT_NAME.tar.zst" 35 | URLS+=("$URL") 36 | echo "--> Will download from $URL" 37 | done 38 | echo 39 | 40 | echo "--> Downloading artifacts" 41 | WORK_DIR=$(mktemp -d "$TMPDIR/XXXXXX") 42 | mkdir -p "$ARTIFACT_PATH" 43 | gsutil -m cp "${URLS[@]}" "$WORK_DIR/" 44 | echo 45 | 46 | echo "--> Extracting artifacts" 47 | # shellcheck disable=SC2153 48 | for ARTIFACT_NAME in $ARTIFACT_NAMES; do 49 | mkdir -p "$ARTIFACT_PATH/$ARTIFACT_NAME" 50 | zstd -d < "$WORK_DIR/$ARTIFACT_NAME.tar.zst" | tar -C "$ARTIFACT_PATH/$ARTIFACT_NAME" -xf - 51 | done 52 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/download-rbenv-source/download.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | set -o pipefail 4 | 5 | SELFDIR=$(dirname "$0") 6 | ROOTDIR=$(cd "$SELFDIR/../../.." && pwd) 7 | # shellcheck source=../../../lib/library.sh 8 | source "$ROOTDIR/lib/library.sh" 9 | 10 | require_envvar RBENV_REPO_URL 11 | require_envvar RBENV_REPO_REF 12 | 13 | 14 | if [[ -e output/rbenv-src.tar.gz ]]; then 15 | echo "Source fetched from cache, no need to download." 16 | exit 17 | fi 18 | 19 | mkdir output 20 | 21 | set -x 22 | git clone "$RBENV_REPO_URL" rbenv 23 | cd rbenv 24 | git reset --hard "$RBENV_REPO_REF" 25 | # The tarball must include .git too because `rbenv --version` 26 | # scans the Git history in order to determine its version 27 | # number. 28 | tar -zcf ../output/rbenv-src.tar.gz . 29 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/download-rbenv-source/prepare.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | set -o pipefail 4 | 5 | REPO_URL=$(ruby -ryaml -e 'puts YAML.load_file("config.yml")["rbenv"]["repo"]') 6 | REF=$(ruby -ryaml -e 'puts YAML.load_file("config.yml")["rbenv"]["ref"]') 7 | CACHE_BODY="$REPO_URL"$'\n'"$REF" 8 | CACHE_KEY=$(md5sum <<<"$CACHE_BODY" | awk '{ print $1 }') 9 | 10 | echo "Repo: $REPO_URL" 11 | echo "Ref: $REF" 12 | echo "Cache key: $CACHE_KEY" 13 | 14 | { 15 | echo "repo_url=$REPO_URL" 16 | echo "ref=$REF" 17 | echo "cache_key=$CACHE_KEY" 18 | } >> "$GITHUB_OUTPUT" 19 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/download-ruby-sources/download.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | ROOTDIR=$(cd "$SELFDIR/../../.." && pwd) 6 | # shellcheck source=../../../lib/library.sh 7 | source "$ROOTDIR/lib/library.sh" 8 | 9 | require_envvar RUBY_VERSION 10 | 11 | 12 | MINOR_VERSION=$(sed -E 's/(.+)\..*/\1/' <<<"$RUBY_VERSION") 13 | 14 | if [[ -e output/ruby-src.tar.gz ]]; then 15 | echo "Source fetched from cache, no need to download." 16 | else 17 | run mkdir output 18 | run wget --output-document output/ruby-src.tar.gz \ 19 | "https://cache.ruby-lang.org/pub/ruby/$MINOR_VERSION/ruby-$RUBY_VERSION.tar.gz" 20 | fi 21 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/load-docker-image.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | set -o pipefail 4 | 5 | SELFDIR=$(dirname "$0") 6 | ROOTDIR=$(cd "$SELFDIR/../.." && pwd) 7 | # shellcheck source=../../lib/library.sh 8 | source "$ROOTDIR/lib/library.sh" 9 | 10 | 11 | require_envvar TARBALL 12 | 13 | set -x 14 | zstd -d < "$TARBALL" | docker load 15 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/publish/clean-disk-space.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | set -e 3 | 4 | echo "### Free disk space before cleaning:" 5 | df -h 6 | 7 | # Based on https://stackoverflow.com/questions/75536771/github-runner-out-of-disk-space-after-building-docker-image 8 | echo 9 | echo "### Cleaning...." 10 | sudo rm -rf "$AGENT_TOOLSDIRECTORY" /usr/share/dotnet /usr/local/lib/android /opt/ghc \ 11 | /usr/local/share/powershell /usr/share/swift /usr/local/.ghcup /usr/lib/jvm 12 | sudo apt purge -y aria2 shellcheck zsync google-chrome-stable \ 13 | ant ant-optional kubectl mercurial apt-transport-https yarn libssl-dev \ 14 | libfreetype6-dev libfontconfig1 snmp pollinate libpq-dev sphinxsearch 15 | sudo apt purge -y '^mysql' 16 | sudo apt purge -y '^php' 17 | sudo apt purge -y '^dotnet' 18 | sudo apt autoremove -y 19 | sudo apt autoclean -y 20 | 21 | echo 22 | echo "### Free disk space after cleaning:" 23 | df -h 24 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/publish/install-aptly.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | ROOTDIR=$(cd "$SELFDIR/../../.." && pwd) 6 | # shellcheck source=../../../lib/library.sh 7 | source "$ROOTDIR/lib/library.sh" 8 | 9 | echo '+ Adding Aptly repo' 10 | echo deb http://repo.aptly.info/ squeeze main | sudo tee /etc/apt/sources.list.d/aptly.list 11 | echo '+ Adding Aptly public key' 12 | curl -fsSL https://www.aptly.info/pubkey.txt | sudo apt-key add - 13 | run sudo apt update 14 | run sudo apt install -y aptly 15 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/publish/restart-web-server.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | # frozen_string_literal: true 3 | 4 | require_relative '../../../lib/shell_scripting_support' 5 | require 'net/http' 6 | require 'net/https' 7 | 8 | class RestartWebServer 9 | include ShellScriptingSupport 10 | 11 | # There's only one web server, so no need to call yum.fullstaqruby.org 12 | RESTART_WEB_SERVER_API_URL = 'https://apt.fullstaqruby.org/admin/restart_web_server' 13 | LATEST_REPO_QUERY_TIMESTAMP_URL = 'https://apt.fullstaqruby.org/admin/repo_query_time' 14 | 15 | TIMEOUT_SECS = 60 16 | 17 | class QueryError < StandardError; end 18 | 19 | def main 20 | require_envvar('ID_TOKEN') 21 | 22 | begin 23 | orig_timestamp = get_latest_production_repo_query_timestamp 24 | rescue QueryError, SystemCallError => e 25 | abort("ERROR: failed to query latest web server timestamp: #{e.message}") 26 | end 27 | 28 | log_notice 'Restarting web servers' 29 | initiate_restart_web_server 30 | 31 | log_notice 'Waiting until web server is restarted' 32 | deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + TIMEOUT_SECS 33 | while true 34 | if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline 35 | abort('ERROR: Timed out waiting for web server to restart') 36 | end 37 | 38 | begin 39 | timestamp = get_latest_production_repo_query_timestamp 40 | rescue QueryError, SystemCallError => e 41 | log_info "Error querying latest web server timestamp: #{e.message}" 42 | log_info 'Retrying in 4s...' 43 | sleep 4 44 | next 45 | end 46 | 47 | if timestamp == orig_timestamp 48 | log_info 'Web server has not restarted yet; waiting...' 49 | sleep 4 50 | next 51 | end 52 | 53 | log_info 'Web server has restarted' 54 | break 55 | end 56 | end 57 | 58 | def initiate_restart_web_server 59 | log_info "POSTing to #{RESTART_WEB_SERVER_API_URL}" 60 | uri = URI(RESTART_WEB_SERVER_API_URL) 61 | req = Net::HTTP::Post.new(uri) 62 | req['Authorization'] = "Bearer #{ENV['ID_TOKEN']}" 63 | 64 | resp = Net::HTTP.start(uri.hostname, uri.port, use_ssl: true) do |http| 65 | http.request(req) 66 | end 67 | 68 | if resp.code.to_i / 100 != 2 69 | abort("ERROR: failed to restart web server: #{resp.code} #{resp.message}: #{resp.body}") 70 | end 71 | end 72 | 73 | def get_latest_production_repo_query_timestamp 74 | uri = URI(LATEST_REPO_QUERY_TIMESTAMP_URL) 75 | resp = Net::HTTP.get_response(uri) 76 | if resp.code.to_i / 100 == 2 77 | resp.body 78 | else 79 | raise QueryError, "#{resp.code} #{resp.message}: #{resp.body}" 80 | end 81 | end 82 | end 83 | 84 | RestartWebServer.new.main 85 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/test-packages/run-tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | ROOTDIR=$(cd "$SELFDIR/../../.." && pwd) 6 | # shellcheck source=../../../lib/library.sh 7 | source "$ROOTDIR/lib/library.sh" 8 | 9 | require_envvar DISTRIBUTION_NAME 10 | require_envvar RUBY_PACKAGE_ID 11 | require_envvar PACKAGE_FORMAT 12 | require_envvar VARIANT_NAME 13 | require_envvar TEST_IMAGE_NAME 14 | require_envvar APT_REPO_URL 15 | require_envvar YUM_REPO_URL 16 | # Optional envvar: VARIANT_PACKAGE_SUFFIX 17 | 18 | 19 | mkdir repo 20 | echo '--- Entering main Docker container ---' 21 | 22 | if [[ "$PACKAGE_FORMAT" == DEB ]]; then 23 | set -x 24 | exec docker run --rm --init \ 25 | -v "$ROOTDIR:/system:ro" \ 26 | -v "$(pwd)/repo:/input/repo:ro" \ 27 | -e "SERVER=$APT_REPO_URL" \ 28 | -e "APT_DISTRO_NAME=$DISTRIBUTION_NAME" \ 29 | -e "RUBY_PACKAGE_VERSION=$RUBY_PACKAGE_ID$VARIANT_PACKAGE_SUFFIX" \ 30 | -e "EXPECTED_VARIANT=$VARIANT_NAME" \ 31 | -e "DEBUG_AFTER_TESTS=false" \ 32 | --user root \ 33 | --entrypoint /system/container-entrypoints/test-debs \ 34 | "$TEST_IMAGE_NAME" 35 | else 36 | set -x 37 | exec docker run --rm --init \ 38 | -v "$ROOTDIR:/system:ro" \ 39 | -v "$(pwd)/repo:/input/repo:ro" \ 40 | -e "SERVER=$YUM_REPO_URL/$DISTRIBUTION_NAME" \ 41 | -e "RUBY_PACKAGE_VERSION=$RUBY_PACKAGE_ID$VARIANT_PACKAGE_SUFFIX" \ 42 | -e "EXPECTED_VARIANT=$VARIANT_NAME" \ 43 | -e "DEBUG_AFTER_TESTS=false" \ 44 | --user root \ 45 | --entrypoint /system/container-entrypoints/test-rpms \ 46 | "$TEST_IMAGE_NAME" 47 | fi 48 | -------------------------------------------------------------------------------- /internal-scripts/ci-cd/upload-artifact.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | set -o pipefail 4 | 5 | SELFDIR=$(dirname "$0") 6 | ROOTDIR=$(cd "$SELFDIR/../.." && pwd) 7 | # shellcheck source=../../lib/library.sh 8 | source "$ROOTDIR/lib/library.sh" 9 | 10 | 11 | require_envvar GITHUB_RUN_NUMBER 12 | require_envvar CI_ARTIFACTS_BUCKET 13 | require_envvar ARTIFACT_NAME 14 | require_envvar ARTIFACT_PATH 15 | CI_ARTIFACTS_RUN_NUMBER=${CI_ARTIFACTS_RUN_NUMBER:-$GITHUB_RUN_NUMBER} 16 | 17 | URL="gs://$CI_ARTIFACTS_BUCKET/$CI_ARTIFACTS_RUN_NUMBER/$ARTIFACT_NAME.tar.zst" 18 | echo "--> Will upload to $URL" 19 | if ! tar -C "$ARTIFACT_PATH" -cf - . | zstd -T0 | gsutil cp - "$URL"; then 20 | echo "--> Artifact upload failed; cleaning up" 21 | gsutil rm "$URL" || true 22 | exit 1 23 | fi 24 | -------------------------------------------------------------------------------- /internal-scripts/generate-ci-cd-yaml.rb: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env ruby 2 | require 'erb' 3 | require_relative '../lib/general_support' 4 | require_relative '../lib/ci_workflow_support' 5 | 6 | class TemplateContext 7 | include GeneralSupport 8 | include CiWorkflowSupport 9 | 10 | def get_binding 11 | binding 12 | end 13 | end 14 | 15 | class CiCdYamlGenerationApp 16 | ROOT = GeneralSupport::ROOT 17 | WORKFLOWS_SUBPATH = ".github/workflows" 18 | 19 | include CiWorkflowSupport 20 | 21 | def main 22 | load_config 23 | 24 | remove_yaml_files('ci-cd-build-packages*') 25 | 26 | generate_yaml_file_from_template(template_name: 'ci-cd-main') 27 | generate_yaml_file_from_template(template_name: 'ci-cd-prepare') 28 | distribution_buckets.each_with_index do |distributions, i| 29 | generate_yaml_file_from_template( 30 | template_name: 'ci-cd-build-packages', 31 | output_name: "ci-cd-build-packages-#{i + 1}", 32 | part_number: i + 1, 33 | distributions: distributions, 34 | total_distribution_buckets_num: distribution_buckets.size) 35 | end 36 | generate_yaml_file_from_template(template_name: 'ci-cd-publish-test-test') 37 | generate_yaml_file_from_template(template_name: 'ci-cd-publish-test-production') 38 | end 39 | 40 | private 41 | def remove_yaml_files(pattern) 42 | Dir["#{ROOT}/#{WORKFLOWS_SUBPATH}/#{pattern}.yml"].each do |path| 43 | puts "Removing #{WORKFLOWS_SUBPATH}/#{File.basename(path)}" 44 | File.unlink(path) 45 | end 46 | end 47 | 48 | def generate_yaml_file_from_template(template_name:, output_name: nil, **vars) 49 | input_path = "#{ROOT}/#{WORKFLOWS_SUBPATH}/#{template_name}.yml.erb" 50 | output_path = "#{ROOT}/#{WORKFLOWS_SUBPATH}/#{output_name || template_name}.yml" 51 | puts "Regenerating #{WORKFLOWS_SUBPATH}/#{output_name || template_name}.yml" 52 | 53 | template_src = File.read(input_path, mode: 'r:utf-8') 54 | erb = ERB.new(template_src, trim_mode: '-', eoutvar: '@erb_out') 55 | erb.location = input_path 56 | 57 | context = TemplateContext.new 58 | vars.each_pair do |name, value| 59 | context.define_singleton_method(name) do 60 | value 61 | end 62 | end 63 | result = erb.result(context.get_binding) 64 | 65 | File.open(output_path, 'w:utf-8') do |f| 66 | f.write(result) 67 | end 68 | end 69 | end 70 | 71 | CiCdYamlGenerationApp.new.main 72 | -------------------------------------------------------------------------------- /lib/general_support.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module GeneralSupport 4 | ROOT = File.absolute_path(File.dirname(__FILE__) + '/..') 5 | 6 | # Make instance methods available as class methods 7 | extend self 8 | 9 | 10 | def unindent(amount, &block) 11 | indentation = /^#{' ' * amount}/ 12 | lines = capture(&block).split("\n") 13 | lines.map! { |l| l.sub(indentation, '') } 14 | @erb_out << lines.join("\n") 15 | end 16 | 17 | def slug(str) 18 | str.to_s.scan(/[a-z0-9]+/).join('_') 19 | end 20 | 21 | def capture 22 | pos = @erb_out.size 23 | yield 24 | @erb_out.slice!(pos..@erb_out.size) 25 | end 26 | 27 | # A single-value file is a file such as environments/ubuntu-18.04/image_tag. 28 | # It contains exactly 1 line of usable value, and may optionally contain 29 | # comments that start with '#', which are ignored. 30 | def read_single_value_file(path) 31 | contents = File.read(path, mode: 'r:utf-8') 32 | contents.split("\n").grep_v(/^#/).first.strip 33 | end 34 | 35 | # Puts an array's elements into at most N buckets. For example: 36 | # 37 | # bucketize([:a, :b, :c, :d, :e], 3) 38 | # 39 | # Outputs: 40 | # 41 | # [ 42 | # [:a, :d], 43 | # [:b, :e], 44 | # [:c], 45 | # ] 46 | def bucketize(ary, n) 47 | result = [] 48 | ary.each_with_index do |elem, i| 49 | bucket = (result[i % n] ||= []) 50 | bucket << elem 51 | end 52 | result 53 | end 54 | end 55 | -------------------------------------------------------------------------------- /lib/library.sh: -------------------------------------------------------------------------------- 1 | RESET=$'\033[0m' 2 | BOLD=$'\033[1m' 3 | # shellcheck disable=SC2034 4 | RED=$'\031[36m' 5 | YELLOW=$'\033[33m' 6 | # shellcheck disable=SC2034 7 | CYAN=$'\033[36m' 8 | BLUE_BG=$'\033[44m' 9 | 10 | if [[ "$VERBOSE" = "" ]]; then 11 | VERBOSE=false 12 | fi 13 | 14 | function header() 15 | { 16 | local title="$1" 17 | echo "${BLUE_BG}${YELLOW}${BOLD}${title}${RESET}" 18 | echo "------------------------------------------" 19 | } 20 | 21 | function run() 22 | { 23 | echo "+ $*" 24 | "$@" 25 | } 26 | 27 | function verbose_run() 28 | { 29 | if $VERBOSE; then 30 | echo "+ $*" 31 | fi 32 | "$@" 33 | } 34 | 35 | function verbose_exec() 36 | { 37 | if $VERBOSE; then 38 | echo "+ $*" 39 | fi 40 | exec "$@" 41 | } 42 | 43 | function run_with_retries() 44 | { 45 | local TRY_NUM=1 46 | local MAX_TRIES=3 47 | while true; do 48 | echo "+ (Try $TRY_NUM/$MAX_TRIES) $*" 49 | if "$@"; then 50 | return 0 51 | else 52 | (( TRY_NUM++ )) || true 53 | if [[ $TRY_NUM -gt $MAX_TRIES ]]; then 54 | return 1 55 | else 56 | echo "${BOLD}${YELLOW}*** WARNING: command failed, retrying...${RESET}" 57 | fi 58 | fi 59 | done 60 | } 61 | 62 | # A single-value file is a file such as environments/ubuntu-18.04/image_tag. 63 | # It contains exactly 1 line of usable value, and may optionally contain 64 | # comments that start with '#', which are ignored. 65 | function read_single_value_file() 66 | { 67 | grep -v '^#' "$1" | head -n 1 68 | } 69 | 70 | function require_args_exact() 71 | { 72 | local count="$1" 73 | shift 74 | if [[ $# -ne $count ]]; then 75 | echo "ERROR: $count arguments expected, but got $#." 76 | exit 1 77 | fi 78 | } 79 | 80 | function require_envvar() 81 | { 82 | local name="$1" 83 | local value=$(eval "echo \$$name") 84 | if [[ "$value" = "" ]]; then 85 | echo "ERROR: please pass the '$name' environment variable to this script." 86 | exit 1 87 | fi 88 | } 89 | 90 | function require_container_envvar() 91 | { 92 | local name="$1" 93 | local value=$(eval "echo \$$name") 94 | if [[ "$value" = "" ]]; then 95 | echo "ERROR: please pass the '$name' environment variable to the container." 96 | exit 1 97 | fi 98 | } 99 | 100 | function require_container_mount() 101 | { 102 | local path="$1" 103 | if [[ ! -e "$path" ]]; then 104 | echo "ERROR: please ensure $path is mounted in the container." 105 | exit 1 106 | fi 107 | } 108 | 109 | if command -v realpath &>/dev/null; then 110 | function absolute_path() 111 | { 112 | realpath -L "$1" 113 | } 114 | else 115 | function absolute_path() 116 | { 117 | local dir 118 | local name 119 | 120 | dir=$(dirname "$1") 121 | name=$(basename "$1") 122 | dir=$(cd "$dir" && pwd) 123 | echo "$dir/$name" 124 | } 125 | fi 126 | 127 | function list_environment_names() 128 | { 129 | ls -1 "$1" | grep -v '^utility$' | sort | xargs echo 130 | } 131 | 132 | function create_file_if_missing() 133 | { 134 | if [[ ! -e "$1" ]]; then 135 | run touch "$1" 136 | fi 137 | } 138 | 139 | function cleanup() 140 | { 141 | set +e 142 | local pids 143 | pids=$(jobs -p) 144 | if [[ "$pids" != "" ]]; then 145 | # shellcheck disable=SC2086 146 | kill $pids 2>/dev/null 147 | fi 148 | if [[ $(type -t _cleanup) == function ]]; then 149 | _cleanup 150 | fi 151 | } 152 | 153 | trap cleanup EXIT 154 | -------------------------------------------------------------------------------- /lib/publishing_support.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | module PublishingSupport 4 | private 5 | def pull_utility_image_if_not_exists 6 | # We need the utility image in order to invoke createrepo. 7 | # We pull it now so that a 'docker run' invocation later doesn't 8 | # need to pull. This makes the output of the whole script 9 | # easier to read. 10 | if !docker_image_exists?(utility_image_name) 11 | log_notice "Pulling Docker image #{utility_image_name}" 12 | run_command( 13 | 'docker', 'pull', utility_image_name, 14 | log_invocation: true, 15 | check_error: true, 16 | passthru_output: true 17 | ) 18 | end 19 | end 20 | 21 | def docker_image_exists?(image_name) 22 | _, stderr_output, status = run_command_capture_output( 23 | 'docker', 'inspect', image_name, 24 | log_invocation: false, 25 | check_error: false 26 | ) 27 | if status.success? 28 | true 29 | elsif stderr_output =~ /No such object/ 30 | false 31 | else 32 | abort("Command failed: docker inspect #{image_name}: #{stderr_output.chomp}") 33 | end 34 | end 35 | 36 | 37 | def fetch_signing_key 38 | key, _, _ = run_command_capture_output( 39 | 'az', 'keyvault', 'secret', 'show', 40 | '-o', 'tsv', 41 | '--query', 'value', 42 | '--vault-name', 'fsruby2infraowners', 43 | '--name', 'server-edition-gpg-private-key', 44 | log_invocation: true, 45 | check_error: true 46 | ) 47 | key 48 | end 49 | 50 | def infer_gpg_key_id(gpg_home, key_path) 51 | stdout_output, _, _ = run_command_capture_output( 52 | 'gpg', '--homedir', gpg_home, '--show-keys', key_path, 53 | log_invocation: true, 54 | check_error: true 55 | ) 56 | stdout_output.split("\n")[1].strip 57 | end 58 | 59 | def import_gpg_key(gpg_home, key_path) 60 | run_command( 61 | 'gpg', '--homedir', gpg_home, '--import', key_path, 62 | log_invocation: true, 63 | check_error: true 64 | ) 65 | end 66 | 67 | 68 | def utility_image_name 69 | "ghcr.io/fullstaq-ruby/server-edition-ci-images:utility-v#{utility_image_version}" 70 | end 71 | 72 | def utility_image_version 73 | read_single_value_file("#{GeneralSupport::ROOT}/environments/utility/image_tag") 74 | end 75 | 76 | 77 | def get_latest_production_repo_version 78 | if (version = getenv_integer('LATEST_PRODUCTION_REPO_VERSION')).nil? 79 | stdout_output, stderr_output, status = run_command_capture_output( 80 | 'gsutil', 'cp', latest_production_version_note_url, '-', 81 | log_invocation: false, 82 | check_error: false 83 | ) 84 | if status.success? 85 | version = stdout_output.strip 86 | if version =~ /\A[0-9]+\Z/ 87 | version = version.to_i 88 | else 89 | abort("ERROR: invalid version number stored in #{latest_production_version_note_url}") 90 | end 91 | elsif stderr_output =~ /No URLs matched/ 92 | version = 0 93 | else 94 | abort("ERROR: error fetching #{latest_production_version_note_url}: #{stderr_output.chomp}") 95 | end 96 | end 97 | 98 | log_notice "Latest state/repository version is: #{version}" 99 | version 100 | end 101 | 102 | 103 | def cache_control_policy 104 | if testing? 105 | 'no-store' 106 | else 107 | 'public' 108 | end 109 | end 110 | end 111 | -------------------------------------------------------------------------------- /lib/shell_scripting_support.rb: -------------------------------------------------------------------------------- 1 | # frozen_string_literal: true 2 | 3 | require 'open3' 4 | require 'shellwords' 5 | 6 | module ShellScriptingSupport 7 | private 8 | RESET = "\033[0m" 9 | BOLD = "\033[1m" 10 | BLUE_BG = "\033[44m" 11 | GREEN = "\033[32m" 12 | CYAN = "\033[36m" 13 | YELLOW = "\033[33m" 14 | 15 | 16 | def require_envvar(name) 17 | if ENV[name].to_s.empty? 18 | abort "ERROR: please pass the '#{name}' environment variable to this script." 19 | end 20 | end 21 | 22 | def require_envvar_enum(name, values) 23 | require_envvar(name) 24 | if !values.include?(ENV[name]) 25 | abort "ERROR: the '#{name}' environment variable must be set to one of: #{values.join(', ')}" 26 | end 27 | end 28 | 29 | def optional_envvar(name) 30 | # Does nothing. Only exists to signal intent. 31 | end 32 | 33 | def getenv_boolean(name) 34 | value = ENV[name].to_s.downcase 35 | ['true', 't', 'yes', 'y', '1', 'on'].include?(value) 36 | end 37 | 38 | def getenv_integer(name, default = nil) 39 | value = ENV[name].to_s 40 | if value =~ /\A[0-9]+\Z/ 41 | value.to_i 42 | else 43 | default 44 | end 45 | end 46 | 47 | 48 | def print_header(title) 49 | puts 50 | puts "#{BLUE_BG}#{YELLOW}#{BOLD}#{title}#{RESET}" 51 | puts '------------------------------------------' 52 | end 53 | 54 | def log_notice(message) 55 | puts " --> #{message}" 56 | end 57 | 58 | def log_info(message) 59 | puts " #{message}" 60 | end 61 | 62 | def abort(message) 63 | puts " #{message}" 64 | exit 1 65 | end 66 | 67 | 68 | def run_command(*command, log_invocation:, check_error:, passthru_output: false) 69 | log_info "Running: #{Shellwords.shelljoin(command)}" if log_invocation 70 | if passthru_output 71 | # Force system() to run without shell 72 | success = system([command[0], command[0]], *command[1..-1]) 73 | abort('ERROR: command failed') if check_error && !success 74 | else 75 | output, status = Open3.capture2e(*command) 76 | abort("ERROR: #{output.chomp}") if check_error && !status.success? 77 | end 78 | end 79 | 80 | def run_bash(script, *args, log_invocation:, check_error:, pipefail:, passthru_output: false) 81 | log_info "Running: #{script}" if log_invocation 82 | script = "set -o pipefail && #{script}" if pipefail 83 | if passthru_output 84 | success = system('bash', '-ec', script, 'bash', *args) 85 | abort('ERROR: command failed') if check_error && !success 86 | else 87 | output, status = Open3.capture2e('bash', '-ec', script, 'bash', *args) 88 | abort("ERROR: #{output.chomp}") if check_error && !status.success? 89 | end 90 | end 91 | 92 | def run_command_capture_output(*command, log_invocation:, check_error:) 93 | log_info "Running: #{Shellwords.shelljoin(command)}" if log_invocation 94 | stdout_output, stderr_output, status = Open3.capture3(*command) 95 | abort("ERROR: #{stderr_output.chomp}") if check_error && !status.success? 96 | [stdout_output, stderr_output, status] 97 | end 98 | 99 | 100 | class HardLinkError < StandardError 101 | attr_reader :source_path, :target_path, :cause 102 | 103 | def initialize(message, source_path, target_path, cause) 104 | super(message) 105 | @source_path = source_path 106 | @target_path = target_path 107 | @cause = cause 108 | end 109 | end 110 | 111 | # @return [Boolean] 112 | # @raise [HardLinkError, SystemCallError] 113 | def create_hardlink(source_path, target_path) 114 | File.unlink(target_path) if File.exist?(target_path) 115 | begin 116 | File.link(source_path, target_path) 117 | true 118 | rescue Errno::EXDEV, Errno::EPERM 119 | # These errors potentially indicate that hard linking is not supported. 120 | false 121 | rescue SystemCallError => e 122 | raise HardLinkError.new( 123 | "Error hard linking #{source_path} into #{target_path}: #{e}", 124 | source_path, 125 | target_path, 126 | e 127 | ) 128 | end 129 | end 130 | 131 | # @raise [HardLinkError, SystemCallError] 132 | def hardlink_or_copy_file(source_path, target_path) 133 | if !create_hardlink(source_path, target_path) 134 | FileUtils.cp(source_path, target_path, preserve: true) 135 | end 136 | end 137 | 138 | # @raise [HardLinkError, SystemCallError] 139 | def hardlink_or_copy_files(paths, target_dir) 140 | paths.each do |source_path| 141 | target_path = "#{target_dir}/#{File.basename(source_path)}" 142 | hardlink_or_copy_file(source_path, target_path) 143 | end 144 | end 145 | 146 | def delete_files(glob:) 147 | Dir[glob].each do |path| 148 | File.unlink(path) 149 | end 150 | end 151 | end 152 | -------------------------------------------------------------------------------- /push-environment-images: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | set -e 3 | 4 | SELFDIR=$(dirname "$0") 5 | SELFDIR=$(cd "$SELFDIR" && pwd) 6 | # shellcheck source=lib/library.sh 7 | source "$SELFDIR/lib/library.sh" 8 | 9 | cd "$SELFDIR/environments" 10 | for ENV in *; do 11 | IMAGE_VERSION=$(read_single_value_file "$ENV/image_tag") 12 | run docker push "ghcr.io/fullstaq-ruby/server-edition-ci-images:$ENV-v$IMAGE_VERSION" 13 | done 14 | -------------------------------------------------------------------------------- /resources/jemalloc_cxx_fix.patch: -------------------------------------------------------------------------------- 1 | diff --git a/include/jemalloc/jemalloc_macros.h.in b/include/jemalloc/jemalloc_macros.h.in 2 | index 13dbdd9..b8187e1 100644 3 | --- a/include/jemalloc/jemalloc_macros.h.in 4 | +++ b/include/jemalloc/jemalloc_macros.h.in 5 | @@ -8,6 +8,12 @@ 6 | #define JEMALLOC_VERSION_NREV @jemalloc_version_nrev@ 7 | #define JEMALLOC_VERSION_GID "@jemalloc_version_gid@" 8 | 9 | +#if defined(__cplusplus) 10 | +# define JEMALLOC_CXX_THROW throw() 11 | +#else 12 | +# define JEMALLOC_CXX_THROW 13 | +#endif 14 | + 15 | # define MALLOCX_LG_ALIGN(la) (la) 16 | # if LG_SIZEOF_PTR == 2 17 | # define MALLOCX_ALIGN(a) (ffs(a)-1) 18 | diff --git a/include/jemalloc/jemalloc_protos.h.in b/include/jemalloc/jemalloc_protos.h.in 19 | index 25446de..52f8aaa 100644 20 | --- a/include/jemalloc/jemalloc_protos.h.in 21 | +++ b/include/jemalloc/jemalloc_protos.h.in 22 | @@ -7,15 +7,15 @@ extern JEMALLOC_EXPORT const char *@je_@malloc_conf; 23 | extern JEMALLOC_EXPORT void (*@je_@malloc_message)(void *cbopaque, 24 | const char *s); 25 | 26 | -JEMALLOC_EXPORT void *@je_@malloc(size_t size) JEMALLOC_ATTR(malloc); 27 | +JEMALLOC_EXPORT void *@je_@malloc(size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc); 28 | JEMALLOC_EXPORT void *@je_@calloc(size_t num, size_t size) 29 | - JEMALLOC_ATTR(malloc); 30 | + JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc); 31 | JEMALLOC_EXPORT int @je_@posix_memalign(void **memptr, size_t alignment, 32 | - size_t size) JEMALLOC_ATTR(nonnull(1)); 33 | + size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(nonnull(1)); 34 | JEMALLOC_EXPORT void *@je_@aligned_alloc(size_t alignment, size_t size) 35 | - JEMALLOC_ATTR(malloc); 36 | -JEMALLOC_EXPORT void *@je_@realloc(void *ptr, size_t size); 37 | -JEMALLOC_EXPORT void @je_@free(void *ptr); 38 | + JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc); 39 | +JEMALLOC_EXPORT void *@je_@realloc(void *ptr, size_t size) JEMALLOC_CXX_THROW; 40 | +JEMALLOC_EXPORT void @je_@free(void *ptr) JEMALLOC_CXX_THROW; 41 | 42 | JEMALLOC_EXPORT void *@je_@mallocx(size_t size, int flags); 43 | JEMALLOC_EXPORT void *@je_@rallocx(void *ptr, size_t size, int flags); 44 | @@ -34,15 +34,15 @@ JEMALLOC_EXPORT int @je_@mallctlbymib(const size_t *mib, size_t miblen, 45 | JEMALLOC_EXPORT void @je_@malloc_stats_print(void (*write_cb)(void *, 46 | const char *), void *@je_@cbopaque, const char *opts); 47 | JEMALLOC_EXPORT size_t @je_@malloc_usable_size( 48 | - JEMALLOC_USABLE_SIZE_CONST void *ptr); 49 | + JEMALLOC_USABLE_SIZE_CONST void *ptr) JEMALLOC_CXX_THROW; 50 | 51 | #ifdef JEMALLOC_OVERRIDE_MEMALIGN 52 | JEMALLOC_EXPORT void * @je_@memalign(size_t alignment, size_t size) 53 | - JEMALLOC_ATTR(malloc); 54 | + JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc); 55 | #endif 56 | 57 | #ifdef JEMALLOC_OVERRIDE_VALLOC 58 | -JEMALLOC_EXPORT void * @je_@valloc(size_t size) JEMALLOC_ATTR(malloc); 59 | +JEMALLOC_EXPORT void * @je_@valloc(size_t size) JEMALLOC_CXX_THROW JEMALLOC_ATTR(malloc); 60 | #endif 61 | 62 | #ifdef JEMALLOC_EXPERIMENTAL 63 | -------------------------------------------------------------------------------- /resources/ruby_31_jemalloc.patch: -------------------------------------------------------------------------------- 1 | diff --git a/version.c b/version.c 2 | index 7d4478eeb6..3370b1778c 100644 3 | --- a/version.c 4 | +++ b/version.c 5 | @@ -41,9 +41,9 @@ const char ruby_revision[] = RUBY_FULL_REVISION; 6 | const char ruby_release_date[] = RUBY_RELEASE_DATE; 7 | const char ruby_platform[] = RUBY_PLATFORM; 8 | const int ruby_patchlevel = RUBY_PATCHLEVEL; 9 | -const char ruby_description[] = RUBY_DESCRIPTION_WITH(""); 10 | -static const char ruby_description_with_mjit[] = RUBY_DESCRIPTION_WITH(" +MJIT"); 11 | -static const char ruby_description_with_yjit[] = RUBY_DESCRIPTION_WITH(" +YJIT"); 12 | +const char ruby_description[] = RUBY_DESCRIPTION_WITH(" +jemalloc"); 13 | +static const char ruby_description_with_mjit[] = RUBY_DESCRIPTION_WITH(" +jemalloc +MJIT"); 14 | +static const char ruby_description_with_yjit[] = RUBY_DESCRIPTION_WITH(" +jemalloc +YJIT"); 15 | const char ruby_copyright[] = RUBY_COPYRIGHT; 16 | const char ruby_engine[] = "ruby"; 17 | 18 | -------------------------------------------------------------------------------- /resources/ruby_31_malloctrim.patch: -------------------------------------------------------------------------------- 1 | diff --git a/configure.ac b/configure.ac 2 | index d251da9915..5e2a67db1b 100644 3 | --- a/configure.ac 4 | +++ b/configure.ac 5 | @@ -1825,6 +1825,7 @@ AC_CHECK_FUNCS(lstat) 6 | AC_CHECK_FUNCS(lutimes) 7 | AC_CHECK_FUNCS(malloc_usable_size) 8 | AC_CHECK_FUNCS(malloc_size) 9 | +AC_CHECK_FUNCS(malloc_trim) 10 | AC_CHECK_FUNCS(mblen) 11 | AC_CHECK_FUNCS(memalign) 12 | AC_CHECK_FUNCS(memset_s) 13 | diff --git a/gc.c b/gc.c 14 | index 1331ef21dc..12caa162e7 100644 15 | --- a/gc.c 16 | +++ b/gc.c 17 | @@ -6660,7 +6660,15 @@ gc_start(rb_objspace_t *objspace, int reason) 18 | 19 | gc_prof_timer_start(objspace); 20 | { 21 | - gc_marks(objspace, do_full_mark); 22 | + gc_marks(objspace, do_full_mark); 23 | +#ifdef HAVE_MALLOC_TRIM 24 | + /* [Experimental] Explicitly free all eligible pages to the kernel. See: 25 | + * 26 | + * - https://www.joyfulbikeshedding.com/blog/2019-03-14-what-causes-ruby-memory-bloat.html 27 | + * - https://bugs.ruby-lang.org/issues/15667 28 | + */ 29 | + if (do_full_mark) malloc_trim(0); 30 | +#endif 31 | } 32 | gc_prof_timer_stop(objspace); 33 | 34 | 35 | diff --git a/version.c b/version.c 36 | index 7d4478eeb6..5f42c02b1a 100644 37 | --- a/version.c 38 | +++ b/version.c 39 | @@ -41,9 +41,15 @@ const char ruby_revision[] = RUBY_FULL_REVISION; 40 | const char ruby_release_date[] = RUBY_RELEASE_DATE; 41 | const char ruby_platform[] = RUBY_PLATFORM; 42 | const int ruby_patchlevel = RUBY_PATCHLEVEL; 43 | +#ifdef HAVE_MALLOC_TRIM 44 | +const char ruby_description[] = RUBY_DESCRIPTION_WITH(" +malloctrim"); 45 | +static const char ruby_description_with_mjit[] = RUBY_DESCRIPTION_WITH(" +malloctrim +MJIT"); 46 | +static const char ruby_description_with_yjit[] = RUBY_DESCRIPTION_WITH(" +malloctrim +YJIT"); 47 | +#else 48 | const char ruby_description[] = RUBY_DESCRIPTION_WITH(""); 49 | static const char ruby_description_with_mjit[] = RUBY_DESCRIPTION_WITH(" +MJIT"); 50 | static const char ruby_description_with_yjit[] = RUBY_DESCRIPTION_WITH(" +YJIT"); 51 | +#endif 52 | const char ruby_copyright[] = RUBY_COPYRIGHT; 53 | const char ruby_engine[] = "ruby"; 54 | 55 | -------------------------------------------------------------------------------- /resources/ruby_32_jemalloc.patch: -------------------------------------------------------------------------------- 1 | diff --git a/version.c b/version.c 2 | index 7d4478eeb6..3370b1778c 100644 3 | --- a/version.c 4 | +++ b/version.c 5 | @@ -41,9 +41,9 @@ const char ruby_revision[] = RUBY_FULL_REVISION; 6 | const char ruby_release_date[] = RUBY_RELEASE_DATE; 7 | const char ruby_platform[] = RUBY_PLATFORM; 8 | const int ruby_patchlevel = RUBY_PATCHLEVEL; 9 | -const char ruby_description[] = RUBY_DESCRIPTION_WITH(""); 10 | -static const char ruby_description_with_mjit[] = RUBY_DESCRIPTION_WITH(" +MJIT"); 11 | -static const char ruby_description_with_yjit[] = RUBY_DESCRIPTION_WITH(YJIT_DESCRIPTION); 12 | +const char ruby_description[] = RUBY_DESCRIPTION_WITH(" +jemalloc"); 13 | +static const char ruby_description_with_mjit[] = RUBY_DESCRIPTION_WITH(" +jemalloc +MJIT"); 14 | +static const char ruby_description_with_yjit[] = RUBY_DESCRIPTION_WITH(" +jemalloc " YJIT_DESCRIPTION); 15 | const char ruby_copyright[] = "ruby - Copyright (C) " 16 | RUBY_BIRTH_YEAR_STR "-" RUBY_RELEASE_YEAR_STR " " 17 | RUBY_AUTHOR; 18 | const char ruby_engine[] = "ruby"; 19 | 20 | -------------------------------------------------------------------------------- /resources/ruby_32_malloctrim.patch: -------------------------------------------------------------------------------- 1 | diff --git a/configure.ac b/configure.ac 2 | index d251da9915..5e2a67db1b 100644 3 | --- a/configure.ac 4 | +++ b/configure.ac 5 | @@ -1825,6 +1825,7 @@ AC_CHECK_FUNCS(lstat) 6 | AC_CHECK_FUNCS(lutimes) 7 | AC_CHECK_FUNCS(malloc_usable_size) 8 | AC_CHECK_FUNCS(malloc_size) 9 | +AC_CHECK_FUNCS(malloc_trim) 10 | AC_CHECK_FUNCS(mblen) 11 | AC_CHECK_FUNCS(memalign) 12 | AC_CHECK_FUNCS(memset_s) 13 | diff --git a/gc.c b/gc.c 14 | index 0fccc17..ec46289 100644 15 | --- a/gc.c 16 | +++ b/gc.c 17 | @@ -9545,6 +9545,14 @@ gc_start(rb_objspace_t *objspace, unsigned int reason) 18 | gc_prof_timer_start(objspace); 19 | { 20 | gc_marks(objspace, do_full_mark); 21 | +#ifdef HAVE_MALLOC_TRIM 22 | + /* [Experimental] Explicitly free all eligible pages to the kernel. See: 23 | + * 24 | + * - https://www.joyfulbikeshedding.com/blog/2019-03-14-what-causes-ruby-memory-bloat.html 25 | + * - https://bugs.ruby-lang.org/issues/15667 26 | + */ 27 | + if (do_full_mark) malloc_trim(0); 28 | +#endif 29 | } 30 | gc_prof_timer_stop(objspace); 31 | 32 | diff --git a/version-old.c b/version.c 33 | index 653a423..92ed984 100644 34 | --- a/version-old.c 35 | +++ b/version.c 36 | @@ -70,9 +70,15 @@ const char ruby_revision[] = RUBY_FULL_REVISION; 37 | const char ruby_release_date[] = RUBY_RELEASE_DATE; 38 | const char ruby_platform[] = RUBY_PLATFORM; 39 | const int ruby_patchlevel = RUBY_PATCHLEVEL; 40 | +#ifdef HAVE_MALLOC_TRIM 41 | +const char ruby_description[] = RUBY_DESCRIPTION_WITH(" +malloctrim"); 42 | +static const char ruby_description_with_mjit[] = RUBY_DESCRIPTION_WITH(" +malloctrim +MJIT"); 43 | +static const char ruby_description_with_yjit[] = RUBY_DESCRIPTION_WITH(" +malloctrim " YJIT_DESCRIPTION); 44 | +#else 45 | const char ruby_description[] = RUBY_DESCRIPTION_WITH(""); 46 | static const char ruby_description_with_mjit[] = RUBY_DESCRIPTION_WITH(" +MJIT"); 47 | static const char ruby_description_with_yjit[] = RUBY_DESCRIPTION_WITH(YJIT_DESCRIPTION); 48 | +#endif 49 | const char ruby_copyright[] = "ruby - Copyright (C) " 50 | RUBY_BIRTH_YEAR_STR "-" RUBY_RELEASE_YEAR_STR " " 51 | RUBY_AUTHOR; 52 | -------------------------------------------------------------------------------- /resources/ruby_33_jemalloc.patch: -------------------------------------------------------------------------------- 1 | diff --git a/version.c b/version.c 2 | index e719a59..05544ad 100644 3 | --- a/version.c 4 | +++ b/version.c 5 | @@ -67,7 +67,7 @@ const char ruby_platform[] = RUBY_PLATFORM; 6 | const int ruby_patchlevel = RUBY_PATCHLEVEL; 7 | const char ruby_description[] = 8 | "ruby " RUBY_VERSION RUBY_PATCHLEVEL_STR " " 9 | - "(" RUBY_RELEASE_DATETIME RUBY_REVISION_STR ") " 10 | + "(" RUBY_RELEASE_DATETIME RUBY_REVISION_STR ") +jemalloc " 11 | "[" RUBY_PLATFORM "]"; 12 | static const int ruby_description_opt_point = 13 | (int)(sizeof(ruby_description) - sizeof(" [" RUBY_PLATFORM "]")); 14 | -------------------------------------------------------------------------------- /resources/test-env/Gemfile: -------------------------------------------------------------------------------- 1 | # This Gemfile will be installed into the test container 2 | # when running a test suite. 3 | source 'https://rubygems.org' 4 | 5 | # Must allow for the minimum Ruby version that we support in config.yml. 6 | ruby '>= 3.1' 7 | 8 | # Test engine to use 9 | gem 'rspec' 10 | 11 | # Tests whether installing native extensions works 12 | gem 'bcrypt' 13 | -------------------------------------------------------------------------------- /resources/test-env/Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | bcrypt (3.1.19) 5 | diff-lcs (1.5.0) 6 | rspec (3.12.0) 7 | rspec-core (~> 3.12.0) 8 | rspec-expectations (~> 3.12.0) 9 | rspec-mocks (~> 3.12.0) 10 | rspec-core (3.12.2) 11 | rspec-support (~> 3.12.0) 12 | rspec-expectations (3.12.3) 13 | diff-lcs (>= 1.2.0, < 2.0) 14 | rspec-support (~> 3.12.0) 15 | rspec-mocks (3.12.5) 16 | diff-lcs (>= 1.2.0, < 2.0) 17 | rspec-support (~> 3.12.0) 18 | rspec-support (3.12.1) 19 | 20 | PLATFORMS 21 | arm-darwin-21 22 | ruby 23 | x86_64-darwin-21 24 | x86_64-linux 25 | 26 | DEPENDENCIES 27 | bcrypt 28 | rspec 29 | 30 | RUBY VERSION 31 | ruby 3.2.1p31 32 | 33 | BUNDLED WITH 34 | 2.4.15 35 | -------------------------------------------------------------------------------- /resources/test-env/jemalloc-cxx.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | using namespace std; 7 | 8 | int 9 | main() { 10 | void *data; 11 | int ret = posix_memalign(&data, 1024, 32); 12 | if (ret == 0) { 13 | // Perform an arbitrary I/O operation to 14 | // ensure that the compiler doesn't optimize 15 | // away the posix_memalign() call. 16 | FILE *f; 17 | 18 | f = fopen("/dev/zero", "w"); 19 | if (f != NULL) { 20 | fread(data, 1, 1, f); 21 | fclose(f); 22 | } 23 | 24 | f = fopen("/dev/null", "w"); 25 | if (f != NULL) { 26 | fwrite(data, 1, 1, f); 27 | fclose(f); 28 | } 29 | } else { 30 | perror("posix_memalign() failed"); 31 | } 32 | return 0; 33 | } 34 | --------------------------------------------------------------------------------