├── .github ├── scripts │ ├── auto-backport.py │ └── search_commits.py └── workflows │ ├── add-label-when-promoted.yaml │ ├── build.yml │ └── trigger_jenkins.yaml ├── .gitignore ├── LICENSE ├── README.md ├── SCYLLA-VERSION-GEN ├── aws └── cloudformation │ └── scylla.yaml.j2 ├── common ├── .bash_profile ├── aws_io_params.yaml ├── scylla-image-post-start.service ├── scylla-image-setup.service ├── scylla_cloud_io_setup ├── scylla_configure.py ├── scylla_create_devices ├── scylla_ec2_check ├── scylla_image_setup ├── scylla_login └── scylla_post_start.py ├── dist ├── debian │ ├── build_deb.sh │ ├── changelog.template │ ├── control.template │ ├── debian │ │ ├── compat │ │ ├── copyright │ │ ├── rules │ │ ├── scylla-image-post-start.service │ │ ├── scylla-image-setup.service │ │ ├── scylla-machine-image.install │ │ ├── scylla-machine-image.postinst │ │ ├── scylla-machine-image.postrm │ │ └── source │ │ │ ├── format │ │ │ └── options │ └── debian_files_gen.py └── redhat │ ├── build_rpm.sh │ └── scylla-machine-image.spec ├── lib ├── __init__.py ├── log.py ├── scylla_cloud.py └── user_data.py ├── packer ├── ami_variables.json ├── apply_cis_rules ├── azure_variables.json ├── build_image.sh ├── files │ └── .gitkeep ├── gce_variables.json ├── scylla.json ├── scylla_install_image └── user_data.txt ├── renovate.json ├── test-requirements.txt ├── tests ├── test_aws_instance.py ├── test_azure_instance.py ├── test_gcp_instance.py ├── test_scylla_configure.py ├── test_scylla_post_start.py └── tests-data │ └── scylla.yaml └── tools └── relocate_python_scripts.py /.github/scripts/auto-backport.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import argparse 4 | import os 5 | import re 6 | import sys 7 | import tempfile 8 | import logging 9 | 10 | from github import Github, GithubException 11 | from git import Repo, GitCommandError 12 | 13 | logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') 14 | try: 15 | github_token = os.environ["GITHUB_TOKEN"] 16 | except KeyError: 17 | print("Please set the 'GITHUB_TOKEN' environment variable") 18 | sys.exit(1) 19 | 20 | 21 | def is_pull_request(): 22 | return '--pull-request' in sys.argv[1:] 23 | 24 | 25 | def parse_args(): 26 | parser = argparse.ArgumentParser() 27 | parser.add_argument('--repo', type=str, required=True, help='Github repository name') 28 | parser.add_argument('--base-branch', type=str, default='refs/heads/next', help='Base branch') 29 | parser.add_argument('--commits', default=None, type=str, help='Range of promoted commits.') 30 | parser.add_argument('--pull-request', type=int, help='Pull request number to be backported') 31 | parser.add_argument('--head-commit', type=str, required=is_pull_request(), help='The HEAD of target branch after the pull request specified by --pull-request is merged') 32 | parser.add_argument('--label', type=str, required=is_pull_request(), help='Backport label name when --pull-request is defined') 33 | return parser.parse_args() 34 | 35 | 36 | def create_pull_request(repo, new_branch_name, base_branch_name, pr, backport_pr_title, commits, is_draft=False): 37 | pr_body = f'{pr.body}\n\n' 38 | for commit in commits: 39 | pr_body += f'- (cherry picked from commit {commit})\n\n' 40 | pr_body += f'Parent PR: #{pr.number}' 41 | try: 42 | backport_pr = repo.create_pull( 43 | title=backport_pr_title, 44 | body=pr_body, 45 | head=f'scylladbbot:{new_branch_name}', 46 | base=base_branch_name, 47 | draft=is_draft 48 | ) 49 | logging.info(f"Pull request created: {backport_pr.html_url}") 50 | backport_pr.add_to_assignees(pr.user) 51 | if is_draft: 52 | backport_pr.add_to_labels("conflicts") 53 | pr_comment = f"@{pr.user.login} - This PR has conflicts, therefore it was moved to `draft` \n" 54 | pr_comment += "Please resolve them and mark this PR as ready for review" 55 | backport_pr.create_issue_comment(pr_comment) 56 | logging.info(f"Assigned PR to original author: {pr.user}") 57 | return backport_pr 58 | except GithubException as e: 59 | if 'A pull request already exists' in str(e): 60 | logging.warning(f'A pull request already exists for {pr.user}:{new_branch_name}') 61 | else: 62 | logging.error(f'Failed to create PR: {e}') 63 | 64 | 65 | def get_pr_commits(repo, pr, stable_branch, start_commit=None): 66 | commits = [] 67 | if pr.merged: 68 | merge_commit = repo.get_commit(pr.merge_commit_sha) 69 | if len(merge_commit.parents) > 1: # Check if this merge commit includes multiple commits 70 | commits.append(pr.merge_commit_sha) 71 | else: 72 | if start_commit: 73 | promoted_commits = repo.compare(start_commit, stable_branch).commits 74 | else: 75 | promoted_commits = repo.get_commits(sha=stable_branch) 76 | for commit in pr.get_commits(): 77 | for promoted_commit in promoted_commits: 78 | commit_title = commit.commit.message.splitlines()[0] 79 | # In Scylla-pkg and scylla-dtest, for example, 80 | # we don't create a merge commit for a PR with multiple commits, 81 | # according to the GitHub API, the last commit will be the merge commit, 82 | # which is not what we need when backporting (we need all the commits). 83 | # So here, we are validating the correct SHA for each commit so we can cherry-pick 84 | if promoted_commit.commit.message.startswith(commit_title): 85 | commits.append(promoted_commit.sha) 86 | 87 | elif pr.state == 'closed': 88 | events = pr.get_issue_events() 89 | for event in events: 90 | if event.event == 'closed': 91 | commits.append(event.commit_id) 92 | return commits 93 | 94 | 95 | def backport(repo, pr, version, commits, backport_base_branch): 96 | new_branch_name = f'backport/{pr.number}/to-{version}' 97 | backport_pr_title = f'[Backport {version}] {pr.title}' 98 | repo_url = f'https://scylladbbot:{github_token}@github.com/{repo.full_name}.git' 99 | fork_repo = f'https://scylladbbot:{github_token}@github.com/scylladbbot/{repo.name}.git' 100 | with (tempfile.TemporaryDirectory() as local_repo_path): 101 | try: 102 | repo_local = Repo.clone_from(repo_url, local_repo_path, branch=backport_base_branch) 103 | repo_local.git.checkout(b=new_branch_name) 104 | is_draft = False 105 | for commit in commits: 106 | try: 107 | repo_local.git.cherry_pick(commit, '-m1', '-x') 108 | except GitCommandError as e: 109 | logging.warning(f'Cherry-pick conflict on commit {commit}: {e}') 110 | is_draft = True 111 | repo_local.git.add(A=True) 112 | repo_local.git.cherry_pick('--continue') 113 | repo_local.git.push(fork_repo, new_branch_name, force=True) 114 | create_pull_request(repo, new_branch_name, backport_base_branch, pr, backport_pr_title, commits, 115 | is_draft=is_draft) 116 | except GitCommandError as e: 117 | logging.warning(f"GitCommandError: {e}") 118 | 119 | 120 | def main(): 121 | args = parse_args() 122 | base_branch = args.base_branch.split('/')[2] 123 | promoted_label = 'promoted-to-master' 124 | repo_name = args.repo 125 | if 'scylla-enterprise-machine-image' in args.repo: 126 | promoted_label = 'promoted-to-enterprise' 127 | 128 | backport_branch = 'next-' 129 | stable_branch = 'master' if base_branch == 'next' else 'enterprise' if base_branch == 'next-enterprise' else base_branch.replace('next', 'branch') 130 | backport_label_pattern = re.compile(r'backport/\d+\.\d+$') 131 | 132 | g = Github(github_token) 133 | repo = g.get_repo(repo_name) 134 | closed_prs = [] 135 | start_commit = None 136 | 137 | if args.commits: 138 | start_commit, end_commit = args.commits.split('..') 139 | commits = repo.compare(start_commit, end_commit).commits 140 | for commit in commits: 141 | for pr in commit.get_pulls(): 142 | closed_prs.append(pr) 143 | if args.pull_request: 144 | start_commit = args.head_commit 145 | pr = repo.get_pull(args.pull_request) 146 | closed_prs = [pr] 147 | 148 | for pr in closed_prs: 149 | labels = [label.name for label in pr.labels] 150 | if args.pull_request: 151 | backport_labels = [args.label] 152 | else: 153 | backport_labels = [label for label in labels if backport_label_pattern.match(label)] 154 | if promoted_label not in labels: 155 | print(f'no {promoted_label} label: {pr.number}') 156 | continue 157 | if not backport_labels: 158 | print(f'no backport label: {pr.number}') 159 | continue 160 | commits = get_pr_commits(repo, pr, stable_branch, start_commit) 161 | logging.info(f"Found PR #{pr.number} with commit {commits} and the following labels: {backport_labels}") 162 | for backport_label in backport_labels: 163 | version = backport_label.replace('backport/', '') 164 | backport_base_branch = backport_label.replace('backport/', backport_branch) 165 | backport(repo, pr, version, commits, backport_base_branch) 166 | 167 | 168 | if __name__ == "__main__": 169 | main() 170 | -------------------------------------------------------------------------------- /.github/scripts/search_commits.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | 3 | import re 4 | import requests 5 | from github import Github 6 | import argparse 7 | import sys 8 | import os 9 | 10 | try: 11 | github_token = os.environ["GITHUB_TOKEN"] 12 | except KeyError: 13 | print("Please set the 'GITHUB_TOKEN' environment variable") 14 | sys.exit(1) 15 | 16 | 17 | def parser(): 18 | parser = argparse.ArgumentParser() 19 | parser.add_argument('--repository', type=str, default='scylladb/scylla-pkg', help='Github repository name') 20 | parser.add_argument('--commits', type=str, required=True, help='Range of promoted commits.') 21 | parser.add_argument('--label', type=str, default='promoted-to-master', help='Label to use') 22 | parser.add_argument('--ref', type=str, required=True, help='PR target branch') 23 | return parser.parse_args() 24 | 25 | 26 | def main(): 27 | args = parser() 28 | g = Github(github_token) 29 | repo = g.get_repo(args.repository, lazy=False) 30 | start_commit, end_commit = args.commits.split('..') 31 | commits = repo.compare(start_commit, end_commit).commits 32 | processed_prs = set() 33 | for commit in commits: 34 | search_url = f'https://api.github.com/search/issues' 35 | query = f"repo:{args.repository} is:pr is:merged sha:{commit.sha}" 36 | params = { 37 | "q": query, 38 | } 39 | headers = { 40 | "Authorization": f"token {github_token}", 41 | "Accept": "application/vnd.github.v3+json" 42 | } 43 | response = requests.get(search_url, headers=headers, params=params) 44 | prs = response.json().get("items", []) 45 | for pr in prs: 46 | match = re.findall(r'Parent PR: #(\d+)', pr["body"]) 47 | if match: 48 | pr_number = int(match[0]) 49 | if pr_number in processed_prs: 50 | continue 51 | ref = re.search(r'-(\d+\.\d+)', args.ref) 52 | label_to_add = f'backport/{ref.group(1)}-done' 53 | label_to_remove = f'backport/{ref.group(1)}' 54 | remove_label_url = f'https://api.github.com/repos/{args.repository}/issues/{pr_number}/labels/{label_to_remove}' 55 | del_data = { 56 | "labels": [f'{label_to_remove}'] 57 | } 58 | response = requests.delete(remove_label_url, headers=headers, json=del_data) 59 | if response.ok: 60 | print(f'Label {label_to_remove} removed successfully') 61 | else: 62 | print(f'Label {label_to_remove} cant be removed') 63 | else: 64 | pr_number = pr["number"] 65 | label_to_add = args.label 66 | data = { 67 | "labels": [f'{label_to_add}'] 68 | } 69 | add_label_url = f'https://api.github.com/repos/{args.repository}/issues/{pr_number}/labels' 70 | response = requests.post(add_label_url, headers=headers, json=data) 71 | if response.ok: 72 | print(f"Label added successfully to {add_label_url}") 73 | else: 74 | print(f"No label was added to {add_label_url}") 75 | processed_prs.add(pr_number) 76 | 77 | 78 | if __name__ == "__main__": 79 | main() 80 | -------------------------------------------------------------------------------- /.github/workflows/add-label-when-promoted.yaml: -------------------------------------------------------------------------------- 1 | name: Check if commits are promoted 2 | 3 | on: 4 | push: 5 | branches: 6 | - master 7 | - next-*.* 8 | pull_request_target: 9 | types: [labeled] 10 | branches: [master, next] 11 | 12 | env: 13 | DEFAULT_BRANCH: 'master' 14 | 15 | jobs: 16 | check-commit: 17 | runs-on: ubuntu-latest 18 | permissions: 19 | pull-requests: write 20 | issues: write 21 | steps: 22 | - name: Dump GitHub context 23 | env: 24 | GITHUB_CONTEXT: ${{ toJson(github) }} 25 | run: echo "$GITHUB_CONTEXT" 26 | - name: Checkout repository 27 | uses: actions/checkout@v4 28 | with: 29 | repository: ${{ github.repository }} 30 | ref: ${{ env.DEFAULT_BRANCH }} 31 | token: ${{ secrets.AUTO_BACKPORT_TOKEN }} 32 | fetch-depth: 0 # Fetch all history for all tags and branches 33 | - name: Set up Git identity 34 | run: | 35 | git config --global user.name "GitHub Action" 36 | git config --global user.email "action@github.com" 37 | git config --global merge.conflictstyle diff3 38 | - name: Install dependencies 39 | run: sudo apt-get install -y python3-github python3-git 40 | - name: Run python script 41 | if: github.event_name == 'push' 42 | env: 43 | GITHUB_TOKEN: ${{ secrets.AUTO_BACKPORT_TOKEN }} 44 | run: python .github/scripts/search_commits.py --commits ${{ github.event.before }}..${{ github.sha }} --repository ${{ github.repository }} --ref ${{ github.ref }} 45 | - name: Run auto-backport.py when promotion completed 46 | if: github.event_name == 'push' && github.ref == format('refs/heads/{0}', env.DEFAULT_BRANCH) 47 | env: 48 | GITHUB_TOKEN: ${{ secrets.AUTO_BACKPORT_TOKEN }} 49 | run: python .github/scripts/auto-backport.py --repo ${{ github.repository }} --base-branch ${{ github.ref }} --commits ${{ github.event.before }}..${{ github.sha }} 50 | - name: Check if label starts with 'backport/' and contains digits 51 | id: check_label 52 | run: | 53 | label_name="${{ github.event.label.name }}" 54 | if [[ "$label_name" =~ ^backport/[0-9]+\.[0-9]+$ ]]; then 55 | echo "Label matches backport/X.X pattern." 56 | echo "backport_label=true" >> $GITHUB_OUTPUT 57 | else 58 | echo "Label does not match the required pattern." 59 | echo "backport_label=false" >> $GITHUB_OUTPUT 60 | fi 61 | - name: Run auto-backport.py when label was added 62 | if: github.event_name == 'pull_request_target' && steps.check_label.outputs.backport_label == 'true' && github.event.pull_request.merged == true 63 | env: 64 | GITHUB_TOKEN: ${{ secrets.AUTO_BACKPORT_TOKEN }} 65 | run: python .github/scripts/auto-backport.py --repo ${{ github.repository }} --base-branch ${{ github.ref }} --pull-request ${{ github.event.pull_request.number }} --head-commit ${{ github.event.pull_request.base.sha }} --label ${{ github.event.label.name }} 66 | -------------------------------------------------------------------------------- /.github/workflows/build.yml: -------------------------------------------------------------------------------- 1 | on: 2 | push: 3 | branches: 4 | - master 5 | - next 6 | 7 | pull_request: 8 | branches: 9 | - next 10 | 11 | jobs: 12 | build: 13 | runs-on: ubuntu-latest 14 | name: Unittest and Build RPMs 15 | steps: 16 | - uses: actions/checkout@v4 17 | - name: Setup python 18 | uses: actions/setup-python@v5 19 | with: 20 | python-version: 3.11 21 | architecture: x64 22 | 23 | - name: unittest 24 | 25 | run: | 26 | pip install -r test-requirements.txt 27 | pytest ./tests 28 | 29 | - name: Build RPM (Rockylinux:8) 30 | run: docker run -v `pwd`:/scylla-machine-image -w /scylla-machine-image --rm rockylinux:8 bash -c 'dnf update -y; dnf install -y git ; git config --global --add safe.directory "*"; ./dist/redhat/build_rpm.sh -t centos8' 31 | 32 | - name: Build DEB (Ubuntu:22.04) 33 | run: docker run -v `pwd`:/scylla-machine-image -w /scylla-machine-image --rm ubuntu:22.04 bash -c 'apt update -y; apt install -y git ; git config --global --add safe.directory "*"; ./dist/debian/build_deb.sh' 34 | -------------------------------------------------------------------------------- /.github/workflows/trigger_jenkins.yaml: -------------------------------------------------------------------------------- 1 | name: Trigger next-machine-image gating 2 | 3 | on: 4 | push: 5 | branches: 6 | - next** 7 | 8 | jobs: 9 | trigger-jenkins: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - name: Determine Jenkins Job Name 13 | run: | 14 | if [[ "${{ github.ref_name }}" == "next" ]]; then 15 | FOLDER_NAME="scylla-master" 16 | elif [[ "${{ github.ref_name }}" == "next-enterprise" ]]; then 17 | FOLDER_NAME="scylla-enterprise" 18 | else 19 | VERSION=$(echo "${{ github.ref_name }}" | awk -F'-' '{print $2}') 20 | if [[ "$VERSION" =~ ^202[0-4]\.[0-9]+$ ]]; then 21 | FOLDER_NAME="enterprise-$VERSION" 22 | elif [[ "$VERSION" =~ ^[0-9]+\.[0-9]+$ ]]; then 23 | FOLDER_NAME="scylla-$VERSION" 24 | fi 25 | fi 26 | echo "JOB_NAME=${FOLDER_NAME}/job/next-machine-image" >> $GITHUB_ENV 27 | 28 | - name: Trigger Jenkins Job 29 | env: 30 | JENKINS_USER: ${{ secrets.JENKINS_USERNAME }} 31 | JENKINS_API_TOKEN: ${{ secrets.JENKINS_TOKEN }} 32 | JENKINS_URL: "https://jenkins.scylladb.com" 33 | SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} 34 | run: | 35 | echo "Triggering Jenkins Job: $JOB_NAME" 36 | if ! curl -X POST "$JENKINS_URL/job/$JOB_NAME/buildWithParameters" --fail --user "$JENKINS_USER:$JENKINS_API_TOKEN" -i -v; then 37 | echo "Error: Jenkins job trigger failed" 38 | 39 | # Send Slack message 40 | curl -X POST -H 'Content-type: application/json' \ 41 | -H "Authorization: Bearer $SLACK_BOT_TOKEN" \ 42 | --data '{ 43 | "channel": "#releng-team", 44 | "text": "🚨 @here '$JOB_NAME' failed to be triggered, please check https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} for more details", 45 | "icon_emoji": ":warning:" 46 | }' \ 47 | https://slack.com/api/chat.postMessage 48 | 49 | exit 1 50 | fi 51 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build 2 | __pycache__ 3 | .venv 4 | 5 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Scylla Machine Image 2 | Provides following 3 | - Create an image with pre-installed Scylla 4 | - Allow to configure the database when an instance is launched first time 5 | - Easy cluster creation 6 | 7 | ## OS Package 8 | RPM/DEB package that is pre-installed in the image. 9 | Responsible for configuring Scylla during first boot of the instance. 10 | 11 | ## Create an image 12 | ### AWS 13 | ```shell script 14 | aws/ami/build_ami.sh 15 | ``` 16 | 17 | ## Scylla AMI user-data Format v2 18 | 19 | Scylla AMI user-data should be passed as a json object, as described below 20 | 21 | see AWS docs for how to pass user-data into ec2 instances: 22 | [https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-add-user-data.html](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/instancedata-add-user-data.html) 23 | --- 24 | ### EC2 User-Data 25 | User Data that can pass when create EC2 instances 26 | 27 | * **Object Properties** 28 | * **scylla_yaml** ([`Scylla YAML`](#scylla_yaml)) – Mapping of all fields that would pass down to scylla.yaml configuration file 29 | * **scylla_startup_args** (*list*) – embedded information about the user that created the issue (NOT YET IMPLEMENTED) (*default=’[]’*) 30 | * **developer_mode** ([*boolean*](https://docs.python.org/library/stdtypes.html#boolean-values)) – Enables developer mode (*default=’false’*) 31 | * **post_configuration_script** ([*string*](https://docs.python.org/library/stdtypes.html#str)) – A script to run once AMI first configuration is finished, can be a string encoded in base64. (*default=’’*) 32 | * **post_configuration_script_timeout** ([*int*](https://docs.python.org/library/stdtypes.html#int)) – Time in seconds to limit the post_configuration_script (*default=’600’*) 33 | * **start_scylla_on_first_boot** ([*boolean*](https://docs.python.org/library/stdtypes.html#boolean-values)) – If true, scylla-server would boot at AMI boot (*default=’true’*) 34 | 35 | ### Scylla YAML 36 | All fields that would pass down to scylla.yaml configuration file 37 | 38 | see [https://docs.scylladb.com/operating-scylla/scylla-yaml/](https://docs.scylladb.com/operating-scylla/scylla-yaml/) for all the possible configuration availble 39 | listed here only the one get defaults scylla AMI 40 | 41 | * **Object Properties** 42 | * **cluster_name** ([*string*](https://docs.python.org/library/stdtypes.html#str)) – Name of the cluster (*default=`generated name that would work for only one node cluster`*) 43 | * **auto_bootstrap** ([*boolean*](https://docs.python.org/library/stdtypes.html#boolean-values)) – Enable auto bootstrap (*default=’true’*) 44 | * **listen_address** ([*string*](https://docs.python.org/library/stdtypes.html#str)) – Defaults to ec2 instance private ip 45 | * **broadcast_rpc_address** ([*string*](https://docs.python.org/library/stdtypes.html#str)) – Defaults to ec2 instance private ip 46 | * **endpoint_snitch** ([*string*](https://docs.python.org/library/stdtypes.html#str)) – Defaults to ‘org.apache.cassandra.locator.Ec2Snitch’ 47 | * **rpc_address** ([*string*](https://docs.python.org/library/stdtypes.html#str)) – Defaults to ‘0.0.0.0’ 48 | * **seed_provider** (*mapping*) – Defaults to ec2 instance private ip 49 | 50 | ### Example usage of user-data 51 | 52 | Spinning a new node connecting to “10.0.219.209” as a seed, and installing cloud-init-cfn package at first boot. 53 | 54 | #### using json 55 | ```json 56 | { 57 | "scylla_yaml": { 58 | "cluster_name": "test-cluster", 59 | "experimental": true, 60 | "seed_provider": [{"class_name": "org.apache.cassandra.locator.SimpleSeedProvider", 61 | "parameters": [{"seeds": "10.0.219.209"}]}], 62 | }, 63 | "post_configuration_script": "#! /bin/bash\nyum install cloud-init-cfn", 64 | "start_scylla_on_first_boot": true 65 | } 66 | ``` 67 | 68 | #### using yaml 69 | ```yaml 70 | scylla_yaml: 71 | cluster_name: test-cluster 72 | experimental: true 73 | seed_provider: 74 | - class_name: org.apache.cassandra.locator.SimpleSeedProvider 75 | parameters: 76 | - seeds: 10.0.219.209 77 | post_configuration_script: "#! /bin/bash\nyum install cloud-init-cfn" 78 | start_scylla_on_first_boot: true 79 | ``` 80 | 81 | #### using mimemultipart 82 | 83 | If other feature of cloud-init are needed, one can use mimemultipart, and pass 84 | a json/yaml with `x-scylla/yaml` or `x-scylla/json` 85 | 86 | more information on cloud-init multipart user-data: 87 | 88 | https://cloudinit.readthedocs.io/en/latest/topics/format.html#mime-multi-part-archive 89 | 90 | ```mime 91 | Content-Type: multipart/mixed; boundary="===============5438789820677534874==" 92 | MIME-Version: 1.0 93 | 94 | --===============5438789820677534874== 95 | Content-Type: x-scylla/yaml 96 | MIME-Version: 1.0 97 | Content-Disposition: attachment; filename="scylla_machine_image.yaml" 98 | 99 | scylla_yaml: 100 | cluster_name: test-cluster 101 | experimental: true 102 | seed_provider: 103 | - class_name: org.apache.cassandra.locator.SimpleSeedProvider 104 | parameters: 105 | - seeds: 10.0.219.209 106 | post_configuration_script: "#! /bin/bash\nyum install cloud-init-cfn" 107 | start_scylla_on_first_boot: true 108 | 109 | --===============5438789820677534874== 110 | Content-Type: text/cloud-config; charset="us-ascii" 111 | MIME-Version: 1.0 112 | Content-Transfer-Encoding: 7bit 113 | Content-Disposition: attachment; filename="cloud-config.txt" 114 | 115 | #cloud-config 116 | cloud_final_modules: 117 | - [scripts-user, always] 118 | 119 | --===============5438789820677534874==-- 120 | ``` 121 | 122 | example of creating the multipart message by python code: 123 | 124 | ```python 125 | import json 126 | from email.mime.base import MIMEBase 127 | from email.mime.multipart import MIMEMultipart 128 | 129 | msg = MIMEMultipart() 130 | 131 | scylla_image_configuration = dict( 132 | scylla_yaml=dict( 133 | cluster_name="test_cluster", 134 | listen_address="10.23.20.1", 135 | broadcast_rpc_address="10.23.20.1", 136 | seed_provider=[{ 137 | "class_name": "org.apache.cassandra.locator.SimpleSeedProvider", 138 | "parameters": [{"seeds": "10.23.20.1"}]}], 139 | ) 140 | ) 141 | part = MIMEBase('x-scylla', 'json') 142 | part.set_payload(json.dumps(scylla_image_configuration, indent=4, sort_keys=True)) 143 | part.add_header('Content-Disposition', 'attachment; filename="scylla_machine_image.json"') 144 | msg.attach(part) 145 | 146 | cloud_config = """ 147 | #cloud-config 148 | cloud_final_modules: 149 | - [scripts-user, always] 150 | """ 151 | part = MIMEBase('text', 'cloud-config') 152 | part.set_payload(cloud_config) 153 | part.add_header('Content-Disposition', 'attachment; filename="cloud-config.txt"') 154 | msg.attach(part) 155 | 156 | print(msg) 157 | ``` 158 | 159 | ## Creating a Scylla cluster using the Machine Image 160 | ### AWS - CloudFormation 161 | Use template `aws/cloudformation/scylla.yaml`. 162 | Currently, maximum 10 nodes cluster is supported. 163 | 164 | ## Building scylla-machine-image package 165 | 166 | ### RedHat like - RPM 167 | 168 | Currently the only supported mode is: 169 | 170 | ``` 171 | dist/redhat/build_rpm.sh --target centos7 --cloud-provider aws 172 | ``` 173 | 174 | Build using Docker 175 | 176 | ``` 177 | docker run -it -v $PWD:/scylla-machine-image -w /scylla-machine-image --rm centos:7.2.1511 bash -c './dist/redhat/build_rpm.sh -t centos7 -c aws' 178 | ``` 179 | 180 | ### Ubuntu - DEB 181 | 182 | ``` 183 | dist/debian/build_deb.sh 184 | ``` 185 | 186 | Build using Docker 187 | 188 | ``` 189 | docker run -it -v $PWD:/scylla-machine-image -w /scylla-machine-image --rm ubuntu:20.04 bash -c './dist/debian/build_deb.sh' 190 | ``` 191 | 192 | ## Building docs 193 | 194 | ```bash 195 | python3 -m venv .venv 196 | source .venv/bin/activate 197 | pip install sphinx sphinx-jsondomain sphinx-markdown-builder 198 | make html 199 | make markdown 200 | ``` 201 | 202 | -------------------------------------------------------------------------------- /SCYLLA-VERSION-GEN: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | PRODUCT=scylla 4 | VERSION=2025.3.0-dev 5 | 6 | if test -f version 7 | then 8 | SCYLLA_VERSION=$(cat version | awk -F'-' '{print $1}') 9 | SCYLLA_RELEASE=$(cat version | awk -F'-' '{print $2}') 10 | else 11 | DATE=$(date +%Y%m%d) 12 | GIT_COMMIT=$(git log --pretty=format:'%h' -n 1) 13 | SCYLLA_VERSION=$VERSION 14 | SCYLLA_RELEASE=$DATE.$GIT_COMMIT 15 | fi 16 | 17 | echo "$SCYLLA_VERSION-$SCYLLA_RELEASE" 18 | mkdir -p build 19 | echo "$SCYLLA_VERSION" > build/SCYLLA-VERSION-FILE 20 | echo "$SCYLLA_RELEASE" > build/SCYLLA-RELEASE-FILE 21 | echo "$PRODUCT" > build/SCYLLA-PRODUCT-FILE 22 | -------------------------------------------------------------------------------- /aws/cloudformation/scylla.yaml.j2: -------------------------------------------------------------------------------- 1 | {%- set total_num_node = 10 %} 2 | AWSTemplateFormatVersion: 2010-09-09 3 | Description: >- 4 | AWS CloudFormation Scylla Sample Template: This would create a new Scylla Cluster 5 | Including it's own VPC and subnet, Elastic IPs are used for accessing those node publicly 6 | 7 | Use `SGAdmin` security group to enable access from outside to this cluster 8 | By default only SSH port is out to the outside world. 9 | 10 | The deployment should take a couple of minutes, usually less than 10 minutes. 11 | You do not need AWS account root user for this deployment and you should avoid using it for such. 12 | 13 | NOTE: the cluster password for the default user (cassandra) is the instance-id of 14 | the first node, therefore connecting to the cluster should be something like 15 | `cqlsh -u cassandra -p i-00a9d141da09ba159`. 16 | 17 | Metadata: 18 | AWS::CloudFormation::Interface: 19 | ParameterGroups: 20 | - Label: 21 | default: Scylla Parameters 22 | Parameters: 23 | - ScyllaClusterName 24 | - ScyllaAmi 25 | - ScyllaSeedIPs 26 | - Label: 27 | default: AWS Parameters 28 | Parameters: 29 | - InstanceType 30 | - InstanceCount 31 | - AvailabilityZone 32 | - CIDR 33 | - EnablePublicAccess 34 | - PublicAccessCIDR 35 | - KeyName 36 | ParameterLabels: 37 | ScyllaClusterName: 38 | default: Scylla cluster name 39 | ScyllaAmi: 40 | default: Scylla AMI ID 41 | ScyllaSeedIPs: 42 | default: Scylla seed nodes IPs 43 | InstanceType: 44 | default: EC2 instance type 45 | InstanceCount: 46 | default: Number of Scylla nodes (EC2 Instances) 47 | AvailabilityZone: 48 | default: Availability Zone 49 | CIDR: 50 | default: CIDR block for Scylla VPC 51 | EnablePublicAccess: 52 | default: Allow public access (SSH) 53 | PublicAccessCIDR: 54 | default: Allowed subnet for public access (SSH) 55 | 56 | Parameters: 57 | ScyllaClusterName: 58 | Type: String 59 | 60 | PublicAccessCIDR: 61 | Type: String 62 | Description: | 63 | The IP address range that can be used to SSH to the EC2 instances 64 | (x.x.x.x/32 for specific IP, 0.0.0.0/0 to allow all IP addresses) 65 | 66 | InstanceCount: 67 | Description: Must be between 1 and {{ total_num_node }} 68 | Type: String 69 | Default: 1 70 | AllowedValues: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] 71 | ConstraintDescription: Must be a number between 1 and {{ total_num_node }}. 72 | 73 | EnablePublicAccess: 74 | Description: Sef true to enable public access to the Scylla cluster nodes 75 | Type: String 76 | AllowedValues: 77 | - 'false' 78 | - 'true' 79 | Default: 'false' 80 | 81 | {% if 'x86_64' in arch %} 82 | InstanceType: 83 | Type: String 84 | Default: i4i.2xlarge 85 | AllowedValues: 86 | - i4i.large 87 | - i4i.xlarge 88 | - i4i.2xlarge 89 | - i4i.4xlarge 90 | - i4i.8xlarge 91 | - i4i.16xlarge 92 | - i4i.32xlarge 93 | - i4i.metal 94 | - i3en.large 95 | - i3en.xlarge 96 | - i3en.2xlarge 97 | - i3en.3xlarge 98 | - i3en.6xlarge 99 | - i3en.12xlarge 100 | - i3en.24xlarge 101 | - i3en.metal 102 | {% elif 'aarch64' in arch %} 103 | InstanceType: 104 | Type: String 105 | Default: im4gn.xlarge 106 | AllowedValues: 107 | - im4gn.xlarge 108 | - im4gn.2xlarge 109 | - im4gn.4xlarge 110 | - im4gn.8xlarge 111 | - im4gn.16xlarge 112 | - is4gen.xlarge 113 | - is4gen.2xlarge 114 | - is4gen.4xlarge 115 | {% endif %} 116 | ConstraintDescription: must be a valid EC2 instance type. 117 | 118 | AvailabilityZone: 119 | Type: 'AWS::EC2::AvailabilityZone::Name' 120 | ConstraintDescription: must be the name of available AvailabilityZone. 121 | 122 | KeyName: 123 | Description: Name of an existing EC2 KeyPair to enable SSH access to the instances 124 | Type: 'AWS::EC2::KeyPair::KeyName' 125 | ConstraintDescription: must be the name of an existing EC2 KeyPair. 126 | 127 | CIDR: 128 | Description: | 129 | Currently supports 8, 16, or 24 netmask. 130 | The node IPs will be x.x.x.10, x.x.x.11, x.x.x.12 etc. 131 | Type: 'String' 132 | Default: '172.31.0.0/16' 133 | AllowedPattern: (\d{1,3})\.(\d{1,3})\.(\d{1,3})\.(\d{1,3})/(\d{1,2}) 134 | ConstraintDescription: must be a valid CIDR (ex. 172.31.0.0/16) 135 | 136 | ScyllaSeedIPs: 137 | Description: | 138 | Will be set as `seeds` on /etc/scylla/scylla.yaml. 139 | NOTE: The first four IP addresses and the last IP address in each subnet reserved by AWS, 140 | for more information, see https://docs.aws.amazon.com/vpc/latest/userguide/VPC_Subnets.html#VPC_Sizing 141 | Type: CommaDelimitedList 142 | Default: '172.31.0.10, 172.31.0.11, 172.31.0.12' 143 | 144 | # Those conditions would be used to enable nodes based on InstanceCount parameter 145 | Conditions: 146 | Launch1: !Equals [1, 1] 147 | Launch2: !Not [!Equals [1, !Ref InstanceCount]] 148 | {%- for node_index in range(3, total_num_node) %} 149 | Launch{{ node_index }}: !And 150 | {%- for i in range(1, node_index) %} 151 | - !Not [!Equals [{{ i }}, !Ref InstanceCount]] 152 | {%- endfor %} 153 | {%- endfor %} 154 | Launch{{ total_num_node }}: !Equals [{{ total_num_node }}, !Ref InstanceCount] 155 | 156 | Mappings: 157 | RegionMap: 158 | ap-northeast-2: 159 | HVM64: placeholder-ap-northeast-2 160 | ap-southeast-2: 161 | HVM64: placeholder-ap-southeast-2 162 | ap-southeast-1: 163 | HVM64: placeholder-ap-southeast-1 164 | ca-central-1: 165 | HVM64: placeholder-ca-central-1 166 | us-east-2: 167 | HVM64: placeholder-us-east-2 168 | us-east-1: 169 | HVM64: placeholder-us-east-1 170 | sa-east-1: 171 | HVM64: placeholder-sa-east-1 172 | eu-west-1: 173 | HVM64: placeholder-eu-west-1 174 | eu-west-2: 175 | HVM64: placeholder-eu-west-2 176 | eu-central-1: 177 | HVM64: placeholder-eu-central-1 178 | us-west-2: 179 | HVM64: placeholder-us-west-2 180 | eu-west-3: 181 | HVM64: placeholder-eu-west-3 182 | eu-north-1: 183 | HVM64: placeholder-eu-north-1 184 | ap-northeast-1: 185 | HVM64: placeholder-ap-northeast-1 186 | ap-northeast-3: 187 | HVM64: placeholder-ap-northeast-3 188 | ap-south-1: 189 | HVM64: placeholder-ap-south-1 190 | us-west-1: 191 | HVM64: placeholder-us-west-1 192 | 193 | Resources: 194 | GatewayAttachment: 195 | Type: 'AWS::EC2::VPCGatewayAttachment' 196 | Properties: 197 | InternetGatewayId: !Ref InternetGateway 198 | VpcId: !Ref VPC 199 | 200 | InternetGateway: 201 | Type: 'AWS::EC2::InternetGateway' 202 | Properties: 203 | Tags: 204 | - Key: Name 205 | Value: !Sub '${ScyllaClusterName}-Gateway' 206 | - Key: ScyllaClusterName 207 | Value: !Ref ScyllaClusterName 208 | 209 | {%- for node_index in range(total_num_node) %} 210 | Node{{ node_index }}: 211 | Condition: Launch{{ node_index + 1 }} 212 | Type: 'AWS::EC2::Instance' 213 | CreationPolicy: 214 | ResourceSignal: 215 | Timeout: PT10M 216 | DependsOn: CfnEndpoint 217 | Properties: 218 | BlockDeviceMappings: 219 | - DeviceName: /dev/sda1 220 | Ebs: 221 | DeleteOnTermination: true 222 | VolumeSize: 50 223 | ImageId: !FindInMap [RegionMap, !Ref "AWS::Region", HVM64] 224 | InstanceType: !Ref InstanceType 225 | KeyName: !Ref KeyName 226 | Tags: 227 | - Key: Name 228 | Value: !Sub '${ScyllaClusterName}-Node-{{ node_index + 1 }}' 229 | - Key: ScyllaClusterName 230 | Value: !Ref ScyllaClusterName 231 | {%- if node_index == 0 %} 232 | UserData: !Base64 233 | 'Fn::Join': 234 | - '' 235 | - - '{"scylla_yaml": {"seed_provider": [{"class_name": "org.apache.cassandra.locator.SimpleSeedProvider", "parameters": [{"seeds": "' 236 | - !Join 237 | - ',' 238 | - - !If [Launch1, !Select [0, !Ref ScyllaSeedIPs], !Ref "AWS::NoValue"] 239 | - !If [Launch2, !Select [1, !Ref ScyllaSeedIPs], !Ref "AWS::NoValue"] 240 | - !If [Launch3, !Select [2, !Ref ScyllaSeedIPs], !Ref "AWS::NoValue"] 241 | - '"}]}]' 242 | - !Sub ', "cluster_name": "${ScyllaClusterName}", ' 243 | - '"endpoint_snitch": "org.apache.cassandra.locator.Ec2Snitch", ' 244 | - '"authenticator": "PasswordAuthenticator"}, ' 245 | - '"start_scylla_on_first_boot": true, ' 246 | - '"post_configuration_script" : "' 247 | - !Base64 248 | 'Fn::Join': 249 | - '' 250 | - - !Sub | 251 | #!/bin/bash -ex 252 | /usr/local/bin/cfn-signal --exit-code 0 --resource Node{{ node_index }} --region ${AWS::Region} --stack ${AWS::StackName} 253 | - '", ' 254 | - '"post_start_script": "' 255 | - !Base64 256 | 'Fn::Join': 257 | - '' 258 | - - !Sub | 259 | #!/bin/bash -ex 260 | export INSTANCE_ID=$(curl -sS http://169.254.169.254/latest/meta-data/instance-id) 261 | /usr/bin/cqlsh -u cassandra -p cassandra -e "ALTER USER cassandra WITH PASSWORD '$INSTANCE_ID';" 262 | /usr/bin/cqlsh -u cassandra -p $INSTANCE_ID -e "ALTER KEYSPACE system_auth WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor': ${InstanceCount}};" 263 | - '"}' 264 | {%- else %} 265 | UserData: !Base64 266 | 'Fn::Join': 267 | - '' 268 | - - '{"scylla_yaml": {"seed_provider": [{"class_name": "org.apache.cassandra.locator.SimpleSeedProvider", "parameters": [{"seeds": "' 269 | - !Join 270 | - ',' 271 | - - !If [Launch1, !Select [0, !Ref ScyllaSeedIPs], !Ref "AWS::NoValue"] 272 | - !If [Launch2, !Select [1, !Ref ScyllaSeedIPs], !Ref "AWS::NoValue"] 273 | - !If [Launch3, !Select [2, !Ref ScyllaSeedIPs], !Ref "AWS::NoValue"] 274 | - '"}]}]' 275 | - !Sub ', "cluster_name": "${ScyllaClusterName}", ' 276 | - '"endpoint_snitch": "org.apache.cassandra.locator.Ec2Snitch", ' 277 | - '"authenticator": "PasswordAuthenticator"}, ' 278 | - '"start_scylla_on_first_boot": true, ' 279 | - '"post_configuration_script" : "' 280 | - !Base64 281 | 'Fn::Join': 282 | - '' 283 | - - !Sub | 284 | #!/bin/bash -ex 285 | /usr/local/bin/cfn-signal --exit-code 0 --resource Node{{ node_index }} --region ${AWS::Region} --stack ${AWS::StackName} 286 | - '"}' 287 | {%- endif %} 288 | NetworkInterfaces: 289 | - AssociatePublicIpAddress: !Ref EnablePublicAccess 290 | PrivateIpAddress: !Join 291 | - '.' 292 | - - !Select [0, !Split ['.', !Select [0, !Split ['/', !Ref CIDR]]]] 293 | - !Select [1, !Split ['.', !Select [0, !Split ['/', !Ref CIDR]]]] 294 | - !Select [2, !Split ['.', !Select [0, !Split ['/', !Ref CIDR]]]] 295 | - {{ node_index + 10 }} 296 | SubnetId: !Ref Subnet 297 | DeviceIndex: '0' 298 | Description: 'Primary network interface' 299 | GroupSet: 300 | - !Ref SGCluster 301 | - !Ref SGAdmin 302 | - !Ref SGExternal 303 | {%- endfor %} 304 | Route: 305 | Type: 'AWS::EC2::Route' 306 | DependsOn: GatewayAttachment 307 | Properties: 308 | DestinationCidrBlock: 0.0.0.0/0 309 | GatewayId: !Ref InternetGateway 310 | RouteTableId: !Ref RouteTable 311 | RouteTable: 312 | Type: 'AWS::EC2::RouteTable' 313 | Properties: 314 | Tags: 315 | - Key: Name 316 | Value: !Sub '${ScyllaClusterName}-RT' 317 | - Key: ScyllaClusterName 318 | Value: !Ref ScyllaClusterName 319 | VpcId: !Ref VPC 320 | 321 | SGExternal: 322 | Type: 'AWS::EC2::SecurityGroup' 323 | Properties: 324 | GroupDescription: Security group for outbound rules 325 | Tags: 326 | - Key: Name 327 | Value: !Sub '${ScyllaClusterName}-SGExternal' 328 | - Key: ScyllaClusterName 329 | Value: !Ref ScyllaClusterName 330 | SecurityGroupEgress: 331 | - CidrIp: 0.0.0.0/0 332 | IpProtocol: '-1' 333 | VpcId: !Ref VPC 334 | 335 | SGAdmin: 336 | Type: 'AWS::EC2::SecurityGroup' 337 | Properties: 338 | GroupDescription: Security group for the admin 339 | Tags: 340 | - Key: Name 341 | Value: !Sub '${ScyllaClusterName}-SGAdmin' 342 | - Key: ScyllaClusterName 343 | Value: !Ref ScyllaClusterName 344 | SecurityGroupIngress: 345 | - CidrIp: !Ref PublicAccessCIDR 346 | FromPort: 22 347 | ToPort: 22 348 | IpProtocol: tcp 349 | VpcId: !Ref VPC 350 | 351 | SGCluster: 352 | Type: 'AWS::EC2::SecurityGroup' 353 | Properties: 354 | GroupDescription: Security group for the cluster 355 | Tags: 356 | - Key: Name 357 | Value: !Sub '${ScyllaClusterName}-SGCluster' 358 | - Key: ScyllaClusterName 359 | Value: !Ref ScyllaClusterName 360 | SecurityGroupIngress: 361 | - CidrIp: !Ref CIDR 362 | IpProtocol: 'tcp' 363 | FromPort: 9042 364 | ToPort: 9042 365 | - CidrIp: !Ref CIDR 366 | IpProtocol: 'tcp' 367 | FromPort: 9142 368 | ToPort: 9142 369 | - CidrIp: !Ref CIDR 370 | IpProtocol: 'tcp' 371 | FromPort: 7000 372 | ToPort: 7001 373 | - CidrIp: !Ref CIDR 374 | IpProtocol: 'tcp' 375 | FromPort: 7199 376 | ToPort: 7199 377 | - CidrIp: !Ref CIDR 378 | IpProtocol: 'tcp' 379 | FromPort: 10000 380 | ToPort: 10000 381 | - CidrIp: !Ref CIDR 382 | IpProtocol: 'tcp' 383 | FromPort: 9180 384 | ToPort: 9180 385 | - CidrIp: !Ref CIDR 386 | IpProtocol: 'tcp' 387 | FromPort: 9100 388 | ToPort: 9100 389 | - CidrIp: !Ref CIDR 390 | IpProtocol: 'tcp' 391 | FromPort: 9160 392 | ToPort: 9160 393 | - CidrIp: !Ref CIDR 394 | IpProtocol: 'tcp' 395 | FromPort: 19042 396 | ToPort: 19042 397 | - CidrIp: !Ref CIDR 398 | IpProtocol: 'tcp' 399 | FromPort: 19142 400 | ToPort: 19142 401 | - CidrIp: !Ref CIDR 402 | IpProtocol: 'icmp' 403 | FromPort: 8 404 | ToPort: '-1' 405 | - CidrIp: !Ref CIDR 406 | IpProtocol: 'tcp' 407 | FromPort: 443 408 | ToPort: 443 409 | VpcId: !Ref VPC 410 | 411 | Subnet: 412 | Type: 'AWS::EC2::Subnet' 413 | Properties: 414 | AvailabilityZone: !Ref AvailabilityZone 415 | CidrBlock: !Ref CIDR 416 | MapPublicIpOnLaunch: !Ref EnablePublicAccess 417 | Tags: 418 | - Key: Name 419 | Value: !Sub '${ScyllaClusterName}-Subnet' 420 | - Key: ScyllaClusterName 421 | Value: !Ref ScyllaClusterName 422 | VpcId: !Ref VPC 423 | 424 | SubnetRouteTableAssociation: 425 | Type: 'AWS::EC2::SubnetRouteTableAssociation' 426 | Properties: 427 | RouteTableId: !Ref RouteTable 428 | SubnetId: !Ref Subnet 429 | 430 | VPC: 431 | Type: 'AWS::EC2::VPC' 432 | Properties: 433 | CidrBlock: !Ref CIDR 434 | EnableDnsSupport: true 435 | EnableDnsHostnames: true 436 | Tags: 437 | - Key: Name 438 | Value: !Sub '${ScyllaClusterName}-VPC' 439 | - Key: ScyllaClusterName 440 | Value: !Ref ScyllaClusterName 441 | 442 | CfnEndpoint: 443 | Type: AWS::EC2::VPCEndpoint 444 | DependsOn: SGCluster 445 | Properties: 446 | VpcId: !Ref VPC 447 | ServiceName: !Sub "com.amazonaws.${AWS::Region}.cloudformation" 448 | VpcEndpointType: "Interface" 449 | PrivateDnsEnabled: true 450 | SubnetIds: 451 | - !Ref Subnet 452 | SecurityGroupIds: 453 | - !Ref SGCluster 454 | - !Ref SGAdmin 455 | - !Ref SGExternal 456 | 457 | Outputs: 458 | {%- for node_index in range(total_num_node) %} 459 | Node{{ node_index }}: 460 | Condition: Launch{{ node_index + 1 }} 461 | Value: !Ref Node{{ node_index }} 462 | Node{{ node_index }}PrivateDnsName: 463 | Condition: Launch{{ node_index + 1 }} 464 | Value: !GetAtt 465 | - Node{{ node_index }} 466 | - PrivateDnsName 467 | Node{{ node_index }}PrivateIp: 468 | Condition: Launch{{ node_index + 1 }} 469 | Value: !GetAtt 470 | - Node{{ node_index }} 471 | - PrivateIp 472 | {%- endfor %} 473 | SGExternal: 474 | Value: !Ref SGExternal 475 | SGAdmin: 476 | Value: !Ref SGAdmin 477 | SGCluster: 478 | Value: !Ref SGCluster 479 | Subnet: 480 | Value: !Ref Subnet 481 | VPC: 482 | Value: !Ref VPC 483 | -------------------------------------------------------------------------------- /common/.bash_profile: -------------------------------------------------------------------------------- 1 | # .bash_profile 2 | 3 | # Get the aliases and functions 4 | if [ -f ~/.bashrc ]; then 5 | . ~/.bashrc 6 | fi 7 | 8 | # User specific environment and startup programs 9 | 10 | PATH=$PATH:$HOME/.local/bin:$HOME/bin 11 | 12 | export PATH 13 | 14 | /opt/scylladb/scylla-machine-image/scylla_login 15 | -------------------------------------------------------------------------------- /common/aws_io_params.yaml: -------------------------------------------------------------------------------- 1 | i3.large: 2 | read_iops: 111000 3 | read_bandwidth: 653925080 4 | write_iops: 36800 5 | write_bandwidth: 215066473 6 | i3.xlarge: 7 | read_iops: 200800 8 | read_bandwidth: 1185106376 9 | write_iops: 53180 10 | write_bandwidth: 423621267 11 | i3.ALL: 12 | read_iops: 411200 13 | read_bandwidth: 2015342735 14 | write_iops: 181500 15 | write_bandwidth: 808775652 16 | i3en.large: 17 | read_iops: 43315 18 | read_bandwidth: 330301440 19 | write_iops: 33177 20 | write_bandwidth: 165675008 21 | i3en.xlarge: 22 | read_iops: 84480 23 | read_bandwidth: 666894336 24 | write_iops: 66969 25 | write_bandwidth: 333447168 26 | i3en.2xlarge: 27 | read_iops: 84480 28 | read_bandwidth: 666894336 29 | write_iops: 66969 30 | write_bandwidth: 333447168 31 | i3en.ALL: 32 | read_iops: 257024 33 | read_bandwidth: 2043674624 34 | write_iops: 174080 35 | write_bandwidth: 1024458752 36 | i2.ALL: 37 | read_iops: 64000 38 | read_bandwidth: 507338935 39 | write_iops: 57100 40 | write_bandwidth: 483141731 41 | m5d.large: 42 | read_iops: 33271 43 | read_bandwidth: 158538149 44 | write_iops: 16820 45 | write_bandwidth: 70219810 46 | m5d.xlarge: 47 | read_iops: 65979 48 | read_bandwidth: 260654293 49 | write_iops: 32534 50 | write_bandwidth: 135897424 51 | m5d.2xlarge: 52 | read_iops: 130095 53 | read_bandwidth: 621758272 54 | write_iops: 63644 55 | write_bandwidth: 267667525 56 | m5d.4xlarge: 57 | read_iops: 129822 58 | read_bandwidth: 620878826 59 | write_iops: 63212 60 | write_bandwidth: 267703397 61 | m5d.8xlarge: 62 | read_iops: 257069 63 | read_bandwidth: 1250134869 64 | write_iops: 115433 65 | write_bandwidth: 532868032 66 | m5d.12xlarge: 67 | read_iops: 381626 68 | read_bandwidth: 1865794816 69 | write_iops: 115333 70 | write_bandwidth: 795884800 71 | m5d.16xlarge: 72 | read_iops: 257054 73 | read_bandwidth: 1254133802 74 | write_iops: 108163 75 | write_bandwidth: 532996277 76 | m5d.24xlarge: 77 | read_iops: 374737 78 | read_bandwidth: 1855833386 79 | write_iops: 125214 80 | write_bandwidth: 796082133 81 | m5d.metal: 82 | read_iops: 381441 83 | read_bandwidth: 1874585429 84 | write_iops: 108789 85 | write_bandwidth: 796443221 86 | r5d.large: 87 | read_iops: 33271 88 | read_bandwidth: 158538149 89 | write_iops: 16820 90 | write_bandwidth: 70219810 91 | r5d.xlarge: 92 | read_iops: 65979 93 | read_bandwidth: 260654293 94 | write_iops: 32534 95 | write_bandwidth: 135897424 96 | r5d.2xlarge: 97 | read_iops: 130095 98 | read_bandwidth: 621758272 99 | write_iops: 63644 100 | write_bandwidth: 267667525 101 | r5d.4xlarge: 102 | read_iops: 129822 103 | read_bandwidth: 620878826 104 | write_iops: 63212 105 | write_bandwidth: 267703397 106 | r5d.8xlarge: 107 | read_iops: 257069 108 | read_bandwidth: 1250134869 109 | write_iops: 115433 110 | write_bandwidth: 532868032 111 | r5d.12xlarge: 112 | read_iops: 381626 113 | read_bandwidth: 1865794816 114 | write_iops: 115333 115 | write_bandwidth: 795884800 116 | r5d.16xlarge: 117 | read_iops: 257054 118 | read_bandwidth: 1254133802 119 | write_iops: 108163 120 | write_bandwidth: 532996277 121 | r5d.24xlarge: 122 | read_iops: 374737 123 | read_bandwidth: 1855833386 124 | write_iops: 125214 125 | write_bandwidth: 796082133 126 | r5d.metal: 127 | read_iops: 381441 128 | read_bandwidth: 1874585429 129 | write_iops: 108789 130 | write_bandwidth: 796443221 131 | m5ad.large: 132 | read_iops: 33306 133 | read_bandwidth: 158338864 134 | write_iops: 16817 135 | write_bandwidth: 70194288 136 | m5ad.xlarge: 137 | read_iops: 66127 138 | read_bandwidth: 260377466 139 | write_iops: 32893 140 | write_bandwidth: 135897696 141 | m5ad.2xlarge: 142 | read_iops: 129977 143 | read_bandwidth: 621997248 144 | write_iops: 63442 145 | write_bandwidth: 267648736 146 | m5ad.4xlarge: 147 | read_iops: 129937 148 | read_bandwidth: 620231082 149 | write_iops: 62666 150 | write_bandwidth: 267639125 151 | m5ad.8xlarge: 152 | read_iops: 257095 153 | read_bandwidth: 1249927637 154 | write_iops: 114446 155 | write_bandwidth: 532821760 156 | m5ad.12xlarge: 157 | read_iops: 376431 158 | read_bandwidth: 1865866709 159 | write_iops: 115985 160 | write_bandwidth: 796003477 161 | m5ad.16xlarge: 162 | read_iops: 256358 163 | read_bandwidth: 1250889770 164 | write_iops: 114707 165 | write_bandwidth: 532998506 166 | m5ad.24xlarge: 167 | read_iops: 258951 168 | read_bandwidth: 1865871317 169 | write_iops: 116030 170 | write_bandwidth: 796217706 171 | c5d.large: 172 | read_iops: 22095 173 | read_bandwidth: 104797834 174 | write_iops: 10125 175 | write_bandwidth: 41982906 176 | c5d.xlarge: 177 | read_iops: 44355 178 | read_bandwidth: 212593018 179 | write_iops: 20025 180 | write_bandwidth: 84213472 181 | c5d.2xlarge: 182 | read_iops: 89036 183 | read_bandwidth: 426821429 184 | write_iops: 41697 185 | write_bandwidth: 173730709 186 | c5d.4xlarge: 187 | read_iops: 193970 188 | read_bandwidth: 928278314 189 | write_iops: 83058 190 | write_bandwidth: 351839733 191 | c5d.9xlarge: 192 | read_iops: 381800 193 | read_bandwidth: 1865831893 194 | write_iops: 112264 195 | write_bandwidth: 795731264 196 | c5d.12xlarge: 197 | read_iops: 381775 198 | read_bandwidth: 1866481792 199 | write_iops: 114302 200 | write_bandwidth: 795607616 201 | c5d.18xlarge: 202 | read_iops: 381270 203 | read_bandwidth: 1856972330 204 | write_iops: 125638 205 | write_bandwidth: 795813866 206 | c5d.24xlarge: 207 | read_iops: 381355 208 | read_bandwidth: 1876056704 209 | write_iops: 104946 210 | write_bandwidth: 795901013 211 | c5d.metal: 212 | read_iops: 381330 213 | read_bandwidth: 1865216426 214 | write_iops: 115484 215 | write_bandwidth: 796109546 216 | z1d.large: 217 | read_iops: 33286 218 | read_bandwidth: 158206858 219 | write_iops: 16956 220 | write_bandwidth: 70226280 221 | z1d.xlarge: 222 | read_iops: 66076 223 | read_bandwidth: 260565488 224 | write_iops: 32769 225 | write_bandwidth: 135891989 226 | z1d.2xlarge: 227 | read_iops: 130235 228 | read_bandwidth: 622297194 229 | write_iops: 63891 230 | write_bandwidth: 267679509 231 | z1d.3xlarge: 232 | read_iops: 193840 233 | read_bandwidth: 927493696 234 | write_iops: 82864 235 | write_bandwidth: 351608480 236 | z1d.6xlarge: 237 | read_iops: 381902 238 | read_bandwidth: 1865543381 239 | write_iops: 117874 240 | write_bandwidth: 795786901 241 | z1d.12xlarge: 242 | read_iops: 381648 243 | read_bandwidth: 1865706538 244 | write_iops: 115834 245 | write_bandwidth: 795876778 246 | z1d.metal: 247 | read_iops: 381378 248 | read_bandwidth: 1857873109 249 | write_iops: 122453 250 | write_bandwidth: 795593024 251 | c6gd.medium: 252 | read_iops: 14808 253 | read_bandwidth: 77869147 254 | write_iops: 5972 255 | write_bandwidth: 32820302 256 | c6gd.large: 257 | read_iops: 29690 258 | read_bandwidth: 157712240 259 | write_iops: 12148 260 | write_bandwidth: 65978069 261 | c6gd.xlarge: 262 | read_iops: 59688 263 | read_bandwidth: 318762880 264 | write_iops: 24449 265 | write_bandwidth: 133311808 266 | c6gd.2xlarge: 267 | read_iops: 119353 268 | read_bandwidth: 634795733 269 | write_iops: 49069 270 | write_bandwidth: 266841680 271 | c6gd.4xlarge: 272 | read_iops: 237196 273 | read_bandwidth: 1262309504 274 | write_iops: 98884 275 | write_bandwidth: 533938080 276 | c6gd.8xlarge: 277 | read_iops: 442945 278 | read_bandwidth: 2522688939 279 | write_iops: 166021 280 | write_bandwidth: 1063041152 281 | c6gd.12xlarge: 282 | read_iops: 353691 283 | read_bandwidth: 1908192256 284 | write_iops: 146732 285 | write_bandwidth: 806399360 286 | c6gd.16xlarge: 287 | read_iops: 426893 288 | read_bandwidth: 2525781589 289 | write_iops: 161740 290 | write_bandwidth: 1063389952 291 | c6gd.metal: 292 | read_iops: 416257 293 | read_bandwidth: 2527296683 294 | write_iops: 156326 295 | write_bandwidth: 1063657088 296 | m6gd.medium: 297 | read_iops: 14808 298 | read_bandwidth: 77869147 299 | write_iops: 5972 300 | write_bandwidth: 32820302 301 | m6gd.large: 302 | read_iops: 29690 303 | read_bandwidth: 157712240 304 | write_iops: 12148 305 | write_bandwidth: 65978069 306 | m6gd.xlarge: 307 | read_iops: 59688 308 | read_bandwidth: 318762880 309 | write_iops: 24449 310 | write_bandwidth: 133311808 311 | m6gd.2xlarge: 312 | read_iops: 119353 313 | read_bandwidth: 634795733 314 | write_iops: 49069 315 | write_bandwidth: 266841680 316 | m6gd.4xlarge: 317 | read_iops: 237196 318 | read_bandwidth: 1262309504 319 | write_iops: 98884 320 | write_bandwidth: 533938080 321 | m6gd.8xlarge: 322 | read_iops: 442945 323 | read_bandwidth: 2522688939 324 | write_iops: 166021 325 | write_bandwidth: 1063041152 326 | m6gd.12xlarge: 327 | read_iops: 353691 328 | read_bandwidth: 1908192256 329 | write_iops: 146732 330 | write_bandwidth: 806399360 331 | m6gd.16xlarge: 332 | read_iops: 426893 333 | read_bandwidth: 2525781589 334 | write_iops: 161740 335 | write_bandwidth: 1063389952 336 | m6gd.metal: 337 | read_iops: 416257 338 | read_bandwidth: 2527296683 339 | write_iops: 156326 340 | write_bandwidth: 1063657088 341 | r6gd.medium: 342 | read_iops: 14808 343 | read_bandwidth: 77869147 344 | write_iops: 5972 345 | write_bandwidth: 32820302 346 | r6gd.large: 347 | read_iops: 29690 348 | read_bandwidth: 157712240 349 | write_iops: 12148 350 | write_bandwidth: 65978069 351 | r6gd.xlarge: 352 | read_iops: 59688 353 | read_bandwidth: 318762880 354 | write_iops: 24449 355 | write_bandwidth: 133311808 356 | r6gd.2xlarge: 357 | read_iops: 119353 358 | read_bandwidth: 634795733 359 | write_iops: 49069 360 | write_bandwidth: 266841680 361 | r6gd.4xlarge: 362 | read_iops: 237196 363 | read_bandwidth: 1262309504 364 | write_iops: 98884 365 | write_bandwidth: 533938080 366 | r6gd.8xlarge: 367 | read_iops: 442945 368 | read_bandwidth: 2522688939 369 | write_iops: 166021 370 | write_bandwidth: 1063041152 371 | r6gd.12xlarge: 372 | read_iops: 353691 373 | read_bandwidth: 1908192256 374 | write_iops: 146732 375 | write_bandwidth: 806399360 376 | r6gd.16xlarge: 377 | read_iops: 426893 378 | read_bandwidth: 2525781589 379 | write_iops: 161740 380 | write_bandwidth: 1063389952 381 | r6gd.metal: 382 | read_iops: 416257 383 | read_bandwidth: 2527296683 384 | write_iops: 156326 385 | write_bandwidth: 1063657088 386 | x2gd.medium: 387 | read_iops: 14808 388 | read_bandwidth: 77869147 389 | write_iops: 5972 390 | write_bandwidth: 32820302 391 | x2gd.large: 392 | read_iops: 29690 393 | read_bandwidth: 157712240 394 | write_iops: 12148 395 | write_bandwidth: 65978069 396 | x2gd.xlarge: 397 | read_iops: 59688 398 | read_bandwidth: 318762880 399 | write_iops: 24449 400 | write_bandwidth: 133311808 401 | x2gd.2xlarge: 402 | read_iops: 119353 403 | read_bandwidth: 634795733 404 | write_iops: 49069 405 | write_bandwidth: 266841680 406 | x2gd.4xlarge: 407 | read_iops: 237196 408 | read_bandwidth: 1262309504 409 | write_iops: 98884 410 | write_bandwidth: 533938080 411 | x2gd.8xlarge: 412 | read_iops: 442945 413 | read_bandwidth: 2522688939 414 | write_iops: 166021 415 | write_bandwidth: 1063041152 416 | x2gd.12xlarge: 417 | read_iops: 353691 418 | read_bandwidth: 1908192256 419 | write_iops: 146732 420 | write_bandwidth: 806399360 421 | x2gd.16xlarge: 422 | read_iops: 426893 423 | read_bandwidth: 2525781589 424 | write_iops: 161740 425 | write_bandwidth: 1063389952 426 | x2gd.metal: 427 | read_iops: 416257 428 | read_bandwidth: 2527296683 429 | write_iops: 156326 430 | write_bandwidth: 1063657088 431 | i4g.large: 432 | read_iops: 34035 433 | read_bandwidth: 288471904 434 | write_iops: 27943 435 | write_bandwidth: 126763269 436 | i4g.xlarge: 437 | read_iops: 68111 438 | read_bandwidth: 571766890 439 | write_iops: 47622 440 | write_bandwidth: 254230192 441 | i4g.2xlarge: 442 | read_iops: 136352 443 | read_bandwidth: 1148509696 444 | write_iops: 82746 445 | write_bandwidth: 508828810 446 | i4g.4xlarge: 447 | read_iops: 272704 448 | read_bandwidth: 2297019392 449 | write_iops: 165492 450 | write_bandwidth: 1017657620 451 | i4g.8xlarge: 452 | read_iops: 271495 453 | read_bandwidth: 2293024938 454 | write_iops: 93653 455 | write_bandwidth: 1031956586 456 | i4g.16xlarge: 457 | read_iops: 250489 458 | read_bandwidth: 2286635861 459 | write_iops: 93737 460 | write_bandwidth: 1034256042 461 | im4gn.large: 462 | read_iops: 33943 463 | read_bandwidth: 288433525 464 | write_iops: 27877 465 | write_bandwidth: 126864680 466 | im4gn.xlarge: 467 | read_iops: 68122 468 | read_bandwidth: 576603520 469 | write_iops: 55246 470 | write_bandwidth: 254534954 471 | im4gn.2xlarge: 472 | read_iops: 136422 473 | read_bandwidth: 1152663765 474 | write_iops: 92184 475 | write_bandwidth: 508926453 476 | im4gn.4xlarge: 477 | read_iops: 273050 478 | read_bandwidth: 1638427264 479 | write_iops: 92173 480 | write_bandwidth: 1027966826 481 | im4gn.8xlarge: 482 | read_iops: 250241 483 | read_bandwidth: 1163130709 484 | write_iops: 86374 485 | write_bandwidth: 977617664 486 | im4gn.16xlarge: 487 | read_iops: 273030 488 | read_bandwidth: 1638211413 489 | write_iops: 92607 490 | write_bandwidth: 1028340266 491 | is4gen.medium: 492 | read_iops: 33965 493 | read_bandwidth: 288462506 494 | write_iops: 27876 495 | write_bandwidth: 126954200 496 | is4gen.large: 497 | read_iops: 68131 498 | read_bandwidth: 576654869 499 | write_iops: 55257 500 | write_bandwidth: 254551002 501 | is4gen.xlarge: 502 | read_iops: 136413 503 | read_bandwidth: 1152747904 504 | write_iops: 92180 505 | write_bandwidth: 508889546 506 | is4gen.2xlarge: 507 | read_iops: 273038 508 | read_bandwidth: 1628982613 509 | write_iops: 92182 510 | write_bandwidth: 1027983530 511 | is4gen.4xlarge: 512 | read_iops: 260493 513 | read_bandwidth: 1217396928 514 | write_iops: 83169 515 | write_bandwidth: 1000390784 516 | is4gen.8xlarge: 517 | read_iops: 273021 518 | read_bandwidth: 1656354602 519 | write_iops: 92233 520 | write_bandwidth: 1028010325 521 | i4i.large: 522 | read_iops: 54987 523 | read_bandwidth: 378494048 524 | write_iops: 30459 525 | write_bandwidth: 279713216 526 | i4i.xlarge: 527 | read_iops: 109954 528 | read_bandwidth: 763580096 529 | write_iops: 61008 530 | write_bandwidth: 561926784 531 | i4i.2xlarge: 532 | read_iops: 218786 533 | read_bandwidth: 1542559872 534 | write_iops: 121499 535 | write_bandwidth: 1130867072 536 | i4i.4xlarge: 537 | read_iops: 385400 538 | read_bandwidth: 3087631104 539 | write_iops: 240628 540 | write_bandwidth: 2289281280 541 | i4i.8xlarge: 542 | read_iops: 384561 543 | read_bandwidth: 3115819008 544 | write_iops: 239980 545 | write_bandwidth: 2289285120 546 | i4i.12xlarge: 547 | read_iops: 294982 548 | read_bandwidth: 3116245760 549 | write_iops: 67283 550 | write_bandwidth: 2287695786 551 | i4i.16xlarge: 552 | read_iops: 374273 553 | read_bandwidth: 3088962816 554 | write_iops: 240185 555 | write_bandwidth: 2292813568 556 | i4i.24xlarge: 557 | read_iops: 282557 558 | read_bandwidth: 3116171434 559 | write_iops: 67003 560 | write_bandwidth: 2288658688 561 | i4i.32xlarge: 562 | read_iops: 374273 563 | read_bandwidth: 3095612416 564 | write_iops: 239413 565 | write_bandwidth: 2296702976 566 | i4i.metal: 567 | read_iops: 379565 568 | read_bandwidth: 3088599296 569 | write_iops: 239549 570 | write_bandwidth: 2302438912 571 | i7ie.large: 572 | read_iops: 58449 573 | read_bandwidth: 574854656 574 | write_iops: 47145 575 | write_bandwidth: 253132917 576 | i7ie.xlarge: 577 | read_iops: 117257 578 | read_bandwidth: 1148572714 579 | write_iops: 94180 580 | write_bandwidth: 505684885 581 | i7ie.2xlarge: 582 | read_iops: 117257 583 | read_bandwidth: 1148572714 584 | write_iops: 94180 585 | write_bandwidth: 505684885 586 | i7ie.ALL: 587 | read_iops: 352834 588 | read_bandwidth: 3422623232 589 | write_iops: 119327 590 | write_bandwidth: 1526442410 591 | -------------------------------------------------------------------------------- /common/scylla-image-post-start.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Scylla cloud image post start service 3 | After=scylla-server.service 4 | ConditionPathExists=!/etc/scylla/machine_image_post_start_configured 5 | 6 | [Service] 7 | Type=oneshot 8 | # The ExecStartPre makes sure the scylla_post_start.py script 9 | # will run only after scylla-server is up and running 10 | ExecStartPre=/bin/sleep 30 11 | ExecStart=/opt/scylladb/scylla-machine-image/scylla_post_start.py 12 | RemainAfterExit=yes 13 | TimeoutStartSec=900 14 | 15 | [Install] 16 | RequiredBy=scylla-server.service 17 | -------------------------------------------------------------------------------- /common/scylla-image-setup.service: -------------------------------------------------------------------------------- 1 | [Unit] 2 | Description=Scylla Cloud Image Setup service 3 | Before=scylla-server.service 4 | After=network.target 5 | 6 | [Service] 7 | Type=oneshot 8 | ExecStart=/opt/scylladb/scylla-machine-image/scylla_image_setup 9 | RemainAfterExit=yes 10 | TimeoutStartSec=900 11 | 12 | [Install] 13 | RequiredBy=scylla-server.service 14 | -------------------------------------------------------------------------------- /common/scylla_cloud_io_setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2022 ScyllaDB 5 | # 6 | # SPDX-License-Identifier: Apache-2.0 7 | 8 | import os 9 | import subprocess 10 | import yaml 11 | import logging 12 | import sys 13 | from abc import ABCMeta, abstractmethod 14 | from lib.scylla_cloud import get_cloud_instance, is_ec2, is_gce, is_azure 15 | 16 | 17 | def UnsupportedInstanceClassError(Exception): 18 | pass 19 | 20 | 21 | class cloud_io_setup(metaclass=ABCMeta): 22 | @abstractmethod 23 | def generate(self): 24 | pass 25 | 26 | def save(self): 27 | assert "read_iops" in self.disk_properties 28 | properties_file = open("/etc/scylla.d/io_properties.yaml", "w") 29 | yaml.dump({"disks": [self.disk_properties]}, properties_file, default_flow_style=False) 30 | ioconf = open("/etc/scylla.d/io.conf", "w") 31 | ioconf.write("SEASTAR_IO=\"--io-properties-file={}\"\n".format(properties_file.name)) 32 | os.chmod('/etc/scylla.d/io_properties.yaml', 0o644) 33 | os.chmod('/etc/scylla.d/io.conf', 0o644) 34 | 35 | 36 | class aws_io_setup(cloud_io_setup): 37 | def __init__(self, idata): 38 | self.idata = idata 39 | self.disk_properties = {} 40 | 41 | def generate(self): 42 | if not self.idata.is_supported_instance_class(): 43 | logging.error('This is not a recommended EC2 instance setup for auto local disk tuning.') 44 | raise UnsupportedInstanceClassError() 45 | self.disk_properties["mountpoint"] = '/var/lib/scylla' 46 | nr_disks = len(self.idata.get_local_disks()) 47 | instance_type = self.idata.instancetype 48 | instance_class_all = self.idata.instance_class() + '.ALL' 49 | with open('/opt/scylladb/scylla-machine-image/aws_io_params.yaml') as f: 50 | io_params = yaml.safe_load(f) 51 | 52 | t = None 53 | if instance_type in io_params: 54 | t = instance_type 55 | elif instance_class_all in io_params: 56 | t = instance_class_all 57 | if t: 58 | for p in ["read_iops", "read_bandwidth", "write_iops", "write_bandwidth"]: 59 | self.disk_properties[p] = io_params[t][p] * nr_disks 60 | self.save() 61 | else: 62 | logging.warning("This is a supported instance but with no pre-configured io, scylla_io_setup will be run") 63 | subprocess.run('scylla_io_setup', shell=True, check=True, capture_output=True, timeout=300) 64 | 65 | 66 | class gcp_io_setup(cloud_io_setup): 67 | def __init__(self, idata): 68 | self.idata = idata 69 | self.disk_properties = {} 70 | 71 | def generate(self): 72 | if not self.idata.is_supported_instance_class(): 73 | logging.error('This is not a recommended Google Cloud instance setup for auto local disk tuning.') 74 | raise UnsupportedInstanceClassError() 75 | self.disk_properties = {} 76 | self.disk_properties["mountpoint"] = '/var/lib/scylla' 77 | nr_disks = self.idata.nvme_disk_count 78 | # below is based on https://cloud.google.com/compute/docs/disks/local-ssd#performance 79 | # and https://cloud.google.com/compute/docs/disks/local-ssd#nvme 80 | # note that scylla iotune might measure more, this is GCP recommended 81 | mbs=1024*1024 82 | if nr_disks >= 1 and nr_disks < 4: 83 | self.disk_properties["read_iops"] = 170000 * nr_disks 84 | self.disk_properties["read_bandwidth"] = 660 * mbs * nr_disks 85 | self.disk_properties["write_iops"] = 90000 * nr_disks 86 | self.disk_properties["write_bandwidth"] = 350 * mbs * nr_disks 87 | elif nr_disks >= 4 and nr_disks <= 8: 88 | self.disk_properties["read_iops"] = 680000 89 | self.disk_properties["read_bandwidth"] = 2650 * mbs 90 | self.disk_properties["write_iops"] = 360000 91 | self.disk_properties["write_bandwidth"] = 1400 * mbs 92 | elif nr_disks == 16: 93 | self.disk_properties["read_iops"] = 1600000 94 | self.disk_properties["read_bandwidth"] = 4521251328 95 | #below is google, above is our measured 96 | #self.disk_properties["read_bandwidth"] = 6240 * mbs 97 | self.disk_properties["write_iops"] = 800000 98 | self.disk_properties["write_bandwidth"] = 2759452672 99 | #below is google, above is our measured 100 | #self.disk_properties["write_bandwidth"] = 3120 * mbs 101 | elif nr_disks == 24: 102 | self.disk_properties["read_iops"] = 2400000 103 | self.disk_properties["read_bandwidth"] = 5921532416 104 | #below is google, above is our measured 105 | #self.disk_properties["read_bandwidth"] = 9360 * mbs 106 | self.disk_properties["write_iops"] = 1200000 107 | self.disk_properties["write_bandwidth"] = 4663037952 108 | #below is google, above is our measured 109 | #self.disk_properties["write_bandwidth"] = 4680 * mbs 110 | 111 | if "read_iops" in self.disk_properties: 112 | self.save() 113 | else: 114 | logging.warning("This is a supported instance but with no pre-configured io, scylla_io_setup will be run") 115 | subprocess.run('scylla_io_setup', shell=True, check=True, capture_output=True, timeout=300) 116 | 117 | 118 | class azure_io_setup(cloud_io_setup): 119 | def __init__(self, idata): 120 | self.idata = idata 121 | self.disk_properties = {} 122 | 123 | def generate(self): 124 | if not self.idata.is_supported_instance_class(): 125 | logging.error('This is not a recommended Azure Cloud instance setup for auto local disk tuning.') 126 | raise UnsupportedInstanceClassError() 127 | 128 | self.disk_properties = {} 129 | self.disk_properties["mountpoint"] = '/var/lib/scylla' 130 | nr_disks = self.idata.nvme_disk_count 131 | # below is based on https://docs.microsoft.com/en-us/azure/virtual-machines/lsv2-series 132 | # note that scylla iotune might measure more, this is Azure recommended 133 | # since write properties are not defined, they come from our iotune tests 134 | mbs = 1024*1024 135 | if nr_disks == 1: 136 | self.disk_properties["read_iops"] = 400000 137 | self.disk_properties["read_bandwidth"] = 2000 * mbs 138 | self.disk_properties["write_iops"] = 271696 139 | self.disk_properties["write_bandwidth"] = 1314 * mbs 140 | elif nr_disks == 2: 141 | self.disk_properties["read_iops"] = 800000 142 | self.disk_properties["read_bandwidth"] = 4000 * mbs 143 | self.disk_properties["write_iops"] = 552434 144 | self.disk_properties["write_bandwidth"] = 2478 * mbs 145 | elif nr_disks == 4: 146 | self.disk_properties["read_iops"] = 1500000 147 | self.disk_properties["read_bandwidth"] = 8000 * mbs 148 | self.disk_properties["write_iops"] = 1105063 149 | self.disk_properties["write_bandwidth"] = 4948 * mbs 150 | elif nr_disks == 6: 151 | self.disk_properties["read_iops"] = 2200000 152 | self.disk_properties["read_bandwidth"] = 14000 * mbs 153 | self.disk_properties["write_iops"] = 1616847 154 | self.disk_properties["write_bandwidth"] = 7892 * mbs 155 | elif nr_disks == 8: 156 | self.disk_properties["read_iops"] = 2900000 157 | self.disk_properties["read_bandwidth"] = 16000 * mbs 158 | self.disk_properties["write_iops"] = 2208081 159 | self.disk_properties["write_bandwidth"] = 9694 * mbs 160 | elif nr_disks == 10: 161 | self.disk_properties["read_iops"] = 3800000 162 | self.disk_properties["read_bandwidth"] = 20000 * mbs 163 | self.disk_properties["write_iops"] = 2546511 164 | self.disk_properties["write_bandwidth"] = 11998 * mbs 165 | 166 | if "read_iops" in self.disk_properties: 167 | self.save() 168 | else: 169 | logging.warning("This is a supported instance but with no pre-configured io, scylla_io_setup will be run") 170 | subprocess.run('scylla_io_setup', shell=True, check=True, capture_output=True, timeout=300) 171 | 172 | 173 | if __name__ == '__main__': 174 | if not os.path.ismount('/var/lib/scylla'): 175 | logging.error('RAID volume not mounted') 176 | sys.exit(1) 177 | cloud_instance = get_cloud_instance() 178 | if is_ec2(): 179 | io = aws_io_setup(cloud_instance) 180 | elif is_gce(): 181 | io = gcp_io_setup(cloud_instance) 182 | elif is_azure(): 183 | io = azure_io_setup(cloud_instance) 184 | try: 185 | io.generate() 186 | except UnsupportedInstanceClassError as e: 187 | sys.exit(1) 188 | -------------------------------------------------------------------------------- /common/scylla_configure.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright 2020 ScyllaDB 4 | # 5 | # SPDX-License-Identifier: Apache-2.0 6 | 7 | import base64 8 | import subprocess 9 | import yaml 10 | import time 11 | import logging 12 | import sys 13 | from textwrap import dedent 14 | from datetime import datetime 15 | from lib.scylla_cloud import scylla_excepthook 16 | from lib.log import setup_logging 17 | from lib.user_data import UserData 18 | from pathlib import Path 19 | 20 | LOGGER = logging.getLogger(__name__) 21 | 22 | 23 | class ScyllaMachineImageConfigurator(UserData): 24 | CONF_DEFAULTS = { 25 | 'scylla_yaml': { 26 | 'cluster_name': "scylladb-cluster-%s" % int(time.time()), 27 | 'auto_bootstrap': True, 28 | 'listen_address': "", # will be configured as a private IP when instance meta data is read 29 | 'broadcast_rpc_address': "", # will be configured as a private IP when instance meta data is read 30 | 'rpc_address': "0.0.0.0", 31 | 'seed_provider': [{'class_name': 'org.apache.cassandra.locator.SimpleSeedProvider', 32 | 'parameters': [{'seeds': ""}]}], # will be configured as a private IP when 33 | # instance meta data is read 34 | }, 35 | 'scylla_startup_args': [], # TODO: implement 36 | 'developer_mode': False, 37 | 'post_configuration_script': '', 38 | 'post_configuration_script_timeout': 600, # seconds 39 | 'start_scylla_on_first_boot': True, 40 | 'data_device': 'auto', # Supported options: 41 | # instance_store - find all ephemeral devices (only for AWS) 42 | # attached - find all attached devices and use them 43 | # auto - automatically select devices using following strategy: 44 | # GCE: select attached NVMe. 45 | # AWS: 46 | # if ephemeral found - use them 47 | # else if attached EBS found use them 48 | # else: fail create_devices 49 | 'raid_level': 0 # Default raid level is 0, supported raid 0, 5 50 | } 51 | 52 | def __init__(self, scylla_yaml_path="/etc/scylla/scylla.yaml"): 53 | self.scylla_yaml_path = Path(scylla_yaml_path) 54 | self.scylla_yaml_example_path = Path(scylla_yaml_path + ".example") 55 | self._scylla_yaml = {} 56 | super().__init__() 57 | 58 | @property 59 | def scylla_yaml(self): 60 | if not self._scylla_yaml: 61 | with self.scylla_yaml_path.open() as scylla_yaml_file: 62 | self._scylla_yaml = yaml.load(scylla_yaml_file, Loader=yaml.SafeLoader) 63 | return self._scylla_yaml 64 | 65 | def save_scylla_yaml(self): 66 | LOGGER.info("Saving %s", self.scylla_yaml_path) 67 | with self.scylla_yaml_path.open("w") as scylla_yaml_file: 68 | now = datetime.now() 69 | scylla_yaml_file.write(dedent(f""" 70 | # Generated by Scylla Machine Image at {now} 71 | # See '/etc/scylla/scylla.yaml.example' with the full list of supported configuration 72 | # options and their descriptions.\n"""[1:])) 73 | return yaml.dump(data=self.scylla_yaml, stream=scylla_yaml_file) 74 | 75 | def updated_ami_conf_defaults(self): 76 | private_ip = self.cloud_instance.private_ipv4() 77 | self.CONF_DEFAULTS["scylla_yaml"]["listen_address"] = private_ip 78 | self.CONF_DEFAULTS["scylla_yaml"]["broadcast_rpc_address"] = private_ip 79 | self.CONF_DEFAULTS["scylla_yaml"]["seed_provider"][0]['parameters'][0]['seeds'] = private_ip 80 | self.CONF_DEFAULTS["scylla_yaml"]["endpoint_snitch"] = self.cloud_instance.endpoint_snitch 81 | 82 | def configure_scylla_yaml(self): 83 | self.updated_ami_conf_defaults() 84 | LOGGER.info("Going to create scylla.yaml...") 85 | new_scylla_yaml_config = self.instance_user_data.get("scylla_yaml", {}) 86 | if new_scylla_yaml_config: 87 | LOGGER.info("Setting params from user-data...") 88 | for param in new_scylla_yaml_config: 89 | param_value = new_scylla_yaml_config[param] 90 | LOGGER.info("Setting {param}={param_value}".format(**locals())) 91 | self.scylla_yaml[param] = param_value 92 | 93 | for param in self.CONF_DEFAULTS["scylla_yaml"]: 94 | if param not in new_scylla_yaml_config: 95 | default_param_value = self.CONF_DEFAULTS["scylla_yaml"][param] 96 | LOGGER.info("Setting default {param}={default_param_value}".format(**locals())) 97 | self.scylla_yaml[param] = default_param_value 98 | self.scylla_yaml_path.rename(str(self.scylla_yaml_example_path)) 99 | self.save_scylla_yaml() 100 | 101 | def configure_scylla_startup_args(self): 102 | default_scylla_startup_args = self.CONF_DEFAULTS["scylla_startup_args"] 103 | if self.instance_user_data.get("scylla_startup_args", default_scylla_startup_args): 104 | LOGGER.warning("Setting of Scylla startup args currently unsupported") 105 | 106 | def set_developer_mode(self): 107 | default_developer_mode = self.CONF_DEFAULTS["developer_mode"] 108 | if self.instance_user_data.get("developer_mode", default_developer_mode) or self.cloud_instance.is_dev_instance_type(): 109 | LOGGER.info("Setting up developer mode") 110 | subprocess.run(['/usr/sbin/scylla_dev_mode_setup', '--developer-mode', '1'], timeout=60, check=True) 111 | 112 | def run_post_configuration_script(self): 113 | post_configuration_script = self.instance_user_data.get("post_configuration_script") 114 | if post_configuration_script: 115 | try: 116 | default_timeout = self.CONF_DEFAULTS["post_configuration_script_timeout"] 117 | script_timeout = self.instance_user_data.get("post_configuration_script_timeout", default_timeout) 118 | try: 119 | decoded_script = base64.b64decode(post_configuration_script) 120 | except binascii.Error: 121 | decoded_script = post_configuration_script 122 | LOGGER.info("Running post configuration script:\n%s", decoded_script) 123 | subprocess.run(decoded_script, check=True, shell=True, timeout=int(script_timeout)) 124 | except Exception as e: 125 | scylla_excepthook(*sys.exc_info()) 126 | LOGGER.error("Post configuration script failed: %s", e) 127 | sys.exit(1) 128 | 129 | def start_scylla_on_first_boot(self): 130 | default_start_scylla_on_first_boot = self.CONF_DEFAULTS["start_scylla_on_first_boot"] 131 | if not self.instance_user_data.get("start_scylla_on_first_boot", default_start_scylla_on_first_boot): 132 | LOGGER.info("Disabling Scylla start on first boot") 133 | subprocess.run("/usr/bin/systemctl stop scylla-server.service", shell=True, check=True) 134 | 135 | def create_devices(self): 136 | device_type = self.instance_user_data.get("data_device", self.CONF_DEFAULTS['data_device']) 137 | raid_level = self.instance_user_data.get("raid_level", self.CONF_DEFAULTS['raid_level']) 138 | cmd_create_devices = f"/opt/scylladb/scylla-machine-image/scylla_create_devices --data-device {device_type} --raid-level {raid_level}" 139 | try: 140 | LOGGER.info(f"Create scylla data devices as {device_type}") 141 | subprocess.run(cmd_create_devices, shell=True, check=True) 142 | except Exception as e: 143 | if self.cloud_instance.is_dev_instance_type(): 144 | LOGGER.info("Skipping to create devices: %s", e) 145 | else: 146 | scylla_excepthook(*sys.exc_info()) 147 | LOGGER.error("Failed to create devices: %s", e) 148 | sys.exit(1) 149 | 150 | def configure(self): 151 | self.configure_scylla_yaml() 152 | self.configure_scylla_startup_args() 153 | self.set_developer_mode() 154 | self.run_post_configuration_script() 155 | self.start_scylla_on_first_boot() 156 | self.create_devices() 157 | 158 | 159 | if __name__ == "__main__": 160 | setup_logging() 161 | smi_configurator = ScyllaMachineImageConfigurator() 162 | smi_configurator.configure() 163 | -------------------------------------------------------------------------------- /common/scylla_create_devices: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright 2020 ScyllaDB 4 | # 5 | # SPDX-License-Identifier: Apache-2.0 6 | 7 | import argparse 8 | import sys 9 | from pathlib import Path 10 | from subprocess import run 11 | from lib.scylla_cloud import is_ec2, is_gce, is_azure, get_cloud_instance, out 12 | 13 | class DiskIsNotEmptyError(Exception): 14 | def __init__(self, disk): 15 | self.disk = disk 16 | pass 17 | 18 | def __str__(self): 19 | return f"{self.disk} is not empty, abort setup" 20 | 21 | class NoDiskFoundError(Exception): 22 | def __init__(self): 23 | pass 24 | 25 | def __str__(self): 26 | return "No data disk found, abort setup" 27 | 28 | def create_raid(devices, raid_level=0): 29 | if len(devices) == 0: 30 | raise(NoDiskFoundError) 31 | scylla_path = Path("/var/lib/scylla") 32 | print(f"Devices: {devices}") 33 | if scylla_path.is_mount(): 34 | print(f"{scylla_path} is already mounted. Will not run 'scylla_raid_setup'!") 35 | sys.exit(0) 36 | run(["/opt/scylladb/scripts/scylla_raid_setup", "--raiddev", "/dev/md0", "--disks", ",".join(devices), 37 | "--root", "/var/lib/scylla", "--raid-level", f"{raid_level}", "--volume-role", "all", "--update-fstab"], check=True) 38 | 39 | 40 | def check_persistent_disks_are_empty(disks): 41 | for disk in disks: 42 | part = out(f'lsblk -dpnr -o PTTYPE /dev/{disk}') 43 | fs = out(f'lsblk -dpnr -o FSTYPE /dev/{disk}') 44 | if part != '' or fs != '': 45 | raise DiskIsNotEmptyError(f'/dev/{disk}') 46 | 47 | 48 | def get_disk_devices(instance, device_type): 49 | if is_ec2(): 50 | devices = [] 51 | if device_type == "attached": 52 | check_persistent_disks_are_empty(instance.get_remote_disks()) 53 | devices = [str(Path('/dev', name)) for name in instance.get_remote_disks() if Path('/dev', name).exists()] 54 | if not devices or device_type == "instance_store": 55 | devices = [str(Path('/dev', name)) for name in instance.get_local_disks()] 56 | if not devices: 57 | raise Exception(f"No block devices were found for '{device_type}' device type") 58 | return devices 59 | elif is_gce(): 60 | return get_default_devices(instance) 61 | elif is_azure(): 62 | return get_default_devices(instance) 63 | else: 64 | raise Exception("Running in unknown cloud environment") 65 | 66 | 67 | def get_default_devices(instance): 68 | disk_names = [] 69 | disk_names = instance.get_local_disks() 70 | if not disk_names: 71 | disk_names = instance.get_remote_disks() 72 | check_persistent_disks_are_empty(disk_names) 73 | return [str(Path('/dev', name)) for name in disk_names] 74 | 75 | 76 | if __name__ == "__main__": 77 | parser = argparse.ArgumentParser(description='Disk creation script for Scylla.') 78 | parser.add_argument('--data-device', dest='data_device', action='store', 79 | choices=["auto", "attached", "instance_store"], 80 | help='Define type of device to use for scylla data: attached|instance_store') 81 | parser.add_argument('--raid-level', dest='raid_level', action='store', 82 | choices=[0, 5], default=0, type=int, 83 | help='Define raid level to use: RAID0 or RAID5') 84 | args = parser.parse_args() 85 | 86 | instance = get_cloud_instance() 87 | 88 | try: 89 | if not args.data_device or args.data_device == "auto": 90 | disk_devices = get_default_devices(instance) 91 | else: 92 | disk_devices = get_disk_devices(instance, args.data_device) 93 | create_raid(disk_devices, args.raid_level) 94 | except (DiskIsNotEmptyError, NoDiskFoundError) as e: 95 | print(e) 96 | sys.exit(1) 97 | -------------------------------------------------------------------------------- /common/scylla_ec2_check: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2022 ScyllaDB 5 | # 6 | # SPDX-License-Identifier: Apache-2.0 7 | 8 | import os 9 | import sys 10 | import re 11 | import argparse 12 | from lib.scylla_cloud import is_ec2, aws_instance, colorprint, out 13 | from subprocess import run 14 | 15 | if __name__ == '__main__': 16 | if not is_ec2(): 17 | sys.exit(0) 18 | parser = argparse.ArgumentParser(description='Verify EC2 configuration is optimized.') 19 | parser.add_argument('--nic', default='eth0', 20 | help='specify NIC') 21 | args = parser.parse_args() 22 | 23 | if not os.path.exists(f'/sys/class/net/{args.nic}'): 24 | print('NIC {} doesn\'t exist.'.format(args.nic)) 25 | sys.exit(1) 26 | 27 | aws = aws_instance() 28 | instance_class = aws.instance_class() 29 | en = aws.get_en_interface_type() 30 | match = re.search(r'^driver: (\S+)$', out(f'ethtool -i {args.nic}'), flags=re.MULTILINE) 31 | driver = match.group(1) 32 | 33 | if not en: 34 | colorprint('{red}{instance_class} doesn\'t support enhanced networking!{nocolor}', instance_class=instance_class) 35 | print('''To enable enhanced networking, please use the instance type which supports it. 36 | More documentation available at: 37 | http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/enhanced-networking.html#enabling_enhanced_networking''') 38 | sys.exit(1) 39 | elif not aws.is_vpc_enabled(args.nic): 40 | colorprint('{red}VPC is not enabled!{nocolor}') 41 | print('To enable enhanced networking, please enable VPC.') 42 | sys.exit(1) 43 | elif driver != en: 44 | colorprint('{red}Enhanced networking is disabled!{nocolor}') 45 | print('''More documentation available at: 46 | http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/enhanced-networking.html''') 47 | sys.exit(1) 48 | 49 | colorprint('{green}This EC2 instance is optimized for Scylla.{nocolor}') 50 | -------------------------------------------------------------------------------- /common/scylla_image_setup: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # 3 | # Copyright 2020 ScyllaDB 4 | # 5 | # SPDX-License-Identifier: Apache-2.0 6 | 7 | import os 8 | import sys 9 | from pathlib import Path 10 | from lib.scylla_cloud import get_cloud_instance, is_gce, is_azure, is_redhat_variant 11 | from subprocess import run 12 | 13 | if __name__ == '__main__': 14 | if is_azure(): 15 | swap_directory = Path('/mnt') 16 | swap_unit = Path('/etc/systemd/system/mnt-swapfile.swap') 17 | else: 18 | swap_directory = Path('/') 19 | swap_unit = Path('/etc/systemd/system/swapfile.swap') 20 | swapfile = swap_directory / 'swapfile' 21 | if not swapfile.exists(): 22 | swap_unit.unlink(missing_ok=True) 23 | run(f'/opt/scylladb/scripts/scylla_swap_setup --swap-directory {swap_directory}', shell=True, check=True) 24 | machine_image_configured = Path('/etc/scylla/machine_image_configured') 25 | if not machine_image_configured.exists(): 26 | run('/opt/scylladb/scripts/scylla_cpuscaling_setup', shell=True, check=True) 27 | cloud_instance = get_cloud_instance() 28 | run('/opt/scylladb/scylla-machine-image/scylla_configure.py', shell=True, check=True) 29 | 30 | run('/opt/scylladb/scripts/scylla_sysconfig_setup --nic eth0 --setup-nic', shell=True, check=True) 31 | if os.path.ismount('/var/lib/scylla'): 32 | if cloud_instance.is_supported_instance_class(): 33 | # We run io_setup only when ehpemeral disks are available 34 | if is_gce(): 35 | nr_disks = cloud_instance.nvme_disk_count 36 | if nr_disks > 0: 37 | cloud_instance.io_setup() 38 | else: 39 | cloud_instance.io_setup() 40 | run('systemctl daemon-reload', shell=True, check=True) 41 | run('systemctl enable var-lib-systemd-coredump.mount', shell=True, check=True) 42 | run('systemctl start var-lib-systemd-coredump.mount', shell=True, check=True) 43 | # some distro has fstrim enabled by default, since we are using XFS with online discard, we don't need fstrim 44 | run('systemctl is-active -q fstrim.timer && systemctl disable fstrim.timer', shell=True, check=True) 45 | 46 | if not os.path.ismount('/var/lib/scylla') and not cloud_instance.is_dev_instance_type(): 47 | print('Failed to initialize RAID volume!') 48 | machine_image_configured.touch() 49 | -------------------------------------------------------------------------------- /common/scylla_login: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # 3 | # Copyright 2020 ScyllaDB 4 | # 5 | # SPDX-License-Identifier: Apache-2.0 6 | 7 | 8 | import os 9 | import sys 10 | from lib.scylla_cloud import get_cloud_instance, colorprint, out 11 | from subprocess import run 12 | 13 | MSG_HEADER = r''' 14 | 15 | _____ _ _ _____ ____ 16 | / ____| | | | | __ \| _ \ 17 | | (___ ___ _ _| | | __ _| | | | |_) | 18 | \___ \ / __| | | | | |/ _` | | | | _ < 19 | ____) | (__| |_| | | | (_| | |__| | |_) | 20 | |_____/ \___|\__, |_|_|\__,_|_____/|____/ 21 | __/ | 22 | |___/ 23 | 24 | Version: 25 | {scylla_version} 26 | Nodetool: 27 | nodetool help 28 | CQL Shell: 29 | cqlsh 30 | More documentation available at: 31 | https://docs.scylladb.com/ 32 | By default, Scylla sends certain information about this node to a data collection server. For more details, see https://www.scylladb.com/privacy/ 33 | 34 | '''[1:-1] 35 | MSG_UNSUPPORTED_INSTANCE_TYPE = ''' 36 | {red}{type} is not eligible for optimized automatic tuning!{nocolor} 37 | 38 | To continue the setup of Scylla on this instance, run 'sudo scylla_io_setup' then 'sudo systemctl start scylla-server'. 39 | For a list of optimized instance types and more instructions, see the requirements section in the 40 | ScyllaDB documentation at https://docs.scylladb.com 41 | 42 | '''[1:-1] 43 | MSG_UNSUPPORTED_INSTANCE_TYPE_NO_DISKS = ''' 44 | {red}{type} is not eligible for optimized automatic tuning!{nocolor} 45 | 46 | To continue the setup of Scylla on this instance, you need to attach additional disks, next run 'sudo scylla_create_devices', 47 | 'sudo scylla_io_setup' then 'sudo systemctl start scylla-server'. 48 | For a list of optimized instance types and more instructions, see the requirements section in the 49 | ScyllaDB documentation at https://docs.scylladb.com 50 | '''[1:-1] 51 | MSG_DEV_INSTANCE_TYPE = ''' 52 | {yellow}WARNING: {type} is intended for development purposes only and should not be used in production environments! 53 | This ScyllaDB instance is running in developer-mode.{nocolor} 54 | 55 | For a list of optimized instance types and more instructions, see the requirements section in the 56 | ScyllaDB documentation at https://docs.scylladb.com 57 | '''[1:-1] 58 | MSG_SETUP_ACTIVATING = ''' 59 | {green}Constructing RAID volume...{nocolor} 60 | 61 | Please wait for the setup to finish. To see its status, run 62 | 'systemctl status scylla-image-setup' 63 | 64 | After the setup has finished, scylla-server service will launch. 65 | To see the status of scylla-server, run 66 | 'systemctl status scylla-server' 67 | 68 | '''[1:-1] 69 | MSG_SETUP_FAILED = ''' 70 | {red}Initial image configuration has failed!{nocolor} 71 | 72 | To see the status of setup service, run 73 | 'systemctl status scylla-image-setup' 74 | 75 | '''[1:-1] 76 | MSG_SCYLLA_ACTIVATING = ''' 77 | {green}Scylla is starting...{nocolor} 78 | 79 | Please wait for Scylla startup to finish. To see its status, run 80 | 'systemctl status scylla-server' 81 | 82 | '''[1:-1] 83 | MSG_SCYLLA_FAILED = ''' 84 | {red}Scylla has not started!{nocolor} 85 | 86 | To see the status of Scylla, run 87 | 'systemctl status scylla-server' 88 | 89 | '''[1:-1] 90 | MSG_SCYLLA_MOUNT_FAILED = ''' 91 | {red}Failed mounting RAID volume!{nocolor} 92 | 93 | Scylla has aborted startup because of a missing RAID volume. 94 | To see the status of Scylla, run 95 | 'systemctl status scylla-server' 96 | 97 | '''[1:-1] 98 | MSG_SCYLLA_UNKNOWN = ''' 99 | {red}Scylla has not started!{nocolor} 100 | 101 | To see the status of Scylla, run 102 | 'systemctl status scylla-server' 103 | 104 | '''[1:-1] 105 | MSG_SCYLLA_ACTIVE = ''' 106 | {green}Scylla is active.{nocolor} 107 | 108 | $ nodetool status 109 | 110 | '''[1:-1] 111 | 112 | if __name__ == '__main__': 113 | colorprint(MSG_HEADER.format(scylla_version=out("scylla --version"))) 114 | cloud_instance = get_cloud_instance() 115 | if cloud_instance.is_dev_instance_type(): 116 | colorprint(MSG_DEV_INSTANCE_TYPE, type=cloud_instance.instancetype) 117 | elif not cloud_instance.is_supported_instance_class(): 118 | non_root_disks = cloud_instance.get_local_disks() + cloud_instance.get_remote_disks() 119 | if len(non_root_disks) == 0: 120 | colorprint(MSG_UNSUPPORTED_INSTANCE_TYPE_NO_DISKS, type=cloud_instance.instance_class()) 121 | else: 122 | colorprint(MSG_UNSUPPORTED_INSTANCE_TYPE, type=cloud_instance.instance_class()) 123 | else: 124 | skip_scylla_server = False 125 | if not os.path.exists('/etc/scylla/machine_image_configured'): 126 | res = out('systemctl is-active scylla-image-setup.service', ignore_error=True) 127 | if res == 'activating': 128 | colorprint(MSG_SETUP_ACTIVATING) 129 | skip_scylla_server = True 130 | elif res == 'failed': 131 | colorprint(MSG_SETUP_FAILED) 132 | skip_scylla_server = True 133 | if not skip_scylla_server: 134 | res = out('systemctl is-active scylla-server.service', ignore_error=True) 135 | if res == 'activating': 136 | colorprint(MSG_SCYLLA_ACTIVATING) 137 | elif res == 'failed': 138 | colorprint(MSG_SCYLLA_FAILED) 139 | elif res == 'inactive': 140 | if os.path.exists('/etc/systemd/system/scylla-server.service.d/mounts.conf'): 141 | colorprint(MSG_SCYLLA_MOUNT_FAILED) 142 | else: 143 | colorprint(MSG_SCYLLA_UNKNOWN) 144 | else: 145 | colorprint(MSG_SCYLLA_ACTIVE) 146 | run('nodetool status', shell=True) 147 | cloud_instance.check() 148 | print('\n', end='') 149 | -------------------------------------------------------------------------------- /common/scylla_post_start.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright 2021 ScyllaDB 4 | # 5 | # SPDX-License-Identifier: Apache-2.0 6 | 7 | import base64 8 | import logging 9 | import pathlib 10 | import subprocess 11 | import sys 12 | 13 | from lib.log import setup_logging 14 | from lib.user_data import UserData 15 | from lib.scylla_cloud import scylla_excepthook 16 | 17 | LOGGER = logging.getLogger(__name__) 18 | DISABLE_FILE_PATH = '/etc/scylla/machine_image_post_start_configured' 19 | 20 | 21 | class ScyllaMachineImagePostStart(UserData): 22 | def run_post_start_script(self): 23 | post_start_script = self.instance_user_data.get("post_start_script") 24 | if post_start_script: 25 | try: 26 | decoded_script = base64.b64decode(post_start_script) 27 | LOGGER.info(f"Running post start script:\n{decoded_script}") 28 | subprocess.run(decoded_script, check=True, shell=True, timeout=600) 29 | return True 30 | except Exception as e: 31 | scylla_excepthook(*sys.exc_info()) 32 | LOGGER.error(f"Post start script failed: {e}") 33 | sys.exit(1) 34 | 35 | 36 | if __name__ == "__main__": 37 | setup_logging() 38 | init = ScyllaMachineImagePostStart() 39 | if init.run_post_start_script(): 40 | pathlib.Path(DISABLE_FILE_PATH).touch() 41 | -------------------------------------------------------------------------------- /dist/debian/build_deb.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Copyright 2021 ScyllaDB 4 | # 5 | # SPDX-License-Identifier: Apache-2.0 6 | 7 | . /etc/os-release 8 | 9 | is_redhat() { 10 | [ -f /etc/redhat-release ] 11 | } 12 | 13 | is_debian() { 14 | [ -f /etc/debian_version ] 15 | } 16 | 17 | print_usage() { 18 | echo "build_deb.sh" 19 | echo " no option available for this script." 20 | exit 1 21 | } 22 | 23 | while [ $# -gt 0 ]; do 24 | case "$1" in 25 | *) 26 | print_usage 27 | ;; 28 | esac 29 | done 30 | 31 | apt_updated=false 32 | 33 | # On clean Docker sudo command is not installed 34 | if is_redhat && ! rpm -q sudo; then 35 | yum install -y sudo 36 | elif is_debian && ! dpkg -s sudo > /dev/null 2>&1; then 37 | apt-get update 38 | apt_updated=true 39 | apt-get install -y sudo 40 | fi 41 | 42 | yum_install() { 43 | if ! rpm -q $1; then 44 | sudo yum install -y $1 45 | else 46 | echo "$1 already installed." 47 | fi 48 | } 49 | 50 | apt_install() { 51 | if ! dpkg -s $1 > /dev/null 2>&1; then 52 | if ! $apt_updated; then 53 | sudo apt-get update 54 | apt_updated=true 55 | fi 56 | sudo env DEBIAN_FRONTEND=noninteractive apt-get install -y $1 57 | else 58 | echo "$1 already installed." 59 | fi 60 | } 61 | 62 | pkg_install() { 63 | if is_redhat; then 64 | yum_install $1 65 | elif is_debian; then 66 | apt_install ${1/-devel/-dev} 67 | fi 68 | } 69 | 70 | if [[ ! -e dist/debian/build_deb.sh ]]; then 71 | echo "run build_deb.sh in top of scylla-machine-image dir" 72 | exit 1 73 | fi 74 | 75 | pkg_install devscripts 76 | pkg_install debhelper 77 | pkg_install fakeroot 78 | pkg_install dpkg-dev 79 | pkg_install git 80 | pkg_install python3 81 | pkg_install python3-devel 82 | pkg_install python3-pip 83 | 84 | echo "Building in $PWD..." 85 | 86 | VERSION=$(./SCYLLA-VERSION-GEN) 87 | SCYLLA_VERSION=$(sed 's/-/~/' build/SCYLLA-VERSION-FILE) 88 | SCYLLA_RELEASE=$(cat build/SCYLLA-RELEASE-FILE) 89 | PRODUCT=$(cat build/SCYLLA-PRODUCT-FILE) 90 | BUILDDIR=build/debian 91 | PACKAGE_NAME="$PRODUCT-machine-image" 92 | 93 | rm -rf "$BUILDDIR" 94 | mkdir -p "$BUILDDIR"/scylla-machine-image 95 | 96 | git archive --format=tar.gz HEAD -o "$BUILDDIR"/"$PACKAGE_NAME"_"$SCYLLA_VERSION"-"$SCYLLA_RELEASE".orig.tar.gz 97 | cd "$BUILDDIR"/scylla-machine-image 98 | tar -C ./ -xpf ../"$PACKAGE_NAME"_"$SCYLLA_VERSION"-"$SCYLLA_RELEASE".orig.tar.gz 99 | cd - 100 | ./dist/debian/debian_files_gen.py 101 | cd "$BUILDDIR"/scylla-machine-image 102 | debuild -rfakeroot -us -uc 103 | -------------------------------------------------------------------------------- /dist/debian/changelog.template: -------------------------------------------------------------------------------- 1 | %{product}-machine-image (%{version}-%{release}-%{revision}) %{codename}; urgency=medium 2 | 3 | * Initial release. 4 | 5 | -- Takuya ASADA Mon, 24 Aug 2015 09:22:55 +0000 6 | -------------------------------------------------------------------------------- /dist/debian/control.template: -------------------------------------------------------------------------------- 1 | Source: %{product}-machine-image 2 | Maintainer: Takuya ASADA 3 | Homepage: http://scylladb.com 4 | Section: database 5 | Priority: optional 6 | Standards-Version: 3.9.5 7 | Rules-Requires-Root: no 8 | 9 | Package: %{product}-machine-image 10 | Architecture: all 11 | Depends: %{product}, %{product}-python3, ${shlibs:Depends}, ${misc:Depends} 12 | Replaces: scylla-enterprise-machine-image (<< 2025.1.0~) 13 | Breaks: scylla-enterprise-machine-image (<< 2025.1.0~) 14 | Description: Scylla Machine Image 15 | Scylla is a highly scalable, eventually consistent, distributed, 16 | partitioned row DB. 17 | 18 | Package: scylla-enterprise-machine-image 19 | Depends: %{product}-machine-image (= ${binary:Version}) 20 | Architecture: all 21 | Priority: optional 22 | Section: oldlibs 23 | Description: transitional package 24 | This is a transitional package. It can safely be removed. 25 | -------------------------------------------------------------------------------- /dist/debian/debian/compat: -------------------------------------------------------------------------------- 1 | 9 2 | -------------------------------------------------------------------------------- /dist/debian/debian/copyright: -------------------------------------------------------------------------------- 1 | Format: http://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ 2 | Upstream-Name: Scylla DB 3 | Upstream-Contact: http://www.scylladb.com/ 4 | Source: https://github.com/scylladb/scylla-jmx 5 | 6 | Files: * 7 | Copyright: Copyright (C) 2015 ScyllaDB 8 | License: Apache-2.0 9 | 10 | Files: debian/* 11 | Copyright: Copyright (C) 2015 ScyllaDB 12 | License: Apache-2.0 13 | 14 | Files: scripts/git-archive-all 15 | Copyright: Copyright (c) 2010 Ilya Kulakov 16 | License: MIT 17 | -------------------------------------------------------------------------------- /dist/debian/debian/rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/make -f 2 | 3 | include /usr/share/dpkg/pkg-info.mk 4 | 5 | override_dh_auto_build: 6 | 7 | override_dh_auto_clean: 8 | 9 | override_dh_auto_install: 10 | install -d -m755 $(CURDIR)/debian/tmp/opt/scylladb/scylla-machine-image/lib 11 | install -m644 lib/log.py lib/scylla_cloud.py lib/user_data.py $(CURDIR)/debian/tmp/opt/scylladb/scylla-machine-image/lib 12 | install -m644 common/aws_io_params.yaml $(CURDIR)/debian/tmp/opt/scylladb/scylla-machine-image 13 | install -m755 common/scylla_configure.py common/scylla_post_start.py common/scylla_create_devices $(CURDIR)/debian/tmp/opt/scylladb/scylla-machine-image 14 | ./tools/relocate_python_scripts.py \ 15 | --installroot $(CURDIR)/debian/tmp/opt/scylladb/scylla-machine-image/ \ 16 | --with-python3 $(CURDIR)/debian/tmp/opt/scylladb/python3/bin/python3 \ 17 | common/scylla_image_setup common/scylla_login common/scylla_configure.py \ 18 | common/scylla_create_devices common/scylla_post_start.py \ 19 | common/scylla_cloud_io_setup common/scylla_ec2_check 20 | 21 | override_dh_installinit: 22 | dh_installinit --no-start --name scylla-image-setup 23 | dh_installinit --no-start --name scylla-image-post-start 24 | 25 | override_dh_auto_test: 26 | 27 | override_dh_strip_nondeterminism: 28 | 29 | %: 30 | dh $@ 31 | -------------------------------------------------------------------------------- /dist/debian/debian/scylla-image-post-start.service: -------------------------------------------------------------------------------- 1 | ../../../common/scylla-image-post-start.service -------------------------------------------------------------------------------- /dist/debian/debian/scylla-image-setup.service: -------------------------------------------------------------------------------- 1 | ../../../common/scylla-image-setup.service -------------------------------------------------------------------------------- /dist/debian/debian/scylla-machine-image.install: -------------------------------------------------------------------------------- 1 | opt/scylladb/scylla-machine-image/* 2 | -------------------------------------------------------------------------------- /dist/debian/debian/scylla-machine-image.postinst: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ -d /run/systemd/system ]; then 4 | systemctl --system daemon-reload >/dev/null || true 5 | fi 6 | 7 | #DEBHELPER# 8 | -------------------------------------------------------------------------------- /dist/debian/debian/scylla-machine-image.postrm: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | 3 | if [ -d /run/systemd/system ]; then 4 | systemctl --system daemon-reload >/dev/null || true 5 | fi 6 | 7 | #DEBHELPER# 8 | -------------------------------------------------------------------------------- /dist/debian/debian/source/format: -------------------------------------------------------------------------------- 1 | 3.0 (quilt) 2 | -------------------------------------------------------------------------------- /dist/debian/debian/source/options: -------------------------------------------------------------------------------- 1 | extend-diff-ignore = ^build/ 2 | -------------------------------------------------------------------------------- /dist/debian/debian_files_gen.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright (C) 2021 ScyllaDB 5 | # 6 | 7 | # SPDX-License-Identifier: Apache-2.0 8 | 9 | import string 10 | import os 11 | import shutil 12 | import re 13 | from pathlib import Path 14 | 15 | class DebianFilesTemplate(string.Template): 16 | delimiter = '%' 17 | 18 | scriptdir = os.path.dirname(__file__) 19 | 20 | with open(os.path.join(scriptdir, 'changelog.template')) as f: 21 | changelog_template = f.read() 22 | 23 | with open(os.path.join(scriptdir, 'control.template')) as f: 24 | control_template = f.read() 25 | 26 | with open('build/SCYLLA-PRODUCT-FILE') as f: 27 | product = f.read().strip() 28 | 29 | with open('build/SCYLLA-VERSION-FILE') as f: 30 | version = f.read().strip().replace('-', '~') 31 | with open('build/SCYLLA-RELEASE-FILE') as f: 32 | release = f.read().strip() 33 | 34 | if os.path.exists('build/debian/scylla-machine-image/debian'): 35 | shutil.rmtree('build/debian/scylla-machine-image/debian') 36 | shutil.copytree('dist/debian/debian', 'build/debian/scylla-machine-image/debian') 37 | 38 | if product != 'scylla': 39 | # Unlike other packages, scylla-machine-image is not relocatable package, 40 | # so we don't generate debian direcotry on build/debian/debian 41 | # to relocatable tar.gz 42 | for p in Path('build/debian/scylla-machine-image/debian').glob('scylla-*'): 43 | # pat1: scylla-server.service 44 | # -> scylla-enterprise-server.scylla-server.service 45 | # pat2: scylla-server.scylla-fstrim.service 46 | # -> scylla-enterprise-server.scylla-fstrim.service 47 | # pat3: scylla-conf.install 48 | # -> scylla-enterprise-conf.install 49 | 50 | if m := re.match(r'^scylla(-[^.]+)\.service$', p.name): 51 | p.rename(p.parent / f'{p.name}') 52 | elif m := re.match(r'^scylla(-[^.]+\.scylla-[^.]+\.[^.]+)$', p.name): 53 | p.rename(p.parent / f'{product}{m.group(1)}') 54 | else: 55 | p.rename(p.parent / p.name.replace('scylla', product, 1)) 56 | 57 | s = DebianFilesTemplate(changelog_template) 58 | changelog_applied = s.substitute(product=product, version=version, release=release, revision='1', codename='stable') 59 | 60 | s = DebianFilesTemplate(control_template) 61 | control_applied = s.substitute(product=product) 62 | 63 | with open('build/debian/scylla-machine-image/debian/changelog', 'w') as f: 64 | f.write(changelog_applied) 65 | 66 | with open('build/debian/scylla-machine-image/debian/control', 'w') as f: 67 | f.write(control_applied) 68 | -------------------------------------------------------------------------------- /dist/redhat/build_rpm.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Copyright 2020 ScyllaDB 4 | # 5 | # SPDX-License-Identifier: Apache-2.0 6 | 7 | . /etc/os-release 8 | 9 | TARGET= 10 | 11 | print_usage() { 12 | echo "build_rpm.sh -t [centos7/8|redhat]" 13 | echo " -t target target distribution" 14 | exit 1 15 | } 16 | while getopts t:c: option 17 | do 18 | case "${option}" 19 | in 20 | t) TARGET=${OPTARG};; 21 | *) print_usage;; 22 | esac 23 | done 24 | 25 | 26 | 27 | if [[ -n "${TARGET}" ]] ; then 28 | echo ${TARGET} 29 | else 30 | echo "please provide valid target (-t)" 31 | exit 1 32 | fi 33 | 34 | if [[ ! -f /etc/redhat-release ]]; then 35 | echo "Need Redhat like OS to build RPM" 36 | fi 37 | 38 | # Centos8 is EOL, need to change mirrors 39 | if [ "$VERSION" = "8" ] ; then 40 | sed -i 's/mirrorlist/#mirrorlist/g' /etc/yum.repos.d/CentOS-Linux-* ; 41 | sed -i 's|#baseurl=http://mirror.centos.org|baseurl=http://vault.centos.org|g' /etc/yum.repos.d/CentOS-Linux-* ; 42 | fi 43 | 44 | # On clean CentOS Docker sudo command is not installed 45 | if ! rpm -q sudo; then 46 | yum install -y sudo 47 | fi 48 | 49 | 50 | pkg_install() { 51 | if ! rpm -q $1; then 52 | sudo yum install -y $1 53 | else 54 | echo "$1 already installed." 55 | fi 56 | } 57 | 58 | if [[ ! -e dist/redhat/build_rpm.sh ]]; then 59 | echo "run build_rpm.sh in top of scylla-machine-image dir" 60 | exit 1 61 | fi 62 | 63 | pkg_install rpm-build 64 | pkg_install git 65 | pkg_install python3 66 | 67 | echo "Building in $PWD..." 68 | 69 | VERSION=$(./SCYLLA-VERSION-GEN) 70 | SCYLLA_VERSION=$(sed 's/-/~/' build/SCYLLA-VERSION-FILE) 71 | SCYLLA_RELEASE=$(cat build/SCYLLA-RELEASE-FILE) 72 | PRODUCT=$(cat build/SCYLLA-PRODUCT-FILE) 73 | 74 | PACKAGE_NAME="$PRODUCT-machine-image" 75 | 76 | RPMBUILD=$(readlink -f build/) 77 | mkdir -pv ${RPMBUILD}/{BUILD,BUILDROOT,RPMS,SOURCES,SPECS,SRPMS} 78 | 79 | git archive --format=tar --prefix=$PACKAGE_NAME-$SCYLLA_VERSION/ HEAD -o $RPMBUILD/SOURCES/$PACKAGE_NAME-$SCYLLA_VERSION-$SCYLLA_RELEASE.tar 80 | cp dist/redhat/scylla-machine-image.spec $RPMBUILD/SPECS/$PACKAGE_NAME.spec 81 | 82 | parameters=( 83 | -D"product $PRODUCT" 84 | -D"version $SCYLLA_VERSION" 85 | -D"release $SCYLLA_RELEASE" 86 | -D"package_name $PACKAGE_NAME" 87 | -D"scylla true" 88 | ) 89 | 90 | if [[ "$TARGET" = "centos7" ]] || [[ "$TARGET" = "centos8" ]]; then 91 | rpmbuild "${parameters[@]}" -ba --define '_binary_payload w2.xzdio' --define "_topdir $RPMBUILD" --define "dist .el7" $RPM_JOBS_OPTS $RPMBUILD/SPECS/$PACKAGE_NAME.spec 92 | else 93 | rpmbuild "${parameters[@]}" -ba --define '_binary_payload w2.xzdio' --define "_topdir $RPMBUILD" $RPM_JOBS_OPTS $RPMBUILD/SPECS/$PACKAGE_NAME.spec 94 | fi 95 | -------------------------------------------------------------------------------- /dist/redhat/scylla-machine-image.spec: -------------------------------------------------------------------------------- 1 | Name: %{package_name} 2 | Version: %{version} 3 | Release: %{release} 4 | Summary: Scylla Machine Image 5 | Group: Applications/Databases 6 | 7 | License: Apache-2.0 8 | URL: http://www.scylladb.com/ 9 | Source0: %{name}-%{version}-%{release}.tar 10 | Requires: %{product} = %{version} %{product}-python3 curl 11 | Provides: scylla-enterprise-machine-image = %{version}-%{release} 12 | Obsoletes: scylla-enterprise-machine-image < 2025.1.0 13 | 14 | BuildArch: noarch 15 | 16 | %global _python_bytecompile_errors_terminate_build 0 17 | %global __brp_python_bytecompile %{nil} 18 | %global __brp_mangle_shebangs %{nil} 19 | 20 | %description 21 | 22 | 23 | %prep 24 | %setup -q 25 | 26 | 27 | %build 28 | 29 | %install 30 | rm -rf $RPM_BUILD_ROOT 31 | 32 | install -d m755 $RPM_BUILD_ROOT%{_unitdir} 33 | install -m644 common/scylla-image-setup.service common/scylla-image-post-start.service $RPM_BUILD_ROOT%{_unitdir}/ 34 | install -d -m755 $RPM_BUILD_ROOT/opt/scylladb 35 | install -d -m755 $RPM_BUILD_ROOT/opt/scylladb/scylla-machine-image 36 | install -d -m755 $RPM_BUILD_ROOT/opt/scylladb/scylla-machine-image/lib 37 | install -m644 lib/log.py lib/scylla_cloud.py lib/user_data.py $RPM_BUILD_ROOT/opt/scylladb/scylla-machine-image/lib 38 | install -m644 common/aws_io_params.yaml $RPM_BUILD_ROOT/opt/scylladb/scylla-machine-image/ 39 | install -m755 common/scylla_configure.py common/scylla_post_start.py common/scylla_create_devices \ 40 | $RPM_BUILD_ROOT/opt/scylladb/scylla-machine-image/ 41 | ./tools/relocate_python_scripts.py \ 42 | --installroot $RPM_BUILD_ROOT/opt/scylladb/scylla-machine-image/ \ 43 | --with-python3 ${RPM_BUILD_ROOT}/opt/scylladb/python3/bin/python3 \ 44 | common/scylla_image_setup common/scylla_login common/scylla_configure.py \ 45 | common/scylla_create_devices common/scylla_post_start.py \ 46 | common/scylla_cloud_io_setup common/scylla_ec2_check 47 | 48 | %pre 49 | /usr/sbin/groupadd scylla 2> /dev/null || : 50 | /usr/sbin/useradd -g scylla -s /sbin/nologin -r -d ${_sharedstatedir}/scylla scylla 2> /dev/null || : 51 | 52 | %post 53 | %systemd_post scylla-image-setup.service 54 | %systemd_post scylla-image-post-start.service 55 | 56 | %preun 57 | %systemd_preun scylla-image-setup.service 58 | %systemd_preun scylla-image-post-start.service 59 | 60 | %postun 61 | %systemd_postun scylla-image-setup.service 62 | %systemd_postun scylla-image-post-start.service 63 | 64 | %posttrans 65 | if [ -L /home/scyllaadm/.bash_profile ] && [ ! -e /home/scyllaadm/.bash_profile ]; then 66 | rm /home/scyllaadm/.bash_profile 67 | cp /etc/skel/.bash_profile /home/scyllaadm/ 68 | chown scyllaadm:scyllaadm /home/scyllaadm/.bash_profile 69 | echo -e '\n' >> /home/scyllaadm/.bash_profile 70 | echo "/opt/scylladb/scylla-machine-image/scylla_login" >> /home/scyllaadm/.bash_profile 71 | fi 72 | 73 | %clean 74 | rm -rf $RPM_BUILD_ROOT 75 | 76 | 77 | %files 78 | %license LICENSE 79 | %defattr(-,root,root) 80 | 81 | %{_unitdir}/scylla-image-setup.service 82 | %{_unitdir}/scylla-image-post-start.service 83 | /opt/scylladb/scylla-machine-image/* 84 | 85 | %changelog 86 | * Sun Nov 1 2020 Bentsi Magidovich 87 | - generalize scylla_create_devices 88 | * Sun Jun 28 2020 Bentsi Magidovich 89 | - generalize code and support GCE image 90 | * Wed Nov 20 2019 Bentsi Magidovich 91 | - Rename package to scylla-machine-image 92 | * Mon Aug 20 2018 Takuya ASADA 93 | - inital version of scylla-ami.spec 94 | 95 | -------------------------------------------------------------------------------- /lib/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scylladb/scylla-machine-image/c32cb717a252f560383df463d7e2a89f8c86cfb2/lib/__init__.py -------------------------------------------------------------------------------- /lib/log.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # 3 | # Copyright 2020 ScyllaDB 4 | # 5 | # SPDX-License-Identifier: Apache-2.0 6 | 7 | import sys 8 | import logging 9 | import pathlib 10 | 11 | 12 | class ExitOnExceptionHandler(logging.StreamHandler): 13 | def emit(self, record): 14 | super().emit(record) 15 | # exit the application when logged message is above error logging level 16 | if record.levelno >= logging.ERROR: 17 | sys.exit(1) 18 | 19 | 20 | def setup_logging(log_level=logging.INFO, log_dir_path="/var/lib/scylla/logs"): 21 | log_dir = pathlib.Path(log_dir_path) 22 | log_path = log_dir / "smi.log" 23 | 24 | log_dir.mkdir(parents=True, exist_ok=True) 25 | root_logger = logging.getLogger() 26 | formatter = logging.Formatter("%(asctime)s - [%(module)s] - %(levelname)s - %(message)s") 27 | 28 | file_handler = logging.FileHandler(str(log_path)) 29 | file_handler.setFormatter(formatter) 30 | root_logger.addHandler(file_handler) 31 | 32 | console_handler = logging.StreamHandler(sys.stdout) 33 | console_handler.setFormatter(formatter) 34 | root_logger.addHandler(console_handler) 35 | root_logger.addHandler(ExitOnExceptionHandler()) 36 | root_logger.setLevel(log_level) 37 | -------------------------------------------------------------------------------- /lib/user_data.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2022 ScyllaDB 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | import logging 6 | from email import message_from_string 7 | 8 | import yaml 9 | 10 | LOGGER = logging.getLogger(__name__) 11 | 12 | 13 | class UserData: 14 | def __init__(self, *arg, **kwargs): 15 | self._instance_user_data = None 16 | self._cloud_instance = None 17 | 18 | @property 19 | def cloud_instance(self): 20 | if not self._cloud_instance: 21 | from lib.scylla_cloud import get_cloud_instance 22 | self._cloud_instance = get_cloud_instance() 23 | return self._cloud_instance 24 | 25 | @property 26 | def instance_user_data(self): 27 | if self._instance_user_data is None: 28 | try: 29 | raw_user_data = self.cloud_instance.user_data.strip() 30 | LOGGER.info("Got user-data: %s", raw_user_data) 31 | 32 | # Try reading mime multipart message, and extract 33 | # scylla-machine-image configuration out of it 34 | message = message_from_string(raw_user_data) 35 | if message.is_multipart(): 36 | for part in message.walk(): 37 | if part.get_content_type() in ('x-scylla/json', 'x-scylla/yaml'): 38 | # we'll pick here the last seen json or yaml file, 39 | # if multiple of them exists the last one wins, we are not merging them together 40 | raw_user_data = part.get_payload() 41 | 42 | # try parse yaml, and fallback to parsing json 43 | self._instance_user_data = {} 44 | if raw_user_data: 45 | self._instance_user_data = yaml.safe_load(raw_user_data) 46 | LOGGER.debug("parsed user-data: %s", self._instance_user_data) 47 | except Exception as e: 48 | LOGGER.warning("Error getting user data: %s. Will use defaults!", e) 49 | self._instance_user_data = {} 50 | return self._instance_user_data 51 | -------------------------------------------------------------------------------- /packer/ami_variables.json: -------------------------------------------------------------------------------- 1 | { 2 | "security_group_id": "sg-088128b2712c264d1", 3 | "region": "us-east-1", 4 | "associate_public_ip_address": "true", 5 | "instance_type": "c4.xlarge" 6 | } 7 | -------------------------------------------------------------------------------- /packer/apply_cis_rules: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020 ScyllaDB 5 | # 6 | # SPDX-License-Identifier: Apache-2.0 7 | 8 | import os 9 | import sys 10 | import re 11 | import argparse 12 | from subprocess import run 13 | 14 | if __name__ == '__main__': 15 | if os.getuid() > 0: 16 | print('Requires root permission.') 17 | sys.exit(1) 18 | 19 | parser = argparse.ArgumentParser() 20 | parser.add_argument('--target-cloud', choices=['aws', 'gce', 'azure'], help='specify target cloud') 21 | args = parser.parse_args() 22 | 23 | # xccdf_org.ssgproject.content_rule_grub2_audit_argument 24 | kernel_opt = 'audit=1' 25 | # xccdf_org.ssgproject.content_rule_grub2_audit_backlog_limit_argument 26 | kernel_opt += ' audit_backlog_limit=8192' 27 | if args.target_cloud == 'aws' or args.target_cloud == 'gce': 28 | grub_variable = 'GRUB_CMDLINE_LINUX_DEFAULT' 29 | elif args.target_cloud == 'azure': 30 | grub_variable = 'GRUB_CMDLINE_LINUX' 31 | with open('/etc/default/grub.d/50-cloudimg-settings.cfg') as f: 32 | grub = f.read() 33 | grub = re.sub(fr'^{grub_variable}="(.+)"$', 34 | fr'{grub_variable}="\1 {kernel_opt}"', grub, 35 | flags=re.MULTILINE) 36 | with open('/etc/default/grub.d/50-cloudimg-settings.cfg', 'w') as f: 37 | f.write(grub) 38 | run('update-grub2', shell=True, check=True) 39 | 40 | 41 | run('apt-get install -y auditd', shell=True, check=True) 42 | 43 | auditd_rules = ''' 44 | ## xccdf_org.ssgproject.content_rule_audit_rules_privileged_commands_insmod 45 | -w /sbin/insmod -p x -k modules 46 | 47 | ## xccdf_org.ssgproject.content_rule_audit_rules_privileged_commands_modprobe 48 | -w /sbin/modprobe -p x -k modules 49 | 50 | ## xccdf_org.ssgproject.content_rule_audit_rules_privileged_commands_rmmod 51 | -w /sbin/rmmod -p x -k modules 52 | 53 | ## xccdf_org.ssgproject.content_rule_audit_rules_mac_modification 54 | -w /etc/selinux/ -p wa -k MAC-policy 55 | 56 | ## xccdf_org.ssgproject.content_rule_audit_rules_networkconfig_modification 57 | -a always,exit -F arch=b32 -S sethostname,setdomainname -F key=audit_rules_networkconfig_modification 58 | -a always,exit -F arch=b64 -S sethostname,setdomainname -F key=audit_rules_networkconfig_modification 59 | -w /etc/issue -p wa -k audit_rules_networkconfig_modification 60 | -w /etc/issue.net -p wa -k audit_rules_networkconfig_modification 61 | -w /etc/hosts -p wa -k audit_rules_networkconfig_modification 62 | -w /etc/networks -p wa -k audit_rules_networkconfig_modification 63 | -w /etc/network/ -p wa -k audit_rules_networkconfig_modification 64 | 65 | ## xccdf_org.ssgproject.content_rule_audit_rules_session_events 66 | -w /var/run/utmp -p wa -k session 67 | -w /var/log/btmp -p wa -k session 68 | -w /var/log/wtmp -p wa -k session 69 | 70 | ## xccdf_org.ssgproject.content_rule_audit_rules_suid_privilege_function 71 | -a always,exit -F arch=b32 -S execve -C uid!=euid -F euid=0 -k setuid 72 | -a always,exit -F arch=b64 -S execve -C uid!=euid -F euid=0 -k setuid 73 | -a always,exit -F arch=b32 -S execve -C gid!=egid -F egid=0 -k setgid 74 | -a always,exit -F arch=b64 -S execve -C gid!=egid -F egid=0 -k setgid 75 | 76 | '''[1:-1] 77 | with open('/etc/audit/rules.d/70-cis-rules.rules', 'w') as f: 78 | f.write(auditd_rules) 79 | os.chmod('/etc/audit/rules.d/70-cis-rules.rules', 0o640) 80 | run('augenrules --load', shell=True, check=True) 81 | 82 | with open('/etc/audit/auditd.conf') as f: 83 | auditd = f.read() 84 | # xccdf_org.ssgproject.content_rule_auditd_data_retention_max_log_file_action 85 | auditd = re.sub(r'^max_log_file_action = .+$', 'max_log_file_action = KEEP_LOGS', auditd, flags=re.MULTILINE) 86 | # xccdf_org.ssgproject.content_rule_auditd_data_retention_space_left_action 87 | auditd = re.sub(r'^space_left_action = .+$', 'space_left_action = EMAIL', auditd, flags=re.MULTILINE) 88 | # xccdf_org.ssgproject.content_rule_auditd_data_retention_admin_space_left_action 89 | auditd = re.sub(r'^admin_space_left_action = .+$', 'admin_space_left_action = suspend', auditd, flags=re.MULTILINE) 90 | with open('/etc/audit/auditd.conf', 'w') as f: 91 | f.write(auditd) 92 | 93 | 94 | sysctl_conf = ''' 95 | # xccdf_org.ssgproject.content_rule_sysctl_net_ipv6_conf_all_accept_ra 96 | net.ipv6.conf.all.accept_ra = 0 97 | 98 | # xccdf_org.ssgproject.content_rule_sysctl_net_ipv6_conf_all_accept_redirects 99 | net.ipv6.conf.all.accept_redirects = 0 100 | 101 | # xccdf_org.ssgproject.content_rule_sysctl_net_ipv6_conf_all_accept_source_route 102 | net.ipv6.conf.all.accept_source_route = 0 103 | 104 | # xccdf_org.ssgproject.content_rule_sysctl_net_ipv6_conf_all_forwarding 105 | net.ipv6.conf.all.forwarding = 0 106 | 107 | # xccdf_org.ssgproject.content_rule_sysctl_net_ipv6_conf_default_accept_ra 108 | net.ipv6.conf.default.accept_ra = 0 109 | 110 | # xccdf_org.ssgproject.content_rule_sysctl_net_ipv6_conf_default_accept_redirects 111 | net.ipv6.conf.default.accept_redirects = 0 112 | 113 | # xccdf_org.ssgproject.content_rule_sysctl_net_ipv6_conf_default_accept_source_route 114 | net.ipv6.conf.default.accept_source_route = 0 115 | 116 | # xccdf_org.ssgproject.content_rule_sysctl_net_ipv4_conf_all_accept_redirects 117 | net.ipv4.conf.all.accept_redirects = 0 118 | 119 | # xccdf_org.ssgproject.content_rule_sysctl_net_ipv4_conf_all_accept_source_route 120 | net.ipv4.conf.all.accept_source_route = 0 121 | 122 | # xccdf_org.ssgproject.content_rule_sysctl_net_ipv4_conf_all_log_martians 123 | net.ipv4.conf.all.log_martians = 1 124 | 125 | # xccdf_org.ssgproject.content_rule_sysctl_net_ipv4_conf_all_rp_filter 126 | net.ipv4.conf.all.rp_filter = 1 127 | 128 | # xccdf_org.ssgproject.content_rule_sysctl_net_ipv4_conf_all_secure_redirects 129 | net.ipv4.conf.all.secure_redirects = 0 130 | 131 | # xccdf_org.ssgproject.content_rule_sysctl_net_ipv4_conf_default_accept_redirects 132 | net.ipv4.conf.default.accept_redirects = 0 133 | 134 | # xccdf_org.ssgproject.content_rule_sysctl_net_ipv4_conf_default_accept_source_route 135 | net.ipv4.conf.default.accept_source_route = 0 136 | 137 | # xccdf_org.ssgproject.content_rule_sysctl_net_ipv4_conf_default_log_martians 138 | net.ipv4.conf.default.log_martians = 1 139 | 140 | # xccdf_org.ssgproject.content_rule_sysctl_net_ipv4_conf_default_rp_filter 141 | net.ipv4.conf.default.rp_filter = 1 142 | 143 | # xccdf_org.ssgproject.content_rule_sysctl_net_ipv4_conf_default_secure_redirects 144 | net.ipv4.conf.default.secure_redirects = 0 145 | 146 | # xccdf_org.ssgproject.content_rule_sysctl_net_ipv4_icmp_echo_ignore_broadcasts 147 | net.ipv4.icmp_echo_ignore_broadcasts = 1 148 | 149 | # xccdf_org.ssgproject.content_rule_sysctl_net_ipv4_icmp_ignore_bogus_error_responses 150 | net.ipv4.icmp_ignore_bogus_error_responses = 1 151 | 152 | # xccdf_org.ssgproject.content_rule_sysctl_net_ipv4_tcp_syncookies 153 | net.ipv4.tcp_syncookies = 1 154 | 155 | # xccdf_org.ssgproject.content_rule_sysctl_net_ipv4_conf_all_send_redirects 156 | net.ipv4.conf.all.send_redirects = 0 157 | 158 | # xccdf_org.ssgproject.content_rule_sysctl_net_ipv4_conf_default_send_redirects 159 | net.ipv4.conf.default.send_redirects = 0 160 | 161 | # xccdf_org.ssgproject.content_rule_sysctl_net_ipv4_ip_forward 162 | net.ipv4.ip_forward = 0 163 | 164 | # xccdf_org.ssgproject.content_rule_sysctl_fs_suid_dumpable 165 | fs.suid_dumpable = 0 166 | 167 | # xccdf_org.ssgproject.content_rule_sysctl_kernel_randomize_va_space 168 | kernel.randomize_va_space = 2 169 | '''[1:-1] 170 | with open('/etc/sysctl.d/99-cis-rules.conf', 'w') as f: 171 | f.write(sysctl_conf) 172 | run('sysctl -p /etc/sysctl.d/99-cis-rules.conf', shell=True, check=True) 173 | 174 | 175 | # xccdf_org.ssgproject.content_rule_partition_for_tmp 176 | # xccdf_org.ssgproject.content_rule_mount_option_tmp_nodev 177 | # xccdf_org.ssgproject.content_rule_mount_option_tmp_noexec 178 | # xccdf_org.ssgproject.content_rule_mount_option_tmp_nosuid 179 | tmp_dot_mount = ''' 180 | [Unit] 181 | ConditionPathIsSymbolicLink=!/tmp 182 | DefaultDependencies=no 183 | Conflicts=umount.target 184 | Before=local-fs.target umount.target 185 | After=swap.target 186 | 187 | [Mount] 188 | What=tmpfs 189 | Where=/tmp 190 | Type=tmpfs 191 | Options=mode=1777,strictatime,nosuid,nodev,noexec,size=50%%,nr_inodes=1m 192 | 193 | [Install] 194 | WantedBy=local-fs.target 195 | '''[1:-1] 196 | with open('/etc/systemd/system/tmp.mount', 'w') as f: 197 | f.write(tmp_dot_mount) 198 | run('systemctl daemon-reload', shell=True, check=True) 199 | run('systemctl enable tmp.mount', shell=True, check=True) 200 | 201 | 202 | # xccdf_org.ssgproject.content_rule_mount_option_var_tmp_nodev 203 | # xccdf_org.ssgproject.content_rule_mount_option_var_tmp_noexec 204 | # xccdf_org.ssgproject.content_rule_mount_option_var_tmp_nosuid 205 | run('fallocate -l 1024MiB /vartmpfile', shell=True, check=True) 206 | os.chmod('/vartmpfile', 0o600) 207 | run('mke2fs -t ext4 /vartmpfile', shell=True, check=True) 208 | var_tmp_dot_mount = ''' 209 | [Unit] 210 | Before=local-fs.target 211 | Requires=-.mount 212 | After=-.mount 213 | 214 | [Mount] 215 | What=/vartmpfile 216 | Where=/var/tmp 217 | Type=ext4 218 | Options=strictatime,nosuid,nodev,noexec 219 | ReadWriteOnly=True 220 | 221 | [Install] 222 | WantedBy=multi-user.target 223 | '''[1:-1] 224 | with open('/etc/systemd/system/var-tmp.mount', 'w') as f: 225 | f.write(var_tmp_dot_mount) 226 | run('systemctl daemon-reload', shell=True, check=True) 227 | run('systemctl enable var-tmp.mount', shell=True, check=True) 228 | 229 | 230 | # xccdf_org.ssgproject.content_rule_sshd_use_strong_ciphers 231 | # xccdf_org.ssgproject.content_rule_sshd_use_strong_kex 232 | # xccdf_org.ssgproject.content_rule_sshd_use_strong_macs 233 | sshd_config = ''' 234 | Ciphers aes128-ctr,aes192-ctr,aes256-ctr,chacha20-poly1305@openssh.com,aes256-gcm@openssh.com,aes128-gcm@openssh.com 235 | KexAlgorithms ecdh-sha2-nistp256,ecdh-sha2-nistp384,ecdh-sha2-nistp521,diffie-hellman-group-exchange-sha256,diffie-hellman-group16-sha512,diffie-hellman-group18-sha512,diffie-hellman-group14-sha256 236 | MACs hmac-sha2-512-etm@openssh.com,hmac-sha2-256-etm@openssh.com,hmac-sha2-512,hmac-sha2-256 237 | '''[1:-1] 238 | with open('/etc/ssh/sshd_config.d/99-cis-rules.conf', 'w') as f: 239 | f.write(sshd_config) 240 | run('systemctl restart ssh.service', shell=True, check=True) 241 | -------------------------------------------------------------------------------- /packer/azure_variables.json: -------------------------------------------------------------------------------- 1 | { 2 | "client_id": "", 3 | "client_secret": "", 4 | "tenant_id": "", 5 | "subscription_id": "", 6 | "security_group_id": "", 7 | "region": "EAST US", 8 | "vm_size": "Standard_D4_v4" 9 | } 10 | -------------------------------------------------------------------------------- /packer/build_image.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash -e 2 | # 3 | # Copyright 2021 ScyllaDB 4 | # 5 | # SPDX-License-Identifier: Apache-2.0 6 | 7 | DIR=$(dirname $(readlink -f "$0")) 8 | source "$DIR"/../SCYLLA-VERSION-GEN 9 | 10 | CREATION_TIMESTAMP=$(date -u '+%FT%H-%M-%S') 11 | OPERATING_SYSTEM="ubuntu24.04" 12 | EXIT_STATUS=0 13 | DRY_RUN=false 14 | DEBUG=false 15 | BUILD_MODE='release' 16 | TARGET= 17 | ENV_TAG="debug" 18 | 19 | print_usage() { 20 | echo "$0 --repo [URL] --target [distribution]" 21 | echo " --repo Repository for both install and update, specify .repo/.list file URL" 22 | echo " --repo-for-install Repository for install, specify .repo/.list file URL" 23 | echo " --repo-for-update Repository for update, specify .repo/.list file URL" 24 | echo " [--product] scylla or scylla-enterprise, default from SCYLLA-PRODUCT-FILE" 25 | echo " [--dry-run] Validate template only (image is not built). Default: false" 26 | echo " [--scylla-build-sha-id] Scylla build SHA id form metadata file" 27 | echo " [--branch] Set the release branch for GCE label. Default: master" 28 | echo " [--ami-regions] Set regions to copy the AMI when done building it (including permissions and tags)" 29 | echo " [--build-tag] Jenkins Build tag" 30 | echo " [--env-tag] Environment tag for our images. default: debug. Valid options: daily(master)|candidate(releases)|production(releases)|private(custom images for customers)" 31 | echo " [--build-mode] Choose which build mode to use for Scylla installation. Default: release. Valid options: release|debug" 32 | echo " [--debug] Build debug image with special prefix for image name. Default: false." 33 | echo " [--log-file] Path for log. Default build/ami.log on current dir. Default: build/packer.log" 34 | echo " --target Target cloud (aws/gce/azure), mandatory when using this script directly, and not by soft links" 35 | echo " --arch Set the image build architecture. Valid options: x86_64 | aarch64 . if use didn't pass this parameter it will use local node architecture" 36 | echo " --ec2-instance-type Set EC2 instance type to use while building the AMI. If empty will use defaults per architecture" 37 | exit 1 38 | } 39 | PACKER_SUB_CMD="build" 40 | REPO_FOR_INSTALL= 41 | PACKER_LOG_PATH=build/packer.log 42 | 43 | while [ $# -gt 0 ]; do 44 | case "$1" in 45 | "--repo") 46 | REPO_FOR_INSTALL="https://$2" 47 | echo "--repo parameter: REPO_FOR_INSTALL $REPO_FOR_INSTALL" 48 | INSTALL_ARGS="$INSTALL_ARGS --repo https://$2" 49 | shift 2 50 | ;; 51 | "--repo-for-install") 52 | REPO_FOR_INSTALL=$2 53 | echo "--repo-for-install parameter: REPO_FOR_INSTALL $REPO_FOR_INSTALL" 54 | INSTALL_ARGS="$INSTALL_ARGS --repo-for-install $2" 55 | shift 2 56 | ;; 57 | "--repo-for-update") 58 | echo "--repo-for-update parameter: |$2|" 59 | INSTALL_ARGS="$INSTALL_ARGS --repo-for-update $2" 60 | shift 2 61 | ;; 62 | "--product") 63 | PRODUCT=$2 64 | echo "--product parameter: PRODUCT |$PRODUCT|" 65 | shift 2 66 | ;; 67 | "--scylla-build-sha-id") 68 | SCYLLA_BUILD_SHA_ID=$2 69 | echo "--scylla-build-sha-id parameter: SCYLLA_BUILD_SHA_ID |$SCYLLA_BUILD_SHA_ID|" 70 | shift 2 71 | ;; 72 | "--build-tag") 73 | BUILD_TAG=$2 74 | echo "--build-tag parameter: BUILD_TAG |$BUILD_TAG|" 75 | shift 2 76 | ;; 77 | "--env-tag") 78 | ENV_TAG=$2 79 | echo "--env-tag parameter: ENV_TAG |$ENV_TAG|" 80 | shift 2 81 | ;; 82 | "--version") 83 | VERSION=$2 84 | echo "--version: VERSION |$VERSION|" 85 | shift 2 86 | ;; 87 | "--scylla-release") 88 | SCYLLA_RELEASE=$2 89 | echo "--scylla-release: SCYLLA_RELEASE |$SCYLLA_RELEASE|" 90 | shift 2 91 | ;; 92 | "--scylla-machine-image-release") 93 | SCYLLA_MACHINE_IMAGE_RELEASE=$2 94 | echo "--scylla-machine-image-release: SCYLLA_MACHINE_IMAGE_RELEASE |$SCYLLA_MACHINE_IMAGE_RELEASE|" 95 | shift 2 96 | ;; 97 | "--branch") 98 | BRANCH=$2 99 | echo "--branch parameter: BRANCH |$BRANCH|" 100 | shift 2 101 | ;; 102 | "--ami-regions"): 103 | AMI_REGIONS=$2 104 | echo "--ami-regions parameter: AMI_REGIONS |$AMI_REGIONS|" 105 | shift 2 106 | ;; 107 | "--log-file") 108 | PACKER_LOG_PATH=$2 109 | echo "--log-file parameter: PACKER_LOG_PATH |$PACKER_LOG_PATH|" 110 | shift 2 111 | ;; 112 | "--build-mode") 113 | BUILD_MODE=$2 114 | shift 2 115 | ;; 116 | "--debug") 117 | echo "!!! Building image for debug !!!" 118 | DEBUG=true 119 | shift 1 120 | ;; 121 | "--dry-run") 122 | echo "!!! Running in DRY-RUN mode !!!" 123 | PACKER_SUB_CMD="validate" 124 | DRY_RUN=true 125 | shift 1 126 | ;; 127 | "--target") 128 | TARGET="$2" 129 | shift 2 130 | echo "--target parameter TARGET: |$TARGET|" 131 | case "$TARGET" in 132 | "aws") 133 | JSON_FILE="ami_variables.json" 134 | ;; 135 | "gce") 136 | JSON_FILE="gce_variables.json" 137 | ;; 138 | "azure") 139 | JSON_FILE="azure_variables.json" 140 | ;; 141 | *) 142 | print_usage 143 | ;; 144 | esac 145 | ;; 146 | "--arch") 147 | ARCH="$2" 148 | shift 2 149 | ;; 150 | "--ec2-instance-type") 151 | INSTANCE_TYPE="$2" 152 | shift 2 153 | ;; 154 | *) 155 | echo "ERROR: Illegal option: $1" 156 | print_usage 157 | ;; 158 | esac 159 | done 160 | 161 | if [ -z "$PRODUCT" ]; then 162 | PRODUCT=$(cat build/SCYLLA-PRODUCT-FILE) 163 | fi 164 | INSTALL_ARGS="$INSTALL_ARGS --product $PRODUCT" 165 | 166 | echo "INSTALL_ARGS: |$INSTALL_ARGS|" 167 | 168 | if [ -z "$TARGET" ]; then 169 | echo "Missing --target parameter. Please specify target cloud (aws/gce/azure)" 170 | exit 1 171 | fi 172 | 173 | SSH_USERNAME=ubuntu 174 | 175 | SCYLLA_FULL_VERSION="$VERSION-$SCYLLA_RELEASE" 176 | SCYLLA_MACHINE_IMAGE_VERSION="$VERSION-$SCYLLA_MACHINE_IMAGE_RELEASE" 177 | 178 | if [ -z "$REPO_FOR_INSTALL" ]; then 179 | echo "ERROR: No --repo or --repo-for-install were given." 180 | print_usage 181 | exit 1 182 | fi 183 | 184 | if [ "$TARGET" = "aws" ]; then 185 | 186 | SSH_USERNAME=ubuntu 187 | SOURCE_AMI_OWNER=099720109477 188 | REGION=us-east-1 189 | 190 | arch="$ARCH" 191 | case "$arch" in 192 | "x86_64") 193 | SOURCE_AMI_FILTER="ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-amd64*" 194 | if [ -z "$INSTANCE_TYPE" ]; then 195 | INSTANCE_TYPE="c4.xlarge" 196 | fi 197 | ;; 198 | "aarch64") 199 | SOURCE_AMI_FILTER="ubuntu-minimal/images/hvm-ssd-gp3/ubuntu-noble-24.04-arm64*" 200 | if [ -z "$INSTANCE_TYPE" ]; then 201 | INSTANCE_TYPE="im4gn.2xlarge" 202 | fi 203 | ;; 204 | *) 205 | echo "Unsupported architecture: $arch" 206 | exit 1 207 | esac 208 | 209 | SCYLLA_AMI_DESCRIPTION="scylla-$SCYLLA_FULL_VERSION scylla-machine-image-$SCYLLA_MACHINE_IMAGE_VERSION scylla-python3-$SCYLLA_FULL_VERSION" 210 | 211 | PACKER_ARGS+=(-var region="$REGION") 212 | PACKER_ARGS+=(-var buildMode="$BUILD_MODE") 213 | PACKER_ARGS+=(-var instance_type="$INSTANCE_TYPE") 214 | PACKER_ARGS+=(-var source_ami_filter="$SOURCE_AMI_FILTER") 215 | PACKER_ARGS+=(-var source_ami_owner="$SOURCE_AMI_OWNER") 216 | PACKER_ARGS+=(-var scylla_ami_description="${SCYLLA_AMI_DESCRIPTION:0:255}") 217 | elif [ "$TARGET" = "gce" ]; then 218 | SSH_USERNAME=ubuntu 219 | SOURCE_IMAGE_FAMILY="ubuntu-minimal-2404-lts-amd64" 220 | 221 | PACKER_ARGS+=(-var source_image_family="$SOURCE_IMAGE_FAMILY") 222 | elif [ "$TARGET" = "azure" ]; then 223 | REGION="EAST US" 224 | SSH_USERNAME=azureuser 225 | SCYLLA_IMAGE_DESCRIPTION="scylla-$SCYLLA_FULL_VERSION scylla-machine-image-$SCYLLA_MACHINE_IMAGE_VERSION scylla-python3-$SCYLLA_FULL_VERSION" 226 | 227 | PACKER_ARGS+=(-var scylla_image_description="${SCYLLA_IMAGE_DESCRIPTION:0:255}") 228 | PACKER_ARGS+=(-var client_id="$AZURE_CLIENT_ID") 229 | PACKER_ARGS+=(-var client_secret="$AZURE_CLIENT_SECRET") 230 | PACKER_ARGS+=(-var tenant_id="$AZURE_TENANT_ID") 231 | PACKER_ARGS+=(-var subscription_id="$AZURE_SUBSCRIPTION_ID") 232 | fi 233 | 234 | if [ "$TARGET" = "azure" ]; then 235 | if [ "$BUILD_MODE" = "debug" ]; then 236 | IMAGE_NAME="scylla-debug-$VERSION-$ARCH-$(date '+%FT%T')" 237 | else 238 | IMAGE_NAME="scylla-$VERSION-$ARCH-$(date '+%FT%T')" 239 | fi 240 | else 241 | IMAGE_NAME="$PRODUCT-$VERSION-$ARCH-$(date '+%FT%T')" 242 | fi 243 | if [ "$BUILD_MODE" = "debug" ]; then 244 | IMAGE_NAME="$PRODUCT-debug-$VERSION-$ARCH-$(date '+%FT%T')" 245 | fi 246 | if $DEBUG ; then 247 | IMAGE_NAME="debug-$IMAGE_NAME" 248 | fi 249 | 250 | if [ ! -f $JSON_FILE ]; then 251 | echo "'$JSON_FILE not found. Please create it before start building Image." 252 | echo "See variables.json.example" 253 | exit 1 254 | fi 255 | 256 | mkdir -p build 257 | 258 | export PACKER_LOG=1 259 | export PACKER_LOG_PATH 260 | 261 | set -x 262 | /usr/bin/packer ${PACKER_SUB_CMD} \ 263 | -only="$TARGET" \ 264 | -var-file="$JSON_FILE" \ 265 | -var install_args="$INSTALL_ARGS" \ 266 | -var ssh_username="$SSH_USERNAME" \ 267 | -var scylla_full_version="$SCYLLA_FULL_VERSION" \ 268 | -var scylla_version="$VERSION" \ 269 | -var scylla_machine_image_version="$SCYLLA_MACHINE_IMAGE_VERSION" \ 270 | -var scylla_python3_version="$SCYLLA_FULL_VERSION" \ 271 | -var creation_timestamp="$CREATION_TIMESTAMP" \ 272 | -var scylla_build_sha_id="$SCYLLA_BUILD_SHA_ID" \ 273 | -var build_tag="$BUILD_TAG" \ 274 | -var environment="$ENV_TAG" \ 275 | -var operating_system="$OPERATING_SYSTEM" \ 276 | -var branch="$BRANCH" \ 277 | -var ami_regions="$AMI_REGIONS" \ 278 | -var arch="$ARCH" \ 279 | -var product="$PRODUCT" \ 280 | -var build_mode="$BUILD_MODE" \ 281 | -var image_name="$IMAGE_NAME" \ 282 | "${PACKER_ARGS[@]}" \ 283 | "$DIR"/scylla.json 284 | set +x 285 | # For some errors packer gives a success status even if fails. 286 | # Search log for errors 287 | if $DRY_RUN ; then 288 | echo "DryRun: No need to grep errors on log" 289 | else 290 | GREP_STATUS=0 291 | case "$TARGET" in 292 | "aws") 293 | grep "us-east-1:" $PACKER_LOG_PATH 294 | GREP_STATUS=$? 295 | ;; 296 | "gce") 297 | grep "A disk image was created" $PACKER_LOG_PATH 298 | GREP_STATUS=$? 299 | ;; 300 | "azure") 301 | grep "Builds finished. The artifacts of successful builds are:" $PACKER_LOG_PATH 302 | GREP_STATUS=$? 303 | ;; 304 | *) 305 | echo "No Target is defined" 306 | exit 1 307 | esac 308 | 309 | if [ $GREP_STATUS -ne 0 ] ; then 310 | echo "Error: No image line found on log." 311 | exit 1 312 | else 313 | echo "Success: image line found on log" 314 | fi 315 | fi 316 | 317 | exit $EXIT_STATUS 318 | -------------------------------------------------------------------------------- /packer/files/.gitkeep: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/scylladb/scylla-machine-image/c32cb717a252f560383df463d7e2a89f8c86cfb2/packer/files/.gitkeep -------------------------------------------------------------------------------- /packer/gce_variables.json: -------------------------------------------------------------------------------- 1 | { 2 | "project_id": "scylla-images", 3 | "region": "europe-west1", 4 | "zone": "europe-west1-b", 5 | "image_storage_location": "europe-west1", 6 | "instance_type": "n2-standard-2" 7 | } 8 | -------------------------------------------------------------------------------- /packer/scylla.json: -------------------------------------------------------------------------------- 1 | { 2 | "builders": [ 3 | { 4 | "name": "aws", 5 | "type": "amazon-ebs", 6 | "access_key": "{{user `access_key`}}", 7 | "ami_block_device_mappings": [ 8 | { 9 | "device_name": "/dev/sdb", 10 | "virtual_name": "ephemeral0" 11 | }, 12 | { 13 | "device_name": "/dev/sdc", 14 | "virtual_name": "ephemeral1" 15 | }, 16 | { 17 | "device_name": "/dev/sdd", 18 | "virtual_name": "ephemeral2" 19 | }, 20 | { 21 | "device_name": "/dev/sde", 22 | "virtual_name": "ephemeral3" 23 | }, 24 | { 25 | "device_name": "/dev/sdf", 26 | "virtual_name": "ephemeral4" 27 | }, 28 | { 29 | "device_name": "/dev/sdg", 30 | "virtual_name": "ephemeral5" 31 | }, 32 | { 33 | "device_name": "/dev/sdh", 34 | "virtual_name": "ephemeral6" 35 | }, 36 | { 37 | "device_name": "/dev/sdi", 38 | "virtual_name": "ephemeral7" 39 | } 40 | ], 41 | "ami_name": "{{user `image_name`| clean_resource_name}}", 42 | "associate_public_ip_address": "{{user `associate_public_ip_address`}}", 43 | "sriov_support": true, 44 | "ena_support": true, 45 | "instance_type": "{{user `instance_type`}}", 46 | "launch_block_device_mappings": [ 47 | { 48 | "delete_on_termination": true, 49 | "device_name": "/dev/sda1", 50 | "volume_type": "gp3", 51 | "volume_size": 30 52 | } 53 | ], 54 | "region": "{{user `region`}}", 55 | "secret_key": "{{user `secret_key`}}", 56 | "security_group_id": "{{user `security_group_id`}}", 57 | "source_ami_filter": { 58 | "filters": { 59 | "name": "{{user `source_ami_filter`}}" 60 | }, 61 | "owners": ["{{user `source_ami_owner`}}"], 62 | "most_recent": true 63 | }, 64 | "ssh_timeout": "5m", 65 | "ssh_read_write_timeout": "5m", 66 | "ssh_username": "{{user `ssh_username`}}", 67 | "ssh_clear_authorized_keys": true, 68 | "subnet_filter": { 69 | "filters": { 70 | "tag:Name": "image-build-subnet*" 71 | }, 72 | "random": true 73 | }, 74 | "user_data_file": "user_data.txt", 75 | "ami_description": "{{user `scylla_ami_description`}}", 76 | "tags": { 77 | "Name": "{{user `image_name`| clean_resource_name}}", 78 | "scylla_version": "{{user `scylla_full_version`}}", 79 | "scylla_machine_image_version": "{{user `scylla_machine_image_version`}}", 80 | "scylla_python3_version": "{{user `scylla_python3_version`}}", 81 | "user_data_format_version": "3", 82 | "creation_timestamp": "{{user `creation_timestamp`| clean_resource_name}}", 83 | "branch": "{{user `branch`| clean_resource_name}}", 84 | "operating_system": "{{user `operating_system`| clean_resource_name}}", 85 | "scylla_build_sha_id": "{{user `scylla_build_sha_id`| clean_resource_name}}", 86 | "arch": "{{user `arch`| clean_resource_name}}", 87 | "build_tag": "{{user `build_tag`| clean_resource_name}}", 88 | "environment": "{{user `environment`| clean_resource_name}}", 89 | "build_mode": "{{user `build_mode`| clean_resource_name}}" 90 | }, 91 | "ami_regions": "{{user `ami_regions`}}", 92 | "aws_polling": { 93 | "delay_seconds": "30", 94 | "max_attempts": "100" 95 | }, 96 | "shutdown_behavior": "terminate", 97 | "ami_org_arns": [ 98 | "arn:aws:organizations::978072043225:organization/o-o561yy1rs6" 99 | ] 100 | }, 101 | { 102 | "name": "gce", 103 | "type": "googlecompute", 104 | "source_image_family": "{{user `source_image_family`}}", 105 | "ssh_username": "{{user `ssh_username`}}", 106 | "ssh_timeout": "6m", 107 | "ssh_read_write_timeout": "5m", 108 | "project_id": "{{user `project_id`}}", 109 | "zone": "{{user `zone`}}", 110 | "image_storage_locations": ["{{user `image_storage_location`}}"], 111 | "machine_type": "{{user `instance_type`}}", 112 | "metadata": {"block-project-ssh-keys": "TRUE"}, 113 | "image_family": "scylla", 114 | "image_name": "{{user `image_name`| clean_resource_name}}", 115 | "image_description": "Official ScyllaDB image v-{{user `scylla_version`| clean_resource_name}}", 116 | "use_internal_ip": false, 117 | "preemptible": true, 118 | "omit_external_ip": false, 119 | "disk_size": 30, 120 | "image_labels": { 121 | "scylla_version": "{{user `scylla_full_version`| clean_resource_name}}", 122 | "scylla_machine_image_version": "{{user `scylla_machine_image_version`| clean_resource_name}}", 123 | "scylla_python3_version": "{{user `scylla_python3_version`| clean_resource_name}}", 124 | "user_data_format_version": "3", 125 | "creation_timestamp": "{{user `creation_timestamp`| clean_resource_name}}", 126 | "branch": "{{user `branch`| clean_resource_name}}", 127 | "operating_system": "{{user `operating_system`| clean_resource_name}}", 128 | "scylla_build_sha_id": "{{user `scylla_build_sha_id`| clean_resource_name}}", 129 | "arch": "{{user `arch`| clean_resource_name}}", 130 | "build_tag": "{{user `build_tag`| clean_resource_name}}", 131 | "environment": "{{user `environment`| clean_resource_name}}", 132 | "build_mode": "{{user `build_mode`| clean_resource_name}}" 133 | }, 134 | "labels": { 135 | "keep": 1, 136 | "keep_action": "terminate" 137 | } 138 | }, 139 | { 140 | "name": "azure", 141 | "type": "azure-arm", 142 | "ssh_username": "{{user `ssh_username`}}", 143 | "ssh_timeout": "5m", 144 | "ssh_read_write_timeout": "5m", 145 | "client_id": "{{user `client_id`}}", 146 | "client_secret": "{{user `client_secret`}}", 147 | "tenant_id": "{{user `tenant_id`}}", 148 | "subscription_id": "{{user `subscription_id`}}", 149 | "managed_image_resource_group_name": "scylla-images", 150 | "managed_image_name": "{{user `image_name`| clean_resource_name}}", 151 | "os_type": "Linux", 152 | "image_publisher": "Canonical", 153 | "image_offer": "ubuntu-24_04-lts-daily", 154 | "image_sku": "minimal", 155 | "azure_tags": { 156 | "scylla_version": "{{user `scylla_full_version`}}", 157 | "scylla_machine_image_version": "{{user `scylla_machine_image_version`}}", 158 | "scylla_python3_version": "{{user `scylla_python3_version`}}", 159 | "user_data_format_version": "3", 160 | "creation_timestamp": "{{user `creation_timestamp`| clean_resource_name}}", 161 | "branch": "{{user `branch`| clean_resource_name}}", 162 | "operating_system": "{{user `operating_system`| clean_resource_name}}", 163 | "scylla_build_sha_id": "{{user `scylla_build_sha_id`| clean_resource_name}}", 164 | "arch": "{{user `arch`| clean_resource_name}}", 165 | "build_tag": "{{user `build_tag`| clean_resource_name}}", 166 | "environment": "{{user `environment`| clean_resource_name}}", 167 | "build_mode": "{{user `build_mode`| clean_resource_name}}" 168 | }, 169 | "vm_size": "{{user `vm_size`}}", 170 | "build_resource_group_name": "scylla-images", 171 | "keep_os_disk": true, 172 | "virtual_network_name": "scylla-images", 173 | "private_virtual_network_with_public_ip": true 174 | } 175 | ], 176 | "provisioners": [ 177 | { 178 | "destination": "/home/{{user `ssh_username`}}/", 179 | "source": "files/", 180 | "type": "file", 181 | "pause_before": "40s" 182 | }, 183 | { 184 | "destination": "/home/{{user `ssh_username`}}/", 185 | "source": "scylla_install_image", 186 | "type": "file" 187 | }, 188 | { 189 | "destination": "/tmp/", 190 | "source": "apply_cis_rules", 191 | "type": "file" 192 | }, 193 | { 194 | "inline": [ 195 | "sudo /usr/bin/cloud-init status --wait", 196 | "sudo /home/{{user `ssh_username`}}/scylla_install_image --target-cloud {{build_name}} --scylla-version {{user `scylla_full_version`}} {{user `install_args`}}" 197 | ], 198 | "type": "shell" 199 | }, 200 | { 201 | "inline": [ 202 | "sudo /tmp/apply_cis_rules --target-cloud {{build_name}}" 203 | ], 204 | "type": "shell" 205 | }, 206 | { 207 | "inline": [ 208 | "curl -sSfL https://raw.githubusercontent.com/anchore/syft/main/install.sh | sudo sh -s -- -b /usr/local/bin", 209 | "sudo syft -o cyclonedx-json@1.4 / > /home/{{user `ssh_username`}}/sbom_report.json", 210 | "python3 -c 'import json; d=json.load(open(\"/home/{{user `ssh_username`}}/sbom_report.json\")); d[\"components\"]=[c for c in d[\"components\"] if c.get(\"type\")!=\"file\"]; json.dump(d, open(\"/home/{{user `ssh_username`}}/ami_sbom_report_{{user `scylla_version`}}_{{user `arch`}}.json\", \"w\"), indent=2)'", 211 | "sudo rm -f /usr/local/bin/syft /home/{{user `ssh_username`}}/sbom_report.json" 212 | ], 213 | "only": ["aws"], 214 | "type": "shell" 215 | }, 216 | { 217 | "source": "/home/{{user `ssh_username`}}/ami_sbom_report_{{user `scylla_version`}}_{{user `arch`}}.json", 218 | "destination": "build/", 219 | "direction": "download", 220 | "only": ["aws"], 221 | "type": "file" 222 | }, 223 | { 224 | "source": "/home/{{user `ssh_username`}}/{{user `product`}}-{{build_name}}-kernel-{{user `scylla_full_version`}}-{{user `arch`}}.txt", 225 | "destination": "build/", 226 | "direction": "download", 227 | "type": "file" 228 | }, 229 | { 230 | "inline": [ 231 | "if [ {{build_name}} = gce -o {{build_name}} = azure ]; then sudo userdel -r -f {{user `ssh_username`}}; fi" 232 | ], 233 | "type": "shell" 234 | } 235 | ], 236 | "variables": { 237 | "access_key": "", 238 | "associate_public_ip_address": "", 239 | "install_args": "", 240 | "instance_type": "", 241 | "region": "", 242 | "secret_key": "", 243 | "security_group_id": "", 244 | "source_ami": "", 245 | "ssh_username": "", 246 | "subnet_id": "", 247 | "project_id": "", 248 | "zone": "", 249 | "image_storage_location": "", 250 | "source_image_family": "" 251 | } 252 | } 253 | -------------------------------------------------------------------------------- /packer/scylla_install_image: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020 ScyllaDB 5 | # 6 | # SPDX-License-Identifier: Apache-2.0 7 | 8 | import os 9 | import sys 10 | import glob 11 | import re 12 | import shutil 13 | import shlex 14 | import tarfile 15 | import argparse 16 | import platform 17 | import yaml 18 | from io import StringIO 19 | from subprocess import run, PIPE, STDOUT 20 | 21 | my_env = os.environ.copy() 22 | my_env['DEBIAN_FRONTEND']='noninteractive' 23 | apt_keys_dir = '/etc/apt/keyrings' 24 | 25 | def get_kver(pattern): 26 | for k in glob.glob(pattern): 27 | return re.sub(r'^/boot/vmlinuz-(.+)$', r'\1', k) 28 | 29 | def arch(): 30 | return platform.machine() 31 | 32 | def deb_arch(): 33 | darch={'x86_64': 'amd64', 'aarch64': 'arm64'} 34 | return darch[arch()] 35 | 36 | if __name__ == '__main__': 37 | if os.getuid() > 0: 38 | print('Requires root permission.') 39 | sys.exit(1) 40 | 41 | homedir = os.path.abspath(os.path.join(__file__, os.pardir)) 42 | parser = argparse.ArgumentParser(description='Construct AMI') 43 | parser.add_argument('--localdeb', action='store_true', default=False, 44 | help='deploy locally built rpms') 45 | parser.add_argument('--product', 46 | help='name of the product', default='scylla') 47 | parser.add_argument('--repo', 48 | help='repository for both install and update, specify .repo/.list file URL') 49 | parser.add_argument('--repo-for-install', 50 | help='repository for install, specify .repo/.list file URL') 51 | parser.add_argument('--repo-for-update', 52 | help='repository for update, specify .repo/.list file URL') 53 | parser.add_argument('--target-cloud', choices=['aws', 'gce', 'azure'], help='specify target cloud') 54 | parser.add_argument('--scylla-version', 55 | help='Scylla version to be added to manifest file') 56 | args = parser.parse_args() 57 | 58 | if args.repo: 59 | args.repo_for_install = args.repo_for_update = args.repo 60 | 61 | if not args.localdeb and not args.repo_for_install: 62 | print('Error: need to specify --localdeb or --repo/--repo-for-install') 63 | sys.exit(1) 64 | 65 | run('apt-get update --allow-insecure-repositories -y', shell=True, check=True) 66 | run('apt-get install -y gnupg2', shell=True, check=True) 67 | run(f'mkdir -p {apt_keys_dir}; gpg --homedir /tmp --no-default-keyring --keyring {apt_keys_dir}/scylladb.gpg ' 68 | f'--keyserver hkp://keyserver.ubuntu.com:80 --recv-keys a43e06657bac99e3', shell=True, check=True) 69 | 70 | if args.repo_for_install: 71 | run(f'curl -L -o /etc/apt/sources.list.d/scylla_install.list {args.repo_for_install}', shell=True, check=True) 72 | elif args.localdeb: 73 | with open('/etc/apt/sources.list.d/scylla_install.list', 'w') as f: 74 | f.write('deb file:/home/ubuntu ./') 75 | else: 76 | print('no scylla package found.') 77 | sys.exit(1) 78 | 79 | run('apt-get update --allow-insecure-repositories -y', shell=True, check=True) 80 | run('apt-get full-upgrade -y', shell=True, check=True) 81 | run('apt-get purge -y apport python3-apport fuse', shell=True, check=True) 82 | run('apt-get install -y systemd-coredump vim.tiny nmap ncat tmux jq python3-boto xfsprogs mdadm initramfs-tools ethtool vim-nox sysstat cpufrequtils nload htop traceroute dnsutils net-tools netcat-openbsd', shell=True, check=True) 83 | run(f'apt-get install -y --auto-remove --allow-unauthenticated {args.product}-machine-image {args.product}-server-dbg', shell=True, check=True) 84 | 85 | os.remove('/etc/apt/sources.list.d/scylla_install.list') 86 | if args.repo_for_update: 87 | run(f'curl -L -o /etc/apt/sources.list.d/scylla.list {args.repo_for_update}', shell=True, check=True) 88 | 89 | # disable unattended-upgrades 90 | run('apt-get purge -y unattended-upgrades update-notifier-common', shell=True, check=True) 91 | # remove snapd 92 | run('apt-get purge -y snapd', shell=True, check=True) 93 | run('apt-get purge -y modemmanager', shell=True, check=True) 94 | run('apt-get purge -y accountsservice', shell=True, check=True) 95 | run('apt-get purge -y acpid motd-news-config fwupd-signed', shell=True, check=True) 96 | run('apt-get purge -y udisks2 htop', shell=True, check=True) 97 | 98 | # drop packages does not need anymore 99 | run('apt-get autoremove --purge -y', shell=True, check=True) 100 | 101 | if args.target_cloud == 'aws': 102 | run('apt-get install -y pipx', shell=True, check=True) 103 | run('pipx install https://s3.amazonaws.com/cloudformation-examples/aws-cfn-bootstrap-py3-2.0-18.tar.gz', shell=True, check=True) 104 | 105 | # install .deb version of ssm-agent since we dropped snapd version 106 | run(f'curl -L -o /tmp/amazon-ssm-agent.deb https://s3.amazonaws.com/ec2-downloads-windows/SSMAgent/latest/debian_{deb_arch()}/amazon-ssm-agent.deb', shell=True, check=True) 107 | run('dpkg -i /tmp/amazon-ssm-agent.deb', shell=True, check=True) 108 | run('systemctl enable amazon-ssm-agent', shell=True, check=True) 109 | with open('/etc/chrony/chrony.conf') as f: 110 | chrony_conf = f.read() 111 | 112 | chrony_conf = re.sub(r'^(pool .*$)', '# \\1', chrony_conf, flags=re.MULTILINE) 113 | with open('/etc/chrony/chrony.conf', 'w') as f: 114 | f.write(chrony_conf) 115 | 116 | with open('/etc/chrony/sources.d/ntp-pool.sources', 'w') as f: 117 | f.write('pool time.aws.com iburst\n') 118 | 119 | kernel_opt = '' 120 | grub_variable = 'GRUB_CMDLINE_LINUX_DEFAULT' 121 | run('systemctl mask amazon-ssm-agent', shell=True, check=True) 122 | elif args.target_cloud == 'gce': 123 | # align with other clouds image 124 | run('apt-get purge -y rsyslog', shell=True, check=True) 125 | kernel_opt = '' 126 | grub_variable = 'GRUB_CMDLINE_LINUX_DEFAULT' 127 | run('systemctl mask google-osconfig-agent', shell=True, check=True) 128 | elif args.target_cloud == 'azure': 129 | kernel_opt = ' rootdelay=300' 130 | grub_variable = 'GRUB_CMDLINE_LINUX' 131 | run('systemctl mask walinuxagent', shell=True, check=True) 132 | 133 | run('systemctl disable apt-daily-upgrade.timer apt-daily.timer dpkg-db-backup.timer motd-news.timer', shell=True, check=True) 134 | run('systemctl daemon-reload', shell=True, check=True) 135 | run('systemctl enable scylla-image-setup.service', shell=True, check=True) 136 | run('systemctl enable scylla-image-post-start.service', shell=True, check=True) 137 | run('/opt/scylladb/scripts/scylla_setup --no-coredump-setup --no-sysconfig-setup --no-raid-setup --no-io-setup --no-ec2-check --no-swap-setup --no-cpuscaling-setup --no-ntp-setup', shell=True, check=True) 138 | 139 | # On Ubuntu, 'cpufrequtils' never fails even CPU scaling is not supported, 140 | # so we want to enable it here 141 | run('/opt/scylladb/scripts/scylla_cpuscaling_setup --force', shell=True, check=True) 142 | 143 | run(f'/opt/scylladb/scripts/scylla_sysconfig_setup --set-clocksource', shell=True, check=True) 144 | run('/opt/scylladb/scripts/scylla_coredump_setup', shell=True, check=True) 145 | dot_mount = ''' 146 | [Unit] 147 | Description=Save coredump to scylla data directory 148 | Conflicts=umount.target 149 | Before=scylla-server.service 150 | After=local-fs.target 151 | DefaultDependencies=no 152 | 153 | [Mount] 154 | What=/var/lib/scylla/coredump 155 | Where=/var/lib/systemd/coredump 156 | Type=none 157 | Options=bind 158 | 159 | [Install] 160 | WantedBy=multi-user.target 161 | '''[1:-1] 162 | with open('/etc/systemd/system/var-lib-systemd-coredump.mount', 'w') as f: 163 | f.write(dot_mount) 164 | os.makedirs('/var/lib/scylla/coredump', exist_ok=True) 165 | 166 | os.remove('{}/.ssh/authorized_keys'.format(homedir)) 167 | os.remove('/var/lib/scylla-housekeeping/housekeeping.uuid') 168 | os.remove('/var/cache/debconf/config.dat') 169 | 170 | with open('/etc/default/grub.d/50-cloudimg-settings.cfg') as f: 171 | grub = f.read() 172 | grub = re.sub(fr'^{grub_variable}="(.+)"$', 173 | fr'{grub_variable}="\1 net.ifnames=0 clocksource=tsc tsc=reliable intel_idle.max_cstate=1 processor.max_cstate=1 {kernel_opt}"', grub, 174 | flags=re.MULTILINE) 175 | with open('/etc/default/grub.d/50-cloudimg-settings.cfg', 'w') as f: 176 | f.write(grub) 177 | run('update-grub2', shell=True, check=True) 178 | 179 | profile = '/etc/skel/.profile' 180 | with open(profile, 'a') as f: 181 | f.write('\n\n/opt/scylladb/scylla-machine-image/scylla_login\n') 182 | 183 | # On AWS, ssh user is statically created at AMI building time, so we need to 184 | # change it to 'scyllaadm`. 185 | # However, on GCE and Azure ssh user is dynamically created at instance startup 186 | # time, and username is specified while launching the instance, we have nothing 187 | # to do. 188 | if args.target_cloud == 'aws': 189 | with open('/etc/cloud/cloud.cfg') as f: 190 | y = yaml.safe_load(f) 191 | groups = ','.join(y['system_info']['default_user']['groups']) 192 | y['cloud_init_modules'].remove('mounts') 193 | y['ssh_deletekeys'] = True 194 | y['system_info']['default_user']['name'] = 'scyllaadm' 195 | y['system_info']['default_user']['gecos'] = 'scyllaadm' 196 | with open('/etc/cloud/cloud.cfg', 'w') as f: 197 | yaml.dump(y, f) 198 | # before deleting home directory, need to change current directory 199 | os.chdir('/tmp') 200 | run('userdel -r -f ubuntu', shell=True, check=True) 201 | run('cloud-init clean', shell=True, check=True) 202 | run('cloud-init init', shell=True, check=True) 203 | for skel in glob.glob('/etc/skel/.*'): 204 | shutil.copy(skel, '/home/scyllaadm') 205 | os.chown(skel, 1000, 1000) 206 | run(f'useradd -o -u 1000 -g scyllaadm -G {groups} -s /bin/bash -d /home/scyllaadm centos', shell=True, check=True) 207 | run('groupadd -o -g 1000 centos', shell=True, check=True) 208 | os.symlink('/home/scyllaadm', '/home/centos') 209 | run(f'useradd -o -u 1000 -g scyllaadm -G {groups} -s /bin/bash -d /home/scyllaadm ubuntu', shell=True, check=True) 210 | run('groupadd -o -g 1000 ubuntu', shell=True, check=True) 211 | os.symlink('/home/scyllaadm', '/home/ubuntu') 212 | 213 | if args.target_cloud == 'azure': 214 | with open('/etc/hosts', 'a') as f: 215 | f.write('\n\n169.254.169.254 metadata.azure.internal\n') 216 | with open('/etc/ssh/sshd_config.d/50-cloudimg-settings.conf', 'w') as f: 217 | f.write('ClientAliveInterval 180 \nHostKeyAlgorithms +ssh-rsa \nPubkeyAcceptedKeyTypes +ssh-rsa') 218 | 219 | kver = run('uname -r', shell=True, check=True, capture_output=True, encoding='utf-8').stdout.strip() 220 | with open('{}/{}-{}-kernel-{}-{}.txt'.format(homedir, args.product, args.target_cloud, args.scylla_version, arch()), 'a+') as f: 221 | f.write(f'kernel-version: {kver}\n') 222 | print('{}/{}-{}-kernel-{}-{}.txt generated.'.format(homedir, args.product, args.target_cloud, args.scylla_version, arch())) 223 | -------------------------------------------------------------------------------- /packer/user_data.txt: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | sed -i 's/Defaults requiretty/#Defaults requiretty/g' /etc/sudoers 3 | -------------------------------------------------------------------------------- /renovate.json: -------------------------------------------------------------------------------- 1 | { 2 | "$schema": "https://docs.renovatebot.com/renovate-schema.json", 3 | "extends": [ 4 | "config:recommended" 5 | ], 6 | "schedule": ["every weekend"] 7 | } 8 | -------------------------------------------------------------------------------- /test-requirements.txt: -------------------------------------------------------------------------------- 1 | distro==1.9.0 2 | httpretty==1.1.4 3 | psutil==7.0.0 4 | pytest==8.3.5 5 | PyYAML==6.0.2 6 | traceback-with-variables==2.2.0 7 | boto3==1.38.23 -------------------------------------------------------------------------------- /tests/test_gcp_instance.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import logging 3 | import httpretty 4 | import unittest.mock 5 | import json 6 | from unittest import TestCase, IsolatedAsyncioTestCase 7 | from collections import namedtuple 8 | from socket import AddressFamily, SocketKind 9 | from pathlib import Path 10 | 11 | sys.path.append(str(Path(__file__).parent.parent)) 12 | import lib.scylla_cloud 13 | from lib.scylla_cloud import gcp_instance 14 | 15 | LOGGER = logging.getLogger(__name__) 16 | 17 | svmem = namedtuple('svmem', ['total']) 18 | 19 | sdiskpart = namedtuple('sdiskpart', ['device', 'mountpoint']) 20 | mock_disk_partitions = [ 21 | sdiskpart('/dev/root', '/'), 22 | sdiskpart('/dev/sda15', '/boot/efi'), 23 | sdiskpart('/dev/md0', '/var/lib/scylla'), 24 | sdiskpart('/dev/md0', '/var/lib/systemd/coredump') 25 | ] 26 | 27 | mock_listdevdir_n2_standard_8 = ['md0', 'root', 'sda15', 'sda14', 'sda1', 'sda', 'sg0', 'zero', 'null'] 28 | mock_listdevdir_n2_standard_8_4ssd = ['md0', 'root', 'nvme0n4', 'nvme0n3', 'nvme0n2', 'sda15', 'sda14', 'sda1', 'sda', 'sg0', 'nvme0n1', 'nvme0', 'zero', 'null'] 29 | mock_listdevdir_n2_highcpu_8_4ssd = mock_listdevdir_n2_standard_8_4ssd 30 | mock_listdevdir_n2_standard_8_24ssd = ['md0', 'root', 'nvme0n24', 'nvme0n23', 'nvme0n22', 'nvme0n21', 'nvme0n20', 'nvme0n19', 'nvme0n18', 'nvme0n17', 'nvme0n16', 'nvme0n15', 'nvme0n14', 'nvme0n13', 'nvme0n12', 'nvme0n11', 'nvme0n10', 'nvme0n9', 'nvme0n8', 'nvme0n7', 'nvme0n6', 'nvme0n5', 'nvme0n4', 'nvme0n3', 'nvme0n2', 'nvme0n1', 'sda15', 'sda14', 'sda1', 'sda', 'sg0','nvme0', 'zero', 'null'] 31 | mock_listdevdir_n2_standard_8_4ssd_2persistent = ['sdc', 'sg2', 'sdb', 'sg1', 'md0', 'root', 'nvme0n4', 'nvme0n3', 'nvme0n2', 'sda15', 'sda14', 'sda1', 'sda', 'sg0', 'nvme0n1', 'nvme0', 'zero', 'null'] 32 | mock_glob_glob_dev_n2_standard_8 = ['/dev/sda15', '/dev/sda14', '/dev/sda1', '/dev/sda'] 33 | mock_glob_glob_dev_n2_standard_8_4ssd = mock_glob_glob_dev_n2_standard_8 34 | mock_glob_glob_dev_n2_standard_8_24ssd = mock_glob_glob_dev_n2_standard_8 35 | mock_glob_glob_dev_n2_highcpu_8_4ssd = mock_glob_glob_dev_n2_standard_8 36 | mock_glob_glob_dev_n2_standard_8_4ssd_2persistent = ['/dev/sdc', '/dev/sdb', '/dev/sda15', '/dev/sda14', '/dev/sda1', '/dev/sda'] 37 | 38 | def _mock_multi_open(files, filename, *args, **kwargs): 39 | if filename in files: 40 | return unittest.mock.mock_open(read_data=files[filename]).return_value 41 | else: 42 | raise FileNotFoundError(f'Unable to open {filename}') 43 | 44 | def mock_multi_open_n2(filename, *args, **kwargs): 45 | files = { 46 | '/sys/class/dmi/id/product_name': 'Google Compute Engine' 47 | } 48 | return _mock_multi_open(files, filename, *args, **kwargs) 49 | 50 | 51 | class GcpMetadata: 52 | def httpretty_gcp_metadata(self, instance_type='n2-standard-8', project_number='431729375847', instance_name='testcase_1', num_local_disks=4, num_remote_disks=0, with_userdata=False): 53 | httpretty.register_uri( 54 | httpretty.GET, 55 | 'http://metadata.google.internal/computeMetadata/v1/instance/machine-type?recursive=false', 56 | f'projects/{project_number}/machineTypes/{instance_type}' 57 | ) 58 | httpretty.register_uri( 59 | httpretty.GET, 60 | 'http://metadata.google.internal/computeMetadata/v1/instance/network-interfaces/0/ip?recursive=false', 61 | '172.16.0.1' 62 | ) 63 | disks = [] 64 | i = 0 65 | disks.append({"deviceName": instance_name, "index": i, "interface": "SCSI", "mode": "READ_WRITE", "type": "PERSISTENT-BALANCED"}) 66 | i += 1 67 | for j in range(num_local_disks): 68 | disks.append({"deviceName": f"local-ssd-{j}", "index": i, "interface": "NVME", "mode": "READ_WRITE", "type": "LOCAL-SSD"}) 69 | i += 1 70 | for j in range(num_remote_disks): 71 | disks.append({"deviceName": f"disk-{j}", "index": i, "interface": "SCSI", "mode": "READ_WRITE", "type": "PERSISTENT-BALANCED"}) 72 | i += 1 73 | httpretty.register_uri( 74 | httpretty.GET, 75 | 'http://metadata.google.internal/computeMetadata/v1/instance/disks?recursive=true', 76 | json.dumps(disks) 77 | ) 78 | if not with_userdata: 79 | httpretty.register_uri( 80 | httpretty.GET, 81 | 'http://metadata.google.internal/computeMetadata/v1/instance/attributes/user-data?recursive=false', 82 | status = 404 83 | ) 84 | else: 85 | httpretty.register_uri( 86 | httpretty.GET, 87 | 'http://metadata.google.internal/computeMetadata/v1/instance/attributes/user-data?recursive=false', 88 | '{"scylla_yaml": {"cluster_name": "test-cluster"}}' 89 | ) 90 | 91 | class TestAsyncGcpInstance(IsolatedAsyncioTestCase, GcpMetadata): 92 | def setUp(self): 93 | httpretty.enable(verbose=True, allow_net_connect=False) 94 | 95 | def tearDown(self): 96 | httpretty.disable() 97 | httpretty.reset() 98 | 99 | async def test_identify_metadata(self): 100 | self.httpretty_gcp_metadata() 101 | with unittest.mock.patch('socket.getaddrinfo', return_value=[(AddressFamily.AF_INET, SocketKind.SOCK_STREAM, 6, '', ('169.254.169.254', 80))]): 102 | assert await gcp_instance.identify_metadata() 103 | 104 | async def test_not_identify_metadata(self): 105 | assert not await gcp_instance.identify_metadata() 106 | 107 | class TestGcpInstance(TestCase, GcpMetadata): 108 | def setUp(self): 109 | httpretty.enable(verbose=True, allow_net_connect=False) 110 | 111 | def tearDown(self): 112 | httpretty.disable() 113 | httpretty.reset() 114 | 115 | def test_identify_dmi(self): 116 | with unittest.mock.patch('builtins.open', unittest.mock.MagicMock(side_effect=mock_multi_open_n2)): 117 | assert gcp_instance.identify_dmi() 118 | 119 | def test_endpoint_snitch(self): 120 | self.httpretty_gcp_metadata() 121 | ins = gcp_instance() 122 | assert ins.endpoint_snitch == 'GoogleCloudSnitch' 123 | 124 | def test_instancetype_n2_standard_8(self): 125 | self.httpretty_gcp_metadata() 126 | ins = gcp_instance() 127 | assert ins.instancetype == 'n2-standard-8' 128 | 129 | def test_instancetype_n2d_highmem_4(self): 130 | self.httpretty_gcp_metadata(instance_type='n2d-highmem-4') 131 | ins = gcp_instance() 132 | assert ins.instancetype == 'n2d-highmem-4' 133 | 134 | def test_instancetype_e2_micro(self): 135 | self.httpretty_gcp_metadata(instance_type='e2-micro') 136 | ins = gcp_instance() 137 | assert ins.instancetype == 'e2-micro' 138 | 139 | def test_cpu_n2_standard_8(self): 140 | self.httpretty_gcp_metadata() 141 | ins = gcp_instance() 142 | with unittest.mock.patch('psutil.cpu_count', return_value=8): 143 | assert ins.cpu == 8 144 | 145 | def test_cpu_n2d_highmem_4(self): 146 | self.httpretty_gcp_metadata(instance_type='n2d-highmem-4') 147 | ins = gcp_instance() 148 | with unittest.mock.patch('psutil.cpu_count', return_value=4): 149 | assert ins.cpu == 4 150 | 151 | def test_memoryGB_n2_standard_8(self): 152 | self.httpretty_gcp_metadata() 153 | ins = gcp_instance() 154 | # XXX: the value is little bit less than 32GB 155 | with unittest.mock.patch('psutil.virtual_memory', return_value=svmem(33663647744)): 156 | assert ins.memoryGB > 31 and ins.memoryGB <= 32 157 | 158 | def test_memoryGB_n2d_highmem_4(self): 159 | self.httpretty_gcp_metadata(instance_type='n2d-highmem-4') 160 | ins = gcp_instance() 161 | # XXX: the value is little bit less than 32GB 162 | with unittest.mock.patch('psutil.virtual_memory', return_value=svmem(33664700416)): 163 | assert ins.memoryGB > 31 and ins.memoryGB <= 32 164 | 165 | def test_memoryGB_n1_standard_1(self): 166 | self.httpretty_gcp_metadata(instance_type='n1-standard-1') 167 | ins = gcp_instance() 168 | # XXX: the value is little bit less than 3.75GB 169 | with unittest.mock.patch('psutil.virtual_memory', return_value=svmem(3850301440)): 170 | assert ins.memoryGB > 3 and ins.memoryGB < 4 171 | 172 | def test_instance_size_n2_standard_8(self): 173 | self.httpretty_gcp_metadata() 174 | ins = gcp_instance() 175 | assert ins.instance_size() == '8' 176 | 177 | def test_instance_size_n2d_highmem_4(self): 178 | self.httpretty_gcp_metadata(instance_type='n2d-highmem-4') 179 | ins = gcp_instance() 180 | assert ins.instance_size() == '4' 181 | 182 | def test_instance_size_e2_micro(self): 183 | self.httpretty_gcp_metadata(instance_type='e2-micro') 184 | ins = gcp_instance() 185 | assert not ins.instance_size() 186 | 187 | def test_instance_class_n2_standard_8(self): 188 | self.httpretty_gcp_metadata() 189 | ins = gcp_instance() 190 | assert ins.instance_class() == 'n2' 191 | 192 | def test_instance_class_n2d_highmem_4(self): 193 | self.httpretty_gcp_metadata(instance_type='n2d-highmem-4') 194 | ins = gcp_instance() 195 | assert ins.instance_class() == 'n2d' 196 | 197 | def test_instance_class_e2_micro(self): 198 | self.httpretty_gcp_metadata(instance_type='e2-micro') 199 | ins = gcp_instance() 200 | assert ins.instance_class() == 'e2' 201 | 202 | def test_instance_purpose_n2_standard_8(self): 203 | self.httpretty_gcp_metadata() 204 | ins = gcp_instance() 205 | assert ins.instance_purpose() == 'standard' 206 | 207 | def test_instance_purpose_n2d_highmem_4(self): 208 | self.httpretty_gcp_metadata(instance_type='n2d-highmem-4') 209 | ins = gcp_instance() 210 | assert ins.instance_purpose() == 'highmem' 211 | 212 | def test_instance_purpose_e2_micro(self): 213 | self.httpretty_gcp_metadata(instance_type='e2-micro') 214 | ins = gcp_instance() 215 | assert ins.instance_purpose() == 'micro' 216 | 217 | def test_is_not_unsupported_instance_class_n2_standard_8(self): 218 | self.httpretty_gcp_metadata() 219 | ins = gcp_instance() 220 | assert not ins.is_unsupported_instance_class() 221 | 222 | def test_is_not_unsupported_instance_class_n2d_highmem_4(self): 223 | self.httpretty_gcp_metadata(instance_type='n2d-highmem-4') 224 | ins = gcp_instance() 225 | assert not ins.is_unsupported_instance_class() 226 | 227 | def test_is_unsupported_instance_class_e2_micro(self): 228 | self.httpretty_gcp_metadata(instance_type='e2-micro') 229 | ins = gcp_instance() 230 | assert ins.is_unsupported_instance_class() 231 | 232 | def test_is_not_unsupported_instance_class_m1_megamem_96(self): 233 | self.httpretty_gcp_metadata(instance_type='m1-megamem-96') 234 | ins = gcp_instance() 235 | assert not ins.is_unsupported_instance_class() 236 | 237 | def test_is_supported_instance_class_n2_standard_8(self): 238 | self.httpretty_gcp_metadata() 239 | ins = gcp_instance() 240 | assert ins.is_supported_instance_class() 241 | 242 | def test_is_supported_instance_class_n2d_highmem_4(self): 243 | self.httpretty_gcp_metadata(instance_type='n2d-highmem-4') 244 | ins = gcp_instance() 245 | assert ins.is_supported_instance_class() 246 | 247 | def test_is_not_supported_instance_class_e2_micro(self): 248 | self.httpretty_gcp_metadata(instance_type='e2-micro') 249 | ins = gcp_instance() 250 | assert not ins.is_supported_instance_class() 251 | 252 | def test_is_supported_instance_class_m1_megamem_96(self): 253 | self.httpretty_gcp_metadata(instance_type='m1-megamem-96') 254 | ins = gcp_instance() 255 | assert ins.is_supported_instance_class() 256 | 257 | def test_is_recommended_instance_size_n2_standard_8(self): 258 | self.httpretty_gcp_metadata() 259 | ins = gcp_instance() 260 | assert ins.is_recommended_instance_size() 261 | 262 | def test_is_not_recommended_instance_size_n1_standard_1(self): 263 | self.httpretty_gcp_metadata(instance_type='n1-standard-1') 264 | ins = gcp_instance() 265 | assert not ins.is_recommended_instance_size() 266 | 267 | # Unsupported class, but recommended size 268 | def test_is_recommended_instance_size_e2_standard_8(self): 269 | self.httpretty_gcp_metadata(instance_type='e2-standard-8') 270 | ins = gcp_instance() 271 | assert ins.is_recommended_instance_size() 272 | 273 | def test_private_ipv4(self): 274 | self.httpretty_gcp_metadata() 275 | ins = gcp_instance() 276 | assert ins.private_ipv4() == '172.16.0.1' 277 | 278 | def test_user_data(self): 279 | self.httpretty_gcp_metadata(with_userdata=True) 280 | ins = gcp_instance() 281 | assert ins.user_data == '{"scylla_yaml": {"cluster_name": "test-cluster"}}' 282 | 283 | def test_no_user_data(self): 284 | self.httpretty_gcp_metadata() 285 | ins = gcp_instance() 286 | real_curl = lib.scylla_cloud.curl 287 | 288 | def mocked_curl(*args, **kwargs): 289 | kwargs['timeout'] = 0.001 290 | kwargs['retry_interval'] = 0.0001 291 | return real_curl(*args, **kwargs) 292 | 293 | with unittest.mock.patch('lib.scylla_cloud.curl', new=mocked_curl): 294 | assert ins.user_data == '' 295 | 296 | def test_non_root_nvmes_n2_standard_8_4ssd(self): 297 | self.httpretty_gcp_metadata() 298 | with unittest.mock.patch('psutil.disk_partitions', return_value=mock_disk_partitions),\ 299 | unittest.mock.patch('os.listdir', return_value=mock_listdevdir_n2_standard_8_4ssd): 300 | ins = gcp_instance() 301 | assert ins._non_root_nvmes() == {'root': ['/dev/root'], 'ephemeral': ['nvme0n4', 'nvme0n3', 'nvme0n2', 'nvme0n1']} 302 | 303 | def test_non_root_nvmes_n2_standard_8_4ssd_2persistent(self): 304 | self.httpretty_gcp_metadata(num_remote_disks=2) 305 | with unittest.mock.patch('psutil.disk_partitions', return_value=mock_disk_partitions),\ 306 | unittest.mock.patch('os.listdir', return_value=mock_listdevdir_n2_standard_8_4ssd_2persistent): 307 | ins = gcp_instance() 308 | assert ins._non_root_nvmes() == {'root': ['/dev/root'], 'ephemeral': ['nvme0n4', 'nvme0n3', 'nvme0n2', 'nvme0n1']} 309 | 310 | def test_non_root_disks_n2_standard_8_4ssd(self): 311 | self.httpretty_gcp_metadata() 312 | with unittest.mock.patch('psutil.disk_partitions', return_value=mock_disk_partitions),\ 313 | unittest.mock.patch('glob.glob', return_value=mock_glob_glob_dev_n2_standard_8_4ssd): 314 | ins = gcp_instance() 315 | assert ins._non_root_disks() == {'persistent': []} 316 | 317 | def test_non_root_disks_n2_standard_8_4ssd_2persistent(self): 318 | self.httpretty_gcp_metadata(num_remote_disks=2) 319 | with unittest.mock.patch('psutil.disk_partitions', return_value=mock_disk_partitions),\ 320 | unittest.mock.patch('glob.glob', return_value=mock_glob_glob_dev_n2_standard_8_4ssd_2persistent): 321 | ins = gcp_instance() 322 | assert ins._non_root_disks() == {'persistent': ['sdc','sdb']} 323 | 324 | def test_os_disks_n2_standard_8_4ssd(self): 325 | self.httpretty_gcp_metadata() 326 | with unittest.mock.patch('psutil.disk_partitions', return_value=mock_disk_partitions),\ 327 | unittest.mock.patch('os.listdir', return_value=mock_listdevdir_n2_standard_8_4ssd),\ 328 | unittest.mock.patch('glob.glob', return_value=mock_glob_glob_dev_n2_standard_8_4ssd): 329 | ins = gcp_instance() 330 | assert ins.os_disks == {'root': ['/dev/root'], 'ephemeral': ['nvme0n4', 'nvme0n3', 'nvme0n2', 'nvme0n1'], 'persistent': []} 331 | 332 | def test_os_disks_n2_standard_8_4ssd_2persistent(self): 333 | self.httpretty_gcp_metadata(num_remote_disks=2) 334 | with unittest.mock.patch('psutil.disk_partitions', return_value=mock_disk_partitions),\ 335 | unittest.mock.patch('os.listdir', return_value=mock_listdevdir_n2_standard_8_4ssd_2persistent),\ 336 | unittest.mock.patch('glob.glob', return_value=mock_glob_glob_dev_n2_standard_8_4ssd_2persistent): 337 | ins = gcp_instance() 338 | assert ins.os_disks == {'root': ['/dev/root'], 'ephemeral': ['nvme0n4', 'nvme0n3', 'nvme0n2', 'nvme0n1'], 'persistent': ['sdc','sdb']} 339 | 340 | def test_get_local_disks_n2_standard_8_4ssd(self): 341 | self.httpretty_gcp_metadata() 342 | with unittest.mock.patch('psutil.disk_partitions', return_value=mock_disk_partitions),\ 343 | unittest.mock.patch('os.listdir', return_value=mock_listdevdir_n2_standard_8_4ssd),\ 344 | unittest.mock.patch('glob.glob', return_value=mock_glob_glob_dev_n2_standard_8_4ssd): 345 | ins = gcp_instance() 346 | assert ins.get_local_disks() == ['nvme0n4', 'nvme0n3', 'nvme0n2', 'nvme0n1'] 347 | 348 | def test_get_local_disks_n2_standard_8_4ssd_2persistent(self): 349 | self.httpretty_gcp_metadata(num_remote_disks=2) 350 | with unittest.mock.patch('psutil.disk_partitions', return_value=mock_disk_partitions),\ 351 | unittest.mock.patch('os.listdir', return_value=mock_listdevdir_n2_standard_8_4ssd_2persistent),\ 352 | unittest.mock.patch('glob.glob', return_value=mock_glob_glob_dev_n2_standard_8_4ssd_2persistent): 353 | ins = gcp_instance() 354 | assert ins.get_local_disks() == ['nvme0n4', 'nvme0n3', 'nvme0n2', 'nvme0n1'] 355 | 356 | def test_get_remote_disks_n2_standard_8_4ssd(self): 357 | self.httpretty_gcp_metadata() 358 | with unittest.mock.patch('psutil.disk_partitions', return_value=mock_disk_partitions),\ 359 | unittest.mock.patch('os.listdir', return_value=mock_listdevdir_n2_standard_8_4ssd),\ 360 | unittest.mock.patch('glob.glob', return_value=mock_glob_glob_dev_n2_standard_8_4ssd): 361 | ins = gcp_instance() 362 | assert ins.get_remote_disks() == [] 363 | 364 | def test_get_remote_disks_n2_standard_8_4ssd_2persistent(self): 365 | self.httpretty_gcp_metadata(num_remote_disks=2) 366 | with unittest.mock.patch('psutil.disk_partitions', return_value=mock_disk_partitions),\ 367 | unittest.mock.patch('os.listdir', return_value=mock_listdevdir_n2_standard_8_4ssd_2persistent),\ 368 | unittest.mock.patch('glob.glob', return_value=mock_glob_glob_dev_n2_standard_8_4ssd_2persistent): 369 | ins = gcp_instance() 370 | assert ins.get_remote_disks() == ['sdc','sdb'] 371 | 372 | def test_get_nvme_disks_from_metadata_n2_standard_8_4ssd(self): 373 | self.httpretty_gcp_metadata() 374 | ins = gcp_instance() 375 | assert ins._gcp_instance__get_nvme_disks_from_metadata() == [{'deviceName': 'local-ssd-0', 'index': 1, 'interface': 'NVME', 'mode': 'READ_WRITE', 'type': 'LOCAL-SSD'}, {'deviceName': 'local-ssd-1', 'index': 2, 'interface': 'NVME', 'mode': 'READ_WRITE', 'type': 'LOCAL-SSD'}, {'deviceName': 'local-ssd-2', 'index': 3, 'interface': 'NVME', 'mode': 'READ_WRITE', 'type': 'LOCAL-SSD'}, {'deviceName': 'local-ssd-3', 'index': 4, 'interface': 'NVME', 'mode': 'READ_WRITE', 'type': 'LOCAL-SSD'}] 376 | 377 | def test_get_nvme_disks_from_metadata_n2_standard_8_4ssd_2persistent(self): 378 | self.httpretty_gcp_metadata(num_remote_disks=2) 379 | ins = gcp_instance() 380 | assert ins._gcp_instance__get_nvme_disks_from_metadata() == [{'deviceName': 'local-ssd-0', 'index': 1, 'interface': 'NVME', 'mode': 'READ_WRITE', 'type': 'LOCAL-SSD'}, {'deviceName': 'local-ssd-1', 'index': 2, 'interface': 'NVME', 'mode': 'READ_WRITE', 'type': 'LOCAL-SSD'}, {'deviceName': 'local-ssd-2', 'index': 3, 'interface': 'NVME', 'mode': 'READ_WRITE', 'type': 'LOCAL-SSD'}, {'deviceName': 'local-ssd-3', 'index': 4, 'interface': 'NVME', 'mode': 'READ_WRITE', 'type': 'LOCAL-SSD'}] 381 | 382 | def test_nvme_disk_count_n2_standard_8_4ssd(self): 383 | self.httpretty_gcp_metadata() 384 | with unittest.mock.patch('psutil.disk_partitions', return_value=mock_disk_partitions),\ 385 | unittest.mock.patch('os.listdir', return_value=mock_listdevdir_n2_standard_8_4ssd),\ 386 | unittest.mock.patch('glob.glob', return_value=mock_glob_glob_dev_n2_standard_8_4ssd): 387 | ins = gcp_instance() 388 | assert ins.nvme_disk_count == 4 389 | 390 | def test_nvme_disk_count_n2_standard_8_4ssd_2persistent(self): 391 | self.httpretty_gcp_metadata(num_remote_disks=2) 392 | with unittest.mock.patch('psutil.disk_partitions', return_value=mock_disk_partitions),\ 393 | unittest.mock.patch('os.listdir', return_value=mock_listdevdir_n2_standard_8_4ssd_2persistent),\ 394 | unittest.mock.patch('glob.glob', return_value=mock_glob_glob_dev_n2_standard_8_4ssd_2persistent): 395 | ins = gcp_instance() 396 | assert ins.nvme_disk_count == 4 397 | 398 | def test_firstNvmeSize_n2_standard_8_4ssd(self): 399 | self.httpretty_gcp_metadata() 400 | with unittest.mock.patch('psutil.disk_partitions', return_value=mock_disk_partitions),\ 401 | unittest.mock.patch('os.listdir', return_value=mock_listdevdir_n2_standard_8_4ssd),\ 402 | unittest.mock.patch('glob.glob', return_value=mock_glob_glob_dev_n2_standard_8_4ssd),\ 403 | unittest.mock.patch('lib.scylla_cloud.gcp_instance.get_file_size_by_seek', return_value=402653184000): 404 | ins = gcp_instance() 405 | assert ins.firstNvmeSize == 375.0 406 | 407 | def test_is_recommended_instance_n2_standard_8_4ssd(self): 408 | self.httpretty_gcp_metadata() 409 | with unittest.mock.patch('psutil.cpu_count', return_value=8),\ 410 | unittest.mock.patch('psutil.virtual_memory', return_value=svmem(33663647744)),\ 411 | unittest.mock.patch('psutil.disk_partitions', return_value=mock_disk_partitions),\ 412 | unittest.mock.patch('os.listdir', return_value=mock_listdevdir_n2_standard_8_4ssd),\ 413 | unittest.mock.patch('glob.glob', return_value=mock_glob_glob_dev_n2_standard_8_4ssd),\ 414 | unittest.mock.patch('lib.scylla_cloud.gcp_instance.get_file_size_by_seek', return_value=402653184000): 415 | ins = gcp_instance() 416 | assert ins.is_recommended_instance() == True 417 | 418 | def test_is_not_recommended_instance_n2_highcpu_8_4ssd(self): 419 | self.httpretty_gcp_metadata() 420 | with unittest.mock.patch('psutil.cpu_count', return_value=8),\ 421 | unittest.mock.patch('psutil.virtual_memory', return_value=svmem(8334258176)),\ 422 | unittest.mock.patch('psutil.disk_partitions', return_value=mock_disk_partitions),\ 423 | unittest.mock.patch('os.listdir', return_value=mock_listdevdir_n2_highcpu_8_4ssd),\ 424 | unittest.mock.patch('glob.glob', return_value=mock_glob_glob_dev_n2_highcpu_8_4ssd),\ 425 | unittest.mock.patch('lib.scylla_cloud.gcp_instance.get_file_size_by_seek', return_value=402653184000): 426 | ins = gcp_instance() 427 | # Not enough memory 428 | assert ins.is_recommended_instance() == False 429 | 430 | def test_is_not_recommended_instance_n2_standard_8_24ssd(self): 431 | self.httpretty_gcp_metadata(num_local_disks=24) 432 | with unittest.mock.patch('psutil.cpu_count', return_value=8),\ 433 | unittest.mock.patch('psutil.virtual_memory', return_value=svmem(33663647744)),\ 434 | unittest.mock.patch('psutil.disk_partitions', return_value=mock_disk_partitions),\ 435 | unittest.mock.patch('lib.scylla_cloud.gcp_instance.get_file_size_by_seek', return_value=402653184000): 436 | ins = gcp_instance() 437 | # Requires more CPUs to use this number of SSDs 438 | assert ins.is_recommended_instance() == False 439 | 440 | def test_is_not_recommended_instance_n2_standard_8(self): 441 | self.httpretty_gcp_metadata(num_local_disks=0) 442 | with unittest.mock.patch('psutil.cpu_count', return_value=8),\ 443 | unittest.mock.patch('psutil.virtual_memory', return_value=svmem(33663647744)),\ 444 | unittest.mock.patch('psutil.disk_partitions', return_value=mock_disk_partitions),\ 445 | unittest.mock.patch('os.listdir', return_value=mock_listdevdir_n2_standard_8),\ 446 | unittest.mock.patch('glob.glob', return_value=mock_glob_glob_dev_n2_standard_8),\ 447 | unittest.mock.patch('lib.scylla_cloud.gcp_instance.get_file_size_by_seek', return_value=402653184000): 448 | ins = gcp_instance() 449 | # No SSD 450 | assert ins.is_recommended_instance() == False 451 | -------------------------------------------------------------------------------- /tests/test_scylla_configure.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2020 ScyllaDB 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | import random 6 | import sys 7 | import base64 8 | import json 9 | import shutil 10 | import tempfile 11 | import yaml 12 | import logging 13 | import unittest.mock 14 | from textwrap import dedent 15 | from unittest import TestCase 16 | from pathlib import Path 17 | from email.mime.multipart import MIMEMultipart 18 | from email.mime.base import MIMEBase 19 | 20 | sys.path.append(str(Path(__file__).parent.parent)) 21 | 22 | from lib.log import setup_logging 23 | from common.scylla_configure import ScyllaMachineImageConfigurator 24 | 25 | LOGGER = logging.getLogger(__name__) 26 | 27 | 28 | SNITCHS = ["GoogleCloudSnitch", "Ec2Snitch"] 29 | 30 | 31 | class DummyCloudInstance: 32 | 33 | ENDPOINT_SNITCH = random.choice(SNITCHS) 34 | 35 | def __init__(self, user_data, private_ipv4): 36 | self._user_data = user_data 37 | self.private_ip_v4 = private_ipv4 38 | 39 | def private_ipv4(self): 40 | return self.private_ip_v4 41 | 42 | @property 43 | def user_data(self): 44 | return self._user_data 45 | 46 | @property 47 | def endpoint_snitch(self): 48 | return self.ENDPOINT_SNITCH 49 | 50 | 51 | class TestScyllaConfigurator(TestCase): 52 | 53 | def setUp(self): 54 | LOGGER.info("Setting up test dir") 55 | self.temp_dir = tempfile.TemporaryDirectory() 56 | self.temp_dir_path = Path(self.temp_dir.name) 57 | setup_logging(log_level=logging.DEBUG, log_dir_path=str(self.temp_dir_path)) 58 | LOGGER.info("Test dir: %s", self.temp_dir_path) 59 | shutil.copyfile(Path(__file__).parent / "tests-data/scylla.yaml", str(self.temp_dir_path / "scylla.yaml")) 60 | self.private_ip = "172.16.16.1" 61 | self.configurator = ScyllaMachineImageConfigurator(scylla_yaml_path=str(self.temp_dir_path / "scylla.yaml")) 62 | self.test_cluster_name = "test-cluster" 63 | 64 | def tearDown(self): 65 | self.temp_dir.cleanup() 66 | 67 | def default_instance_metadata(self): 68 | return dict( 69 | user_data="", 70 | private_ipv4=self.private_ip 71 | ) 72 | 73 | def check_yaml_files_exist(self): 74 | assert self.configurator.scylla_yaml_example_path.exists(), "scylla.yaml example file not created" 75 | assert self.configurator.scylla_yaml_path.exists(), "scylla.yaml file not created" 76 | 77 | def run_scylla_configure(self, user_data, private_ipv4): 78 | self.configurator._cloud_instance = DummyCloudInstance(user_data=user_data, private_ipv4=private_ipv4) 79 | self.configurator.configure_scylla_yaml() 80 | 81 | @staticmethod 82 | def multipart_user_data(scylla_data, data_type): 83 | raw_user_data = json.dumps(scylla_data) if data_type == 'json' else yaml.safe_dump(scylla_data) 84 | 85 | msg = MIMEMultipart() 86 | 87 | filename = f"scylla_machine_image.{data_type}" 88 | part = MIMEBase('x-scylla', data_type) 89 | part.set_payload(raw_user_data) 90 | part.add_header('Content-Disposition', 'attachment; filename="%s"' % filename) 91 | msg.attach(part) 92 | 93 | return msg 94 | 95 | def test_empty_user_data(self): 96 | self.run_scylla_configure(**self.default_instance_metadata()) 97 | self.check_yaml_files_exist() 98 | with self.configurator.scylla_yaml_path.open() as scylla_yaml_file: 99 | LOGGER.info("Checking that defaults are set as expected...") 100 | scylla_yaml = yaml.load(scylla_yaml_file, Loader=yaml.SafeLoader) 101 | assert scylla_yaml["listen_address"] == self.private_ip 102 | assert scylla_yaml["broadcast_rpc_address"] == self.private_ip 103 | self.assertIn(scylla_yaml["endpoint_snitch"], self.configurator.cloud_instance.endpoint_snitch) 104 | assert scylla_yaml["rpc_address"] == "0.0.0.0" 105 | assert scylla_yaml["seed_provider"][0]['parameters'][0]['seeds'] == self.private_ip 106 | assert "scylladb-cluster-" in scylla_yaml["cluster_name"], "Cluster name was not autogenerated" 107 | 108 | def test_user_data_params_are_set(self): 109 | ip_to_set = "172.16.16.84" 110 | raw_user_data = json.dumps( 111 | dict( 112 | scylla_yaml=dict( 113 | cluster_name=self.test_cluster_name, 114 | listen_address=ip_to_set, 115 | broadcast_rpc_address=ip_to_set, 116 | seed_provider=[{ 117 | "class_name": "org.apache.cassandra.locator.SimpleSeedProvider", 118 | "parameters": [{"seeds": ip_to_set}]}], 119 | ) 120 | ) 121 | ) 122 | self.run_scylla_configure(user_data=raw_user_data, private_ipv4=ip_to_set) 123 | self.check_yaml_files_exist() 124 | with self.configurator.scylla_yaml_path.open() as scylla_yaml_file: 125 | scylla_yaml = yaml.load(scylla_yaml_file, Loader=yaml.SafeLoader) 126 | assert scylla_yaml["cluster_name"] == self.test_cluster_name 127 | assert scylla_yaml["listen_address"] == ip_to_set 128 | assert scylla_yaml["broadcast_rpc_address"] == ip_to_set 129 | assert scylla_yaml["seed_provider"][0]["parameters"][0]["seeds"] == ip_to_set 130 | # check defaults 131 | assert scylla_yaml["auto_bootstrap"] is True 132 | 133 | def test_postconfig_script(self): 134 | test_file = "scylla_configure_test" 135 | script = dedent(""" 136 | touch {0.temp_dir_path}/{1} 137 | """.format(self, test_file)) 138 | raw_user_data = json.dumps( 139 | dict( 140 | post_configuration_script=base64.b64encode(bytes(script, "utf-8")).decode("utf-8") 141 | ) 142 | ) 143 | self.run_scylla_configure(user_data=raw_user_data, private_ipv4=self.private_ip) 144 | self.configurator.run_post_configuration_script() 145 | assert (self.temp_dir_path / test_file).exists(), "Post configuration script didn't run" 146 | 147 | def test_postconfig_script_with_timeout(self): 148 | test_file = "scylla_configure_test" 149 | script_timeout = 5 150 | script = dedent(""" 151 | sleep {0} 152 | touch {1.temp_dir_path}/{2} 153 | """.format(script_timeout, self, test_file)) 154 | raw_user_data = json.dumps( 155 | dict( 156 | post_configuration_script=base64.b64encode(bytes(script, "utf-8")).decode("utf-8"), 157 | post_configuration_script_timeout=script_timeout - 4.5, 158 | ) 159 | ) 160 | 161 | self.run_scylla_configure(user_data=raw_user_data, private_ipv4=self.private_ip) 162 | 163 | with self.assertRaises(expected_exception=SystemExit): 164 | self.configurator.run_post_configuration_script() 165 | assert not (self.temp_dir_path / test_file).exists(), "Post configuration script didn't fail with timeout" 166 | 167 | def test_postconfig_script_with_bad_exit_code(self): 168 | script = dedent(""" 169 | exit 84 170 | """) 171 | raw_user_data = json.dumps( 172 | dict( 173 | post_configuration_script=base64.b64encode(bytes(script, "utf-8")).decode("utf-8"), 174 | ) 175 | ) 176 | self.run_scylla_configure(user_data=raw_user_data, private_ipv4=self.private_ip) 177 | with self.assertRaises(expected_exception=SystemExit): 178 | self.configurator.run_post_configuration_script() 179 | 180 | def test_multipart_user_data_params_are_set(self): 181 | ip_to_set = "172.16.16.84" 182 | 183 | scylla_user_data = dict( 184 | scylla_yaml=dict( 185 | cluster_name=self.test_cluster_name, 186 | listen_address=ip_to_set, 187 | broadcast_rpc_address=ip_to_set, 188 | seed_provider=[{ 189 | "class_name": "org.apache.cassandra.locator.SimpleSeedProvider", 190 | "parameters": [{"seeds": ip_to_set}]}], 191 | ) 192 | ) 193 | for data_type in ('json', 'yaml'): 194 | msg = self.multipart_user_data(scylla_user_data, data_type) 195 | self.run_scylla_configure(user_data=str(msg), private_ipv4=ip_to_set) 196 | self.check_yaml_files_exist() 197 | with self.configurator.scylla_yaml_path.open() as scylla_yaml_file: 198 | scylla_yaml = yaml.load(scylla_yaml_file, Loader=yaml.SafeLoader) 199 | assert scylla_yaml["cluster_name"] == self.test_cluster_name 200 | assert scylla_yaml["listen_address"] == ip_to_set 201 | assert scylla_yaml["broadcast_rpc_address"] == ip_to_set 202 | assert scylla_yaml["seed_provider"][0]["parameters"][0]["seeds"] == ip_to_set 203 | # check defaults 204 | assert scylla_yaml["auto_bootstrap"] is True 205 | 206 | def test_do_not_start_on_first_boot(self): 207 | raw_user_data = json.dumps( 208 | dict( 209 | start_scylla_on_first_boot=False, 210 | ) 211 | ) 212 | self.run_scylla_configure(user_data=raw_user_data, private_ipv4=self.private_ip) 213 | with unittest.mock.patch("subprocess.run") as mocked_run: 214 | self.configurator.start_scylla_on_first_boot() 215 | 216 | def test_default_raid0(self): 217 | self.run_scylla_configure(**self.default_instance_metadata()) 218 | with unittest.mock.patch("subprocess.run") as mocked_run: 219 | self.configurator.create_devices() 220 | assert "--raid-level 0" in str(mocked_run.call_args) 221 | 222 | def test_set_raid0(self): 223 | raw_user_data = json.dumps(dict(raid_level=0)) 224 | self.run_scylla_configure(user_data=raw_user_data, private_ipv4=self.private_ip) 225 | with unittest.mock.patch("subprocess.run") as mocked_run: 226 | self.configurator.create_devices() 227 | assert "--raid-level 0" in str(mocked_run.call_args) 228 | 229 | def test_set_raid5(self): 230 | raw_user_data = json.dumps(dict(raid_level=5)) 231 | self.run_scylla_configure(user_data=raw_user_data, private_ipv4=self.private_ip) 232 | with unittest.mock.patch("subprocess.run") as mocked_run: 233 | self.configurator.create_devices() 234 | assert "--raid-level 5" in str(mocked_run.call_args) 235 | -------------------------------------------------------------------------------- /tests/test_scylla_post_start.py: -------------------------------------------------------------------------------- 1 | # 2 | # Copyright 2020 ScyllaDB 3 | # 4 | # SPDX-License-Identifier: Apache-2.0 5 | import sys 6 | import logging 7 | from unittest import TestCase 8 | from pathlib import Path 9 | import tempfile 10 | import base64 11 | import json 12 | 13 | sys.path.append(str(Path(__file__).parent.parent)) 14 | 15 | from lib.log import setup_logging 16 | from common.scylla_post_start import ScyllaMachineImagePostStart 17 | 18 | from test_scylla_configure import DummyCloudInstance, TestScyllaConfigurator 19 | LOGGER = logging.getLogger(__name__) 20 | 21 | 22 | def b64(s): 23 | return base64.b64encode(s.encode()).decode() 24 | 25 | 26 | class TestScyllaPostStart(TestCase): 27 | 28 | def setUp(self): 29 | LOGGER.info("Setting up test dir") 30 | self.temp_dir = tempfile.TemporaryDirectory() 31 | self.temp_dir_path = Path(self.temp_dir.name) 32 | setup_logging(log_level=logging.DEBUG, log_dir_path=str(self.temp_dir_path)) 33 | LOGGER.info("Test dir: %s", self.temp_dir_path) 34 | self.post_start = ScyllaMachineImagePostStart() 35 | 36 | def tearDown(self): 37 | self.temp_dir.cleanup() 38 | 39 | def run_scylla_post_start(self, user_data): 40 | self.post_start._cloud_instance = DummyCloudInstance(user_data=user_data, private_ipv4='127.0.0.1') 41 | self.post_start.run_post_start_script() 42 | 43 | def test_no_user_data(self): 44 | self.run_scylla_post_start('') 45 | 46 | def test_empty_script(self): 47 | self.run_scylla_post_start('{"post_start_script": ""}') 48 | 49 | def test_base64_encoding(self): 50 | self.run_scylla_post_start(json.dumps({"post_start_script": b64("echo start")})) 51 | 52 | def test_base64_encoding_error(self): 53 | with self.assertRaises(SystemExit): 54 | self.run_scylla_post_start( 55 | json.dumps({"post_start_script": b64("non-existing-command")})) 56 | 57 | def test_mime_multipart(self): 58 | scylla_user_data = {"post_start_script": b64("echo start")} 59 | for data_type in ('json', 'yaml'): 60 | msg = TestScyllaConfigurator.multipart_user_data(scylla_user_data, data_type) 61 | self.run_scylla_post_start(user_data=str(msg)) 62 | -------------------------------------------------------------------------------- /tools/relocate_python_scripts.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python3 2 | # -*- coding: utf-8 -*- 3 | # 4 | # Copyright 2020 ScyllaDB 5 | # 6 | # SPDX-License-Identifier: Apache-2.0 7 | 8 | 9 | import argparse 10 | import pathlib 11 | import os 12 | import shutil 13 | import io 14 | 15 | class FilesystemFixup: 16 | def __init__(self, python_path, installroot): 17 | self.thunk = '''\ 18 | #!/usr/bin/env bash 19 | x="$(readlink -f "$0")" 20 | b="$(basename "$x")" 21 | d="$(dirname "$x")" 22 | PYTHONPATH="${{d}}:${{d}}/libexec:$PYTHONPATH" PATH="${{d}}/{pythonpath}:${{PATH}}" exec -a "$0" "${{d}}/libexec/${{b}}" "$@" 23 | ''' 24 | self.python_path = python_path 25 | self.installroot = installroot 26 | pathlib.Path(self.installroot).mkdir(parents=False, exist_ok=True) 27 | 28 | def relocated_file(self, filename): 29 | basename = os.path.basename(filename) 30 | return os.path.join("libexec", basename) 31 | 32 | def gen_thunk_contents(self, filename): 33 | base_dir = os.path.dirname(os.path.realpath(filename)) 34 | python_path = os.path.dirname(os.path.relpath(self.python_path, base_dir)) 35 | return self.thunk.format(pythonpath=python_path) 36 | 37 | def fix_shebang(self, dest, original_stat, bytes_stream): 38 | dest = os.path.join(self.installroot, self.relocated_file(dest)) 39 | pathlib.Path(os.path.dirname(dest)).mkdir(parents=True, exist_ok=True) 40 | with open(dest, "wb") as out: 41 | bytes_stream.seek(0) 42 | os.chmod(dest, original_stat.st_mode) 43 | out.write(bytes_stream.read()) 44 | 45 | def copy_as_is(self, orig, dest): 46 | dest = os.path.join(self.installroot, os.path.basename(dest)) 47 | pathlib.Path(os.path.dirname(dest)).mkdir(parents=True, exist_ok=True) 48 | shutil.copy2(orig, dest) 49 | 50 | def generate_thunk(self, original_stat, out): 51 | bash_thunk = os.path.join(self.installroot, os.path.basename(out)) 52 | with open(bash_thunk, "w") as f: 53 | os.chmod(bash_thunk, original_stat.st_mode) 54 | f.write(self.gen_thunk_contents(bash_thunk)) 55 | 56 | def fixup_script(output, script_name): 57 | '''Given a script as a parameter, fixup the script so it can be transparently called with a non-standard python3 interpreter. 58 | This will generate a copy of the script in the libexec/ inside the script's directory location with the shebang pointing to env 59 | instead of a hardcoded python location, and will replace the main script with a thunk that calls into the right interpreter 60 | ''' 61 | 62 | script = os.path.realpath(script_name) 63 | newpath = "libexec" 64 | orig_stat = os.stat(script) 65 | 66 | if not os.access(script, os.X_OK): 67 | output.copy_as_is(script, script_name) 68 | return 69 | 70 | with open(script, "r") as f: 71 | firstline = f.readline() 72 | if not firstline.startswith("#!") or not "python3" in firstline: 73 | output.copy_as_is(script, script_name) 74 | return 75 | 76 | obj = io.BytesIO() 77 | shebang = "#!/usr/bin/env python3\n" 78 | 79 | obj.write(shebang.encode()) 80 | for l in f: 81 | obj.write(l.encode()) 82 | 83 | output.fix_shebang(script_name, orig_stat, obj) 84 | 85 | output.generate_thunk(orig_stat, script_name) 86 | 87 | def fixup_scripts(output, scripts): 88 | for script in scripts: 89 | fixup_script(output, script) 90 | 91 | if __name__ == "__main__": 92 | ap = argparse.ArgumentParser(description='Modify a python3 script adding indirection to its execution so it can run with a non-standard interpreter.') 93 | ap.add_argument('--with-python3', required=True, 94 | help='path of the python3 interepreter') 95 | ap.add_argument('--installroot', required=True, 96 | help='directory where to copy the files') 97 | ap.add_argument('scripts', nargs='+', help='list of python modules scripts to modify') 98 | 99 | args = ap.parse_args() 100 | archive = FilesystemFixup(args.with_python3, args.installroot) 101 | fixup_scripts(archive, args.scripts) 102 | --------------------------------------------------------------------------------