├── .coverage_tests.sh ├── .github ├── ISSUE_TEMPLATE │ ├── bug-report.md │ └── feature-request.md └── workflows │ ├── go.yml │ └── issues.yml ├── .gitignore ├── LICENSE ├── NOTICE ├── README.md ├── cmd └── timescaledb-tune │ └── main.go ├── codecov.yml ├── go.mod ├── go.sum ├── internal └── parse │ ├── parse.go │ ├── parse.md │ ├── parse_test.go │ ├── time.go │ └── time_test.go └── pkg ├── pgtune ├── background_writer.go ├── background_writer_test.go ├── float_parser.go ├── float_parser_test.go ├── memory.go ├── memory_test.go ├── misc.go ├── misc_test.go ├── null_recommender.go ├── null_recommender_test.go ├── parallel.go ├── parallel_test.go ├── tune.go ├── tune_test.go ├── wal.go └── wal_test.go ├── pgutils ├── utils.go └── utils_test.go └── tstune ├── backup.go ├── backup_test.go ├── checker.go ├── checker_test.go ├── config_file.go ├── config_file_test.go ├── io_handler.go ├── io_handler_test.go ├── print.go ├── print_test.go ├── shared_preload_libs.go ├── shared_preload_libs_test.go ├── tune_settings.go ├── tune_settings_test.go ├── tuner.go ├── tuner_test.go ├── utils.go └── utils_test.go /.coverage_tests.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | set -e 4 | echo "" > coverage.txt 5 | 6 | for d in $(go list ./... | grep -v vendor); do 7 | go test -race -coverprofile=profile.out -covermode=atomic $d 8 | if [ -f profile.out ]; then 9 | cat profile.out >> coverage.txt 10 | rm profile.out 11 | fi 12 | done 13 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug-report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug Report 3 | about: Something is not working as expected 4 | title: '' 5 | labels: bug 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Relevant system information:** 11 | - OS: [e.g. Ubuntu 16.04, Windows 10 x64, etc] 12 | - PostgreSQL version (output of `SELECT version();`): [e.g. 12.0, 13.2, etc] 13 | - TimescaleDB Tune version (output of `timescaledb-tune --version`): [e.g. 0.14.3] 14 | 15 | **Describe the bug** 16 | A clear and concise description of what the bug is. 17 | 18 | **To Reproduce** 19 | Steps to reproduce the behavior: 20 | 1. Go to '...' 21 | 2. Click on '....' 22 | 3. Scroll down to '....' 23 | 4. See error 24 | 25 | **Expected behavior** 26 | A clear and concise description of what you _expected_ to happen. 27 | 28 | **Actual behavior** 29 | A clear and concise description of what _actually_ happened. 30 | 31 | **Screenshots** 32 | If applicable, add screenshots to help explain your problem. 33 | 34 | **Additional context** 35 | Add any other context about the problem here. 36 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature-request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature Request 3 | about: Suggest an idea for this project 4 | title: '' 5 | labels: feature-request 6 | assignees: '' 7 | 8 | --- 9 | 10 | **Is your feature request related to a problem? Please describe.** 11 | A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] 12 | 13 | **Describe the solution you'd like** 14 | A clear and concise description of what you would like to happen. 15 | 16 | **Describe alternatives you've considered** 17 | A clear and concise description of any alternative solutions or features you've considered. 18 | 19 | **Additional context** 20 | Add any other context or screenshots about the feature request here. 21 | -------------------------------------------------------------------------------- /.github/workflows/go.yml: -------------------------------------------------------------------------------- 1 | name: Go 2 | 3 | on: 4 | push: 5 | pull_request: 6 | 7 | jobs: 8 | 9 | build: 10 | runs-on: ubuntu-latest 11 | steps: 12 | - uses: actions/checkout@v3 13 | 14 | - name: Set up Go 15 | uses: actions/setup-go@v3 16 | with: 17 | go-version: 1.18 18 | 19 | - name: Build 20 | run: go build -v ./... 21 | 22 | - name: Test 23 | run: go test -v ./... 24 | -------------------------------------------------------------------------------- /.github/workflows/issues.yml: -------------------------------------------------------------------------------- 1 | name: Add bugs to bugs project 2 | 3 | "on": 4 | issues: 5 | types: [opened, labeled] 6 | issue_comment: 7 | types: [created, edited] 8 | 9 | jobs: 10 | add-to-project: 11 | name: Add issue to project 12 | runs-on: ubuntu-latest 13 | steps: 14 | - uses: actions/add-to-project@v0.4.0 15 | with: 16 | project-url: https://github.com/orgs/timescale/projects/55 17 | github-token: ${{ secrets.ORG_AUTOMATION_TOKEN }} 18 | labeled: bug 19 | 20 | waiting-for-author: 21 | name: Waiting for Author 22 | runs-on: ubuntu-latest 23 | if: github.event_name == 'issues' && github.event.action == 'labeled' 24 | && github.event.label.name == 'waiting-for-author' 25 | steps: 26 | - uses: leonsteinhaeuser/project-beta-automations@v2.0.0 27 | with: 28 | gh_token: ${{ secrets.ORG_AUTOMATION_TOKEN }} 29 | organization: timescale 30 | project_id: 55 31 | resource_node_id: ${{ github.event.issue.node_id }} 32 | status_value: 'Waiting for Author' 33 | 34 | waiting-for-engineering: 35 | name: Waiting for Engineering 36 | runs-on: ubuntu-latest 37 | if: github.event_name == 'issue_comment' && !github.event.issue.pull_request 38 | steps: 39 | - name: Install dependencies 40 | run: | 41 | sudo apt-get update 42 | sudo apt-get install jq 43 | - name: Get board column of issue 44 | id: extract_board_column 45 | continue-on-error: true 46 | run: | 47 | # Request all issues from project (server-side filtering is not supported bit GitHub yet) 48 | gh api graphql --paginate -F issue=$ISSUE -F project=$PROJECT -F owner=$OWNER -F repo=$REPO -f query=' 49 | query board_column($issue: Int!, $project: Int!, $owner: String!, $repo: String!, $endCursor: String) { 50 | repository(owner: $owner, name: $repo) { 51 | issue(number: $issue) { 52 | projectV2(number: $project) { 53 | items(first: 100, after: $endCursor) { 54 | nodes { 55 | fieldValueByName(name: "Status") { 56 | ... on ProjectV2ItemFieldSingleSelectValue { 57 | name 58 | } 59 | } 60 | content { 61 | ... on Issue { 62 | id 63 | title 64 | number 65 | repository { 66 | name 67 | owner { 68 | login 69 | } 70 | } 71 | } 72 | } 73 | } 74 | pageInfo { 75 | hasNextPage 76 | endCursor 77 | } 78 | } 79 | } 80 | } 81 | } 82 | } 83 | ' > api_result 84 | # Get board column for issue 85 | board_column=$(cat api_result | jq -r ".data.repository.issue.projectV2.items.nodes[] | 86 | select (.content.number == $ISSUE and .content.repository.name == \"$REPO\" and .content.repository.owner.login == \"$OWNER\") | 87 | .fieldValueByName.name") 88 | echo "Issue is in column: $board_column" 89 | echo "issue_board_column=$board_column" >> "$GITHUB_OUTPUT" 90 | env: 91 | OWNER: timescale 92 | REPO: ${{ github.event.repository.name }} 93 | PROJECT: 55 94 | ISSUE: ${{ github.event.issue.number }} 95 | GITHUB_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} 96 | 97 | - name: Check if organization member 98 | uses: tspascoal/get-user-teams-membership@v2 99 | id: checkUserMember 100 | with: 101 | username: ${{ github.actor }} 102 | organization: timescale 103 | team: 'database-eng' 104 | GITHUB_TOKEN: ${{ secrets.ORG_AUTOMATION_TOKEN }} 105 | - name: Remove waiting-for-author label 106 | if: ${{ steps.checkUserMember.outputs.isTeamMember == 'false' 107 | && steps.extract_board_column.outputs.issue_board_column == 'Waiting for Author' }} 108 | uses: andymckay/labeler@3a4296e9dcdf9576b0456050db78cfd34853f260 109 | with: 110 | remove-labels: 'waiting-for-author, no-activity' 111 | repo-token: ${{ secrets.ORG_AUTOMATION_TOKEN }} 112 | - name: Move to waiting for engineering column 113 | if: ${{ steps.checkUserMember.outputs.isTeamMember == 'false' 114 | && steps.extract_board_column.outputs.issue_board_column == 'Waiting for Author' }} 115 | uses: leonsteinhaeuser/project-beta-automations@v2.0.0 116 | with: 117 | gh_token: ${{ secrets.ORG_AUTOMATION_TOKEN }} 118 | organization: timescale 119 | project_id: 55 120 | resource_node_id: ${{ github.event.issue.node_id }} 121 | status_value: 'Waiting for Engineering' 122 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.log 2 | 3 | # Binaries for programs and plugins 4 | *.exe 5 | *.exe~ 6 | *.dll 7 | *.so 8 | *.dylib 9 | timescaledb-tune 10 | 11 | # Test binary, built with `go test -c` 12 | *.test 13 | 14 | # Output of the go coverage tool, specifically when used with LiteIDE 15 | *.out 16 | coverage.txt 17 | 18 | # Popular IDEs 19 | .idea/ 20 | .vscode/ 21 | 22 | # OSX internals 23 | .DS_Store 24 | 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | TimescaleDB (TM) Tune 2 | 3 | Copyright (c) 2018-2021 Timescale, Inc. All Rights Reserved. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); 6 | you may not use this file except in compliance with the License. 7 | You may obtain a copy of the License at 8 | 9 | http://www.apache.org/licenses/LICENSE-2.0 10 | 11 | Unless required by applicable law or agreed to in writing, software 12 | distributed under the License is distributed on an "AS IS" BASIS, 13 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | See the License for the specific language governing permissions and 15 | limitations under the License. 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | ## timescaledb-tune 2 | 3 | `timescaledb-tune` is a program for tuning a 4 | [TimescaleDB](//github.com/timescale/timescaledb) database to perform 5 | its best based on the host's resources such as memory and number of CPUs. 6 | It parses the existing `postgresql.conf` file to ensure that the TimescaleDB 7 | extension is appropriately installed and provides recommendations for 8 | memory, parallelism, WAL, and other settings. 9 | 10 | ### Getting started 11 | You need the Go runtime (1.12+) installed, then simply `go install` this repo: 12 | ```bash 13 | $ go install github.com/timescale/timescaledb-tune/cmd/timescaledb-tune@main 14 | ``` 15 | 16 | It is also available as a binary package on a variety systems using 17 | Homebrew, `yum`, or `apt`. Search for `timescaledb-tools`. 18 | 19 | ### Using timescaledb-tune 20 | By default, `timescaledb-tune` attempts to locate your `postgresql.conf` 21 | file for parsing by using heuristics based on the operating system, so the 22 | simplest invocation would be: 23 | ```bash 24 | $ timescaledb-tune 25 | ``` 26 | 27 | You'll then be given a series of prompts that require minimal user input to 28 | make sure your config file is up to date: 29 | ```text 30 | Using postgresql.conf at this path: 31 | /usr/local/var/postgres/postgresql.conf 32 | 33 | Is this correct? [(y)es/(n)o]: y 34 | Writing backup to: 35 | /var/folders/cr/zpgdkv194vz1g5smxl_5tggm0000gn/T/timescaledb_tune.backup201901071520 36 | 37 | shared_preload_libraries needs to be updated 38 | Current: 39 | #shared_preload_libraries = 'timescaledb' 40 | Recommended: 41 | shared_preload_libraries = 'timescaledb' 42 | Is this okay? [(y)es/(n)o]: y 43 | success: shared_preload_libraries will be updated 44 | 45 | Tune memory/parallelism/WAL and other settings? [(y)es/(n)o]: y 46 | Recommendations based on 8.00 GB of available memory and 4 CPUs for PostgreSQL 11 47 | 48 | Memory settings recommendations 49 | Current: 50 | shared_buffers = 128MB 51 | #effective_cache_size = 4GB 52 | #maintenance_work_mem = 64MB 53 | #work_mem = 4MB 54 | Recommended: 55 | shared_buffers = 2GB 56 | effective_cache_size = 6GB 57 | maintenance_work_mem = 1GB 58 | work_mem = 26214kB 59 | Is this okay? [(y)es/(s)kip/(q)uit]: 60 | ``` 61 | 62 | If you have moved the configuration file to a different location, or 63 | auto-detection fails (file an issue please!), you can provide the location 64 | with the `--conf-path` flag: 65 | ```bash 66 | $ timescaledb-tune --conf-path=/path/to/postgresql.conf 67 | ``` 68 | 69 | At the end, your `postgresql.conf` will be overwritten with the changes 70 | that you accepted from the prompts. 71 | 72 | #### Other invocations 73 | 74 | By default, timescaledb-tune provides recommendations for a typical timescaledb workload. The `--profile` flag can be 75 | used to tailor the recommendations for other workload types. Currently, the only non-default profile is "promscale". 76 | The `TSTUNE_PROFILE` environment variable can also be used to affect this behavior. 77 | 78 | ```bash 79 | $ timescaledb-tune --profile promscale 80 | ``` 81 | 82 | If you want recommendations for a specific amount of memory and/or CPUs: 83 | ```bash 84 | $ timescaledb-tune --memory="4GB" --cpus=2 85 | ``` 86 | 87 | If you want to set a specific number of background workers (`timescaledb.max_background_workers`): 88 | ```bash 89 | $ timescaledb-tune --max-bg-workers=16 90 | ``` 91 | 92 | If you have a dedicated disk for WAL, or want to specify how much of a 93 | shared disk should be used for WAL: 94 | ```bash 95 | $ timescaledb-tune --wal-disk-size="10GB" 96 | ``` 97 | 98 | If you want to accept all recommendations, you can use `--yes`: 99 | ```bash 100 | $ timescaledb-tune --yes 101 | ``` 102 | 103 | If you just want to see the recommendations without writing: 104 | ```bash 105 | $ timescaledb-tune --dry-run 106 | ``` 107 | 108 | If there are too many prompts: 109 | ```bash 110 | $ timescaledb-tune --quiet 111 | ``` 112 | 113 | And if you want to skip all prompts and get quiet output: 114 | ```bash 115 | $ timescaledb-tune --quiet --yes 116 | ``` 117 | 118 | And if you want to append the recommendations to the end of your conf file 119 | instead of in-place replacement: 120 | ```bash 121 | $ timescaledb-tune --quiet --yes --dry-run >> /path/to/postgresql.conf 122 | ``` 123 | 124 | ### Restoring backups 125 | 126 | `timescaledb-tune` makes a backup of your `postgresql.conf` file each time 127 | it runs (without the `--dry-run` flag) in your temp directory. If you find 128 | that the configuration given is not working well, you can restore a backup 129 | by using the `--restore` flag: 130 | ```bash 131 | $ timescaledb-tune --restore 132 | ``` 133 | ```text 134 | Using postgresql.conf at this path: 135 | /usr/local/var/postgres/postgresql.conf 136 | 137 | Is this correct? [(y)es/(n)o]: y 138 | Available backups (most recent first): 139 | 1) timescaledb_tune.backup201901222056 (14 hours ago) 140 | 2) timescaledb_tune.backup201901221640 (18 hours ago) 141 | 3) timescaledb_tune.backup201901221050 (24 hours ago) 142 | 4) timescaledb_tune.backup201901211817 (41 hours ago) 143 | 144 | Use which backup? Number or (q)uit: 1 145 | Restoring 'timescaledb_tune.backup201901222056'... 146 | success: restored successfully 147 | ``` 148 | 149 | ### Contributing 150 | We welcome contributions to this utility, which like TimescaleDB is 151 | released under the Apache2 Open Source License. The same [Contributors Agreement](//github.com/timescale/timescaledb/blob/master/CONTRIBUTING.md) 152 | applies; please sign the [Contributor License Agreement](https://cla-assistant.io/timescale/timescaledb-tune) (CLA) if you're a new contributor. 153 | 154 | ### Releasing 155 | 156 | Please follow the instructions [here](https://github.com/timescale/eng-database/wiki/Releasing-timescaledb-tune) 157 | to publish a release of this tool. 158 | -------------------------------------------------------------------------------- /cmd/timescaledb-tune/main.go: -------------------------------------------------------------------------------- 1 | // timescaledb-tune analyzes a user's postgresql.conf file to make sure it is 2 | // ready and tuned to use TimescaleDB. It checks that the TimescaleDB library 3 | // ('timescaledb') is properly listed as a shared preload library and analyzes 4 | // various groups of settings to make sure they are reasonably set for the 5 | // machine's resources. 6 | // 7 | // The groups of settings deal with memory usage, parallelism, the WAL, and 8 | // other miscellaneous settings that have been found to be useful when tuning. 9 | package main 10 | 11 | import ( 12 | "flag" 13 | "fmt" 14 | "os" 15 | "runtime" 16 | "strings" 17 | 18 | "github.com/timescale/timescaledb-tune/pkg/pgtune" 19 | "github.com/timescale/timescaledb-tune/pkg/tstune" 20 | ) 21 | 22 | const ( 23 | binName = "timescaledb-tune" 24 | version = tstune.Version 25 | ) 26 | 27 | var ( 28 | f tstune.TunerFlags 29 | showVersion bool 30 | ) 31 | 32 | // Parse args 33 | func init() { 34 | flag.StringVar(&f.Memory, "memory", "", "Amount of memory to base recommendations on in the PostgreSQL format , e.g., 4GB. Default is to use all memory") 35 | flag.UintVar(&f.NumCPUs, "cpus", 0, "Number of CPU cores to base recommendations on. Default is equal to number of cores") 36 | flag.StringVar(&f.PGVersion, "pg-version", "", "Major version of PostgreSQL to base recommendations on. Default is determined via pg_config. Valid values: "+strings.Join(tstune.ValidPGVersions, ", ")) 37 | flag.StringVar(&f.WALDiskSize, "wal-disk-size", "", "Size of the disk where the WAL resides, in PostgreSQL format , e.g., 4GB. Using this flag helps tune WAL behavior.") 38 | flag.Uint64Var(&f.MaxConns, "max-conns", 0, "Max number of connections for the database. Default is equal to our best recommendation") 39 | flag.IntVar(&f.MaxBGWorkers, "max-bg-workers", pgtune.MaxBackgroundWorkersDefault, "Max number of background workers") 40 | flag.StringVar(&f.ConfPath, "conf-path", "", "Path to postgresql.conf. If blank, heuristics will be used to find it") 41 | flag.StringVar(&f.DestPath, "out-path", "", "Path to write the new configuration file. If blank, will use the same file that is read from") 42 | flag.StringVar(&f.PGConfig, "pg-config", "pg_config", "Path to the pg_config binary") 43 | flag.BoolVar(&f.YesAlways, "yes", false, "Answer 'yes' to every prompt") 44 | flag.BoolVar(&f.Quiet, "quiet", false, "Show only the total recommendations at the end") 45 | flag.BoolVar(&f.UseColor, "color", true, "Use color in output (works best on dark terminals)") 46 | flag.BoolVar(&f.DryRun, "dry-run", false, "Whether to just show the changes without overwriting the configuration file") 47 | flag.BoolVar(&f.Restore, "restore", false, "Whether to restore a previously made conf file backup") 48 | flag.StringVar(&f.Profile, "profile", "", "a specific \"mode\" for tailoring recommendations to a special workload type. If blank or unspecified, a default is used unless the TSTUNE_PROFILE environment variable is set. Valid values: \"promscale\"") 49 | 50 | flag.BoolVar(&showVersion, "version", false, "Show the version of this tool") 51 | flag.Parse() 52 | 53 | // the TSTUNE_PROFILE environment variable overrides the --profile flag if the --profile is blank or unset 54 | // this is designed for docker usage 55 | if val := os.Getenv("TSTUNE_PROFILE"); val != "" && f.Profile == "" { 56 | f.Profile = val 57 | } 58 | } 59 | 60 | func main() { 61 | if showVersion { 62 | fmt.Printf("%s %s (%s %s)\n", binName, version, runtime.GOOS, runtime.GOARCH) 63 | } else { 64 | tuner := tstune.Tuner{} 65 | tuner.Run(&f, os.Stdin, os.Stdout, os.Stderr) 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /codecov.yml: -------------------------------------------------------------------------------- 1 | coverage: 2 | status: 3 | project: off 4 | patch: off 5 | -------------------------------------------------------------------------------- /go.mod: -------------------------------------------------------------------------------- 1 | module github.com/timescale/timescaledb-tune 2 | 3 | go 1.18 4 | 5 | require ( 6 | github.com/fatih/color v1.17.0 7 | github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 8 | ) 9 | 10 | require ( 11 | github.com/mattn/go-colorable v0.1.13 // indirect 12 | github.com/mattn/go-isatty v0.0.20 // indirect 13 | golang.org/x/sys v0.25.0 // indirect 14 | ) 15 | -------------------------------------------------------------------------------- /go.sum: -------------------------------------------------------------------------------- 1 | github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4= 2 | github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI= 3 | github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= 4 | github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= 5 | github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= 6 | github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= 7 | github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= 8 | github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= 9 | github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= 10 | golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 11 | golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= 12 | golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= 13 | golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= 14 | -------------------------------------------------------------------------------- /internal/parse/parse.go: -------------------------------------------------------------------------------- 1 | // Package parse provides internal constants and functions for parsing byte 2 | // totals as presented in string of uint64 forms. 3 | package parse 4 | 5 | import ( 6 | "fmt" 7 | "math" 8 | "regexp" 9 | "strconv" 10 | "strings" 11 | "time" 12 | ) 13 | 14 | // Byte equivalents (using 1024) of common byte measurements 15 | const ( 16 | Terabyte = 1 << 40 17 | Gigabyte = 1 << 30 18 | Megabyte = 1 << 20 19 | Kilobyte = 1 << 10 20 | ) 21 | 22 | // Suffixes for byte measurements that are valid to PostgreSQL 23 | const ( 24 | TB = "TB" // terabyte 25 | GB = "GB" // gigabyte 26 | MB = "MB" // megabyte 27 | KB = "kB" // kilobyte 28 | B = "" // no unit, therefore: bytes 29 | ) 30 | 31 | // TimeUnit represents valid suffixes for time measurements used by PostgreSQL settings 32 | // https://www.postgresql.org/docs/current/config-setting.html#20.1.1.%20Parameter%20Names%20and%20Values 33 | type TimeUnit int64 34 | 35 | const ( 36 | Microseconds TimeUnit = iota 37 | Milliseconds 38 | Seconds 39 | Minutes 40 | Hours 41 | Days 42 | ) 43 | 44 | func (tu TimeUnit) String() string { 45 | switch tu { 46 | case Microseconds: 47 | return "us" 48 | case Milliseconds: 49 | return "ms" 50 | case Seconds: 51 | return "s" 52 | case Minutes: 53 | return "min" 54 | case Hours: 55 | return "h" 56 | case Days: 57 | return "d" 58 | default: 59 | return "unrecognized" 60 | } 61 | } 62 | 63 | func ParseTimeUnit(val string) (TimeUnit, error) { 64 | switch strings.ToLower(val) { 65 | case "us": 66 | return Microseconds, nil 67 | case "ms": 68 | return Milliseconds, nil 69 | case "s": 70 | return Seconds, nil 71 | case "min": 72 | return Minutes, nil 73 | case "h": 74 | return Hours, nil 75 | case "d": 76 | return Days, nil 77 | default: 78 | return TimeUnit(-1), fmt.Errorf(errUnrecognizedTimeUnitsFmt, val) 79 | } 80 | } 81 | 82 | // VarType represents the values from the vartype column of the pg_settings table 83 | type VarType int64 84 | 85 | const ( 86 | VarTypeReal VarType = iota 87 | VarTypeInteger 88 | ) 89 | 90 | func (v VarType) String() string { 91 | switch v { 92 | case VarTypeReal: 93 | return "real" 94 | case VarTypeInteger: 95 | return "integer" 96 | default: 97 | return "unrecognized" 98 | } 99 | } 100 | 101 | const ( 102 | errIncorrectBytesFormatFmt = "incorrect PostgreSQL bytes format: '%s'" 103 | errIncorrectTimeFormatFmt = "incorrect PostgreSQL time format: '%s'" 104 | errCouldNotParseBytesFmt = "could not parse bytes number: %v" 105 | errUnrecognizedTimeUnitsFmt = "unrecognized time units: %s" 106 | ) 107 | 108 | var ( 109 | pgBytesRegex = regexp.MustCompile("^(?:')?([0-9]+)((?:k|M|G|T)B)?(?:')?$") 110 | pgTimeRegex = regexp.MustCompile(`^(?:')?([0-9]+(\.[0-9]+)?)(?:\s*)(us|ms|s|min|h|d)?(?:')?$`) 111 | ) 112 | 113 | func parseIntToFloatUnits(bytes uint64) (float64, string) { 114 | if bytes <= 0 { 115 | panic(fmt.Sprintf("bytes must be at least 1 byte (got %d)", bytes)) 116 | } 117 | divisor := 1.0 118 | units := B 119 | if bytes >= Terabyte { 120 | divisor = float64(Terabyte) 121 | units = TB 122 | } else if bytes >= Gigabyte { 123 | divisor = float64(Gigabyte) 124 | units = GB 125 | } else if bytes >= Megabyte { 126 | divisor = float64(Megabyte) 127 | units = MB 128 | } else if bytes >= Kilobyte { 129 | divisor = float64(Kilobyte) 130 | units = KB 131 | } 132 | return float64(bytes) / divisor, units 133 | } 134 | 135 | // BytesToDecimalFormat converts a given amount of bytes into string with a two decimal 136 | // precision float value. 137 | func BytesToDecimalFormat(bytes uint64) string { 138 | val, units := parseIntToFloatUnits(bytes) 139 | return fmt.Sprintf("%0.2f %s", val, units) 140 | } 141 | 142 | // BytesToPGFormat converts a given amount of bytes into an acceptable PostgreSQL 143 | // string, such as 1024 -> 1kB. 144 | func BytesToPGFormat(bytes uint64) string { 145 | val, units := parseIntToFloatUnits(bytes) 146 | if units == B { // nothing less than 1kB allowed 147 | val = 1.0 148 | units = KB 149 | } else if units == KB { 150 | val = math.Round(val) 151 | } else { 152 | if val-float64(uint64(val)) > 0.001 { // (anything less than .001 is not going to meaningfully change at 1024x) 153 | val = val * 1024 154 | if units == TB { 155 | units = GB 156 | } else if units == GB { 157 | units = MB 158 | } else if units == MB { 159 | units = KB 160 | } else { 161 | panic(fmt.Sprintf("unknown units: %s", units)) 162 | } 163 | } 164 | } 165 | return fmt.Sprintf("%d%s", uint64(val), units) 166 | } 167 | 168 | // PGFormatToBytes parses a string to match it to the PostgreSQL byte string format, 169 | // which is , e.g., 10GB, 1520MB, 20kB, etc. 170 | func PGFormatToBytes(val string) (uint64, error) { 171 | res := pgBytesRegex.FindStringSubmatch(val) 172 | if len(res) != 3 { 173 | return 0.0, fmt.Errorf(errIncorrectBytesFormatFmt, val) 174 | } 175 | num, err := strconv.ParseInt(res[1], 10, 64) 176 | if err != nil { 177 | return 0.0, fmt.Errorf(errCouldNotParseBytesFmt, err) 178 | } 179 | units := res[2] 180 | var ret uint64 181 | if units == KB { 182 | ret = uint64(num) * Kilobyte 183 | } else if units == MB { 184 | ret = uint64(num) * Megabyte 185 | } else if units == GB { 186 | ret = uint64(num) * Gigabyte 187 | } else if units == TB { 188 | ret = uint64(num) * Terabyte 189 | } else if units == B { 190 | ret = uint64(num) 191 | } else { 192 | return 0, fmt.Errorf("unknown units: %s", units) 193 | } 194 | return ret, nil 195 | } 196 | 197 | func NextSmallerTimeUnits(units TimeUnit) TimeUnit { 198 | switch units { 199 | case Microseconds: 200 | return Microseconds 201 | case Milliseconds: 202 | return Microseconds 203 | case Seconds: 204 | return Milliseconds 205 | case Minutes: 206 | return Seconds 207 | case Hours: 208 | return Minutes 209 | default: // Days 210 | return Hours 211 | } 212 | } 213 | 214 | func UnitsToDuration(units TimeUnit) time.Duration { 215 | switch units { 216 | case Microseconds: 217 | return time.Microsecond 218 | case Milliseconds: 219 | return time.Millisecond 220 | case Seconds: 221 | return time.Second 222 | case Minutes: 223 | return time.Minute 224 | case Hours: 225 | return time.Hour 226 | case Days: 227 | return 24 * time.Hour 228 | default: 229 | return time.Nanosecond 230 | } 231 | } 232 | 233 | func TimeConversion(fromUnits, toUnits TimeUnit) (float64, error) { 234 | return float64(UnitsToDuration(fromUnits)) / float64(UnitsToDuration(toUnits)), nil 235 | } 236 | 237 | func PGFormatToTime(val string, defaultUnits TimeUnit, vt VarType) (float64, TimeUnit, error) { 238 | // the default units, whether units were specified, and the variable type ALL impact how the value is interpreted 239 | // https://www.postgresql.org/docs/current/config-setting.html#20.1.1.%20Parameter%20Names%20and%20Values 240 | 241 | // select unit, vartype, array_agg(name) from pg_settings where unit in ('us', 'ms', 's', 'm', 'h', 'd') group by 1, 2; 242 | 243 | // parse it 244 | res := pgTimeRegex.FindStringSubmatch(val) 245 | if res == nil || len(res) < 2 { 246 | return -1.0, TimeUnit(-1), fmt.Errorf(errIncorrectTimeFormatFmt, val) 247 | } 248 | 249 | // extract the numeric portion 250 | v, err := strconv.ParseFloat(res[1], 64) 251 | if err != nil { 252 | return -1.0, TimeUnit(-1), fmt.Errorf(errIncorrectTimeFormatFmt, val) 253 | } 254 | 255 | // extract the units or use the default 256 | unitsWereUnspecified := true 257 | units := defaultUnits 258 | if len(res) >= 4 && res[3] != "" { 259 | unitsWereUnspecified = false 260 | units, err = ParseTimeUnit(res[3]) 261 | if err != nil { 262 | return -1.0, TimeUnit(-1), err 263 | } 264 | } 265 | 266 | convert := func(v float64, units TimeUnit, vt VarType) (float64, TimeUnit, error) { 267 | if _, fract := math.Modf(v); fract < math.Nextafter(0.0, 1.0) { 268 | // not distinguishable as a fractional value 269 | switch vt { 270 | case VarTypeInteger: 271 | return math.Trunc(v), units, nil 272 | case VarTypeReal: 273 | return v, units, nil 274 | } 275 | } else { 276 | // IS a fractional value. it had a decimal component 277 | toUnits := NextSmallerTimeUnits(units) 278 | if err != nil { 279 | return -1.0, TimeUnit(-1), err 280 | } 281 | conv, err := TimeConversion(units, toUnits) 282 | if err != nil { 283 | return -1.0, TimeUnit(-1), err 284 | } 285 | return math.Round(v * conv), toUnits, nil 286 | } 287 | return -1.0, TimeUnit(-1), fmt.Errorf(errIncorrectTimeFormatFmt, val) 288 | } 289 | 290 | if unitsWereUnspecified { 291 | switch vt { 292 | case VarTypeInteger: 293 | return math.Round(v), units, nil 294 | case VarTypeReal: 295 | return convert(v, units, vt) 296 | } 297 | } else /* units WERE specified */ { 298 | if units == defaultUnits { 299 | switch vt { 300 | case VarTypeInteger: 301 | return math.Round(v), units, nil 302 | case VarTypeReal: 303 | return convert(v, units, vt) 304 | } 305 | } else /* specified units are different from the default units */ { 306 | return convert(v, units, vt) 307 | } 308 | } 309 | // should never get here! 310 | return -1.0, TimeUnit(-1), fmt.Errorf(errIncorrectTimeFormatFmt, val) 311 | } 312 | -------------------------------------------------------------------------------- /internal/parse/parse.md: -------------------------------------------------------------------------------- 1 | 2 | https://www.postgresql.org/docs/current/config-setting.html#20.1.1.%20Parameter%20Names%20and%20Values 3 | 4 | Postgres settings come in one of several variable types: 5 | 6 | * enum 7 | * string 8 | * bool 9 | * integer 10 | * real 11 | 12 | Some postgres settings are measured in time. They have default units, but postgres can interpret the following units if provided: 13 | 14 | * us - microseconds 15 | * ms - milliseconds 16 | * s - seconds 17 | * min - minutes 18 | * h - hours 19 | * d - days 20 | 21 | In postgres 14, time-based settings come in three flavors: 22 | 23 | * real milliseconds 24 | * integer milliseconds 25 | * integer seconds 26 | 27 | How postgres interprets a setting's value depends upon: 28 | 29 | 1. whether the setting is a real or integer 30 | 2. the default units 31 | 3. whether units were specified in the input and whether those units are larger, smaller, or equal to the defaults 32 | 4. whether the input was a fractional value 33 | 34 | statement_timeout is an integer setting with default units of milliseconds. 35 | 36 | ``` 37 | -- no units 38 | set statement_timeout to '155'; show statement_timeout; 39 | ┌───────────────────┐ 40 | │ statement_timeout │ 41 | ├───────────────────┤ 42 | │ 155ms │ 43 | └───────────────────┘ 44 | 45 | -- default units 46 | set statement_timeout to '100ms'; show statement_timeout; 47 | ┌───────────────────┐ 48 | │ statement_timeout │ 49 | ├───────────────────┤ 50 | │ 100ms │ 51 | └───────────────────┘ 52 | 53 | -- non-default units 54 | set statement_timeout to '100s'; show statement_timeout; 55 | ┌───────────────────┐ 56 | │ statement_timeout │ 57 | ├───────────────────┤ 58 | │ 100s │ 59 | └───────────────────┘ 60 | 61 | -- non-default units with fraction 62 | set statement_timeout to '100.3s'; show statement_timeout; 63 | ┌───────────────────┐ 64 | │ statement_timeout │ 65 | ├───────────────────┤ 66 | │ 100300ms │ 67 | └───────────────────┘ 68 | 69 | -- default units with fraction 70 | set statement_timeout to '100.7ms'; show statement_timeout; 71 | ┌───────────────────┐ 72 | │ statement_timeout │ 73 | ├───────────────────┤ 74 | │ 101ms │ 75 | └───────────────────┘ 76 | 77 | -- no units with fraction 78 | set statement_timeout to '155.3'; show statement_timeout; 79 | ┌───────────────────┐ 80 | │ statement_timeout │ 81 | ├───────────────────┤ 82 | │ 155ms │ 83 | └───────────────────┘ 84 | ``` 85 | 86 | vacuum_cost_delay is a real setting with default units of milliseconds. 87 | 88 | ``` 89 | -- no units 90 | set vacuum_cost_delay to '99'; show vacuum_cost_delay; 91 | ┌───────────────────┐ 92 | │ vacuum_cost_delay │ 93 | ├───────────────────┤ 94 | │ 99ms │ 95 | └───────────────────┘ 96 | 97 | -- default units 98 | set vacuum_cost_delay to '100ms'; show vacuum_cost_delay; 99 | ┌───────────────────┐ 100 | │ vacuum_cost_delay │ 101 | ├───────────────────┤ 102 | │ 100ms │ 103 | └───────────────────┘ 104 | 105 | -- non-default units 106 | set vacuum_cost_delay to '1000us'; show vacuum_cost_delay; 107 | ┌───────────────────┐ 108 | │ vacuum_cost_delay │ 109 | ├───────────────────┤ 110 | │ 1ms │ 111 | └───────────────────┘ 112 | 113 | -- non-default units with fraction 114 | set vacuum_cost_delay to '100.3us'; show vacuum_cost_delay; 115 | ┌───────────────────┐ 116 | │ vacuum_cost_delay │ 117 | ├───────────────────┤ 118 | │ 100.3us │ 119 | └───────────────────┘ 120 | 121 | -- default units with fraction 122 | set vacuum_cost_delay to '50.7ms'; show vacuum_cost_delay; 123 | ┌───────────────────┐ 124 | │ vacuum_cost_delay │ 125 | ├───────────────────┤ 126 | │ 50700us │ 127 | └───────────────────┘ 128 | 129 | -- no units with fraction 130 | set vacuum_cost_delay to '55.3'; show vacuum_cost_delay; 131 | ┌───────────────────┐ 132 | │ vacuum_cost_delay │ 133 | ├───────────────────┤ 134 | │ 55300us │ 135 | └───────────────────┘ 136 | ``` 137 | -------------------------------------------------------------------------------- /internal/parse/time.go: -------------------------------------------------------------------------------- 1 | package parse 2 | 3 | import ( 4 | "fmt" 5 | "time" 6 | ) 7 | 8 | const lessThanMin = "less than a minute" 9 | 10 | // PrettyDuration returns a human-readable duration that should fit the 11 | // phrase "X ago", e.g., "less than a minute ago", "2 minutes ago", etc. 12 | func PrettyDuration(d time.Duration) string { 13 | if d < time.Minute { 14 | return lessThanMin 15 | } else if d < time.Hour { 16 | mins := int64(d.Minutes()) 17 | ending := "" 18 | if mins > 1 { 19 | ending = "s" 20 | } 21 | return fmt.Sprintf("%d minute%s", mins, ending) 22 | } else if d < 48*time.Hour { 23 | hrs := int64(d.Hours()) 24 | ending := "" 25 | if hrs > 1 { 26 | ending = "s" 27 | } 28 | return fmt.Sprintf("%d hour%s", hrs, ending) 29 | } 30 | 31 | days := int64(d.Hours()) / 24 32 | return fmt.Sprintf("%d days", days) 33 | } 34 | -------------------------------------------------------------------------------- /internal/parse/time_test.go: -------------------------------------------------------------------------------- 1 | package parse 2 | 3 | import ( 4 | "testing" 5 | "time" 6 | ) 7 | 8 | func TestPrettyDuration(t *testing.T) { 9 | now := time.Now() 10 | cases := []struct { 11 | given time.Time 12 | want string 13 | }{ 14 | { 15 | given: now, 16 | want: lessThanMin, 17 | }, 18 | { 19 | given: now.Add(59 * time.Second), 20 | want: lessThanMin, 21 | }, 22 | { 23 | given: now.Add(60 * time.Second), 24 | want: "1 minute", 25 | }, 26 | { 27 | given: now.Add(61 * time.Second), 28 | want: "1 minute", 29 | }, 30 | { 31 | given: now.Add(2 * time.Minute), 32 | want: "2 minutes", 33 | }, 34 | { 35 | given: now.Add(59 * time.Minute), 36 | want: "59 minutes", 37 | }, 38 | { 39 | given: now.Add(60 * time.Minute), 40 | want: "1 hour", 41 | }, 42 | { 43 | given: now.Add(61 * time.Minute), 44 | want: "1 hour", 45 | }, 46 | { 47 | given: now.Add(2 * time.Hour), 48 | want: "2 hours", 49 | }, 50 | { 51 | given: now.Add(47 * time.Hour), 52 | want: "47 hours", 53 | }, 54 | { 55 | given: now.Add(48 * time.Hour), 56 | want: "2 days", 57 | }, 58 | } 59 | 60 | for _, c := range cases { 61 | d := c.given.Sub(now) 62 | if got := PrettyDuration(d); got != c.want { 63 | t.Errorf("incorrect value for %v: got\n%s\nwant\n%s", c.given, got, c.want) 64 | } 65 | } 66 | } 67 | -------------------------------------------------------------------------------- /pkg/pgtune/background_writer.go: -------------------------------------------------------------------------------- 1 | package pgtune 2 | 3 | const ( 4 | BgwriterFlushAfterKey = "bgwriter_flush_after" 5 | 6 | promscaleDefaultBgwriterFlushAfter = "0" 7 | ) 8 | 9 | // BgwriterLabel is the label used to refer to the background writer settings group 10 | const BgwriterLabel = "background writer" 11 | 12 | var BgwriterKeys = []string{ 13 | BgwriterFlushAfterKey, 14 | } 15 | 16 | // PromscaleBgwriterRecommender gives recommendations for the background writer for the promscale profile 17 | type PromscaleBgwriterRecommender struct{} 18 | 19 | // IsAvailable returns whether this Recommender is usable given the system resources. Always true. 20 | func (r *PromscaleBgwriterRecommender) IsAvailable() bool { 21 | return true 22 | } 23 | 24 | // Recommend returns the recommended PostgreSQL formatted value for the conf 25 | // file for a given key. 26 | func (r *PromscaleBgwriterRecommender) Recommend(key string) string { 27 | switch key { 28 | case BgwriterFlushAfterKey: 29 | return promscaleDefaultBgwriterFlushAfter 30 | default: 31 | return NoRecommendation 32 | } 33 | } 34 | 35 | // BgwriterSettingsGroup is the SettingsGroup to represent settings that affect the background writer. 36 | type BgwriterSettingsGroup struct { 37 | totalMemory uint64 38 | cpus int 39 | maxConns uint64 40 | } 41 | 42 | // Label should always return the value BgwriterLabel. 43 | func (sg *BgwriterSettingsGroup) Label() string { return BgwriterLabel } 44 | 45 | // Keys should always return the BgwriterKeys slice. 46 | func (sg *BgwriterSettingsGroup) Keys() []string { return BgwriterKeys } 47 | 48 | // GetRecommender should return a new Recommender. 49 | func (sg *BgwriterSettingsGroup) GetRecommender(profile Profile) Recommender { 50 | switch profile { 51 | case PromscaleProfile: 52 | return &PromscaleBgwriterRecommender{} 53 | default: 54 | return &NullRecommender{} 55 | } 56 | } 57 | -------------------------------------------------------------------------------- /pkg/pgtune/background_writer_test.go: -------------------------------------------------------------------------------- 1 | package pgtune 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestBgwriterSettingsGroup_GetRecommender(t *testing.T) { 9 | cases := []struct { 10 | profile Profile 11 | recommender string 12 | }{ 13 | {DefaultProfile, "*pgtune.NullRecommender"}, 14 | {PromscaleProfile, "*pgtune.PromscaleBgwriterRecommender"}, 15 | } 16 | 17 | sg := BgwriterSettingsGroup{} 18 | for _, k := range cases { 19 | r := sg.GetRecommender(k.profile) 20 | y := fmt.Sprintf("%T", r) 21 | if y != k.recommender { 22 | t.Errorf("Expected to get a %s using the %s profile but got %s", k.recommender, k.profile, y) 23 | } 24 | } 25 | } 26 | 27 | func TestBgwriterSettingsGroupRecommend(t *testing.T) { 28 | sg := BgwriterSettingsGroup{} 29 | 30 | // the default profile should provide no recommendations 31 | r := sg.GetRecommender(DefaultProfile) 32 | if val := r.Recommend(BgwriterFlushAfterKey); val != NoRecommendation { 33 | t.Errorf("Expected no recommendation for key %s but got %s", BgwriterFlushAfterKey, val) 34 | } 35 | 36 | // the promscale profile should have recommendations 37 | r = sg.GetRecommender(PromscaleProfile) 38 | if val := r.Recommend(BgwriterFlushAfterKey); val != promscaleDefaultBgwriterFlushAfter { 39 | t.Errorf("Expected %s for key %s but got %s", promscaleDefaultBgwriterFlushAfter, BgwriterFlushAfterKey, val) 40 | } 41 | } 42 | 43 | func TestPromscaleBgwriterRecommender(t *testing.T) { 44 | r := PromscaleBgwriterRecommender{} 45 | if !r.IsAvailable() { 46 | t.Error("PromscaleBgwriterRecommender should always be available") 47 | } 48 | if val := r.Recommend(BgwriterFlushAfterKey); val != promscaleDefaultBgwriterFlushAfter { 49 | t.Errorf("Expected %s for key %s but got %s", promscaleDefaultBgwriterFlushAfter, BgwriterFlushAfterKey, val) 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /pkg/pgtune/float_parser.go: -------------------------------------------------------------------------------- 1 | package pgtune 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | "strings" 7 | 8 | "github.com/timescale/timescaledb-tune/internal/parse" 9 | ) 10 | 11 | const ( 12 | errUnrecognizedBoolValue = "unrecognized bool value: %s" 13 | ) 14 | 15 | type FloatParser interface { 16 | ParseFloat(string, string) (float64, error) 17 | } 18 | 19 | type bytesFloatParser struct{} 20 | 21 | func (v *bytesFloatParser) ParseFloat(key string, s string) (float64, error) { 22 | temp, err := parse.PGFormatToBytes(s) 23 | return float64(temp), err 24 | } 25 | 26 | type numericFloatParser struct{} 27 | 28 | func (v *numericFloatParser) ParseFloat(key string, s string) (float64, error) { 29 | return strconv.ParseFloat(s, 64) 30 | } 31 | 32 | type boolFloatParser struct{} 33 | 34 | func (v *boolFloatParser) ParseFloat(key string, s string) (float64, error) { 35 | s = strings.ToLower(s) 36 | s = strings.TrimLeft(s, `"'`) 37 | s = strings.TrimRight(s, `"'`) 38 | switch s { 39 | case "on": 40 | return 1.0, nil 41 | case "off": 42 | return 0.0, nil 43 | case "true": 44 | return 1.0, nil 45 | case "false": 46 | return 0.0, nil 47 | case "yes": 48 | return 1.0, nil 49 | case "no": 50 | return 0.0, nil 51 | case "1": 52 | return 1.0, nil 53 | case "0": 54 | return 0.0, nil 55 | default: 56 | return 0.0, fmt.Errorf(errUnrecognizedBoolValue, s) 57 | } 58 | } 59 | 60 | // GetFloatParser returns the correct FloatParser for a given Recommender. 61 | func GetFloatParser(r Recommender) FloatParser { 62 | switch r.(type) { 63 | case *MemoryRecommender: 64 | return &bytesFloatParser{} 65 | case *WALRecommender: 66 | return &WALFloatParser{} 67 | case *PromscaleWALRecommender: 68 | return &WALFloatParser{} 69 | case *PromscaleBgwriterRecommender: 70 | return &numericFloatParser{} 71 | case *ParallelRecommender: 72 | return &numericFloatParser{} 73 | default: 74 | return &numericFloatParser{} 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /pkg/pgtune/float_parser_test.go: -------------------------------------------------------------------------------- 1 | package pgtune 2 | 3 | import ( 4 | "testing" 5 | 6 | "github.com/timescale/timescaledb-tune/internal/parse" 7 | ) 8 | 9 | func TestBytesFloatParserParseFloat(t *testing.T) { 10 | s := "8" + parse.GB 11 | want := float64(8 * parse.Gigabyte) 12 | v := &bytesFloatParser{} 13 | got, err := v.ParseFloat("foo", s) 14 | if err != nil { 15 | t.Errorf("unexpected error: %v", err) 16 | } 17 | if got != want { 18 | t.Errorf("incorrect result: got %f want %f", got, want) 19 | } 20 | } 21 | 22 | func TestNumericFloatParserParseFloat(t *testing.T) { 23 | s := "8.245" 24 | want := 8.245 25 | v := &numericFloatParser{} 26 | got, err := v.ParseFloat("foo", s) 27 | if err != nil { 28 | t.Errorf("unexpected error: %v", err) 29 | } 30 | if got != want { 31 | t.Errorf("incorrect result: got %f want %f", got, want) 32 | } 33 | } 34 | 35 | func Test_boolFloatParser_ParseFloat(t *testing.T) { 36 | tests := []struct { 37 | name string 38 | arg string 39 | want float64 40 | wantErr bool 41 | }{ 42 | { 43 | name: "on", 44 | arg: "on", 45 | want: 1.0, 46 | wantErr: false, 47 | }, 48 | { 49 | name: "oN", 50 | arg: "oN", 51 | want: 1.0, 52 | wantErr: false, 53 | }, 54 | { 55 | name: "'ON'", 56 | arg: "'ON'", 57 | want: 1.0, 58 | wantErr: false, 59 | }, 60 | { 61 | name: "off", 62 | arg: "off", 63 | want: 0.0, 64 | wantErr: false, 65 | }, 66 | { 67 | name: "OfF", 68 | arg: "OfF", 69 | want: 0.0, 70 | wantErr: false, 71 | }, 72 | { 73 | name: "'OFF'", 74 | arg: "'OFF'", 75 | want: 0.0, 76 | wantErr: false, 77 | }, 78 | { 79 | name: "true", 80 | arg: "true", 81 | want: 1.0, 82 | wantErr: false, 83 | }, 84 | { 85 | name: "false", 86 | arg: "false", 87 | want: 0.0, 88 | wantErr: false, 89 | }, 90 | { 91 | name: "yes", 92 | arg: "yes", 93 | want: 1.0, 94 | wantErr: false, 95 | }, 96 | { 97 | name: "no", 98 | arg: "no", 99 | want: 0.0, 100 | wantErr: false, 101 | }, 102 | { 103 | name: "1", 104 | arg: "1", 105 | want: 1.0, 106 | wantErr: false, 107 | }, 108 | { 109 | name: "0", 110 | arg: "0", 111 | want: 0.0, 112 | wantErr: false, 113 | }, 114 | { 115 | name: "bob", 116 | arg: "bob", 117 | want: 0.0, 118 | wantErr: true, 119 | }, 120 | { 121 | name: "99", 122 | arg: "99", 123 | want: 0.0, 124 | wantErr: true, 125 | }, 126 | { 127 | name: "0.1", 128 | arg: "0.1", 129 | want: 0.0, 130 | wantErr: true, 131 | }, 132 | } 133 | for _, tt := range tests { 134 | t.Run(tt.name, func(t *testing.T) { 135 | v := &boolFloatParser{} 136 | got, err := v.ParseFloat("", tt.arg) 137 | if (err != nil) != tt.wantErr { 138 | t.Errorf("ParseFloat() error = %v, wantErr %v", err, tt.wantErr) 139 | return 140 | } 141 | if got != tt.want { 142 | t.Errorf("ParseFloat() got = %v, want %v", got, tt.want) 143 | } 144 | }) 145 | } 146 | } 147 | 148 | func TestGetFloatParser(t *testing.T) { 149 | switch x := (GetFloatParser(&MemoryRecommender{})).(type) { 150 | case *bytesFloatParser: 151 | default: 152 | t.Errorf("wrong validator type for MemoryRecommender: got %T", x) 153 | } 154 | 155 | switch x := (GetFloatParser(&WALRecommender{})).(type) { 156 | case *WALFloatParser: 157 | default: 158 | t.Errorf("wrong validator type for WALRecommender: got %T", x) 159 | } 160 | 161 | switch x := (GetFloatParser(&PromscaleWALRecommender{})).(type) { 162 | case *WALFloatParser: 163 | default: 164 | t.Errorf("wrong validator type for PromscaleWALRecommender: got %T", x) 165 | } 166 | 167 | switch x := (GetFloatParser(&ParallelRecommender{})).(type) { 168 | case *numericFloatParser: 169 | default: 170 | t.Errorf("wrong validator type for ParallelRecommender: got %T", x) 171 | } 172 | 173 | switch x := (GetFloatParser(&PromscaleBgwriterRecommender{})).(type) { 174 | case *numericFloatParser: 175 | default: 176 | t.Errorf("wrong validator type for PromscaleBgwriterRecommender: got %T", x) 177 | } 178 | 179 | switch x := (GetFloatParser(&MiscRecommender{})).(type) { 180 | case *numericFloatParser: 181 | default: 182 | t.Errorf("wrong validator type for MiscRecommender: got %T", x) 183 | } 184 | 185 | switch x := (GetFloatParser(&NullRecommender{})).(type) { 186 | case *numericFloatParser: 187 | default: 188 | t.Errorf("wrong validator type for NullRecommender: got %T", x) 189 | } 190 | } 191 | -------------------------------------------------------------------------------- /pkg/pgtune/memory.go: -------------------------------------------------------------------------------- 1 | package pgtune 2 | 3 | import ( 4 | "math" 5 | "runtime" 6 | 7 | "github.com/timescale/timescaledb-tune/internal/parse" 8 | ) 9 | 10 | // Keys in the conf file that are tuned related to memory 11 | const ( 12 | SharedBuffersKey = "shared_buffers" 13 | EffectiveCacheKey = "effective_cache_size" 14 | MaintenanceWorkMemKey = "maintenance_work_mem" 15 | WorkMemKey = "work_mem" 16 | 17 | // the limit is 2GB on Unix, but 2047MB on Windows, so using 2047MB is easier all around 18 | maintenanceWorkMemLimit = 2047 * parse.Megabyte 19 | sharedBuffersWindows = 512 * parse.Megabyte 20 | baseConns = 20 21 | workMemMin = 64 * parse.Kilobyte 22 | workMemPerGigPerConn = 6.4 * baseConns // derived from pgtune results 23 | workMemPerGigPerConnWindows = 8.53336 * baseConns // derived from pgtune results 24 | ) 25 | 26 | // MemoryLabel is the label used to refer to the memory settings group 27 | const MemoryLabel = "memory" 28 | 29 | // MemoryKeys is an array of keys that are tunable for memory 30 | var MemoryKeys = []string{ 31 | SharedBuffersKey, 32 | EffectiveCacheKey, 33 | MaintenanceWorkMemKey, 34 | WorkMemKey, 35 | } 36 | 37 | // MemoryRecommender gives recommendations for ParallelKeys based on system resources 38 | type MemoryRecommender struct { 39 | totalMemory uint64 40 | cpus int 41 | conns uint64 42 | } 43 | 44 | // NewMemoryRecommender returns a MemoryRecommender that recommends based on the given 45 | // number of cpus and system memory 46 | func NewMemoryRecommender(totalMemory uint64, cpus int, maxConns uint64) *MemoryRecommender { 47 | conns := maxConns 48 | if conns == 0 { 49 | conns = getMaxConns(totalMemory) 50 | } 51 | return &MemoryRecommender{totalMemory, cpus, conns} 52 | } 53 | 54 | // IsAvailable returns whether this Recommender is usable given the system resources. Always true. 55 | func (r *MemoryRecommender) IsAvailable() bool { 56 | return true 57 | } 58 | 59 | // Recommend returns the recommended PostgreSQL formatted value for the conf 60 | // file for a given key. 61 | func (r *MemoryRecommender) Recommend(key string) string { 62 | var val string 63 | if key == SharedBuffersKey { 64 | if runtime.GOOS == osWindows { 65 | val = parse.BytesToPGFormat(sharedBuffersWindows) 66 | } else { 67 | val = parse.BytesToPGFormat(r.totalMemory / 4) 68 | } 69 | } else if key == EffectiveCacheKey { 70 | val = parse.BytesToPGFormat((r.totalMemory * 3) / 4) 71 | } else if key == MaintenanceWorkMemKey { 72 | temp := (float64(r.totalMemory) / float64(parse.Gigabyte)) * (128.0 * float64(parse.Megabyte)) 73 | if temp > maintenanceWorkMemLimit { 74 | temp = maintenanceWorkMemLimit 75 | } 76 | val = parse.BytesToPGFormat(uint64(temp)) 77 | } else if key == WorkMemKey { 78 | if runtime.GOOS == osWindows { 79 | val = r.recommendWindows() 80 | } else { 81 | cpuFactor := math.Round(float64(r.cpus) / 2.0) 82 | gigs := float64(r.totalMemory) / float64(parse.Gigabyte) 83 | temp := uint64(gigs * (workMemPerGigPerConn * float64(parse.Megabyte) / float64(r.conns)) / cpuFactor) 84 | if temp < workMemMin { 85 | temp = workMemMin 86 | } 87 | val = parse.BytesToPGFormat(temp) 88 | } 89 | 90 | } else { 91 | val = NoRecommendation 92 | } 93 | return val 94 | } 95 | 96 | func (r *MemoryRecommender) recommendWindows() string { 97 | cpuFactor := math.Round(float64(r.cpus) / 2.0) 98 | var temp uint64 99 | 100 | if r.totalMemory <= 2*parse.Gigabyte { 101 | gigs := float64(r.totalMemory) / float64(parse.Gigabyte) 102 | temp = uint64(gigs * (workMemPerGigPerConn * float64(parse.Megabyte) / float64(r.conns)) / cpuFactor) 103 | } else { 104 | base := 2.0 * workMemPerGigPerConn * float64(parse.Megabyte) 105 | gigs := float64(r.totalMemory)/float64(parse.Gigabyte) - 2.0 106 | temp = uint64(((gigs*(workMemPerGigPerConnWindows*float64(parse.Megabyte)) + base) / float64(r.conns)) / cpuFactor) 107 | } 108 | if temp < workMemMin { 109 | temp = workMemMin 110 | } 111 | return parse.BytesToPGFormat(temp) 112 | } 113 | 114 | // PromscaleMemoryRecommender gives recommendations for ParallelKeys based on system resources 115 | type PromscaleMemoryRecommender struct { 116 | *MemoryRecommender 117 | } 118 | 119 | // NewPromscaleMemoryRecommender returns a PromscaleMemoryRecommender that recommends based on the given 120 | // number of cpus and system memory 121 | func NewPromscaleMemoryRecommender(totalMemory uint64, cpus int, maxConns uint64) *PromscaleMemoryRecommender { 122 | return &PromscaleMemoryRecommender{ 123 | MemoryRecommender: NewMemoryRecommender(totalMemory, cpus, maxConns), 124 | } 125 | } 126 | 127 | // IsAvailable returns whether this Recommender is usable given the system resources. Always true. 128 | func (r *PromscaleMemoryRecommender) IsAvailable() bool { 129 | return true 130 | } 131 | 132 | // Recommend returns the recommended PostgreSQL formatted value for the conf 133 | // file for a given key. 134 | func (r *PromscaleMemoryRecommender) Recommend(key string) string { 135 | var val string 136 | switch key { 137 | case SharedBuffersKey: 138 | if runtime.GOOS == osWindows { 139 | val = parse.BytesToPGFormat(sharedBuffersWindows) 140 | } else { 141 | val = parse.BytesToPGFormat(r.totalMemory / 2) 142 | } 143 | default: 144 | val = r.MemoryRecommender.Recommend(key) 145 | } 146 | return val 147 | } 148 | 149 | // MemorySettingsGroup is the SettingsGroup to represent settings that affect memory usage. 150 | type MemorySettingsGroup struct { 151 | totalMemory uint64 152 | cpus int 153 | maxConns uint64 154 | } 155 | 156 | // Label should always return the value MemoryLabel. 157 | func (sg *MemorySettingsGroup) Label() string { return MemoryLabel } 158 | 159 | // Keys should always return the MemoryKeys slice. 160 | func (sg *MemorySettingsGroup) Keys() []string { return MemoryKeys } 161 | 162 | // GetRecommender should return a new MemoryRecommender. 163 | func (sg *MemorySettingsGroup) GetRecommender(profile Profile) Recommender { 164 | switch profile { 165 | case PromscaleProfile: 166 | return NewPromscaleMemoryRecommender(sg.totalMemory, sg.cpus, sg.maxConns) 167 | default: 168 | return NewMemoryRecommender(sg.totalMemory, sg.cpus, sg.maxConns) 169 | } 170 | } 171 | -------------------------------------------------------------------------------- /pkg/pgtune/memory_test.go: -------------------------------------------------------------------------------- 1 | package pgtune 2 | 3 | import ( 4 | "math" 5 | "math/rand" 6 | "testing" 7 | 8 | "github.com/timescale/timescaledb-tune/internal/parse" 9 | ) 10 | 11 | // defaultMemoryToBaseVals provides a memory from test memory levels to expected "base" 12 | // memory settings. These "base" values are the values if there is only 1 CPU 13 | // and 20 max connections to the database. Most settings are actually 14 | // unaffected by number of CPUs and max connections; the exception is work_mem, 15 | // so the adjustment is done in the init function 16 | var defaultMemoryToBaseVals = map[uint64]map[string]uint64{ 17 | 10 * parse.Gigabyte: { 18 | SharedBuffersKey: 2560 * parse.Megabyte, 19 | EffectiveCacheKey: 7680 * parse.Megabyte, 20 | MaintenanceWorkMemKey: 1280 * parse.Megabyte, 21 | WorkMemKey: 64 * parse.Megabyte, 22 | }, 23 | 12 * parse.Gigabyte: { 24 | SharedBuffersKey: 3 * parse.Gigabyte, 25 | EffectiveCacheKey: 9 * parse.Gigabyte, 26 | MaintenanceWorkMemKey: 1536 * parse.Megabyte, 27 | WorkMemKey: 78643 * parse.Kilobyte, 28 | }, 29 | 32 * parse.Gigabyte: { 30 | SharedBuffersKey: 8 * parse.Gigabyte, 31 | EffectiveCacheKey: 24 * parse.Gigabyte, 32 | MaintenanceWorkMemKey: maintenanceWorkMemLimit, 33 | WorkMemKey: 209715 * parse.Kilobyte, 34 | }, 35 | } 36 | 37 | // promscaleMemoryToBaseVals provides a memory from test memory levels to expected "base" 38 | // memory settings. These "base" values are the values if there is only 1 CPU 39 | // and 20 max connections to the database. Most settings are actually 40 | // unaffected by number of CPUs and max connections; the exception is work_mem, 41 | // so the adjustment is done in the init function 42 | var promscaleMemoryToBaseVals = map[uint64]map[string]uint64{ 43 | 10 * parse.Gigabyte: { 44 | SharedBuffersKey: 5120 * parse.Megabyte, 45 | EffectiveCacheKey: 7680 * parse.Megabyte, 46 | MaintenanceWorkMemKey: 1280 * parse.Megabyte, 47 | WorkMemKey: 64 * parse.Megabyte, 48 | }, 49 | 12 * parse.Gigabyte: { 50 | SharedBuffersKey: 6 * parse.Gigabyte, 51 | EffectiveCacheKey: 9 * parse.Gigabyte, 52 | MaintenanceWorkMemKey: 1536 * parse.Megabyte, 53 | WorkMemKey: 78643 * parse.Kilobyte, 54 | }, 55 | 32 * parse.Gigabyte: { 56 | SharedBuffersKey: 16 * parse.Gigabyte, 57 | EffectiveCacheKey: 24 * parse.Gigabyte, 58 | MaintenanceWorkMemKey: maintenanceWorkMemLimit, 59 | WorkMemKey: 209715 * parse.Kilobyte, 60 | }, 61 | } 62 | 63 | // highCPUs is the number of CPUs that is high enough that work_mem would normally 64 | // fall below the minimum (64KB) using the standard formula 65 | const highCPUs = 9000 66 | 67 | var ( 68 | // cpuVals is the different amounts of CPUs to test 69 | cpuVals = []int{1, 4, 5, highCPUs} 70 | // connVals is the different number of conns to test 71 | connVals = []uint64{0, 19, 20, 50} 72 | // defaultMemorySettingsMatrix stores the test cases for MemoryRecommend along with 73 | // the expected values 74 | defaultMemorySettingsMatrix = map[uint64]map[int]map[uint64]map[string]string{} 75 | // promscaleMemorySettingsMatrix stores the test cases for PromscaleMemoryRecommend along with 76 | // the expected values 77 | promscaleMemorySettingsMatrix = map[uint64]map[int]map[uint64]map[string]string{} 78 | ) 79 | 80 | func init() { 81 | for mem, baseMatrix := range defaultMemoryToBaseVals { 82 | defaultMemorySettingsMatrix[mem] = make(map[int]map[uint64]map[string]string) 83 | for _, cpus := range cpuVals { 84 | defaultMemorySettingsMatrix[mem][cpus] = make(map[uint64]map[string]string) 85 | for _, conns := range connVals { 86 | defaultMemorySettingsMatrix[mem][cpus][conns] = make(map[string]string) 87 | 88 | defaultMemorySettingsMatrix[mem][cpus][conns][SharedBuffersKey] = parse.BytesToPGFormat(baseMatrix[SharedBuffersKey]) 89 | defaultMemorySettingsMatrix[mem][cpus][conns][EffectiveCacheKey] = parse.BytesToPGFormat(baseMatrix[EffectiveCacheKey]) 90 | defaultMemorySettingsMatrix[mem][cpus][conns][MaintenanceWorkMemKey] = parse.BytesToPGFormat(baseMatrix[MaintenanceWorkMemKey]) 91 | 92 | if cpus == highCPUs { 93 | defaultMemorySettingsMatrix[mem][cpus][conns][WorkMemKey] = parse.BytesToPGFormat(workMemMin) 94 | } else { 95 | // CPU only affects work_mem in groups of 2 (i.e. 2 and 3 CPUs are treated as the same) 96 | cpuFactor := math.Round(float64(cpus) / 2.0) 97 | // Our work_mem values are derivied by a certain amount of memory lost/gained when 98 | // moving away from baseConns 99 | connFactor := float64(MaxConnectionsDefault) / float64(baseConns) 100 | if conns != 0 { 101 | connFactor = float64(conns) / float64(baseConns) 102 | } 103 | 104 | defaultMemorySettingsMatrix[mem][cpus][conns][WorkMemKey] = 105 | parse.BytesToPGFormat(uint64(float64(baseMatrix[WorkMemKey]) / connFactor / cpuFactor)) 106 | } 107 | } 108 | } 109 | } 110 | 111 | for mem, baseMatrix := range promscaleMemoryToBaseVals { 112 | promscaleMemorySettingsMatrix[mem] = make(map[int]map[uint64]map[string]string) 113 | for _, cpus := range cpuVals { 114 | promscaleMemorySettingsMatrix[mem][cpus] = make(map[uint64]map[string]string) 115 | for _, conns := range connVals { 116 | promscaleMemorySettingsMatrix[mem][cpus][conns] = make(map[string]string) 117 | 118 | promscaleMemorySettingsMatrix[mem][cpus][conns][SharedBuffersKey] = parse.BytesToPGFormat(baseMatrix[SharedBuffersKey]) 119 | promscaleMemorySettingsMatrix[mem][cpus][conns][EffectiveCacheKey] = parse.BytesToPGFormat(baseMatrix[EffectiveCacheKey]) 120 | promscaleMemorySettingsMatrix[mem][cpus][conns][MaintenanceWorkMemKey] = parse.BytesToPGFormat(baseMatrix[MaintenanceWorkMemKey]) 121 | 122 | if cpus == highCPUs { 123 | promscaleMemorySettingsMatrix[mem][cpus][conns][WorkMemKey] = parse.BytesToPGFormat(workMemMin) 124 | } else { 125 | // CPU only affects work_mem in groups of 2 (i.e. 2 and 3 CPUs are treated as the same) 126 | cpuFactor := math.Round(float64(cpus) / 2.0) 127 | // Our work_mem values are derivied by a certain amount of memory lost/gained when 128 | // moving away from baseConns 129 | connFactor := float64(MaxConnectionsDefault) / float64(baseConns) 130 | if conns != 0 { 131 | connFactor = float64(conns) / float64(baseConns) 132 | } 133 | 134 | promscaleMemorySettingsMatrix[mem][cpus][conns][WorkMemKey] = 135 | parse.BytesToPGFormat(uint64(float64(baseMatrix[WorkMemKey]) / connFactor / cpuFactor)) 136 | } 137 | } 138 | } 139 | } 140 | } 141 | 142 | func TestNewMemoryRecommender(t *testing.T) { 143 | for i := 0; i < 1000000; i++ { 144 | mem := rand.Uint64() 145 | cpus := rand.Intn(128) 146 | r := NewMemoryRecommender(mem, cpus, MaxConnectionsDefault) 147 | if r == nil { 148 | t.Errorf("unexpected nil recommender") 149 | } 150 | if got := r.totalMemory; got != mem { 151 | t.Errorf("recommender has incorrect cpus: got %d want %d", got, cpus) 152 | } 153 | if got := r.cpus; got != cpus { 154 | t.Errorf("recommender has incorrect cpus: got %d want %d", got, cpus) 155 | } 156 | 157 | if !r.IsAvailable() { 158 | t.Errorf("unexpectedly not available") 159 | } 160 | } 161 | } 162 | 163 | func TestNewPromscaleMemoryRecommender(t *testing.T) { 164 | for i := 0; i < 1000000; i++ { 165 | mem := rand.Uint64() 166 | cpus := rand.Intn(128) 167 | r := NewPromscaleMemoryRecommender(mem, cpus, MaxConnectionsDefault) 168 | if r == nil { 169 | t.Errorf("unexpected nil recommender") 170 | } 171 | if got := r.totalMemory; got != mem { 172 | t.Errorf("recommender has incorrect cpus: got %d want %d", got, cpus) 173 | } 174 | if got := r.cpus; got != cpus { 175 | t.Errorf("recommender has incorrect cpus: got %d want %d", got, cpus) 176 | } 177 | 178 | if !r.IsAvailable() { 179 | t.Errorf("unexpectedly not available") 180 | } 181 | } 182 | } 183 | 184 | func TestMemoryRecommenderRecommendWindows(t *testing.T) { 185 | cases := []struct { 186 | desc string 187 | totalMemory uint64 188 | cpus int 189 | conns uint64 190 | want string 191 | }{ 192 | { 193 | desc: "1GB", 194 | totalMemory: 1 * parse.Gigabyte, 195 | cpus: 1, 196 | conns: baseConns, 197 | want: "6553" + parse.KB, // from pgtune 198 | }, 199 | { 200 | desc: "1GB, 10 conns", 201 | totalMemory: 1 * parse.Gigabyte, 202 | cpus: 1, 203 | conns: 10, 204 | want: "13107" + parse.KB, // from pgtune 205 | }, 206 | { 207 | desc: "1GB, 4 cpus", 208 | totalMemory: 1 * parse.Gigabyte, 209 | cpus: 4, 210 | conns: baseConns, 211 | want: "3276" + parse.KB, // from pgtune 212 | }, 213 | { 214 | desc: "2GB", 215 | totalMemory: 2 * parse.Gigabyte, 216 | cpus: 1, 217 | conns: baseConns, 218 | want: "13107" + parse.KB, // from pgtune 219 | }, 220 | { 221 | desc: "2GB, 5 cpus", 222 | totalMemory: 2 * parse.Gigabyte, 223 | cpus: 5, 224 | conns: baseConns, 225 | want: "4369" + parse.KB, // from pgtune 226 | }, 227 | { 228 | desc: "3GB", 229 | totalMemory: 3 * parse.Gigabyte, 230 | cpus: 1, 231 | conns: baseConns, 232 | want: "21845" + parse.KB, // from pgtune 233 | }, 234 | { 235 | desc: "3GB, 3 cpus", 236 | totalMemory: 3 * parse.Gigabyte, 237 | cpus: 3, 238 | conns: baseConns, 239 | want: "10922" + parse.KB, // from pgtune 240 | }, 241 | { 242 | desc: "8GB", 243 | totalMemory: 8 * parse.Gigabyte, 244 | cpus: 1, 245 | conns: baseConns, 246 | want: "64" + parse.MB, // from pgtune 247 | }, 248 | { 249 | desc: "8GB, 8 cpus", 250 | totalMemory: 8 * parse.Gigabyte, 251 | cpus: 8, 252 | conns: baseConns, 253 | want: "16" + parse.MB, // from pgtune 254 | }, 255 | { 256 | desc: "16GB", 257 | totalMemory: 16 * parse.Gigabyte, 258 | cpus: 1, 259 | conns: baseConns, 260 | want: "135441" + parse.KB, // from pgtune 261 | }, 262 | { 263 | desc: "16GB, 10 cpus", 264 | totalMemory: 16 * parse.Gigabyte, 265 | cpus: 10, 266 | conns: baseConns, 267 | want: "27088" + parse.KB, // from pgtune 268 | }, 269 | { 270 | desc: "1GB, 9000 cpus", 271 | totalMemory: parse.Gigabyte, 272 | cpus: highCPUs, 273 | conns: baseConns, 274 | want: "64" + parse.KB, 275 | }, 276 | } 277 | 278 | for _, c := range cases { 279 | mr := NewMemoryRecommender(c.totalMemory, c.cpus, c.conns) 280 | if got := mr.recommendWindows(); got != c.want { 281 | t.Errorf("%s: incorrect value: got %s want %s", c.desc, got, c.want) 282 | } 283 | } 284 | } 285 | 286 | func TestMemoryRecommenderRecommend(t *testing.T) { 287 | for totalMemory, cpuMatrix := range defaultMemorySettingsMatrix { 288 | for cpus, connMatrix := range cpuMatrix { 289 | for conns, cases := range connMatrix { 290 | mr := NewMemoryRecommender(totalMemory, cpus, conns) 291 | testRecommender(t, mr, MemoryKeys, cases) 292 | } 293 | } 294 | } 295 | } 296 | 297 | func TestPromscaleMemoryRecommenderRecommend(t *testing.T) { 298 | for totalMemory, cpuMatrix := range promscaleMemorySettingsMatrix { 299 | for cpus, connMatrix := range cpuMatrix { 300 | for conns, cases := range connMatrix { 301 | mr := NewPromscaleMemoryRecommender(totalMemory, cpus, conns) 302 | testRecommender(t, mr, MemoryKeys, cases) 303 | } 304 | } 305 | } 306 | } 307 | 308 | func TestMemoryRecommenderNoRecommendation(t *testing.T) { 309 | r := NewMemoryRecommender(1, 1, 1) 310 | if r.Recommend("foo") != NoRecommendation { 311 | t.Error("Recommendation was provided when there should have been none") 312 | } 313 | } 314 | 315 | func TestPromscaleMemoryRecommenderNoRecommendation(t *testing.T) { 316 | r := NewPromscaleMemoryRecommender(1, 1, 1) 317 | if r.Recommend("foo") != NoRecommendation { 318 | t.Error("Recommendation was provided when there should have been none") 319 | } 320 | } 321 | 322 | func TestMemorySettingsGroup(t *testing.T) { 323 | for totalMemory, cpuMatrix := range defaultMemorySettingsMatrix { 324 | for cpus, connMatrix := range cpuMatrix { 325 | for conns, matrix := range connMatrix { 326 | config := getDefaultTestSystemConfig(t) 327 | config.CPUs = cpus 328 | config.Memory = totalMemory 329 | config.maxConns = conns 330 | 331 | sg := GetSettingsGroup(MemoryLabel, config) 332 | testSettingGroup(t, sg, DefaultProfile, matrix, MemoryLabel, MemoryKeys) 333 | } 334 | } 335 | } 336 | } 337 | 338 | func TestPromscaleMemorySettingsGroup(t *testing.T) { 339 | for totalMemory, cpuMatrix := range promscaleMemorySettingsMatrix { 340 | for cpus, connMatrix := range cpuMatrix { 341 | for conns, matrix := range connMatrix { 342 | config := getDefaultTestSystemConfig(t) 343 | config.CPUs = cpus 344 | config.Memory = totalMemory 345 | config.maxConns = conns 346 | 347 | sg := GetSettingsGroup(MemoryLabel, config) 348 | testSettingGroup(t, sg, PromscaleProfile, matrix, MemoryLabel, MemoryKeys) 349 | } 350 | } 351 | } 352 | } 353 | -------------------------------------------------------------------------------- /pkg/pgtune/misc.go: -------------------------------------------------------------------------------- 1 | package pgtune 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "runtime" 7 | 8 | "github.com/timescale/timescaledb-tune/internal/parse" 9 | "github.com/timescale/timescaledb-tune/pkg/pgutils" 10 | ) 11 | 12 | // Keys in the conf file that are tunable but not in the other groupings 13 | const ( 14 | CheckpointKey = "checkpoint_completion_target" 15 | StatsTargetKey = "default_statistics_target" 16 | MaxConnectionsKey = "max_connections" 17 | RandomPageCostKey = "random_page_cost" 18 | MaxLocksPerTxKey = "max_locks_per_transaction" 19 | AutovacuumMaxWorkersKey = "autovacuum_max_workers" 20 | AutovacuumNaptimeKey = "autovacuum_naptime" 21 | EffectiveIOKey = "effective_io_concurrency" // linux only 22 | 23 | // nonnumeric 24 | DefaultToastCompression = "default_toast_compression" 25 | Jit = "jit" 26 | 27 | checkpointDefault = "0.9" 28 | statsTargetDefault = "100" 29 | randomPageCostDefault = "1.1" 30 | autovacuumMaxWorkersDefault = "10" 31 | autovacuumNaptimeDefault = "10" 32 | // effective io concurrency has changed in v13: https://www.postgresql.org/docs/13/release-13.html 33 | // However, our previous value of 200 is translated to 1176, which seems excessively high 34 | // (the upper limit is 1000. For the SSDs we'll follow up the wise man's advice here: 35 | // https://www.postgresql.org/message-id/20210422195232.GA25061%40momjian.us 36 | effectiveIODefaultOldVersions = "200" 37 | effectiveIODefault = "256" 38 | lz4Compression = "lz4" 39 | off = "off" 40 | 41 | // If you want to lower this value, consider that Patroni will not accept anything less than 25 as 42 | // a valid max_connections and will replace it with 100, per 43 | // https://github.com/zalando/patroni/blob/00cc62726d6df25d31f9b0baa082c83cd3f7bef9/patroni/postgresql/config.py#L280 44 | minMaxConns = 25 45 | ) 46 | 47 | // MaxConnectionsDefault is the recommended default value for max_connections. 48 | const MaxConnectionsDefault uint64 = 100 49 | 50 | // MaxBackgroundWorkersDefault is the recommended default value for timescaledb.max_background_workers. 51 | const MaxBackgroundWorkersDefault int = 16 52 | 53 | // getMaxConns gives a default amount of connections based on a memory step 54 | // function. 55 | func getMaxConns(totalMemory uint64) uint64 { 56 | switch { 57 | case totalMemory <= 2*parse.Gigabyte: 58 | return minMaxConns 59 | case totalMemory <= 4*parse.Gigabyte: 60 | return 50 61 | case totalMemory <= 6*parse.Gigabyte: 62 | return 75 63 | default: 64 | return MaxConnectionsDefault 65 | } 66 | } 67 | 68 | func getValueForVersion(currentVersion string, oldVersions []string, oldVersionValue, newVersionValue string) string { 69 | for _, ov := range oldVersions { 70 | if ov == currentVersion { 71 | return oldVersionValue 72 | } 73 | } 74 | return newVersionValue 75 | } 76 | 77 | func getEffectiveIOConcurrency(pgMajorVersion string) string { 78 | switch pgMajorVersion { 79 | case pgutils.MajorVersion96, pgutils.MajorVersion10, pgutils.MajorVersion11, pgutils.MajorVersion12: 80 | return effectiveIODefaultOldVersions 81 | } 82 | return effectiveIODefault 83 | } 84 | 85 | // maxLocksValues gives the number of locks for a power-2 memory starting 86 | // with sub-8GB. i.e.: 87 | // < 8GB = 128 88 | // >=8GB, < 16GB = 256 89 | // >=16GB, < 32GB = 512 90 | // >=32GB = 1024 91 | var maxLocksValues = []string{"128", "256", "512", "1024"} 92 | 93 | // MiscLabel is the label used to refer to the miscellaneous settings group 94 | const MiscLabel = "miscellaneous" 95 | 96 | // MiscKeys is an array of miscellaneous keys that are tunable 97 | var MiscKeys = []string{ 98 | StatsTargetKey, 99 | RandomPageCostKey, 100 | CheckpointKey, 101 | MaxConnectionsKey, 102 | MaxLocksPerTxKey, 103 | AutovacuumMaxWorkersKey, 104 | AutovacuumNaptimeKey, 105 | DefaultToastCompression, 106 | Jit, 107 | 108 | EffectiveIOKey, //linux only 109 | } 110 | 111 | // MiscRecommender gives recommendations for MiscKeys based on system resources. 112 | type MiscRecommender struct { 113 | totalMemory uint64 114 | maxConns uint64 115 | pgMajorVersion string 116 | } 117 | 118 | // NewMiscRecommender returns a MiscRecommender (unaffected by system resources). 119 | func NewMiscRecommender(totalMemory, maxConns uint64, pgMajorVersion string) *MiscRecommender { 120 | return &MiscRecommender{totalMemory, maxConns, pgMajorVersion} 121 | } 122 | 123 | // IsAvailable returns whether this Recommender is usable given the system resources. Always true. 124 | func (r *MiscRecommender) IsAvailable() bool { 125 | return true 126 | } 127 | 128 | // Recommend returns the recommended PostgreSQL formatted value for the conf 129 | // file for a given key. 130 | func (r *MiscRecommender) Recommend(key string) string { 131 | switch key { 132 | case CheckpointKey: 133 | return checkpointDefault 134 | case StatsTargetKey: 135 | return statsTargetDefault 136 | case AutovacuumMaxWorkersKey: 137 | return autovacuumMaxWorkersDefault 138 | case AutovacuumNaptimeKey: 139 | return autovacuumNaptimeDefault 140 | case RandomPageCostKey: 141 | return randomPageCostDefault 142 | case EffectiveIOKey: 143 | return getValueForVersion(r.pgMajorVersion, []string{ 144 | pgutils.MajorVersion96, pgutils.MajorVersion10, pgutils.MajorVersion11, pgutils.MajorVersion12}, 145 | effectiveIODefaultOldVersions, effectiveIODefault, 146 | ) 147 | case DefaultToastCompression: 148 | return getValueForVersion(r.pgMajorVersion, []string{ 149 | pgutils.MajorVersion96, pgutils.MajorVersion10, pgutils.MajorVersion11, pgutils.MajorVersion12, pgutils.MajorVersion13}, 150 | NoRecommendation, lz4Compression, 151 | ) 152 | case Jit: 153 | return getValueForVersion(r.pgMajorVersion, []string{ 154 | pgutils.MajorVersion96, pgutils.MajorVersion10, pgutils.MajorVersion11}, 155 | NoRecommendation, off, 156 | ) 157 | case MaxConnectionsKey: 158 | if r.maxConns != 0 { 159 | return fmt.Sprintf("%d", r.maxConns) 160 | } 161 | return fmt.Sprintf("%d", getMaxConns(r.totalMemory)) 162 | case MaxLocksPerTxKey: 163 | for i := len(maxLocksValues) - 1; i >= 1; i-- { 164 | limit := uint64(math.Pow(2.0, float64(2+i))) 165 | if r.totalMemory >= limit*parse.Gigabyte { 166 | return maxLocksValues[i] 167 | } 168 | } 169 | return maxLocksValues[0] 170 | } 171 | 172 | //unknown key no recommendation 173 | return NoRecommendation 174 | } 175 | 176 | // MiscSettingsGroup is the SettingsGroup to represent settings that do not fit in other SettingsGroups. 177 | type MiscSettingsGroup struct { 178 | totalMemory uint64 179 | maxConns uint64 180 | pgMajorVersion string 181 | } 182 | 183 | // Label should always return the value MiscLabel. 184 | func (sg *MiscSettingsGroup) Label() string { return MiscLabel } 185 | 186 | // Keys should always return the MiscKeys slice. 187 | func (sg *MiscSettingsGroup) Keys() []string { 188 | if runtime.GOOS != "linux" { 189 | return MiscKeys[:len(MiscKeys)-1] 190 | } 191 | return MiscKeys 192 | } 193 | 194 | // GetRecommender should return a new MiscRecommender. 195 | func (sg *MiscSettingsGroup) GetRecommender(profile Profile) Recommender { 196 | return NewMiscRecommender(sg.totalMemory, sg.maxConns, sg.pgMajorVersion) 197 | } 198 | -------------------------------------------------------------------------------- /pkg/pgtune/misc_test.go: -------------------------------------------------------------------------------- 1 | package pgtune 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "testing" 7 | 8 | "github.com/timescale/timescaledb-tune/internal/parse" 9 | "github.com/timescale/timescaledb-tune/pkg/pgutils" 10 | ) 11 | 12 | // memoryToLocks is a mapping of the different memory levels we want tested and 13 | // the corresponding number of locks at that level. 14 | var memoryToLocks = map[uint64]string{ 15 | 7 * parse.Gigabyte: maxLocksValues[0], 16 | 8 * parse.Gigabyte: maxLocksValues[1], 17 | 15 * parse.Gigabyte: maxLocksValues[1], 18 | 16 * parse.Gigabyte: maxLocksValues[2], 19 | 24 * parse.Gigabyte: maxLocksValues[2], 20 | 32 * parse.Gigabyte: maxLocksValues[3], 21 | 80 * parse.Gigabyte: maxLocksValues[3], 22 | } 23 | 24 | // connsToMaxConns is a mapping of the user given connection values we want 25 | // tested and the corresponding number of actual max connections assigned. 26 | var connsToMaxConns = map[uint64]uint64{ 27 | MaxConnectionsDefault - 10: MaxConnectionsDefault - 10, 28 | MaxConnectionsDefault: MaxConnectionsDefault, 29 | MaxConnectionsDefault + 10: MaxConnectionsDefault + 10, 30 | } 31 | 32 | // miscSettingsMatrix is a matrix that holds the test cases and desired key/value 33 | // pairs. The first key is the memory level (uint64), the second is the user 34 | // given connections (uint64), and the innermost map is the key-value pairs 35 | // we expect 36 | var miscSettingsMatrix = map[uint64]map[uint64]map[string]string{} 37 | 38 | func init() { 39 | // Initialize the miscSettingsMatrix by creating a key-value map for every 40 | // memory level for every connections given 41 | for mem, maxLocks := range memoryToLocks { 42 | miscSettingsMatrix[mem] = make(map[uint64]map[string]string) 43 | for conns, maxConns := range connsToMaxConns { 44 | miscSettingsMatrix[mem][conns] = make(map[string]string) 45 | miscSettingsMatrix[mem][conns][MaxLocksPerTxKey] = maxLocks 46 | miscSettingsMatrix[mem][conns][MaxConnectionsKey] = fmt.Sprintf("%d", maxConns) 47 | 48 | miscSettingsMatrix[mem][conns][CheckpointKey] = checkpointDefault 49 | miscSettingsMatrix[mem][conns][StatsTargetKey] = statsTargetDefault 50 | miscSettingsMatrix[mem][conns][RandomPageCostKey] = randomPageCostDefault 51 | miscSettingsMatrix[mem][conns][EffectiveIOKey] = effectiveIODefaultOldVersions 52 | miscSettingsMatrix[mem][conns][AutovacuumMaxWorkersKey] = autovacuumMaxWorkersDefault 53 | miscSettingsMatrix[mem][conns][AutovacuumNaptimeKey] = autovacuumNaptimeDefault 54 | } 55 | } 56 | } 57 | 58 | func TestGetMaxConns(t *testing.T) { 59 | cases := []struct { 60 | desc string 61 | mem uint64 62 | want uint64 63 | }{ 64 | { 65 | desc: "really small instance (1GB)", 66 | mem: 1 * parse.Gigabyte, 67 | want: minMaxConns, 68 | }, 69 | { 70 | desc: "small instance boundary (2GB)", 71 | mem: 2 * parse.Gigabyte, 72 | want: minMaxConns, 73 | }, 74 | { 75 | desc: "medium instance (3GB)", 76 | mem: 3 * parse.Gigabyte, 77 | want: 50, 78 | }, 79 | { 80 | desc: "medium instance boundary (4GB)", 81 | mem: 4 * parse.Gigabyte, 82 | want: 50, 83 | }, 84 | { 85 | desc: "big instance", 86 | mem: 5 * parse.Gigabyte, 87 | want: 75, 88 | }, 89 | { 90 | desc: "big instance boundary (6GB)", 91 | mem: 6 * parse.Gigabyte, 92 | want: 75, 93 | }, 94 | { 95 | desc: "large instance", 96 | mem: 7 * parse.Gigabyte, 97 | want: MaxConnectionsDefault, 98 | }, 99 | } 100 | 101 | for _, c := range cases { 102 | t.Run(c.desc, func(t *testing.T) { 103 | if got := getMaxConns(c.mem); got != c.want { 104 | t.Errorf("incorrect conns: got %d want %d", got, c.want) 105 | } 106 | }) 107 | } 108 | } 109 | 110 | func TestGetEffectiveIOConcurrency(t *testing.T) { 111 | cases := []struct { 112 | pgMajorVersion string 113 | want string 114 | }{ 115 | { 116 | pgutils.MajorVersion96, 117 | effectiveIODefaultOldVersions, 118 | }, 119 | { 120 | pgutils.MajorVersion10, 121 | effectiveIODefaultOldVersions, 122 | }, 123 | { 124 | pgutils.MajorVersion11, 125 | effectiveIODefaultOldVersions, 126 | }, 127 | { 128 | pgutils.MajorVersion12, 129 | effectiveIODefaultOldVersions, 130 | }, 131 | { 132 | pgutils.MajorVersion13, 133 | effectiveIODefault, 134 | }, 135 | { 136 | /* a new version, not yet released */ 137 | "15", 138 | effectiveIODefault, 139 | }, 140 | } 141 | for _, c := range cases { 142 | t.Run(fmt.Sprintf("test effective_io_concurrency (v%s)", c.pgMajorVersion), func(t *testing.T) { 143 | if got := getEffectiveIOConcurrency(c.pgMajorVersion); got != c.want { 144 | t.Errorf("incorrect effective_io_concurrency: got %s, want %s", got, c.want) 145 | } 146 | }) 147 | } 148 | } 149 | 150 | func TestDefaultToastCompression(t *testing.T) { 151 | cases := []struct { 152 | pgMajorVersions []string 153 | want string 154 | }{ 155 | { 156 | []string{pgutils.MajorVersion96, pgutils.MajorVersion10, pgutils.MajorVersion11, pgutils.MajorVersion12, pgutils.MajorVersion13}, 157 | NoRecommendation, 158 | }, 159 | { 160 | []string{pgutils.MajorVersion14, pgutils.MajorVersion15, pgutils.MajorVersion16, pgutils.MajorVersion17, 161 | "18", //future versions 162 | }, 163 | "lz4", 164 | }, 165 | } 166 | for _, c := range cases { 167 | for _, v := range c.pgMajorVersions { 168 | t.Run("default_toast_compression:"+v, func(t *testing.T) { 169 | r := NewMiscRecommender(1000, 32, v) 170 | 171 | rec := r.Recommend(DefaultToastCompression) 172 | if rec != c.want { 173 | t.Errorf("wanted %s got: %s", c.want, rec) 174 | } 175 | 176 | }) 177 | } 178 | } 179 | } 180 | 181 | func TestJIT(t *testing.T) { 182 | cases := []struct { 183 | pgMajorVersions []string 184 | want string 185 | }{ 186 | { 187 | []string{pgutils.MajorVersion96, pgutils.MajorVersion10, pgutils.MajorVersion11}, 188 | NoRecommendation, 189 | }, 190 | { 191 | []string{pgutils.MajorVersion12, pgutils.MajorVersion13, pgutils.MajorVersion14, pgutils.MajorVersion15, pgutils.MajorVersion16, pgutils.MajorVersion17, 192 | "18", //future versions 193 | }, 194 | "off", 195 | }, 196 | } 197 | for _, c := range cases { 198 | for _, v := range c.pgMajorVersions { 199 | t.Run("default_toast_compression:"+v, func(t *testing.T) { 200 | r := NewMiscRecommender(1000, 32, v) 201 | 202 | rec := r.Recommend(Jit) 203 | if rec != c.want { 204 | t.Errorf("wanted %s got: %s", c.want, rec) 205 | } 206 | 207 | }) 208 | } 209 | } 210 | } 211 | 212 | func TestNewMiscRecommender(t *testing.T) { 213 | for i := 0; i < 1000000; i++ { 214 | mem := rand.Uint64() 215 | conns := rand.Uint64() 216 | r := NewMiscRecommender(mem, conns, pgutils.MajorVersion12) 217 | if r == nil { 218 | t.Errorf("unexpected nil recommender") 219 | continue 220 | } 221 | 222 | if got := r.totalMemory; got != mem { 223 | t.Errorf("recommender has incorrect memory: got %d want %d", got, mem) 224 | } 225 | if got := r.maxConns; got != conns { 226 | t.Errorf("recommender has incorrect conns: got %d want %d", got, conns) 227 | } 228 | 229 | if !r.IsAvailable() { 230 | t.Errorf("unexpectedly not available") 231 | } 232 | } 233 | } 234 | 235 | func TestMiscRecommenderRecommend(t *testing.T) { 236 | for totalMemory, outerMatrix := range miscSettingsMatrix { 237 | for maxConns, matrix := range outerMatrix { 238 | r := &MiscRecommender{totalMemory, maxConns, pgutils.MajorVersion10} 239 | testRecommender(t, r, MiscKeys, matrix) 240 | } 241 | } 242 | } 243 | 244 | func TestMiscRecommenderNoRecommendation(t *testing.T) { 245 | r := &MiscRecommender{} 246 | if r.Recommend("foo") != NoRecommendation { 247 | t.Error("Recommendation was provided when there should have been none") 248 | } 249 | } 250 | 251 | func TestMiscSettingsGroup(t *testing.T) { 252 | for totalMemory, outerMatrix := range miscSettingsMatrix { 253 | for maxConns, matrix := range outerMatrix { 254 | config, err := NewSystemConfig(totalMemory, 8, "10", walDiskUnset, maxConns, MaxBackgroundWorkersDefault) 255 | if err != nil { 256 | t.Errorf("unexpected error on system config creation: got %v", err) 257 | } 258 | sg := GetSettingsGroup(MiscLabel, config) 259 | 260 | testSettingGroup(t, sg, DefaultProfile, matrix, MiscLabel, MiscKeys) 261 | } 262 | } 263 | } 264 | -------------------------------------------------------------------------------- /pkg/pgtune/null_recommender.go: -------------------------------------------------------------------------------- 1 | package pgtune 2 | 3 | // NullRecommender is a Recommender that returns NoRecommendation for all keys 4 | type NullRecommender struct { 5 | } 6 | 7 | // IsAvailable returns whether this Recommender is usable given the system resources. Always true. 8 | func (r *NullRecommender) IsAvailable() bool { 9 | return true 10 | } 11 | 12 | // Recommend returns the recommended PostgreSQL formatted value for the conf 13 | // file for a given key. 14 | func (r *NullRecommender) Recommend(key string) string { 15 | return NoRecommendation 16 | } 17 | -------------------------------------------------------------------------------- /pkg/pgtune/null_recommender_test.go: -------------------------------------------------------------------------------- 1 | package pgtune 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestNullRecommender_Recommend(t *testing.T) { 9 | r := &NullRecommender{} 10 | // NullRecommender should ALWAYS return NoRecommendation 11 | for i := 0; i < 1000; i++ { 12 | key := fmt.Sprintf("key%d", i) 13 | if val := r.Recommend(key); val != NoRecommendation { 14 | t.Errorf("Expected no recommendation for key %s but got %s", key, val) 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /pkg/pgtune/parallel.go: -------------------------------------------------------------------------------- 1 | package pgtune 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | ) 7 | 8 | // Keys in the conf file that are tuned related to parallelism 9 | const ( 10 | MaxBackgroundWorkers = "timescaledb.max_background_workers" 11 | MaxWorkerProcessesKey = "max_worker_processes" 12 | MaxParallelWorkersGatherKey = "max_parallel_workers_per_gather" 13 | MaxParallelWorkers = "max_parallel_workers" // pg10+ 14 | 15 | minBuiltInProcesses = 3 // at least checkpointer, WALwriter, vacuum 16 | 17 | errOneCPU = "cannot make recommendations with just 1 CPU" 18 | errWorkers = "cannot make recommendations with less than %d workers" 19 | ) 20 | 21 | // ParallelLabel is the label used to refer to the parallelism settings group 22 | const ParallelLabel = "parallelism" 23 | 24 | // ParallelKeys is an array of keys that are tunable for parallelism 25 | var ParallelKeys = []string{ 26 | MaxBackgroundWorkers, 27 | MaxWorkerProcessesKey, 28 | MaxParallelWorkersGatherKey, 29 | MaxParallelWorkers, 30 | } 31 | 32 | // ParallelRecommender gives recommendations for ParallelKeys based on system resources. 33 | type ParallelRecommender struct { 34 | cpus int 35 | maxBGWorkers int 36 | } 37 | 38 | // NewParallelRecommender returns a ParallelRecommender that recommends based on 39 | // the given number of cpus. 40 | func NewParallelRecommender(cpus, maxBGWorkers int) *ParallelRecommender { 41 | return &ParallelRecommender{cpus, maxBGWorkers} 42 | } 43 | 44 | // IsAvailable returns whether this Recommender is usable given the system 45 | // resources. True when number of CPUS > 1. 46 | func (r *ParallelRecommender) IsAvailable() bool { 47 | return r.cpus > 1 48 | } 49 | 50 | // Recommend returns the recommended PostgreSQL formatted value for the conf 51 | // file for a given key. 52 | func (r *ParallelRecommender) Recommend(key string) string { 53 | var val string 54 | if r.cpus <= 1 { 55 | panic(errOneCPU) 56 | } 57 | if r.maxBGWorkers < MaxBackgroundWorkersDefault { 58 | panic(fmt.Sprintf(errWorkers, MaxBackgroundWorkersDefault)) 59 | } 60 | if key == MaxWorkerProcessesKey { 61 | // Need enough processes to handle built-ins (e.g., autovacuum), 62 | // TimescaleDB background workers, and the number of parallel workers 63 | // (equal to the number of CPUs). 64 | val = fmt.Sprintf("%d", minBuiltInProcesses+r.maxBGWorkers+r.cpus) 65 | } else if key == MaxParallelWorkers { 66 | val = fmt.Sprintf("%d", r.cpus) 67 | } else if key == MaxParallelWorkersGatherKey { 68 | val = fmt.Sprintf("%d", int(math.Round(float64(r.cpus)/2.0))) 69 | } else if key == MaxBackgroundWorkers { 70 | val = fmt.Sprintf("%d", r.maxBGWorkers) 71 | } else { 72 | val = NoRecommendation 73 | } 74 | return val 75 | } 76 | 77 | // ParallelSettingsGroup is the SettingsGroup to represent parallelism settings. 78 | type ParallelSettingsGroup struct { 79 | pgVersion string 80 | cpus int 81 | maxBGWorkers int 82 | } 83 | 84 | // Label should always return the value ParallelLabel. 85 | func (sg *ParallelSettingsGroup) Label() string { return ParallelLabel } 86 | 87 | // Keys should always return the ParallelKeys slice. 88 | func (sg *ParallelSettingsGroup) Keys() []string { 89 | if sg.pgVersion == "9.6" { 90 | return ParallelKeys[:len(ParallelKeys)-1] 91 | } 92 | return ParallelKeys 93 | } 94 | 95 | // GetRecommender should return a new ParallelRecommender. 96 | func (sg *ParallelSettingsGroup) GetRecommender(profile Profile) Recommender { 97 | return NewParallelRecommender(sg.cpus, sg.maxBGWorkers) 98 | } 99 | -------------------------------------------------------------------------------- /pkg/pgtune/parallel_test.go: -------------------------------------------------------------------------------- 1 | package pgtune 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "testing" 7 | "time" 8 | 9 | "github.com/timescale/timescaledb-tune/pkg/pgutils" 10 | ) 11 | 12 | // parallelSettingsMatrix stores the test cases for ParallelRecommender along 13 | // with the expected values for its keys 14 | var parallelSettingsMatrix = map[int]map[int]map[string]string{ 15 | 2: { 16 | MaxBackgroundWorkersDefault: { 17 | MaxBackgroundWorkers: fmt.Sprintf("%d", MaxBackgroundWorkersDefault), 18 | MaxWorkerProcessesKey: fmt.Sprintf("%d", 2+minBuiltInProcesses+MaxBackgroundWorkersDefault), 19 | MaxParallelWorkersGatherKey: "1", 20 | MaxParallelWorkers: "2", 21 | }, 22 | MaxBackgroundWorkersDefault * 2: { 23 | MaxBackgroundWorkers: fmt.Sprintf("%d", MaxBackgroundWorkersDefault*2), 24 | MaxWorkerProcessesKey: fmt.Sprintf("%d", 2+minBuiltInProcesses+MaxBackgroundWorkersDefault*2), 25 | MaxParallelWorkersGatherKey: "1", 26 | MaxParallelWorkers: "2", 27 | }, 28 | }, 29 | 4: { 30 | MaxBackgroundWorkersDefault: { 31 | MaxBackgroundWorkers: fmt.Sprintf("%d", MaxBackgroundWorkersDefault), 32 | MaxWorkerProcessesKey: fmt.Sprintf("%d", 4+minBuiltInProcesses+MaxBackgroundWorkersDefault), 33 | MaxParallelWorkersGatherKey: "2", 34 | MaxParallelWorkers: "4", 35 | }, 36 | MaxBackgroundWorkersDefault * 4: { 37 | MaxBackgroundWorkers: fmt.Sprintf("%d", MaxBackgroundWorkersDefault*4), 38 | MaxWorkerProcessesKey: fmt.Sprintf("%d", 4+minBuiltInProcesses+MaxBackgroundWorkersDefault*4), 39 | MaxParallelWorkersGatherKey: "2", 40 | MaxParallelWorkers: "4", 41 | }, 42 | }, 43 | 5: { 44 | MaxBackgroundWorkersDefault: { 45 | MaxBackgroundWorkers: fmt.Sprintf("%d", MaxBackgroundWorkersDefault), 46 | MaxWorkerProcessesKey: fmt.Sprintf("%d", 5+minBuiltInProcesses+MaxBackgroundWorkersDefault), 47 | MaxParallelWorkersGatherKey: "3", 48 | MaxParallelWorkers: "5", 49 | }, 50 | MaxBackgroundWorkersDefault * 5: { 51 | MaxBackgroundWorkers: fmt.Sprintf("%d", MaxBackgroundWorkersDefault*5), 52 | MaxWorkerProcessesKey: fmt.Sprintf("%d", 5+minBuiltInProcesses+MaxBackgroundWorkersDefault*5), 53 | MaxParallelWorkersGatherKey: "3", 54 | MaxParallelWorkers: "5", 55 | }, 56 | }, 57 | } 58 | 59 | func TestNewParallelRecommender(t *testing.T) { 60 | rand.Seed(time.Now().UnixNano()) 61 | for i := 0; i < 1000000; i++ { 62 | cpus := rand.Intn(128) 63 | // ensure a minimum of background workers 64 | workers := rand.Intn(128-MaxBackgroundWorkersDefault+1) + MaxBackgroundWorkersDefault 65 | r := NewParallelRecommender(cpus, workers) 66 | if r == nil { 67 | t.Errorf("unexpected nil recommender") 68 | } 69 | if got := r.cpus; got != cpus { 70 | t.Errorf("recommender has incorrect cpus: got %d want %d", got, cpus) 71 | } 72 | if got := r.maxBGWorkers; got != workers { 73 | t.Errorf("recommender has incorrect workers: got %d want %d", got, workers) 74 | } 75 | } 76 | } 77 | 78 | func TestParallelRecommenderIsAvailable(t *testing.T) { 79 | if r := NewParallelRecommender(0, MaxBackgroundWorkersDefault); r.IsAvailable() { 80 | t.Errorf("unexpectedly available for 0 cpus") 81 | } 82 | if r := NewParallelRecommender(1, MaxBackgroundWorkersDefault); r.IsAvailable() { 83 | t.Errorf("unexpectedly available for 1 cpus") 84 | } 85 | 86 | for i := 2; i < 1000; i++ { 87 | if r := NewParallelRecommender(i, MaxBackgroundWorkersDefault); !r.IsAvailable() { 88 | t.Errorf("unexpected UNavailable for %d cpus", i) 89 | } 90 | } 91 | } 92 | 93 | func TestParallelRecommenderRecommend(t *testing.T) { 94 | for cpus, tempMatrix := range parallelSettingsMatrix { 95 | for workers, matrix := range tempMatrix { 96 | r := &ParallelRecommender{cpus, workers} 97 | testRecommender(t, r, ParallelKeys, matrix) 98 | } 99 | } 100 | } 101 | 102 | func TestParallelRecommenderNoRecommendation(t *testing.T) { 103 | r := &ParallelRecommender{5, MaxBackgroundWorkersDefault} 104 | if r.Recommend("foo") != NoRecommendation { 105 | t.Error("Recommendation was provided when there should have been none") 106 | } 107 | } 108 | 109 | func TestParallelRecommenderRecommendPanics(t *testing.T) { 110 | // test invalid CPU panic 111 | func() { 112 | defer func() { 113 | if re := recover(); re == nil { 114 | t.Errorf("did not panic when should") 115 | } 116 | }() 117 | r := &ParallelRecommender{1, MaxBackgroundWorkersDefault} 118 | r.Recommend("foo") 119 | }() 120 | 121 | // test invalid worker panic 122 | func() { 123 | defer func() { 124 | if re := recover(); re == nil { 125 | t.Errorf("did not panic when should") 126 | } 127 | }() 128 | r := &ParallelRecommender{5, MaxBackgroundWorkersDefault - 1} 129 | r.Recommend("foo") 130 | }() 131 | } 132 | 133 | func TestParallelSettingsGroup(t *testing.T) { 134 | keyCount := len(ParallelKeys) 135 | for cpus, tempMatrix := range parallelSettingsMatrix { 136 | for workers, matrix := range tempMatrix { 137 | config := getDefaultTestSystemConfig(t) 138 | config.CPUs = cpus 139 | config.PGMajorVersion = pgutils.MajorVersion96 // 9.6 lacks one key 140 | config.MaxBGWorkers = workers 141 | sg := GetSettingsGroup(ParallelLabel, config) 142 | if got := len(sg.Keys()); got != keyCount-1 { 143 | t.Errorf("incorrect number of keys for PG %s: got %d want %d", pgutils.MajorVersion96, got, keyCount-1) 144 | } 145 | testSettingGroup(t, sg, DefaultProfile, matrix, ParallelLabel, ParallelKeys) 146 | 147 | // PG10 adds a key 148 | config.PGMajorVersion = pgutils.MajorVersion10 149 | sg = GetSettingsGroup(ParallelLabel, config) 150 | if got := len(sg.Keys()); got != keyCount { 151 | t.Errorf("incorrect number of keys for PG %s: got %d want %d", pgutils.MajorVersion10, got, keyCount) 152 | } 153 | testSettingGroup(t, sg, DefaultProfile, matrix, ParallelLabel, ParallelKeys) 154 | 155 | config.PGMajorVersion = pgutils.MajorVersion11 156 | sg = GetSettingsGroup(ParallelLabel, config) 157 | if got := len(sg.Keys()); got != keyCount { 158 | t.Errorf("incorrect number of keys for PG %s: got %d want %d", pgutils.MajorVersion11, got, keyCount) 159 | } 160 | testSettingGroup(t, sg, DefaultProfile, matrix, ParallelLabel, ParallelKeys) 161 | } 162 | } 163 | 164 | } 165 | -------------------------------------------------------------------------------- /pkg/pgtune/tune.go: -------------------------------------------------------------------------------- 1 | // Package pgtune provides the resources and interfaces for getting tuning 2 | // recommendations based on heuristics and knowledge from the online pgtune tool 3 | // for groups of settings in a PostgreSQL conf file. 4 | package pgtune 5 | 6 | import ( 7 | "fmt" 8 | "strings" 9 | ) 10 | 11 | const ( 12 | osWindows = "windows" 13 | errMaxConnsTooLowFmt = "maxConns must be 0 OR >= %d: got %d" 14 | errMaxBGWorkersTooLowFmt = "maxBGWorkers must be >= %d: got %d" 15 | errUnrecognizedProfile = "unrecognized profile: %s" 16 | ) 17 | 18 | // Profile is a specific "mode" in which timescaledb-tune can be run to provide recommendations tailored to a 19 | // special workload type, e.g. "promscale" 20 | type Profile int64 21 | 22 | const ( 23 | DefaultProfile Profile = iota 24 | PromscaleProfile 25 | ) 26 | 27 | func ParseProfile(s string) (Profile, error) { 28 | switch strings.ToLower(s) { 29 | case "": 30 | return DefaultProfile, nil 31 | case "promscale": 32 | return PromscaleProfile, nil 33 | default: 34 | return DefaultProfile, fmt.Errorf(errUnrecognizedProfile, s) 35 | } 36 | } 37 | 38 | func (p Profile) String() string { 39 | switch p { 40 | case DefaultProfile: 41 | return "" 42 | case PromscaleProfile: 43 | return "promscale" 44 | default: 45 | return "unrecognized" 46 | } 47 | } 48 | 49 | const NoRecommendation = "" 50 | 51 | // Recommender is an interface that gives setting recommendations for a given 52 | // key, usually grouped by logical settings groups (e.g. MemoryRecommender for memory settings). 53 | type Recommender interface { 54 | // IsAvailable returns whether this Recommender is usable given the system resources. 55 | IsAvailable() bool 56 | // Recommend returns the recommended PostgreSQL formatted value for the conf file for a given key. 57 | Recommend(string) string 58 | } 59 | 60 | // SettingsGroup is an interface that defines a group of related settings that share 61 | // a Recommender and can be processed together. 62 | type SettingsGroup interface { 63 | // Label is the canonical name for the group and should be grammatically correct when 64 | // followed by 'settings', e.g., "memory" -> "memory settings", "parallelism" -> "parallelism settings", etc. 65 | Label() string 66 | // Keys are the parameter names/keys as they appear in the PostgreSQL conf file, e.g. "shared_buffers". 67 | Keys() []string 68 | // GetRecommender returns the Recommender that should be used for this group of settings. 69 | GetRecommender(Profile) Recommender 70 | } 71 | 72 | // SystemConfig represents a system's resource configuration, to be used when generating 73 | // recommendations for different SettingsGroups. 74 | type SystemConfig struct { 75 | Memory uint64 76 | CPUs int 77 | PGMajorVersion string 78 | WALDiskSize uint64 79 | maxConns uint64 80 | MaxBGWorkers int 81 | } 82 | 83 | // NewSystemConfig returns a new SystemConfig with the given parameters. 84 | func NewSystemConfig(totalMemory uint64, cpus int, pgVersion string, walDiskSize uint64, maxConns uint64, maxBGWorkers int) (*SystemConfig, error) { 85 | if maxConns != 0 && maxConns < minMaxConns { 86 | return nil, fmt.Errorf(errMaxConnsTooLowFmt, minMaxConns, maxConns) 87 | } 88 | if maxBGWorkers < MaxBackgroundWorkersDefault { 89 | return nil, fmt.Errorf(errMaxBGWorkersTooLowFmt, MaxBackgroundWorkersDefault, maxBGWorkers) 90 | } 91 | return &SystemConfig{ 92 | Memory: totalMemory, 93 | CPUs: cpus, 94 | PGMajorVersion: pgVersion, 95 | WALDiskSize: walDiskSize, 96 | maxConns: maxConns, 97 | MaxBGWorkers: maxBGWorkers, 98 | }, nil 99 | } 100 | 101 | // GetSettingsGroup returns the corresponding SettingsGroup for a given label, initialized 102 | // according to the system resources of totalMemory and cpus. Panics if unknown label. 103 | func GetSettingsGroup(label string, config *SystemConfig) SettingsGroup { 104 | switch { 105 | case label == MemoryLabel: 106 | return &MemorySettingsGroup{config.Memory, config.CPUs, config.maxConns} 107 | case label == ParallelLabel: 108 | return &ParallelSettingsGroup{config.PGMajorVersion, config.CPUs, config.MaxBGWorkers} 109 | case label == WALLabel: 110 | return &WALSettingsGroup{config.Memory, config.WALDiskSize} 111 | case label == BgwriterLabel: 112 | return &BgwriterSettingsGroup{} 113 | case label == MiscLabel: 114 | return &MiscSettingsGroup{config.Memory, config.maxConns, config.PGMajorVersion} 115 | } 116 | panic("unknown label: " + label) 117 | } 118 | -------------------------------------------------------------------------------- /pkg/pgtune/tune_test.go: -------------------------------------------------------------------------------- 1 | package pgtune 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "strings" 7 | "testing" 8 | ) 9 | 10 | const ( 11 | testMaxConnsSpecial = 0 12 | testMaxConnsBad = 1 13 | testMaxConns = minMaxConns 14 | ) 15 | 16 | func getDefaultTestSystemConfig(t *testing.T) *SystemConfig { 17 | config, err := NewSystemConfig(1024, 4, "10", walDiskUnset, testMaxConns, MaxBackgroundWorkersDefault) 18 | if err != nil { 19 | t.Errorf("unexpected error: got %v", err) 20 | } 21 | return config 22 | } 23 | 24 | func TestNewSystemConfig(t *testing.T) { 25 | for i := 0; i < 1000; i++ { 26 | mem := rand.Uint64() 27 | cpus := rand.Intn(32) 28 | pgVersion := "10" 29 | if i%2 == 0 { 30 | pgVersion = "9.6" 31 | } 32 | 33 | config, err := NewSystemConfig(mem, cpus, pgVersion, walDiskUnset, testMaxConns, MaxBackgroundWorkersDefault) 34 | if err != nil { 35 | t.Errorf("unexpected error: got %v", err) 36 | } 37 | if config.Memory != mem { 38 | t.Errorf("incorrect memory: got %d want %d", config.Memory, mem) 39 | } 40 | if config.CPUs != cpus { 41 | t.Errorf("incorrect cpus: got %d want %d", config.CPUs, cpus) 42 | } 43 | if config.PGMajorVersion != pgVersion { 44 | t.Errorf("incorrect pg version: got %s want %s", config.PGMajorVersion, pgVersion) 45 | } 46 | if config.maxConns != testMaxConns { 47 | t.Errorf("incorrect max conns: got %d want %d", config.maxConns, testMaxConns) 48 | } 49 | if config.MaxBGWorkers != MaxBackgroundWorkersDefault { 50 | t.Errorf("incorrect max background workers: got %d want %d", config.MaxBGWorkers, MaxBackgroundWorkersDefault) 51 | } 52 | 53 | // test invalid number of connections 54 | _, err = NewSystemConfig(mem, cpus, pgVersion, walDiskUnset, testMaxConnsBad, MaxBackgroundWorkersDefault) 55 | wantErr := fmt.Sprintf(errMaxConnsTooLowFmt, minMaxConns, testMaxConnsBad) 56 | if err == nil { 57 | t.Errorf("unexpected lack of error") 58 | } else if got := err.Error(); got != wantErr { 59 | t.Errorf("unexpected error: got\n%s\nwant\n%s", got, wantErr) 60 | } 61 | 62 | // test 0 connections 63 | config, err = NewSystemConfig(mem, cpus, pgVersion, walDiskUnset, testMaxConnsSpecial, MaxBackgroundWorkersDefault) 64 | if err != nil { 65 | t.Errorf("unexpected error: got %v", err) 66 | } 67 | if config.maxConns != testMaxConnsSpecial { 68 | t.Errorf("incorrect max conns: got %d want %d", config.maxConns, testMaxConnsSpecial) 69 | } 70 | 71 | // test invalid number of background workers 72 | _, err = NewSystemConfig(mem, cpus, pgVersion, walDiskUnset, testMaxConns, MaxBackgroundWorkersDefault-1) 73 | wantErr = fmt.Sprintf(errMaxBGWorkersTooLowFmt, MaxBackgroundWorkersDefault, MaxBackgroundWorkersDefault-1) 74 | if err == nil { 75 | t.Errorf("unexpected lack of error") 76 | } else if got := err.Error(); got != wantErr { 77 | t.Errorf("unexpected error: got\n%s\nwant\n%s", got, wantErr) 78 | } 79 | 80 | } 81 | } 82 | 83 | func TestGetSettingsGroup(t *testing.T) { 84 | okLabels := []string{MemoryLabel, ParallelLabel, WALLabel, BgwriterLabel, MiscLabel} 85 | config := getDefaultTestSystemConfig(t) 86 | for _, label := range okLabels { 87 | sg := GetSettingsGroup(label, config) 88 | if sg == nil { 89 | t.Errorf("settings group unexpectedly nil for label %s", label) 90 | } 91 | switch x := sg.(type) { 92 | case *MemorySettingsGroup: 93 | if x.totalMemory != config.Memory || x.cpus != config.CPUs { 94 | t.Errorf("memory group incorrect (memory): got %d want %d", x.totalMemory, config.Memory) 95 | } 96 | if x.cpus != config.CPUs { 97 | t.Errorf("memory group incorrect (CPUs): got %d want %d", x.cpus, config.CPUs) 98 | } 99 | case *ParallelSettingsGroup: 100 | if x.cpus != config.CPUs { 101 | t.Errorf("parallel group incorrect (CPUs): got %d want %d", x.cpus, config.CPUs) 102 | } 103 | if x.pgVersion != config.PGMajorVersion { 104 | t.Errorf("parallel group incorrect (PG version): got %s want %s", x.pgVersion, config.PGMajorVersion) 105 | } 106 | case *WALSettingsGroup: 107 | if x.totalMemory != config.Memory { 108 | t.Errorf("WAL group incorrect (memory): got %d want %d", x.totalMemory, config.Memory) 109 | } 110 | if x.walDiskSize != config.WALDiskSize { 111 | t.Errorf("WAL group incorrect (wal disk): got %d want %d", x.walDiskSize, config.WALDiskSize) 112 | } 113 | case *BgwriterSettingsGroup: 114 | // nothing to check here 115 | continue 116 | case *MiscSettingsGroup: 117 | if x.totalMemory != config.Memory { 118 | t.Errorf("Misc group incorrect (memory): got %d want %d", x.totalMemory, config.Memory) 119 | } 120 | if x.maxConns != config.maxConns { 121 | t.Errorf("Misc group incorrect (max conns): got %d want %d", x.maxConns, config.maxConns) 122 | } 123 | default: 124 | t.Errorf("unexpected type for settings group %T", x) 125 | } 126 | } 127 | 128 | // this should panic on unknown label 129 | func() { 130 | defer func() { 131 | if re := recover(); re == nil { 132 | t.Errorf("did not panic when should") 133 | } 134 | }() 135 | GetSettingsGroup("foo", config) 136 | }() 137 | } 138 | 139 | func testSettingGroup(t *testing.T, sg SettingsGroup, profile Profile, cases map[string]string, wantLabel string, wantKeys []string) { 140 | t.Helper() 141 | 142 | // No matter how many calls, all calls should return the same 143 | for i := 0; i < 1000; i++ { 144 | if got := sg.Label(); got != wantLabel { 145 | t.Errorf("incorrect label: got %s want %s", got, wantLabel) 146 | } 147 | if got := sg.Keys(); got == nil { 148 | t.Errorf("keys is nil") 149 | } 150 | for i, k := range sg.Keys() { 151 | if k != wantKeys[i] { 152 | t.Errorf("incorrect key at %d: got %s want %s", i, k, wantKeys[i]) 153 | } 154 | } 155 | 156 | r := sg.GetRecommender(profile) 157 | 158 | testRecommender(t, r, sg.Keys(), cases) 159 | } 160 | } 161 | 162 | // testRecommender is a helper method for testing whether a Recommender gives 163 | // the appropriate values for a set of keys. 164 | // 165 | // Rather than iterating over the 'wants' map to get the keys, we iterate over 166 | // a separate 'keys' parameter that should include _all_ keys a Recommender 167 | // handles. This makes sure that when new keys are added, our tests are comprehensive, 168 | // since otherwise the Recommender will panic on an unknown key. 169 | func testRecommender(t *testing.T, r Recommender, keys []string, wants map[string]string) { 170 | t.Helper() 171 | 172 | for _, key := range keys { 173 | want := wants[key] 174 | if got := r.Recommend(key); got != want { 175 | t.Errorf("%T: incorrect result for key %s: got\n%s\nwant\n%s", r, key, got, want) 176 | } 177 | } 178 | } 179 | 180 | func TestParseProfile(t *testing.T) { 181 | cases := []struct { 182 | input string 183 | expected Profile 184 | }{ 185 | {input: DefaultProfile.String(), expected: DefaultProfile}, 186 | {input: PromscaleProfile.String(), expected: PromscaleProfile}, 187 | {input: strings.ToUpper(PromscaleProfile.String()), expected: PromscaleProfile}, 188 | } 189 | for _, kase := range cases { 190 | actual, err := ParseProfile(kase.input) 191 | if err != nil { 192 | t.Errorf("expected %v for input %s but got an error: %v", kase.expected, kase.input, err) 193 | } 194 | if actual != kase.expected { 195 | t.Errorf("expected %v for input %s but got %v", kase.expected, kase.input, actual) 196 | } 197 | } 198 | 199 | if actual, err := ParseProfile("garbage"); err == nil { 200 | t.Errorf("expected to get an error for unrecognized input, but did not. got %v", actual) 201 | } 202 | } 203 | -------------------------------------------------------------------------------- /pkg/pgtune/wal.go: -------------------------------------------------------------------------------- 1 | package pgtune 2 | 3 | import ( 4 | "github.com/timescale/timescaledb-tune/internal/parse" 5 | ) 6 | 7 | // Keys in the conf file that are tuned related to the WAL 8 | const ( 9 | WALBuffersKey = "wal_buffers" 10 | MinWALKey = "min_wal_size" 11 | MaxWALKey = "max_wal_size" 12 | CheckpointTimeoutKey = "checkpoint_timeout" 13 | WALCompressionKey = "wal_compression" 14 | 15 | walMaxDiskPct = 60 // max_wal_size should be 60% of the WAL disk 16 | walBuffersThreshold = 2 * parse.Gigabyte 17 | walBuffersDefault = 16 * parse.Megabyte 18 | defaultMaxWALBytes = 1 * parse.Gigabyte 19 | promscaleDefaultMaxWALBytes = 4 * parse.Gigabyte 20 | promscaleDefaultCheckpointTimeout = "900" // 15 minutes expressed in seconds 21 | promscaleDefaultWALCompression = "1" 22 | ) 23 | 24 | // WALLabel is the label used to refer to the WAL settings group 25 | const WALLabel = "WAL" 26 | 27 | // WALKeys is an array of keys that are tunable for the WAL 28 | var WALKeys = []string{ 29 | WALBuffersKey, 30 | MinWALKey, 31 | MaxWALKey, 32 | CheckpointTimeoutKey, 33 | WALCompressionKey, 34 | } 35 | 36 | // WALRecommender gives recommendations for WALKeys based on system resources 37 | type WALRecommender struct { 38 | totalMemory uint64 39 | walDiskSize uint64 40 | } 41 | 42 | // NewWALRecommender returns a WALRecommender that recommends based on the given 43 | // totalMemory bytes. 44 | func NewWALRecommender(totalMemory, walDiskSize uint64) *WALRecommender { 45 | return &WALRecommender{ 46 | totalMemory: totalMemory, 47 | walDiskSize: walDiskSize, 48 | } 49 | } 50 | 51 | // IsAvailable returns whether this Recommender is usable given the system resources. Always true. 52 | func (r *WALRecommender) IsAvailable() bool { 53 | return true 54 | } 55 | 56 | // Recommend returns the recommended PostgreSQL formatted value for the conf 57 | // file for a given key. 58 | func (r *WALRecommender) Recommend(key string) string { 59 | var val string 60 | if key == WALBuffersKey { 61 | if r.totalMemory < walBuffersThreshold { 62 | temp := (float64(r.totalMemory) / float64(parse.Gigabyte)) * (7864.0 * float64(parse.Kilobyte)) 63 | val = parse.BytesToPGFormat(uint64(temp)) 64 | } else { 65 | val = parse.BytesToPGFormat(walBuffersDefault) 66 | } 67 | } else if key == MinWALKey { 68 | temp := r.calcMaxWALBytes() / 2 69 | val = parse.BytesToPGFormat(temp) 70 | } else if key == MaxWALKey { 71 | temp := r.calcMaxWALBytes() 72 | val = parse.BytesToPGFormat(temp) 73 | } else { 74 | val = NoRecommendation 75 | } 76 | return val 77 | } 78 | 79 | func (r *WALRecommender) calcMaxWALBytes() uint64 { 80 | // If disk size is not given, just use default 81 | if r.walDiskSize == 0 { 82 | return defaultMaxWALBytes 83 | } 84 | 85 | // With size given, we want to take up at most walMaxDiskPct, to give 86 | // additional room for safety. 87 | max := uint64(r.walDiskSize*walMaxDiskPct) / 100 88 | 89 | // WAL segments are 16MB, so it doesn't make sense not to round 90 | // up to the nearest 16MB boundary. 91 | if max%(16*parse.Megabyte) != 0 { 92 | max = (max/(16*parse.Megabyte) + 1) * 16 * parse.Megabyte 93 | } 94 | return max 95 | } 96 | 97 | // WALSettingsGroup is the SettingsGroup to represent settings that affect WAL usage. 98 | type WALSettingsGroup struct { 99 | totalMemory uint64 100 | walDiskSize uint64 101 | } 102 | 103 | // Label should always return the value WALLabel. 104 | func (sg *WALSettingsGroup) Label() string { return WALLabel } 105 | 106 | // Keys should always return the WALKeys slice. 107 | func (sg *WALSettingsGroup) Keys() []string { return WALKeys } 108 | 109 | // GetRecommender should return a new WALRecommender. 110 | func (sg *WALSettingsGroup) GetRecommender(profile Profile) Recommender { 111 | switch profile { 112 | case PromscaleProfile: 113 | return NewPromscaleWALRecommender(sg.totalMemory, sg.walDiskSize) 114 | default: 115 | return NewWALRecommender(sg.totalMemory, sg.walDiskSize) 116 | } 117 | } 118 | 119 | // PromscaleWALRecommender gives recommendations for WALKeys based on system resources 120 | type PromscaleWALRecommender struct { 121 | WALRecommender 122 | } 123 | 124 | // NewPromscaleWALRecommender returns a PromscaleWALRecommender that recommends based on the given 125 | // totalMemory bytes. 126 | func NewPromscaleWALRecommender(totalMemory, walDiskSize uint64) *PromscaleWALRecommender { 127 | return &PromscaleWALRecommender{ 128 | WALRecommender: WALRecommender{ 129 | totalMemory: totalMemory, 130 | walDiskSize: walDiskSize, 131 | }, 132 | } 133 | } 134 | 135 | // IsAvailable returns whether this Recommender is usable given the system resources. Always true. 136 | func (r *PromscaleWALRecommender) IsAvailable() bool { 137 | return true 138 | } 139 | 140 | // Recommend returns the recommended PostgreSQL formatted value for the conf 141 | // file for a given key. 142 | func (r *PromscaleWALRecommender) Recommend(key string) string { 143 | switch key { 144 | case MinWALKey: 145 | temp := r.promscaleCalcMaxWALBytes() / 2 146 | return parse.BytesToPGFormat(temp) 147 | case MaxWALKey: 148 | temp := r.promscaleCalcMaxWALBytes() 149 | return parse.BytesToPGFormat(temp) 150 | case CheckpointTimeoutKey: 151 | return promscaleDefaultCheckpointTimeout 152 | case WALCompressionKey: 153 | return promscaleDefaultWALCompression 154 | default: 155 | return r.WALRecommender.Recommend(key) 156 | } 157 | } 158 | 159 | func (r *WALRecommender) promscaleCalcMaxWALBytes() uint64 { 160 | // If disk size is not given, just use default 161 | if r.walDiskSize == 0 { 162 | return promscaleDefaultMaxWALBytes 163 | } 164 | 165 | // With size given, we want to take up at most walMaxDiskPct, to give 166 | // additional room for safety. 167 | max := uint64(r.walDiskSize*walMaxDiskPct) / 100 168 | 169 | // WAL segments are 16MB, so it doesn't make sense not to round 170 | // up to the nearest 16MB boundary. 171 | if max%(16*parse.Megabyte) != 0 { 172 | max = (max/(16*parse.Megabyte) + 1) * 16 * parse.Megabyte 173 | } 174 | return max 175 | } 176 | 177 | type WALFloatParser struct{} 178 | 179 | func (v *WALFloatParser) ParseFloat(key string, s string) (float64, error) { 180 | switch key { 181 | case WALCompressionKey: 182 | bfp := &boolFloatParser{} 183 | return bfp.ParseFloat(key, s) 184 | case CheckpointTimeoutKey: 185 | val, units, err := parse.PGFormatToTime(s, parse.Milliseconds, parse.VarTypeInteger) 186 | if err != nil { 187 | return val, err 188 | } 189 | conv, err := parse.TimeConversion(units, parse.Milliseconds) 190 | if err != nil { 191 | return val, err 192 | } 193 | return val * conv, nil 194 | default: 195 | bfp := &bytesFloatParser{} 196 | return bfp.ParseFloat(key, s) 197 | } 198 | } 199 | -------------------------------------------------------------------------------- /pkg/pgtune/wal_test.go: -------------------------------------------------------------------------------- 1 | package pgtune 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "testing" 7 | 8 | "github.com/timescale/timescaledb-tune/internal/parse" 9 | ) 10 | 11 | const ( 12 | walDiskUnset = 0 13 | walDiskDivideUnevenly = 8 * parse.Gigabyte 14 | walDiskDivideEvenly = 8800 * parse.Megabyte 15 | ) 16 | 17 | // memoryToWALBuffers provides a mapping from test case memory levels to the 18 | // expected WAL buffers setting. This is used to generate the test cases for 19 | // WALRecommender, stored in walSettingsMatrix. 20 | var memoryToWALBuffers = map[uint64]uint64{ 21 | 1 * parse.Gigabyte: 7864 * parse.Kilobyte, 22 | uint64(1.5 * float64(parse.Gigabyte)): 11796 * parse.Kilobyte, 23 | 2 * parse.Gigabyte: walBuffersDefault, 24 | 10 * parse.Gigabyte: walBuffersDefault, 25 | } 26 | 27 | var walDiskToMaxBytes = map[uint64]uint64{ 28 | walDiskUnset: defaultMaxWALBytes, 29 | walDiskDivideUnevenly: 4928 * parse.Megabyte, // nearest 16MB segment 30 | walDiskDivideEvenly: 5280 * parse.Megabyte, 31 | } 32 | 33 | var promscaleWALDiskToMaxBytes = map[uint64]uint64{ 34 | walDiskUnset: promscaleDefaultMaxWALBytes, 35 | walDiskDivideUnevenly: 4928 * parse.Megabyte, // nearest 16MB segment 36 | walDiskDivideEvenly: 5280 * parse.Megabyte, 37 | } 38 | 39 | // walSettingsMatrix stores the test cases for WALRecommender along with the 40 | // expected values for WAL keys. 41 | var walSettingsMatrix = map[uint64]map[uint64]map[string]string{} 42 | 43 | // walSettingsMatrix stores the test cases for WALRecommender along with the 44 | // expected values for WAL keys. 45 | var promscaleWalSettingsMatrix = map[uint64]map[uint64]map[string]string{} 46 | 47 | func init() { 48 | for memory, walBuffers := range memoryToWALBuffers { 49 | walSettingsMatrix[memory] = make(map[uint64]map[string]string) 50 | for walSize := range walDiskToMaxBytes { 51 | walSettingsMatrix[memory][walSize] = make(map[string]string) 52 | walSettingsMatrix[memory][walSize][MinWALKey] = parse.BytesToPGFormat(walDiskToMaxBytes[walSize] / 2) 53 | walSettingsMatrix[memory][walSize][MaxWALKey] = parse.BytesToPGFormat(walDiskToMaxBytes[walSize]) 54 | walSettingsMatrix[memory][walSize][WALBuffersKey] = parse.BytesToPGFormat(walBuffers) 55 | walSettingsMatrix[memory][walSize][CheckpointTimeoutKey] = NoRecommendation 56 | walSettingsMatrix[memory][walSize][WALCompressionKey] = NoRecommendation 57 | } 58 | } 59 | 60 | for memory, walBuffers := range memoryToWALBuffers { 61 | promscaleWalSettingsMatrix[memory] = make(map[uint64]map[string]string) 62 | for walSize := range walDiskToMaxBytes { 63 | promscaleWalSettingsMatrix[memory][walSize] = make(map[string]string) 64 | promscaleWalSettingsMatrix[memory][walSize][MinWALKey] = parse.BytesToPGFormat(promscaleWALDiskToMaxBytes[walSize] / 2) 65 | promscaleWalSettingsMatrix[memory][walSize][MaxWALKey] = parse.BytesToPGFormat(promscaleWALDiskToMaxBytes[walSize]) 66 | promscaleWalSettingsMatrix[memory][walSize][WALBuffersKey] = parse.BytesToPGFormat(walBuffers) 67 | promscaleWalSettingsMatrix[memory][walSize][CheckpointTimeoutKey] = promscaleDefaultCheckpointTimeout 68 | promscaleWalSettingsMatrix[memory][walSize][WALCompressionKey] = promscaleDefaultWALCompression 69 | } 70 | } 71 | } 72 | 73 | func TestWALSettingsGroup_GetRecommender(t *testing.T) { 74 | cases := []struct { 75 | profile Profile 76 | recommender string 77 | }{ 78 | {DefaultProfile, "*pgtune.WALRecommender"}, 79 | {PromscaleProfile, "*pgtune.PromscaleWALRecommender"}, 80 | } 81 | 82 | sg := WALSettingsGroup{totalMemory: 1, walDiskSize: 1} 83 | for _, k := range cases { 84 | r := sg.GetRecommender(k.profile) 85 | y := fmt.Sprintf("%T", r) 86 | if y != k.recommender { 87 | t.Errorf("Expected to get a %s using the %s profile but got %s", k.recommender, k.profile, y) 88 | } 89 | } 90 | } 91 | 92 | func TestNewWALRecommender(t *testing.T) { 93 | for i := 0; i < 1000000; i++ { 94 | mem := rand.Uint64() 95 | r := NewWALRecommender(mem, walDiskUnset) 96 | if r == nil { 97 | t.Errorf("unexpected nil recommender") 98 | } 99 | if got := r.totalMemory; got != mem { 100 | t.Errorf("recommender has incorrect memory: got %d want %d", got, mem) 101 | } 102 | 103 | if !r.IsAvailable() { 104 | t.Errorf("unexpectedly not available") 105 | } 106 | } 107 | } 108 | 109 | func TestWALRecommenderRecommend(t *testing.T) { 110 | for totalMemory, outerMatrix := range walSettingsMatrix { 111 | for walSize, matrix := range outerMatrix { 112 | r := NewWALRecommender(totalMemory, walSize) 113 | testRecommender(t, r, WALKeys, matrix) 114 | } 115 | } 116 | } 117 | 118 | func TestPromscaleWALRecommenderRecommend(t *testing.T) { 119 | for totalMemory, outerMatrix := range promscaleWalSettingsMatrix { 120 | for walSize, matrix := range outerMatrix { 121 | r := NewPromscaleWALRecommender(totalMemory, walSize) 122 | testRecommender(t, r, WALKeys, matrix) 123 | } 124 | } 125 | } 126 | 127 | func TestPromscaleWALRecommenderCheckpointTimeout(t *testing.T) { 128 | // recommendation for checkpoint timeout should not be impacted by totalMemory or walDiskSize 129 | for i := uint64(0); i < 1000000; i++ { 130 | r := NewPromscaleWALRecommender(i, i) 131 | if v := r.Recommend(CheckpointTimeoutKey); v != promscaleDefaultCheckpointTimeout { 132 | t.Errorf("Expected %s for %s, but got %s", promscaleDefaultCheckpointTimeout, CheckpointTimeoutKey, v) 133 | } 134 | } 135 | } 136 | 137 | func TestWALRecommenderNoRecommendation(t *testing.T) { 138 | r := NewWALRecommender(0, 0) 139 | if r.Recommend("foo") != NoRecommendation { 140 | t.Errorf("Recommendation was provided for %s when there should have been none", "foo") 141 | } 142 | 143 | if r.Recommend(CheckpointTimeoutKey) != NoRecommendation { 144 | t.Errorf("Recommendation was provided for %s when there should have been none", CheckpointTimeoutKey) 145 | } 146 | } 147 | 148 | func TestWALSettingsGroup(t *testing.T) { 149 | for totalMemory, outerMatrix := range walSettingsMatrix { 150 | for walSize, matrix := range outerMatrix { 151 | config := getDefaultTestSystemConfig(t) 152 | config.Memory = totalMemory 153 | config.WALDiskSize = walSize 154 | sg := GetSettingsGroup(WALLabel, config) 155 | testSettingGroup(t, sg, DefaultProfile, matrix, WALLabel, WALKeys) 156 | } 157 | } 158 | 159 | for totalMemory, outerMatrix := range promscaleWalSettingsMatrix { 160 | for walSize, matrix := range outerMatrix { 161 | config := getDefaultTestSystemConfig(t) 162 | config.Memory = totalMemory 163 | config.WALDiskSize = walSize 164 | sg := GetSettingsGroup(WALLabel, config) 165 | testSettingGroup(t, sg, PromscaleProfile, matrix, WALLabel, WALKeys) 166 | } 167 | } 168 | } 169 | 170 | func TestWALFloatParserParseFloat(t *testing.T) { 171 | v := &WALFloatParser{} 172 | 173 | s := "8" + parse.GB 174 | want := float64(8 * parse.Gigabyte) 175 | got, err := v.ParseFloat(MaxWALKey, s) 176 | if err != nil { 177 | t.Errorf("unexpected error: %v", err) 178 | } 179 | if got != want { 180 | t.Errorf("incorrect result: got %f want %f", got, want) 181 | } 182 | 183 | s = "1000" 184 | want = 1000.0 185 | got, err = v.ParseFloat(WALBuffersKey, s) 186 | if err != nil { 187 | t.Errorf("unexpected error: %v", err) 188 | } 189 | if got != want { 190 | t.Errorf("incorrect result: got %f want %f", got, want) 191 | } 192 | 193 | s = "33" + parse.Minutes.String() 194 | conversion, _ := parse.TimeConversion(parse.Minutes, parse.Milliseconds) 195 | want = 33.0 * conversion 196 | got, err = v.ParseFloat(CheckpointTimeoutKey, s) 197 | if err != nil { 198 | t.Errorf("unexpected error: %v", err) 199 | } 200 | if got != want { 201 | t.Errorf("incorrect result: got %f want %f", got, want) 202 | } 203 | } 204 | -------------------------------------------------------------------------------- /pkg/pgutils/utils.go: -------------------------------------------------------------------------------- 1 | package pgutils 2 | 3 | import ( 4 | "fmt" 5 | "os/exec" 6 | "regexp" 7 | ) 8 | 9 | // Major version strings for recent PostgreSQL versions 10 | const ( 11 | MajorVersion96 = "9.6" 12 | MajorVersion10 = "10" 13 | MajorVersion11 = "11" 14 | MajorVersion12 = "12" 15 | MajorVersion13 = "13" 16 | MajorVersion14 = "14" 17 | MajorVersion15 = "15" 18 | MajorVersion16 = "16" 19 | MajorVersion17 = "17" 20 | ) 21 | 22 | const ( 23 | defaultBinName = "pg_config" 24 | versionFlag = "--version" 25 | 26 | errCouldNotParseVersionFmt = "unable to parse PG version string: %s" 27 | errUnknownMajorVersionFmt = "unknown major PG version: %s" 28 | ) 29 | 30 | var ( 31 | pgVersionRegex = regexp.MustCompile("^PostgreSQL ([0-9]+)([.][0-9]+|devel|beta)") 32 | 33 | execFn = func(name string, args ...string) ([]byte, error) { 34 | return exec.Command(name, args...).Output() 35 | } 36 | ) 37 | 38 | // ToPGMajorVersion returns the major PostgreSQL version associated with a given 39 | // version string, as given from an invocation of `pg_config --version`. This 40 | // string has the form of "PostgreSQL X.Y[.Z (extra)]". For versions before 10, 41 | // the major version is defined as X.Y, whereas starting with 10, it is defined 42 | // as just X. That is, "PostgreSQL 10.3" returns "10" and "PostgreSQL 9.6.4" 43 | // returns "9.6". 44 | func ToPGMajorVersion(val string) (string, error) { 45 | res := pgVersionRegex.FindStringSubmatch(val) 46 | if len(res) != 3 { 47 | return "", fmt.Errorf(errCouldNotParseVersionFmt, val) 48 | } 49 | switch res[1] { 50 | case MajorVersion10, MajorVersion11, MajorVersion12, MajorVersion13, MajorVersion14, MajorVersion15, MajorVersion16, MajorVersion17: 51 | return res[1], nil 52 | case "7", "8", "9": 53 | return res[1] + res[2], nil 54 | default: 55 | return "", fmt.Errorf(errUnknownMajorVersionFmt, val) 56 | } 57 | } 58 | 59 | // GetPGConfigVersion executes the pg_config binary (assuming it is in PATH) to 60 | // get the version of PostgreSQL associated with it. 61 | func GetPGConfigVersion() (string, error) { 62 | return GetPGConfigVersionAtPath(defaultBinName) 63 | } 64 | 65 | // GetPGConfigVersionAtPath executes the (pg_config) binary at path to get the 66 | // version of PostgreSQL associated with it. 67 | func GetPGConfigVersionAtPath(path string) (string, error) { 68 | output, err := execFn(path, versionFlag) 69 | if err != nil { 70 | return "", err 71 | } 72 | return string(output), nil 73 | } 74 | -------------------------------------------------------------------------------- /pkg/pgutils/utils_test.go: -------------------------------------------------------------------------------- 1 | package pgutils 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | func TestToPGMajorVersion(t *testing.T) { 9 | okPrefix := "PostgreSQL " 10 | cases := []struct { 11 | desc string 12 | version string 13 | want string 14 | errMsg string 15 | }{ 16 | { 17 | desc: "pg12", 18 | version: okPrefix + "12.4", 19 | want: "12", 20 | }, 21 | { 22 | desc: "pg12 w/ extra", 23 | version: okPrefix + "12.4 (Debian 10.1+foo)", 24 | want: "12", 25 | }, 26 | { 27 | desc: "pg11", 28 | version: okPrefix + "11.1", 29 | want: "11", 30 | }, 31 | { 32 | desc: "pg11 w/ extra", 33 | version: okPrefix + "11.1 (Debian)", 34 | want: "11", 35 | }, 36 | { 37 | desc: "pg10", 38 | version: okPrefix + "10.5", 39 | want: "10", 40 | }, 41 | { 42 | desc: "pg10 w/ extra", 43 | version: okPrefix + "10.2 (Debian)", 44 | want: "10", 45 | }, 46 | { 47 | desc: "9.6", 48 | version: okPrefix + "9.6.3", 49 | want: "9.6", 50 | }, 51 | 52 | { 53 | desc: "9.5", 54 | version: okPrefix + "9.5.9", 55 | want: "9.5", 56 | }, 57 | { 58 | desc: "8.1", 59 | version: okPrefix + "8.1.9", 60 | want: "8.1", 61 | }, 62 | { 63 | desc: "7.3", 64 | version: okPrefix + "7.3.9", 65 | want: "7.3", 66 | }, 67 | { 68 | desc: "bad parse", 69 | version: "10.0", 70 | want: "", 71 | errMsg: fmt.Sprintf(errCouldNotParseVersionFmt, "10.0"), 72 | }, 73 | { 74 | desc: "old version", 75 | version: "PostgreSQL 6.3.2", 76 | want: "", 77 | errMsg: fmt.Sprintf(errUnknownMajorVersionFmt, "PostgreSQL 6.3.2"), 78 | }, 79 | } 80 | 81 | for _, c := range cases { 82 | got, err := ToPGMajorVersion(c.version) 83 | if got != c.want { 84 | t.Errorf("%s: incorrect version: got %s want %s", c.desc, got, c.want) 85 | } 86 | if len(c.errMsg) > 0 { 87 | if err == nil { 88 | t.Errorf("%s: unexpected lack of error", c.desc) 89 | } 90 | if got := err.Error(); got != c.errMsg { 91 | t.Errorf("%s: incorrect error msg: got\n%s\nwant\n%s", c.desc, got, c.errMsg) 92 | } 93 | } else { 94 | if err != nil { 95 | t.Errorf("%s: unexpected error: %v", c.desc, err) 96 | } 97 | } 98 | } 99 | } 100 | 101 | func TestGetPGConfigVersionAtPath(t *testing.T) { 102 | wantStr := "test success" 103 | goodName := "foo" 104 | badName := "bad" 105 | errStr := "error" 106 | 107 | oldExecFn := execFn 108 | var calledName string 109 | var calledArgs []string 110 | execFn = func(name string, args ...string) ([]byte, error) { 111 | calledName = name 112 | calledArgs = args 113 | if name == badName { 114 | return nil, fmt.Errorf(errStr) 115 | } 116 | return []byte(wantStr), nil 117 | } 118 | out, err := GetPGConfigVersionAtPath(goodName) 119 | if err != nil { 120 | t.Errorf("unexpected error: %v", err) 121 | } else { 122 | if got := string(out); got != wantStr { 123 | t.Errorf("unexpected result: got %s want %s", got, wantStr) 124 | } 125 | if got := calledName; got != "foo" { 126 | t.Errorf("incorrect calledName: got %s want %s", got, goodName) 127 | } 128 | if got := len(calledArgs); got != 1 { 129 | t.Errorf("incorrect calledArgs len: got %d want %d", got, 1) 130 | } 131 | if got := calledArgs[0]; got != versionFlag { 132 | t.Errorf("incorrect calledArgs: got %s want %s", got, versionFlag) 133 | } 134 | } 135 | 136 | out, err = GetPGConfigVersionAtPath(badName) 137 | if err == nil { 138 | t.Errorf("unexpected lack of error") 139 | } else if out != "" { 140 | t.Errorf("unexpected output: got %s", out) 141 | } else if got := err.Error(); got != errStr { 142 | t.Errorf("unexpected error: got %s want %s", got, errStr) 143 | } 144 | 145 | out, err = GetPGConfigVersion() 146 | if err != nil { 147 | t.Errorf("unexpected error: %v", err) 148 | } else { 149 | if got := string(out); got != wantStr { 150 | t.Errorf("unexpected result: got %s want %s", got, wantStr) 151 | } 152 | if got := calledName; got != defaultBinName { 153 | t.Errorf("incorrect calledName: got %s want %s", got, defaultBinName) 154 | } 155 | if got := len(calledArgs); got != 1 { 156 | t.Errorf("incorrect calledArgs len: got %d want %d", got, 1) 157 | } 158 | if got := calledArgs[0]; got != versionFlag { 159 | t.Errorf("incorrect calledArgs: got %s want %s", got, versionFlag) 160 | } 161 | } 162 | 163 | execFn = oldExecFn 164 | } 165 | -------------------------------------------------------------------------------- /pkg/tstune/backup.go: -------------------------------------------------------------------------------- 1 | package tstune 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "os" 7 | "path" 8 | "path/filepath" 9 | "strings" 10 | "time" 11 | ) 12 | 13 | const ( 14 | backupFilePrefix = "timescaledb_tune.backup" 15 | backupDateFmt = "200601021504" 16 | 17 | errBackupNotCreatedFmt = "could not create backup at %s: %v" 18 | ) 19 | 20 | // allows us to substitute mock versions in tests 21 | var filepathGlobFn = filepath.Glob 22 | var osCreateFn = func(path string) (io.WriteCloser, error) { 23 | return os.Create(path) 24 | } 25 | 26 | // backup writes the conf file state to the system's temporary directory 27 | // with a well known name format so it can potentially be restored. 28 | func backup(cfs *configFileState) (string, error) { 29 | backupName := backupFilePrefix + time.Now().Format(backupDateFmt) 30 | backupPath := path.Join(os.TempDir(), backupName) 31 | bf, err := osCreateFn(backupPath) 32 | if err != nil { 33 | return backupPath, fmt.Errorf(errBackupNotCreatedFmt, backupPath, err) 34 | } 35 | _, err = cfs.WriteTo(bf) 36 | return backupPath, err 37 | } 38 | 39 | // getBackups returns a list of files that match timescaledb-tune's backup 40 | // filename format. 41 | func getBackups() ([]string, error) { 42 | backupPattern := path.Join(os.TempDir(), backupFilePrefix+"*") 43 | files, err := filepathGlobFn(backupPattern) 44 | if err != nil { 45 | return nil, err 46 | } 47 | ret := []string{} 48 | stripPrefix := path.Join(os.TempDir(), backupFilePrefix) 49 | for _, f := range files { 50 | datePart := strings.Replace(f, stripPrefix, "", -1) 51 | _, err := time.Parse(backupDateFmt, datePart) 52 | if err != nil { 53 | continue 54 | } 55 | ret = append(ret, f) 56 | } 57 | return ret, nil 58 | } 59 | 60 | type restorer interface { 61 | Restore(string, string) error 62 | } 63 | 64 | type fsRestorer struct{} 65 | 66 | func (r *fsRestorer) Restore(backupPath, confPath string) error { 67 | backupFile, err := os.Open(backupPath) 68 | if err != nil { 69 | return err 70 | } 71 | defer backupFile.Close() 72 | 73 | backupCFS, err := getConfigFileState(backupFile) 74 | if err != nil { 75 | return err 76 | } 77 | 78 | confFile, err := os.OpenFile(confPath, os.O_WRONLY, 0644) 79 | if err != nil { 80 | return err 81 | } 82 | defer confFile.Close() 83 | 84 | _, err = backupCFS.WriteTo(confFile) 85 | if err != nil { 86 | return err 87 | } 88 | 89 | return nil 90 | } 91 | -------------------------------------------------------------------------------- /pkg/tstune/backup_test.go: -------------------------------------------------------------------------------- 1 | package tstune 2 | 3 | import ( 4 | "bufio" 5 | "bytes" 6 | "fmt" 7 | "io" 8 | "io/ioutil" 9 | "os" 10 | "path" 11 | "testing" 12 | "time" 13 | ) 14 | 15 | type testBufferCloser struct { 16 | b bytes.Buffer 17 | shouldErr bool 18 | } 19 | 20 | func (b *testBufferCloser) Write(p []byte) (int, error) { 21 | if b.shouldErr { 22 | return 0, fmt.Errorf(errTestWriter) 23 | } 24 | return b.b.Write(p) 25 | } 26 | 27 | func (b *testBufferCloser) Close() error { return nil } 28 | 29 | func TestBackup(t *testing.T) { 30 | oldOSCreateFn := osCreateFn 31 | now := time.Now() 32 | lines := []string{"foo", "bar", "baz", "quaz"} 33 | r := stringSliceToBytesReader(lines) 34 | cfs, err := getConfigFileState(r) 35 | if err != nil { 36 | t.Fatalf("unexpected err: %v", err) 37 | } 38 | wantFileName := backupFilePrefix + now.Format(backupDateFmt) 39 | wantPath := path.Join(os.TempDir(), wantFileName) 40 | 41 | osCreateFn = func(_ string) (io.WriteCloser, error) { 42 | return nil, fmt.Errorf("erroring") 43 | } 44 | 45 | path, err := backup(cfs) 46 | if path != wantPath { 47 | t.Errorf("incorrect path in error case: got\n%s\nwant\n%s", path, wantPath) 48 | } 49 | if err == nil { 50 | t.Errorf("unexpected lack of error for bad create") 51 | } 52 | want := fmt.Sprintf(errBackupNotCreatedFmt, wantPath, "erroring") 53 | if got := err.Error(); got != want { 54 | t.Errorf("incorrect error: got\n%s\nwant\n%s", got, want) 55 | } 56 | 57 | var buf testBufferCloser 58 | osCreateFn = func(p string) (io.WriteCloser, error) { 59 | if p != wantPath { 60 | t.Errorf("incorrect backup path: got %s want %s", p, wantPath) 61 | } 62 | return &buf, nil 63 | } 64 | path, err = backup(cfs) 65 | if path != wantPath { 66 | t.Errorf("incorrect path in correct case: got\n%s\nwant\n%s", path, wantPath) 67 | } 68 | if err != nil { 69 | t.Errorf("unexpected error for backup: %v", err) 70 | } 71 | 72 | scanner := bufio.NewScanner(bytes.NewReader(buf.b.Bytes())) 73 | i := 0 74 | for scanner.Scan() { 75 | if scanner.Err() != nil { 76 | t.Errorf("unexpected error while scanning: %v", scanner.Err()) 77 | } 78 | got := scanner.Text() 79 | if want := lines[i]; got != want { 80 | t.Errorf("incorrect line at %d: got\n%s\nwant\n%s", i, got, want) 81 | } 82 | i++ 83 | } 84 | 85 | osCreateFn = oldOSCreateFn 86 | } 87 | 88 | func TestGetBackups(t *testing.T) { 89 | errGlob := "glob error" 90 | correctFile1 := path.Join(os.TempDir(), "timescaledb_tune.backup201901181122") 91 | correctFile2 := path.Join(os.TempDir(), "timescaledb_tune.backup201901191200") 92 | cases := []struct { 93 | desc string 94 | onDiskFiles []string 95 | globErr bool 96 | want []string 97 | errMsg string 98 | }{ 99 | { 100 | desc: "error on glob", 101 | onDiskFiles: []string{"foo"}, 102 | globErr: true, 103 | errMsg: errGlob, 104 | }, 105 | { 106 | desc: "no matching files", 107 | onDiskFiles: []string{}, 108 | want: []string{}, 109 | }, 110 | { 111 | desc: "invalid file", 112 | onDiskFiles: []string{"foo"}, 113 | want: []string{}, 114 | }, 115 | { 116 | desc: "one correct file", 117 | onDiskFiles: []string{correctFile1}, 118 | want: []string{correctFile1}, 119 | }, 120 | { 121 | desc: "two correct files", 122 | onDiskFiles: []string{correctFile1, correctFile2}, 123 | want: []string{correctFile1, correctFile2}, 124 | }, 125 | { 126 | desc: "two correct files with wrong files", 127 | onDiskFiles: []string{"foo", correctFile1, "bar", correctFile2, "baz"}, 128 | want: []string{correctFile1, correctFile2}, 129 | }, 130 | } 131 | oldFilepathGlobFn := filepathGlobFn 132 | for _, c := range cases { 133 | filepathGlobFn = func(_ string) ([]string, error) { 134 | if c.globErr { 135 | return nil, fmt.Errorf(errGlob) 136 | } 137 | return c.onDiskFiles, nil 138 | } 139 | 140 | files, err := getBackups() 141 | if c.errMsg == "" && err != nil { 142 | t.Errorf("%s: unexpected error: got %v", c.desc, err) 143 | } else if c.errMsg != "" { 144 | if err == nil { 145 | t.Errorf("%s: unexpected lack of error", c.desc) 146 | } 147 | if got := err.Error(); got != c.errMsg { 148 | t.Errorf("%s: unexpected error msg: got\n%s\nwant\n%s", c.desc, got, c.errMsg) 149 | } 150 | } 151 | if got := len(files); got != len(c.want) { 152 | t.Errorf("%s: incorrect size of files: got %d want %d", c.desc, got, len(c.want)) 153 | } else { 154 | for i, wantFile := range c.want { 155 | if got := files[i]; got != wantFile { 156 | t.Errorf("%s: incorrect file at index %d: got %s want %s", c.desc, i, got, wantFile) 157 | } 158 | } 159 | } 160 | } 161 | filepathGlobFn = oldFilepathGlobFn 162 | } 163 | 164 | func TestFSRestorer(t *testing.T) { 165 | fileContents := []byte("oneline\ntwoline\nthreeline\n") 166 | tmpfile, err := ioutil.TempFile("", "timescaledb-tune-test") 167 | if err != nil { 168 | t.Fatal(err) 169 | } 170 | defer os.Remove(tmpfile.Name()) // clean up 171 | 172 | if _, err = tmpfile.Write(fileContents); err != nil { 173 | t.Fatal(err) 174 | } 175 | if err = tmpfile.Close(); err != nil { 176 | t.Fatal(err) 177 | } 178 | backupPath := tmpfile.Name() 179 | tmpfile2, err := ioutil.TempFile("", "timescaledb-tune-test") 180 | if err != nil { 181 | t.Fatal(err) 182 | } 183 | defer os.Remove(tmpfile2.Name()) // clean up 184 | 185 | confPath := tmpfile2.Name() 186 | 187 | r := &fsRestorer{} 188 | err = r.Restore("", "") 189 | if err == nil { 190 | t.Fatalf("expected an error 1") 191 | } 192 | err = r.Restore(backupPath, "") 193 | if err == nil { 194 | t.Fatalf("expected an error 2") 195 | } 196 | err = r.Restore(backupPath, confPath) 197 | if err != nil { 198 | t.Errorf("unexpected error: %v", err) 199 | } 200 | 201 | backupContents, err := ioutil.ReadFile(tmpfile.Name()) 202 | if err != nil { 203 | t.Fatalf("unexpected error: %v", err) 204 | } 205 | 206 | destContents, err := ioutil.ReadFile(tmpfile2.Name()) 207 | if err != nil { 208 | t.Fatalf("unexpected error: %v", err) 209 | } 210 | 211 | if got := string(destContents); got != string(backupContents) { 212 | t.Errorf("contents not the same: got\n%s\nwant\n%s", got, string(backupContents)) 213 | } 214 | } 215 | -------------------------------------------------------------------------------- /pkg/tstune/checker.go: -------------------------------------------------------------------------------- 1 | package tstune 2 | 3 | import ( 4 | "fmt" 5 | "strconv" 6 | ) 7 | 8 | var ( 9 | errSkip = fmt.Errorf("skip err") 10 | ) 11 | 12 | func isYes(s string) bool { 13 | return s == "y" || s == "yes" 14 | } 15 | 16 | func isNo(s string) bool { 17 | return s == "n" || s == "no" 18 | } 19 | 20 | func isSkip(s string) bool { 21 | return s == "s" || s == "skip" 22 | } 23 | 24 | func isQuit(s string) bool { 25 | return s == "q" || s == "quit" 26 | } 27 | 28 | type promptChecker interface { 29 | Check(string) (bool, error) 30 | } 31 | 32 | type yesNoChecker struct { 33 | errMsg string 34 | args []interface{} 35 | } 36 | 37 | func newYesNoChecker(errMsg string, args ...interface{}) *yesNoChecker { 38 | return &yesNoChecker{errMsg, args} 39 | } 40 | 41 | func (c *yesNoChecker) Check(r string) (bool, error) { 42 | if isNo(r) { 43 | return false, fmt.Errorf(c.errMsg, c.args...) 44 | } else if isYes(r) { 45 | return true, nil 46 | } 47 | return false, nil 48 | } 49 | 50 | type skipChecker struct { 51 | err error 52 | } 53 | 54 | func newSkipChecker(errMsg string, args ...interface{}) *skipChecker { 55 | return &skipChecker{fmt.Errorf(errMsg, args...)} 56 | } 57 | 58 | func (c *skipChecker) Check(r string) (bool, error) { 59 | if isQuit(r) || isNo(r) { 60 | return false, c.err 61 | } else if isYes(r) { 62 | return true, nil 63 | } else if isSkip(r) { 64 | return true, errSkip 65 | } 66 | return false, nil 67 | } 68 | 69 | type numberedListChecker struct { 70 | limit int 71 | err error 72 | response int 73 | } 74 | 75 | func newNumberedListChecker(limit int, errMsg string, args ...interface{}) *numberedListChecker { 76 | return &numberedListChecker{limit, fmt.Errorf(errMsg, args...), 0} 77 | } 78 | 79 | func (c *numberedListChecker) Check(r string) (bool, error) { 80 | if isQuit(r) { 81 | return false, c.err 82 | } 83 | num, err := strconv.ParseInt(r, 10, 0) 84 | if err != nil { 85 | return false, err 86 | } else if num < 1 || int(num) > c.limit { 87 | return false, nil 88 | } 89 | c.response = int(num) 90 | return true, nil 91 | } 92 | -------------------------------------------------------------------------------- /pkg/tstune/checker_test.go: -------------------------------------------------------------------------------- 1 | package tstune 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | type isCases struct { 9 | s string 10 | want bool 11 | } 12 | 13 | func testIsFunc(t *testing.T, cases []isCases, fn func(string) bool) { 14 | for _, c := range cases { 15 | if got := fn(c.s); got != c.want { 16 | t.Errorf("'%s' returned wrong answer: got %v want %v", c.s, got, c.want) 17 | } 18 | } 19 | } 20 | 21 | func TestIsYes(t *testing.T) { 22 | cases := []isCases{ 23 | { 24 | s: "y", 25 | want: true, 26 | }, 27 | { 28 | s: "yes", 29 | want: true, 30 | }, 31 | { 32 | s: "ye", 33 | want: false, 34 | }, 35 | { 36 | s: "n", 37 | want: false, 38 | }, 39 | { 40 | s: "no", 41 | want: false, 42 | }, 43 | { 44 | s: "", 45 | want: false, 46 | }, 47 | } 48 | 49 | testIsFunc(t, cases, isYes) 50 | } 51 | 52 | func TestIsNo(t *testing.T) { 53 | cases := []isCases{ 54 | { 55 | s: "y", 56 | want: false, 57 | }, 58 | { 59 | s: "yes", 60 | want: false, 61 | }, 62 | { 63 | s: "ye", 64 | want: false, 65 | }, 66 | { 67 | s: "n", 68 | want: true, 69 | }, 70 | { 71 | s: "no", 72 | want: true, 73 | }, 74 | { 75 | s: "", 76 | want: false, 77 | }, 78 | } 79 | 80 | testIsFunc(t, cases, isNo) 81 | } 82 | 83 | func TestIsSkip(t *testing.T) { 84 | cases := []isCases{ 85 | { 86 | s: "s", 87 | want: true, 88 | }, 89 | { 90 | s: "skip", 91 | want: true, 92 | }, 93 | { 94 | s: "sk", 95 | want: false, 96 | }, 97 | { 98 | s: "y", 99 | want: false, 100 | }, 101 | { 102 | s: "yes", 103 | want: false, 104 | }, 105 | { 106 | s: "ye", 107 | want: false, 108 | }, 109 | { 110 | s: "n", 111 | want: false, 112 | }, 113 | { 114 | s: "no", 115 | want: false, 116 | }, 117 | { 118 | s: "", 119 | want: false, 120 | }, 121 | } 122 | 123 | testIsFunc(t, cases, isSkip) 124 | } 125 | 126 | func TestIsQuit(t *testing.T) { 127 | cases := []isCases{ 128 | { 129 | s: "q", 130 | want: true, 131 | }, 132 | { 133 | s: "quit", 134 | want: true, 135 | }, 136 | { 137 | s: "qu", 138 | want: false, 139 | }, 140 | { 141 | s: "y", 142 | want: false, 143 | }, 144 | { 145 | s: "yes", 146 | want: false, 147 | }, 148 | { 149 | s: "ye", 150 | want: false, 151 | }, 152 | { 153 | s: "n", 154 | want: false, 155 | }, 156 | { 157 | s: "no", 158 | want: false, 159 | }, 160 | { 161 | s: "", 162 | want: false, 163 | }, 164 | } 165 | 166 | testIsFunc(t, cases, isQuit) 167 | } 168 | 169 | func TestNewNumberedListCheckerCheck(t *testing.T) { 170 | defaultErrMsg := "default error" 171 | defaultLimit := 3 172 | cases := []struct { 173 | s string 174 | want bool 175 | errMsg string 176 | }{ 177 | { 178 | s: "q", 179 | want: false, 180 | errMsg: defaultErrMsg, 181 | }, 182 | { 183 | s: "not a number", 184 | want: false, 185 | errMsg: "strconv.ParseInt: parsing \"not a number\": invalid syntax", 186 | }, 187 | { 188 | s: "0", 189 | want: false, 190 | }, 191 | { 192 | s: fmt.Sprintf("%d", defaultLimit+1), 193 | want: false, 194 | }, 195 | { 196 | s: "1", 197 | want: true, 198 | }, 199 | { 200 | s: "2", 201 | want: true, 202 | }, 203 | { 204 | s: fmt.Sprintf("%d", defaultLimit), 205 | want: true, 206 | }, 207 | } 208 | 209 | for _, c := range cases { 210 | checker := newNumberedListChecker(defaultLimit, defaultErrMsg) 211 | got, err := checker.Check(c.s) 212 | if c.errMsg == "" && err != nil { 213 | t.Errorf("%s: unexpected err: got %v", c.s, err) 214 | } else if c.errMsg != "" { 215 | if err == nil { 216 | t.Errorf("%s: unexpected lack of error", c.s) 217 | } else if got := err.Error(); got != c.errMsg { 218 | t.Errorf("%s: incorrect error: got %s want %s", c.s, got, c.errMsg) 219 | } 220 | } else if got != c.want { 221 | t.Errorf("%s: incorrect value: got %v want %v", c.s, got, c.want) 222 | } 223 | } 224 | } 225 | -------------------------------------------------------------------------------- /pkg/tstune/config_file.go: -------------------------------------------------------------------------------- 1 | package tstune 2 | 3 | import ( 4 | "bufio" 5 | "fmt" 6 | "io" 7 | "os" 8 | "regexp" 9 | "strings" 10 | ) 11 | 12 | const ( 13 | osMac = "darwin" 14 | osLinux = "linux" 15 | 16 | fileNameMac = "/usr/local/var/postgres/postgresql.conf" 17 | fileNameMacFmt = "/usr/local/var/postgresql@%s/postgresql.conf" 18 | fileNameMacM1 = "/opt/homebrew/var/postgres/postgresql.conf" 19 | fileNameMacM1Fmt = "/opt/homebrew/var/postgresql@%s/postgresql.conf" 20 | fileNameDebianFmt = "/etc/postgresql/%s/main/postgresql.conf" 21 | fileNameRPMFmt = "/var/lib/pgsql/%s/data/postgresql.conf" 22 | fileNameArch = "/var/lib/postgres/data/postgresql.conf" 23 | fileNameAlpine = "/var/lib/postgresql/data/postgresql.conf" 24 | 25 | errConfigNotFoundFmt = "could not find postgresql.conf at any of these locations:\n%v" 26 | ) 27 | 28 | type truncateWriter interface { 29 | io.Writer 30 | Seek(int64, int) (int64, error) 31 | Truncate(int64) error 32 | } 33 | 34 | // getConfigFilePath attempts to find the postgresql.conf file using path heuristics 35 | // for different operating systems. If successful it returns the full path to 36 | // the file; otherwise, it returns with an empty path and error. 37 | func getConfigFilePath(system, pgVersion string) (string, error) { 38 | tried := []string{} 39 | try := func(format string, args ...interface{}) string { 40 | fileName := fmt.Sprintf(format, args...) 41 | tried = append(tried, fileName) 42 | if fileExists(fileName) { 43 | return fileName 44 | } 45 | return "" 46 | } 47 | pgdata := os.Getenv("PGDATA") 48 | if pgdata != "" { 49 | fileName := try(pgdata + "/postgresql.conf") 50 | if fileName != "" { 51 | return fileName, nil 52 | } 53 | } 54 | switch system { 55 | case osMac: 56 | fileName := try(fileNameMac) 57 | if fileName != "" { 58 | return fileName, nil 59 | } 60 | fileName = try(fileNameMacFmt, pgVersion) 61 | if fileName != "" { 62 | return fileName, nil 63 | } 64 | fileName = try(fileNameMacM1) 65 | if fileName != "" { 66 | return fileName, nil 67 | } 68 | fileName = try(fileNameMacM1Fmt, pgVersion) 69 | if fileName != "" { 70 | return fileName, nil 71 | } 72 | case osLinux: 73 | fileName := try(fileNameDebianFmt, pgVersion) 74 | if fileName != "" { 75 | return fileName, nil 76 | } 77 | fileName = try(fileNameRPMFmt, pgVersion) 78 | if fileName != "" { 79 | return fileName, nil 80 | } 81 | fileName = try(fileNameArch) 82 | if fileName != "" { 83 | return fileName, nil 84 | } 85 | fileName = try(fileNameAlpine) 86 | if fileName != "" { 87 | return fileName, nil 88 | } 89 | } 90 | return "", fmt.Errorf(errConfigNotFoundFmt, strings.Join(tried, "\n")) 91 | } 92 | 93 | type tunableParseResult struct { 94 | idx int 95 | commented bool 96 | missing bool 97 | key string 98 | value string 99 | extra string 100 | } 101 | 102 | // configLine represents a line in the conf file with some associated metadata 103 | // on how it should be processed when written. 104 | type configLine struct { 105 | content string 106 | remove bool 107 | } 108 | 109 | // configLineProcessor is an interface that processes a line of the conf file to 110 | // do some manipulation, typically called in a loop or multiple times on different 111 | // lines. For example, a dupes remover could be called on all lines and mark 112 | // the lines that, for some key, appear multiple times for deletion. 113 | type configLineProcessor interface { 114 | Process(*configLine) error 115 | } 116 | 117 | // removesDuplicatesProcessor is used to track and mark duplicates for removal that 118 | // match a particular regex. 119 | type removeDuplicatesProcessor struct { 120 | prev *configLine 121 | regex *regexp.Regexp 122 | } 123 | 124 | // Process takes the input l to determine if any previous lines should be marked 125 | // for removal. The processor alays tracks the last instance matching the regex 126 | // it has seen, so if it encounters a new one, it can mark the previous instance 127 | // for removal. 128 | func (p *removeDuplicatesProcessor) Process(l *configLine) error { 129 | if found := parseWithRegex(l.content, p.regex); found != nil { 130 | if p.prev != nil { 131 | p.prev.remove = true 132 | } 133 | p.prev = l 134 | } 135 | return nil 136 | } 137 | 138 | // getRemoveDuplicatesProcessors is a convenience function for creating a slice 139 | // of configLineProcessors (of the removeDuplicatesProcessor type) for a set of 140 | // keys. 141 | func getRemoveDuplicatesProcessors(keys []string) []configLineProcessor { 142 | ret := []configLineProcessor{} 143 | for _, key := range keys { 144 | ret = append(ret, &removeDuplicatesProcessor{regex: keyToRegexQuoted(key)}) 145 | } 146 | return ret 147 | } 148 | 149 | // configFileState represents the postgresql.conf file, including all of its 150 | // lines, the parsed result of the shared_preload_libraries line, and parse results 151 | // for parameters we care about tuning 152 | type configFileState struct { 153 | lines []*configLine // all the lines, to be updated for output 154 | sharedLibResult *sharedLibResult // parsing result for shared lib line 155 | tuneParseResults map[string]*tunableParseResult // mapping of each tunable param to its parsed line result 156 | } 157 | 158 | // getConfigFileState returns the current state of the configuration file by 159 | // reading it line by line and parsing those lines we particularly care about. 160 | func getConfigFileState(r io.Reader) (*configFileState, error) { 161 | cfs := &configFileState{ 162 | lines: []*configLine{}, 163 | tuneParseResults: make(map[string]*tunableParseResult), 164 | } 165 | i := 0 166 | scanner := bufio.NewScanner(r) 167 | for scanner.Scan() { 168 | if scanner.Err() != nil { 169 | return nil, fmt.Errorf("could not read postgresql.conf: %v", scanner.Err()) 170 | } 171 | line := scanner.Text() 172 | temp := parseLineForSharedLibResult(line) 173 | if temp != nil { 174 | temp.idx = i 175 | cfs.sharedLibResult = temp 176 | } else { 177 | for k, regex := range regexes { 178 | tpr := parseWithRegex(line, regex) 179 | if tpr != nil { 180 | tpr.idx = i 181 | cfs.tuneParseResults[k] = tpr 182 | } 183 | } 184 | } 185 | cfs.lines = append(cfs.lines, &configLine{content: line}) 186 | i++ 187 | } 188 | return cfs, nil 189 | } 190 | 191 | func (cfs *configFileState) ProcessLines(processors ...configLineProcessor) error { 192 | var err error 193 | for _, line := range cfs.lines { 194 | for _, p := range processors { 195 | err = p.Process(line) 196 | if err != nil { 197 | return err 198 | } 199 | } 200 | } 201 | return nil 202 | } 203 | 204 | func (cfs *configFileState) WriteTo(w io.Writer) (int64, error) { 205 | // in case new output is shorter than old, need to truncate first 206 | switch t := w.(type) { 207 | case truncateWriter: 208 | err := t.Truncate(0) 209 | if err != nil { 210 | return 0, err 211 | } 212 | _, err = t.Seek(0, 0) 213 | if err != nil { 214 | return 0, err 215 | } 216 | } 217 | ret := int64(0) 218 | for _, l := range cfs.lines { 219 | if l.remove { 220 | continue 221 | } 222 | 223 | n, err := w.Write([]byte(l.content + "\n")) 224 | if err != nil { 225 | return 0, err 226 | } 227 | ret += int64(n) 228 | } 229 | return ret, nil 230 | } 231 | -------------------------------------------------------------------------------- /pkg/tstune/config_file_test.go: -------------------------------------------------------------------------------- 1 | package tstune 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "io" 7 | "os" 8 | "strings" 9 | "testing" 10 | 11 | "github.com/timescale/timescaledb-tune/pkg/pgtune" 12 | "github.com/timescale/timescaledb-tune/pkg/pgutils" 13 | ) 14 | 15 | func stringSliceToBytesReader(lines []string) *bytes.Buffer { 16 | return bytes.NewBufferString(strings.Join(lines, "\n")) 17 | } 18 | 19 | func TestRemoveDuplicatesProcessor(t *testing.T) { 20 | lines := []*configLine{ 21 | {content: "foo = 'bar'"}, 22 | {content: "foo = 'baz'"}, 23 | {content: "foo = 'quaz'"}, 24 | } 25 | p := &removeDuplicatesProcessor{regex: keyToRegexQuoted("foo")} 26 | p.Process(lines[0]) 27 | if lines[0].remove { 28 | t.Errorf("first instance incorrectly marked for remove") 29 | } 30 | 31 | check := func(idx int) { 32 | err := p.Process(lines[idx]) 33 | if err != nil { 34 | t.Errorf("unexpected error on test %d: %v", idx, err) 35 | } 36 | if !lines[idx-1].remove { 37 | t.Errorf("configLine not marked to remove on test %d", idx) 38 | } 39 | if lines[idx].remove { 40 | t.Errorf("configLine incorrectly marked to remove on test %d", idx) 41 | } 42 | } 43 | 44 | check(1) 45 | check(2) 46 | } 47 | 48 | func TestGetRemoveDuplicatesProcessors(t *testing.T) { 49 | cases := []struct { 50 | desc string 51 | keys []string 52 | }{ 53 | { 54 | desc: "no keys", 55 | keys: []string{}, 56 | }, 57 | { 58 | desc: "one key", 59 | keys: []string{"foo"}, 60 | }, 61 | { 62 | desc: "two keys", 63 | keys: []string{"foo", "bar"}, 64 | }, 65 | } 66 | 67 | for _, c := range cases { 68 | procs := getRemoveDuplicatesProcessors(c.keys) 69 | if got := len(procs); got != len(c.keys) { 70 | t.Errorf("%s: incorrect length: got %d want %d", c.desc, got, len(c.keys)) 71 | } else { 72 | for i, key := range c.keys { 73 | rdp := procs[i].(*removeDuplicatesProcessor) 74 | want := keyToRegexQuoted(key).String() 75 | if got := rdp.regex.String(); got != want { 76 | t.Errorf("%s: incorrect proc at %d: got %s want %s", c.desc, i, got, want) 77 | } 78 | } 79 | } 80 | } 81 | } 82 | 83 | func TestGetConfigFilePath(t *testing.T) { 84 | cases := []struct { 85 | desc string 86 | os string 87 | pgVersion string 88 | files []string 89 | wantFile string 90 | shouldErr bool 91 | }{ 92 | { 93 | desc: "mac - yes", 94 | os: osMac, 95 | files: []string{fileNameMac}, 96 | wantFile: fileNameMac, 97 | shouldErr: false, 98 | }, 99 | { 100 | desc: "mac - no", 101 | os: osMac, 102 | files: []string{"/etc"}, 103 | wantFile: "", 104 | shouldErr: true, 105 | }, 106 | { 107 | desc: "linux - pg10+debian", 108 | os: osLinux, 109 | pgVersion: pgutils.MajorVersion10, 110 | files: []string{fmt.Sprintf(fileNameDebianFmt, "10")}, 111 | wantFile: fmt.Sprintf(fileNameDebianFmt, "10"), 112 | shouldErr: false, 113 | }, 114 | { 115 | desc: "linux - pg9.6+debian", 116 | os: osLinux, 117 | pgVersion: pgutils.MajorVersion96, 118 | files: []string{fmt.Sprintf(fileNameDebianFmt, "9.6")}, 119 | wantFile: fmt.Sprintf(fileNameDebianFmt, "9.6"), 120 | shouldErr: false, 121 | }, 122 | { 123 | desc: "linux - mismatch+debian", 124 | os: osLinux, 125 | pgVersion: pgutils.MajorVersion96, 126 | files: []string{fmt.Sprintf(fileNameDebianFmt, "10")}, 127 | wantFile: "", 128 | shouldErr: true, 129 | }, 130 | { 131 | desc: "linux - pg10+rpm", 132 | os: osLinux, 133 | pgVersion: pgutils.MajorVersion10, 134 | files: []string{fmt.Sprintf(fileNameRPMFmt, "10")}, 135 | wantFile: fmt.Sprintf(fileNameRPMFmt, "10"), 136 | shouldErr: false, 137 | }, 138 | { 139 | desc: "linux - pg9.6+rpm", 140 | os: osLinux, 141 | pgVersion: pgutils.MajorVersion96, 142 | files: []string{fmt.Sprintf(fileNameDebianFmt, "9.6")}, 143 | wantFile: fmt.Sprintf(fileNameDebianFmt, "9.6"), 144 | shouldErr: false, 145 | }, 146 | 147 | { 148 | desc: "linux - mismatch+rpm", 149 | os: osLinux, 150 | pgVersion: pgutils.MajorVersion96, 151 | files: []string{fmt.Sprintf(fileNameRPMFmt, "10")}, 152 | wantFile: "", 153 | shouldErr: true, 154 | }, 155 | { 156 | desc: "linux - arch", 157 | os: osLinux, 158 | files: []string{fileNameArch}, 159 | wantFile: fileNameArch, 160 | shouldErr: false, 161 | }, 162 | { 163 | desc: "linux - alpine", 164 | os: osLinux, 165 | files: []string{fileNameAlpine}, 166 | wantFile: fileNameAlpine, 167 | shouldErr: false, 168 | }, 169 | 170 | { 171 | desc: "linux - no", 172 | os: osLinux, 173 | files: []string{fmt.Sprintf(fileNameDebianFmt, "9.0")}, 174 | wantFile: "", 175 | shouldErr: true, 176 | }, 177 | } 178 | 179 | oldOSStatFn := osStatFn 180 | for _, c := range cases { 181 | osStatFn = func(fn string) (os.FileInfo, error) { 182 | for _, s := range c.files { 183 | if fn == s { 184 | return nil, nil 185 | } 186 | } 187 | return nil, os.ErrNotExist 188 | } 189 | filename, err := getConfigFilePath(c.os, c.pgVersion) 190 | if err != nil && !c.shouldErr { 191 | t.Errorf("%s: unexpected error: %v", c.desc, err) 192 | } else if err == nil && c.shouldErr { 193 | t.Errorf("%s: unexpected lack of error", c.desc) 194 | } 195 | 196 | if c.shouldErr && filename != "" { 197 | t.Errorf("%s: unexpected filename in error case: got %s", c.desc, filename) 198 | } 199 | 200 | if got := filename; got != c.wantFile { 201 | t.Errorf("%s: incorrect filename: got %s want %s", c.desc, got, c.wantFile) 202 | } 203 | } 204 | osStatFn = oldOSStatFn 205 | } 206 | 207 | func newConfigFileStateFromSlice(t *testing.T, lines []string) *configFileState { 208 | r := stringSliceToBytesReader(lines) 209 | cfs, err := getConfigFileState(r) 210 | if err != nil { 211 | t.Fatalf("could not parse config lines: %v\nlines: %v", err, lines) 212 | } 213 | 214 | return cfs 215 | } 216 | 217 | func TestGetConfigFileState(t *testing.T) { 218 | sharedLibLine := "shared_preload_libraries = 'timescaledb' # comment" 219 | memoryLine := "#shared_buffers = 64MB" 220 | walLine := "min_wal_size = 0GB # weird" 221 | cases := []struct { 222 | desc string 223 | lines []string 224 | want *configFileState 225 | }{ 226 | { 227 | desc: "empty file", 228 | lines: []string{}, 229 | want: &configFileState{ 230 | lines: []*configLine{}, 231 | tuneParseResults: make(map[string]*tunableParseResult), 232 | sharedLibResult: nil, 233 | }, 234 | }, 235 | { 236 | desc: "single irrelevant line", 237 | lines: []string{"foo"}, 238 | want: &configFileState{ 239 | lines: []*configLine{{content: "foo"}}, 240 | tuneParseResults: make(map[string]*tunableParseResult), 241 | sharedLibResult: nil, 242 | }, 243 | }, 244 | { 245 | desc: "shared lib line only", 246 | lines: []string{sharedLibLine}, 247 | want: &configFileState{ 248 | lines: []*configLine{{content: sharedLibLine}}, 249 | tuneParseResults: make(map[string]*tunableParseResult), 250 | sharedLibResult: &sharedLibResult{ 251 | idx: 0, 252 | commented: false, 253 | hasTimescale: true, 254 | commentGroup: "", 255 | libs: "timescaledb", 256 | }, 257 | }, 258 | }, 259 | { 260 | desc: "multi-line", 261 | lines: []string{"foo", sharedLibLine, "bar", memoryLine, walLine, "baz"}, 262 | want: &configFileState{ 263 | lines: []*configLine{ 264 | {content: "foo"}, 265 | {content: sharedLibLine}, 266 | {content: "bar"}, 267 | {content: memoryLine}, 268 | {content: walLine}, 269 | {content: "baz"}, 270 | }, 271 | tuneParseResults: map[string]*tunableParseResult{ 272 | pgtune.SharedBuffersKey: { 273 | idx: 3, 274 | commented: true, 275 | key: pgtune.SharedBuffersKey, 276 | value: "64MB", 277 | extra: "", 278 | }, 279 | pgtune.MinWALKey: { 280 | idx: 4, 281 | commented: false, 282 | key: pgtune.MinWALKey, 283 | value: "0GB", 284 | extra: " # weird", 285 | }, 286 | }, 287 | sharedLibResult: &sharedLibResult{ 288 | idx: 1, 289 | commented: false, 290 | hasTimescale: true, 291 | commentGroup: "", 292 | libs: "timescaledb", 293 | }, 294 | }, 295 | }, 296 | } 297 | 298 | for _, c := range cases { 299 | cfs := newConfigFileStateFromSlice(t, c.lines) 300 | if got := len(cfs.lines); got != len(c.want.lines) { 301 | t.Errorf("%s: incorrect number of cfs lines: got %d want %d", c.desc, got, len(c.want.lines)) 302 | } else { 303 | for i, got := range cfs.lines { 304 | if want := c.want.lines[i].content; got.content != want { 305 | t.Errorf("%s: incorrect line at %d: got\n%s\nwant\n%s", c.desc, i, got.content, want) 306 | } 307 | } 308 | } 309 | 310 | if c.want.sharedLibResult != nil { 311 | if cfs.sharedLibResult == nil { 312 | t.Errorf("%s: unexpected nil shared lib result", c.desc) 313 | } else { 314 | want := fmt.Sprintf("%v", c.want.sharedLibResult) 315 | if got := fmt.Sprintf("%v", cfs.sharedLibResult); got != want { 316 | t.Errorf("%s: incorrect sharedLibResult: got %s want %s", c.desc, got, want) 317 | } 318 | } 319 | } 320 | 321 | if len(c.want.tuneParseResults) > 0 { 322 | if got := len(cfs.tuneParseResults); got != len(c.want.tuneParseResults) { 323 | t.Errorf("%s: incorrect tuneParseResults size: got %d want %d", c.desc, got, len(c.want.tuneParseResults)) 324 | } else { 325 | for k, v := range c.want.tuneParseResults { 326 | want := fmt.Sprintf("%v", v) 327 | if got, ok := cfs.tuneParseResults[k]; fmt.Sprintf("%v", got) != want || !ok { 328 | t.Errorf("%s: incorrect tuneParseResults for %s: got %s want %s", c.desc, k, fmt.Sprintf("%v", got), want) 329 | } 330 | } 331 | } 332 | } 333 | } 334 | } 335 | 336 | type errReader struct { 337 | count uint64 338 | } 339 | 340 | func (r *errReader) Read(p []byte) (int, error) { 341 | if r.count > 1 { 342 | return 0, fmt.Errorf("erroring") 343 | } 344 | p[len(p)-1] = '\n' 345 | r.count++ 346 | return 1, nil 347 | } 348 | 349 | func TestGetConfigFileStateErr(t *testing.T) { 350 | r := &errReader{} 351 | cfs, err := getConfigFileState(r) 352 | if cfs != nil { 353 | t.Errorf("cfs not nil: %v", cfs) 354 | } 355 | if err == nil { 356 | t.Errorf("err is nil") 357 | } 358 | } 359 | 360 | const errProcess = "process error" 361 | 362 | type countProcessor struct { 363 | count int 364 | shouldErr bool 365 | } 366 | 367 | func (p *countProcessor) Process(_ *configLine) error { 368 | if p.shouldErr { 369 | return fmt.Errorf(errProcess) 370 | } 371 | p.count++ 372 | return nil 373 | } 374 | 375 | func TestConfigFileStateProcessLines(t *testing.T) { 376 | countProc1 := &countProcessor{} 377 | countProc2 := &countProcessor{} 378 | procs := []configLineProcessor{countProc1, countProc2} 379 | lines := []string{"foo", "bar", "baz"} 380 | wantCount := len(lines) 381 | 382 | cfs := newConfigFileStateFromSlice(t, lines) 383 | err := cfs.ProcessLines(procs...) 384 | if err != nil { 385 | t.Errorf("unexpected error in processing: %v", err) 386 | } 387 | if got := countProc1.count; got != wantCount { 388 | t.Errorf("incorrect count for countProc1: got %d want %d", got, wantCount) 389 | } 390 | if got := countProc2.count; got != wantCount { 391 | t.Errorf("incorrect count for countProc2: got %d want %d", got, wantCount) 392 | } 393 | 394 | badCountProc := &countProcessor{shouldErr: true} 395 | procs = append(procs, badCountProc) 396 | err = cfs.ProcessLines(procs...) 397 | if err == nil { 398 | t.Errorf("unexpected lack of error") 399 | } 400 | if got := err.Error(); got != errProcess { 401 | t.Errorf("unexpected error: got %s want %s", got, errProcess) 402 | } 403 | } 404 | 405 | const ( 406 | errTestTruncate = "truncate error" 407 | errTestSeek = "seek error" 408 | ) 409 | 410 | type testTruncateWriter struct { 411 | *testWriter 412 | seekErr bool 413 | truncateErr bool 414 | } 415 | 416 | func (w *testTruncateWriter) Seek(_ int64, _ int) (int64, error) { 417 | if w.seekErr { 418 | return 0, fmt.Errorf(errTestSeek) 419 | } 420 | return 0, nil 421 | } 422 | 423 | func (w *testTruncateWriter) Truncate(_ int64) error { 424 | if w.truncateErr { 425 | return fmt.Errorf(errTestTruncate) 426 | } 427 | return nil 428 | } 429 | 430 | func TestConfigFileStateWriteTo(t *testing.T) { 431 | cases := []struct { 432 | desc string 433 | lines []string 434 | removeIdx int 435 | errMsg string 436 | w io.Writer 437 | }{ 438 | { 439 | desc: "empty", 440 | lines: []string{}, 441 | removeIdx: -1, 442 | w: &testWriter{false, []string{}}, 443 | }, 444 | { 445 | desc: "one line", 446 | lines: []string{"foo"}, 447 | removeIdx: -1, 448 | w: &testWriter{false, []string{}}, 449 | }, 450 | { 451 | desc: "many lines", 452 | lines: []string{"foo", "bar", "baz", "quaz"}, 453 | removeIdx: -1, 454 | w: &testWriter{false, []string{}}, 455 | }, 456 | { 457 | desc: "many lines w/ truncating", 458 | lines: []string{"foo", "bar", "baz", "quaz"}, 459 | removeIdx: -1, 460 | w: &testTruncateWriter{&testWriter{false, []string{}}, false, false}, 461 | }, 462 | { 463 | desc: "many lines, remove middle line", 464 | lines: []string{"foo", "bar", "baz"}, 465 | removeIdx: 1, 466 | w: &testWriter{false, []string{}}, 467 | }, 468 | { 469 | desc: "error in truncate", 470 | lines: []string{"foo"}, 471 | removeIdx: -1, 472 | errMsg: errTestTruncate, 473 | w: &testTruncateWriter{&testWriter{true, []string{}}, false, true}, 474 | }, 475 | { 476 | desc: "error in seek", 477 | lines: []string{"foo"}, 478 | removeIdx: -1, 479 | errMsg: errTestSeek, 480 | w: &testTruncateWriter{&testWriter{true, []string{}}, true, false}, 481 | }, 482 | { 483 | desc: "error in write w/o truncating", 484 | lines: []string{"foo"}, 485 | removeIdx: -1, 486 | errMsg: errTestWriter, 487 | w: &testWriter{true, []string{}}, 488 | }, 489 | { 490 | desc: "error in write w/ truncating", 491 | lines: []string{"foo"}, 492 | removeIdx: -1, 493 | errMsg: errTestWriter, 494 | w: &testTruncateWriter{&testWriter{true, []string{}}, false, false}, 495 | }, 496 | } 497 | 498 | for _, c := range cases { 499 | cfs := newConfigFileStateFromSlice(t, c.lines) 500 | if c.removeIdx >= 0 { 501 | cfs.lines[c.removeIdx].remove = true 502 | } 503 | 504 | _, err := cfs.WriteTo(c.w) 505 | if c.errMsg == "" && err != nil { 506 | t.Errorf("%s: unexpected error: %v", c.desc, err) 507 | } else if c.errMsg != "" { 508 | if err == nil { 509 | t.Errorf("%s: unexpected lack of error", c.desc) 510 | } else if got := err.Error(); got != c.errMsg { 511 | t.Errorf("%s: unexpected type of error: %v", c.desc, err) 512 | } 513 | } 514 | 515 | var w *testWriter 516 | switch temp := c.w.(type) { 517 | case *testWriter: 518 | w = temp 519 | case *testTruncateWriter: 520 | w = temp.testWriter 521 | } 522 | 523 | lineCntModifier := 0 524 | if c.removeIdx >= 0 { 525 | lineCntModifier = 1 526 | } 527 | 528 | if len(c.lines) > 0 && c.errMsg == "" { 529 | if got := len(w.lines); got != len(c.lines)-lineCntModifier { 530 | t.Errorf("%s: incorrect output len: got %d want %d", c.desc, got, len(c.lines)-lineCntModifier) 531 | } 532 | idxModifier := 0 533 | for i, want := range c.lines { 534 | if i == c.removeIdx { 535 | idxModifier = 1 536 | continue 537 | } 538 | if got := w.lines[i-idxModifier]; got != want+"\n" { 539 | t.Errorf("%s: incorrect line at %d: got %s want %s", c.desc, i, got, want+"\n") 540 | } 541 | } 542 | } 543 | } 544 | } 545 | -------------------------------------------------------------------------------- /pkg/tstune/io_handler.go: -------------------------------------------------------------------------------- 1 | package tstune 2 | 3 | import ( 4 | "bufio" 5 | "io" 6 | "os" 7 | ) 8 | 9 | const exitLabel = "exit" 10 | 11 | var exitFn = os.Exit 12 | 13 | // ioHandler manages the reading and writing for a Tuner 14 | type ioHandler struct { 15 | p printer // handles output 16 | br *bufio.Reader // handles input 17 | out io.Writer 18 | outErr io.Writer 19 | } 20 | 21 | func (h *ioHandler) exit(errCode int, format string, args ...interface{}) { 22 | h.p.Error(exitLabel, format, args...) 23 | exitFn(errCode) 24 | } 25 | 26 | func (h *ioHandler) errorExit(err error) { 27 | h.exit(1, err.Error()) 28 | } 29 | -------------------------------------------------------------------------------- /pkg/tstune/io_handler_test.go: -------------------------------------------------------------------------------- 1 | package tstune 2 | 3 | import ( 4 | "fmt" 5 | "testing" 6 | ) 7 | 8 | const errTestWriter = "error on write" 9 | 10 | type testWriter struct { 11 | shouldErr bool 12 | lines []string 13 | } 14 | 15 | func (w *testWriter) Write(buf []byte) (int, error) { 16 | if w.shouldErr { 17 | return 0, fmt.Errorf(errTestWriter) 18 | } 19 | w.lines = append(w.lines, string(buf)) 20 | return 0, nil 21 | } 22 | 23 | func TestIOHandlerExit(t *testing.T) { 24 | p := &testPrinter{} 25 | 26 | oldExitFn := exitFn 27 | exitCalls := 0 28 | exitLastCode := -1 29 | exitFn = func(code int) { 30 | exitCalls++ 31 | exitLastCode = code 32 | } 33 | 34 | handler := &ioHandler{p, nil, nil, nil} 35 | for i := 0; i < 100; i++ { 36 | handler.exit(i*2, "bye") 37 | if got := p.errorCalls; got != uint64(i+1) { 38 | t.Errorf("incorrect number of error calls: got %d want %d", got, i+1) 39 | } 40 | if got := exitCalls; got != i+1 { 41 | t.Errorf("incorrect number of exit calls: got %d want %d", got, i+1) 42 | } 43 | if got := exitLastCode; got != i*2 { 44 | t.Errorf("incorrect last code: got %d want %d", got, i*2) 45 | } 46 | 47 | want := fmt.Sprintf(exitLabel + ": bye") 48 | if got := p.errors[len(p.errors)-1]; got != want { 49 | t.Errorf("incorrect error: got %v want %v", got, want) 50 | } 51 | } 52 | 53 | exitFn = oldExitFn 54 | } 55 | 56 | func TestIOHandlerErrorExit(t *testing.T) { 57 | p := &testPrinter{} 58 | 59 | oldExitFn := exitFn 60 | exitCalls := 0 61 | exitLastCode := -1 62 | exitFn = func(code int) { 63 | exitCalls++ 64 | exitLastCode = code 65 | } 66 | 67 | handler := &ioHandler{p, nil, nil, nil} 68 | for i := 0; i < 100; i++ { 69 | handler.errorExit(fmt.Errorf("error %d", i*3)) 70 | if got := p.errorCalls; got != uint64(i+1) { 71 | t.Errorf("incorrect number of error calls: got %d want %d", got, i+1) 72 | } 73 | if got := exitCalls; got != i+1 { 74 | t.Errorf("incorrect number of exit calls: got %d want %d", got, i+1) 75 | } 76 | if got := exitLastCode; got != 1 { 77 | t.Errorf("incorrect last code: got %d want %d", got, 1) 78 | } 79 | want := fmt.Sprintf(exitLabel+": error %d", i*3) 80 | if got := p.errors[len(p.errors)-1]; got != want { 81 | t.Errorf("incorrect error: got %v want %v", got, want) 82 | } 83 | } 84 | 85 | exitFn = oldExitFn 86 | } 87 | -------------------------------------------------------------------------------- /pkg/tstune/print.go: -------------------------------------------------------------------------------- 1 | package tstune 2 | 3 | import ( 4 | "fmt" 5 | "io" 6 | "strings" 7 | 8 | "github.com/fatih/color" 9 | ) 10 | 11 | const ( 12 | successLabel = "success: " 13 | noColorPrefixStatement = "== " 14 | noColorPrefixPrompt = "-- " 15 | ) 16 | 17 | var ( 18 | statementColor = color.New(color.FgWhite, color.Bold) // color for directions / statements 19 | promptColor = color.New(color.FgMagenta, color.Bold) // color for prompt/questions requiring user input 20 | successColor = color.New(color.FgGreen, color.Bold) 21 | errorColor = color.New(color.FgRed, color.Bold) 22 | ) 23 | 24 | type printer interface { 25 | Statement(string, ...interface{}) 26 | Prompt(string, ...interface{}) 27 | Success(string, ...interface{}) 28 | Error(string, string, ...interface{}) 29 | } 30 | 31 | type colorPrinter struct { 32 | w io.Writer 33 | } 34 | 35 | func (p *colorPrinter) printWithColor(c *color.Color, format string, args ...interface{}) { 36 | color.NoColor = false 37 | c.Fprintf(p.w, format, args...) 38 | } 39 | 40 | func (p *colorPrinter) Statement(format string, args ...interface{}) { 41 | p.printWithColor(statementColor, format+"\n", args...) 42 | } 43 | 44 | func (p *colorPrinter) Prompt(format string, args ...interface{}) { 45 | p.printWithColor(promptColor, format, args...) 46 | } 47 | 48 | func (p *colorPrinter) Success(format string, args ...interface{}) { 49 | p.printWithColor(successColor, successLabel) 50 | fmt.Fprintf(p.w, format+"\n", args...) 51 | } 52 | 53 | func (p *colorPrinter) Error(label string, format string, args ...interface{}) { 54 | p.printWithColor(errorColor, label+": ") 55 | fmt.Fprintf(p.w, format+"\n", args...) 56 | } 57 | 58 | type noColorPrinter struct { 59 | w io.Writer 60 | } 61 | 62 | func (p *noColorPrinter) Statement(format string, args ...interface{}) { 63 | fmt.Fprintf(p.w, noColorPrefixStatement+format+"\n", args...) 64 | } 65 | 66 | func (p *noColorPrinter) Prompt(format string, args ...interface{}) { 67 | fmt.Fprintf(p.w, noColorPrefixPrompt+format, args...) 68 | } 69 | 70 | func (p *noColorPrinter) Success(format string, args ...interface{}) { 71 | fmt.Fprintf(p.w, strings.ToUpper(successLabel)+format+"\n", args...) 72 | } 73 | 74 | func (p *noColorPrinter) Error(label string, format string, args ...interface{}) { 75 | fmt.Fprintf(p.w, strings.ToUpper(label)+": "+format+"\n", args...) 76 | } 77 | -------------------------------------------------------------------------------- /pkg/tstune/print_test.go: -------------------------------------------------------------------------------- 1 | package tstune 2 | 3 | import ( 4 | "bytes" 5 | "fmt" 6 | "strings" 7 | "testing" 8 | ) 9 | 10 | type testPrinter struct { 11 | statementCalls uint64 12 | statements []string 13 | promptCalls uint64 14 | prompts []string 15 | successCalls uint64 16 | successes []string 17 | errorCalls uint64 18 | errors []string 19 | } 20 | 21 | func (p *testPrinter) Statement(format string, args ...interface{}) { 22 | p.statementCalls++ 23 | p.statements = append(p.statements, fmt.Sprintf(format, args...)) 24 | } 25 | 26 | func (p *testPrinter) Prompt(format string, args ...interface{}) { 27 | p.promptCalls++ 28 | p.prompts = append(p.prompts, fmt.Sprintf(format, args...)) 29 | } 30 | 31 | func (p *testPrinter) Success(format string, args ...interface{}) { 32 | p.successCalls++ 33 | p.successes = append(p.successes, fmt.Sprintf(format, args...)) 34 | } 35 | 36 | func (p *testPrinter) Error(label string, format string, args ...interface{}) { 37 | p.errorCalls++ 38 | p.errors = append(p.errors, fmt.Sprintf(label+": "+format, args...)) 39 | } 40 | 41 | const whiteBoldSeq = "\x1b[37;1m" 42 | const purpleBoldSeq = "\x1b[35;1m" 43 | const greenBoldSeq = "\x1b[32;1m" 44 | const redBoldSeq = "\x1b[31;1m" 45 | const resetSeq = "\x1b[0m" 46 | 47 | func TestColorPrinterStatement(t *testing.T) { 48 | var buf bytes.Buffer 49 | p := &colorPrinter{&buf} 50 | stmt := "This is a statement with %d" 51 | p.Statement(stmt, 1) 52 | want := whiteBoldSeq + fmt.Sprintf(stmt+"\n", 1) + resetSeq 53 | if got := string(buf.Bytes()); got != want { 54 | t.Errorf("incorrect statement: got\n%s\nwant\n%s", got, want) 55 | } 56 | } 57 | 58 | func TestColorPrinterPrompt(t *testing.T) { 59 | var buf bytes.Buffer 60 | p := &colorPrinter{&buf} 61 | stmt := "This is a prompt with %d" 62 | p.Prompt(stmt, 1) 63 | want := purpleBoldSeq + fmt.Sprintf(stmt, 1) + resetSeq 64 | if got := string(buf.Bytes()); got != want { 65 | t.Errorf("incorrect prompt: got\n%s\nwant\n%s", got, want) 66 | } 67 | } 68 | 69 | func TestColorPrinterSuccess(t *testing.T) { 70 | var buf bytes.Buffer 71 | p := &colorPrinter{&buf} 72 | stmt := "This is a success with %d" 73 | p.Success(stmt, 1) 74 | want := greenBoldSeq + successLabel + resetSeq + fmt.Sprintf(stmt+"\n", 1) 75 | if got := string(buf.Bytes()); got != want { 76 | t.Errorf("incorrect success: got\n%s\nwant\n%s", got, want) 77 | } 78 | } 79 | 80 | func TestColorPrinterError(t *testing.T) { 81 | var buf bytes.Buffer 82 | p := &colorPrinter{&buf} 83 | stmt := "This is a error with %d" 84 | label := "yikes" 85 | p.Error(label, stmt, 1) 86 | want := redBoldSeq + label + ": " + resetSeq + fmt.Sprintf(stmt+"\n", 1) 87 | if got := string(buf.Bytes()); got != want { 88 | t.Errorf("incorrect error: got\n%s\nwant\n%s", got, want) 89 | } 90 | } 91 | 92 | func TestNoColorPrinterStatement(t *testing.T) { 93 | var buf bytes.Buffer 94 | p := &noColorPrinter{&buf} 95 | stmt := "This is a statement with %d" 96 | p.Statement(stmt, 1) 97 | want := noColorPrefixStatement + fmt.Sprintf(stmt+"\n", 1) 98 | if got := string(buf.Bytes()); got != want { 99 | t.Errorf("incorrect statement: got\n%s\nwant\n%s", got, want) 100 | } 101 | } 102 | 103 | func TestNoColorPrinterPrompt(t *testing.T) { 104 | var buf bytes.Buffer 105 | p := &noColorPrinter{&buf} 106 | stmt := "This is a prompt with %d" 107 | p.Prompt(stmt, 1) 108 | want := noColorPrefixPrompt + fmt.Sprintf(stmt, 1) 109 | if got := string(buf.Bytes()); got != want { 110 | t.Errorf("incorrect prompt: got\n%s\nwant\n%s", got, want) 111 | } 112 | } 113 | 114 | func TestNoColorPrinterSuccess(t *testing.T) { 115 | var buf bytes.Buffer 116 | p := &noColorPrinter{&buf} 117 | stmt := "This is a success with %d" 118 | p.Success(stmt, 1) 119 | want := strings.ToUpper(successLabel) + fmt.Sprintf(stmt+"\n", 1) 120 | if got := string(buf.Bytes()); got != want { 121 | t.Errorf("incorrect success: got\n%s\nwant\n%s", got, want) 122 | } 123 | } 124 | 125 | func TestNoColorPrinterError(t *testing.T) { 126 | var buf bytes.Buffer 127 | p := &noColorPrinter{&buf} 128 | stmt := "This is a error with %d" 129 | label := "yikes" 130 | p.Error(label, stmt, 1) 131 | want := strings.ToUpper(label) + ": " + fmt.Sprintf(stmt+"\n", 1) 132 | if got := string(buf.Bytes()); got != want { 133 | t.Errorf("incorrect error: got\n%s\nwant\n%s", got, want) 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /pkg/tstune/shared_preload_libs.go: -------------------------------------------------------------------------------- 1 | package tstune 2 | 3 | import ( 4 | "regexp" 5 | "strings" 6 | ) 7 | 8 | const extName = "timescaledb" 9 | 10 | var sharedRegex = regexp.MustCompile("(#+?\\s*)?shared_preload_libraries = '(.*?)'.*") 11 | 12 | // sharedLibResult holds the results of extracting/parsing the shared_preload_libraries 13 | // line of a postgresql.conf file. 14 | type sharedLibResult struct { 15 | idx int // the line index where this result was parsed 16 | commented bool // whether the line is currently commented out (i.e., prepended by #) 17 | hasTimescale bool // whether 'timescaledb' appears in the list of libraries 18 | commentGroup string // the combination of # + spaces that appear before the key / value 19 | libs string // the string value of the libraries currently set in the config file 20 | } 21 | 22 | // parseLineForSharedLibResult attempts to parse a line of the config file by 23 | // matching it against the shared_preload_libraries regex. If the line is 24 | // parseable by the regex, then the representation of that line is returned; 25 | // otherwise, nil. 26 | func parseLineForSharedLibResult(line string) *sharedLibResult { 27 | res := sharedRegex.FindStringSubmatch(line) 28 | if len(res) > 0 { 29 | return &sharedLibResult{ 30 | commented: len(res[1]) > 0, 31 | hasTimescale: strings.Contains(res[2], extName), 32 | commentGroup: res[1], 33 | libs: res[2], 34 | } 35 | } 36 | return nil 37 | } 38 | 39 | // updateSharedLibLine takes a given line that matched the shared_preload_libraries 40 | // regex and updates it to validly include the 'timescaledb' extension. 41 | func updateSharedLibLine(line string, parseResult *sharedLibResult) string { 42 | res := line 43 | if parseResult.commented { 44 | res = strings.Replace(res, parseResult.commentGroup, "", 1) 45 | } 46 | 47 | if parseResult.hasTimescale { 48 | return res 49 | } 50 | newLibsVal := "= '" 51 | if len(parseResult.libs) > 0 { 52 | newLibsVal += parseResult.libs + "," 53 | } 54 | newLibsVal += extName + "'" 55 | replaceVal := "= '" + parseResult.libs + "'" 56 | res = strings.Replace(res, replaceVal, newLibsVal, 1) 57 | 58 | return res 59 | } 60 | -------------------------------------------------------------------------------- /pkg/tstune/shared_preload_libs_test.go: -------------------------------------------------------------------------------- 1 | package tstune 2 | 3 | import "testing" 4 | 5 | func TestParseLineForSharedLibResult(t *testing.T) { 6 | cases := []struct { 7 | desc string 8 | input string 9 | want *sharedLibResult 10 | }{ 11 | { 12 | desc: "initial config value", 13 | input: "#shared_preload_libraries = '' # (change requires restart)", 14 | want: &sharedLibResult{ 15 | commented: true, 16 | hasTimescale: false, 17 | libs: "", 18 | }, 19 | }, 20 | { 21 | desc: "extra commented out", 22 | input: "###shared_preload_libraries = '' # (change requires restart)", 23 | want: &sharedLibResult{ 24 | commented: true, 25 | hasTimescale: false, 26 | libs: "", 27 | }, 28 | }, 29 | { 30 | desc: "commented with space after", 31 | input: "# shared_preload_libraries = '' # (change requires restart)", 32 | want: &sharedLibResult{ 33 | commented: true, 34 | hasTimescale: false, 35 | libs: "", 36 | }, 37 | }, 38 | { 39 | desc: "extra commented with space after", 40 | input: "## shared_preload_libraries = '' # (change requires restart)", 41 | want: &sharedLibResult{ 42 | commented: true, 43 | hasTimescale: false, 44 | libs: "", 45 | }, 46 | }, 47 | { 48 | desc: "initial config value, uncommented", 49 | input: "shared_preload_libraries = '' # (change requires restart)", 50 | want: &sharedLibResult{ 51 | commented: false, 52 | hasTimescale: false, 53 | libs: "", 54 | }, 55 | }, 56 | { 57 | desc: "initial config value, uncommented with leading space", 58 | input: " shared_preload_libraries = '' # (change requires restart)", 59 | want: &sharedLibResult{ 60 | commented: false, 61 | hasTimescale: false, 62 | libs: "", 63 | }, 64 | }, 65 | { 66 | desc: "timescaledb already there but commented", 67 | input: "#shared_preload_libraries = 'timescaledb' # (change requires restart)", 68 | want: &sharedLibResult{ 69 | commented: true, 70 | hasTimescale: true, 71 | libs: "timescaledb", 72 | }, 73 | }, 74 | { 75 | desc: "other libraries besides timescaledb, commented", 76 | input: "#shared_preload_libraries = 'pg_stats' # (change requires restart) ", 77 | want: &sharedLibResult{ 78 | commented: true, 79 | hasTimescale: false, 80 | libs: "pg_stats", 81 | }, 82 | }, 83 | { 84 | desc: "no string after the quotes", 85 | input: "shared_preload_libraries = 'pg_stats,timescaledb'", 86 | want: &sharedLibResult{ 87 | commented: false, 88 | hasTimescale: true, 89 | libs: "pg_stats,timescaledb", 90 | }, 91 | }, 92 | { 93 | desc: "don't be greedy with things between single quotes", 94 | input: "#shared_preload_libraries = 'timescaledb' # comment with single quote ' test", 95 | want: &sharedLibResult{ 96 | commented: true, 97 | hasTimescale: true, 98 | libs: "timescaledb", 99 | }, 100 | }, 101 | { 102 | desc: "not shared preload line", 103 | input: "data_dir = '/path/to/data'", 104 | want: nil, 105 | }, 106 | } 107 | for _, c := range cases { 108 | res := parseLineForSharedLibResult(c.input) 109 | if res == nil && c.want != nil { 110 | t.Errorf("%s: result was unexpectedly nil: want %v", c.desc, c.want) 111 | } else if res != nil && c.want == nil { 112 | t.Errorf("%s: result was unexpectedly non-nil: got %v", c.desc, res) 113 | } else if c.want != nil { 114 | if got := res.commented; got != c.want.commented { 115 | t.Errorf("%s: incorrect commented: got %v want %v", c.desc, got, c.want.commented) 116 | } 117 | if got := res.hasTimescale; got != c.want.hasTimescale { 118 | t.Errorf("%s: incorrect hasTimescale: got %v want %v", c.desc, got, c.want.hasTimescale) 119 | } 120 | if got := res.libs; got != c.want.libs { 121 | t.Errorf("%s: incorrect libs: got %s want %s", c.desc, got, c.want.libs) 122 | } 123 | } 124 | } 125 | } 126 | 127 | func TestUpdateSharedLibLine(t *testing.T) { 128 | confKey := "shared_preload_libraries = " 129 | simpleOkayCase := confKey + "'" + extName + "'" 130 | simpleOkayCaseExtra := simpleOkayCase + " # (change requires restart)" 131 | cases := []struct { 132 | desc string 133 | original string 134 | want string 135 | }{ 136 | { 137 | desc: "original = ok", 138 | original: simpleOkayCase, 139 | want: simpleOkayCase, 140 | }, 141 | { 142 | desc: "original = ok w/ ending comments", 143 | original: simpleOkayCaseExtra, 144 | want: simpleOkayCaseExtra, 145 | }, 146 | { 147 | desc: "original = ok w/ prepended spaces", 148 | original: " " + simpleOkayCase, 149 | want: " " + simpleOkayCase, 150 | }, 151 | { 152 | desc: "just need to uncomment", 153 | original: "#" + simpleOkayCase, 154 | want: simpleOkayCase, 155 | }, 156 | { 157 | desc: "just need to uncomment w/ ending comments", 158 | original: "#" + simpleOkayCaseExtra, 159 | want: simpleOkayCaseExtra, 160 | }, 161 | { 162 | desc: "just need to uncomment multiple times", 163 | original: "###" + simpleOkayCase, 164 | want: simpleOkayCase, 165 | }, 166 | { 167 | desc: "uncomment + spaces", 168 | original: "### " + simpleOkayCase, 169 | want: simpleOkayCase, 170 | }, 171 | { 172 | desc: "needs to be added, empty list", 173 | original: confKey + "''", 174 | want: simpleOkayCase, 175 | }, 176 | { 177 | desc: "needs to be added, empty list, commented out", 178 | original: "#" + confKey + "''", 179 | want: simpleOkayCase, 180 | }, 181 | { 182 | desc: "needs to be added, empty list, trailing comment", 183 | original: confKey + "'' # (change requires restart)", 184 | want: simpleOkayCaseExtra, 185 | }, 186 | { 187 | desc: "needs to be added, one item", 188 | original: confKey + "'pg_stats'", 189 | want: confKey + "'pg_stats," + extName + "'", 190 | }, 191 | { 192 | desc: "needs to be added, t item, commented out", 193 | original: "#" + confKey + "'pg_stats,ext2'", 194 | want: confKey + "'pg_stats,ext2," + extName + "'", 195 | }, 196 | { 197 | desc: "needs to be added, two items", 198 | original: confKey + "'pg_stats'", 199 | want: confKey + "'pg_stats," + extName + "'", 200 | }, 201 | { 202 | desc: "needs to be added, two items, commented out", 203 | original: "#" + confKey + "'pg_stats,ext2'", 204 | want: confKey + "'pg_stats,ext2," + extName + "'", 205 | }, 206 | { 207 | desc: "in list with others", 208 | original: confKey + "'timescaledb,pg_stats'", 209 | want: confKey + "'timescaledb,pg_stats'", 210 | }, 211 | { 212 | desc: "in list with others, commented out", 213 | original: "#" + confKey + "'timescaledb,pg_stats'", 214 | want: confKey + "'timescaledb,pg_stats'", 215 | }, 216 | } 217 | 218 | for _, c := range cases { 219 | res := parseLineForSharedLibResult(c.original) 220 | if res == nil { 221 | t.Errorf("%s: parsing gave unexpected nil", c.desc) 222 | } 223 | got := updateSharedLibLine(c.original, res) 224 | if got != c.want { 225 | t.Errorf("%s: incorrect result: got\n%s\nwant\n%s", c.desc, got, c.want) 226 | } 227 | } 228 | } 229 | -------------------------------------------------------------------------------- /pkg/tstune/tune_settings.go: -------------------------------------------------------------------------------- 1 | package tstune 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "time" 7 | 8 | "github.com/timescale/timescaledb-tune/pkg/pgtune" 9 | ) 10 | 11 | // Names of parameters that this tuning tool will add to the conf file. 12 | const ( 13 | fmtOurParam = "%s = '%s'" 14 | lastTunedParam = "timescaledb.last_tuned" 15 | lastTunedVersionParam = "timescaledb.last_tuned_version" 16 | ) 17 | 18 | // ourParams is a list of parameters that the tuning program adds to the conf file 19 | var ourParams = []string{lastTunedParam, lastTunedVersionParam} 20 | 21 | // ourParamToValue returns the configuration file line for a given 22 | // timescaledb-tune parameter, e.g., timescaledb.last_tuned. 23 | func ourParamString(param string) string { 24 | var val string 25 | switch param { 26 | case lastTunedParam: 27 | val = time.Now().Format(time.RFC3339) 28 | case lastTunedVersionParam: 29 | val = Version 30 | default: 31 | panic("unknown param: " + param) 32 | } 33 | return fmt.Sprintf(fmtOurParam, param, val) 34 | } 35 | 36 | const ( 37 | // tuneRegexFmt is a regular expression that is used to match a line in the 38 | // conf file that just needs to be Sprintf'd with the key name. That is, its 39 | // usage is usually: 40 | // regex := fmt.Sprintf(tuneRegexFmt, "key_name") 41 | tuneRegexFmt = `^(\s*#+?\s*)?(%s)\s*=\s*(\S+?)(\s*(?:#.*|))$` 42 | // tuneRegexQuotedFmt is similar to the format above but for string parameters 43 | // that need single quotes around them 44 | tuneRegexQuotedFmt = `^(\s*#+?\s*)?(%s)\s*=\s*'(.+?)'(\s*(?:#.*|))$` 45 | ) 46 | 47 | var regexes = make(map[string]*regexp.Regexp) 48 | 49 | func init() { 50 | setup := func(arr []string) { 51 | for _, k := range arr { 52 | regexes[k] = keyToRegex(k) 53 | } 54 | } 55 | 56 | setup(pgtune.MemoryKeys) 57 | setup(pgtune.ParallelKeys) 58 | setup(pgtune.WALKeys) 59 | setup(pgtune.MiscKeys) 60 | setup(pgtune.BgwriterKeys) 61 | } 62 | 63 | // keyToRegex takes a conf file key/param name and creates the correct regular 64 | // expression. 65 | func keyToRegex(key string) *regexp.Regexp { 66 | return regexp.MustCompile(fmt.Sprintf(tuneRegexFmt, regexp.QuoteMeta(key))) 67 | } 68 | 69 | // keyToRegexQuoted takes a conf file key/param name and creates the correct 70 | // regular expression, assuming the values need to be single quoted. 71 | func keyToRegexQuoted(key string) *regexp.Regexp { 72 | return regexp.MustCompile(fmt.Sprintf(tuneRegexQuotedFmt, regexp.QuoteMeta(key))) 73 | } 74 | 75 | // parseWithRegex takes a line and attempts to parse it using a given regular 76 | // expression regex. The regex is expected to produce 5 capture groups: 77 | // 1) the full result, 2) whether the line is preceded by # or not, 3) the 78 | // parameter name/key, 4) the parameter value, and 5) any comments at the end. 79 | // If successful, a tunableParseResult is returned based on the contents of the 80 | // line; otherwise, nil. Panics if the regex parsing returns and unexpected 81 | // result (i.e., too many capture groups). 82 | func parseWithRegex(line string, regex *regexp.Regexp) *tunableParseResult { 83 | res := regex.FindStringSubmatch(line) 84 | if len(res) > 0 { 85 | if len(res) != 5 { 86 | panic(fmt.Sprintf("unexpected regex parse result: %v (len = %d)", res, len(res))) 87 | } 88 | 89 | return &tunableParseResult{ 90 | commented: len(res[1]) > 0, 91 | key: res[2], 92 | value: res[3], 93 | extra: res[4], 94 | } 95 | } 96 | return nil 97 | } 98 | -------------------------------------------------------------------------------- /pkg/tstune/tune_settings_test.go: -------------------------------------------------------------------------------- 1 | package tstune 2 | 3 | import ( 4 | "fmt" 5 | "regexp" 6 | "testing" 7 | "time" 8 | ) 9 | 10 | // To make the test less flaky, we 0 out the seconds to make the comparison 11 | // more likely to succeed. 12 | func removeSecsFromLastTuned(time string) string { 13 | runes := []rune(time) 14 | start := len(lastTunedParam + " = '") 15 | runes[start+17] = '0' 16 | runes[start+18] = '0' 17 | return string(runes) 18 | } 19 | 20 | func TestOurParamToValue(t *testing.T) { 21 | now := time.Now().Format(time.RFC3339) 22 | want := removeSecsFromLastTuned(fmt.Sprintf(fmtOurParam, lastTunedParam, now)) 23 | got := removeSecsFromLastTuned(ourParamString(lastTunedParam)) 24 | if got != want { 25 | t.Errorf("incorrect value for %s: got %s want %s", lastTunedParam, got, want) 26 | } 27 | 28 | want = fmt.Sprintf(fmtOurParam, lastTunedVersionParam, Version) 29 | got = ourParamString(lastTunedVersionParam) 30 | if got != want { 31 | t.Errorf("incorrect value for %s: got %s want %s", lastTunedVersionParam, got, want) 32 | } 33 | 34 | defer func() { 35 | if re := recover(); re == nil { 36 | t.Errorf("did not panic when should") 37 | } 38 | }() 39 | _ = ourParamString("not_a_real_param") 40 | } 41 | 42 | const ( 43 | testKey = "test_setting" 44 | testKeyMeta = "test.setting" 45 | testKeyMetaCorrect = "test\\.setting" 46 | ) 47 | 48 | func TestKeyToRegex(t *testing.T) { 49 | regex := keyToRegex(testKey) 50 | want := fmt.Sprintf(tuneRegexFmt, testKey) 51 | if got := regex.String(); got != want { 52 | t.Errorf("incorrect regex: got %s want %s", got, want) 53 | } 54 | 55 | regex = keyToRegex(testKeyMeta) 56 | want = fmt.Sprintf(tuneRegexFmt, testKeyMetaCorrect) 57 | if got := regex.String(); got != want { 58 | t.Errorf("incorrect regex (meta symbols): got %s want %s", got, want) 59 | } 60 | } 61 | 62 | func TestKeyToRegexQuoted(t *testing.T) { 63 | regex := keyToRegexQuoted(testKey) 64 | want := fmt.Sprintf(tuneRegexQuotedFmt, testKey) 65 | if got := regex.String(); got != want { 66 | t.Errorf("incorrect regex: got %s want %s", got, want) 67 | } 68 | 69 | regex = keyToRegexQuoted(testKeyMeta) 70 | want = fmt.Sprintf(tuneRegexQuotedFmt, testKeyMetaCorrect) 71 | if got := regex.String(); got != want { 72 | t.Errorf("incorrect regex (meta symbols): got %s want %s", got, want) 73 | } 74 | } 75 | 76 | var testRegex = keyToRegex(testKey) 77 | 78 | func TestParseWithRegex(t *testing.T) { 79 | cases := []struct { 80 | desc string 81 | input string 82 | want *tunableParseResult 83 | }{ 84 | { 85 | desc: "simple correct", 86 | input: testKey + " = 50.0", 87 | want: &tunableParseResult{ 88 | commented: false, 89 | key: testKey, 90 | value: "50.0", 91 | extra: "", 92 | }, 93 | }, 94 | { 95 | desc: "correct, no whitespace surrounding =", 96 | input: testKey + "=50.0", 97 | want: &tunableParseResult{ 98 | commented: false, 99 | key: testKey, 100 | value: "50.0", 101 | extra: "", 102 | }, 103 | }, 104 | { 105 | desc: "correct, much whitespace surrounding =", 106 | input: testKey + " = 50.0", 107 | want: &tunableParseResult{ 108 | commented: false, 109 | key: testKey, 110 | value: "50.0", 111 | extra: "", 112 | }, 113 | }, 114 | 115 | { 116 | desc: "correct, comment at end", 117 | input: testKey + " = 50.0 # do not change!", 118 | want: &tunableParseResult{ 119 | commented: false, 120 | key: testKey, 121 | value: "50.0", 122 | extra: " # do not change!", 123 | }, 124 | }, 125 | { 126 | desc: "correct, comment at end no space", 127 | input: testKey + " = 50.0# do not change!", 128 | want: &tunableParseResult{ 129 | commented: false, 130 | key: testKey, 131 | value: "50.0", 132 | extra: "# do not change!", 133 | }, 134 | }, 135 | { 136 | desc: "correct, comment at end more space", 137 | input: testKey + " = 50.0 # do not change!", 138 | want: &tunableParseResult{ 139 | commented: false, 140 | key: testKey, 141 | value: "50.0", 142 | extra: " # do not change!", 143 | }, 144 | }, 145 | { 146 | desc: "correct, comment at end tabs", 147 | input: testKey + " = 50.0 # do not change!", 148 | want: &tunableParseResult{ 149 | commented: false, 150 | key: testKey, 151 | value: "50.0", 152 | extra: " # do not change!", 153 | }, 154 | }, 155 | { 156 | desc: "correct, tabs at the end", 157 | input: testKey + " = 50.0 ", 158 | want: &tunableParseResult{ 159 | commented: false, 160 | key: testKey, 161 | value: "50.0", 162 | extra: " ", 163 | }, 164 | }, 165 | { 166 | desc: "simple correct, commented", 167 | input: "#" + testKey + " = 50.0", 168 | want: &tunableParseResult{ 169 | commented: true, 170 | key: testKey, 171 | value: "50.0", 172 | extra: "", 173 | }, 174 | }, 175 | { 176 | desc: "commented with spaces", 177 | input: " # " + testKey + " = 50.0", 178 | want: &tunableParseResult{ 179 | commented: true, 180 | key: testKey, 181 | value: "50.0", 182 | extra: "", 183 | }, 184 | }, 185 | { 186 | desc: "commented with ending comment", 187 | input: "# " + testKey + " = 50.0 # do not change", 188 | want: &tunableParseResult{ 189 | commented: true, 190 | key: testKey, 191 | value: "50.0", 192 | extra: " # do not change", 193 | }, 194 | }, 195 | { 196 | desc: "incorrect, do not accept comments with starting #", 197 | input: testKey + " = 50.0 do not change!", 198 | want: nil, 199 | }, 200 | } 201 | 202 | for _, c := range cases { 203 | res := parseWithRegex(c.input, testRegex) 204 | if res == nil && c.want != nil { 205 | t.Errorf("%s: result was unexpectedly nil: want %v", c.desc, c.want) 206 | } else if res != nil && c.want == nil { 207 | t.Errorf("%s: result was unexpectedly non-nil: got %v", c.desc, res) 208 | } else if c.want != nil { 209 | if got := res.commented; got != c.want.commented { 210 | t.Errorf("%s: incorrect commented: got %v want %v", c.desc, got, c.want.commented) 211 | } 212 | if got := res.key; got != c.want.key { 213 | t.Errorf("%s: incorrect key: got %v want %v", c.desc, got, c.want.key) 214 | } 215 | if got := res.value; got != c.want.value { 216 | t.Errorf("%s: incorrect value: got %s want %s", c.desc, got, c.want.value) 217 | } 218 | if got := res.extra; got != c.want.extra { 219 | t.Errorf("%s: incorrect extra: got %s want %s", c.desc, got, c.want.extra) 220 | } 221 | } 222 | } 223 | } 224 | 225 | func TestParseWithRegexPanic(t *testing.T) { 226 | // Don't use regexp.QuoteMeta so that we can sneak meta chars into and cause 227 | // an extra capture group, (bar)+ 228 | badRegex := regexp.MustCompile(fmt.Sprintf(tuneRegexFmt, "foo(bar)+")) 229 | line := "#foobar = 5 #commented" 230 | 231 | defer func() { 232 | if re := recover(); re == nil { 233 | t.Errorf("did not panic when should") 234 | } 235 | }() 236 | parseWithRegex(line, badRegex) 237 | } 238 | -------------------------------------------------------------------------------- /pkg/tstune/utils.go: -------------------------------------------------------------------------------- 1 | package tstune 2 | 3 | import ( 4 | "fmt" 5 | "math" 6 | "os" 7 | "path/filepath" 8 | 9 | "github.com/timescale/timescaledb-tune/pkg/pgutils" 10 | ) 11 | 12 | // ValidPGVersions is a slice representing the major versions of PostgreSQL 13 | // for which recommendations can be generated. 14 | var ValidPGVersions = []string{ 15 | pgutils.MajorVersion17, 16 | pgutils.MajorVersion16, 17 | pgutils.MajorVersion15, 18 | pgutils.MajorVersion14, 19 | pgutils.MajorVersion13, 20 | pgutils.MajorVersion12, 21 | pgutils.MajorVersion11, 22 | pgutils.MajorVersion10, 23 | pgutils.MajorVersion96, 24 | } 25 | 26 | // allows us to substitute mock versions in tests 27 | var getPGConfigVersionFn = pgutils.GetPGConfigVersionAtPath 28 | 29 | // allows us to substitute mock versions in tests 30 | var osStatFn = os.Stat 31 | 32 | // fileExists is a simple check for stating if a file exists and if any error 33 | // occurs it returns false. 34 | func fileExists(name string) bool { 35 | // for our purposes, any error is a problem, so assume it does not exist 36 | if _, err := osStatFn(name); err != nil { 37 | return false 38 | } 39 | return true 40 | } 41 | 42 | func pathIsDir(name string) bool { 43 | fi, err := osStatFn(name) 44 | // for our purposes, any error is a problem, so it is not a directory 45 | if err != nil { 46 | return false 47 | } 48 | return fi.IsDir() 49 | } 50 | 51 | // dirPathToFile will construct a full path if the given path 52 | // is a directory. This allows us to also accept directory paths for 53 | // well-known files (postgresql.conf, pg_config) 54 | func dirPathToFile(path string, defaultFilename string) string { 55 | if len(path) > 0 && pathIsDir(path) { 56 | return filepath.Join(path, defaultFilename) 57 | } 58 | return path 59 | } 60 | 61 | // isCloseEnough checks whether a provided value actual is within +/- the 62 | // fudge factor fudge of target. 63 | func isCloseEnough(actual, target, fudge float64) bool { 64 | return math.Abs((target-actual)/target) <= fudge 65 | } 66 | 67 | // isIn checks whether a given string s is inside the []string arr. 68 | func isIn(s string, arr []string) bool { 69 | for _, x := range arr { 70 | if s == x { 71 | return true 72 | } 73 | } 74 | return false 75 | } 76 | 77 | // validatePGMajorVersion tests whether majorVersion is a major version of 78 | // PostgreSQL that this package knows how to handle. 79 | func validatePGMajorVersion(majorVersion string) error { 80 | if !isIn(majorVersion, ValidPGVersions) { 81 | return fmt.Errorf(errUnsupportedMajorFmt, majorVersion) 82 | } 83 | return nil 84 | } 85 | 86 | // getPGMajorVersion extracts the major version of PostgreSQL according to 87 | // the output of pg_config located at binPath. It validates that it is a 88 | // version that tstune knows how to handle. 89 | func getPGMajorVersion(binPath string) (string, error) { 90 | version, err := getPGConfigVersionFn(binPath) 91 | if err != nil { 92 | return "", fmt.Errorf(errCouldNotExecuteFmt, binPath, err) 93 | } 94 | majorVersion, err := pgutils.ToPGMajorVersion(string(version)) 95 | if err != nil { 96 | return "", err 97 | } 98 | if err = validatePGMajorVersion(majorVersion); err != nil { 99 | return "", err 100 | } 101 | return majorVersion, nil 102 | } 103 | -------------------------------------------------------------------------------- /pkg/tstune/utils_test.go: -------------------------------------------------------------------------------- 1 | package tstune 2 | 3 | import ( 4 | "fmt" 5 | "math/rand" 6 | "os" 7 | "os/exec" 8 | "path/filepath" 9 | "testing" 10 | 11 | "github.com/timescale/timescaledb-tune/pkg/pgutils" 12 | ) 13 | 14 | func TestFileExists(t *testing.T) { 15 | existsName := "exists.txt" 16 | errorName := "error.txt" 17 | cases := []struct { 18 | desc string 19 | filename string 20 | want bool 21 | }{ 22 | { 23 | desc: "found file", 24 | filename: existsName, 25 | want: true, 26 | }, 27 | { 28 | desc: "not found file", 29 | filename: "ghost.txt", 30 | want: false, 31 | }, 32 | { 33 | desc: "error in stat", 34 | filename: errorName, 35 | want: false, 36 | }, 37 | } 38 | 39 | oldOSStatFn := osStatFn 40 | osStatFn = func(name string) (os.FileInfo, error) { 41 | if name == existsName { 42 | return nil, nil 43 | } else if name == errorName { 44 | return nil, fmt.Errorf("this is an error") 45 | } else { 46 | return nil, os.ErrNotExist 47 | } 48 | } 49 | 50 | for _, c := range cases { 51 | if got := fileExists(c.filename); got != c.want { 52 | t.Errorf("%s: incorrect result: got %v want %v", c.desc, got, c.want) 53 | } 54 | } 55 | 56 | osStatFn = oldOSStatFn 57 | } 58 | 59 | func TestDirPathToFile(t *testing.T) { 60 | currentDir := "." 61 | missingFile := "ghost.txt" 62 | defaultFile := "replace.txt" 63 | cases := []struct { 64 | desc string 65 | dirname string 66 | filename string 67 | want string 68 | }{ 69 | { 70 | desc: "augmented directory with default file", 71 | dirname: currentDir, 72 | filename: defaultFile, 73 | want: filepath.Join(currentDir, defaultFile), 74 | }, 75 | { 76 | desc: "could not find file - no action", 77 | dirname: missingFile, 78 | filename: defaultFile, 79 | want: missingFile, 80 | }, 81 | } 82 | 83 | for _, c := range cases { 84 | if got := dirPathToFile(c.dirname, c.filename); got != c.want { 85 | t.Errorf("%s: incorrect result: got %v want %v", c.desc, got, c.want) 86 | } 87 | } 88 | } 89 | 90 | func TestIsIn(t *testing.T) { 91 | limit := 1000 92 | arr := []string{} 93 | for i := 0; i < limit; i++ { 94 | arr = append(arr, fmt.Sprintf("str%d", i)) 95 | } 96 | 97 | // Should always be in the arr 98 | for i := 0; i < limit*10; i++ { 99 | s := fmt.Sprintf("str%d", rand.Intn(limit)) 100 | if !isIn(s, arr) { 101 | t.Errorf("should be in the arr: %s", s) 102 | } 103 | } 104 | 105 | // Should never be in the arr 106 | for i := 0; i < limit*10; i++ { 107 | s := fmt.Sprintf("str%d", limit+rand.Intn(limit)) 108 | if isIn(s, arr) { 109 | t.Errorf("should not be in the arr: %s", s) 110 | } 111 | } 112 | } 113 | 114 | func TestGetPGMajorVersion(t *testing.T) { 115 | okPath96 := "pg_config_9.6" 116 | okPath10 := "pg_config_10" 117 | okPath11 := "pg_config_11" 118 | okPath12 := "pg_config_12" 119 | okPath13 := "pg_config_13" 120 | okPath14 := "pg_config_14" 121 | okPath15 := "pg_config_15" 122 | okPath16 := "pg_config_16" 123 | okPath17 := "pg_config_17" 124 | okPath95 := "pg_config_9.5" 125 | okPath60 := "pg_config_6.0" 126 | cases := []struct { 127 | desc string 128 | binPath string 129 | want string 130 | errMsg string 131 | }{ 132 | { 133 | desc: "failed execute", 134 | binPath: "pg_config_bad", 135 | errMsg: fmt.Sprintf(errCouldNotExecuteFmt, "pg_config_bad", exec.ErrNotFound), 136 | }, 137 | { 138 | desc: "failed major parse", 139 | binPath: okPath60, 140 | errMsg: fmt.Sprintf("unknown major PG version: PostgreSQL 6.0.5"), 141 | }, 142 | { 143 | desc: "failed unsupported", 144 | binPath: okPath95, 145 | errMsg: fmt.Sprintf(errUnsupportedMajorFmt, "9.5"), 146 | }, 147 | { 148 | desc: "success 9.6", 149 | binPath: okPath96, 150 | want: pgutils.MajorVersion96, 151 | }, 152 | { 153 | desc: "success 10", 154 | binPath: okPath10, 155 | want: pgutils.MajorVersion10, 156 | }, 157 | { 158 | desc: "success 11", 159 | binPath: okPath11, 160 | want: pgutils.MajorVersion11, 161 | }, 162 | { 163 | desc: "success 12", 164 | binPath: okPath12, 165 | want: pgutils.MajorVersion12, 166 | }, 167 | { 168 | desc: "success 13", 169 | binPath: okPath13, 170 | want: pgutils.MajorVersion13, 171 | }, 172 | { 173 | desc: "success 14", 174 | binPath: okPath14, 175 | want: pgutils.MajorVersion14, 176 | }, 177 | } 178 | 179 | oldVersionFn := getPGConfigVersionFn 180 | getPGConfigVersionFn = func(binPath string) (string, error) { 181 | switch binPath { 182 | case okPath60: 183 | return "PostgreSQL 6.0.5", nil 184 | case okPath95: 185 | return "PostgreSQL 9.5.10", nil 186 | case okPath96: 187 | return "PostgreSQL 9.6.6", nil 188 | case okPath10: 189 | return "PostgreSQL 10.5 (Debian7)", nil 190 | case okPath11: 191 | return "PostgreSQL 11.1", nil 192 | case okPath12: 193 | return "PostgreSQL 12.4", nil 194 | case okPath13: 195 | return "PostgreSQL 13.2", nil 196 | case okPath14: 197 | return "PostgreSQL 14.0", nil 198 | case okPath15: 199 | return "PostgreSQL 15.0", nil 200 | case okPath16: 201 | return "PostgreSQL 16.0", nil 202 | case okPath17: 203 | return "PostgreSQL 17.0", nil 204 | default: 205 | return "", exec.ErrNotFound 206 | } 207 | } 208 | 209 | for _, c := range cases { 210 | got, err := getPGMajorVersion(c.binPath) 211 | if len(c.errMsg) == 0 { 212 | if err != nil { 213 | t.Errorf("%s: unexpected error: got %v", c.desc, err) 214 | } 215 | if got != c.want { 216 | t.Errorf("%s: incorrect major version: got %s want %s", c.desc, got, c.want) 217 | } 218 | } else { 219 | if err == nil { 220 | t.Errorf("%s: unexpected lack of error", c.desc) 221 | } 222 | if got := err.Error(); got != c.errMsg { 223 | t.Errorf("%s: incorrect error:\ngot\n%s\nwant\n%s", c.desc, got, c.errMsg) 224 | } 225 | } 226 | } 227 | 228 | getPGConfigVersionFn = oldVersionFn 229 | } 230 | 231 | func TestValidatePGMajorVersion(t *testing.T) { 232 | cases := map[string]bool{ 233 | pgutils.MajorVersion96: true, 234 | pgutils.MajorVersion10: true, 235 | pgutils.MajorVersion11: true, 236 | pgutils.MajorVersion12: true, 237 | pgutils.MajorVersion13: true, 238 | pgutils.MajorVersion14: true, 239 | pgutils.MajorVersion15: true, 240 | pgutils.MajorVersion16: true, 241 | pgutils.MajorVersion17: true, 242 | "9.5": false, 243 | "1.2.3": false, 244 | "9.6.6": false, 245 | "10.2": false, 246 | "11.0": false, 247 | } 248 | for majorVersion, valid := range cases { 249 | err := validatePGMajorVersion(majorVersion) 250 | if valid && err != nil { 251 | t.Errorf("unexpected error: got %v", err) 252 | } else if !valid { 253 | if err == nil { 254 | t.Errorf("unexpected lack of error") 255 | } 256 | want := fmt.Errorf(errUnsupportedMajorFmt, majorVersion).Error() 257 | if got := err.Error(); got != want { 258 | t.Errorf("unexpected error: got %v want %v", got, want) 259 | } 260 | } 261 | } 262 | } 263 | --------------------------------------------------------------------------------