├── .github └── workflows │ └── pages.yml ├── .gitignore ├── .pylintrc ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── config_from_pg ├── __init__.py ├── create_pgconfig_settings_view.sql ├── db.py └── generate.py ├── docs ├── Makefile ├── __init__.py ├── conf.py ├── index.rst └── make.bat ├── logo └── pgconfig-logo.xcf ├── requirements.txt ├── run_server.py ├── tests ├── __init__.py └── test_pgconfig.py └── webapp ├── __init__.py ├── config.py ├── config ├── pg10.pkl ├── pg11.pkl ├── pg12.pkl ├── pg13.pkl ├── pg14.pkl ├── pg15.pkl ├── pg16.pkl └── pg17.pkl ├── pgconfig.py ├── routes.py ├── static ├── logo.png └── style.css └── templates ├── _layout_header.html ├── about.html ├── custom_config.html ├── footer.html ├── header.html ├── layout.html ├── param.html └── param_change.html /.github/workflows/pages.yml: -------------------------------------------------------------------------------- 1 | name: Deploy Sphinx documentation to Pages 2 | 3 | on: 4 | push: 5 | branches: [main] 6 | 7 | jobs: 8 | pages: 9 | runs-on: ubuntu-latest 10 | environment: 11 | name: github-pages 12 | url: ${{ steps.deployment.outputs.page_url }} 13 | permissions: 14 | pages: write 15 | id-token: write 16 | steps: 17 | - id: deployment 18 | uses: sphinx-notes/pages@v3 19 | with: 20 | requirements_path: ./requirements.txt 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | pytest.xml 2 | **/__pycache__/ 3 | docs/_build/ 4 | .coverage 5 | *.log 6 | **/.ipynb_checkpoints 7 | *.ipynb 8 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | # Specify a configuration file. 4 | #rcfile= 5 | 6 | # Python code to execute, usually for sys.path manipulation such as 7 | # pygtk.require(). 8 | #init-hook= 9 | 10 | # Add files or directories to the blacklist. They should be base names, not 11 | # paths. 12 | ignore=CVS 13 | 14 | # Pickle collected data for later comparisons. 15 | persistent=yes 16 | 17 | # List of plugins (as comma separated values of python modules names) to load, 18 | # usually to register additional checkers. 19 | load-plugins= 20 | 21 | # Use multiple processes to speed up Pylint. 22 | jobs=1 23 | 24 | # Allow loading of arbitrary C extensions. Extensions are imported into the 25 | # active Python interpreter and may run arbitrary code. 26 | unsafe-load-any-extension=no 27 | 28 | # A comma-separated list of package or module names from where C extensions may 29 | # be loaded. Extensions are loading into the active Python interpreter and may 30 | # run arbitrary code 31 | extension-pkg-whitelist= 32 | 33 | # Allow optimization of some AST trees. This will activate a peephole AST 34 | # optimizer, which will apply various small optimizations. For instance, it can 35 | # be used to obtain the result of joining multiple strings with the addition 36 | # operator. Joining a lot of strings can lead to a maximum recursion error in 37 | # Pylint and this flag can prevent that. It has one side effect, the resulting 38 | # AST will be different than the one from reality. 39 | optimize-ast=no 40 | 41 | 42 | [MESSAGES CONTROL] 43 | 44 | # Only show warnings with the listed confidence levels. Leave empty to show 45 | # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED 46 | confidence= 47 | 48 | # Enable the message, report, category or checker with the given id(s). You can 49 | # either give multiple identifier separated by comma (,) or put this option 50 | # multiple time. See also the "--disable" option for examples. 51 | #enable= 52 | 53 | # Disable the message, report, category or checker with the given id(s). You 54 | # can either give multiple identifiers separated by comma (,) or put this 55 | # option multiple times (only on the command line, not in the configuration 56 | # file where it should appear only once).You can also use "--disable=all" to 57 | # disable everything first and then reenable specific checks. For example, if 58 | # you want to run only the similarities checker, you can use "--disable=all 59 | # --enable=similarities". If you want to run only the classes checker, but have 60 | # no Warning level messages displayed, use"--disable=all --enable=classes 61 | # --disable=W" 62 | #disable=old-octal-literal,parameter-unpacking,backtick,old-raise-syntax,old-ne-operator,long-suffix,dict-view-method,dict-iter-method,metaclass-assignment,next-method-called,raising-string,indexing-exception,raw_input-builtin,long-builtin,file-builtin,execfile-builtin,coerce-builtin,cmp-builtin,buffer-builtin,basestring-builtin,apply-builtin,filter-builtin-not-iterating,using-cmp-argument,useless-suppression,range-builtin-not-iterating,suppressed-message,no-absolute-import,old-division,cmp-method,reload-builtin,zip-builtin-not-iterating,intern-builtin,unichr-builtin,reduce-builtin,standarderror-builtin,unicode-builtin,xrange-builtin,coerce-method,delslice-method,getslice-method,setslice-method,input-builtin,round-builtin,hex-method,nonzero-method,map-builtin-not-iterating 63 | 64 | 65 | [REPORTS] 66 | 67 | # Set the output format. Available formats are text, parseable, colorized, msvs 68 | # (visual studio) and html. You can also give a reporter class, eg 69 | # mypackage.mymodule.MyReporterClass. 70 | output-format=text 71 | 72 | # Put messages in a separate file for each module / package specified on the 73 | # command line instead of printing them on stdout. Reports (if any) will be 74 | # written in a file name "pylint_global.[txt|html]". 75 | files-output=no 76 | 77 | # Tells whether to display a full report or only the messages 78 | reports=yes 79 | 80 | # Python expression which should return a note less than 10 (10 is the highest 81 | # note). You have access to the variables errors warning, statement which 82 | # respectively contain the number of errors / warnings messages and the total 83 | # number of statements analyzed. This is used by the global evaluation report 84 | # (RP0004). 85 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 86 | 87 | # Template used to display messages. This is a python new-style format string 88 | # used to format the message information. See doc for all details 89 | #msg-template= 90 | 91 | 92 | [VARIABLES] 93 | 94 | # Tells whether we should check for unused import in __init__ files. 95 | init-import=no 96 | 97 | # A regular expression matching the name of dummy variables (i.e. expectedly 98 | # not used). 99 | dummy-variables-rgx=_$|dummy 100 | 101 | # List of additional names supposed to be defined in builtins. Remember that 102 | # you should avoid to define new builtins when possible. 103 | additional-builtins= 104 | 105 | # List of strings which can identify a callback function by name. A callback 106 | # name must start or end with one of those strings. 107 | callbacks=cb_,_cb 108 | 109 | 110 | [LOGGING] 111 | 112 | # Logging modules to check that the string format arguments are in logging 113 | # function parameter format 114 | logging-modules=logging 115 | 116 | 117 | [SIMILARITIES] 118 | 119 | # Minimum lines number of a similarity. 120 | min-similarity-lines=4 121 | 122 | # Ignore comments when computing similarities. 123 | ignore-comments=yes 124 | 125 | # Ignore docstrings when computing similarities. 126 | ignore-docstrings=yes 127 | 128 | # Ignore imports when computing similarities. 129 | ignore-imports=no 130 | 131 | 132 | [SPELLING] 133 | 134 | # Spelling dictionary name. Available dictionaries: none. To make it working 135 | # install python-enchant package. 136 | spelling-dict= 137 | 138 | # List of comma separated words that should not be checked. 139 | spelling-ignore-words= 140 | 141 | # A path to a file that contains private dictionary; one word per line. 142 | spelling-private-dict-file= 143 | 144 | # Tells whether to store unknown words to indicated private dictionary in 145 | # --spelling-private-dict-file option instead of raising a message. 146 | spelling-store-unknown-words=no 147 | 148 | 149 | [TYPECHECK] 150 | 151 | # Tells whether missing members accessed in mixin class should be ignored. A 152 | # mixin class is detected if its name ends with "mixin" (case insensitive). 153 | ignore-mixin-members=yes 154 | 155 | # List of module names for which member attributes should not be checked 156 | # (useful for modules/projects where namespaces are manipulated during runtime 157 | # and thus existing member attributes cannot be deduced by static analysis. It 158 | # supports qualified module names, as well as Unix pattern matching. 159 | ignored-modules= 160 | 161 | # List of classes names for which member attributes should not be checked 162 | # (useful for classes with attributes dynamically set). This supports can work 163 | # with qualified names. 164 | ignored-classes= 165 | 166 | # List of members which are set dynamically and missed by pylint inference 167 | # system, and so shouldn't trigger E1101 when accessed. Python regular 168 | # expressions are accepted. 169 | generated-members= 170 | 171 | 172 | [FORMAT] 173 | 174 | # Maximum number of characters on a single line. 175 | max-line-length=100 176 | 177 | # Regexp for a line that is allowed to be longer than the limit. 178 | ignore-long-lines=^\s*(# )??$ 179 | 180 | # Allow the body of an if to be on the same line as the test if there is no 181 | # else. 182 | single-line-if-stmt=no 183 | 184 | # List of optional constructs for which whitespace checking is disabled. `dict- 185 | # separator` is used to allow tabulation in dicts, etc.: {1 : 1,\n222: 2}. 186 | # `trailing-comma` allows a space between comma and closing bracket: (a, ). 187 | # `empty-line` allows space-only lines. 188 | no-space-check=trailing-comma,dict-separator 189 | 190 | # Maximum number of lines in a module 191 | max-module-lines=1000 192 | 193 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 194 | # tab). 195 | indent-string=' ' 196 | 197 | # Number of spaces of indent required inside a hanging or continued line. 198 | indent-after-paren=4 199 | 200 | # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. 201 | expected-line-ending-format= 202 | 203 | 204 | [MISCELLANEOUS] 205 | 206 | # List of note tags to take in consideration, separated by a comma. 207 | notes=FIXME,XXX,TODO 208 | 209 | 210 | [BASIC] 211 | 212 | # List of builtins function names that should not be used, separated by a comma 213 | bad-functions=map,filter,input 214 | 215 | # Good variable names which should always be accepted, separated by a comma 216 | good-names=i,j,k,ex,Run,_ 217 | 218 | # Bad variable names which should always be refused, separated by a comma 219 | bad-names=foo,bar,baz,toto,tutu,tata 220 | 221 | # Colon-delimited sets of names that determine each other's naming style when 222 | # the name regexes allow several styles. 223 | name-group= 224 | 225 | # Include a hint for the correct naming format with invalid-name 226 | include-naming-hint=no 227 | 228 | # Regular expression matching correct function names 229 | function-rgx=[a-z_][a-z0-9_]{2,30}$ 230 | 231 | # Naming hint for function names 232 | function-name-hint=[a-z_][a-z0-9_]{2,30}$ 233 | 234 | # Regular expression matching correct variable names 235 | variable-rgx=[a-z_][a-z0-9_]{2,30}$ 236 | 237 | # Naming hint for variable names 238 | variable-name-hint=[a-z_][a-z0-9_]{2,30}$ 239 | 240 | # Regular expression matching correct constant names 241 | const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 242 | 243 | # Naming hint for constant names 244 | const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 245 | 246 | # Regular expression matching correct attribute names 247 | attr-rgx=[a-z_][a-z0-9_]{2,30}$ 248 | 249 | # Naming hint for attribute names 250 | attr-name-hint=[a-z_][a-z0-9_]{2,30}$ 251 | 252 | # Regular expression matching correct argument names 253 | argument-rgx=[a-z_][a-z0-9_]{2,30}$ 254 | 255 | # Naming hint for argument names 256 | argument-name-hint=[a-z_][a-z0-9_]{2,30}$ 257 | 258 | # Regular expression matching correct class attribute names 259 | class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 260 | 261 | # Naming hint for class attribute names 262 | class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 263 | 264 | # Regular expression matching correct inline iteration names 265 | inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ 266 | 267 | # Naming hint for inline iteration names 268 | inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ 269 | 270 | # Regular expression matching correct class names 271 | class-rgx=[A-Z_][a-zA-Z0-9]+$ 272 | 273 | # Naming hint for class names 274 | class-name-hint=[A-Z_][a-zA-Z0-9]+$ 275 | 276 | # Regular expression matching correct module names 277 | module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 278 | 279 | # Naming hint for module names 280 | module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 281 | 282 | # Regular expression matching correct method names 283 | method-rgx=[a-z_][a-z0-9_]{2,30}$ 284 | 285 | # Naming hint for method names 286 | method-name-hint=[a-z_][a-z0-9_]{2,30}$ 287 | 288 | # Regular expression which should only match function or class names that do 289 | # not require a docstring. 290 | no-docstring-rgx=^_ 291 | 292 | # Minimum line length for functions/classes that require docstrings, shorter 293 | # ones are exempt. 294 | docstring-min-length=-1 295 | 296 | 297 | [ELIF] 298 | 299 | # Maximum number of nested blocks for function / method body 300 | max-nested-blocks=5 301 | 302 | 303 | [DESIGN] 304 | 305 | # Maximum number of arguments for function / method 306 | max-args=5 307 | 308 | # Argument names that match this expression will be ignored. Default to name 309 | # with leading underscore 310 | ignored-argument-names=_.* 311 | 312 | # Maximum number of locals for function / method body 313 | max-locals=15 314 | 315 | # Maximum number of return / yield for function / method body 316 | max-returns=6 317 | 318 | # Maximum number of branch for function / method body 319 | max-branches=12 320 | 321 | # Maximum number of statements in function / method body 322 | max-statements=50 323 | 324 | # Maximum number of parents for a class (see R0901). 325 | max-parents=7 326 | 327 | # Maximum number of attributes for a class (see R0902). 328 | max-attributes=7 329 | 330 | # Minimum number of public methods for a class (see R0903). 331 | min-public-methods=2 332 | 333 | # Maximum number of public methods for a class (see R0904). 334 | max-public-methods=20 335 | 336 | # Maximum number of boolean expressions in a if statement 337 | max-bool-expr=5 338 | 339 | 340 | [IMPORTS] 341 | 342 | # Deprecated modules which should not be used, separated by a comma 343 | deprecated-modules=regsub,TERMIOS,Bastion,rexec 344 | 345 | # Create a graph of every (i.e. internal and external) dependencies in the 346 | # given file (report RP0402 must not be disabled) 347 | import-graph= 348 | 349 | # Create a graph of external dependencies in the given file (report RP0402 must 350 | # not be disabled) 351 | ext-import-graph= 352 | 353 | # Create a graph of internal dependencies in the given file (report RP0402 must 354 | # not be disabled) 355 | int-import-graph= 356 | 357 | 358 | [CLASSES] 359 | 360 | # List of method names used to declare (i.e. assign) instance attributes. 361 | defining-attr-methods=__init__,__new__,setUp 362 | 363 | # List of valid names for the first argument in a class method. 364 | valid-classmethod-first-arg=cls 365 | 366 | # List of valid names for the first argument in a metaclass class method. 367 | valid-metaclass-classmethod-first-arg=mcs 368 | 369 | # List of member names, which should be excluded from the protected access 370 | # warning. 371 | exclude-protected=_asdict,_fields,_replace,_source,_make 372 | 373 | 374 | [EXCEPTIONS] 375 | 376 | # Exceptions that will emit a warning when being caught. Defaults to 377 | # "Exception" 378 | overgeneral-exceptions=Exception 379 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | We encourage pull requests from everyone. 4 | 5 | You should fork the project into your own repo, create a topic branch there and then make one or more pull requests back to the RustProof Labs repository. Your pull requests will then be reviewed and discussed. 6 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2018 - 2024 Ryan Lambert 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # postgresql.conf compare 2 | 3 | Compares parameters defined and their default values between PostgreSQL major versions. [Hosted by RustProof Labs](https://pgconfig.rustprooflabs.com/). 4 | 5 | [API reference](https://rustprooflabs.github.io/pgconfig-ce/index.html) 6 | available. 7 | 8 | ## Deployment Instructions 9 | 10 | > Note: Need to update the sub-version of Python over time. Can use simply 11 | `python3` but that can lead to using older unsupported versions based on distro-defaults. 12 | 13 | 14 | ```bash 15 | cd ~/venv 16 | python3.8 -m venv pgconfig 17 | source ~/venv/pgconfig/bin/activate 18 | ``` 19 | 20 | Install requirements 21 | 22 | ```bash 23 | source ~/venv/pgconfig/bin/activate 24 | cd ~/git/pgconfig-ce 25 | pip install -r requirements.txt 26 | ``` 27 | 28 | Run web server w/ uWSGI. 29 | 30 | ```bash 31 | source ~/venv/pgconfig/bin/activate 32 | cd ~/git/pgconfig-ce 33 | python run_server.py 34 | ``` 35 | 36 | ## Add new config files 37 | 38 | To add a new configuration version you need a Postgres database instance running 39 | that you can connect to. Activate the Python venv and start `ipython`. 40 | 41 | ```bash 42 | source ~/venv/pgconfig/bin/activate 43 | cd ~/git/pgconfig-ce/config_from_pg 44 | ipython 45 | ``` 46 | 47 | Import 48 | ```python 49 | import generate 50 | ``` 51 | 52 | You'll be prompted for the database connection parameters. Ideally you are using 53 | a `~/.pgpass` file, but the option is there to enter your password. 54 | 55 | ``` 56 | Database host [127.0.0.1]: 57 | Database port [5432]: 58 | Database name: postgres 59 | Enter PgSQL username: your_username 60 | Enter password (empty for pgpass): 61 | ``` 62 | 63 | Run the generation. Will create a file in the appropriate spot for the webapp. 64 | When adding a new version you need to add it to `webapp/pgconfig.py` as well 65 | as generating this file. 66 | 67 | ```python 68 | generate.run() 69 | ``` 70 | 71 | Preparing database objects... 72 | Database objects ready. 73 | Pickled config data saved to: ../webapp/config/pg15.pkl 74 | 75 | 76 | 77 | ## Unit tests 78 | 79 | Run unit tests. 80 | 81 | ``` 82 | python -m unittest tests/*.py 83 | ``` 84 | 85 | Or run unit tests with coverage. 86 | 87 | ``` 88 | coverage run -m unittest tests/*.py 89 | ``` 90 | 91 | Generate report. 92 | 93 | ``` 94 | coverage report -m webapp/*.py 95 | 96 | Name Stmts Miss Cover Missing 97 | -------------------------------------------------- 98 | webapp/__init__.py 15 0 100% 99 | webapp/config.py 25 0 100% 100 | webapp/forms.py 6 0 100% 101 | webapp/pgconfig.py 150 37 75% 26-27, 53, 71-73, 87-94, 115-122, 140-145, 162-170, 222-223, 300, 393, 417 102 | webapp/routes.py 83 58 30% 20, 24, 30, 35, 40-43, 51, 56-75, 87, 92-119, 127-140, 143-155 103 | -------------------------------------------------- 104 | TOTAL 279 95 66% 105 | ``` 106 | 107 | ## Pylint 108 | 109 | Run pylint. 110 | 111 | ``` 112 | pylint --rcfile=./.pylintrc -f parseable ./webapp/*.py ./config_from_pg/*.py 113 | ``` 114 | 115 | ## History 116 | 117 | The open source (Community Edition) version of this project started as a manual fork 118 | of RustProof Labs' internal project, commit `fcc0619`. The original internal project will 119 | be retired as this project evolves. 120 | 121 | -------------------------------------------------------------------------------- /config_from_pg/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rustprooflabs/pgconfig-ce/10946d43989c3c26fb6ec24f6f85a1a212d48d41/config_from_pg/__init__.py -------------------------------------------------------------------------------- /config_from_pg/create_pgconfig_settings_view.sql: -------------------------------------------------------------------------------- 1 | 2 | DROP VIEW IF EXISTS pgconfig.settings; 3 | CREATE VIEW pgconfig.settings AS 4 | SELECT name, setting, unit, context, source, category, 5 | reset_val, boot_val, 6 | CASE WHEN vartype IN ('string', 'enum') 7 | THEN 8 | name || ' = ' || CHR(39) || current_setting(name) || CHR(39) 9 | ELSE 10 | name || ' = ' || current_setting(name) 11 | END AS postgresconf_line, 12 | name || ' = ' || CHR(39) || 13 | -- Recalculates 8kB units to more comprehensible kB units. 14 | -- above line gets to use the current_setting() func, didn't find any 15 | -- such option for this one 16 | CASE WHEN boot_val = '-1' THEN boot_val 17 | WHEN unit = '8kB' THEN ((boot_val::numeric * 8)::BIGINT)::TEXT 18 | ELSE boot_val 19 | END 20 | || COALESCE(CASE WHEN boot_val = '-1' THEN NULL 21 | WHEN unit = '8kB' THEN 'kB' 22 | ELSE unit 23 | END, '') || CHR(39) 24 | AS default_config_line, 25 | short_desc, 26 | CASE WHEN name LIKE 'lc%' 27 | THEN True 28 | WHEN name LIKE 'unix%' 29 | THEN True 30 | WHEN name IN ('application_name', 'TimeZone', 'timezone_abbreviations', 31 | 'default_text_search_config') 32 | THEN True 33 | WHEN category IN ('File Locations') 34 | THEN True 35 | ELSE False 36 | END AS frequent_override, 37 | CASE WHEN boot_val = setting THEN True 38 | ELSE False 39 | END AS system_default, 40 | CASE WHEN reset_val = setting THEN False 41 | ELSE True 42 | END AS session_override, 43 | pending_restart, 44 | vartype, min_val, max_val, enumvals 45 | FROM pg_catalog.pg_settings 46 | ; 47 | 48 | COMMENT ON COLUMN pgconfig.settings.postgresconf_line IS 'Current configuration in format suitable for postgresql.conf. All setting values quoted in single quotes since that always works, and omitting the quotes does not. Uses pg_catalog.current_setting() which converts settings into sensible units for display.'; 49 | COMMENT ON COLUMN pgconfig.settings.default_config_line IS 'Postgres default configuration for setting. Some are hard coded, some are determined at build time.'; 50 | COMMENT ON COLUMN pgconfig.settings.setting IS 'Raw setting value in the units defined in the "units" column.'; 51 | -------------------------------------------------------------------------------- /config_from_pg/db.py: -------------------------------------------------------------------------------- 1 | """Database helper module. 2 | Modified from https://gist.github.com/rustprooflabs/3b8564a8e7b7fe611436b30a95b7cd17, 3 | adapted to psycopg 3 from psycopg2. 4 | """ 5 | import getpass 6 | import psycopg 7 | 8 | 9 | def prepare(): 10 | """Ensures latest `pgconfig.settings` view exists in DB to generate config. 11 | """ 12 | print('Preparing database objects...') 13 | ensure_schema_exists() 14 | ensure_view_exists() 15 | print('Database objects ready.') 16 | 17 | 18 | def ensure_schema_exists(): 19 | """Ensures the `pgconfig` schema exists.""" 20 | sql_raw = 'CREATE SCHEMA IF NOT EXISTS pgconfig;' 21 | _execute_query(sql_raw, params=None, qry_type='ddl') 22 | 23 | 24 | def ensure_view_exists(): 25 | """Ensures the view `pgconfig.settings` exists.""" 26 | sql_file = 'create_pgconfig_settings_view.sql' 27 | with open(sql_file) as f: 28 | sql_raw = f.read() 29 | 30 | _execute_query(sql_raw, params=None, qry_type='ddl') 31 | 32 | 33 | def select_one(sql_raw: str, params: dict) -> dict: 34 | """ Runs SELECT query that will return zero or 1 rows. 35 | 36 | Parameters 37 | ----------------- 38 | sql_raw : str 39 | params : dict 40 | Params is required, can be `None` if query returns a single row 41 | such as `SELECT version();` 42 | 43 | Returns 44 | ----------------- 45 | data : dict 46 | """ 47 | return _execute_query(sql_raw, params, 'sel_single') 48 | 49 | 50 | def select_multi(sql_raw, params=None) -> list: 51 | """ Runs SELECT query that will return multiple. `params` is optional. 52 | 53 | Parameters 54 | ----------------- 55 | sql_raw : str 56 | params : dict 57 | Params is optional, defaults to `None`. 58 | 59 | Returns 60 | ------------------ 61 | data : list 62 | List of dictionaries. 63 | """ 64 | return _execute_query(sql_raw, params, 'sel_multi') 65 | 66 | 67 | def get_db_string() -> str: 68 | """Prompts user for details to create connection string 69 | 70 | Returns 71 | ------------------------ 72 | database_string : str 73 | """ 74 | db_host = input('Database host [127.0.0.1]: ') or '127.0.0.1' 75 | db_port = input('Database port [5432]: ') or '5432' 76 | db_name = input('Database name: ') 77 | db_user = input('Enter PgSQL username: ') 78 | db_pw = getpass.getpass('Enter password (empty for pgpass): ') or None 79 | 80 | if db_pw is None: 81 | database_string = 'postgresql://{user}@{host}:{port}/{dbname}' 82 | else: 83 | database_string = 'postgresql://{user}:{pw}@{host}:{port}/{dbname}' 84 | 85 | return database_string.format(user=db_user, pw=db_pw, host=db_host, 86 | port=db_port, dbname=db_name) 87 | 88 | DB_STRING = get_db_string() 89 | 90 | def get_db_conn(): 91 | """Uses DB_STRING to establish psycopg connection.""" 92 | db_string = DB_STRING 93 | 94 | try: 95 | conn = psycopg.connect(db_string) 96 | except psycopg.OperationalError as err: 97 | err_msg = f'DB Connection Error - Error: {err}' 98 | print(err_msg) 99 | return False 100 | 101 | return conn 102 | 103 | 104 | def _execute_query(sql_raw, params, qry_type): 105 | """ Handles executing all types of queries based on the `qry_type` passed in. 106 | Returns False if there are errors during connection or execution. 107 | if results == False: 108 | print('Database error') 109 | else: 110 | print(results) 111 | You cannot use `if not results:` b/c 0 results is a false negative. 112 | """ 113 | try: 114 | conn = get_db_conn() 115 | except psycopg.ProgrammingError as err: 116 | print(f'Connection not configured properly. Err: {err}') 117 | return False 118 | 119 | if not conn: 120 | return False 121 | 122 | cur = conn.cursor(row_factory=psycopg.rows.dict_row) 123 | 124 | try: 125 | cur.execute(sql_raw, params) 126 | if qry_type == 'sel_single': 127 | results = cur.fetchone() 128 | elif qry_type == 'sel_multi': 129 | results = cur.fetchall() 130 | elif qry_type == 'ddl': 131 | conn.commit() 132 | results = True 133 | else: 134 | raise Exception('Invalid query type defined.') 135 | 136 | except psycopg.BINARYProgrammingError as err: 137 | print('Database error via psycopg. %s', err) 138 | results = False 139 | except psycopg.IntegrityError as err: 140 | print('PostgreSQL integrity error via psycopg. %s', err) 141 | results = False 142 | finally: 143 | conn.close() 144 | 145 | return results 146 | -------------------------------------------------------------------------------- /config_from_pg/generate.py: -------------------------------------------------------------------------------- 1 | """Generates config based on pgconfig.settings and pickles for reuse in webapp. 2 | 3 | This code is expected to be used on Postgres 10 and newer. 4 | """ 5 | import pickle 6 | import db 7 | 8 | 9 | def run(): 10 | """Saves pickled config data from defined database connection. 11 | """ 12 | db.prepare() 13 | pg_version_num = get_pg_version_num() 14 | pg_config_data = get_config_data() 15 | save_config_data(data=pg_config_data, pg_version_num=pg_version_num) 16 | 17 | 18 | def get_pg_version_num() -> int: 19 | """Returns the Postgres version number as an integer. 20 | 21 | Expected to be used on Postgres 10 and newer only. 22 | 23 | Returns 24 | ------------------------ 25 | pg_version_num : int 26 | """ 27 | sql_raw = """SELECT current_setting('server_version_num')::BIGINT / 10000 28 | AS pg_version_num; 29 | """ 30 | results = db.select_one(sql_raw, params=None) 31 | pg_version_num = results['pg_version_num'] 32 | return pg_version_num 33 | 34 | 35 | def get_config_data() -> list: 36 | """Query Postgres for data about default settings. 37 | 38 | Returns 39 | -------------------- 40 | results : list 41 | """ 42 | sql_raw = """ 43 | SELECT default_config_line, name, unit, context, category, 44 | boot_val, short_desc, frequent_override, 45 | vartype, min_val, max_val, enumvals, 46 | boot_val || COALESCE(' ' || unit, '') AS boot_val_display 47 | FROM pgconfig.settings 48 | /* Excluding read-only present options. Not included in delivered 49 | postgresql.conf files per docs: 50 | https://www.postgresql.org/docs/current/runtime-config-preset.html 51 | */ 52 | WHERE category != 'Preset Options' 53 | /* Configuration options that typically are customized such as 54 | application_name do not make sense to compare against "defaults" 55 | */ 56 | AND NOT frequent_override 57 | ORDER BY name 58 | ; 59 | """ 60 | results = db.select_multi(sql_raw) 61 | return results 62 | 63 | def save_config_data(data: list, pg_version_num: int): 64 | """Pickles config data for reuse later. 65 | 66 | Parameters 67 | ---------------------- 68 | data : list 69 | List of dictionaries to pickle. 70 | 71 | pg_version_num : int 72 | Integer of Postgres version. 73 | """ 74 | filename = f'../webapp/config/pg{pg_version_num}.pkl' 75 | with open(filename, 'wb') as data_file: 76 | pickle.dump(data, data_file) 77 | print(f'Pickled config data saved to: {filename}') 78 | 79 | 80 | if __name__ == "__main__": 81 | run() 82 | -------------------------------------------------------------------------------- /docs/Makefile: -------------------------------------------------------------------------------- 1 | # Minimal makefile for Sphinx documentation 2 | # 3 | 4 | # You can set these variables from the command line, and also 5 | # from the environment for the first two. 6 | SPHINXOPTS ?= 7 | SPHINXBUILD ?= sphinx-build 8 | SOURCEDIR = . 9 | BUILDDIR = _build 10 | 11 | # Put it first so that "make" without argument is like "make help". 12 | help: 13 | @$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 14 | 15 | .PHONY: help Makefile 16 | 17 | # Catch-all target: route all unknown targets to Sphinx using the new 18 | # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). 19 | %: Makefile 20 | @$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) 21 | -------------------------------------------------------------------------------- /docs/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rustprooflabs/pgconfig-ce/10946d43989c3c26fb6ec24f6f85a1a212d48d41/docs/__init__.py -------------------------------------------------------------------------------- /docs/conf.py: -------------------------------------------------------------------------------- 1 | # Configuration file for the Sphinx documentation builder. 2 | # 3 | # This file only contains a selection of the most common options. For a full 4 | # list see the documentation: 5 | # http://www.sphinx-doc.org/en/master/config 6 | 7 | # -- Path setup -------------------------------------------------------------- 8 | 9 | # If extensions (or modules to document with autodoc) are in another directory, 10 | # add these directories to sys.path here. If the directory is relative to the 11 | # documentation root, use os.path.abspath to make it absolute, like shown here. 12 | # 13 | import os 14 | import sys 15 | sys.path.insert(0, os.path.abspath('../')) 16 | 17 | 18 | # -- Project information ----------------------------------------------------- 19 | 20 | project = 'pgConfig' 21 | copyright = '2018 - 2024, Ryan Lambert, RustProof Labs' 22 | author = 'Ryan Lambert, RustProof Labs' 23 | 24 | version = '0.1.0' 25 | 26 | 27 | # -- General configuration --------------------------------------------------- 28 | 29 | # Add any Sphinx extension module names here, as strings. They can be 30 | # extensions coming with Sphinx (named 'sphinx.ext.*') or your custom 31 | # ones. 32 | extensions = ['sphinx.ext.autodoc', 33 | 'sphinx.ext.coverage', 34 | 'sphinx.ext.napoleon', 35 | 'sphinx.ext.viewcode' 36 | ] 37 | 38 | # Add any paths that contain templates here, relative to this directory. 39 | templates_path = ['_templates'] 40 | 41 | # List of patterns, relative to source directory, that match files and 42 | # directories to ignore when looking for source files. 43 | # This pattern also affects html_static_path and html_extra_path. 44 | exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store'] 45 | 46 | 47 | # -- Options for HTML output ------------------------------------------------- 48 | 49 | # The theme to use for HTML and HTML Help pages. See the documentation for 50 | # a list of builtin themes. 51 | # 52 | html_theme = 'sphinx_rtd_theme' 53 | 54 | # Add any paths that contain custom static files (such as style sheets) here, 55 | # relative to this directory. They are copied after the builtin static files, 56 | # so a file named "default.css" will overwrite the builtin "default.css". 57 | html_static_path = ['_static'] 58 | 59 | html_show_sphinx = False 60 | 61 | html_theme_options = { 62 | 'canonical_url': '', 63 | 'logo_only': False, 64 | 'display_version': True 65 | } 66 | 67 | extensions.append('autoapi.extension') 68 | autoapi_type = 'python' 69 | autoapi_dirs = ['../config_from_pg', '../webapp'] 70 | -------------------------------------------------------------------------------- /docs/index.rst: -------------------------------------------------------------------------------- 1 | .. pgConfig documentation master file, created by 2 | sphinx-quickstart on Mon Sep 23 14:47:37 2019. 3 | You can adapt this file completely to your liking, but it should at least 4 | contain the root `toctree` directive. 5 | 6 | pgConfig documentation! 7 | ==================================== 8 | 9 | .. toctree:: 10 | :maxdepth: 2 11 | :caption: Contents: 12 | 13 | 14 | 15 | Indices and tables 16 | ================== 17 | 18 | * :ref:`genindex` 19 | * :ref:`modindex` 20 | * :ref:`search` 21 | -------------------------------------------------------------------------------- /docs/make.bat: -------------------------------------------------------------------------------- 1 | @ECHO OFF 2 | 3 | pushd %~dp0 4 | 5 | REM Command file for Sphinx documentation 6 | 7 | if "%SPHINXBUILD%" == "" ( 8 | set SPHINXBUILD=sphinx-build 9 | ) 10 | set SOURCEDIR=. 11 | set BUILDDIR=_build 12 | 13 | if "%1" == "" goto help 14 | 15 | %SPHINXBUILD% >NUL 2>NUL 16 | if errorlevel 9009 ( 17 | echo. 18 | echo.The 'sphinx-build' command was not found. Make sure you have Sphinx 19 | echo.installed, then set the SPHINXBUILD environment variable to point 20 | echo.to the full path of the 'sphinx-build' executable. Alternatively you 21 | echo.may add the Sphinx directory to PATH. 22 | echo. 23 | echo.If you don't have Sphinx installed, grab it from 24 | echo.http://sphinx-doc.org/ 25 | exit /b 1 26 | ) 27 | 28 | %SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 29 | goto end 30 | 31 | :help 32 | %SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS% %O% 33 | 34 | :end 35 | popd 36 | -------------------------------------------------------------------------------- /logo/pgconfig-logo.xcf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rustprooflabs/pgconfig-ce/10946d43989c3c26fb6ec24f6f85a1a212d48d41/logo/pgconfig-logo.xcf -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | Click==8.1.7 2 | coverage==7.3.1 3 | Flask==3.0.0 4 | Flask-WTF==1.2.1 5 | itsdangerous==2.1.2 6 | Jinja2==3.1.3 7 | MarkupSafe==2.1.3 8 | numpy==1.26.1 9 | pandas==2.1.2 10 | psycopg==3.1.16 11 | psycopg-binary==3.1.16 12 | pylint==2.17.5 13 | pytest==7.4.3 14 | python-dateutil==2.8.2 15 | pytz==2023.3.post1 16 | six==1.16.0 17 | Sphinx==7.2.6 18 | sphinx-autoapi==2.1.1 19 | sphinx-rtd-theme==1.3.0 20 | uWSGI==2.0.22 21 | Werkzeug==3.0.1 22 | WTForms==3.0.1 23 | -------------------------------------------------------------------------------- /run_server.py: -------------------------------------------------------------------------------- 1 | from webapp import app 2 | 3 | if __name__ == '__main__': 4 | app.run(host='0.0.0.0') 5 | 6 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rustprooflabs/pgconfig-ce/10946d43989c3c26fb6ec24f6f85a1a212d48d41/tests/__init__.py -------------------------------------------------------------------------------- /tests/test_pgconfig.py: -------------------------------------------------------------------------------- 1 | """ Unit tests to cover the pgconfig module.""" 2 | import unittest 3 | import pandas as pd 4 | from webapp import pgconfig 5 | 6 | 7 | class PGConfigTests(unittest.TestCase): 8 | 9 | def test_pgconfig_check_redirect_returns_same_version_with_no_redirect(self): 10 | version = '10' 11 | expected = version 12 | new_version = pgconfig.check_redirect(version) 13 | self.assertEqual(expected, new_version) 14 | 15 | def test_pgconfig_check_redirect_returns_updated_redirect_version(self): 16 | version = '12beta4' 17 | expected = '12' 18 | new_version = pgconfig.check_redirect(version) 19 | self.assertEqual(expected, new_version) 20 | 21 | 22 | def test_pgconfig_config_changes_returns_expected_type(self): 23 | vers1 = 12 24 | vers2 = 16 25 | result = pgconfig.config_changes(vers1=vers1, vers2=vers2) 26 | actual = type(result) 27 | expected = pd.DataFrame 28 | self.assertEqual(expected, actual) 29 | 30 | def test_pgconfig_config_changes_raises_ValueError_when_misordered_versions(self): 31 | vers1 = 16 32 | vers2 = 12 33 | self.assertRaises(ValueError, pgconfig.config_changes, vers1, vers2) 34 | 35 | def test_load_config_data_raises_ValueError_when_invalid_version(self): 36 | vers1 = -999 37 | self.assertRaises(ValueError, pgconfig.load_config_data, vers1) 38 | -------------------------------------------------------------------------------- /webapp/__init__.py: -------------------------------------------------------------------------------- 1 | import logging 2 | from flask import Flask 3 | from webapp import config 4 | 5 | # App settings 6 | app = Flask(__name__) 7 | app.config['DEBUG'] = config.APP_DEBUG 8 | app.config['SECRET_KEY'] = config.APP_SECRET_KEY 9 | app.config['WTF_CSRF_ENABLED'] = True 10 | 11 | # Setup Logging 12 | LOG_PATH = config.LOG_PATH 13 | LOGGER = logging.getLogger(__name__) 14 | HANDLER = logging.FileHandler(filename=LOG_PATH, mode='a+') 15 | FORMATTER = logging.Formatter(config.LOG_FORMAT) 16 | HANDLER.setFormatter(FORMATTER) 17 | LOGGER.addHandler(HANDLER) 18 | LOGGER.setLevel(logging.DEBUG) 19 | 20 | from webapp import routes 21 | -------------------------------------------------------------------------------- /webapp/config.py: -------------------------------------------------------------------------------- 1 | """Config module for PgConfig webapp.""" 2 | import os 3 | import logging 4 | 5 | CURR_PATH = os.path.abspath(os.path.dirname(__file__)) 6 | PROJECT_BASE_PATH = os.path.abspath(os.path.join(CURR_PATH, os.pardir)) 7 | 8 | APP_NAME = 'PG Configuration Tracking' 9 | 10 | LOG_FORMAT = '%(levelname)s - %(asctime)s - %(name)s - %(message)s' 11 | 12 | LOGGER = logging.getLogger(__name__) 13 | 14 | try: 15 | LOG_PATH = os.environ['LOG_PATH'] 16 | except KeyError: 17 | LOG_PATH = PROJECT_BASE_PATH + '/pgconfig_app.log' 18 | 19 | try: 20 | APP_DEBUG = os.environ['APP_DEBUG'] 21 | except KeyError: 22 | APP_DEBUG = True 23 | 24 | 25 | # Required for CSRF protection in Flask, please change to something secret! 26 | try: 27 | APP_SECRET_KEY = os.environ['APP_SECRET_KEY'] 28 | except KeyError: 29 | ERR_MSG = '\nSECURITY WARNING: To ensure security please set the APP_SECRET_KEY' 30 | ERR_MSG += ' environment variable.\n' 31 | LOGGER.warning(ERR_MSG) 32 | print(ERR_MSG) 33 | APP_SECRET_KEY = 'YourApplicationShouldBeServedFreshWithASecureAndSecretKey!' 34 | 35 | 36 | SESSION_COOKIE_SECURE = True 37 | REMEMBER_COOKIE_SECURE = True 38 | -------------------------------------------------------------------------------- /webapp/config/pg10.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rustprooflabs/pgconfig-ce/10946d43989c3c26fb6ec24f6f85a1a212d48d41/webapp/config/pg10.pkl -------------------------------------------------------------------------------- /webapp/config/pg11.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rustprooflabs/pgconfig-ce/10946d43989c3c26fb6ec24f6f85a1a212d48d41/webapp/config/pg11.pkl -------------------------------------------------------------------------------- /webapp/config/pg12.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rustprooflabs/pgconfig-ce/10946d43989c3c26fb6ec24f6f85a1a212d48d41/webapp/config/pg12.pkl -------------------------------------------------------------------------------- /webapp/config/pg13.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rustprooflabs/pgconfig-ce/10946d43989c3c26fb6ec24f6f85a1a212d48d41/webapp/config/pg13.pkl -------------------------------------------------------------------------------- /webapp/config/pg14.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rustprooflabs/pgconfig-ce/10946d43989c3c26fb6ec24f6f85a1a212d48d41/webapp/config/pg14.pkl -------------------------------------------------------------------------------- /webapp/config/pg15.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rustprooflabs/pgconfig-ce/10946d43989c3c26fb6ec24f6f85a1a212d48d41/webapp/config/pg15.pkl -------------------------------------------------------------------------------- /webapp/config/pg16.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rustprooflabs/pgconfig-ce/10946d43989c3c26fb6ec24f6f85a1a212d48d41/webapp/config/pg16.pkl -------------------------------------------------------------------------------- /webapp/config/pg17.pkl: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rustprooflabs/pgconfig-ce/10946d43989c3c26fb6ec24f6f85a1a212d48d41/webapp/config/pg17.pkl -------------------------------------------------------------------------------- /webapp/pgconfig.py: -------------------------------------------------------------------------------- 1 | """Parses PostgreSQL config data from pickled files""" 2 | import os 3 | import logging 4 | import pandas as pd 5 | import pickle 6 | 7 | 8 | LOGGER = logging.getLogger(__name__) 9 | VERSIONS = ['10', '11', '12', '13', '14', '15', '16', '17'] 10 | """list : Versions with data included. Add new versions to this list. 11 | When including pre-production versions include a non-production designation, 12 | e.g. 16beta1 13 | """ 14 | 15 | VERSION_REDIRECTS = [{'version': '12beta4', 'redirect': '12'}, 16 | {'version': '16beta1', 'redirect': '16'} 17 | ] 18 | """list : List of dictionaries. Dict keys are 'version' and 'redirect'.""" 19 | 20 | NEW_STRING = 'Configuration parameter added' 21 | REMOVED_STRING = 'Configuration parameter removed' 22 | 23 | 24 | def check_redirect(version): 25 | """Checks version for defined redirects. 26 | 27 | e.g. 12beta4 redirects to 12 once the production-ready version 28 | is released. 29 | 30 | Parameters 31 | --------------- 32 | version : str 33 | Version to check for redirects for. 34 | 35 | Returns 36 | --------------- 37 | version : str 38 | Redirected if necessary, original version if not. 39 | """ 40 | for redirect in VERSION_REDIRECTS: 41 | if version == redirect['version']: 42 | LOGGER.info('Redirecting version %s to %s', version, 43 | redirect['redirect']) 44 | return redirect['redirect'] 45 | return version 46 | 47 | 48 | def config_changes(vers1: int, vers2: int) -> pd.DataFrame: 49 | """Find changes between `vers1` and `vers2`. 50 | 51 | Parameters 52 | ---------------- 53 | vers1 : int 54 | Version number, e.g. 11 or 16 55 | 56 | vers2 : int 57 | Version number, e.g. 11 or 16 58 | 59 | Returns 60 | ----------------- 61 | changed : pd.DataFrame 62 | """ 63 | if vers2 <= vers1: 64 | raise ValueError('Version 1 must be lower (before) version 2.') 65 | 66 | # Have to drop enumvals in comparison to avoid errors comparing using pandas 67 | data1 = load_config_data(pg_version=vers1) 68 | data2 = load_config_data(pg_version=vers2) 69 | 70 | data2 = data2.add_suffix('2') 71 | 72 | combined = pd.concat([data1, data2], axis=1) 73 | 74 | combined['summary'] = combined.apply(classify_changes, axis=1) 75 | combined['change_display'] = combined.apply(calculate_change_display, axis=1) 76 | 77 | # Create combined columns. Passing in 2nd version first displays latest 78 | # version if there are any differences. 79 | squash_column_names = ['short_desc2', 'short_desc'] 80 | new_column = 'short_desc' 81 | combined = squash_columns(data=combined, 82 | original_columns=squash_column_names, 83 | new_column=new_column) 84 | 85 | squash_column_names = ['frequent_override2', 'frequent_override'] 86 | new_column = 'frequent_override' 87 | combined = squash_columns(data=combined, 88 | original_columns=squash_column_names, 89 | new_column=new_column) 90 | 91 | squash_column_names = ['category2', 'category'] 92 | new_column = 'category' 93 | combined = squash_columns(data=combined, 94 | original_columns=squash_column_names, 95 | new_column=new_column) 96 | 97 | squash_column_names = ['history_url2', 'history_url'] 98 | new_column = 'history_url' 99 | combined = squash_columns(data=combined, 100 | original_columns=squash_column_names, 101 | new_column=new_column) 102 | 103 | # Limit the columns 104 | columns = ['summary', 'frequent_override', 105 | 'category', 'short_desc', 106 | 'vartype', 'vartype2', 107 | 'boot_val_display', 'boot_val_display2', 108 | 'enumvals', 'enumvals2', 'change_display', 109 | 'history_url' 110 | ] 111 | changed = combined[combined['summary'] != ''][columns] 112 | 113 | return changed 114 | 115 | 116 | def squash_columns(data: pd.DataFrame, original_columns: list, new_column: str): 117 | """Coalesces the values from DataFrame columns in `original_columns` list 118 | into the `new_column` name. Drops `original_columns`, so can reuse one of 119 | the column names if desired. e.g `short_desc` and `short_desc2` combined 120 | into the `short_desc` column. 121 | 122 | Note: This is useful for added and removed items, NOT changed items. 123 | 124 | Parameters 125 | --------------------- 126 | data : pd.DataFrame 127 | original_columns : list 128 | new_column : str 129 | 130 | Returns 131 | --------------------- 132 | data_new : pd.DataFrame 133 | """ 134 | data['tmp'] = data[original_columns].bfill(axis=1).iloc[:, 0] 135 | data_mid = data.drop(columns=original_columns) 136 | data_new = data_mid.rename(columns={'tmp': new_column}) 137 | return data_new 138 | 139 | 140 | def load_config_data(pg_version: int) -> pd.DataFrame: 141 | """Loads the pickled config data for `pg_version` into DataFrame with the 142 | config name as the index. 143 | 144 | Returns empty DataFrame on file read error. 145 | 146 | Parameters 147 | ---------------------- 148 | pg_version : int 149 | 150 | Returns 151 | ---------------------- 152 | df : pd.DataFrame 153 | """ 154 | base_path = os.path.dirname(os.path.realpath(__file__)) 155 | # Checking user input against configured versions to avoid security concerns 156 | if str(pg_version) not in VERSIONS: 157 | raise ValueError(f'Invalid Postgres version. Options are {VERSIONS}') 158 | 159 | filename = os.path.join(base_path, 'config', f'pg{pg_version}.pkl') 160 | 161 | try: 162 | with open(filename, 'rb') as data_file: 163 | config_data = pickle.load(data_file) 164 | 165 | df = pd.DataFrame(config_data) 166 | except FileNotFoundError: 167 | msg = f'File not found for Postgres version {pg_version}' 168 | print(msg) 169 | LOGGER.error(msg) 170 | df = pd.DataFrame() 171 | return df 172 | 173 | # Add hyperlink to the parameter history page 174 | html_part1 = '' 176 | df['history_url'] = html_part1 + df['name'] + html_part2 177 | 178 | df.set_index('name', inplace=True) 179 | return df 180 | 181 | 182 | def is_NaN(input: str) -> bool: 183 | """Checks string values for NaN, aka it isn't equal to itself. 184 | 185 | Parameters 186 | ------------------------ 187 | input : str 188 | 189 | Returns 190 | ------------------------ 191 | is_nan : bool 192 | """ 193 | return input != input 194 | 195 | 196 | def classify_changes(row: pd.Series) -> str: 197 | """Used by dataFrame.apply on the combined DataFrame to check version1 and 198 | version2 values, types, etc. for differences. 199 | 200 | Parameters 201 | -------------------------- 202 | row : pd.Series 203 | Row from combined DataFrame to check details. 204 | 205 | Returns 206 | ------------------------- 207 | changes : str 208 | Changes are built as a list internally and returned as a string 209 | with a comma separated list of changes. 210 | """ 211 | changes = [] 212 | delim = ', ' 213 | 214 | # When old is empty, and new is not, it's a new parameters 215 | if is_NaN(row['default_config_line']) and not is_NaN(row['default_config_line2']): 216 | changes.append('Configuration parameter added') 217 | return delim.join(changes) 218 | 219 | # When new is empty and old is not, it was removed 220 | if is_NaN(row['default_config_line2']) and not is_NaN(row['default_config_line']): 221 | changes.append('Configuration parameter removed') 222 | return delim.join(changes) 223 | 224 | if row['boot_val'] != row['boot_val2']: 225 | changes.append('Changed default value') 226 | if row['vartype'] != row['vartype2']: 227 | changes.append('Changed variable type') 228 | return delim.join(changes) 229 | 230 | 231 | def calculate_change_display(row: pd.Series) -> str: 232 | """Used by dataFrame.apply on the combined DataFrame to create the columns 233 | to display for changes. 234 | 235 | Parameters 236 | -------------------------- 237 | row : pd.Series 238 | Row from combined DataFrame to check details. 239 | 240 | Returns 241 | ------------------------- 242 | changes : str 243 | Changes are built as a list internally and returned as a string 244 | with a comma separated list of changes. 245 | """ 246 | changes = [] 247 | delim = ', ' 248 | 249 | # If either is Nan, don't calculate 250 | if is_NaN(row['default_config_line']) or is_NaN(row['default_config_line2']): 251 | return None 252 | 253 | if row.boot_val != row.boot_val2: 254 | changes.append(f'Default value: {row.boot_val} -> {row.boot_val2}') 255 | if row['vartype'] != row['vartype2']: 256 | changes.append(f'Variable type: {row.vartype} -> {row.vartype2}') 257 | return delim.join(changes) 258 | 259 | 260 | def config_changes_html(changes: pd.DataFrame) -> dict: 261 | """Splits `changes` data into new, removed, and changed. 262 | 263 | Parameters 264 | ------------------- 265 | changes : pd.DataFrame 266 | 267 | Returns 268 | ------------------- 269 | changes_html : dict 270 | Dictionary with keys: 271 | * new 272 | * removed 273 | * changed 274 | 275 | Each item holds the string HTML for the table of the data from input 276 | DataFrame 277 | """ 278 | # New Section 279 | columns_new = ['category', 'short_desc', 'boot_val_display2', 280 | 'vartype2', 'enumvals2', 'history_url'] 281 | rename_columns_new = {'vartype2': 'Var Type', 282 | 'boot_val_display2': 'Default Value', 283 | 'enumvals2': 'Enum Values' 284 | } 285 | new = changes[changes.summary == NEW_STRING][columns_new].rename(columns=rename_columns_new) 286 | 287 | new_html = _df_to_html(new) 288 | 289 | # Removed Section 290 | columns_removed = ['category', 'short_desc', 'boot_val_display', 291 | 'vartype', 'enumvals', 'history_url'] 292 | rename_columns_removed = {'vartype': 'Var Type', 293 | 'boot_val_display': 'Default Value', 294 | 'enumvals': 'Enum Values' 295 | } 296 | removed = changes[changes.summary == REMOVED_STRING][columns_removed].rename(columns=rename_columns_removed) 297 | removed_html = _df_to_html(removed) 298 | 299 | # Changed section 300 | columns_changed = ['category', 'short_desc', 'change_display', 301 | 'history_url'] 302 | rename_columns_changed = {'vartype': 'Var Type', 303 | 'change_display': 'Changed:', 304 | 'enumvals': 'Enum Values' 305 | } 306 | changed = changes[~changes.summary.isin([NEW_STRING, REMOVED_STRING])][columns_changed].rename(columns=rename_columns_changed) 307 | changed_html = _df_to_html(changed) 308 | 309 | return {'new': new_html, 'removed': removed_html, 'changed': changed_html} 310 | 311 | 312 | def config_changes_stats(changes: dict) -> dict: 313 | """Provides counts of changes by type (new, removed, updated) to display. 314 | 315 | Parameters 316 | --------------------- 317 | changes : dict 318 | 319 | Returns 320 | --------------------- 321 | stats : dict 322 | """ 323 | new = changes[changes.summary == NEW_STRING].count().max() 324 | removed = changes[changes.summary == REMOVED_STRING].count().max() 325 | total = changes.count().max() 326 | updated = total - new - removed 327 | stats = {'new': new, 328 | 'updated': updated, 329 | 'removed': removed} 330 | return stats 331 | 332 | 333 | def _df_to_html(df): 334 | """Converts DataFrame to HTML table with classes for formatting. 335 | 336 | Parameters 337 | --------------- 338 | df : pandas.DataFrame 339 | 340 | Returns 341 | --------------- 342 | str 343 | HTML table for display. 344 | """ 345 | classes = ['table', 'table-hover'] 346 | html_raw = '
{src}
' 347 | default_renames = {'category': 'Category', 'short_desc': 'Description'} 348 | df.rename(columns=default_renames, inplace=True) 349 | src = df.to_html(index=True, 350 | classes=classes, 351 | justify='center', 352 | escape=False) 353 | html = html_raw.format(src=src) 354 | return html 355 | 356 | 357 | def get_all_postgres_parameters() -> set: 358 | """Returns sorted set of Postgres parameters from all versions. 359 | 360 | Returns 361 | ---------------------- 362 | all_parameters : set 363 | """ 364 | all_version_data = {} 365 | all_parameters = set() 366 | 367 | for version in VERSIONS: 368 | all_version_data[version] = load_config_data(version) 369 | version_parameters = all_version_data[version].index.values.tolist() 370 | all_parameters.update(version_parameters) 371 | 372 | return sorted(all_parameters) 373 | 374 | 375 | def get_pg_param_over_versions(pg_param: str) -> dict: 376 | """Return details about pg_param over Postgres major versions. 377 | 378 | Parameters 379 | ------------------------- 380 | pg_param : str 381 | Name of the Postgres parameter 382 | 383 | Returns 384 | ------------------------- 385 | param_details : dict 386 | """ 387 | param_details = {'name': pg_param} 388 | max_version = max(VERSIONS) 389 | value_history = {} 390 | vartype_history = {} 391 | 392 | for version in VERSIONS: 393 | config_data = load_config_data(version) 394 | filtered_data = config_data[config_data.index == pg_param] 395 | 396 | try: 397 | boot_val = filtered_data['boot_val_display'].iloc[0] 398 | except IndexError: 399 | boot_val = '' 400 | 401 | try: 402 | vartype = filtered_data['vartype'].iloc[0] 403 | except IndexError: 404 | vartype = '' 405 | 406 | try: 407 | category = filtered_data['category'].iloc[0] 408 | except IndexError: 409 | category = '' 410 | 411 | try: 412 | short_desc = filtered_data['short_desc'].iloc[0] 413 | except IndexError: 414 | short_desc = '' 415 | 416 | try: 417 | frequent_override = filtered_data['frequent_override'].iloc[0] 418 | except IndexError: 419 | frequent_override = '' 420 | 421 | if version == max_version: 422 | details = {'vartype': vartype, 423 | 'boot_val': boot_val, 424 | 'category': category, 425 | 'short_desc': short_desc, 426 | 'frequent_override': frequent_override 427 | } 428 | param_details['details'] = details 429 | 430 | value_history[version] = boot_val 431 | vartype_history[version] = vartype 432 | 433 | value_history_html = history_df_to_html(history=value_history, 434 | pg_param=pg_param) 435 | vartype_history_html = history_df_to_html(history=vartype_history, 436 | pg_param=pg_param) 437 | 438 | param_details['value_history'] = value_history_html 439 | param_details['vartype_history'] = vartype_history_html 440 | 441 | return param_details 442 | 443 | 444 | def history_df_to_html(history: dict, pg_param: str) -> str: 445 | """Converts a dictionary of details across versions to HTML for display on 446 | frontend. 447 | 448 | Parameters 449 | ------------------------ 450 | history : dict 451 | pg_param : str 452 | 453 | Returns 454 | ------------------------ 455 | history_html : str 456 | """ 457 | history_df = pd.DataFrame([history]) 458 | history_df['name'] = pg_param 459 | history_df.set_index('name', inplace=True) 460 | history_html = _df_to_html(history_df) 461 | return history_html 462 | -------------------------------------------------------------------------------- /webapp/routes.py: -------------------------------------------------------------------------------- 1 | """Routes for pgconfig-ce webapp.""" 2 | from datetime import datetime as dt 3 | import logging 4 | from flask import render_template, redirect 5 | from webapp import app, pgconfig 6 | 7 | 8 | LOGGER = logging.getLogger(__name__) 9 | 10 | 11 | VERSION_PRIOR = '16' 12 | VERSION_CURRENT = '17' 13 | 14 | 15 | def get_year(): 16 | """ Gets the current year. Used for providing dynamic 17 | copyright year in the footer. 18 | 19 | Returns 20 | ---------------- 21 | int 22 | Current year 23 | """ 24 | return dt.now().year 25 | 26 | @app.route('/about') 27 | def view_about(): 28 | return render_template('about.html', 29 | year=get_year()) 30 | 31 | 32 | @app.route('/') 33 | def view_root_url(): 34 | return redirect_param_change() 35 | 36 | 37 | @app.route('/param') 38 | def view_app_param_not_set(): 39 | return redirect('/param/{}'.format('max_parallel_workers_per_gather')) 40 | 41 | 42 | 43 | @app.route('/param/') 44 | def view_app_params(pg_param): 45 | select_html = get_param_select_html(pg_param) 46 | param_details = pgconfig.get_pg_param_over_versions(pg_param) 47 | return render_template('param.html', 48 | year=get_year(), 49 | param_details=param_details, 50 | select_html=select_html) 51 | 52 | 53 | 54 | @app.route('/param/change//') 55 | def view_app_param_changes_v2(vers1, vers2): 56 | vers1_redirect = pgconfig.check_redirect(version=vers1) 57 | vers2_redirect = pgconfig.check_redirect(version=vers2) 58 | 59 | if vers1 == vers2: 60 | return redirect('/param/change') 61 | elif vers1 != vers1_redirect or vers2 != vers2_redirect: 62 | redirect_url = '/param/change/{}/{}' 63 | return redirect(redirect_url.format(vers1_redirect, vers2_redirect)) 64 | 65 | vers1_html = _version_select_html(name='version_1', filter_default=vers1) 66 | vers2_html = _version_select_html(name='version_2', filter_default=vers2) 67 | 68 | try: 69 | config_changes = pgconfig.config_changes(vers1, vers2) 70 | except ValueError: 71 | return redirect('/param/change') 72 | 73 | config_changes_html = pgconfig.config_changes_html(config_changes) 74 | changes_stats = pgconfig.config_changes_stats(config_changes) 75 | return render_template('param_change.html', 76 | year=get_year(), 77 | config_changes=config_changes_html, 78 | changes_stats=changes_stats, 79 | vers1=vers1, 80 | vers2=vers2, 81 | vers1_html=vers1_html, 82 | vers2_html=vers2_html) 83 | 84 | 85 | @app.route('/param/change') 86 | def redirect_param_change(): 87 | return redirect('/param/change/{}/{}'.format(VERSION_PRIOR, VERSION_CURRENT)) 88 | 89 | 90 | @app.route('/custom') 91 | def redirect_custom_with_defaults(): 92 | """Route supporting removed feature. If users have this route bookmarked, 93 | don't just 404 on them. Gives hint at query to run to get the data directly 94 | in their database. 95 | """ 96 | return redirect('/custom/{}'.format(VERSION_CURRENT)) 97 | 98 | 99 | @app.route('/custom/', methods=['GET', 'POST']) 100 | def view_custom_config_comparison(vers1): 101 | """Route supporting removed feature. If users have this route bookmarked, 102 | don't just 404 on them. Gives hint at query to run to get the data directly 103 | in their database. 104 | """ 105 | return render_template('custom_config.html', 106 | year=get_year(), 107 | vers1=None, 108 | vers1_html=None, 109 | form=None) 110 | 111 | 112 | 113 | def get_param_select_html(filter_default: str='max_parallel_workers_per_gather') -> str: 114 | """Returns HTML of parameter options to select from on the "single parameter" 115 | page. 116 | 117 | Parameters 118 | ------------------------------------ 119 | filter_default : str 120 | Default value: max_parallel_workers_per_gather 121 | 122 | Returns 123 | ------------------------------------ 124 | html : str 125 | """ 126 | html = '' 139 | return html 140 | 141 | 142 | def _version_select_html(name, filter_default): 143 | html = '' 155 | return html 156 | -------------------------------------------------------------------------------- /webapp/static/logo.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/rustprooflabs/pgconfig-ce/10946d43989c3c26fb6ec24f6f85a1a212d48d41/webapp/static/logo.png -------------------------------------------------------------------------------- /webapp/static/style.css: -------------------------------------------------------------------------------- 1 | .page { 2 | background-color: #e6f0ff; 3 | color: #1a1a1a; 4 | } 5 | 6 | header { 7 | background-color: #66a3ff; 8 | } 9 | 10 | body { 11 | background-color: #e6e6e6; 12 | font-family: "Gill Sans", sans-serif; 13 | text-rendering: optimizeLegibility; 14 | font-variant: normal; 15 | } 16 | 17 | h1, h2 { 18 | font-variant: small-caps; 19 | } 20 | 21 | body html { 22 | height: 100%; 23 | font-size: 1.4em; 24 | } 25 | 26 | nav li { 27 | font-size: 1.6rem; 28 | margin: 0.4rem; 29 | } 30 | 31 | label { 32 | font-size: 1.5rem; 33 | } 34 | 35 | .footer { 36 | padding-top: 20px; 37 | padding-bottom: 20px; 38 | font-size:12px; 39 | text-align:center 40 | } 41 | 42 | .fa { 43 | font-size: 30px; 44 | text-align: center; 45 | text-decoration: none; 46 | padding: 5px; 47 | } 48 | 49 | .fa:hover { 50 | opacity: 0.7; 51 | } 52 | 53 | .error { 54 | color: red; 55 | } 56 | 57 | .text-center { 58 | justify-content: center; 59 | } 60 | -------------------------------------------------------------------------------- /webapp/templates/_layout_header.html: -------------------------------------------------------------------------------- 1 | postgresql.conf compare 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 22 | 23 | 24 | -------------------------------------------------------------------------------- /webapp/templates/about.html: -------------------------------------------------------------------------------- 1 | {% extends 'layout.html' %} 2 | {% block main_content %} 3 | 4 |
5 |

6 | About 7 |

8 | 9 |
10 |

11 | The postgresql.conf comparison tool was created to 12 | make it easier to track changes to the database's main configuration file. 13 | These configuration options, referred to by the Pg hackers as 14 | GUCs 15 | ( 16 | Grand Unified Configuration), 17 | do change over time with major versions. These changes typically reflect new features added, existing features 18 | improved, and other changes. 19 |

20 | 21 |

22 | This project is open source and available on 23 | GitHub. 24 |

25 | 26 |

27 | 28 | This blog post 29 | illustrates how this tool can be used to research 30 | and prepare for upgrading PostgreSQL major versions. 31 | 32 |

33 |
34 | 35 |

36 | Versions Available 37 |

38 | 39 |
40 |

41 | Configuration data is available for Postgres 10 and newer. 42 | Configurations for new versions will be added as they are released. 43 |

44 |

45 | NOTE: This tool previously included Postgres versions 9.2 through 9.6. 46 | Those versions were not included during the rewrite from using 47 | `postgresql.conf` directly to using the more robust comparison 48 | from `pg_catalog.pg_settings` data. 49 |

50 |
51 | 52 |
53 | 54 | {% endblock main_content %} -------------------------------------------------------------------------------- /webapp/templates/custom_config.html: -------------------------------------------------------------------------------- 1 | {% extends 'layout.html' %} 2 | {% block main_content %} 3 | 4 |
5 |

6 | compare your Configuration 7 |

8 |
9 |

10 | The compare your Configuration feature has been removed. 11 | To compare your Postgres configuration against defaults for your version 12 | explore the data returned by the following query. 13 |

14 | 15 | 
16 |     SELECT *
17 |         FROM pg_catalog.pg_settings
18 |         WHERE boot_val != setting
19 |     ;
20 |
21 | 22 |
23 | 24 | 25 |
26 | 27 | {% endblock main_content %} -------------------------------------------------------------------------------- /webapp/templates/footer.html: -------------------------------------------------------------------------------- 1 |
2 | About this tool 3 |
4 | © RustProof Labs, {{ year }} 5 | 6 |
7 | Maintained by 8 | RustProof Labs. 9 |
-------------------------------------------------------------------------------- /webapp/templates/header.html: -------------------------------------------------------------------------------- 1 |
2 | 3 |
4 |

5 | 9 | postgresql.conf comparison 10 |

11 |
12 | 13 | 47 | 48 |
-------------------------------------------------------------------------------- /webapp/templates/layout.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | {% include '_layout_header.html' %} 6 | 7 | 8 | 9 | 10 |
11 | {% block body %} 12 |
13 | {% include 'header.html' %} 14 |
15 | 16 |
17 | {% block main_content %} 18 | Main content will go here... 19 | {% endblock main_content %} 20 |
21 | 22 |
23 | {% include 'footer.html' %} 24 |
25 | {% endblock body %} 26 |
27 | 28 | 29 | 30 | -------------------------------------------------------------------------------- /webapp/templates/param.html: -------------------------------------------------------------------------------- 1 | {% extends 'layout.html' %} 2 | {% block main_content %} 3 | 4 |
5 |

6 | Postgres Parameter across Versions 7 |

8 | 9 |

10 | Select a parameter to view the defaults across versions of PostgreSQL. 11 |

12 | 13 |
14 | {{ select_html | safe }} 15 |
16 | 17 |
18 |
19 | 20 |

21 | Latest version details 22 |

23 |
24 | Category 25 | {{ param_details['details']['category'] }} 26 |
27 | 28 |
29 | 30 | Variable type: 31 | {{ param_details['details']['vartype'] }} 32 |
33 | Short Description: 34 | {{ param_details['details']['short_desc'] }} 35 | 36 |
37 | 38 | 39 |

40 | Value History 41 |

42 |
43 | {{ param_details['value_history'] | safe }} 44 |
45 | 46 |

47 | Vartype History 48 |

49 |
50 | {{ param_details['vartype_history'] | safe }} 51 |
52 |
53 |
54 | 55 | 56 |
57 | 58 | 66 | 67 | 68 | {% endblock main_content %} -------------------------------------------------------------------------------- /webapp/templates/param_change.html: -------------------------------------------------------------------------------- 1 | {% extends 'layout.html' %} 2 | {% block main_content %} 3 | 4 |
5 |

6 | Postgres Config Changes: {{ vers1 }} to {{ vers2 }} 7 |

8 | 9 |

10 | Select two versions of Postgres to see the differences between their 11 |  postgresql.conf  parameters and defaults. Parameters that remain 12 | the same in both versions are not displayed. 13 |

14 | 15 |

16 | Upgrade from {{ vers1_html | safe }} to {{ vers2_html | safe }} 17 |

18 | 19 | 20 |
21 | 22 |

23 | Updated: {{ changes_stats['updated'] }} 24 |

25 | 26 |

27 | New: {{ changes_stats['new'] }} 28 |

29 | 30 |

31 | Removed: {{ changes_stats['removed'] }} 32 |

33 | 34 |
35 | 36 | 37 |
38 | 39 |

40 | Updated Parameters 41 |

42 |
43 | 44 |
45 | {{ config_changes['changed'] | safe }} 46 |
47 |
48 | 49 | 50 |

51 | New Parameters 52 |

53 |
54 | 55 |
56 | {{ config_changes['new'] | safe }} 57 |
58 |
59 | 60 |

61 | Removed Parameters 62 |

63 |
64 | 65 |
66 | {{ config_changes['removed'] | safe }} 67 |
68 |
69 |
70 | 71 | 72 |
73 | 74 | 75 | 84 | {% endblock main_content %} --------------------------------------------------------------------------------