├── .editorconfig ├── .github ├── ISSUE_TEMPLATE │ ├── bug_report.md │ ├── feature_request.md │ └── other.md ├── dependabot.yml └── workflows │ └── tests.yml ├── .gitignore ├── .pylintrc ├── CONTRIBUTING.md ├── LICENSE ├── README.md ├── bors.toml ├── requirements.txt └── tools ├── build_image.py ├── config.py ├── destroy_image.py ├── publish_image.py ├── test_image.py └── utils.py /.editorconfig: -------------------------------------------------------------------------------- 1 | root = true 2 | 3 | [*] 4 | charset = utf-8 5 | end_of_line = lf 6 | indent_size = 4 7 | indent_style = space 8 | insert_final_newline = true 9 | 10 | # 4 space indentation 11 | [*.{py,java,r,R}] 12 | indent_style = space 13 | indent_size = 4 14 | 15 | # 2 space indentation 16 | [*.{js,json,y{a,}ml,html,cwl}] 17 | indent_style = space 18 | indent_size = 2 19 | 20 | [*.{md,Rmd,rst}] 21 | trim_trailing_whitespace = false 22 | indent_style = space 23 | indent_size = 2 24 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/bug_report.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Bug Report 🐞 3 | about: Create a report to help us improve. 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | --- 8 | 9 | 10 | 11 | **Description** 12 | Description of what the bug is about. 13 | 14 | **Expected behavior** 15 | What you expected to happen. 16 | 17 | **Current behavior** 18 | What happened. 19 | 20 | **Screenshots or Logs** 21 | If applicable, add screenshots or logs to help explain your problem. 22 | 23 | **Environment (please complete the following information):** 24 | - OS and version: [e.g. Debian 9] 25 | - Browser: [e.g. Chrome version 90.0] 26 | - Meilisearch version: [e.g. v.0.20.0] 27 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/feature_request.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Feature Request 💡 3 | about: Suggest a new idea for the project. 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | --- 8 | 9 | 10 | 11 | **Description** 12 | Brief explanation of the feature. 13 | 14 | **Basic example** 15 | If the proposal involves something new or a change, include a basic example. How would you use the feature? In which context? 16 | 17 | **Other** 18 | Any other things you want to add. 19 | -------------------------------------------------------------------------------- /.github/ISSUE_TEMPLATE/other.md: -------------------------------------------------------------------------------- 1 | --- 2 | name: Other 3 | about: Any other topic you want to talk about. 4 | title: '' 5 | labels: '' 6 | assignees: '' 7 | --- 8 | -------------------------------------------------------------------------------- /.github/dependabot.yml: -------------------------------------------------------------------------------- 1 | version: 2 2 | updates: 3 | - package-ecosystem: "github-actions" 4 | directory: "/" 5 | schedule: 6 | interval: "monthly" 7 | labels: 8 | - 'skip-changelog' 9 | - 'dependencies' 10 | rebase-strategy: disabled 11 | 12 | - package-ecosystem: pip 13 | directory: "/" 14 | schedule: 15 | interval: "weekly" 16 | time: "04:00" 17 | open-pull-requests-limit: 10 18 | labels: 19 | - skip-changelog 20 | - dependencies 21 | rebase-strategy: disabled 22 | -------------------------------------------------------------------------------- /.github/workflows/tests.yml: -------------------------------------------------------------------------------- 1 | name: Tests 2 | 3 | on: 4 | pull_request: 5 | push: 6 | # trying and staging branches are for BORS config 7 | branches: 8 | - trying 9 | - staging 10 | - main 11 | 12 | jobs: 13 | tests: 14 | # Will not run if the event is a PR to bump-meilisearch-v* (so a pre-release PR) 15 | # Will still run for each push to bump-meilisearch-v* 16 | if: github.event_name != 'pull_request' || !startsWith(github.base_ref, 'bump-meilisearch-v') 17 | name: build-image-test 18 | runs-on: ubuntu-latest 19 | steps: 20 | - uses: actions/checkout@v3 21 | - name: Set up Python 3.7 22 | uses: actions/setup-python@v4 23 | with: 24 | python-version: 3.7 25 | - name: Install dependencies 26 | run: pip3 install -r requirements.txt 27 | - name: Set up Cloud SDK 28 | uses: google-github-actions/setup-gcloud@v0.6.2 29 | with: 30 | service_account_key: ${{ secrets.GOOGLE_APPLICATION_CREDENTIALS }} 31 | export_default_credentials: true 32 | - name: Set image name as env variable 33 | run: echo "IMAGE_NAME=meilisearch-gcp-ci-test-$(date +'%d-%m-%Y-%H-%M-%S')" >> $GITHUB_ENV 34 | - name: Build image 35 | run: python3 tools/build_image.py ${{ env.IMAGE_NAME }} --no-analytics 36 | - name: Test image 37 | run: python3 tools/test_image.py ${{ env.IMAGE_NAME }} 38 | - name: Clean image 39 | if: ${{ always() }} 40 | run: python3 tools/destroy_image.py ${{ env.IMAGE_NAME }} 41 | 42 | pylint: 43 | name: pylint 44 | runs-on: ubuntu-latest 45 | steps: 46 | - uses: actions/checkout@v3 47 | - name: Set up Python 3.7 48 | uses: actions/setup-python@v4 49 | with: 50 | python-version: 3.7 51 | - name: Install dependencies 52 | run: pip3 install -r requirements.txt 53 | - name: Linter with pylint 54 | run: pylint tools 55 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pem 2 | 3 | # Byte-compiled / optimized / DLL files 4 | __pycache__/ 5 | *.py[cod] 6 | *$py.class 7 | 8 | # C extensions 9 | *.so 10 | 11 | # Distribution / packaging 12 | .Python 13 | build/ 14 | develop-eggs/ 15 | dist/ 16 | downloads/ 17 | eggs/ 18 | .eggs/ 19 | lib/ 20 | lib64/ 21 | parts/ 22 | sdist/ 23 | var/ 24 | wheels/ 25 | share/python-wheels/ 26 | *.egg-info/ 27 | .installed.cfg 28 | *.egg 29 | MANIFEST 30 | 31 | # PyInstaller 32 | # Usually these files are written by a python script from a template 33 | # before PyInstaller builds the exe, so as to inject date/other infos into it. 34 | *.manifest 35 | *.spec 36 | 37 | # Installer logs 38 | pip-log.txt 39 | pip-delete-this-directory.txt 40 | 41 | # Unit test / coverage reports 42 | htmlcov/ 43 | .tox/ 44 | .nox/ 45 | .coverage 46 | .coverage.* 47 | .cache 48 | nosetests.xml 49 | coverage.xml 50 | *.cover 51 | *.py,cover 52 | .hypothesis/ 53 | .pytest_cache/ 54 | cover/ 55 | 56 | # Translations 57 | *.mo 58 | *.pot 59 | 60 | # Django stuff: 61 | *.log 62 | local_settings.py 63 | db.sqlite3 64 | db.sqlite3-journal 65 | 66 | # Flask stuff: 67 | instance/ 68 | .webassets-cache 69 | 70 | # Scrapy stuff: 71 | .scrapy 72 | 73 | # Sphinx documentation 74 | docs/_build/ 75 | 76 | # PyBuilder 77 | .pybuilder/ 78 | target/ 79 | 80 | # Jupyter Notebook 81 | .ipynb_checkpoints 82 | 83 | # IPython 84 | profile_default/ 85 | ipython_config.py 86 | 87 | # pyenv 88 | # For a library or package, you might want to ignore these files since the code is 89 | # intended to run in multiple environments; otherwise, check them in: 90 | # .python-version 91 | 92 | # pipenv 93 | # According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control. 94 | # However, in case of collaboration, if having platform-specific dependencies or dependencies 95 | # having no cross-platform support, pipenv may install dependencies that don't work, or not 96 | # install all needed dependencies. 97 | #Pipfile.lock 98 | 99 | # PEP 582; used by e.g. github.com/David-OConnor/pyflow 100 | __pypackages__/ 101 | 102 | # Celery stuff 103 | celerybeat-schedule 104 | celerybeat.pid 105 | 106 | # SageMath parsed files 107 | *.sage.py 108 | 109 | # Environments 110 | .env 111 | .venv 112 | env/ 113 | venv/ 114 | ENV/ 115 | env.bak/ 116 | venv.bak/ 117 | 118 | # Spyder project settings 119 | .spyderproject 120 | .spyproject 121 | 122 | # Rope project settings 123 | .ropeproject 124 | 125 | # mkdocs documentation 126 | /site 127 | 128 | # mypy 129 | .mypy_cache/ 130 | .dmypy.json 131 | dmypy.json 132 | 133 | # Pyre type checker 134 | .pyre/ 135 | 136 | # pytype static type analyzer 137 | .pytype/ 138 | 139 | # Cython debug symbols 140 | cython_debug/ 141 | 142 | env_digitalocean 143 | 144 | .DS_Store 145 | .vscode/ 146 | -------------------------------------------------------------------------------- /.pylintrc: -------------------------------------------------------------------------------- 1 | [MASTER] 2 | 3 | # Specify a configuration file. 4 | #rcfile= 5 | 6 | # Python code to execute, usually for sys.path manipulation such as 7 | # pygtk.require(). 8 | #init-hook= 9 | 10 | # Pickle collected data for later comparisons. 11 | persistent=yes 12 | 13 | # List of plugins (as comma separated values of python modules names) to load, 14 | # usually to register additional checkers. 15 | # load-plugins= 16 | 17 | # Use multiple processes to speed up Pylint. 18 | jobs=1 19 | 20 | # Allow loading of arbitrary C extensions. Extensions are imported into the 21 | # active Python interpreter and may run arbitrary code. 22 | unsafe-load-any-extension=no 23 | 24 | # A comma-separated list of package or module names from where C extensions may 25 | # be loaded. Extensions are loading into the active Python interpreter and may 26 | # run arbitrary code 27 | extension-pkg-whitelist= 28 | 29 | 30 | [MESSAGES CONTROL] 31 | 32 | # Only show warnings with the listed confidence levels. Leave empty to show 33 | # all. Valid levels: HIGH, INFERENCE, INFERENCE_FAILURE, UNDEFINED 34 | confidence= 35 | 36 | # Enable the message, report, category or checker with the given id(s). You can 37 | # either give multiple identifier separated by comma (,) or put this option 38 | # multiple time. See also the "--disable" option for examples. 39 | enable=use-symbolic-message-instead,useless-supression,fixme 40 | 41 | # Disable the message, report, category or checker with the given id(s). You 42 | # can either give multiple identifiers separated by comma (,) or put this 43 | # option multiple times (only on the command line, not in the configuration 44 | # file where it should appear only once).You can also use "--disable=all" to 45 | # disable everything first and then reenable specific checks. For example, if 46 | # you want to run only the similarities checker, you can use "--disable=all 47 | # --enable=similarities". If you want to run only the classes checker, but have 48 | # no Warning level messages displayed, use"--disable=all --enable=classes 49 | # --disable=W" 50 | 51 | disable= 52 | attribute-defined-outside-init, 53 | duplicate-code, 54 | missing-docstring, 55 | too-few-public-methods, 56 | line-too-long, 57 | broad-except 58 | 59 | 60 | [REPORTS] 61 | 62 | # Set the output format. Available formats are text, parseable, colorized, msvs 63 | # (visual studio) and html. You can also give a reporter class, eg 64 | # mypackage.mymodule.MyReporterClass. 65 | output-format=text 66 | 67 | # Put messages in a separate file for each module / package specified on the 68 | # command line instead of printing them on stdout. Reports (if any) will be 69 | # written in a file name "pylint_global.[txt|html]". 70 | files-output=no 71 | 72 | # Tells whether to display a full report or only the messages 73 | reports=no 74 | 75 | # Python expression which should return a note less than 10 (10 is the highest 76 | # note). You have access to the variables errors warning, statement which 77 | # respectively contain the number of errors / warnings messages and the total 78 | # number of statements analyzed. This is used by the global evaluation report 79 | # (RP0004). 80 | evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) 81 | 82 | # Template used to display messages. This is a python new-style format string 83 | # used to format the message information. See doc for all details 84 | #msg-template= 85 | 86 | 87 | [LOGGING] 88 | 89 | # Logging modules to check that the string format arguments are in logging 90 | # function parameter format 91 | logging-modules=logging 92 | 93 | 94 | [MISCELLANEOUS] 95 | 96 | # List of note tags to take in consideration, separated by a comma. 97 | notes=FIXME,XXX,TODO 98 | 99 | 100 | [SIMILARITIES] 101 | 102 | # Minimum lines number of a similarity. 103 | min-similarity-lines=4 104 | 105 | # Ignore comments when computing similarities. 106 | ignore-comments=yes 107 | 108 | # Ignore docstrings when computing similarities. 109 | ignore-docstrings=yes 110 | 111 | # Ignore imports when computing similarities. 112 | ignore-imports=no 113 | 114 | 115 | [VARIABLES] 116 | 117 | # Tells whether we should check for unused import in __init__ files. 118 | init-import=no 119 | 120 | # A regular expression matching the name of dummy variables (i.e. expectedly 121 | # not used). 122 | dummy-variables-rgx=_$|dummy 123 | 124 | # List of additional names supposed to be defined in builtins. Remember that 125 | # you should avoid defining new builtins when possible. 126 | additional-builtins= 127 | 128 | # List of strings which can identify a callback function by name. A callback 129 | # name must start or end with one of those strings. 130 | callbacks=cb_,_cb 131 | 132 | 133 | [FORMAT] 134 | 135 | # Maximum number of characters on a single line. 136 | # max-line-length=100 137 | 138 | # Regexp for a line that is allowed to be longer than the limit. 139 | # ignore-long-lines=^\s*(# )??$ 140 | 141 | # Allow the body of an if to be on the same line as the test if there is no 142 | # else. 143 | single-line-if-stmt=no 144 | 145 | # List of optional constructs for which whitespace checking is disabled 146 | no-space-check=trailing-comma,dict-separator 147 | 148 | # Maximum number of lines in a module 149 | max-module-lines=2000 150 | 151 | # String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 152 | # tab). 153 | indent-string=' ' 154 | 155 | # Number of spaces of indent required inside a hanging or continued line. 156 | indent-after-paren=4 157 | 158 | # Expected format of line ending, e.g. empty (any line ending), LF or CRLF. 159 | expected-line-ending-format= 160 | 161 | 162 | [BASIC] 163 | 164 | # List of builtins function names that should not be used, separated by a comma 165 | bad-functions=map,filter,input 166 | 167 | # Good variable names which should always be accepted, separated by a comma 168 | good-names=i,j,k,ex,Run,_ 169 | 170 | # Bad variable names which should always be refused, separated by a comma 171 | bad-names=foo,bar,baz,toto,tutu,tata 172 | 173 | # Colon-delimited sets of names that determine each other's naming style when 174 | # the name regexes allow several styles. 175 | name-group= 176 | 177 | # Include a hint for the correct naming format with invalid-name 178 | include-naming-hint=no 179 | 180 | # Regular expression matching correct function names 181 | function-rgx=[a-z_][a-z0-9_]{2,30}$ 182 | 183 | # Naming hint for function names 184 | function-name-hint=[a-z_][a-z0-9_]{2,30}$ 185 | 186 | # Regular expression matching correct variable names 187 | variable-rgx=[a-z_][a-z0-9_]{2,30}$ 188 | 189 | # Naming hint for variable names 190 | variable-name-hint=[a-z_][a-z0-9_]{2,30}$ 191 | 192 | # Regular expression matching correct constant names 193 | const-rgx=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 194 | 195 | # Naming hint for constant names 196 | const-name-hint=(([A-Z_][A-Z0-9_]*)|(__.*__))$ 197 | 198 | # Regular expression matching correct attribute names 199 | attr-rgx=[a-z_][a-z0-9_]{2,}$ 200 | 201 | # Naming hint for attribute names 202 | attr-name-hint=[a-z_][a-z0-9_]{2,}$ 203 | 204 | # Regular expression matching correct argument names 205 | argument-rgx=[a-z_][a-z0-9_]{2,30}$ 206 | 207 | # Naming hint for argument names 208 | argument-name-hint=[a-z_][a-z0-9_]{2,30}$ 209 | 210 | # Regular expression matching correct class attribute names 211 | class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 212 | 213 | # Naming hint for class attribute names 214 | class-attribute-name-hint=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ 215 | 216 | # Regular expression matching correct inline iteration names 217 | inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ 218 | 219 | # Naming hint for inline iteration names 220 | inlinevar-name-hint=[A-Za-z_][A-Za-z0-9_]*$ 221 | 222 | # Regular expression matching correct class names 223 | class-rgx=[A-Z_][a-zA-Z0-9]+$ 224 | 225 | # Naming hint for class names 226 | class-name-hint=[A-Z_][a-zA-Z0-9]+$ 227 | 228 | # Regular expression matching correct module names 229 | module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 230 | 231 | # Naming hint for module names 232 | module-name-hint=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ 233 | 234 | # Regular expression matching correct method names 235 | method-rgx=[a-z_][a-z0-9_]{2,}$ 236 | 237 | # Naming hint for method names 238 | method-name-hint=[a-z_][a-z0-9_]{2,}$ 239 | 240 | # Regular expression which should only match function or class names that do 241 | # not require a docstring. 242 | no-docstring-rgx=__.*__ 243 | 244 | # Minimum line length for functions/classes that require docstrings, shorter 245 | # ones are exempt. 246 | docstring-min-length=-1 247 | 248 | # List of decorators that define properties, such as abc.abstractproperty. 249 | property-classes=abc.abstractproperty 250 | 251 | 252 | [TYPECHECK] 253 | 254 | # Tells whether missing members accessed in mixin class should be ignored. A 255 | # mixin class is detected if its name ends with "mixin" (case insensitive). 256 | ignore-mixin-members=yes 257 | 258 | # List of module names for which member attributes should not be checked 259 | # (useful for modules/projects where namespaces are manipulated during runtime 260 | # and thus existing member attributes cannot be deduced by static analysis 261 | ignored-modules= 262 | 263 | # List of classes names for which member attributes should not be checked 264 | # (useful for classes with attributes dynamically set). 265 | ignored-classes=SQLObject, optparse.Values, thread._local, _thread._local, googleapiclient.discovery.Resource 266 | 267 | # List of members which are set dynamically and missed by pylint inference 268 | # system, and so shouldn't trigger E1101 when accessed. Python regular 269 | # expressions are accepted. 270 | generated-members=REQUEST,acl_users,aq_parent 271 | 272 | # List of decorators that create context managers from functions, such as 273 | # contextlib.contextmanager. 274 | contextmanager-decorators=contextlib.contextmanager 275 | 276 | 277 | [SPELLING] 278 | 279 | # Spelling dictionary name. Available dictionaries: none. To make it working 280 | # install python-enchant package. 281 | spelling-dict= 282 | 283 | # List of comma separated words that should not be checked. 284 | spelling-ignore-words= 285 | 286 | # A path to a file that contains private dictionary; one word per line. 287 | spelling-private-dict-file= 288 | 289 | # Tells whether to store unknown words to indicated private dictionary in 290 | # --spelling-private-dict-file option instead of raising a message. 291 | spelling-store-unknown-words=no 292 | 293 | 294 | [DESIGN] 295 | 296 | # Maximum number of arguments for function / method 297 | max-args=10 298 | 299 | # Argument names that match this expression will be ignored. Default to name 300 | # with leading underscore 301 | ignored-argument-names=_.* 302 | 303 | # Maximum number of locals for function / method body 304 | max-locals=25 305 | 306 | # Maximum number of return / yield for function / method body 307 | max-returns=11 308 | 309 | # Maximum number of branch for function / method body 310 | max-branches=26 311 | 312 | # Maximum number of statements in function / method body 313 | max-statements=100 314 | 315 | # Maximum number of parents for a class (see R0901). 316 | max-parents=7 317 | 318 | # Maximum number of attributes for a class (see R0902). 319 | max-attributes=11 320 | 321 | # Minimum number of public methods for a class (see R0903). 322 | min-public-methods=2 323 | 324 | # Maximum number of public methods for a class (see R0904). 325 | max-public-methods=25 326 | 327 | 328 | [CLASSES] 329 | 330 | # List of method names used to declare (i.e. assign) instance attributes. 331 | defining-attr-methods=__init__,__new__,setUp,__post_init__ 332 | 333 | # List of valid names for the first argument in a class method. 334 | valid-classmethod-first-arg=cls 335 | 336 | # List of valid names for the first argument in a metaclass class method. 337 | valid-metaclass-classmethod-first-arg=mcs 338 | 339 | # List of member names, which should be excluded from the protected access 340 | # warning. 341 | exclude-protected=_asdict,_fields,_replace,_source,_make 342 | 343 | 344 | [IMPORTS] 345 | 346 | # Deprecated modules which should not be used, separated by a comma 347 | deprecated-modules=regsub,TERMIOS,Bastion,rexec 348 | 349 | # Create a graph of every (i.e. internal and external) dependencies in the 350 | # given file (report RP0402 must not be disabled) 351 | import-graph= 352 | 353 | # Create a graph of external dependencies in the given file (report RP0402 must 354 | # not be disabled) 355 | ext-import-graph= 356 | 357 | # Create a graph of internal dependencies in the given file (report RP0402 must 358 | # not be disabled) 359 | int-import-graph= 360 | 361 | 362 | [EXCEPTIONS] 363 | 364 | # Exceptions that will emit a warning when being caught. Defaults to 365 | # "Exception" 366 | overgeneral-exceptions=Exception 367 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | # Contributing 2 | 3 | First of all, thank you for contributing to Meilisearch! The goal of this document is to provide everything you need to know in order to contribute to Meilisearch and its different integrations. 4 | 5 | - [Assumptions](#assumptions) 6 | - [How to Contribute](#how-to-contribute) 7 | - [Development Workflow](#development-workflow) 8 | - [Git Guidelines](#git-guidelines) 9 | - [Release Process (for internal team only)](#release-process-for-internal-team-only) 10 | 11 | ## Assumptions 12 | 13 | 1. **You're familiar with [GitHub](https://github.com) and the [Pull Request](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/about-pull-requests)(PR) workflow.** 14 | 2. **You've read the Meilisearch [documentation](https://docs.meilisearch.com) and the [README](/README.md).** 15 | 3. **You know about the [Meilisearch community](https://docs.meilisearch.com/resources/contact.html). Please use this for help.** 16 | 17 | ## How to Contribute 18 | 19 | 1. Make sure that the contribution you want to make is explained or detailed in a GitHub issue! Find an [existing issue](https://github.com/meilisearch/meilisearch-gcp/issues/) or [open a new one](https://github.com/meilisearch/meilisearch-gcp/issues/new). 20 | 2. Once done, [fork the meilisearch-gcp repository](https://help.github.com/en/github/getting-started-with-github/fork-a-repo) in your own GitHub account. Ask a maintainer if you want your issue to be checked before making a PR. 21 | 3. [Create a new Git branch](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-and-deleting-branches-within-your-repository). 22 | 4. Make the changes on your branch. 23 | 5. [Submit the branch as a PR](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/creating-a-pull-request-from-a-fork) pointing to the `main` branch of the main meilisearch-gcp repository. A maintainer should comment and/or review your Pull Request within a few days. Although depending on the circumstances, it may take longer.
24 | We do not enforce a naming convention for the PRs, but **please use something descriptive of your changes**, having in mind that the title of your PR will be automatically added to the next [release changelog](https://github.com/meilisearch/meilisearch-gcp/releases/). 25 | 26 | ## Development Workflow 27 | 28 | ### Setup 29 | 30 | ```bash 31 | pip3 install -r requirements.txt 32 | ``` 33 | 34 | ### Tests and Linter 35 | 36 | Each PR should pass the tests and the linter to be accepted. 37 | 38 | ```bash 39 | # Linter 40 | pylint tools 41 | ``` 42 | 43 | ## Git Guidelines 44 | 45 | ### Git Branches 46 | 47 | All changes must be made in a branch and submitted as PR. 48 | We do not enforce any branch naming style, but please use something descriptive of your changes. 49 | 50 | ### Git Commits 51 | 52 | As minimal requirements, your commit message should: 53 | - be capitalized 54 | - not finish by a dot or any other punctuation character (!,?) 55 | - start with a verb so that we can read your commit message this way: "This commit will ...", where "..." is the commit message. 56 | e.g.: "Fix the home page button" or "Add more tests for create_index method" 57 | 58 | We don't follow any other convention, but if you want to use one, we recommend [this one](https://chris.beams.io/posts/git-commit/). 59 | 60 | ### GitHub Pull Requests 61 | 62 | Some notes on GitHub PRs: 63 | 64 | - [Convert your PR as a draft](https://help.github.com/en/github/collaborating-with-issues-and-pull-requests/changing-the-stage-of-a-pull-request) if your changes are a work in progress: no one will review it until you pass your PR as ready for review.
65 | The draft PR can be very useful if you want to show that you are working on something and make your work visible. 66 | - The branch related to the PR must be **up-to-date with `main`** before merging. If it's not, you have to rebase your branch. Check out this [quick tutorial](https://gist.github.com/curquiza/5f7ce615f85331f083cd467fc4e19398) to successfully apply the rebase from a forked repository. 67 | - All PRs must be reviewed and approved by at least one maintainer. 68 | - The PR title should be accurate and descriptive of the changes. 69 | 70 | ## Release Process (for the internal team only) 71 | 72 | ⚠️ The [cloud-scripts](https://github.com/meilisearch/cloud-scripts) repository should be upgraded to the new version before this repository can be updated and released. 73 | 74 | The release tags of this package follow exactly the Meilisearch versions.
75 | It means that, for example, the `v0.17.0` tag in this repository corresponds to the GCP image running Meilisearch `v0.17.0`. 76 | 77 | This repository currently does not provide any automated way to test the GCP Meilisearch image.
78 | **Please, follow carefully the steps in the next sections before any release.** 79 | 80 | ### Set your environment 81 | 82 | After cloning this repository, install python dependencies with the following command: 83 | 84 | ```bash 85 | pip3 install -r requirements.txt 86 | ``` 87 | 88 | Before running any script, make sure to [set your GCP credentials](https://cloud.google.com/docs/authentication/getting-started) locally. Make sure that youhave access to the project `meilisearchimage`, or ask a maintainer for a Service Account access.
89 | 90 | ### Test before Releasing 91 | 92 | 1. Add your SSH key to the Compute Engine platform, on the [metadata section](https://console.cloud.google.com/compute/metadata/sshKeys?project=meilisearchimage&authuser=1&folder&organizationId=710828868196). Set the name of the user defined on your SSH key as a value for the `SSH_USER` variable in the [`tools/config.py`](tools/config.py) script. (I.E. If your SSH key finishes by `myname@mylaptop` use `myname` as the value of `SSH_USER`) 93 | 94 | 2. In [`tools/config.py`](tools/config.py), update the `MEILI_CLOUD_SCRIPTS_VERSION_TAG` variable value with the new Meilisearch version you want to release, in the format: `vX.X.X`. If you want to test with a Meilisearch RC, replace it by the right RC version tag (`vX.X.XrcX`). 95 | 96 | 3. Run the [`tools/build_image.py`](tools/build_image.py) script to build the GCP custom image without analytics: 97 | 98 | ```bash 99 | python3 tools/build_image.py --no-analytics 100 | ``` 101 | 102 | This command will create an GCP Compute Instance on Meilisearch's account and configure it in order to prepare the Meilisearch custom image. It will then create an Image, which should be ready to be published in the following steps. The instance will automatically be terminated after the custom image creation.
103 | The image name will be `Meilisearch-vX-X-X-ubuntu-X--lts-build--XX-XX-XXXX`. 104 | 105 | 4. Test the image: create a new GCP Compute instance based on the new image `Meilisearch-vX-X-X-ubuntu-X--lts-build--XX-XX-XXXX`, and make sure everything is running smoothly. Remember to check the following checkboxes: `Allow HTTP traffic` and `Allow HTTPS traffic`. Connect via SSH to the droplet and test the configuration script that is run automatically on login. Use the same username defined in the step 1 and in the SSH key you have set on GCP.
106 | 🗑 Don't forget to destroy the Droplet after the test. 107 | 108 | 5. When you are ready to create the final image to release juste remove the `--no-analytics` option 109 | 110 | ```bash 111 | python3 tools/build_image.py 112 | ``` 113 | 114 | ### Publish the GCP image and Release 115 | 116 | ⚠️ The GCP image should never be published with a `RC` version of Meilisearch. 117 | 118 | Once the tests in the previous section have been done: 119 | 120 | 1. Set the image name that you TESTED and you want to publish as the value of the `PUBLISH_IMAGE_NAME` in [`tools/config.py`](tools/config.py). Use the exact name of the IMAGE that you built in the previous step: `Meilisearch-vX-X-X-ubuntu-X--lts-build--XX-XX-XXXX`. 121 | 122 | 2. Run the [`tools/publish_image.py`](tools/publish_image.py) script to export and publish the GCP image: 123 | 124 | ```bash 125 | python3 tools/publish_image.py 126 | ``` 127 | 128 | 3. Commit your changes on a new branch. 129 | 130 | 4. Open a PR from the branch where changes where done and merge it. 131 | 132 | 5. Create a git tag on the last `main` commit: 133 | 134 | ```bash 135 | git checkout main 136 | git pull origin main 137 | git tag vX.X.X 138 | git push origin vX.X.X 139 | ``` 140 | 141 | ⚠️ If changes where made to the repository between your testing branch was created and the moment it was merged, you should consider building the image and testing it again. Some important changes may have been introduced, unexpectedly changing the behavior of the image that will be published to the Marketplace. 142 | 143 | ### Clean old GCP images 144 | 145 | Make sure that the last 2 versions of the Meilisearch image are available and public. Our goal is to always offer the latest Meilisearch version to GCP users, but we are keeping the previous version in case there is a bug or a problem in the latest one. Any other older version of the image must be deleted. 146 | 147 | To proceed to delete older images that should no longer be available, use the GCP user interface, since this task hasn't yet been automated. 148 | 149 | ### Update the GCP image between two Meilisearch Releases 150 | 151 | It can happen that you need to release a new GCP image but you cannot wait for the new Meilisearch release.
152 | For example, the `v0.17.0` is already pushed but you find out you need to fix the installation script: you can't wait for the `v0.18.0` release and need to re-publish the `v0.17.0` GCP image. 153 | 154 | In this case: 155 | 156 | - Apply your changes and reproduce the testing process (see [Test before Releasing](#test-before-releasing)). 157 | - Delete the current tag remotely and locally: 158 | 159 | ```bash 160 | git push --delete origin vX.X.X 161 | git tag -d vX.X.X 162 | ``` 163 | 164 | - Publish the GCP image again (see [Publish the GCP image and Release](#publish-the-gcp-image-and-release)) 165 | 166 |
167 | 168 | Thank you again for reading this through. We can not wait to begin to work with you if you make your way through this contributing guide ❤️ 169 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | MIT License 2 | 3 | Copyright (c) 2020-2022 Meili SAS 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 |

2 | Meilisearch-GCP 3 |

4 | 5 |

[DEPRECATED] Meilisearch GCP

6 | 7 |

8 | Meilisearch | 9 | Documentation | 10 | Discord | 11 | Roadmap | 12 | Website | 13 | FAQ 14 |

15 | 16 |

17 | License 18 | Bors enabled 19 |

20 | 21 |

⚡ Meilisearch tools for GCP integration

22 | 23 | --------- 24 | 25 | ⚠️ **Please note that this repository is no longer being maintained and has been deprecated. GCP is still a provider we maintain integration with, and this is the new repository [cloud-providers](https://github.com/meilisearch/cloud-providers).** 26 | 27 | --------- 28 | 29 | **Meilisearch GCP** is a set of tools and scripts to improve user deployment of Meilisearch on [Google Cloud Platform](https://cloud.google.com). 30 | 31 | **Meilisearch** is an open-source search engine. [Discover what Meilisearch is!](https://github.com/meilisearch/meilisearch) 32 | 33 | ## Table of Contents 34 | 35 | - [🚀 How to deploy Meilisearch on GCP](#-how-to-deploy-meilisearch-on-gcp) 36 | - [🎁 Content of this repository](#-content-of-this-repository) 37 | - [📖 Documentation](#-documentation) 38 | - [⚙️ Development Workflow and Contributing](#️-development-workflow-and-contributing) 39 | 40 | ## 🚀 How to deploy Meilisearch on GCP 41 | 42 | [Coming soon...] 43 | 44 | ## 🎁 Content of this repository 45 | 46 | This repository contains a few tools and scripts used mainly by the Meilisearch team, aiming to provide our users simple ways to deploy and configure Meilisearch in the cloud. As our heart resides on the Open Source community, we maintain several of this tools as open source repository. 47 | 48 | ## 📖 Documentation 49 | 50 | See our [Documentation](https://docs.meilisearch.com/learn/tutorials/getting_started.html) or our [API References](https://docs.meilisearch.com/reference/api/). 51 | 52 | ## ⚙️ Development Workflow and Contributing 53 | 54 | Any new contribution is more than welcome in this project! 55 | 56 | If you want to know more about the development workflow or want to contribute, please visit our [contributing guidelines](/CONTRIBUTING.md) for detailed instructions! 57 | 58 |
59 | 60 | **Meilisearch** provides and maintains many **SDKs and Integration tools** like this one. We want to provide everyone with an **amazing search experience for any kind of project**. If you want to contribute, make suggestions, or just know what's going on right now, visit us in the [integration-guides](https://github.com/meilisearch/integration-guides) repository. 61 | -------------------------------------------------------------------------------- /bors.toml: -------------------------------------------------------------------------------- 1 | status = [ 2 | 'pylint', 3 | 'build-image-test' 4 | ] 5 | # 1 hour timeout 6 | timeout-sec = 3600 7 | -------------------------------------------------------------------------------- /requirements.txt: -------------------------------------------------------------------------------- 1 | cachetools==5.3.0 2 | certifi==2022.12.7 3 | cffi==1.15.1 4 | chardet==5.1.0 5 | google-api-core==2.11.0 6 | google-api-python-client==2.84.0 7 | google-auth==2.17.2 8 | google-auth-httplib2==0.1.0 9 | google-cloud-core==2.3.2 10 | google-cloud-storage==2.8.0 11 | google-crc32c==1.5.0 12 | google-resumable-media==2.4.1 13 | googleapis-common-protos==1.59.0 14 | httplib2==0.22.0 15 | idna==3.4 16 | packaging==23.0 17 | protobuf==4.22.1 18 | pyasn1==0.4.8 19 | pyasn1-modules==0.2.8 20 | pycparser==2.21 21 | pylint==2.14.3 22 | pyparsing==3.0.9 23 | pytz==2023.3 24 | requests==2.28.2 25 | rsa==4.9 26 | six==1.16.0 27 | uritemplate==4.1.1 28 | urllib3==1.26.15 29 | -------------------------------------------------------------------------------- /tools/build_image.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import time 3 | import sys 4 | import googleapiclient.discovery 5 | import config as conf 6 | import utils 7 | 8 | compute = googleapiclient.discovery.build('compute', 'v1') 9 | 10 | # Remove analytics for CI jobs 11 | 12 | if len(sys.argv) > 1 and '--no-analytics' in sys.argv: 13 | print('Launch build image without analytics.') 14 | config = conf.BUILD_INSTANCE_CONFIG.get('metadata').get('items')[0].get('value') 15 | test = conf.BUILD_INSTANCE_CONFIG['metadata']['items'][0]['value'] 16 | if '--env development' in conf.BUILD_INSTANCE_CONFIG['metadata']['items'][0]['value']: 17 | index = conf.BUILD_INSTANCE_CONFIG['metadata']['items'][0]['value'].find('--env development') 18 | conf.BUILD_INSTANCE_CONFIG['metadata']['items'][0]['value'] = conf.BUILD_INSTANCE_CONFIG['metadata']['items'][0]['value'][:index] + '--no-analytics ' + conf.BUILD_INSTANCE_CONFIG['metadata']['items'][0]['value'][index:] 19 | 20 | # Create GCP Compute instance to setup Meilisearch 21 | 22 | print('Creating GCP Compute instance') 23 | 24 | source_image = compute.images().getFromFamily( 25 | project=conf.DEBIAN_BASE_IMAGE_PROJECT, 26 | family=conf.DEBIAN_BASE_IMAGE_FAMILY 27 | ).execute() 28 | 29 | instance_config = conf.BUILD_INSTANCE_CONFIG 30 | instance_config['disks'][0]['initializeParams']['sourceImage'] = source_image['selfLink'] 31 | 32 | create = compute.instances().insert( 33 | project=conf.GCP_DEFAULT_PROJECT, 34 | zone=conf.GCP_DEFAULT_ZONE, 35 | body=instance_config).execute() 36 | print(f' Instance created. Name: {conf.INSTANCE_BUILD_NAME}') 37 | 38 | 39 | # Wait for GCP instance to be 'RUNNING' 40 | 41 | print('Waiting for GCP Compute instance state to be "RUNNING"') 42 | state_code, state, instance = utils.wait_for_instance_running( 43 | conf.GCP_DEFAULT_PROJECT, conf.GCP_DEFAULT_ZONE, timeout_seconds=600) 44 | print(f' Instance state: {state}') 45 | 46 | if state_code == utils.STATUS_OK: 47 | instance = compute.instances().get(project=conf.GCP_DEFAULT_PROJECT, 48 | zone=conf.GCP_DEFAULT_ZONE, instance=conf.INSTANCE_BUILD_NAME).execute() 49 | instance_ip = instance['networkInterfaces'][0]['accessConfigs'][0]['natIP'] 50 | print(f' Instance IP: {instance_ip}') 51 | else: 52 | print(f' Error: {state_code}. State: {state}.') 53 | utils.terminate_instance_and_exit( 54 | compute=compute, 55 | project=conf.GCP_DEFAULT_PROJECT, 56 | zone=conf.GCP_DEFAULT_ZONE, 57 | instance=conf.INSTANCE_BUILD_NAME 58 | ) 59 | 60 | # Wait for Health check after configuration is finished 61 | 62 | print('Waiting for Meilisearch health check (may take a few minutes: config and reboot)') 63 | HEALTH = utils.wait_for_health_check(instance_ip, timeout_seconds=600) 64 | if HEALTH == utils.STATUS_OK: 65 | print(' Instance is healthy') 66 | else: 67 | print(' Timeout waiting for health check') 68 | utils.terminate_instance_and_exit( 69 | compute=compute, 70 | project=conf.GCP_DEFAULT_PROJECT, 71 | zone=conf.GCP_DEFAULT_ZONE, 72 | instance=conf.INSTANCE_BUILD_NAME 73 | ) 74 | 75 | # Check version 76 | 77 | print('Waiting for Version check') 78 | try: 79 | utils.check_meilisearch_version( 80 | instance_ip, conf.MEILI_CLOUD_SCRIPTS_VERSION_TAG) 81 | except Exception as err: 82 | print(f" Exception: {err}") 83 | utils.terminate_instance_and_exit( 84 | compute=compute, 85 | project=conf.GCP_DEFAULT_PROJECT, 86 | zone=conf.GCP_DEFAULT_ZONE, 87 | instance=conf.INSTANCE_BUILD_NAME 88 | ) 89 | print(' Version of Meilisearch match!') 90 | 91 | # Stop instance before image creation 92 | 93 | print('Stopping GCP instance...') 94 | 95 | TIMEOUT_SECONDS=60 96 | start_time = datetime.datetime.now() 97 | IS_TIMEOUT=0 98 | while IS_TIMEOUT is utils.STATUS_OK: 99 | try: 100 | print('Trying to stop instance ...') 101 | stop_instance_operation = compute.instances().stop( 102 | project=conf.GCP_DEFAULT_PROJECT, 103 | zone=conf.GCP_DEFAULT_ZONE, 104 | instance=conf.INSTANCE_BUILD_NAME 105 | ).execute() 106 | break 107 | except Exception as err: 108 | print(f' Exception: {err}') 109 | time.sleep(1) 110 | IS_TIMEOUT=utils.check_timeout(start_time, TIMEOUT_SECONDS) 111 | if IS_TIMEOUT is not utils.STATUS_OK: 112 | print('Timeout or an error occurred when trying to stop the instance') 113 | utils.terminate_instance_and_exit( 114 | compute=compute, 115 | project=conf.GCP_DEFAULT_PROJECT, 116 | zone=conf.GCP_DEFAULT_ZONE, 117 | instance=conf.INSTANCE_BUILD_NAME 118 | ) 119 | 120 | print('Successfully stopped the instance') 121 | 122 | STOPPED = utils.wait_for_zone_operation( 123 | compute=compute, 124 | project=conf.GCP_DEFAULT_PROJECT, 125 | zone=conf.GCP_DEFAULT_ZONE, 126 | operation=stop_instance_operation['name'] 127 | ) 128 | if STOPPED == utils.STATUS_OK: 129 | print(' Instance stopped') 130 | else: 131 | print(' Timeout waiting for instace stop operation') 132 | utils.terminate_instance_and_exit( 133 | compute=compute, 134 | project=conf.GCP_DEFAULT_PROJECT, 135 | zone=conf.GCP_DEFAULT_ZONE, 136 | instance=conf.INSTANCE_BUILD_NAME 137 | ) 138 | 139 | # Create GCP Snapshot 140 | 141 | if len(sys.argv) > 1 and sys.argv[1] != '--no-analytics': 142 | SNAPSHOT_NAME = sys.argv[1] 143 | else: 144 | SNAPSHOT_NAME = conf.INSTANCE_BUILD_NAME 145 | 146 | print('Triggering Meilisearch GCP Snapshot creation...') 147 | create_image_operation = compute.images().insert( 148 | project=conf.GCP_DEFAULT_PROJECT, 149 | body={ 150 | 'name': SNAPSHOT_NAME, 151 | 'sourceDisk': instance['disks'][0]['source'], 152 | 'description': conf.IMAGE_DESCRIPTION_NAME, 153 | } 154 | ).execute() 155 | 156 | IMAGE_CREATION = utils.wait_for_global_operation( 157 | compute=compute, 158 | project=conf.GCP_DEFAULT_PROJECT, 159 | operation=create_image_operation['name'] 160 | ) 161 | if IMAGE_CREATION == utils.STATUS_OK: 162 | print(' Image created') 163 | else: 164 | print(' Timeout waiting for image creation') 165 | utils.terminate_instance_and_exit( 166 | compute=compute, 167 | project=conf.GCP_DEFAULT_PROJECT, 168 | zone=conf.GCP_DEFAULT_ZONE, 169 | instance=conf.INSTANCE_BUILD_NAME 170 | ) 171 | 172 | # Delete instance 173 | 174 | print('Delete instance...') 175 | compute.instances().delete( 176 | project=conf.GCP_DEFAULT_PROJECT, 177 | zone=conf.GCP_DEFAULT_ZONE, 178 | instance=conf.INSTANCE_BUILD_NAME 179 | ).execute() 180 | print('Instance deleted') 181 | -------------------------------------------------------------------------------- /tools/config.py: -------------------------------------------------------------------------------- 1 | from datetime import datetime 2 | import requests 3 | 4 | # Update with the Meilisearch version TAG you want to build the image with 5 | 6 | MEILI_CLOUD_SCRIPTS_VERSION_TAG = 'v1.0.0' 7 | 8 | # Update with the custom image name that you want to publish after TESTING 9 | 10 | PUBLISH_IMAGE_NAME = 'meilisearch-v1-0-0-debian-10-build--06-02-2023-12-07-26' 11 | 12 | # Setup environment and settings 13 | 14 | DEBIAN_BASE_IMAGE_PROJECT = 'debian-cloud' 15 | DEBIAN_BASE_IMAGE_FAMILY = 'debian-10' 16 | IMAGE_DESCRIPTION_NAME = f'Meilisearch-{MEILI_CLOUD_SCRIPTS_VERSION_TAG} running on {DEBIAN_BASE_IMAGE_FAMILY}' 17 | IMAGE_FORMAT = 'vmdk' 18 | IMAGE_DESTINATION_URI = f'gs://meilisearch-image/meilisearch-{MEILI_CLOUD_SCRIPTS_VERSION_TAG}-{DEBIAN_BASE_IMAGE_FAMILY}.{IMAGE_FORMAT}' 19 | IMAGE_DESTINATION_BUCKET_NAME = 'meilisearch-image' 20 | SERVICE_ACCOUNT_EMAIL = '591812945139@cloudbuild.gserviceaccount.com' 21 | 22 | USER_DATA = requests.get( 23 | f'https://raw.githubusercontent.com/meilisearch/cloud-scripts/{MEILI_CLOUD_SCRIPTS_VERSION_TAG}/scripts/providers/gcp/cloud-config.yaml' 24 | ).text 25 | 26 | SNAPSHOT_NAME = f'meilisearch-{MEILI_CLOUD_SCRIPTS_VERSION_TAG}-{DEBIAN_BASE_IMAGE_FAMILY}'.replace('.', '-') 27 | 28 | INSTANCE_BUILD_NAME = f"{SNAPSHOT_NAME}-build-{datetime.now().strftime('-%d-%m-%Y-%H-%M-%S')}" 29 | 30 | GCP_DEFAULT_PROJECT = 'meilisearchimage' 31 | GCP_DEFAULT_ZONE = 'us-central1-a' 32 | 33 | INSTANCE_TYPE = f'zones/{GCP_DEFAULT_ZONE}/machineTypes/n1-standard-1' 34 | 35 | MEILISEARCH_LOGO_URL = 'https://github.com/meilisearch/integration-guides/blob/main/assets/logos/logo.svg' 36 | 37 | # SEE: https://blog.woohoosvcs.com/2019/11/cloud-init-on-google-compute-engine/ 38 | STARTUP_SCRIPT = """ 39 | #!/bin/bash 40 | 41 | if ! type cloud-init > /dev/null 2>&1 ; then 42 | echo "Ran - `date`" >> /root/startup 43 | sleep 30 44 | apt install -y cloud-init 45 | 46 | if [ $? == 0 ]; then 47 | echo "Ran - Success - `date`" >> /root/startup 48 | systemctl enable cloud-init 49 | else 50 | echo "Ran - Fail - `date`" >> /root/startup 51 | fi 52 | 53 | # Reboot either way 54 | reboot 55 | fi 56 | """ 57 | 58 | # DOCS: https://cloud.google.com/compute/docs/reference/rest/v1/instances/insert 59 | BUILD_INSTANCE_CONFIG = { 60 | 'name': INSTANCE_BUILD_NAME, 61 | 'machineType': INSTANCE_TYPE, 62 | 'disks': [ 63 | { 64 | 'boot': True, 65 | 'autoDelete': True, 66 | 'initializeParams': { 67 | 'sourceImage': '', 68 | } 69 | } 70 | ], 71 | 'tags': { 72 | 'items': [ 73 | 'http-server', 74 | 'https-server' 75 | ], 76 | }, 77 | 'networkInterfaces': [{ 78 | 'network': 'global/networks/default', 79 | 'accessConfigs': [ 80 | {'type': 'ONE_TO_ONE_NAT', 'name': 'External NAT'} 81 | ] 82 | }], 83 | # user-data 84 | 'metadata': { 85 | 'items': [ 86 | { 87 | 'key': 'user-data', 88 | 'value': USER_DATA, 89 | }, 90 | { 91 | 'key': 'startup-script', 92 | 'value': STARTUP_SCRIPT 93 | }, 94 | { 95 | 'key': 'block-project-ssh-keys', 96 | 'value': False 97 | } 98 | ] 99 | } 100 | } 101 | 102 | # DOCS: https://cloud.google.com/compute/docs/reference/rest/v1/instances/insert 103 | BUILD_INSTANCE_TEST_CONFIG = { 104 | 'name': INSTANCE_BUILD_NAME, 105 | 'machineType': INSTANCE_TYPE, 106 | 'disks': [ 107 | { 108 | 'boot': True, 109 | 'autoDelete': True, 110 | 'initializeParams': { 111 | 'sourceImage': '', 112 | } 113 | } 114 | ], 115 | 'tags': { 116 | 'items': [ 117 | 'http-server', 118 | 'https-server' 119 | ], 120 | }, 121 | 'networkInterfaces': [{ 122 | 'network': 'global/networks/default', 123 | 'accessConfigs': [ 124 | {'type': 'ONE_TO_ONE_NAT', 'name': 'External NAT'} 125 | ] 126 | }], 127 | 'metadata': { 128 | 'items': [ 129 | { 130 | 'key': 'block-project-ssh-keys', 131 | 'value': False 132 | } 133 | ] 134 | } 135 | } 136 | 137 | EXPORT_IMAGE_CONFIG = { 138 | 'steps': [ 139 | { 140 | 'args': [ 141 | '-timeout=7000s', 142 | f'-source_image={PUBLISH_IMAGE_NAME}', 143 | '-client_id=api', 144 | f'-format={IMAGE_FORMAT}', 145 | f'-destination_uri={IMAGE_DESTINATION_URI}', 146 | f'-compute_service_account={SERVICE_ACCOUNT_EMAIL}' 147 | ], 148 | 'name':'gcr.io/compute-image-tools/gce_vm_image_export:release', 149 | 'env':[ 150 | 'BUILD_ID=$BUILD_ID' 151 | ] 152 | } 153 | ] 154 | } 155 | -------------------------------------------------------------------------------- /tools/destroy_image.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import googleapiclient.discovery 3 | import config as conf 4 | 5 | compute = googleapiclient.discovery.build('compute', 'v1') 6 | 7 | if len(sys.argv) > 1: 8 | SNAPSHOT_NAME = sys.argv[1] 9 | else: 10 | raise Exception('No snapshot name specified') 11 | 12 | print(f'Destroying image named: {SNAPSHOT_NAME}...') 13 | 14 | # Destroy image 15 | 16 | print('Deleting image...') 17 | delete = compute.images().delete(project=conf.GCP_DEFAULT_PROJECT, 18 | image=SNAPSHOT_NAME).execute() 19 | print('Image deleted') 20 | -------------------------------------------------------------------------------- /tools/publish_image.py: -------------------------------------------------------------------------------- 1 | # Build image using projects.builds() 2 | # 3 | # https://cloud.google.com/compute/docs/images/export-image#exporting_an_image_with_a_single_command 4 | # https://cloud.google.com/build/docs/api/reference/rest/v1/projects.builds 5 | 6 | # Export image to bucket 7 | 8 | import googleapiclient.discovery 9 | from google.cloud import storage 10 | 11 | import config as conf 12 | import utils 13 | 14 | cloudbuild = googleapiclient.discovery.build('cloudbuild', 'v1') 15 | compute = googleapiclient.discovery.build('compute', 'v1') 16 | 17 | export_image = cloudbuild.projects().builds().create( 18 | projectId='meilisearchimage', 19 | body=conf.EXPORT_IMAGE_CONFIG 20 | ).execute() 21 | 22 | print('Waiting for image export operation') 23 | IMAGE_EXPORT_OPERATION = utils.wait_for_build_operation( 24 | cloudbuild=cloudbuild, 25 | project=conf.GCP_DEFAULT_PROJECT, 26 | operation=export_image['metadata']['build']['id'] 27 | ) 28 | if IMAGE_EXPORT_OPERATION == utils.STATUS_OK: 29 | print(f' Image exported: {conf.IMAGE_DESTINATION_URI}') 30 | else: 31 | print(' Timeout waiting for image export') 32 | 33 | # Make image public 34 | 35 | print('Publishing image') 36 | bucket = storage.Client().get_bucket(conf.IMAGE_DESTINATION_BUCKET_NAME) 37 | policy = bucket.get_iam_policy(requested_policy_version=3) 38 | policy.bindings.append( 39 | {'role': 'roles/storage.objectViewer', 'members': {'allUsers'}}) 40 | bucket.set_iam_policy(policy) 41 | print(' Image is now public') 42 | 43 | # Delete custom image 44 | 45 | delete_image_operation = compute.images().delete( 46 | project=conf.GCP_DEFAULT_PROJECT, 47 | image=conf.PUBLISH_IMAGE_NAME 48 | ).execute() 49 | -------------------------------------------------------------------------------- /tools/test_image.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import time 3 | import sys 4 | import googleapiclient.discovery 5 | import config as conf 6 | import utils 7 | 8 | compute = googleapiclient.discovery.build('compute', 'v1') 9 | 10 | if len(sys.argv) > 1: 11 | SNAPSHOT_NAME = sys.argv[1] 12 | else: 13 | raise Exception('No snapshot name specified') 14 | 15 | print(f'Running test for image named: {SNAPSHOT_NAME}...') 16 | 17 | # Get the image for the test 18 | 19 | source_image = compute.images().get(project=conf.GCP_DEFAULT_PROJECT, 20 | image=SNAPSHOT_NAME).execute() 21 | 22 | # Create GCP Compute instance to test Meilisearch image 23 | 24 | print('Creating GCP Compute instance') 25 | 26 | instance_config = conf.BUILD_INSTANCE_TEST_CONFIG 27 | instance_config['disks'][0]['initializeParams']['sourceImage'] = source_image['selfLink'] 28 | 29 | create = compute.instances().insert( 30 | project=conf.GCP_DEFAULT_PROJECT, 31 | zone=conf.GCP_DEFAULT_ZONE, 32 | body=instance_config).execute() 33 | print(f' Instance created. Name: {conf.INSTANCE_BUILD_NAME}') 34 | 35 | # Wait for GCP instance to be 'RUNNING' 36 | 37 | print('Waiting for GCP Compute instance state to be "RUNNING"') 38 | state_code, state, instance = utils.wait_for_instance_running( 39 | conf.GCP_DEFAULT_PROJECT, conf.GCP_DEFAULT_ZONE, timeout_seconds=600) 40 | print(f' Instance state: {state}') 41 | 42 | if state_code == utils.STATUS_OK: 43 | instance_ip = instance['networkInterfaces'][0]['accessConfigs'][0]['natIP'] 44 | print(f' Instance IP: {instance_ip}') 45 | else: 46 | print(f' Error: {state_code}. State: {state}.') 47 | utils.terminate_instance_and_exit( 48 | compute=compute, 49 | project=conf.GCP_DEFAULT_PROJECT, 50 | zone=conf.GCP_DEFAULT_ZONE, 51 | instance=conf.INSTANCE_BUILD_NAME 52 | ) 53 | 54 | # Wait for Health check after configuration is finished 55 | 56 | print('Waiting for Meilisearch health check (may take a few minutes: config and reboot)') 57 | HEALTH = utils.wait_for_health_check(instance_ip, timeout_seconds=600) 58 | if HEALTH == utils.STATUS_OK: 59 | print(' Instance is healthy') 60 | else: 61 | print(' Timeout waiting for health check') 62 | utils.terminate_instance_and_exit( 63 | compute=compute, 64 | project=conf.GCP_DEFAULT_PROJECT, 65 | zone=conf.GCP_DEFAULT_ZONE, 66 | instance=conf.INSTANCE_BUILD_NAME 67 | ) 68 | 69 | # Check version 70 | 71 | print('Waiting for Version check') 72 | try: 73 | utils.check_meilisearch_version( 74 | instance_ip, conf.MEILI_CLOUD_SCRIPTS_VERSION_TAG) 75 | except Exception as err: 76 | print(f' Exception: {err}') 77 | utils.terminate_instance_and_exit( 78 | compute=compute, 79 | project=conf.GCP_DEFAULT_PROJECT, 80 | zone=conf.GCP_DEFAULT_ZONE, 81 | instance=conf.INSTANCE_BUILD_NAME 82 | ) 83 | print(' Version of Meilisearch match!') 84 | 85 | # Stop instance 86 | 87 | print('Stopping GCP instance...') 88 | 89 | TIMEOUT_SECONDS=60 90 | start_time = datetime.datetime.now() 91 | IS_TIMEOUT=0 92 | while IS_TIMEOUT is utils.STATUS_OK: 93 | try: 94 | print('Trying to stop the instance ...') 95 | stop_instance_operation = compute.instances().stop( 96 | project=conf.GCP_DEFAULT_PROJECT, 97 | zone=conf.GCP_DEFAULT_ZONE, 98 | instance=conf.INSTANCE_BUILD_NAME 99 | ).execute() 100 | break 101 | except Exception as err: 102 | print(f' Exception: {err}') 103 | time.sleep(1) 104 | IS_TIMEOUT=utils.check_timeout(start_time, TIMEOUT_SECONDS) 105 | if IS_TIMEOUT is not utils.STATUS_OK: 106 | print('Timeout or an error occurred when trying to stop the instance') 107 | utils.terminate_instance_and_exit( 108 | compute=compute, 109 | project=conf.GCP_DEFAULT_PROJECT, 110 | zone=conf.GCP_DEFAULT_ZONE, 111 | instance=conf.INSTANCE_BUILD_NAME 112 | ) 113 | 114 | print('Successfully stopped the instance') 115 | 116 | STOPPED = utils.wait_for_zone_operation( 117 | compute=compute, 118 | project=conf.GCP_DEFAULT_PROJECT, 119 | zone=conf.GCP_DEFAULT_ZONE, 120 | operation=stop_instance_operation['name'] 121 | ) 122 | if STOPPED == utils.STATUS_OK: 123 | print(' Instance stopped') 124 | else: 125 | print(' Timeout waiting for instace stop operation') 126 | utils.terminate_instance_and_exit( 127 | compute=compute, 128 | project=conf.GCP_DEFAULT_PROJECT, 129 | zone=conf.GCP_DEFAULT_ZONE, 130 | instance=conf.INSTANCE_BUILD_NAME 131 | ) 132 | 133 | # Delete instance 134 | 135 | print('Delete instance...') 136 | compute.instances().delete( 137 | project=conf.GCP_DEFAULT_PROJECT, 138 | zone=conf.GCP_DEFAULT_ZONE, 139 | instance=conf.INSTANCE_BUILD_NAME 140 | ).execute() 141 | print('Instance deleted') 142 | -------------------------------------------------------------------------------- /tools/utils.py: -------------------------------------------------------------------------------- 1 | import datetime 2 | import time 3 | import sys 4 | import requests 5 | import googleapiclient.discovery 6 | 7 | import config as conf 8 | 9 | STATUS_OK = 0 10 | STATUS_TIMEOUT = 1 11 | STATUS_ERROR = 2 12 | 13 | # INSTANCE 14 | 15 | 16 | def wait_for_instance_running(project, zone, timeout_seconds=None): 17 | compute = googleapiclient.discovery.build('compute', 'v1') 18 | start_time = datetime.datetime.now() 19 | while timeout_seconds is None \ 20 | or check_timeout(start_time, timeout_seconds) is not STATUS_TIMEOUT: 21 | instance = compute.instances().get(project=project, zone=zone, 22 | instance=conf.INSTANCE_BUILD_NAME).execute() 23 | if instance['status'] not in ['STAGING', 'PROVISIONING']: 24 | if instance['status'] == 'RUNNING': 25 | return STATUS_OK, instance['status'], instance 26 | return STATUS_ERROR, instance['status'], instance 27 | time.sleep(1) 28 | return STATUS_TIMEOUT, None, instance 29 | 30 | 31 | def wait_for_health_check(instance_ip, timeout_seconds=None): 32 | start_time = datetime.datetime.now() 33 | while timeout_seconds is None \ 34 | or check_timeout(start_time, timeout_seconds) is not STATUS_TIMEOUT: 35 | try: 36 | resp = requests.get(f'http://{instance_ip}/health') 37 | if resp.status_code >= 200 and resp.status_code < 300: 38 | return STATUS_OK 39 | except Exception: 40 | pass 41 | time.sleep(1) 42 | return STATUS_TIMEOUT 43 | 44 | 45 | def check_meilisearch_version(instance_ip, version): 46 | resp = requests.get( 47 | f'http://{instance_ip}/version').json() 48 | if resp['pkgVersion'] in version: 49 | return 50 | raise Exception( 51 | f" The version of meilisearch ({version}) does not match the instance ({resp['pkgVersion']})") 52 | 53 | 54 | def wait_for_zone_operation(compute, project, zone, operation, timeout_seconds=None): 55 | start_time = datetime.datetime.now() 56 | while timeout_seconds is None \ 57 | or check_timeout(start_time, timeout_seconds) is not STATUS_TIMEOUT: 58 | try: 59 | result = compute.zoneOperations().get( 60 | project=project, 61 | zone=zone, 62 | operation=operation).execute() 63 | if result['status'] == 'DONE': 64 | if 'error' in result: 65 | raise Exception(result['error']) 66 | return STATUS_OK 67 | except Exception as err: 68 | print(err) 69 | time.sleep(1) 70 | return STATUS_TIMEOUT 71 | 72 | 73 | def wait_for_global_operation(compute, project, operation, timeout_seconds=None): 74 | start_time = datetime.datetime.now() 75 | while timeout_seconds is None \ 76 | or check_timeout(start_time, timeout_seconds) is not STATUS_TIMEOUT: 77 | try: 78 | result = compute.globalOperations().get( 79 | project=project, 80 | operation=operation).execute() 81 | if result['status'] == 'DONE': 82 | if 'error' in result: 83 | raise Exception(result['error']) 84 | return STATUS_OK 85 | except Exception as err: 86 | print(err) 87 | time.sleep(1) 88 | return STATUS_TIMEOUT 89 | 90 | 91 | def terminate_instance_and_exit(compute, project, zone, instance): 92 | print(f' Terminating instance {instance}') 93 | compute.instances().delete( 94 | project=project, 95 | zone=zone, 96 | instance=instance 97 | ).execute() 98 | print('ENDING PROCESS WITH EXIT CODE 1') 99 | sys.exit(1) 100 | 101 | # BUILD AND PUBLISH 102 | 103 | 104 | def wait_for_build_operation(cloudbuild, project, operation, timeout_seconds=None): 105 | start_time = datetime.datetime.now() 106 | while timeout_seconds is None \ 107 | or check_timeout(start_time, timeout_seconds) is not STATUS_TIMEOUT: 108 | try: 109 | result = cloudbuild.projects().builds().get( 110 | projectId=project, 111 | id=operation 112 | ).execute() 113 | except Exception as err: 114 | print(err) 115 | if result['status'] == 'SUCCESS': 116 | return STATUS_OK 117 | if result['status'] == 'FAILURE': 118 | print(result) 119 | raise Exception('Error on build operation') 120 | time.sleep(1) 121 | return STATUS_TIMEOUT 122 | 123 | # GENERAL 124 | 125 | 126 | def check_timeout(start_time, timeout_seconds): 127 | elapsed_time = datetime.datetime.now() - start_time 128 | if elapsed_time.total_seconds() > timeout_seconds: 129 | return STATUS_TIMEOUT 130 | return STATUS_OK 131 | --------------------------------------------------------------------------------