├── .gitignore ├── .travis.yml ├── CONTRIBUTING.md ├── LICENSE ├── MANIFEST.in ├── README.md ├── appstart ├── __init__.py ├── cli │ ├── __init__.py │ ├── parsing.py │ └── start_script.py ├── constants.py ├── devappserver_init │ ├── Dockerfile │ ├── __init__.py │ ├── app.yaml │ └── das.sh ├── pinger │ ├── Dockerfile │ ├── __init__.py │ └── pinger.py ├── sandbox │ ├── __init__.py │ ├── app.yaml │ ├── configuration.py │ ├── container.py │ └── container_sandbox.py ├── utils.py └── validator │ ├── __init__.py │ ├── color_formatting.py │ ├── color_logging.py │ ├── contract.py │ ├── errors.py │ ├── parsing.py │ └── runtime_contract.py ├── run_tests.sh ├── setup.cfg ├── setup.py └── tests ├── __init__.py ├── fakes ├── __init__.py └── fake_docker.py ├── system_tests ├── Dockerfile ├── __init__.py ├── app.yaml ├── appstart_systemtests.py └── services_test_app.py ├── test_data └── certs │ ├── ca.pem │ ├── cert.pem │ └── key.pem └── unit_tests ├── __init__.py ├── sandbox ├── __init__.py ├── configuration_test.py ├── container_sandbox_test.py └── container_test.py ├── utils_test.py └── validator ├── __init__.py └── contract_test.py /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: python 2 | python: "2.7" 3 | 4 | sudo: required 5 | services: 6 | - docker 7 | 8 | script: 9 | - python setup.py sdist 10 | - pip install dist/appstart-0.8.tar.gz 11 | - ./run_tests.sh 12 | -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | Want to contribute? Great! First, read this page (including the small print at the end). 2 | 3 | ### Before you contribute 4 | Before we can use your code, you must sign the 5 | [Google Individual Contributor License Agreement](https://developers.google.com/open-source/cla/individual?csw=1) 6 | (CLA), which you can do online. The CLA is necessary mainly because you own the 7 | copyright to your changes, even after your contribution becomes part of our 8 | codebase, so we need your permission to use and distribute your code. We also 9 | need to be sure of various other things—for instance that you'll tell us if you 10 | know that your code infringes on other people's patents. You don't have to sign 11 | the CLA until after you've submitted your code for review and a member has 12 | approved it, but you must do it before we can put your code into our codebase. 13 | Before you start working on a larger contribution, you should get in touch with 14 | us first through the issue tracker with your idea so that we can help out and 15 | possibly guide you. Coordinating up front makes it much easier to avoid 16 | frustration later on. 17 | 18 | ### Code reviews 19 | All submissions, including submissions by project members, require review. We 20 | use Github pull requests for this purpose. 21 | 22 | ### The small print 23 | Contributions made by corporations are covered by a different agreement than 24 | the one above, the Software Grant and Corporate Contributor License Agreement. 25 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /MANIFEST.in: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | include README.md 16 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Appstart 2 | 3 | ## Introduction 4 | 5 | appstart is a local tool that allows Managed VM applications to be deployed into 6 | a local Docker cluster in which App Engine services such as Task Queues and 7 | Datastatore are emulated. It can be useful for advanced users who wish to perform 8 | additional local testing before deploying applications to App Engine. 9 | 10 | *Important: This project is experimental and is not officially supported by the 11 | Google Cloud Platform.* 12 | 13 | ## Installation (Linux) 14 | 15 | From its root directory, appstart can be installed like so: 16 | 17 | $ python setup.py sdist 18 | $ sudo pip install dist/appstart-0.8.tar.gz 19 | 20 | ## Requirements 21 | 22 | Appstart requires a running Docker server. The server can be running in a 23 | docker-machine instance, as long as the docker environment variables are set 24 | correctly. The latest version of Appstart is known to work for Docker server 25 | versions 1.8.0 to 1.8.3 and will accept any version less than 1.9. For 26 | information about installing docker and docker-machine, see: 27 | 28 | * docker: https://docs.docker.com/installation/ 29 | * docker-machine: https://docs.docker.com/machine/ 30 | 31 | ## Usage 32 | 33 | Before using appstart for the first time, run: 34 | 35 | $ appstart init 36 | 37 | This generates a 'devappserver base image', which Appstart will later use to 38 | run the API server. 39 | 40 | For a list of permissible command line options, you can run: 41 | 42 | $ appstart --help 43 | 44 | ### Default invocation 45 | 46 | Appstart can be used to start the application from the command line. 47 | It is invoked as follows: 48 | 49 | $ appstart run PATH_TO_CONFIG_FILE 50 | 51 | `PATH_TO_CONFIG_FILE` must be a path to the application's configuration file, 52 | which is either `appengine-web.xml` or a `.yaml` file. For Java standard runtime 53 | applications, the `appengine-web.xml` file must be inside WEB-INF, along with a 54 | web.xml file. For all other applications, the `.yaml` file must be in the root 55 | directory of the application. By default, Appstart will attempt to locate the 56 | Dockerfile in the application's root directory and use it to build the 57 | application's image. 58 | 59 | As stated earlier, Appstart will also run an api server to provide stubs for 60 | Google Cloud Platform. 61 | 62 | ### Specifying an image 63 | 64 | The `--image_name` flag can be used to specify an existing image rather than 65 | having Appstart build one from the Dockerfile. When `--image_name` is specified, 66 | a Dockerfile is not needed: 67 | 68 | $ appstart PATH_TO_CONFIG_FILE --image_name=IMAGE_NAME 69 | 70 | Appstart can also start an image without a configuration file like so: 71 | 72 | $ appstart --image_name=IMAGE_NAME 73 | 74 | In this case, appstart uses a "phony" app.yaml file as the application's 75 | configuration file. 76 | 77 | ### Turning off the api server 78 | 79 | By default, Appstart runs an api server so that the application can make calls 80 | to Google Cloud Platform services (datastore, taskqueue, logging, etc). If you 81 | don't consume these services, you can run appstart like this: 82 | 83 | $ appstart PATH_TO_CONFIG_FILE --no_api_server 84 | 85 | Be aware that the api server also functions as a proxy and static file server, 86 | so if you turn it off you'll need to serve any static files from your 87 | application. 88 | 89 | ## Options 90 | 91 | To see all command line options, run: 92 | 93 | $ appstart run --help 94 | 95 | ## Under the hood 96 | 97 | Appstart runs the aforementioned api server in the devappserver container. The 98 | `appstart init` command builds the 'devappserver base image', which contains all 99 | the source files necessary for the api server to run. 100 | 101 | Appstart will also build a layer on top of the devappserver image, populating 102 | the devappserver image with the application's configuration files. As was 103 | mentioned earlier, if Appstart is not provided with a configuration file, it 104 | adds a "phony" app.yaml file to the devappserver base image. 105 | 106 | After building images for devappserver and the application, appstart will start 107 | containers based on these images, using the correct environment variables. The 108 | environment variables allow the application container to locate the devappserver 109 | container, and allow the devappserver container to locate the application 110 | container. The containers currently run on the same network stack for 111 | simplicity, but that's subject to change in the future. 112 | 113 | All of the functionality described above is implemented by the ContainerSandbox 114 | class. This class constructs a sandbox consisting of an application container 115 | and a devappserver container, and it connects the two together. Upon exiting, it 116 | will stop these containers and remove them. It's quite resilient, so it won't 117 | litter the docker environment with old containers. 118 | 119 | # The Validator 120 | 121 | The validator is a framework built to validate whether or not a container 122 | conforms to the runtime contract. The runtime contract consists of a set of 123 | requirements imposed on it by the Google Cloud Platform. For instance, a 124 | container must respond to health checks on the `_ah/health` endpoint. 125 | 126 | The validator can test an application container to see if it meets the 127 | requirements of the runtime contract. 128 | 129 | ## Default Invocation 130 | 131 | The validator can be invoked like this: 132 | 133 | $ appstart validate 134 | 135 | The validator gets the container running in the same way as `appstart run` 136 | does. It then runs through a series of 'clauses', each of which test if 137 | the container is meeting a specific expectation. For example, a particular 138 | clause might send an HTTP request to the `_ah/health` endpoint to see if the 139 | application container properly responds to health checks. Another clause 140 | might check if the application is writing logs correctly. 141 | 142 | ## Lifecycle points 143 | 144 | The validator evaluates clauses at very specific points of the container's 145 | lifecycle. The time period in which a clause is evaluated is called its 146 | 'lifecycle point'. Lifecycle points include the following: 147 | 148 | * `PRE_START`: the time before a start request is sent to the container 149 | * `START`: the time during which a start request is sent to the container. 150 | Note that only one clause can be defined for this lifecycle point. 151 | * `POST_START`: the time after the container has received the start request. 152 | * `STOP`: the time during which a stop request is sent to the container. 153 | Note that only one clause can be defined for this lifecycle point. 154 | * `POST_STOP`: the time after the container has received the stop request. 155 | 156 | ## Error levels 157 | 158 | Each clause is marked with a specific error level. The error level denotes the 159 | severity of the error that would occur, should the clause fail to pass. Error 160 | levels include the following: 161 | 162 | * FATAL: If the container fails a clause marked as FATAL, the container will 163 | absolutely not work. FATAL errors include not listening on port 8080, not 164 | responding properly to health checks, etc. 165 | 166 | * WARNING: If the container fails a clause marked as WARNING, 167 | it will possibly exhibit unexpected behavior. WARNING errors include 168 | not writing logs in the correct format. 169 | 170 | * UNUSED: If the container does not pass a clause marked as UNUSED, no real 171 | error has occurred. It just means that the container isn't taking full 172 | advantage of the runtime contract. UNUSED level errors include not writing 173 | access or diagnostic logs. Other errors (namely WARNING errors) might be 174 | dependent on UNUSED-level clauses. For instance, logging format is 175 | contingent on the existence of logs in the proper location. 176 | 177 | By default, validation will fail if any clauses with error level WARNING or 178 | higher fail. This behavior can be changed by specifying a threshold. See 179 | `appstart validate --help` for more info. 180 | 181 | ## Options 182 | 183 | The validator accepts all of the same options as `appstart run` does. In 184 | addition, the validator provides several options specific to its own 185 | functionality. To see all available options, run: 186 | 187 | $ appstart validate --help 188 | 189 | ## Custom Hook Clauses 190 | 191 | The validator provides functionality to write "hook clauses". These are 192 | user-supplied clauses generated at runtime. 193 | 194 | ### Adding a hook clause 195 | 196 | To find hook clauses, the validator looks in the application's root for a 197 | directory by the name of `validator_hooks`. If such a directory is present, 198 | the validator recursively walks it, looking for any configuration files that 199 | end with `.conf.yaml`. For every such file found, the validator generates a 200 | hook clause, which will be evaluated along with all of the validator's default 201 | clauses. Note that for the validator to discover hook clauses, it must be run 202 | on the application's configuration file. In other words, hook clauses will 203 | not be discovered if the following command is run: 204 | 205 | $ appstart validate --image_name= 206 | 207 | ### Writing a hook clause 208 | 209 | Specifying the behavior of a hook clause is very simple. As an example, let's 210 | walk through how this is done. Suppose we have an application with a very 211 | sophisticated `/foo` url endpoint. When we hit the `/foo` endpoint, we expect 212 | it to return a response whose body consists of the word, 'bar'. 213 | 214 | To write a hook clause to test this behavior, create the file 215 | `validator_hooks/test.py.conf.yaml` with the following content: 216 | 217 | name: TestClause 218 | title: Foo endpoint test 219 | description: Test that /foo endpoint of the application returns 'bar' 220 | lifecycle_point: POST_START 221 | 222 | Upon encountering our test.py.conf.yaml file, the validator will search for 223 | a script called `test.py` in the same directory. If it finds one, the validator 224 | will execute it at runtime, providing it with the following environment 225 | variables: 226 | 227 | * `APP_CONTAINER_ID`: The application's Docker container ID. This can be used 228 | to perform docker commands on the application. 229 | * `APP_CONTAINER_HOST`: The host where the application container is running. 230 | * `APP_CONTAINER_PORT`: The port where the application container is running. 231 | 232 | These environment variables can be used to test the container by sending it 233 | requests or even examining its internal state with docker. Since we're testing 234 | the `/foo` endpoint, our `validator_hooks/test.py` might look like this: 235 | 236 | #!/usr/bin/python 237 | import os 238 | import requests 239 | 240 | host = os.environ.get('APP_CONTAINER_HOST') 241 | port = os.environ.get('APP_CONTAINER_PORT') 242 | 243 | response = requests.get('http://{0}:{1}/foo'.format(host, port)) 244 | assert response.text == 'bar' 245 | 246 | Make sure that test.py is an executable. To do this, you can run: 247 | 248 | $ chmod u+x test.py 249 | 250 | The hook clause will only be considered a failure if the executable returns a 251 | nonzero exit code. In the case of failure, the stdout and stderr of the 252 | executable will be reported in the test results. 253 | 254 | ### Configuring Hook Clauses 255 | 256 | In our `.conf.yaml` file, we specified only a few key-value pairs. Those were 257 | the minimum parameters a hook clause needs. Of course, we can make our hook 258 | clause more configurable than that. Here's a list of configurable keys that can 259 | be put into a hook clause's `.conf.yaml` file: 260 | 261 | * name: The name of the clause, used to identify the clause by other clauses. 262 | * title: The title of the clause, displayed in test results 263 | * description: A brief description of the thing the clause is validating. This 264 | description is also displayed in test results. 265 | * lifecycle\_point: The point of the container's lifecycle in which the hook 266 | clause should be executed. 267 | * error\_level: The severity of the error that would occur, should the clause 268 | fail. Defaults to `UNUSED`. 269 | * tags: A list of string tags to mark the hook clause with. The validator can 270 | be invoked with the `--tags' option, which specifies a subset of clauses to 271 | be evaluated. 272 | * command: A string representing the command used to "evaluate" the hook clause. 273 | By default, the validator will search for an executable of the same name, 274 | less the `.conf.yaml` suffix. The hook clause is considered a success if it 275 | returns with an exit code of 0. 276 | * dependencies: A list of clauses that must have passed before the hook clause 277 | is evaluated. If any of the hook clauses dependencies have failed, the hook 278 | clause will be skipped. 279 | * dependents: A list of clauses whose evaluation should be contingent on the 280 | success of the hook clause. If the hook clause fails, none of its dependents 281 | will run. 282 | * before: A list of clauses that should be evaluated BEFORE the hook clause. 283 | Similar to dependencies, but the hook clause will run even if any of its 284 | "before" clauses have failed. 285 | * after: A list of clauses that should be evaluated AFTER the hook clause. 286 | 287 | To specify a list of clauses for the last four keys, simply supply their names 288 | as they appear in the "name" key. For example, here's our old `.conf.yaml` file 289 | with a little more configuration: 290 | 291 | name: TestClause 292 | title: Foo endpoint test 293 | description: Test that /foo endpoint of the application returns 'bar' 294 | lifecycle_point: POST_START 295 | error_level: WARNING 296 | tags: 297 | - test 298 | - foo 299 | - baz 300 | dependencies: 301 | - StartClause 302 | - HealthChecksEnabledClause 303 | after: 304 | - HealthCheckClause 305 | command: /path/to/some/executable 306 | 307 | ### Clause Names 308 | 309 | As stated above, dependencies among clauses are expressed by name. To see the 310 | list of all clause names currently available to the validator, including hook 311 | clauses, run: 312 | 313 | $ appstart validate --list 314 | -------------------------------------------------------------------------------- /appstart/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Utilities for starting and manipulating GAE Managed VM containers.""" 16 | -------------------------------------------------------------------------------- /appstart/cli/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """CLI to locally deploy a managed vm application for testing.""" 16 | -------------------------------------------------------------------------------- /appstart/cli/parsing.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | """Command line argument parsers for appstart.""" 17 | 18 | # This file conforms to the external style guide. 19 | # pylint: disable=bad-indentation 20 | 21 | import argparse 22 | from ..validator import contract 23 | 24 | 25 | class StorePortMapAction(argparse.Action): 26 | """Arg parser action to store a port map. 27 | 28 | Accepts an option value consisting of a comma-separated list of 29 | host_port:container_port mappings. 30 | """ 31 | 32 | def __call__(self, parser, namespace, values, option_string): 33 | vals = values.split(',') 34 | result = {} 35 | try: 36 | for val in vals: 37 | try: 38 | host_port, container_port = val.split(':') 39 | except ValueError: 40 | host_port = container_port = val 41 | result[int(container_port)] = int(host_port) 42 | except ValueError: 43 | parser.error('Bad value for option {0}. Expected a comma ' 44 | 'separated list of colon-separated port ' 45 | 'mappings.'.format(option_string)) 46 | setattr(namespace, self.dest, result) 47 | 48 | 49 | def make_appstart_parser(): 50 | """Make an argument parser to take in command line arguments. 51 | 52 | Returns: 53 | (argparse.ArgumentParser) the parser. 54 | """ 55 | parser = argparse.ArgumentParser( 56 | description='Wrapper to run a managed vm container. If ' 57 | 'using for the first time, run \'appstart init\' ' 58 | 'to generate a devappserver base image.') 59 | subparsers = parser.add_subparsers(dest='parser_type') 60 | 61 | run_parser = subparsers.add_parser('run', 62 | help='Run a Managed VM application') 63 | add_appstart_args(run_parser) 64 | 65 | init_parser = subparsers.add_parser('init', 66 | help='Initialize the Docker ' 67 | 'environment for Appstart. Must be ' 68 | 'run before the first use of ' 69 | '"appstart run"') 70 | add_init_args(init_parser) 71 | 72 | validate_parser = subparsers.add_parser('validate') 73 | validate_parser.set_defaults(parser_type='validate') 74 | add_validate_args(validate_parser) 75 | add_appstart_args(validate_parser) 76 | return parser 77 | 78 | 79 | def add_validate_args(parser): 80 | """Adds command line arguments for the validator. 81 | 82 | Args: 83 | parser: the argparse.ArgumentParser to add the args to. 84 | """ 85 | parser.add_argument('--log_file', 86 | default=None, 87 | help='Logfile to collect validation results.') 88 | 89 | parser.add_argument('--threshold', 90 | default='WARNING', 91 | choices=[name for _, name in 92 | contract.LEVEL_NAMES_TO_NUMBERS.iteritems()], 93 | help='The threshold at which validation should fail.') 94 | parser.add_argument('--tags', 95 | nargs='*', 96 | help='Tag names of the tests to run') 97 | parser.add_argument('--verbose', 98 | action='store_true', 99 | dest='verbose', 100 | help='Whether to emit verbose output to stdout.') 101 | parser.set_defaults(verbose=False) 102 | 103 | parser.add_argument('--list', 104 | action='store_true', 105 | dest='list_clauses', 106 | help='List the clauses available to the validator.') 107 | parser.set_defaults(list_clauses=False) 108 | 109 | 110 | def add_init_args(parser): 111 | parser.add_argument('--use_cache', 112 | action='store_false', 113 | dest='nocache', 114 | help='Flag to enable usage of cache during init.') 115 | parser.set_defaults(nocache=True) 116 | 117 | 118 | def add_appstart_args(parser): 119 | """Add Appstart's command line options to the parser.""" 120 | parser.add_argument('--image_name', 121 | default=None, 122 | help='The name of the docker image to run. ' 123 | 'If no image is specified, one will be ' 124 | "built from the application's Dockerfile.") 125 | parser.add_argument('--application_port', 126 | default=8080, 127 | type=int, 128 | help='The port on the Docker host machine where ' 129 | 'your application should be reached. Defaults to ' 130 | '8080.') 131 | parser.add_argument('--admin_port', 132 | default='8000', 133 | type=int, 134 | help='The port on the Docker host machine where ' 135 | 'the admin panel should be reached. Defaults to ' 136 | '8000.') 137 | parser.add_argument('--proxy_port', 138 | default='8088', 139 | type=int, 140 | help='The port on the Docker host machine where ' 141 | 'the application proxy server can be reached (this ' 142 | 'is generally the port you should use to access ' 143 | 'your application). Defaults to 8088.') 144 | 145 | parser.add_argument('--application_id', 146 | default=None, 147 | help='The api server uses this ID to maintain an ' 148 | 'isolated state for each application. Thus, the ' 149 | 'Application ID determines which Datastore ' 150 | 'the application container has access to. ' 151 | 'In theory, the ID should be the same as the Google ' 152 | 'App Engine ID found in the Google Developers ' 153 | 'Console. However, in practice, an arbitrary ID can ' 154 | 'be chosen during development. By default, if the ID ' 155 | 'is not specified, Appstart chooses a new, ' 156 | 'timestamped ID for every invocation.') 157 | parser.add_argument('--storage_path', 158 | default='/tmp/appengine/storage', 159 | help='The api server creates files to store the ' 160 | "state of the application's datastore, taskqueue, " 161 | 'etc. By default, these files are stored in ' 162 | '/tmp/app_engine/storage on the docker host. ' 163 | 'An alternative storage path can be specified with ' 164 | 'this flag. A good use of this flag is to maintain ' 165 | 'multiple sets of test data.') 166 | 167 | # The port that the admin panel should bind to inside the container. 168 | parser.add_argument('--internal_admin_port', 169 | type=int, 170 | default=32768, 171 | help=argparse.SUPPRESS) 172 | 173 | # The port that the api server should bind to inside the container. 174 | parser.add_argument('--internal_api_port', 175 | type=int, 176 | default=32769, 177 | help=argparse.SUPPRESS) 178 | 179 | # The port that the proxy should bind to inside the container. 180 | parser.add_argument('--internal_proxy_port', 181 | type=int, 182 | default=32770, 183 | help=argparse.SUPPRESS) 184 | 185 | parser.add_argument('--log_path', 186 | default=None, 187 | help='Managed VM application containers are expected ' 188 | 'to write logs to /var/log/app_engine. Appstart binds ' 189 | 'the /var/log/app_engine directory inside the ' 190 | 'application container to a directory in the Docker ' 191 | 'host machine. This option specifies which directory ' 192 | 'on the host machine to use for the binding. Defaults ' 193 | 'to a timestamped directory inside ' 194 | '/tmp/log/app_engine.') 195 | parser.add_argument('--timeout', 196 | type=int, 197 | default=30, 198 | help='How many seconds to wait for the application ' 199 | 'to start listening on port 8080. Defaults to 30 ' 200 | 'seconds.') 201 | parser.add_argument('config_file', 202 | nargs='?', 203 | default=None, 204 | help='The relative or absolute path to the ' 205 | "application\'s .yaml or .xml file.") 206 | 207 | ################################ Flags ############################### 208 | parser.add_argument('--no_cache', 209 | action='store_true', 210 | dest='nocache', 211 | help="Stop Appstart from using Docker's cache during " 212 | 'image builds.') 213 | parser.set_defaults(nocache=False) 214 | 215 | parser.add_argument('--no_api_server', 216 | action='store_false', 217 | dest='run_api_server', 218 | help='Stop Appstart from running an API server. ' 219 | 'You do not need one if you do not consume ' 220 | 'standard Google services such as taskqueue, ' 221 | 'datastore, logging, etc.') 222 | parser.set_defaults(run_api_server=True) 223 | 224 | parser.add_argument('--force_version', 225 | action='store_true', 226 | dest='force_version', 227 | help='Force Appstart to run with mismatched Docker ' 228 | 'version.') 229 | parser.set_defaults(force_version=False) 230 | 231 | parser.add_argument('--clear_datastore', 232 | action='store_true', 233 | dest='clear_datastore', 234 | help='Clear the contents of the datastore before ' 235 | 'running the application.') 236 | parser.set_defaults(clear_datastore=False) 237 | 238 | parser.add_argument('--extra_ports', 239 | action=StorePortMapAction, 240 | dest='extra_ports', 241 | help='A comma separated map of ' 242 | 'host_port:container_port pairs defining the mapping ' 243 | 'from host ports to container ports on the ' 244 | 'application container for an arbitrary set of ports.' 245 | 'If a single value is provided instead of a colon ' 246 | 'separated pair, that value will be used for both the ' 247 | 'host port and the container port.') 248 | parser.set_defaults(extra_ports=None) 249 | -------------------------------------------------------------------------------- /appstart/cli/start_script.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Copyright 2015 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | """A python wrapper to start devappserver and a managed vm application. 17 | 18 | Both the devappserver and the application will run in their respective 19 | containers. 20 | """ 21 | # This file conforms to the external style guide 22 | # pylint: disable=bad-indentation, g-bad-import-order 23 | 24 | import logging 25 | import os 26 | import sys 27 | import time 28 | import warnings 29 | 30 | from .. import constants 31 | from .. import devappserver_init 32 | from .. import pinger 33 | from .. import utils 34 | from ..sandbox import container_sandbox 35 | from ..validator import contract 36 | from ..validator import runtime_contract 37 | 38 | import parsing 39 | 40 | 41 | def main(): 42 | """Run devappserver and the user's application in separate containers. 43 | 44 | The application must be started with the proper environment variables, 45 | port bindings, and volume bindings. The devappserver image runs a 46 | standalone api server. 47 | """ 48 | logging.getLogger('appstart').setLevel(logging.INFO) 49 | 50 | # args should include all the args to the sandbox, as well as a 51 | # parser_type arg, which indicates which subparser was used. 52 | args = vars(parsing.make_appstart_parser().parse_args()) 53 | 54 | # Find out what parser was used (and remove the entry from the args). 55 | parser_type = args.pop('parser_type') 56 | 57 | # In response to 'appstart init', create a new devappserver base image. 58 | if parser_type == 'init': 59 | utils.build_from_directory(os.path.dirname(devappserver_init.__file__), 60 | constants.DEVAPPSERVER_IMAGE, 61 | **args) 62 | utils.build_from_directory(os.path.dirname(pinger.__file__), 63 | constants.PINGER_IMAGE, 64 | **args) 65 | 66 | # In response to 'appstart run', create a container sandbox and run it. 67 | elif parser_type == 'run': 68 | try: 69 | with warnings.catch_warnings(): 70 | # Suppress the InsecurePlatformWarning generated by urllib3 71 | # see: http://stackoverflow.com/questions/29134512/ 72 | warnings.simplefilter('ignore') 73 | with container_sandbox.ContainerSandbox(**args): 74 | while True: 75 | # Sleeping like this is hacky, but it works. Note 76 | # that signal.pause is not compatible with Windows... 77 | time.sleep(10000) 78 | 79 | except KeyboardInterrupt: 80 | utils.get_logger().info('Exiting') 81 | sys.exit(0) 82 | except utils.AppstartAbort as err: 83 | if err.message: 84 | utils.get_logger().warning(str(err.message)) 85 | sys.exit(1) 86 | 87 | # In response to 'appstart validate', attempt to perform validation. 88 | elif parser_type == 'validate': 89 | logfile = args.pop('log_file') 90 | threshold = args.pop('threshold') 91 | tags = args.pop('tags') 92 | verbose = args.pop('verbose') 93 | list_clauses = args.pop('list_clauses') 94 | success = False 95 | utils.get_logger().setLevel(logging.INFO) 96 | try: 97 | with warnings.catch_warnings(): 98 | warnings.simplefilter('ignore') 99 | validator = contract.ContractValidator(runtime_contract, **args) 100 | if list_clauses: 101 | validator.list_clauses() 102 | sys.exit(0) 103 | success = validator.validate(tags, threshold, logfile, verbose) 104 | except KeyboardInterrupt: 105 | utils.get_logger().info('Exiting') 106 | except utils.AppstartAbort as err: 107 | if err.message: 108 | utils.get_logger().warning(err.message) 109 | if success: 110 | sys.exit(0) 111 | sys.exit('Validation failed') 112 | 113 | else: 114 | # This should not be reached 115 | sys.exit(1) 116 | 117 | if __name__ == '__main__': 118 | main() 119 | -------------------------------------------------------------------------------- /appstart/constants.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Appstart constants used by multiple packages.""" 16 | 17 | # Devappserver base image name 18 | DEVAPPSERVER_IMAGE = 'appstart_devappserver_base' 19 | 20 | # Pinger image name 21 | PINGER_IMAGE = 'appstart_pinger' 22 | -------------------------------------------------------------------------------- /appstart/devappserver_init/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | # This is the Dockerfile for building a devappserver base image. 16 | FROM debian 17 | RUN apt-get update 18 | RUN apt-get install -y python curl python-pip 19 | RUN curl https://dl.google.com/dl/cloudsdk/release/install_google_cloud_sdk.bash > install.sh 20 | RUN bash ./install.sh --disable-prompts --install-dir ./sdk 21 | RUN SDK_ROOT=$(echo /sdk/$(ls sdk/)); $(echo $SDK_ROOT/bin/gcloud components update --quiet app app-engine-python app-engine-php app-engine-java) 22 | ADD ./das.sh / 23 | ENTRYPOINT /das.sh 24 | -------------------------------------------------------------------------------- /appstart/devappserver_init/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Package to construct a devappserver base image. 16 | 17 | Appstart uses this as a base to run the api servers. 18 | """ 19 | -------------------------------------------------------------------------------- /appstart/devappserver_init/app.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | 3 | # Phony app.yaml file for devappserver. 4 | 5 | module: default 6 | runtime: python27 7 | vm: true 8 | api_version: 1 9 | threadsafe: true 10 | 11 | resources: 12 | cpu: .5 13 | memory_gb: 1.3 14 | 15 | manual_scaling: 16 | instances: 1 17 | 18 | handlers: 19 | - url: /.* 20 | script: phony.app 21 | -------------------------------------------------------------------------------- /appstart/devappserver_init/das.sh: -------------------------------------------------------------------------------- 1 | SDK_ROOT=/sdk/$(ls /sdk/) 2 | export PYTHONPATH=$SDK_ROOT/lib/ 3 | python $(find $SDK_ROOT -name dev_appserver.py | head -1) \ 4 | --allow_skipped_files=False \ 5 | --api_host=0.0.0.0 \ 6 | --api_port=$API_PORT \ 7 | --admin_host=0.0.0.0 \ 8 | --admin_port=$ADMIN_PORT \ 9 | --application=$APP_ID \ 10 | --auth_domain=gmail.com \ 11 | --clear_datastore=$CLEAR_DATASTORE \ 12 | --datastore_consistency_policy=time \ 13 | --dev_appserver_log_level=info \ 14 | --enable_cloud_datastore=False \ 15 | --enable_mvm_logs=False \ 16 | --enable_sendmail=False \ 17 | --log_level=info \ 18 | --require_indexes=False \ 19 | --show_mail_body=False \ 20 | --skip_sdk_update_check=True \ 21 | --port=$PROXY_PORT \ 22 | --smtp_allow_tls=False \ 23 | --use_mtime_file_watcher=False \ 24 | --external_port=8080 \ 25 | --host=0.0.0.0 \ 26 | --storage_path=/storage \ 27 | --logs_path=./log.txt \ 28 | --automatic_restart=False \ 29 | /app/$CONFIG_FILE 30 | -------------------------------------------------------------------------------- /appstart/pinger/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | # This is the Dockerfile for building a pinger. The pinger checks if the 16 | # application is listening on port 8080 by connecting to its network stack. 17 | FROM debian 18 | RUN apt-get update && apt-get install -y python 19 | ADD ./pinger.py / 20 | ENTRYPOINT while true; do sleep 1000; done; 21 | -------------------------------------------------------------------------------- /appstart/pinger/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Package that contains the files necessary to build the pinger. 16 | 17 | The pinger is intended to ping the application container to check if it's 18 | listening on various ports. It does so by connecting to the container's 19 | network stack and attempting to establish a socket with the desired port. 20 | """ 21 | -------------------------------------------------------------------------------- /appstart/pinger/pinger.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/python 2 | # Copyright 2015 Google Inc. All Rights Reserved. 3 | # 4 | # Licensed under the Apache License, Version 2.0 (the "License"); 5 | # you may not use this file except in compliance with the License. 6 | # You may obtain a copy of the License at 7 | # 8 | # http://www.apache.org/licenses/LICENSE-2.0 9 | # 10 | # Unless required by applicable law or agreed to in writing, software 11 | # distributed under the License is distributed on an "AS IS" BASIS, 12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 | # See the License for the specific language governing permissions and 14 | # limitations under the License. 15 | 16 | # This file conforms to the external style guide. 17 | # pylint: disable=bad-indentation 18 | 19 | """Pings the app by trying to establish a connection on the specified port. 20 | 21 | When Appstart actually starts the application container, it exposes port 8080 22 | on the container by mapping some port X on the Docker host to 8080 within the 23 | application, where X is determined at runtime. For proper behavior, the 24 | application needs to actually be listening on port 8080. To determine if the 25 | application is in fact listening on the port, it's not enough to simply 26 | establish a connection with port X on the Docker host. Due to the way port 27 | mappings are done, a connection can always be established with port X, even if 28 | there's nothing listening on 8080 inside the container. To bypass this issue, 29 | the pinger tries to establish a connection on 8080 from INSIDE the same network 30 | stack as the application. 31 | 32 | The alternative is simply to send an actual request to port X. Due to the port 33 | mapping, docker would attempt to forward the request to 8080 inside the 34 | container. The response can then be examined to see if a service is listening. 35 | The problem with this approach is that the request may actually cause the 36 | container to change state in an unpredictable way. 37 | 38 | To actually run the pinger, a container is created and put on the same network 39 | stack as the application container. It's then possible to run the pinger via 40 | docker exec and see its exit status. The actual running of pinger.py is done in 41 | appstart.sandbox.container.PingerContainer. 42 | """ 43 | 44 | import httplib 45 | import logging 46 | import socket 47 | import sys 48 | 49 | 50 | def ping(): 51 | """Check if container is listening on the specified port.""" 52 | try: 53 | host = sys.argv[1] 54 | port = int(sys.argv[2]) 55 | except (IndexError, ValueError): 56 | host = '0.0.0.0' 57 | port = 8080 58 | 59 | con = None 60 | success = True 61 | try: 62 | con = httplib.HTTPConnection(host, port) 63 | con.connect() 64 | except (socket.error, httplib.HTTPException): 65 | success = False 66 | finally: 67 | if con: 68 | con.close() 69 | if success: 70 | logging.info('success') 71 | sys.exit(0) 72 | logging.info('failure') 73 | sys.exit(1) 74 | 75 | 76 | if __name__ == '__main__': 77 | logging.basicConfig(level=logging.INFO) 78 | ping() 79 | -------------------------------------------------------------------------------- /appstart/sandbox/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Utilities to deploy a managed VM application locally for testing.""" 16 | -------------------------------------------------------------------------------- /appstart/sandbox/app.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | 3 | # Phony app.yaml file for devappserver. 4 | 5 | module: default 6 | runtime: python27 7 | vm: true 8 | api_version: 1 9 | threadsafe: true 10 | 11 | resources: 12 | cpu: .5 13 | memory_gb: 1.3 14 | 15 | manual_scaling: 16 | instances: 1 17 | 18 | handlers: 19 | - url: /.* 20 | script: phony.app 21 | -------------------------------------------------------------------------------- /appstart/sandbox/configuration.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | 3 | """Parser of application configuration files. 4 | 5 | These include appengine-web.xml files and *.yaml files. 6 | """ 7 | 8 | # This file conforms to the external style guide. 9 | # pylint: disable=bad-indentation, g-bad-import-order 10 | 11 | import os 12 | import xml.dom.minidom 13 | from xml.parsers import expat 14 | import yaml 15 | 16 | from .. import utils 17 | 18 | 19 | class ApplicationConfiguration(object): 20 | """Class to parse an xml or yaml config file. 21 | 22 | Extract the necessary configuration details. Currently, only health 23 | check information is required. 24 | """ 25 | 26 | def __init__(self, config_file): 27 | """Initializer for ApplicationConfiguration. 28 | 29 | Args: 30 | config_file: (basestring) The absolute path to the configuration 31 | file. 32 | 33 | Raises: 34 | utils.AppstartAbort: If the config file neither ends with .yaml 35 | nor is an appengine-web.xml file. 36 | """ 37 | self._verify_structure(config_file) 38 | if config_file.endswith('.yaml'): 39 | self._init_from_yaml_config(config_file) 40 | self.is_java = False 41 | elif os.path.basename(config_file) == 'appengine-web.xml': 42 | self._init_from_xml_config(config_file) 43 | self.is_java = True 44 | else: 45 | raise utils.AppstartAbort('{0} is not a valid ' 46 | 'configuration file. Use either a .yaml ' 47 | 'file or .xml file.'.format(config_file)) 48 | 49 | def _init_from_xml_config(self, xml_config): 50 | """Initialize from an xml file. 51 | 52 | Args: 53 | xml_config: (basestring) The absolute path to an appengine-web.xml 54 | file. 55 | 56 | Raises: 57 | utils.AppstartAbort: If "true" is not set in the 58 | configuration. 59 | """ 60 | try: 61 | root = xml.dom.minidom.parse(xml_config).firstChild 62 | except expat.ExpatError: 63 | raise utils.AppstartAbort('Malformed xml file: ' 64 | '{0}'.format(xml_config)) 65 | try: 66 | vm = root.getElementsByTagName('vm')[0] 67 | assert vm.firstChild.nodeValue == 'true' 68 | except (IndexError, AttributeError, AssertionError): 69 | raise utils.AppstartAbort( 70 | '"true" must be set in ' 71 | '{0}'.format(os.path.basename(xml_config))) 72 | 73 | # Assume that health checks are enabled. 74 | self.health_checks_enabled = True 75 | health = root.getElementsByTagName('health-check') 76 | if health: 77 | checks = health[0].getElementsByTagName('enable-health-check') 78 | if checks: 79 | value = checks[0].firstChild 80 | if value and value.nodeValue != 'true': 81 | self.health_checks_enabled = False 82 | 83 | def _init_from_yaml_config(self, yaml_config): 84 | """Initialize from a yaml file. 85 | 86 | Args: 87 | yaml_config: (basestring) The absolute path to a *.yaml 88 | file. 89 | 90 | Raises: 91 | utils.AppstartAbort: if "vm: true" is not set in the configuration. 92 | """ 93 | yaml_dict = yaml.load(open(yaml_config)) 94 | if not isinstance(yaml_dict, dict): 95 | raise utils.AppstartAbort('Malformed yaml file: ' 96 | '{0}'.format(yaml_config)) 97 | if not yaml_dict.get('vm'): 98 | raise utils.AppstartAbort( 99 | '"vm: true" must be set in ' 100 | '{0}'.format(os.path.basename(yaml_config))) 101 | hc_options = yaml_dict.get('health_check') 102 | if hc_options and not hc_options.get('enable_health_check', True): 103 | self.health_checks_enabled = False 104 | else: 105 | self.health_checks_enabled = True 106 | 107 | @staticmethod 108 | def _verify_structure(full_config_file_path): 109 | """Verify the existence of the configuration files. 110 | 111 | If the config file is an xml file, there also 112 | needs to be a web.xml file in the same directory. 113 | 114 | Args: 115 | full_config_file_path: (basestring) The absolute path to a 116 | .xml or .yaml config file. 117 | 118 | Raises: 119 | utils.AppstartAbort: If the application is a Java app, and 120 | the web.xml file cannot be found, or the config 121 | file cannot be found. 122 | """ 123 | if not os.path.exists(full_config_file_path): 124 | raise utils.AppstartAbort('The path %s could not be resolved.' % 125 | full_config_file_path) 126 | 127 | if full_config_file_path.endswith('.xml'): 128 | webxml = os.path.join(os.path.dirname(full_config_file_path), 129 | 'web.xml') 130 | if not os.path.exists(webxml): 131 | raise utils.AppstartAbort('Could not find web.xml at: ' 132 | '{}'.format(webxml)) 133 | -------------------------------------------------------------------------------- /appstart/sandbox/container.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Wrapper around docker.Client to create semblance of a container.""" 16 | 17 | # This file conforms to the external style guide. 18 | # pylint: disable=bad-indentation, g-bad-import-order 19 | 20 | import requests 21 | import signal 22 | import StringIO 23 | import tarfile 24 | import threading 25 | import urlparse 26 | 27 | import docker 28 | 29 | from .. import utils 30 | 31 | 32 | _EXITING = False 33 | 34 | 35 | def sig_handler(unused_signo, unused_frame): 36 | global _EXITING 37 | _EXITING = True 38 | 39 | 40 | class Container(object): 41 | """Wrapper around docker container.""" 42 | 43 | def __init__(self, dclient): 44 | """Initializer for Container. 45 | 46 | Args: 47 | dclient: (docker.Client) The docker client that is managing 48 | the container. 49 | """ 50 | self._container_id = None 51 | self._dclient = dclient 52 | res = urlparse.urlparse(self._dclient.base_url) 53 | self.host = (res.hostname if res.hostname != 'localunixsocket' 54 | else 'localhost') 55 | self.name = None 56 | 57 | def create(self, **docker_kwargs): 58 | """Do the work of calling docker.Client.create_container. 59 | 60 | The purpose of separating this functionality from __init__ is to be 61 | safe from the race condition (as documented below). 62 | 63 | Args: 64 | **docker_kwargs: (dict) Keyword arguments that can be supplied 65 | to docker.Client.create_container. 66 | 67 | Raises: 68 | KeyboardInterrupt: If SIGINT is caught during 69 | docker.client.create_container. 70 | """ 71 | # Anticipate the possibility of SIGINT during construction. 72 | # Note that graceful behavior is guaranteed only for SIGINT. 73 | prev = signal.signal(signal.SIGINT, sig_handler) 74 | 75 | # Protecting create_container in this manner ensures that there 76 | # is GUARANTEED to be a container_id after this call. Then, 77 | # if there was a KeyboardInterrupt, it'll bubble up to higher 78 | # level error handling, which should remove the container. 79 | # This solves the problem where create_container gets interrupted 80 | # AFTER the container is created but BEFORE a result is returned. 81 | try: 82 | self._container_id = ( 83 | self._dclient.create_container(**docker_kwargs).get('Id')) 84 | except docker.errors.APIError as err: 85 | raise utils.AppstartAbort('Could not create container because: ' 86 | '{0}'.format(err)) 87 | 88 | # Restore previous handler 89 | signal.signal(signal.SIGINT, prev) 90 | 91 | # If _EXITING is True, then the signal handler was called. 92 | if _EXITING: 93 | raise KeyboardInterrupt 94 | 95 | self.name = docker_kwargs.get('name') 96 | 97 | def kill(self): 98 | """Kill the underlying container.""" 99 | 100 | # "removed" container is occasionally killed in ContainerSandbox. 101 | # Stay silent about this scenario. 102 | if self._container_id: 103 | self._dclient.kill(self._container_id) 104 | 105 | def remove(self): 106 | """Remove the underlying container.""" 107 | 108 | # Containers are occasionally removed twice in ContainerSandbox. 109 | # Stay silent about this scenario. 110 | if self._container_id: 111 | self._dclient.remove_container(self._container_id) 112 | self._container_id = None 113 | 114 | def start(self, **start_kwargs): 115 | """Start the container. 116 | 117 | Args: 118 | **start_kwargs: (dict) Additional kwargs to be supplied to 119 | docker.Client.start. 120 | """ 121 | try: 122 | self._dclient.start(self._container_id, **start_kwargs) 123 | utils.get_logger().info('Starting container: {0}'.format(self.name)) 124 | except docker.errors.APIError as err: 125 | raise utils.AppstartAbort('Docker error: {0}'.format(err)) 126 | 127 | def stream_logs(self, stream=True): 128 | """Print the container's stdout/stderr. 129 | 130 | Args: 131 | stream: (bool) Whether or not to continue streaming stdout/stderr. 132 | If False, only the current stdout/stderr buffer will be 133 | collected from the container. If True, stdout/stderr collection 134 | will continue as a subprocess. 135 | """ 136 | 137 | def log_streamer(): 138 | # This loop tackles the problem of request timeouts. When the 139 | # docker client is created, it establishes a timeout. The 140 | # default is 60 seconds. If docker.Client.logs hangs for 141 | # more than 60 seconds, this is considered a "timeout". 142 | name = (self._dclient.inspect_container(self._container_id) 143 | .get('Name')) 144 | while True: 145 | try: 146 | # If a timeout happens, an error will be raise from inside 147 | # the log generator. 148 | logs = self._dclient.logs(container=self._container_id, 149 | stream=True) 150 | for line in logs: 151 | utils.get_logger().debug('{0}: {1}'.format( 152 | name, line.strip())) 153 | 154 | # In the case of a timeout, try to start collecting logs again. 155 | except requests.exceptions.ReadTimeout: 156 | pass 157 | 158 | # An APIError/NullResource occurs if the container doesn't 159 | # exist anymore. This indicates that we're shutting down, so 160 | # we can break and terminate the thread. (The older versions 161 | # produce an APIError). 162 | except (docker.errors.APIError, docker.errors.NullResource): 163 | break 164 | 165 | if stream: 166 | # If we want to stream the log output of the container, 167 | # start another thread. There's no need to join this thread, 168 | # because it's supposed to live until the container is removed. 169 | thread = threading.Thread(target=log_streamer) 170 | thread.start() 171 | else: 172 | logs = self._dclient.logs(container=self._container_id, 173 | stream=False) 174 | for line in logs.split('\n'): 175 | utils.get_logger().debug(line.strip()) 176 | 177 | def running(self): 178 | """Check if the container is still running. 179 | 180 | Returns: 181 | (bool) Whether or not the container is running. 182 | """ 183 | if not self._container_id: 184 | return False 185 | res = self._dclient.inspect_container(self._container_id) 186 | return res['State']['Running'] 187 | 188 | def get_id(self): 189 | return self._container_id 190 | 191 | def execute(self, cmd, **create_kwargs): 192 | """Execute the command specified by cmd inside the container. 193 | 194 | Args: 195 | cmd: (basestring) The command to execute. 196 | **create_kwargs: (dict) Arguments that can be supplied to 197 | docker.Client.exec_create. 198 | 199 | Returns: 200 | (dict) A dict of values as returned by docker.Client.exec_inspect. 201 | """ 202 | exec_id = self._dclient.exec_create(container=self._container_id, 203 | cmd=cmd, 204 | **create_kwargs).get('Id') 205 | self._dclient.exec_start(exec_id) 206 | return self._dclient.exec_inspect(exec_id) 207 | 208 | def extract_tar(self, path): 209 | """Extract the file/directory specified by path as a TarWrapper object. 210 | 211 | Args: 212 | path: (basestring) The path (within the container) 213 | to the file/directory to extract. 214 | 215 | Raises: 216 | IOError: If path cannot be resolved within the container. 217 | 218 | Returns: 219 | (utils.TarWrapper) The tar archive. 220 | """ 221 | try: 222 | reply = self._dclient.copy(self._container_id, path) 223 | except docker.errors.APIError: 224 | raise IOError('File could not be found at {0}.'.format(path)) 225 | 226 | fileobj = StringIO.StringIO(reply.read()) 227 | 228 | # Wrap the TarFile for more user-friendliness 229 | return utils.TarWrapper(tarfile.open(fileobj=fileobj)) 230 | 231 | 232 | class PingerContainer(Container): 233 | """Give devappserver the ability to ping the application. 234 | 235 | Relies on container having a pinger.py in the root directory. 236 | """ 237 | 238 | def ping_application_container(self): 239 | """Return True iff the application is listening on port 8080.""" 240 | return self.execute('python /pinger.py')['ExitCode'] == 0 241 | 242 | 243 | class ApplicationContainer(Container): 244 | """Explicitly give the application container a configuration file. 245 | 246 | This will be useful for validation. 247 | """ 248 | 249 | def __init__(self, app_config, *args, **kwargs): 250 | super(ApplicationContainer, self).__init__(*args, **kwargs) 251 | self.configuration = app_config 252 | -------------------------------------------------------------------------------- /appstart/sandbox/container_sandbox.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """A ContainerSandbox manages the application and devappserver containers. 16 | 17 | This includes their creation, termination, and destruction. 18 | ContainerSandbox is intended to be used inside a "with" statement. Inside 19 | the interior of the "with" statement, the user interact with the containers 20 | via the docker api. It may also be beneficial for the user to perform 21 | system tests in this manner. 22 | """ 23 | 24 | # This file conforms to the external style guide 25 | # pylint: disable=bad-indentation, g-bad-import-order 26 | 27 | import io 28 | import os 29 | import sys 30 | import time 31 | 32 | import docker 33 | 34 | import configuration 35 | import container 36 | from .. import utils 37 | from .. import constants 38 | from ..utils import get_logger 39 | 40 | 41 | # Maximum attempts to health check application container. 42 | MAX_ATTEMPTS = 30 43 | 44 | # Default port that the application is expected to listen on inside 45 | # the application container. 46 | DEFAULT_APPLICATION_PORT = 8080 47 | 48 | # Time format for naming images/containers 49 | TIME_FMT = '%Y.%m.%d_%H.%M.%S' 50 | 51 | # Java offset for the xml file's location, relative to the root 52 | # diretory of the WAR archive 53 | JAVA_OFFSET = 'WEB-INF/' 54 | 55 | 56 | class ContainerSandbox(object): 57 | """Sandbox to manage the user application & devappserver containers. 58 | 59 | This sandbox aims to leave the docker container space untouched. 60 | Proper usage ensures that application & devappserver containers will 61 | be created, started, stopped, and destroyed. For proper usage, the 62 | ContainerSandbox should be used as a context manager (inside a "with" 63 | statement), or the start and stop functions should be invoked from 64 | within a try-finally context. 65 | """ 66 | # pylint: disable=too-many-instance-attributes 67 | # pylint: disable=too-many-arguments 68 | 69 | def __init__(self, 70 | config_file=None, 71 | image_name=None, 72 | application_id=None, 73 | application_port=8080, 74 | admin_port=8000, 75 | proxy_port=8088, 76 | clear_datastore=False, 77 | internal_admin_port=32768, 78 | internal_api_port=32769, 79 | internal_proxy_port=32770, 80 | log_path=None, 81 | run_api_server=True, 82 | storage_path='/tmp/app_engine/storage', 83 | nocache=False, 84 | timeout=MAX_ATTEMPTS, 85 | force_version=False, 86 | devbase_image=constants.DEVAPPSERVER_IMAGE, 87 | extra_ports=None): 88 | """Get the sandbox ready to construct and run the containers. 89 | 90 | Args: 91 | config_file: (basestring or None) The relative or full path 92 | to the config_file of the application. At least one of 93 | image_name and config_file must be specified. If image_name is 94 | not specified, this path will be used to help find the 95 | Dockerfile and build the application container. 96 | Therefore, if image_name is not specified, there should 97 | be a Dockerfile in the correct location: 98 | 99 | Non-java apps (apps that use .yaml files) 100 | 1) The .yaml file must be in the root of the app 101 | directory. 102 | 2) The Dockerfile must be in the root of the app 103 | directory. 104 | 105 | Java apps (apps that are built off java-compat): 106 | 1) The appengine-web.xml file must be in 107 | /WEB-INF/ (where is the root 108 | directory of the WAR archive.) 109 | 2) The Dockerfile must be in the root of the WAR 110 | archive. 111 | 3) There must be a web.xml file in the same 112 | directory as the appengine-web.xml file. 113 | image_name: (basestring or None) If specified, the sandbox 114 | will run the image associated with image_name instead of 115 | building an image from the specified application_directory. 116 | application_id: (basestring) The application ID is 117 | the unique "appengine application ID" that the app is 118 | identified by, and can be found in the developer's 119 | console. While for deployment purposes, this ID is 120 | important, it's not as important in development. This 121 | ID only controls which datastore, blobstore, etc the 122 | sandbox will use. If the sandbox is run consecutively 123 | with the same application_id, (and of course, the same 124 | storage_path) the datastore, blobstore, taskqueue, etc 125 | will persist assuming their data has not been deleted. 126 | application_port: (int) The port on the docker host that should be 127 | mapped to the application. The application will be 128 | accessible through this port. 129 | admin_port: (int) The port on the docker server host that 130 | should be mapped to the admin server, which runs inside 131 | the devappserver container. The admin panel will be 132 | accessible through this port. 133 | devbase_image: (basestring or None): If specified, the sandbox 134 | will build the devappserver on the specified base_image 135 | clear_datastore: (bool) Whether or not to clear the datastore. 136 | If True, this eliminates all of the data from the datastore 137 | before running the api server. 138 | internal_admin_port: (int) The port INSIDE the devappserver 139 | container that the admin panel binds to. Because this 140 | is internal to the container, it can be defaulted. 141 | In fact, you shouldn't change it from the default unless 142 | you have a reason to. 143 | internal_api_port: (int) The port INSIDE the devappserver 144 | container that the api server should bind to. 145 | ~Same disclaimer as the one for internal_admin_port.~ 146 | internal_proxy_port: (int) The port INSIDE the devappserver 147 | container that the proxy should bind to. 148 | ~Same disclaimer as the one for internal_admin_port.~ 149 | log_path: (basestring or None) The path where the application's 150 | logs should be collected. Note that the application's logs 151 | will be collected EXTERNALLY (ie they will collect in the 152 | docker host's file system) and log_path specifies where 153 | these logs should go. If log_path is None, a timestamped 154 | name will be generated for the log directory. 155 | run_api_server: (bool) Whether or not to run the api server. 156 | If this argument is set to false, the sandbox won't start 157 | a devappserver. 158 | storage_path: (basestring) The path (external to the 159 | containers) where the data associated with the api 160 | server's services - datastore, blobstore, etc - should 161 | collect. Note that this path defaults to 162 | /tmp/appengine/storage, so it should be changed if the data 163 | is intended to persist. 164 | nocache: (bool) Whether or not to use the cache when building 165 | images. 166 | timeout: (int) How many seconds to wait for the application 167 | container to start. 168 | force_version: (bool) Whether or not to continue in the case 169 | of mismatched docker versions. 170 | extra_ports: ({int: int, ...} or None) A mapping from application 171 | docker container ports to host ports, allowing 172 | additional application ports to be exposed. 173 | """ 174 | self.cur_time = time.strftime(TIME_FMT) 175 | self.app_id = (application_id or None) 176 | self.internal_api_port = internal_api_port 177 | self.internal_proxy_port = internal_proxy_port 178 | self.internal_admin_port = internal_admin_port 179 | self.clear_datastore = clear_datastore 180 | self.port = application_port 181 | self.storage_path = storage_path 182 | self.log_path = ( 183 | log_path or self.make_timestamped_name( 184 | '/tmp/log/app_engine/app_logs', 185 | self.cur_time)) 186 | self.image_name = image_name 187 | self.admin_port = admin_port 188 | self.proxy_port = proxy_port 189 | self.dclient = utils.get_docker_client() 190 | self.devappserver_container = None 191 | self.app_container = None 192 | self.pinger_container = None 193 | self.nocache = nocache 194 | self.run_devappserver = run_api_server 195 | self.timeout = timeout 196 | self.devbase_image=constants.DEVAPPSERVER_IMAGE 197 | self.extra_ports = extra_ports 198 | 199 | if devbase_image: 200 | self.devbase_image=devbase_image 201 | 202 | if config_file: 203 | self.conf_path = os.path.abspath(config_file) 204 | else: 205 | if not image_name: 206 | raise utils.AppstartAbort('At least one of config_file and ' 207 | 'image_name must be specified.') 208 | self.conf_path = os.path.join(os.path.dirname(__file__), 209 | 'app.yaml') 210 | self.application_configuration = ( 211 | configuration.ApplicationConfiguration(self.conf_path)) 212 | 213 | self.app_dir = self.app_directory_from_config(self.conf_path) 214 | 215 | # For Java apps, the xml file must be offset by WEB-INF. 216 | # Otherwise, devappserver will think that it's a non-java app. 217 | self.das_offset = (JAVA_OFFSET if 218 | self.application_configuration.is_java else '') 219 | 220 | if not force_version: 221 | utils.check_docker_version(self.dclient) 222 | 223 | def __enter__(self): 224 | self.start() 225 | return self 226 | 227 | def start(self): 228 | """Start the sandbox.""" 229 | try: 230 | self.create_and_run_containers() 231 | except: # pylint: disable=bare-except 232 | self.stop() 233 | raise 234 | 235 | def create_and_run_containers(self): 236 | """Creates and runs app and (optionally) devappserver containers. 237 | 238 | This includes the creation of a new devappserver image, unless 239 | self.run_devappserver is False. If image_name isn't specified, an 240 | image is created for the application as well. Newly made containers 241 | are cleaned up, but newly made images are not. 242 | """ 243 | 244 | if self.run_devappserver: 245 | # Devappserver must know APP_ID to properly interface with 246 | # services like datastore, blobstore, etc. It also needs 247 | # to know where to find the config file, which port to 248 | # run the proxy on, and which port to run the api server on. 249 | das_env = {'CLEAR_DATASTORE': self.clear_datastore, 250 | 'PROXY_PORT': self.internal_proxy_port, 251 | 'API_PORT': self.internal_api_port, 252 | 'ADMIN_PORT': self.internal_admin_port, 253 | 'CONFIG_FILE': os.path.join( 254 | self.das_offset, 255 | os.path.basename(self.conf_path))} 256 | 257 | if self.app_id: 258 | das_env['APP_ID'] = self.app_id 259 | 260 | devappserver_image = self.build_devappserver_image( 261 | devbase_image=self.devbase_image 262 | ) 263 | devappserver_container_name = ( 264 | self.make_timestamped_name('devappserver', 265 | self.cur_time)) 266 | 267 | port_bindings = { 268 | DEFAULT_APPLICATION_PORT: self.port, 269 | self.internal_admin_port: self.admin_port, 270 | self.internal_proxy_port: self.proxy_port, 271 | } 272 | if self.extra_ports: 273 | port_bindings.update(self.extra_ports) 274 | 275 | # The host_config specifies port bindings and volume bindings. 276 | # /storage is bound to the storage_path. Internally, the 277 | # devappserver writes all the db files to /storage. The mapping 278 | # thus allows these files to appear on the host machine. As for 279 | # port mappings, we only want to expose the application (via the 280 | # proxy), and the admin panel. 281 | devappserver_hconf = docker.utils.create_host_config( 282 | port_bindings=port_bindings, 283 | binds={ 284 | self.storage_path: {'bind': '/storage'}, 285 | } 286 | ) 287 | 288 | self.devappserver_container = container.Container(self.dclient) 289 | self.devappserver_container.create( 290 | name=devappserver_container_name, 291 | image=devappserver_image, 292 | ports=port_bindings.keys(), 293 | volumes=['/storage'], 294 | host_config=devappserver_hconf, 295 | environment=das_env) 296 | 297 | self.devappserver_container.start() 298 | get_logger().info('Starting container: %s', 299 | devappserver_container_name) 300 | 301 | # The application container needs several environment variables 302 | # in order to start up the application properly, as well as 303 | # look for the api server in the correct place. Notes: 304 | # 305 | # GAE_PARTITION is always dev for development modules. 306 | # GAE_LONG_APP_ID is the "application ID". When devappserver 307 | # is invoked, it can be passed a "--application" flag. This 308 | # application must be consistent with GAE_LONG_APP_ID. 309 | # API_HOST is 0.0.0.0 because application container runs on the 310 | # same network stack as devappserver. 311 | # MODULE_YAML_PATH specifies the path to the app from the 312 | # app directory 313 | 314 | # TODO (find in g3 and link to here via comment) 315 | app_env = {'API_HOST': '0.0.0.0', 316 | 'API_PORT': self.internal_api_port, 317 | 'GAE_LONG_APP_ID': self.app_id, 318 | 'GAE_PARTITION': 'dev', 319 | 'GAE_MODULE_INSTANCE': '0', 320 | 'MODULE_YAML_PATH': os.path.basename(self.conf_path), 321 | 'GAE_MODULE_NAME': 'default', # TODO(gouzenko) parse app.yaml 322 | 'GAE_MODULE_VERSION': '1', 323 | 'GAE_SERVER_PORT': '8080', 324 | 'USE_MVM_AGENT': 'true'} 325 | 326 | # Build from the application directory iff image_name is not 327 | # specified. 328 | app_image = self.image_name or self.build_app_image() 329 | app_container_name = self.make_timestamped_name('test_app', 330 | self.cur_time) 331 | 332 | # If devappserver is running, hook up the app to it. 333 | if self.run_devappserver: 334 | network_mode = ('container:%s' % 335 | self.devappserver_container.get_id()) 336 | ports = port_bindings = None 337 | else: 338 | port_bindings = {DEFAULT_APPLICATION_PORT: self.port} 339 | ports = [DEFAULT_APPLICATION_PORT] 340 | network_mode = None 341 | 342 | app_hconf = docker.utils.create_host_config( 343 | port_bindings=port_bindings, 344 | binds={ 345 | self.log_path: {'bind': '/var/log/app_engine'} 346 | }, 347 | 348 | ) 349 | 350 | self.app_container = container.ApplicationContainer( 351 | self.application_configuration, 352 | self.dclient) 353 | self.app_container.create( 354 | name=app_container_name, 355 | image=app_image, 356 | ports=ports, 357 | volumes=['/var/log/app_engine'], 358 | host_config=app_hconf, 359 | environment=app_env) 360 | 361 | # Start as a shared network container, putting the application 362 | # on devappserver's network stack. (If devappserver is not 363 | # running, network_mode is None). 364 | try: 365 | self.app_container.start(network_mode=network_mode) 366 | except utils.AppstartAbort: 367 | if self.run_devappserver: 368 | self.abort_if_not_running(self.devappserver_container) 369 | raise 370 | 371 | # Construct a pinger container and bind it to the application's network 372 | # stack. This will allow the pinger to attempt to connect to the 373 | # application's ports. 374 | pinger_name = self.make_timestamped_name('pinger', self.cur_time) 375 | self.pinger_container = container.PingerContainer(self.dclient) 376 | try: 377 | self.pinger_container.create(name=pinger_name, 378 | image=constants.PINGER_IMAGE) 379 | except utils.AppstartAbort: 380 | if not utils.find_image(constants.PINGER_IMAGE): 381 | raise utils.AppstartAbort('No pinger image found. ' 382 | 'Did you forget to run "appstart ' 383 | 'init"? ') 384 | raise 385 | 386 | try: 387 | self.pinger_container.start( 388 | network_mode='container:{}'.format(self.app_container.get_id())) 389 | except utils.AppstartAbort: 390 | self.abort_if_not_running(self.app_container) 391 | raise 392 | 393 | self.wait_for_start() 394 | self.app_container.stream_logs() 395 | 396 | def stop(self): 397 | """Remove containers to clean up the environment.""" 398 | self.stop_and_remove_containers() 399 | 400 | @staticmethod 401 | def abort_if_not_running(cont): 402 | if not cont.running(): 403 | cont.stream_logs(stream=False) 404 | raise utils.AppstartAbort('{0} stopped ' 405 | 'prematurely'.format(cont.name)) 406 | 407 | def __exit__(self, etype, value, traceback): 408 | self.stop() 409 | 410 | def stop_and_remove_containers(self): 411 | """Stop and remove application containers.""" 412 | containers_to_remove = [self.app_container, 413 | self.devappserver_container, 414 | self.pinger_container] 415 | for cont in containers_to_remove: 416 | if cont and cont.running(): 417 | cont_id = cont.get_id() 418 | get_logger().info('Stopping %s', cont_id) 419 | cont.kill() 420 | 421 | get_logger().info('Removing %s', cont_id) 422 | cont.remove() 423 | 424 | def wait_for_start(self): 425 | """Wait for the app container to start. 426 | 427 | Raises: 428 | utils.AppstartAbort: If the application server doesn't 429 | start after timeout reach it on 8080. 430 | """ 431 | host = self.app_container.host 432 | 433 | get_logger().info('Waiting for application to listen on port 8080') 434 | attempt = 1 435 | graphical = sys.stdout.isatty() 436 | 437 | def print_if_graphical(message): 438 | if graphical: 439 | sys.stdout.write(message) 440 | sys.stdout.flush() 441 | 442 | def exit_loop_with_error(error): 443 | print_if_graphical('\n') 444 | raise utils.AppstartAbort(error) 445 | 446 | print_if_graphical('Waiting ') 447 | while True: 448 | if attempt > self.timeout: 449 | exit_loop_with_error('The application server timed out.') 450 | 451 | if self.run_devappserver: 452 | self.abort_if_not_running(self.devappserver_container) 453 | 454 | self.abort_if_not_running(self.app_container) 455 | 456 | if attempt % 4 == 0: 457 | # \033[3D moves the cursor left 3 times. \033[K clears to the 458 | # end of the line. So, every 4th ping, clear the dots. 459 | print_if_graphical('\033[3D\033[K') 460 | else: 461 | print_if_graphical('.') 462 | 463 | if self.pinger_container.ping_application_container(): 464 | print_if_graphical('\n') 465 | break 466 | 467 | attempt += 1 468 | time.sleep(1) 469 | 470 | # Tell the user where to connect, depending on whether or not the 471 | # devappserver is running. 472 | if self.run_devappserver: 473 | port = self.proxy_port 474 | else: 475 | port = self.port 476 | get_logger().info('Your application is live. ' 477 | 'Access it at: {0}:{1}'.format(host, port)) 478 | if self.run_devappserver: 479 | get_logger().info('(port {0} goes through the dev_appserver ' 480 | 'proxy, for direct access use {1})'.format( 481 | self.proxy_port, 482 | self.port)) 483 | 484 | @staticmethod 485 | def app_directory_from_config(full_config_file_path): 486 | """Get the application root directory based on the config file. 487 | 488 | Args: 489 | full_config_file_path: (basestring) The absolute path to a 490 | config file. 491 | 492 | Returns: 493 | (basestring): The application's root directory. 494 | """ 495 | conf_file_dir = os.path.dirname(full_config_file_path) 496 | if full_config_file_path.endswith('.yaml'): 497 | return conf_file_dir 498 | else: 499 | return os.path.dirname(conf_file_dir) 500 | 501 | def build_app_image(self): 502 | """Build the app image from the Dockerfile in the root directory. 503 | 504 | Returns: 505 | (basestring) The name of the new app image. 506 | """ 507 | name = self.make_timestamped_name('app_image', self.cur_time) 508 | utils.build_from_directory(self.app_dir, name) 509 | return name 510 | 511 | def build_devappserver_image(self,devbase_image=constants.DEVAPPSERVER_IMAGE): 512 | """Build a layer over devappserver to include application files. 513 | 514 | The new image contains the user's config files. 515 | 516 | Returns: 517 | (basestring) The name of the new devappserver image. 518 | """ 519 | # Collect the files that should be added to the docker build 520 | # context. 521 | files_to_add = {self.conf_path: None} 522 | if self.application_configuration.is_java: 523 | files_to_add[self.get_web_xml(self.conf_path)] = None 524 | utils.add_files_from_static_dirs(files_to_add, self.conf_path) 525 | 526 | # The Dockerfile should add the config files to 527 | # the /app folder in devappserver's container. 528 | dockerfile = """ 529 | FROM %(das_repo)s 530 | ADD %(path)s/ %(dest)s 531 | WORKDIR /app/ 532 | """ % {'das_repo': devbase_image, 533 | 'path': os.path.dirname(self.conf_path), 534 | 'dest': os.path.join('/app', self.das_offset)} 535 | 536 | # Construct a file-like object from the Dockerfile. 537 | dockerfile_obj = io.BytesIO(dockerfile.encode('utf-8')) 538 | build_context = utils.make_tar_build_context(dockerfile_obj, 539 | files_to_add) 540 | image_name = self.make_timestamped_name('devappserver_image', 541 | self.cur_time) 542 | 543 | # Build the devappserver image. 544 | res = self.dclient.build(fileobj=build_context, 545 | custom_context=True, 546 | rm=True, 547 | nocache=self.nocache, 548 | tag=image_name) 549 | 550 | # Log the output of the build. 551 | try: 552 | utils.log_and_check_build_results(res, image_name) 553 | except utils.AppstartAbort: 554 | if not utils.find_image(constants.DEVAPPSERVER_IMAGE): 555 | raise utils.AppstartAbort('No devappserver base image found. ' 556 | 'Did you forget to run "appstart ' 557 | 'init"?') 558 | raise 559 | return image_name 560 | 561 | @staticmethod 562 | def get_web_xml(full_config_file_path): 563 | """Get (what should be) the path of the web.xml file. 564 | 565 | Args: 566 | full_config_file_path: (basestring) The absolute path to a 567 | .xml config file. 568 | 569 | Returns: 570 | (basestring) The full path to the web.xml file. 571 | """ 572 | return os.path.join(os.path.dirname(full_config_file_path), 573 | 'web.xml') 574 | 575 | @staticmethod 576 | def make_timestamped_name(base, time_str): 577 | """Construct a name for an image or container. 578 | 579 | Note that naming is functionally unimportant and 580 | serves only to make the output of 'docker images' 581 | and 'docker ps' look cleaner. 582 | 583 | Args: 584 | base: (basestring) The prefix of the name. 585 | time_str: (basestring) The name's timestamp. 586 | Returns: 587 | (basestring) The name of the image or container. 588 | """ 589 | return '%s.%s' % (base, time_str) 590 | -------------------------------------------------------------------------------- /appstart/utils.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Helper functions for components of appstart.""" 16 | 17 | # This file follows the external style guide. 18 | # pylint: disable=bad-indentation, g-bad-import-order 19 | 20 | import logging 21 | import io 22 | import json 23 | import os 24 | import re 25 | import requests 26 | import socket 27 | import ssl 28 | import sys 29 | import tarfile 30 | import tempfile 31 | import yaml 32 | 33 | import docker 34 | 35 | 36 | # HTTP timeout for docker client 37 | TIMEOUT_SECS = 60 38 | 39 | # Default docker host if user isn't using boot2docker 40 | LINUX_DOCKER_HOST = '/var/run/docker.sock' 41 | 42 | # Supported docker versions 43 | DOCKER_API_VERSION = '1.17' 44 | MIN_DOCKER_VERSION = [1, 8, 0] 45 | MAX_DOCKER_VERSION = [1, 9, 1000] 46 | 47 | # Logger that is shared accross all components of appstart 48 | _logger = None 49 | 50 | # Logging format 51 | FMT = '[%(levelname).1s: %(asctime)s] %(message)s' 52 | DATE_FMT = '%H:%M:%S' 53 | 54 | INT_RX = re.compile(r'\d+') 55 | 56 | 57 | class AppstartAbort(Exception): 58 | pass 59 | 60 | 61 | def get_logger(): 62 | """Configures the appstart logger if it doesn't exist yet. 63 | 64 | Returns: 65 | (logging.Logger) a logger used to log messages on behalf of 66 | appstart. 67 | """ 68 | global _logger 69 | if _logger is None: 70 | _logger = logging.getLogger('appstart') 71 | _logger.setLevel(logging.DEBUG) 72 | sh = logging.StreamHandler() 73 | sh.setLevel(logging.DEBUG) 74 | formatter = logging.Formatter(fmt=FMT, datefmt=DATE_FMT) 75 | sh.setFormatter(formatter) 76 | _logger.addHandler(sh) 77 | return _logger 78 | 79 | 80 | def _soft_int(val): 81 | """Convert strings to integers without dying on non-integer values.""" 82 | m = INT_RX.match(val) 83 | if m: 84 | return int(m.group()) 85 | else: 86 | return 0 87 | 88 | 89 | def format_version(version): 90 | """Converts a version specified as a list of integers to a string. 91 | 92 | e.g. [1, 2, 3] -> '1.2.3' 93 | 94 | Args: 95 | version: ([int, ...]) Version as a list of integers. 96 | 97 | Returns: 98 | (str) Stringified version. 99 | """ 100 | return '.'.join(str(x) for x in version) 101 | 102 | 103 | def check_docker_version(dclient): 104 | """Check version of docker server and log errors if it's too old/new. 105 | 106 | The currently supported versions of docker are specified in 107 | {MIN,MAX}_DOCKER_VERSION above. 108 | 109 | Args: 110 | dclient: (docker.Client) The docker client to use to connect to the 111 | docker server. 112 | 113 | Raises: 114 | AppstartAbort: If user's docker server version is not correct. 115 | """ 116 | version = dclient.version() 117 | server_version = [_soft_int(x) for x in version.get('Version').split('.')] 118 | if (server_version < MIN_DOCKER_VERSION or 119 | server_version > MAX_DOCKER_VERSION): 120 | raise AppstartAbort('Expected docker server version between {0} and {1}. ' 121 | 'Found server version {2}. Use --force_version ' 122 | 'flag to run Appstart ' 123 | 'anyway'.format(format_version(MIN_DOCKER_VERSION), 124 | format_version(MAX_DOCKER_VERSION), 125 | format_version(server_version))) 126 | 127 | # TODO(mmuller): "ClientWrapper" is a pretty horrible kludge. Rewrite it so 128 | # that it doesn't indiscriminately reconnect every time an attribute is 129 | # accessed. 130 | 131 | 132 | class ClientWrapper(object): 133 | 134 | def __init__(self, **params): 135 | self.__params = params 136 | 137 | def __getattr__(self, attrname): 138 | return getattr(docker.Client(**self.__params), attrname) 139 | 140 | 141 | def get_docker_client(): 142 | """Get the user's docker client. 143 | 144 | Raises: 145 | AppstartAbort: If there was an error in connecting to the 146 | Docker Daemon. 147 | 148 | Returns: 149 | (docker.Client) a docker client that can be used to manage 150 | containers and images. 151 | """ 152 | host = os.environ.get('DOCKER_HOST') 153 | cert_path = os.environ.get('DOCKER_CERT_PATH') 154 | tls_verify = int(os.environ.get('DOCKER_TLS_VERIFY', 0)) 155 | 156 | params = {} 157 | if host: 158 | params['base_url'] = (host.replace('tcp://', 'https://') 159 | if tls_verify else host) 160 | elif sys.platform.startswith('linux'): 161 | # if this is a linux user, the default value of DOCKER_HOST 162 | # should be the unix socket. first check if the socket is 163 | # valid to give a better feedback to the user. 164 | if os.path.exists(LINUX_DOCKER_HOST): 165 | sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) 166 | try: 167 | sock.connect(LINUX_DOCKER_HOST) 168 | params['base_url'] = 'unix://' + LINUX_DOCKER_HOST 169 | except socket.error: 170 | get_logger().warning('Found a stale ' 171 | '/var/run/docker.sock, ' 172 | 'did you forget to start ' 173 | 'your docker daemon?') 174 | finally: 175 | sock.close() 176 | 177 | if tls_verify and cert_path: 178 | # assert_hostname=False is needed for boot2docker to work with 179 | # our custom registry. 180 | params['tls'] = docker.tls.TLSConfig( 181 | client_cert=(os.path.join(cert_path, 'cert.pem'), 182 | os.path.join(cert_path, 'key.pem')), 183 | ca_cert=os.path.join(cert_path, 'ca.pem'), 184 | verify=True, 185 | ssl_version=ssl.PROTOCOL_TLSv1, 186 | assert_hostname=False) 187 | 188 | # pylint: disable=star-args 189 | client = ClientWrapper(version=DOCKER_API_VERSION, 190 | timeout=TIMEOUT_SECS, 191 | **params) 192 | try: 193 | client.ping() 194 | except requests.exceptions.ConnectionError as excep: 195 | raise AppstartAbort('Failed to connect to Docker ' 196 | 'Daemon due to: {0}'.format(excep.message)) 197 | return client 198 | 199 | 200 | def build_from_directory(dirname, image_name, nocache=False): 201 | """Builds devappserver base image from source using a Dockerfile.""" 202 | dclient = get_docker_client() 203 | 204 | res = dclient.build(path=dirname, 205 | rm=True, 206 | nocache=nocache, 207 | tag=image_name) 208 | 209 | try: 210 | log_and_check_build_results(res, image_name) 211 | except docker.errors.DockerException as err: 212 | raise AppstartAbort(err.message) 213 | 214 | 215 | def make_tar_build_context(dockerfile, context_files): 216 | """Compose tar file for the new devappserver layer's build context. 217 | 218 | Args: 219 | dockerfile: (io.BytesIO or file) a file-like object 220 | representing the Dockerfile. 221 | context_files: ({basestring: basestring, ...}) a dictionary 222 | mapping absolute filepaths to their destination name in 223 | the tar build context. This is used to specify other files 224 | that should be added to the build context. 225 | 226 | Returns: 227 | (tempfile.NamedTemporaryFile) a temporary tarfile 228 | representing the docker build context. 229 | """ 230 | 231 | f = tempfile.NamedTemporaryFile() 232 | t = tarfile.open(mode='w', fileobj=f) 233 | 234 | # Add dockerfile to top level under the name "Dockerfile" 235 | if isinstance(dockerfile, io.BytesIO): 236 | dfinfo = tarfile.TarInfo('Dockerfile') 237 | dfinfo.size = len(dockerfile.getvalue()) 238 | dockerfile.seek(0) 239 | else: 240 | dfinfo = t.gettarinfo(fileobj=dockerfile, arcname='Dockerfile') 241 | t.addfile(dfinfo, dockerfile) 242 | 243 | # Open all of the context files and add them to the tarfile. 244 | for path in context_files: 245 | with open(path) as file_object: 246 | file_info = t.gettarinfo(fileobj=file_object, 247 | arcname=context_files[path]) 248 | t.addfile(file_info, file_object) 249 | 250 | t.close() 251 | f.seek(0) 252 | return f 253 | 254 | 255 | def add_files_from_static_dirs(file_dict, config_name): 256 | """Add all files from static directories specified in the config file. 257 | 258 | Args: 259 | file_dict: ({str: NoneType}) A dictionary who's keys are filenames. 260 | config_name: (str) Name of the config file. 261 | 262 | Raises: 263 | AppstartAbort: An invalid field type was discovered. 264 | """ 265 | config = yaml.load(open(config_name)) 266 | root_dir = os.path.dirname(config_name) 267 | handlers = config.get('handlers') 268 | if handlers and isinstance(handlers, list): 269 | for handler in handlers: 270 | if not isinstance(handler, dict): 271 | raise AppstartAbort('"handlers" section of {!r} contains an ' 272 | 'illegal value'.format(config_name)) 273 | static_dir = handler.get('static_dir') 274 | if static_dir: 275 | if not isinstance(static_dir, basestring): 276 | raise AppstartAbort('"handlers" section of {!r} contains a ' 277 | 'non-string static_dir.' % config_name) 278 | print 'walking %s' % static_dir 279 | for dirname, subdirs, files in os.walk( 280 | os.path.join(root_dir, static_dir)): 281 | for filename in files: 282 | file_dict[os.path.join(dirname, filename)] = None 283 | 284 | 285 | class TarWrapper(object): 286 | """A convenience wrapper around a tar archive. 287 | 288 | Helps to list contents of directories and read contents of files. 289 | """ 290 | 291 | def __init__(self, tar_file): 292 | """Initializer for TarWrapper.""" 293 | self.tarfile = tar_file 294 | 295 | def list(self, path): 296 | """Return the contents of dir_path as a list of file/directory names. 297 | 298 | Args: 299 | path: (basestring) The path to the directory, 300 | relative to the root of the tar archive. 301 | 302 | Raises: 303 | ValueError: If dir_path resolves to something other than 304 | a directory. 305 | KeyError: If path cannot be found. 306 | 307 | Returns: 308 | ([basestring, ...], [basestring, ...]) 309 | A tuple of two lists, collectively representing the files and 310 | directories contained within the directory specified by dir_path. 311 | The first element of the tuple is a list of files and the second 312 | a list of directories. 313 | """ 314 | tinfo = self.tarfile.getmember(path.lstrip('/')) 315 | if not tinfo.isdir(): 316 | raise ValueError('"{0}" is not a directory.'.format(path)) 317 | 318 | if not path.endswith('/'): 319 | path += '/' 320 | 321 | files = [] 322 | dirs = [] 323 | 324 | # Find all files rooted at path. 325 | names = [n for n in self.tarfile.getnames() if n.startswith(path)] 326 | 327 | # Calculate the number of components in the path. 328 | path_len = len(path.strip(os.sep).split(os.sep)) 329 | 330 | for name in names: 331 | # If the name is one component longer, it must be directly inside 332 | # the directory specified by path (as opposed to being inside a 333 | # hierarchy of subdirectories that begin at path). 334 | if len(name.split(os.sep)) - path_len == 1: 335 | if self.tarfile.getmember(name).isfile(): 336 | files.append(os.path.basename(name)) 337 | elif self.tarfile.getmember(name).isdir(): 338 | dirs.append(os.path.basename(name)) 339 | 340 | return files, dirs 341 | 342 | def get_file(self, path): 343 | """Return a file-like object from within the tar archive. 344 | 345 | Args: 346 | path: (basestring) The path to the file, relative to 347 | the root of the tar archive. 348 | 349 | Raises: 350 | ValueError: If path resolves to something other than 351 | a file. 352 | KeyError: If path cannot be found. 353 | 354 | Returns: 355 | (basestring) The contents of the file. 356 | """ 357 | tinfo = self.tarfile.getmember(path) 358 | if not tinfo.isfile(): 359 | raise ValueError('"{0}" is not a file.'.format(path)) 360 | return self.tarfile.extractfile(tinfo) 361 | 362 | 363 | def find_image(image_name): 364 | dclient = get_docker_client() 365 | for image in dclient.images(): 366 | if image_name in image['RepoTags']: 367 | return True 368 | return False 369 | 370 | 371 | def log_and_check_build_results(build_res, image_name): 372 | """Log the results of a docker build. 373 | 374 | Args: 375 | build_res: ([basestring, ...]) a generator of build results, 376 | as returned by docker.Client.build 377 | image_name: (basestring) the name of the image associated 378 | with the build results (for logging purposes only) 379 | 380 | Raises: 381 | AppstartAbort: if the build failed. 382 | """ 383 | get_logger().info(' BUILDING IMAGE '.center(80, '-')) 384 | get_logger().info('IMAGE : %s', image_name) 385 | 386 | success = True 387 | try: 388 | for chunk in build_res: 389 | if not chunk: 390 | continue 391 | line = json.loads(chunk) 392 | if 'stream' in line: 393 | logmsg = line['stream'].strip() 394 | get_logger().info(logmsg) 395 | elif 'error' in line: 396 | success = False 397 | logmsg = line['error'].strip() 398 | get_logger().error(logmsg) 399 | elif 'errorDetail' in line: 400 | success = False 401 | logmsg = line['errorDetail']['message'].strip() 402 | get_logger().error(logmsg) 403 | finally: 404 | get_logger().info('-' * 80) 405 | 406 | # Docker build doesn't raise exceptions, so raise one here if the 407 | # build was not successful. 408 | if not success: 409 | raise AppstartAbort('Image build failed.') 410 | -------------------------------------------------------------------------------- /appstart/validator/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """The Validator performs runtime contract validation on GAE containers.""" 16 | -------------------------------------------------------------------------------- /appstart/validator/color_formatting.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Color formatting for the validator's logging stream handlers.""" 16 | 17 | # This file conforms to the external style guide. 18 | # see: https://www.python.org/dev/peps/pep-0008/. 19 | # pylint: disable=bad-indentation, g-bad-import-order 20 | 21 | import logging 22 | 23 | # Color escapes. 24 | GREEN = '\033[92m' 25 | RED = '\033[91m' 26 | WARN = '\033[93m' 27 | END = '\033[0m' 28 | BOLD = '\033[1m' 29 | 30 | 31 | class ColorFormatter(logging.Formatter): 32 | """Formats log messages with or without colors.""" 33 | 34 | def __init__(self, tty=True, **kwargs): 35 | super(ColorFormatter, self).__init__(**kwargs) 36 | self.tty = tty 37 | 38 | def format(self, record): 39 | # Let the base Formatter do all of the heavy lifting. 40 | message = super(ColorFormatter, self).format(record) 41 | 42 | # If the destination is a tty, replace all the color replacement fields 43 | # with the appropriate ansi escape pattern. 44 | if self.tty: 45 | return message % {'red': RED, 46 | 'green': GREEN, 47 | 'warn': WARN, 48 | 'end': END, 49 | 'bold': BOLD} 50 | 51 | # Otherwise (if we're printing to a log file) eliminate the colors. 52 | else: 53 | return message % {'red': '', 54 | 'green': '', 55 | 'warn': '', 56 | 'end': '', 57 | 'bold': ''} 58 | -------------------------------------------------------------------------------- /appstart/validator/color_logging.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Color logging for the validator.""" 16 | 17 | # This file conforms to the external style guide. 18 | # see: https://www.python.org/dev/peps/pep-0008/. 19 | # pylint: disable=bad-indentation, g-bad-import-order 20 | 21 | import logging 22 | 23 | import color_formatting 24 | 25 | 26 | _logger = None 27 | 28 | 29 | def get_validator_logger(): 30 | global _logger 31 | if not _logger: 32 | _logger = logging.getLogger('appstart.validator') 33 | return _logger 34 | 35 | 36 | class LogfileHandler(logging.FileHandler): 37 | 38 | def emit(self, record): 39 | # Only emit the record if it wasn't an empty string or separator. 40 | if str(record.msg).replace('=', ''): 41 | super(LogfileHandler, self).emit(record) 42 | 43 | 44 | class LoggingStream(object): 45 | """A fake 'stream' to be used for logging in tests.""" 46 | 47 | def __init__(self, logfile, verbose_printing, formatter=None): 48 | self.__logger = get_validator_logger() 49 | self.__logger.handlers = [] 50 | self.__logger.setLevel(logging.DEBUG) 51 | 52 | # Don't send messages to root logger. 53 | self.__logger.propagate = False 54 | 55 | # Stream handler prints to console. 56 | stream_handler = logging.StreamHandler() 57 | stream_handler.setLevel(logging.DEBUG if verbose_printing 58 | else logging.INFO) 59 | 60 | # Color formatter replaces colors (like {red}, {warn}, etc) with ansi 61 | # escape sequences. 62 | stream_handler.setFormatter(fmt=formatter or 63 | color_formatting.ColorFormatter()) 64 | self.__logger.addHandler(stream_handler) 65 | 66 | if logfile: 67 | # This special logfile handler doesn't emit empty records. 68 | logfile_handler = LogfileHandler(logfile) 69 | logfile_handler.setLevel(logging.DEBUG) 70 | 71 | # Use a colorless formatter since this handler logs to a file. 72 | logfile_handler.setFormatter( 73 | fmt=color_formatting.ColorFormatter(tty=False)) 74 | self.__logger.addHandler(logfile_handler) 75 | 76 | def writeln(self, message=None, lvl=logging.INFO): 77 | """Write logs, but do proper formatting first. 78 | 79 | Args: 80 | message: (basestring) A message that may or may not contain. 81 | unformatted replacement fields. 82 | lvl: (int) The logging level. 83 | """ 84 | if message is None: 85 | message = '' 86 | 87 | self.__logger.log(lvl, msg=message) 88 | -------------------------------------------------------------------------------- /appstart/validator/errors.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """This file contains the error classes for the validator.""" 16 | 17 | # This file conforms to the external style guide. 18 | # pylint: disable=bad-indentation 19 | 20 | from .. import utils 21 | 22 | 23 | class CircularDependencyError(utils.AppstartAbort): 24 | pass 25 | 26 | 27 | class ContractAttributeError(utils.AppstartAbort): 28 | pass 29 | -------------------------------------------------------------------------------- /appstart/validator/parsing.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | 16 | """Command line argument parsers for validator.""" 17 | 18 | # This file conforms to the external style guide. 19 | # pylint: disable=bad-indentation, g-bad-import-order 20 | 21 | import argparse 22 | 23 | import appstart 24 | import contract 25 | 26 | 27 | def make_validator_parser(): 28 | parser = argparse.ArgumentParser( 29 | description='Utility to validate whether or not a docker image ' 30 | 'fulfills the runtime contract specified by the Google Cloud ' 31 | 'Platform.') 32 | add_validate_args(parser) 33 | appstart.parsing.add_appstart_args(parser) 34 | return parser 35 | 36 | 37 | def add_validate_args(parser): 38 | parser.add_argument('--log_file', 39 | default=None, 40 | help='Logfile to collect validation results.') 41 | parser.add_argument('--threshold', 42 | default='WARNING', 43 | choices=[name for _, name in 44 | contract.LEVEL_NAMES.iteritems()], 45 | help='The threshold at which validation should fail.') 46 | -------------------------------------------------------------------------------- /appstart/validator/runtime_contract.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | 3 | """The runtime contract is a set of requirements on GAE app containers.""" 4 | 5 | # This file conforms to the external style guide. 6 | # see: https://www.python.org/dev/peps/pep-0008/. 7 | # pylint: disable=bad-indentation, g-bad-import-order 8 | 9 | import json 10 | import os 11 | import re 12 | import requests 13 | 14 | import contract 15 | 16 | 17 | # Fields that diagnostic log entries are required to have. 18 | _DIAGNOSTIC_FIELDS = ['timestamp', 'severity', 'thread', 'message'] 19 | 20 | # Fields that a diagnostic log's timestamp are supposed to have. 21 | _TIMESTAMP_FIELDS = ['seconds', 'nanos'] 22 | 23 | # Absolute path to directory where the application is expected to write logs. 24 | _LOG_LOCATION = '/var/log/app_engine' 25 | 26 | # Diagnostic log location 27 | _DLOG_LOCATION = os.path.join(_LOG_LOCATION, 'app.log.json') 28 | 29 | # Access log location 30 | _ALOG_LOCATION = os.path.join(_LOG_LOCATION, 'request.log') 31 | 32 | # Custom log directory 33 | _CLOG_LOCATION = os.path.join(_LOG_LOCATION, 'custom_logs') 34 | 35 | # Permissible status codes for a container to return from _ah/start 36 | _STATUS_CODES = [200, 202, 404, 503] 37 | 38 | 39 | class LogFormatChecker(object): 40 | """Class to give clauses the ability to check the format of logs. 41 | 42 | 43 | For json logs, there must be one json object per line. Furthermore, all 44 | entries must have the following fields: 45 | 46 | - timestamp 47 | - seconds 48 | - nanos 49 | - severity 50 | - thread 51 | - message 52 | 53 | For access logs, it seems that they're supposed to be 54 | in Common Log or Extended format. It's not immediately clear if or how 55 | this is enforced. 56 | """ 57 | 58 | def check_json_log_format(self, logfile): 59 | """Check if a log file conforms to the proper json format. 60 | 61 | Args: 62 | logfile: (file-like object) The log file to be checked. 63 | """ 64 | for line in logfile: 65 | if line: 66 | try: 67 | logmsg = json.loads(line) 68 | except ValueError: 69 | self.fail('Improperly formatted line: "{0}"'.format(line)) 70 | 71 | for field in _DIAGNOSTIC_FIELDS: 72 | self.assertIn( 73 | field, 74 | logmsg, 75 | 'Log message missing "{0}" field: {1}'.format(field, 76 | line)) 77 | 78 | ts = logmsg['timestamp'] 79 | if not isinstance(ts, dict) or ts.keys() != _TIMESTAMP_FIELDS: 80 | self.fail('{0}\nTimestamps must have fields: ' 81 | '"{1}"'.format(line, _TIMESTAMP_FIELDS)) 82 | 83 | def check_access_log_format(self, logfile): 84 | """Check if a log file conforms to the Common Log or Extended formats. 85 | 86 | (Currently not implemented) 87 | 88 | Args: 89 | logfile: (file-like object) The log file to be checked. 90 | """ 91 | 92 | # TODO(gouzenko): *ACTUALLY* figure out how to do this in the best way 93 | common_log_format = re.compile(r'(\S*) ' 94 | r'(\S*) ' 95 | r'(\S*) ' 96 | r'\[([^]]*)\] ' 97 | r'"([^"]*)" ' 98 | r'(\S*) ' 99 | r'(\S*)') 100 | full_line = False 101 | for line in logfile: 102 | if line: 103 | full_line = True 104 | match = common_log_format.match(line) 105 | if not match: 106 | self.fail('Improperly formatted line:"{0}"'.format(line)) 107 | if not full_line: 108 | self.fail('No access logs found in log file.') 109 | 110 | 111 | class HealthChecksEnabledClause(contract.ContractClause): 112 | """Validate that health checking is turned on.""" 113 | title = 'Health checking enabled' 114 | description = 'Container can enable health checks in configuration' 115 | lifecycle_point = contract.PRE_START 116 | error_level = contract.UNUSED 117 | tags = {'health'} 118 | 119 | def evaluate_clause(self, app_container): 120 | self.assertTrue(app_container.configuration.health_checks_enabled) 121 | 122 | 123 | class HealthCheckClause(contract.ContractClause): 124 | """Validate that the application responds to '_ah/health' endpoint.""" 125 | 126 | title = 'Health checking' 127 | description = 'Endpoint /_ah/health must respond with status code 200' 128 | lifecycle_point = contract.POST_START 129 | error_level = contract.FATAL 130 | dependencies = {HealthChecksEnabledClause} 131 | tags = {'health'} 132 | 133 | def evaluate_clause(self, app_container): 134 | url = 'http://{0}:{1}/_ah/health'.format(app_container.host, 135 | 8080) 136 | rep = requests.get(url) 137 | self.assertEqual(rep.status_code, 138 | 200, 139 | 'the container did not ' 140 | 'properly respond to ' 141 | 'health checks.') 142 | 143 | 144 | class AccessLogLocationClause(contract.ContractClause): 145 | """Validate that the application writes access logs to correct location. 146 | 147 | Access logs should be written to _ALOG_LOCATION. 148 | """ 149 | 150 | title = 'Access log location' 151 | description = ('Container should write access logs to ' 152 | '{0}'.format(_ALOG_LOCATION)) 153 | lifecycle_point = contract.POST_START 154 | error_level = contract.UNUSED 155 | tags = {'logging'} 156 | 157 | def evaluate_clause(self, app_container): 158 | try: 159 | _ = app_container.extract_tar(_ALOG_LOCATION) 160 | except IOError: 161 | self.fail('No log file found at {0}'.format(_ALOG_LOCATION)) 162 | 163 | 164 | class AccessLogFormatClause(contract.ContractClause, LogFormatChecker): 165 | """Validate that the application writes access logs in the correct format. 166 | 167 | Logs must be in Common Log Format or Extended Format. 168 | 169 | Common Log Format: http://httpd.apache.org/docs/1.3/logs.html#common 170 | Extended Format: http://www.w3.org/TR/WD-logfile.html 171 | """ 172 | 173 | title = 'Access log format' 174 | description = 'Access logs should be in Common or Extended formats' 175 | lifecycle_point = contract.POST_START 176 | error_level = contract.WARNING 177 | dependencies = {AccessLogLocationClause} 178 | tags = {'logging'} 179 | 180 | def evaluate_clause(self, app_container): 181 | logfile_tar = app_container.extract_tar(_ALOG_LOCATION) 182 | logfile = logfile_tar.get_file(os.path.basename(_ALOG_LOCATION)) 183 | self.check_access_log_format(logfile) 184 | 185 | 186 | class CustomLogLocationClause(contract.ContractClause): 187 | """Validate that the application writes custom logs in correct location. 188 | 189 | The application should write custom logs to the directory _CLOG_LOCATION. 190 | """ 191 | 192 | title = 'Custom log location' 193 | description = 'Custom logs can be written to {0}'.format(_CLOG_LOCATION) 194 | lifecycle_point = contract.POST_START 195 | error_level = contract.UNUSED 196 | tags = {'logging'} 197 | 198 | def evaluate_clause(self, app_container): 199 | try: 200 | _ = app_container.extract_tar(_CLOG_LOCATION) 201 | except IOError: 202 | self.fail('Custom logs directory not found at ' 203 | '{0}'.format(_CLOG_LOCATION)) 204 | 205 | 206 | class CustomLogFormatClause(contract.ContractClause, LogFormatChecker): 207 | """Custom logs must have either .log or .log.json extensions.""" 208 | 209 | title = 'Custom log format' 210 | description = ('Json logs must have "timestamp", ' 211 | '"thread", "severity" and "message" fields. Other logs ' 212 | 'can be plain text.') 213 | lifecycle_point = contract.POST_START 214 | error_level = contract.WARNING 215 | dependencies = {CustomLogLocationClause} 216 | tags = {'logging'} 217 | 218 | def evaluate_clause(self, app_container): 219 | custom_logs_tar = app_container.extract_tar(_CLOG_LOCATION) 220 | custom_logs_root = os.path.basename(_CLOG_LOCATION) 221 | files, dirs = custom_logs_tar.list(custom_logs_root) 222 | 223 | for f in files: 224 | if f.endswith('.log.json'): 225 | logfile = custom_logs_tar.get_file( 226 | os.path.join(custom_logs_root, f)) 227 | self.check_json_log_format(logfile) 228 | 229 | elif not f.endswith('.log'): 230 | self.fail('File "{0}" does not end in .log or ' 231 | '.log.json'.format(f)) 232 | 233 | self.assertEqual( 234 | len(dirs), 235 | 0, 236 | ('Directories inside {0} will not have their logs ' 237 | 'ingested.').format(_CLOG_LOCATION)) 238 | 239 | 240 | class DiagnosticLogLocationClause(contract.ContractClause): 241 | """Validate that the application writes diagnostic log to correct location. 242 | 243 | App must write diagnostic logs to _DLOG_LOCATION. 244 | """ 245 | 246 | title = 'Diagnostic log location' 247 | description = ('Container should write diagnostic logs to ' 248 | '{0}'.format(_DLOG_LOCATION)) 249 | error_level = contract.UNUSED 250 | lifecycle_point = contract.POST_START 251 | tags = {'logging'} 252 | 253 | def evaluate_clause(self, app_container): 254 | try: 255 | _ = app_container.extract_tar(_DLOG_LOCATION) 256 | except (ValueError, IOError): 257 | self.fail('Could not find log file at {0}'.format(_DLOG_LOCATION)) 258 | 259 | 260 | class DiagnosticLogFormatClause(contract.ContractClause, LogFormatChecker): 261 | """Validate that the application writes diagnostic logs correctly.""" 262 | title = 'Diagnostic log format' 263 | description = ('Json logs must have "timestamp", ' 264 | '"thread", "severity" and "message" fields.') 265 | lifecycle_point = contract.POST_START 266 | error_level = contract.WARNING 267 | dependencies = {DiagnosticLogLocationClause} 268 | tags = {'logging'} 269 | 270 | def evaluate_clause(self, app_container): 271 | logfile_tar = app_container.extract_tar(_DLOG_LOCATION) 272 | logfile = logfile_tar.get_file(os.path.basename(_DLOG_LOCATION)) 273 | self.check_json_log_format(logfile) 274 | 275 | 276 | class HostnameClause(contract.ContractClause): 277 | """Validate that container executes /bin/hostname cleanly.""" 278 | 279 | title = 'Hostname' 280 | description = 'Container must make hostname available through /bin/hostname' 281 | error_level = contract.WARNING 282 | lifecycle_point = contract.PRE_START 283 | 284 | def evaluate_clause(self, app_container): 285 | res = app_container.execute('/bin/hostname') 286 | self.assertEqual(res['ExitCode'], 287 | 0, 288 | 'Error while executing /bin/hostname') 289 | 290 | 291 | class StartClause(contract.ContractClause): 292 | """Validate that the application responds correctly to _ah/start. 293 | 294 | The application shouldn't respond with status code 500 but all other 295 | status codes are fine. 296 | """ 297 | 298 | title = 'Start request' 299 | description = 'Container must respond 200 OK on _ah/start endpoint' 300 | lifecycle_point = contract.START 301 | error_level = contract.FATAL 302 | 303 | def evaluate_clause(self, app_container): 304 | url = 'http://{0}:{1}/_ah/start'.format(app_container.host, 305 | 8080) 306 | r = requests.get(url) 307 | self.assertIn(r.status_code, 308 | _STATUS_CODES, 309 | 'Request to _ah/start failed.') 310 | 311 | 312 | class StopClause(contract.ContractClause): 313 | """Validate that the application responds correctly to _ah/stop. 314 | 315 | The application shouldn't respond with status code 500 but all other 316 | status codes are fine. 317 | """ 318 | 319 | title = 'Stop request' 320 | description = 'Send a request to _ah/stop.' 321 | lifecycle_point = contract.STOP 322 | error_level = contract.WARNING 323 | 324 | def evaluate_clause(self, app_container): 325 | """Ensure that the status code is not 500.""" 326 | url = 'http://{0}:{1}/_ah/stop'.format(app_container.host, 8080) 327 | r = requests.get(url) 328 | self.assertIn(r.status_code, 329 | _STATUS_CODES, 330 | 'Request to _ah/stop failed.') 331 | -------------------------------------------------------------------------------- /run_tests.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | python -m unittest discover ./tests/ -p *test.py 3 | -------------------------------------------------------------------------------- /setup.cfg: -------------------------------------------------------------------------------- 1 | [bdist_wheel] 2 | # This flag says that the code is written to work on both Python 2 and Python 3 | # 3. If at all possible, it is good practice to do this. If you cannot, you 4 | # will need to generate wheels for each Python version that you support. 5 | universal=1 6 | -------------------------------------------------------------------------------- /setup.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """A setuptools based setup module. 16 | 17 | See: 18 | https://packaging.python.org/en/latest/distributing.html 19 | https://github.com/pypa/sampleproject 20 | """ 21 | 22 | # This file conforms to the external style guide. 23 | # pylint: disable=bad-indentation 24 | 25 | import codecs 26 | import os 27 | import setuptools 28 | 29 | 30 | here = os.path.abspath(os.path.dirname(__file__)) 31 | 32 | with codecs.open(os.path.join(here, 'README.md'), encoding='utf-8') as f: 33 | long_description = f.read() 34 | 35 | setuptools.setup( 36 | name='appstart', 37 | version='0.8', 38 | description='A utility to start GAE Managed VMs in containers.', 39 | long_description=long_description, 40 | url='https://github.com/GoogleCloudPlatform/appstart', 41 | author='Mitchell Gouzenko', 42 | author_email='mgouzenko@gmail.com', 43 | license='APACHE', 44 | classifiers=[ 45 | 'Development Status :: 3 - Alpha', 46 | 'Intended Audience :: Developers', 47 | 'Topic :: Software Development :: Development Tools', 48 | 'License :: OSI Approved :: Apache Software License', 49 | 'Programming Language :: Python :: 2', 50 | 'Programming Language :: Python :: 2.6', 51 | 'Programming Language :: Python :: 2.7', 52 | 'Programming Language :: Python :: 3', 53 | 'Programming Language :: Python :: 3.2', 54 | 'Programming Language :: Python :: 3.3', 55 | 'Programming Language :: Python :: 3.4', 56 | ], 57 | keywords='GAE Google App Engine appengine development docker', 58 | packages=setuptools.find_packages(exclude='tests'), 59 | package_data={'appstart.devappserver_init': ['Dockerfile', 'das.sh'], 60 | 'appstart.pinger': ['Dockerfile'], 61 | 'appstart.sandbox': ['app.yaml']}, 62 | install_requires=[ 63 | 'backports.ssl-match-hostname==3.4.0.2', 64 | 'docker-py==1.5.0', 65 | 'mox==0.5.3', 66 | 'PyYAML==3.11', 67 | 'requests==2.8.1', 68 | 'six==1.10.0', 69 | 'websocket-client==0.32.0', 70 | 'wheel==0.24.0', 71 | ], 72 | entry_points={ 73 | 'console_scripts': [ 74 | 'appstart=appstart.cli.start_script:main', 75 | ], 76 | }, 77 | ) 78 | -------------------------------------------------------------------------------- /tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Tests for Appstart.""" 16 | -------------------------------------------------------------------------------- /tests/fakes/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Fakes for use in testing.""" 16 | -------------------------------------------------------------------------------- /tests/fakes/fake_docker.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """An imitation of docker.Client. 16 | 17 | This class is used for unit testing ContainerSandbox. 18 | """ 19 | # This file conforms to the external style guide. 20 | # pylint: disable=bad-indentation 21 | 22 | import requests 23 | import stubout 24 | import unittest 25 | import uuid 26 | 27 | import docker 28 | 29 | from appstart import constants 30 | from appstart import utils 31 | 32 | DEFAULT_IMAGES = [constants.DEVAPPSERVER_IMAGE, 33 | constants.PINGER_IMAGE] 34 | images = list(DEFAULT_IMAGES) 35 | containers = [] 36 | removed_containers = [] 37 | 38 | 39 | def reset(): 40 | global containers, images, removed_containers 41 | containers = [] 42 | images = list(DEFAULT_IMAGES) 43 | removed_containers = [] 44 | 45 | 46 | # Fake build results, mimicking those that appear from docker.Client.build 47 | BUILD_RES = [ 48 | '{"stream":"Step 1 : MAINTAINER first last, name@example.com\\n"}', 49 | '{"stream":" ---\\u003e Running in 08787d0ee8b1\\n"}', 50 | '{"stream":" ---\\u003e 23e5e66a4494\\n"}', 51 | '{"stream":"Removing intermediate container 08787d0ee8b1\\n"}', 52 | '{"stream":"Step 2 : VOLUME /data\\n"}', 53 | '{"stream":" ---\\u003e Running in abdc1e6896c6\\n"}', 54 | '{"stream":" ---\\u003e 713bca62012e\\n"}', 55 | '{"stream":"Removing intermediate container abdc1e6896c6\\n"}', 56 | '{"stream":"Step 3 : CMD [\\"/bin/sh\\"]\\n"}', 57 | '{"stream":" ---\\u003e Running in dba30f2a1a7e\\n"}', 58 | '{"stream":" ---\\u003e 032b8b2855fc\\n"}', 59 | '{"stream":"Removing intermediate container dba30f2a1a7e\\n"}', 60 | '{"stream":"Successfully built 032b8b2855fc\\n"}'] 61 | 62 | FAILED_BUILD_RES = [ 63 | '{"stream":"Step 1 : MAINTAINER first last, name@example.com\\n"}', 64 | '{"stream":" ---\\u003e Running in 08787d0ee8b1\\n"}', 65 | '{"stream":" ---\\u003e 23e5e66a4494\\n"}', 66 | '{"stream":"Removing intermediate container 08787d0ee8b1\\n"}', 67 | '{"stream":"Step 2 : VOLUME /data\\n"}', 68 | '{"stream":" ---\\u003e Running in abdc1e6896c6\\n"}', 69 | '{"stream":" ---\\u003e 713bca62012e\\n"}', 70 | '{"stream":"Removing intermediate container abdc1e6896c6\\n"}', 71 | '{"stream":"Step 3 : CMD [\\"/bin/sh\\"]\\n"}', 72 | '{"stream":" ---\\u003e Running in dba30f2a1a7e\\n"}', 73 | '{"stream":" ---\\u003e 032b8b2855fc\\n"}', 74 | '{"stream":"Removing intermediate container dba30f2a1a7e\\n"}', 75 | '{"error":"Could not build 032b8b2855fc\\n"}'] 76 | 77 | 78 | class FakeDockerClient(object): 79 | """Fake the functionality of docker.Client.""" 80 | 81 | def __init__(self, **kwargs): # pylint: disable=unused-argument 82 | """Keep lists for images, containers, and removed containers.""" 83 | self.base_url = 'http://0.0.0.0:1234' 84 | self.kwargs = kwargs 85 | 86 | def version(self): 87 | return {'Version': utils.format_version(utils.MIN_DOCKER_VERSION)} 88 | 89 | def ping(self): 90 | """Do nothing.""" 91 | pass 92 | 93 | def build(self, **kwargs): 94 | """Imitate docker.Client.build.""" 95 | if 'custom_context' in kwargs and 'fileobj' not in kwargs: 96 | raise TypeError('fileobj must be passed with custom_context.') 97 | if 'tag' not in kwargs: 98 | raise KeyError('tag must be specified in docker build.') 99 | if 'nocache' not in kwargs: 100 | raise KeyError('appstart must specify nocache in builds.') 101 | 102 | # "Store" the newly "built" image 103 | images.append(kwargs['tag']) 104 | return BUILD_RES 105 | 106 | def inspect_container(self, container_id): 107 | cont = find_container(container_id) 108 | return {'Name': cont['Name'], 109 | 'Id': cont['Id'], 110 | 'State': {'Running': cont['Running']}} 111 | 112 | def create_container(self, **kwargs): 113 | """Imitiate docker.Client.create_container.""" 114 | if 'image' not in kwargs: 115 | raise KeyError('image was not specified.') 116 | if 'name' not in kwargs: 117 | raise KeyError('appstart should not make unnamed containers.') 118 | if kwargs['image'] not in images: 119 | raise docker.errors.APIError('the specified image does not exist.', 120 | requests.Response()) 121 | 122 | # Create a unique id for the container. 123 | container_id = str(uuid.uuid4()) 124 | 125 | # Create a new container and append it to the list of containers. 126 | new_container = {'Id': container_id, 127 | 'Running': False, 128 | 'Options': kwargs, 129 | 'Name': kwargs['name']} 130 | containers.append(new_container) 131 | return {'Id': container_id, 'Warnings': None} 132 | 133 | def kill(self, cont_id): 134 | """Imitate docker.Client.kill.""" 135 | cont_to_kill = find_container(cont_id) 136 | cont_to_kill['Running'] = False 137 | 138 | def remove_container(self, cont_id): 139 | """Imitate docker.Client.remove_container.""" 140 | cont_to_rm = find_container(cont_id) 141 | if cont_to_rm['Running']: 142 | raise RuntimeError('tried to remove a running container.') 143 | removed_containers.append(cont_to_rm) 144 | containers.remove(cont_to_rm) 145 | 146 | def start(self, cont_id, **kwargs): # pylint: disable=unused-argument 147 | """Imitate docker.Client.start.""" 148 | cont_to_start = find_container(cont_id) 149 | cont_to_start['Running'] = True 150 | 151 | def images(*args, **kwargs): 152 | return [{'RepoTags': [image_name]} for image_name in images] 153 | 154 | 155 | 156 | def find_container(cont_id): 157 | """Helper function to find a container based on id.""" 158 | for cont in containers: 159 | if cont['Id'] == cont_id: 160 | return cont 161 | raise docker.errors.APIError('container was not found.', requests.Response()) 162 | 163 | 164 | class FakeDockerTestBase(unittest.TestCase): 165 | 166 | def setUp(self): 167 | self.stubs = stubout.StubOutForTesting() 168 | self.stubs.Set(docker, 'Client', FakeDockerClient) 169 | reset() 170 | 171 | def tearDown(self): 172 | """Restore docker.Client and requests.get.""" 173 | reset() 174 | self.stubs.UnsetAll() 175 | -------------------------------------------------------------------------------- /tests/system_tests/Dockerfile: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | # 15 | # Dockerfile extending the generic Python image with application files for a 16 | # single application. 17 | FROM gcr.io/google_appengine/python-compat 18 | 19 | RUN apt-get install -y coreutils 20 | 21 | ADD . /app 22 | -------------------------------------------------------------------------------- /tests/system_tests/__init__.py: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/appstart/f08d4867cd115c458b151b1414d9833fadc63bf1/tests/system_tests/__init__.py -------------------------------------------------------------------------------- /tests/system_tests/app.yaml: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # yaml file for a test application 3 | module: default 4 | runtime: python27 5 | api_version: 1 6 | threadsafe: true 7 | vm: true 8 | 9 | resources: 10 | cpu: .5 11 | memory_gb: 1.3 12 | 13 | manual_scaling: 14 | instances: 1 15 | 16 | handlers: 17 | - url: /static 18 | static_dir: static 19 | - url: /.* 20 | script: services_test_app.app 21 | -------------------------------------------------------------------------------- /tests/system_tests/appstart_systemtests.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Set up real application and devappserver containers. 16 | 17 | All endpoints of the application are probed. The application 18 | respond with 200 OK if the endpoint is healthy, and any other 19 | status code if the endpoint is broken. 20 | """ 21 | # This file conforms to the external style guide 22 | # pylint: disable=bad-indentation, g-bad-import-order 23 | 24 | import logging 25 | import os 26 | import socket 27 | import requests 28 | import time 29 | import unittest 30 | from appstart import utils, devappserver_init 31 | 32 | from appstart.sandbox import container_sandbox 33 | 34 | APPSTART_BASE_IMAGE = "appstart_systemtest_devappserver" 35 | 36 | # pylint: disable=too-many-public-methods 37 | class SystemTests(unittest.TestCase): 38 | """Probe endpoints on a running application container. 39 | 40 | In addition to the tests defined here, this class will be populated with 41 | tests from the endpoints using make_endpoint_test() below. 42 | """ 43 | 44 | @classmethod 45 | def setUpClass(cls): 46 | """Create an actual sandbox. 47 | 48 | This depends on a properly set up docker environment. 49 | """ 50 | 51 | utils.build_from_directory(os.path.dirname(devappserver_init.__file__), 52 | APPSTART_BASE_IMAGE, 53 | nocache=True) 54 | 55 | test_directory = os.path.dirname(os.path.realpath(__file__)) 56 | cls.conf_file = os.path.join(test_directory, 'app.yaml') 57 | 58 | # Use temporary storage, generating unique name with a timestamp. 59 | temp_storage_path = '/tmp/storage/%s' % str(time.time()) 60 | cls.sandbox = container_sandbox.ContainerSandbox( 61 | cls.conf_file, 62 | storage_path=temp_storage_path, 63 | devbase_image=APPSTART_BASE_IMAGE, 64 | force_version=True, 65 | extra_ports={1000: 1111}) 66 | 67 | # Set up the containers 68 | cls.sandbox.start() 69 | 70 | @classmethod 71 | def tearDownClass(cls): 72 | """Clean up the docker environment.""" 73 | cls.sandbox.stop() 74 | 75 | def test_extra_ports(self): 76 | res = requests.get('http://%s:%i/openport' % 77 | (self.sandbox.devappserver_container.host, 78 | self.sandbox.proxy_port)) 79 | self.assertEqual(res.status_code, 200) 80 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) 81 | s.connect((self.sandbox.devappserver_container.host, 1111)) 82 | s.send('test string') 83 | self.assertEqual(s.recv(1024), 'test string') 84 | 85 | def test_static_files(self): 86 | res = requests.get('http://%s:%i/static/file.txt' % 87 | (self.sandbox.devappserver_container.host, 88 | self.sandbox.proxy_port)) 89 | self.assertEqual(res.status_code, 200) 90 | self.assertEqual(res.text, 'file contents') 91 | 92 | 93 | def make_endpoint_test(endpoint): 94 | """Create and return a function that tests the endpoint. 95 | 96 | Args: 97 | endpoint: (basestring) the endpoint to be tested (starting with /) 98 | handler: (webapp2.RequestHandler) the handler (from the test 99 | application) that services endpoint. 100 | 101 | Returns: 102 | (callable()) a function to test the endpoint. 103 | """ 104 | def _endpoint_test(self): 105 | """Hit the endpoint and assert that it responds with 200 OK.""" 106 | res = requests.get('http://%s:%i%s' % 107 | (self.sandbox.devappserver_container.host, 108 | self.sandbox.port, 109 | endpoint)) 110 | self.assertEqual(res.status_code, 111 | 200, 112 | '%s failed with error \"%s\"' % 113 | (endpoint, res.text)) 114 | _endpoint_test.__name__ = 'test_%s_endpoint' % endpoint.strip('/') 115 | return _endpoint_test 116 | 117 | 118 | if __name__ == '__main__': 119 | logging.getLogger('appstart').setLevel(logging.INFO) 120 | 121 | # Sync with urls in services_test_app.py 122 | # Keeping handler as None for later on customizing of tests 123 | urls = [ 124 | '/datastore', 125 | '/logging', 126 | '/memcache' 127 | ] 128 | 129 | 130 | # Get all the endpoints from the test app and turn them into tests. 131 | for ep in urls: 132 | endpoint_test = make_endpoint_test(ep) 133 | setattr(SystemTests, endpoint_test.__name__, endpoint_test) 134 | unittest.main() 135 | -------------------------------------------------------------------------------- /tests/system_tests/services_test_app.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """App that tests api server to see if services are connected. 16 | 17 | Each endpoint tests a different service. An endpoints respond with 200 18 | if it is working, and 500 if there are any exceptions 19 | """ 20 | # This file conforms to the external style guide 21 | # pylint: disable=bad-indentation 22 | 23 | import sys 24 | import logging 25 | import socket 26 | import threading 27 | 28 | import webapp2 29 | 30 | from google.appengine.api import memcache 31 | from google.appengine.api.logservice import logservice 32 | from google.appengine.ext import ndb 33 | 34 | 35 | def respond_with_error(func): 36 | """Wraps func so that it writes all Exceptions to response.""" 37 | def get_func(self): 38 | """Handle a get request respond with 500 status code on error.""" 39 | try: 40 | func(self) 41 | except Exception as excep: # pylint: disable=broad-except 42 | self.response.set_status(500) 43 | self.response.write(str(excep)) 44 | return get_func 45 | 46 | 47 | # pylint: disable=no-member 48 | class Message(ndb.Model): # pylint: disable=too-few-public-methods 49 | """Models a simple message.""" 50 | content = ndb.StringProperty() 51 | 52 | 53 | # pylint: disable=no-self-use 54 | class DataStoreTest(webapp2.RequestHandler): 55 | """Test that the datastore is connected.""" 56 | 57 | @respond_with_error 58 | def get(self): 59 | """Ensure that the datastore works.""" 60 | Message(content='Hi', parent=ndb.Key(Message, 'test')).put() 61 | 62 | msg = Message.query(ancestor=ndb.Key(Message, 'test')).get() 63 | assert msg.content == 'Hi', ('\"%s\" is not \"%s\"' % 64 | (msg.content, 'Hi')) 65 | 66 | 67 | class LoggingTest(webapp2.RequestHandler): 68 | """Test that logservice is connected.""" 69 | 70 | @respond_with_error 71 | def get(self): 72 | """Ensure that the log service works.""" 73 | logservice.write('Hi') 74 | logservice.flush() 75 | 76 | 77 | class MemcacheTest(webapp2.RequestHandler): 78 | """Test that memcache is connected.""" 79 | 80 | @respond_with_error 81 | def get(self): 82 | """Ensure that memcache works.""" 83 | memcache.set('test', 'hi') 84 | assert memcache.get('test') == 'hi', 'Memcache failure' 85 | 86 | 87 | def socket_thread(): 88 | # Function that runs a little server on port 1000 that just echoes back 89 | # the first chunk of data that it receives. 90 | logging.info('In socket thread') 91 | s = socket.socket(socket.AF_INET, socket.SOCK_STREAM, 0) 92 | s.bind(('', 1000)) 93 | s.listen(5) 94 | while True: 95 | c, addr = s.accept() 96 | data = c.recv(1024) 97 | c.send(data) 98 | c.close() 99 | 100 | 101 | class OpenPort(webapp2.RequestHandler): 102 | """Open port 1000.""" 103 | 104 | def get(self): 105 | logging.info('Starting socket thread') 106 | threading.Thread(target=socket_thread).start() 107 | self.content_type = 'text/plain' 108 | self.response.write('started thread.') 109 | 110 | 111 | # pylint: disable=invalid-name 112 | urls = [('/datastore', DataStoreTest), 113 | ('/logging', LoggingTest), 114 | ('/memcache', MemcacheTest), 115 | ('/openport', OpenPort)] 116 | 117 | app = webapp2.WSGIApplication(urls, debug=True) 118 | -------------------------------------------------------------------------------- /tests/test_data/certs/ca.pem: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/appstart/f08d4867cd115c458b151b1414d9833fadc63bf1/tests/test_data/certs/ca.pem -------------------------------------------------------------------------------- /tests/test_data/certs/cert.pem: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/appstart/f08d4867cd115c458b151b1414d9833fadc63bf1/tests/test_data/certs/cert.pem -------------------------------------------------------------------------------- /tests/test_data/certs/key.pem: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/GoogleCloudPlatform/appstart/f08d4867cd115c458b151b1414d9833fadc63bf1/tests/test_data/certs/key.pem -------------------------------------------------------------------------------- /tests/unit_tests/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Unittests for appstart.""" 16 | -------------------------------------------------------------------------------- /tests/unit_tests/sandbox/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Tests for appstart.sandbox.""" 16 | -------------------------------------------------------------------------------- /tests/unit_tests/sandbox/configuration_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Unit tests for sandbox.configuration.""" 16 | 17 | # This file conforms to the external style guide. 18 | # pylint: disable=bad-indentation, g-bad-import-order 19 | 20 | import os 21 | import tempfile 22 | import textwrap 23 | import unittest 24 | 25 | from appstart.sandbox import configuration 26 | from appstart import utils 27 | 28 | 29 | class ConfigurationTest(unittest.TestCase): 30 | 31 | def setUp(self): 32 | self.temp_dir = tempfile.mkdtemp() 33 | 34 | def _make_xml_configs(self, xml_config_file): 35 | """Make an xml config file and web.xml file in self.temp_dir. 36 | 37 | Args: 38 | xml_config_file: (basestring) The string contents of the 39 | xml file. 40 | 41 | Returns: 42 | (basestring) The path to the appengine-web.xml file. 43 | """ 44 | conf_file_name = os.path.join(self.temp_dir, 'appengine-web.xml') 45 | web_xml_name = os.path.join(self.temp_dir, 'web.xml') 46 | 47 | conf_file = open(conf_file_name, 'w') 48 | conf_file.write(xml_config_file) 49 | conf_file.close() 50 | 51 | web_xml = open(web_xml_name, 'w') 52 | web_xml.close() 53 | 54 | return conf_file_name 55 | 56 | def _make_yaml_config(self, yaml_config_file): 57 | """Make a yaml config file in self.temp_dir. 58 | 59 | Args: 60 | yaml_config_file: (basestring) The string contents of the yaml file. 61 | 62 | Returns: 63 | (basestring) The path to the app.yaml file. 64 | """ 65 | conf_file_name = os.path.join(self.temp_dir, 'app.yaml') 66 | 67 | conf_file = open(conf_file_name, 'w') 68 | conf_file.write(yaml_config_file) 69 | conf_file.close() 70 | 71 | return conf_file_name 72 | 73 | def test_init_from_xml_health_checks_on(self): 74 | xml_files = [textwrap.dedent("""\ 75 | 76 | true 77 | 78 | true 79 | 80 | """), 81 | textwrap.dedent("""\ 82 | 83 | true 84 | 85 | 86 | 87 | """), 88 | textwrap.dedent("""\ 89 | 90 | true 91 | """)] 92 | 93 | for xml_file in xml_files: 94 | conf_file_name = self._make_xml_configs(xml_file) 95 | conf = configuration.ApplicationConfiguration(conf_file_name) 96 | self.assertTrue(conf.health_checks_enabled, 97 | 'Health checks should be ' 98 | 'enabled in file:\n{0}'.format(xml_file)) 99 | 100 | def test_init_from_xml_health_checks_off(self): 101 | xml_file = textwrap.dedent("""\ 102 | 103 | true 104 | 105 | false 106 | 107 | """) 108 | 109 | conf_file_name = self._make_xml_configs(xml_file) 110 | conf = configuration.ApplicationConfiguration(conf_file_name) 111 | self.assertFalse(conf.health_checks_enabled) 112 | 113 | def test_init_from_xml_vm_false(self): 114 | xml_files = [textwrap.dedent("""\ 115 | 116 | """), 117 | textwrap.dedent("""\ 118 | 119 | 120 | """), 121 | textwrap.dedent("""\ 122 | 123 | false 124 | """)] 125 | 126 | for xml_file in xml_files: 127 | conf_file_name = self._make_xml_configs(xml_file) 128 | with self.assertRaises(utils.AppstartAbort): 129 | configuration.ApplicationConfiguration(conf_file_name) 130 | 131 | def test_malformed_xml(self): 132 | xml_file = 'malformed xml file' 133 | conf_file_name = self._make_xml_configs(xml_file) 134 | with self.assertRaises(utils.AppstartAbort): 135 | configuration.ApplicationConfiguration(conf_file_name) 136 | 137 | def test_init_from_yaml_health_checks_on(self): 138 | yaml_files = [textwrap.dedent("""\ 139 | vm: true 140 | health_check: 141 | enable_health_check: True 142 | check_interval_sec: 5 143 | timeout_sec: 4 144 | unhealthy_threshold: 2 145 | healthy_threshold: 2 146 | restart_threshold: 60"""), 147 | textwrap.dedent("""vm: true""")] 148 | 149 | for yaml_file in yaml_files: 150 | conf_file_name = self._make_yaml_config(yaml_file) 151 | conf = configuration.ApplicationConfiguration(conf_file_name) 152 | self.assertTrue(conf.health_checks_enabled) 153 | 154 | def test_init_from_yaml_health_checks_off(self): 155 | yaml_file = textwrap.dedent("""\ 156 | vm: true 157 | health_check: 158 | enable_health_check: False""") 159 | 160 | conf_file_name = self._make_yaml_config(yaml_file) 161 | conf = configuration.ApplicationConfiguration(conf_file_name) 162 | self.assertFalse(conf.health_checks_enabled) 163 | 164 | def test_init_from_yaml_vm_false(self): 165 | yaml_files = [textwrap.dedent("""\ 166 | health_check: 167 | enable_health_check: True 168 | check_interval_sec: 5 169 | timeout_sec: 4 170 | unhealthy_threshold: 2 171 | healthy_threshold: 2 172 | restart_threshold: 60"""), 173 | textwrap.dedent("""vm: false""")] 174 | 175 | for yaml_file in yaml_files: 176 | conf_file_name = self._make_yaml_config(yaml_file) 177 | with self.assertRaises(utils.AppstartAbort): 178 | configuration.ApplicationConfiguration(conf_file_name) 179 | 180 | def test_malformed_yaml(self): 181 | yaml_file = 'malformed yaml file' 182 | conf_file_name = self._make_yaml_config(yaml_file) 183 | with self.assertRaises(utils.AppstartAbort): 184 | configuration.ApplicationConfiguration(conf_file_name) 185 | -------------------------------------------------------------------------------- /tests/unit_tests/sandbox/container_sandbox_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | """Unit tests for ContainerSandbox.""" 3 | # This file conforms to the external style guide 4 | # pylint: disable=bad-indentation, g-bad-import-order 5 | 6 | import logging 7 | import os 8 | import stubout 9 | import tempfile 10 | import unittest 11 | 12 | import docker 13 | import mox 14 | 15 | from appstart.sandbox import container_sandbox 16 | from appstart.sandbox import container 17 | from appstart import utils 18 | 19 | from fakes import fake_docker 20 | 21 | 22 | class TestBase(fake_docker.FakeDockerTestBase): 23 | 24 | def setUp(self): 25 | super(TestBase, self).setUp() 26 | test_directory = tempfile.mkdtemp() 27 | app_yaml = 'vm: true' 28 | self.conf_file = open(os.path.join(test_directory, 'app.yaml'), 'w') 29 | self.conf_file.write(app_yaml) 30 | self.conf_file.close() 31 | 32 | self.mocker = mox.Mox() 33 | 34 | def tearDown(self): 35 | """Restore docker.Client and requests.get.""" 36 | super(TestBase, self).tearDown() 37 | self.mocker.VerifyAll() 38 | self.mocker.UnsetStubs() 39 | self.stubs.UnsetAll() 40 | 41 | 42 | # pylint: disable=too-many-public-methods 43 | class CreateAndRemoveContainersTest(TestBase): 44 | """Test the full code paths associated with starting the sandbox.""" 45 | 46 | def setUp(self): 47 | super(CreateAndRemoveContainersTest, self).setUp() 48 | 49 | # Fake out ping. Under the hood, this is a docker exec. 50 | self.stubs.Set(container.PingerContainer, 51 | 'ping_application_container', 52 | lambda self: True) 53 | 54 | # Fake out stream_logs, as this will try to start another thread. 55 | self.stubs.Set(container.Container, 56 | 'stream_logs', 57 | lambda unused_self, unused_stream=True: None) 58 | 59 | def test_start_from_conf(self): 60 | """Test ContainerSandbox.start.""" 61 | sb = container_sandbox.ContainerSandbox(self.conf_file.name) 62 | sb.start() 63 | 64 | self.assertIsNotNone(sb.app_container) 65 | self.assertIsNotNone(sb.devappserver_container) 66 | self.assertIsNotNone(sb.app_container) 67 | 68 | def test_start_no_api_server(self): 69 | """Test ContainerSandbox.start (with no api server).""" 70 | sb = container_sandbox.ContainerSandbox(self.conf_file.name, 71 | run_api_server=False) 72 | sb.start() 73 | self.assertIsNotNone(sb.app_container) 74 | self.assertIsNotNone(sb.app_container) 75 | self.assertIsNone(sb.devappserver_container) 76 | 77 | def test_start_from_image(self): 78 | sb = container_sandbox.ContainerSandbox(image_name='test_image') 79 | with self.assertRaises(utils.AppstartAbort): 80 | sb.start() 81 | 82 | fake_docker.reset() 83 | fake_docker.images.append('test_image') 84 | sb.start() 85 | 86 | self.assertEqual(len(fake_docker.images), 87 | len(fake_docker.DEFAULT_IMAGES) + 2, 88 | 'Too many images created') 89 | 90 | def test_start_no_image_no_conf(self): 91 | with self.assertRaises(utils.AppstartAbort): 92 | container_sandbox.ContainerSandbox() 93 | 94 | 95 | class BadVersionTest(unittest.TestCase): 96 | 97 | def setUp(self): 98 | fake_docker.reset() 99 | 100 | def test_bad_version(self): 101 | """Test ContainerSandbox.create_and_run_containers. 102 | 103 | With a bad version, construction of the sandbox should fail. 104 | """ 105 | docker.Client.version = lambda _: {'Version': '1.6.0'} 106 | with self.assertRaises(utils.AppstartAbort): 107 | container_sandbox.ContainerSandbox(image_name='temp') 108 | 109 | 110 | class ExitTest(TestBase): 111 | """Ensure the ContainerSandbox exits properly.""" 112 | 113 | def setUp(self): 114 | """Populate the sandbox fake containers. 115 | 116 | This simulates the scenario where create_and_run_containers() has 117 | just run successfully. 118 | """ 119 | super(ExitTest, self).setUp() 120 | self.sandbox = container_sandbox.ContainerSandbox( 121 | self.conf_file.name) 122 | # Add the containers to the sandbox. Mock them out (we've tested the 123 | # containers elsewhere, and we just need the appropriate methods to be 124 | # called). 125 | self.sandbox.app_container = ( 126 | self.mocker.CreateMock(container.ApplicationContainer)) 127 | 128 | self.sandbox.devappserver_container = ( 129 | self.mocker.CreateMock(container.Container)) 130 | 131 | self.sandbox.pinger_container = ( 132 | self.mocker.CreateMock(container.PingerContainer)) 133 | 134 | # TODO(gouzenko): Figure out how to make order not matter (among the 135 | # three containers). 136 | self.sandbox.app_container.running().AndReturn(True) 137 | self.sandbox.app_container.get_id().AndReturn('456') 138 | self.sandbox.app_container.kill() 139 | self.sandbox.app_container.remove() 140 | 141 | self.sandbox.devappserver_container.running().AndReturn(True) 142 | self.sandbox.devappserver_container.get_id().AndReturn('123') 143 | self.sandbox.devappserver_container.kill() 144 | self.sandbox.devappserver_container.remove() 145 | 146 | self.sandbox.pinger_container.running().AndReturn(True) 147 | self.sandbox.pinger_container.get_id().AndReturn('123') 148 | self.sandbox.pinger_container.kill() 149 | self.sandbox.pinger_container.remove() 150 | self.mocker.ReplayAll() 151 | 152 | def test_stop(self): 153 | self.sandbox.stop() 154 | 155 | def test_exception_handling(self): 156 | """Test the case where an exception was raised in start(). 157 | 158 | The sandbox should stop and remove all containers before 159 | re-raising the exception. 160 | """ 161 | 162 | def excep_func(): 163 | """Simulate arbitrary exception.""" 164 | raise Exception 165 | 166 | self.sandbox.create_and_run_containers = excep_func 167 | 168 | with self.assertRaises(Exception): 169 | self.sandbox.start() 170 | 171 | 172 | class StaticTest(unittest.TestCase): 173 | 174 | def setUp(self): 175 | self.sandbox = container_sandbox.ContainerSandbox 176 | 177 | def test_get_web_xml(self): 178 | self.assertEqual(self.sandbox.get_web_xml('/conf/appengine-web.xml'), 179 | '/conf/web.xml', 180 | 'web.xml must be in same folder as appengine-web.xml') 181 | 182 | def test_app_directory_from_config(self): 183 | self.assertEqual( 184 | self.sandbox.app_directory_from_config('/app/blah/app-web.xml'), 185 | '/app') 186 | 187 | self.assertEqual( 188 | self.sandbox.app_directory_from_config('/app/app.yaml'), 189 | '/app') 190 | 191 | if __name__ == '__main__': 192 | logging.basicConfig(level=logging.CRITICAL) 193 | unittest.main() 194 | -------------------------------------------------------------------------------- /tests/unit_tests/sandbox/container_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Unit tests for appstart.sandbox.container.""" 16 | 17 | # This file conforms to the external style guide. 18 | # pylint: disable=bad-indentation, g-bad-import-order 19 | 20 | import unittest 21 | 22 | from appstart.sandbox import container 23 | from fakes import fake_docker 24 | 25 | 26 | class TestContainerExit(fake_docker.FakeDockerTestBase): 27 | 28 | def setUp(self): 29 | # Simulate that a SIGINT was caught by setting global _EXITING var 30 | super(TestContainerExit, self).setUp() 31 | self.stubs.Set(container, '_EXITING', True) 32 | 33 | # Pretend that we've created an image called 'temp' 34 | fake_docker.images.append('temp') 35 | 36 | def test_exit_from_create(self): 37 | dclient = fake_docker.FakeDockerClient() 38 | 39 | # container should detect that a KeyboardInterrupt was raised and 40 | # manually raise it again. 41 | cont = container.Container(dclient) 42 | with self.assertRaises(KeyboardInterrupt): 43 | cont.create(name='temp', image='temp') 44 | self.assertIsNotNone(cont._container_id) 45 | 46 | 47 | class TestContainer(fake_docker.FakeDockerTestBase): 48 | 49 | def setUp(self): 50 | super(TestContainer, self).setUp() 51 | self.dclient = fake_docker.FakeDockerClient() 52 | fake_docker.images.append('temp') 53 | self.cont = container.Container(self.dclient) 54 | self.cont.create(name='temp', 55 | image='temp') 56 | 57 | def test_create(self): 58 | # Ensure that only one container was created. 59 | self.assertEqual(len(fake_docker.containers), 60 | 1, 61 | 'Too many containers') 62 | 63 | created_container = fake_docker.containers[0] 64 | 65 | # Ensure that Container used the name supplied to it. 66 | self.assertEqual(created_container['Name'], 67 | 'temp', 68 | 'Container name did not match.') 69 | 70 | # Ensure that the Container is not yet running. 71 | self.assertFalse(created_container['Running'], 72 | '__init__ should not start container') 73 | 74 | # Ensure the correct host. (Note that the host is a result of 75 | # hardcoding the base_url in fake_docker.py 76 | self.assertEqual(self.cont.host, 77 | '0.0.0.0', 78 | 'Hosts do not match.') 79 | 80 | def test_kill(self): 81 | # Ensure that the container stops running in response to 'kill' 82 | fake_docker.containers[0]['Running'] = True 83 | self.cont.kill() 84 | self.assertFalse(self.cont.running(), 85 | 'Container believes itself to be running') 86 | self.assertFalse(fake_docker.containers[0]['Running'], 87 | 'Container was left running') 88 | 89 | # Containers can be more than once, so this should pass without error 90 | self.cont.kill() 91 | 92 | def test_remove(self): 93 | # Ensure that the container is removed when 'remove' is called. 94 | self.cont.remove() 95 | self.assertEqual(len(fake_docker.containers), 96 | 0, 97 | 'Container was not removed') 98 | 99 | # Containers can be removed more than once 100 | self.cont.remove() 101 | 102 | def test_start(self): 103 | self.cont.start() 104 | self.assertTrue(self.cont.running(), 105 | 'Container does not think itself to be running') 106 | self.assertTrue(fake_docker.containers[0]['Running'], 107 | 'Container is not running') 108 | 109 | def test_get_id(self): 110 | self.assertEqual(self.cont.get_id(), 111 | fake_docker.containers[0]['Id'], 112 | 'Container IDs do not match') 113 | 114 | if __name__ == '__main__': 115 | unittest.main() 116 | -------------------------------------------------------------------------------- /tests/unit_tests/utils_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Unit tests for utils.""" 16 | 17 | # This file conforms to the external style guide. 18 | # pylint: disable=bad-indentation, g-bad-import-order 19 | 20 | import io 21 | import logging 22 | import os 23 | import shutil 24 | import tarfile 25 | import tempfile 26 | import textwrap 27 | import unittest 28 | 29 | import docker 30 | 31 | from fakes import fake_docker 32 | from appstart import utils 33 | 34 | 35 | CERT_PATH = os.path.join(os.path.dirname(os.path.dirname(__file__)), 36 | 'test_data/certs') 37 | APP_DIR = os.path.join(os.path.dirname(__file__), 'system_tests') 38 | 39 | 40 | class DockerTest(fake_docker.FakeDockerTestBase): 41 | """Test error detection in Docker build results.""" 42 | 43 | def test_get_docker_client(self): 44 | os.environ['DOCKER_HOST'] = 'tcp://192.168.59.103:2376' 45 | os.environ['DOCKER_TLS_VERIFY'] = '1' 46 | os.environ['DOCKER_CERT_PATH'] = CERT_PATH 47 | 48 | dclient = utils.get_docker_client() 49 | self.assertIn('tls', dclient.kwargs) 50 | self.assertIn('base_url', dclient.kwargs) 51 | 52 | def test_build_from_directory(self): 53 | utils.build_from_directory(APP_DIR, 'test') 54 | self.assertEqual(len(fake_docker.images), 55 | 1 + len(fake_docker.DEFAULT_IMAGES)) 56 | self.assertIn('test', fake_docker.images) 57 | 58 | def test_failed_build(self): 59 | bad_build_res = fake_docker.FAILED_BUILD_RES 60 | with self.assertRaises(utils.AppstartAbort): 61 | utils.log_and_check_build_results(bad_build_res, 'temp') 62 | 63 | def test_successful_build(self): 64 | good_build_res = fake_docker.BUILD_RES 65 | utils.log_and_check_build_results(good_build_res, 'temp') 66 | 67 | def test_good_version(self): 68 | dclient = fake_docker.FakeDockerClient() 69 | utils.check_docker_version(dclient) 70 | 71 | def test_bad_version(self): 72 | dclient = fake_docker.FakeDockerClient() 73 | dclient.version = lambda: {'Version': '1.4.0'} 74 | with self.assertRaises(utils.AppstartAbort): 75 | utils.check_docker_version(dclient) 76 | 77 | def test_find_image(self): 78 | dclient = fake_docker.FakeDockerClient() 79 | fake_docker.images.append('test') 80 | self.assertTrue(utils.find_image('test')) 81 | 82 | 83 | class TarTest(unittest.TestCase): 84 | """Test the feature in utils that deal with tarfiles.""" 85 | 86 | def setUp(self): 87 | self.tempfile1 = tempfile.NamedTemporaryFile() 88 | self.tempfile1.write('foo') 89 | self.tempfile1.seek(0) 90 | 91 | self.tempfile2 = tempfile.NamedTemporaryFile() 92 | self.tempfile2.write('bar') 93 | self.tempfile2.seek(0) 94 | 95 | def test_make_build_context(self): 96 | dockerfile = io.BytesIO('FROM debian'.encode('utf-8')) 97 | context_files = {self.tempfile1.name: 'foo.txt', 98 | self.tempfile2.name: '/baz/bar.txt'} 99 | 100 | context = utils.make_tar_build_context(dockerfile, context_files) 101 | tar = tarfile.TarFile(fileobj=context) 102 | 103 | self.assertEqual(tar.extractfile('foo.txt').read(), 'foo') 104 | self.assertEqual(tar.extractfile('baz/bar.txt').read(), 'bar') 105 | 106 | def test_tar_wrapper(self): 107 | temp = tempfile.NamedTemporaryFile() 108 | tar = tarfile.open(mode='w', fileobj=temp) 109 | 110 | tinfo1 = tar.gettarinfo(fileobj=self.tempfile1, 111 | arcname='/root/baz/foo.txt') 112 | tar.addfile(tinfo1, self.tempfile1) 113 | 114 | tinfo2 = tar.gettarinfo(fileobj=self.tempfile2, 115 | arcname='/root/bar.txt') 116 | tar.addfile(tinfo2, self.tempfile2) 117 | 118 | fake_root = tarfile.TarInfo('root') 119 | fake_root.type = tarfile.DIRTYPE 120 | tar.addfile(fake_root) 121 | 122 | fake_baz = tarfile.TarInfo('root/baz') 123 | fake_baz.type = tarfile.DIRTYPE 124 | tar.addfile(fake_baz) 125 | 126 | tar.close() 127 | temp.seek(0) 128 | 129 | wrapped_tar = utils.TarWrapper(tarfile.open(mode='r', fileobj=temp)) 130 | self.assertEqual(wrapped_tar.get_file('root/bar.txt').read(), 'bar') 131 | self.assertEqual(wrapped_tar.get_file('root/baz/foo.txt').read(), 'foo') 132 | with self.assertRaises(ValueError): 133 | wrapped_tar.get_file('root') 134 | 135 | files, dirs = wrapped_tar.list('root') 136 | self.assertEqual(files, ['bar.txt']) 137 | self.assertEqual(dirs, ['baz']) 138 | with self.assertRaises(ValueError): 139 | wrapped_tar.list('root/bar.txt') 140 | 141 | 142 | class FileCollectionTest(unittest.TestCase): 143 | 144 | def setUp(self): 145 | self.temp_dir = tempfile.mkdtemp() 146 | self.files = [] 147 | for name in ('foo', 'bar', 'baz'): 148 | name_dir = os.path.join(self.temp_dir, name) 149 | os.mkdir(name_dir) 150 | file_name = os.path.join(name_dir, name + '.txt') 151 | with open(file_name, 'w') as f: 152 | f.write('example file') 153 | 154 | # 'baz' is excluded from the static dirs. 155 | if name != 'baz': 156 | self.files.append(file_name) 157 | self.config_file = os.path.join(self.temp_dir, 'app.yaml') 158 | with open(self.config_file, 'w') as f: 159 | f.write(textwrap.dedent("""\ 160 | handlers: 161 | - url: foo 162 | static_dir: foo 163 | - url: bar 164 | static_dir: bar 165 | """)) 166 | 167 | def tearDown(self): 168 | shutil.rmtree(self.temp_dir) 169 | 170 | def test_add_files(self): 171 | data = {} 172 | utils.add_files_from_static_dirs(data, self.config_file) 173 | self.assertEqual( 174 | data, 175 | dict((name, None) for name in self.files)) 176 | 177 | 178 | class LoggerTest(unittest.TestCase): 179 | 180 | def test_get_logger(self): 181 | logger = utils.get_logger() 182 | self.assertIsInstance(logger, logging.Logger) 183 | 184 | if __name__ == '__main__': 185 | unittest.main() 186 | -------------------------------------------------------------------------------- /tests/unit_tests/validator/__init__.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Tests for appstart.validator.""" 16 | -------------------------------------------------------------------------------- /tests/unit_tests/validator/contract_test.py: -------------------------------------------------------------------------------- 1 | # Copyright 2015 Google Inc. All Rights Reserved. 2 | # 3 | # Licensed under the Apache License, Version 2.0 (the "License"); 4 | # you may not use this file except in compliance with the License. 5 | # You may obtain a copy of the License at 6 | # 7 | # http://www.apache.org/licenses/LICENSE-2.0 8 | # 9 | # Unless required by applicable law or agreed to in writing, software 10 | # distributed under the License is distributed on an "AS IS" BASIS, 11 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12 | # See the License for the specific language governing permissions and 13 | # limitations under the License. 14 | 15 | """Unit tests for validator.container.""" 16 | 17 | # This file conforms to the external style guide. 18 | # pylint: disable=bad-indentation, g-bad-import-order 19 | 20 | import logging 21 | import os 22 | import stat 23 | import tempfile 24 | import textwrap 25 | 26 | from appstart import utils 27 | from appstart.sandbox import container_sandbox 28 | from appstart.validator import contract 29 | from appstart.validator import errors 30 | 31 | from fakes import fake_docker 32 | 33 | 34 | class ClauseTestBase(fake_docker.FakeDockerTestBase): 35 | 36 | def setUp(self): 37 | """Prepare a temporary application directory.""" 38 | super(ClauseTestBase, self).setUp() 39 | self.app_dir = tempfile.mkdtemp() 40 | self._add_file('app.yaml', 'vm: true') 41 | self.conf_file = os.path.join(self.app_dir, 'app.yaml') 42 | os.mkdir(os.path.join(self.app_dir, 'validator_tests')) 43 | 44 | def _add_file(self, name, string): 45 | f = open(os.path.join(self.app_dir, name), 'w') 46 | f.write(string) 47 | f.close() 48 | 49 | 50 | class ClauseTest(ClauseTestBase): 51 | 52 | def test_bad_clause_definitions(self): 53 | """Ensure that it's impossible to define bad clause classes.""" 54 | 55 | # pylint: disable=unused-variable, function-redefined 56 | 57 | # Clause doesn't have any required attributes. 58 | with self.assertRaises(errors.ContractAttributeError): 59 | 60 | class TestClause(contract.ContractClause): 61 | pass 62 | 63 | # Clause doesn't have description or lifecycle_point 64 | with self.assertRaises(errors.ContractAttributeError): 65 | 66 | class TestClause(contract.ContractClause): 67 | title = 'Test' 68 | 69 | # Clause doesn't have lifecycle_point 70 | with self.assertRaises(errors.ContractAttributeError): 71 | 72 | class TestClause(contract.ContractClause): 73 | title = 'Test' 74 | description = 'foobar' 75 | 76 | # Clause has invalid lifecycle_point 77 | with self.assertRaises(errors.ContractAttributeError): 78 | 79 | class TestClause(contract.ContractClause): 80 | title = 'Test' 81 | description = 'foobar' 82 | lifecycle_point = -10 83 | 84 | # Clause has invalid error_level 85 | with self.assertRaises(errors.ContractAttributeError): 86 | 87 | class TestClause(contract.ContractClause): 88 | title = 'Test' 89 | description = 'foobar' 90 | lifecycle_point = contract.POST_START 91 | error_level = -10 92 | 93 | # Clause has improper type for unresolved_dependencies 94 | with self.assertRaises(errors.ContractAttributeError): 95 | 96 | class Test4(contract.ContractClause): 97 | title = 'Test' 98 | description = 'foobar' 99 | lifecycle_point = contract.POST_START 100 | _unresolved_dependencies = 'blah' 101 | 102 | # Clause is ok. 103 | class TestClause(contract.ContractClause): 104 | title = 'Test' 105 | description = 'foobar' 106 | lifecycle_point = contract.POST_START 107 | _unresolved_dependencies = {'blah'} 108 | 109 | def test_extract_clauses(self): 110 | """Test that validator extracts only clauses from module.""" 111 | 112 | class TestClause(contract.ContractClause): 113 | title = 'Test' 114 | description = 'foobar' 115 | lifecycle_point = contract.POST_START 116 | 117 | class TestClause2(contract.ContractClause): 118 | title = 'Test2' 119 | description = 'baz' 120 | lifecycle_point = contract.POST_START 121 | 122 | class NotAClause(object): 123 | pass 124 | 125 | class Module(object): 126 | 127 | # Should be extracted 128 | testclause = TestClause 129 | testclause2 = TestClause2 130 | notaclause = NotAClause 131 | 132 | # Should not be extracted 133 | foo = 'foo' 134 | bar = 1 135 | 136 | clauses = contract.ContractValidator._extract_clauses(Module) 137 | clauses.sort() 138 | 139 | expected = [TestClause, TestClause2] 140 | expected.sort() 141 | 142 | self.assertEqual(clauses, expected) 143 | 144 | def test_normalize_clauses(self): 145 | """Test that clauses are properly normalized.""" 146 | 147 | class TestClause(contract.ContractClause): 148 | title = 'Test' 149 | description = 'foo' 150 | lifecycle_point = contract.POST_START 151 | _unresolved_dependents = {'TestClause2'} 152 | _unresolved_after = {'TestClause3'} 153 | 154 | class TestClause2(contract.ContractClause): 155 | title = 'Test2' 156 | description = 'bar' 157 | lifecycle_point = contract.POST_START 158 | _unresolved_before = {'TestClause'} 159 | _unresolved_dependencies = {'TestClause'} 160 | 161 | class TestClause3(contract.ContractClause): 162 | title = 'Test3' 163 | description = 'baz' 164 | lifecycle_point = contract.POST_START 165 | _unresolved_dependencies = {'TestClause', 'TestClause2'} 166 | 167 | clause_dict = {'TestClause': TestClause, 168 | 'TestClause2': TestClause2, 169 | 'TestClause3': TestClause3} 170 | 171 | contract.ContractValidator._normalize_clause_dict(clause_dict) 172 | self.assertEqual(TestClause.dependents, {TestClause2}) 173 | self.assertEqual(TestClause.after, {TestClause3}) 174 | self.assertEqual(TestClause2.before, {TestClause}) 175 | self.assertEqual(TestClause2.dependencies, {TestClause}) 176 | 177 | 178 | class HookClauseTest(ClauseTestBase): 179 | 180 | def setUp(self): 181 | super(HookClauseTest, self).setUp() 182 | 183 | # Disable excessively verbose output from the validator (for now) 184 | logging.getLogger('appstart.validator').disabled = True 185 | 186 | class FakeAppContainer(object): 187 | host = 'localhost' 188 | 189 | def get_id(self): 190 | return '123' 191 | 192 | class FakeSandbox(object): 193 | app_dir = self.app_dir 194 | app_container = FakeAppContainer() 195 | port = 8080 196 | 197 | def __init__(self, *args, **kwargs): 198 | pass 199 | 200 | def start(self): 201 | pass 202 | 203 | def stop(self): 204 | pass 205 | 206 | class Module(object): 207 | pass 208 | 209 | self.module = Module 210 | self.old_sandbox = container_sandbox.ContainerSandbox 211 | container_sandbox.ContainerSandbox = FakeSandbox 212 | 213 | # Should result in a successful hook clause 214 | self.successful_hook = textwrap.dedent('''\ 215 | #!/usr/bin/python 216 | import sys 217 | sys.exit(0)''') 218 | 219 | # Should result in a failed hook clause. 220 | self.unsuccessful_hook = textwrap.dedent('''\ 221 | #!/usr/bin/python 222 | import sys 223 | sys.exit(1)''') 224 | 225 | self.default_test_file = 'validator_tests/test1.py' 226 | 227 | def test_make_hook_clauses(self): 228 | """Test the construction of hook clauses from a .conf.yaml file.""" 229 | 230 | test_config = textwrap.dedent('''\ 231 | name: Test1 232 | title: Test number 1 233 | lifecycle_point: POST_START''') 234 | self._add_file('validator_tests/test1.py.conf.yaml', test_config) 235 | 236 | # The default test file does not exist. 237 | with self.assertRaises(utils.AppstartAbort): 238 | contract.ContractValidator(self.module, config_file=self.conf_file) 239 | 240 | self._add_file(self.default_test_file, self.successful_hook) 241 | 242 | # The default test file is not executable 243 | with self.assertRaises(utils.AppstartAbort): 244 | contract.ContractValidator(self.module, config_file=self.conf_file) 245 | 246 | os.chmod(os.path.join(self.app_dir, self.default_test_file), 247 | stat.S_IEXEC | stat.S_IREAD) 248 | 249 | # The 'description' attribute is missing from the configuration file. 250 | with self.assertRaises(utils.AppstartAbort): 251 | contract.ContractValidator(self.module, config_file=self.conf_file) 252 | 253 | test_config = textwrap.dedent('''\ 254 | name: Test1 255 | title: Test number 1 256 | lifecycle_point: POST_START 257 | description: This is a test.''') 258 | self._add_file('validator_tests/test1.py.conf.yaml', test_config) 259 | 260 | # The initialization should be okay now. 261 | contract.ContractValidator( 262 | self.module, 263 | config_file=os.path.join(self.app_dir, 'app.yaml')) 264 | 265 | def test_evaluate_hook_clauses(self): 266 | """Test that hook clauses are actually being evaluated.""" 267 | 268 | test_config = textwrap.dedent('''\ 269 | name: Test1 270 | title: Test number 1 271 | lifecycle_point: POST_START 272 | description: This is a test.''') 273 | self._add_file('validator_tests/test1.py.conf.yaml', test_config) 274 | self._add_file(self.default_test_file, self.successful_hook) 275 | os.chmod(os.path.join(self.app_dir, self.default_test_file), 276 | stat.S_IEXEC | stat.S_IREAD | stat.S_IWRITE) 277 | 278 | validator = contract.ContractValidator(self.module, 279 | config_file=self.conf_file) 280 | self.assertTrue(validator.validate()) 281 | 282 | self._add_file(self.default_test_file, self.unsuccessful_hook) 283 | os.chmod(os.path.join(self.app_dir, self.default_test_file), 284 | stat.S_IEXEC | stat.S_IREAD | stat.S_IWRITE) 285 | 286 | # Validator should still pass because the default threshold is higher 287 | # than UNUSED. 288 | self.assertTrue(validator.validate()) 289 | 290 | self.assertFalse(validator.validate(threshold='UNUSED')) 291 | 292 | test_config = textwrap.dedent('''\ 293 | name: Test1 294 | title: Test number 1 295 | lifecycle_point: POST_START 296 | description: This is a test. 297 | error_level: FATAL''') 298 | self._add_file('validator_tests/test1.py.conf.yaml', test_config) 299 | validator = contract.ContractValidator(self.module, 300 | config_file=self.conf_file) 301 | 302 | # Validator should fail because the defaul threshold should be less 303 | # than FATAL. 304 | self.assertFalse(validator.validate()) 305 | 306 | def test_loop_detection(self): 307 | """Test that the validator detects dependency loops.""" 308 | 309 | class Test1(contract.ContractClause): 310 | title = 'test' 311 | description = 'test' 312 | lifecycle_point = contract.POST_START 313 | _unresolved_dependencies = {'Test2'} 314 | 315 | class Test2(contract.ContractClause): 316 | title = 'test' 317 | description = 'test' 318 | lifecycle_point = contract.POST_START 319 | dependencies = {Test1} 320 | 321 | class LoopyModule(object): 322 | test1 = Test1 323 | test2 = Test2 324 | 325 | class Test3(contract.ContractClause): 326 | title = 'test' 327 | description = 'test' 328 | lifecycle_point = contract.POST_START 329 | _unresolved_dependencies = {'Test3'} 330 | 331 | class LoopyModule2(object): 332 | test3 = Test3 333 | 334 | class Test4(contract.ContractClause): 335 | title = 'test' 336 | description = 'test' 337 | lifecycle_point = contract.POST_START 338 | dependencies = {Test1} 339 | 340 | class LoopyModule3(object): 341 | test4 = Test4 342 | 343 | class Test5(contract.ContractClause): 344 | title = 'test' 345 | description = 'test' 346 | lifecycle_point = contract.POST_START 347 | 348 | class Test6(contract.ContractClause): 349 | title = 'test' 350 | description = 'test' 351 | lifecycle_point = contract.PRE_START 352 | dependencies = {Test5} 353 | 354 | class LoopyModule4(object): 355 | test5 = Test5 356 | test6 = Test6 357 | 358 | for mod in [LoopyModule, LoopyModule2, LoopyModule3, LoopyModule4]: 359 | with self.assertRaises(errors.CircularDependencyError): 360 | contract.ContractValidator(mod, config_file=self.conf_file) 361 | 362 | def test_dependency_order(self): 363 | """Test that dependencies get executed in correct order.""" 364 | ordering = [] 365 | 366 | class Test1(contract.ContractClause): 367 | title = 'test' 368 | description = 'test' 369 | lifecycle_point = contract.POST_START 370 | 371 | def evaluate_clause(self, app_container): 372 | ordering.append(self) 373 | 374 | class Test3(contract.ContractClause): 375 | title = 'test' 376 | description = 'test' 377 | lifecycle_point = contract.POST_START 378 | dependencies = {Test1} 379 | 380 | def evaluate_clause(self, app_container): 381 | ordering.append(self) 382 | 383 | class Test2(contract.ContractClause): 384 | title = 'test' 385 | description = 'test' 386 | lifecycle_point = contract.POST_START 387 | dependents = {Test3} 388 | 389 | def evaluate_clause(self, app_container): 390 | ordering.append(self) 391 | 392 | class Test0(contract.ContractClause): 393 | title = 'test' 394 | description = 'test' 395 | lifecycle_point = contract.POST_START 396 | after = {Test1} 397 | 398 | def evaluate_clause(self, app_container): 399 | ordering.append(self) 400 | 401 | class GoodModule(object): 402 | test1 = Test1 403 | test3 = Test3 404 | test2 = Test2 405 | test0 = Test0 406 | 407 | validator = contract.ContractValidator(GoodModule, 408 | config_file=self.conf_file) 409 | validator.validate() 410 | types = [type(obj) for obj in ordering] 411 | self.assertEqual(types, [Test0, Test1, Test2, Test3]) 412 | 413 | def tearDown(self): 414 | super(HookClauseTest, self).tearDown() 415 | logging.getLogger('appstart.validator').disabled = False 416 | container_sandbox.ContainerSandbox = self.old_sandbox 417 | --------------------------------------------------------------------------------