├── .gitignore ├── Jenkinsfile ├── LICENSE ├── README.md ├── pom.xml ├── src ├── main │ ├── java │ │ └── com │ │ │ └── amazon │ │ │ └── jenkins │ │ │ └── ec2fleet │ │ │ ├── CloudNanny.java │ │ │ ├── EC2Api.java │ │ │ ├── EC2FleetAutoResubmitComputerLauncher.java │ │ │ ├── EC2FleetCloud.java │ │ │ ├── EC2FleetCloudAware.java │ │ │ ├── EC2FleetCloudAwareUtils.java │ │ │ ├── EC2FleetNode.java │ │ │ ├── EC2FleetNodeComputer.java │ │ │ ├── EC2FleetOnlineChecker.java │ │ │ ├── EC2FleetStatusInfo.java │ │ │ ├── EC2FleetStatusWidget.java │ │ │ ├── EC2TerminationCause.java │ │ │ ├── FleetStateStats.java │ │ │ ├── IdleRetentionStrategy.java │ │ │ ├── LazyUuid.java │ │ │ ├── NoDelayProvisionStrategy.java │ │ │ └── Registry.java │ └── resources │ │ ├── Message.properties │ │ ├── com │ │ └── amazon │ │ │ └── jenkins │ │ │ └── ec2fleet │ │ │ ├── EC2FleetCloud │ │ │ ├── config.jelly │ │ │ ├── help-addNodeOnlyIfRunning.html │ │ │ ├── help-cloudStatusIntervalSec.html │ │ │ ├── help-disableTaskResubmit.html │ │ │ ├── help-endpoint.html │ │ │ ├── help-fleet.html │ │ │ ├── help-idleMinutes.html │ │ │ ├── help-initOnlineTimeoutSec.html │ │ │ ├── help-name.html │ │ │ ├── help-noDelayProvision.html │ │ │ ├── help-restrictUsage.html │ │ │ └── help-scaleExecutorsByWeight.html │ │ │ ├── EC2FleetNode │ │ │ └── configure-entries.jelly │ │ │ └── EC2FleetStatusWidget │ │ │ └── index.jelly │ │ └── index.jelly └── test │ └── java │ └── com │ └── amazon │ └── jenkins │ └── ec2fleet │ ├── AutoResubmitIntegrationTest.java │ ├── CloudNannyTest.java │ ├── EC2ApiTest.java │ ├── EC2FleetAutoResubmitComputerLauncherTest.java │ ├── EC2FleetCloudAwareUtilsTest.java │ ├── EC2FleetCloudTest.java │ ├── EC2FleetCloudWithHistory.java │ ├── EC2FleetCloudWithMeter.java │ ├── EC2FleetNodeComputerTest.java │ ├── EC2FleetOnlineCheckerTest.java │ ├── EmptyAmazonEC2.java │ ├── FleetStateStatsTest.java │ ├── IdleRetentionStrategyTest.java │ ├── IntegrationTest.java │ ├── LazyUuidTest.java │ ├── LocalComputerConnector.java │ ├── Meter.java │ ├── NoDelayProvisionStrategyPerformanceTest.java │ ├── NoDelayProvisionStrategyTest.java │ ├── ProvisionIntegrationTest.java │ ├── ProvisionPerformanceTest.java │ ├── RealEc2ApiIntegrationTest.java │ └── UiIntegrationTest.java └── test.xml /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | *.iml 3 | build 4 | target 5 | work 6 | credentials.txt 7 | -------------------------------------------------------------------------------- /Jenkinsfile: -------------------------------------------------------------------------------- 1 | #!groovy 2 | 3 | buildPlugin(jenkinsVersions: [null, '2.60.1', '2.107.1'], failFast: false) 4 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "{}" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright {yyyy} {name of copyright owner} 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # ec2-spot-jenkins-plugin 2 | 3 | [![Build Status](https://ci.jenkins.io/buildStatus/icon?job=Plugins/ec2-fleet-plugin/master)](https://ci.jenkins.io/blue/organizations/jenkins/Plugins%2Fec2-fleet-plugin/activity) [![](https://img.shields.io/jenkins/plugin/v/ec2-fleet.svg)](https://github.com/jenkinsci/ec2-fleet-plugin/releases) 4 | 5 | Use [jenkinsci/ec2-fleet-plugin](https://github.com/jenkinsci/ec2-fleet-plugin) instead of [awslabs/ec2-spot-jenkins-plugin](https://github.com/awslabs/ec2-spot-jenkins-plugin) 6 | 7 | The EC2 Spot Jenkins plugin launches EC2 Spot instances as worker nodes for Jenkins CI server, 8 | automatically scaling the capacity with the load. 9 | 10 | * [Jenkins Page](https://wiki.jenkins.io/display/JENKINS/Amazon+EC2+Fleet+Plugin) 11 | * [Report Issue](https://github.com/jenkinsci/ec2-fleet-plugin/issues/new) 12 | * [Overview](#overview) 13 | * [Change Log](#change-log) 14 | * [Usage](#usage) 15 | * [Setup](#setup) 16 | * [Scaling](#scaling) 17 | * [Groovy](#groovy) 18 | * [Preconfigure Slave](#preconfigure-slave) 19 | * [Development](#development) 20 | 21 | # Overview 22 | This plugin uses Spot Fleet to launch instances instead of directly launching them by itself. 23 | Amazon EC2 attempts to maintain your Spot fleet's target capacity as Spot prices change to maintain 24 | the fleet within the specified price range. For more information, see 25 | [How Spot Fleet Works](http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-fleet.html). 26 | 27 | # Change Log 28 | 29 | This plugin is using [SemVersion](https://semver.org/) which means that each plugin version looks like 30 | ``` 31 | .. 32 | 33 | major = increase only if non back compatible changes 34 | minor = increase when new features 35 | bugfix = increase when bug fixes 36 | ``` 37 | 38 | As result you safe to update plugin to any version until first number is the same with what you have. 39 | 40 | https://github.com/jenkinsci/ec2-fleet-plugin/releases 41 | 42 | # Usage 43 | 44 | ## Setup 45 | 46 | #### 1. Get AWS Account 47 | 48 | [AWS account](http://aws.amazon.com/ec2/) 49 | 50 | #### 2. Create IAM User 51 | 52 | Specify ```programmatic access``` during creation, and record credentials 53 | which will be used by Jenkins EC2 Fleet Plugin to connect to your Spot Fleet 54 | 55 | #### 3. Configure User permissions 56 | 57 | Add inline policy to the user to allow it use EC2 Spot Fleet 58 | [AWS documentation about that](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-fleet-requests.html#spot-fleet-prerequisites) 59 | 60 | ```json 61 | { 62 | "Version": "2012-10-17", 63 | "Statement": [ 64 | { 65 | "Effect": "Allow", 66 | "Action": [ 67 | "ec2:*" 68 | ], 69 | "Resource": "*" 70 | }, 71 | { 72 | "Effect": "Allow", 73 | "Action": [ 74 | "iam:ListRoles", 75 | "iam:PassRole", 76 | "iam:ListInstanceProfiles" 77 | ], 78 | "Resource": "*" 79 | } 80 | ] 81 | } 82 | ``` 83 | 84 | #### 4. Create EC2 Spot Fleet 85 | 86 | https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-fleet-requests.html#create-spot-fleet 87 | 88 | Make sure that you: 89 | - Checked ```Maintain target capacity``` [why](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/ec2-fleet-configuration-strategies.html#ec2-fleet-request-type) 90 | - specify an SSH key that will be used later by Jenkins. 91 | 92 | #### 5. Configure Jenkins 93 | 94 | Once the fleet is launched, you can set it up by adding a new **EC2 Fleet** cloud in the Jenkins 95 | 96 | 1. Goto ```Manage Jenkins > Plugin Manager``` 97 | 1. Install ```EC2 Fleet Jenkins Plugin``` 98 | 1. Goto ```Manage Jenkins > Configure System``` 99 | 1. Click ```Add a new cloud``` and select ```Amazon SpotFleet``` 100 | 1. Configure credentials and specify EC2 Spot Fleet which you want to use 101 | 102 | ## Scaling 103 | You can specify the scaling limits in your cloud settings. By default, Jenkins will try to scale fleet up 104 | if there are enough tasks waiting in the build queue and scale down idle nodes after a specified idleness period. 105 | 106 | You can use the History tab in the AWS console to view the scaling history. 107 | 108 | ## Groovy 109 | 110 | Below Groovy script to setup EC2 Spot Fleet Plugin for Jenkins and configure it, you can 111 | run it by [Jenkins Script Console](https://wiki.jenkins.io/display/JENKINS/Jenkins+Script+Console) 112 | 113 | ```groovy 114 | import com.amazonaws.services.ec2.model.InstanceType 115 | import com.cloudbees.jenkins.plugins.sshcredentials.impl.BasicSSHUserPrivateKey.DirectEntryPrivateKeySource 116 | import com.cloudbees.jenkins.plugins.sshcredentials.impl.BasicSSHUserPrivateKey 117 | import com.cloudbees.jenkins.plugins.awscredentials.AWSCredentialsImpl 118 | import hudson.plugins.sshslaves.SSHConnector 119 | import hudson.plugins.sshslaves.verifiers.NonVerifyingKeyVerificationStrategy 120 | import com.cloudbees.plugins.credentials.* 121 | import com.cloudbees.plugins.credentials.domains.Domain 122 | import hudson.model.* 123 | import com.amazon.jenkins.ec2fleet.EC2FleetCloud 124 | import jenkins.model.Jenkins 125 | 126 | // just modify this config other code just logic 127 | config = [ 128 | region: "us-east-1", 129 | fleetId: "...", 130 | idleMinutes: 10, 131 | minSize: 0, 132 | maxSize: 10, 133 | numExecutors: 1, 134 | awsKeyId: "...", 135 | secretKey: "...", 136 | ec2PrivateKey: '''-----BEGIN RSA PRIVATE KEY----- 137 | ... 138 | -----END RSA PRIVATE KEY-----''' 139 | ] 140 | 141 | // https://github.com/jenkinsci/aws-credentials-plugin/blob/aws-credentials-1.23/src/main/java/com/cloudbees/jenkins/plugins/awscredentials/AWSCredentialsImpl.java 142 | AWSCredentialsImpl awsCredentials = new AWSCredentialsImpl( 143 | CredentialsScope.GLOBAL, 144 | "aws-credentials", 145 | config.awsKeyId, 146 | config.secretKey, 147 | "my aws credentials" 148 | ) 149 | 150 | BasicSSHUserPrivateKey instanceCredentials = new BasicSSHUserPrivateKey( 151 | CredentialsScope.GLOBAL, 152 | "instance-ssh-key", 153 | "ec2-user", 154 | new DirectEntryPrivateKeySource(config.ec2PrivateKey), 155 | "", 156 | "my private key to ssh ec2 for jenkins" 157 | ) 158 | 159 | // find detailed information about parameters on plugin config page or 160 | // https://github.com/jenkinsci/ec2-fleet-plugin/blob/master/src/main/java/com/amazon/jenkins/ec2fleet/EC2FleetCloud.java 161 | EC2FleetCloud ec2FleetCloud = new EC2FleetCloud( 162 | "", // fleetCloudName 163 | awsCredentials.id, 164 | "", 165 | config.region, 166 | config.fleetId, 167 | "ec2-fleet", // labels 168 | "", // fs root 169 | new SSHConnector(22, 170 | instanceCredentials.id, "", "", "", "", null, 0, 0, 171 | // consult doc for line below, this one say no host verification, but you can use more strict mode 172 | // https://github.com/jenkinsci/ssh-slaves-plugin/blob/master/src/main/java/hudson/plugins/sshslaves/verifiers/NonVerifyingKeyVerificationStrategy.java 173 | new NonVerifyingKeyVerificationStrategy()), 174 | false, // if need to use privateIpUsed 175 | false, // if need alwaysReconnect 176 | config.idleMinutes, // if need to allow downscale set > 0 in min 177 | config.minSize, // minSize 178 | config.maxSize, // maxSize 179 | config.numExecutors, // numExecutors 180 | false, // addNodeOnlyIfRunning 181 | false, // restrictUsage allow execute only jobs with proper label 182 | ) 183 | 184 | // get Jenkins instance 185 | Jenkins jenkins = Jenkins.getInstance() 186 | // get credentials domain 187 | def domain = Domain.global() 188 | // get credentials store 189 | def store = jenkins.getExtensionList('com.cloudbees.plugins.credentials.SystemCredentialsProvider')[0].getStore() 190 | // add credential to store 191 | store.addCredentials(domain, awsCredentials) 192 | store.addCredentials(domain, instanceCredentials) 193 | // add cloud configuration to Jenkins 194 | jenkins.clouds.add(ec2FleetCloud) 195 | // save current Jenkins state to disk 196 | jenkins.save() 197 | ``` 198 | 199 | ## Preconfigure Slave 200 | 201 | Sometimes you need to prepare slave (which is EC2 instance) before Jenkins could use it. 202 | For example install some software which will be required by your builds like Maven etc. 203 | 204 | For those cases you have a few options, described below: 205 | 206 | ### Amazon EC2 AMI 207 | 208 | **Greate for static preconfiguration** 209 | 210 | [AMI](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html) allows you to 211 | create custom images for your EC2 instances. For example you can create image with 212 | Linux plus Java, Maven etc. as result when EC2 fleet will launch new EC2 instance with 213 | this AMI it will automatically get all required software. Nice =) 214 | 215 | 1. Create custom AMI as described [here](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AMIs.html#creating-an-ami) 216 | 1. Create EC2 Spot Fleet with this AMI 217 | 218 | ### EC2 instance User Data 219 | 220 | EC2 instance allows to specify special script [User Data](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/user-data.html) 221 | which will be executed when EC2 instance is created. That's allow you to do some customization 222 | for particular instance. 223 | 224 | However, EC2 instance doesn't provide any information about User Data execution status, 225 | as result Jenkins could start task on new instances while User Data still in progress. 226 | 227 | To avoid that you can use Jenkins SSH Launcher ```Prefix Start Agent Command``` setting 228 | to specify command which should fail if User Data is not finished, in that way Jenkins will 229 | not be able to connect to instance until User Data is not done [more](https://github.com/jenkinsci/ssh-slaves-plugin/blob/master/doc/CONFIGURE.md) 230 | 231 | 1. Prepare [User Data script](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/user-data.html) 232 | 1. Open Jenkins 233 | 1. Goto ```Manage Jenkins > Configure Jenkins``` 234 | 1. Find proper fleet configuration and click ```Advance``` for SSH Launcher 235 | 1. Add checking command into field ``` Prefix Start Slave Command``` 236 | - example ```java -version && ``` 237 | 1. To apply for existent instances restart Jenkins or Delete Nodes from Jenkins so they will be reconnected 238 | 239 | # Development 240 | 241 | Plugin usage statistic per Jenkins version [here](https://stats.jenkins.io/pluginversions/ec2-fleet.html) 242 | 243 | ## Releasing 244 | 245 | https://jenkins.io/doc/developer/publishing/releasing/ 246 | 247 | ```bash 248 | mvn release:prepare release:perform 249 | ``` 250 | 251 | ### Jenkins 2 can't connect by SSH 252 | 253 | https://issues.jenkins-ci.org/browse/JENKINS-53954 254 | 255 | ### Install Java 8 on EC2 instance 256 | 257 | ```bash 258 | sudo yum install java-1.8.0 259 | sudo yum remove java-1.7.0-openjdk 260 | java -version 261 | ``` 262 | 263 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | 5 | 6 | org.jenkins-ci.plugins 7 | plugin 8 | 3.6 9 | 10 | 11 | 12 | com.amazon.jenkins.fleet 13 | ec2-fleet 14 | 1.13.1-SNAPSHOT 15 | hpi 16 | 17 | 18 | 1.625.3 19 | 7 20 | 21 | 22 | EC2 Fleet Jenkins Plugin 23 | Support EC2 SpotFleet for Jenkins 24 | https://wiki.jenkins.io/display/JENKINS/Amazon+EC2+Fleet+Plugin 25 | 26 | 27 | MIT License 28 | http://opensource.org/licenses/MIT 29 | 30 | 31 | 32 | 33 | 34 | terma 35 | Artem Stasiuk 36 | artem.stasuk@gmail.com 37 | 38 | 39 | schmutze 40 | Chad Schmutzer 41 | schmutze@amazon.com 42 | 43 | 44 | 45 | 46 | scm:git:ssh://github.com/jenkinsci/ec2-fleet-plugin.git 47 | scm:git:ssh://git@github.com/jenkinsci/ec2-fleet-plugin.git 48 | https://github.com/jenkinsci/ec2-fleet-plugin 49 | HEAD 50 | 51 | 52 | 53 | 54 | repo.jenkins-ci.org 55 | https://repo.jenkins-ci.org/public/ 56 | 57 | 58 | 59 | 60 | 61 | repo.jenkins-ci.org 62 | https://repo.jenkins-ci.org/public/ 63 | 64 | 65 | 66 | 67 | 68 | org.jenkins-ci.plugins 69 | credentials 70 | 2.3.19 71 | 72 | 73 | org.jenkins-ci.plugins 74 | aws-java-sdk 75 | 1.11.341 76 | 77 | 78 | org.jenkins-ci.plugins 79 | aws-credentials 80 | 1.24 81 | 82 | 83 | org.jenkins-ci.plugins 84 | ssh-slaves 85 | 1.20 86 | 87 | 88 | org.jenkins-ci.plugins 89 | jackson2-api 90 | 2.7.3 91 | 92 | 93 | 94 | 95 | 96 | org.powermock 97 | powermock-module-junit4 98 | 2.0.2 99 | test 100 | 101 | 102 | org.powermock 103 | powermock-api-mockito2 104 | 2.0.2 105 | test 106 | 107 | 108 | org.objenesis 109 | objenesis 110 | 111 | 112 | 113 | 114 | 115 | 116 | 117 | 118 | 119 | 120 | org.codehaus.mojo 121 | findbugs-maven-plugin 122 | 123 | false 124 | 125 | 126 | 127 | 128 | 129 | 130 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/jenkins/ec2fleet/CloudNanny.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import com.google.common.annotations.VisibleForTesting; 4 | import com.google.common.base.Objects; 5 | import com.google.common.collect.MapMaker; 6 | import hudson.Extension; 7 | import hudson.model.PeriodicWork; 8 | import hudson.slaves.Cloud; 9 | import hudson.widgets.Widget; 10 | import jenkins.model.Jenkins; 11 | 12 | import java.util.ArrayList; 13 | import java.util.List; 14 | import java.util.concurrent.ConcurrentMap; 15 | import java.util.concurrent.atomic.AtomicInteger; 16 | import java.util.logging.Level; 17 | import java.util.logging.Logger; 18 | 19 | /** 20 | * @see EC2FleetCloud 21 | * @see EC2FleetStatusWidget 22 | */ 23 | @Extension 24 | @SuppressWarnings("unused") 25 | public class CloudNanny extends PeriodicWork { 26 | 27 | private static final Logger LOGGER = Logger.getLogger(CloudNanny.class.getName()); 28 | 29 | private final ConcurrentMap recurrenceCounters = new MapMaker() 30 | .weakKeys() // the map should not hold onto fleet instances to allow deletion of fleets. 31 | .concurrencyLevel(1) 32 | .makeMap(); 33 | 34 | @Override 35 | public long getRecurrencePeriod() { 36 | return 1000L; 37 | } 38 | 39 | /** 40 | *

Exceptions

41 | * This method will be executed by {@link PeriodicWork} inside {@link java.util.concurrent.ScheduledExecutorService} 42 | * by default it stops execution if task throws exception, however {@link PeriodicWork} fix that 43 | * by catch any exception and just log it, so we safe to throw exception here. 44 | */ 45 | @Override 46 | protected void doRun() { 47 | final List info = new ArrayList<>(); 48 | for (final Cloud cloud : getClouds()) { 49 | if (!(cloud instanceof EC2FleetCloud)) continue; 50 | final EC2FleetCloud fleetCloud = (EC2FleetCloud) cloud; 51 | 52 | AtomicInteger recurrenceCounter = getRecurrenceCounter(fleetCloud); 53 | 54 | if (recurrenceCounter.decrementAndGet() > 0) { 55 | continue; 56 | } 57 | 58 | recurrenceCounter.set(fleetCloud.getCloudStatusIntervalSec()); 59 | 60 | try { 61 | // Update the cluster states 62 | final FleetStateStats stats = fleetCloud.update(); 63 | info.add(new EC2FleetStatusInfo( 64 | fleetCloud.getFleet(), stats.getState(), fleetCloud.getLabelString(), 65 | stats.getNumActive(), stats.getNumDesired())); 66 | } catch (Exception e) { 67 | // could bad configuration or real exception, we can't do too much here 68 | LOGGER.log(Level.INFO, String.format("Error during fleet %s stats update", fleetCloud.name), e); 69 | } 70 | } 71 | 72 | for (final Widget w : getWidgets()) { 73 | if (w instanceof EC2FleetStatusWidget) ((EC2FleetStatusWidget) w).setStatusList(info); 74 | } 75 | } 76 | 77 | /** 78 | * Will be mocked by tests to avoid deal with jenkins 79 | * 80 | * @return widgets 81 | */ 82 | @VisibleForTesting 83 | private static List getWidgets() { 84 | return Jenkins.getActiveInstance().getWidgets(); 85 | } 86 | 87 | /** 88 | * We return {@link List} instead of original {@link jenkins.model.Jenkins.CloudList} 89 | * to simplify testing as jenkins list requires actual {@link Jenkins} instance. 90 | * 91 | * @return basic java list 92 | */ 93 | @VisibleForTesting 94 | private static List getClouds() { 95 | return Jenkins.getActiveInstance().clouds; 96 | } 97 | 98 | @VisibleForTesting 99 | private AtomicInteger getRecurrenceCounter(EC2FleetCloud fleetCloud) { 100 | AtomicInteger counter = new AtomicInteger(fleetCloud.getCloudStatusIntervalSec()); 101 | // If a counter already exists, return the value, otherwise set the new counter value and return it. 102 | return Objects.firstNonNull(recurrenceCounters.putIfAbsent(fleetCloud, counter), counter); 103 | } 104 | } 105 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/jenkins/ec2fleet/EC2Api.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import com.amazonaws.regions.Region; 4 | import com.amazonaws.regions.RegionUtils; 5 | import com.amazonaws.services.ec2.AmazonEC2; 6 | import com.amazonaws.services.ec2.AmazonEC2Client; 7 | import com.amazonaws.services.ec2.model.AmazonEC2Exception; 8 | import com.amazonaws.services.ec2.model.DescribeInstancesRequest; 9 | import com.amazonaws.services.ec2.model.DescribeInstancesResult; 10 | import com.amazonaws.services.ec2.model.Instance; 11 | import com.amazonaws.services.ec2.model.InstanceStateName; 12 | import com.amazonaws.services.ec2.model.Reservation; 13 | import com.amazonaws.services.ec2.model.TerminateInstancesRequest; 14 | import com.cloudbees.jenkins.plugins.awscredentials.AWSCredentialsHelper; 15 | import com.cloudbees.jenkins.plugins.awscredentials.AmazonWebServicesCredentials; 16 | import com.google.common.collect.ImmutableSet; 17 | import com.google.common.collect.Lists; 18 | import jenkins.model.Jenkins; 19 | import org.apache.commons.lang.StringUtils; 20 | 21 | import javax.annotation.Nullable; 22 | import java.util.ArrayList; 23 | import java.util.Collections; 24 | import java.util.HashMap; 25 | import java.util.List; 26 | import java.util.Map; 27 | import java.util.Set; 28 | import java.util.regex.Matcher; 29 | import java.util.regex.Pattern; 30 | 31 | @SuppressWarnings("WeakerAccess") 32 | public class EC2Api { 33 | 34 | private static final ImmutableSet TERMINATED_STATES = ImmutableSet.of( 35 | InstanceStateName.Terminated.toString(), 36 | InstanceStateName.Stopped.toString(), 37 | InstanceStateName.Stopping.toString(), 38 | InstanceStateName.ShuttingDown.toString() 39 | ); 40 | 41 | private static final int BATCH_SIZE = 900; 42 | 43 | private static final String NOT_FOUND_ERROR_CODE = "InvalidInstanceID.NotFound"; 44 | private static final Pattern INSTANCE_ID_PATTERN = Pattern.compile("(i-[0-9a-zA-Z]+)"); 45 | 46 | private static List parseInstanceIdsFromNotFoundException(final String errorMessage) { 47 | final Matcher fullMessageMatcher = INSTANCE_ID_PATTERN.matcher(errorMessage); 48 | 49 | final List instanceIds = new ArrayList<>(); 50 | while (fullMessageMatcher.find()) { 51 | instanceIds.add(fullMessageMatcher.group(1)); 52 | } 53 | 54 | return instanceIds; 55 | } 56 | 57 | public Map describeInstances(final AmazonEC2 ec2, final Set instanceIds) { 58 | return describeInstances(ec2, instanceIds, BATCH_SIZE); 59 | } 60 | 61 | public Map describeInstances(final AmazonEC2 ec2, final Set instanceIds, final int batchSize) { 62 | final Map described = new HashMap<>(); 63 | // don't do actual call if no data 64 | if (instanceIds.isEmpty()) return described; 65 | 66 | final List> batches = Lists.partition(new ArrayList<>(instanceIds), batchSize); 67 | for (final List batch : batches) { 68 | describeInstancesBatch(ec2, described, batch); 69 | } 70 | return described; 71 | } 72 | 73 | private static void describeInstancesBatch( 74 | final AmazonEC2 ec2, final Map described, final List batch) { 75 | // we are going to modify list, so copy 76 | final List copy = new ArrayList<>(batch); 77 | 78 | // just to simplify debug by having consist order 79 | Collections.sort(copy); 80 | 81 | // because instances could be terminated at any time we do multiple 82 | // retry to get status and all time remove from request all non found instances if any 83 | while (copy.size() > 0) { 84 | try { 85 | final DescribeInstancesRequest request = new DescribeInstancesRequest().withInstanceIds(copy); 86 | 87 | DescribeInstancesResult result; 88 | do { 89 | result = ec2.describeInstances(request); 90 | request.setNextToken(result.getNextToken()); 91 | 92 | for (final Reservation r : result.getReservations()) { 93 | for (final Instance instance : r.getInstances()) { 94 | // if instance not in terminated state, add it to described 95 | if (!TERMINATED_STATES.contains(instance.getState().getName())) { 96 | described.put(instance.getInstanceId(), instance); 97 | } 98 | } 99 | } 100 | } while (result.getNextToken() != null); 101 | 102 | // all good, clear request batch to stop 103 | copy.clear(); 104 | } catch (final AmazonEC2Exception exception) { 105 | // if we cannot find instance, that's fine assume them as terminated 106 | // remove from request and try again 107 | if (exception.getErrorCode().equals(NOT_FOUND_ERROR_CODE)) { 108 | final List notFoundInstanceIds = parseInstanceIdsFromNotFoundException(exception.getMessage()); 109 | if (notFoundInstanceIds.isEmpty()) { 110 | // looks like we cannot parse correctly, rethrow 111 | throw exception; 112 | } 113 | copy.removeAll(notFoundInstanceIds); 114 | } 115 | } 116 | } 117 | } 118 | 119 | /** 120 | * Auto handle instance not found exception if any and assume those instances as already terminated 121 | * 122 | * @param ec2 123 | * @param instanceIds 124 | */ 125 | public void terminateInstances(final AmazonEC2 ec2, final Set instanceIds) { 126 | final List temp = new ArrayList<>(instanceIds); 127 | while (temp.size() > 0) { 128 | // terminateInstances is idempotent so it can be called until it's successful 129 | try { 130 | ec2.terminateInstances(new TerminateInstancesRequest(temp)); 131 | // clear as removed so we can finish 132 | temp.clear(); 133 | } catch (AmazonEC2Exception exception) { 134 | // if we cannot find instance, that's fine assume them as terminated 135 | // remove from request and try again 136 | if (exception.getErrorCode().equals(NOT_FOUND_ERROR_CODE)) { 137 | final List notFoundInstanceIds = parseInstanceIdsFromNotFoundException(exception.getMessage()); 138 | if (notFoundInstanceIds.isEmpty()) { 139 | // looks like we cannot parse correctly, rethrow 140 | throw exception; 141 | } 142 | temp.removeAll(notFoundInstanceIds); 143 | } 144 | } 145 | } 146 | } 147 | 148 | public AmazonEC2 connect(final String awsCredentialsId, final String regionName, final String endpoint) { 149 | final AmazonWebServicesCredentials credentials = AWSCredentialsHelper.getCredentials(awsCredentialsId, Jenkins.getInstance()); 150 | final AmazonEC2Client client = 151 | credentials != null ? 152 | new AmazonEC2Client(credentials) : 153 | new AmazonEC2Client(); 154 | 155 | final String effectiveEndpoint = getEndpoint(regionName, endpoint); 156 | if (effectiveEndpoint != null) client.setEndpoint(effectiveEndpoint); 157 | return client; 158 | } 159 | 160 | /** 161 | * Derive EC2 API endpoint. If endpoint parameter not empty will use 162 | * it as first priority, otherwise will try to find region in {@link RegionUtils} by regionName 163 | * and use endpoint from it, if not available will generate endpoint as string and check if 164 | * region name looks like China cn- prefix. 165 | *

166 | * Implementation details 167 | *

168 | * {@link RegionUtils} is static information, and to get new region required to be updated, 169 | * as it's not possible too fast as you need to check new version of lib, moreover new version of lib 170 | * could be pointed to new version of Jenkins which is not a case for our plugin as some of installation 171 | * still on 1.6.x 172 | *

173 | * For example latest AWS SDK lib depends on Jackson2 plugin which starting from version 2.8.7.0 174 | * require Jenkins at least 2.60 https://plugins.jenkins.io/jackson2-api 175 | *

176 | * List of all AWS endpoints 177 | * https://docs.aws.amazon.com/general/latest/gr/rande.html 178 | * 179 | * @param regionName like us-east-1 not a airport code, could be null 180 | * @param endpoint custom endpoint could be null 181 | * @return null or actual endpoint 182 | */ 183 | @Nullable 184 | public String getEndpoint(@Nullable final String regionName, @Nullable final String endpoint) { 185 | if (StringUtils.isNotEmpty(endpoint)) { 186 | return endpoint; 187 | } else if (StringUtils.isNotEmpty(regionName)) { 188 | final Region region = RegionUtils.getRegion(regionName); 189 | if (region != null && region.isServiceSupported(endpoint)) { 190 | return region.getServiceEndpoint(endpoint); 191 | } else { 192 | final String domain = regionName.startsWith("cn-") ? "amazonaws.com.cn" : "amazonaws.com"; 193 | return "https://ec2." + regionName + "." + domain; 194 | } 195 | } else { 196 | return null; 197 | } 198 | } 199 | } 200 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/jenkins/ec2fleet/EC2FleetAutoResubmitComputerLauncher.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import hudson.model.Action; 4 | import hudson.model.Actionable; 5 | import hudson.model.Executor; 6 | import hudson.model.Queue; 7 | import hudson.model.Result; 8 | import hudson.model.TaskListener; 9 | import hudson.model.queue.SubTask; 10 | import hudson.slaves.ComputerLauncher; 11 | import hudson.slaves.DelegatingComputerLauncher; 12 | import hudson.slaves.OfflineCause; 13 | import hudson.slaves.SlaveComputer; 14 | 15 | import javax.annotation.concurrent.ThreadSafe; 16 | import java.util.ArrayList; 17 | import java.util.List; 18 | import java.util.logging.Level; 19 | import java.util.logging.Logger; 20 | 21 | /** 22 | * This is wrapper for {@link ComputerLauncher} to get notification when slave was disconnected 23 | * and automatically resubmit {@link hudson.model.Queue.Task} if reason is unexpected termination 24 | * which usually means EC2 instance was interrupted. 25 | *

26 | * This is optional feature, it's enabled by default, but could be disabled by 27 | * {@link EC2FleetCloud#isDisableTaskResubmit()} 28 | * 29 | * @see EC2FleetNode 30 | * @see EC2FleetNodeComputer 31 | */ 32 | @SuppressWarnings("WeakerAccess") 33 | @ThreadSafe 34 | public class EC2FleetAutoResubmitComputerLauncher extends DelegatingComputerLauncher { 35 | 36 | private static final Level LOG_LEVEL = Level.INFO; 37 | private static final Logger LOGGER = Logger.getLogger(EC2FleetAutoResubmitComputerLauncher.class.getName()); 38 | 39 | /** 40 | * Delay which will be applied when job {@link Queue#scheduleInternal(Queue.Task, int, List)} 41 | * rescheduled after offline 42 | */ 43 | private static final int RESCHEDULE_QUIET_PERIOD_SEC = 10; 44 | 45 | public EC2FleetAutoResubmitComputerLauncher(final ComputerLauncher launcher) { 46 | super(launcher); 47 | } 48 | 49 | /** 50 | * {@link ComputerLauncher#afterDisconnect(SlaveComputer, TaskListener)} 51 | *

52 | * EC2 Fleet plugin overrides this method to detect jobs which were failed because of 53 | * EC2 instance was terminated/stopped. It could be manual stop or because of Spot marked. 54 | * In all cases as soon as job aborted because of broken connection and slave is offline 55 | * it will try to resubmit aborted job back to the queue, so user doesn't need to do that manually 56 | * and another slave could take it. 57 | *

58 | * Implementation details 59 | *

60 | * There is no official recommendation about way how to resubmit job according to 61 | * https://issues.jenkins-ci.org/browse/JENKINS-49707 moreover some of Jenkins code says it impossible. 62 | *

63 | * method checks {@link SlaveComputer#getOfflineCause()} for disconnect because of EC2 instance termination 64 | * it returns 65 | * 66 | * result = {OfflineCause$ChannelTermination@13708} "Connection was broken: java.io.IOException: 67 | * Unexpected termination of the channel\n\tat hudson.remoting.SynchronousCommandTransport$ReaderThread... 68 | * cause = {IOException@13721} "java.io.IOException: Unexpected termination of the channel" 69 | * timestamp = 1561067177837 70 | * 71 | * 72 | * @param computer computer 73 | * @param listener listener 74 | */ 75 | @Override 76 | public void afterDisconnect(final SlaveComputer computer, final TaskListener listener) { 77 | // according to jenkins docs could be null in edge cases, check ComputerLauncher.afterDisconnect 78 | if (computer == null) return; 79 | 80 | // in some multi-thread edge cases cloud could be null for some time, just be ok with that 81 | final EC2FleetCloud cloud = ((EC2FleetNodeComputer) computer).getCloud(); 82 | if (cloud == null) { 83 | LOGGER.warning("Edge case cloud is null for computer " + computer.getDisplayName() 84 | + " should be autofixed in a few minutes, if no please create issue for plugin"); 85 | return; 86 | } 87 | 88 | final boolean unexpectedDisconnect = computer.isOffline() && computer.getOfflineCause() instanceof OfflineCause.ChannelTermination; 89 | if (!cloud.isDisableTaskResubmit() && unexpectedDisconnect) { 90 | final List executors = computer.getExecutors(); 91 | LOGGER.log(LOG_LEVEL, "Unexpected " + computer.getDisplayName() 92 | + " termination, resubmit"); 93 | 94 | for (Executor executor : executors) { 95 | if (executor.getCurrentExecutable() != null) { 96 | executor.interrupt(Result.ABORTED, new EC2TerminationCause(computer.getDisplayName())); 97 | 98 | final Queue.Executable executable = executor.getCurrentExecutable(); 99 | // if executor is not idle 100 | if (executable != null) { 101 | final SubTask subTask = executable.getParent(); 102 | final Queue.Task task = subTask.getOwnerTask(); 103 | 104 | List actions = new ArrayList<>(); 105 | if (executable instanceof Actionable) { 106 | actions = ((Actionable) executable).getActions(); 107 | } 108 | 109 | Queue.getInstance().schedule2(task, RESCHEDULE_QUIET_PERIOD_SEC, actions); 110 | LOGGER.log(LOG_LEVEL, "Unexpected " + computer.getDisplayName() 111 | + " termination, resubmit " + task + " with actions " + actions); 112 | } 113 | } 114 | } 115 | LOGGER.log(LOG_LEVEL, "Unexpected " + computer.getDisplayName() 116 | + " termination, resubmit finished"); 117 | } else { 118 | LOGGER.log(LOG_LEVEL, "Unexpected " + computer.getDisplayName() 119 | + " termination but resubmit disabled, no actions, disableTaskResubmit: " 120 | + cloud.isDisableTaskResubmit() + ", offline: " + computer.isOffline() 121 | + ", offlineCause: " + (computer.getOfflineCause() != null ? computer.getOfflineCause().getClass() : "null")); 122 | } 123 | 124 | // call parent 125 | super.afterDisconnect(computer, listener); 126 | } 127 | 128 | } 129 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/jenkins/ec2fleet/EC2FleetCloudAware.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import javax.annotation.Nonnull; 4 | 5 | /** 6 | * Interface to mark object that it's require cloud change update. Jenkins always creates 7 | * new instance of {@link hudson.slaves.Cloud} each time when you save Jenkins configuration page 8 | * regardless of was it actually changed or not. 9 | *

10 | * Jenkins never mutate existent {@link hudson.slaves.Cloud} instance 11 | *

12 | * As result all objects which depends on info from cloud 13 | * should be start to consume new instance of object to be able get new configuration if any. 14 | *

15 | * {@link EC2FleetCloud} is responsible to update all dependencies with new reference 16 | */ 17 | public interface EC2FleetCloudAware { 18 | 19 | EC2FleetCloud getCloud(); 20 | 21 | void setCloud(@Nonnull EC2FleetCloud cloud); 22 | 23 | } 24 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/jenkins/ec2fleet/EC2FleetCloudAwareUtils.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import hudson.model.Computer; 4 | import hudson.model.Node; 5 | import jenkins.model.Jenkins; 6 | 7 | import javax.annotation.Nonnull; 8 | import java.util.logging.Logger; 9 | 10 | /** 11 | * @see EC2FleetCloudAware 12 | */ 13 | @SuppressWarnings("WeakerAccess") 14 | public class EC2FleetCloudAwareUtils { 15 | 16 | private static final Logger LOGGER = Logger.getLogger(EC2FleetCloudAwareUtils.class.getName()); 17 | 18 | public static void reassign(final @Nonnull String oldId, @Nonnull final EC2FleetCloud cloud) { 19 | for (final Computer computer : Jenkins.getActiveInstance().getComputers()) { 20 | checkAndReassign(oldId, cloud, computer); 21 | } 22 | 23 | for (final Node node : Jenkins.getActiveInstance().getNodes()) { 24 | checkAndReassign(oldId, cloud, node); 25 | } 26 | 27 | LOGGER.info("Finish to reassign resources from old cloud with id " + oldId + " to " + cloud.getDisplayName()); 28 | } 29 | 30 | private static void checkAndReassign(final String oldId, final EC2FleetCloud cloud, final Object object) { 31 | if (object instanceof EC2FleetCloudAware) { 32 | final EC2FleetCloudAware cloudAware = (EC2FleetCloudAware) object; 33 | final EC2FleetCloud oldCloud = cloudAware.getCloud(); 34 | if (oldCloud != null && oldId.equals(oldCloud.getOldId())) { 35 | ((EC2FleetCloudAware) object).setCloud(cloud); 36 | LOGGER.info("Reassign " + object + " from " + oldCloud.getDisplayName() + " to " + cloud.getDisplayName()); 37 | } 38 | } 39 | } 40 | } 41 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/jenkins/ec2fleet/EC2FleetNode.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import hudson.Extension; 4 | import hudson.model.Computer; 5 | import hudson.model.Descriptor; 6 | import hudson.model.Node; 7 | import hudson.model.Slave; 8 | import hudson.slaves.ComputerLauncher; 9 | import hudson.slaves.EphemeralNode; 10 | import hudson.slaves.NodeProperty; 11 | import hudson.slaves.RetentionStrategy; 12 | 13 | import javax.annotation.Nonnull; 14 | import java.io.IOException; 15 | import java.util.List; 16 | 17 | public class EC2FleetNode extends Slave implements EphemeralNode, EC2FleetCloudAware { 18 | 19 | private volatile EC2FleetCloud cloud; 20 | 21 | public EC2FleetNode(final String name, final String nodeDescription, final String remoteFS, final int numExecutors, final Mode mode, final String label, 22 | final List> nodeProperties, final EC2FleetCloud cloud, ComputerLauncher launcher) throws IOException, Descriptor.FormException { 23 | super(name, nodeDescription, remoteFS, numExecutors, mode, label, 24 | launcher, RetentionStrategy.NOOP, nodeProperties); 25 | this.cloud = cloud; 26 | } 27 | 28 | @Override 29 | public Node asNode() { 30 | return this; 31 | } 32 | 33 | @Override 34 | public String getDisplayName() { 35 | // in some multi-thread edge cases cloud could be null for some time, just be ok with that 36 | return (cloud == null ? "unknown fleet" : cloud.getDisplayName()) + " " + name; 37 | } 38 | 39 | @Override 40 | public Computer createComputer() { 41 | return new EC2FleetNodeComputer(this, name, cloud); 42 | } 43 | 44 | @Override 45 | public EC2FleetCloud getCloud() { 46 | return cloud; 47 | } 48 | 49 | /** 50 | * {@inheritDoc} 51 | */ 52 | @Override 53 | public void setCloud(@Nonnull EC2FleetCloud cloud) { 54 | this.cloud = cloud; 55 | } 56 | 57 | @Extension 58 | public static final class DescriptorImpl extends SlaveDescriptor { 59 | public DescriptorImpl() { 60 | super(); 61 | } 62 | 63 | public String getDisplayName() { 64 | return "Fleet Slave"; 65 | } 66 | 67 | /** 68 | * We only create this kind of nodes programmatically. 69 | */ 70 | @Override 71 | public boolean isInstantiable() { 72 | return false; 73 | } 74 | } 75 | } 76 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/jenkins/ec2fleet/EC2FleetNodeComputer.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import hudson.model.Slave; 4 | import hudson.slaves.SlaveComputer; 5 | 6 | import javax.annotation.Nonnull; 7 | import javax.annotation.concurrent.ThreadSafe; 8 | 9 | /** 10 | * @see EC2FleetNode 11 | * @see EC2FleetAutoResubmitComputerLauncher 12 | */ 13 | @ThreadSafe 14 | public class EC2FleetNodeComputer extends SlaveComputer implements EC2FleetCloudAware { 15 | 16 | private final String name; 17 | 18 | private volatile EC2FleetCloud cloud; 19 | 20 | public EC2FleetNodeComputer(final Slave slave, @Nonnull final String name, @Nonnull final EC2FleetCloud cloud) { 21 | super(slave); 22 | this.name = name; 23 | this.cloud = cloud; 24 | } 25 | 26 | @Override 27 | public EC2FleetNode getNode() { 28 | return (EC2FleetNode) super.getNode(); 29 | } 30 | 31 | /** 32 | * Return label which will represent executor in "Build Executor Status" 33 | * section of Jenkins UI. 34 | * 35 | * @return node display name 36 | */ 37 | @Nonnull 38 | @Override 39 | public String getDisplayName() { 40 | // in some multi-thread edge cases cloud could be null for some time, just be ok with that 41 | return (cloud == null ? "unknown fleet" : cloud.getDisplayName()) + " " + name; 42 | } 43 | 44 | /** 45 | * {@inheritDoc} 46 | */ 47 | @Override 48 | public void setCloud(@Nonnull final EC2FleetCloud cloud) { 49 | this.cloud = cloud; 50 | } 51 | 52 | @Override 53 | public EC2FleetCloud getCloud() { 54 | return cloud; 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/jenkins/ec2fleet/EC2FleetOnlineChecker.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import com.google.common.util.concurrent.SettableFuture; 4 | import hudson.model.Computer; 5 | import hudson.model.Node; 6 | import hudson.util.DaemonThreadFactory; 7 | 8 | import javax.annotation.concurrent.ThreadSafe; 9 | import java.util.concurrent.Executors; 10 | import java.util.concurrent.ScheduledExecutorService; 11 | import java.util.concurrent.TimeUnit; 12 | import java.util.logging.Level; 13 | import java.util.logging.Logger; 14 | 15 | /** 16 | * Keep {@link hudson.slaves.NodeProvisioner.PlannedNode#future} not resolved until node will not be online 17 | * or timeout reached. 18 | *

19 | * Default Jenkins node capacity planner {@link hudson.slaves.NodeProvisioner.Strategy} count planned nodes 20 | * as available capacity, but exclude offline computers {@link Computer#isOnline()} from available capacity. 21 | * Because EC2 instance requires some time when it was added into fleet to start up, next situation happens: 22 | * plugin add described capacity as node into Jenkins pool, but Jenkins keeps it as offline as no way to connect, 23 | * during time when node is offline, Jenkins will try to request more nodes from plugin as offline nodes 24 | * excluded from capacity. 25 | *

26 | * This class fix this situation and keep planned node until instance is really online, so Jenkins planner 27 | * count planned node as available capacity and doesn't request more. 28 | *

29 | * Before each wait it will try to {@link Computer#connect(boolean)}, because by default Jenkins is trying to 30 | * make a few short interval reconnection initially (when EC2 instance still is not ready) after that 31 | * with big interval, experiment shows a few minutes and more. 32 | *

33 | * Based on https://github.com/jenkinsci/ec2-plugin/blob/master/src/main/java/hudson/plugins/ec2/EC2Cloud.java#L640 34 | * 35 | * @see EC2FleetCloud 36 | * @see EC2FleetNode 37 | */ 38 | @SuppressWarnings("WeakerAccess") 39 | @ThreadSafe 40 | class EC2FleetOnlineChecker implements Runnable { 41 | 42 | private static final Logger LOGGER = Logger.getLogger(EC2FleetOnlineChecker.class.getName()); 43 | // use daemon thread, so no problem when stop jenkins 44 | private static final ScheduledExecutorService EXECUTOR = Executors.newSingleThreadScheduledExecutor(new DaemonThreadFactory()); 45 | 46 | public static void start(final Node node, final SettableFuture future, final long timeout, final long interval) { 47 | EXECUTOR.execute(new EC2FleetOnlineChecker(node, future, timeout, interval)); 48 | } 49 | 50 | private final long start; 51 | private final Node node; 52 | private final SettableFuture future; 53 | private final long timeout; 54 | private final long interval; 55 | 56 | private EC2FleetOnlineChecker( 57 | final Node node, final SettableFuture future, final long timeout, final long interval) { 58 | this.start = System.currentTimeMillis(); 59 | this.node = node; 60 | this.future = future; 61 | this.timeout = timeout; 62 | this.interval = interval; 63 | } 64 | 65 | @Override 66 | public void run() { 67 | if (future.isCancelled()) { 68 | return; 69 | } 70 | 71 | if (timeout < 1 || interval < 1) { 72 | future.set(node); 73 | LOGGER.log(Level.INFO, String.format("%s connection check disabled, resolve planned node", node.getNodeName())); 74 | return; 75 | } 76 | 77 | final Computer computer = node.toComputer(); 78 | if (computer != null) { 79 | if (computer.isOnline()) { 80 | future.set(node); 81 | LOGGER.log(Level.INFO, String.format("%s connected, resolve planned node", node.getNodeName())); 82 | return; 83 | } 84 | } 85 | 86 | if (System.currentTimeMillis() - start > timeout) { 87 | future.setException(new IllegalStateException( 88 | "Fail to provision node, cannot connect to " + node.getNodeName() + " in " + timeout + " msec")); 89 | return; 90 | } 91 | 92 | if (computer == null) { 93 | LOGGER.log(Level.INFO, String.format("%s no connection, wait before retry", node.getNodeName())); 94 | } else { 95 | computer.connect(false); 96 | LOGGER.log(Level.INFO, String.format("%s no connection, connect and wait before retry", node.getNodeName())); 97 | } 98 | EXECUTOR.schedule(this, interval, TimeUnit.MILLISECONDS); 99 | } 100 | 101 | } 102 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/jenkins/ec2fleet/EC2FleetStatusInfo.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import hudson.widgets.Widget; 4 | 5 | import javax.annotation.concurrent.ThreadSafe; 6 | import java.util.Objects; 7 | 8 | /** 9 | * This consumed by jelly file EC2FleetStatusWidget/index.jelly 10 | * to render fleet information about all fleets, don't forget to update it 11 | * if you change fields name 12 | * 13 | * @see EC2FleetStatusWidget 14 | * @see CloudNanny 15 | */ 16 | @SuppressWarnings({"WeakerAccess", "unused"}) 17 | @ThreadSafe 18 | public class EC2FleetStatusInfo extends Widget { 19 | 20 | private final String id; 21 | private final String state; 22 | private final String label; 23 | private final int numActive; 24 | private final int numDesired; 25 | 26 | public EC2FleetStatusInfo(String id, String state, String label, int numActive, int numDesired) { 27 | this.id = id; 28 | this.state = state; 29 | this.label = label; 30 | this.numActive = numActive; 31 | this.numDesired = numDesired; 32 | } 33 | 34 | public String getId() { 35 | return id; 36 | } 37 | 38 | @Override 39 | public boolean equals(Object o) { 40 | if (this == o) return true; 41 | if (o == null || getClass() != o.getClass()) return false; 42 | EC2FleetStatusInfo that = (EC2FleetStatusInfo) o; 43 | return numActive == that.numActive && 44 | numDesired == that.numDesired && 45 | Objects.equals(id, that.id) && 46 | Objects.equals(state, that.state) && 47 | Objects.equals(label, that.label); 48 | } 49 | 50 | @Override 51 | public int hashCode() { 52 | return Objects.hash(id, state, label, numActive, numDesired); 53 | } 54 | 55 | public String getLabel() { 56 | return label; 57 | } 58 | 59 | public String getState() { 60 | return state; 61 | } 62 | 63 | public int getNumActive() { 64 | return numActive; 65 | } 66 | 67 | public int getNumDesired() { 68 | return numDesired; 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/jenkins/ec2fleet/EC2FleetStatusWidget.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import hudson.Extension; 4 | import hudson.widgets.Widget; 5 | 6 | import javax.annotation.concurrent.ThreadSafe; 7 | import java.util.Collections; 8 | import java.util.List; 9 | 10 | /** 11 | * This class should be thread safe, consumed by Jenkins and updated 12 | * by {@link CloudNanny} 13 | */ 14 | @Extension 15 | @ThreadSafe 16 | public class EC2FleetStatusWidget extends Widget { 17 | 18 | private volatile List statusList = Collections.emptyList(); 19 | 20 | public void setStatusList(final List statusList) { 21 | this.statusList = statusList; 22 | } 23 | 24 | @SuppressWarnings("unused") 25 | public List getStatusList() { 26 | return statusList; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/jenkins/ec2fleet/EC2TerminationCause.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import jenkins.model.CauseOfInterruption; 4 | 5 | import javax.annotation.Nonnull; 6 | 7 | public class EC2TerminationCause extends CauseOfInterruption { 8 | 9 | @Nonnull 10 | private final String nodeName; 11 | 12 | @SuppressWarnings("WeakerAccess") 13 | public EC2TerminationCause(@Nonnull String nodeName) { 14 | this.nodeName = nodeName; 15 | } 16 | 17 | @Override 18 | public String getShortDescription() { 19 | return "EC2 instance for node " + nodeName + " was terminated"; 20 | } 21 | 22 | @Override 23 | public boolean equals(Object o) { 24 | if (o == null || getClass() != o.getClass()) return false; 25 | EC2TerminationCause that = (EC2TerminationCause) o; 26 | return nodeName.equals(that.nodeName); 27 | } 28 | 29 | @Override 30 | public int hashCode() { 31 | return nodeName.hashCode(); 32 | } 33 | 34 | @Override 35 | public String toString() { 36 | return getShortDescription(); 37 | } 38 | 39 | } 40 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/jenkins/ec2fleet/FleetStateStats.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import com.amazonaws.services.ec2.AmazonEC2; 4 | import com.amazonaws.services.ec2.model.ActiveInstance; 5 | import com.amazonaws.services.ec2.model.DescribeSpotFleetInstancesRequest; 6 | import com.amazonaws.services.ec2.model.DescribeSpotFleetInstancesResult; 7 | import com.amazonaws.services.ec2.model.DescribeSpotFleetRequestsRequest; 8 | import com.amazonaws.services.ec2.model.DescribeSpotFleetRequestsResult; 9 | import com.amazonaws.services.ec2.model.SpotFleetLaunchSpecification; 10 | import com.amazonaws.services.ec2.model.SpotFleetRequestConfig; 11 | import com.amazonaws.services.ec2.model.SpotFleetRequestConfigData; 12 | 13 | import javax.annotation.Nonnegative; 14 | import javax.annotation.Nonnull; 15 | import javax.annotation.concurrent.ThreadSafe; 16 | import java.util.Collections; 17 | import java.util.HashMap; 18 | import java.util.HashSet; 19 | import java.util.Map; 20 | import java.util.Set; 21 | 22 | /** 23 | * @see EC2FleetCloud 24 | */ 25 | @SuppressWarnings({"unused", "WeakerAccess"}) 26 | @ThreadSafe 27 | public final class FleetStateStats { 28 | 29 | @Nonnull 30 | private final String fleetId; 31 | @Nonnegative 32 | private final int numActive; 33 | @Nonnegative 34 | private final int numDesired; 35 | @Nonnull 36 | private final String state; 37 | @Nonnull 38 | private final Set instances; 39 | @Nonnull 40 | private final Map instanceTypeWeights; 41 | 42 | public FleetStateStats(final @Nonnull String fleetId, 43 | final int numDesired, final @Nonnull String state, 44 | final @Nonnull Set instances, 45 | final @Nonnull Map instanceTypeWeights) { 46 | this.fleetId = fleetId; 47 | this.numActive = instances.size(); 48 | this.numDesired = numDesired; 49 | this.state = state; 50 | this.instances = instances; 51 | this.instanceTypeWeights = instanceTypeWeights; 52 | } 53 | 54 | @Nonnull 55 | public String getFleetId() { 56 | return fleetId; 57 | } 58 | 59 | public int getNumActive() { 60 | return numActive; 61 | } 62 | 63 | public int getNumDesired() { 64 | return numDesired; 65 | } 66 | 67 | @Nonnull 68 | public String getState() { 69 | return state; 70 | } 71 | 72 | @Nonnull 73 | public Set getInstances() { 74 | return instances; 75 | } 76 | 77 | @Nonnull 78 | public Map getInstanceTypeWeights() { 79 | return instanceTypeWeights; 80 | } 81 | 82 | public static FleetStateStats readClusterState(final AmazonEC2 ec2, final String fleetId, final String label) { 83 | String token = null; 84 | final Set instances = new HashSet<>(); 85 | do { 86 | final DescribeSpotFleetInstancesRequest request = new DescribeSpotFleetInstancesRequest(); 87 | request.setSpotFleetRequestId(fleetId); 88 | request.setNextToken(token); 89 | final DescribeSpotFleetInstancesResult res = ec2.describeSpotFleetInstances(request); 90 | for (final ActiveInstance instance : res.getActiveInstances()) { 91 | instances.add(instance.getInstanceId()); 92 | } 93 | 94 | token = res.getNextToken(); 95 | } while (token != null); 96 | 97 | final DescribeSpotFleetRequestsRequest request = new DescribeSpotFleetRequestsRequest(); 98 | request.setSpotFleetRequestIds(Collections.singleton(fleetId)); 99 | final DescribeSpotFleetRequestsResult fleet = ec2.describeSpotFleetRequests(request); 100 | if (fleet.getSpotFleetRequestConfigs().isEmpty()) 101 | throw new IllegalStateException("Fleet " + fleetId + " can't be described"); 102 | 103 | final SpotFleetRequestConfig fleetConfig = fleet.getSpotFleetRequestConfigs().get(0); 104 | final SpotFleetRequestConfigData fleetRequestConfig = fleetConfig.getSpotFleetRequestConfig(); 105 | 106 | // Index configured instance types by weight: 107 | final Map instanceTypeWeights = new HashMap<>(); 108 | for (SpotFleetLaunchSpecification launchSpecification : fleetRequestConfig.getLaunchSpecifications()) { 109 | final String instanceType = launchSpecification.getInstanceType(); 110 | if (instanceType == null) continue; 111 | 112 | final Double instanceWeight = launchSpecification.getWeightedCapacity(); 113 | final Double existingWeight = instanceTypeWeights.get(instanceType); 114 | if (instanceWeight == null || (existingWeight != null && existingWeight > instanceWeight)) { 115 | continue; 116 | } 117 | instanceTypeWeights.put(instanceType, instanceWeight); 118 | } 119 | 120 | return new FleetStateStats(fleetId, 121 | fleetRequestConfig.getTargetCapacity(), 122 | fleetConfig.getSpotFleetRequestState(), instances, 123 | instanceTypeWeights); 124 | } 125 | } 126 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/jenkins/ec2fleet/IdleRetentionStrategy.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import hudson.model.Computer; 4 | import hudson.model.Node; 5 | import hudson.slaves.RetentionStrategy; 6 | import hudson.slaves.SlaveComputer; 7 | 8 | import java.util.concurrent.TimeUnit; 9 | import java.util.logging.Level; 10 | import java.util.logging.Logger; 11 | 12 | /** 13 | * @see EC2FleetCloud 14 | */ 15 | public class IdleRetentionStrategy extends RetentionStrategy { 16 | 17 | private static final int RE_CHECK_IN_MINUTE = 1; 18 | 19 | private static final Logger LOGGER = Logger.getLogger(IdleRetentionStrategy.class.getName()); 20 | 21 | /** 22 | * Will be called under {@link hudson.model.Queue#withLock(Runnable)} 23 | * 24 | * @param computer computer 25 | * @return delay in min before next run 26 | */ 27 | @Override 28 | public long check(final SlaveComputer computer) { 29 | final EC2FleetNodeComputer fc = (EC2FleetNodeComputer) computer; 30 | final EC2FleetCloud cloud = fc.getCloud(); 31 | 32 | LOGGER.log(Level.INFO, "Check if node idle " + computer.getName()); 33 | 34 | // in some multi-thread edge cases cloud could be null for some time, just be ok with that 35 | if (cloud == null) { 36 | LOGGER.warning("Edge case cloud is null for computer " + fc.getDisplayName() 37 | + " should be autofixed in a few minutes, if no please create issue for plugin"); 38 | return RE_CHECK_IN_MINUTE; 39 | } 40 | 41 | // Ensure that the EC2FleetCloud cannot be mutated from under us while 42 | // we're doing this check 43 | // Ensure nobody provisions onto this node until we've done 44 | // checking 45 | boolean shouldAcceptTasks = fc.isAcceptingTasks(); 46 | boolean justTerminated = false; 47 | fc.setAcceptingTasks(false); 48 | try { 49 | if (fc.isIdle() && isIdleForTooLong(cloud, fc)) { 50 | // Find instance ID 51 | Node compNode = fc.getNode(); 52 | if (compNode == null) { 53 | return 0; 54 | } 55 | 56 | final String instanceId = compNode.getNodeName(); 57 | if (cloud.scheduleToTerminate(instanceId)) { 58 | // Instance successfully terminated, so no longer accept tasks 59 | shouldAcceptTasks = false; 60 | justTerminated = true; 61 | } 62 | } 63 | 64 | if (cloud.isAlwaysReconnect() && !justTerminated && fc.isOffline() && !fc.isConnecting() && fc.isLaunchSupported()) { 65 | LOGGER.log(Level.INFO, "Reconnecting to instance: " + fc.getDisplayName()); 66 | fc.tryReconnect(); 67 | } 68 | } finally { 69 | fc.setAcceptingTasks(shouldAcceptTasks); 70 | } 71 | 72 | return RE_CHECK_IN_MINUTE; 73 | } 74 | 75 | @Override 76 | public void start(SlaveComputer c) { 77 | LOGGER.log(Level.INFO, "Connecting to instance: " + c.getDisplayName()); 78 | c.connect(false); 79 | } 80 | 81 | private boolean isIdleForTooLong(final EC2FleetCloud cloud, final Computer computer) { 82 | final int idleMinutes = cloud.getIdleMinutes(); 83 | if (idleMinutes <= 0) return false; 84 | final long idleTime = System.currentTimeMillis() - computer.getIdleStartMilliseconds(); 85 | final long maxIdle = TimeUnit.MINUTES.toMillis(idleMinutes); 86 | LOGGER.log(Level.INFO, "Instance: " + computer.getDisplayName() + " Age: " + idleTime + " Max Age:" + maxIdle); 87 | return idleTime > maxIdle; 88 | } 89 | } 90 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/jenkins/ec2fleet/LazyUuid.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import javax.annotation.concurrent.ThreadSafe; 4 | import java.util.UUID; 5 | 6 | /** 7 | * Provide uuid string, create it when first call happens 8 | */ 9 | @ThreadSafe 10 | @SuppressWarnings("WeakerAccess") 11 | public class LazyUuid { 12 | 13 | private String value; 14 | 15 | public synchronized String getValue() { 16 | if (value == null) { 17 | value = UUID.randomUUID().toString(); 18 | } 19 | return value; 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /src/main/java/com/amazon/jenkins/ec2fleet/NoDelayProvisionStrategy.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import com.google.common.annotations.VisibleForTesting; 4 | import hudson.Extension; 5 | import hudson.model.Label; 6 | import hudson.model.LoadStatistics; 7 | import hudson.slaves.Cloud; 8 | import hudson.slaves.NodeProvisioner; 9 | import jenkins.model.Jenkins; 10 | 11 | import java.util.Collection; 12 | import java.util.Collections; 13 | import java.util.List; 14 | import java.util.logging.Level; 15 | import java.util.logging.Logger; 16 | 17 | /** 18 | * Implementation of {@link NodeProvisioner.Strategy} which will provision a new node immediately as 19 | * a task enter the queue. 20 | * Now that EC2 is billed by the minute, we don't really need to wait before provisioning a new node. 21 | *

22 | * As based we are used 23 | * EC2 Jenkins Plugin 24 | */ 25 | @Extension(ordinal = 100) 26 | public class NoDelayProvisionStrategy extends NodeProvisioner.Strategy { 27 | 28 | private static final Logger LOGGER = Logger.getLogger(NoDelayProvisionStrategy.class.getName()); 29 | 30 | @Override 31 | public NodeProvisioner.StrategyDecision apply(final NodeProvisioner.StrategyState strategyState) { 32 | final Label label = strategyState.getLabel(); 33 | 34 | final LoadStatistics.LoadStatisticsSnapshot snapshot = strategyState.getSnapshot(); 35 | final int availableCapacity = 36 | snapshot.getAvailableExecutors() // live executors 37 | + snapshot.getConnectingExecutors() // executors present but not yet connected 38 | + strategyState.getPlannedCapacitySnapshot() // capacity added by previous strategies from previous rounds 39 | + strategyState.getAdditionalPlannedCapacity(); // capacity added by previous strategies _this round_ 40 | 41 | int currentDemand = snapshot.getQueueLength() - availableCapacity; 42 | LOGGER.log(Level.INFO, "Available capacity={0}, currentDemand={1}", 43 | new Object[]{availableCapacity, currentDemand}); 44 | 45 | for (final Cloud cloud : getClouds()) { 46 | if (currentDemand < 1) break; 47 | 48 | if (!(cloud instanceof EC2FleetCloud)) continue; 49 | if (!cloud.canProvision(label)) continue; 50 | 51 | final EC2FleetCloud ec2 = (EC2FleetCloud) cloud; 52 | if (!ec2.isNoDelayProvision()) continue; 53 | 54 | final Collection plannedNodes = cloud.provision(label, currentDemand); 55 | currentDemand -= plannedNodes.size(); 56 | LOGGER.log(Level.FINE, "Planned {0} new nodes", plannedNodes.size()); 57 | strategyState.recordPendingLaunches(plannedNodes); 58 | LOGGER.log(Level.FINE, "After provisioning, available capacity={0}, currentDemand={1}", 59 | new Object[]{availableCapacity, currentDemand}); 60 | } 61 | 62 | if (currentDemand < 1) { 63 | LOGGER.log(Level.FINE, "Provisioning completed"); 64 | return NodeProvisioner.StrategyDecision.PROVISIONING_COMPLETED; 65 | } else { 66 | LOGGER.log(Level.FINE, "Provisioning not complete, consulting remaining strategies"); 67 | return NodeProvisioner.StrategyDecision.CONSULT_REMAINING_STRATEGIES; 68 | } 69 | } 70 | 71 | @VisibleForTesting 72 | protected List getClouds() { 73 | final Jenkins jenkins = Jenkins.getInstance(); 74 | return jenkins == null ? Collections.emptyList() : jenkins.clouds; 75 | } 76 | 77 | } -------------------------------------------------------------------------------- /src/main/java/com/amazon/jenkins/ec2fleet/Registry.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | /** 4 | * Decouple plugin code from dependencies for easy testing. We cannot just make transient fields 5 | * in required classes as they usually restored by Jenkins without constructor call. Instead 6 | * we are using registry pattern. 7 | * 8 | * @see EC2FleetCloud 9 | */ 10 | @SuppressWarnings("WeakerAccess") 11 | public class Registry { 12 | 13 | private static EC2Api ec2Api = new EC2Api(); 14 | 15 | public static void setEc2Api(EC2Api ec2Api) { 16 | Registry.ec2Api = ec2Api; 17 | } 18 | 19 | public static EC2Api getEc2Api() { 20 | return ec2Api; 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /src/main/resources/Message.properties: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/awslabs/ec2-spot-jenkins-plugin/9cd2668cb772eb7f28a419512205e4f9e763b2f1/src/main/resources/Message.properties -------------------------------------------------------------------------------- /src/main/resources/com/amazon/jenkins/ec2fleet/EC2FleetCloud/config.jelly: -------------------------------------------------------------------------------- 1 | 2 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | Select China region for China credentials. 23 | 24 | 25 | 26 | 27 | Endpoint like https://ec2.us-east-2.amazonaws.com 28 | 29 | 30 | 31 | 32 | Fleet list will be available once region and credentials are specified. Only maintain supported, see help 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | Connect using private IP 48 | 49 | 50 | 51 | 52 | Always reconnect to offline nodes 53 | 54 | 55 | 56 | 57 | Only build jobs with label expressions matching this node 58 | 59 | 60 | 61 | 62 | 63 | Labels to add to instances in this fleet 64 | 65 | 66 | 67 | 68 | Default is /tmp/jenkins-<random ID> 69 | 70 | 71 | 72 | 73 | Number of executors per instance 74 | 75 | 76 | 77 | 78 | 79 | 80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 | 90 | 91 | 92 | 93 | 94 | Experimental Add EC2 instance to slaves only when state is running 95 | 96 | 97 | 98 | 99 | Disable auto resubmit build if failed because of EC2 instance termination like Spot 100 | 101 | 102 | 103 | 104 | Maximum time to wait for EC2 instance startup 105 | 106 | 107 | 108 | 109 | Interval for updating EC2 cloud status 110 | 111 | 112 | 113 | 114 | Enable faster provision when queue grow 115 | 116 | 117 | 118 | 119 | 120 | 121 | -------------------------------------------------------------------------------- /src/main/resources/com/amazon/jenkins/ec2fleet/EC2FleetCloud/help-addNodeOnlyIfRunning.html: -------------------------------------------------------------------------------- 1 |

2 |

3 | Regulate when plugin should add new EC2 Instances described in EC2 Spot Fleet into Jenkins as slave. 4 |

5 | 6 | When unchecked. 7 |

Default behavior. Old versions of plugin use this approach.

8 |

Plugin will immediately add new EC2 Instances described in EC2 Spot Fleet into Jenkins slaves. 9 | Plugin will not check state 10 | of EC2 Instance

11 |

It takes some time to EC2 Instance to run and be able accept connections, as result Jenkins will move 12 | just added slaves to offline and will try to reconnect later. At the same time Jenkins will 13 | request more capacity from EC2 Fleet Plugin. This could over-provision your EC2 Spot Fleet, but guarantee 14 | that if new EC2 Instance has some problem to start, Jenkins will not be stuck and request move capacity, 15 | new instances. 16 |

17 | 18 | When checked 19 |

20 | This option is experimental, if you have questions/problems, please report 21 | GitHub Issue 22 |

23 |

24 | Plugin will add new EC2 Spot Instances to Jenkins slaves only when EC2 Instance will be in running 25 | state. 26 |

27 |

28 | As result Jenkins will not request over-provision capacity. Make sure that you have 29 | enabled health check 30 | for EC2 Spot Fleet to avoid case when EC2 Instance cannot be running but Jenkins wait it 31 | and doesn't request additional capacity. 32 |

33 |
-------------------------------------------------------------------------------- /src/main/resources/com/amazon/jenkins/ec2fleet/EC2FleetCloud/help-cloudStatusIntervalSec.html: -------------------------------------------------------------------------------- 1 |
2 | Set the interval for checking the EC2 clouds status. 3 |

4 | Every check fetches the current state of the cloud from EC2 and scales the size of the cloud up or down. 5 |

6 |

7 | Longer intervals will reduce the number of EC2 API calls, shorter intervals will allow the cloud to faster scale up and down. 8 |

9 |

10 | The default is to check every 10 seconds. 11 |

12 |
13 | -------------------------------------------------------------------------------- /src/main/resources/com/amazon/jenkins/ec2fleet/EC2FleetCloud/help-disableTaskResubmit.html: -------------------------------------------------------------------------------- 1 | If unchecked which is default, plugin will try to resubmit job if it was failed due to 2 | EC2 Spot instance interruption. If checked auto resubmit 3 | will be disabled. 4 | 5 |

6 | Interrupted job will be shown as failed or aborted see pipeline notes below. 7 | While copy will be submitted into jenkins queue to execute or any possible node. 8 |

9 | 10 |

11 | Parametrized job will be resubmitted with same parameters as defined for interrupted. 12 |

13 | 14 |

15 | In case if job is pipeline (workflow) interrupted 16 | task will be aborted as workflow usually don't fail in case of node interruption and wait until 17 | node will be online. Plugin force job to abort and resubmit. 18 |

19 | 20 |

21 | Please report any problem with this feature to GitHub Issue 22 |

23 | -------------------------------------------------------------------------------- /src/main/resources/com/amazon/jenkins/ec2fleet/EC2FleetCloud/help-endpoint.html: -------------------------------------------------------------------------------- 1 | Optional field, only required if you cannot find proper region in regions list. 2 |

3 | Please specify endpoint for service EC2, example: 4 |

5 | 6 | https://ec2.cn-northwest-1.amazonaws.com.cn
7 | https://ec2.ap-northeast-1.amazonaws.com 8 | 9 |

10 |

11 | List of all available endpoints for EC2 you can find: 12 | Amazon AWS documentation 13 |

-------------------------------------------------------------------------------- /src/main/resources/com/amazon/jenkins/ec2fleet/EC2FleetCloud/help-fleet.html: -------------------------------------------------------------------------------- 1 | Select fleet which will be used to launch Jenkins workers. 2 |

3 | By default this list include only active 4 | fleets. If you need to see all fleet include cancelled etc. click checkbox below. 5 | Selected fleet will be in list regardless of fleet state. 6 |

7 |

8 | All non maintain fleets will be filter out from this list. 9 | Plugin requires stable capacity. 10 | More 11 | about 12 | types 13 | 14 | If you are not sure about your fleet check its type by API or 15 | check if Maintain target capacity enabled in UI for this spot fleet. 16 |

17 | -------------------------------------------------------------------------------- /src/main/resources/com/amazon/jenkins/ec2fleet/EC2FleetCloud/help-idleMinutes.html: -------------------------------------------------------------------------------- 1 | 0 for no scaledown 2 | -------------------------------------------------------------------------------- /src/main/resources/com/amazon/jenkins/ec2fleet/EC2FleetCloud/help-initOnlineTimeoutSec.html: -------------------------------------------------------------------------------- 1 | Specify maximum time which Jenkins will wait for EC2 instance startup before 2 | trying to request new instance from fleet. 3 | 4 |

5 | By default Jenkins expect that node will be available immediately, however EC2 instance 6 | requires some time to be provision, start and be ready for actual builds. In result 7 | Jenkins might request more capacity then actually required (over-provision). This setting 8 | avoids over-provision by force Jenkins to wait until EC2 instance will be online and 9 | don't request more capacity. 10 |

11 | 12 |

13 | In case of positive value, will force Jenkins to wait when EC2 instance will be online, 14 | no more then specified time in seconds. Cannot be negative. 15 | In case of 0, timeout will be disabled and Jenkins will 16 | not wait any time to start up EC2 instance, which could lead to over-provision. 17 |

18 |

19 | Note Be careful with very big values (more 5 min) for this settings. In case of any problem 20 | with node (no java, wrong version of java, Jenkins agent cannot start), so it cannot be online. 21 | Jenkins will wait, without requesting additional capacity. 22 |

23 |

24 | Behavior of previous versions of plugin could be represented as this version 25 | with timeout set to 0. 26 |

-------------------------------------------------------------------------------- /src/main/resources/com/amazon/jenkins/ec2fleet/EC2FleetCloud/help-name.html: -------------------------------------------------------------------------------- 1 |
2 | Provide a name for this EC2 Fleet Cloud 3 | 4 |

5 | Could be used in Groovy as cloud identifier. 6 |

7 | 8 |

9 | Note Due to Jenkins organization. 10 | Build Executor Status section in Jenkins UI could show old cloud name 11 | for slaves, to fix that just wait when slaves will be recreated (could be long) 12 | or delete slave (will not delete underline EC2 instances). 13 | It will request plugin to recreate slaves with correct cloud name. 14 |

15 |
-------------------------------------------------------------------------------- /src/main/resources/com/amazon/jenkins/ec2fleet/EC2FleetCloud/help-noDelayProvision.html: -------------------------------------------------------------------------------- 1 | Enable no delay provision strategy. 2 |

3 | Disabled by default. 4 |

5 |

6 | Enabling of this setting will disable default Jenkins behavior for particular cloud 7 | and ask this cloud to provision new agents almost as soon as new jobs appear in queue. 8 |

9 |

10 | When this option is disable (by default). 11 | Each time when Jenkins doesn't have enough agent to execute all jobs in a queue. 12 | Jenkins starts provision of new agents. Default strategy is trying to keep amount of agents 13 | as small as possible. It waits a little bit until queue will not reach some limit or 14 | jobs in a queue will not reach some age. As result before scheduled job will be executed you can 15 | see some delay. 16 |

17 |

18 | Note Enabling of this setting could increase chance of over provision, because of that 19 | make sure that your Max Idle Minutes Before Scaledown has expected value so 20 | free capacity could be scale in. While Linux capacity billing per minute, Windows capacity 21 | billing per hour, so be careful with this option. 22 |

23 | -------------------------------------------------------------------------------- /src/main/resources/com/amazon/jenkins/ec2fleet/EC2FleetCloud/help-restrictUsage.html: -------------------------------------------------------------------------------- 1 |
2 |

Controls how Jenkins schedules builds on provisioned nodes.

3 | When unchecked 4 |

5 | Jenkins will use provisioned nodes as much as possible. 6 | This is the default setting. In this mode, Jenkins uses provisioned nodes freely. 7 | Whenever there is a build that can be done by using provisioned node, Jenkins will use it. 8 |

9 | 10 | When checked 11 |

12 | In this mode, Jenkins will only build a project on provisioned node when that project is restricted to certain nodes 13 | using a label expression, and that expression matches this node's labels. 14 |

15 |

16 | This allows a node to be reserved for certain kinds of jobs. For example, if you have jobs that run performance 17 | tests, you may want them to only run on a specially configured machine, while preventing all other jobs from 18 | using that machine. To do so, you would restrict where the test jobs may run by giving them a label expression 19 | matching that machine. 20 |

21 |

22 | Furthermore, if you set the # of executors value to 1, you can ensure that only one performance test will 23 | execute at any given time on that machine; no other builds will interfere. 24 |

25 |
26 | -------------------------------------------------------------------------------- /src/main/resources/com/amazon/jenkins/ec2fleet/EC2FleetCloud/help-scaleExecutorsByWeight.html: -------------------------------------------------------------------------------- 1 | If unchecked which is default, plugin will not use 2 | instance 3 | weight 4 | information to scale number of executors per node, and just set number of executors defined in 5 | configuration field Number of Executors 6 | 7 |

8 | When it's checked, plugin consumes instance weight information provided by Launch Specification 9 | and uses it to scale node number of executors from configuration field Number of Executors 10 | Note current implementation doesn't support Launch Template only Launch Specification 11 |

12 | 13 |

14 | Example (here instance type from launch specification match with 15 | launched instance type): 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 |
Number of ExecutorsInstance WeightEffective
Number of Executors
111
10.51
10.11
100.11
11.52
11.441
58 |

59 | 60 |

61 | Plugin always set number of executors at least one. 62 | If launched instance type doesn't match any weight in launch specification 63 | regular number of executors will be used without any scale. 64 |

65 | -------------------------------------------------------------------------------- /src/main/resources/com/amazon/jenkins/ec2fleet/EC2FleetNode/configure-entries.jelly: -------------------------------------------------------------------------------- 1 | 2 | 3 | EC2 Spot Fleet Instance 4 | 5 | -------------------------------------------------------------------------------- /src/main/resources/com/amazon/jenkins/ec2fleet/EC2FleetStatusWidget/index.jelly: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 |
  • ${fleet.id}
    • 8 |
    State: ${fleet.state}, 9 | label: "${fleet.label}", nodes: ${fleet.numActive}, target: ${fleet.numDesired} 10 |
    11 |
    12 |     13 | 14 | 15 |     16 |     17 | -------------------------------------------------------------------------------- /src/main/resources/index.jelly: -------------------------------------------------------------------------------- 1 | 2 |
    3 | Use EC2 SpotFleet to launch builders 4 |
    5 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/jenkins/ec2fleet/AutoResubmitIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import com.amazonaws.services.ec2.AmazonEC2; 4 | import com.amazonaws.services.ec2.model.ActiveInstance; 5 | import com.amazonaws.services.ec2.model.DescribeInstancesRequest; 6 | import com.amazonaws.services.ec2.model.DescribeInstancesResult; 7 | import com.amazonaws.services.ec2.model.DescribeSpotFleetInstancesRequest; 8 | import com.amazonaws.services.ec2.model.DescribeSpotFleetInstancesResult; 9 | import com.amazonaws.services.ec2.model.DescribeSpotFleetRequestsRequest; 10 | import com.amazonaws.services.ec2.model.DescribeSpotFleetRequestsResult; 11 | import com.amazonaws.services.ec2.model.Instance; 12 | import com.amazonaws.services.ec2.model.InstanceState; 13 | import com.amazonaws.services.ec2.model.InstanceStateName; 14 | import com.amazonaws.services.ec2.model.Reservation; 15 | import com.amazonaws.services.ec2.model.SpotFleetRequestConfig; 16 | import com.amazonaws.services.ec2.model.SpotFleetRequestConfigData; 17 | import hudson.Functions; 18 | import hudson.model.FreeStyleProject; 19 | import hudson.model.Node; 20 | import hudson.model.ParametersAction; 21 | import hudson.model.ParametersDefinitionProperty; 22 | import hudson.model.Result; 23 | import hudson.model.StringParameterDefinition; 24 | import hudson.model.StringParameterValue; 25 | import hudson.model.labels.LabelAtom; 26 | import hudson.model.queue.QueueTaskFuture; 27 | import hudson.slaves.OfflineCause; 28 | import hudson.tasks.BatchFile; 29 | import hudson.tasks.Shell; 30 | import org.junit.Assert; 31 | import org.junit.Before; 32 | import org.junit.Test; 33 | import org.mockito.Mockito; 34 | 35 | import java.util.ArrayList; 36 | import java.util.Arrays; 37 | import java.util.List; 38 | 39 | import static org.mockito.ArgumentMatchers.any; 40 | import static org.mockito.ArgumentMatchers.anyString; 41 | import static org.mockito.Mockito.mock; 42 | import static org.mockito.Mockito.spy; 43 | import static org.mockito.Mockito.when; 44 | 45 | @SuppressWarnings({"ArraysAsListWithZeroOrOneArgument", "deprecation"}) 46 | public class AutoResubmitIntegrationTest extends IntegrationTest { 47 | 48 | @Before 49 | public void before() { 50 | EC2Api ec2Api = spy(EC2Api.class); 51 | Registry.setEc2Api(ec2Api); 52 | 53 | AmazonEC2 amazonEC2 = mock(AmazonEC2.class); 54 | when(ec2Api.connect(anyString(), anyString(), Mockito.nullable(String.class))).thenReturn(amazonEC2); 55 | 56 | final Instance instance = new Instance() 57 | .withState(new InstanceState().withName(InstanceStateName.Running)) 58 | .withPublicIpAddress("public-io") 59 | .withInstanceId("i-1"); 60 | 61 | when(amazonEC2.describeInstances(any(DescribeInstancesRequest.class))).thenReturn( 62 | new DescribeInstancesResult().withReservations( 63 | new Reservation().withInstances( 64 | instance 65 | ))); 66 | 67 | when(amazonEC2.describeSpotFleetInstances(any(DescribeSpotFleetInstancesRequest.class))) 68 | .thenReturn(new DescribeSpotFleetInstancesResult() 69 | .withActiveInstances(new ActiveInstance().withInstanceId("i-1"))); 70 | 71 | DescribeSpotFleetRequestsResult describeSpotFleetRequestsResult = new DescribeSpotFleetRequestsResult(); 72 | describeSpotFleetRequestsResult.setSpotFleetRequestConfigs(Arrays.asList( 73 | new SpotFleetRequestConfig() 74 | .withSpotFleetRequestState("active") 75 | .withSpotFleetRequestConfig( 76 | new SpotFleetRequestConfigData().withTargetCapacity(1)))); 77 | when(amazonEC2.describeSpotFleetRequests(any(DescribeSpotFleetRequestsRequest.class))) 78 | .thenReturn(describeSpotFleetRequestsResult); 79 | } 80 | 81 | @Test 82 | public void should_successfully_resubmit_freestyle_task() throws Exception { 83 | EC2FleetCloud cloud = new EC2FleetCloud(null, null, "credId", null, "region", 84 | null, "fId", "momo", null, new LocalComputerConnector(j), false, false, 85 | 0, 0, 10, 1, false, false, 86 | false, 0, 0, false, 87 | 10, false); 88 | j.jenkins.clouds.add(cloud); 89 | 90 | List rs = getQueueTaskFutures(1); 91 | 92 | System.out.println("check if zero nodes!"); 93 | Assert.assertEquals(0, j.jenkins.getNodes().size()); 94 | 95 | assertAtLeastOneNode(); 96 | 97 | final Node node = j.jenkins.getNodes().get(0); 98 | assertQueueIsEmpty(); 99 | 100 | System.out.println("disconnect node"); 101 | node.toComputer().disconnect(new OfflineCause.ChannelTermination(new UnsupportedOperationException("Test"))); 102 | 103 | // due to test nature job could be failed if started or aborted as we call disconnect 104 | // in prod code it's not matter 105 | assertLastBuildResult(Result.FAILURE, Result.ABORTED); 106 | 107 | node.toComputer().connect(true); 108 | assertNodeIsOnline(node); 109 | assertQueueAndNodesIdle(node); 110 | 111 | Assert.assertEquals(1, j.jenkins.getProjects().size()); 112 | Assert.assertEquals(Result.SUCCESS, j.jenkins.getProjects().get(0).getLastBuild().getResult()); 113 | Assert.assertEquals(2, j.jenkins.getProjects().get(0).getBuilds().size()); 114 | 115 | cancelTasks(rs); 116 | } 117 | 118 | @Test 119 | public void should_successfully_resubmit_parametrized_task() throws Exception { 120 | EC2FleetCloud cloud = new EC2FleetCloud(null, null, "credId", null, "region", 121 | null, "fId", "momo", null, new LocalComputerConnector(j), false, false, 122 | 0, 0, 10, 1, false, false, 123 | false, 0, 0, false, 124 | 10, false); 125 | j.jenkins.clouds.add(cloud); 126 | 127 | List rs = new ArrayList<>(); 128 | final FreeStyleProject project = j.createFreeStyleProject(); 129 | project.setAssignedLabel(new LabelAtom("momo")); 130 | project.addProperty(new ParametersDefinitionProperty(new StringParameterDefinition("number", "opa"))); 131 | /* 132 | example of actions for project 133 | 134 | actions = {CopyOnWriteArrayList@14845} size = 2 135 | 0 = {ParametersAction@14853} 136 | safeParameters = {TreeSet@14855} size = 0 137 | parameters = {ArrayList@14856} size = 1 138 | 0 = {StringParameterValue@14862} "(StringParameterValue) number='1'" 139 | value = "1" 140 | name = "number" 141 | description = "" 142 | parameterDefinitionNames = {ArrayList@14857} size = 1 143 | 0 = "number" 144 | build = null 145 | run = {FreeStyleBuild@14834} "parameter #14" 146 | */ 147 | project.getBuildersList().add(Functions.isWindows() ? new BatchFile("Ping -n %number% 127.0.0.1 > nul") : new Shell("sleep ${number}")); 148 | 149 | rs.add(project.scheduleBuild2(0, new ParametersAction(new StringParameterValue("number", "30")))); 150 | 151 | System.out.println("check if zero nodes!"); 152 | Assert.assertEquals(0, j.jenkins.getNodes().size()); 153 | 154 | assertAtLeastOneNode(); 155 | 156 | final Node node = j.jenkins.getNodes().get(0); 157 | assertQueueIsEmpty(); 158 | 159 | System.out.println("disconnect node"); 160 | node.toComputer().disconnect(new OfflineCause.ChannelTermination(new UnsupportedOperationException("Test"))); 161 | 162 | assertLastBuildResult(Result.FAILURE, Result.ABORTED); 163 | 164 | node.toComputer().connect(true); 165 | assertNodeIsOnline(node); 166 | assertQueueAndNodesIdle(node); 167 | 168 | Assert.assertEquals(1, j.jenkins.getProjects().size()); 169 | Assert.assertEquals(Result.SUCCESS, j.jenkins.getProjects().get(0).getLastBuild().getResult()); 170 | Assert.assertEquals(2, j.jenkins.getProjects().get(0).getBuilds().size()); 171 | 172 | cancelTasks(rs); 173 | } 174 | 175 | @Test 176 | public void should_not_resubmit_if_disabled() throws Exception { 177 | EC2FleetCloud cloud = new EC2FleetCloud(null, null, "credId", null, "region", 178 | null, "fId", "momo", null, new LocalComputerConnector(j), false, false, 179 | 0, 0, 10, 1, false, false, 180 | true, 0, 0, false, 10, false); 181 | j.jenkins.clouds.add(cloud); 182 | 183 | List rs = getQueueTaskFutures(1); 184 | 185 | System.out.println("check if zero nodes!"); 186 | Assert.assertEquals(0, j.jenkins.getNodes().size()); 187 | 188 | assertAtLeastOneNode(); 189 | 190 | final Node node = j.jenkins.getNodes().get(0); 191 | assertQueueIsEmpty(); 192 | 193 | System.out.println("disconnect node"); 194 | node.toComputer().disconnect(new OfflineCause.ChannelTermination(new UnsupportedOperationException("Test"))); 195 | 196 | assertLastBuildResult(Result.FAILURE, Result.ABORTED); 197 | 198 | node.toComputer().connect(true); 199 | assertNodeIsOnline(node); 200 | assertQueueAndNodesIdle(node); 201 | 202 | Assert.assertEquals(1, j.jenkins.getProjects().size()); 203 | Assert.assertEquals(Result.FAILURE, j.jenkins.getProjects().get(0).getLastBuild().getResult()); 204 | Assert.assertEquals(1, j.jenkins.getProjects().get(0).getBuilds().size()); 205 | 206 | cancelTasks(rs); 207 | } 208 | 209 | } 210 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/jenkins/ec2fleet/CloudNannyTest.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import com.google.common.collect.ImmutableList; 4 | import com.google.common.collect.ImmutableSet; 5 | import com.google.common.collect.MapMaker; 6 | import hudson.slaves.Cloud; 7 | import hudson.widgets.Widget; 8 | import org.junit.Before; 9 | import org.junit.Test; 10 | import org.junit.runner.RunWith; 11 | import org.mockito.Mock; 12 | import org.powermock.api.mockito.PowerMockito; 13 | import org.powermock.core.classloader.annotations.PrepareForTest; 14 | import org.powermock.modules.junit4.PowerMockRunner; 15 | import org.powermock.reflect.Whitebox; 16 | 17 | import java.util.ArrayList; 18 | import java.util.Collections; 19 | import java.util.List; 20 | import java.util.concurrent.ConcurrentMap; 21 | import java.util.concurrent.atomic.AtomicInteger; 22 | 23 | import static org.junit.Assert.assertEquals; 24 | import static org.mockito.ArgumentMatchers.any; 25 | import static org.mockito.Mockito.atLeastOnce; 26 | import static org.mockito.Mockito.mock; 27 | import static org.mockito.Mockito.verify; 28 | import static org.mockito.Mockito.verifyNoMoreInteractions; 29 | import static org.mockito.Mockito.verifyZeroInteractions; 30 | import static org.mockito.Mockito.when; 31 | 32 | @RunWith(PowerMockRunner.class) 33 | @PrepareForTest(CloudNanny.class) 34 | public class CloudNannyTest { 35 | 36 | @Mock 37 | private EC2FleetCloud cloud1; 38 | 39 | @Mock 40 | private EC2FleetCloud cloud2; 41 | 42 | @Mock 43 | private EC2FleetStatusWidget widget1; 44 | 45 | @Mock 46 | private EC2FleetStatusWidget widget2; 47 | 48 | private List widgets = new ArrayList<>(); 49 | 50 | private List clouds = new ArrayList<>(); 51 | 52 | private FleetStateStats stats1 = new FleetStateStats( 53 | "f1", 1, "a", ImmutableSet.of(), Collections.emptyMap()); 54 | 55 | private FleetStateStats stats2 = new FleetStateStats( 56 | "f2", 1, "a", ImmutableSet.of(), Collections.emptyMap()); 57 | 58 | private int recurrencePeriod = 45; 59 | 60 | private AtomicInteger recurrenceCounter1 = new AtomicInteger(); 61 | private AtomicInteger recurrenceCounter2 = new AtomicInteger(); 62 | 63 | private ConcurrentMap recurrenceCounters = new MapMaker() 64 | .weakKeys() 65 | .concurrencyLevel(2) 66 | .makeMap(); 67 | 68 | @Before 69 | public void before() throws Exception { 70 | PowerMockito.mockStatic(CloudNanny.class); 71 | PowerMockito.when(CloudNanny.class, "getClouds").thenReturn(clouds); 72 | PowerMockito.when(CloudNanny.class, "getWidgets").thenReturn(widgets); 73 | 74 | when(cloud1.getLabelString()).thenReturn("a"); 75 | when(cloud2.getLabelString()).thenReturn(""); 76 | when(cloud1.getFleet()).thenReturn("f1"); 77 | when(cloud2.getFleet()).thenReturn("f2"); 78 | 79 | when(cloud1.update()).thenReturn(stats1); 80 | when(cloud2.update()).thenReturn(stats2); 81 | 82 | when(cloud1.getCloudStatusIntervalSec()).thenReturn(recurrencePeriod); 83 | when(cloud2.getCloudStatusIntervalSec()).thenReturn(recurrencePeriod * 2); 84 | 85 | recurrenceCounters.put(cloud1, recurrenceCounter1); 86 | recurrenceCounters.put(cloud2, recurrenceCounter2); 87 | } 88 | 89 | private CloudNanny getMockCloudNannyInstance() { 90 | CloudNanny cloudNanny = Whitebox.newInstance(CloudNanny.class); 91 | 92 | // next execution should trigger running the status check. 93 | recurrenceCounter1.set(1); 94 | recurrenceCounter2.set(1); 95 | 96 | Whitebox.setInternalState(cloudNanny, "recurrenceCounters", recurrenceCounters); 97 | 98 | return cloudNanny; 99 | } 100 | 101 | @Test 102 | public void shouldDoNothingIfNoCloudsAndWidgets() throws Exception { 103 | getMockCloudNannyInstance().doRun(); 104 | } 105 | 106 | @Test 107 | public void shouldUpdateCloudAndDoNothingIfNoWidgets() throws Exception { 108 | clouds.add(cloud1); 109 | clouds.add(cloud2); 110 | 111 | getMockCloudNannyInstance().doRun(); 112 | } 113 | 114 | @Test 115 | public void shouldUpdateCloudCollectResultAndUpdateWidgets() throws Exception { 116 | clouds.add(cloud1); 117 | 118 | widgets.add(widget1); 119 | 120 | getMockCloudNannyInstance().doRun(); 121 | 122 | verify(widget1).setStatusList(ImmutableList.of(new EC2FleetStatusInfo( 123 | cloud1.getFleet(), stats1.getState(), cloud1.getLabelString(), stats1.getNumActive(), stats1.getNumDesired()))); 124 | } 125 | 126 | @Test 127 | public void shouldUpdateCloudCollectResultAndUpdateAllEC2FleetWidgets() throws Exception { 128 | clouds.add(cloud1); 129 | 130 | widgets.add(widget1); 131 | widgets.add(widget2); 132 | 133 | getMockCloudNannyInstance().doRun(); 134 | 135 | verify(widget1).setStatusList(ImmutableList.of(new EC2FleetStatusInfo( 136 | cloud1.getFleet(), stats1.getState(), cloud1.getLabelString(), stats1.getNumActive(), stats1.getNumDesired()))); 137 | verify(widget2).setStatusList(ImmutableList.of(new EC2FleetStatusInfo( 138 | cloud1.getFleet(), stats1.getState(), cloud1.getLabelString(), stats1.getNumActive(), stats1.getNumDesired()))); 139 | } 140 | 141 | @Test 142 | public void shouldIgnoreNonEC2FleetClouds() throws Exception { 143 | clouds.add(cloud1); 144 | 145 | Cloud nonEc2FleetCloud = mock(Cloud.class); 146 | clouds.add(nonEc2FleetCloud); 147 | 148 | widgets.add(widget2); 149 | 150 | getMockCloudNannyInstance().doRun(); 151 | 152 | verify(cloud1).update(); 153 | verifyZeroInteractions(nonEc2FleetCloud); 154 | } 155 | 156 | @Test 157 | public void shouldUpdateCloudCollectAllResultAndUpdateWidgets() throws Exception { 158 | clouds.add(cloud1); 159 | clouds.add(cloud2); 160 | 161 | widgets.add(widget1); 162 | 163 | getMockCloudNannyInstance().doRun(); 164 | 165 | verify(widget1).setStatusList(ImmutableList.of( 166 | new EC2FleetStatusInfo(cloud1.getFleet(), stats1.getState(), cloud1.getLabelString(), stats1.getNumActive(), stats1.getNumDesired()), 167 | new EC2FleetStatusInfo(cloud2.getFleet(), stats2.getState(), cloud2.getLabelString(), stats2.getNumActive(), stats2.getNumDesired()) 168 | )); 169 | } 170 | 171 | @Test 172 | public void shouldIgnoreExceptionsFromUpdateForOneofCloudAndUpdateOther() throws Exception { 173 | clouds.add(cloud1); 174 | clouds.add(cloud2); 175 | 176 | when(cloud1.update()).thenThrow(new IllegalArgumentException("test")); 177 | 178 | widgets.add(widget1); 179 | 180 | getMockCloudNannyInstance().doRun(); 181 | 182 | verify(widget1).setStatusList(ImmutableList.of( 183 | new EC2FleetStatusInfo(cloud2.getFleet(), stats2.getState(), cloud2.getLabelString(), stats2.getNumActive(), stats2.getNumDesired()) 184 | )); 185 | } 186 | 187 | @SuppressWarnings("unchecked") 188 | @Test 189 | public void shouldIgnoreNonEc2FleetWidgets() throws Exception { 190 | clouds.add(cloud1); 191 | 192 | Widget nonEc2FleetWidget = mock(Widget.class); 193 | widgets.add(nonEc2FleetWidget); 194 | 195 | widgets.add(widget1); 196 | 197 | getMockCloudNannyInstance().doRun(); 198 | 199 | verify(widget1).setStatusList(any(List.class)); 200 | verifyZeroInteractions(nonEc2FleetWidget); 201 | } 202 | 203 | @Test 204 | public void resetCloudInterval() throws Exception { 205 | clouds.add(cloud1); 206 | clouds.add(cloud2); 207 | CloudNanny cloudNanny = getMockCloudNannyInstance(); 208 | 209 | cloudNanny.doRun(); 210 | 211 | verify(cloud1).update(); 212 | verify(cloud1, atLeastOnce()).getCloudStatusIntervalSec(); 213 | verify(cloud2).update(); 214 | verify(cloud2, atLeastOnce()).getCloudStatusIntervalSec(); 215 | 216 | 217 | assertEquals(cloud1.getCloudStatusIntervalSec(), recurrenceCounter1.get()); 218 | assertEquals(cloud2.getCloudStatusIntervalSec(), recurrenceCounter2.get()); 219 | } 220 | 221 | @Test 222 | public void skipCloudIntervalExecution() throws Exception { 223 | clouds.add(cloud1); 224 | clouds.add(cloud2); 225 | CloudNanny cloudNanny = getMockCloudNannyInstance(); 226 | recurrenceCounter1.set(2); 227 | recurrenceCounter2.set(3); 228 | 229 | cloudNanny.doRun(); 230 | 231 | verify(cloud1, atLeastOnce()).getCloudStatusIntervalSec(); 232 | verify(cloud2, atLeastOnce()).getCloudStatusIntervalSec(); 233 | verifyNoMoreInteractions(cloud1, cloud2); 234 | 235 | assertEquals(1, recurrenceCounter1.get()); 236 | assertEquals(2, recurrenceCounter2.get()); 237 | } 238 | 239 | @Test 240 | public void updateOnlyOneCloud() throws Exception { 241 | clouds.add(cloud1); 242 | clouds.add(cloud2); 243 | CloudNanny cloudNanny = getMockCloudNannyInstance(); 244 | recurrenceCounter1.set(2); 245 | recurrenceCounter2.set(1); 246 | 247 | cloudNanny.doRun(); 248 | 249 | verify(cloud2, atLeastOnce()).getCloudStatusIntervalSec(); 250 | verify(cloud2).update(); 251 | 252 | verify(cloud1, atLeastOnce()).getCloudStatusIntervalSec(); 253 | verifyNoMoreInteractions(cloud1); 254 | 255 | assertEquals(1, recurrenceCounter1.get()); 256 | assertEquals(cloud2.getCloudStatusIntervalSec(), recurrenceCounter2.get()); 257 | } 258 | } 259 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/jenkins/ec2fleet/EC2ApiTest.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import com.amazonaws.services.ec2.AmazonEC2; 4 | import com.amazonaws.services.ec2.model.AmazonEC2Exception; 5 | import com.amazonaws.services.ec2.model.DescribeInstancesRequest; 6 | import com.amazonaws.services.ec2.model.DescribeInstancesResult; 7 | import com.amazonaws.services.ec2.model.Instance; 8 | import com.amazonaws.services.ec2.model.InstanceState; 9 | import com.amazonaws.services.ec2.model.InstanceStateName; 10 | import com.amazonaws.services.ec2.model.Reservation; 11 | import com.google.common.collect.ImmutableMap; 12 | import org.junit.Assert; 13 | import org.junit.Test; 14 | import org.junit.runner.RunWith; 15 | import org.mockito.Mock; 16 | import org.mockito.junit.MockitoJUnitRunner; 17 | 18 | import java.util.Arrays; 19 | import java.util.Collections; 20 | import java.util.HashSet; 21 | import java.util.Map; 22 | import java.util.Set; 23 | 24 | import static org.mockito.Mockito.any; 25 | import static org.mockito.Mockito.times; 26 | import static org.mockito.Mockito.verify; 27 | import static org.mockito.Mockito.verifyNoMoreInteractions; 28 | import static org.mockito.Mockito.verifyZeroInteractions; 29 | import static org.mockito.Mockito.when; 30 | 31 | @SuppressWarnings("ArraysAsListWithZeroOrOneArgument") 32 | @RunWith(MockitoJUnitRunner.class) 33 | public class EC2ApiTest { 34 | 35 | @Mock 36 | private AmazonEC2 amazonEC2; 37 | 38 | @Test 39 | public void describeInstances_shouldReturnEmptyResultAndNoCallIfEmptyListOfInstances() { 40 | Map described = new EC2Api().describeInstances(amazonEC2, Collections.emptySet()); 41 | 42 | Assert.assertEquals(Collections.emptyMap(), described); 43 | verifyZeroInteractions(amazonEC2); 44 | } 45 | 46 | @Test 47 | public void describeInstances_shouldReturnAllInstancesIfStillActive() { 48 | // given 49 | Set instanceIds = new HashSet<>(); 50 | instanceIds.add("i-1"); 51 | instanceIds.add("i-2"); 52 | 53 | DescribeInstancesResult describeInstancesResult = new DescribeInstancesResult(); 54 | Reservation reservation = new Reservation(); 55 | Instance instance1 = new Instance() 56 | .withInstanceId("i-1") 57 | .withState(new InstanceState().withName(InstanceStateName.Running)); 58 | Instance instance2 = new Instance() 59 | .withInstanceId("i-2") 60 | .withState(new InstanceState().withName(InstanceStateName.Running)); 61 | reservation.setInstances(Arrays.asList(instance1, instance2)); 62 | describeInstancesResult.setReservations(Arrays.asList(reservation)); 63 | 64 | when(amazonEC2.describeInstances(any(DescribeInstancesRequest.class))).thenReturn(describeInstancesResult); 65 | 66 | // when 67 | Map described = new EC2Api().describeInstances(amazonEC2, instanceIds); 68 | 69 | // then 70 | Assert.assertEquals(ImmutableMap.of("i-1", instance1, "i-2", instance2), described); 71 | verify(amazonEC2, times(1)) 72 | .describeInstances(any(DescribeInstancesRequest.class)); 73 | } 74 | 75 | @Test 76 | public void describeInstances_shouldProcessAllPagesUntilNextTokenIsAvailable() { 77 | // given 78 | Set instanceIds = new HashSet<>(); 79 | instanceIds.add("i-1"); 80 | instanceIds.add("i-2"); 81 | instanceIds.add("i-3"); 82 | 83 | final Instance instance1 = new Instance() 84 | .withInstanceId("i-1") 85 | .withState(new InstanceState().withName(InstanceStateName.Running)); 86 | DescribeInstancesResult describeInstancesResult1 = 87 | new DescribeInstancesResult() 88 | .withReservations( 89 | new Reservation().withInstances(instance1)) 90 | .withNextToken("a"); 91 | 92 | final Instance instance2 = new Instance() 93 | .withInstanceId("i-2") 94 | .withState(new InstanceState().withName(InstanceStateName.Running)); 95 | DescribeInstancesResult describeInstancesResult2 = 96 | new DescribeInstancesResult() 97 | .withReservations(new Reservation().withInstances( 98 | instance2, 99 | new Instance() 100 | .withInstanceId("i-3") 101 | .withState(new InstanceState().withName(InstanceStateName.Terminated)) 102 | )); 103 | 104 | when(amazonEC2.describeInstances(any(DescribeInstancesRequest.class))) 105 | .thenReturn(describeInstancesResult1) 106 | .thenReturn(describeInstancesResult2); 107 | 108 | // when 109 | Map described = new EC2Api().describeInstances(amazonEC2, instanceIds); 110 | 111 | // then 112 | Assert.assertEquals(ImmutableMap.of("i-1", instance1, "i-2", instance2), described); 113 | verify(amazonEC2, times(2)) 114 | .describeInstances(any(DescribeInstancesRequest.class)); 115 | } 116 | 117 | @Test 118 | public void describeInstances_shouldNotDescribeMissedInResultInstanceOrTerminatedOrStoppedOrStoppingOrShuttingDownAs() { 119 | // given 120 | Set instanceIds = new HashSet<>(); 121 | instanceIds.add("missed"); 122 | instanceIds.add("stopped"); 123 | instanceIds.add("terminated"); 124 | instanceIds.add("stopping"); 125 | instanceIds.add("shutting-down"); 126 | 127 | DescribeInstancesResult describeInstancesResult1 = 128 | new DescribeInstancesResult() 129 | .withReservations( 130 | new Reservation().withInstances(new Instance() 131 | .withInstanceId("stopped") 132 | .withState(new InstanceState().withName(InstanceStateName.Stopped)), 133 | new Instance() 134 | .withInstanceId("stopping") 135 | .withState(new InstanceState().withName(InstanceStateName.Stopping)), 136 | new Instance() 137 | .withInstanceId("shutting-down") 138 | .withState(new InstanceState().withName(InstanceStateName.ShuttingDown)), 139 | new Instance() 140 | .withInstanceId("terminated") 141 | .withState(new InstanceState().withName(InstanceStateName.Terminated)) 142 | )); 143 | 144 | 145 | when(amazonEC2.describeInstances(any(DescribeInstancesRequest.class))) 146 | .thenReturn(describeInstancesResult1); 147 | 148 | // when 149 | Map described = new EC2Api().describeInstances(amazonEC2, instanceIds); 150 | 151 | // then 152 | Assert.assertEquals(Collections.emptyMap(), described); 153 | verify(amazonEC2, times(1)) 154 | .describeInstances(any(DescribeInstancesRequest.class)); 155 | } 156 | 157 | @Test 158 | public void describeInstances_shouldSendInOneCallNoMoreThenBatchSizeOfInstance() { 159 | // given 160 | Set instanceIds = new HashSet<>(); 161 | instanceIds.add("i1"); 162 | instanceIds.add("i2"); 163 | instanceIds.add("i3"); 164 | 165 | DescribeInstancesResult describeInstancesResult1 = 166 | new DescribeInstancesResult() 167 | .withReservations( 168 | new Reservation().withInstances(new Instance() 169 | .withInstanceId("stopped") 170 | .withState(new InstanceState().withName(InstanceStateName.Running)), 171 | new Instance() 172 | .withInstanceId("stopping") 173 | .withState(new InstanceState().withName(InstanceStateName.Running)) 174 | )); 175 | 176 | DescribeInstancesResult describeInstancesResult2 = new DescribeInstancesResult(); 177 | 178 | when(amazonEC2.describeInstances(any(DescribeInstancesRequest.class))) 179 | .thenReturn(describeInstancesResult1) 180 | .thenReturn(describeInstancesResult2); 181 | 182 | // when 183 | new EC2Api().describeInstances(amazonEC2, instanceIds, 2); 184 | 185 | // then 186 | verify(amazonEC2).describeInstances(new DescribeInstancesRequest().withInstanceIds(Arrays.asList("i1", "i2"))); 187 | verify(amazonEC2).describeInstances(new DescribeInstancesRequest().withInstanceIds(Arrays.asList("i3"))); 188 | verifyNoMoreInteractions(amazonEC2); 189 | } 190 | 191 | /** 192 | * NotFound exception example data 193 | *

    194 | * 195 | * Single instance 196 | * requestId = "0fd56c54-e11a-4928-843c-9a80a24bedd1" 197 | * errorCode = "InvalidInstanceID.NotFound" 198 | * errorType = {AmazonServiceException$ErrorType@11247} "Unknown" 199 | * errorMessage = "The instance ID 'i-1233f' does not exist" 200 | * 201 | *

    202 | * Multiple instances 203 | * 204 | * ex = {AmazonEC2Exception@11233} "com.amazonaws.services.ec2.model.AmazonEC2Exception: The instance IDs 'i-1233f, i-ffffff' do not exist (Service: AmazonEC2; Status Code: 400; Error Code: InvalidInstanceID.NotFound; Request ID:)" 205 | * requestId = "1a353313-ef52-4626-b87b-fd828db6343f" 206 | * errorCode = "InvalidInstanceID.NotFound" 207 | * errorType = {AmazonServiceException$ErrorType@11251} "Unknown" 208 | * errorMessage = "The instance IDs 'i-1233f, i-ffffff' do not exist" 209 | * 210 | */ 211 | @Test 212 | public void describeInstances_shouldHandleAmazonEc2NotFoundErrorAsTerminatedInstancesAndRetry() { 213 | // given 214 | Set instanceIds = new HashSet<>(); 215 | instanceIds.add("i-1"); 216 | instanceIds.add("i-f"); 217 | instanceIds.add("i-3"); 218 | 219 | AmazonEC2Exception notFoundException = new AmazonEC2Exception( 220 | "The instance IDs 'i-1, i-f' do not exist"); 221 | notFoundException.setErrorCode("InvalidInstanceID.NotFound"); 222 | 223 | final Instance instance3 = new Instance().withInstanceId("i-3") 224 | .withState(new InstanceState().withName(InstanceStateName.Running)); 225 | DescribeInstancesResult describeInstancesResult2 = new DescribeInstancesResult() 226 | .withReservations(new Reservation().withInstances( 227 | instance3)); 228 | 229 | when(amazonEC2.describeInstances(any(DescribeInstancesRequest.class))) 230 | .thenThrow(notFoundException) 231 | .thenReturn(describeInstancesResult2); 232 | 233 | // when 234 | final Map described = new EC2Api().describeInstances(amazonEC2, instanceIds); 235 | 236 | // then 237 | Assert.assertEquals(ImmutableMap.of("i-3", instance3), described); 238 | verify(amazonEC2).describeInstances(new DescribeInstancesRequest().withInstanceIds(Arrays.asList("i-1", "i-3", "i-f"))); 239 | verify(amazonEC2).describeInstances(new DescribeInstancesRequest().withInstanceIds(Arrays.asList("i-3"))); 240 | verifyNoMoreInteractions(amazonEC2); 241 | } 242 | 243 | @Test 244 | public void describeInstances_shouldFailIfNotAbleToParseNotFoundExceptionFromEc2Api() { 245 | // given 246 | Set instanceIds = new HashSet<>(); 247 | instanceIds.add("i-1"); 248 | instanceIds.add("i-f"); 249 | instanceIds.add("i-3"); 250 | 251 | AmazonEC2Exception notFoundException = new AmazonEC2Exception( 252 | "unparseable"); 253 | notFoundException.setErrorCode("InvalidInstanceID.NotFound"); 254 | 255 | when(amazonEC2.describeInstances(any(DescribeInstancesRequest.class))) 256 | .thenThrow(notFoundException); 257 | 258 | // when 259 | try { 260 | new EC2Api().describeInstances(amazonEC2, instanceIds); 261 | Assert.fail(); 262 | } catch (AmazonEC2Exception exception) { 263 | Assert.assertSame(notFoundException, exception); 264 | } 265 | } 266 | 267 | @Test 268 | public void describeInstances_shouldThrowExceptionIfEc2DescribeFailsWithException() { 269 | // given 270 | Set instanceIds = new HashSet<>(); 271 | instanceIds.add("a"); 272 | 273 | UnsupportedOperationException exception = new UnsupportedOperationException("test"); 274 | when(amazonEC2.describeInstances(any(DescribeInstancesRequest.class))) 275 | .thenThrow(exception); 276 | 277 | // when 278 | try { 279 | new EC2Api().describeInstances(amazonEC2, instanceIds); 280 | Assert.fail(); 281 | } catch (UnsupportedOperationException e) { 282 | Assert.assertSame(exception, e); 283 | } 284 | } 285 | 286 | @Test 287 | public void getEndpoint_returnNullIfRegionNameOrEndpointAreEmpty() { 288 | Assert.assertNull(new EC2Api().getEndpoint(null, null)); 289 | } 290 | 291 | @Test 292 | public void getEndpoint_returnEnpointAsIsIfProvided() { 293 | Assert.assertEquals("mymy", new EC2Api().getEndpoint(null, "mymy")); 294 | } 295 | 296 | @Test 297 | public void getEndpoint_returnCraftedIfRegionNotInStatic() { 298 | Assert.assertEquals("https://ec2.non-real-region.amazonaws.com", 299 | new EC2Api().getEndpoint("non-real-region", null)); 300 | } 301 | 302 | @Test 303 | public void getEndpoint_returnCraftedChinaIfRegionNotInStatic() { 304 | Assert.assertEquals("https://ec2.cn-non-real.amazonaws.com.cn", 305 | new EC2Api().getEndpoint("cn-non-real", null)); 306 | } 307 | 308 | @Test 309 | public void getEndpoint_returnStaticRegionEndpoint() { 310 | Assert.assertEquals("https://ec2.cn-north-1.amazonaws.com.cn", 311 | new EC2Api().getEndpoint("cn-north-1", null)); 312 | } 313 | 314 | } 315 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/jenkins/ec2fleet/EC2FleetAutoResubmitComputerLauncherTest.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import hudson.model.Action; 4 | import hudson.model.Actionable; 5 | import hudson.model.Executor; 6 | import hudson.model.Queue; 7 | import hudson.model.Result; 8 | import hudson.model.Slave; 9 | import hudson.model.TaskListener; 10 | import hudson.model.queue.SubTask; 11 | import hudson.slaves.ComputerLauncher; 12 | import hudson.slaves.OfflineCause; 13 | import jenkins.model.Jenkins; 14 | import org.junit.Before; 15 | import org.junit.Test; 16 | import org.junit.runner.RunWith; 17 | import org.mockito.Mock; 18 | import org.powermock.api.mockito.PowerMockito; 19 | import org.powermock.core.classloader.annotations.PrepareForTest; 20 | import org.powermock.modules.junit4.PowerMockRunner; 21 | 22 | import java.util.Arrays; 23 | import java.util.Collections; 24 | 25 | import static org.mockito.ArgumentMatchers.anyInt; 26 | import static org.mockito.ArgumentMatchers.eq; 27 | import static org.mockito.Mockito.mock; 28 | import static org.mockito.Mockito.verify; 29 | import static org.mockito.Mockito.verifyZeroInteractions; 30 | import static org.mockito.Mockito.when; 31 | import static org.mockito.Mockito.withSettings; 32 | 33 | @SuppressWarnings("ArraysAsListWithZeroOrOneArgument") 34 | @RunWith(PowerMockRunner.class) 35 | @PrepareForTest({Jenkins.class, Queue.class}) 36 | public class EC2FleetAutoResubmitComputerLauncherTest { 37 | 38 | @Mock 39 | private ComputerLauncher baseComputerLauncher; 40 | 41 | @Mock 42 | private TaskListener taskListener; 43 | 44 | @Mock 45 | private Action action1; 46 | 47 | @Mock 48 | private Executor executor1; 49 | 50 | @Mock 51 | private Executor executor2; 52 | 53 | private Actionable executable1; 54 | 55 | @Mock 56 | private Queue.Executable executable2; 57 | 58 | @Mock 59 | private Slave slave; 60 | 61 | @Mock 62 | private EC2FleetNodeComputer computer; 63 | 64 | @Mock 65 | private Jenkins jenkins; 66 | 67 | @Mock 68 | private Queue queue; 69 | 70 | @Mock 71 | private SubTask subTask1; 72 | 73 | @Mock 74 | private SubTask subTask2; 75 | 76 | @Mock 77 | private Queue.Task task1; 78 | 79 | @Mock 80 | private Queue.Task task2; 81 | 82 | @Mock 83 | private EC2FleetNode fleetNode; 84 | 85 | @Mock 86 | private EC2FleetCloud cloud; 87 | 88 | @Before 89 | public void before() { 90 | executable1 = mock(Actionable.class, withSettings().extraInterfaces(Queue.Executable.class)); 91 | 92 | when(computer.getDisplayName()).thenReturn("i-12"); 93 | 94 | PowerMockito.mockStatic(Jenkins.class); 95 | when(Jenkins.getInstance()).thenReturn(jenkins); 96 | when(Queue.getInstance()).thenReturn(queue); 97 | 98 | when(slave.getNumExecutors()).thenReturn(1); 99 | 100 | when(fleetNode.getDisplayName()).thenReturn("fleet node name"); 101 | 102 | when(((Queue.Executable) executable1).getParent()).thenReturn(subTask1); 103 | when(executable2.getParent()).thenReturn(subTask2); 104 | 105 | when(subTask1.getOwnerTask()).thenReturn(task1); 106 | when(subTask2.getOwnerTask()).thenReturn(task2); 107 | 108 | when(executor1.getCurrentExecutable()).thenReturn((Queue.Executable) executable1); 109 | when(executor2.getCurrentExecutable()).thenReturn(executable2); 110 | 111 | when(computer.getExecutors()).thenReturn(Arrays.asList(executor1, executor2)); 112 | when(computer.isOffline()).thenReturn(true); 113 | 114 | when(computer.getCloud()).thenReturn(cloud); 115 | } 116 | 117 | @Test 118 | public void afterDisconnect_should_do_nothing_if_task_finished_without_cause() { 119 | new EC2FleetAutoResubmitComputerLauncher(baseComputerLauncher) 120 | .afterDisconnect(computer, taskListener); 121 | verifyZeroInteractions(queue); 122 | } 123 | 124 | @Test 125 | public void afterDisconnect_should_do_nothing_if_task_finished_offline_but_no_cause() { 126 | when(computer.isOffline()).thenReturn(true); 127 | new EC2FleetAutoResubmitComputerLauncher(baseComputerLauncher) 128 | .afterDisconnect(computer, taskListener); 129 | verifyZeroInteractions(queue); 130 | } 131 | 132 | @Test 133 | public void afterDisconnect_should_do_nothing_if_task_finished_cause_but_still_online() { 134 | when(computer.isOffline()).thenReturn(false); 135 | when(computer.getOfflineCause()).thenReturn(new OfflineCause.ChannelTermination(null)); 136 | new EC2FleetAutoResubmitComputerLauncher(baseComputerLauncher) 137 | .afterDisconnect(computer, taskListener); 138 | verifyZeroInteractions(queue); 139 | } 140 | 141 | @Test 142 | public void taskCompleted_should_resubmit_task_if_offline_and_cause_disconnect() { 143 | when(computer.getExecutors()).thenReturn(Arrays.asList(executor1)); 144 | when(computer.getOfflineCause()).thenReturn(new OfflineCause.ChannelTermination(null)); 145 | new EC2FleetAutoResubmitComputerLauncher(baseComputerLauncher) 146 | .afterDisconnect(computer, taskListener); 147 | verify(queue).schedule2(eq(task1), anyInt(), eq(Collections.emptyList())); 148 | verifyZeroInteractions(queue); 149 | } 150 | 151 | @Test 152 | public void taskCompleted_should_not_resubmit_task_if_offline_and_cause_disconnect_but_disabled() { 153 | when(cloud.isDisableTaskResubmit()).thenReturn(true); 154 | when(computer.getExecutors()).thenReturn(Arrays.asList(executor1)); 155 | when(computer.getOfflineCause()).thenReturn(new OfflineCause.ChannelTermination(null)); 156 | new EC2FleetAutoResubmitComputerLauncher(baseComputerLauncher) 157 | .afterDisconnect(computer, taskListener); 158 | verifyZeroInteractions(queue); 159 | } 160 | 161 | @Test 162 | public void taskCompleted_should_resubmit_task_for_all_executors() { 163 | when(computer.getOfflineCause()).thenReturn(new OfflineCause.ChannelTermination(null)); 164 | new EC2FleetAutoResubmitComputerLauncher(baseComputerLauncher) 165 | .afterDisconnect(computer, taskListener); 166 | verify(queue).schedule2(eq(task1), anyInt(), eq(Collections.emptyList())); 167 | verify(queue).schedule2(eq(task2), anyInt(), eq(Collections.emptyList())); 168 | verifyZeroInteractions(queue); 169 | } 170 | 171 | @Test 172 | public void taskCompleted_should_abort_executors_during_resubmit() { 173 | when(computer.getOfflineCause()).thenReturn(new OfflineCause.ChannelTermination(null)); 174 | new EC2FleetAutoResubmitComputerLauncher(baseComputerLauncher) 175 | .afterDisconnect(computer, taskListener); 176 | verify(executor1).interrupt(Result.ABORTED, new EC2TerminationCause("i-12")); 177 | verify(executor2).interrupt(Result.ABORTED, new EC2TerminationCause("i-12")); 178 | } 179 | 180 | @Test 181 | public void taskCompleted_should_resubmit_task_with_actions() { 182 | when(computer.getExecutors()).thenReturn(Arrays.asList(executor1)); 183 | when(executable1.getActions()).thenReturn(Arrays.asList(action1)); 184 | when(computer.getOfflineCause()).thenReturn(new OfflineCause.ChannelTermination(null)); 185 | new EC2FleetAutoResubmitComputerLauncher(baseComputerLauncher) 186 | .afterDisconnect(computer, taskListener); 187 | verify(queue).schedule2(eq(task1), anyInt(), eq(Arrays.asList(action1))); 188 | verifyZeroInteractions(queue); 189 | } 190 | 191 | } 192 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/jenkins/ec2fleet/EC2FleetCloudAwareUtilsTest.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import hudson.model.Computer; 4 | import hudson.model.LabelFinder; 5 | import hudson.model.Node; 6 | import jenkins.model.Jenkins; 7 | import org.junit.Before; 8 | import org.junit.Test; 9 | import org.junit.runner.RunWith; 10 | import org.mockito.Mock; 11 | import org.powermock.api.mockito.PowerMockito; 12 | import org.powermock.core.classloader.annotations.PrepareForTest; 13 | import org.powermock.modules.junit4.PowerMockRunner; 14 | 15 | import java.util.Arrays; 16 | import java.util.Collections; 17 | 18 | import static org.mockito.ArgumentMatchers.any; 19 | import static org.mockito.Mockito.times; 20 | import static org.mockito.Mockito.verify; 21 | import static org.mockito.Mockito.when; 22 | 23 | @SuppressWarnings("ArraysAsListWithZeroOrOneArgument") 24 | @RunWith(PowerMockRunner.class) 25 | @PrepareForTest({Jenkins.class, LabelFinder.class}) 26 | public class EC2FleetCloudAwareUtilsTest { 27 | 28 | @Mock 29 | private Jenkins jenkins; 30 | 31 | @Mock 32 | private EC2FleetCloud cloud; 33 | 34 | @Mock 35 | private EC2FleetCloud oldCloud; 36 | 37 | @Mock 38 | private EC2FleetCloud otherCloud; 39 | 40 | @Mock 41 | private EC2FleetNodeComputer computer; 42 | 43 | @Mock 44 | private EC2FleetNode node; 45 | 46 | @Before 47 | public void before() { 48 | PowerMockito.mockStatic(LabelFinder.class); 49 | 50 | PowerMockito.mockStatic(Jenkins.class); 51 | PowerMockito.when(Jenkins.getActiveInstance()).thenReturn(jenkins); 52 | 53 | when(oldCloud.getOldId()).thenReturn("cloud"); 54 | when(computer.getCloud()).thenReturn(oldCloud); 55 | when(node.getCloud()).thenReturn(oldCloud); 56 | 57 | when(cloud.getOldId()).thenReturn("cloud"); 58 | when(otherCloud.getOldId()).thenReturn("other"); 59 | 60 | when(jenkins.getNodes()).thenReturn(Collections.emptyList()); 61 | when(jenkins.getComputers()).thenReturn(new Computer[0]); 62 | } 63 | 64 | @Test 65 | public void reassign_nothing_if_no_nodes_or_computers() { 66 | EC2FleetCloudAwareUtils.reassign("cloud", cloud); 67 | } 68 | 69 | @Test 70 | public void reassign_nothing_if_computers_belong_to_diff_cloud_id() { 71 | when(jenkins.getNodes()).thenReturn(Collections.emptyList()); 72 | when(computer.getCloud()).thenReturn(otherCloud); 73 | when(jenkins.getComputers()).thenReturn(new Computer[]{computer}); 74 | 75 | EC2FleetCloudAwareUtils.reassign("cloud", cloud); 76 | 77 | verify(computer, times(0)).setCloud(any(EC2FleetCloud.class)); 78 | } 79 | 80 | @Test 81 | public void reassign_nothing_if_computer_cloud_is_null() { 82 | when(computer.getCloud()).thenReturn(null); 83 | when(jenkins.getComputers()).thenReturn(new Computer[]{computer}); 84 | 85 | EC2FleetCloudAwareUtils.reassign("cloud", cloud); 86 | 87 | verify(computer, times(0)).setCloud(any(EC2FleetCloud.class)); 88 | } 89 | 90 | @Test 91 | public void reassign_if_computer_belong_to_old_cloud() { 92 | when(jenkins.getComputers()).thenReturn(new Computer[]{computer}); 93 | 94 | EC2FleetCloudAwareUtils.reassign("cloud", cloud); 95 | 96 | verify(computer, times(1)).setCloud(cloud); 97 | } 98 | 99 | @Test 100 | public void reassign_if_node_belong_to_same_cloud() { 101 | when(computer.getCloud()).thenReturn(cloud); 102 | when(jenkins.getNodes()).thenReturn(Arrays.asList((Node) node)); 103 | 104 | EC2FleetCloudAwareUtils.reassign("cloud", cloud); 105 | 106 | verify(node, times(1)).setCloud(cloud); 107 | } 108 | 109 | @Test 110 | public void reassign_nothing_if_node_belong_to_other_cloud_id() { 111 | when(computer.getCloud()).thenReturn(cloud); 112 | when(node.getCloud()).thenReturn(otherCloud); 113 | when(jenkins.getNodes()).thenReturn(Arrays.asList((Node) node)); 114 | 115 | EC2FleetCloudAwareUtils.reassign("cloud", cloud); 116 | 117 | verify(node, times(0)).setCloud(cloud); 118 | } 119 | 120 | } 121 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/jenkins/ec2fleet/EC2FleetCloudWithHistory.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import hudson.model.Label; 4 | import hudson.slaves.ComputerConnector; 5 | import hudson.slaves.NodeProvisioner; 6 | 7 | import java.util.Collection; 8 | import java.util.concurrent.CopyOnWriteArrayList; 9 | 10 | 11 | public class EC2FleetCloudWithHistory extends EC2FleetCloud { 12 | 13 | public CopyOnWriteArrayList provisionTimes = new CopyOnWriteArrayList<>(); 14 | 15 | public EC2FleetCloudWithHistory(String name, String oldId, String awsCredentialsId, String credentialsId, String region, String endpoint, String fleet, String labelString, String fsRoot, ComputerConnector computerConnector, boolean privateIpUsed, boolean alwaysReconnect, Integer idleMinutes, Integer minSize, Integer maxSize, Integer numExecutors, boolean addNodeOnlyIfRunning, boolean restrictUsage, boolean disableTaskResubmit, Integer initOnlineTimeoutSec, Integer initOnlineCheckIntervalSec, boolean scaleExecutorsByWeight, Integer cloudStatusIntervalSec, boolean immediatelyProvision) { 16 | super(name, oldId, awsCredentialsId, credentialsId, region, endpoint, fleet, labelString, fsRoot, computerConnector, privateIpUsed, alwaysReconnect, idleMinutes, minSize, maxSize, numExecutors, addNodeOnlyIfRunning, restrictUsage, disableTaskResubmit, initOnlineTimeoutSec, initOnlineCheckIntervalSec, scaleExecutorsByWeight, cloudStatusIntervalSec, immediatelyProvision); 17 | } 18 | 19 | @Override 20 | public Collection provision( 21 | final Label label, final int excessWorkload) { 22 | final Collection r = super.provision(label, excessWorkload); 23 | for (NodeProvisioner.PlannedNode ignore : r) provisionTimes.add(System.currentTimeMillis()); 24 | return r; 25 | } 26 | 27 | // @Override 28 | // public FleetStateStats update() { 29 | // try (Meter.Shot s = updateMeter.start()) { 30 | // return super.update(); 31 | // } 32 | // } 33 | 34 | // @Override 35 | // public boolean scheduleToTerminate(final String instanceId) { 36 | // try (Meter.Shot s = removeMeter.start()) { 37 | // return super.scheduleToTerminate(instanceId); 38 | // } 39 | // } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/jenkins/ec2fleet/EC2FleetCloudWithMeter.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import hudson.model.Label; 4 | import hudson.slaves.ComputerConnector; 5 | import hudson.slaves.NodeProvisioner; 6 | 7 | import java.util.Collection; 8 | 9 | 10 | public class EC2FleetCloudWithMeter extends EC2FleetCloud { 11 | 12 | public final Meter updateMeter = new Meter("update"); 13 | public final Meter provisionMeter = new Meter("provision"); 14 | public final Meter removeMeter = new Meter("remove"); 15 | 16 | public EC2FleetCloudWithMeter(String name, String oldId, String awsCredentialsId, String credentialsId, String region, String endpoint, String fleet, String labelString, String fsRoot, ComputerConnector computerConnector, boolean privateIpUsed, boolean alwaysReconnect, Integer idleMinutes, Integer minSize, Integer maxSize, Integer numExecutors, boolean addNodeOnlyIfRunning, boolean restrictUsage, boolean disableTaskResubmit, Integer initOnlineTimeoutSec, Integer initOnlineCheckIntervalSec, boolean scaleExecutorsByWeight, Integer cloudStatusIntervalSec, boolean immediatelyProvision) { 17 | super(name, oldId, awsCredentialsId, credentialsId, region, endpoint, fleet, labelString, fsRoot, computerConnector, privateIpUsed, alwaysReconnect, idleMinutes, minSize, maxSize, numExecutors, addNodeOnlyIfRunning, restrictUsage, disableTaskResubmit, initOnlineTimeoutSec, initOnlineCheckIntervalSec, scaleExecutorsByWeight, cloudStatusIntervalSec, immediatelyProvision); 18 | } 19 | 20 | @Override 21 | public Collection provision( 22 | final Label label, final int excessWorkload) { 23 | try (Meter.Shot s = provisionMeter.start()) { 24 | return super.provision(label, excessWorkload); 25 | } 26 | } 27 | 28 | @Override 29 | public FleetStateStats update() { 30 | try (Meter.Shot s = updateMeter.start()) { 31 | return super.update(); 32 | } 33 | } 34 | 35 | @Override 36 | public boolean scheduleToTerminate(final String instanceId) { 37 | try (Meter.Shot s = removeMeter.start()) { 38 | return super.scheduleToTerminate(instanceId); 39 | } 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/jenkins/ec2fleet/EC2FleetNodeComputerTest.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import hudson.model.Queue; 4 | import hudson.model.Slave; 5 | import jenkins.model.Jenkins; 6 | import org.junit.Assert; 7 | import org.junit.Before; 8 | import org.junit.Test; 9 | import org.junit.runner.RunWith; 10 | import org.mockito.Mock; 11 | import org.powermock.api.mockito.PowerMockito; 12 | import org.powermock.core.classloader.annotations.PrepareForTest; 13 | import org.powermock.modules.junit4.PowerMockRunner; 14 | 15 | import static org.mockito.Mockito.spy; 16 | import static org.mockito.Mockito.when; 17 | 18 | @RunWith(PowerMockRunner.class) 19 | @PrepareForTest({Jenkins.class}) 20 | public class EC2FleetNodeComputerTest { 21 | 22 | @Mock 23 | private Slave slave; 24 | 25 | @Mock 26 | private Jenkins jenkins; 27 | 28 | @Mock 29 | private Queue queue; 30 | 31 | @Mock 32 | private EC2FleetCloud cloud; 33 | 34 | @Before 35 | public void before() { 36 | PowerMockito.mockStatic(Jenkins.class); 37 | when(Jenkins.getInstance()).thenReturn(jenkins); 38 | when(Queue.getInstance()).thenReturn(queue); 39 | 40 | when(slave.getNumExecutors()).thenReturn(1); 41 | } 42 | 43 | @Test 44 | public void getDisplayName_should_be_ok_with_init_null_cloud() { 45 | EC2FleetNodeComputer computer = spy(new EC2FleetNodeComputer(slave, "a", null)); 46 | Assert.assertEquals("unknown fleet a", computer.getDisplayName()); 47 | } 48 | 49 | @Test 50 | public void getDisplayName_should_be_ok_with_set_null_cloud() { 51 | EC2FleetNodeComputer computer = spy(new EC2FleetNodeComputer(slave, "a", cloud)); 52 | computer.setCloud(null); 53 | Assert.assertEquals("unknown fleet a", computer.getDisplayName()); 54 | } 55 | 56 | @Test 57 | public void getDisplayName_returns_node_display_name() { 58 | when(cloud.getDisplayName()).thenReturn("a"); 59 | EC2FleetNodeComputer computer = spy(new EC2FleetNodeComputer(slave, "n", cloud)); 60 | Assert.assertEquals("a n", computer.getDisplayName()); 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/jenkins/ec2fleet/EC2FleetOnlineCheckerTest.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import com.google.common.util.concurrent.SettableFuture; 4 | import hudson.model.Computer; 5 | import hudson.model.Node; 6 | import jenkins.model.Jenkins; 7 | import org.junit.Assert; 8 | import org.junit.Before; 9 | import org.junit.Test; 10 | import org.junit.runner.RunWith; 11 | import org.mockito.Mock; 12 | import org.powermock.api.mockito.PowerMockito; 13 | import org.powermock.core.classloader.annotations.PrepareForTest; 14 | import org.powermock.modules.junit4.PowerMockRunner; 15 | 16 | import java.util.concurrent.CancellationException; 17 | import java.util.concurrent.ExecutionException; 18 | import java.util.concurrent.TimeUnit; 19 | 20 | import static org.mockito.Mockito.atLeast; 21 | import static org.mockito.Mockito.times; 22 | import static org.mockito.Mockito.verify; 23 | import static org.mockito.Mockito.verifyZeroInteractions; 24 | import static org.mockito.Mockito.when; 25 | 26 | 27 | @RunWith(PowerMockRunner.class) 28 | @PrepareForTest({EC2FleetOnlineChecker.class, EC2FleetNode.class, Jenkins.class, Computer.class}) 29 | public class EC2FleetOnlineCheckerTest { 30 | 31 | private SettableFuture future = SettableFuture.create(); 32 | 33 | @Mock 34 | private EC2FleetNode node; 35 | 36 | @Mock 37 | private Computer computer; 38 | 39 | @Mock 40 | private Jenkins jenkins; 41 | 42 | @Before 43 | public void before() throws Exception { 44 | when(node.getNodeName()).thenReturn("i-1"); 45 | 46 | PowerMockito.mockStatic(Jenkins.class); 47 | 48 | when(Jenkins.getInstance()).thenReturn(jenkins); 49 | 50 | // final method 51 | PowerMockito.when(node.toComputer()).thenReturn(computer); 52 | 53 | PowerMockito.whenNew(EC2FleetNode.class).withAnyArguments().thenReturn(node); 54 | } 55 | 56 | @Test 57 | public void shouldStopImmediatelyIfFutureIsCancelled() throws InterruptedException, ExecutionException { 58 | future.cancel(true); 59 | 60 | EC2FleetOnlineChecker.start(node, future, 0, 0); 61 | try { 62 | future.get(); 63 | Assert.fail(); 64 | } catch (CancellationException e) { 65 | // ok 66 | } 67 | } 68 | 69 | @Test 70 | public void shouldStopAndFailFutureIfTimeout() { 71 | EC2FleetOnlineChecker.start(node, future, 100, 50); 72 | try { 73 | future.get(); 74 | Assert.fail(); 75 | } catch (InterruptedException | ExecutionException e) { 76 | Assert.assertEquals("Fail to provision node, cannot connect to i-1 in 100 msec", e.getCause().getMessage()); 77 | Assert.assertEquals(IllegalStateException.class, e.getCause().getClass()); 78 | verify(computer, atLeast(2)).isOnline(); 79 | } 80 | } 81 | 82 | @Test 83 | public void shouldFinishWithNodeWhenSuccessfulConnect() throws InterruptedException, ExecutionException { 84 | PowerMockito.when(computer.isOnline()).thenReturn(true); 85 | 86 | EC2FleetOnlineChecker.start(node, future, TimeUnit.MINUTES.toMillis(1), 0); 87 | 88 | Assert.assertSame(node, future.get()); 89 | } 90 | 91 | @Test 92 | public void shouldFinishWithNodeWhenTimeoutIsZeroWithoutCheck() throws InterruptedException, ExecutionException { 93 | EC2FleetOnlineChecker.start(node, future, 0, 0); 94 | 95 | Assert.assertSame(node, future.get()); 96 | verifyZeroInteractions(computer); 97 | } 98 | 99 | @Test 100 | public void shouldSuccessfullyFinishAndNoWaitIfIntervalIsZero() throws ExecutionException, InterruptedException { 101 | PowerMockito.when(computer.isOnline()).thenReturn(true); 102 | 103 | EC2FleetOnlineChecker.start(node, future, 10, 0); 104 | 105 | Assert.assertSame(node, future.get()); 106 | verifyZeroInteractions(computer); 107 | } 108 | 109 | @Test 110 | public void shouldWaitIfOffline() throws InterruptedException, ExecutionException { 111 | PowerMockito.when(computer.isOnline()) 112 | .thenReturn(false) 113 | .thenReturn(false) 114 | .thenReturn(false) 115 | .thenReturn(true); 116 | 117 | EC2FleetOnlineChecker.start(node, future, 100, 10); 118 | 119 | Assert.assertSame(node, future.get()); 120 | verify(computer, times(3)).connect(false); 121 | } 122 | 123 | @Test 124 | public void shouldWaitIfComputerIsNull() throws InterruptedException, ExecutionException { 125 | PowerMockito.when(computer.isOnline()).thenReturn(true); 126 | 127 | PowerMockito.when(node.toComputer()) 128 | .thenReturn(null) 129 | .thenReturn(null) 130 | .thenReturn(computer); 131 | 132 | EC2FleetOnlineChecker.start(node, future, 100, 10); 133 | 134 | Assert.assertSame(node, future.get()); 135 | verify(computer, times(1)).isOnline(); 136 | } 137 | 138 | } 139 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/jenkins/ec2fleet/FleetStateStatsTest.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import com.amazonaws.services.ec2.AmazonEC2; 4 | import com.amazonaws.services.ec2.model.ActiveInstance; 5 | import com.amazonaws.services.ec2.model.BatchState; 6 | import com.amazonaws.services.ec2.model.DescribeSpotFleetInstancesRequest; 7 | import com.amazonaws.services.ec2.model.DescribeSpotFleetInstancesResult; 8 | import com.amazonaws.services.ec2.model.DescribeSpotFleetRequestsRequest; 9 | import com.amazonaws.services.ec2.model.DescribeSpotFleetRequestsResult; 10 | import com.amazonaws.services.ec2.model.LaunchTemplateConfig; 11 | import com.amazonaws.services.ec2.model.SpotFleetLaunchSpecification; 12 | import com.amazonaws.services.ec2.model.SpotFleetRequestConfig; 13 | import com.amazonaws.services.ec2.model.SpotFleetRequestConfigData; 14 | import com.google.common.collect.ImmutableMap; 15 | import com.google.common.collect.ImmutableSet; 16 | import org.junit.Assert; 17 | import org.junit.Before; 18 | import org.junit.Test; 19 | import org.junit.runner.RunWith; 20 | import org.mockito.Mock; 21 | import org.mockito.junit.MockitoJUnitRunner; 22 | 23 | import java.util.Collections; 24 | 25 | import static org.mockito.ArgumentMatchers.any; 26 | import static org.mockito.Mockito.verify; 27 | import static org.mockito.Mockito.when; 28 | 29 | @RunWith(MockitoJUnitRunner.class) 30 | public class FleetStateStatsTest { 31 | 32 | @Mock 33 | private AmazonEC2 ec2; 34 | 35 | @Before 36 | public void before() { 37 | when(ec2.describeSpotFleetInstances(any(DescribeSpotFleetInstancesRequest.class))) 38 | .thenReturn(new DescribeSpotFleetInstancesResult()); 39 | 40 | when(ec2.describeSpotFleetRequests(any(DescribeSpotFleetRequestsRequest.class))) 41 | .thenReturn(new DescribeSpotFleetRequestsResult() 42 | .withSpotFleetRequestConfigs( 43 | new SpotFleetRequestConfig() 44 | .withSpotFleetRequestConfig( 45 | new SpotFleetRequestConfigData() 46 | .withTargetCapacity(0)))); 47 | } 48 | 49 | @Test(expected = IllegalStateException.class) 50 | public void readClusterState_failIfNoFleet() { 51 | when(ec2.describeSpotFleetRequests(any(DescribeSpotFleetRequestsRequest.class))) 52 | .thenReturn(new DescribeSpotFleetRequestsResult()); 53 | 54 | FleetStateStats.readClusterState(ec2, "f", ""); 55 | } 56 | 57 | @Test 58 | public void readClusterState_returnFleetInfo() { 59 | when(ec2.describeSpotFleetRequests(any(DescribeSpotFleetRequestsRequest.class))) 60 | .thenReturn(new DescribeSpotFleetRequestsResult() 61 | .withSpotFleetRequestConfigs( 62 | new SpotFleetRequestConfig() 63 | .withSpotFleetRequestState(BatchState.Active) 64 | .withSpotFleetRequestConfig( 65 | new SpotFleetRequestConfigData() 66 | .withTargetCapacity(12)))); 67 | 68 | FleetStateStats stats = FleetStateStats.readClusterState(ec2, "f-id", ""); 69 | 70 | Assert.assertEquals("f-id", stats.getFleetId()); 71 | Assert.assertEquals("active", stats.getState()); 72 | Assert.assertEquals(12, stats.getNumDesired()); 73 | } 74 | 75 | @Test 76 | public void readClusterState_returnEmptyIfNoInstancesForFleet() { 77 | FleetStateStats stats = FleetStateStats.readClusterState(ec2, "f", ""); 78 | 79 | Assert.assertEquals(Collections.emptySet(), stats.getInstances()); 80 | Assert.assertEquals(0, stats.getNumActive()); 81 | } 82 | 83 | @Test 84 | public void readClusterState_returnAllDescribedInstancesForFleet() { 85 | when(ec2.describeSpotFleetInstances(any(DescribeSpotFleetInstancesRequest.class))) 86 | .thenReturn(new DescribeSpotFleetInstancesResult() 87 | .withActiveInstances( 88 | new ActiveInstance().withInstanceId("i-1"), 89 | new ActiveInstance().withInstanceId("i-2"))); 90 | 91 | FleetStateStats stats = FleetStateStats.readClusterState(ec2, "f", ""); 92 | 93 | Assert.assertEquals(ImmutableSet.of("i-1", "i-2"), stats.getInstances()); 94 | Assert.assertEquals(2, stats.getNumActive()); 95 | verify(ec2).describeSpotFleetInstances(new DescribeSpotFleetInstancesRequest() 96 | .withSpotFleetRequestId("f")); 97 | } 98 | 99 | @Test 100 | public void readClusterState_returnAllPagesDescribedInstancesForFleet() { 101 | when(ec2.describeSpotFleetInstances(any(DescribeSpotFleetInstancesRequest.class))) 102 | .thenReturn(new DescribeSpotFleetInstancesResult() 103 | .withNextToken("p1") 104 | .withActiveInstances(new ActiveInstance().withInstanceId("i-1"))) 105 | .thenReturn(new DescribeSpotFleetInstancesResult() 106 | .withActiveInstances(new ActiveInstance().withInstanceId("i-2"))); 107 | 108 | FleetStateStats stats = FleetStateStats.readClusterState(ec2, "f", ""); 109 | 110 | Assert.assertEquals(ImmutableSet.of("i-1", "i-2"), stats.getInstances()); 111 | Assert.assertEquals(2, stats.getNumActive()); 112 | verify(ec2).describeSpotFleetInstances(new DescribeSpotFleetInstancesRequest() 113 | .withSpotFleetRequestId("f").withNextToken("p1")); 114 | verify(ec2).describeSpotFleetInstances(new DescribeSpotFleetInstancesRequest() 115 | .withSpotFleetRequestId("f")); 116 | } 117 | 118 | @Test 119 | public void readClusterState_returnEmptyInstanceTypeWeightsIfNoInformation() { 120 | FleetStateStats stats = FleetStateStats.readClusterState(ec2, "f", ""); 121 | 122 | Assert.assertEquals(Collections.emptyMap(), stats.getInstanceTypeWeights()); 123 | } 124 | 125 | @Test 126 | public void readClusterState_returnInstanceTypeWeightsFromLaunchSpecification() { 127 | when(ec2.describeSpotFleetRequests(any(DescribeSpotFleetRequestsRequest.class))) 128 | .thenReturn(new DescribeSpotFleetRequestsResult() 129 | .withSpotFleetRequestConfigs(new SpotFleetRequestConfig() 130 | .withSpotFleetRequestState(BatchState.Active) 131 | .withSpotFleetRequestConfig(new SpotFleetRequestConfigData() 132 | .withTargetCapacity(1) 133 | .withLaunchSpecifications( 134 | new SpotFleetLaunchSpecification().withInstanceType("t1").withWeightedCapacity(0.1), 135 | new SpotFleetLaunchSpecification().withInstanceType("t2").withWeightedCapacity(12.0))))); 136 | 137 | FleetStateStats stats = FleetStateStats.readClusterState(ec2, "f", ""); 138 | 139 | Assert.assertEquals(ImmutableMap.of("t1", 0.1, "t2", 12.0), stats.getInstanceTypeWeights()); 140 | } 141 | 142 | @Test 143 | public void readClusterState_returnInstanceTypeWeightsForLaunchSpecificationIfItHasIt() { 144 | when(ec2.describeSpotFleetRequests(any(DescribeSpotFleetRequestsRequest.class))) 145 | .thenReturn(new DescribeSpotFleetRequestsResult() 146 | .withSpotFleetRequestConfigs(new SpotFleetRequestConfig() 147 | .withSpotFleetRequestState(BatchState.Active) 148 | .withSpotFleetRequestConfig(new SpotFleetRequestConfigData() 149 | .withTargetCapacity(1) 150 | .withLaunchSpecifications( 151 | new SpotFleetLaunchSpecification().withInstanceType("t1"), 152 | new SpotFleetLaunchSpecification().withWeightedCapacity(12.0))))); 153 | 154 | FleetStateStats stats = FleetStateStats.readClusterState(ec2, "f", ""); 155 | 156 | Assert.assertEquals(Collections.emptyMap(), stats.getInstanceTypeWeights()); 157 | } 158 | 159 | } 160 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/jenkins/ec2fleet/IdleRetentionStrategyTest.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import hudson.slaves.SlaveComputer; 4 | import org.junit.Before; 5 | import org.junit.Test; 6 | import org.junit.runner.RunWith; 7 | import org.mockito.Mock; 8 | import org.powermock.api.mockito.PowerMockito; 9 | import org.powermock.core.classloader.annotations.PrepareForTest; 10 | import org.powermock.modules.junit4.PowerMockRunner; 11 | 12 | import java.util.concurrent.TimeUnit; 13 | 14 | import static org.junit.Assert.assertEquals; 15 | import static org.junit.Assert.fail; 16 | import static org.mockito.ArgumentMatchers.anyString; 17 | import static org.mockito.Mockito.never; 18 | import static org.mockito.Mockito.times; 19 | import static org.mockito.Mockito.verify; 20 | import static org.mockito.Mockito.when; 21 | 22 | @RunWith(PowerMockRunner.class) 23 | @PrepareForTest(SlaveComputer.class) 24 | public class IdleRetentionStrategyTest { 25 | 26 | @Mock 27 | private EC2FleetCloud cloud; 28 | 29 | @Mock 30 | private EC2FleetNodeComputer slaveComputer; 31 | 32 | @Mock 33 | private EC2FleetNode slave; 34 | 35 | @Before 36 | public void before() { 37 | when(cloud.getIdleMinutes()).thenReturn(10); 38 | PowerMockito.when(slaveComputer.getIdleStartMilliseconds()).thenReturn(System.currentTimeMillis() - TimeUnit.MINUTES.toMillis(11)); 39 | when(slaveComputer.getNode()).thenReturn(slave); 40 | PowerMockito.when(slaveComputer.isIdle()).thenReturn(true); 41 | when(slave.getNodeName()).thenReturn("n-a"); 42 | when(slaveComputer.isAcceptingTasks()).thenReturn(true); 43 | when(slaveComputer.getCloud()).thenReturn(cloud); 44 | } 45 | 46 | @Test 47 | public void if_idle_time_not_configured_should_do_nothing() { 48 | when(cloud.getIdleMinutes()).thenReturn(0); 49 | 50 | new IdleRetentionStrategy().check(slaveComputer); 51 | 52 | verify(slaveComputer, times(0)).getNode(); 53 | verify(cloud, times(0)).scheduleToTerminate(anyString()); 54 | verify(slaveComputer).setAcceptingTasks(false); 55 | verify(slaveComputer).setAcceptingTasks(true); 56 | } 57 | 58 | @Test 59 | public void if_idle_time_configured_should_do_nothing_if_node_idle_less_time() { 60 | when(slaveComputer.getIdleStartMilliseconds()).thenReturn(System.currentTimeMillis()); 61 | 62 | new IdleRetentionStrategy().check(slaveComputer); 63 | 64 | verify(slaveComputer, never()).getNode(); 65 | verify(cloud, never()).scheduleToTerminate(anyString()); 66 | verify(slaveComputer).setAcceptingTasks(false); 67 | verify(slaveComputer).setAcceptingTasks(true); 68 | } 69 | 70 | @Test 71 | public void if_node_not_execute_anything_yet_idle_time_negative_do_nothing() { 72 | when(slaveComputer.getIdleStartMilliseconds()).thenReturn(Long.MIN_VALUE); 73 | 74 | new IdleRetentionStrategy().check(slaveComputer); 75 | 76 | verify(slaveComputer, times(0)).getNode(); 77 | verify(cloud, times(0)).scheduleToTerminate(anyString()); 78 | verify(slaveComputer).setAcceptingTasks(false); 79 | verify(slaveComputer).setAcceptingTasks(true); 80 | } 81 | 82 | @Test 83 | public void if_idle_time_configured_should_terminate_node_if_idle_time_more_then_allowed() { 84 | new IdleRetentionStrategy().check(slaveComputer); 85 | 86 | verify(cloud, times(1)).scheduleToTerminate("n-a"); 87 | verify(slaveComputer, times(1)).setAcceptingTasks(true); 88 | verify(slaveComputer, times(1)).setAcceptingTasks(false); 89 | } 90 | 91 | @Test 92 | public void if_computer_has_no_cloud_should_do_nothing() { 93 | when(slaveComputer.getCloud()).thenReturn(null); 94 | 95 | new IdleRetentionStrategy().check(slaveComputer); 96 | 97 | verify(cloud, times(0)).scheduleToTerminate(anyString()); 98 | verify(slaveComputer, times(0)).setAcceptingTasks(true); 99 | verify(slaveComputer, times(0)).setAcceptingTasks(false); 100 | } 101 | 102 | @Test 103 | public void if_node_not_idle_should_do_nothing() { 104 | when(slaveComputer.getIdleStartMilliseconds()).thenReturn(0L); 105 | when(slaveComputer.isIdle()).thenReturn(false); 106 | 107 | new IdleRetentionStrategy().check(slaveComputer); 108 | 109 | verify(cloud, never()).scheduleToTerminate("n-a"); 110 | verify(slaveComputer, times(1)).setAcceptingTasks(true); 111 | verify(slaveComputer, times(1)).setAcceptingTasks(false); 112 | } 113 | 114 | @Test 115 | public void if_node_idle_time_more_them_allowed_but_not_idle_should_do_nothing() { 116 | when(slaveComputer.isIdle()).thenReturn(false); 117 | 118 | new IdleRetentionStrategy().check(slaveComputer); 119 | 120 | verify(cloud, never()).scheduleToTerminate("n-a"); 121 | verify(slaveComputer, times(1)).setAcceptingTasks(true); 122 | verify(slaveComputer, times(1)).setAcceptingTasks(false); 123 | } 124 | 125 | @Test 126 | public void if_exception_happen_during_termination_should_throw_it_and_restore_task_accepting() { 127 | when(cloud.scheduleToTerminate(anyString())).thenThrow(new IllegalArgumentException("test")); 128 | 129 | try { 130 | new IdleRetentionStrategy().check(slaveComputer); 131 | fail(); 132 | } catch (IllegalArgumentException e) { 133 | assertEquals("test", e.getMessage()); 134 | verify(cloud, times(1)).scheduleToTerminate("n-a"); 135 | verify(slaveComputer).setAcceptingTasks(false); 136 | verify(slaveComputer).setAcceptingTasks(true); 137 | } 138 | } 139 | 140 | // todo we do nothing if computer doesn't have node 141 | 142 | } 143 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/jenkins/ec2fleet/LazyUuidTest.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import org.junit.Assert; 4 | import org.junit.Test; 5 | 6 | import java.util.HashSet; 7 | import java.util.Set; 8 | import java.util.concurrent.TimeUnit; 9 | 10 | public class LazyUuidTest { 11 | 12 | @Test 13 | public void getValue_provides_uuid() { 14 | Assert.assertEquals(36, new LazyUuid().getValue().length()); 15 | } 16 | 17 | @Test 18 | public void getValue_provides_same_value_if_multiplecall() { 19 | final LazyUuid lazyUuid = new LazyUuid(); 20 | Assert.assertEquals(lazyUuid.getValue(), lazyUuid.getValue()); 21 | Assert.assertEquals(lazyUuid.getValue(), lazyUuid.getValue()); 22 | } 23 | 24 | @Test 25 | public void getValue_is_thread_safe() throws InterruptedException { 26 | final LazyUuid lazyUuid = new LazyUuid(); 27 | 28 | final LazyUuidGetter[] threads = new LazyUuidGetter[3]; 29 | 30 | for (int i = 0; i < threads.length; i++) { 31 | threads[i] = new LazyUuidGetter(lazyUuid); 32 | threads[i].start(); 33 | } 34 | 35 | Thread.sleep(TimeUnit.SECONDS.toMillis(10)); 36 | 37 | final Set history = new HashSet<>(); 38 | 39 | for (final LazyUuidGetter thread : threads) { 40 | thread.interrupt(); 41 | 42 | history.addAll(thread.history); 43 | } 44 | 45 | Assert.assertEquals(1, history.size()); 46 | Assert.assertNotNull(history.iterator().next()); 47 | } 48 | 49 | private static class LazyUuidGetter extends Thread { 50 | 51 | private final Set history; 52 | private final LazyUuid lazyUuid; 53 | 54 | LazyUuidGetter(LazyUuid lazyUuid) { 55 | this.lazyUuid = lazyUuid; 56 | history = new HashSet<>(); 57 | } 58 | 59 | @Override 60 | public void run() { 61 | while (true) { 62 | history.add(lazyUuid.getValue()); 63 | 64 | try { 65 | Thread.sleep(100); 66 | } catch (InterruptedException e) { 67 | return; // stop 68 | } 69 | } 70 | } 71 | } 72 | } 73 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/jenkins/ec2fleet/LocalComputerConnector.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import hudson.model.TaskListener; 4 | import hudson.slaves.ComputerConnector; 5 | import hudson.slaves.ComputerLauncher; 6 | import org.jvnet.hudson.test.JenkinsRule; 7 | 8 | import javax.annotation.Nonnull; 9 | import java.io.IOException; 10 | import java.io.Serializable; 11 | import java.net.URISyntaxException; 12 | 13 | /** 14 | * For testing only. 15 | * 16 | * @see AutoResubmitIntegrationTest 17 | */ 18 | class LocalComputerConnector extends ComputerConnector implements Serializable { 19 | 20 | @Nonnull 21 | private transient final JenkinsRule j; 22 | 23 | LocalComputerConnector(final JenkinsRule j) { 24 | this.j = j; 25 | } 26 | 27 | @Override 28 | public ComputerLauncher launch(@Nonnull String host, TaskListener listener) throws IOException { 29 | System.out.println("Creating computer launcher"); 30 | try { 31 | return j.createComputerLauncher(null); 32 | } catch (URISyntaxException e) { 33 | throw new RuntimeException(e); 34 | } 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/jenkins/ec2fleet/Meter.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import javax.annotation.concurrent.ThreadSafe; 4 | import java.io.Closeable; 5 | import java.util.concurrent.TimeUnit; 6 | import java.util.concurrent.atomic.AtomicInteger; 7 | import java.util.concurrent.atomic.AtomicLong; 8 | 9 | @ThreadSafe 10 | public class Meter { 11 | 12 | private final String name; 13 | private final AtomicInteger count = new AtomicInteger(); 14 | private final AtomicLong total = new AtomicLong(); 15 | 16 | public Meter(final String name) { 17 | this.name = name; 18 | } 19 | 20 | public Shot start() { 21 | return new Shot(this); 22 | } 23 | 24 | private void time(long time) { 25 | count.incrementAndGet(); 26 | total.addAndGet(time); 27 | } 28 | 29 | public String toString() { 30 | final long tempTotal = total.get(); 31 | final int tempCount = count.get(); 32 | return name + " meter, " + 33 | "total " + TimeUnit.MILLISECONDS.toSeconds(tempTotal) + ", sec" + 34 | " count " + tempCount + 35 | " avg " + (tempCount == 0 ? "~" : tempTotal / tempCount) + " msec"; 36 | } 37 | 38 | @ThreadSafe 39 | public static class Shot implements Closeable { 40 | 41 | private final long start; 42 | private final Meter meter; 43 | 44 | private Shot(Meter meter) { 45 | this.start = System.currentTimeMillis(); 46 | this.meter = meter; 47 | } 48 | 49 | public void close() { 50 | meter.time(System.currentTimeMillis() - start); 51 | } 52 | 53 | } 54 | 55 | } 56 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/jenkins/ec2fleet/NoDelayProvisionStrategyPerformanceTest.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import com.amazonaws.services.ec2.model.InstanceStateName; 4 | import hudson.model.FreeStyleBuild; 5 | import hudson.model.queue.QueueTaskFuture; 6 | import hudson.slaves.ComputerConnector; 7 | import hudson.slaves.NodeProvisioner; 8 | import org.apache.commons.lang3.tuple.ImmutableTriple; 9 | import org.junit.Assert; 10 | import org.junit.BeforeClass; 11 | import org.junit.Ignore; 12 | import org.junit.Test; 13 | 14 | import java.io.IOException; 15 | import java.util.ArrayList; 16 | import java.util.Date; 17 | import java.util.List; 18 | import java.util.concurrent.ExecutionException; 19 | import java.util.concurrent.TimeUnit; 20 | 21 | /** 22 | * https://support.cloudbees.com/hc/en-us/articles/115000060512-New-Shared-Agents-Clouds-are-not-being-provisioned-for-my-jobs-in-the-queue-when-I-have-agents-that-are-suspended 23 | *

    24 | * Run example: 25 | * https://docs.google.com/spreadsheets/d/e/2PACX-1vSuPWeDJD8xAbvHpyJPigAIMYJyL0YvljjAatutNqaFqUQofTx2PxY-sfqZgfsWqRxMGl2elJErbH5n/pubchart?oid=983520837&format=interactive 26 | */ 27 | @Ignore 28 | public class NoDelayProvisionStrategyPerformanceTest extends IntegrationTest { 29 | 30 | @BeforeClass 31 | public static void beforeClass() { 32 | // zero is unlimited timeout 33 | System.setProperty("jenkins.test.timeout", "0"); 34 | // set default MARGIN for Jenkins 35 | System.setProperty(NodeProvisioner.class.getName() + ".MARGIN", Integer.toString(10)); 36 | } 37 | 38 | @Test 39 | public void noDelayProvisionStrategy() throws Exception { 40 | test(true); 41 | } 42 | 43 | @Test 44 | public void defaultProvisionStrategy() throws Exception { 45 | test(false); 46 | } 47 | 48 | private void test(final boolean noDelay) throws IOException, InterruptedException { 49 | final int maxWorkers = 100; 50 | final int scheduleInterval = 15; 51 | final int batchSize = 9; 52 | 53 | mockEc2ApiToDescribeInstancesWhenModifiedWithDelay(InstanceStateName.Running, 500); 54 | 55 | final ComputerConnector computerConnector = new LocalComputerConnector(j); 56 | final String label = "momo"; 57 | final EC2FleetCloudWithHistory cloud = new EC2FleetCloudWithHistory(null, null, "credId", null, "region", 58 | null, "fId", label, null, computerConnector, false, false, 59 | 1, 0, maxWorkers, 1, true, false, 60 | false, 0, 0, false, 61 | 15, noDelay); 62 | j.jenkins.clouds.add(cloud); 63 | 64 | System.out.println("waiting cloud start"); 65 | // updated plugin requires some init time to get first update 66 | // so wait this event to be really correct with perf comparison as old version is not require init time 67 | tryUntil(new Runnable() { 68 | @Override 69 | public void run() { 70 | Assert.assertNotNull(cloud.getStats()); 71 | } 72 | }); 73 | 74 | // warm up jenkins queue, as it takes some time when jenkins run first task and start scale in/out 75 | // so let's run one task and wait it finish 76 | System.out.println("waiting warm up task execution"); 77 | final List warmUpTasks = getQueueTaskFutures(1); 78 | waitTasksFinish(warmUpTasks); 79 | 80 | final List> metrics = new ArrayList<>(); 81 | final Thread monitor = new Thread(new Runnable() { 82 | 83 | @Override 84 | public void run() { 85 | while (!Thread.interrupted()) { 86 | final int queueSize = j.jenkins.getQueue().countBuildableItems() // tasks to build 87 | + j.jenkins.getQueue().getPendingItems().size() // tasks to start 88 | + j.jenkins.getLabelAtom(label).getBusyExecutors(); // tasks in progress 89 | final int executors = j.jenkins.getLabelAtom(label).getTotalExecutors(); 90 | final ImmutableTriple data = new ImmutableTriple<>( 91 | System.currentTimeMillis(), queueSize, executors); 92 | metrics.add(data); 93 | System.out.println(new Date(data.left) + " " + data.middle + " " + data.right); 94 | 95 | try { 96 | Thread.sleep(TimeUnit.SECONDS.toMillis(5)); 97 | } catch (InterruptedException e) { 98 | throw new RuntimeException("stopped"); 99 | } 100 | } 101 | } 102 | }); 103 | monitor.start(); 104 | 105 | System.out.println("start test"); 106 | int taskCount = 0; 107 | final List tasks = new ArrayList<>(); 108 | for (int i = 0; i < 15; i++) { 109 | tasks.addAll((List) getQueueTaskFutures(batchSize)); 110 | taskCount += batchSize; 111 | System.out.println("schedule " + taskCount + " tasks, waiting " + scheduleInterval + " sec"); 112 | Thread.sleep(TimeUnit.SECONDS.toMillis(scheduleInterval)); 113 | } 114 | 115 | waitTasksFinish(tasks); 116 | 117 | monitor.interrupt(); 118 | monitor.join(); 119 | 120 | for (ImmutableTriple data : metrics) { 121 | System.out.println(data.middle + " " + data.right); 122 | } 123 | } 124 | 125 | private static void waitTasksFinish(List tasks) { 126 | for (final QueueTaskFuture task : tasks) { 127 | try { 128 | task.get(); 129 | } catch (InterruptedException | ExecutionException e) { 130 | throw new RuntimeException(e); 131 | } 132 | } 133 | } 134 | 135 | } 136 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/jenkins/ec2fleet/NoDelayProvisionStrategyTest.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import hudson.model.Label; 4 | import hudson.model.LoadStatistics; 5 | import hudson.slaves.Cloud; 6 | import hudson.slaves.NodeProvisioner; 7 | import org.junit.Assert; 8 | import org.junit.Before; 9 | import org.junit.Test; 10 | import org.junit.runner.RunWith; 11 | import org.mockito.Mock; 12 | import org.powermock.core.classloader.annotations.PrepareForTest; 13 | import org.powermock.modules.junit4.PowerMockRunner; 14 | 15 | import java.util.ArrayList; 16 | import java.util.Arrays; 17 | import java.util.List; 18 | 19 | import static org.mockito.ArgumentMatchers.any; 20 | import static org.mockito.ArgumentMatchers.anyInt; 21 | import static org.mockito.Mockito.mock; 22 | import static org.mockito.Mockito.never; 23 | import static org.mockito.Mockito.spy; 24 | import static org.mockito.Mockito.times; 25 | import static org.mockito.Mockito.verify; 26 | import static org.mockito.Mockito.when; 27 | 28 | @RunWith(PowerMockRunner.class) 29 | @PrepareForTest({NodeProvisioner.StrategyState.class}) 30 | public class NoDelayProvisionStrategyTest { 31 | 32 | @Mock 33 | private NodeProvisioner.StrategyState state; 34 | 35 | @Mock 36 | private LoadStatistics.LoadStatisticsSnapshot snapshot; 37 | 38 | @Mock 39 | private Label label; 40 | 41 | private NoDelayProvisionStrategy strategy; 42 | 43 | private List clouds = new ArrayList<>(); 44 | 45 | @Before 46 | public void before() { 47 | strategy = spy(new NoDelayProvisionStrategy()); 48 | when(strategy.getClouds()).thenReturn(clouds); 49 | when(state.getSnapshot()).thenReturn(snapshot); 50 | } 51 | 52 | @Test 53 | public void givenNoRequiredCapacity_shouldDoNotScale() { 54 | final EC2FleetCloud ec2FleetCloud = mock(EC2FleetCloud.class); 55 | clouds.add(ec2FleetCloud); 56 | 57 | strategy.apply(state); 58 | 59 | verify(ec2FleetCloud, never()).canProvision(any(Label.class)); 60 | } 61 | 62 | @Test 63 | public void givenAvailableSameAsRequiredCapacity_shouldDoNotScale() { 64 | final EC2FleetCloud ec2FleetCloud = mock(EC2FleetCloud.class); 65 | clouds.add(ec2FleetCloud); 66 | when(snapshot.getQueueLength()).thenReturn(10); 67 | when(snapshot.getAvailableExecutors()).thenReturn(10); 68 | 69 | Assert.assertEquals( 70 | NodeProvisioner.StrategyDecision.PROVISIONING_COMPLETED, 71 | strategy.apply(state)); 72 | 73 | verify(ec2FleetCloud, never()).canProvision(any(Label.class)); 74 | } 75 | 76 | @Test 77 | public void givenNoEC2Cloud_shouldDoNotScale() { 78 | when(snapshot.getQueueLength()).thenReturn(10); 79 | 80 | Assert.assertEquals( 81 | NodeProvisioner.StrategyDecision.CONSULT_REMAINING_STRATEGIES, 82 | strategy.apply(state)); 83 | } 84 | 85 | @Test 86 | public void givenNonEC2Cloud_shouldDoNotScale() { 87 | when(snapshot.getQueueLength()).thenReturn(10); 88 | clouds.add(mock(Cloud.class)); 89 | 90 | Assert.assertEquals( 91 | NodeProvisioner.StrategyDecision.CONSULT_REMAINING_STRATEGIES, 92 | strategy.apply(state)); 93 | } 94 | 95 | @Test 96 | public void givenEC2CloudWithDisabledNoDelay_shouldDoNotScale() { 97 | when(snapshot.getQueueLength()).thenReturn(10); 98 | when(state.getLabel()).thenReturn(label); 99 | 100 | final EC2FleetCloud ec2FleetCloud = mock(EC2FleetCloud.class); 101 | clouds.add(ec2FleetCloud); 102 | when(ec2FleetCloud.canProvision(any(Label.class))).thenReturn(true); 103 | when(ec2FleetCloud.isNoDelayProvision()).thenReturn(false); 104 | 105 | Assert.assertEquals( 106 | NodeProvisioner.StrategyDecision.CONSULT_REMAINING_STRATEGIES, 107 | strategy.apply(state)); 108 | verify(ec2FleetCloud, never()).provision(any(Label.class), anyInt()); 109 | } 110 | 111 | @Test 112 | public void givenEC2CloudWhichCannotProvision_shouldDoNotScale() { 113 | when(snapshot.getQueueLength()).thenReturn(10); 114 | when(state.getLabel()).thenReturn(label); 115 | 116 | final EC2FleetCloud ec2FleetCloud = mock(EC2FleetCloud.class); 117 | clouds.add(ec2FleetCloud); 118 | when(ec2FleetCloud.canProvision(any(Label.class))).thenReturn(false); 119 | 120 | Assert.assertEquals( 121 | NodeProvisioner.StrategyDecision.CONSULT_REMAINING_STRATEGIES, 122 | strategy.apply(state)); 123 | verify(ec2FleetCloud, never()).provision(any(Label.class), anyInt()); 124 | } 125 | 126 | @Test 127 | public void givenEC2CloudsWithEnabledNoDelayAndWithout_shouldDoScalingForOne() { 128 | when(snapshot.getQueueLength()).thenReturn(10); 129 | when(state.getLabel()).thenReturn(label); 130 | 131 | final EC2FleetCloud ec2FleetCloud1 = mock(EC2FleetCloud.class); 132 | clouds.add(ec2FleetCloud1); 133 | final EC2FleetCloud ec2FleetCloud2 = mock(EC2FleetCloud.class); 134 | clouds.add(ec2FleetCloud2); 135 | when(ec2FleetCloud1.canProvision(any(Label.class))).thenReturn(true); 136 | when(ec2FleetCloud2.canProvision(any(Label.class))).thenReturn(true); 137 | when(ec2FleetCloud1.isNoDelayProvision()).thenReturn(true); 138 | when(ec2FleetCloud2.isNoDelayProvision()).thenReturn(false); 139 | 140 | Assert.assertEquals( 141 | NodeProvisioner.StrategyDecision.CONSULT_REMAINING_STRATEGIES, 142 | strategy.apply(state)); 143 | verify(ec2FleetCloud1, times(1)).provision(label, 10); 144 | verify(ec2FleetCloud2, never()).provision(any(Label.class), anyInt()); 145 | } 146 | 147 | @Test 148 | public void givenEC2CloudsWhenOneCanCoverCapacity_shouldDoScalingForFirstOnly() { 149 | when(snapshot.getQueueLength()).thenReturn(2); 150 | when(state.getLabel()).thenReturn(label); 151 | 152 | final EC2FleetCloud ec2FleetCloud1 = mock(EC2FleetCloud.class); 153 | clouds.add(ec2FleetCloud1); 154 | final EC2FleetCloud ec2FleetCloud2 = mock(EC2FleetCloud.class); 155 | clouds.add(ec2FleetCloud2); 156 | when(ec2FleetCloud1.canProvision(any(Label.class))).thenReturn(true); 157 | when(ec2FleetCloud2.canProvision(any(Label.class))).thenReturn(true); 158 | when(ec2FleetCloud1.isNoDelayProvision()).thenReturn(true); 159 | when(ec2FleetCloud2.isNoDelayProvision()).thenReturn(true); 160 | when(ec2FleetCloud1.provision(any(Label.class), anyInt())).thenReturn(Arrays.asList( 161 | mock(NodeProvisioner.PlannedNode.class), 162 | mock(NodeProvisioner.PlannedNode.class) 163 | )); 164 | 165 | Assert.assertEquals( 166 | NodeProvisioner.StrategyDecision.PROVISIONING_COMPLETED, 167 | strategy.apply(state)); 168 | verify(ec2FleetCloud1, times(1)).provision(label, 2); 169 | verify(ec2FleetCloud2, never()).provision(any(Label.class), anyInt()); 170 | } 171 | 172 | @Test 173 | public void givenEC2Clouds_shouldDoScalingAndReduceForNextOne() { 174 | when(snapshot.getQueueLength()).thenReturn(5); 175 | when(state.getLabel()).thenReturn(label); 176 | 177 | final EC2FleetCloud ec2FleetCloud1 = mock(EC2FleetCloud.class); 178 | clouds.add(ec2FleetCloud1); 179 | final EC2FleetCloud ec2FleetCloud2 = mock(EC2FleetCloud.class); 180 | clouds.add(ec2FleetCloud2); 181 | when(ec2FleetCloud1.canProvision(any(Label.class))).thenReturn(true); 182 | when(ec2FleetCloud2.canProvision(any(Label.class))).thenReturn(true); 183 | when(ec2FleetCloud1.isNoDelayProvision()).thenReturn(true); 184 | when(ec2FleetCloud2.isNoDelayProvision()).thenReturn(true); 185 | when(ec2FleetCloud1.provision(any(Label.class), anyInt())).thenReturn(Arrays.asList( 186 | mock(NodeProvisioner.PlannedNode.class), 187 | mock(NodeProvisioner.PlannedNode.class) 188 | )); 189 | 190 | Assert.assertEquals( 191 | NodeProvisioner.StrategyDecision.CONSULT_REMAINING_STRATEGIES, 192 | strategy.apply(state)); 193 | verify(ec2FleetCloud1, times(1)).provision(label, 5); 194 | verify(ec2FleetCloud2, times(1)).provision(label, 3); 195 | } 196 | 197 | } 198 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/jenkins/ec2fleet/ProvisionIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import com.amazonaws.services.ec2.AmazonEC2; 4 | import com.amazonaws.services.ec2.model.ActiveInstance; 5 | import com.amazonaws.services.ec2.model.DescribeInstancesRequest; 6 | import com.amazonaws.services.ec2.model.DescribeInstancesResult; 7 | import com.amazonaws.services.ec2.model.DescribeSpotFleetInstancesRequest; 8 | import com.amazonaws.services.ec2.model.DescribeSpotFleetInstancesResult; 9 | import com.amazonaws.services.ec2.model.DescribeSpotFleetRequestsRequest; 10 | import com.amazonaws.services.ec2.model.DescribeSpotFleetRequestsResult; 11 | import com.amazonaws.services.ec2.model.Instance; 12 | import com.amazonaws.services.ec2.model.InstanceState; 13 | import com.amazonaws.services.ec2.model.InstanceStateName; 14 | import com.amazonaws.services.ec2.model.Reservation; 15 | import com.amazonaws.services.ec2.model.SpotFleetRequestConfig; 16 | import com.amazonaws.services.ec2.model.SpotFleetRequestConfigData; 17 | import com.google.common.collect.ImmutableSet; 18 | import hudson.model.Computer; 19 | import hudson.model.FreeStyleBuild; 20 | import hudson.model.Label; 21 | import hudson.model.Node; 22 | import hudson.model.Result; 23 | import hudson.model.TaskListener; 24 | import hudson.model.queue.QueueTaskFuture; 25 | import hudson.slaves.ComputerConnector; 26 | import hudson.slaves.ComputerLauncher; 27 | import org.hamcrest.Matchers; 28 | import org.junit.Assert; 29 | import org.junit.BeforeClass; 30 | import org.junit.Test; 31 | import org.mockito.Mockito; 32 | 33 | import java.io.IOException; 34 | import java.util.ArrayList; 35 | import java.util.Arrays; 36 | import java.util.Collections; 37 | import java.util.List; 38 | import java.util.concurrent.ExecutionException; 39 | import java.util.concurrent.TimeUnit; 40 | 41 | import static org.mockito.ArgumentMatchers.any; 42 | import static org.mockito.ArgumentMatchers.anyInt; 43 | import static org.mockito.ArgumentMatchers.anyString; 44 | import static org.mockito.Mockito.atLeast; 45 | import static org.mockito.Mockito.mock; 46 | import static org.mockito.Mockito.spy; 47 | import static org.mockito.Mockito.times; 48 | import static org.mockito.Mockito.verify; 49 | import static org.mockito.Mockito.when; 50 | 51 | public class ProvisionIntegrationTest extends IntegrationTest { 52 | 53 | @BeforeClass 54 | public static void beforeClass() { 55 | System.setProperty("jenkins.test.timeout", "720"); 56 | } 57 | 58 | @Test 59 | public void dont_provide_any_planned_if_empty_and_reached_max_capacity() throws Exception { 60 | ComputerLauncher computerLauncher = mock(ComputerLauncher.class); 61 | ComputerConnector computerConnector = mock(ComputerConnector.class); 62 | when(computerConnector.launch(anyString(), any(TaskListener.class))).thenReturn(computerLauncher); 63 | 64 | EC2FleetCloud cloud = new EC2FleetCloud(null, null, "credId", null, "region", 65 | null, "fId", "momo", null, computerConnector, false, false, 66 | 0, 0, 0, 1, false, false, 67 | false, 0, 0, false, 68 | 2, false); 69 | j.jenkins.clouds.add(cloud); 70 | 71 | EC2Api ec2Api = spy(EC2Api.class); 72 | Registry.setEc2Api(ec2Api); 73 | 74 | AmazonEC2 amazonEC2 = mock(AmazonEC2.class); 75 | when(ec2Api.connect(anyString(), anyString(), Mockito.nullable(String.class))).thenReturn(amazonEC2); 76 | 77 | when(amazonEC2.describeSpotFleetInstances(any(DescribeSpotFleetInstancesRequest.class))) 78 | .thenReturn(new DescribeSpotFleetInstancesResult()); 79 | 80 | DescribeSpotFleetRequestsResult describeSpotFleetRequestsResult = new DescribeSpotFleetRequestsResult(); 81 | describeSpotFleetRequestsResult.setSpotFleetRequestConfigs(Arrays.asList( 82 | new SpotFleetRequestConfig() 83 | .withSpotFleetRequestState("active") 84 | .withSpotFleetRequestConfig( 85 | new SpotFleetRequestConfigData().withTargetCapacity(0)))); 86 | when(amazonEC2.describeSpotFleetRequests(any(DescribeSpotFleetRequestsRequest.class))) 87 | .thenReturn(describeSpotFleetRequestsResult); 88 | 89 | List rs = getQueueTaskFutures(5); 90 | 91 | Assert.assertEquals(0, j.jenkins.getNodes().size()); 92 | 93 | triggerSuggestReviewNow("momo"); 94 | 95 | Thread.sleep(TimeUnit.SECONDS.toMillis(30)); 96 | 97 | Assert.assertEquals(0, j.jenkins.getNodes().size()); 98 | 99 | cancelTasks(rs); 100 | } 101 | 102 | @Test 103 | public void should_add_planned_if_capacity_required_but_not_described_yet() throws Exception { 104 | ComputerLauncher computerLauncher = mock(ComputerLauncher.class); 105 | ComputerConnector computerConnector = mock(ComputerConnector.class); 106 | when(computerConnector.launch(anyString(), any(TaskListener.class))).thenReturn(computerLauncher); 107 | 108 | mockEc2ApiToDescribeFleetNotInstanceWhenModified(); 109 | 110 | EC2FleetCloud cloud = new EC2FleetCloud(null, null, "credId", null, "region", 111 | null, "fId", "momo", null, computerConnector, false, false, 112 | 0, 0, 10, 1, false, false, 113 | false, 0, 0, false, 114 | 2, false); 115 | j.jenkins.clouds.add(cloud); 116 | 117 | List rs = getQueueTaskFutures(1); 118 | 119 | triggerSuggestReviewNow("momo"); 120 | 121 | Assert.assertEquals(0, j.jenkins.getNodes().size()); 122 | 123 | tryUntil(new Runnable() { 124 | @Override 125 | public void run() { 126 | Assert.assertEquals(0, j.jenkins.getNodes().size()); 127 | Assert.assertEquals(2, j.jenkins.getLabels().size()); 128 | Assert.assertEquals(1, j.jenkins.getLabelAtom("momo").nodeProvisioner.getPendingLaunches().size()); 129 | } 130 | }); 131 | 132 | cancelTasks(rs); 133 | } 134 | 135 | @Test 136 | public void should_keep_planned_node_until_node_will_not_be_online_so_jenkins_will_not_request_overprovision() throws Exception { 137 | ComputerLauncher computerLauncher = mock(ComputerLauncher.class); 138 | ComputerConnector computerConnector = mock(ComputerConnector.class); 139 | when(computerConnector.launch(anyString(), any(TaskListener.class))).thenReturn(computerLauncher); 140 | 141 | EC2FleetCloud cloud = spy(new EC2FleetCloud(null, null, "credId", null, "region", 142 | null, "fId", "momo", null, computerConnector, false, false, 143 | 0, 0, 10, 1, false, false, 144 | false, 300, 15, false, 145 | 2, false)); 146 | 147 | // provide init state 148 | cloud.setStats(new FleetStateStats("", 0, "active", 149 | Collections.emptySet(), Collections.emptyMap())); 150 | 151 | j.jenkins.clouds.add(cloud); 152 | 153 | mockEc2ApiToDescribeInstancesWhenModified(InstanceStateName.Running); 154 | 155 | List rs = getQueueTaskFutures(1); 156 | 157 | final String labelString = "momo"; 158 | triggerSuggestReviewNow(labelString); 159 | 160 | Thread.sleep(TimeUnit.MINUTES.toMillis(2)); 161 | 162 | verify(cloud, times(1)).provision(any(Label.class), anyInt()); 163 | 164 | cancelTasks(rs); 165 | } 166 | 167 | @Test 168 | public void should_not_keep_planned_node_if_configured_so_jenkins_will_overprovision() throws Exception { 169 | ComputerLauncher computerLauncher = mock(ComputerLauncher.class); 170 | ComputerConnector computerConnector = mock(ComputerConnector.class); 171 | when(computerConnector.launch(anyString(), any(TaskListener.class))).thenReturn(computerLauncher); 172 | 173 | final EC2FleetCloud cloud = spy(new EC2FleetCloud(null, null, "credId", null, "region", 174 | null, "fId", "momo", null, computerConnector, false, false, 175 | 0, 0, 10, 1, false, false, 176 | false, 0, 0, false, 177 | 10, false)); 178 | j.jenkins.clouds.add(cloud); 179 | 180 | mockEc2ApiToDescribeInstancesWhenModified(InstanceStateName.Running); 181 | 182 | getQueueTaskFutures(1); 183 | 184 | tryUntil(new Runnable() { 185 | @Override 186 | public void run() { 187 | j.jenkins.getLabelAtom("momo").nodeProvisioner.suggestReviewNow(); 188 | verify(cloud, atLeast(2)).provision(any(Label.class), anyInt()); 189 | } 190 | }); 191 | } 192 | 193 | @Test 194 | public void should_not_allow_jenkins_to_provision_if_address_not_available() throws Exception { 195 | ComputerLauncher computerLauncher = mock(ComputerLauncher.class); 196 | ComputerConnector computerConnector = mock(ComputerConnector.class); 197 | when(computerConnector.launch(anyString(), any(TaskListener.class))).thenReturn(computerLauncher); 198 | 199 | EC2FleetCloud cloud = spy(new EC2FleetCloud(null, null, "credId", null, "region", 200 | null, "fId", "momo", null, computerConnector, false, false, 201 | 0, 0, 10, 1, false, false, 202 | false, 0, 0, false, 203 | 10, false)); 204 | 205 | cloud.setStats(new FleetStateStats("", 0, "active", 206 | Collections.emptySet(), Collections.emptyMap())); 207 | 208 | j.jenkins.clouds.add(cloud); 209 | 210 | EC2Api ec2Api = spy(EC2Api.class); 211 | Registry.setEc2Api(ec2Api); 212 | 213 | AmazonEC2 amazonEC2 = mock(AmazonEC2.class); 214 | when(ec2Api.connect(anyString(), anyString(), Mockito.nullable(String.class))).thenReturn(amazonEC2); 215 | 216 | when(amazonEC2.describeInstances(any(DescribeInstancesRequest.class))).thenReturn( 217 | new DescribeInstancesResult().withReservations( 218 | new Reservation().withInstances( 219 | new Instance() 220 | .withState(new InstanceState().withName(InstanceStateName.Running)) 221 | // .withPublicIpAddress("public-io") 222 | .withInstanceId("i-1") 223 | ))); 224 | 225 | when(amazonEC2.describeSpotFleetInstances(any(DescribeSpotFleetInstancesRequest.class))).thenReturn( 226 | new DescribeSpotFleetInstancesResult().withActiveInstances(new ActiveInstance().withInstanceId("i-1"))); 227 | 228 | DescribeSpotFleetRequestsResult describeSpotFleetRequestsResult = new DescribeSpotFleetRequestsResult(); 229 | describeSpotFleetRequestsResult.setSpotFleetRequestConfigs(Arrays.asList( 230 | new SpotFleetRequestConfig() 231 | .withSpotFleetRequestState("active") 232 | .withSpotFleetRequestConfig( 233 | new SpotFleetRequestConfigData().withTargetCapacity(1)))); 234 | when(amazonEC2.describeSpotFleetRequests(any(DescribeSpotFleetRequestsRequest.class))) 235 | .thenReturn(describeSpotFleetRequestsResult); 236 | 237 | List rs = getQueueTaskFutures(1); 238 | 239 | j.jenkins.getLabelAtom("momo").nodeProvisioner.suggestReviewNow(); 240 | 241 | Assert.assertEquals(0, j.jenkins.getNodes().size()); 242 | 243 | Thread.sleep(TimeUnit.MINUTES.toMillis(2)); 244 | 245 | cancelTasks(rs); 246 | 247 | verify(cloud, times(1)).provision(any(Label.class), anyInt()); 248 | } 249 | 250 | @Test 251 | public void should_not_convert_planned_to_node_if_state_is_not_running_and_check_state_enabled() throws Exception { 252 | ComputerLauncher computerLauncher = mock(ComputerLauncher.class); 253 | ComputerConnector computerConnector = mock(ComputerConnector.class); 254 | when(computerConnector.launch(anyString(), any(TaskListener.class))).thenReturn(computerLauncher); 255 | 256 | EC2FleetCloud cloud = new EC2FleetCloud(null, null, "credId", null, "region", 257 | null, "fId", "momo", null, computerConnector, false, false, 258 | 0, 0, 10, 1, true, false, 259 | false, 0, 0, false, 260 | 2, false); 261 | j.jenkins.clouds.add(cloud); 262 | 263 | mockEc2ApiToDescribeInstancesWhenModified(InstanceStateName.Pending); 264 | 265 | List rs = getQueueTaskFutures(1); 266 | 267 | triggerSuggestReviewNow("momo"); 268 | 269 | Assert.assertEquals(0, j.jenkins.getNodes().size()); 270 | 271 | tryUntil(new Runnable() { 272 | @Override 273 | public void run() { 274 | Assert.assertEquals(ImmutableSet.of("master", "momo"), labelsToNames(j.jenkins.getLabels())); 275 | Assert.assertEquals(1, j.jenkins.getLabelAtom("momo").nodeProvisioner.getPendingLaunches().size()); 276 | Assert.assertEquals(0, j.jenkins.getNodes().size()); 277 | } 278 | }); 279 | 280 | cancelTasks(rs); 281 | } 282 | 283 | @Test 284 | public void should_continue_update_after_termination() throws IOException { 285 | mockEc2ApiToDescribeInstancesWhenModified(InstanceStateName.Running, 5); 286 | 287 | final ComputerConnector computerConnector = new LocalComputerConnector(j); 288 | final EC2FleetCloud cloud = new EC2FleetCloud(null, null, "credId", null, "region", 289 | null, "fId", "momo", null, computerConnector, false, false, 290 | 1, 0, 5, 1, true, false, 291 | false, 0, 0, false, 292 | 10, false); 293 | j.jenkins.clouds.add(cloud); 294 | 295 | // wait while all nodes will be ok 296 | // tryUntil(new Runnable() { 297 | // @Override 298 | // public void run() { 299 | // for (Node node : j.jenkins.getNodes()) { 300 | // final Computer computer = node.toComputer(); 301 | // Assert.assertNotNull(computer); 302 | // Assert.assertTrue(computer.isOnline()); 303 | // } 304 | // } 305 | // }); 306 | 307 | final List> tasks = new ArrayList<>(); 308 | tasks.addAll((List) getQueueTaskFutures(5)); 309 | System.out.println("tasks submitted"); 310 | 311 | // wait full execution 312 | for (final QueueTaskFuture task : tasks) { 313 | try { 314 | Assert.assertEquals(task.get().getResult(), Result.SUCCESS); 315 | } catch (InterruptedException | ExecutionException e) { 316 | throw new RuntimeException(e); 317 | } 318 | } 319 | 320 | // wait until downscale happens 321 | tryUntil(new Runnable() { 322 | @Override 323 | public void run() { 324 | // defect in termination logic, that why 1 325 | Assert.assertThat(j.jenkins.getLabel("momo").getNodes().size(), Matchers.lessThanOrEqualTo(1)); 326 | } 327 | }, TimeUnit.MINUTES.toMillis(3)); 328 | 329 | final FleetStateStats oldStats = cloud.getStats(); 330 | tryUntil(new Runnable() { 331 | @Override 332 | public void run() { 333 | System.out.println("stats should be updated"); 334 | Assert.assertNotSame(oldStats, cloud.getStats()); 335 | } 336 | }); 337 | } 338 | 339 | } 340 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/jenkins/ec2fleet/ProvisionPerformanceTest.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import com.amazonaws.services.ec2.model.InstanceStateName; 4 | import hudson.model.FreeStyleBuild; 5 | import hudson.model.Result; 6 | import hudson.model.queue.QueueTaskFuture; 7 | import hudson.slaves.ComputerConnector; 8 | import org.hamcrest.Matchers; 9 | import org.junit.Assert; 10 | import org.junit.BeforeClass; 11 | import org.junit.Ignore; 12 | import org.junit.Test; 13 | 14 | import java.io.IOException; 15 | import java.util.ArrayList; 16 | import java.util.List; 17 | import java.util.concurrent.ExecutionException; 18 | import java.util.concurrent.TimeUnit; 19 | 20 | @Ignore 21 | public class ProvisionPerformanceTest extends IntegrationTest { 22 | 23 | @BeforeClass 24 | public static void beforeClass() { 25 | System.setProperty("jenkins.test.timeout", "720"); 26 | } 27 | 28 | @Test 29 | public void spikeLoadWorkers10Tasks30() throws Exception { 30 | test(10, 30); 31 | } 32 | 33 | @Test 34 | public void spikeLoadWorkers20Tasks60() throws Exception { 35 | test(20, 60); 36 | } 37 | 38 | private void test(int workers, int maxTasks) throws IOException, InterruptedException { 39 | mockEc2ApiToDescribeInstancesWhenModifiedWithDelay(InstanceStateName.Running, 500); 40 | 41 | final ComputerConnector computerConnector = new LocalComputerConnector(j); 42 | final EC2FleetCloudWithMeter cloud = new EC2FleetCloudWithMeter(null, null, "credId", null, "region", 43 | null, "fId", "momo", null, computerConnector, false, false, 44 | 1, 0, workers, 1, true, false, 45 | false, 0, 0, false, 46 | 2, false); 47 | j.jenkins.clouds.add(cloud); 48 | 49 | // updated plugin requires some init time to get first update 50 | // so wait this event to be really correct with perf comparison as old version is not require init time 51 | tryUntil(new Runnable() { 52 | @Override 53 | public void run() { 54 | Assert.assertNotNull(cloud.getStats()); 55 | } 56 | }); 57 | 58 | System.out.println("start test"); 59 | final long start = System.currentTimeMillis(); 60 | 61 | final List> tasks = new ArrayList<>(); 62 | 63 | final int taskBatch = 5; 64 | 65 | while (tasks.size() < maxTasks) { 66 | tasks.addAll((List) getQueueTaskFutures(taskBatch)); 67 | triggerSuggestReviewNow("momo"); 68 | System.out.println(taskBatch + " added into queue, " + (maxTasks - tasks.size()) + " remain"); 69 | } 70 | 71 | for (final QueueTaskFuture task : tasks) { 72 | try { 73 | Assert.assertEquals(task.get().getResult(), Result.SUCCESS); 74 | } catch (InterruptedException | ExecutionException e) { 75 | throw new RuntimeException(e); 76 | } 77 | } 78 | 79 | System.out.println("downscale"); 80 | final long finish = System.currentTimeMillis(); 81 | 82 | // wait until downscale happens 83 | tryUntil(new Runnable() { 84 | @Override 85 | public void run() { 86 | // defect in termination logic, that why 1 87 | Assert.assertThat(j.jenkins.getLabel("momo").getNodes().size(), Matchers.lessThanOrEqualTo(1)); 88 | } 89 | }, TimeUnit.MINUTES.toMillis(3)); 90 | 91 | final long upTime = TimeUnit.MILLISECONDS.toSeconds(finish - start); 92 | final long downTime = TimeUnit.MILLISECONDS.toSeconds(System.currentTimeMillis() - finish); 93 | final long totalTime = upTime + downTime; 94 | final long ideaUpTime = (maxTasks / workers) * JOB_SLEEP_TIME; 95 | final int idealDownTime = 60; 96 | final long ideaTime = ideaUpTime + idealDownTime; 97 | 98 | System.out.println(maxTasks + " up in " + upTime + " sec, ideal time is " + ideaUpTime + " sec, overhead is " + (upTime - ideaUpTime) + " sec"); 99 | System.out.println(maxTasks + " down in " + downTime + " sec, ideal time is " + idealDownTime + " sec, overhead is " + (downTime - idealDownTime) + " sec"); 100 | System.out.println(maxTasks + " completed in " + totalTime + " sec, ideal time is " + ideaTime + " sec, overhead is " + (totalTime - ideaTime) + " sec"); 101 | System.out.println(cloud.provisionMeter); 102 | System.out.println(cloud.removeMeter); 103 | System.out.println(cloud.updateMeter); 104 | } 105 | 106 | } 107 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/jenkins/ec2fleet/RealEc2ApiIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import com.amazonaws.auth.AWSCredentials; 4 | import com.amazonaws.auth.BasicAWSCredentials; 5 | import com.amazonaws.services.ec2.AmazonEC2; 6 | import com.amazonaws.services.ec2.AmazonEC2Client; 7 | import com.amazonaws.services.ec2.model.CancelSpotFleetRequestsRequest; 8 | import com.amazonaws.services.ec2.model.DescribeSpotFleetRequestsRequest; 9 | import com.amazonaws.services.ec2.model.RequestSpotFleetRequest; 10 | import com.amazonaws.services.ec2.model.RequestSpotFleetResult; 11 | import com.amazonaws.services.ec2.model.SpotFleetLaunchSpecification; 12 | import com.amazonaws.services.ec2.model.SpotFleetRequestConfig; 13 | import com.amazonaws.services.ec2.model.SpotFleetRequestConfigData; 14 | import com.cloudbees.jenkins.plugins.awscredentials.AWSCredentialsImpl; 15 | import com.cloudbees.plugins.credentials.CredentialsScope; 16 | import com.cloudbees.plugins.credentials.SystemCredentialsProvider; 17 | import org.apache.commons.lang.StringUtils; 18 | import org.junit.ClassRule; 19 | import org.junit.Ignore; 20 | import org.junit.Rule; 21 | import org.junit.Test; 22 | import org.jvnet.hudson.test.BuildWatcher; 23 | import org.jvnet.hudson.test.JenkinsRule; 24 | 25 | import java.util.Arrays; 26 | import java.util.List; 27 | import java.util.concurrent.TimeUnit; 28 | 29 | @Ignore("for manual run as you need to provide real AWS EC2 API credentials") 30 | public class RealEc2ApiIntegrationTest { 31 | 32 | @Rule 33 | public JenkinsRule j = new JenkinsRule(); 34 | 35 | @ClassRule 36 | public static BuildWatcher bw = new BuildWatcher(); 37 | 38 | @Test 39 | public void shouldSuccessfullyUpdatePluginWithFleetStatus() throws Exception { 40 | int targetCapacity = 0; 41 | 42 | final AWSCredentials awsCredentials = getAwsCredentials(); 43 | 44 | SystemCredentialsProvider.getInstance().getCredentials().add( 45 | new AWSCredentialsImpl(CredentialsScope.SYSTEM, "credId", 46 | awsCredentials.getAWSAccessKeyId(), awsCredentials.getAWSSecretKey(), "d")); 47 | 48 | withFleet(awsCredentials, targetCapacity, new WithFleetBody() { 49 | @Override 50 | public void run(AmazonEC2 amazonEC2, String fleetId) throws Exception { 51 | EC2FleetCloud cloud = new EC2FleetCloud("", null, "credId", null, null, null, fleetId, 52 | null, null, null, false, false, 53 | 0, 0, 0, 0, false, false, 54 | false, 0, 0, false, 55 | 10, false); 56 | j.jenkins.clouds.add(cloud); 57 | 58 | // 10 sec refresh time so wait 59 | Thread.sleep(TimeUnit.SECONDS.toMillis(60)); 60 | 61 | // assertEquals(0, cloud.getStatusCache().getNumActive()); 62 | // assertEquals(fleetId, cloud.getStatusCache().getFleetId()); 63 | } 64 | }); 65 | } 66 | 67 | /** 68 | * Related to https://github.com/jenkinsci/ec2-fleet-plugin/issues/60 69 | * 70 | * @throws Exception e 71 | */ 72 | @Test 73 | public void shouldSuccessfullyUpdateBigFleetPluginWithFleetStatus() throws Exception { 74 | final int targetCapacity = 30; 75 | 76 | final AWSCredentials awsCredentials = getAwsCredentials(); 77 | 78 | SystemCredentialsProvider.getInstance().getCredentials().add( 79 | new AWSCredentialsImpl(CredentialsScope.SYSTEM, "credId", 80 | awsCredentials.getAWSAccessKeyId(), awsCredentials.getAWSSecretKey(), "d")); 81 | 82 | withFleet(awsCredentials, targetCapacity, new WithFleetBody() { 83 | @Override 84 | public void run(AmazonEC2 amazonEC2, String fleetId) throws Exception { 85 | EC2FleetCloud cloud = new EC2FleetCloud(null, null, "credId", null, null, null, fleetId, 86 | null, null, null, false, false, 87 | 0, 0, 0, 0, false, false, 88 | false, 0, 0, false, 10, false); 89 | j.jenkins.clouds.add(cloud); 90 | 91 | final long start = System.currentTimeMillis(); 92 | final long max = TimeUnit.MINUTES.toMillis(2); 93 | while (System.currentTimeMillis() - start < max) { 94 | // if (cloud.getStatusCache().getNumActive() >= targetCapacity) break; 95 | Thread.sleep(TimeUnit.SECONDS.toMillis(10)); 96 | } 97 | 98 | // todo replace with proper accessor assertEquals(targetCapacity, cloud.getStatusCache().getNumActive()); 99 | // assertEquals(fleetId, cloud.getStatusCache().getFleetId()); 100 | } 101 | }); 102 | } 103 | 104 | private interface WithFleetBody { 105 | void run(AmazonEC2 amazonEC2, String fleetId) throws Exception; 106 | } 107 | 108 | private void withFleet(AWSCredentials awsCredentials, int targetCapacity, WithFleetBody body) throws Exception { 109 | final AmazonEC2 amazonEC2 = new AmazonEC2Client(awsCredentials); 110 | 111 | final SpotFleetRequestConfigData data = new SpotFleetRequestConfigData(); 112 | data.setLaunchSpecifications(Arrays.asList( 113 | new SpotFleetLaunchSpecification() 114 | .withInstanceType("t2.micro") 115 | .withImageId("ami-009d6802948d06e52") 116 | )); 117 | data.setIamFleetRole("arn:aws:iam::...:role/aws-service-role/spotfleet.amazonaws.com/AWSServiceRoleForEC2SpotFleet"); 118 | data.setTargetCapacity(targetCapacity); 119 | 120 | final RequestSpotFleetResult result = amazonEC2.requestSpotFleet( 121 | new RequestSpotFleetRequest().withSpotFleetRequestConfig(data)); 122 | 123 | try { 124 | final List configs = amazonEC2.describeSpotFleetRequests( 125 | new DescribeSpotFleetRequestsRequest().withSpotFleetRequestIds( 126 | result.getSpotFleetRequestId())).getSpotFleetRequestConfigs(); 127 | 128 | if (configs.isEmpty()) throw new IllegalArgumentException(); 129 | 130 | final int f = configs.get(0).getSpotFleetRequestConfig().getFulfilledCapacity().intValue(); 131 | System.out.println("Fulfilment " + f); 132 | 133 | body.run(amazonEC2, result.getSpotFleetRequestId()); 134 | } finally { 135 | amazonEC2.cancelSpotFleetRequests(new CancelSpotFleetRequestsRequest() 136 | .withSpotFleetRequestIds(result.getSpotFleetRequestId()).withTerminateInstances(true)); 137 | } 138 | } 139 | 140 | private AWSCredentials getAwsCredentials() { 141 | final String accessKey = System.getProperty("AWS_ACCESS_KEY"); 142 | final String secretKey = System.getProperty("AWS_SECRET_KEY"); 143 | 144 | if (StringUtils.isBlank(accessKey) || StringUtils.isBlank(secretKey)) { 145 | throw new IllegalArgumentException("AWS_ACCESS_KEY or AWS_SECRET_KEY is not specified in system properties, -D"); 146 | } 147 | 148 | return new BasicAWSCredentials(accessKey, secretKey); 149 | } 150 | 151 | } 152 | -------------------------------------------------------------------------------- /src/test/java/com/amazon/jenkins/ec2fleet/UiIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package com.amazon.jenkins.ec2fleet; 2 | 3 | import com.gargoylesoftware.htmlunit.html.DomElement; 4 | import com.gargoylesoftware.htmlunit.html.HtmlForm; 5 | import com.gargoylesoftware.htmlunit.html.HtmlFormUtil; 6 | import com.gargoylesoftware.htmlunit.html.HtmlPage; 7 | import com.gargoylesoftware.htmlunit.html.HtmlTextInput; 8 | import hudson.PluginWrapper; 9 | import hudson.model.Node; 10 | import hudson.slaves.Cloud; 11 | import hudson.slaves.NodeProperty; 12 | import org.apache.commons.lang.StringUtils; 13 | import org.junit.ClassRule; 14 | import org.junit.Rule; 15 | import org.junit.Test; 16 | import org.jvnet.hudson.test.BuildWatcher; 17 | import org.jvnet.hudson.test.JenkinsRule; 18 | import org.xml.sax.SAXException; 19 | 20 | import java.io.IOException; 21 | import java.util.ArrayList; 22 | import java.util.List; 23 | 24 | import static org.junit.Assert.assertEquals; 25 | import static org.junit.Assert.assertNotNull; 26 | import static org.junit.Assert.assertNotSame; 27 | import static org.junit.Assert.assertSame; 28 | import static org.junit.Assert.assertTrue; 29 | 30 | /** 31 | * Detailed guides https://jenkins.io/doc/developer/testing/ 32 | * https://wiki.jenkins.io/display/JENKINS/Unit+Test#UnitTest-DealingwithproblemsinJavaScript 33 | */ 34 | public class UiIntegrationTest { 35 | 36 | @Rule 37 | public JenkinsRule j = new JenkinsRule(); 38 | 39 | @ClassRule 40 | public static BuildWatcher bw = new BuildWatcher(); 41 | 42 | @Test 43 | public void shouldFindThePluginByShortName() { 44 | PluginWrapper wrapper = j.getPluginManager().getPlugin("ec2-fleet"); 45 | assertNotNull("should have a valid plugin", wrapper); 46 | } 47 | 48 | @Test 49 | public void shouldShowAsHiddenCloudIdAsOldId() throws IOException, SAXException { 50 | Cloud cloud = new EC2FleetCloud(null, null, null, null, null, null, null, 51 | null, null, null, false, false, 52 | 0, 0, 0, 0, false, false, 53 | false, 0, 0, false, 54 | 10, false); 55 | j.jenkins.clouds.add(cloud); 56 | 57 | HtmlPage page = j.createWebClient().goTo("configure"); 58 | 59 | assertTrue(StringUtils.isNotBlank(((HtmlTextInput) getElementsByNameWithoutJdk(page, "_.oldId").get(0)).getText())); 60 | } 61 | 62 | @Test 63 | public void shouldShowNodeConfigurationPage() throws Exception { 64 | EC2FleetCloud cloud = new EC2FleetCloud(null, null, null, null, null, null, null, 65 | null, null, null, false, false, 66 | 0, 0, 0, 0, false, false, 67 | false, 0, 0, false, 68 | 10, false); 69 | j.jenkins.clouds.add(cloud); 70 | 71 | j.jenkins.addNode(new EC2FleetNode("node-name", "", "", 1, 72 | Node.Mode.EXCLUSIVE, "", new ArrayList>(), cloud, 73 | j.createComputerLauncher(null))); 74 | 75 | HtmlPage page = j.createWebClient().goTo("computer/node-name/configure"); 76 | 77 | assertTrue(StringUtils.isNotBlank(((HtmlTextInput) getElementsByNameWithoutJdk(page, "_.name").get(0)).getText())); 78 | } 79 | 80 | @Test 81 | public void shouldReplaceCloudForNodesAfterConfigurationSave() throws Exception { 82 | EC2FleetCloud cloud = new EC2FleetCloud(null, null, null, null, null, null, null, 83 | null, null, null, false, false, 84 | 0, 0, 0, 0, false, false, 85 | false, 0, 0, false, 86 | 10, false); 87 | j.jenkins.clouds.add(cloud); 88 | 89 | j.jenkins.addNode(new EC2FleetNode("mock", "", "", 1, 90 | Node.Mode.EXCLUSIVE, "", new ArrayList>(), cloud, 91 | j.createComputerLauncher(null))); 92 | 93 | HtmlPage page = j.createWebClient().goTo("configure"); 94 | HtmlForm form = page.getFormByName("config"); 95 | 96 | ((HtmlTextInput) getElementsByNameWithoutJdk(page, "_.name").get(0)).setText("a"); 97 | 98 | HtmlFormUtil.submit(form); 99 | 100 | final Cloud newCloud = j.jenkins.clouds.get(0); 101 | assertNotSame(cloud, newCloud); 102 | 103 | assertSame(newCloud, ((EC2FleetNode) j.jenkins.getNode("mock")).getCloud()); 104 | } 105 | 106 | @Test 107 | public void shouldShowInConfigurationClouds() throws IOException, SAXException { 108 | Cloud cloud = new EC2FleetCloud(null, null, null, null, null, null, null, 109 | null, null, null, false, false, 110 | 0, 0, 0, 0, false, false, 111 | false, 0, 0, false, 112 | 10, false); 113 | j.jenkins.clouds.add(cloud); 114 | 115 | HtmlPage page = j.createWebClient().goTo("configure"); 116 | 117 | assertEquals("ec2-fleet", ((HtmlTextInput) getElementsByNameWithoutJdk(page, "_.labelString").get(1)).getText()); 118 | } 119 | 120 | @Test 121 | public void shouldShowMultipleClouds() throws IOException, SAXException { 122 | Cloud cloud1 = new EC2FleetCloud("a", null, null, null, null, null, 123 | null, null, null, null, false, false, 124 | 0, 0, 0, 0, false, false, 125 | false, 0, 0, false, 126 | 10, false); 127 | j.jenkins.clouds.add(cloud1); 128 | 129 | Cloud cloud2 = new EC2FleetCloud("b", null, null, null, null, null, 130 | null, null, null, null, false, false, 131 | 0, 0, 0, 0, false, false, 132 | false, 0, 0, false, 133 | 10, false); 134 | j.jenkins.clouds.add(cloud2); 135 | 136 | HtmlPage page = j.createWebClient().goTo("configure"); 137 | 138 | List elementsByName = getElementsByNameWithoutJdk(page, "_.name"); 139 | assertEquals(2, elementsByName.size()); 140 | assertEquals("a", ((HtmlTextInput) elementsByName.get(0)).getText()); 141 | assertEquals("b", ((HtmlTextInput) elementsByName.get(1)).getText()); 142 | } 143 | 144 | @Test 145 | public void shouldShowMultipleCloudsWithDefaultName() throws IOException, SAXException { 146 | Cloud cloud1 = new EC2FleetCloud(null, null, null, null, null, null, 147 | null, null, null, null, false, false, 148 | 0, 0, 0, 0, false, false, 149 | false, 0, 0, false, 150 | 10, false); 151 | j.jenkins.clouds.add(cloud1); 152 | 153 | Cloud cloud2 = new EC2FleetCloud(null, null, null, null, null, null, 154 | null, null, null, null, false, false, 155 | 0, 0, 0, 0, false, false, 156 | false, 0, 0, false, 157 | 10, false); 158 | j.jenkins.clouds.add(cloud2); 159 | 160 | HtmlPage page = j.createWebClient().goTo("configure"); 161 | 162 | List elementsByName = getElementsByNameWithoutJdk(page, "_.name"); 163 | assertEquals(2, elementsByName.size()); 164 | assertEquals("FleetCloud", ((HtmlTextInput) elementsByName.get(0)).getText()); 165 | assertEquals("FleetCloud", ((HtmlTextInput) elementsByName.get(1)).getText()); 166 | } 167 | 168 | @Test 169 | public void shouldUpdateProperCloudWhenMultiple() throws IOException, SAXException { 170 | EC2FleetCloud cloud1 = new EC2FleetCloud(null, null, null, null, null, null, 171 | null, null, null, null, false, false, 172 | 0, 0, 0, 0, false, false, 173 | false, 0, 0, false, 174 | 10, false); 175 | j.jenkins.clouds.add(cloud1); 176 | 177 | EC2FleetCloud cloud2 = new EC2FleetCloud(null, null, null, null, null, null, 178 | null, null, null, null, false, false, 179 | 0, 0, 0, 0, false, false, 180 | false, 0, 0, false, 181 | 10, false); 182 | j.jenkins.clouds.add(cloud2); 183 | 184 | HtmlPage page = j.createWebClient().goTo("configure"); 185 | HtmlForm form = page.getFormByName("config"); 186 | 187 | ((HtmlTextInput) getElementsByNameWithoutJdk(page, "_.name").get(0)).setText("a"); 188 | 189 | HtmlFormUtil.submit(form); 190 | 191 | assertEquals("a", j.jenkins.clouds.get(0).name); 192 | assertEquals("FleetCloud", j.jenkins.clouds.get(1).name); 193 | } 194 | 195 | @Test 196 | public void shouldGetFirstWhenMultipleCloudWithSameName() { 197 | EC2FleetCloud cloud1 = new EC2FleetCloud(null, null, null, null, null, null, 198 | null, null, null, null, false, false, 199 | 0, 0, 0, 0, false, false, 200 | false, 0, 0, false, 201 | 10, false); 202 | j.jenkins.clouds.add(cloud1); 203 | 204 | EC2FleetCloud cloud2 = new EC2FleetCloud(null, null, null, null, null, null, 205 | null, null, null, null, false, false, 206 | 0, 0, 0, 0, false, false, 207 | false, 0, 0, false, 208 | 10, false); 209 | j.jenkins.clouds.add(cloud2); 210 | 211 | assertSame(cloud1, j.jenkins.getCloud("FleetCloud")); 212 | } 213 | 214 | @Test 215 | public void shouldGetProperWhenMultipleWithDiffName() { 216 | EC2FleetCloud cloud1 = new EC2FleetCloud("a", null, null, null, null, null, 217 | null, null, null, null, false, false, 218 | 0, 0, 0, 0, false, false, 219 | false, 0, 0, false, 220 | 10, false); 221 | j.jenkins.clouds.add(cloud1); 222 | 223 | EC2FleetCloud cloud2 = new EC2FleetCloud("b", null, null, null, null, null, 224 | null, null, null, null, false, false, 225 | 0, 0, 0, 0, false, false, 226 | false, 0, 0, false, 227 | 10, false); 228 | j.jenkins.clouds.add(cloud2); 229 | 230 | assertSame(cloud1, j.jenkins.getCloud("a")); 231 | assertSame(cloud2, j.jenkins.getCloud("b")); 232 | } 233 | 234 | private static List getElementsByNameWithoutJdk(HtmlPage page, String name) { 235 | String jdkCheckUrl = "/jenkins/descriptorByName/hudson.model.JDK/checkName"; 236 | List r = new ArrayList<>(); 237 | for (DomElement domElement : page.getElementsByName(name)) { 238 | if (!jdkCheckUrl.equals(domElement.getAttribute("checkurl"))) { 239 | r.add(domElement); 240 | } 241 | } 242 | return r; 243 | } 244 | 245 | } 246 | -------------------------------------------------------------------------------- /test.xml: -------------------------------------------------------------------------------- 1 | 2 | jenkins 3 | Jenkins 4 | This service runs Jenkins continuous integration system. 5 | 6 | C:\Windows\Temp\jenkins-slave\jdk.exe\java-se-8u40-ri\bin\java 7 | -Xrs -Xmx256m -jar "%BASE%\jenkins.war" --httpPort=8080 8 | rotate 9 | 10 | --------------------------------------------------------------------------------