├── .arcconfig ├── .gitignore ├── README.md ├── distributed-docker-appmaster ├── pom.xml └── src │ └── main │ ├── java │ └── com │ │ └── sogou │ │ └── dockeronyarn │ │ ├── appmaster │ │ └── DockerRunnerApplicationMaster.java │ │ ├── common │ │ ├── DSConstants.java │ │ ├── DistributedDockerConfiguration.java │ │ ├── ExitCode.java │ │ ├── Log4jPropertyHelper.java │ │ └── Utils.java │ │ └── docker │ │ ├── Constants.java │ │ ├── DockerContainerRunner.java │ │ ├── DockerContainerRunnerParam.java │ │ ├── DockerShareSpaceMonitor.java │ │ ├── ExitCode.java │ │ └── Test.java │ └── resources │ └── META-INF │ └── services │ └── javax.ws.rs.ext.RuntimeDelegate ├── distributed-docker-submit-client ├── .classpath ├── .project ├── .settings │ ├── org.eclipse.core.resources.prefs │ ├── org.eclipse.jdt.core.prefs │ └── org.eclipse.m2e.core.prefs ├── DockerApplicationMaster_24.java ├── README.MD ├── hadoop-env.sh ├── log4j.properties ├── pom.xml ├── run_YarnDockerClient.sh ├── src │ └── main │ │ ├── java │ │ └── com │ │ │ └── sogou │ │ │ └── dockeronyarn │ │ │ └── client │ │ │ ├── DockerApplicationMaster_23.java │ │ │ ├── DockerApplicationSubmitter.java │ │ │ ├── DockerClient.java │ │ │ ├── DockerOnYarnAppDescriptor.java │ │ │ ├── DockerOnYarnClient.java │ │ │ ├── DockerOnYarnClientCli.java │ │ │ └── model │ │ │ ├── SubmitException.java │ │ │ ├── SubmitItem.java │ │ │ └── SubmitReturnItem.java │ │ └── resources │ │ ├── META-INF │ │ └── services │ │ │ └── javax.ws.rs.ext.RuntimeDelegate │ │ ├── distributed-docker-site.xml │ │ └── log4j.properties └── test_docker_client.sh ├── distributed-docker-submit-server ├── .classpath ├── .project ├── .settings │ ├── org.eclipse.core.resources.prefs │ ├── org.eclipse.jdt.core.prefs │ └── org.eclipse.m2e.core.prefs ├── assembly.xml ├── bin │ ├── monitor_docker_submit_server.sh │ ├── start_server.sh │ └── test.sh ├── conf │ ├── log.properties │ └── server.properties ├── pom.xml └── src │ └── main │ └── java │ └── com │ └── sogou │ └── dockeronyarn │ ├── server │ ├── DockerServer.java │ └── DockerSubmitMain.java │ ├── service │ └── DockerService.java │ └── util │ ├── LogUtil.java │ ├── PropertyManager.java │ └── StaticData.java └── pom.xml /.arcconfig: -------------------------------------------------------------------------------- 1 | { 2 | "phabricator.uri" : "http://review.dev.sogou-inc.com/" 3 | } 4 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | .idea 2 | target/ 3 | *.iml 4 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Docker on yarn - Unified Resource Management Submit System 2 | 3 | --- 4 | 5 | Docker ont yarn is a simply system to submit your hadoop job. 6 | 7 | --- 8 | 9 | ## User Guide 10 | 11 | ### Maven Dependency 12 | 13 | ``` 14 | 15 | distributed-docker-submit-project 16 | distributed-docker-submit-project 17 | 0.0.1-SNAPSHOT 18 | 19 | ``` 20 | 21 | --- 22 | 23 | ## Developer Guide 24 | 25 | ### Requirements 26 | 27 | * JDK-1.7 28 | * Maven 3.0.3 above 29 | 30 | ### Building 31 | 32 | ``` 33 | $ mvn clean package 34 | ``` 35 | 36 | * jar location: distributed-docker-appmaster/target/,distributed-docker-submit-client-0.0.1-SNAPSHOT.jar 37 | 38 | ### Deploy To Remote Maven Repository 39 | 40 | ``` 41 | $ mvn deploy 42 | ``` 43 | 44 | --- 45 | -------------------------------------------------------------------------------- /distributed-docker-appmaster/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | distributed-docker-submit-project 7 | distributed-docker-submit-project 8 | 0.0.1-SNAPSHOT 9 | 10 | 4.0.0 11 | 12 | distributed-docker-appmaster 13 | 14 | 15 | 16 | true 17 | 18 | UTF-8 19 | true 20 | false 21 | 1.7 22 | 1.7 23 | 24 | 25 | 26 | 27 | 28 | org.apache.hadoop 29 | hadoop-common 30 | 2.5.0-cdh5.3.2 31 | 32 | 33 | 34 | jersey-core 35 | com.sun.jersey 36 | 37 | 38 | com.sun.jersey 39 | jersey-server 40 | 41 | 42 | 43 | 44 | 45 | 46 | org.apache.hadoop 47 | hadoop-yarn-client 48 | 2.5.0-cdh5.3.2 49 | 50 | 51 | 52 | jersey-core 53 | com.sun.jersey 54 | 55 | 56 | com.sun.jersey 57 | jersey-server 58 | 59 | 60 | 61 | 62 | 63 | com.github.docker-java 64 | docker-java 65 | 2.2.3 66 | 67 | 68 | 69 | org.apache.httpcomponents 70 | httpcore 71 | 4.3.1 72 | 73 | 74 | 75 | 76 | 77 | 78 | maven-compiler-plugin 79 | 3.0 80 | 81 | 1.6 82 | 1.6 83 | GB18030 84 | 85 | 86 | 87 | 88 | org.apache.maven.plugins 89 | maven-jar-plugin 90 | 2.4 91 | 92 | 93 | 94 | true 95 | 96 | 97 | 98 | 99 | 100 | 101 | org.apache.maven.plugins 102 | maven-assembly-plugin 103 | 2.5.4 104 | 105 | 106 | jar-with-dependencies 107 | 108 | 109 | 110 | 111 | 112 | assemble-all 113 | package 114 | 115 | single 116 | 117 | 118 | 119 | 120 | 121 | 122 | 123 | 124 | -------------------------------------------------------------------------------- /distributed-docker-appmaster/src/main/java/com/sogou/dockeronyarn/appmaster/DockerRunnerApplicationMaster.java: -------------------------------------------------------------------------------- 1 | package com.sogou.dockeronyarn.appmaster; 2 | 3 | import com.sogou.dockeronyarn.docker.DockerContainerRunner; 4 | import com.sogou.dockeronyarn.common.Log4jPropertyHelper; 5 | import com.sogou.dockeronyarn.common.Utils; 6 | import com.sogou.dockeronyarn.docker.DockerContainerRunnerParam; 7 | import com.sogou.dockeronyarn.common.DistributedDockerConfiguration; 8 | import org.apache.commons.cli.*; 9 | import org.apache.commons.logging.Log; 10 | import org.apache.commons.logging.LogFactory; 11 | import org.apache.hadoop.conf.Configuration; 12 | import org.apache.hadoop.io.DataOutputBuffer; 13 | import org.apache.hadoop.io.IOUtils; 14 | import org.apache.hadoop.net.NetUtils; 15 | import org.apache.hadoop.security.Credentials; 16 | import org.apache.hadoop.security.UserGroupInformation; 17 | import org.apache.hadoop.security.token.Token; 18 | import org.apache.hadoop.util.Shell; 19 | import org.apache.hadoop.yarn.api.ApplicationConstants; 20 | import org.apache.hadoop.yarn.api.protocolrecords.RegisterApplicationMasterResponse; 21 | import org.apache.hadoop.yarn.api.records.*; 22 | import org.apache.hadoop.yarn.client.api.async.AMRMClientAsync; 23 | import org.apache.hadoop.yarn.conf.YarnConfiguration; 24 | import org.apache.hadoop.yarn.exceptions.YarnException; 25 | import org.apache.hadoop.yarn.security.AMRMTokenIdentifier; 26 | import org.apache.hadoop.yarn.util.ConverterUtils; 27 | 28 | import java.io.BufferedReader; 29 | import java.io.File; 30 | import java.io.IOException; 31 | import java.io.StringReader; 32 | import java.util.Iterator; 33 | import java.util.List; 34 | import java.util.Map; 35 | import java.util.concurrent.atomic.AtomicInteger; 36 | 37 | import static com.sogou.dockeronyarn.common.Utils.checkNotEmpty; 38 | 39 | /** 40 | * The DockerRunnerApplicationMaster DONOT require more container from RM. 41 | * It only create and run a new container to docker service on the local host 42 | * where the AM is launched. 43 | * 44 | * Created by guoshiwei on 15/4/25. 45 | */ 46 | public class DockerRunnerApplicationMaster { 47 | public static final String LOCAL_RUNNER_NAME = "runner.py"; 48 | private static final Log LOG = LogFactory.getLog(DockerRunnerApplicationMaster.class); 49 | private Configuration conf; 50 | // Handle to communicate with the Resource Manager 51 | @SuppressWarnings("rawtypes") 52 | private AMRMClientAsync amRMClient; 53 | private ApplicationAttemptId appAttemptID; 54 | // Hostname of the container 55 | private String appMasterHostname = "localhost"; 56 | // Port on which the app master listens for status updates from clients 57 | private int appMasterRpcPort = 9999; 58 | // Tracking url to which app master publishes info for clients to monitor 59 | private String appMasterTrackingUrl = ""; //"http://localhost:9999/someurl"; 60 | private String appHistoryTrackingUrlBase = ""; 61 | // Hardcoded path to custom log_properties 62 | private static final String log4jPath = "log4j.properties"; 63 | protected AtomicInteger numRetryCount = new AtomicInteger(); 64 | protected AtomicInteger MAX_RETRY_COUNT = new AtomicInteger(3); 65 | private String mountVolume =""; 66 | private String workDir=""; 67 | private DockerContainerRunner dockerContainerRunner; 68 | private volatile boolean done; 69 | private String workingDirectory ; 70 | private String dockerImage; 71 | private String[] commandToRun; 72 | private String jobName; 73 | private String[] dockerArgs ; 74 | 75 | public String getRunnerAbsolutePath() { 76 | return workingDirectory + "/" + LOCAL_RUNNER_NAME; 77 | } 78 | 79 | private DockerContainerRunnerParam buildYarnDockerClientParam() { 80 | DockerContainerRunnerParam p = new DockerContainerRunnerParam(); 81 | DistributedDockerConfiguration dockerConf = new DistributedDockerConfiguration(); 82 | p.cmdAndArgs = commandToRun; 83 | p.containerMemory = dockerConf.getInt(DistributedDockerConfiguration.CONTAINER_MEMORY, 84 | DistributedDockerConfiguration.DEFAULT_CONTAINER_MEMORY); 85 | p.containerVirtualCores = dockerConf.getInt(DistributedDockerConfiguration.CONTAINER_CORES, 86 | DistributedDockerConfiguration.DEFAULT_CONTAINER_CORES); 87 | 88 | p.runnerScriptPath = getRunnerAbsolutePath(); 89 | p.dockerCertPath = dockerConf.get(DistributedDockerConfiguration.DOCKER_CERT_PATH); 90 | Utils.checkNotEmpty(p.dockerCertPath, DistributedDockerConfiguration.DOCKER_CERT_PATH + " Not given"); 91 | p.dockerHost = dockerConf.get(DistributedDockerConfiguration.DOCKER_HOST, 92 | DistributedDockerConfiguration.DEFAULT_DOCKER_HOST); 93 | 94 | p.dockerImage = this.dockerImage; 95 | p.mountVolume = this.mountVolume ; 96 | p.workingDir = this.workDir; 97 | p.setDockerArgs(dockerArgs); 98 | return p; 99 | } 100 | //init the docker options 101 | public static Options InitDockerOptions() { 102 | Options opts = new Options(); 103 | opts.addOption("v", true, "the mount volume"); 104 | opts.addOption(OptionBuilder.withLongOpt("rm") 105 | .withDescription("rm the container after execute").create()); 106 | opts.addOption(new Option("m","memory",true,"the memory of container")); 107 | opts.addOption(new Option("c","cpu-shares",true,"the cpu of the container")); 108 | opts.addOption(new Option("H","cpu-shares",true,"the host of the container")); 109 | return opts; 110 | } 111 | 112 | public DockerRunnerApplicationMaster() { 113 | // Set up the configuration 114 | conf = new YarnConfiguration(); 115 | appHistoryTrackingUrlBase = conf.get("yarn.timeline-service.webapp.address"); 116 | if (appHistoryTrackingUrlBase == null || appHistoryTrackingUrlBase.isEmpty()) { 117 | throw new IllegalArgumentException("No yarn.timeline-service.webapp.address is found." + 118 | "Check your yarn-site.xml"); 119 | } 120 | 121 | appHistoryTrackingUrlBase = "http://" + appHistoryTrackingUrlBase + "/applicationhistory/app/"; 122 | workingDirectory = System.getenv("PWD"); 123 | checkNotEmpty(workingDirectory, "PWD enviroment is not set"); 124 | if (!new File(getRunnerAbsolutePath()).isFile()) { 125 | throw new IllegalArgumentException("Runner cannot be found: " + getRunnerAbsolutePath()); 126 | } 127 | } 128 | 129 | public static void main(String[] args) { 130 | boolean result = false; 131 | try { 132 | DockerRunnerApplicationMaster appMaster = new DockerRunnerApplicationMaster(); 133 | LOG.info("Initializing ApplicationMaster. " 134 | + "yarn.timeline-service.webapp.address" 135 | + " = " + appMaster.appHistoryTrackingUrlBase); 136 | boolean doRun = appMaster.init(args); 137 | if (!doRun) { 138 | System.exit(0); 139 | } 140 | result = appMaster.run(); 141 | } catch (Throwable t) { 142 | LOG.fatal("Error running ApplicationMaster", t); 143 | System.exit(1); 144 | } 145 | 146 | if (result) { 147 | LOG.info("Application Master completed successfully. exiting"); 148 | System.exit(0); 149 | } else { 150 | LOG.info("Application Master failed. exiting"); 151 | System.exit(2); 152 | } 153 | } 154 | 155 | private boolean run() throws IOException, YarnException { 156 | LOG.info("Starting ApplicationMaster"); 157 | Credentials credentials = 158 | UserGroupInformation.getCurrentUser().getCredentials(); 159 | DataOutputBuffer dob = new DataOutputBuffer(); 160 | credentials.writeTokenStorageToStream(dob); 161 | // Now remove the AM->RM token so that containers cannot access it. 162 | Iterator> iter = credentials.getAllTokens().iterator(); 163 | while (iter.hasNext()) { 164 | Token token = iter.next(); 165 | if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) { 166 | iter.remove(); 167 | } 168 | } 169 | 170 | AMRMClientAsync.CallbackHandler allocListener = new RMCallbackHandler(); 171 | amRMClient = AMRMClientAsync.createAMRMClientAsync(1000, allocListener); 172 | amRMClient.init(conf); 173 | amRMClient.start(); 174 | 175 | // Register self with ResourceManager 176 | // This will start heartbeating to the RM 177 | appMasterHostname = NetUtils.getHostname(); 178 | RegisterApplicationMasterResponse response = amRMClient 179 | .registerApplicationMaster(appMasterHostname, appMasterRpcPort, appMasterTrackingUrl); 180 | // Dump out information about cluster capability as seen by the 181 | // resource manager 182 | int maxMem = response.getMaximumResourceCapability().getMemory(); 183 | LOG.info("Max mem capabililty of resources in this cluster " + maxMem); 184 | 185 | int maxVCores = response.getMaximumResourceCapability().getVirtualCores(); 186 | LOG.info("Max vcores capabililty of resources in this cluster " + maxVCores); 187 | 188 | try { 189 | dockerContainerRunner.startContainer(String.format("%s-%s", appAttemptID, jobName)); 190 | dockerContainerRunner.waitContainerExit(); 191 | } 192 | catch (Exception e) { 193 | LOG.error("dockerContainerRunner exited with exception: " + e.getMessage(), e); 194 | } 195 | 196 | return finish(); 197 | } 198 | 199 | /** 200 | * Parse command line options 201 | * 202 | * @param args Command line args 203 | * @return Whether init successful and run should be invoked 204 | * @throws ParseException 205 | * @throws IOException 206 | */ 207 | private boolean init(String[] args) throws ParseException { 208 | Options opts = new Options(); 209 | opts.addOption("priority", true, "Application Priority. Default 0"); 210 | opts.addOption("container_retry", true, "Application container_retry. Default 3"); 211 | opts.addOption("image", true, "Docker image to run"); 212 | opts.addOption("job_name", true, "Uniq name of this job"); 213 | opts.addOption("v", true, "the file needed to be mounted into docker container"); 214 | opts.addOption("w", true, "the working dir of docker container "); 215 | opts.addOption("debug", false, "Dump out debug information"); 216 | opts.addOption(OptionBuilder.withLongOpt("docker-args").hasArgs().create()); 217 | opts.addOption("help", false, "Print usage"); 218 | CommandLine cliParser = new GnuParser().parse(opts, args, true); 219 | 220 | //Check whether customer log4j.properties file exists 221 | if (fileExist(log4jPath)) { 222 | try { 223 | Log4jPropertyHelper.updateLog4jConfiguration( 224 | DockerRunnerApplicationMaster.class, log4jPath); 225 | } catch (Exception e) { 226 | LOG.warn("Can not set up custom log4j properties. " + e); 227 | } 228 | } 229 | 230 | if(!cliParser.hasOption("job_name")){ 231 | LOG.error("job_name param not found"); 232 | return false; 233 | } 234 | jobName = cliParser.getOptionValue("job_name"); 235 | if (cliParser.hasOption("help")) { 236 | printUsage(opts); 237 | return false; 238 | } 239 | 240 | if (cliParser.hasOption("debug")) { 241 | dumpOutDebugInfo(); 242 | } 243 | 244 | Map envs = System.getenv(); 245 | if (!envs.containsKey(ApplicationConstants.Environment.CONTAINER_ID.name())) { 246 | if (cliParser.hasOption("app_attempt_id")) { 247 | String appIdStr = cliParser.getOptionValue("app_attempt_id", ""); 248 | appAttemptID = ConverterUtils.toApplicationAttemptId(appIdStr); 249 | } else { 250 | throw new IllegalArgumentException( 251 | "Application Attempt Id not set in the environment"); 252 | } 253 | } 254 | else { 255 | ContainerId containerId = ConverterUtils.toContainerId(envs 256 | .get(ApplicationConstants.Environment.CONTAINER_ID.name())); 257 | appAttemptID = containerId.getApplicationAttemptId(); 258 | } 259 | 260 | if (!envs.containsKey(ApplicationConstants.APP_SUBMIT_TIME_ENV)) { 261 | throw new RuntimeException(ApplicationConstants.APP_SUBMIT_TIME_ENV 262 | + " not set in the environment"); 263 | } 264 | if (!envs.containsKey(ApplicationConstants.Environment.NM_HOST.name())) { 265 | throw new RuntimeException(ApplicationConstants.Environment.NM_HOST.name() 266 | + " not set in the environment"); 267 | } 268 | if (!envs.containsKey(ApplicationConstants.Environment.NM_HTTP_PORT.name())) { 269 | throw new RuntimeException(ApplicationConstants.Environment.NM_HTTP_PORT 270 | + " not set in the environment"); 271 | } 272 | if (!envs.containsKey(ApplicationConstants.Environment.NM_PORT.name())) { 273 | throw new RuntimeException(ApplicationConstants.Environment.NM_PORT.name() 274 | + " not set in the environment"); 275 | } 276 | 277 | LOG.info("Application master for app" + ", appId=" 278 | + appAttemptID.getApplicationId().getId() + ", clustertimestamp=" 279 | + appAttemptID.getApplicationId().getClusterTimestamp() 280 | + ", attemptId=" + appAttemptID.getAttemptId()); 281 | 282 | this.MAX_RETRY_COUNT.set(Integer.parseInt(cliParser.getOptionValue( 283 | "container_retry", "3"))); 284 | 285 | dockerImage = cliParser.getOptionValue("image"); 286 | checkNotEmpty(dockerImage, "No image argument given"); 287 | mountVolume = cliParser.getOptionValue("v"); 288 | workDir = cliParser.getOptionValue("w"); 289 | commandToRun = cliParser.getArgs(); 290 | dockerArgs = cliParser.getOptionValue("docker-args").split(" "); 291 | this.dockerContainerRunner = new DockerContainerRunner(buildYarnDockerClientParam()); 292 | return true; 293 | } 294 | 295 | /** 296 | * Dump out contents of $CWD and the environment to stdout for debugging 297 | */ 298 | private void dumpOutDebugInfo() { 299 | LOG.info("Dump debug output"); 300 | Map envs = System.getenv(); 301 | for (Map.Entry env : envs.entrySet()) { 302 | LOG.info("System env: key=" + env.getKey() + ", val=" + env.getValue()); 303 | System.out.println("System env: key=" + env.getKey() + ", val=" 304 | + env.getValue()); 305 | } 306 | 307 | BufferedReader buf = null; 308 | try { 309 | String lines = Shell.WINDOWS ? Shell.execCommand("cmd", "/c", "dir") : 310 | Shell.execCommand("ls", "-al"); 311 | buf = new BufferedReader(new StringReader(lines)); 312 | String line = ""; 313 | while ((line = buf.readLine()) != null) { 314 | LOG.info("System CWD content: " + line); 315 | System.out.println("System CWD content: " + line); 316 | } 317 | } catch (IOException e) { 318 | LOG.warn(e); 319 | } finally { 320 | IOUtils.cleanup(LOG, buf); 321 | } 322 | } 323 | 324 | private boolean finish(){ 325 | // When the application completes, it should send a shutdown application 326 | // signal to the RM 327 | LOG.info("Application completed. Signalling shutdown to RM"); 328 | 329 | FinalApplicationStatus appStatus; 330 | String appMessage = null; 331 | boolean success = true; 332 | if (dockerContainerRunner.getExitStatus() == 0) { 333 | appStatus = FinalApplicationStatus.SUCCEEDED; 334 | } else { 335 | appStatus = FinalApplicationStatus.FAILED; 336 | appMessage = "Diagnostics." + ", docker Container exited code: " + 337 | dockerContainerRunner.getExitStatus(); 338 | success = false; 339 | } 340 | 341 | try { 342 | amRMClient.unregisterApplicationMaster(appStatus, appMessage, 343 | appHistoryTrackingUrlBase + appAttemptID.getApplicationId().toString()); 344 | } catch (YarnException ex) { 345 | LOG.error("Failed to unregister application", ex); 346 | } catch (IOException e) { 347 | LOG.error("Failed to unregister application", e); 348 | } 349 | 350 | amRMClient.stop(); 351 | return success; 352 | } 353 | 354 | private boolean fileExist(String filePath) { 355 | return new File(filePath).exists(); 356 | } 357 | 358 | private void printUsage(Options opts) { 359 | new HelpFormatter().printHelp("[options] command [commandArgs]", opts); 360 | } 361 | 362 | private class RMCallbackHandler implements AMRMClientAsync.CallbackHandler{ 363 | public void onContainersCompleted(List list) { 364 | // Nothing to do since we do not require more container 365 | } 366 | 367 | public void onContainersAllocated(List list) { 368 | // Nothing to do since we do not require more container 369 | } 370 | 371 | public void onShutdownRequest() { 372 | done = true; 373 | dockerContainerRunner.stopContainer(); 374 | } 375 | 376 | public void onNodesUpdated(List list) { 377 | // Nothing to do since we do not require more container 378 | } 379 | 380 | public float getProgress() { 381 | 382 | return dockerContainerRunner.getProgress(); 383 | } 384 | 385 | public void onError(Throwable throwable) { 386 | done = true; 387 | dockerContainerRunner.stopContainer(); 388 | amRMClient.stop(); 389 | } 390 | } 391 | } 392 | -------------------------------------------------------------------------------- /distributed-docker-appmaster/src/main/java/com/sogou/dockeronyarn/common/DSConstants.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.sogou.dockeronyarn.common; 20 | 21 | import org.apache.hadoop.classification.InterfaceAudience; 22 | import org.apache.hadoop.classification.InterfaceStability; 23 | 24 | /** 25 | * Constants used in both Client and Application Master 26 | */ 27 | @InterfaceAudience.Public 28 | @InterfaceStability.Unstable 29 | public class DSConstants { 30 | 31 | /** 32 | * Environment key name pointing to the shell script's location 33 | */ 34 | public static final String DISTRIBUTEDSHELLSCRIPTLOCATION = "DISTRIBUTEDSHELLSCRIPTLOCATION"; 35 | 36 | /** 37 | * Environment key name denoting the file timestamp for the shell script. 38 | * Used to validate the local resource. 39 | */ 40 | public static final String DISTRIBUTEDSHELLSCRIPTTIMESTAMP = "DISTRIBUTEDSHELLSCRIPTTIMESTAMP"; 41 | 42 | /** 43 | * Environment key name denoting the file content length for the shell script. 44 | * Used to validate the local resource. 45 | */ 46 | public static final String DISTRIBUTEDSHELLSCRIPTLEN = "DISTRIBUTEDSHELLSCRIPTLEN"; 47 | } 48 | -------------------------------------------------------------------------------- /distributed-docker-appmaster/src/main/java/com/sogou/dockeronyarn/common/DistributedDockerConfiguration.java: -------------------------------------------------------------------------------- 1 | package com.sogou.dockeronyarn.common; 2 | 3 | import org.apache.hadoop.conf.Configuration; 4 | 5 | import java.util.Properties; 6 | 7 | /** 8 | * Created by guoshiwei on 15/5/10. 9 | */ 10 | public class DistributedDockerConfiguration extends Configuration { 11 | private static String DISTRIBUTED_DOCKER_CONFIGURATION_FILE 12 | = "distributed-docker-site.xml"; 13 | 14 | public static String DDOCKER_PREFIX = "ddocker."; 15 | public static String DDOCKER_RUNNER_PATH = DDOCKER_PREFIX + "runner.path"; 16 | 17 | public static final String CONTAINER_MEMORY = DDOCKER_PREFIX + "container.memeory"; 18 | public static final int DEFAULT_CONTAINER_MEMORY = 512; // MB 19 | 20 | public static final String CONTAINER_CORES = DDOCKER_PREFIX + "container.cores"; 21 | public static final int DEFAULT_CONTAINER_CORES = 1; 22 | 23 | public static final String DOCKER_CERT_PATH = DDOCKER_PREFIX + "docker.cert.path"; 24 | 25 | public static final String DOCKER_HOST = DDOCKER_PREFIX + "docker.host"; 26 | public static final String DEFAULT_DOCKER_HOST = "127.0.0.1:2376"; 27 | 28 | public static final String REQUIRED_RESOURCE_HDFS_PATH = DDOCKER_PREFIX + "resources.hdfs.dir.path"; 29 | static { 30 | Configuration.addDefaultResource(DISTRIBUTED_DOCKER_CONFIGURATION_FILE); 31 | } 32 | 33 | /** 34 | * Load system properties which startswith `DDOCKER_PREFIX` 35 | */ 36 | public void loadSystemProperties(){ 37 | final Properties properties = System.getProperties(); 38 | for(String p: properties.stringPropertyNames()){ 39 | if(p.startsWith(DDOCKER_PREFIX)){ 40 | set(p, properties.getProperty(p)); 41 | } 42 | } 43 | } 44 | 45 | public DistributedDockerConfiguration() { 46 | super(); 47 | loadSystemProperties(); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /distributed-docker-appmaster/src/main/java/com/sogou/dockeronyarn/common/ExitCode.java: -------------------------------------------------------------------------------- 1 | package com.sogou.dockeronyarn.common; 2 | 3 | public enum ExitCode{ 4 | 5 | CONTAINER_NOT_CREATE(254), ILLEGAL_ARGUMENT(253), FAIL(1), SUCC(0), TIMEOUT(250), KILLED(255); 6 | private int value; 7 | public int getValue() { 8 | return value; 9 | } 10 | public void setValue(int value) { 11 | this.value = value; 12 | } 13 | private ExitCode(int value){ 14 | this.value = value; 15 | } 16 | 17 | } 18 | 19 | 20 | 21 | 22 | -------------------------------------------------------------------------------- /distributed-docker-appmaster/src/main/java/com/sogou/dockeronyarn/common/Log4jPropertyHelper.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Licensed to the Apache Software Foundation (ASF) under one 3 | * or more contributor license agreements. See the NOTICE file 4 | * distributed with this work for additional information 5 | * regarding copyright ownership. The ASF licenses this file 6 | * to you under the Apache License, Version 2.0 (the 7 | * "License"); you may not use this file except in compliance 8 | * with the License. You may obtain a copy of the License at 9 | * 10 | * http://www.apache.org/licenses/LICENSE-2.0 11 | * 12 | * Unless required by applicable law or agreed to in writing, software 13 | * distributed under the License is distributed on an "AS IS" BASIS, 14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 15 | * See the License for the specific language governing permissions and 16 | * limitations under the License. 17 | */ 18 | 19 | package com.sogou.dockeronyarn.common; 20 | 21 | import java.io.FileInputStream; 22 | import java.io.InputStream; 23 | import java.util.Map.Entry; 24 | import java.util.Properties; 25 | 26 | import org.apache.commons.io.IOUtils; 27 | import org.apache.log4j.LogManager; 28 | import org.apache.log4j.PropertyConfigurator; 29 | 30 | 31 | public class Log4jPropertyHelper { 32 | 33 | public static void updateLog4jConfiguration(Class targetClass, 34 | String log4jPath) throws Exception { 35 | Properties customProperties = new Properties(); 36 | FileInputStream fs = null; 37 | InputStream is = null; 38 | try { 39 | fs = new FileInputStream(log4jPath); 40 | is = targetClass.getResourceAsStream("/log4j.properties"); 41 | customProperties.load(fs); 42 | Properties originalProperties = new Properties(); 43 | originalProperties.load(is); 44 | for (Entry entry : customProperties.entrySet()) { 45 | originalProperties.setProperty(entry.getKey().toString(), entry 46 | .getValue().toString()); 47 | } 48 | LogManager.resetConfiguration(); 49 | PropertyConfigurator.configure(originalProperties); 50 | }finally { 51 | IOUtils.closeQuietly(is); 52 | IOUtils.closeQuietly(fs); 53 | } 54 | } 55 | } 56 | -------------------------------------------------------------------------------- /distributed-docker-appmaster/src/main/java/com/sogou/dockeronyarn/common/Utils.java: -------------------------------------------------------------------------------- 1 | package com.sogou.dockeronyarn.common; 2 | 3 | /** 4 | * Created by guoshiwei on 15/5/11. 5 | */ 6 | public class Utils { 7 | 8 | public static void checkNotEmpty(String v, String msg) throws IllegalArgumentException{ 9 | if(v == null || v.trim().isEmpty()) 10 | throw new IllegalArgumentException(msg); 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /distributed-docker-appmaster/src/main/java/com/sogou/dockeronyarn/docker/Constants.java: -------------------------------------------------------------------------------- 1 | package com.sogou.dockeronyarn.docker; 2 | 3 | public class Constants { 4 | public static String DOCKER_USER_SPACE = "/search/hadoop/docker_user_space"; 5 | } 6 | -------------------------------------------------------------------------------- /distributed-docker-appmaster/src/main/java/com/sogou/dockeronyarn/docker/DockerContainerRunner.java: -------------------------------------------------------------------------------- 1 | package com.sogou.dockeronyarn.docker; 2 | 3 | import com.github.dockerjava.api.DockerClient; 4 | import com.github.dockerjava.api.DockerException; 5 | import com.github.dockerjava.api.NotModifiedException; 6 | import com.github.dockerjava.api.command.*; 7 | import com.github.dockerjava.api.model.AccessMode; 8 | import com.github.dockerjava.api.model.Bind; 9 | import com.github.dockerjava.api.model.Frame; 10 | import com.github.dockerjava.api.model.Volume; 11 | import com.github.dockerjava.core.DockerClientBuilder; 12 | import com.github.dockerjava.core.DockerClientConfig; 13 | import com.github.dockerjava.core.command.LogContainerResultCallback; 14 | import com.github.dockerjava.core.command.PullImageResultCallback; 15 | 16 | import org.apache.commons.cli.*; 17 | import org.apache.commons.logging.Log; 18 | import org.apache.commons.logging.LogFactory; 19 | import com.github.dockerjava.api.NotFoundException; 20 | import java.io.*; 21 | import java.util.ArrayList; 22 | import java.util.Collections; 23 | import java.util.List; 24 | import java.util.concurrent.TimeUnit; 25 | 26 | public class DockerContainerRunner { 27 | 28 | private static final Log LOG = LogFactory.getLog(DockerContainerRunner.class); 29 | private static String CONTAINER_RUNNER_SCRIPT_PATH = "/runner.py"; 30 | private static String[] RUN_CMD = new String[]{"/usr/bin/python", CONTAINER_RUNNER_SCRIPT_PATH}; 31 | 32 | private final DockerContainerRunnerParam param; 33 | private int stopTimeout = 150; 34 | 35 | private final DockerClient docker; 36 | 37 | private Thread stderrThread; 38 | private Thread waitThread; 39 | 40 | private String containerId; 41 | private int exitcode = ExitCode.TIMEOUT.getValue(); 42 | private volatile boolean containerStopped = false; 43 | private volatile boolean isStopContainerRequested = false; 44 | private List volumeBinds = new ArrayList(); 45 | 46 | private static final long DEFAULT_CONTAINER_MEMORY = 2 *1024 *1024 *1024L ; 47 | private static final int DEFAULT_CONTAINER_CPU_SHARES = 512 ; 48 | 49 | 50 | public DockerContainerRunner(DockerContainerRunnerParam param) { 51 | this.param = param; 52 | this.docker = createDockerClient(); 53 | 54 | Runtime.getRuntime().addShutdownHook( 55 | new Thread("shutdown DockerContainerRunner") { 56 | public void run() { 57 | LOG.info("shutdownhook start"); 58 | try { 59 | shutdown(); 60 | } catch (IOException e) { 61 | LOG.warn(e); 62 | } 63 | LOG.info("shutdownhook end"); 64 | } 65 | } 66 | ); 67 | } 68 | 69 | /** 70 | * Start container, non block. 71 | * 72 | * @throws IOException 73 | * @throws DockerException 74 | */ 75 | public void startContainer(String containerName) throws IOException, DockerException { 76 | LOG.info("Pulling docker image: " + param.dockerImage); 77 | try { 78 | docker.pullImageCmd(param.dockerImage).exec(new PullImageResultCallback()) 79 | .awaitCompletion(181, TimeUnit.SECONDS).close(); 80 | } catch (InterruptedException e) { 81 | throw new RuntimeException("Pull docker image failed.", e); 82 | } 83 | 84 | CreateContainerCmd createContainerCmd = getCreateContainerCmd(containerName); 85 | LOG.info("Creating docker container: " + createContainerCmd.toString()); 86 | this.containerId = createContainerCmd.exec().getId(); 87 | 88 | LOG.info("Start docker container: " + containerId); 89 | docker.startContainerCmd(containerId).exec(); 90 | startLogTailingThreads(containerId); 91 | 92 | this.waitThread = new Thread(new Runnable() { 93 | @Override 94 | public void run() { 95 | //we create a new client to wait the container return 96 | DockerClient checkContainerClient = createDockerClient(); 97 | WaitContainerCmd wc = checkContainerClient.waitContainerCmd(containerId); 98 | try { 99 | exitcode = wc.exec(); 100 | LOG.info(String.format("Container %s exited with exitCode=%d", containerId, exitcode)); 101 | checkContainerClient.close(); 102 | LOG.info("WaittingClient is closed"); 103 | } catch (NotFoundException e) { 104 | LOG.error(String.format("Container %s not found", containerId), e); 105 | exitcode = ExitCode.CONTAINER_NOT_CREATE.getValue(); 106 | } catch (IOException e) { 107 | e.printStackTrace(); 108 | } finally { 109 | wc.close(); 110 | } 111 | } 112 | }, "waitThread-" + containerId); 113 | 114 | waitThread.start(); 115 | } 116 | 117 | /** 118 | * Block until the docker container exit 119 | * 120 | * @return Exit code of the container. 121 | */ 122 | public int waitContainerExit() throws IOException { 123 | final long WAIT_INTERVAL = 100; 124 | long waitedMilliSecs = 0; 125 | 126 | while (true) { 127 | if (isStopContainerRequested) { 128 | doStopContainer("user requested"); 129 | } 130 | 131 | if((param.clientTimeout > 0) && (waitedMilliSecs >= param.clientTimeout)) { 132 | doStopContainer(String.format("Timeout for %d seconds", waitedMilliSecs/1000)); 133 | } 134 | 135 | try { 136 | long waitStart = System.currentTimeMillis(); 137 | waitThread.join(WAIT_INTERVAL); 138 | waitedMilliSecs += System.currentTimeMillis() - waitStart; 139 | } catch (InterruptedException e) { 140 | LOG.info("Interrupted when waiting container to exit"); 141 | break; 142 | } 143 | 144 | if(waitThread.isAlive()) { 145 | // container is still running, keep waiting 146 | continue; 147 | } 148 | else { 149 | LOG.info(String.format("Container %s running for %d secs and stopped.", 150 | containerId, waitedMilliSecs/1000)); 151 | //container is stoped now 152 | containerStopped = true; 153 | break; 154 | } 155 | } 156 | 157 | DockerClient rmClient = createDockerClient(); 158 | try { 159 | rmClient.removeContainerCmd(containerId).exec(); 160 | LOG.info(String.format("Container %s removed.", containerId)); 161 | } catch (NotFoundException e) { 162 | LOG.error(e); 163 | } finally { 164 | rmClient.close(); 165 | LOG.info("removeClient is closed"); 166 | containerId=null; 167 | } 168 | return exitcode; 169 | } 170 | 171 | private void doStopContainer(String reason) { 172 | if (this.containerStopped) { 173 | return; 174 | } 175 | 176 | if (this.containerId == null) { 177 | throw new IllegalStateException("containerId is null when call doStopContainer"); 178 | } 179 | LOG.info(String.format("Stopping Container %s cause %s", containerId, reason)); 180 | // When stopping container, we just send the request to docker service, 181 | // and continue wait the waitThread to exit, which in turn wait the docker container exit. 182 | // If something wrong, cause the container never exit, so our process is blocked forever, 183 | // we just let it be. This situation need to be handled by the admins. 184 | StopContainerCmd stopContainerCmd = docker.stopContainerCmd(containerId); 185 | stopContainerCmd.withTimeout(stopTimeout); 186 | 187 | LOG.info(String.format("Executing stop command: %s", stopContainerCmd.toString())); 188 | try { 189 | stopContainerCmd.exec(); 190 | } catch(NotFoundException nfe) { 191 | handleDockerException(nfe); 192 | }catch(NotModifiedException nme) { 193 | handleDockerException(nme); 194 | }finally { 195 | stopContainerCmd.close(); 196 | } 197 | } 198 | 199 | private void handleDockerException(DockerException e) { 200 | LOG.warn(e); 201 | } 202 | 203 | private int runTask(String containerName) throws IOException, DockerException { 204 | startContainer(containerName); 205 | return waitContainerExit(); 206 | } 207 | 208 | private DockerClient createDockerClient() { 209 | LOG.info("Initializing Docker Client"); 210 | DockerClientConfig.DockerClientConfigBuilder configBuilder = DockerClientConfig 211 | .createDefaultConfigBuilder(); 212 | configBuilder.withUri("" + param.dockerHost); 213 | DockerClientConfig config = configBuilder.build(); 214 | return DockerClientBuilder.getInstance(config).build(); 215 | } 216 | 217 | private CreateContainerCmd getCreateContainerCmd(String containerName) { 218 | CreateContainerCmd con = docker.createContainerCmd(this.param.dockerImage); 219 | Options opts = new Options(); 220 | opts.addOption(OptionBuilder.withLongOpt("rm") 221 | .withDescription("rm the container after execute").create()); 222 | opts.addOption(new Option("v","volume",true,"the memory of container")); 223 | opts.addOption(new Option("m","memory",true,"the memory of container")); 224 | opts.addOption(new Option("c","cpu-shares",true,"the cpu of the container")); 225 | opts.addOption(new Option("H","net",true,"the host of the container")); 226 | CommandLine dockerArgsParser = null; 227 | try { 228 | dockerArgsParser = new GnuParser().parse(opts, param.getDockerArgs(), true); 229 | } catch (ParseException e) { 230 | e.printStackTrace(); 231 | } 232 | 233 | if (dockerArgsParser.hasOption("m")) { 234 | String memoryArgs = dockerArgsParser.getOptionValue("m") ; 235 | LOG.info("Set container memory to " +memoryArgs); 236 | int memorySize = Integer.parseInt(memoryArgs.split("[\\D]+")[0]); 237 | if (memoryArgs.contains("m") || memoryArgs.contains("M")) { 238 | con.withMemoryLimit(new Long(memorySize * 1024 * 1024L )); 239 | } 240 | else if (memoryArgs.contains("g") || memoryArgs.contains("G")) { 241 | con.withMemoryLimit(new Long(memorySize * 1024 * 1024* 1024L)); 242 | } 243 | } 244 | else { 245 | con.withMemoryLimit(DEFAULT_CONTAINER_MEMORY); 246 | } 247 | 248 | if (dockerArgsParser.hasOption("c")) { 249 | int cpushares = Integer.parseInt(dockerArgsParser.getOptionValue("c")); 250 | con.withCpuShares(cpushares); 251 | } 252 | else { 253 | con.withCpuShares(DEFAULT_CONTAINER_CPU_SHARES); 254 | } 255 | 256 | //set --net=host as default 257 | con.withNetworkMode("host"); 258 | if(dockerArgsParser.hasOption("H")) { 259 | String net = dockerArgsParser.getOptionValue("H"); 260 | con.withNetworkMode(net); 261 | } 262 | 263 | con.withName(containerName); 264 | con.withAttachStderr(true); 265 | con.withAttachStdin(false); 266 | con.withAttachStdout(true); 267 | this.volumeBinds.add(new Bind(param.runnerScriptPath, 268 | new Volume(CONTAINER_RUNNER_SCRIPT_PATH), AccessMode.ro)); 269 | for(String mountPath : param.mountVolume.split("\\+")) { 270 | Bind localPath = new Bind(mountPath.split(":")[0], new Volume( 271 | mountPath.split(":")[1]), AccessMode.rw); 272 | volumeBinds.add(localPath); 273 | } 274 | 275 | con.withBinds(volumeBinds.toArray(new Bind[volumeBinds.size()])); 276 | ArrayList cmds = new ArrayList(); 277 | Collections.addAll(cmds, RUN_CMD); 278 | Collections.addAll(cmds, param.cmdAndArgs); 279 | param.cmdAndArgs = cmds.toArray(param.cmdAndArgs); 280 | con.withCmd(this.param.cmdAndArgs); 281 | con.withWorkingDir(param.workingDir); 282 | return con; 283 | } 284 | 285 | public void shutdown() throws IOException { 286 | LOG.info("Finishing"); 287 | // Container should be stopped first 288 | if(!containerStopped) { 289 | LOG.warn(String.format( 290 | "Docker Container not stopped when shutting down, will stop it now", containerId)); 291 | stopContainer(); 292 | waitContainerExit(); 293 | } 294 | 295 | this.docker.close(); 296 | LOG.info("Docker client closed"); 297 | } 298 | 299 | public static class LogContainerYarnCallback extends LogContainerResultCallback { 300 | protected PrintStream out; 301 | public LogContainerYarnCallback() { 302 | super(); 303 | out = System.err; 304 | } 305 | 306 | @Override 307 | public void onNext(Frame item) { 308 | out.println(new String(item.getPayload()).trim()); 309 | } 310 | 311 | @Override 312 | public void onError(Throwable throwable) { 313 | super.onError(throwable); 314 | out.println("LogContainerYarnCallback on Error"); 315 | if (out != null) { 316 | out.close(); 317 | } 318 | } 319 | 320 | @Override 321 | public void onComplete() { 322 | super.onComplete(); 323 | out.println("LogConterYarnCallback on complete"); 324 | if ( out != null ) { 325 | out.close(); 326 | } 327 | } 328 | } 329 | 330 | 331 | private void startLogTailingThreads(final String containerId) { 332 | this.stderrThread = createTailingThread(containerId); 333 | stderrThread.start(); 334 | } 335 | 336 | private Thread createTailingThread(final String containerId) { 337 | Thread thread = new Thread() { 338 | @Override 339 | public void run() { 340 | try { 341 | docker.logContainerCmd(containerId) 342 | .withFollowStream(true) 343 | .withStdErr(true) 344 | .withStdOut(true) 345 | .withTimestamps(false) 346 | .exec(new LogContainerYarnCallback()) 347 | .awaitCompletion(); 348 | 349 | LOG.info(String.format("Tailing STDOUT/STDERR of container %s stopped", 350 | containerId)); 351 | } catch (Exception e) { 352 | LOG.error(e); 353 | } 354 | } 355 | }; 356 | 357 | thread.setDaemon(true); 358 | return thread; 359 | } 360 | 361 | 362 | public static void main(String[] args) { 363 | 364 | int result = -1; 365 | try { 366 | DockerContainerRunnerParam dockerContainerRunnerParam = new DockerContainerRunnerParam(); 367 | try { 368 | dockerContainerRunnerParam.initFromCmdlineArgs(args); 369 | if (dockerContainerRunnerParam.isPrintHelp) { 370 | dockerContainerRunnerParam.printUsage(); 371 | System.exit(ExitCode.SUCC.getValue()); 372 | } 373 | } catch (IllegalArgumentException e) { 374 | System.err.println(e.getLocalizedMessage()); 375 | dockerContainerRunnerParam.printUsage(); 376 | System.exit(ExitCode.ILLEGAL_ARGUMENT.getValue()); 377 | } 378 | 379 | DockerContainerRunner client = new DockerContainerRunner(dockerContainerRunnerParam); 380 | 381 | result = client.runTask(String.format("dockerClientRunner-%d", System.currentTimeMillis())); 382 | 383 | } catch (Throwable t) { 384 | LOG.fatal("Error running CLient", t); 385 | System.exit(ExitCode.FAIL.getValue()); 386 | } 387 | 388 | if (result == 0) { 389 | LOG.info("docker task completed successfully"); 390 | System.exit(ExitCode.SUCC.getValue()); 391 | } 392 | 393 | LOG.info("Application failed to complete successfully"); 394 | LOG.info("client ends with value: " + result); 395 | System.exit(result); 396 | } 397 | 398 | public int getExitStatus() { 399 | return exitcode; 400 | } 401 | 402 | public void stopContainer(){ 403 | isStopContainerRequested = true; 404 | } 405 | 406 | public float getProgress() { 407 | // TODO Implement getProgress 408 | return 0; 409 | } 410 | 411 | } 412 | -------------------------------------------------------------------------------- /distributed-docker-appmaster/src/main/java/com/sogou/dockeronyarn/docker/DockerContainerRunnerParam.java: -------------------------------------------------------------------------------- 1 | package com.sogou.dockeronyarn.docker; 2 | 3 | import org.apache.commons.cli.*; 4 | 5 | import java.util.ArrayList; 6 | import java.util.HashMap; 7 | import java.util.Map; 8 | 9 | public class DockerContainerRunnerParam {// Debug flag 10 | public boolean debugFlag = false;// Args to be passed to the shell command 11 | public String[] cmdAndArgs;// Env variables to be setup for the shell command 12 | public Map cmdEnv = new HashMap(); 13 | public String dockerImage; 14 | public String dockerHost; 15 | public String dockerCertPath; 16 | public String mountVolume; 17 | 18 | 19 | public String workingDir; 20 | 21 | //the path to be mounted into docker container,eg: {"/root/ugi_config:/root/uig_config"} 22 | private String[] mountPaths ; 23 | public String[] virtualDirs;// memory to request for container in docker will be executed 24 | public long containerMemory = 10;// virtual cores to request for container in docker will be executed,to do 25 | 26 | public String[] getDockerArgs() { 27 | return dockerArgs; 28 | } 29 | 30 | public void setDockerArgs(String[] dockerArgs) { 31 | this.dockerArgs = dockerArgs; 32 | } 33 | 34 | public int containerVirtualCores = 512;// Timeout threshold for client. Kill app after time interval expires. 35 | public long clientTimeout = 24 * 3600 * 1000; 36 | public String runnerScriptPath; // The absolute path of runner script on host local filesystem 37 | private Options opts = new Options(); 38 | public boolean isPrintHelp = false; // TODO: get value from command line options. 39 | private String[] dockerArgs = {}; 40 | 41 | 42 | public DockerContainerRunnerParam(){ 43 | opts.addOption("timeout", true, "Application timeout in milliseconds"); 44 | opts.addOption("docker_image", true, "docker image to be executed"); 45 | opts.addOption("docker_host", true, "ip:port of docker server, example: localhost:2376"); 46 | opts.addOption("docker_cert_path", true, "Example: /some/path/certs"); 47 | 48 | opts.addOption("runner_path", true, "The absolute path of runner script on host local filesystem"); 49 | opts.addOption("workingdir", true, "working dir for command"); 50 | opts.addOption("cmd_env", true, 51 | "Environment. Specified as env_key=env_val pairs"); 52 | opts.addOption("container_memory", true, 53 | "Amount of memory in MB to be requested to run the docker container"); 54 | opts.addOption("container_vcores", true, 55 | "Amount of virtual cores to be requested to run the docker container"); 56 | opts.addOption("debug", false, "Dump out debug information"); 57 | } 58 | 59 | private static void requireOptionValue(String optionValue, String optionName){ 60 | if(optionValue == null || optionValue.trim().isEmpty()){ 61 | throw new IllegalArgumentException("No value given for '" + optionName 62 | + "' option."); 63 | } 64 | } 65 | public void initFromCmdlineArgs(String[] args) throws ParseException { 66 | 67 | Options opts = getOptions(); 68 | CommandLine cmdLine = new GnuParser().parse(opts, args); 69 | ArrayList cmds = new ArrayList(); 70 | 71 | if (args.length == 0) { 72 | throw new IllegalArgumentException( 73 | "No args specified for DockerContainerRunner to initialize"); 74 | } 75 | 76 | if (!cmdLine.hasOption("docker_image")) { 77 | throw new IllegalArgumentException( 78 | "No image specified for DockerContainerRunner to initialize"); 79 | } else { 80 | dockerImage = cmdLine.getOptionValue("docker_image"); 81 | if (dockerImage == null 82 | || dockerImage.trim().length() == 0) { 83 | throw new IllegalArgumentException( 84 | "No image specified for DockerContainerRunner to initialize"); 85 | } 86 | } 87 | dockerHost = cmdLine.getOptionValue("docker_host"); 88 | requireOptionValue(dockerHost, "docker_host"); 89 | 90 | dockerCertPath = cmdLine.getOptionValue("docker_cert_path"); 91 | requireOptionValue(dockerCertPath, "docker_cert_path"); 92 | 93 | runnerScriptPath = cmdLine.getOptionValue("runner_path", null); 94 | 95 | if (runnerScriptPath == null || runnerScriptPath.trim().length() == 0) { 96 | throw new IllegalArgumentException( 97 | "No runpath specified for DockerContainerRunner to run cmds"); 98 | } 99 | 100 | 101 | workingDir = cmdLine.getOptionValue("workingdir", 102 | "/search"); 103 | 104 | if (cmdLine.hasOption("debug")) { 105 | debugFlag = true; 106 | try { 107 | Thread.sleep(10000l); 108 | } catch (InterruptedException e) { 109 | // TODO Auto-generated catch block 110 | e.printStackTrace(); 111 | } 112 | } 113 | 114 | if (cmdLine.hasOption("cmd_env")) { 115 | String envs[] = cmdLine.getOptionValues("cmd_env"); 116 | for (String env : envs) { 117 | env = env.trim(); 118 | int index = env.indexOf('='); 119 | if (index == -1) { 120 | cmdEnv.put(env, ""); 121 | continue; 122 | } 123 | String key = env.substring(0, index); 124 | String val = ""; 125 | if (index < (env.length() - 1)) { 126 | val = env.substring(index + 1); 127 | } 128 | cmdEnv.put(key, val); 129 | } 130 | } 131 | 132 | containerMemory = Integer.parseInt(cmdLine.getOptionValue( 133 | "container_memory", "10")); 134 | containerVirtualCores = Integer.parseInt(cmdLine.getOptionValue( 135 | "container_vcores", "1")); 136 | 137 | if (containerMemory < 0 || containerVirtualCores < 0) { 138 | throw new IllegalArgumentException( 139 | "Invalid no. of containers or container memory/vcores specified," 140 | + " exiting." + " Specified containerMemory=" 141 | + containerMemory + ", containerVirtualCores=" 142 | + containerVirtualCores); 143 | } 144 | 145 | clientTimeout = Integer.parseInt(cmdLine.getOptionValue("timeout", 146 | "86400000")); 147 | 148 | if (clientTimeout <= 0) { 149 | throw new IllegalArgumentException( 150 | " Illegal timeout specified for DockerContainerRunner to initialize"); 151 | } 152 | 153 | cmdAndArgs = cmdLine.getArgs(); 154 | } 155 | 156 | private Options getOptions() { 157 | 158 | return opts; 159 | } 160 | 161 | public void printUsage() { 162 | new HelpFormatter().printHelp("DockerContainerRunner [options] command [args ... ]", opts); 163 | } 164 | } -------------------------------------------------------------------------------- /distributed-docker-appmaster/src/main/java/com/sogou/dockeronyarn/docker/DockerShareSpaceMonitor.java: -------------------------------------------------------------------------------- 1 | package com.sogou.dockeronyarn.docker; 2 | 3 | import java.io.File; 4 | import java.io.IOException; 5 | import java.text.SimpleDateFormat; 6 | import java.util.Date; 7 | import java.util.List; 8 | 9 | import com.github.dockerjava.core.DockerClientBuilder; 10 | import org.apache.commons.logging.Log; 11 | import org.apache.commons.logging.LogFactory; 12 | 13 | import com.github.dockerjava.api.DockerClient; 14 | import com.github.dockerjava.api.command.InspectContainerCmd; 15 | import com.github.dockerjava.api.command.InspectContainerResponse; 16 | import com.github.dockerjava.api.command.InspectContainerResponse.ContainerState; 17 | import com.github.dockerjava.api.command.ListContainersCmd; 18 | import com.github.dockerjava.api.command.RemoveContainerCmd; 19 | import com.github.dockerjava.api.model.Container; 20 | import com.github.dockerjava.core.DockerClientConfig; 21 | import com.github.dockerjava.core.DockerClientConfig.DockerClientConfigBuilder; 22 | 23 | 24 | public class DockerShareSpaceMonitor { 25 | private static final Log LOG = LogFactory.getLog(DockerShareSpaceMonitor.class); 26 | public static long cLeanInterv = 86400000 * 3; 27 | 28 | public static void main(String[] args) { 29 | //CleanDockerShareSpaceRunner cdsr = new CleanDockerShareSpaceRunner(); 30 | CleanDockerSpaceRunner cdsr = new CleanDockerSpaceRunner(); 31 | Runtime.getRuntime().addShutdownHook(new Thread(new ShutDownHook(cdsr), "shutdownhook")); 32 | 33 | Thread thread = new Thread(cdsr, "dockerSpacecleaner"); 34 | thread.start(); 35 | } 36 | 37 | public static class ShutDownHook implements Runnable{ 38 | public CleanDockerSpaceRunner cleanDockerShareSpaceRunner; 39 | 40 | public ShutDownHook(CleanDockerSpaceRunner cleanDockerShareSpaceRunner){ 41 | this.cleanDockerShareSpaceRunner = cleanDockerShareSpaceRunner; 42 | } 43 | 44 | public void run(){ 45 | cleanDockerShareSpaceRunner.stoped(); 46 | while(!cleanDockerShareSpaceRunner.isKilled()){ 47 | try { 48 | Thread.sleep(1000L); 49 | } catch (InterruptedException e) { 50 | // TODO Auto-generated catch block 51 | e.printStackTrace(); 52 | } 53 | } 54 | } 55 | } 56 | 57 | public static class CleanDockerShareSpaceRunner implements Runnable{ 58 | volatile boolean stoped = false; 59 | volatile boolean killed = false; 60 | 61 | @Override 62 | public void run() { 63 | while (!stoped) { 64 | File file = new File(Constants.DOCKER_USER_SPACE); 65 | if (!file.isDirectory()) { 66 | LOG.error(Constants.DOCKER_USER_SPACE + "is not dir"); 67 | try { 68 | Thread.sleep(1000L); 69 | } catch (InterruptedException e) { 70 | // TODO Auto-generated catch block 71 | e.printStackTrace(); 72 | } 73 | continue; 74 | } 75 | 76 | File[] files = file.listFiles(); 77 | if (files == null) { 78 | try { 79 | Thread.sleep(1000L); 80 | } catch (InterruptedException e) { 81 | // TODO Auto-generated catch block 82 | e.printStackTrace(); 83 | } 84 | continue; 85 | } 86 | for (int i = 0; i < files.length; ++i) { 87 | if(this.stoped) 88 | continue; 89 | if(files[i] == null) 90 | continue; 91 | if (files[i].isFile()) { 92 | if (!files[i].delete()) { 93 | LOG.error(files[i].getAbsolutePath() + "can not be deleted"); 94 | } 95 | } 96 | else if (files[i].isDirectory()) { 97 | long lastModified = files[i].lastModified(); 98 | if (System.currentTimeMillis() - lastModified > cLeanInterv) { 99 | deleteDir(files[i]); 100 | } 101 | } 102 | } 103 | 104 | if (this.stoped == false) { 105 | try { 106 | Thread.sleep(10000L); 107 | } catch (InterruptedException e) { 108 | // TODO Auto-generated catch block 109 | e.printStackTrace(); 110 | } 111 | } 112 | } 113 | killed = true; 114 | } 115 | 116 | private void deleteDir(File file) { 117 | if (this.stoped) 118 | return; 119 | 120 | if (file.isDirectory()) { 121 | File[] files = file.listFiles(); 122 | if (files == null) { 123 | boolean del = file.delete(); 124 | if (!del) { 125 | LOG.error(file.getAbsolutePath() + "can not be deleted"); 126 | } 127 | return; 128 | } 129 | for (int i = 0; i < files.length; ++i) { 130 | if(files[i] == null) continue; 131 | deleteDir(files[i]); 132 | } 133 | if (this.stoped) 134 | return; 135 | boolean del = file.delete(); 136 | if (!del) { 137 | LOG.error(file.getAbsolutePath() + "can not be deleted"); 138 | } 139 | } 140 | else { 141 | if (this.stoped) 142 | return; 143 | boolean del = file.delete(); 144 | if (!del) { 145 | LOG.error(file.getAbsolutePath() + "can not be deleted"); 146 | } 147 | } 148 | 149 | } 150 | 151 | public void stoped() { 152 | this.stoped = true; 153 | } 154 | 155 | public boolean isKilled() { 156 | return this.killed; 157 | } 158 | 159 | } 160 | 161 | 162 | public static class CleanDockerSpaceRunner implements Runnable{ 163 | volatile boolean stoped = false; 164 | volatile boolean killed = false; 165 | 166 | @Override 167 | public void run() { 168 | while (!stoped) { 169 | DockerClientConfigBuilder configBuilder = DockerClientConfig 170 | .createDefaultConfigBuilder(); 171 | DockerClientConfig config = configBuilder.build(); 172 | DockerClient docker = DockerClientBuilder 173 | .getInstance(config).build(); 174 | do { 175 | try { 176 | ListContainersCmd listContainerCmd = docker.listContainersCmd(); 177 | listContainerCmd.withShowAll(true); 178 | List containers = listContainerCmd.exec(); 179 | 180 | if (containers == null || containers.size() == 0) 181 | break; 182 | 183 | for (int i = 0; i < containers.size(); ++i) { 184 | if (this.stoped) 185 | break; 186 | 187 | Container container = containers.get(i); 188 | if (container == null) 189 | continue; 190 | 191 | if (System.currentTimeMillis() - container.getCreated() * 1000 < cLeanInterv) { 192 | continue; 193 | } 194 | try { 195 | removeContainer(docker, container); 196 | } catch(Exception e) { 197 | e.printStackTrace(); 198 | } 199 | } 200 | } catch(Exception e) { 201 | e.printStackTrace(); 202 | LOG.warn("cleanning docker meets exception " + e.getMessage()); 203 | } 204 | } while(false); 205 | 206 | try { 207 | docker.close(); 208 | } catch (IOException e) { 209 | // TODO Auto-generated catch block 210 | e.printStackTrace(); 211 | } 212 | 213 | if (!this.stoped) { 214 | try { 215 | Thread.sleep(10000L); 216 | } catch (InterruptedException e) { 217 | // TODO Auto-generated catch block 218 | e.printStackTrace(); 219 | } 220 | } 221 | } 222 | killed = true; 223 | } 224 | 225 | private void removeContainer(DockerClient docker , Container container) { 226 | SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); 227 | InspectContainerCmd inspectContainerCmd = docker.inspectContainerCmd(container.getId()); 228 | InspectContainerResponse response = inspectContainerCmd.exec(); 229 | if (response == null) 230 | return; 231 | 232 | ContainerState state = response.getState(); 233 | if(state == null) 234 | return; 235 | 236 | if (state.isRunning()) 237 | return; 238 | 239 | String finishTime = state.getFinishedAt(); 240 | finishTime = finishTime.replace('T', ' ').replace('Z', ' '); 241 | 242 | Date date = null; 243 | try { 244 | date = sdf.parse(finishTime); 245 | } catch (java.text.ParseException e) { 246 | LOG.warn("date parse exception: "+ e.getMessage()); 247 | } 248 | 249 | if (date == null || System.currentTimeMillis() - date.getTime() > cLeanInterv) { 250 | LOG.info("deleting container id: " + container.getId() + " finishTime: " + finishTime); 251 | RemoveContainerCmd removeCmd = docker.removeContainerCmd(container.getId()); 252 | removeCmd.withRemoveVolumes(true); 253 | removeCmd.withForce(true); 254 | removeCmd.exec(); 255 | File file = new File(Constants.DOCKER_USER_SPACE + "/" + container.getId()); 256 | if (file != null && file.isDirectory()) { 257 | Process deleteProcess = null; 258 | try { 259 | deleteProcess = Runtime.getRuntime().exec("rm -rf " + Constants.DOCKER_USER_SPACE + "/" + container.getId()); 260 | deleteProcess.waitFor(); 261 | LOG.warn("deleting " + Constants.DOCKER_USER_SPACE + "/" + container.getId() + " succ: " + (deleteProcess.exitValue() == 0) ); 262 | } catch (IOException e) { 263 | // TODO Auto-generated catch block 264 | e.printStackTrace(); 265 | } catch (InterruptedException e) { 266 | // TODO Auto-generated catch block 267 | e.printStackTrace(); 268 | } 269 | } 270 | } 271 | } 272 | 273 | public void stoped() { 274 | this.stoped = true; 275 | } 276 | 277 | public boolean isKilled() { 278 | return this.killed; 279 | } 280 | } 281 | } 282 | -------------------------------------------------------------------------------- /distributed-docker-appmaster/src/main/java/com/sogou/dockeronyarn/docker/ExitCode.java: -------------------------------------------------------------------------------- 1 | package com.sogou.dockeronyarn.docker; 2 | 3 | public enum ExitCode{ 4 | 5 | CONTAINER_NOT_CREATE(254), ILLEGAL_ARGUMENT(253), FAIL(1), SUCC(0), TIMEOUT(250), KILLED(255), IMAGE_NOTFOUND(252), RUNTIME_EXCEPTION(251); 6 | 7 | private int value; 8 | public int getValue() { 9 | return value; 10 | } 11 | public void setValue(int value) { 12 | this.value = value; 13 | } 14 | private ExitCode(int value){ 15 | this.value = value; 16 | } 17 | 18 | } 19 | 20 | 21 | 22 | 23 | -------------------------------------------------------------------------------- /distributed-docker-appmaster/src/main/java/com/sogou/dockeronyarn/docker/Test.java: -------------------------------------------------------------------------------- 1 | package com.sogou.dockeronyarn.docker; 2 | 3 | import java.util.ArrayList; 4 | 5 | public class Test { 6 | 7 | public static void main(String[] args) { 8 | ArrayList list = new ArrayList(); 9 | while(true){ 10 | try { 11 | Thread.sleep(1000); 12 | } catch (InterruptedException e) { 13 | // TODO Auto-generated catch block 14 | e.printStackTrace(); 15 | } 16 | } 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /distributed-docker-appmaster/src/main/resources/META-INF/services/javax.ws.rs.ext.RuntimeDelegate: -------------------------------------------------------------------------------- 1 | org.glassfish.jersey.internal.RuntimeDelegateImpl -------------------------------------------------------------------------------- /distributed-docker-submit-client/.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /distributed-docker-submit-client/.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | distributed-docker-submit-client 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | org.eclipse.m2e.core.maven2Builder 15 | 16 | 17 | 18 | 19 | 20 | org.eclipse.jdt.core.javanature 21 | org.eclipse.m2e.core.maven2Nature 22 | 23 | 24 | -------------------------------------------------------------------------------- /distributed-docker-submit-client/.settings/org.eclipse.core.resources.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | encoding//src/main/java=GB18030 3 | encoding/=UTF-8 4 | -------------------------------------------------------------------------------- /distributed-docker-submit-client/.settings/org.eclipse.jdt.core.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 3 | org.eclipse.jdt.core.compiler.compliance=1.6 4 | org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning 5 | org.eclipse.jdt.core.compiler.source=1.6 6 | -------------------------------------------------------------------------------- /distributed-docker-submit-client/.settings/org.eclipse.m2e.core.prefs: -------------------------------------------------------------------------------- 1 | activeProfiles= 2 | eclipse.preferences.version=1 3 | resolveWorkspaceProjects=true 4 | version=1 5 | -------------------------------------------------------------------------------- /distributed-docker-submit-client/README.MD: -------------------------------------------------------------------------------- 1 | # 关于Docker容器相关垃圾的清理 2 | 3 | * 已退出的Container，虽然DockerRunnerApplicationMaster会在container退出时删除，但还是可能由于某种原因没有清理，变为垃圾。 4 | 可启动一定时流程，清理一定时间（如一周）前退出的container 5 | * 如果是Container中应用，通过volume挂载向外输出的中间文件，则应确保volume挂载的是YARN AppCache目录下的子目录，因而交给YARN来清理 6 | 7 | # TODO 8 | 9 | * [DONE] Integrate History Server 10 | * [DONE] Implement docker client 11 | * [CANCEL] Implement AM Tracking url 12 | * [DONE] runner.py 通过YarnClient 指定 13 | * [DONE] 给这个新Framework想个名字 DistributedDocker 14 | * [DONE] 常用配置通过Config文件指定 15 | * [DONE] Client提供submit接口，并提供异步追踪进度的方法 16 | 17 | -------------------------------------------------------------------------------- /distributed-docker-submit-client/hadoop-env.sh: -------------------------------------------------------------------------------- 1 | export JAVA_HOME=/usr/lib/jvm/java-1.7.0-oracle.x86_64 2 | #export HADOOP_PREFIX=/Users/guoshiwei/DEV/ENV/hadoop-2.7.0/ 3 | -------------------------------------------------------------------------------- /distributed-docker-submit-client/log4j.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # Define some default values that can be overridden by system properties 18 | hadoop.root.logger=INFO,console 19 | hadoop.log.dir=. 20 | hadoop.log.file=hadoop.log 21 | 22 | # Define the root logger to the system property "hadoop.root.logger". 23 | log4j.rootLogger=${hadoop.root.logger}, EventCounter 24 | 25 | # Logging Threshold 26 | log4j.threshold=ALL 27 | 28 | # Null Appender 29 | log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender 30 | 31 | # 32 | # Rolling File Appender - cap space usage at 5gb. 33 | # 34 | hadoop.log.maxfilesize=256MB 35 | hadoop.log.maxbackupindex=20 36 | log4j.appender.RFA=org.apache.log4j.RollingFileAppender 37 | log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file} 38 | 39 | log4j.appender.RFA.MaxFileSize=${hadoop.log.maxfilesize} 40 | log4j.appender.RFA.MaxBackupIndex=${hadoop.log.maxbackupindex} 41 | 42 | log4j.appender.RFA.layout=org.apache.log4j.PatternLayout 43 | 44 | # Pattern format: Date LogLevel LoggerName LogMessage 45 | log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 46 | # Debugging Pattern format 47 | #log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 48 | 49 | 50 | # 51 | # Daily Rolling File Appender 52 | # 53 | 54 | log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender 55 | log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file} 56 | 57 | # Rollover at midnight 58 | log4j.appender.DRFA.DatePattern=.yyyy-MM-dd 59 | 60 | log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout 61 | 62 | # Pattern format: Date LogLevel LoggerName LogMessage 63 | log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 64 | # Debugging Pattern format 65 | #log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 66 | 67 | 68 | # 69 | # console 70 | # Add "console" to rootlogger above if you want to use this 71 | # 72 | 73 | log4j.appender.console=org.apache.log4j.ConsoleAppender 74 | log4j.appender.console.target=System.err 75 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 76 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n 77 | 78 | # 79 | # TaskLog Appender 80 | # 81 | 82 | #Default values 83 | hadoop.tasklog.taskid=null 84 | hadoop.tasklog.iscleanup=false 85 | hadoop.tasklog.noKeepSplits=4 86 | hadoop.tasklog.totalLogFileSize=100 87 | hadoop.tasklog.purgeLogSplits=true 88 | hadoop.tasklog.logsRetainHours=12 89 | 90 | log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender 91 | log4j.appender.TLA.taskId=${hadoop.tasklog.taskid} 92 | log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup} 93 | log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize} 94 | 95 | log4j.appender.TLA.layout=org.apache.log4j.PatternLayout 96 | log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 97 | 98 | # 99 | # HDFS block state change log from block manager 100 | # 101 | # Uncomment the following to suppress normal block state change 102 | # messages from BlockManager in NameNode. 103 | #log4j.logger.BlockStateChange=WARN 104 | 105 | # 106 | #Security appender 107 | # 108 | hadoop.security.logger=INFO,NullAppender 109 | hadoop.security.log.maxfilesize=256MB 110 | hadoop.security.log.maxbackupindex=20 111 | log4j.category.SecurityLogger=${hadoop.security.logger} 112 | hadoop.security.log.file=SecurityAuth-${user.name}.audit 113 | log4j.appender.RFAS=org.apache.log4j.RollingFileAppender 114 | log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} 115 | log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout 116 | log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 117 | log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize} 118 | log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex} 119 | 120 | # 121 | # Daily Rolling Security appender 122 | # 123 | log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender 124 | log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} 125 | log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout 126 | log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 127 | log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd 128 | 129 | # 130 | # hadoop configuration logging 131 | # 132 | 133 | # Uncomment the following line to turn off configuration deprecation warnings. 134 | # log4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN 135 | 136 | # 137 | # hdfs audit logging 138 | # 139 | hdfs.audit.logger=INFO,NullAppender 140 | hdfs.audit.log.maxfilesize=256MB 141 | hdfs.audit.log.maxbackupindex=20 142 | log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger} 143 | log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false 144 | log4j.appender.RFAAUDIT=org.apache.log4j.RollingFileAppender 145 | log4j.appender.RFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log 146 | log4j.appender.RFAAUDIT.layout=org.apache.log4j.PatternLayout 147 | log4j.appender.RFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 148 | log4j.appender.RFAAUDIT.MaxFileSize=${hdfs.audit.log.maxfilesize} 149 | log4j.appender.RFAAUDIT.MaxBackupIndex=${hdfs.audit.log.maxbackupindex} 150 | 151 | # 152 | # mapred audit logging 153 | # 154 | mapred.audit.logger=INFO,NullAppender 155 | mapred.audit.log.maxfilesize=256MB 156 | mapred.audit.log.maxbackupindex=20 157 | log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger} 158 | log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false 159 | log4j.appender.MRAUDIT=org.apache.log4j.RollingFileAppender 160 | log4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log 161 | log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout 162 | log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 163 | log4j.appender.MRAUDIT.MaxFileSize=${mapred.audit.log.maxfilesize} 164 | log4j.appender.MRAUDIT.MaxBackupIndex=${mapred.audit.log.maxbackupindex} 165 | 166 | # Custom Logging levels 167 | 168 | #log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG 169 | #log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG 170 | #log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=DEBUG 171 | 172 | # Jets3t library 173 | log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR 174 | 175 | # AWS SDK & S3A FileSystem 176 | log4j.logger.com.amazonaws=ERROR 177 | log4j.logger.com.amazonaws.http.AmazonHttpClient=ERROR 178 | log4j.logger.org.apache.hadoop.fs.s3a.S3AFileSystem=WARN 179 | 180 | # 181 | # Event Counter Appender 182 | # Sends counts of logging messages at different severity levels to Hadoop Metrics. 183 | # 184 | log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter 185 | 186 | # 187 | # Job Summary Appender 188 | # 189 | # Use following logger to send summary to separate file defined by 190 | # hadoop.mapreduce.jobsummary.log.file : 191 | # hadoop.mapreduce.jobsummary.logger=INFO,JSA 192 | # 193 | hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger} 194 | hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log 195 | hadoop.mapreduce.jobsummary.log.maxfilesize=256MB 196 | hadoop.mapreduce.jobsummary.log.maxbackupindex=20 197 | log4j.appender.JSA=org.apache.log4j.RollingFileAppender 198 | log4j.appender.JSA.File=${hadoop.log.dir}/${hadoop.mapreduce.jobsummary.log.file} 199 | log4j.appender.JSA.MaxFileSize=${hadoop.mapreduce.jobsummary.log.maxfilesize} 200 | log4j.appender.JSA.MaxBackupIndex=${hadoop.mapreduce.jobsummary.log.maxbackupindex} 201 | log4j.appender.JSA.layout=org.apache.log4j.PatternLayout 202 | log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n 203 | log4j.logger.org.apache.hadoop.mapred.JobInProgress$JobSummary=${hadoop.mapreduce.jobsummary.logger} 204 | log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false 205 | 206 | # 207 | # Yarn ResourceManager Application Summary Log 208 | # 209 | # Set the ResourceManager summary log filename 210 | yarn.server.resourcemanager.appsummary.log.file=rm-appsummary.log 211 | # Set the ResourceManager summary log level and appender 212 | yarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger} 213 | #yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY 214 | 215 | # To enable AppSummaryLogging for the RM, 216 | # set yarn.server.resourcemanager.appsummary.logger to 217 | # ,RMSUMMARY in hadoop-env.sh 218 | 219 | # Appender for ResourceManager Application Summary Log 220 | # Requires the following properties to be set 221 | # - hadoop.log.dir (Hadoop Log directory) 222 | # - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename) 223 | # - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender) 224 | 225 | log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger} 226 | log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false 227 | log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender 228 | log4j.appender.RMSUMMARY.File=${hadoop.log.dir}/${yarn.server.resourcemanager.appsummary.log.file} 229 | log4j.appender.RMSUMMARY.MaxFileSize=256MB 230 | log4j.appender.RMSUMMARY.MaxBackupIndex=20 231 | log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout 232 | log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 233 | 234 | # HS audit log configs 235 | #mapreduce.hs.audit.logger=INFO,HSAUDIT 236 | #log4j.logger.org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger=${mapreduce.hs.audit.logger} 237 | #log4j.additivity.org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger=false 238 | #log4j.appender.HSAUDIT=org.apache.log4j.DailyRollingFileAppender 239 | #log4j.appender.HSAUDIT.File=${hadoop.log.dir}/hs-audit.log 240 | #log4j.appender.HSAUDIT.layout=org.apache.log4j.PatternLayout 241 | #log4j.appender.HSAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 242 | #log4j.appender.HSAUDIT.DatePattern=.yyyy-MM-dd 243 | 244 | # Http Server Request Logs 245 | #log4j.logger.http.requests.namenode=INFO,namenoderequestlog 246 | #log4j.appender.namenoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender 247 | #log4j.appender.namenoderequestlog.Filename=${hadoop.log.dir}/jetty-namenode-yyyy_mm_dd.log 248 | #log4j.appender.namenoderequestlog.RetainDays=3 249 | 250 | #log4j.logger.http.requests.datanode=INFO,datanoderequestlog 251 | #log4j.appender.datanoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender 252 | #log4j.appender.datanoderequestlog.Filename=${hadoop.log.dir}/jetty-datanode-yyyy_mm_dd.log 253 | #log4j.appender.datanoderequestlog.RetainDays=3 254 | 255 | #log4j.logger.http.requests.resourcemanager=INFO,resourcemanagerrequestlog 256 | #log4j.appender.resourcemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender 257 | #log4j.appender.resourcemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-resourcemanager-yyyy_mm_dd.log 258 | #log4j.appender.resourcemanagerrequestlog.RetainDays=3 259 | 260 | #log4j.logger.http.requests.jobhistory=INFO,jobhistoryrequestlog 261 | #log4j.appender.jobhistoryrequestlog=org.apache.hadoop.http.HttpRequestLogAppender 262 | #log4j.appender.jobhistoryrequestlog.Filename=${hadoop.log.dir}/jetty-jobhistory-yyyy_mm_dd.log 263 | #log4j.appender.jobhistoryrequestlog.RetainDays=3 264 | 265 | #log4j.logger.http.requests.nodemanager=INFO,nodemanagerrequestlog 266 | #log4j.appender.nodemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender 267 | #log4j.appender.nodemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-nodemanager-yyyy_mm_dd.log 268 | #log4j.appender.nodemanagerrequestlog.RetainDays=3 269 | -------------------------------------------------------------------------------- /distributed-docker-submit-client/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | distributed-docker-submit-client 5 | distributed-docker-submit-client 6 | 7 | distributed-docker-submit-project 8 | distributed-docker-submit-project 9 | 0.0.1-SNAPSHOT 10 | ../../distributed-docker-submit-project 11 | 12 | 13 | 0.0.1-SNAPSHOT 14 | distributed-docker-submit-client 15 | distributed-docker system of cloud_dev@sogou-inc.com 16 | 17 | true 18 | 19 | UTF-8 20 | true 21 | false 22 | 1.6 23 | 1.6 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | org.apache.hadoop 48 | hadoop-yarn-client 49 | 2.5.0-cdh5.3.2 50 | 51 | 52 | 53 | jersey-core 54 | com.sun.jersey 55 | 56 | 57 | com.sun.jersey 58 | jersey-server 59 | 60 | 61 | 62 | 63 | 64 | 65 | distributed-docker-submit-project 66 | distributed-docker-appmaster 67 | 0.0.1-SNAPSHOT 68 | 69 | 70 | 71 | 72 | 73 | 74 | maven-compiler-plugin 75 | 3.0 76 | 77 | 1.6 78 | 1.6 79 | GB18030 80 | 81 | 82 | 83 | 84 | 85 | 86 | org.apache.maven.plugins 87 | maven-jar-plugin 88 | 2.4 89 | 90 | 91 | 92 | true 93 | 94 | 95 | 96 | 97 | 98 | 99 | 100 | 101 | 102 | 103 | 104 | -------------------------------------------------------------------------------- /distributed-docker-submit-client/run_YarnDockerClient.sh: -------------------------------------------------------------------------------- 1 | export JAVA_HOME=/Library/Java/JavaVirtualMachines/jdk1.7.0_71.jdk/Contents/Home 2 | 3 | CP=. 4 | while read line 5 | do 6 | if [ ${line:0:1} == "#" ]; then 7 | continue 8 | fi 9 | CP=$CP:$line 10 | done The distributed shell client allows an application master to be launched that in turn would run 86 | * the provided shell command on a set of containers.

87 | * 88 | *

This client is meant to act as an example on how to write yarn-based applications.

89 | * 90 | *

To submit an application, a client first needs to connect to the ResourceManager 91 | * aka ApplicationsManager or ASM via the {@link ApplicationClientProtocol}. The {@link ApplicationClientProtocol} 92 | * provides a way for the client to get access to cluster information and to request for a 93 | * new {@link ApplicationId}.

94 | * 95 | *

For the actual job submission, the client first has to create an {@link ApplicationSubmissionContext}. 96 | * The {@link ApplicationSubmissionContext} defines the application details such as {@link ApplicationId} 97 | * and application name, the priority assigned to the application and the queue 98 | * to which this application needs to be assigned. In addition to this, the {@link ApplicationSubmissionContext} 99 | * also defines the {@link ContainerLaunchContext} which describes the Container with which 100 | * the {@link DockerApplicationMaster_23} is launched.

101 | * 102 | *

The {@link ContainerLaunchContext} in this scenario defines the resources to be allocated for the 103 | * {@link DockerApplicationMaster_23}'s container, the local resources (jars, configuration files) to be made available 104 | * and the environment to be set for the {@link DockerApplicationMaster_23} and the commands to be executed to run the 105 | * {@link DockerApplicationMaster_23}.

106 | * 107 | *

Using the {@link ApplicationSubmissionContext}, the client submits the application to the 108 | * ResourceManager and then monitors the application by requesting the ResourceManager 109 | * for an {@link ApplicationReport} at regular time intervals. In case of the application taking too long, the client 110 | * kills the application by submitting a {@link KillApplicationRequest} to the ResourceManager.

111 | * 112 | */ 113 | @InterfaceAudience.Public 114 | @InterfaceStability.Unstable 115 | public class DockerClient { 116 | 117 | private static final Log LOG = LogFactory.getLog(DockerClient.class); 118 | 119 | // Configuration 120 | private Configuration conf; 121 | private YarnClient yarnClient; 122 | // Application master specific info to register a new Application with RM/ASM 123 | private String appName = ""; 124 | // App master priority 125 | private int amPriority = 0; 126 | // Queue for App master 127 | private String amQueue = ""; 128 | // Amt. of memory resource to request for to run the App Master 129 | private int amMemory = 10; 130 | // Amt. of virtual core resource to request for to run the App Master 131 | private int amVCores = 1; 132 | 133 | private int container_retry = 3; 134 | 135 | // Application master jar file 136 | private String appMasterJar = ""; 137 | // Main class to invoke application master 138 | private final String appMasterMainClass; 139 | 140 | // Shell command to be executed 141 | // private String shellCommand = ""; 142 | // Location of shell script 143 | //private String shellScriptPath = ""; 144 | // Args to be passed to the shell command 145 | private String[] shellArgs = new String[] {}; 146 | // Env variables to be setup for the shell command 147 | private Map shellEnv = new HashMap(); 148 | // Shell Command Container priority 149 | private int shellCmdPriority = 0; 150 | 151 | // Amt of memory to request for container in which shell script will be executed 152 | private int containerMemory = 10; 153 | // Amt. of virtual cores to request for container in which shell script will be executed 154 | private int containerVirtualCores = 1; 155 | // No. of containers in which the shell script needs to be executed 156 | private int numContainers = 1; 157 | 158 | // log4j.properties file 159 | // if available, add to local resources and set into classpath 160 | private String log4jPropFile = ""; 161 | 162 | // Start time for client 163 | private final long clientStartTime = System.currentTimeMillis(); 164 | // Timeout threshold for client. Kill app after time interval expires. 165 | private long clientTimeout = 600000; 166 | 167 | // flag to indicate whether to keep containers across application attempts. 168 | private boolean keepContainers = false; 169 | 170 | // Debug flag 171 | boolean debugFlag = false; 172 | 173 | // Command line options 174 | private Options opts; 175 | 176 | private static final String shellCommandPath = "shellCommands"; 177 | private static final String shellArgsPath = "shellArgs"; 178 | private static final String appMasterJarPath = "AppMaster.jar"; 179 | // Hardcoded path to custom log_properties 180 | private static final String log4jPath = "log4j.properties"; 181 | 182 | public static final String SCRIPT_PATH = "common.sh"; 183 | 184 | public YarnClient getYarnClient(){ 185 | return this.yarnClient; 186 | } 187 | /** 188 | * @param args Command line arguments 189 | */ 190 | public static void main(String[] args) { 191 | boolean result = false; 192 | try { 193 | DockerClient client = new DockerClient(); 194 | LOG.info("Initializing Client"); 195 | try { 196 | boolean doRun = client.init(args); 197 | if (!doRun) { 198 | System.exit(0); 199 | } 200 | } catch (IllegalArgumentException e) { 201 | System.err.println(e.getLocalizedMessage()); 202 | client.printUsage(); 203 | System.exit(-1); 204 | } 205 | //result = client.run(); 206 | } catch (Throwable t) { 207 | LOG.fatal("Error running CLient", t); 208 | System.exit(1); 209 | } 210 | if (result) { 211 | LOG.info("Application completed successfully"); 212 | System.exit(0); 213 | } 214 | LOG.error("Application failed to complete successfully"); 215 | System.exit(2); 216 | } 217 | 218 | 219 | 220 | /** 221 | */ 222 | public DockerClient(Configuration conf) throws Exception { 223 | this( 224 | "com.sogou.docker.client.DockerRunnerApplicationMaster", 225 | conf); 226 | } 227 | 228 | DockerClient(String appMasterMainClass, Configuration conf) { 229 | this.conf = conf; 230 | this.appMasterMainClass = appMasterMainClass; 231 | yarnClient = YarnClient.createYarnClient(); 232 | yarnClient.init(conf); 233 | opts = new Options(); 234 | opts.addOption("appname", true, "Application Name. Default value - DistributedShell"); 235 | opts.addOption("priority", true, "Application Priority. Default 0"); 236 | opts.addOption("queue", true, "RM Queue in which this application is to be submitted"); 237 | opts.addOption("timeout", true, "Application timeout in milliseconds"); 238 | opts.addOption("master_memory", true, "Amount of memory in MB to be requested to run the application master"); 239 | opts.addOption("master_vcores", true, "Amount of virtual cores to be requested to run the application master"); 240 | opts.addOption("jar", true, "Jar file containing the application master"); 241 | opts.addOption("shell_command", true, "Shell command to be executed by " + 242 | "the Application Master. Can only specify either --shell_command " + 243 | "or --shell_script"); 244 | opts.addOption("shell_script", true, "Location of the shell script to be " + 245 | "executed. Can only specify either --shell_command or --shell_script"); 246 | opts.addOption("shell_args", true, "Command line args for the shell script." + 247 | "Multiple args can be separated by empty space."); 248 | opts.getOption("shell_args").setArgs(Option.UNLIMITED_VALUES); 249 | opts.addOption("shell_env", true, "Environment for shell script. Specified as env_key=env_val pairs"); 250 | opts.addOption("shell_cmd_priority", true, "Priority for the shell command containers"); 251 | opts.addOption("container_memory", true, "Amount of memory in MB to be requested to run the shell command"); 252 | opts.addOption("container_vcores", true, "Amount of virtual cores to be requested to run the shell command"); 253 | opts.addOption("num_containers", true, "No. of containers on which the shell command needs to be executed"); 254 | opts.addOption("log_properties", true, "log4j.properties file"); 255 | opts.addOption("container_retry", true, "container retry count"); 256 | opts.addOption("keep_containers_across_application_attempts", false, 257 | "Flag to indicate whether to keep containers across application attempts." + 258 | " If the flag is true, running containers will not be killed when" + 259 | " application attempt fails and these containers will be retrieved by" + 260 | " the new application attempt "); 261 | opts.addOption("debug", false, "Dump out debug information"); 262 | opts.addOption("help", false, "Print usage"); 263 | 264 | } 265 | 266 | /** 267 | */ 268 | public DockerClient() throws Exception { 269 | this(new YarnConfiguration()); 270 | } 271 | 272 | /** 273 | * Helper function to print out usage 274 | */ 275 | public void printUsage() { 276 | new HelpFormatter().printHelp("Client", opts); 277 | } 278 | 279 | /** 280 | * Parse command line options 281 | * @param args Parsed command line options 282 | * @return Whether the init was successful to run the client 283 | * @throws ParseException 284 | */ 285 | public boolean init(String[] args) throws ParseException { 286 | 287 | CommandLine cliParser = new GnuParser().parse(opts, args); 288 | 289 | if (args.length == 0) { 290 | throw new IllegalArgumentException("No args specified for client to initialize"); 291 | } 292 | 293 | if (cliParser.hasOption("log_properties")) { 294 | String log4jPath = cliParser.getOptionValue("log_properties"); 295 | try { 296 | Log4jPropertyHelper.updateLog4jConfiguration(DockerClient.class, log4jPath); 297 | } catch (Exception e) { 298 | LOG.warn("Can not set up custom log4j properties. " + e); 299 | } 300 | } 301 | 302 | if (cliParser.hasOption("help")) { 303 | printUsage(); 304 | return false; 305 | } 306 | 307 | if (cliParser.hasOption("debug")) { 308 | debugFlag = true; 309 | 310 | } 311 | 312 | if (cliParser.hasOption("keep_containers_across_application_attempts")) { 313 | LOG.info("keep_containers_across_application_attempts"); 314 | keepContainers = true; 315 | } 316 | 317 | appName = cliParser.getOptionValue("appname", "DistributedShell"); 318 | amPriority = Integer.parseInt(cliParser.getOptionValue("priority", "0")); 319 | amQueue = cliParser.getOptionValue("queue", "default"); 320 | amMemory = Integer.parseInt(cliParser.getOptionValue("master_memory", "10")); 321 | amVCores = Integer.parseInt(cliParser.getOptionValue("master_vcores", "1")); 322 | 323 | container_retry = Integer.parseInt(cliParser.getOptionValue("container_retry", "3")); 324 | 325 | if (amMemory < 0) { 326 | throw new IllegalArgumentException("Invalid memory specified for application master, exiting." 327 | + " Specified memory=" + amMemory); 328 | } 329 | if (amVCores < 0) { 330 | throw new IllegalArgumentException("Invalid virtual cores specified for application master, exiting." 331 | + " Specified virtual cores=" + amVCores); 332 | } 333 | 334 | if (!cliParser.hasOption("jar")) { 335 | throw new IllegalArgumentException("No jar file specified for application master"); 336 | } 337 | 338 | appMasterJar = cliParser.getOptionValue("jar"); 339 | 340 | if (cliParser.hasOption("shell_args")) { 341 | shellArgs = cliParser.getOptionValues("shell_args"); 342 | } 343 | if (cliParser.hasOption("shell_env")) { 344 | String envs[] = cliParser.getOptionValues("shell_env"); 345 | for (String env : envs) { 346 | env = env.trim(); 347 | int index = env.indexOf('='); 348 | if (index == -1) { 349 | shellEnv.put(env, ""); 350 | continue; 351 | } 352 | String key = env.substring(0, index); 353 | String val = ""; 354 | if (index < (env.length()-1)) { 355 | val = env.substring(index+1); 356 | } 357 | shellEnv.put(key, val); 358 | } 359 | } 360 | shellCmdPriority = Integer.parseInt(cliParser.getOptionValue("shell_cmd_priority", "0")); 361 | 362 | containerMemory = Integer.parseInt(cliParser.getOptionValue("container_memory", "10")); 363 | containerVirtualCores = Integer.parseInt(cliParser.getOptionValue("container_vcores", "1")); 364 | numContainers = Integer.parseInt(cliParser.getOptionValue("num_containers", "1")); 365 | 366 | if (containerMemory < 0 || containerVirtualCores < 0 || numContainers < 1) { 367 | throw new IllegalArgumentException("Invalid no. of containers or container memory/vcores specified," 368 | + " exiting." 369 | + " Specified containerMemory=" + containerMemory 370 | + ", containerVirtualCores=" + containerVirtualCores 371 | + ", numContainer=" + numContainers); 372 | } 373 | 374 | clientTimeout = Integer.parseInt(cliParser.getOptionValue("timeout", "600000")); 375 | 376 | log4jPropFile = cliParser.getOptionValue("log_properties", ""); 377 | 378 | return true; 379 | } 380 | 381 | /** 382 | * Main run function for the client 383 | * @return true if application completed successfully 384 | * @throws IOException 385 | * @throws YarnException 386 | */ 387 | public ApplicationId run() throws IOException, YarnException { 388 | 389 | LOG.info("Running Client"); 390 | yarnClient.start(); 391 | 392 | YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics(); 393 | LOG.info("Got Cluster metric info from ASM" 394 | + ", numNodeManagers=" + clusterMetrics.getNumNodeManagers()); 395 | 396 | List clusterNodeReports = yarnClient.getNodeReports( 397 | NodeState.RUNNING); 398 | LOG.info("Got Cluster node info from ASM"); 399 | for (NodeReport node : clusterNodeReports) { 400 | LOG.info("Got node report from ASM for" 401 | + ", nodeId=" + node.getNodeId() 402 | + ", nodeAddress" + node.getHttpAddress() 403 | + ", nodeRackName" + node.getRackName() 404 | + ", nodeNumContainers" + node.getNumContainers()); 405 | } 406 | 407 | QueueInfo queueInfo = yarnClient.getQueueInfo(this.amQueue); 408 | LOG.info("Queue info" 409 | + ", queueName=" + queueInfo.getQueueName() 410 | + ", queueCurrentCapacity=" + queueInfo.getCurrentCapacity() 411 | + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity() 412 | + ", queueApplicationCount=" + queueInfo.getApplications().size() 413 | + ", queueChildQueueCount=" + queueInfo.getChildQueues().size()); 414 | 415 | List listAclInfo = yarnClient.getQueueAclsInfo(); 416 | for (QueueUserACLInfo aclInfo : listAclInfo) { 417 | for (QueueACL userAcl : aclInfo.getUserAcls()) { 418 | LOG.info("User ACL Info for Queue" 419 | + ", queueName=" + aclInfo.getQueueName() 420 | + ", userAcl=" + userAcl.name()); 421 | } 422 | } 423 | 424 | // Get a new application id 425 | YarnClientApplication app = yarnClient.createApplication(); 426 | GetNewApplicationResponse appResponse = app.getNewApplicationResponse(); 427 | // TODO get min/max resource capabilities from RM and change memory ask if needed 428 | // If we do not have min/max, we may not be able to correctly request 429 | // the required resources from the RM for the app master 430 | // Memory ask has to be a multiple of min and less than max. 431 | // Dump out information about cluster capability as seen by the resource manager 432 | int maxMem = appResponse.getMaximumResourceCapability().getMemory(); 433 | LOG.info("Max mem capabililty of resources in this cluster " + maxMem); 434 | 435 | // A resource ask cannot exceed the max. 436 | if (amMemory > maxMem) { 437 | LOG.info("AM memory specified above max threshold of cluster. Using max value." 438 | + ", specified=" + amMemory 439 | + ", max=" + maxMem); 440 | amMemory = maxMem; 441 | } 442 | 443 | int maxVCores = appResponse.getMaximumResourceCapability().getVirtualCores(); 444 | LOG.info("Max virtual cores capabililty of resources in this cluster " + maxVCores); 445 | 446 | if (amVCores > maxVCores) { 447 | LOG.info("AM virtual cores specified above max threshold of cluster. " 448 | + "Using max value." + ", specified=" + amVCores 449 | + ", max=" + maxVCores); 450 | amVCores = maxVCores; 451 | } 452 | 453 | // set the application name 454 | ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); 455 | ApplicationId appId = appContext.getApplicationId(); 456 | 457 | //appContext.setKeepContainersAcrossApplicationAttempts(keepContainers); 458 | appContext.setApplicationName(appName); 459 | 460 | // set local resources for the application master 461 | // local files or archives as needed 462 | // In this scenario, the jar file for the application master is part of the local resources 463 | Map localResources = new HashMap(); 464 | 465 | LOG.info("Copy App Master jar from local filesystem and add to local environment"); 466 | // Copy the application master jar to the filesystem 467 | // Create a local resource to point to the destination jar path 468 | FileSystem fs = FileSystem.get(conf); 469 | addToLocalResources(fs, appMasterJar, appMasterJarPath, appId.toString(), 470 | localResources, null); 471 | 472 | // Set the log4j properties if needed 473 | if (!log4jPropFile.isEmpty()) { 474 | addToLocalResources(fs, log4jPropFile, log4jPath, appId.toString(), 475 | localResources, null); 476 | } 477 | 478 | // The shell script has to be made available on the final container(s) 479 | // where it will be executed. 480 | // To do this, we need to first copy into the filesystem that is visible 481 | // to the yarn framework. 482 | // We do not need to set this as a local resource for the application 483 | // master as the application master does not need it. 484 | String hdfsShellScriptLocation = ""; 485 | long hdfsShellScriptLen = 0; 486 | long hdfsShellScriptTimestamp = 0; 487 | //if (!shellScriptPath.isEmpty()) { 488 | // Path shellSrc = new Path(fs.getHomeDirectory(), SCRIPT_PATH); 489 | String shellPathSuffix = SCRIPT_PATH; 490 | Path shellDst = 491 | new Path(fs.getHomeDirectory(), shellPathSuffix); 492 | //fs.copyFromLocalFile(false, true, shellSrc, shellDst); 493 | hdfsShellScriptLocation = shellDst.toUri().toString(); 494 | FileStatus shellFileStatus = fs.getFileStatus(shellDst); 495 | hdfsShellScriptLen = shellFileStatus.getLen(); 496 | hdfsShellScriptTimestamp = shellFileStatus.getModificationTime(); 497 | //} 498 | 499 | if (shellArgs.length > 0) { 500 | addToLocalResources(fs, null, shellArgsPath, appId.toString(), 501 | localResources, StringUtils.join(shellArgs, " ")); 502 | } 503 | 504 | // Set the necessary security tokens as needed 505 | //amContainer.setContainerTokens(containerToken); 506 | 507 | // Set the env variables to be setup in the env where the application master will be run 508 | LOG.info("Set the environment for the application master"); 509 | Map env = new HashMap(); 510 | 511 | // put location of shell script into env 512 | // using the env info, the application master will create the correct local resource for the 513 | // eventual containers that will be launched to execute the shell scripts 514 | env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTLOCATION, hdfsShellScriptLocation); 515 | env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTTIMESTAMP, Long.toString(hdfsShellScriptTimestamp)); 516 | env.put(DSConstants.DISTRIBUTEDSHELLSCRIPTLEN, Long.toString(hdfsShellScriptLen)); 517 | 518 | // Add AppMaster.jar location to classpath 519 | // At some point we should not be required to add 520 | // the hadoop specific classpaths to the env. 521 | // It should be provided out of the box. 522 | // For now setting all required classpaths including 523 | // the classpath to "." for the application jar 524 | StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$()) 525 | .append(File.pathSeparatorChar).append("./*"); 526 | 527 | // StringBuilder classPathEnv = new StringBuilder(Environment.CLASSPATH.$$()) 528 | // .append(ApplicationConstants.CLASS_PATH_SEPARATOR).append("./*"); 529 | 530 | for (String c : conf.getStrings( 531 | YarnConfiguration.YARN_APPLICATION_CLASSPATH, 532 | YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { 533 | classPathEnv.append(File.pathSeparatorChar); 534 | classPathEnv.append(c.trim()); 535 | } 536 | classPathEnv.append(File.pathSeparatorChar).append("./log4j.properties"); 537 | 538 | // for (String c : conf.getStrings( 539 | // YarnConfiguration.YARN_APPLICATION_CLASSPATH, 540 | // YarnConfiguration.DEFAULT_YARN_CROSS_PLATFORM_APPLICATION_CLASSPATH)) { 541 | // classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR); 542 | // classPathEnv.append(c.trim()); 543 | // } 544 | // classPathEnv.append(ApplicationConstants.CLASS_PATH_SEPARATOR).append( 545 | // "./log4j.properties"); 546 | 547 | // add the runtime classpath needed for tests to work 548 | if (conf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { 549 | classPathEnv.append(':'); 550 | classPathEnv.append(System.getProperty("java.class.path")); 551 | } 552 | 553 | env.put("CLASSPATH", classPathEnv.toString()); 554 | 555 | // Set the necessary command to execute the application master 556 | Vector vargs = new Vector(30); 557 | 558 | // Set java executable command 559 | LOG.info("Setting up app master command"); 560 | vargs.add(Environment.JAVA_HOME.$() + "/bin/java"); 561 | //vargs.add(Environment.JAVA_HOME.$$() + "/bin/java"); 562 | // Set Xmx based on am memory size 563 | vargs.add("-Xmx" + amMemory + "m"); 564 | // Set class name 565 | vargs.add(appMasterMainClass); 566 | // Set params for Application Master 567 | vargs.add("--container_memory " + String.valueOf(containerMemory)); 568 | vargs.add("--container_vcores " + String.valueOf(containerVirtualCores)); 569 | vargs.add("--num_containers " + String.valueOf(numContainers)); 570 | vargs.add("--priority " + String.valueOf(shellCmdPriority)); 571 | vargs.add("--container_retry " + String.valueOf(this.container_retry)); 572 | 573 | for (Map.Entry entry : shellEnv.entrySet()) { 574 | vargs.add("--shell_env " + entry.getKey() + "=" + entry.getValue()); 575 | } 576 | if (debugFlag) { 577 | vargs.add("--debug"); 578 | } 579 | 580 | vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout"); 581 | vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr"); 582 | 583 | // Get final commmand 584 | StringBuilder command = new StringBuilder(); 585 | for (CharSequence str : vargs) { 586 | command.append(str).append(" "); 587 | } 588 | 589 | LOG.info("Completed setting up app master command " + command.toString()); 590 | List commands = new ArrayList(); 591 | commands.add(command.toString()); 592 | 593 | // Set up the container launch context for the application master 594 | ContainerLaunchContext amContainer = ContainerLaunchContext.newInstance( 595 | localResources, env, commands, null, null, null); 596 | 597 | // Set up resource type requirements 598 | // For now, both memory and vcores are supported, so we set memory and 599 | // vcores requirements 600 | Resource capability = Resource.newInstance(amMemory, amVCores); 601 | appContext.setResource(capability); 602 | 603 | // Service data is a binary blob that can be passed to the application 604 | // Not needed in this scenario 605 | // amContainer.setServiceData(serviceData); 606 | 607 | // Setup security tokens 608 | if (UserGroupInformation.isSecurityEnabled()) { 609 | // Note: Credentials class is marked as LimitedPrivate for HDFS and MapReduce 610 | Credentials credentials = new Credentials(); 611 | String tokenRenewer = conf.get(YarnConfiguration.RM_PRINCIPAL); 612 | if (tokenRenewer == null || tokenRenewer.length() == 0) { 613 | throw new IOException( 614 | "Can't get Master Kerberos principal for the RM to use as renewer"); 615 | } 616 | 617 | // For now, only getting tokens for the default file-system. 618 | final Token tokens[] = 619 | fs.addDelegationTokens(tokenRenewer, credentials); 620 | if (tokens != null) { 621 | for (Token token : tokens) { 622 | LOG.info("Got dt for " + fs.getUri() + "; " + token); 623 | } 624 | } 625 | DataOutputBuffer dob = new DataOutputBuffer(); 626 | credentials.writeTokenStorageToStream(dob); 627 | ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); 628 | amContainer.setTokens(fsTokens); 629 | } 630 | 631 | appContext.setAMContainerSpec(amContainer); 632 | 633 | // Set the priority for the application master 634 | // TODO - what is the range for priority? how to decide? 635 | Priority pri = Priority.newInstance(amPriority); 636 | appContext.setPriority(pri); 637 | 638 | // Set the queue to which this application is to be submitted in the RM 639 | appContext.setQueue(amQueue); 640 | 641 | // Submit the application to the applications manager 642 | // SubmitApplicationResponse submitResp = applicationsManager.submitApplication(appRequest); 643 | // Ignore the response as either a valid response object is returned on success 644 | // or an exception thrown to denote some form of a failure 645 | LOG.info("Submitting application to ASM"); 646 | 647 | yarnClient.submitApplication(appContext); 648 | 649 | // TODO 650 | // Try submitting the same request again 651 | // app submission failure? 652 | 653 | // Monitor the application 654 | return appId; 655 | 656 | } 657 | 658 | /** 659 | * Monitor the submitted application for completion. 660 | * Kill application if time expires. 661 | * @param appId Application Id of application to be monitored 662 | * @return true if application completed successfully 663 | * @throws YarnException 664 | * @throws IOException 665 | */ 666 | private boolean monitorApplication(ApplicationId appId) 667 | throws YarnException, IOException { 668 | 669 | while (true) { 670 | 671 | // Check app status every 1 second. 672 | try { 673 | Thread.sleep(1000); 674 | } catch (InterruptedException e) { 675 | LOG.debug("Thread sleep in monitoring loop interrupted"); 676 | } 677 | // Get application report for the appId we are interested in 678 | ApplicationReport report = yarnClient.getApplicationReport(appId); 679 | 680 | LOG.info("Got application report from ASM for" 681 | + ", appId=" + appId.getId() 682 | + ", clientToAMToken=" + report.getClientToAMToken() 683 | + ", appDiagnostics=" + report.getDiagnostics() 684 | + ", appMasterHost=" + report.getHost() 685 | + ", appQueue=" + report.getQueue() 686 | + ", appMasterRpcPort=" + report.getRpcPort() 687 | + ", appStartTime=" + report.getStartTime() 688 | + ", yarnAppState=" + report.getYarnApplicationState().toString() 689 | + ", distributedFinalState=" + report.getFinalApplicationStatus().toString() 690 | + ", appTrackingUrl=" + report.getTrackingUrl() 691 | + ", appUser=" + report.getUser()); 692 | 693 | YarnApplicationState state = report.getYarnApplicationState(); 694 | FinalApplicationStatus dsStatus = report.getFinalApplicationStatus(); 695 | if (YarnApplicationState.FINISHED == state) { 696 | if (FinalApplicationStatus.SUCCEEDED == dsStatus) { 697 | LOG.info("Application has completed successfully. Breaking monitoring loop"); 698 | return true; 699 | } 700 | else { 701 | LOG.info("Application did finished unsuccessfully." 702 | + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() 703 | + ". Breaking monitoring loop"); 704 | return false; 705 | } 706 | } 707 | else if (YarnApplicationState.KILLED == state 708 | || YarnApplicationState.FAILED == state) { 709 | LOG.info("Application did not shutdown." 710 | + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() 711 | + ". Breaking monitoring loop"); 712 | return false; 713 | } 714 | 715 | if (System.currentTimeMillis() > (clientStartTime + clientTimeout)) { 716 | LOG.info("Reached client specified timeout for application. Killing application"); 717 | forceKillApplication(appId); 718 | return false; 719 | } 720 | } 721 | 722 | } 723 | 724 | /** 725 | * Kill a submitted application by sending a call to the ASM 726 | * @param appId Application Id to be killed. 727 | * @throws YarnException 728 | * @throws IOException 729 | */ 730 | private void forceKillApplication(ApplicationId appId) 731 | throws YarnException, IOException { 732 | // TODO clarify whether multiple jobs with the same app id can be submitted and be running at 733 | // the same time. 734 | // If yes, can we kill a particular attempt only? 735 | 736 | // Response can be ignored as it is non-null on success or 737 | // throws an exception in case of failures 738 | yarnClient.killApplication(appId); 739 | } 740 | 741 | private void addToLocalResources(FileSystem fs, String fileSrcPath, 742 | String fileDstPath, String appId, Map localResources, 743 | String resources) throws IOException { 744 | String suffix = 745 | appName + "/" + appId + "/" + fileDstPath; 746 | Path dst = 747 | new Path(fs.getHomeDirectory(), suffix); 748 | if (fileSrcPath == null) { 749 | FSDataOutputStream ostream = null; 750 | try { 751 | ostream = FileSystem 752 | .create(fs, dst, new FsPermission((short) 0710)); 753 | ostream.writeUTF(resources); 754 | } finally { 755 | IOUtils.closeQuietly(ostream); 756 | } 757 | } else { 758 | fs.copyFromLocalFile(new Path(fileSrcPath), dst); 759 | } 760 | FileStatus scFileStatus = fs.getFileStatus(dst); 761 | LocalResource scRsrc = 762 | LocalResource.newInstance( 763 | ConverterUtils.getYarnUrlFromURI(dst.toUri()), 764 | LocalResourceType.FILE, LocalResourceVisibility.APPLICATION, 765 | scFileStatus.getLen(), scFileStatus.getModificationTime()); 766 | localResources.put(fileDstPath, scRsrc); 767 | } 768 | } 769 | -------------------------------------------------------------------------------- /distributed-docker-submit-client/src/main/java/com/sogou/dockeronyarn/client/DockerOnYarnAppDescriptor.java: -------------------------------------------------------------------------------- 1 | package com.sogou.dockeronyarn.client; 2 | 3 | /** 4 | * Created by guoshiwei on 15/5/16. 5 | */ 6 | public class DockerOnYarnAppDescriptor { 7 | // Start time for client 8 | private final long clientStartTime = System.currentTimeMillis(); 9 | 10 | public String[] getDockerArgs() { 11 | return dockerArgs; 12 | } 13 | 14 | public DockerOnYarnAppDescriptor setDockerArgs(String[] dockerArgs) { 15 | this.dockerArgs = dockerArgs; 16 | return this ; 17 | } 18 | 19 | // Application master specific info to register a new Application with RM/ASM 20 | private String appName = ""; 21 | // App master priority 22 | private int amPriority = 0; 23 | 24 | private String[] dockerArgs = {}; 25 | // Queue for App master 26 | 27 | public String getMemory() { 28 | return memory; 29 | } 30 | 31 | public DockerOnYarnAppDescriptor setMemory(String memory) { 32 | this.memory = memory; 33 | return this ; 34 | } 35 | 36 | public String getCpuShares() { 37 | return cpuShares; 38 | } 39 | 40 | public DockerOnYarnAppDescriptor setCpuShares(String cpuShares) { 41 | this.cpuShares = cpuShares; 42 | return this ; 43 | } 44 | 45 | private String amQueue = ""; 46 | 47 | // log4j.properties file 48 | // if available, add to local resources and set into classpath 49 | private String log4jPropFile = ""; 50 | private long clientTimeout; 51 | private String dockerImage; 52 | private int container_retry = 3; 53 | private String memory =""; 54 | private String cpuShares="" ; 55 | 56 | 57 | private String workDir =""; 58 | 59 | public String getWorkDir() { 60 | return workDir; 61 | } 62 | 63 | public DockerOnYarnAppDescriptor setWorkDir(String workDir) { 64 | this.workDir = workDir; 65 | return this; 66 | } 67 | 68 | private String mountVolume; 69 | 70 | public String getMountVolume() { 71 | return mountVolume; 72 | } 73 | 74 | public void setMountVolume(String mountVolume) { 75 | this.mountVolume = mountVolume; 76 | } 77 | 78 | public String getAmJarPath() { 79 | return amJarPath; 80 | } 81 | 82 | public DockerOnYarnAppDescriptor setAmJarPath(String amJarPath) { 83 | this.amJarPath = amJarPath; 84 | return this ; 85 | } 86 | 87 | private String commandToRun; 88 | private String amJarPath ; 89 | 90 | public DockerOnYarnAppDescriptor setAppName(String appName) { 91 | this.appName = appName; 92 | return this; 93 | } 94 | 95 | public DockerOnYarnAppDescriptor setAmPriority(int amPriority) { 96 | this.amPriority = amPriority; return this; 97 | } 98 | 99 | public DockerOnYarnAppDescriptor setAmQueue(String amQueue) { 100 | this.amQueue = amQueue; return this; 101 | } 102 | 103 | public DockerOnYarnAppDescriptor setContainer_retry(int container_retry) { 104 | this.container_retry = container_retry; return this; 105 | } 106 | 107 | public DockerOnYarnAppDescriptor setLog4jPropFile(String log4jPropFile) { 108 | this.log4jPropFile = log4jPropFile; return this; 109 | } 110 | 111 | public DockerOnYarnAppDescriptor setClientTimeout(long clientTimeout) { 112 | this.clientTimeout = clientTimeout; return this; 113 | } 114 | 115 | public DockerOnYarnAppDescriptor setDockerImage(String dockerImage) { 116 | this.dockerImage = dockerImage; return this; 117 | } 118 | 119 | public DockerOnYarnAppDescriptor setCommandToRun(String commandToRun) { 120 | this.commandToRun = commandToRun; return this; 121 | } 122 | 123 | public long getClientStartTime() { 124 | return clientStartTime; 125 | } 126 | 127 | public String getAppName() { 128 | return appName; 129 | } 130 | 131 | public int getAmPriority() { 132 | return amPriority; 133 | } 134 | 135 | public String getAmQueue() { 136 | return amQueue; 137 | } 138 | 139 | public String getLog4jPropFile() { 140 | return log4jPropFile; 141 | } 142 | 143 | public long getClientTimeout() { 144 | return clientTimeout; 145 | } 146 | 147 | public String getDockerImage() { 148 | return dockerImage; 149 | } 150 | 151 | public int getContainer_retry() { 152 | return container_retry; 153 | } 154 | 155 | public String getCommandToRun() { 156 | return commandToRun; 157 | } 158 | 159 | 160 | 161 | } 162 | -------------------------------------------------------------------------------- /distributed-docker-submit-client/src/main/java/com/sogou/dockeronyarn/client/DockerOnYarnClient.java: -------------------------------------------------------------------------------- 1 | package com.sogou.dockeronyarn.client; 2 | 3 | import com.google.common.base.Preconditions; 4 | import com.sogou.dockeronyarn.appmaster.DockerRunnerApplicationMaster; 5 | import com.sogou.dockeronyarn.common.DistributedDockerConfiguration; 6 | import org.apache.commons.lang.StringUtils; 7 | import org.apache.commons.logging.Log; 8 | import org.apache.commons.logging.LogFactory; 9 | import org.apache.hadoop.conf.Configuration; 10 | import org.apache.hadoop.fs.FileStatus; 11 | import org.apache.hadoop.fs.FileSystem; 12 | import org.apache.hadoop.fs.Path; 13 | import org.apache.hadoop.io.DataOutputBuffer; 14 | import org.apache.hadoop.security.Credentials; 15 | import org.apache.hadoop.security.UserGroupInformation; 16 | import org.apache.hadoop.yarn.api.ApplicationConstants; 17 | import org.apache.hadoop.yarn.api.records.*; 18 | import org.apache.hadoop.yarn.client.api.YarnClient; 19 | import org.apache.hadoop.yarn.client.api.YarnClientApplication; 20 | import org.apache.hadoop.yarn.conf.YarnConfiguration; 21 | import org.apache.hadoop.yarn.exceptions.YarnException; 22 | import org.apache.hadoop.yarn.util.ConverterUtils; 23 | import org.apache.hadoop.yarn.util.Records; 24 | import org.mortbay.util.StringUtil; 25 | 26 | import java.io.File; 27 | import java.io.IOException; 28 | import java.nio.ByteBuffer; 29 | import java.util.ArrayList; 30 | import java.util.HashMap; 31 | import java.util.List; 32 | import java.util.Map; 33 | 34 | /** 35 | * Created by guoshiwei on 15/4/25. 36 | */ 37 | public class DockerOnYarnClient { 38 | private static final Log LOG = LogFactory.getLog(DockerOnYarnClient.class); 39 | 40 | // Configuration 41 | private final Configuration yarnConf; 42 | 43 | private final DistributedDockerConfiguration ddockerConf; 44 | private final YarnClient yarnClient; 45 | 46 | // Hardcoded path to custom log_properties 47 | private static final String log4jHdfsPath = "log4j.properties"; 48 | 49 | // Main class to invoke application master 50 | private final String appMasterMainClass; 51 | 52 | private boolean keepContainers; 53 | 54 | private static final String appMasterJarHdfsPath = "AppMaster.jar"; 55 | 56 | // Amt. of memory resource to request for to run the App Master 57 | private int amMemory = 10; 58 | // Amt. of virtual core resource to request for to run the App Master 59 | private int amVCores = 1; 60 | 61 | private static final String libDir = "lib"; 62 | 63 | private boolean debugFlag; 64 | private FileSystem fs; 65 | 66 | public DockerOnYarnClient() { 67 | this.yarnConf = new YarnConfiguration(); 68 | this.ddockerConf = new DistributedDockerConfiguration(); 69 | this.appMasterMainClass = DockerRunnerApplicationMaster.class.getName(); 70 | yarnClient = YarnClient.createYarnClient(); 71 | yarnClient.init(yarnConf); 72 | 73 | } 74 | 75 | private String getAppMasterJarfilePath() { 76 | return null; 77 | } 78 | 79 | public void start() throws IOException, YarnException { 80 | LOG.info("Running Client"); 81 | fs = FileSystem.get(yarnConf); 82 | yarnClient.start(); 83 | YarnClusterMetrics clusterMetrics = yarnClient.getYarnClusterMetrics(); 84 | LOG.info("Got Cluster metric info from ASM" 85 | + ", numNodeManagers=" + clusterMetrics.getNumNodeManagers()); 86 | 87 | List clusterNodeReports = yarnClient.getNodeReports( 88 | NodeState.RUNNING); 89 | LOG.info("Got Cluster node info from ASM"); 90 | for (NodeReport node : clusterNodeReports) { 91 | LOG.info("Got node report from ASM for" 92 | + ", nodeId=" + node.getNodeId() 93 | + ", nodeAddress" + node.getHttpAddress() 94 | + ", nodeRackName" + node.getRackName() 95 | + ", nodeNumContainers" + node.getNumContainers()); 96 | } 97 | 98 | List listAclInfo = yarnClient.getQueueAclsInfo(); 99 | for (QueueUserACLInfo aclInfo : listAclInfo) { 100 | for (QueueACL userAcl : aclInfo.getUserAcls()) { 101 | LOG.info("User ACL Info for Queue" 102 | + ", queueName=" + aclInfo.getQueueName() 103 | + ", userAcl=" + userAcl.name()); 104 | } 105 | } 106 | } 107 | 108 | 109 | public void stop() { 110 | yarnClient.stop(); 111 | } 112 | 113 | /** 114 | * Submit app descripted by @appDescriptor and wait it for shutdown. 115 | */ 116 | public boolean run(DockerOnYarnAppDescriptor appDescriptor) throws IOException, YarnException { 117 | // LOG.info("Running Client"); 118 | // yarnClient.start(); 119 | start(); 120 | ApplicationId appId = submitYarnApplication(appDescriptor); 121 | return monitorApplication(appId, appDescriptor); 122 | } 123 | 124 | public ApplicationId submitYarnApplication(DockerOnYarnAppDescriptor appDescriptor) throws IOException, YarnException { 125 | 126 | QueueInfo queueInfo = yarnClient.getQueueInfo(appDescriptor.getAmQueue()); 127 | LOG.info("Queue info" 128 | + ", queueName=" + queueInfo.getQueueName() 129 | + ", queueCurrentCapacity=" + queueInfo.getCurrentCapacity() 130 | + ", queueMaxCapacity=" + queueInfo.getMaximumCapacity() 131 | + ", queueApplicationCount=" + queueInfo.getApplications().size() 132 | + ", queueChildQueueCount=" + queueInfo.getChildQueues().size()); 133 | 134 | // Get a new application id 135 | YarnClientApplication app = yarnClient.createApplication(); 136 | 137 | int maxMem = app.getNewApplicationResponse().getMaximumResourceCapability().getMemory(); 138 | LOG.info("Max mem capabililty of resources in this cluster " + maxMem); 139 | 140 | ApplicationSubmissionContext appContext = createApplicationSubmissionContext(appDescriptor, app); 141 | 142 | LOG.info("Submitting application to ASM"); 143 | return yarnClient.submitApplication(appContext); 144 | } 145 | 146 | private ApplicationSubmissionContext createApplicationSubmissionContext( 147 | DockerOnYarnAppDescriptor appDescriptor, YarnClientApplication app) throws IOException { 148 | ApplicationSubmissionContext appContext = app.getApplicationSubmissionContext(); 149 | 150 | appContext.setApplicationName(appDescriptor.getAppName()); 151 | appContext.setPriority(createPriorityRecord(appDescriptor)); 152 | appContext.setQueue(appDescriptor.getAmQueue()); 153 | 154 | appContext.setAMContainerSpec(createContainerLaunchContext(appDescriptor, appContext)); 155 | 156 | return appContext; 157 | } 158 | 159 | private Priority createPriorityRecord(DockerOnYarnAppDescriptor appDescriptor) { 160 | Priority pri = Records.newRecord(Priority.class); 161 | pri.setPriority(appDescriptor.getAmPriority()); 162 | return pri; 163 | } 164 | 165 | /** 166 | * Setup required resources for AM. 167 | * 168 | * All required resources should be placed on HDFS first. 169 | * The value of DistributedDockerConfiguration.REQUIRED_RESOURCE_HDFS_PATH will be used. 170 | * You can set a value other than the default one to use different version of AppMaster. 171 | * 172 | * Under REQUIRED_RESOURCE_HDFS_PATH dir, must have the following file structure: 173 | * - lib/appmaster.jar 174 | * - runner.py 175 | * - conf/distributed-docker-default.xml 176 | * - conf/log4j.properties 177 | * 178 | * If a local log4j.properties is given, the default one on hdfs will not be used, 179 | * but the local one will be uploaded every time the app is submitted. Upload every time 180 | * is slow, so we prefer to use the default one. 181 | * 182 | * @param appDescriptor 183 | * @param appId 184 | * @return 185 | * @throws IOException 186 | */ 187 | private Map createLocaleResourceMap( 188 | DockerOnYarnAppDescriptor appDescriptor, ApplicationId appId) throws IOException { 189 | Map localResources = new HashMap(); 190 | 191 | LOG.info("Copy App Master jar from local filesystem and add to local environment"); 192 | // Copy the application master jar to the filesystem 193 | // Create a local resource to point to the destination jar path 194 | 195 | loadLocalResources(appDescriptor.getAmJarPath(), appMasterJarHdfsPath,localResources); 196 | 197 | loadLocalResources(ddockerConf.get(DistributedDockerConfiguration.DDOCKER_RUNNER_PATH), 198 | DockerRunnerApplicationMaster.LOCAL_RUNNER_NAME,localResources); 199 | 200 | // Set the log4j properties if needed 201 | if (!appDescriptor.getLog4jPropFile().isEmpty()) { 202 | addToLocalResources(appDescriptor.getLog4jPropFile(), log4jHdfsPath, appId.toString(), 203 | localResources, appDescriptor); 204 | } 205 | return localResources; 206 | } 207 | 208 | private ContainerLaunchContext createContainerLaunchContext( 209 | DockerOnYarnAppDescriptor appDescriptor, 210 | ApplicationSubmissionContext appContext) throws IOException { 211 | ContainerLaunchContext amContainer = Records.newRecord(ContainerLaunchContext.class); 212 | 213 | 214 | // set local resources for the application master 215 | // local files or archives as needed 216 | // In this scenario, the jar file for the application master is part of the local resources 217 | Map localResources = createLocaleResourceMap(appDescriptor, appContext.getApplicationId()); 218 | 219 | amContainer.setLocalResources(localResources); 220 | 221 | // Set the env variables to be setup in the env where the application master will be run 222 | LOG.info("Set the environment for the application master"); 223 | Map env = getEnv(); 224 | 225 | // env.put("mount.volume",appDescriptor.getMountVolume()); 226 | 227 | amContainer.setEnvironment(env); 228 | amContainer.setCommands(getAppMasterCommands(appDescriptor)); 229 | 230 | // Set up resource type requirements 231 | // For now, both memory and vcores are supported, so we set memory and 232 | // vcores requirements 233 | Resource capability = Records.newRecord(Resource.class); 234 | capability.setMemory(amMemory); 235 | capability.setVirtualCores(amVCores); 236 | appContext.setResource(capability); 237 | 238 | // Setup security tokens 239 | if (UserGroupInformation.isSecurityEnabled()) { 240 | Credentials credentials = new Credentials(); 241 | String tokenRenewer = yarnConf.get(YarnConfiguration.RM_PRINCIPAL); 242 | if (tokenRenewer == null || tokenRenewer.length() == 0) { 243 | throw new IOException( 244 | "Can't get Master Kerberos principal for the RM to use as renewer"); 245 | } 246 | 247 | // For now, only getting tokens for the default file-system. 248 | final org.apache.hadoop.security.token.Token tokens[] = 249 | fs.addDelegationTokens(tokenRenewer, credentials); 250 | if (tokens != null) { 251 | for (org.apache.hadoop.security.token.Token token : tokens) { 252 | LOG.info("Got dt for " + fs.getUri() + "; " + token); 253 | } 254 | } 255 | DataOutputBuffer dob = new DataOutputBuffer(); 256 | credentials.writeTokenStorageToStream(dob); 257 | ByteBuffer fsTokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength()); 258 | amContainer.setTokens(fsTokens); 259 | } 260 | return amContainer; 261 | } 262 | 263 | private List getAppMasterCommands(DockerOnYarnAppDescriptor appDescriptor) { 264 | // Set the necessary command to execute the application master 265 | List vargs = new ArrayList(30); 266 | 267 | // Set java executable command 268 | LOG.info("Setting up app master command"); 269 | vargs.add("java"); 270 | 271 | // Set Xmx based on am memory size 272 | vargs.add("-Xmx" + amMemory + "m"); 273 | // vargs.add("-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=12345"); 274 | 275 | // Pass DistributedDockerConfiguration as Properties 276 | for (Map.Entry e : ddockerConf) { 277 | if (e.getKey().startsWith(DistributedDockerConfiguration.DDOCKER_PREFIX)) { 278 | vargs.add(String.format("-D%s=%s", e.getKey(), e.getValue())); 279 | } 280 | } 281 | 282 | // Set class name 283 | vargs.add(appMasterMainClass); 284 | 285 | // Set params for Application Master 286 | vargs.add("-job_name"); 287 | vargs.add(appDescriptor.getAppName()); 288 | 289 | 290 | vargs.add("-v"); 291 | vargs.add(appDescriptor.getMountVolume()); 292 | 293 | vargs.add("-w"); 294 | vargs.add(appDescriptor.getWorkDir()); 295 | 296 | vargs.add("--docker-args"); 297 | vargs.add("'"+StringUtils.join(appDescriptor.getDockerArgs()," ")+"\'"); 298 | 299 | vargs.add("-image"); 300 | vargs.add(appDescriptor.getDockerImage()); 301 | if (debugFlag) { 302 | vargs.add("--debug"); 303 | } 304 | 305 | vargs.add("1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stdout"); 306 | vargs.add("2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/AppMaster.stderr"); 307 | 308 | vargs.add(appDescriptor.getCommandToRun()); 309 | 310 | Object object ; 311 | LOG.info("AppMasterCommand: " + StringUtils.join(vargs, " ")); 312 | 313 | return vargs; 314 | } 315 | 316 | private Map getEnv() { 317 | Map env = new HashMap(); 318 | 319 | StringBuilder classPathEnv = new StringBuilder("./*"); 320 | for (String c : yarnConf.getStrings( 321 | YarnConfiguration.YARN_APPLICATION_CLASSPATH, 322 | YarnConfiguration.DEFAULT_YARN_APPLICATION_CLASSPATH)) { 323 | classPathEnv.append(File.pathSeparatorChar).append(c.trim()); 324 | } 325 | 326 | classPathEnv.append(File.pathSeparatorChar).append( 327 | ApplicationConstants.Environment.CLASSPATH.$()); 328 | 329 | 330 | // add the runtime classpath needed for tests to work 331 | if (yarnConf.getBoolean(YarnConfiguration.IS_MINI_YARN_CLUSTER, false)) { 332 | classPathEnv.append(':'); 333 | classPathEnv.append(System.getProperty("java.class.path")); 334 | } 335 | 336 | env.put("CLASSPATH", classPathEnv.toString()); 337 | return env; 338 | } 339 | 340 | public YarnClient getYarnClient() { 341 | return yarnClient; 342 | } 343 | 344 | /** 345 | * Monitor the submitted application for completion. 346 | * Kill application if time expires. 347 | * 348 | * @param appId Application Id of application to be monitored 349 | * @return true if application completed successfully 350 | * @throws YarnException 351 | * @throws IOException 352 | */ 353 | public boolean monitorApplication(ApplicationId appId, DockerOnYarnAppDescriptor appDescriptor) 354 | throws YarnException, IOException { 355 | 356 | while (true) { 357 | 358 | // Check app status every 1 second. 359 | try { 360 | Thread.sleep(1000); 361 | } catch (InterruptedException e) { 362 | LOG.debug("Thread sleep in monitoring loop interrupted"); 363 | } 364 | 365 | // Get application report for the appId we are interested in 366 | ApplicationReport report = yarnClient.getApplicationReport(appId); 367 | 368 | LOG.info("Got application report from ASM for" 369 | + ", appId=" + appId.getId() 370 | + ", clientToAMToken=" + report.getClientToAMToken() 371 | + ", appDiagnostics=" + report.getDiagnostics() 372 | + ", appMasterHost=" + report.getHost() 373 | + ", appQueue=" + report.getQueue() 374 | + ", appMasterRpcPort=" + report.getRpcPort() 375 | + ", appStartTime=" + report.getStartTime() 376 | + ", yarnAppState=" + report.getYarnApplicationState().toString() 377 | + ", distributedFinalState=" + report.getFinalApplicationStatus().toString() 378 | + ", appTrackingUrl=" + report.getTrackingUrl() 379 | + ", appUser=" + report.getUser()); 380 | 381 | YarnApplicationState state = report.getYarnApplicationState(); 382 | FinalApplicationStatus dsStatus = report.getFinalApplicationStatus(); 383 | if (YarnApplicationState.FINISHED == state) { 384 | if (FinalApplicationStatus.SUCCEEDED == dsStatus) { 385 | LOG.info("Application has completed successfully. Breaking monitoring loop"); 386 | return true; 387 | } else { 388 | LOG.info("Application did finished unsuccessfully." 389 | + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() 390 | + ". Breaking monitoring loop"); 391 | return false; 392 | } 393 | } else if (YarnApplicationState.KILLED == state 394 | || YarnApplicationState.FAILED == state) { 395 | LOG.info("Application did not shutdown." 396 | + " YarnState=" + state.toString() + ", DSFinalStatus=" + dsStatus.toString() 397 | + ". Breaking monitoring loop"); 398 | return false; 399 | } 400 | 401 | if (System.currentTimeMillis() > (appDescriptor.getClientStartTime() + appDescriptor.getClientTimeout())) { 402 | LOG.info("Reached client specified timeout for application. Killing application"); 403 | forceKillApplication(appId); 404 | return false; 405 | } 406 | } 407 | 408 | } 409 | 410 | /** 411 | * Kill a submitted application by sending a call to the ASM 412 | * 413 | * @param appId Application Id to be killed. 414 | * @throws YarnException 415 | * @throws IOException 416 | */ 417 | public void forceKillApplication(ApplicationId appId) 418 | throws YarnException, IOException { 419 | // TODO clarify whether multiple jobs with the same app id can be submitted and be running at 420 | // the same time. 421 | // If yes, can we kill a particular attempt only? 422 | 423 | // Response can be ignored as it is non-null on success or 424 | // throws an exception in case of failures 425 | yarnClient.killApplication(appId); 426 | } 427 | 428 | private void addToLocalResources(String fileSrcPath, 429 | String fileDstPath, String appId, 430 | Map localResources, 431 | DockerOnYarnAppDescriptor appDescriptor) throws IOException { 432 | Preconditions.checkNotNull(fileDstPath); 433 | 434 | // TODO Skip upload appMaster resources if it is not changed. 435 | String suffix = 436 | appDescriptor.getAppName() + "/" + appId + "/" + fileDstPath; 437 | Path dst = 438 | new Path(fs.getHomeDirectory(), suffix); 439 | 440 | fs.copyFromLocalFile(new Path(fileSrcPath), dst); 441 | 442 | FileStatus scFileStatus = fs.getFileStatus(dst); 443 | LocalResource scRsrc = 444 | LocalResource.newInstance( 445 | ConverterUtils.getYarnUrlFromURI(dst.toUri()), 446 | LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 447 | scFileStatus.getLen(), scFileStatus.getModificationTime()); 448 | localResources.put(fileDstPath, scRsrc); 449 | } 450 | 451 | public void setDebugFlag(boolean debugFlag) { 452 | this.debugFlag = debugFlag; 453 | } 454 | 455 | private void loadLocalResources(String fileSrcPath, 456 | String fileDstPath, 457 | Map localResources) throws IOException { 458 | Preconditions.checkNotNull(fileDstPath); 459 | String suffix = 460 | libDir + "/" + fileDstPath ; 461 | Path dst = 462 | new Path(fs.getHomeDirectory(),suffix); 463 | if(!fs.exists(dst)){ 464 | fs.copyFromLocalFile(new Path(fileSrcPath), dst); 465 | } 466 | FileStatus scFileStatus = fs.getFileStatus(dst); 467 | LocalResource scRsrc = 468 | LocalResource.newInstance( 469 | ConverterUtils.getYarnUrlFromURI(dst.toUri()), 470 | LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 471 | scFileStatus.getLen(), scFileStatus.getModificationTime()); 472 | localResources.put(fileDstPath, scRsrc); 473 | } 474 | 475 | 476 | public ApplicationReport getApplicationReport(ApplicationId appId) 477 | { 478 | try { 479 | return yarnClient.getApplicationReport(appId); 480 | } catch (YarnException e) { 481 | e.printStackTrace(); 482 | 483 | } catch (IOException e) { 484 | e.printStackTrace(); 485 | } 486 | return null ; 487 | } 488 | 489 | public void killApplication(ApplicationId appId) 490 | throws YarnException, IOException { 491 | // TODO clarify whether multiple jobs with the same app id can be submitted and be running at 492 | // the same time. 493 | // If yes, can we kill a particular attempt only? 494 | 495 | // Response can be ignored as it is non-null on success or 496 | // throws an exception in case of failures 497 | yarnClient.killApplication(appId); 498 | } 499 | 500 | 501 | 502 | } 503 | -------------------------------------------------------------------------------- /distributed-docker-submit-client/src/main/java/com/sogou/dockeronyarn/client/DockerOnYarnClientCli.java: -------------------------------------------------------------------------------- 1 | package com.sogou.dockeronyarn.client; 2 | 3 | import com.sogou.dockeronyarn.common.Log4jPropertyHelper; 4 | import org.apache.commons.cli.*; 5 | import org.apache.commons.lang.StringUtils; 6 | import org.apache.commons.logging.Log; 7 | import org.apache.commons.logging.LogFactory; 8 | 9 | import static com.sogou.dockeronyarn.common.Utils.checkNotEmpty; 10 | 11 | /** 12 | * Created by guoshiwei on 15/5/16. 13 | */ 14 | public class DockerOnYarnClientCli { 15 | private static final Log LOG = LogFactory.getLog(DockerOnYarnClientCli.class); 16 | private final DockerOnYarnClient dockerClient; 17 | private DockerOnYarnAppDescriptor appDescriptor; 18 | private Options opts; 19 | private String amJarPath ; 20 | 21 | public DockerOnYarnClientCli() { 22 | dockerClient = new DockerOnYarnClient(); 23 | appDescriptor = new DockerOnYarnAppDescriptor(); 24 | this.setupOpts(); 25 | } 26 | 27 | public String getAmJarPath() { 28 | return amJarPath; 29 | } 30 | 31 | public void setAmJarPath(String amJarPath) { 32 | this.amJarPath = amJarPath; 33 | } 34 | 35 | public static void main(String [] args){ 36 | boolean result = false; 37 | try { 38 | DockerOnYarnClientCli client = new DockerOnYarnClientCli(); 39 | LOG.info("Initializing Client"); 40 | try { 41 | 42 | boolean doRun = client.init(args); 43 | if (!doRun) { 44 | System.exit(0); 45 | } 46 | } catch (IllegalArgumentException e) { 47 | System.err.println(e.getLocalizedMessage()); 48 | client.printUsage(); 49 | System.exit(-1); 50 | } 51 | result = client.dockerClient.run(client.appDescriptor); 52 | } catch (Throwable t) { 53 | LOG.fatal("Error running CLient", t); 54 | System.exit(1); 55 | } 56 | if (result) { 57 | LOG.info("Application completed successfully"); 58 | System.exit(0); 59 | } 60 | LOG.error("Application failed to complete successfully"); 61 | System.exit(2); 62 | } 63 | 64 | private void setupOpts(){ 65 | opts = new Options(); 66 | opts.addOption("appname", true, "Application Name. Default value - DistributedShell"); 67 | opts.addOption("priority", true, "Application Priority. Default 0"); 68 | opts.addOption("queue", true, "RM Queue in which this application is to be submitted"); 69 | opts.addOption("timeout", true, "Application timeout in milliseconds"); 70 | opts.addOption("master_memory", true, "Amount of memory in MB to be requested to run the application master"); 71 | opts.addOption("master_vcores", true, "Amount of virtual cores to be requested to run the application master"); 72 | opts.addOption("jar", true, "Jar file containing the application master"); 73 | 74 | opts.addOption("log_properties", true, "log4j.properties file"); 75 | opts.addOption("container_retry", true, "container retry count"); 76 | opts.addOption("image", true, "docker image name"); 77 | 78 | opts.addOption("debug", false, "Dump out debug information"); 79 | opts.addOption("help", false, "Print usage"); 80 | } 81 | 82 | /** 83 | * Parse command line options 84 | * @param args Parsed command line options 85 | * @return Whether the init was successful to run the client 86 | * @throws ParseException 87 | */ 88 | public boolean init(String[] args) throws ParseException { 89 | 90 | CommandLine cliParser = new GnuParser().parse(opts, args, true); 91 | 92 | if (args.length == 0) { 93 | throw new IllegalArgumentException("No args specified for client to initialize"); 94 | } 95 | 96 | if (cliParser.hasOption("log_properties")) { 97 | String log4jPath = cliParser.getOptionValue("log_properties"); 98 | try { 99 | Log4jPropertyHelper.updateLog4jConfiguration(DockerClient.class, log4jPath); 100 | } catch (Exception e) { 101 | LOG.warn("Can not set up custom log4j properties. " + e); 102 | } 103 | } 104 | 105 | if (cliParser.hasOption("help")) { 106 | printUsage(); 107 | return false; 108 | } 109 | 110 | if (cliParser.hasOption("debug")) { 111 | dockerClient.setDebugFlag(true); 112 | } 113 | 114 | appDescriptor.setAppName(cliParser.getOptionValue("appname", "DistributedShell")); 115 | appDescriptor.setAmPriority(Integer.parseInt(cliParser.getOptionValue("priority", "0"))); 116 | appDescriptor.setAmQueue(cliParser.getOptionValue("queue", "default")); 117 | 118 | appDescriptor.setContainer_retry(Integer.parseInt(cliParser.getOptionValue("container_retry", "3"))); 119 | 120 | appDescriptor.setClientTimeout(Integer.parseInt(cliParser.getOptionValue("timeout", "600000"))); 121 | 122 | appDescriptor.setLog4jPropFile(cliParser.getOptionValue("log_properties", "")); 123 | appDescriptor.setAmJarPath(cliParser.getOptionValue("jar")); 124 | 125 | String dockerImage = cliParser.getOptionValue("image"); 126 | checkNotEmpty(dockerImage, "No dockerImage given"); 127 | appDescriptor.setDockerImage(dockerImage); 128 | 129 | String commandToRun = StringUtils.join(cliParser.getArgs(), " "); 130 | checkNotEmpty(commandToRun, "No command given"); 131 | 132 | appDescriptor.setCommandToRun(commandToRun); 133 | return true; 134 | } 135 | 136 | private void printUsage() { 137 | new HelpFormatter().printHelp("[options] command [commandArgs]", opts); 138 | } 139 | } 140 | -------------------------------------------------------------------------------- /distributed-docker-submit-client/src/main/java/com/sogou/dockeronyarn/client/model/SubmitException.java: -------------------------------------------------------------------------------- 1 | package com.sogou.dockeronyarn.client.model; 2 | 3 | /** 4 | * Created by guoshiwei on 14/11/28. 5 | */ 6 | public class SubmitException extends RuntimeException{ 7 | public SubmitException(String message){ 8 | super(message); 9 | } 10 | public SubmitException(String message, Exception e){ 11 | super(message, e); 12 | } 13 | } 14 | -------------------------------------------------------------------------------- /distributed-docker-submit-client/src/main/java/com/sogou/dockeronyarn/client/model/SubmitItem.java: -------------------------------------------------------------------------------- 1 | package com.sogou.dockeronyarn.client.model; 2 | 3 | import javax.xml.bind.annotation.XmlRootElement; 4 | 5 | @XmlRootElement 6 | public class SubmitItem { 7 | private String appname = "defaultApp"; 8 | private String docker_image = null; 9 | private String workingdir = null; 10 | private String command = null; 11 | private long container_memory = 512; 12 | private long master_memory = 128; 13 | private long container_retry = 3; 14 | private int container_vcores = 1; 15 | private int priority = 10; 16 | private long timeout = 86400000; 17 | private String queue = "default"; 18 | private String[] virualdirs = null; 19 | private boolean debug = false; 20 | 21 | 22 | public SubmitItem() { 23 | super(); 24 | } 25 | 26 | public SubmitItem(String appname, String docker_image, String workingdir, String command, long container_memory, long master_memory, long container_retry, int container_vcores, int priority, 27 | long timeout, String queue, String[] virualdirs, boolean debug) { 28 | super(); 29 | this.appname = appname; 30 | this.docker_image = docker_image; 31 | this.workingdir = workingdir; 32 | this.command = command; 33 | this.container_memory = container_memory; 34 | this.master_memory = master_memory; 35 | this.container_retry = container_retry; 36 | this.container_vcores = container_vcores; 37 | this.priority = priority; 38 | this.timeout = timeout; 39 | this.queue = queue; 40 | this.virualdirs = virualdirs; 41 | this.debug = debug; 42 | } 43 | 44 | public String[] getVirualdirs() { 45 | return virualdirs; 46 | } 47 | 48 | public void setVirualdirs(String[] virualdirs) { 49 | this.virualdirs = virualdirs; 50 | } 51 | 52 | public String getAppname() { 53 | return appname; 54 | } 55 | 56 | public void setAppname(String appname) { 57 | this.appname = appname; 58 | } 59 | 60 | public String getDocker_image() { 61 | return docker_image; 62 | } 63 | 64 | public void setDocker_image(String docker_image) { 65 | this.docker_image = docker_image; 66 | } 67 | 68 | public String getWorkingdir() { 69 | return workingdir; 70 | } 71 | 72 | public void setWorkingdir(String workingdir) { 73 | this.workingdir = workingdir; 74 | } 75 | 76 | public String getCommand() { 77 | return command; 78 | } 79 | 80 | public void setCommand(String command) { 81 | this.command = command; 82 | } 83 | 84 | public long getContainer_memory() { 85 | return container_memory; 86 | } 87 | 88 | public void setContainer_memory(long container_memory) { 89 | this.container_memory = container_memory; 90 | } 91 | 92 | public long getMaster_memory() { 93 | return master_memory; 94 | } 95 | 96 | public void setMaster_memory(long master_memory) { 97 | this.master_memory = master_memory; 98 | } 99 | 100 | public long getContainer_retry() { 101 | return container_retry; 102 | } 103 | 104 | public void setContainer_retry(long container_retry) { 105 | this.container_retry = container_retry; 106 | } 107 | 108 | public int getContainer_vcores() { 109 | return container_vcores; 110 | } 111 | 112 | public void setContainer_vcores(int container_vcores) { 113 | this.container_vcores = container_vcores; 114 | } 115 | 116 | public int getPriority() { 117 | return priority; 118 | } 119 | 120 | public void setPriority(int priority) { 121 | this.priority = priority; 122 | } 123 | 124 | public long getTimeout() { 125 | return timeout; 126 | } 127 | 128 | public void setTimeout(long timeout) { 129 | this.timeout = timeout; 130 | } 131 | 132 | public String getQueue() { 133 | return queue; 134 | } 135 | 136 | public void setQueue(String queue) { 137 | this.queue = queue; 138 | } 139 | 140 | public boolean isDebug() { 141 | return debug; 142 | } 143 | 144 | public void setDebug(boolean debug) { 145 | this.debug = debug; 146 | } 147 | 148 | 149 | } 150 | -------------------------------------------------------------------------------- /distributed-docker-submit-client/src/main/java/com/sogou/dockeronyarn/client/model/SubmitReturnItem.java: -------------------------------------------------------------------------------- 1 | package com.sogou.dockeronyarn.client.model; 2 | 3 | import org.apache.hadoop.yarn.api.records.ApplicationId; 4 | 5 | import javax.xml.bind.annotation.XmlRootElement; 6 | 7 | @XmlRootElement 8 | public class SubmitReturnItem { 9 | private String appid = null; 10 | private ApplicationId applicationIdObject; 11 | 12 | public ApplicationId getApplicationIdObject() { 13 | return applicationIdObject; 14 | } 15 | 16 | public void setApplicationIdObject(ApplicationId applicationIdObject) { 17 | this.applicationIdObject = applicationIdObject; 18 | } 19 | 20 | private String apptrackingurl = null; 21 | private int status = 0; 22 | private String message = null; 23 | 24 | public SubmitReturnItem() { 25 | super(); 26 | } 27 | 28 | public SubmitReturnItem(String appid, String apptrackingurl, int status, String message) { 29 | super(); 30 | this.appid = appid; 31 | this.apptrackingurl = apptrackingurl; 32 | this.status = status; 33 | this.message = message; 34 | } 35 | 36 | public String getMessage() { 37 | return message; 38 | } 39 | 40 | public void setMessage(String message) { 41 | this.message = message; 42 | } 43 | 44 | public String getAppid() { 45 | return appid; 46 | } 47 | 48 | public void setAppid(String appid) { 49 | this.appid = appid; 50 | } 51 | 52 | public String getApptrackingurl() { 53 | return apptrackingurl; 54 | } 55 | 56 | public void setApptrackingurl(String apptrackingurl) { 57 | this.apptrackingurl = apptrackingurl; 58 | } 59 | 60 | public int getStatus() { 61 | return status; 62 | } 63 | 64 | public void setStatus(int status) { 65 | this.status = status; 66 | } 67 | 68 | 69 | 70 | 71 | } 72 | -------------------------------------------------------------------------------- /distributed-docker-submit-client/src/main/resources/META-INF/services/javax.ws.rs.ext.RuntimeDelegate: -------------------------------------------------------------------------------- 1 | org.glassfish.jersey.internal.RuntimeDelegateImpl -------------------------------------------------------------------------------- /distributed-docker-submit-client/src/main/resources/distributed-docker-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | ddocker.runner.path 5 | /usr/lib/docker-on-yarn/runner.py 6 | 7 | 8 | ddocker.docker.cert.path 9 | /Users/guoshiwei/.boot2docker/certs/boot2docker-vm 10 | 11 | 12 | ddocker.docker.host 13 | unix:///var/run/docker.sock 14 | 15 | 16 | -------------------------------------------------------------------------------- /distributed-docker-submit-client/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Licensed to the Apache Software Foundation (ASF) under one 2 | # or more contributor license agreements. See the NOTICE file 3 | # distributed with this work for additional information 4 | # regarding copyright ownership. The ASF licenses this file 5 | # to you under the Apache License, Version 2.0 (the 6 | # "License"); you may not use this file except in compliance 7 | # with the License. You may obtain a copy of the License at 8 | # 9 | # http://www.apache.org/licenses/LICENSE-2.0 10 | # 11 | # Unless required by applicable law or agreed to in writing, software 12 | # distributed under the License is distributed on an "AS IS" BASIS, 13 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 14 | # See the License for the specific language governing permissions and 15 | # limitations under the License. 16 | 17 | # Define some default values that can be overridden by system properties 18 | hadoop.root.logger=INFO,console 19 | hadoop.log.dir=. 20 | hadoop.log.file=hadoop.log 21 | 22 | # Define the root logger to the system property "hadoop.root.logger". 23 | log4j.rootLogger=${hadoop.root.logger}, EventCounter 24 | 25 | # Logging Threshold 26 | log4j.threshold=ALL 27 | 28 | # Null Appender 29 | log4j.appender.NullAppender=org.apache.log4j.varia.NullAppender 30 | 31 | # 32 | # Rolling File Appender - cap space usage at 5gb. 33 | # 34 | hadoop.log.maxfilesize=256MB 35 | hadoop.log.maxbackupindex=20 36 | log4j.appender.RFA=org.apache.log4j.RollingFileAppender 37 | log4j.appender.RFA.File=${hadoop.log.dir}/${hadoop.log.file} 38 | 39 | log4j.appender.RFA.MaxFileSize=${hadoop.log.maxfilesize} 40 | log4j.appender.RFA.MaxBackupIndex=${hadoop.log.maxbackupindex} 41 | 42 | log4j.appender.RFA.layout=org.apache.log4j.PatternLayout 43 | 44 | # Pattern format: Date LogLevel LoggerName LogMessage 45 | log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 46 | # Debugging Pattern format 47 | #log4j.appender.RFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 48 | 49 | 50 | # 51 | # Daily Rolling File Appender 52 | # 53 | 54 | log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender 55 | log4j.appender.DRFA.File=${hadoop.log.dir}/${hadoop.log.file} 56 | 57 | # Rollover at midnight 58 | log4j.appender.DRFA.DatePattern=.yyyy-MM-dd 59 | 60 | log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout 61 | 62 | # Pattern format: Date LogLevel LoggerName LogMessage 63 | log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 64 | # Debugging Pattern format 65 | #log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p %c{2} (%F:%M(%L)) - %m%n 66 | 67 | 68 | # 69 | # console 70 | # Add "console" to rootlogger above if you want to use this 71 | # 72 | 73 | log4j.appender.console=org.apache.log4j.ConsoleAppender 74 | log4j.appender.console.target=System.err 75 | log4j.appender.console.layout=org.apache.log4j.PatternLayout 76 | log4j.appender.console.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n 77 | 78 | # 79 | # TaskLog Appender 80 | # 81 | 82 | #Default values 83 | hadoop.tasklog.taskid=null 84 | hadoop.tasklog.iscleanup=false 85 | hadoop.tasklog.noKeepSplits=4 86 | hadoop.tasklog.totalLogFileSize=100 87 | hadoop.tasklog.purgeLogSplits=true 88 | hadoop.tasklog.logsRetainHours=12 89 | 90 | log4j.appender.TLA=org.apache.hadoop.mapred.TaskLogAppender 91 | log4j.appender.TLA.taskId=${hadoop.tasklog.taskid} 92 | log4j.appender.TLA.isCleanup=${hadoop.tasklog.iscleanup} 93 | log4j.appender.TLA.totalLogFileSize=${hadoop.tasklog.totalLogFileSize} 94 | 95 | log4j.appender.TLA.layout=org.apache.log4j.PatternLayout 96 | log4j.appender.TLA.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 97 | 98 | # 99 | # HDFS block state change log from block manager 100 | # 101 | # Uncomment the following to suppress normal block state change 102 | # messages from BlockManager in NameNode. 103 | #log4j.logger.BlockStateChange=WARN 104 | 105 | # 106 | #Security appender 107 | # 108 | hadoop.security.logger=INFO,NullAppender 109 | hadoop.security.log.maxfilesize=256MB 110 | hadoop.security.log.maxbackupindex=20 111 | log4j.category.SecurityLogger=${hadoop.security.logger} 112 | hadoop.security.log.file=SecurityAuth-${user.name}.audit 113 | log4j.appender.RFAS=org.apache.log4j.RollingFileAppender 114 | log4j.appender.RFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} 115 | log4j.appender.RFAS.layout=org.apache.log4j.PatternLayout 116 | log4j.appender.RFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 117 | log4j.appender.RFAS.MaxFileSize=${hadoop.security.log.maxfilesize} 118 | log4j.appender.RFAS.MaxBackupIndex=${hadoop.security.log.maxbackupindex} 119 | 120 | # 121 | # Daily Rolling Security appender 122 | # 123 | log4j.appender.DRFAS=org.apache.log4j.DailyRollingFileAppender 124 | log4j.appender.DRFAS.File=${hadoop.log.dir}/${hadoop.security.log.file} 125 | log4j.appender.DRFAS.layout=org.apache.log4j.PatternLayout 126 | log4j.appender.DRFAS.layout.ConversionPattern=%d{ISO8601} %p %c: %m%n 127 | log4j.appender.DRFAS.DatePattern=.yyyy-MM-dd 128 | 129 | # 130 | # hadoop configuration logging 131 | # 132 | 133 | # Uncomment the following line to turn off configuration deprecation warnings. 134 | # log4j.logger.org.apache.hadoop.conf.Configuration.deprecation=WARN 135 | 136 | # 137 | # hdfs audit logging 138 | # 139 | hdfs.audit.logger=INFO,NullAppender 140 | hdfs.audit.log.maxfilesize=256MB 141 | hdfs.audit.log.maxbackupindex=20 142 | log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=${hdfs.audit.logger} 143 | log4j.additivity.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=false 144 | log4j.appender.RFAAUDIT=org.apache.log4j.RollingFileAppender 145 | log4j.appender.RFAAUDIT.File=${hadoop.log.dir}/hdfs-audit.log 146 | log4j.appender.RFAAUDIT.layout=org.apache.log4j.PatternLayout 147 | log4j.appender.RFAAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 148 | log4j.appender.RFAAUDIT.MaxFileSize=${hdfs.audit.log.maxfilesize} 149 | log4j.appender.RFAAUDIT.MaxBackupIndex=${hdfs.audit.log.maxbackupindex} 150 | 151 | # 152 | # mapred audit logging 153 | # 154 | mapred.audit.logger=INFO,NullAppender 155 | mapred.audit.log.maxfilesize=256MB 156 | mapred.audit.log.maxbackupindex=20 157 | log4j.logger.org.apache.hadoop.mapred.AuditLogger=${mapred.audit.logger} 158 | log4j.additivity.org.apache.hadoop.mapred.AuditLogger=false 159 | log4j.appender.MRAUDIT=org.apache.log4j.RollingFileAppender 160 | log4j.appender.MRAUDIT.File=${hadoop.log.dir}/mapred-audit.log 161 | log4j.appender.MRAUDIT.layout=org.apache.log4j.PatternLayout 162 | log4j.appender.MRAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 163 | log4j.appender.MRAUDIT.MaxFileSize=${mapred.audit.log.maxfilesize} 164 | log4j.appender.MRAUDIT.MaxBackupIndex=${mapred.audit.log.maxbackupindex} 165 | 166 | # Custom Logging levels 167 | 168 | #log4j.logger.org.apache.hadoop.mapred.JobTracker=DEBUG 169 | #log4j.logger.org.apache.hadoop.mapred.TaskTracker=DEBUG 170 | #log4j.logger.org.apache.hadoop.hdfs.server.namenode.FSNamesystem.audit=DEBUG 171 | 172 | # Jets3t library 173 | log4j.logger.org.jets3t.service.impl.rest.httpclient.RestS3Service=ERROR 174 | 175 | # AWS SDK & S3A FileSystem 176 | log4j.logger.com.amazonaws=ERROR 177 | log4j.logger.com.amazonaws.http.AmazonHttpClient=ERROR 178 | log4j.logger.org.apache.hadoop.fs.s3a.S3AFileSystem=WARN 179 | 180 | # 181 | # Event Counter Appender 182 | # Sends counts of logging messages at different severity levels to Hadoop Metrics. 183 | # 184 | log4j.appender.EventCounter=org.apache.hadoop.log.metrics.EventCounter 185 | 186 | # 187 | # Job Summary Appender 188 | # 189 | # Use following logger to send summary to separate file defined by 190 | # hadoop.mapreduce.jobsummary.log.file : 191 | # hadoop.mapreduce.jobsummary.logger=INFO,JSA 192 | # 193 | hadoop.mapreduce.jobsummary.logger=${hadoop.root.logger} 194 | hadoop.mapreduce.jobsummary.log.file=hadoop-mapreduce.jobsummary.log 195 | hadoop.mapreduce.jobsummary.log.maxfilesize=256MB 196 | hadoop.mapreduce.jobsummary.log.maxbackupindex=20 197 | log4j.appender.JSA=org.apache.log4j.RollingFileAppender 198 | log4j.appender.JSA.File=${hadoop.log.dir}/${hadoop.mapreduce.jobsummary.log.file} 199 | log4j.appender.JSA.MaxFileSize=${hadoop.mapreduce.jobsummary.log.maxfilesize} 200 | log4j.appender.JSA.MaxBackupIndex=${hadoop.mapreduce.jobsummary.log.maxbackupindex} 201 | log4j.appender.JSA.layout=org.apache.log4j.PatternLayout 202 | log4j.appender.JSA.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} %p %c{2}: %m%n 203 | log4j.logger.org.apache.hadoop.mapred.JobInProgress$JobSummary=${hadoop.mapreduce.jobsummary.logger} 204 | log4j.additivity.org.apache.hadoop.mapred.JobInProgress$JobSummary=false 205 | 206 | # 207 | # Yarn ResourceManager Application Summary Log 208 | # 209 | # Set the ResourceManager summary log filename 210 | yarn.server.resourcemanager.appsummary.log.file=rm-appsummary.log 211 | # Set the ResourceManager summary log level and appender 212 | yarn.server.resourcemanager.appsummary.logger=${hadoop.root.logger} 213 | #yarn.server.resourcemanager.appsummary.logger=INFO,RMSUMMARY 214 | 215 | # To enable AppSummaryLogging for the RM, 216 | # set yarn.server.resourcemanager.appsummary.logger to 217 | # ,RMSUMMARY in hadoop-env.sh 218 | 219 | # Appender for ResourceManager Application Summary Log 220 | # Requires the following properties to be set 221 | # - hadoop.log.dir (Hadoop Log directory) 222 | # - yarn.server.resourcemanager.appsummary.log.file (resource manager app summary log filename) 223 | # - yarn.server.resourcemanager.appsummary.logger (resource manager app summary log level and appender) 224 | 225 | log4j.logger.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=${yarn.server.resourcemanager.appsummary.logger} 226 | log4j.additivity.org.apache.hadoop.yarn.server.resourcemanager.RMAppManager$ApplicationSummary=false 227 | log4j.appender.RMSUMMARY=org.apache.log4j.RollingFileAppender 228 | log4j.appender.RMSUMMARY.File=${hadoop.log.dir}/${yarn.server.resourcemanager.appsummary.log.file} 229 | log4j.appender.RMSUMMARY.MaxFileSize=256MB 230 | log4j.appender.RMSUMMARY.MaxBackupIndex=20 231 | log4j.appender.RMSUMMARY.layout=org.apache.log4j.PatternLayout 232 | log4j.appender.RMSUMMARY.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 233 | 234 | # HS audit log configs 235 | #mapreduce.hs.audit.logger=INFO,HSAUDIT 236 | #log4j.logger.org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger=${mapreduce.hs.audit.logger} 237 | #log4j.additivity.org.apache.hadoop.mapreduce.v2.hs.HSAuditLogger=false 238 | #log4j.appender.HSAUDIT=org.apache.log4j.DailyRollingFileAppender 239 | #log4j.appender.HSAUDIT.File=${hadoop.log.dir}/hs-audit.log 240 | #log4j.appender.HSAUDIT.layout=org.apache.log4j.PatternLayout 241 | #log4j.appender.HSAUDIT.layout.ConversionPattern=%d{ISO8601} %p %c{2}: %m%n 242 | #log4j.appender.HSAUDIT.DatePattern=.yyyy-MM-dd 243 | 244 | # Http Server Request Logs 245 | #log4j.logger.http.requests.namenode=INFO,namenoderequestlog 246 | #log4j.appender.namenoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender 247 | #log4j.appender.namenoderequestlog.Filename=${hadoop.log.dir}/jetty-namenode-yyyy_mm_dd.log 248 | #log4j.appender.namenoderequestlog.RetainDays=3 249 | 250 | #log4j.logger.http.requests.datanode=INFO,datanoderequestlog 251 | #log4j.appender.datanoderequestlog=org.apache.hadoop.http.HttpRequestLogAppender 252 | #log4j.appender.datanoderequestlog.Filename=${hadoop.log.dir}/jetty-datanode-yyyy_mm_dd.log 253 | #log4j.appender.datanoderequestlog.RetainDays=3 254 | 255 | #log4j.logger.http.requests.resourcemanager=INFO,resourcemanagerrequestlog 256 | #log4j.appender.resourcemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender 257 | #log4j.appender.resourcemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-resourcemanager-yyyy_mm_dd.log 258 | #log4j.appender.resourcemanagerrequestlog.RetainDays=3 259 | 260 | #log4j.logger.http.requests.jobhistory=INFO,jobhistoryrequestlog 261 | #log4j.appender.jobhistoryrequestlog=org.apache.hadoop.http.HttpRequestLogAppender 262 | #log4j.appender.jobhistoryrequestlog.Filename=${hadoop.log.dir}/jetty-jobhistory-yyyy_mm_dd.log 263 | #log4j.appender.jobhistoryrequestlog.RetainDays=3 264 | 265 | #log4j.logger.http.requests.nodemanager=INFO,nodemanagerrequestlog 266 | #log4j.appender.nodemanagerrequestlog=org.apache.hadoop.http.HttpRequestLogAppender 267 | #log4j.appender.nodemanagerrequestlog.Filename=${hadoop.log.dir}/jetty-nodemanager-yyyy_mm_dd.log 268 | #log4j.appender.nodemanagerrequestlog.RetainDays=3 269 | -------------------------------------------------------------------------------- /distributed-docker-submit-client/test_docker_client.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | source ./hadoop-env.sh 3 | export HADOOP_CLASSPATH="/search/distributed-docker-submit-project/distributed-docker-appmaster/target/distributed-docker-appmaster-0.0.1-SNAPSHOT-jar-with-dependencies.jar" 4 | AM_JAR=../distributed-docker-appmaster/target/distributed-docker-appmaster-0.0.1-SNAPSHOT-jar-with-dependencies.jar 5 | CLIENT_JAR=target/distributed-docker-submit-client-0.0.1-SNAPSHOT.jar 6 | sh -x /usr/lib/hadoop/bin/hadoop jar $CLIENT_JAR com.sogou.dockeronyarn.client.DockerOnYarnClientCli \ 7 | -jar $AM_JAR \ 8 | -appname nimei \ 9 | -queue root.hdfs \ 10 | -image registry.docker.dev.sogou-inc.com:5000/clouddev/sogou-rhel-base:6.5 \ 11 | echo hello world 12 | 13 | -------------------------------------------------------------------------------- /distributed-docker-submit-server/.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | -------------------------------------------------------------------------------- /distributed-docker-submit-server/.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | distributed-docker-submit-server 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | org.eclipse.m2e.core.maven2Builder 15 | 16 | 17 | 18 | 19 | 20 | org.eclipse.jdt.core.javanature 21 | org.eclipse.m2e.core.maven2Nature 22 | 23 | 24 | -------------------------------------------------------------------------------- /distributed-docker-submit-server/.settings/org.eclipse.core.resources.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | encoding/=UTF-8 3 | -------------------------------------------------------------------------------- /distributed-docker-submit-server/.settings/org.eclipse.jdt.core.prefs: -------------------------------------------------------------------------------- 1 | eclipse.preferences.version=1 2 | org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.6 3 | org.eclipse.jdt.core.compiler.compliance=1.6 4 | org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning 5 | org.eclipse.jdt.core.compiler.source=1.6 6 | -------------------------------------------------------------------------------- /distributed-docker-submit-server/.settings/org.eclipse.m2e.core.prefs: -------------------------------------------------------------------------------- 1 | activeProfiles= 2 | eclipse.preferences.version=1 3 | resolveWorkspaceProjects=true 4 | version=1 5 | -------------------------------------------------------------------------------- /distributed-docker-submit-server/assembly.xml: -------------------------------------------------------------------------------- 1 | 2 | bin 3 | 4 | tar.gz 5 | dir 6 | 7 | 8 | 9 | true 10 | lib 11 | 12 | 13 | 14 | 15 | / 16 | 17 | README.md 18 | 19 | 20 | 21 | bin 22 | / 23 | 24 | 25 | conf 26 | /conf 27 | 28 | * 29 | 30 | 31 | 32 | scripts 33 | /scripts 34 | 35 | * 36 | init.d/* 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /distributed-docker-submit-server/bin/monitor_docker_submit_server.sh: -------------------------------------------------------------------------------- 1 | flag=`jps -m | awk '{print "\$2"}' | grep -c '^DockerSubmitMain$'` 2 | if [ $flag = 0 ] 3 | then 4 | echo [`date`] Start docker submit server 5 | cd /search/eclipse/workspace/distributed-docker-submit-server/; sh start_server.sh 6 | fi 7 | -------------------------------------------------------------------------------- /distributed-docker-submit-server/bin/start_server.sh: -------------------------------------------------------------------------------- 1 | #!/bin/sh 2 | project=docker-submit-server 3 | main_class=com.sogou.docker.server.DockerSubmitMain 4 | 5 | crontab=$(cat "/var/spool/cron/root" | grep "monitor_docker_submit_server" | wc -l) 6 | if [ $crontab -eq 0 ] 7 | then 8 | echo "*/1 * * * * cd /search/eclipse/distributed-docker-submit-server/; sh monitor_docker_submit_server.sh >> monitor.log 2>&1" >> /var/spool/cron/root 9 | fi 10 | 11 | 12 | running_num=`jps -ml | grep -c $main_class` 13 | if [ $running_num -ne 0 ] 14 | then 15 | echo "$project $main_class is already running..." 16 | exit 0 17 | fi 18 | 19 | dir='/search/eclipse/workspace/distributed-docker-submit-server/target' 20 | dir=`cd $dir/; pwd` 21 | 22 | bin_dir=$dir/bin 23 | lib_dir=$dir/lib 24 | conf_dir=$dir/conf 25 | log_dir=$dir/log 26 | cp *.properties target/ 27 | cp target/*.jar target/lib/ 28 | rm -fr $lib_dir/hadoop* 29 | 30 | JOBQUEUE_WORKER_HEAP_SIZE=128m 31 | 32 | for f in $lib_dir/* 33 | do 34 | CLASSPATH=$f:$CLASSPATH 35 | done 36 | 37 | CLASSPATH=$conf_dir:$CLASSPATH:`hadoop classpath` 38 | 39 | # set JVM heap size 40 | HEAP_SIZE=512m 41 | if [ "x${JOBQUEUE_WORKER_HEAP_SIZE}" != "x" ] 42 | then 43 | HEAP_SIZE=$JOBQUEUE_WORKER_HEAP_SIZE 44 | fi 45 | JAVA_OPTS="-Xmx${HEAP_SIZE}" 46 | 47 | # set jmx remote port 48 | JMX_REMOTE_OPTS="" 49 | JAVA_OPTS="$JAVA_OPTS $JMX_REMOTE_OPTS" 50 | 51 | cd $dir 52 | echo classpath: $CLASSPATH 53 | #nohup java $JAVA_OPTS -Xrunjdwp:transport=dt_socket,server=y,suspend=n,address=17891 -cp $CLASSPATH $main_class >$log_dir/$project.log 2>&1 & 54 | nohup java $JAVA_OPTS -cp $CLASSPATH $main_class >>submit.log & 55 | code=$? 56 | echo 'code: ' $code 57 | exit $code 58 | 59 | -------------------------------------------------------------------------------- /distributed-docker-submit-server/bin/test.sh: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | curl -H "Accept: application/json" -H "Content-type: application/json" -X POST \ 5 | -d '{"virualdir":"/search/working/", "docker_image":"10.134.69.187:5000/hadoop/cdh5client:1.2", "workingdir":"/search/sogouquery/", "command":"sh time_website.sh abcde 20141001"}' \ 6 | http://127.0.0.1:18088/docker/submit 7 | -------------------------------------------------------------------------------- /distributed-docker-submit-server/conf/log.properties: -------------------------------------------------------------------------------- 1 | log4j.rootLogger= DEBUG, console ,file 2 | log4j.appender.file.layout=org.apache.log4j.PatternLayout 3 | log4j.appender.file=org.apache.log4j.RollingFileAppender 4 | log4j.appender.file.File=docker.log 5 | log4j.appender.file.MaxFileSize=30000KB 6 | log4j.appender.file.MaxBackupIndex=100 7 | #log4j.appender.file.DatePattern='.'yyyy-MM 8 | log4j.appender.file.layout.ConversionPattern= %-d{yyyy-MM-dd HH:mm:ss:SSS} [%p] (%t) %m%n 9 | log4j.logger.org.apache.zookeeper=ERROR,file 10 | -------------------------------------------------------------------------------- /distributed-docker-submit-server/conf/server.properties: -------------------------------------------------------------------------------- 1 | server.ip=0.0.0.0 2 | server.port=18088 3 | dis.shell.jar=/search/eclipse/workspace/distributed-docker-submit-server/lib/distributed-docker-submit-client-0.0.1-SNAPSHOT.jar -------------------------------------------------------------------------------- /distributed-docker-submit-server/pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | distributed-docker-submit-server 5 | distributed-docker-submit-server 6 | 7 | distributed-docker-submit-project 8 | distributed-docker-submit-project 9 | 0.0.1-SNAPSHOT 10 | ../../distributed-docker-submit-project 11 | 12 | 13 | 0.0.1-SNAPSHOT 14 | distributed-docker-submit-server 15 | distributed-docker system of cloud_dev@sogou-inc.com 16 | 17 | true 18 | 19 | UTF-8 20 | true 21 | false 22 | 1.6 23 | 1.6 24 | 25 | 1.6.1 26 | 27 | 2.1.1 28 | 1.9 29 | 30 | 2.3.3 31 | 32 | 4.2.5 33 | 1.5 34 | 1.8 35 | 2.3 36 | 2.6 37 | 1.7.5 38 | 1.3.9 39 | 0.3 40 | 18.0 41 | 42 | 43 | 1.0.1 44 | 5.12.1 45 | 1.3 46 | 1.6 47 | 2.3.3 48 | 49 | 50 | 2.2 51 | 2.3.1 52 | 2.3.1 53 | 2.8.1 54 | 2.5.1 55 | 1.7 56 | 57 | 58 | 59 | 60 | my-local-repo 61 | file://${basedir}/lib/repo 62 | 63 | 64 | github-releases 65 | http://oss.sonatype.org/content/repositories/github-releases/ 66 | 67 | 68 | clojars.org 69 | http://clojars.org/repo 70 | 71 | 72 | sogou.repo1 73 | http://cloud.sogou-inc.com/nexus/content/groups/public 74 | SogouRepository 75 | 76 | 77 | 78 | 79 | 80 | distributed-docker-submit-client 81 | distributed-docker-submit-client 82 | 0.0.1-SNAPSHOT 83 | 84 | 85 | commons-lang 86 | commons-lang 87 | ${commons-lang.version} 88 | 89 | 90 | commons-io 91 | commons-io 92 | ${commons-io.version} 93 | 94 | 95 | com.google.guava 96 | guava 97 | ${guava.version} 98 | 99 | 100 | org.apache.hadoop 101 | hadoop-yarn-common 102 | 2.5.0-cdh5.3.2 103 | 104 | 105 | jersey-core 106 | com.sun.jersey 107 | 108 | 109 | 110 | 111 | 112 | org.apache.hadoop 113 | hadoop-common 114 | 2.5.0-cdh5.3.2 115 | 116 | 117 | jersey-core 118 | com.sun.jersey 119 | 120 | 121 | 122 | 123 | 124 | org.apache.hadoop 125 | hadoop-yarn-client 126 | 2.5.0-cdh5.3.2 127 | 128 | 129 | jersey-core 130 | com.sun.jersey 131 | 132 | 133 | 134 | 135 | org.glassfish.jersey.containers 136 | jersey-container-servlet-core 137 | 2.0 138 | 139 | 140 | 141 | org.glassfish.jersey.media 142 | jersey-media-json-jackson 143 | 2.0 144 | 145 | 146 | 147 | javax.xml 148 | jaxb-api 149 | 2.1 150 | 151 | 152 | 153 | 154 | org.eclipse.jetty.aggregate 155 | jetty-all 156 | 8.0.4.v20111024 157 | 158 | 159 | 160 | 161 | org.eclipse.jetty 162 | jetty-webapp 163 | 8.0.4.v20111024 164 | 165 | 166 | 167 | 168 | org.glassfish.web 169 | javax.servlet.jsp 170 | 2.2.3 171 | 172 | 173 | 174 | 175 | org.glassfish.web 176 | javax.el 177 | 2.2.3 178 | 179 | 180 | 181 | 182 | org.glassfish.web 183 | javax.servlet.jsp.jstl 184 | 1.2.1 185 | 186 | 187 | jstl-api 188 | javax.servlet.jsp.jstl 189 | 190 | 191 | 192 | 193 | 194 | log4j 195 | log4j 196 | 1.2.17 197 | 198 | 199 | javax.mail 200 | mail 201 | 1.4.1 202 | 203 | 204 | org.apache.commons 205 | commons-exec 206 | 1.2 207 | 208 | 209 | commons-configuration 210 | commons-configuration 211 | 1.9 212 | 213 | 214 | com.sogou 215 | util.sendalert.javatool 216 | 0.0.4 217 | 218 | 219 | 220 | com.sun.jersey 221 | jersey-client 222 | 1.18.1 223 | 224 | 225 | 226 | junit 227 | junit 228 | 4.11 229 | 230 | 231 | 232 | org.mockito 233 | mockito-core 234 | 1.9.5 235 | 236 | 237 | 238 | commons-io 239 | commons-io 240 | 2.4 241 | 242 | 243 | 244 | com.sun.jersey 245 | jersey-server 246 | 1.17 247 | 248 | 249 | org.jboss.resteasy 250 | resteasy-jaxrs 251 | 2.3.3.Final 252 | provided 253 | 254 | 255 | com.sun.jersey 256 | jersey-grizzly2 257 | 1.17 258 | 259 | 260 | javax.ws.rs 261 | jsr311-api 262 | 1.1.1 263 | 264 | 265 | com.sun.jersey 266 | jersey-json 267 | 1.17 268 | 269 | 270 | com.sun.jersey 271 | jersey-client 272 | 1.17 273 | 274 | 275 | org.json 276 | json 277 | 20090211 278 | 279 | 280 | 281 | com.sun.jersey 282 | jersey-servlet 283 | 1.18.1 284 | 285 | 286 | org.jboss.resteasy 287 | resteasy-jaxrs 288 | 2.3.3.Final 289 | provided 290 | 291 | 292 | 293 | 294 | 295 | 296 | 297 | org.apache.maven.plugins 298 | maven-dependency-plugin 299 | 2.7 300 | 301 | 302 | copy-dependencies 303 | package 304 | 305 | copy-dependencies 306 | 307 | 308 | ${project.build.directory}/lib 309 | false 310 | false 311 | true 312 | 313 | 314 | 315 | 316 | 317 | org.apache.maven.plugins 318 | maven-surefire-plugin 319 | 2.17 320 | 321 | 322 | ${project.build.directory}/conf 323 | 324 | 325 | 326 | 327 | maven-compiler-plugin 328 | 3.0 329 | 330 | 1.6 331 | 1.6 332 | GB18030 333 | 334 | 335 | 336 | org.apache.maven.plugins 337 | maven-assembly-plugin 338 | 339 | 340 | assembly.xml 341 | 342 | 343 | 344 | 345 | make-assembly 346 | package 347 | 348 | single 349 | 350 | 351 | 352 | 353 | 354 | 355 | 356 | org.apache.maven.plugins 357 | maven-jar-plugin 358 | 2.4 359 | 360 | 361 | 362 | true 363 | 364 | 365 | 366 | 367 | 368 | 369 | 370 | 371 | 372 | org.eclipse.m2e 373 | lifecycle-mapping 374 | 1.0.0 375 | 376 | 377 | 378 | 379 | 380 | 381 | org.apache.maven.plugins 382 | 383 | 384 | maven-dependency-plugin 385 | 386 | 387 | [2.7,) 388 | 389 | 390 | 391 | copy-dependencies 392 | 393 | 394 | 395 | 396 | 397 | 398 | 399 | 400 | 401 | 402 | 403 | 404 | 405 | 406 | 407 | -------------------------------------------------------------------------------- /distributed-docker-submit-server/src/main/java/com/sogou/dockeronyarn/server/DockerServer.java: -------------------------------------------------------------------------------- 1 | package com.sogou.dockeronyarn.server; 2 | 3 | import java.io.IOException; 4 | import java.net.URI; 5 | 6 | import javax.ws.rs.core.UriBuilder; 7 | 8 | import org.eclipse.jetty.server.handler.StatisticsHandler; 9 | import org.glassfish.grizzly.http.server.HttpServer; 10 | 11 | import com.sogou.dockeronyarn.util.StaticData; 12 | import com.sun.jersey.api.container.grizzly2.GrizzlyServerFactory; 13 | import com.sun.jersey.api.core.PackagesResourceConfig; 14 | import com.sun.jersey.api.core.ResourceConfig; 15 | 16 | public class DockerServer { 17 | 18 | public static String HTTP_CONTEXT = "/"; 19 | private static StatisticsHandler STATIC = null; 20 | 21 | private static URI BASE_URI = UriBuilder 22 | .fromUri(String.format("http://%s/", StaticData.SERVER_IP)) 23 | .port(StaticData.SERVER_PORT).build(); 24 | 25 | 26 | public void startHttpService() throws Exception { 27 | 28 | try { 29 | ResourceConfig rc = new PackagesResourceConfig("com.sogou.docker.service"); 30 | HttpServer httpServer = GrizzlyServerFactory.createHttpServer( 31 | BASE_URI, rc); 32 | httpServer.start(); 33 | 34 | } catch (IOException e) { 35 | e.printStackTrace(); 36 | } 37 | } 38 | 39 | public static StatisticsHandler getStaticHandler() { 40 | return STATIC; 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /distributed-docker-submit-server/src/main/java/com/sogou/dockeronyarn/server/DockerSubmitMain.java: -------------------------------------------------------------------------------- 1 | package com.sogou.dockeronyarn.server; 2 | 3 | import org.apache.commons.lang.math.RandomUtils; 4 | 5 | 6 | public class DockerSubmitMain { 7 | 8 | private static Object lock = new Object(); 9 | 10 | public static void main(String[] args) throws Exception { 11 | Thread.currentThread().setName("Main" + RandomUtils.nextInt()); 12 | Runtime.getRuntime().addShutdownHook(new ShutdownHook()); 13 | try { 14 | DockerServer server = new DockerServer(); 15 | server.startHttpService(); 16 | 17 | synchronized(lock){ 18 | lock.wait(); 19 | } 20 | 21 | 22 | } catch (Exception e) { 23 | System.exit(0); 24 | } 25 | } 26 | 27 | public static class ShutdownHook extends Thread{ 28 | public void run(){ 29 | try{ 30 | lock.notifyAll(); 31 | }catch(IllegalMonitorStateException e){ 32 | 33 | } 34 | 35 | } 36 | } 37 | 38 | 39 | } 40 | -------------------------------------------------------------------------------- /distributed-docker-submit-server/src/main/java/com/sogou/dockeronyarn/service/DockerService.java: -------------------------------------------------------------------------------- 1 | package com.sogou.dockeronyarn.service; 2 | 3 | import javax.ws.rs.Consumes; 4 | import javax.ws.rs.GET; 5 | import javax.ws.rs.POST; 6 | import javax.ws.rs.Path; 7 | import javax.ws.rs.Produces; 8 | import javax.ws.rs.QueryParam; 9 | import javax.ws.rs.core.MediaType; 10 | import javax.ws.rs.core.Response; 11 | import javax.ws.rs.core.Response.Status; 12 | 13 | import com.sogou.dockeronyarn.client.DockerApplicationSubmitter; 14 | import com.sogou.dockeronyarn.client.model.SubmitException; 15 | import com.sogou.dockeronyarn.util.StaticData; 16 | import org.apache.commons.logging.Log; 17 | import org.apache.commons.logging.LogFactory; 18 | 19 | import com.sogou.dockeronyarn.client.model.SubmitItem; 20 | import com.sogou.dockeronyarn.client.model.SubmitReturnItem; 21 | 22 | @Path("/docker") 23 | public class DockerService { 24 | private static final Log logger = LogFactory.getLog(DockerService.class); 25 | 26 | @POST 27 | @Path("submit") 28 | @Consumes(MediaType.APPLICATION_JSON) 29 | @Produces(MediaType.APPLICATION_JSON) 30 | public Response submit(SubmitItem si) { 31 | logger.debug("submit service start"); 32 | Status status = Status.NO_CONTENT; 33 | String message = null; 34 | SubmitReturnItem sri = null; 35 | try{ 36 | sri = new DockerApplicationSubmitter(StaticData.DIS_SHELL_JAR).submitToYarn(si); 37 | status = Status.OK; 38 | message = sri.getMessage(); 39 | }catch(SubmitException e){ 40 | message = e.getMessage(); 41 | status = Status.NO_CONTENT; 42 | } 43 | 44 | logger.debug(message); 45 | 46 | return Response.status(status).header("Message", message).entity(sri).build(); 47 | } 48 | 49 | @GET 50 | @Path("test") 51 | @Consumes(MediaType.APPLICATION_JSON) 52 | @Produces(MediaType.APPLICATION_JSON) 53 | public Response test(@QueryParam("str") String str) { 54 | logger.debug("test service start"); 55 | Status status = Status.NO_CONTENT; 56 | String message = null; 57 | 58 | 59 | status = Status.OK; 60 | message = "submit succeed"; 61 | 62 | logger.debug(message); 63 | 64 | return Response.status(status).header("Message", message).entity(str).build(); 65 | } 66 | 67 | 68 | } 69 | -------------------------------------------------------------------------------- /distributed-docker-submit-server/src/main/java/com/sogou/dockeronyarn/util/LogUtil.java: -------------------------------------------------------------------------------- 1 | package com.sogou.dockeronyarn.util; 2 | 3 | import java.util.concurrent.ConcurrentSkipListMap; 4 | import org.apache.log4j.Logger; 5 | import com.sogou.util.sendalert.bean.AlertLevel; 6 | import com.sogou.util.sendalert.bean.AlertType; 7 | 8 | 9 | public class LogUtil { 10 | private static Logger logger = Logger.getLogger(LogUtil.class); 11 | private static ConcurrentSkipListMap exceptions = new ConcurrentSkipListMap(); 12 | //private static AlertSenderThread alertSender = null; 13 | 14 | // static { 15 | // alertSender = new AlertSenderThread(); 16 | // alertSender.start(); 17 | // } 18 | 19 | public static void debug(String info) { 20 | logger.debug(info); 21 | } 22 | 23 | public static void debug(Object ...infos) { 24 | if (!logger.isDebugEnabled()) { 25 | return ; 26 | } 27 | logger.debug(objsToString(infos)); 28 | } 29 | 30 | public static void log(String info) { 31 | logger.info(info); 32 | } 33 | 34 | public static void log(Object ...infos) { 35 | if (!logger.isInfoEnabled()) { 36 | return ; 37 | } 38 | logger.info(objsToString(infos)); 39 | } 40 | 41 | public static void warn(String info) { 42 | logger.warn(info); 43 | } 44 | 45 | public static void warn(Object ...infos) { 46 | String logInfo = objsToString(infos); 47 | logger.warn(logInfo); 48 | } 49 | 50 | private static String objsToString(Object ...infos) { 51 | StringBuilder logInfo = new StringBuilder(); 52 | for (Object info : infos) { 53 | if (info == null) { 54 | logInfo.append("null"); 55 | } else { 56 | logInfo.append(info.toString()); 57 | } 58 | logInfo.append(' '); 59 | } 60 | return logInfo.toString(); 61 | } 62 | 63 | public static void error(String info) { 64 | logger.error(info); 65 | } 66 | 67 | public static void fatal(String info) { 68 | logger.fatal(info); 69 | } 70 | 71 | public static void error(Exception e) { 72 | exceptions.put(System.currentTimeMillis(), e); 73 | if (exceptions.size() > 20) { 74 | exceptions.remove(exceptions.firstKey()); 75 | } 76 | logger.error("Exception:", e); 77 | } 78 | 79 | public static void error(String info, Exception e) { 80 | exceptions.put(System.currentTimeMillis(), e); 81 | if (exceptions.size() > 20) { 82 | exceptions.remove(exceptions.firstKey()); 83 | } 84 | logger.error(info, e); 85 | } 86 | 87 | public static ConcurrentSkipListMap getExceptions() { 88 | return exceptions; 89 | } 90 | 91 | public static void setExceptions(ConcurrentSkipListMap exceptions) { 92 | LogUtil.exceptions = exceptions; 93 | } 94 | 95 | public static void warnAndSendAlert(String title, Object ...infos) { 96 | String logInfo = objsToString(infos); 97 | logger.warn(logInfo); 98 | } 99 | 100 | private static void sendAlert(String title, String logInfo, AlertType type, AlertLevel level) { 101 | //alertSender.addAlertInfo(new AlertInfo(title, logInfo, type, level)); 102 | } 103 | 104 | public static void errorAndSendAlert(String title, Object ...infos) { 105 | String logInfo = objsToString(infos); 106 | logger.warn(logInfo); 107 | } 108 | 109 | public static void sendNormalAlert(String title, String info) { 110 | // sendAlert(title, info, AlertType.DAILY, AlertLevel.NORMAL); 111 | } 112 | 113 | public static void fatal(Exception e) { 114 | logger.fatal(e); 115 | e.printStackTrace(); 116 | } 117 | 118 | public static void error(Object ...infos) { 119 | String logInfo = objsToString(infos); 120 | logger.error(logInfo); 121 | } 122 | 123 | } 124 | 125 | class AlertInfo { 126 | 127 | String title = null; 128 | String logInfo = null; 129 | AlertType type = null; 130 | AlertLevel level = null; 131 | 132 | public AlertInfo(String title, String logInfo, AlertType type, 133 | AlertLevel level) { 134 | super(); 135 | this.title = title; 136 | this.logInfo = logInfo; 137 | this.type = type; 138 | this.level = level; 139 | } 140 | 141 | } 142 | 143 | -------------------------------------------------------------------------------- /distributed-docker-submit-server/src/main/java/com/sogou/dockeronyarn/util/PropertyManager.java: -------------------------------------------------------------------------------- 1 | package com.sogou.dockeronyarn.util; 2 | 3 | import org.apache.commons.configuration.Configuration; 4 | import org.apache.commons.configuration.ConfigurationException; 5 | import org.apache.commons.configuration.PropertiesConfiguration; 6 | 7 | public class PropertyManager { 8 | private String propertyFile; 9 | private Configuration config; 10 | 11 | public PropertyManager(String propertyFile) { 12 | this.propertyFile = propertyFile; 13 | try { 14 | config = new PropertiesConfiguration(PropertyManager.class 15 | .getClassLoader().getResource(propertyFile)); 16 | } catch (ConfigurationException e) { 17 | e.printStackTrace(); 18 | System.exit(1); 19 | } 20 | } 21 | 22 | public String getProperty(String key) { 23 | if (config.containsKey(key)) { 24 | return config.getString(key); 25 | } else { 26 | return null; 27 | } 28 | } 29 | 30 | public String getPropertyFile() { 31 | return propertyFile; 32 | } 33 | 34 | public void setPropertyFile(String propertyFile) { 35 | this.propertyFile = propertyFile; 36 | } 37 | } 38 | -------------------------------------------------------------------------------- /distributed-docker-submit-server/src/main/java/com/sogou/dockeronyarn/util/StaticData.java: -------------------------------------------------------------------------------- 1 | package com.sogou.dockeronyarn.util; 2 | 3 | import org.apache.log4j.Logger; 4 | import org.apache.log4j.PropertyConfigurator; 5 | 6 | public class StaticData { 7 | public static String LOG4J_CONFIG = "conf/log.properties"; 8 | private static PropertyManager serverPropertyManager = new PropertyManager( 9 | "server.properties"); 10 | 11 | /* server info */ 12 | public static String SERVER_IP = serverPropertyManager 13 | .getProperty("server.ip"); 14 | public static int SERVER_PORT = Integer.parseInt(serverPropertyManager 15 | .getProperty("server.port")); 16 | 17 | public static String DIS_SHELL_JAR = serverPropertyManager 18 | .getProperty("dis.shell.jar"); 19 | 20 | 21 | 22 | 23 | 24 | public static Logger getLogger(Class clazz) { 25 | PropertyConfigurator.configure(clazz.getClassLoader().getResource( 26 | StaticData.LOG4J_CONFIG)); 27 | return Logger.getLogger(clazz.getName()); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /pom.xml: -------------------------------------------------------------------------------- 1 | 3 | 4.0.0 4 | 5 | distributed-docker-submit-project 6 | distributed-docker-submit-project 7 | 8 | 9 | 0.0.1-SNAPSHOT 10 | distributed-docker-submit-project 11 | distributed-docker system of cloud_dev@sogou-inc.com 12 | 13 | pom 14 | 15 | 16 | distributed-docker-submit-server 17 | distributed-docker-submit-client 18 | distributed-docker-appmaster 19 | 20 | 21 | 22 | sogou.repo1 23 | http://cloud.sogou-inc.com/nexus/content/groups/public 24 | SogouRepository 25 | 26 | 27 | 28 | 29 | 30 | maven-deploy-plugin 31 | 32 | true 33 | 34 | 35 | 36 | org.apache.rat 37 | apache-rat-plugin 38 | 39 | 40 | 41 | 42 | 43 | --------------------------------------------------------------------------------