├── jobs-core ├── src │ ├── test │ │ ├── resources │ │ │ ├── jobs │ │ │ │ └── demojob1 │ │ │ │ │ ├── demoscript.sh │ │ │ │ │ └── demojob1.conf │ │ │ ├── log4j.xml │ │ │ └── spring │ │ │ │ └── jobs-context.xml │ │ └── java │ │ │ └── de │ │ │ └── otto │ │ │ └── jobstore │ │ │ ├── common │ │ │ ├── JobExecutionPriorityTest.java │ │ │ ├── example │ │ │ │ ├── StepTwoJobRunnableExample.java │ │ │ │ ├── SimpleAbortableJob.java │ │ │ │ ├── StepOneJobRunnableExample.java │ │ │ │ └── SimpleJobRunnableExample.java │ │ │ ├── JobInfoCacheTest.java │ │ │ ├── JobInfoTest.java │ │ │ └── AbstractRemoteJobRunnableTest.java │ │ │ ├── service │ │ │ ├── JobSchedulerTest.java │ │ │ ├── DirectoryBasedTarArchiveProviderTest.java │ │ │ ├── RemoteJobExecutorServiceWithScriptTransferTest.java │ │ │ ├── JobInfoServiceTest.java │ │ │ ├── JobServiceNotActiveTest.java │ │ │ ├── RemoteJobExecutorServiceIntegrationTest.java │ │ │ ├── RemoteJobExecutorServiceWithScriptTransferIntegrationTest.java │ │ │ └── JobServiceIntegrationTest.java │ │ │ ├── repository │ │ │ └── JobDefinitionRepositoryIntegrationTest.java │ │ │ └── TestSetup.java │ └── main │ │ └── java │ │ └── de │ │ └── otto │ │ └── jobstore │ │ ├── common │ │ ├── properties │ │ │ ├── ItemProperty.java │ │ │ ├── LogLineProperty.java │ │ │ ├── JobDefinitionProperty.java │ │ │ └── JobInfoProperty.java │ │ ├── RunningState.java │ │ ├── ResultCode.java │ │ ├── JobExecutionPriority.java │ │ ├── AbstractRemoteJobDefinition.java │ │ ├── ActiveChecker.java │ │ ├── util │ │ │ └── InternetUtils.java │ │ ├── AbstractLocalJobDefinition.java │ │ ├── DefaultOnException.java │ │ ├── LogLine.java │ │ ├── JobExecutionResult.java │ │ ├── JobLogger.java │ │ ├── RemoteJobResult.java │ │ ├── AbstractItem.java │ │ ├── JobDefinition.java │ │ ├── JobInfoCache.java │ │ ├── JobSchedule.java │ │ ├── RemoteJobStatus.java │ │ ├── RemoteJob.java │ │ ├── JobRunnable.java │ │ ├── AbstractLocalJobRunnable.java │ │ ├── JobExecutionContext.java │ │ ├── StoredJobDefinition.java │ │ ├── AbstractRemoteJobRunnable.java │ │ └── JobInfo.java │ │ ├── service │ │ ├── exception │ │ │ ├── JobAlreadyQueuedException.java │ │ │ ├── JobNotRegisteredException.java │ │ │ ├── JobAlreadyRunningException.java │ │ │ ├── JobExecutionDisabledException.java │ │ │ ├── JobServiceNotActiveException.java │ │ │ ├── JobExecutionNotNecessaryException.java │ │ │ ├── JobExecutionException.java │ │ │ ├── RemoteJobNotRunningException.java │ │ │ ├── JobExecutionAbortedException.java │ │ │ ├── JobExecutionTimeoutException.java │ │ │ ├── RemoteJobAlreadyRunningException.java │ │ │ ├── JobException.java │ │ │ └── RemoteJobFailedException.java │ │ ├── RemoteJobExecutor.java │ │ ├── TarArchiveProvider.java │ │ ├── SimpleJobLogger.java │ │ ├── RemoteJobExecutorStatusRetriever.java │ │ ├── JobExecutionRunnable.java │ │ ├── DirectoryBasedTarArchiveProvider.java │ │ ├── RemoteJobExecutorService.java │ │ ├── JobInfoService.java │ │ ├── JobScheduler.java │ │ └── RemoteJobExecutorWithScriptTransferService.java │ │ └── repository │ │ ├── SortOrder.java │ │ ├── MongoOperator.java │ │ ├── JobDefinitionRepository.java │ │ └── AbstractRepository.java └── jobs-core.gradle ├── gradle ├── wrapper │ ├── gradle-wrapper.jar │ └── gradle-wrapper.properties └── libraries.gradle ├── settings.gradle ├── .gitignore ├── gradle.properties ├── jobs-api ├── jobs-api.gradle └── src │ ├── main │ └── java │ │ └── de │ │ └── otto │ │ └── jobstore │ │ └── web │ │ └── representation │ │ ├── JobNameRepresentation.java │ │ ├── LogLineRepresentation.java │ │ └── JobInfoRepresentation.java │ └── test │ ├── java │ └── de │ │ └── otto │ │ └── jobstore │ │ └── web │ │ ├── representation │ │ ├── LogLineRepresentationTest.java │ │ └── JobInfoRepresentationTest.java │ │ ├── JobInfoResourceIntegrationTest.java │ │ └── JobInfoResourceTest.java │ └── resources │ └── spring │ └── api-context.xml ├── gradlew.bat ├── README.md └── gradlew /jobs-core/src/test/resources/jobs/demojob1/demoscript.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | ping -c 50 $1 -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/otto-de-legacy/jobs/master/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | include ':jobs-core', ':jobs-api' 2 | 3 | project(':jobs-core').buildFileName = 'jobs-core.gradle' 4 | project(':jobs-api').buildFileName = 'jobs-api.gradle' 5 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/common/properties/ItemProperty.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common.properties; 2 | 3 | 4 | /** 5 | * Interface for Properties of Items 6 | */ 7 | public interface ItemProperty { 8 | 9 | String val(); 10 | 11 | } 12 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/common/RunningState.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common; 2 | 3 | /** 4 | * The State a Job can have while it is Running 5 | */ 6 | public enum RunningState { 7 | 8 | QUEUED, 9 | RUNNING, 10 | FINISHED 11 | 12 | } 13 | -------------------------------------------------------------------------------- /jobs-core/src/test/resources/jobs/demojob1/demojob1.conf: -------------------------------------------------------------------------------- 1 | 2 | program ./demoscript.sh $host 3 | socket-name $zsocket 4 | transcript $transcript_file 5 | backoff-limit 3 6 | 7 | 8 | 9 | path $zlog 10 | 11 | -------------------------------------------------------------------------------- /gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Mon Jun 22 14:29:05 CEST 2015 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=http\://services.gradle.org/distributions/gradle-2.4-bin.zip 7 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/common/ResultCode.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common; 2 | 3 | /** 4 | * The Result code of a Job once it is finished. 5 | */ 6 | public enum ResultCode { 7 | 8 | SUCCESSFUL, 9 | 10 | FAILED, 11 | 12 | TIMED_OUT, 13 | 14 | ABORTED, 15 | 16 | } 17 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.pyc 2 | venv 3 | coverage.xml 4 | *.iml 5 | *.iws 6 | *.ipr 7 | out 8 | build 9 | **/*.iml 10 | **/*.iws 11 | **/*.ipr 12 | **/out 13 | **/build 14 | .gradle 15 | .idea 16 | jobs-executor/instances/* 17 | instances/*.conf 18 | job-executor/coverage.xml 19 | jobs-core/jobs-executor 20 | *.log 21 | /.project 22 | -------------------------------------------------------------------------------- /gradle.properties: -------------------------------------------------------------------------------- 1 | nexusReleaseUrl= 2 | nexusReleaseUsername= 3 | nexusReleasePassword= 4 | 5 | nexusSnapshotUrl= 6 | nexusSnapshotUsername= 7 | nexusSnapshotPassword= 8 | 9 | artifactoryUrl= 10 | artifactoryUsername= 11 | artifactoryPassword= 12 | 13 | #org.gradle.jvmargs=-Djavax.net.ssl.trustStore=~/.gradle/cacerts -Djavax.net.ssl.trustStorePassword=changeit 14 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/service/exception/JobAlreadyQueuedException.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service.exception; 2 | 3 | 4 | /** 5 | * Exception which is thrown if a Job is already Queued 6 | */ 7 | public final class JobAlreadyQueuedException extends JobException { 8 | 9 | public JobAlreadyQueuedException(String s) { 10 | super(s); 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/service/exception/JobNotRegisteredException.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service.exception; 2 | 3 | 4 | /** 5 | * Exception which is thrown if a Job is not registered 6 | */ 7 | public final class JobNotRegisteredException extends JobException { 8 | 9 | public JobNotRegisteredException(String s) { 10 | super(s); 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/service/exception/JobAlreadyRunningException.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service.exception; 2 | 3 | 4 | /** 5 | * Exception which is thrown if a job is already running 6 | */ 7 | public final class JobAlreadyRunningException extends JobException { 8 | 9 | public JobAlreadyRunningException(String s) { 10 | super(s); 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/service/exception/JobExecutionDisabledException.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service.exception; 2 | 3 | 4 | /** 5 | * Exception which is thrown if a job is already running 6 | */ 7 | public final class JobExecutionDisabledException extends JobException { 8 | 9 | public JobExecutionDisabledException(String s) { 10 | super(s); 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/service/exception/JobServiceNotActiveException.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service.exception; 2 | 3 | 4 | /** 5 | * Exception which is thrown if the job service is not active 6 | */ 7 | public final class JobServiceNotActiveException extends JobException { 8 | 9 | public JobServiceNotActiveException(String s) { 10 | super(s); 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/service/exception/JobExecutionNotNecessaryException.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service.exception; 2 | 3 | 4 | /** 5 | * Exception which is thrown if execution of a job is not necessary 6 | */ 7 | public final class JobExecutionNotNecessaryException extends JobException { 8 | 9 | public JobExecutionNotNecessaryException(String s) { 10 | super(s); 11 | } 12 | 13 | } 14 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/repository/SortOrder.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.repository; 2 | 3 | /** 4 | * Enumeration for MongoDB Sort Order 5 | * 6 | * @author Sebastian Schroeder 7 | */ 8 | enum SortOrder { 9 | 10 | ASC(1), 11 | DESC(-1); 12 | 13 | private final int val; 14 | 15 | private SortOrder(int key) { 16 | this.val = key; 17 | } 18 | 19 | public int val() { 20 | return val; 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/service/exception/JobExecutionException.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service.exception; 2 | 3 | /** 4 | * Exception for the case something goes bad while a job is executed. 5 | */ 6 | public final class JobExecutionException extends JobException { 7 | 8 | public JobExecutionException(String s) { 9 | super(s); 10 | } 11 | 12 | public JobExecutionException(String s, Throwable throwable) { 13 | super(s, throwable); 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/common/JobExecutionPriority.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common; 2 | 3 | 4 | public enum JobExecutionPriority { 5 | 6 | CHECK_PRECONDITIONS(1), 7 | IGNORE_PRECONDITIONS(2), 8 | FORCE_EXECUTION(3); 9 | 10 | private final int level; 11 | 12 | private JobExecutionPriority(int level) { 13 | this.level = level; 14 | } 15 | 16 | public boolean hasLowerPriority(JobExecutionPriority priority) { 17 | return this.level < priority.level; 18 | } 19 | 20 | } 21 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/service/exception/RemoteJobNotRunningException.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service.exception; 2 | 3 | 4 | import java.net.URI; 5 | 6 | /** 7 | * Exception which is thrown if a job is already running 8 | */ 9 | public final class RemoteJobNotRunningException extends JobException { 10 | 11 | public RemoteJobNotRunningException(String s, Throwable t) { 12 | super(s, t); 13 | } 14 | 15 | public RemoteJobNotRunningException(String s) { 16 | super(s); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/common/properties/LogLineProperty.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common.properties; 2 | 3 | 4 | /** 5 | * Properties of LogLIne 6 | * 7 | * {@link de.otto.jobstore.common.LogLine} 8 | */ 9 | public enum LogLineProperty implements ItemProperty { 10 | 11 | LINE("line"), 12 | TIMESTAMP("timestamp"); 13 | 14 | private final String value; 15 | 16 | private LogLineProperty(String value) { 17 | this.value = value; 18 | } 19 | 20 | public String val() { 21 | return value; 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/service/exception/JobExecutionAbortedException.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service.exception; 2 | 3 | 4 | /** 5 | * Exception which is thrown if a job was aborted 6 | */ 7 | public final class JobExecutionAbortedException extends JobException { 8 | 9 | public JobExecutionAbortedException(String s) { 10 | super(s); 11 | } 12 | 13 | public static JobExecutionAbortedException fromJobName(String name) { 14 | return new JobExecutionAbortedException("Job with name " + name + " was aborted"); 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/service/exception/JobExecutionTimeoutException.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service.exception; 2 | 3 | 4 | /** 5 | * Exception which is thrown if a job was aborted 6 | */ 7 | public final class JobExecutionTimeoutException extends JobException { 8 | 9 | public JobExecutionTimeoutException(String s) { 10 | super(s); 11 | } 12 | 13 | public static JobExecutionTimeoutException fromJobName(String name) { 14 | return new JobExecutionTimeoutException("Job with name " + name + " reached timeout"); 15 | } 16 | 17 | } 18 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/service/RemoteJobExecutor.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service; 2 | 3 | import de.otto.jobstore.common.RemoteJob; 4 | import de.otto.jobstore.common.RemoteJobStatus; 5 | import de.otto.jobstore.service.exception.JobException; 6 | 7 | import java.net.URI; 8 | 9 | public interface RemoteJobExecutor { 10 | String getJobExecutorUri(); 11 | 12 | URI startJob(RemoteJob job) throws JobException; 13 | 14 | void stopJob(URI jobUri) throws JobException; 15 | 16 | RemoteJobStatus getStatus(URI jobUri); 17 | 18 | boolean isAlive(); 19 | } 20 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/service/exception/RemoteJobAlreadyRunningException.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service.exception; 2 | 3 | 4 | import java.net.URI; 5 | 6 | /** 7 | * Exception which is thrown if a job is already running 8 | */ 9 | public final class RemoteJobAlreadyRunningException extends JobException { 10 | 11 | private final URI jobUri; 12 | 13 | public RemoteJobAlreadyRunningException(String s, URI jobUri) { 14 | super(s); 15 | this.jobUri = jobUri; 16 | } 17 | 18 | public URI getJobUri() { 19 | return jobUri; 20 | } 21 | 22 | } 23 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/common/AbstractRemoteJobDefinition.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common; 2 | 3 | 4 | public abstract class AbstractRemoteJobDefinition implements JobDefinition { 5 | 6 | @Override 7 | public long getMaxRetries() { 8 | return 0; 9 | } 10 | 11 | @Override 12 | public long getRetryInterval() { 13 | return -1; 14 | } 15 | 16 | @Override 17 | public final boolean isRemote() { 18 | return true; 19 | } 20 | 21 | @Override 22 | public boolean isAbortable() { 23 | return false; 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /jobs-core/src/test/java/de/otto/jobstore/common/JobExecutionPriorityTest.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common; 2 | 3 | 4 | import org.testng.annotations.Test; 5 | 6 | import static org.testng.AssertJUnit.assertTrue; 7 | 8 | public class JobExecutionPriorityTest { 9 | 10 | @Test 11 | public void testLowerJobPriority() throws Exception { 12 | assertTrue(JobExecutionPriority.CHECK_PRECONDITIONS.hasLowerPriority(JobExecutionPriority.IGNORE_PRECONDITIONS)); 13 | assertTrue(JobExecutionPriority.IGNORE_PRECONDITIONS.hasLowerPriority(JobExecutionPriority.FORCE_EXECUTION)); 14 | } 15 | 16 | } 17 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/repository/MongoOperator.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.repository; 2 | 3 | /** 4 | * Enumeration of MongoDB Query Operators 5 | * 6 | * @author Sebastian Schroeder 7 | */ 8 | enum MongoOperator { 9 | 10 | GTE("$gte"), 11 | IN("$in"), 12 | LT("$lt"), 13 | LTE("$lte"), 14 | NE("$ne"), 15 | NIN("$nin"), 16 | PUSH("$push"), 17 | PUSH_ALL("$pushAll"), 18 | SET("$set"); 19 | 20 | private final String op; 21 | 22 | private MongoOperator(final String op) { 23 | this.op = op; 24 | } 25 | 26 | public String op() { 27 | return op; 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /jobs-api/jobs-api.gradle: -------------------------------------------------------------------------------- 1 | uploadArchives { 2 | configuration = configurations.archives 3 | repositories { 4 | mavenDeployer { 5 | pom { 6 | groupId = 'de.otto' 7 | artifactId = 'jobs-api' 8 | } 9 | } 10 | } 11 | } 12 | 13 | dependencies { 14 | compile project(':jobs-core') 15 | testCompile project(':jobs-core').sourceSets.test.output 16 | compile libs.jerseyAbdera, libs.jerseyClient, libs.jerseyCore 17 | 18 | testCompile libs.testng, libs.mockito 19 | testCompile libs.cobertura 20 | testCompile libs.springContext, libs.springCore, libs.springBeans, libs.springTest 21 | } -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/common/ActiveChecker.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common; 2 | 3 | /** 4 | * This Interface has the purpose to check, if the server is currently active and should execute jobs. 5 | * 6 | * Everytime a job should be executed (Either directly or by executing a queued job) this interface gets called. 7 | * 8 | * Usages may be a green-blue deployment, where only the newly deployed servers should execute jobs. 9 | * Or if you divide your servers in online and batch servers with the same database, but only the batch-server should execute jobs. 10 | */ 11 | public interface ActiveChecker { 12 | 13 | boolean isActive(); 14 | 15 | } 16 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/common/util/InternetUtils.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common.util; 2 | 3 | import java.net.InetAddress; 4 | import java.net.UnknownHostException; 5 | 6 | public final class InternetUtils { 7 | 8 | private InternetUtils() {} 9 | 10 | /** 11 | * Returns the canonical hostname 12 | * @return canonical hostname 13 | */ 14 | public static String getHostName() { 15 | try { 16 | final InetAddress address = InetAddress.getLocalHost(); 17 | return address.getCanonicalHostName(); 18 | } catch (UnknownHostException e) { 19 | return "N/A"; 20 | } 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/service/TarArchiveProvider.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service; 2 | 3 | import de.otto.jobstore.common.RemoteJob; 4 | 5 | import java.io.IOException; 6 | import java.io.InputStream; 7 | import java.net.URISyntaxException; 8 | 9 | /** 10 | * 11 | */ 12 | public interface TarArchiveProvider { 13 | /** 14 | * returns an InputStream on the tar archive for the given job. This shall contain all required files 15 | * for remote execution 16 | * @param remoteJob 17 | * @return 18 | * @throws IOException 19 | * 20 | */ 21 | InputStream getArchiveAsInputStream(RemoteJob remoteJob) throws IOException; 22 | } 23 | -------------------------------------------------------------------------------- /jobs-api/src/main/java/de/otto/jobstore/web/representation/JobNameRepresentation.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.web.representation; 2 | 3 | import javax.xml.bind.annotation.XmlAccessType; 4 | import javax.xml.bind.annotation.XmlAccessorType; 5 | import javax.xml.bind.annotation.XmlRootElement; 6 | 7 | @XmlRootElement(name = "job") 8 | @XmlAccessorType(value = XmlAccessType.FIELD) 9 | public final class JobNameRepresentation { 10 | 11 | private String name; 12 | 13 | public JobNameRepresentation() {} 14 | 15 | public JobNameRepresentation(String name) { 16 | this.name = name; 17 | } 18 | 19 | public String getName() { 20 | return name; 21 | } 22 | 23 | } 24 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/common/AbstractLocalJobDefinition.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common; 2 | 3 | public abstract class AbstractLocalJobDefinition implements JobDefinition { 4 | 5 | @Override 6 | public final long getPollingInterval() { 7 | return -1; 8 | } 9 | 10 | @Override 11 | public long getMaxRetries() { 12 | return 0; 13 | } 14 | 15 | @Override 16 | public long getRetryInterval() { 17 | return -1; 18 | } 19 | 20 | @Override 21 | public final boolean isRemote() { 22 | return false; 23 | } 24 | 25 | @Override 26 | public boolean isAbortable() { 27 | return false; 28 | } 29 | 30 | 31 | } 32 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/common/DefaultOnException.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common; 2 | 3 | import de.otto.jobstore.service.exception.JobException; 4 | 5 | public class DefaultOnException implements JobRunnable.OnException { 6 | private final Exception e; 7 | 8 | public DefaultOnException(Exception e) { 9 | this.e = e; 10 | } 11 | 12 | @Override 13 | public void doThrow() throws JobException { 14 | if (e instanceof JobException) { 15 | throw (JobException) e; 16 | } 17 | throw new JobException("Unexpected exception.", e){}; 18 | } 19 | 20 | @Override 21 | public boolean hasRecovered() { 22 | return false; 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/common/LogLine.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common; 2 | 3 | import com.mongodb.DBObject; 4 | import de.otto.jobstore.common.properties.LogLineProperty; 5 | 6 | import java.util.Date; 7 | 8 | public final class LogLine extends AbstractItem { 9 | 10 | public LogLine(DBObject dbObject) { 11 | super(dbObject); 12 | } 13 | 14 | public LogLine(String line, Date timestamp) { 15 | addProperty(LogLineProperty.LINE, line); 16 | addProperty(LogLineProperty.TIMESTAMP, timestamp); 17 | } 18 | 19 | public String getLine() { 20 | return getProperty(LogLineProperty.LINE); 21 | } 22 | 23 | public Date getTimestamp() { 24 | return getProperty(LogLineProperty.TIMESTAMP); 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/common/JobExecutionResult.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common; 2 | 3 | public final class JobExecutionResult { 4 | 5 | final RunningState runningState; 6 | 7 | final ResultCode resultCode; 8 | 9 | public JobExecutionResult(RunningState runningState) { 10 | this.runningState = runningState; 11 | this.resultCode = null; 12 | } 13 | 14 | public JobExecutionResult(RunningState runningState, ResultCode resultState) { 15 | this.runningState = runningState; 16 | this.resultCode = resultState; 17 | } 18 | 19 | public RunningState getRunningState() { 20 | return runningState; 21 | } 22 | 23 | public ResultCode getResultCode() { 24 | return resultCode; 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/service/exception/JobException.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service.exception; 2 | 3 | 4 | public abstract class JobException extends Exception { 5 | 6 | protected JobException(String s, Throwable t) { 7 | super(s,t); 8 | } 9 | 10 | protected JobException(String s) { 11 | super(s); 12 | } 13 | 14 | //public enum Type { EXECUTION } 15 | 16 | //private final Type type; 17 | 18 | //public Type getType() { 19 | // return type; 20 | //} 21 | 22 | //public JobException(Type type, String s) { 23 | // super(s); 24 | // this.type = type; 25 | //} 26 | 27 | //protected JobException(Type type, String s, Throwable throwable) { 28 | // super(s, throwable); 29 | // this.type = type; 30 | //} 31 | 32 | } 33 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/service/exception/RemoteJobFailedException.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service.exception; 2 | 3 | import de.otto.jobstore.common.JobInfo; 4 | import de.otto.jobstore.common.RemoteJobStatus; 5 | 6 | public class RemoteJobFailedException extends JobException { 7 | private final JobInfo jobInfo; 8 | private final RemoteJobStatus remoteJobStatus; 9 | 10 | public RemoteJobFailedException(JobInfo jobInfo, RemoteJobStatus remoteJobStatus) { 11 | super("Job " + jobInfo.getName() + " failed."); 12 | this.jobInfo = jobInfo; 13 | this.remoteJobStatus = remoteJobStatus; 14 | } 15 | 16 | public RemoteJobStatus getRemoteJobStatus() { 17 | return remoteJobStatus; 18 | } 19 | 20 | public JobInfo getJobInfo() { 21 | return jobInfo; 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /jobs-api/src/test/java/de/otto/jobstore/web/representation/LogLineRepresentationTest.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.web.representation; 2 | 3 | import de.otto.jobstore.common.LogLine; 4 | import org.testng.annotations.Test; 5 | 6 | import javax.xml.bind.JAXBContext; 7 | import javax.xml.bind.Marshaller; 8 | import javax.xml.bind.Unmarshaller; 9 | import java.io.StringWriter; 10 | import java.util.Date; 11 | 12 | import static org.testng.AssertJUnit.assertEquals; 13 | 14 | public class LogLineRepresentationTest { 15 | 16 | @Test 17 | public void testFromLogLine() throws Exception { 18 | LogLine logLine = new LogLine("line", new Date()); 19 | LogLineRepresentation llRep = LogLineRepresentation.fromLogLine(logLine); 20 | assertEquals(logLine.getLine(), llRep.getLine()); 21 | assertEquals(logLine.getTimestamp(), llRep.getTimestamp()); 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/common/JobLogger.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common; 2 | 3 | import java.util.List; 4 | 5 | /** 6 | * Adds logging and/or additional data to a job being executed 7 | */ 8 | public interface JobLogger { 9 | 10 | /** 11 | * Adds logging data to the job 12 | * 13 | * @param log The log line 14 | */ 15 | void addLoggingData(String log); 16 | 17 | List getLoggingData(); 18 | 19 | /** 20 | * Adds additional data to the job 21 | * 22 | * @param key The key of the additional data 23 | * @param value The data to be added 24 | */ 25 | void insertOrUpdateAdditionalData(String key, String value); 26 | 27 | /** 28 | * Return the additional data stored for this job 29 | * @param key the key of the additional data 30 | * @return value The value to be found. 31 | */ 32 | public String getAdditionalData(String key); 33 | 34 | } 35 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/common/RemoteJobResult.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common; 2 | 3 | import javax.xml.bind.annotation.XmlElement; 4 | import javax.xml.bind.annotation.XmlRootElement; 5 | 6 | @XmlRootElement 7 | public final class RemoteJobResult { 8 | 9 | public boolean ok; 10 | 11 | @XmlElement(name = "exit_code") 12 | public int exitCode; 13 | 14 | public String message; 15 | 16 | public RemoteJobResult() {} 17 | 18 | public RemoteJobResult(boolean ok, int exitCode, String message) { 19 | this.ok = ok; 20 | this.exitCode = exitCode; 21 | this.message = message; 22 | } 23 | 24 | @Override 25 | public String toString() { 26 | final StringBuilder sb = new StringBuilder(); 27 | sb.append("RemoteJobResult"); 28 | sb.append("{ok=").append(ok); 29 | sb.append(", exitCode=").append(exitCode); 30 | sb.append(", message='").append(message).append('\''); 31 | sb.append('}'); 32 | return sb.toString(); 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/common/AbstractItem.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common; 2 | 3 | import com.mongodb.BasicDBObject; 4 | import com.mongodb.DBObject; 5 | import de.otto.jobstore.common.properties.ItemProperty; 6 | 7 | import java.io.Serializable; 8 | 9 | /** 10 | * Abstract Class for Objects to be stored in MongoDB 11 | */ 12 | public abstract class AbstractItem implements Serializable { 13 | 14 | private final DBObject dbObject; 15 | 16 | AbstractItem() { 17 | dbObject = new BasicDBObject(); 18 | } 19 | 20 | AbstractItem(DBObject dbObject) { 21 | this.dbObject = dbObject; 22 | } 23 | 24 | public final DBObject toDbObject() { 25 | return dbObject; 26 | } 27 | 28 | final void addProperty(final ItemProperty key, final Object value) { 29 | dbObject.put(key.val(), value); 30 | } 31 | 32 | @SuppressWarnings("unchecked") 33 | final E getProperty(final ItemProperty key) { 34 | return (E) dbObject.get(key.val()); 35 | } 36 | 37 | final boolean hasProperty(final ItemProperty key) { 38 | return dbObject.containsField(key.val()); 39 | } 40 | 41 | } 42 | -------------------------------------------------------------------------------- /jobs-core/src/test/java/de/otto/jobstore/service/JobSchedulerTest.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service; 2 | 3 | import de.otto.jobstore.common.JobSchedule; 4 | import org.testng.annotations.BeforeMethod; 5 | import org.testng.annotations.Test; 6 | 7 | import java.util.Arrays; 8 | 9 | import static org.testng.AssertJUnit.assertTrue; 10 | 11 | public class JobSchedulerTest { 12 | 13 | 14 | @BeforeMethod 15 | public void setUp() throws Exception { 16 | } 17 | 18 | @Test 19 | public void testSchedulingOfJobs() throws Exception { 20 | JobSchedule jobSchedule1 = JobSchedule.create("jobSchedule1",100, new Runnable() { 21 | @Override 22 | public void run() { 23 | throw new RuntimeException(); 24 | } 25 | }); 26 | JobSchedule jobSchedule2 = JobSchedule.create("jobSchedule2",200, null); 27 | 28 | JobScheduler jobScheduler = new JobScheduler(Arrays.asList(jobSchedule1, jobSchedule2)); 29 | jobScheduler.startup(); 30 | Thread.sleep(1200); 31 | jobScheduler.shutdown(); 32 | 33 | assertTrue(jobSchedule1.count() > 10); 34 | assertTrue(jobSchedule2.count() > 5); 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/common/properties/JobDefinitionProperty.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common.properties; 2 | 3 | /** 4 | * Key names used to refer to properties in Job. 5 | * 6 | * {@link de.otto.jobstore.common.StoredJobDefinition} 7 | */ 8 | public enum JobDefinitionProperty implements ItemProperty { 9 | 10 | NAME("name"), 11 | MAX_IDLE_TIME("maxIdleTime"), 12 | MAX_EXECUTION_TIME("maxExecutionTime"), 13 | POLLING_INTERVAL("pollingInterval"), 14 | MAX_RETRIES("maxRetries"), 15 | RETRY_INTERVAL("retryInterval"), 16 | REMOTE("remote"), 17 | DISABLED("disabled", true), 18 | LAST_NOT_EXECUTED("lastNotExecuted", true), 19 | ABORTABLE("abortable"); 20 | 21 | private final String value; 22 | private final boolean dynamic; 23 | 24 | private JobDefinitionProperty(String value, boolean dynamic) { 25 | this.value = value; 26 | this.dynamic = dynamic; 27 | } 28 | 29 | private JobDefinitionProperty(String value) { 30 | this.value = value; 31 | this.dynamic = false; 32 | } 33 | 34 | public String val() { 35 | return value; 36 | } 37 | 38 | public boolean isDynamic() { 39 | return dynamic; 40 | } 41 | } 42 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/common/properties/JobInfoProperty.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common.properties; 2 | 3 | /** 4 | * Key names used to refer to properties in JobInfo. 5 | * 6 | * {@link de.otto.jobstore.common.JobInfo} 7 | */ 8 | public enum JobInfoProperty implements ItemProperty { 9 | 10 | ID("_id"), 11 | NAME("name"), 12 | HOST("host"), 13 | THREAD("thread"), 14 | CREATION_TIME("creationTime"), 15 | START_TIME("startTime"), 16 | FINISH_TIME("finishTime"), 17 | PARAMETERS("parameters"), 18 | EXECUTION_PRIORITY("executionPriority"), 19 | STATUS_MESSAGE("statusMessage"), 20 | RESULT_MESSAGE("resultMessage"), 21 | RUNNING_STATE("runningState"), 22 | RESULT_STATE("resultState"), 23 | MAX_IDLE_TIME("maxIdleTime"), 24 | MAX_EXECUTION_TIME("maxExecutionTime"), 25 | RETRIES("retries"), 26 | LAST_MODIFICATION_TIME("lastModificationTime"), 27 | ADDITIONAL_DATA("additionalData"), 28 | LOG_LINES("logLines"), 29 | REMOTE_JOB_URI("remoteJobUri"), 30 | ABORTED("aborted"); 31 | 32 | private final String value; 33 | 34 | private JobInfoProperty(String value) { 35 | this.value = value; 36 | } 37 | 38 | public String val() { 39 | return value; 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /jobs-api/src/main/java/de/otto/jobstore/web/representation/LogLineRepresentation.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.web.representation; 2 | 3 | import de.otto.jobstore.common.LogLine; 4 | 5 | import javax.xml.bind.annotation.XmlAccessType; 6 | import javax.xml.bind.annotation.XmlAccessorType; 7 | import javax.xml.bind.annotation.XmlRootElement; 8 | import java.util.Date; 9 | 10 | @XmlRootElement 11 | @XmlAccessorType(value = XmlAccessType.FIELD) 12 | public final class LogLineRepresentation { 13 | 14 | private String line; 15 | private Date timestamp; 16 | 17 | public LogLineRepresentation() {} 18 | 19 | private LogLineRepresentation(String line, Date timestamp) { 20 | this.line = line; 21 | this.timestamp = timestamp; 22 | } 23 | 24 | public String getLine() { 25 | return line; 26 | } 27 | 28 | public Date getTimestamp() { 29 | return timestamp; 30 | } 31 | 32 | public static LogLineRepresentation fromLogLine(LogLine ll) { 33 | return new LogLineRepresentation(ll.getLine(), ll.getTimestamp()); 34 | } 35 | 36 | @Override 37 | public String toString() { 38 | return "LogLineRepresentation{" + 39 | "'" + line + '\'' + 40 | " from " + timestamp + 41 | '}'; 42 | } 43 | 44 | } 45 | -------------------------------------------------------------------------------- /gradle/libraries.gradle: -------------------------------------------------------------------------------- 1 | project.ext.libs = [ 2 | "cobertura" : "net.sourceforge.cobertura:cobertura:2.1.1", 3 | "commonsCompress" : "org.apache.commons:commons-compress:1.0", 4 | "httpClient" : 'org.apache.httpcomponents:httpclient:4.3.4', 5 | "httpMime" : "org.apache.httpcomponents:httpmime:4.3.4", 6 | "jerseyAbdera" : "com.sun.jersey.contribs:jersey-atom-abdera:1.17.1", 7 | "jerseyApache" : "com.sun.jersey.contribs:jersey-apache-client4:1.17.1", 8 | "jerseyClient" : "com.sun.jersey:jersey-client:1.17.1", 9 | "jerseyClientJSON" : "com.sun.jersey:jersey-json:1.17.1", 10 | "jerseyCore" : "com.sun.jersey:jersey-core:1.17.1", 11 | "mockito" : "org.mockito:mockito-all:1.8.5", 12 | "mongoDb" : 'org.mongodb:mongo-java-driver:2.13.0', 13 | "slfApi" : "org.slf4j:slf4j-api:1.7.2", 14 | "slfLog4j" : "org.slf4j:slf4j-log4j12:1.7.2", 15 | "springContext" : "org.springframework:spring-context:3.1.3.RELEASE", 16 | "springCore" : "org.springframework:spring-core:3.1.3.RELEASE", 17 | "springBeans" : "org.springframework:spring-beans:3.1.3.RELEASE", 18 | "springTest" : "org.springframework:spring-test:3.1.3.RELEASE", 19 | "testng" : "org.testng:testng:6.8", 20 | "multithreadedtc" : "com.googlecode.multithreadedtc:multithreadedtc:1.01" 21 | ] 22 | -------------------------------------------------------------------------------- /jobs-core/jobs-core.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | compile libs.mongoDb 3 | compile libs.slfApi, libs.slfLog4j 4 | compile libs.jerseyClient, libs.jerseyCore, libs.jerseyClientJSON 5 | compile libs.commonsCompress 6 | compile libs.httpClient 7 | compile libs.httpMime 8 | 9 | testCompile libs.testng, libs.mockito 10 | testCompile libs.multithreadedtc 11 | testCompile libs.cobertura 12 | testCompile libs.springContext, libs.springCore, libs.springBeans, libs.springTest 13 | testCompile libs.jerseyApache 14 | } 15 | 16 | // ~~~~~~~~~~~ 17 | 18 | test { 19 | exclude "**/RemoteJobExecutorServiceIntegrationTest*" 20 | } 21 | 22 | task integrationTest(type: Test) { 23 | useTestNG() { 24 | listeners << 'org.testng.reporters.XMLReporter'// erzeuge neben dem html-report auch einen xml-report 25 | includeGroups 'integration' 26 | } 27 | description "Run integration tests" 28 | testLogging.showStandardStreams = true 29 | reports.html.destination = file('build/reports/integration') 30 | include "**/RemoteJobExecutorServiceIntegrationTest*" 31 | } 32 | // ~~~~~~~~~~~ 33 | 34 | uploadArchives { 35 | configuration = configurations.archives 36 | repositories { 37 | mavenDeployer { 38 | pom { 39 | groupId = 'de.otto' 40 | artifactId = 'jobs-core' 41 | } 42 | } 43 | } 44 | } 45 | -------------------------------------------------------------------------------- /jobs-core/src/test/java/de/otto/jobstore/common/example/StepTwoJobRunnableExample.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common.example; 2 | 3 | import de.otto.jobstore.common.AbstractLocalJobDefinition; 4 | import de.otto.jobstore.common.AbstractLocalJobRunnable; 5 | import de.otto.jobstore.common.JobDefinition; 6 | import de.otto.jobstore.common.JobExecutionContext; 7 | import de.otto.jobstore.service.exception.JobExecutionException; 8 | 9 | import java.util.Map; 10 | 11 | public final class StepTwoJobRunnableExample extends AbstractLocalJobRunnable { 12 | 13 | public static final String STEP_TWO_JOB = "STEP_TWO_JOB"; 14 | 15 | @Override 16 | public JobDefinition getJobDefinition() { 17 | return new AbstractLocalJobDefinition() { 18 | @Override 19 | public String getName() { 20 | return STEP_TWO_JOB; 21 | } 22 | 23 | @Override 24 | public long getMaxExecutionTime() { 25 | return 1000 * 60 * 20; 26 | } 27 | 28 | @Override 29 | public long getMaxIdleTime() { 30 | return 1000 * 60 * 20; 31 | } 32 | }; 33 | } 34 | 35 | /** 36 | * Job always finishes with an error 37 | */ 38 | public void execute(JobExecutionContext executionContext) throws JobExecutionException { 39 | throw new JobExecutionException("I do not want to work"); 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /jobs-core/src/test/java/de/otto/jobstore/common/JobInfoCacheTest.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common; 2 | 3 | 4 | import com.mongodb.DBObject; 5 | import de.otto.jobstore.repository.JobInfoRepository; 6 | import org.testng.annotations.Test; 7 | 8 | import static org.mockito.Mockito.*; 9 | 10 | public class JobInfoCacheTest { 11 | 12 | private JobInfoRepository jobInfoRepository = mock(JobInfoRepository.class); 13 | private final JobInfo jobInfo = new JobInfo(mock(DBObject.class)); 14 | private final String id = "id"; 15 | 16 | @Test 17 | public void testThatRepoIsHitOnlyOnce() throws Exception { 18 | reset(jobInfoRepository); 19 | when(jobInfoRepository.findById(id)).thenReturn(jobInfo); 20 | 21 | JobInfoCache jobInfoCache = new JobInfoCache(id, jobInfoRepository, 10000); 22 | Thread.sleep(100); 23 | jobInfoCache.isAborted(); 24 | Thread.sleep(100); 25 | jobInfoCache.isAborted(); 26 | 27 | verify(jobInfoRepository, times(1)).findById(id); 28 | } 29 | 30 | @Test 31 | public void testThatRepoIsHitTwice() throws Exception { 32 | reset(jobInfoRepository); 33 | when(jobInfoRepository.findById(id)).thenReturn(jobInfo); 34 | 35 | JobInfoCache jobInfoCache = new JobInfoCache(id, jobInfoRepository, 0); 36 | Thread.sleep(100); 37 | jobInfoCache.isAborted(); 38 | 39 | verify(jobInfoRepository, times(2)).findById(id); 40 | } 41 | 42 | } 43 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/common/JobDefinition.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common; 2 | 3 | public interface JobDefinition { 4 | 5 | /** 6 | * The name of the job 7 | */ 8 | String getName(); 9 | 10 | 11 | /** 12 | * The time after which a job is considered to be timed out because it has not updated anymore (in milliseconds). 13 | */ 14 | long getMaxIdleTime(); 15 | 16 | 17 | /** 18 | * The time after which a job is considered to be timed out because its total runningtime has exceeded (in milliseconds). 19 | */ 20 | long getMaxExecutionTime(); 21 | 22 | /** 23 | * The interval after which the job should be polled for new information (in milliseconds). 24 | */ 25 | long getPollingInterval(); 26 | 27 | /** 28 | * return the number of retries before job is flagged as error. 29 | */ 30 | long getMaxRetries(); 31 | 32 | /** 33 | * The interval after which the job should be checked for retry 34 | */ 35 | long getRetryInterval(); 36 | 37 | /** 38 | * Flag if the job is executed locally or remotely 39 | * 40 | * @return true - The job is executed remotely
41 | * false - The job is executed locally 42 | */ 43 | boolean isRemote(); 44 | 45 | /** 46 | * Flag if the job can be aborted 47 | * 48 | * @return true - The job can be aborted
49 | * false - The job cannot be aborted 50 | */ 51 | boolean isAbortable(); 52 | 53 | 54 | } 55 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/common/JobInfoCache.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common; 2 | 3 | import de.otto.jobstore.repository.JobInfoRepository; 4 | 5 | import java.util.Map; 6 | 7 | public class JobInfoCache { 8 | 9 | private final String id; 10 | private final JobInfoRepository jobInfoRepository; 11 | private long updateInterval; 12 | private volatile long lastUpdate = 0; 13 | private volatile JobInfo jobInfo; 14 | 15 | public JobInfoCache(String id, JobInfoRepository jobInfoRepository, long updateInterval) { 16 | this.id = id; 17 | this.jobInfoRepository = jobInfoRepository; 18 | this.jobInfo = getJobInfo(); 19 | this.updateInterval = updateInterval; 20 | } 21 | 22 | public boolean isAborted() { 23 | return getJobInfo().isAborted(); 24 | } 25 | 26 | public boolean isTimedOut() { 27 | return getJobInfo().isTimedOut(); 28 | } 29 | 30 | public Map getParameters() { 31 | return getJobInfo().getParameters(); 32 | } 33 | 34 | private JobInfo getJobInfo() { 35 | final long currentTime = System.currentTimeMillis(); 36 | if (lastUpdate + updateInterval < currentTime) { 37 | synchronized (this) { 38 | if (lastUpdate + updateInterval < currentTime) { 39 | lastUpdate = currentTime; 40 | jobInfo = jobInfoRepository.findById(id); 41 | } 42 | } 43 | } 44 | return jobInfo; 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /jobs-core/src/test/java/de/otto/jobstore/common/example/SimpleAbortableJob.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common.example; 2 | 3 | import de.otto.jobstore.common.*; 4 | import de.otto.jobstore.service.exception.JobException; 5 | import de.otto.jobstore.service.exception.JobExecutionAbortedException; 6 | import de.otto.jobstore.service.exception.JobExecutionException; 7 | 8 | public class SimpleAbortableJob extends AbstractLocalJobRunnable { 9 | 10 | 11 | @Override 12 | public JobDefinition getJobDefinition() { 13 | return new AbstractLocalJobDefinition() { 14 | @Override 15 | public String getName() { 16 | return "SimpleAbortableJob"; 17 | } 18 | 19 | @Override 20 | public long getMaxExecutionTime() { 21 | return 1000; 22 | } 23 | 24 | @Override 25 | public long getMaxIdleTime() { 26 | return 1000; 27 | } 28 | 29 | @Override 30 | public boolean isAbortable() { 31 | return true; 32 | } 33 | }; 34 | } 35 | 36 | @Override 37 | public void execute(JobExecutionContext context) throws JobException { 38 | for (int i = 0; i < 1000000; i++) { 39 | context.checkForAbort(); 40 | try { 41 | Thread.sleep(10); 42 | } catch (InterruptedException e) { 43 | throw new JobExecutionException(e.getMessage(), e); 44 | } 45 | } 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/common/JobSchedule.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common; 2 | 3 | import org.slf4j.Logger; 4 | import org.slf4j.LoggerFactory; 5 | 6 | public abstract class JobSchedule implements Runnable { 7 | 8 | private static final Logger LOGGER = LoggerFactory.getLogger(JobSchedule.class); 9 | 10 | private long count=0; 11 | private long lastExecuted=0; 12 | 13 | public long count() { 14 | return count; 15 | } 16 | 17 | @Override 18 | public synchronized void run() { 19 | try { 20 | count++; 21 | lastExecuted = System.currentTimeMillis(); 22 | LOGGER.info("schedule called on {} ({})", getName(), count); 23 | schedule(); 24 | } catch (Exception e) { 25 | LOGGER.error("error executing JobSchedule {}", getName(), e); 26 | } 27 | LOGGER.info("schedule finished on {} ({})", getName(),count); 28 | } 29 | 30 | public abstract long interval(); 31 | 32 | public abstract void schedule(); 33 | 34 | public abstract String getName(); 35 | 36 | public static JobSchedule create(final String name, final long interval, final Runnable runnable) { 37 | return new JobSchedule() { 38 | @Override 39 | public long interval() { 40 | return interval; 41 | } 42 | 43 | @Override 44 | public void schedule() { 45 | if(runnable == null) { 46 | LOGGER.warn("runnable is null"); 47 | } else { 48 | runnable.run(); 49 | } 50 | } 51 | 52 | @Override 53 | public String getName() { 54 | return name; 55 | } 56 | }; 57 | } 58 | } -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/service/SimpleJobLogger.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service; 2 | 3 | import de.otto.jobstore.common.JobInfo; 4 | import de.otto.jobstore.common.JobLogger; 5 | import de.otto.jobstore.common.RunningState; 6 | import de.otto.jobstore.repository.JobInfoRepository; 7 | 8 | import java.util.ArrayList; 9 | import java.util.List; 10 | import java.util.Map; 11 | 12 | final class SimpleJobLogger implements JobLogger { 13 | 14 | private final String jobId; 15 | private final JobInfoRepository jobInfoRepository; 16 | private List logLines; 17 | 18 | SimpleJobLogger(String jobId, JobInfoRepository jobInfoRepository) { 19 | this(jobId, jobInfoRepository, new ArrayList()); 20 | } 21 | 22 | SimpleJobLogger(String jobId, JobInfoRepository jobInfoRepository, List logLines) { 23 | this.jobId = jobId; 24 | this.jobInfoRepository = jobInfoRepository; 25 | this.logLines = logLines == null ? new ArrayList() : logLines; 26 | } 27 | 28 | @Override 29 | public void addLoggingData(String logLine) { 30 | if (logLine != null && logLine.trim().length() > 0) { 31 | jobInfoRepository.addLogLine(jobId, logLine); 32 | logLines.add(logLine); 33 | } 34 | } 35 | 36 | @Override 37 | public List getLoggingData() { 38 | return logLines; 39 | } 40 | 41 | @Override 42 | public void insertOrUpdateAdditionalData(String key, String value) { 43 | jobInfoRepository.addAdditionalData(jobId, key, value); 44 | } 45 | 46 | @Override 47 | public String getAdditionalData(String key) { 48 | final JobInfo jobInfo = jobInfoRepository.findById(jobId); 49 | Map additionalData = jobInfo.getAdditionalData(); 50 | return additionalData == null ? null : additionalData.get(key); 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /jobs-core/src/test/java/de/otto/jobstore/common/JobInfoTest.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common; 2 | 3 | import de.otto.jobstore.common.properties.JobInfoProperty; 4 | import org.bson.types.ObjectId; 5 | import org.springframework.test.util.ReflectionTestUtils; 6 | import org.springframework.util.ReflectionUtils; 7 | import org.testng.annotations.Test; 8 | 9 | import java.lang.reflect.Field; 10 | import java.util.Date; 11 | 12 | import static org.testng.AssertJUnit.assertEquals; 13 | import static org.testng.AssertJUnit.assertFalse; 14 | import static org.testng.AssertJUnit.assertTrue; 15 | 16 | public class JobInfoTest { 17 | 18 | @Test 19 | public void testIsLongTimeIdleReached() throws Exception { 20 | JobInfo jobInfo = new JobInfo("test", null, null, 60 * 1000L, 60 * 1000L, 0L); //Timeout eine Minute 21 | Date current = new Date(); 22 | Date lastModified = new Date(current.getTime() - 2 * 60 * 1000L); //Last modified vor 2 Minuten 23 | jobInfo.setLastModifiedTime(lastModified); 24 | assertTrue(jobInfo.isIdleTimeExceeded(current)); //Timeout da zuletzt vor zwei Minuten update 25 | assertFalse(jobInfo.isIdleTimeExceeded(new Date(current.getTime() - Math.round(1.5 * 60 * 1000)))); //Kein Timeout, da vor 500ms update 26 | } 27 | 28 | @Test 29 | public void testIsTimeoutReached() throws Exception { 30 | JobInfo jobInfo = new JobInfo("test", null, null, 1000L, 1000L, 0L); //Timeout eine Sekunde 31 | ReflectionTestUtils.invokeMethod(jobInfo, "addProperty", JobInfoProperty.START_TIME, jobInfo.getCreationTime()); 32 | Date startTime = jobInfo.getStartTime(); 33 | 34 | assertFalse(jobInfo.isTimedOut(new Date(startTime.getTime() + 500))); //Kein Timeout da job erst eine halbe Sekunde alt 35 | assertTrue(jobInfo.isTimedOut(new Date(startTime.getTime() + 1500))); //Kein Timeout da job erst eine eineinhalb Sekunde alt 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/common/RemoteJobStatus.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common; 2 | 3 | import javax.xml.bind.annotation.XmlElement; 4 | import javax.xml.bind.annotation.XmlRootElement; 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | 8 | @XmlRootElement 9 | public final class RemoteJobStatus { 10 | 11 | public enum Status { 12 | RUNNING, 13 | FINISHED 14 | } 15 | 16 | public Status status; 17 | 18 | @XmlElement(name = "log_lines") 19 | public List logLines = new ArrayList<>(); 20 | 21 | public RemoteJobResult result; 22 | 23 | @XmlElement(name = "finish_time") 24 | public String finishTime; 25 | 26 | public String message; 27 | 28 | public RemoteJobStatus() { 29 | } 30 | 31 | public RemoteJobStatus(Status status, List logLines, RemoteJobResult result, String finishTime) { 32 | this.status = status; 33 | if (logLines != null) { 34 | this.logLines = logLines; 35 | } 36 | this.result = result; 37 | this.finishTime = finishTime; 38 | } 39 | 40 | /** 41 | * Convenience constructor in case you are not finished yet, 42 | */ 43 | public RemoteJobStatus(Status status, List logLines, String message) { 44 | this.status = status; 45 | if (logLines != null) { 46 | this.logLines = logLines; 47 | } 48 | this.message = message; 49 | } 50 | 51 | 52 | @Override 53 | public String toString() { 54 | final StringBuilder sb = new StringBuilder(); 55 | sb.append("RemoteJobStatus"); 56 | sb.append("{status=").append(status); 57 | sb.append(", result=").append(result); 58 | sb.append(", finishTime='").append(finishTime).append('\''); 59 | sb.append(", message='").append(message).append('\''); 60 | sb.append('}'); 61 | return sb.toString(); 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /jobs-api/src/test/resources/spring/api-context.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /jobs-core/src/test/java/de/otto/jobstore/service/DirectoryBasedTarArchiveProviderTest.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service; 2 | 3 | import de.otto.jobstore.common.RemoteJob; 4 | import org.apache.commons.compress.archivers.tar.TarArchiveEntry; 5 | import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; 6 | import org.testng.annotations.Test; 7 | 8 | import java.io.ByteArrayInputStream; 9 | import java.io.ByteArrayOutputStream; 10 | import java.io.IOException; 11 | import java.io.InputStream; 12 | import java.util.ArrayList; 13 | import java.util.HashMap; 14 | import java.util.List; 15 | import java.util.zip.GZIPInputStream; 16 | 17 | import static org.testng.Assert.assertEquals; 18 | import static org.testng.Assert.assertTrue; 19 | 20 | public class DirectoryBasedTarArchiveProviderTest { 21 | 22 | @Test 23 | public void shouldTarFilesForJobName() throws Exception { 24 | TarArchiveProvider tarArchiveProvider = new DirectoryBasedTarArchiveProvider("/jobs"); 25 | 26 | RemoteJob remoteJob = new RemoteJob("demojob1", "client_id", new HashMap()); 27 | InputStream inputStream = tarArchiveProvider.getArchiveAsInputStream(remoteJob); 28 | 29 | assertArchiveContainsExecutableFiles(inputStream, "demoscript.sh", "demojob1.conf"); 30 | } 31 | 32 | private void assertArchiveContainsExecutableFiles(InputStream inputStream, String... files) throws IOException { 33 | TarArchiveInputStream tarInput = new TarArchiveInputStream(new GZIPInputStream(inputStream)); 34 | 35 | TarArchiveEntry tarEntry = tarInput.getNextTarEntry(); 36 | List fileNames = new ArrayList(); 37 | while(tarEntry != null) { 38 | fileNames.add(tarEntry.getName()); 39 | assertEquals(tarEntry.getMode(), 0100755); 40 | tarEntry = tarInput.getNextTarEntry(); 41 | } 42 | assertEquals(fileNames.size(), files.length); 43 | 44 | for (String filename : files) { 45 | assertTrue(fileNames.contains(filename)); 46 | } 47 | } 48 | } -------------------------------------------------------------------------------- /jobs-core/src/test/java/de/otto/jobstore/service/RemoteJobExecutorServiceWithScriptTransferTest.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service; 2 | 3 | import de.otto.jobstore.common.RemoteJob; 4 | import org.apache.http.client.methods.HttpPost; 5 | import org.apache.http.entity.mime.MultipartEntity; 6 | import org.testng.annotations.BeforeMethod; 7 | import org.testng.annotations.Test; 8 | 9 | import java.io.ByteArrayInputStream; 10 | import java.io.ByteArrayOutputStream; 11 | import java.io.InputStream; 12 | import java.io.OutputStream; 13 | import java.util.HashMap; 14 | import java.util.Map; 15 | 16 | import static org.testng.AssertJUnit.assertTrue; 17 | 18 | public class RemoteJobExecutorServiceWithScriptTransferTest { 19 | 20 | private static final String JOB_NAME = "jobname"; 21 | private static final String JOB_SCRIPT_DIRECTORY = "/jobs"; 22 | 23 | private RemoteJobExecutorWithScriptTransferService remoteJobExecutorService; 24 | 25 | @BeforeMethod 26 | public void setUp() { 27 | remoteJobExecutorService = new RemoteJobExecutorWithScriptTransferService("uri", new DirectoryBasedTarArchiveProvider(JOB_SCRIPT_DIRECTORY)); 28 | } 29 | 30 | @Test 31 | public void shouldCreateMultipartRequestWithScriptsAndParams() throws Exception { 32 | // When 33 | InputStream tarAsByteArray = new ByteArrayInputStream(new byte[0]); 34 | HttpPost request = remoteJobExecutorService.createRemoteExecutorMultipartRequest(createRemoteJob(), "url", tarAsByteArray); 35 | // Then 36 | MultipartEntity multipartEntity = (MultipartEntity) request.getEntity(); 37 | OutputStream os = new ByteArrayOutputStream(); 38 | multipartEntity.writeTo(os); 39 | String requestAsString = os.toString(); 40 | assertTrue(requestAsString.contains("name=\"params\"")); 41 | assertTrue(requestAsString.contains("filename=\"scripts.tar.gz\"")); 42 | } 43 | 44 | private RemoteJob createRemoteJob() { 45 | Map params = new HashMap(); 46 | return new RemoteJob(JOB_NAME, "2311", params); 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /jobs-core/src/test/java/de/otto/jobstore/common/example/StepOneJobRunnableExample.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common.example; 2 | 3 | import de.otto.jobstore.common.*; 4 | import de.otto.jobstore.service.JobService; 5 | import de.otto.jobstore.service.exception.JobException; 6 | import de.otto.jobstore.service.exception.JobExecutionException; 7 | 8 | import java.util.Map; 9 | 10 | public final class StepOneJobRunnableExample extends AbstractLocalJobRunnable { 11 | 12 | private final JobService jobService; 13 | 14 | public StepOneJobRunnableExample(JobService jobService) { 15 | this.jobService = jobService; 16 | } 17 | 18 | @Override 19 | public JobDefinition getJobDefinition() { 20 | return new AbstractLocalJobDefinition() { 21 | @Override 22 | public String getName() { 23 | return "STEP_ONE_JOB"; 24 | } 25 | 26 | @Override 27 | public long getMaxExecutionTime() { 28 | return 1000 * 60 * 10; 29 | } 30 | 31 | @Override 32 | public long getMaxIdleTime() { 33 | return 1000 * 60 * 10; 34 | } 35 | }; 36 | } 37 | 38 | /** 39 | * A very lazy job which triggers job two if done 40 | */ 41 | @Override 42 | public void execute(JobExecutionContext executionContext) throws JobException { 43 | if (JobExecutionPriority.CHECK_PRECONDITIONS.equals(executionContext.getExecutionPriority()) 44 | || jobService.listJobNames().contains(StepTwoJobRunnableExample.STEP_TWO_JOB)) { 45 | executionContext.setResultCode(ResultCode.FAILED); 46 | } 47 | try { 48 | for (int i = 0; i < 10; i++) { 49 | Thread.sleep(i * 1000); 50 | } 51 | } catch (InterruptedException e) { 52 | throw new JobExecutionException("Interrupted: " + e.getMessage()); 53 | } 54 | jobService.executeJob(StepTwoJobRunnableExample.STEP_TWO_JOB); 55 | executionContext.setResultCode(ResultCode.SUCCESSFUL); 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /jobs-core/src/test/resources/log4j.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 62 | 63 | 64 | 65 | 66 | 67 | -------------------------------------------------------------------------------- /jobs-core/src/test/resources/spring/jobs-context.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | -------------------------------------------------------------------------------- /jobs-api/src/test/java/de/otto/jobstore/web/representation/JobInfoRepresentationTest.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.web.representation; 2 | 3 | import de.otto.jobstore.common.JobExecutionPriority; 4 | import de.otto.jobstore.common.JobInfo; 5 | import de.otto.jobstore.common.LogLine; 6 | import de.otto.jobstore.common.RunningState; 7 | import org.testng.annotations.Test; 8 | 9 | import java.util.Date; 10 | import java.util.HashMap; 11 | 12 | import static org.testng.AssertJUnit.assertEquals; 13 | 14 | public class JobInfoRepresentationTest { 15 | 16 | @Test 17 | public void testFromJobInfo() throws Exception { 18 | JobInfo jobInfo = new JobInfo("foo", "host", "thread", 1234L, 1234L, 0L, RunningState.RUNNING, JobExecutionPriority.IGNORE_PRECONDITIONS, new HashMap()); 19 | jobInfo.appendLogLine(new LogLine("line1", new Date())); 20 | jobInfo.appendLogLine(new LogLine("line2", new Date())); 21 | jobInfo.putAdditionalData("key1", "value1"); 22 | jobInfo.putAdditionalData("key2", "value1"); 23 | JobInfoRepresentation jobInfoRep = JobInfoRepresentation.fromJobInfo(jobInfo, 100); 24 | 25 | assertEquals("foo", jobInfoRep.getName()); 26 | assertEquals("host", jobInfoRep.getHost()); 27 | assertEquals("thread", jobInfoRep.getThread()); 28 | assertEquals(2, jobInfoRep.getLogLines().size()); 29 | assertEquals(2, jobInfoRep.getAdditionalData().size()); 30 | } 31 | 32 | @Test 33 | public void testCutoffLogLines() throws Exception { 34 | JobInfo jobInfo = new JobInfo("foo", "host", "thread", 1234L, 1234L, 0L, RunningState.RUNNING, JobExecutionPriority.IGNORE_PRECONDITIONS, new HashMap()); 35 | for (int i = 0; i < 50; i++) { 36 | jobInfo.appendLogLine(new LogLine("line " + i, new Date())); 37 | } 38 | jobInfo.putAdditionalData("key1", "value1"); 39 | jobInfo.putAdditionalData("key2", "value1"); 40 | JobInfoRepresentation jobInfoRep = JobInfoRepresentation.fromJobInfo(jobInfo, 20); 41 | assertEquals(20, jobInfoRep.getLogLines().size()); 42 | assertEquals("line 30", jobInfoRep.getLogLines().get(0).getLine()); 43 | assertEquals("line 49", jobInfoRep.getLogLines().get(19).getLine()); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/common/RemoteJob.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common; 2 | 3 | import org.codehaus.jettison.json.JSONException; 4 | import org.codehaus.jettison.json.JSONObject; 5 | 6 | import java.io.Serializable; 7 | import java.util.Map; 8 | 9 | public final class RemoteJob implements Serializable { 10 | 11 | public String name; 12 | public String client_id; 13 | public Map parameters; 14 | 15 | public RemoteJob(String name, String client_id, Map parameters) { 16 | this.name = name; 17 | this.client_id = client_id; 18 | this.parameters = parameters; 19 | } 20 | 21 | public JSONObject toJsonObject() throws JSONException { 22 | final JSONObject obj = new JSONObject(); 23 | obj.put("name", name); 24 | obj.put("client_id", client_id); 25 | final JSONObject params = new JSONObject(); 26 | for (String paramName : parameters.keySet()) { 27 | params.put(paramName, parameters.get(paramName)); 28 | } 29 | obj.put("parameters", params); 30 | return obj; 31 | } 32 | 33 | @Override 34 | public String toString() { 35 | final StringBuilder sb = new StringBuilder(); 36 | sb.append("RemoteJob"); 37 | sb.append("{name='").append(name).append('\''); 38 | sb.append(", client_id='").append(client_id).append('\''); 39 | sb.append(", parameters=").append(parameters); 40 | sb.append('}'); 41 | return sb.toString(); 42 | } 43 | 44 | @Override 45 | public boolean equals(Object o) { 46 | if (this == o) return true; 47 | if (o == null || getClass() != o.getClass()) return false; 48 | 49 | RemoteJob remoteJob = (RemoteJob) o; 50 | 51 | if (name != null ? !name.equals(remoteJob.name) : remoteJob.name != null) return false; 52 | if (parameters != null ? !parameters.equals(remoteJob.parameters) : remoteJob.parameters != null) return false; 53 | 54 | return true; 55 | } 56 | 57 | @Override 58 | public int hashCode() { 59 | int result = name != null ? name.hashCode() : 0; 60 | result = 31 * result + (parameters != null ? parameters.hashCode() : 0); 61 | return result; 62 | } 63 | 64 | } 65 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/common/JobRunnable.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common; 2 | 3 | import de.otto.jobstore.service.exception.JobException; 4 | 5 | import java.util.Map; 6 | 7 | /** 8 | * A job to be executed by a JobService {@link de.otto.jobstore.service.JobService} 9 | */ 10 | public interface JobRunnable { 11 | 12 | /** 13 | * Returns the defining data of this job 14 | */ 15 | JobDefinition getJobDefinition(); 16 | 17 | /** 18 | * Returns the parameters being used to execute the job. 19 | */ 20 | Map getParameters(); 21 | 22 | /** 23 | * Asks for the current remote status. Is only called on remote jobs. 24 | */ 25 | RemoteJobStatus getRemoteStatus(JobExecutionContext context); 26 | 27 | /** 28 | * Checks preconditions whether job should be executed. 29 | * 30 | * @return true - The job can now be executed
31 | * false - The job should not be executed 32 | */ 33 | boolean prepare(JobExecutionContext context) throws JobException; 34 | 35 | /** 36 | * Executes the job. 37 | * 38 | * @param context The context in which this job is executed 39 | * @throws de.otto.jobstore.service.exception.JobExecutionException Thrown if the execution of the job failed 40 | */ 41 | void execute(JobExecutionContext context) throws JobException; 42 | 43 | /** 44 | * This method is called after the job is executed successfully. 45 | */ 46 | void afterExecution(JobExecutionContext context) throws JobException; 47 | 48 | /** 49 | * Extension point for side effects after an exception occurred. 50 | * Clients decide on rethrowing exceptions. 51 | */ 52 | OnException onException(JobExecutionContext context, Exception e, State state); 53 | 54 | enum State { 55 | PREPARE, EXECUTE, AFTER_EXECUTION 56 | } 57 | 58 | interface OnException { 59 | /** 60 | * Do nothing if successfully recovered from the specific exception. Rethrow if not. 61 | */ 62 | void doThrow() throws JobException; 63 | 64 | /** 65 | * @return whether the job has successfully recovered from the specific exception. 66 | */ 67 | boolean hasRecovered(); 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /jobs-core/src/test/java/de/otto/jobstore/common/example/SimpleJobRunnableExample.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common.example; 2 | 3 | import de.otto.jobstore.common.*; 4 | import de.otto.jobstore.service.exception.JobExecutionException; 5 | 6 | import java.util.Calendar; 7 | import java.util.GregorianCalendar; 8 | import java.util.Map; 9 | import java.util.Random; 10 | 11 | public final class SimpleJobRunnableExample extends AbstractLocalJobRunnable { 12 | 13 | @Override 14 | public JobDefinition getJobDefinition() { 15 | return new AbstractLocalJobDefinition() { 16 | @Override 17 | public String getName() { 18 | return "SimpleJobRunnableExampleToBeUsed"; 19 | } 20 | 21 | @Override 22 | public long getMaxExecutionTime() { 23 | return 1000 * 60 * 10; 24 | } 25 | 26 | @Override 27 | public long getMaxIdleTime() { 28 | return 1000 * 60 * 10; 29 | } 30 | }; 31 | } 32 | 33 | @Override 34 | public Map getParameters() { 35 | return null; 36 | } 37 | 38 | /** 39 | * Computes 100 numbers by dividing each number from 0 to 99 by a random number 40 | * 41 | * @param executionContext The context in which this job is executed 42 | * @throws JobExecutionException An exception is thrown if the random number is zero 43 | * and would thus cause a division by zero error 44 | */ 45 | @Override 46 | public void execute(JobExecutionContext executionContext) throws JobExecutionException { 47 | if (JobExecutionPriority.CHECK_PRECONDITIONS.equals(executionContext.getExecutionPriority()) 48 | || new GregorianCalendar().get(Calendar.DAY_OF_WEEK) != Calendar.SUNDAY) { 49 | executionContext.setResultCode(ResultCode.FAILED); 50 | } 51 | Random r = new Random(); 52 | for (int i = 0; i < 100; i++) { 53 | final int randomNumber = r.nextInt(); 54 | if (randomNumber == 0) { 55 | throw new IllegalArgumentException("Division by Zero"); 56 | } else { 57 | executionContext.getJobLogger().addLoggingData("Computed the number: " + i / randomNumber); 58 | } 59 | } 60 | executionContext.setResultCode(ResultCode.SUCCESSFUL); 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/service/RemoteJobExecutorStatusRetriever.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service; 2 | 3 | import com.sun.jersey.api.client.Client; 4 | import com.sun.jersey.api.client.ClientHandlerException; 5 | import com.sun.jersey.api.client.ClientResponse; 6 | import com.sun.jersey.api.client.UniformInterfaceException; 7 | import de.otto.jobstore.common.RemoteJobStatus; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import javax.ws.rs.core.MediaType; 12 | import java.net.URI; 13 | 14 | public class RemoteJobExecutorStatusRetriever { 15 | 16 | private static final Logger LOGGER = LoggerFactory.getLogger(RemoteJobExecutorStatusRetriever.class); 17 | private final Client client; 18 | 19 | public RemoteJobExecutorStatusRetriever(Client client) { 20 | this.client = client; 21 | } 22 | 23 | public RemoteJobStatus getStatus(final URI jobUri) { 24 | try { 25 | final ClientResponse response = client.resource(jobUri.toString()). 26 | accept(MediaType.APPLICATION_JSON).header("Connection", "close").get(ClientResponse.class); 27 | if (response.getStatus() == 200) { 28 | final RemoteJobStatus status = response.getEntity(RemoteJobStatus.class); 29 | LOGGER.info("ltag=RemoteJobExecutorService.getStatus Response from server: {}", status); 30 | return status; 31 | } else { 32 | response.close(); 33 | } 34 | LOGGER.warn("Received unexpected status code {} when trying to retrieve status for remote job from: {}", response.getStatus(), jobUri); 35 | } catch (UniformInterfaceException | ClientHandlerException e) { 36 | LOGGER.warn("Problem while trying to retrieve status for remote job from: {}", jobUri, e); 37 | } 38 | return null; // TODO: this should be avoided 39 | } 40 | 41 | public boolean isAlive(String jobExecutorUri) { 42 | try { 43 | final ClientResponse response = client.resource(jobExecutorUri).header("Connection", "close").get(ClientResponse.class); 44 | final boolean alive = response.getStatus() == 200; 45 | response.close(); 46 | return alive; 47 | } catch (UniformInterfaceException | ClientHandlerException e) { 48 | LOGGER.warn("Remote Job Executor is not available from: {}", jobExecutorUri, e); 49 | } 50 | return false; 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /jobs-core/src/test/java/de/otto/jobstore/service/JobInfoServiceTest.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service; 2 | 3 | 4 | import de.otto.jobstore.common.JobInfo; 5 | import de.otto.jobstore.common.ResultCode; 6 | import de.otto.jobstore.repository.JobInfoRepository; 7 | import org.testng.annotations.BeforeMethod; 8 | import org.testng.annotations.Test; 9 | 10 | import java.util.Arrays; 11 | import java.util.EnumSet; 12 | import java.util.List; 13 | 14 | import static org.mockito.Mockito.mock; 15 | import static org.mockito.Mockito.when; 16 | import static org.testng.AssertJUnit.assertEquals; 17 | 18 | public class JobInfoServiceTest { 19 | 20 | private JobInfoService jobInfoService; 21 | private JobInfoRepository jobInfoRepository; 22 | 23 | @BeforeMethod 24 | public void setUp() throws Exception { 25 | jobInfoRepository = mock(JobInfoRepository.class); 26 | jobInfoService = new JobInfoService(jobInfoRepository); 27 | } 28 | 29 | @Test 30 | public void testGetMostRecentExecuted() throws Exception { 31 | when(jobInfoRepository.findMostRecentFinished("test")).thenReturn(new JobInfo("test", "host", "thread", 1234L, 1234L, 0L)); 32 | 33 | JobInfo jobInfo = jobInfoService.getMostRecentExecuted("test"); 34 | assertEquals("test", jobInfo.getName()); 35 | assertEquals(Long.valueOf(1234L), jobInfo.getMaxIdleTime()); 36 | } 37 | 38 | @Test 39 | public void testGetMostRecentSuccessful() throws Exception { 40 | when(jobInfoRepository.findMostRecentByNameAndResultState("test", 41 | EnumSet.of(ResultCode.SUCCESSFUL))).thenReturn(new JobInfo("test", "host", "thread", 1234L, 1234L, 0L)); 42 | 43 | JobInfo jobInfo = jobInfoService.getMostRecentSuccessful("test"); 44 | assertEquals("test", jobInfo.getName()); 45 | assertEquals(Long.valueOf(1234L), jobInfo.getMaxIdleTime()); 46 | } 47 | 48 | @Test 49 | public void testGetMostRecentExecutedList() throws Exception { 50 | when(jobInfoRepository.distinctJobNames()).thenReturn(Arrays.asList("test", "test2")); 51 | when(jobInfoRepository.findMostRecentFinished("test")).thenReturn(new JobInfo("test", "host", "thread", 1234L, 1234L, 0L)); 52 | when(jobInfoRepository.findMostRecentFinished("test2")).thenReturn(null); 53 | 54 | List jobInfoList = jobInfoService.getMostRecentExecuted(); 55 | assertEquals(1, jobInfoList.size()); 56 | assertEquals("test", jobInfoList.get(0).getName()); 57 | } 58 | } 59 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/common/AbstractLocalJobRunnable.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common; 2 | 3 | import de.otto.jobstore.service.exception.JobException; 4 | 5 | import java.util.Collections; 6 | import java.util.Map; 7 | 8 | public abstract class AbstractLocalJobRunnable implements JobRunnable { 9 | 10 | @Override 11 | public final RemoteJobStatus getRemoteStatus(JobExecutionContext context) { 12 | throw new UnsupportedOperationException(); 13 | } 14 | 15 | @Override 16 | public Map getParameters() { 17 | return Collections.emptyMap(); 18 | } 19 | 20 | /** 21 | * By default returns true. If an exception occurs, returns false. 22 | */ 23 | @Override 24 | public boolean prepare(JobExecutionContext context) { 25 | try { 26 | return doPrepare(context); 27 | } catch (Exception e) { 28 | return onException(context, e, State.PREPARE).hasRecovered(); 29 | } 30 | } 31 | 32 | @Override 33 | public void execute(JobExecutionContext context) throws JobException { 34 | try { 35 | doExecute(context); 36 | } catch (Exception e) { 37 | onException(context, e, State.EXECUTE).doThrow(); 38 | } 39 | } 40 | 41 | /** 42 | * Template method for de.otto.jobstore.common.AbstractLocalJobRunnable#execute(de.otto.jobstore.common.JobExecutionContext) 43 | */ 44 | protected void doExecute(JobExecutionContext context) throws JobException { 45 | } 46 | 47 | /** 48 | * Template method of de.otto.jobstore.common.AbstractLocalJobRunnable#prepare(de.otto.jobstore.common.JobExecutionContext) 49 | */ 50 | protected boolean doPrepare(JobExecutionContext context) throws JobException { 51 | return true; 52 | } 53 | 54 | /** 55 | * Implementation might want to set the {@link JobExecutionContext#resultCode} 56 | */ 57 | @Override 58 | public void afterExecution(JobExecutionContext context) throws JobException { 59 | try { 60 | doAfterExecution(context); 61 | } catch (Exception e) { 62 | onException(context, e, State.AFTER_EXECUTION).doThrow(); 63 | } 64 | } 65 | 66 | /** 67 | * Template method for de.otto.jobstore.common.AbstractLocalJobRunnable#afterExecution(de.otto.jobstore.common.JobExecutionContext) 68 | */ 69 | protected void doAfterExecution(JobExecutionContext context) throws JobException { 70 | } 71 | 72 | @Override 73 | public OnException onException(JobExecutionContext context, Exception e, State state) { 74 | return new DefaultOnException(e); 75 | } 76 | } 77 | -------------------------------------------------------------------------------- /gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 12 | set DEFAULT_JVM_OPTS= 13 | 14 | set DIRNAME=%~dp0 15 | if "%DIRNAME%" == "" set DIRNAME=. 16 | set APP_BASE_NAME=%~n0 17 | set APP_HOME=%DIRNAME% 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windowz variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | if "%@eval[2+2]" == "4" goto 4NT_args 53 | 54 | :win9xME_args 55 | @rem Slurp the command line arguments. 56 | set CMD_LINE_ARGS= 57 | set _SKIP=2 58 | 59 | :win9xME_args_slurp 60 | if "x%~1" == "x" goto execute 61 | 62 | set CMD_LINE_ARGS=%* 63 | goto execute 64 | 65 | :4NT_args 66 | @rem Get arguments from the 4NT Shell from JP Software 67 | set CMD_LINE_ARGS=%$ 68 | 69 | :execute 70 | @rem Setup the command line 71 | 72 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 73 | 74 | @rem Execute Gradle 75 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 76 | 77 | :end 78 | @rem End local scope for the variables with windows NT shell 79 | if "%ERRORLEVEL%"=="0" goto mainEnd 80 | 81 | :fail 82 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 83 | rem the _cmd.exe /c_ return code! 84 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 85 | exit /b 1 86 | 87 | :mainEnd 88 | if "%OS%"=="Windows_NT" endlocal 89 | 90 | :omega 91 | -------------------------------------------------------------------------------- /jobs-core/src/test/java/de/otto/jobstore/service/JobServiceNotActiveTest.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service; 2 | 3 | import de.otto.jobstore.TestSetup; 4 | import de.otto.jobstore.common.ActiveChecker; 5 | import de.otto.jobstore.common.RunningState; 6 | import de.otto.jobstore.common.StoredJobDefinition; 7 | import de.otto.jobstore.repository.JobDefinitionRepository; 8 | import de.otto.jobstore.repository.JobInfoRepository; 9 | import de.otto.jobstore.service.exception.JobServiceNotActiveException; 10 | import org.testng.annotations.BeforeMethod; 11 | import org.testng.annotations.Test; 12 | 13 | import static org.mockito.Mockito.*; 14 | 15 | public class JobServiceNotActiveTest { 16 | 17 | private JobService jobService; 18 | private JobInfoRepository jobInfoRepository; 19 | private JobDefinitionRepository jobDefinitionRepository; 20 | private static final String JOB_NAME_01 = "test"; 21 | 22 | private class AlwaysFalseActiveChecker implements ActiveChecker { 23 | @Override 24 | public boolean isActive() { 25 | return false; 26 | } 27 | } 28 | 29 | @BeforeMethod 30 | public void setUp() throws Exception { 31 | jobInfoRepository = mock(JobInfoRepository.class); 32 | jobDefinitionRepository = mock(JobDefinitionRepository.class); 33 | when(jobDefinitionRepository.find(StoredJobDefinition.JOB_EXEC_SEMAPHORE.getName())).thenReturn(StoredJobDefinition.JOB_EXEC_SEMAPHORE); 34 | jobService = new JobService(jobDefinitionRepository, jobInfoRepository, new AlwaysFalseActiveChecker()); 35 | jobService.registerJob(TestSetup.localJobRunnable(JOB_NAME_01, 1, 1)); 36 | } 37 | 38 | @Test(expectedExceptions = JobServiceNotActiveException.class) 39 | public void doesNotExecuteJobIfNotActive() throws Exception { 40 | jobService.executeJob(JOB_NAME_01); 41 | } 42 | 43 | @Test 44 | public void doesNotExecuteQueuedJobIfNotActive() throws Exception { 45 | jobService.executeQueuedJobs(); 46 | 47 | verify(jobDefinitionRepository, never()).find(anyString()); 48 | } 49 | 50 | @Test 51 | public void doesNotPollRemoteJobsIfNotActive() throws Exception { 52 | jobService.pollRemoteJobs(); 53 | 54 | verify(jobDefinitionRepository, never()).find(anyString()); 55 | } 56 | 57 | @Test 58 | public void doesNotRetryFailedJobsIfNotActive() throws Exception { 59 | jobService.retryFailedJobs(); 60 | 61 | verify(jobInfoRepository, never()).findMostRecentFinished(JOB_NAME_01); 62 | } 63 | 64 | @Test 65 | public void doesNotCleanupTimedOutJobsIfNotActive() throws Exception { 66 | jobService.cleanupTimedOutJobs(); 67 | 68 | verify(jobInfoRepository, never()).cleanupTimedOutJobs(); 69 | } 70 | 71 | } 72 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/common/JobExecutionContext.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common; 2 | 3 | import de.otto.jobstore.service.exception.JobExecutionAbortedException; 4 | import de.otto.jobstore.service.exception.JobExecutionTimeoutException; 5 | 6 | import java.util.Map; 7 | 8 | public class JobExecutionContext { 9 | 10 | private final String id; 11 | private final JobLogger jobLogger; 12 | private final JobExecutionPriority executionPriority; 13 | private final JobDefinition jobDefinition; 14 | private final JobInfoCache jobInfoCache; 15 | 16 | private volatile ResultCode resultCode = ResultCode.SUCCESSFUL; 17 | private String resultMessage; 18 | 19 | public JobExecutionContext(String id, JobLogger jobLogger, JobInfoCache jobInfoCache, JobExecutionPriority executionPriority, JobDefinition jobDefinition) { 20 | this.id = id; 21 | this.jobLogger = jobLogger; 22 | this.jobInfoCache = jobInfoCache; 23 | this.executionPriority = executionPriority; 24 | this.jobDefinition = jobDefinition; 25 | } 26 | 27 | public JobLogger getJobLogger() { 28 | return jobLogger; 29 | } 30 | 31 | public JobExecutionPriority getExecutionPriority() { 32 | return executionPriority; 33 | } 34 | 35 | public void setResultCode(ResultCode resultCode) { 36 | this.resultCode = resultCode; 37 | } 38 | 39 | public ResultCode getResultCode() { 40 | return resultCode; 41 | } 42 | 43 | public String getId() { 44 | return id; 45 | } 46 | 47 | public String getResultMessage() { 48 | return resultMessage; 49 | } 50 | 51 | public void setResultMessage(String resultMessage) { 52 | this.resultMessage = resultMessage; 53 | } 54 | 55 | /** 56 | * checks if conditions are met to abort the job, either an external abort request or the job reached its timeout condition 57 | * @throws JobExecutionAbortedException 58 | * @throws JobExecutionTimeoutException 59 | */ 60 | public void checkForAbort() throws JobExecutionAbortedException, JobExecutionTimeoutException { 61 | if(jobInfoCache.isAborted()) { 62 | throw JobExecutionAbortedException.fromJobName(getId()); 63 | } 64 | if(jobInfoCache.isTimedOut()) { 65 | throw JobExecutionTimeoutException.fromJobName(getId()); 66 | } 67 | 68 | } 69 | 70 | public Map getParameters() { 71 | return jobInfoCache.getParameters(); 72 | } 73 | 74 | @Override 75 | public String toString() { 76 | return "JobExecutionContext{" + 77 | "id='" + id + '\'' + 78 | ", resultCode=" + resultCode + 79 | ", resultMessage='" + resultMessage + '\'' + 80 | '}'; 81 | } 82 | 83 | public JobDefinition getJobDefinition() { 84 | return jobDefinition; 85 | } 86 | } 87 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Job-Framework using MongoDB 2 | 3 | ## Usecase and Workmode 4 | The framework handles the execution of jobs (local or remote) in a multi-node environment and insures that at a certain point in time only one job of a kind is executed. The node that executes a job is determined at runtime and may change each execution. It is also possible to define constraints in order to disallow execution of two different kinds of jobs at the same time. 5 | 6 | Information collected during the execution of a job may be saved and used for monitoring. The result state of a job is captured as well. The definition of timeouts allows to detect jobs which did not finish in their expected time frame. 7 | 8 | Certain jobs may be deactivated or the whole processing may be deactivated online. Additionally certain servers in a cluster with the same database may be excluded from job execution. 9 | 10 | ## Documentation 11 | In order to use the framework you have to implement a JobRunnable interface for each job which defines their properties and execution logic. For every job executed information on it are stored in the connected MongoDB which is also used as the semaphore to only allow one job to be executed and/or queued. 12 | 13 | ### Jobservice 14 | The Jobservice interface allows the user to control registration and execution of jobs. The executeJob methods returns the id of the executed (or queued) jobs with which the status of the job can be queried. If a job could not be executed or queued an appropriate JobException is thrown. 15 | 16 | When starting a job an execution priority can be supplied. The effect of the execution priority is displayed in the table below. 17 | 18 | | Priority | A job is queued | A job is running | No job running or queued | 19 | | --------------- | ---------------| ---------------| ---------------| 20 | | lower | Job with current priority will be queued | Job with current priority will be queued | Job with current priority will be executed | 21 | | equal | JobAlreadyQueuedException | JobAlreadyQueuedException | Job with current priority will be executed | 22 | | higher | JobAlreadyQueuedException | JobAlreadyQueuedException | Job with current priority will be executed | 23 | 24 | ### JobRunnable 25 | The JobRunnable interface defines the properties of a job, its execute method is executed when running the job. Before execution the prepare method is called which may contain initialization steps necessary for the job execution or check if the execution is necessary. The execute method is thus only called if the prepare method returns successfully. The afterExecution method is called after the successful execution of the execute method to allow execution of additional logic. 26 | 27 | ### JobExecutionContext 28 | The execution context is passed to the prepage, execute and afterExecution method and may contain context information needed during the lifecycle of a job. 29 | 30 | ### JobExecutionPriority 31 | The execution priority defines the priority with which a job is executed. It influences the behavior of the executeJob method in the JobService and also the prepare method as it should always return true for the IGNORE_PRECONDITIONS und FORCE_EXECUTION priority. 32 | 33 | ### JobInfoService 34 | May be used to query information on jobs. 35 | 36 | ### JobInfo 37 | Contains information about currently running and past jobs. -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/service/JobExecutionRunnable.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service; 2 | 3 | import de.otto.jobstore.common.*; 4 | import de.otto.jobstore.repository.JobDefinitionRepository; 5 | import de.otto.jobstore.repository.JobInfoRepository; 6 | import de.otto.jobstore.service.exception.JobExecutionAbortedException; 7 | import de.otto.jobstore.service.exception.JobExecutionTimeoutException; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.util.Date; 12 | 13 | final class JobExecutionRunnable implements Runnable { 14 | 15 | private static final Logger LOGGER = LoggerFactory.getLogger(JobExecutionRunnable.class); 16 | 17 | final JobRunnable jobRunnable; 18 | final JobInfoRepository jobInfoRepository; 19 | final JobDefinitionRepository jobDefinitionRepository; 20 | final JobExecutionContext context; 21 | 22 | JobExecutionRunnable(JobRunnable jobRunnable, JobInfoRepository jobInfoRepository, JobDefinitionRepository jobDefinitionRepository, JobExecutionContext context) { 23 | this.jobRunnable = jobRunnable; 24 | this.jobInfoRepository = jobInfoRepository; 25 | this.jobDefinitionRepository = jobDefinitionRepository; 26 | this.context = context; 27 | 28 | } 29 | 30 | @Override 31 | public void run() { 32 | final JobDefinition jobDefinition = jobRunnable.getJobDefinition(); 33 | final String name = jobDefinition.getName(); 34 | try { 35 | LOGGER.info("ltag=JobService.JobExecutionRunnable.run start jobName={} jobId={}", name, context.getId()); 36 | if (jobRunnable.prepare(context)) { 37 | // add parameters coming from JobRunnable directly before execution, keep old ones! 38 | jobInfoRepository.appendParameters(context.getId(), jobRunnable.getParameters()); 39 | jobRunnable.execute(context); 40 | if (!jobDefinition.isRemote()) { 41 | LOGGER.info("ltag=JobService.JobExecutionRunnable.run finished jobName={} jobId={}", name, context.getId()); 42 | jobRunnable.afterExecution(context); 43 | jobInfoRepository.markAsFinished(context.getId(), context.getResultCode(), context.getResultMessage()); 44 | } 45 | } else { 46 | LOGGER.info("ltag=JobService.JobExecutionRunnable.run skipped jobName={} jobId={}", name, context.getId()); 47 | jobInfoRepository.remove(context.getId()); 48 | jobDefinitionRepository.setLastNotExecuted(name, new Date()); 49 | } 50 | } catch (JobExecutionAbortedException e) { 51 | LOGGER.warn("ltag=JobService.JobExecutionRunnable.run jobName=" + name + " jobId=" + context.getId() + " was aborted"); 52 | jobInfoRepository.markAsFinished(context.getId(), ResultCode.ABORTED); 53 | } catch (JobExecutionTimeoutException e) { 54 | LOGGER.warn("ltag=JobService.JobExecutionRunnable.run jobName=" + name + " jobId=" + context.getId() + " timed out"); 55 | jobInfoRepository.markAsFinished(context.getId(), ResultCode.TIMED_OUT); 56 | } catch (Exception e) { 57 | LOGGER.error("ltag=JobService.JobExecutionRunnable.run jobName=" + name + " jobId=" + context.getId() + " failed: " + e.getMessage(), e); 58 | jobInfoRepository.markAsFinished(context.getId(), e); 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/common/StoredJobDefinition.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common; 2 | 3 | 4 | import com.mongodb.DBObject; 5 | import de.otto.jobstore.common.properties.JobDefinitionProperty; 6 | 7 | import java.util.Date; 8 | import java.util.concurrent.TimeUnit; 9 | 10 | public final class StoredJobDefinition extends AbstractItem implements JobDefinition { 11 | 12 | public static final StoredJobDefinition JOB_EXEC_SEMAPHORE = new StoredJobDefinition("ALL_JOBS", 0, 0, 0, 0, 0, false, false); 13 | 14 | private static final long serialVersionUID = 2454224305569320787L; 15 | 16 | public StoredJobDefinition(DBObject dbObject) { 17 | super(dbObject); 18 | } 19 | 20 | public StoredJobDefinition(String name, long maxIdleTime, long maxExecutionTime, long pollingInterval, long maxRetries, long retryInterval, boolean remote, boolean abortable) { 21 | addProperty(JobDefinitionProperty.NAME, name); 22 | addProperty(JobDefinitionProperty.MAX_IDLE_TIME, maxIdleTime); 23 | addProperty(JobDefinitionProperty.MAX_EXECUTION_TIME, maxExecutionTime); 24 | addProperty(JobDefinitionProperty.POLLING_INTERVAL, pollingInterval); 25 | addProperty(JobDefinitionProperty.MAX_RETRIES, maxRetries); 26 | addProperty(JobDefinitionProperty.RETRY_INTERVAL, retryInterval); 27 | addProperty(JobDefinitionProperty.REMOTE, remote); 28 | addProperty(JobDefinitionProperty.ABORTABLE, abortable); 29 | } 30 | 31 | public StoredJobDefinition(JobDefinition jd) { 32 | this(jd.getName(), jd.getMaxIdleTime(), jd.getMaxExecutionTime(), jd.getPollingInterval(), jd.getMaxRetries(), jd.getRetryInterval(), jd.isRemote(), jd.isAbortable()); 33 | } 34 | 35 | public String getName() { 36 | return getProperty(JobDefinitionProperty.NAME); 37 | } 38 | 39 | 40 | 41 | public long getMaxIdleTime() { 42 | return getProperty(JobDefinitionProperty.MAX_IDLE_TIME); 43 | } 44 | 45 | public long getMaxExecutionTime() { 46 | Long maxExecutionTime = getProperty(JobDefinitionProperty.MAX_EXECUTION_TIME); 47 | if(maxExecutionTime == null){ 48 | maxExecutionTime = TimeUnit.HOURS.toMillis(2); 49 | } 50 | return maxExecutionTime; 51 | } 52 | 53 | public long getPollingInterval() { 54 | return getProperty(JobDefinitionProperty.POLLING_INTERVAL); 55 | } 56 | 57 | public long getMaxRetries() { 58 | return getProperty(JobDefinitionProperty.MAX_RETRIES); 59 | } 60 | 61 | public long getRetryInterval() { 62 | return getProperty(JobDefinitionProperty.RETRY_INTERVAL); 63 | } 64 | 65 | public boolean isRemote() { 66 | final Boolean remote = getProperty(JobDefinitionProperty.REMOTE); 67 | return remote == null ? false : remote; 68 | } 69 | 70 | public boolean isAbortable() { 71 | final Boolean abortable = getProperty(JobDefinitionProperty.ABORTABLE); 72 | return abortable == null ? false : abortable; 73 | } 74 | 75 | public void setDisabled(boolean disabled) { 76 | addProperty(JobDefinitionProperty.DISABLED, disabled); 77 | } 78 | 79 | public boolean isDisabled() { 80 | final Boolean disabled = getProperty(JobDefinitionProperty.DISABLED); 81 | return disabled == null ? false : disabled; 82 | } 83 | 84 | public Date getLastNotExecuted() { 85 | return getProperty(JobDefinitionProperty.LAST_NOT_EXECUTED); 86 | } 87 | 88 | } 89 | -------------------------------------------------------------------------------- /jobs-api/src/test/java/de/otto/jobstore/web/JobInfoResourceIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.web; 2 | 3 | 4 | import de.otto.jobstore.common.*; 5 | import de.otto.jobstore.repository.JobDefinitionRepository; 6 | import de.otto.jobstore.service.JobInfoService; 7 | import de.otto.jobstore.service.JobService; 8 | import de.otto.jobstore.service.exception.JobException; 9 | import org.springframework.test.context.ContextConfiguration; 10 | import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; 11 | import org.testng.annotations.BeforeMethod; 12 | import org.testng.annotations.Test; 13 | 14 | import javax.annotation.Resource; 15 | import javax.ws.rs.core.Response; 16 | 17 | import static org.testng.AssertJUnit.assertEquals; 18 | import static org.testng.AssertJUnit.assertTrue; 19 | 20 | @ContextConfiguration(locations = {"classpath:spring/api-context.xml"}) 21 | public class JobInfoResourceIntegrationTest extends AbstractTestNGSpringContextTests { 22 | 23 | @Resource 24 | private JobInfoResource jobInfoResource; 25 | 26 | @Resource 27 | private JobService jobService; 28 | 29 | @Resource 30 | private JobInfoService jobInfoService; 31 | 32 | @BeforeMethod 33 | public void setUp() throws Exception { 34 | jobService.clean(); 35 | } 36 | 37 | @Test 38 | public void testThatAbortedJobHasAbortFlag() throws Exception { 39 | JobRunnable jobRunnable = mockRunnable(true); 40 | jobService.registerJob(jobRunnable); 41 | final String id = jobService.executeJob(jobRunnable.getJobDefinition().getName()); 42 | jobService.shutdownJobExecutorService(true); 43 | 44 | final Response response = jobInfoResource.abortJob(jobRunnable.getJobDefinition().getName(), id); 45 | assertEquals(200, response.getStatus()); 46 | final JobInfo jobInfo = jobInfoService.getById(id); 47 | assertTrue(jobInfo.isAborted()); 48 | } 49 | 50 | @Test 51 | public void testAbortingNotAbortableJobResultsInError() throws Exception { 52 | JobRunnable jobRunnable = mockRunnable(false); 53 | jobService.registerJob(jobRunnable); 54 | final String id = jobService.executeJob(jobRunnable.getJobDefinition().getName()); 55 | 56 | final Response response = jobInfoResource.abortJob(jobRunnable.getJobDefinition().getName(), id); 57 | assertEquals(403, response.getStatus()); 58 | } 59 | 60 | private AbstractLocalJobRunnable mockRunnable(final boolean abortable) { 61 | return new AbstractLocalJobRunnable() { 62 | @Override 63 | public JobDefinition getJobDefinition() { 64 | return new AbstractLocalJobDefinition() { 65 | @Override 66 | public String getName() { 67 | return "hans"; 68 | } 69 | 70 | @Override 71 | public long getMaxExecutionTime() { 72 | return 0; 73 | } 74 | 75 | @Override 76 | public long getMaxIdleTime() { 77 | return 0; 78 | } 79 | 80 | @Override 81 | public boolean isAbortable() { 82 | return abortable; 83 | } 84 | }; 85 | } 86 | 87 | @Override 88 | public void execute(JobExecutionContext context) throws JobException {} 89 | }; 90 | } 91 | } 92 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/repository/JobDefinitionRepository.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.repository; 2 | 3 | import com.mongodb.*; 4 | import de.otto.jobstore.common.*; 5 | import de.otto.jobstore.common.properties.JobDefinitionProperty; 6 | 7 | import java.util.Date; 8 | 9 | 10 | public class JobDefinitionRepository extends AbstractRepository { 11 | 12 | /** 13 | * @deprecated Please use {@link #JobDefinitionRepository(MongoClient, String, String)} instead} 14 | */ 15 | @Deprecated 16 | public JobDefinitionRepository(Mongo mongo, String dbName, String collectionName) { 17 | super(createMongoClient(mongo, dbName, null, null), dbName, collectionName); 18 | } 19 | 20 | /** 21 | * @deprecated Please use {@link #JobDefinitionRepository(MongoClient, String, String)} instead} 22 | */ 23 | @Deprecated 24 | public JobDefinitionRepository(Mongo mongo, String dbName, String collectionName, String username, String password) { 25 | super(createMongoClient(mongo, dbName, username, password), dbName, collectionName); 26 | } 27 | 28 | /** 29 | * @deprecated Please use {@link #JobDefinitionRepository(MongoClient, String, String, WriteConcern)} instead} 30 | */ 31 | @Deprecated 32 | public JobDefinitionRepository(Mongo mongo, String dbName, String collectionName, String username, String password, WriteConcern safeWriteConcern) { 33 | super(createMongoClient(mongo, dbName, username, password), dbName, collectionName, safeWriteConcern); 34 | } 35 | 36 | public JobDefinitionRepository(MongoClient mongo, String dbName, String collectionName) { 37 | super(mongo, dbName, collectionName); 38 | } 39 | 40 | public JobDefinitionRepository(MongoClient mongo, String dbName, String collectionName, WriteConcern safeWriteConcern) { 41 | super(mongo, dbName, collectionName, safeWriteConcern); 42 | } 43 | 44 | 45 | public StoredJobDefinition find(String name) { 46 | final DBObject object = collection.findOne(new BasicDBObject(JobDefinitionProperty.NAME.val(), name)); 47 | return fromDbObject(object); 48 | } 49 | 50 | @Override 51 | protected void prepareCollection() { 52 | collection.createIndex(new BasicDBObject(JobDefinitionProperty.NAME.val(), 1), "name", true); 53 | } 54 | 55 | @Override 56 | protected StoredJobDefinition fromDbObject(DBObject dbObject) { 57 | if (dbObject == null) { 58 | return null; 59 | } 60 | return new StoredJobDefinition(dbObject); 61 | } 62 | 63 | public void addOrUpdate(StoredJobDefinition jobDefinition) { 64 | final DBObject obj = new BasicDBObject(MongoOperator.SET.op(), buildUpdateObject(jobDefinition)); 65 | collection.update(new BasicDBObject(JobDefinitionProperty.NAME.val(), jobDefinition.getName()), obj, true, false, getSafeWriteConcern()); 66 | } 67 | 68 | private BasicDBObject buildUpdateObject(StoredJobDefinition jobDefinition) { 69 | final BasicDBObject basicDBObject = new BasicDBObject(); 70 | final DBObject jobDefObj = jobDefinition.toDbObject(); 71 | for (JobDefinitionProperty property : JobDefinitionProperty.values()) { 72 | if (!property.isDynamic()) { 73 | basicDBObject.append(property.val(), jobDefObj.get(property.val())); 74 | } 75 | } 76 | return basicDBObject; 77 | } 78 | 79 | public void setJobExecutionEnabled(String name, boolean executionEnabled) { 80 | collection.update(new BasicDBObject(JobDefinitionProperty.NAME.val(), name), 81 | new BasicDBObject(MongoOperator.SET.op(), new BasicDBObject(JobDefinitionProperty.DISABLED.val(), !executionEnabled))); 82 | } 83 | 84 | public void setLastNotExecuted(String name, Date date) { 85 | collection.update(new BasicDBObject(JobDefinitionProperty.NAME.val(), name), 86 | new BasicDBObject(MongoOperator.SET.op(), new BasicDBObject(JobDefinitionProperty.LAST_NOT_EXECUTED.val(), date))); 87 | } 88 | 89 | } 90 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/service/DirectoryBasedTarArchiveProvider.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service; 2 | 3 | import de.otto.jobstore.common.RemoteJob; 4 | import org.apache.commons.compress.archivers.tar.TarArchiveEntry; 5 | import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; 6 | import org.apache.commons.compress.utils.IOUtils; 7 | import org.slf4j.Logger; 8 | import org.slf4j.LoggerFactory; 9 | 10 | import java.io.*; 11 | import java.net.URISyntaxException; 12 | import java.net.URL; 13 | import java.util.Collections; 14 | import java.util.List; 15 | import java.util.zip.GZIPOutputStream; 16 | 17 | /** 18 | * The script archiver generates a tar with all needed artifacts (scripts, config files...) 19 | *

20 | * It is passed the baseDirectory where all jobs reside. Every job has a matching subdirectory in the jobs folder. 21 | *

22 | * The archiver puts all the "global" files of the specific job folder into the tar. 23 | *

24 | * Example: 25 | *

26 | * /jobs 27 | * /jobname1 28 | * /jobname2 29 | *

30 | * 31 | */ 32 | public class DirectoryBasedTarArchiveProvider implements TarArchiveProvider { 33 | 34 | private static final int FOR_ALL_EXECUTABLE_FILE = 0100755; 35 | 36 | private static final Logger LOGGER = LoggerFactory.getLogger(DirectoryBasedTarArchiveProvider.class); 37 | private String baseDirectory; 38 | 39 | public DirectoryBasedTarArchiveProvider(String baseDirectory) { 40 | this.baseDirectory = baseDirectory; 41 | } 42 | 43 | @Override 44 | public InputStream getArchiveAsInputStream(RemoteJob remoteJob) throws IOException { 45 | ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); 46 | 47 | try (TarArchiveOutputStream tarArchive = new TarArchiveOutputStream( 48 | new GZIPOutputStream( 49 | new BufferedOutputStream(byteArrayOutputStream)))) { 50 | 51 | for (String givenDirectory : getTarInputDirectories(remoteJob)) { 52 | writeEntriesForDirectory(givenDirectory, tarArchive); 53 | } 54 | } 55 | return new ByteArrayInputStream(byteArrayOutputStream.toByteArray()); 56 | } 57 | 58 | protected List getTarInputDirectories(RemoteJob remoteJob) { 59 | return Collections.singletonList(getBaseDirectory() + File.separator + remoteJob.name); 60 | } 61 | 62 | protected String getBaseDirectory() { 63 | return baseDirectory; 64 | } 65 | 66 | private void writeEntriesForDirectory(String givenDirectory, TarArchiveOutputStream tarArchive) throws IOException { 67 | for (File file : getResources(givenDirectory)) { 68 | if (file.isFile()) { 69 | writeEntry(tarArchive, file); 70 | } 71 | } 72 | } 73 | 74 | private void writeEntry(TarArchiveOutputStream tarArchive, File file) throws IOException { 75 | try (InputStream fis = new FileInputStream(file)) { 76 | TarArchiveEntry tarArchiveEntry = new TarArchiveEntry(file.getName()); 77 | tarArchiveEntry.setSize(file.length()); 78 | tarArchiveEntry.setMode(FOR_ALL_EXECUTABLE_FILE); 79 | tarArchive.putArchiveEntry(tarArchiveEntry); 80 | IOUtils.copy(fis, tarArchive); 81 | tarArchive.closeArchiveEntry(); 82 | } 83 | } 84 | 85 | private File[] getResources(String path) throws IOException { 86 | File[] result; 87 | URL dirURL = getClass().getResource(path); 88 | if (dirURL == null) { 89 | LOGGER.info("Could not find baseDirectory \"" + path + "\""); 90 | result = new File[0]; 91 | } else { 92 | try { 93 | File directory = new File(dirURL.toURI()); 94 | result = directory.listFiles(); 95 | } catch(URISyntaxException e) { 96 | throw new IOException(e); 97 | } 98 | } 99 | return result; 100 | } 101 | 102 | } 103 | -------------------------------------------------------------------------------- /jobs-core/src/test/java/de/otto/jobstore/common/AbstractRemoteJobRunnableTest.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common; 2 | 3 | import de.otto.jobstore.TestSetup; 4 | import de.otto.jobstore.common.properties.JobInfoProperty; 5 | import de.otto.jobstore.service.JobInfoService; 6 | import de.otto.jobstore.service.RemoteJobExecutorService; 7 | import de.otto.jobstore.service.exception.RemoteJobAlreadyRunningException; 8 | import org.testng.annotations.BeforeMethod; 9 | import org.testng.annotations.Test; 10 | 11 | import java.net.URI; 12 | import java.util.ArrayList; 13 | import java.util.HashMap; 14 | import java.util.List; 15 | import java.util.Map; 16 | 17 | import static org.mockito.Mockito.mock; 18 | import static org.mockito.Mockito.when; 19 | import static org.testng.AssertJUnit.assertEquals; 20 | 21 | public class AbstractRemoteJobRunnableTest { 22 | 23 | private RemoteJobExecutorService remoteJobExecutorService; 24 | private JobInfoService jobInfoService; 25 | 26 | private Map parameters = new HashMap<>(); 27 | private String jobName = "testJob"; 28 | private AbstractRemoteJobDefinition jobDefinition = TestSetup.remoteJobDefinition(jobName, 0, 0); 29 | 30 | @BeforeMethod 31 | public void setUp() throws Exception { 32 | remoteJobExecutorService = mock(RemoteJobExecutorService.class); 33 | jobInfoService = mock(JobInfoService.class); 34 | parameters.put("key", "value"); 35 | } 36 | 37 | @Test 38 | public void testRemoteJobSetup() throws Exception { 39 | URI uri = URI.create("http://www.otto.de"); 40 | JobInfo jobInfo = mock(JobInfo.class); 41 | when(jobInfo.getParameters()).thenReturn(parameters); 42 | when(jobInfoService.getById("4811")).thenReturn(jobInfo); 43 | when(remoteJobExecutorService.startJob(new RemoteJob(jobName, "4811", parameters))).thenReturn(uri); 44 | JobRunnable runnable = TestSetup.remoteJobRunnable(remoteJobExecutorService, jobInfoService, parameters, jobDefinition); 45 | MockJobLogger logger = new MockJobLogger(); 46 | JobExecutionContext context = new JobExecutionContext("4811", logger, mock(JobInfoCache.class), JobExecutionPriority.CHECK_PRECONDITIONS, jobDefinition); 47 | runnable.execute(context); 48 | 49 | assertEquals(uri.toString(), logger.additionalData.get(JobInfoProperty.REMOTE_JOB_URI.val())); 50 | } 51 | 52 | @Test 53 | public void testExecutingJobWhichIsAlreadyRunning() throws Exception { 54 | URI uri = URI.create("http://www.otto.de"); 55 | JobInfo jobInfo = mock(JobInfo.class); 56 | when(jobInfo.getParameters()).thenReturn(parameters); 57 | when(jobInfoService.getById("4711")).thenReturn(jobInfo); 58 | when(remoteJobExecutorService.startJob(new RemoteJob(jobName, "4711", parameters))). 59 | thenThrow(new RemoteJobAlreadyRunningException("", uri)); 60 | JobRunnable runnable = TestSetup.remoteJobRunnable(remoteJobExecutorService, jobInfoService, parameters, jobDefinition); 61 | MockJobLogger logger = new MockJobLogger(); 62 | JobExecutionContext context = new JobExecutionContext("4711", logger, mock(JobInfoCache.class), JobExecutionPriority.CHECK_PRECONDITIONS, jobDefinition); 63 | runnable.execute(context); 64 | 65 | assertEquals(uri.toString(), logger.additionalData.get(JobInfoProperty.REMOTE_JOB_URI.val())); 66 | assertEquals(uri.toString(), logger.additionalData.get("resumedAlreadyRunningJob")); 67 | } 68 | 69 | private class MockJobLogger implements JobLogger { 70 | 71 | public List logs = new ArrayList<>(); 72 | public Map additionalData = new HashMap<>(); 73 | 74 | @Override 75 | public void addLoggingData(String log) { 76 | logs.add(log); 77 | } 78 | 79 | @Override 80 | public List getLoggingData() { 81 | return logs; 82 | } 83 | 84 | @Override 85 | public void insertOrUpdateAdditionalData(String key, String value) { 86 | additionalData.put(key, value); 87 | } 88 | 89 | @Override 90 | public String getAdditionalData(String key) { 91 | return additionalData.get(key); 92 | } 93 | } 94 | 95 | } 96 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/service/RemoteJobExecutorService.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service; 2 | 3 | import com.sun.jersey.api.client.Client; 4 | import com.sun.jersey.api.client.ClientHandlerException; 5 | import com.sun.jersey.api.client.ClientResponse; 6 | import com.sun.jersey.api.client.UniformInterfaceException; 7 | import com.sun.jersey.api.client.config.ClientConfig; 8 | import com.sun.jersey.api.client.config.DefaultClientConfig; 9 | import de.otto.jobstore.common.RemoteJob; 10 | import de.otto.jobstore.common.RemoteJobStatus; 11 | import de.otto.jobstore.service.exception.JobException; 12 | import de.otto.jobstore.service.exception.JobExecutionException; 13 | import de.otto.jobstore.service.exception.RemoteJobAlreadyRunningException; 14 | import de.otto.jobstore.service.exception.RemoteJobNotRunningException; 15 | import org.codehaus.jettison.json.JSONException; 16 | import org.slf4j.Logger; 17 | import org.slf4j.LoggerFactory; 18 | 19 | import javax.ws.rs.core.MediaType; 20 | import java.net.URI; 21 | 22 | public class RemoteJobExecutorService implements RemoteJobExecutor { 23 | 24 | private static final Logger LOGGER = LoggerFactory.getLogger(RemoteJobExecutorService.class); 25 | private final RemoteJobExecutorStatusRetriever remoteJobExecutorStatusRetriever; 26 | 27 | private String jobExecutorUri; 28 | private Client client; 29 | 30 | public RemoteJobExecutorService(String jobExecutorUri) { 31 | this.jobExecutorUri = jobExecutorUri; 32 | 33 | // since Flask (with WSGI) does not suppport HTTP 1.1 chunked encoding, turn it off 34 | // see: https://github.com/mitsuhiko/flask/issues/367 35 | final ClientConfig cc = new DefaultClientConfig(); 36 | cc.getProperties().put(ClientConfig.PROPERTY_CHUNKED_ENCODING_SIZE, null); 37 | this.client = Client.create(cc); 38 | remoteJobExecutorStatusRetriever = new RemoteJobExecutorStatusRetriever(client); 39 | } 40 | 41 | @Override 42 | public String getJobExecutorUri() { 43 | return jobExecutorUri; 44 | } 45 | 46 | @Override 47 | public URI startJob(final RemoteJob job) throws JobException { 48 | final String startUrl = jobExecutorUri + job.name + "/start"; 49 | try { 50 | LOGGER.info("ltag=RemoteJobExecutorService.startJob Going to start job: {} ...", startUrl); 51 | final ClientResponse response = client.resource(startUrl) 52 | .type(MediaType.APPLICATION_JSON).header("Connection", "close").header("User-Agent", "RemoteJobExecutorService") 53 | .post(ClientResponse.class, job.toJsonObject()); 54 | if (response.getStatus() == 201) { 55 | return createJobUri(response.getHeaders().getFirst("Link")); 56 | } else if (response.getStatus() == 303) { 57 | throw new RemoteJobAlreadyRunningException("Remote job is already running, url=" + startUrl, createJobUri(response.getHeaders().getFirst("Link"))); 58 | } 59 | throw new JobExecutionException("Unable to start remote job: url=" + startUrl + " rc=" + response.getStatus()); 60 | } catch (JSONException e) { 61 | throw new JobExecutionException("Could not create JSON object: " + job, e); 62 | } catch (UniformInterfaceException | ClientHandlerException e) { 63 | throw new JobExecutionException("Problem while starting new job: url=" + startUrl, e); 64 | } 65 | } 66 | 67 | @Override 68 | public void stopJob(URI jobUri) throws JobException { 69 | final String stopUrl = jobUri + "/stop"; 70 | try { 71 | LOGGER.info("ltag=RemoteJobExecutorService.stopJob Going to stop job: {} ...", stopUrl); 72 | client.resource(stopUrl).header("Connection", "close").post(); 73 | } catch (UniformInterfaceException e) { 74 | if (e.getResponse().getStatus() == 403) { 75 | throw new RemoteJobNotRunningException("Remote job is not running: url=" + stopUrl); 76 | } 77 | throw e; 78 | } 79 | } 80 | 81 | public RemoteJobStatus getStatus(final URI jobUri) { 82 | return remoteJobExecutorStatusRetriever.getStatus(jobUri); 83 | } 84 | 85 | public boolean isAlive() { 86 | return remoteJobExecutorStatusRetriever.isAlive(jobExecutorUri); 87 | } 88 | 89 | // ~ 90 | 91 | private URI createJobUri(String path) { 92 | return URI.create(jobExecutorUri).resolve(path); 93 | } 94 | 95 | } 96 | -------------------------------------------------------------------------------- /jobs-api/src/main/java/de/otto/jobstore/web/representation/JobInfoRepresentation.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.web.representation; 2 | 3 | import de.otto.jobstore.common.JobInfo; 4 | import de.otto.jobstore.common.LogLine; 5 | import de.otto.jobstore.common.ResultCode; 6 | 7 | import javax.xml.bind.annotation.XmlAccessType; 8 | import javax.xml.bind.annotation.XmlAccessorType; 9 | import javax.xml.bind.annotation.XmlRootElement; 10 | import java.util.ArrayList; 11 | import java.util.Date; 12 | import java.util.List; 13 | import java.util.Map; 14 | 15 | @XmlRootElement(name = "jobInfo") 16 | @XmlAccessorType(value = XmlAccessType.FIELD) 17 | public final class JobInfoRepresentation { 18 | 19 | private String id; 20 | 21 | private String name; 22 | 23 | private String host; 24 | 25 | private String thread; 26 | 27 | private Date creationTime; 28 | 29 | private Date startTime; 30 | 31 | private Date finishTime; 32 | 33 | private String errorMessage; 34 | 35 | private String runningState; 36 | 37 | private ResultCode resultState; 38 | 39 | private Long maxIdleTime; 40 | 41 | private Long maxExecutionTime; 42 | 43 | private Date lastModifiedTime; 44 | 45 | private Map additionalData; 46 | 47 | private List logLines; 48 | 49 | public JobInfoRepresentation() {} 50 | 51 | private JobInfoRepresentation(String id, String name, String host, String thread, Date creationTime, Date startTime, Date finishTime, 52 | String errorMessage, String runningState, ResultCode resultState, Long maxIdleTime, Long maxExecutionTime, 53 | Date lastModifiedTime, Map additionalData, List logLines) { 54 | this.id = id; 55 | this.name = name; 56 | this.host = host; 57 | this.thread = thread; 58 | this.creationTime = creationTime; 59 | this.startTime = startTime; 60 | this.finishTime = finishTime; 61 | this.errorMessage = errorMessage; 62 | this.runningState = runningState; 63 | this.resultState = resultState; 64 | this.maxIdleTime = maxIdleTime; 65 | this.maxExecutionTime = maxExecutionTime; 66 | this.lastModifiedTime = lastModifiedTime; 67 | this.additionalData = additionalData; 68 | this.logLines = logLines; 69 | } 70 | 71 | public String getId() { 72 | return id; 73 | } 74 | 75 | public String getName() { 76 | return name; 77 | } 78 | 79 | public String getHost() { 80 | return host; 81 | } 82 | 83 | public String getThread() { 84 | return thread; 85 | } 86 | 87 | public Date getCreationTime() { 88 | return creationTime; 89 | } 90 | 91 | public Date getStartTime() { 92 | return startTime; 93 | } 94 | 95 | public Date getFinishTime() { 96 | return finishTime; 97 | } 98 | 99 | public String getErrorMessage() { 100 | return errorMessage; 101 | } 102 | 103 | public String getRunningState() { 104 | return runningState; 105 | } 106 | 107 | public ResultCode getResultState() { 108 | return resultState; 109 | } 110 | 111 | public Long getMaxIdleTime() { 112 | return maxIdleTime; 113 | } 114 | 115 | public Long getMaxExecutionTime() { 116 | return maxExecutionTime; 117 | } 118 | 119 | public Date getLastModifiedTime() { 120 | return lastModifiedTime; 121 | } 122 | 123 | public Map getAdditionalData() { 124 | return additionalData; 125 | } 126 | 127 | public List getLogLines() { 128 | return logLines; 129 | } 130 | 131 | public static JobInfoRepresentation fromJobInfo(JobInfo jobInfo, int maxLogLines) { 132 | // Limit to the last recent N loglines 133 | final int nrLogLines = Math.min(maxLogLines, jobInfo.getLogLines().size()); 134 | final List logLines = new ArrayList<>(nrLogLines); 135 | for (LogLine ll : jobInfo.getLastLogLines(nrLogLines)) { 136 | logLines.add(LogLineRepresentation.fromLogLine(ll)); 137 | } 138 | return new JobInfoRepresentation(jobInfo.getId(), jobInfo.getName(), jobInfo.getHost(), 139 | jobInfo.getThread(), jobInfo.getCreationTime(), jobInfo.getStartTime(), jobInfo.getFinishTime(), 140 | jobInfo.getResultMessage(), jobInfo.getRunningState(), jobInfo.getResultState(), 141 | jobInfo.getMaxIdleTime(), jobInfo.getMaxExecutionTime(), jobInfo.getLastModifiedTime(), jobInfo.getAdditionalData(), 142 | logLines); 143 | } 144 | 145 | } 146 | -------------------------------------------------------------------------------- /jobs-core/src/test/java/de/otto/jobstore/service/RemoteJobExecutorServiceIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service; 2 | 3 | import de.otto.jobstore.common.RemoteJob; 4 | import de.otto.jobstore.common.RemoteJobStatus; 5 | import de.otto.jobstore.service.exception.JobException; 6 | import de.otto.jobstore.service.exception.RemoteJobAlreadyRunningException; 7 | import de.otto.jobstore.service.exception.RemoteJobNotRunningException; 8 | import org.springframework.test.context.ContextConfiguration; 9 | import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; 10 | import org.testng.annotations.Test; 11 | 12 | import javax.annotation.Resource; 13 | import java.net.URI; 14 | import java.util.HashMap; 15 | import java.util.Map; 16 | import java.util.concurrent.ExecutorService; 17 | import java.util.concurrent.Executors; 18 | import java.util.concurrent.TimeUnit; 19 | 20 | import static org.testng.AssertJUnit.*; 21 | 22 | @ContextConfiguration(locations = {"classpath:spring/jobs-context.xml"}) 23 | public class RemoteJobExecutorServiceIntegrationTest extends AbstractTestNGSpringContextTests { 24 | 25 | private static final String JOB_NAME = "jobname"; 26 | 27 | @Resource 28 | private RemoteJobExecutorService remoteJobExecutorService; 29 | 30 | @Test(enabled = false) 31 | public void testStartingDemoJob() throws Exception { 32 | URI uri = remoteJobExecutorService.startJob(createRemoteJob()); 33 | assertNotNull(uri); 34 | assertTrue("Expected valid job uri", uri.getPath().startsWith("/jobs/jobname/")); 35 | 36 | remoteJobExecutorService.stopJob(uri); 37 | } 38 | 39 | @Test(enabled = false, expectedExceptions = RemoteJobAlreadyRunningException.class) 40 | public void testStartingDemoJobWhichIsAlreadyRunning() throws Exception { 41 | URI uri = null; 42 | try { 43 | uri = remoteJobExecutorService.startJob(createRemoteJob()); 44 | } catch (JobException e) { 45 | fail("No exception expected when trying to start job"); 46 | } 47 | assert uri != null; 48 | assertTrue("Expected valid job uri", uri.getPath().startsWith("/jobs/jobname/")); 49 | 50 | try { 51 | remoteJobExecutorService.startJob(createRemoteJob()); 52 | } finally { 53 | remoteJobExecutorService.stopJob(uri); 54 | } 55 | } 56 | 57 | @Test(enabled = false, expectedExceptions = RemoteJobNotRunningException.class) 58 | public void testStoppingJobTwice() throws Exception { 59 | URI uri = remoteJobExecutorService.startJob(createRemoteJob()); 60 | remoteJobExecutorService.stopJob(uri); 61 | remoteJobExecutorService.stopJob(uri); 62 | } 63 | 64 | @Test(enabled = false, expectedExceptions = RemoteJobNotRunningException.class) 65 | public void testStoppingNotExistingJob() throws Exception { 66 | remoteJobExecutorService.stopJob(URI.create("http://localhost:5000/jobs/" + JOB_NAME + "/12345")); // TODO: configure URL 67 | } 68 | 69 | 70 | class GetRequest implements Runnable { 71 | 72 | private URI uri; 73 | 74 | public GetRequest(URI uri) { 75 | this.uri = uri; 76 | } 77 | 78 | @Override 79 | public void run() { 80 | RemoteJobStatus status = remoteJobExecutorService.getStatus(uri); 81 | assertNotNull(status); 82 | assertEquals(RemoteJobStatus.Status.RUNNING, status.status); 83 | } 84 | } 85 | 86 | @Test(enabled = false) 87 | public void testGettingStatusOfRunningJob() throws Exception { 88 | final URI uri = remoteJobExecutorService.startJob(createRemoteJob()); 89 | 90 | ExecutorService exec = Executors.newFixedThreadPool(8); 91 | for (int i = 0; i < 100; i++) { 92 | exec.submit(new GetRequest(uri)); 93 | } 94 | exec.shutdown(); 95 | exec.awaitTermination(10, TimeUnit.SECONDS); 96 | 97 | remoteJobExecutorService.stopJob(uri); 98 | 99 | //assertNull(status.result); 100 | } 101 | 102 | @Test(enabled = false) 103 | public void testGettingStatusOfFinishedJob() throws Exception { 104 | URI uri = remoteJobExecutorService.startJob(createRemoteJob()); 105 | remoteJobExecutorService.stopJob(uri); 106 | RemoteJobStatus status = remoteJobExecutorService.getStatus(uri); 107 | 108 | assertNotNull(status); 109 | assertEquals(RemoteJobStatus.Status.FINISHED, status.status); 110 | assertNotNull(status.result); 111 | assertTrue(status.result.ok); 112 | } 113 | 114 | private RemoteJob createRemoteJob() { 115 | Map params = new HashMap(); 116 | params.put("sample_file", "/var/log/mongodb/mongodb.log"); 117 | return new RemoteJob(JOB_NAME, "2311", params); 118 | } 119 | 120 | } 121 | -------------------------------------------------------------------------------- /jobs-core/src/test/java/de/otto/jobstore/repository/JobDefinitionRepositoryIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.repository; 2 | 3 | import de.otto.jobstore.common.StoredJobDefinition; 4 | import org.springframework.test.context.ContextConfiguration; 5 | import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; 6 | import org.testng.annotations.BeforeMethod; 7 | import org.testng.annotations.Test; 8 | 9 | import javax.annotation.Resource; 10 | 11 | import static org.testng.AssertJUnit.*; 12 | 13 | @ContextConfiguration(locations = {"classpath:spring/jobs-context.xml"}) 14 | public class JobDefinitionRepositoryIntegrationTest extends AbstractTestNGSpringContextTests { 15 | 16 | private static final String JOB_NAME = "test"; 17 | 18 | @Resource 19 | private JobDefinitionRepository jobDefinitionRepository; 20 | 21 | @BeforeMethod 22 | public void setUp() throws Exception { 23 | jobDefinitionRepository.clear(true); 24 | } 25 | 26 | @Test 27 | public void testAddingNotExistingJobDefinition() throws Exception { 28 | StoredJobDefinition jd = new StoredJobDefinition(JOB_NAME, 1, 1, 1, 0, 0, true, false); 29 | jobDefinitionRepository.addOrUpdate(jd); 30 | StoredJobDefinition retrievedJobDefinition = jobDefinitionRepository.find(JOB_NAME); 31 | assertNotNull(retrievedJobDefinition); 32 | } 33 | 34 | @Test 35 | public void testUpdatingExistingJobDefinition() throws Exception { 36 | StoredJobDefinition jd = new StoredJobDefinition(JOB_NAME, 1, 1, 1, 0, 0, true, false); 37 | jobDefinitionRepository.addOrUpdate(jd); 38 | StoredJobDefinition retrievedJobDefinition = jobDefinitionRepository.find(JOB_NAME); 39 | assertEquals(1L, retrievedJobDefinition.getPollingInterval()); 40 | 41 | jd = new StoredJobDefinition(JOB_NAME, 1, 1, 2, 0, 0, true, false); 42 | jobDefinitionRepository.addOrUpdate(jd); 43 | retrievedJobDefinition = jobDefinitionRepository.find(JOB_NAME); 44 | assertEquals(2L, retrievedJobDefinition.getPollingInterval()); 45 | } 46 | 47 | @Test 48 | public void testUpdatingExistingJobDefinitionDoesNotOverwriteEnabledStatus() throws Exception { 49 | StoredJobDefinition jd = new StoredJobDefinition(JOB_NAME, 1, 1, 1, 0, 0, true, false); 50 | jd.setDisabled(true); 51 | jobDefinitionRepository.save(jd); 52 | 53 | jd = new StoredJobDefinition(JOB_NAME, 1, 1, 2, 0, 0, true, false); 54 | jobDefinitionRepository.addOrUpdate(jd); 55 | StoredJobDefinition retrievedJobDefinition = jobDefinitionRepository.find(JOB_NAME); 56 | assertTrue(retrievedJobDefinition.isDisabled()); 57 | } 58 | 59 | @Test 60 | public void testUpdatingExistingJobDefinitionOverwritesAbortable() throws Exception { 61 | StoredJobDefinition jd = new StoredJobDefinition(JOB_NAME, 1, 1, 1, 0, 0, true, false); 62 | jobDefinitionRepository.save(jd); 63 | 64 | jd = new StoredJobDefinition(JOB_NAME, 1, 1, 2, 0, 0, true, true); 65 | jobDefinitionRepository.addOrUpdate(jd); 66 | StoredJobDefinition retrievedJobDefinition = jobDefinitionRepository.find(JOB_NAME); 67 | assertTrue(retrievedJobDefinition.isAbortable()); 68 | } 69 | 70 | @Test 71 | public void testDisablingJob() throws Exception { 72 | StoredJobDefinition jd = new StoredJobDefinition(JOB_NAME, 1, 1, 1, 0, 0, true, false); 73 | jobDefinitionRepository.save(jd); 74 | jobDefinitionRepository.setJobExecutionEnabled(JOB_NAME, false); 75 | 76 | StoredJobDefinition retrievedJobDefinition = jobDefinitionRepository.find(JOB_NAME); 77 | assertTrue(retrievedJobDefinition.isDisabled()); 78 | } 79 | 80 | @Test 81 | public void testActivatingJob() throws Exception { 82 | StoredJobDefinition jd = new StoredJobDefinition(JOB_NAME, 1, 1, 1, 0, 0, true, false); 83 | jd.setDisabled(true); 84 | jobDefinitionRepository.save(jd); 85 | jobDefinitionRepository.setJobExecutionEnabled(JOB_NAME, true); 86 | 87 | StoredJobDefinition retrievedJobDefinition = jobDefinitionRepository.find(JOB_NAME); 88 | assertFalse(retrievedJobDefinition.isDisabled()); 89 | } 90 | 91 | @Test 92 | public void testUpdatingPausedJob() throws Exception { 93 | StoredJobDefinition jd = new StoredJobDefinition(JOB_NAME, 1, 1, 1, 0, 0, true, false); 94 | jd.setDisabled(true); 95 | jobDefinitionRepository.save(jd); 96 | 97 | StoredJobDefinition jd2 = new StoredJobDefinition(JOB_NAME, 2, 2, 2, 0, 0, true, false); 98 | jobDefinitionRepository.addOrUpdate(jd2); 99 | StoredJobDefinition retrievedJobDefinition = jobDefinitionRepository.find(JOB_NAME); 100 | assertEquals(2, retrievedJobDefinition.getPollingInterval()); 101 | assertTrue(retrievedJobDefinition.isDisabled()); 102 | } 103 | } 104 | -------------------------------------------------------------------------------- /jobs-core/src/test/java/de/otto/jobstore/service/RemoteJobExecutorServiceWithScriptTransferIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service; 2 | 3 | import de.otto.jobstore.common.RemoteJob; 4 | import de.otto.jobstore.common.RemoteJobStatus; 5 | import de.otto.jobstore.service.exception.JobException; 6 | import de.otto.jobstore.service.exception.RemoteJobAlreadyRunningException; 7 | import de.otto.jobstore.service.exception.RemoteJobNotRunningException; 8 | import org.springframework.test.context.ContextConfiguration; 9 | import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; 10 | import org.testng.annotations.Test; 11 | 12 | import javax.annotation.Resource; 13 | import java.net.URI; 14 | import java.util.HashMap; 15 | import java.util.Map; 16 | import java.util.concurrent.ExecutorService; 17 | import java.util.concurrent.Executors; 18 | import java.util.concurrent.TimeUnit; 19 | 20 | import static org.testng.AssertJUnit.*; 21 | 22 | @ContextConfiguration(locations = {"classpath:spring/jobs-context.xml"}) 23 | public class RemoteJobExecutorServiceWithScriptTransferIntegrationTest extends AbstractTestNGSpringContextTests { 24 | 25 | private static final String JOB_NAME = "demojob1"; 26 | public static final boolean ENABLE_TESTS = false; 27 | 28 | @Resource 29 | private RemoteJobExecutorWithScriptTransferService remoteJobExecutorService; 30 | 31 | @Test(enabled = ENABLE_TESTS) 32 | public void testStartingDemoJob() throws Exception { 33 | 34 | URI uri = remoteJobExecutorService.startJob(createRemoteJob()); 35 | assertNotNull(uri); 36 | 37 | remoteJobExecutorService.stopJob(uri); 38 | } 39 | 40 | @Test(enabled = ENABLE_TESTS, expectedExceptions = RemoteJobAlreadyRunningException.class) 41 | public void testStartingDemoJobWhichIsAlreadyRunning() throws Exception { 42 | URI uri = null; 43 | try { 44 | uri = remoteJobExecutorService.startJob(createRemoteJob()); 45 | } catch (JobException e) { 46 | fail("No exception expected when trying to start job"); 47 | } 48 | assert uri != null; 49 | assertTrue("Expected valid job uri", uri.getPath().startsWith("/jobs/" + JOB_NAME)); 50 | 51 | try { 52 | remoteJobExecutorService.startJob(createRemoteJob()); 53 | } finally { 54 | remoteJobExecutorService.stopJob(uri); 55 | } 56 | } 57 | 58 | @Test(enabled = ENABLE_TESTS, expectedExceptions = RemoteJobNotRunningException.class) 59 | public void testStoppingJobTwice() throws Exception { 60 | URI uri = remoteJobExecutorService.startJob(createRemoteJob()); 61 | remoteJobExecutorService.stopJob(uri); 62 | remoteJobExecutorService.stopJob(uri); 63 | } 64 | 65 | @Test(enabled = ENABLE_TESTS, expectedExceptions = RemoteJobNotRunningException.class) 66 | public void testStoppingNotExistingJob() throws Exception { 67 | remoteJobExecutorService.stopJob(URI.create(remoteJobExecutorService.getJobExecutorUri() + JOB_NAME + "/12345")); 68 | } 69 | 70 | 71 | class GetRequest implements Runnable { 72 | 73 | private URI uri; 74 | 75 | public GetRequest(URI uri) { 76 | this.uri = uri; 77 | } 78 | 79 | @Override 80 | public void run() { 81 | RemoteJobStatus status = remoteJobExecutorService.getStatus(uri); 82 | assertNotNull(status); 83 | assertEquals(RemoteJobStatus.Status.RUNNING, status.status); 84 | } 85 | } 86 | 87 | @Test(enabled = ENABLE_TESTS) 88 | public void testGettingStatusOfRunningJob() throws Exception { 89 | final URI uri = remoteJobExecutorService.startJob(createRemoteJob()); 90 | 91 | ExecutorService exec = Executors.newFixedThreadPool(8); 92 | for (int i = 0; i < 100; i++) { 93 | exec.submit(new GetRequest(uri)); 94 | } 95 | exec.shutdown(); 96 | exec.awaitTermination(10, TimeUnit.SECONDS); 97 | 98 | remoteJobExecutorService.stopJob(uri); 99 | } 100 | 101 | @Test(enabled = ENABLE_TESTS) 102 | public void testGettingStatusOfFinishedJob() throws Exception { 103 | URI uri = remoteJobExecutorService.startJob(createRemoteJob()); 104 | remoteJobExecutorService.stopJob(uri); 105 | RemoteJobStatus status = remoteJobExecutorService.getStatus(uri); 106 | 107 | assertNotNull(status); 108 | assertEquals(RemoteJobStatus.Status.FINISHED, status.status); 109 | assertNotNull(status.result); 110 | // Job was aborted 111 | assertFalse(status.result.ok); 112 | assertEquals(-1, status.result.exitCode); 113 | } 114 | 115 | private RemoteJob createRemoteJob() { 116 | Map params = new HashMap<>(); 117 | params.put("host", "127.0.0.1"); 118 | return new RemoteJob(JOB_NAME, "2311", params); 119 | } 120 | 121 | } 122 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/service/JobInfoService.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service; 2 | 3 | import de.otto.jobstore.common.JobInfo; 4 | import de.otto.jobstore.common.ResultCode; 5 | import de.otto.jobstore.common.RunningState; 6 | import de.otto.jobstore.repository.JobInfoRepository; 7 | 8 | import java.util.*; 9 | 10 | /** 11 | * This service gives access to information on the jobs that have been executed. This allows for example to make 12 | * decisions on whether to execute another job if the previous one timed out or failed. 13 | */ 14 | public class JobInfoService { 15 | 16 | private final JobInfoRepository jobInfoRepository; 17 | 18 | public JobInfoService(JobInfoRepository jobInfoRepository) { 19 | this.jobInfoRepository = jobInfoRepository; 20 | } 21 | 22 | /** 23 | * Returns the information on the most recent job that has been executed. 24 | * 25 | * @param name The name of the job for which to return the information 26 | * @return The most recent executed job 27 | */ 28 | public JobInfo getMostRecentExecuted(String name) { 29 | return jobInfoRepository.findMostRecentFinished(name); 30 | } 31 | 32 | /** 33 | * Returns the information on the most recent job that has been successfully executed. 34 | * 35 | * @param name The name of the job for which to return the information 36 | * @return The most recent successfully executed job 37 | */ 38 | public JobInfo getMostRecentSuccessful(String name) { 39 | return jobInfoRepository.findMostRecentByNameAndResultState(name, EnumSet.of(ResultCode.SUCCESSFUL)); 40 | } 41 | 42 | /** 43 | * Returns for each job name the information on the most recent job that has been executed 44 | * @return The list of job information 45 | */ 46 | public List getMostRecentExecuted() { 47 | final List names = jobInfoRepository.distinctJobNames(); 48 | final List jobInfoList = new ArrayList<>(); 49 | for (String name : names) { 50 | final JobInfo jobInfo = getMostRecentExecuted(name); 51 | if (jobInfo != null) { 52 | jobInfoList.add(jobInfo); 53 | } 54 | } 55 | return jobInfoList; 56 | } 57 | 58 | /** 59 | * Returns for the given name all job information sorted descending by the creation time of the jobs. 60 | * 61 | * @param name The name of the job for which to return the information 62 | * @return The list of job information 63 | */ 64 | public List getByName(String name) { 65 | return jobInfoRepository.findByName(name, null); 66 | } 67 | 68 | /** 69 | * Returns for the given name the job with the given running state or null if none exists 70 | * 71 | * @param name The name of the job for which to return the information 72 | * @param runningState The running state the job to return 73 | * @return The job with the given name and running state, or null 74 | */ 75 | public JobInfo getByNameAndRunningState(String name, RunningState runningState) { 76 | return jobInfoRepository.findByNameAndRunningState(name, runningState); 77 | } 78 | 79 | /** 80 | * Returns for the given name all job information sorted descending by the creation time of the jobs. 81 | * 82 | * @param name The name of the job for which to return the information 83 | * @param limit The maximum number of elements to return 84 | * @return The list of job information 85 | */ 86 | public List getByName(String name, Integer limit) { 87 | return jobInfoRepository.findByName(name, limit); 88 | } 89 | 90 | /** 91 | * Returns for the given id the job information 92 | * 93 | * @param id The id of the job for which to return the information 94 | * @return The job information or null if it does not exist 95 | */ 96 | public JobInfo getById(String id) { 97 | return jobInfoRepository.findById(id); 98 | } 99 | 100 | /** 101 | * Returns all job information for the given name which were last modified after the given after date and before 102 | * the given before date. The result list is sorted descending by the jobs creation date. 103 | * 104 | * @param name The name of the job for which to return the information 105 | * @param after The date after which the last modified date has to be 106 | * @param before The date before which the last modified date has to be 107 | * @return The list of job information 108 | */ 109 | public List getByNameAndTimeRange(String name, Date after, Date before, Set resultCodes) { 110 | return jobInfoRepository.findByNameAndTimeRange(name, after, before, resultCodes); 111 | } 112 | 113 | /** 114 | * Remove all job information. 115 | */ 116 | public void clean() { 117 | jobInfoRepository.clear(false); 118 | } 119 | 120 | } 121 | -------------------------------------------------------------------------------- /jobs-core/src/test/java/de/otto/jobstore/TestSetup.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore; 2 | 3 | 4 | import de.otto.jobstore.common.*; 5 | import de.otto.jobstore.service.JobInfoService; 6 | import de.otto.jobstore.service.RemoteJobExecutorService; 7 | import de.otto.jobstore.service.exception.JobException; 8 | import de.otto.jobstore.service.exception.JobExecutionException; 9 | 10 | import java.util.Map; 11 | 12 | public class TestSetup { 13 | 14 | public static LocalMockJobRunnable localJobRunnable(final String name, final int timeout) { 15 | return localJobRunnable(name, timeout, null, 0); 16 | } 17 | 18 | public static LocalMockJobRunnable localJobRunnable(final String name, final int timeout, final int maxRetries) { 19 | return localJobRunnable(name, timeout, null, maxRetries); 20 | } 21 | 22 | public static LocalMockJobRunnable localJobRunnable(final String name, final int timeout, final JobExecutionException exception, int maxRetries) { 23 | return new LocalMockJobRunnable(name, timeout, exception, maxRetries); 24 | } 25 | 26 | public static LocalMockJobRunnable localJobRunnable(JobDefinition jobDefinition, final JobExecutionException exception) { 27 | return new LocalMockJobRunnable(jobDefinition, exception); 28 | } 29 | 30 | 31 | public static AbstractRemoteJobRunnable remoteJobRunnable(final RemoteJobExecutorService remoteJobExecutorService, final JobInfoService jobInfoService, 32 | final Map parameters, final AbstractRemoteJobDefinition jobDefinition) { 33 | return new AbstractRemoteJobRunnable(remoteJobExecutorService, jobInfoService) { 34 | 35 | @Override 36 | public Map getParameters() { 37 | return parameters; 38 | } 39 | 40 | @Override 41 | public JobDefinition getJobDefinition() { 42 | return jobDefinition; 43 | } 44 | 45 | }; 46 | } 47 | 48 | public static AbstractLocalJobDefinition localJobDefinition(final String name, final long timeoutPeriod) { 49 | return localJobDefinition(name, timeoutPeriod, 0L); 50 | } 51 | 52 | public static AbstractLocalJobDefinition localJobDefinition(final String name, final long timeoutPeriod, final long maxRetries) { 53 | return new AbstractLocalJobDefinition() { 54 | @Override 55 | public String getName() { 56 | return name; 57 | } 58 | 59 | @Override 60 | public long getMaxIdleTime() { 61 | return timeoutPeriod; 62 | } 63 | 64 | @Override 65 | public long getMaxExecutionTime() { 66 | return timeoutPeriod; 67 | } 68 | 69 | @Override 70 | public long getMaxRetries() { 71 | return maxRetries; 72 | } 73 | }; 74 | } 75 | 76 | public static AbstractRemoteJobDefinition remoteJobDefinition(final String name, final long timeoutPeriod, final long pollingInterval) { 77 | return new AbstractRemoteJobDefinition() { 78 | @Override 79 | public String getName() { 80 | return name; 81 | } 82 | 83 | @Override 84 | public long getMaxExecutionTime() { 85 | return timeoutPeriod; 86 | } 87 | 88 | @Override 89 | public long getMaxIdleTime() { 90 | return timeoutPeriod; 91 | } 92 | 93 | @Override 94 | public long getPollingInterval() { 95 | return pollingInterval; 96 | } 97 | }; 98 | } 99 | 100 | public static class LocalMockJobRunnable extends AbstractLocalJobRunnable { 101 | 102 | private volatile boolean executed = false; 103 | private JobException exception; 104 | private JobDefinition localJobDefinition; 105 | 106 | private LocalMockJobRunnable(JobDefinition jobDefinition, JobException exception) { 107 | localJobDefinition = jobDefinition; 108 | this.exception = exception; 109 | } 110 | 111 | private LocalMockJobRunnable(String name, long timeoutPeriod, JobException exception, int maxRetries) { 112 | localJobDefinition = localJobDefinition(name, timeoutPeriod, maxRetries); 113 | this.exception = exception; 114 | } 115 | 116 | @Override 117 | public JobDefinition getJobDefinition() { 118 | return localJobDefinition; 119 | } 120 | 121 | @Override 122 | public void execute(JobExecutionContext executionContext) throws JobException { 123 | executed = true; 124 | if (exception != null) { 125 | throw exception; 126 | } 127 | } 128 | 129 | public boolean isExecuted() { 130 | return executed; 131 | } 132 | 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/common/AbstractRemoteJobRunnable.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common; 2 | 3 | import de.otto.jobstore.common.properties.JobInfoProperty; 4 | import de.otto.jobstore.service.JobInfoService; 5 | import de.otto.jobstore.service.RemoteJobExecutor; 6 | import de.otto.jobstore.service.exception.JobException; 7 | import de.otto.jobstore.service.exception.RemoteJobAlreadyRunningException; 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | import java.net.URI; 12 | 13 | public abstract class AbstractRemoteJobRunnable implements JobRunnable { 14 | 15 | protected Logger log = LoggerFactory.getLogger(this.getClass()); 16 | 17 | protected final RemoteJobExecutor remoteJobExecutorService; 18 | protected final JobInfoService jobInfoService; 19 | 20 | protected AbstractRemoteJobRunnable(RemoteJobExecutor remoteJobExecutorService, JobInfoService jobInfoService) { 21 | this.remoteJobExecutorService = remoteJobExecutorService; 22 | this.jobInfoService = jobInfoService; 23 | } 24 | 25 | @Override 26 | public RemoteJobStatus getRemoteStatus(JobExecutionContext context) { 27 | final String remoteJobUri = context.getJobLogger().getAdditionalData(JobInfoProperty.REMOTE_JOB_URI.val()); 28 | final RemoteJobStatus status = remoteJobExecutorService.getStatus(URI.create(remoteJobUri)); 29 | final JobInfo jobInfo = jobInfoService.getById(context.getId()); 30 | 31 | if (jobInfo != null && status.logLines != null 32 | && jobInfo.getLogLines() != null && !jobInfo.getLogLines().isEmpty()) { 33 | final int currentLength = jobInfo.getLogLines().size(); 34 | // Assume that old lines are already included, and therefore can be cut off 35 | if (currentLength <= status.logLines.size()) { 36 | status.logLines = status.logLines.subList(currentLength, status.logLines.size()); 37 | } 38 | } 39 | return status; 40 | } 41 | 42 | /** 43 | * By default returns true. If an exception occurs, returns false. 44 | */ 45 | @Override 46 | public boolean prepare(JobExecutionContext context) { 47 | try { 48 | return doPrepare(context); 49 | } catch (Exception e) { 50 | return onException(context, e, State.PREPARE).hasRecovered(); 51 | } 52 | } 53 | 54 | /** 55 | * Template method of de.otto.jobstore.common.AbstractRemoteJobRunnable#prepare(de.otto.jobstore.common.JobExecutionContext) 56 | */ 57 | protected boolean doPrepare(JobExecutionContext context) throws JobException { 58 | return true; 59 | } 60 | 61 | /** 62 | * Only triggers the remote job, poll to check wether job is finished or not. 63 | * 64 | * @see de.otto.jobstore.service.JobService#pollRemoteJobs() 65 | */ 66 | @Override 67 | public void execute(JobExecutionContext context) throws JobException { 68 | try { 69 | doExecute(context); 70 | } catch (Exception e) { 71 | onException(context, e, State.EXECUTE).doThrow(); 72 | } 73 | } 74 | 75 | /** 76 | * Template method for de.otto.jobstore.common.AbstractRemoteJobRunnable#execute(de.otto.jobstore.common.JobExecutionContext) 77 | */ 78 | protected void doExecute(JobExecutionContext context) throws JobException { 79 | final JobLogger jobLogger = context.getJobLogger(); 80 | try { 81 | log.info("ltag={}.execute Trigger remote job jobName={} jobId={} ...", 82 | this.getClass().getSimpleName(), getJobDefinition().getName(), context.getId()); 83 | final JobInfo jobInfo = jobInfoService.getById(context.getId()); 84 | final URI uri = remoteJobExecutorService.startJob(new RemoteJob(getJobDefinition().getName(), context.getId(), jobInfo.getParameters())); 85 | jobLogger.insertOrUpdateAdditionalData(JobInfoProperty.REMOTE_JOB_URI.val(), uri.toString()); 86 | } catch (RemoteJobAlreadyRunningException e) { 87 | log.info("ltag={}.execute Remote job jobName={} jobId={} is already running: {}", 88 | this.getClass().getSimpleName(), getJobDefinition().getName(), context.getId(), e.getMessage()); 89 | jobLogger.insertOrUpdateAdditionalData("resumedAlreadyRunningJob", e.getJobUri().toString()); 90 | jobLogger.insertOrUpdateAdditionalData(JobInfoProperty.REMOTE_JOB_URI.val(), e.getJobUri().toString()); 91 | } 92 | } 93 | 94 | /** 95 | * Implementation might want to set the {@link JobExecutionContext#resultCode} 96 | */ 97 | @Override 98 | public void afterExecution(JobExecutionContext context) throws JobException { 99 | try { 100 | doAfterExecution(context); 101 | } catch (Exception e) { 102 | onException(context, e, State.AFTER_EXECUTION).doThrow(); 103 | } 104 | } 105 | 106 | /** 107 | * Template method for de.otto.jobstore.common.AbstractRemoteJobRunnable#afterExecution(de.otto.jobstore.common.JobExecutionContext) 108 | */ 109 | protected void doAfterExecution(JobExecutionContext context) throws JobException { 110 | } 111 | 112 | @Override 113 | public OnException onException(JobExecutionContext context, final Exception e, State state) { 114 | return new DefaultOnException(e); 115 | } 116 | } 117 | -------------------------------------------------------------------------------- /gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 10 | DEFAULT_JVM_OPTS="" 11 | 12 | APP_NAME="Gradle" 13 | APP_BASE_NAME=`basename "$0"` 14 | 15 | # Use the maximum available, or set MAX_FD != -1 to use that value. 16 | MAX_FD="maximum" 17 | 18 | warn ( ) { 19 | echo "$*" 20 | } 21 | 22 | die ( ) { 23 | echo 24 | echo "$*" 25 | echo 26 | exit 1 27 | } 28 | 29 | # OS specific support (must be 'true' or 'false'). 30 | cygwin=false 31 | msys=false 32 | darwin=false 33 | case "`uname`" in 34 | CYGWIN* ) 35 | cygwin=true 36 | ;; 37 | Darwin* ) 38 | darwin=true 39 | ;; 40 | MINGW* ) 41 | msys=true 42 | ;; 43 | esac 44 | 45 | # For Cygwin, ensure paths are in UNIX format before anything is touched. 46 | if $cygwin ; then 47 | [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 48 | fi 49 | 50 | # Attempt to set APP_HOME 51 | # Resolve links: $0 may be a link 52 | PRG="$0" 53 | # Need this for relative symlinks. 54 | while [ -h "$PRG" ] ; do 55 | ls=`ls -ld "$PRG"` 56 | link=`expr "$ls" : '.*-> \(.*\)$'` 57 | if expr "$link" : '/.*' > /dev/null; then 58 | PRG="$link" 59 | else 60 | PRG=`dirname "$PRG"`"/$link" 61 | fi 62 | done 63 | SAVED="`pwd`" 64 | cd "`dirname \"$PRG\"`/" >&- 65 | APP_HOME="`pwd -P`" 66 | cd "$SAVED" >&- 67 | 68 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 69 | 70 | # Determine the Java command to use to start the JVM. 71 | if [ -n "$JAVA_HOME" ] ; then 72 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 73 | # IBM's JDK on AIX uses strange locations for the executables 74 | JAVACMD="$JAVA_HOME/jre/sh/java" 75 | else 76 | JAVACMD="$JAVA_HOME/bin/java" 77 | fi 78 | if [ ! -x "$JAVACMD" ] ; then 79 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 80 | 81 | Please set the JAVA_HOME variable in your environment to match the 82 | location of your Java installation." 83 | fi 84 | else 85 | JAVACMD="java" 86 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 87 | 88 | Please set the JAVA_HOME variable in your environment to match the 89 | location of your Java installation." 90 | fi 91 | 92 | # Increase the maximum file descriptors if we can. 93 | if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then 94 | MAX_FD_LIMIT=`ulimit -H -n` 95 | if [ $? -eq 0 ] ; then 96 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 97 | MAX_FD="$MAX_FD_LIMIT" 98 | fi 99 | ulimit -n $MAX_FD 100 | if [ $? -ne 0 ] ; then 101 | warn "Could not set maximum file descriptor limit: $MAX_FD" 102 | fi 103 | else 104 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 105 | fi 106 | fi 107 | 108 | # For Darwin, add options to specify how the application appears in the dock 109 | if $darwin; then 110 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 111 | fi 112 | 113 | # For Cygwin, switch paths to Windows format before running java 114 | if $cygwin ; then 115 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 116 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 117 | 118 | # We build the pattern for arguments to be converted via cygpath 119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 120 | SEP="" 121 | for dir in $ROOTDIRSRAW ; do 122 | ROOTDIRS="$ROOTDIRS$SEP$dir" 123 | SEP="|" 124 | done 125 | OURCYGPATTERN="(^($ROOTDIRS))" 126 | # Add a user-defined pattern to the cygpath arguments 127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 129 | fi 130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 131 | i=0 132 | for arg in "$@" ; do 133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 135 | 136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 138 | else 139 | eval `echo args$i`="\"$arg\"" 140 | fi 141 | i=$((i+1)) 142 | done 143 | case $i in 144 | (0) set -- ;; 145 | (1) set -- "$args0" ;; 146 | (2) set -- "$args0" "$args1" ;; 147 | (3) set -- "$args0" "$args1" "$args2" ;; 148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 154 | esac 155 | fi 156 | 157 | # Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules 158 | function splitJvmOpts() { 159 | JVM_OPTS=("$@") 160 | } 161 | eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS 162 | JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" 163 | 164 | exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" 165 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/repository/AbstractRepository.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.repository; 2 | 3 | import com.mongodb.*; 4 | import de.otto.jobstore.common.AbstractItem; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import java.util.ArrayList; 9 | import java.util.Collections; 10 | import java.util.List; 11 | 12 | public abstract class AbstractRepository { 13 | 14 | protected final Logger logger = LoggerFactory.getLogger(this.getClass()); 15 | 16 | protected final DBCollection collection; 17 | 18 | private WriteConcern safeWriteConcern = WriteConcern.SAFE; 19 | 20 | public AbstractRepository(MongoClient mongoClient, String dbName, String collectionName) { 21 | this.collection = mongoClient.getDB(dbName).getCollection(collectionName); 22 | logger.info("Prepare access to MongoDB collection '{}' on {}/{}", collectionName, mongoClient, dbName); 23 | prepareCollection(); 24 | } 25 | 26 | public AbstractRepository(MongoClient mongo, String dbName, String collectionName, WriteConcern safeWriteConcern) { 27 | this(mongo, dbName, collectionName); 28 | if(safeWriteConcern == null) { 29 | throw new NullPointerException("writeConcern may not be null"); 30 | } 31 | this.safeWriteConcern = safeWriteConcern; 32 | } 33 | 34 | @Deprecated 35 | static MongoClient createMongoClient(Mongo mongo, String dbName, String username, String password) { 36 | MongoOptions mongoOptions = mongo.getMongoOptions(); 37 | return new MongoClient(mongo.getAllAddress(), 38 | credentials(dbName, username, password), 39 | mongoClientOptions(mongoOptions)); 40 | } 41 | 42 | @Deprecated 43 | private static MongoClientOptions mongoClientOptions(MongoOptions options) { 44 | MongoClientOptions.Builder builder = MongoClientOptions.builder() 45 | .connectionsPerHost(options.getConnectionsPerHost()) 46 | .threadsAllowedToBlockForConnectionMultiplier(options.getThreadsAllowedToBlockForConnectionMultiplier()) 47 | .maxWaitTime(options.getMaxWaitTime()) 48 | .connectTimeout(options.getConnectTimeout()) 49 | .socketTimeout(options.getSocketTimeout()) 50 | .socketKeepAlive(options.isSocketKeepAlive()) 51 | ; 52 | 53 | if (options.getReadPreference() != null) { 54 | builder.readPreference(options.getReadPreference()); 55 | } 56 | if (options.getDbDecoderFactory() != null) { 57 | builder.dbDecoderFactory(options.getDbDecoderFactory()); 58 | } 59 | if (options.getDbEncoderFactory() != null) { 60 | builder.dbEncoderFactory(options.getDbEncoderFactory()); 61 | } 62 | if (options.getSocketFactory() != null) { 63 | builder.socketFactory(options.getSocketFactory()); 64 | } 65 | if (options.getSocketFactory() != null) { 66 | builder.socketFactory(options.getSocketFactory()); 67 | } 68 | if (options.getWriteConcern() != null) { 69 | builder.writeConcern(options.getWriteConcern()); 70 | } 71 | return builder 72 | .description(options.getDescription()) 73 | .cursorFinalizerEnabled(options.isCursorFinalizerEnabled()) 74 | .alwaysUseMBeans(options.isAlwaysUseMBeans()) 75 | .requiredReplicaSetName(options.getRequiredReplicaSetName()) 76 | .build(); 77 | } 78 | 79 | private static List credentials(String dbName, String userName, String password) { 80 | if (userName != null && userName.trim().length() > 0) { 81 | return Collections.singletonList( 82 | MongoCredential.createMongoCRCredential(userName, dbName, password.toCharArray())); 83 | } else { 84 | return Collections.emptyList(); 85 | } 86 | } 87 | 88 | public WriteConcern getSafeWriteConcern() { 89 | return safeWriteConcern; 90 | } 91 | 92 | public void save(E item) { 93 | final DBObject obj = item.toDbObject(); 94 | try { 95 | collection.save(obj, getSafeWriteConcern()); 96 | } catch (MongoException e) { 97 | throw e; 98 | } 99 | } 100 | 101 | /** 102 | * Clears all elements from the repository 103 | * 104 | * @param dropCollection Flag if the collection should be dropped 105 | */ 106 | public final void clear(final boolean dropCollection) { 107 | logger.info("Going to clear all entities on collection: {}", collection.getFullName()); 108 | if (dropCollection) { 109 | collection.drop(); 110 | prepareCollection(); 111 | } else { 112 | try { 113 | collection.remove(new BasicDBObject(), this.safeWriteConcern); 114 | logger.info("Cleared all entities successfully on collection: {}", collection.getFullName()); 115 | } catch (MongoException e) { 116 | logger.error("Could not clear entities on collection {}: {}", collection.getFullName(), e.getMessage()); 117 | throw e; 118 | } 119 | } 120 | } 121 | 122 | // ~~ 123 | 124 | abstract protected void prepareCollection(); 125 | 126 | abstract protected E fromDbObject(DBObject dbObject); 127 | 128 | protected List getAll(final DBCursor cursor) { 129 | final List elements = new ArrayList<>(); 130 | while (cursor.hasNext()) { 131 | elements.add(fromDbObject(cursor.next())); 132 | } 133 | return elements; 134 | } 135 | 136 | protected E getFirst(final DBCursor cursor) { 137 | if (cursor.hasNext()) { 138 | return fromDbObject(cursor.next()); 139 | } 140 | return null; 141 | } 142 | 143 | } 144 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/service/JobScheduler.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service; 2 | 3 | import de.otto.jobstore.common.JobSchedule; 4 | import de.otto.jobstore.repository.JobInfoRepository; 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | import javax.annotation.PostConstruct; 9 | import javax.annotation.PreDestroy; 10 | import java.util.ArrayList; 11 | import java.util.List; 12 | import java.util.concurrent.Executors; 13 | import java.util.concurrent.ScheduledExecutorService; 14 | import java.util.concurrent.ThreadFactory; 15 | import java.util.concurrent.TimeUnit; 16 | import java.util.concurrent.atomic.AtomicInteger; 17 | 18 | /** 19 | * method to unify scheduling. This spawns some extra daemon threads 20 | */ 21 | public class JobScheduler { 22 | private static final Logger LOGGER = LoggerFactory.getLogger(JobScheduler.class); 23 | 24 | private List schedules; 25 | 26 | public JobScheduler(final JobService jobService) { 27 | this(createDefaultSchedules(jobService)); 28 | } 29 | 30 | public JobScheduler(List schedules) { 31 | this.schedules = schedules; 32 | } 33 | 34 | @Deprecated 35 | public JobScheduler(final JobService jobService, final JobInfoRepository jobInfoRepository) { 36 | this(jobService); 37 | } 38 | 39 | private ScheduledExecutorService executorService; 40 | 41 | @PostConstruct 42 | public synchronized void startup() { 43 | LOGGER.info("called startup"); 44 | 45 | if(executorService != null) { 46 | shutdown(); 47 | } 48 | 49 | executorService = Executors.newScheduledThreadPool(schedules.size(),new JobSchedulerThreadFactory()); 50 | 51 | for(JobSchedule schedule: schedules) { 52 | executorService.scheduleAtFixedRate(schedule, 0, schedule.interval(), TimeUnit.MILLISECONDS); 53 | } 54 | 55 | LOGGER.info("finished startup"); 56 | } 57 | 58 | @PreDestroy 59 | public synchronized void shutdown() { 60 | LOGGER.info("called shutdown"); 61 | 62 | if(executorService == null) { 63 | LOGGER.info("shutdown: executor service already removed, stop here."); 64 | return; 65 | } 66 | 67 | 68 | try { 69 | executorService.shutdown(); 70 | executorService.awaitTermination(30, TimeUnit.SECONDS); 71 | } catch (InterruptedException e) { 72 | LOGGER.error("error await termination of tasks: " + e.getMessage(), e); 73 | } 74 | executorService = null; 75 | LOGGER.info("finished shutdown"); 76 | } 77 | 78 | private static List createDefaultSchedules(final JobService jobService) { 79 | List schedules = new ArrayList<>(); 80 | schedules.add(new JobSchedule() { 81 | @Override 82 | public long interval() { 83 | return TimeUnit.MINUTES.toMillis(5); 84 | } 85 | 86 | @Override 87 | public void schedule() { 88 | jobService.cleanupTimedOutJobs(); 89 | } 90 | 91 | @Override 92 | public String getName() { 93 | return "jobInfoRepository.cleanupTimedOutJobs()"; 94 | } 95 | }); 96 | 97 | schedules.add(new JobSchedule() { 98 | @Override 99 | public long interval() { 100 | return TimeUnit.MINUTES.toMillis(1); 101 | } 102 | 103 | @Override 104 | public void schedule() { 105 | jobService.executeQueuedJobs(); 106 | } 107 | @Override 108 | public String getName() { 109 | return "jobService.executeQueuedJobs()"; 110 | } 111 | }); 112 | 113 | schedules.add(new JobSchedule() { 114 | @Override 115 | public long interval() { 116 | return TimeUnit.MINUTES.toMillis(1); 117 | } 118 | @Override 119 | public void schedule() { 120 | jobService.pollRemoteJobs(); 121 | } 122 | @Override 123 | public String getName() { 124 | return "jobService.pollRemoteJobs()"; 125 | } 126 | }); 127 | 128 | schedules.add(new JobSchedule() { 129 | @Override 130 | public long interval() { 131 | return TimeUnit.MINUTES.toMillis(1); 132 | } 133 | @Override 134 | public void schedule() { 135 | jobService.retryFailedJobs(); 136 | } 137 | @Override 138 | public String getName() { 139 | return "jobService.retryFailedJobs()"; 140 | } 141 | }); 142 | 143 | return schedules; 144 | } 145 | 146 | /** 147 | * shameless copy of Executors.DefaultThreadFactory with some adjustments: 148 | * - changed name prefix 149 | * - threads are daemon threads 150 | * - threads run with MAX_PRIORITY 151 | * 152 | */ 153 | private static class JobSchedulerThreadFactory implements ThreadFactory { 154 | private static final AtomicInteger POOL_NUMBER = new AtomicInteger(1); 155 | private final ThreadGroup group; 156 | private final AtomicInteger threadNumber = new AtomicInteger(1); 157 | private final String namePrefix; 158 | 159 | JobSchedulerThreadFactory() { 160 | final SecurityManager s = System.getSecurityManager(); 161 | group = (s == null) ? Thread.currentThread().getThreadGroup() : s.getThreadGroup(); 162 | namePrefix = "jobScheduler-" + POOL_NUMBER.getAndIncrement() + "-thread-"; 163 | } 164 | 165 | public Thread newThread(Runnable r) { 166 | final Thread t = new Thread(group, r, namePrefix + threadNumber.getAndIncrement(), 0); 167 | if (!t.isDaemon()) { 168 | t.setDaemon(true); 169 | } 170 | if (t.getPriority() != Thread.MAX_PRIORITY) { 171 | t.setPriority(Thread.MAX_PRIORITY); 172 | } 173 | return t; 174 | } 175 | } 176 | 177 | 178 | } 179 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/service/RemoteJobExecutorWithScriptTransferService.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service; 2 | 3 | import com.sun.jersey.api.client.Client; 4 | import com.sun.jersey.api.client.ClientHandlerException; 5 | import com.sun.jersey.api.client.UniformInterfaceException; 6 | import com.sun.jersey.api.client.config.ClientConfig; 7 | import com.sun.jersey.api.client.config.DefaultClientConfig; 8 | import de.otto.jobstore.common.RemoteJob; 9 | import de.otto.jobstore.common.RemoteJobStatus; 10 | import de.otto.jobstore.service.exception.JobException; 11 | import de.otto.jobstore.service.exception.JobExecutionException; 12 | import de.otto.jobstore.service.exception.RemoteJobAlreadyRunningException; 13 | import de.otto.jobstore.service.exception.RemoteJobNotRunningException; 14 | import org.apache.commons.compress.utils.IOUtils; 15 | import org.apache.http.Header; 16 | import org.apache.http.HttpResponse; 17 | import org.apache.http.client.HttpClient; 18 | import org.apache.http.client.config.RequestConfig; 19 | import org.apache.http.client.methods.HttpPost; 20 | import org.apache.http.config.SocketConfig; 21 | import org.apache.http.entity.mime.MultipartEntity; 22 | import org.apache.http.entity.mime.content.ByteArrayBody; 23 | import org.apache.http.entity.mime.content.StringBody; 24 | import org.apache.http.impl.client.HttpClients; 25 | import org.apache.http.impl.conn.PoolingHttpClientConnectionManager; 26 | import org.apache.http.util.EntityUtils; 27 | import org.codehaus.jettison.json.JSONException; 28 | import org.slf4j.Logger; 29 | import org.slf4j.LoggerFactory; 30 | 31 | import javax.ws.rs.core.MediaType; 32 | import java.io.ByteArrayOutputStream; 33 | import java.io.IOException; 34 | import java.io.InputStream; 35 | import java.io.UnsupportedEncodingException; 36 | import java.net.URI; 37 | import java.nio.charset.Charset; 38 | 39 | /** 40 | * This class triggers the execution of jobs on a remote server. 41 | * The scripts for execution are sent within the request body. 42 | * 43 | * The remote server has to expose a rest endpoint. 44 | * The multipart request sent to the endpoint consists of two parts: 45 | * 46 | * 1) A binary part containing the tar file: 47 | * Content-Disposition: form-data; name="scripts"; filename="scripts.tar.gz" 48 | * Content-Type: application/octet-stream 49 | * Content-Transfer-Encoding: binary 50 | * 2) A part containing the JSON formatted parameters: 51 | * Content-Disposition: form-data; name="params" 52 | * Content-Type: application/json; charset=UTF-8 53 | * Content-Transfer-Encoding: 8bit 54 | */ 55 | public class RemoteJobExecutorWithScriptTransferService implements RemoteJobExecutor { 56 | 57 | private static final Logger LOGGER = LoggerFactory.getLogger(RemoteJobExecutorWithScriptTransferService.class); 58 | private final RemoteJobExecutorStatusRetriever remoteJobExecutorStatusRetriever; 59 | private String jobExecutorUri; 60 | private Client client; 61 | private HttpClient httpclient; 62 | private TarArchiveProvider tarArchiveProvider; 63 | 64 | @Override 65 | public String getJobExecutorUri() { 66 | return jobExecutorUri; 67 | } 68 | 69 | public RemoteJobExecutorWithScriptTransferService(String jobExecutorUri, TarArchiveProvider tarArchiveProvider) { 70 | this.jobExecutorUri = jobExecutorUri; 71 | this.tarArchiveProvider = tarArchiveProvider; 72 | 73 | // since Flask (with WSGI) does not suppport HTTP 1.1 chunked encoding, turn it off 74 | // see: https://github.com/mitsuhiko/flask/issues/367 75 | final ClientConfig cc = new DefaultClientConfig(); 76 | cc.getProperties().put(ClientConfig.PROPERTY_CHUNKED_ENCODING_SIZE, null); 77 | this.client = Client.create(cc); 78 | remoteJobExecutorStatusRetriever = new RemoteJobExecutorStatusRetriever(client); 79 | 80 | httpclient = createMultithreadSafeClient(); 81 | 82 | } 83 | 84 | private HttpClient createMultithreadSafeClient() { 85 | PoolingHttpClientConnectionManager cm = new PoolingHttpClientConnectionManager(); 86 | cm.setMaxTotal(100); 87 | cm.setDefaultMaxPerRoute(100); 88 | 89 | cm.setDefaultSocketConfig(SocketConfig.custom() 90 | .setSoTimeout(5000) 91 | .build()); 92 | 93 | return HttpClients.custom() 94 | .setConnectionManager(cm) 95 | .build(); 96 | } 97 | 98 | public URI startJob(final RemoteJob job) throws JobException { 99 | final String startUrl = jobExecutorUri + job.name + "/start"; 100 | HttpResponse response = null; 101 | try { 102 | LOGGER.info("ltag=RemoteJobExecutorService.startJob Going to start job: {} ...", startUrl); 103 | 104 | InputStream tarInputStream = createTar(job); 105 | 106 | HttpPost httpPost = createRemoteExecutorMultipartRequest(job, startUrl, tarInputStream); 107 | 108 | response = executeRequest(httpPost); 109 | 110 | int statusCode = response.getStatusLine().getStatusCode(); 111 | String link = extractLink(response); 112 | if (statusCode == 201) { 113 | return createJobUri(link); 114 | } else if (statusCode == 200 || statusCode == 303) { 115 | throw new RemoteJobAlreadyRunningException("Remote job is already running, url=" + startUrl, createJobUri(link)); 116 | } 117 | throw new JobExecutionException("Unable to start remote job: url=" + startUrl + " rc=" + statusCode); 118 | } catch (JSONException e) { 119 | throw new JobExecutionException("Could not create JSON object: " + job, e); 120 | } catch (UniformInterfaceException | ClientHandlerException e) { 121 | throw new JobExecutionException("Problem while starting new job: url=" + startUrl, e); 122 | } finally { 123 | closeResponseConnection(response); 124 | } 125 | } 126 | 127 | private String extractLink(HttpResponse response) { 128 | Header linkHeader = response.getFirstHeader("Link"); 129 | String link; 130 | if (linkHeader == null) { 131 | link = "error"; 132 | } else { 133 | link = linkHeader.getValue(); 134 | } 135 | return link; 136 | } 137 | 138 | private void closeResponseConnection(HttpResponse response) { 139 | if (response != null) { 140 | try { 141 | EntityUtils.consume(response.getEntity()); 142 | } catch (IOException e) { 143 | LOGGER.warn("Could not close response connection", e); 144 | } 145 | } 146 | } 147 | 148 | private HttpResponse executeRequest(HttpPost httpPost) throws JobExecutionException { 149 | HttpResponse response; 150 | try { 151 | httpPost.setConfig(RequestConfig.custom() 152 | .setConnectTimeout(60000) // wait max 60 seconds 153 | .build()); 154 | response = httpclient.execute(httpPost); 155 | } catch (IOException e) { 156 | throw new JobExecutionException("Could not post scripts", e); 157 | } 158 | return response; 159 | } 160 | 161 | private InputStream createTar(RemoteJob job) throws JobExecutionException { 162 | try { 163 | return tarArchiveProvider.getArchiveAsInputStream(job); 164 | } catch (Exception e) { 165 | throw new JobExecutionException("Could not create tar with job scripts (folder: " + job.name + ")", e); 166 | } 167 | } 168 | 169 | public HttpPost createRemoteExecutorMultipartRequest(RemoteJob job, String startUrl, InputStream tarInputStream) throws JSONException, JobExecutionException { 170 | HttpPost httpPost = new HttpPost(startUrl); 171 | 172 | // InputStreamBody tarBody = new InputStreamBody(tarInputStream, "scripts.tar.gz"); 173 | ByteArrayOutputStream baos = new ByteArrayOutputStream(); 174 | try { 175 | IOUtils.copy(tarInputStream, baos); 176 | tarInputStream.close(); 177 | } catch (IOException e) { 178 | throw new JobExecutionException("error copying byte arrays", e); 179 | } 180 | ByteArrayBody tarBody = new ByteArrayBody(baos.toByteArray(), "scripts.tar.gz"); 181 | 182 | MultipartEntity multipartEntity = new MultipartEntity(); 183 | multipartEntity.addPart("scripts", tarBody); 184 | try { 185 | multipartEntity.addPart("params", new StringBody(job.toJsonObject().toString(), MediaType.APPLICATION_JSON, Charset.forName("UTF-8"))); 186 | } catch (UnsupportedEncodingException e) { 187 | throw new JobExecutionException("Could not generate json", e); 188 | } 189 | httpPost.setEntity(multipartEntity); 190 | httpPost.setHeader("Connection", "close"); 191 | httpPost.setHeader("User-Agent", "RemoteJobExecutorService"); 192 | return httpPost; 193 | } 194 | 195 | public void stopJob(URI jobUri) throws JobException { 196 | final String stopUrl = jobUri + "/stop"; 197 | try { 198 | LOGGER.info("ltag=RemoteJobExecutorService.stopJob Going to stop job: {} ...", stopUrl); 199 | client.resource(stopUrl).header("Connection", "close").post(); 200 | } catch (UniformInterfaceException e) { 201 | if (e.getResponse().getStatus() == 403) { 202 | throw new RemoteJobNotRunningException("Remote job is not running: url=" + stopUrl); 203 | } 204 | throw e; 205 | } 206 | } 207 | 208 | public RemoteJobStatus getStatus(final URI jobUri) { 209 | return remoteJobExecutorStatusRetriever.getStatus(jobUri); 210 | } 211 | 212 | public boolean isAlive() { 213 | return remoteJobExecutorStatusRetriever.isAlive(jobExecutorUri); 214 | } 215 | 216 | // ~ 217 | 218 | private URI createJobUri(String path) { 219 | return URI.create(jobExecutorUri).resolve(path); 220 | } 221 | 222 | } 223 | -------------------------------------------------------------------------------- /jobs-core/src/main/java/de/otto/jobstore/common/JobInfo.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.common; 2 | 3 | import com.mongodb.BasicDBObject; 4 | import com.mongodb.DBObject; 5 | import de.otto.jobstore.common.properties.JobInfoProperty; 6 | 7 | import java.util.*; 8 | 9 | public class JobInfo extends AbstractItem { 10 | 11 | private static final long serialVersionUID = 2454224303569320787L; 12 | 13 | public JobInfo(DBObject dbObject) { 14 | super(dbObject); 15 | } 16 | 17 | public JobInfo(String name, String host, String thread, Long maxIdleTime, Long maxExecutionTime, Long retries) { 18 | this(name, host, thread, maxIdleTime, maxExecutionTime, retries, RunningState.QUEUED); 19 | } 20 | 21 | public JobInfo(String name, String host, String thread, Long maxIdleTime, Long maxExecutionTime, Long retries, RunningState state) { 22 | this(name, host, thread, maxIdleTime, maxExecutionTime, retries, state, JobExecutionPriority.CHECK_PRECONDITIONS, Collections.emptyMap()); 23 | } 24 | 25 | public JobInfo(Date dt, String name, String host, String thread, Long maxIdleTime, Long maxExecutionTime, Long retries, RunningState state) { 26 | this(dt, name, host, thread, maxIdleTime, maxExecutionTime, retries, state, JobExecutionPriority.CHECK_PRECONDITIONS, Collections. emptyMap()); 27 | } 28 | 29 | public JobInfo(String name, String host, String thread, Long maxIdleTime, Long maxExecutionTime, Long retries, RunningState state, JobExecutionPriority executionPriority, Map parameters) { 30 | this(new Date(), name, host, thread, maxIdleTime, maxExecutionTime, retries, state, executionPriority, parameters); 31 | } 32 | 33 | public JobInfo(Date dt, String name, String host, String thread, Long maxIdleTime, Long maxExecutionTime, Long retries, RunningState state, JobExecutionPriority executionPriority, Map parameters) { 34 | addProperty(JobInfoProperty.NAME, name); 35 | addProperty(JobInfoProperty.HOST, host); 36 | addProperty(JobInfoProperty.THREAD, thread); 37 | if (state != RunningState.QUEUED) { 38 | addProperty(JobInfoProperty.START_TIME, dt); 39 | } 40 | addProperty(JobInfoProperty.CREATION_TIME, dt); 41 | addProperty(JobInfoProperty.EXECUTION_PRIORITY, executionPriority.name()); 42 | addProperty(JobInfoProperty.RUNNING_STATE, state.name()); 43 | setLastModifiedTime(dt); 44 | addProperty(JobInfoProperty.MAX_IDLE_TIME, maxIdleTime); 45 | addProperty(JobInfoProperty.MAX_EXECUTION_TIME, maxExecutionTime); 46 | addProperty(JobInfoProperty.RETRIES, retries); 47 | 48 | if (parameters != null) { 49 | addProperty(JobInfoProperty.PARAMETERS, new BasicDBObject(parameters)); 50 | } 51 | } 52 | 53 | public boolean hasLowerPriority(JobExecutionPriority priority) { 54 | return getExecutionPriority().hasLowerPriority(priority); 55 | } 56 | 57 | public String getId() { 58 | final Object id = getProperty(JobInfoProperty.ID); 59 | if (id == null) { 60 | return null; 61 | } else { 62 | return id.toString(); 63 | } 64 | } 65 | 66 | public String getName() { 67 | return getProperty(JobInfoProperty.NAME); 68 | } 69 | 70 | public String getHost() { 71 | return getProperty(JobInfoProperty.HOST); 72 | } 73 | 74 | public String getThread() { 75 | return getProperty(JobInfoProperty.THREAD); 76 | } 77 | 78 | public Map getParameters() { 79 | final DBObject parameters = getProperty(JobInfoProperty.PARAMETERS); 80 | if (parameters == null) { 81 | return new HashMap<>(); 82 | } else { 83 | return parameters.toMap(); 84 | } 85 | } 86 | 87 | public void setParameters(Map parameters) { 88 | addProperty(JobInfoProperty.PARAMETERS, parameters); 89 | } 90 | 91 | public Long getMaxIdleTime() { 92 | return getProperty(JobInfoProperty.MAX_IDLE_TIME); 93 | } 94 | 95 | public Long getMaxExecutionTime() { 96 | return getProperty(JobInfoProperty.MAX_EXECUTION_TIME); 97 | } 98 | 99 | public Long getRetries() { 100 | Long retries = getProperty(JobInfoProperty.RETRIES); 101 | 102 | if(retries == null) { 103 | return 0L; 104 | } 105 | return retries; 106 | } 107 | 108 | private Date getJobIdleExceededTime() { 109 | return new Date(getLastModifiedTime().getTime() + getMaxIdleTime()); 110 | } 111 | 112 | private Date getJobExpireTime() { 113 | return new Date(getStartTime().getTime() + getMaxExecutionTime()); 114 | } 115 | 116 | public boolean isTimedOut() { 117 | return isTimedOut(new Date()); 118 | } 119 | 120 | public boolean isTimedOut(Date currentDate) { 121 | if(isStarted()){ 122 | return getJobExpireTime().before(currentDate); 123 | } else { 124 | return false; 125 | } 126 | } 127 | 128 | public boolean isIdleTimeExceeded() { 129 | return isIdleTimeExceeded(new Date()); 130 | } 131 | 132 | public boolean isIdleTimeExceeded(Date currentDate) { 133 | return getJobIdleExceededTime().before(currentDate); 134 | } 135 | 136 | public boolean isStarted() { 137 | return getStartTime() != null; 138 | } 139 | 140 | 141 | public JobExecutionPriority getExecutionPriority() { 142 | final String priority = getProperty(JobInfoProperty.EXECUTION_PRIORITY); 143 | if (priority == null) { 144 | return null; 145 | } else { 146 | return JobExecutionPriority.valueOf(priority); 147 | } 148 | } 149 | 150 | public Date getCreationTime() { 151 | return getProperty(JobInfoProperty.CREATION_TIME); 152 | } 153 | 154 | public Date getStartTime() { 155 | return getProperty(JobInfoProperty.START_TIME); 156 | } 157 | 158 | public Date getFinishTime() { 159 | return getProperty(JobInfoProperty.FINISH_TIME); 160 | } 161 | 162 | public String getResultMessage() { 163 | return getProperty(JobInfoProperty.RESULT_MESSAGE); 164 | } 165 | 166 | public String getStatusMessage() { 167 | return getProperty(JobInfoProperty.STATUS_MESSAGE); 168 | } 169 | 170 | @SuppressWarnings("unchecked") 171 | public Map getAdditionalData() { 172 | final DBObject additionalData = getProperty(JobInfoProperty.ADDITIONAL_DATA); 173 | if (additionalData == null) { 174 | return new HashMap<>(); 175 | } else { 176 | return additionalData.toMap(); 177 | } 178 | } 179 | 180 | public void putAdditionalData(String key, String value) { 181 | final DBObject additionalData; 182 | if (hasProperty(JobInfoProperty.ADDITIONAL_DATA)) { 183 | additionalData = getProperty(JobInfoProperty.ADDITIONAL_DATA); 184 | } else { 185 | additionalData = new BasicDBObject(); 186 | addProperty(JobInfoProperty.ADDITIONAL_DATA, additionalData); 187 | } 188 | additionalData.put(key, value); 189 | } 190 | 191 | public void appendLogLine(LogLine logLine) { 192 | // TODO: should we also only store the most recent MAX_LOGLINES lines? 193 | if (logLine != null) { 194 | final List logLines; 195 | if (hasProperty(JobInfoProperty.LOG_LINES)) { 196 | logLines = getProperty(JobInfoProperty.LOG_LINES); 197 | } else { 198 | logLines = new ArrayList<>(); 199 | addProperty(JobInfoProperty.LOG_LINES, logLines); 200 | } 201 | logLines.add(logLine.toDbObject()); 202 | } 203 | } 204 | 205 | public boolean hasLogLines() { 206 | final List logLines = getProperty(JobInfoProperty.LOG_LINES); 207 | return logLines != null && !logLines.isEmpty(); 208 | } 209 | 210 | public List getLogLines() { 211 | final List logLines = getProperty(JobInfoProperty.LOG_LINES); 212 | if (logLines == null) return Collections.emptyList(); 213 | 214 | final List result = new ArrayList<>(logLines.size()); 215 | for (DBObject logLine : logLines) { 216 | result.add(new LogLine(logLine)); 217 | } 218 | return result; 219 | } 220 | 221 | public List getLastLogLines(int maxLines) { 222 | final List logLines = getProperty(JobInfoProperty.LOG_LINES); 223 | if (logLines == null) return Collections.emptyList(); 224 | 225 | final List result = new ArrayList<>(Math.min(logLines.size(), maxLines)); 226 | int endPos = logLines.size(); 227 | int startPos = Math.max(0, endPos - maxLines); 228 | for (DBObject logLine : logLines.subList(startPos, endPos)) { 229 | result.add(new LogLine(logLine)); 230 | } 231 | return result; 232 | } 233 | 234 | public Date getLastModifiedTime() { 235 | return getProperty(JobInfoProperty.LAST_MODIFICATION_TIME); 236 | } 237 | 238 | public void setLastModifiedTime(Date lastModifiedTime) { 239 | addProperty(JobInfoProperty.LAST_MODIFICATION_TIME, lastModifiedTime); 240 | } 241 | 242 | public String getRunningState() { 243 | return getProperty(JobInfoProperty.RUNNING_STATE); 244 | } 245 | 246 | public boolean isAborted() { 247 | final Boolean aborted = getProperty(JobInfoProperty.ABORTED); 248 | return aborted == null ? false : aborted; 249 | } 250 | 251 | public ResultCode getResultState() { 252 | final String resultState = getProperty(JobInfoProperty.RESULT_STATE); 253 | if (resultState == null) { 254 | return null; 255 | } else { 256 | return ResultCode.valueOf(resultState); 257 | } 258 | } 259 | 260 | public void setResultState(ResultCode resultCode) { 261 | addProperty(JobInfoProperty.RESULT_STATE, resultCode.toString()); 262 | } 263 | 264 | @Override 265 | public String toString() { 266 | return "{\"JobInfo\" : {" + 267 | "\"id\":\"" + getId() + 268 | "\", \"name\":\"" + getName() + 269 | "\", \"host\":\"" + getHost() + 270 | "\", \"thread\":\"" + getThread() + 271 | "\", \"creationTime\":\"" + getCreationTime() + 272 | "\", \"startTime\":\"" + getStartTime() + 273 | "\", \"maxIdleTime\":\"" + getMaxIdleTime() + 274 | "\", \"maxExecutionTime\":\"" + getMaxExecutionTime() + 275 | "\", \"finishTime\":\"" + getFinishTime() + 276 | "\", \"lastModifiedTime\":\"" + getLastModifiedTime() + 277 | "\", \"additionalData\":\"" + getAdditionalData().toString() + 278 | "\"}}"; 279 | } 280 | 281 | } 282 | -------------------------------------------------------------------------------- /jobs-api/src/test/java/de/otto/jobstore/web/JobInfoResourceTest.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.web; 2 | 3 | import com.mongodb.BasicDBObject; 4 | import com.sun.jersey.api.uri.UriBuilderImpl; 5 | import com.sun.jersey.core.util.MultivaluedMapImpl; 6 | import de.otto.jobstore.common.JobExecutionPriority; 7 | import de.otto.jobstore.common.JobInfo; 8 | import de.otto.jobstore.common.RunningState; 9 | import de.otto.jobstore.common.properties.JobInfoProperty; 10 | import de.otto.jobstore.service.JobInfoService; 11 | import de.otto.jobstore.service.JobService; 12 | import de.otto.jobstore.service.exception.JobAlreadyQueuedException; 13 | import de.otto.jobstore.service.exception.JobAlreadyRunningException; 14 | import de.otto.jobstore.service.exception.JobNotRegisteredException; 15 | import de.otto.jobstore.service.exception.JobServiceNotActiveException; 16 | import de.otto.jobstore.web.representation.JobInfoRepresentation; 17 | import de.otto.jobstore.web.representation.JobNameRepresentation; 18 | import org.apache.abdera.model.Entry; 19 | import org.apache.abdera.model.Feed; 20 | import org.testng.annotations.BeforeMethod; 21 | import org.testng.annotations.Test; 22 | 23 | import javax.ws.rs.core.MultivaluedMap; 24 | import javax.ws.rs.core.Response; 25 | import javax.ws.rs.core.UriInfo; 26 | import javax.xml.bind.JAXBContext; 27 | import javax.xml.bind.Unmarshaller; 28 | import java.io.StringReader; 29 | import java.util.*; 30 | 31 | import static de.otto.jobstore.TestSetup.localJobDefinition; 32 | import static de.otto.jobstore.TestSetup.remoteJobDefinition; 33 | import static org.mockito.Matchers.*; 34 | import static org.mockito.Mockito.doThrow; 35 | import static org.mockito.Mockito.mock; 36 | import static org.mockito.Mockito.when; 37 | import static org.testng.AssertJUnit.assertEquals; 38 | import static org.testng.AssertJUnit.assertTrue; 39 | 40 | @SuppressWarnings("unchecked") 41 | public class JobInfoResourceTest { 42 | 43 | private JobInfoResource jobInfoResource; 44 | private JobService jobService; 45 | private JobInfoService jobInfoService; 46 | private UriInfo uriInfo; 47 | private JobInfo JOB_INFO; 48 | 49 | @BeforeMethod 50 | public void setUp() throws Exception { 51 | jobService = mock(JobService.class); 52 | jobInfoService = mock(JobInfoService.class); 53 | jobInfoResource = new JobInfoResource(jobService, jobInfoService); 54 | 55 | uriInfo = mock(UriInfo.class); 56 | when(uriInfo.getBaseUriBuilder()).thenReturn(new UriBuilderImpl()); 57 | JOB_INFO = new JobInfo(new BasicDBObject().append(JobInfoProperty.ID.val(), "1234").append(JobInfoProperty.NAME.val(), "foo")); 58 | } 59 | 60 | @Test 61 | public void testGetJobs() throws Exception { 62 | JAXBContext ctx = JAXBContext.newInstance(JobNameRepresentation.class); 63 | Unmarshaller unmarshaller = ctx.createUnmarshaller(); 64 | 65 | when(jobService.listJobNames()).thenReturn(Arrays.asList("bar", "foo")); 66 | Response response = jobInfoResource.getJobs(uriInfo); 67 | assertEquals(200, response.getStatus()); 68 | Feed feed = (Feed) response.getEntity(); 69 | 70 | List entries = feed.getEntries(); 71 | assertEquals(2, entries.size()); 72 | Entry bar = entries.get(0); 73 | JobNameRepresentation barRep = (JobNameRepresentation) unmarshaller.unmarshal(new StringReader(bar.getContent())); 74 | assertEquals("bar", barRep.getName()); 75 | Entry foo = entries.get(1); 76 | JobNameRepresentation fooRep = (JobNameRepresentation) unmarshaller.unmarshal(new StringReader(foo.getContent())); 77 | assertEquals("foo", fooRep.getName()); 78 | } 79 | 80 | @Test 81 | public void testGetJobsEmpty() throws Exception { 82 | when(jobService.listJobNames()).thenReturn(new HashSet()); 83 | 84 | Response response = jobInfoResource.getJobs(uriInfo); 85 | assertEquals(200, response.getStatus()); 86 | Feed feed = (Feed) response.getEntity(); 87 | 88 | List entries = feed.getEntries(); 89 | assertEquals(0, entries.size()); 90 | } 91 | 92 | @Test 93 | public void testExecuteJobWhichIsNotRegistered() throws Exception { 94 | when(jobService.executeJob(eq("foo"), eq(JobExecutionPriority.FORCE_EXECUTION), anyMap())).thenThrow(new JobNotRegisteredException("")); 95 | //when(jobService.executeJob("foo", false)).thenThrow(new JobNotRegisteredException("")); 96 | 97 | Response response = jobInfoResource.executeJob("foo", uriInfo); 98 | assertEquals(404, response.getStatus()); 99 | } 100 | 101 | @Test 102 | public void testExecuteJobWhichIsAlreadyQueued() throws Exception { 103 | when(jobService.executeJob(eq("foo"), eq(JobExecutionPriority.FORCE_EXECUTION), anyMap())).thenThrow(new JobAlreadyQueuedException("")); 104 | 105 | Response response = jobInfoResource.executeJob("foo", uriInfo); 106 | assertEquals(409, response.getStatus()); 107 | } 108 | 109 | @Test 110 | public void testExecuteJobWhichIsAlreadyRunning() throws Exception { 111 | when(jobService.executeJob(eq("foo"), eq(JobExecutionPriority.FORCE_EXECUTION), anyMap())).thenThrow(new JobAlreadyRunningException("")); 112 | 113 | Response response = jobInfoResource.executeJob("foo", uriInfo); 114 | assertEquals(409, response.getStatus()); 115 | } 116 | 117 | @Test 118 | public void testExecuteJobOnInactiveServiceShouldResultInBadRequestResponse() throws Exception { 119 | when(jobService.executeJob(eq("foo"), eq(JobExecutionPriority.FORCE_EXECUTION), anyMap())).thenThrow(new JobServiceNotActiveException("not active")); 120 | 121 | Response response = jobInfoResource.executeJob("foo", uriInfo); 122 | assertEquals(400, response.getStatus()); 123 | } 124 | 125 | @Test 126 | public void testExecuteJob() throws Exception { 127 | when(jobService.executeJob(eq("foo"), eq(JobExecutionPriority.FORCE_EXECUTION), anyMap())).thenReturn("1234"); 128 | when(jobInfoService.getById("1234")).thenReturn(JOB_INFO); 129 | 130 | Response response = jobInfoResource.executeJob("foo", uriInfo); 131 | assertEquals(201, response.getStatus()); 132 | } 133 | 134 | @Test 135 | public void testGetJob() throws Exception { 136 | when(jobInfoService.getById("1234")).thenReturn(JOB_INFO); 137 | 138 | Response response = jobInfoResource.getJob("foo", "1234"); 139 | assertEquals(200, response.getStatus()); 140 | } 141 | 142 | @Test 143 | public void testGetJobNotExisting() throws Exception { 144 | when(jobInfoService.getById("1234")).thenReturn(null); 145 | 146 | Response response = jobInfoResource.getJob("foo", "1234"); 147 | assertEquals(404, response.getStatus()); 148 | } 149 | 150 | @Test 151 | public void testGetJobMismatchingName() throws Exception { 152 | when(jobInfoService.getById("1234")).thenReturn(JOB_INFO); 153 | 154 | Response response = jobInfoResource.getJob("bar", "1234"); 155 | assertEquals(404, response.getStatus()); 156 | } 157 | 158 | @Test 159 | public void testGetJobsByName() throws Exception { 160 | JAXBContext ctx = JAXBContext.newInstance(JobInfoRepresentation.class); 161 | Unmarshaller unmarshaller = ctx.createUnmarshaller(); 162 | when(jobInfoService.getByName("foo", 5)).thenReturn(createJobs(5, "foo")); 163 | 164 | Response response = jobInfoResource.getJobsByName("foo", 5, uriInfo); 165 | assertEquals(200, response.getStatus()); 166 | Feed feed = (Feed) response.getEntity(); 167 | 168 | List entries = feed.getEntries(); 169 | assertEquals(5, entries.size()); 170 | 171 | Entry foo = entries.get(0); 172 | JobInfoRepresentation fooRep = (JobInfoRepresentation) unmarshaller.unmarshal(new StringReader(foo.getContent())); 173 | assertEquals("0", fooRep.getId()); 174 | assertEquals("foo", fooRep.getName()); 175 | } 176 | 177 | @Test 178 | public void testGetJobsByEmpty() throws Exception { 179 | when(jobInfoService.getByName("foo", 5)).thenReturn(new ArrayList()); 180 | 181 | Response response = jobInfoResource.getJobsByName("foo", 5, uriInfo); 182 | assertEquals(200, response.getStatus()); 183 | Feed feed = (Feed) response.getEntity(); 184 | 185 | List entries = feed.getEntries(); 186 | assertEquals(0, entries.size()); 187 | } 188 | 189 | @Test 190 | @SuppressWarnings("unchecked") 191 | public void testGetJobHistory() throws Exception { 192 | when(jobService.listJobNames()).thenReturn(Arrays.asList("foo")); 193 | when(jobInfoService.getByNameAndTimeRange(anyString(), any(Date.class), any(Date.class), any(Set.class))). 194 | thenReturn(createJobs(5, "foo")); 195 | 196 | Response response = jobInfoResource.getJobsHistory(5, null, new HashSet<>(jobService.listJobNames())); 197 | assertEquals(200, response.getStatus()); 198 | Map> history = (Map>) response.getEntity(); 199 | assertEquals(1, history.size()); 200 | assertEquals(5, history.get("foo").size()); 201 | } 202 | 203 | @Test 204 | @SuppressWarnings("unchecked") 205 | public void testGetJobHistory2() throws Exception { 206 | when(jobService.listJobNames()).thenReturn(Arrays.asList("foo")); 207 | when(jobInfoService.getByNameAndTimeRange(anyString(), any(Date.class), any(Date.class), any(Set.class))). 208 | thenReturn(createJobs(5, "foo")); 209 | 210 | Response response = jobInfoResource.getJobsHistory(5, null, null); 211 | assertEquals(200, response.getStatus()); 212 | Map> history = (Map>) response.getEntity(); 213 | assertEquals(1, history.size()); 214 | assertEquals(0, history.get("foo").size()); 215 | } 216 | 217 | @Test 218 | public void testStatusJob() throws Exception { 219 | Response response = jobInfoResource.statusOfAllJobs(); 220 | assertEquals(200, response.getStatus()); 221 | assertTrue(((String)response.getEntity()).contains("localRunningJobs")); 222 | } 223 | 224 | @Test 225 | public void testStatusWithNoRunningJobs() throws Exception { 226 | when(jobService.listJobNames()).thenReturn(Arrays.asList("local", "remote")); 227 | when(jobService.getJobDefinitionByName("local")).thenReturn(localJobDefinition("local", 10)); 228 | when(jobService.getJobDefinitionByName("remote")).thenReturn(remoteJobDefinition("remote", 10, 10)); 229 | 230 | 231 | Response response = jobInfoResource.statusOfAllJobs(); 232 | assertEquals(200, response.getStatus()); 233 | assertTrue(((String)response.getEntity()).contains("\"localRunningJobs\" : false")); 234 | } 235 | 236 | @Test 237 | public void testStatusWithLocalRunningJobs() throws Exception { 238 | when(jobService.listJobNames()).thenReturn(Arrays.asList("local", "remote")); 239 | when(jobService.getJobDefinitionByName("local")).thenReturn(localJobDefinition("local", 10)); 240 | when(jobService.getJobDefinitionByName("remote")).thenReturn(remoteJobDefinition("remote", 10, 10)); 241 | 242 | JobInfo jobInfo = mock(JobInfo.class); 243 | when(jobInfoService.getByNameAndRunningState("local", RunningState.RUNNING)).thenReturn(jobInfo); 244 | 245 | Response response = jobInfoResource.statusOfAllJobs(); 246 | assertEquals(200, response.getStatus()); 247 | assertTrue(((String)response.getEntity()).contains("\"localRunningJobs\" : true")); 248 | } 249 | 250 | @Test 251 | public void testStatusWithRemoteRunningJobs() throws Exception { 252 | when(jobService.listJobNames()).thenReturn(Arrays.asList("local", "remote")); 253 | when(jobService.getJobDefinitionByName("local")).thenReturn(localJobDefinition("local", 10)); 254 | when(jobService.getJobDefinitionByName("remote")).thenReturn(remoteJobDefinition("remote", 10, 10)); 255 | 256 | JobInfo jobInfo = mock(JobInfo.class); 257 | when(jobInfoService.getByNameAndRunningState("remote", RunningState.RUNNING)).thenReturn(jobInfo); 258 | 259 | Response response = jobInfoResource.statusOfAllJobs(); 260 | assertEquals(200, response.getStatus()); 261 | assertTrue(((String)response.getEntity()).contains("\"localRunningJobs\" : false")); 262 | } 263 | 264 | 265 | @Test 266 | public void testEnablingJob() throws Exception { 267 | Response response = jobInfoResource.enableJob("test"); 268 | assertEquals(200, response.getStatus()); 269 | assertTrue(((String)response.getEntity()).contains("enabled")); 270 | } 271 | 272 | @Test 273 | public void testDisablingJob() throws Exception { 274 | Response response = jobInfoResource.disableJob("test"); 275 | assertEquals(200, response.getStatus()); 276 | assertTrue(((String)response.getEntity()).contains("disabled")); 277 | } 278 | 279 | @Test 280 | public void testDisablingNotRegisteredJob() throws Exception { 281 | doThrow(new JobNotRegisteredException("")).when(jobService).setJobExecutionEnabled("test", false); 282 | Response response = jobInfoResource.disableJob("test"); 283 | assertEquals(404, response.getStatus()); 284 | } 285 | 286 | @Test 287 | public void testExtractParameters() throws Exception { 288 | MultivaluedMap queryParameters = new MultivaluedMapImpl(); 289 | queryParameters.put("key1", Arrays.asList("v1")); 290 | queryParameters.put("key2", Arrays.asList("v2")); 291 | 292 | Map parameters 293 | = jobInfoResource.extractFirstParameters(queryParameters); 294 | assertEquals(2, parameters.size(), 1); 295 | assertEquals("v1", parameters.get("key1")); 296 | assertEquals("v2", parameters.get("key2")); 297 | } 298 | 299 | @Test(expectedExceptions = IllegalArgumentException.class) 300 | public void testExtractParametersFailOnNull() throws Exception { 301 | MultivaluedMap queryParameters = new MultivaluedMapImpl(); 302 | queryParameters.put("key1", Arrays.asList("v1")); 303 | queryParameters.put("key2", null); 304 | 305 | jobInfoResource.extractFirstParameters(queryParameters); 306 | } 307 | 308 | @Test(expectedExceptions = IllegalArgumentException.class) 309 | public void testExtractParametersFailOnMultipleValuesPerKey() throws Exception { 310 | MultivaluedMap queryParameters = new MultivaluedMapImpl(); 311 | queryParameters.put("key1", Arrays.asList("v1", "v2")); 312 | 313 | jobInfoResource.extractFirstParameters(queryParameters); 314 | } 315 | 316 | // ~~ 317 | 318 | private List createJobs(int number, String name) { 319 | List jobs = new ArrayList<>(); 320 | for (int i = 0; i < number; i++) { 321 | jobs.add(new JobInfo(new BasicDBObject().append(JobInfoProperty.ID.val(), String.valueOf(i)). 322 | append(JobInfoProperty.NAME.val(), name))); 323 | } 324 | return jobs; 325 | } 326 | 327 | } 328 | -------------------------------------------------------------------------------- /jobs-core/src/test/java/de/otto/jobstore/service/JobServiceIntegrationTest.java: -------------------------------------------------------------------------------- 1 | package de.otto.jobstore.service; 2 | 3 | import de.otto.jobstore.TestSetup; 4 | import de.otto.jobstore.common.*; 5 | import de.otto.jobstore.common.properties.JobInfoProperty; 6 | import de.otto.jobstore.repository.JobDefinitionRepository; 7 | import de.otto.jobstore.repository.JobInfoRepository; 8 | import de.otto.jobstore.service.exception.JobException; 9 | import de.otto.jobstore.service.exception.JobExecutionException; 10 | import org.springframework.test.context.ContextConfiguration; 11 | import org.springframework.test.context.testng.AbstractTestNGSpringContextTests; 12 | import org.testng.annotations.BeforeMethod; 13 | import org.testng.annotations.Test; 14 | 15 | import javax.annotation.Resource; 16 | import java.net.URI; 17 | import java.util.*; 18 | import java.util.concurrent.CountDownLatch; 19 | import java.util.concurrent.ExecutorService; 20 | import java.util.concurrent.Executors; 21 | 22 | import static org.mockito.Matchers.any; 23 | import static org.mockito.Mockito.*; 24 | import static org.testng.AssertJUnit.*; 25 | import static org.testng.AssertJUnit.assertEquals; 26 | 27 | @ContextConfiguration(locations = {"classpath:spring/jobs-context.xml"}) 28 | public class JobServiceIntegrationTest extends AbstractTestNGSpringContextTests { 29 | 30 | @Resource(name = "jobServiceWithoutRemoteJobExecutorService") 31 | private JobService jobService; 32 | 33 | @Resource 34 | private JobInfoRepository jobInfoRepository; 35 | 36 | @Resource 37 | private JobInfoService jobInfoService; 38 | 39 | @Resource 40 | private JobDefinitionRepository jobDefinitionRepository; 41 | 42 | private RemoteJobExecutorService remoteJobExecutorService = mock(RemoteJobExecutorService.class); 43 | private JobRunnable jobRunnable; 44 | 45 | private static final String JOB_NAME_1 = "test_job_1"; 46 | private static final String JOB_NAME_2 = "test_job_2"; 47 | private static final String JOB_NAME_3 = "test_job_3"; 48 | private static final Map PARAMETERS = Collections.singletonMap("paramK", "paramV"); 49 | private static final URI REMOTE_JOB_URI = URI.create("http://www.example.com"); 50 | 51 | @BeforeMethod 52 | public void setUp() throws Exception { 53 | jobService.clean(); 54 | jobInfoRepository.clear(true); 55 | jobDefinitionRepository.addOrUpdate(StoredJobDefinition.JOB_EXEC_SEMAPHORE); 56 | reset(remoteJobExecutorService); 57 | } 58 | 59 | @Test 60 | public void testExecutingRemoteJob() throws Exception { 61 | jobRunnable = TestSetup.remoteJobRunnable(remoteJobExecutorService, jobInfoService, PARAMETERS, 62 | TestSetup.remoteJobDefinition(JOB_NAME_3, 0, 0)); 63 | jobService.registerJob(jobRunnable); 64 | reset(remoteJobExecutorService); 65 | when(remoteJobExecutorService.startJob(any(RemoteJob.class))).thenReturn(REMOTE_JOB_URI); 66 | String id = jobService.executeJob(JOB_NAME_3); 67 | Thread.sleep(1000); 68 | // job should be started 69 | assertNotNull(id); 70 | JobInfo jobInfo = jobInfoRepository.findById(id); 71 | // job should still be running 72 | assertEquals(RunningState.RUNNING, RunningState.valueOf(jobInfo.getRunningState())); 73 | assertEquals(PARAMETERS, jobInfo.getParameters()); 74 | assertEquals(REMOTE_JOB_URI.toString(), jobInfo.getAdditionalData().get(JobInfoProperty.REMOTE_JOB_URI.val())); 75 | 76 | // testPollingRunningRemoteJob 77 | reset(remoteJobExecutorService); 78 | List logLines = new ArrayList<>(); 79 | Collections.addAll(logLines, "log l.1", "log l.2"); 80 | when(remoteJobExecutorService.getStatus(any(URI.class))).thenReturn(new RemoteJobStatus(RemoteJobStatus.Status.RUNNING, logLines, "bar")); 81 | // verify(remoteJobExecutorService, times(1)). 82 | 83 | jobService.pollRemoteJobs(); 84 | 85 | jobInfo = jobInfoRepository.findByNameAndRunningState(JOB_NAME_3, RunningState.RUNNING); 86 | assertEquals(RunningState.RUNNING, RunningState.valueOf(jobInfo.getRunningState())); 87 | assertEquals(REMOTE_JOB_URI.toString(), jobInfo.getAdditionalData().get(JobInfoProperty.REMOTE_JOB_URI.val())); 88 | assertEquals(2, jobInfo.getLogLines().size()); 89 | assertEquals("bar", jobInfo.getStatusMessage()); 90 | 91 | //testPollingFinishedRemoteJob 92 | reset(remoteJobExecutorService); 93 | when(remoteJobExecutorService.getStatus(any(URI.class))).thenReturn( 94 | new RemoteJobStatus(RemoteJobStatus.Status.FINISHED, logLines, new RemoteJobResult(true, 0, "done"), "date")); 95 | 96 | jobService.pollRemoteJobs(); 97 | 98 | jobInfo = jobInfoRepository.findByName(JOB_NAME_3, 1).get(0); 99 | assertTrue(jobInfo.getRunningState().startsWith("FINISHED")); 100 | assertEquals(REMOTE_JOB_URI.toString(), jobInfo.getAdditionalData().get(JobInfoProperty.REMOTE_JOB_URI.val())); 101 | assertEquals(ResultCode.SUCCESSFUL, jobInfo.getResultState()); 102 | assertEquals(2, jobInfo.getLogLines().size()); 103 | assertEquals("done", jobInfo.getResultMessage()); 104 | } 105 | 106 | @Test 107 | public void testExecuteJobWhichViolatesRunningConstraints() throws Exception { 108 | jobService.registerJob(new LocalJobRunnableMock(JOB_NAME_1)); 109 | jobService.registerJob(new LocalJobRunnableMock(JOB_NAME_2)); 110 | Set constraint = new HashSet<>(); 111 | constraint.add(JOB_NAME_1); constraint.add(JOB_NAME_2); 112 | jobService.addRunningConstraint(constraint); 113 | 114 | String id1 = jobService.executeJob(JOB_NAME_1); 115 | String id2 = jobService.executeJob(JOB_NAME_2); 116 | 117 | JobInfo jobInfo1 = jobInfoRepository.findById(id1); 118 | assertNotNull(jobInfo1); 119 | assertEquals(RunningState.RUNNING.name(), jobInfo1.getRunningState()); 120 | 121 | JobInfo jobInfo2 = jobInfoRepository.findById(id2); 122 | assertNotNull(jobInfo2); 123 | assertEquals(RunningState.QUEUED.name(), jobInfo2.getRunningState()); 124 | } 125 | 126 | @Test 127 | public void testExecuteJobFailsWithConnectionException() throws Exception { 128 | jobRunnable = TestSetup.remoteJobRunnable(remoteJobExecutorService, jobInfoService, PARAMETERS, 129 | TestSetup.remoteJobDefinition(JOB_NAME_3, 0, 0)); 130 | jobService.registerJob(jobRunnable); 131 | reset(remoteJobExecutorService); 132 | when(remoteJobExecutorService.startJob(any(RemoteJob.class))).thenThrow(new JobExecutionException("Error connecting to host")); 133 | String id = jobService.executeJob(JOB_NAME_3); 134 | Thread.sleep(1000); 135 | JobInfo jobInfo = jobInfoRepository.findById(id); 136 | assertTrue("Expected job to be finished but it is: " + jobInfo.getRunningState(), jobInfo.getRunningState().startsWith("FINISHED")); 137 | assertEquals(ResultCode.FAILED, jobInfo.getResultState()); 138 | } 139 | 140 | @Test 141 | public void testIsJobExecutionDisabledReturnsCorrectStatus() throws Exception { 142 | jobRunnable = TestSetup.localJobRunnable(JOB_NAME_1, 1000); 143 | jobService.registerJob(jobRunnable); 144 | 145 | jobService.setJobExecutionEnabled(JOB_NAME_1, false); 146 | assertFalse(jobService.isJobExecutionEnabled(JOB_NAME_1)); 147 | 148 | jobService.setJobExecutionEnabled(JOB_NAME_1, true); 149 | assertTrue(jobService.isJobExecutionEnabled(JOB_NAME_1)); 150 | } 151 | 152 | @Test 153 | public void testIsExecutionDisabledReturnsCorrectStatus() throws Exception { 154 | jobService.setExecutionEnabled(false); 155 | assertFalse(jobService.isExecutionEnabled()); 156 | 157 | jobService.setExecutionEnabled(true); 158 | assertTrue(jobService.isExecutionEnabled()); 159 | } 160 | 161 | @Test 162 | public void testIfJobReactsOnTimeoutConditionAndIsMarkedAsTimeoutAfterwards() throws Throwable { 163 | 164 | final CountDownLatch c = new CountDownLatch(1); 165 | 166 | JobRunnable job = new AbstractLocalJobRunnable() { 167 | private AbstractLocalJobDefinition localJobDefinition = new AbstractLocalJobDefinition() { 168 | @Override 169 | public String getName() { 170 | return JOB_NAME_1; 171 | } 172 | 173 | @Override 174 | public long getMaxIdleTime() { 175 | return 0; 176 | } 177 | 178 | @Override 179 | public long getMaxExecutionTime() { 180 | return 0; 181 | } 182 | 183 | @Override 184 | public boolean isAbortable() { 185 | return true; 186 | } 187 | }; 188 | 189 | @Override 190 | public JobDefinition getJobDefinition() { 191 | return localJobDefinition; 192 | } 193 | 194 | @Override 195 | public void execute(JobExecutionContext context) throws JobException { 196 | 197 | try { 198 | try { 199 | Thread.sleep(100); 200 | } catch (InterruptedException e) { 201 | } 202 | 203 | context.checkForAbort(); 204 | } finally { 205 | c.countDown(); 206 | } 207 | } 208 | }; 209 | 210 | jobService.registerJob(job); 211 | String jobId = jobService.executeJob(job.getJobDefinition().getName()); 212 | c.await(); 213 | 214 | 215 | int count = 10; 216 | while (count-- > 0) { 217 | JobInfo jobInfo = jobInfoRepository.findById(jobId); 218 | if (RunningState.RUNNING.name().equals(jobInfo.getRunningState())) { 219 | try { 220 | Thread.sleep(100); 221 | } catch (InterruptedException e) { 222 | } 223 | } else { 224 | break; 225 | } 226 | } 227 | 228 | JobInfo jobInfo = jobInfoRepository.findById(jobId); 229 | assertEquals(ResultCode.TIMED_OUT, jobInfo.getResultState()); 230 | 231 | } 232 | 233 | 234 | ExecutorService executors = Executors.newFixedThreadPool(2); 235 | 236 | @Test(enabled = false) 237 | public void twoThreadsTryingToExecuteAJobShouldResultInOnlyOneExecution() throws Exception { 238 | for (int i = 0; i < 1000; i++) { 239 | jobService.registerJob(new LocalJobRunnableMock(JOB_NAME_1)); 240 | 241 | executors.submit(new JobExecutionRunnable()); 242 | executors.submit(new JobExecutionRunnable()); 243 | 244 | Thread.sleep(100); 245 | 246 | final List jobInfos = jobInfoRepository.findByName(JOB_NAME_1, 10); 247 | assertEquals("Only one job expected in database as the second thread should not create a job when one already exists.", 1, jobInfos.size()); 248 | jobInfoRepository.clear(false); 249 | } 250 | } 251 | 252 | @Test(enabled = false) 253 | public void twoThreadsTryingToExecuteWhileAnotherJobWhichViolatesRunningConstraints() throws Exception { 254 | Set runningConstraints = new HashSet<>(); runningConstraints.add(JOB_NAME_1); runningConstraints.add(JOB_NAME_2); 255 | for (int i = 0; i < 1000; i++) { 256 | jobService.registerJob(new LocalJobRunnableMock(JOB_NAME_2)); 257 | jobService.executeJob(JOB_NAME_2); 258 | 259 | jobService.registerJob(new LocalJobRunnableMock(JOB_NAME_1)); 260 | 261 | executors.submit(new JobExecutionRunnable()); 262 | executors.submit(new JobExecutionRunnable()); 263 | 264 | Thread.sleep(100); 265 | 266 | final List jobInfos = jobInfoRepository.findByName(JOB_NAME_1, 10); 267 | assertEquals("Only one job expected in database.", 1, jobInfos.size()); 268 | jobInfoRepository.clear(false); 269 | } 270 | 271 | } 272 | 273 | @Test 274 | public void testIfRetryDoesNotOccur() throws Exception { 275 | JobDefinition jobDefinition = TestSetup.localJobDefinition(JOB_NAME_1, 1000, 3); 276 | JobRunnable jobRunnable = TestSetup.localJobRunnable(jobDefinition, null); 277 | jobService.registerJob(jobRunnable); 278 | 279 | // no job yet started, should not start one 280 | jobService.doRetryFailedJobs(); 281 | JobInfo jobInfo1 = jobInfoRepository.findMostRecent(JOB_NAME_1); 282 | assertNull(jobInfo1); 283 | 284 | String id = jobService.executeJob(JOB_NAME_1); 285 | JobInfo jobInfo2 = jobInfoRepository.findById(id); 286 | 287 | // no new job started, so last jobInfo should be same as current found jobInfo 288 | jobService.doRetryFailedJobs(); 289 | JobInfo jobInfo3 = jobInfoRepository.findMostRecent(JOB_NAME_1); 290 | assertEquals(new Long(3), jobInfo3.getRetries()); 291 | assertEquals(jobInfo2.getId(), jobInfo3.getId()); 292 | 293 | } 294 | 295 | public void testIfRetryWorks() throws Exception { 296 | JobDefinition jobDefinition = TestSetup.localJobDefinition(JOB_NAME_1, 1000, 3); 297 | JobRunnable jobRunnable = TestSetup.localJobRunnable(jobDefinition, new JobExecutionException("We shall fail")); 298 | jobService.registerJob(jobRunnable); 299 | 300 | String id1 = jobService.executeJob(JOB_NAME_1); 301 | 302 | JobInfo jobInfo1 = jobInfoRepository.findById(id1); 303 | //assertNotNull(jobInfo1); 304 | //assertEquals(RunningState.RUNNING.name(), jobInfo1.getRunningState()); 305 | 306 | jobService.doRetryFailedJobs(); 307 | JobInfo jobInfo2 = jobInfoRepository.findMostRecent(JOB_NAME_1); 308 | assertEquals(new Long(1), jobInfo2.getRetries()); 309 | 310 | jobService.doRetryFailedJobs(); 311 | JobInfo jobInfo3 = jobInfoRepository.findMostRecent(JOB_NAME_1); 312 | assertEquals(new Long(2), jobInfo3.getRetries()); 313 | 314 | jobService.doRetryFailedJobs(); 315 | JobInfo jobInfo4 = jobInfoRepository.findMostRecent(JOB_NAME_1); 316 | assertEquals(new Long(3), jobInfo4.getRetries()); 317 | 318 | // no new job started, so last jobInfo should be same as current found jobInfo 319 | jobService.doRetryFailedJobs(); 320 | JobInfo jobInfo5 = jobInfoRepository.findMostRecent(JOB_NAME_1); 321 | assertEquals(jobInfo4.getId(), jobInfo5.getId()); 322 | } 323 | 324 | 325 | class JobExecutionRunnable implements Runnable { 326 | 327 | @Override 328 | public void run() { 329 | try { 330 | jobService.executeJob(JOB_NAME_1); 331 | } catch (JobException e) { 332 | throw new RuntimeException(e); 333 | } 334 | } 335 | } 336 | 337 | class LocalJobRunnableMock extends AbstractLocalJobRunnable { 338 | 339 | private JobDefinition localJobDefinition; 340 | 341 | LocalJobRunnableMock(JobDefinition jobDefinition) { 342 | localJobDefinition = jobDefinition; 343 | } 344 | 345 | LocalJobRunnableMock(String name) { 346 | localJobDefinition = TestSetup.localJobDefinition(name, 1000); 347 | } 348 | 349 | @Override 350 | public JobDefinition getJobDefinition() { 351 | return localJobDefinition; 352 | } 353 | 354 | @Override 355 | public void execute(JobExecutionContext context) throws JobException { 356 | try { 357 | Thread.sleep(1000); 358 | } catch (InterruptedException e) {} 359 | } 360 | } 361 | 362 | } 363 | --------------------------------------------------------------------------------