specs = Lists.newArrayList();
51 |
52 | /**
53 | * Return a new builder
54 | *
55 | * @return new builder
56 | */
57 | public static WorkflowManagerBuilder builder()
58 | {
59 | return new WorkflowManagerBuilder();
60 | }
61 |
62 | /**
63 | * required
64 | * Set the Curator instance to use. In addition
65 | * to the Curator instance, specify a namespace for the workflow and a version. The namespace
66 | * and version combine to create a unique workflow. All instances using the same namespace and version
67 | * are logically part of the same workflow.
68 | *
69 | * @param curator Curator instance
70 | * @param namespace workflow namespace
71 | * @param version workflow version
72 | * @return this (for chaining)
73 | */
74 | public WorkflowManagerBuilder withCurator(CuratorFramework curator, String namespace, String version)
75 | {
76 | curator = Preconditions.checkNotNull(curator, "curator cannot be null");
77 | namespace = Preconditions.checkNotNull(namespace, "namespace cannot be null");
78 | version = Preconditions.checkNotNull(version, "version cannot be null");
79 | this.curator = curator.usingNamespace(namespace + "-" + version);
80 | return this;
81 | }
82 |
83 | /**
84 | *
85 | * Adds a pool of task executors for a given task type to this instance of
86 | * the workflow. The specified number of executors are allocated. Call this
87 | * method multiple times to allocate executors for the various types of tasks
88 | * that will be used in this workflow. You can choose to have all workflow
89 | * instances execute all task types or target certain task types to certain instances.
90 | *
91 | *
92 | *
93 | * qty is the maximum concurrency for the given type of task for this instance.
94 | * The logical concurrency for a given task type is the total qty of all instances in the
95 | * workflow. e.g. if there are 3 instances in the workflow and instance A has 2 executors
96 | * for task type "a", instance B has 3 executors for task type "a" and instance C has no
97 | * executors for task type "a", the maximum concurrency for task type "a" is 5.
98 | *
99 | *
100 | *
101 | * IMPORTANT: every workflow cluster must have at least one instance that has task executor(s)
102 | * for each task type that will be submitted to the workflow. i.e workflows will stall
103 | * if there is no executor for a given task type.
104 | *
105 | *
106 | * @param taskExecutor the executor
107 | * @param qty the number of instances for this pool
108 | * @param taskType task type
109 | * @return this (for chaining)
110 | */
111 | public WorkflowManagerBuilder addingTaskExecutor(TaskExecutor taskExecutor, int qty, TaskType taskType)
112 | {
113 | specs.add(new TaskExecutorSpec(taskExecutor, qty, taskType));
114 | return this;
115 | }
116 |
117 | /**
118 | * optional
119 | *
120 | * Used in reporting. This will be the value recorded as tasks are executed. Via reporting,
121 | * you can determine which instance has executed a given task.
122 | *
123 | *
124 | *
125 | * Default is: InetAddress.getLocalHost().getHostName()
126 | *
127 | *
128 | * @param instanceName the name of this instance
129 | * @return this (for chaining)
130 | */
131 | public WorkflowManagerBuilder withInstanceName(String instanceName)
132 | {
133 | this.instanceName = Preconditions.checkNotNull(instanceName, "instanceName cannot be null");
134 | return this;
135 | }
136 |
137 | /**
138 | * Return a new WorkflowManager using the current builder values
139 | *
140 | * @return new WorkflowManager
141 | */
142 | public WorkflowManager build()
143 | {
144 | return new WorkflowManagerImpl(curator, queueFactory, instanceName, specs, autoCleanerHolder, serializer, taskRunnerService);
145 | }
146 |
147 | /**
148 | * optional
149 | * Pluggable queue factory. Default uses ZooKeeper for queuing.
150 | *
151 | * @param queueFactory new queue factory
152 | * @return this (for chaining)
153 | */
154 | public WorkflowManagerBuilder withQueueFactory(QueueFactory queueFactory)
155 | {
156 | this.queueFactory = Preconditions.checkNotNull(queueFactory, "queueFactory cannot be null");
157 | return this;
158 | }
159 |
160 | /**
161 | * optional
162 | * Sets an auto-cleaner that will run every given period. This is used to clean old runs.
163 | * IMPORTANT: the auto cleaner will only run on the instance that is the current scheduler.
164 | *
165 | * @param autoCleaner the auto cleaner to use
166 | * @param runPeriod how often to run
167 | * @return this (for chaining)
168 | */
169 | public WorkflowManagerBuilder withAutoCleaner(AutoCleaner autoCleaner, Duration runPeriod)
170 | {
171 | autoCleanerHolder = (autoCleaner == null) ? newNullHolder() : new AutoCleanerHolder(autoCleaner, runPeriod);
172 | return this;
173 | }
174 |
175 | /**
176 | * optional
177 | * By default, a JSON serializer is used to store data in ZooKeeper. Use this to specify an alternate serializer
178 | *
179 | * @param serializer serializer to use
180 | * @return this (for chaining)
181 | */
182 | public WorkflowManagerBuilder withSerializer(Serializer serializer)
183 | {
184 | this.serializer = Preconditions.checkNotNull(serializer, "serializer cannot be null");
185 | return this;
186 | }
187 |
188 | /**
189 | * optional
190 | * By default, tasks are run in an internal executor service. Use this to specify a custom executor service
191 | * for tasks. This executor does not add any async/concurrency benefit. It's purpose is to allow you to control
192 | * which thread executes your tasks.
193 | *
194 | * @param taskRunnerService custom executor service
195 | * @return this (for chaining)
196 | */
197 | public WorkflowManagerBuilder withTaskRunnerService(Executor taskRunnerService)
198 | {
199 | this.taskRunnerService = Preconditions.checkNotNull(taskRunnerService, "taskRunnerService cannot be null");
200 | return this;
201 | }
202 |
203 | private WorkflowManagerBuilder()
204 | {
205 | try
206 | {
207 | instanceName = InetAddress.getLocalHost().getHostName();
208 | }
209 | catch ( UnknownHostException e )
210 | {
211 | instanceName = "unknown";
212 | }
213 | }
214 |
215 | private AutoCleanerHolder newNullHolder()
216 | {
217 | return new AutoCleanerHolder(null, Duration.ofDays(Integer.MAX_VALUE));
218 | }
219 | }
220 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/admin/AutoCleaner.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.admin;
17 |
18 | @FunctionalInterface
19 | public interface AutoCleaner
20 | {
21 | /**
22 | * Called for each potential run to determine if it can be cleaned
23 | *
24 | * @param runInfo the run
25 | * @return true if it can be cleaned
26 | */
27 | boolean canBeCleaned(RunInfo runInfo);
28 | }
29 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/admin/RunInfo.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.admin;
17 |
18 | import com.google.common.base.Preconditions;
19 | import com.nirmata.workflow.models.RunId;
20 | import java.time.LocalDateTime;
21 | import java.util.Optional;
22 |
23 | /**
24 | * Run information
25 | */
26 | public class RunInfo
27 | {
28 | private final RunId runId;
29 | private final LocalDateTime startTimeUtc;
30 | private final Optional completionTimeUtc;
31 |
32 | public RunInfo(RunId runId, LocalDateTime startTimeUtc)
33 | {
34 | this(runId, startTimeUtc, null);
35 | }
36 |
37 | public RunInfo(RunId runId, LocalDateTime startTimeUtc, LocalDateTime completionTimeUtc)
38 | {
39 | this.runId = Preconditions.checkNotNull(runId, "runId cannot be null");
40 | this.startTimeUtc = Preconditions.checkNotNull(startTimeUtc, "startTimeUtc cannot be null");
41 | this.completionTimeUtc = Optional.ofNullable(completionTimeUtc);
42 | }
43 |
44 | public RunId getRunId()
45 | {
46 | return runId;
47 | }
48 |
49 | public LocalDateTime getStartTimeUtc()
50 | {
51 | return startTimeUtc;
52 | }
53 |
54 | public LocalDateTime getCompletionTimeUtc()
55 | {
56 | //noinspection ConstantConditions
57 | return completionTimeUtc.get(); // exception if empty is desired
58 | }
59 |
60 | public boolean isComplete()
61 | {
62 | return completionTimeUtc.isPresent();
63 | }
64 |
65 | @Override
66 | public boolean equals(Object o)
67 | {
68 | if ( this == o )
69 | {
70 | return true;
71 | }
72 | if ( o == null || getClass() != o.getClass() )
73 | {
74 | return false;
75 | }
76 |
77 | RunInfo runInfo = (RunInfo)o;
78 |
79 | if ( !completionTimeUtc.equals(runInfo.completionTimeUtc) )
80 | {
81 | return false;
82 | }
83 | if ( !runId.equals(runInfo.runId) )
84 | {
85 | return false;
86 | }
87 | //noinspection RedundantIfStatement
88 | if ( !startTimeUtc.equals(runInfo.startTimeUtc) )
89 | {
90 | return false;
91 | }
92 |
93 | return true;
94 | }
95 |
96 | @Override
97 | public int hashCode()
98 | {
99 | int result = runId.hashCode();
100 | result = 31 * result + startTimeUtc.hashCode();
101 | result = 31 * result + completionTimeUtc.hashCode();
102 | return result;
103 | }
104 |
105 | @Override
106 | public String toString()
107 | {
108 | return "RunInfo{" +
109 | "runId=" + runId +
110 | ", startTime=" + startTimeUtc +
111 | ", completionTime=" + completionTimeUtc +
112 | '}';
113 | }
114 | }
115 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/admin/StandardAutoCleaner.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.admin;
17 |
18 | import java.time.Clock;
19 | import java.time.Duration;
20 | import java.time.LocalDateTime;
21 |
22 | /**
23 | * Default auto cleaner. Cleans if the run is completed and was completed past the given minimum age
24 | */
25 | public class StandardAutoCleaner implements AutoCleaner
26 | {
27 | private final Duration minAge;
28 |
29 | public StandardAutoCleaner(Duration minAge)
30 | {
31 | this.minAge = minAge;
32 | }
33 |
34 | @Override
35 | public boolean canBeCleaned(RunInfo runInfo)
36 | {
37 | if ( runInfo.isComplete() )
38 | {
39 | LocalDateTime nowUtc = LocalDateTime.now(Clock.systemUTC());
40 | Duration durationSinceCompletion = Duration.between(runInfo.getCompletionTimeUtc(), nowUtc);
41 | if ( durationSinceCompletion.compareTo(minAge) >= 0 )
42 | {
43 | return true;
44 | }
45 | }
46 | return false;
47 | }
48 | }
49 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/admin/TaskDetails.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.admin;
17 |
18 | import com.google.common.base.Preconditions;
19 | import com.google.common.collect.ImmutableMap;
20 | import com.nirmata.workflow.models.Task;
21 | import com.nirmata.workflow.models.TaskId;
22 | import com.nirmata.workflow.models.TaskType;
23 | import java.util.Map;
24 | import java.util.Optional;
25 |
26 | /**
27 | * Contains the meta-data and type from a submitted {@link Task}
28 | */
29 | public class TaskDetails
30 | {
31 | private final TaskId taskId;
32 | private final Optional taskType;
33 | private final Map metaData;
34 |
35 | public TaskDetails(TaskId taskId, TaskType taskType, Map metaData)
36 | {
37 | this.taskId = Preconditions.checkNotNull(taskId, "taskId cannot be null");
38 | this.taskType = Optional.ofNullable(taskType);
39 | metaData = Preconditions.checkNotNull(metaData, "metaData cannot be null");
40 |
41 | this.metaData = ImmutableMap.copyOf(metaData);
42 | }
43 |
44 | public TaskId getTaskId()
45 | {
46 | return taskId;
47 | }
48 |
49 | public boolean isExecutable()
50 | {
51 | return taskType.isPresent();
52 | }
53 |
54 | public TaskType getTaskType()
55 | {
56 | //noinspection ConstantConditions
57 | return taskType.get(); // exception if empty is desired
58 | }
59 |
60 | public Map getMetaData()
61 | {
62 | return metaData;
63 | }
64 |
65 | public boolean matchesTask(Task task)
66 | {
67 | if ( task == null )
68 | {
69 | return false;
70 | }
71 | TaskDetails rhs = new TaskDetails(task.getTaskId(), task.isExecutable() ? task.getTaskType() : null, task.getMetaData());
72 | return this.equals(rhs);
73 | }
74 |
75 | @Override
76 | public boolean equals(Object o)
77 | {
78 | if ( this == o )
79 | {
80 | return true;
81 | }
82 | if ( o == null || getClass() != o.getClass() )
83 | {
84 | return false;
85 | }
86 |
87 | TaskDetails that = (TaskDetails)o;
88 |
89 | if ( !metaData.equals(that.metaData) )
90 | {
91 | return false;
92 | }
93 | if ( !taskId.equals(that.taskId) )
94 | {
95 | return false;
96 | }
97 | //noinspection RedundantIfStatement
98 | if ( !taskType.equals(that.taskType) )
99 | {
100 | return false;
101 | }
102 |
103 | return true;
104 | }
105 |
106 | @Override
107 | public int hashCode()
108 | {
109 | int result = taskId.hashCode();
110 | result = 31 * result + taskType.hashCode();
111 | result = 31 * result + metaData.hashCode();
112 | return result;
113 | }
114 |
115 | @Override
116 | public String toString()
117 | {
118 | return "TaskDetails{" +
119 | "taskId=" + taskId +
120 | ", taskType=" + taskType +
121 | ", metaData=" + metaData +
122 | '}';
123 | }
124 | }
125 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/admin/TaskInfo.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.admin;
17 |
18 | import com.google.common.base.Preconditions;
19 | import com.nirmata.workflow.models.TaskExecutionResult;
20 | import com.nirmata.workflow.models.TaskId;
21 | import java.time.LocalDateTime;
22 | import java.util.Optional;
23 |
24 | /**
25 | * Task information
26 | */
27 | @SuppressWarnings("ConstantConditions") // exception if empty is desired
28 | public class TaskInfo
29 | {
30 | private final TaskId taskId;
31 | private final Optional instanceName;
32 | private final Optional startDateUtc;
33 | private final Optional progress;
34 | private final Optional result;
35 |
36 | public TaskInfo(TaskId taskId)
37 | {
38 | this(taskId, null, null, 0, null);
39 | }
40 |
41 | public TaskInfo(TaskId taskId, String instanceName, LocalDateTime startDateUtc, int progress)
42 | {
43 | this(taskId, instanceName, startDateUtc, progress, null);
44 | }
45 |
46 | public TaskInfo(TaskId taskId, String instanceName, LocalDateTime startDateUtc, int progress, TaskExecutionResult result)
47 | {
48 | this.taskId = Preconditions.checkNotNull(taskId, "taskId cannot be null");
49 | this.instanceName = Optional.ofNullable(instanceName);
50 | this.startDateUtc = Optional.ofNullable(startDateUtc);
51 | this.result = Optional.ofNullable(result);
52 | this.progress = Optional.of(progress);
53 | }
54 |
55 | public TaskId getTaskId()
56 | {
57 | return taskId;
58 | }
59 |
60 | public String getInstanceName()
61 | {
62 | return instanceName.get();
63 | }
64 |
65 | public LocalDateTime getStartDateUtc()
66 | {
67 | return startDateUtc.get();
68 | }
69 |
70 | public TaskExecutionResult getResult()
71 | {
72 | return result.get();
73 | }
74 |
75 | public int getProgress()
76 | {
77 | return progress.get();
78 | }
79 |
80 | public boolean hasStarted()
81 | {
82 | return startDateUtc.isPresent() && instanceName.isPresent();
83 | }
84 |
85 | public boolean isComplete()
86 | {
87 | return result.isPresent();
88 | }
89 |
90 | @Override
91 | public boolean equals(Object o)
92 | {
93 | if ( this == o )
94 | {
95 | return true;
96 | }
97 | if ( o == null || getClass() != o.getClass() )
98 | {
99 | return false;
100 | }
101 |
102 | TaskInfo taskInfo = (TaskInfo)o;
103 |
104 | if ( !instanceName.equals(taskInfo.instanceName) )
105 | {
106 | return false;
107 | }
108 | if ( !result.equals(taskInfo.result) )
109 | {
110 | return false;
111 | }
112 | if ( !startDateUtc.equals(taskInfo.startDateUtc) )
113 | {
114 | return false;
115 | }
116 | //noinspection RedundantIfStatement
117 | if ( !taskId.equals(taskInfo.taskId) )
118 | {
119 | return false;
120 | }
121 |
122 | return true;
123 | }
124 |
125 | @Override
126 | public int hashCode()
127 | {
128 | int result1 = taskId.hashCode();
129 | result1 = 31 * result1 + instanceName.hashCode();
130 | result1 = 31 * result1 + startDateUtc.hashCode();
131 | result1 = 31 * result1 + result.hashCode();
132 | return result1;
133 | }
134 |
135 | @Override
136 | public String toString()
137 | {
138 | return "TaskInfo{" +
139 | "taskId=" + taskId +
140 | ", instanceName='" + instanceName + '\'' +
141 | ", startDateUtc=" + startDateUtc +
142 | ", progress=" + progress +
143 | ", result=" + result +
144 | '}';
145 | }
146 | }
147 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/admin/WorkflowAdmin.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.admin;
17 |
18 | import com.nirmata.workflow.models.RunId;
19 | import com.nirmata.workflow.models.TaskId;
20 | import java.util.List;
21 | import java.util.Map;
22 |
23 | /**
24 | * Admin operations
25 | */
26 | public interface WorkflowAdmin
27 | {
28 | /**
29 | * Return all run IDs completed or currently executing
30 | * in the workflow manager
31 | *
32 | * @return run infos
33 | */
34 | List getRunIds();
35 |
36 | /**
37 | * Return info about all runs completed or currently executing
38 | * in the workflow manager
39 | *
40 | * @return run infos
41 | */
42 | List getRunInfo();
43 |
44 | /**
45 | * Return info about the given run
46 | *
47 | * @param runId run
48 | * @return info
49 | */
50 | RunInfo getRunInfo(RunId runId);
51 |
52 | /**
53 | * Return info about all the tasks completed, started or waiting for
54 | * the given run
55 | *
56 | * @param runId run
57 | * @return task infos
58 | */
59 | List getTaskInfo(RunId runId);
60 |
61 | /**
62 | * Returns a map of all task details for the given run
63 | *
64 | * @param runId run
65 | * @return task details
66 | */
67 | Map getTaskDetails(RunId runId);
68 |
69 | /**
70 | * Delete all saved data for the given run.
71 | *
72 | * @param runId the run
73 | * @return true if the run was found
74 | */
75 | boolean clean(RunId runId);
76 |
77 | /**
78 | * Return information about the internal run/state of the workflow manager
79 | *
80 | * @return state
81 | */
82 | WorkflowManagerState getWorkflowManagerState();
83 | }
84 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/admin/WorkflowManagerState.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.admin;
17 |
18 | import com.google.common.base.Preconditions;
19 | import com.google.common.collect.ImmutableList;
20 | import java.util.List;
21 |
22 | public class WorkflowManagerState
23 | {
24 | private final boolean isConnectedToZooKeeper;
25 | private final State schedulerState;
26 | private final List consumersState;
27 |
28 | public enum State
29 | {
30 | /**
31 | * Has not started
32 | */
33 | LATENT,
34 |
35 | /**
36 | * Is in a blocking method
37 | */
38 | SLEEPING,
39 |
40 | /**
41 | * Is doing work
42 | */
43 | PROCESSING,
44 |
45 | /**
46 | * Has been closed
47 | */
48 | CLOSED
49 | }
50 |
51 | public WorkflowManagerState(boolean isConnectedToZooKeeper, State schedulerState, List consumersState)
52 | {
53 | this.isConnectedToZooKeeper = isConnectedToZooKeeper;
54 | this.schedulerState = Preconditions.checkNotNull(schedulerState, "schedulerState cannot be null");
55 | this.consumersState = ImmutableList.copyOf(Preconditions.checkNotNull(consumersState, "consumersState cannot be null"));
56 | }
57 |
58 | /**
59 | * Returns true if the connection to ZooKeeper is active
60 | *
61 | * @return true/false
62 | */
63 | public boolean isConnectedToZooKeeper()
64 | {
65 | return isConnectedToZooKeeper;
66 | }
67 |
68 | /**
69 | * Return the state of the scheduler for this manager instance. If {@link State#LATENT} is returned
70 | * this instance is not currently the scheduler.
71 | *
72 | * @return state info
73 | */
74 | public State getSchedulerState()
75 | {
76 | return schedulerState;
77 | }
78 |
79 | /**
80 | * Return state info for each task executor in this manager
81 | *
82 | * @return task executors state
83 | */
84 | public List getExecutorsState()
85 | {
86 | return consumersState;
87 | }
88 |
89 | @Override
90 | public boolean equals(Object o)
91 | {
92 | if ( this == o )
93 | {
94 | return true;
95 | }
96 | if ( o == null || getClass() != o.getClass() )
97 | {
98 | return false;
99 | }
100 |
101 | WorkflowManagerState that = (WorkflowManagerState)o;
102 |
103 | if ( isConnectedToZooKeeper != that.isConnectedToZooKeeper )
104 | {
105 | return false;
106 | }
107 | //noinspection SimplifiableIfStatement
108 | if ( schedulerState != that.schedulerState )
109 | {
110 | return false;
111 | }
112 | return consumersState.equals(that.consumersState);
113 |
114 | }
115 |
116 | @Override
117 | public int hashCode()
118 | {
119 | int result = (isConnectedToZooKeeper ? 1 : 0);
120 | result = 31 * result + schedulerState.hashCode();
121 | result = 31 * result + consumersState.hashCode();
122 | return result;
123 | }
124 |
125 | @Override
126 | public String toString()
127 | {
128 | return "WorkflowManagerState{" +
129 | "isConnectedToZooKeeper=" + isConnectedToZooKeeper +
130 | ", schedulerState=" + schedulerState +
131 | ", consumersState=" + consumersState +
132 | '}';
133 | }
134 | }
135 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/details/AutoCleanerHolder.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.details;
17 |
18 | import com.google.common.base.Preconditions;
19 | import com.nirmata.workflow.admin.AutoCleaner;
20 | import com.nirmata.workflow.admin.WorkflowAdmin;
21 | import org.slf4j.Logger;
22 | import org.slf4j.LoggerFactory;
23 | import java.time.Duration;
24 | import java.time.Instant;
25 | import java.util.concurrent.atomic.AtomicReference;
26 |
27 | public class AutoCleanerHolder
28 | {
29 | private final Logger log = LoggerFactory.getLogger(getClass());
30 | private final AutoCleaner autoCleaner;
31 | private final Duration runPeriod;
32 | private final AtomicReference lastRun = new AtomicReference<>(Instant.now());
33 |
34 | public AutoCleanerHolder(AutoCleaner autoCleaner, Duration runPeriod)
35 | {
36 | this.autoCleaner = autoCleaner;
37 | this.runPeriod = Preconditions.checkNotNull(runPeriod, "runPeriod cannot be null");
38 | }
39 |
40 | public Duration getRunPeriod()
41 | {
42 | return runPeriod;
43 | }
44 |
45 | public void run(WorkflowAdmin admin)
46 | {
47 | Preconditions.checkNotNull(admin, "admin cannot be null");
48 | log.debug("Running");
49 | if ( autoCleaner != null )
50 | {
51 | admin.getRunInfo().stream().filter(autoCleaner::canBeCleaned).forEach(r -> {
52 | log.debug("Auto cleaning: " + r);
53 | admin.clean(r.getRunId());
54 | });
55 | }
56 | lastRun.set(Instant.now());
57 | }
58 |
59 | public boolean shouldRun()
60 | {
61 | if ( autoCleaner == null )
62 | {
63 | return false;
64 | }
65 | Duration periodSinceLast = Duration.between(lastRun.get(), Instant.now());
66 | return (periodSinceLast.compareTo(runPeriod) >= 0);
67 | }
68 | }
69 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/details/RunnableTaskDagBuilder.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.details;
17 |
18 | import com.google.common.collect.ImmutableList;
19 | import com.google.common.collect.ImmutableMap;
20 | import com.google.common.collect.Sets;
21 | import com.nirmata.workflow.details.internalmodels.RunnableTaskDag;
22 | import com.nirmata.workflow.models.Task;
23 | import com.nirmata.workflow.models.TaskId;
24 | import org.jgrapht.alg.CycleDetector;
25 | import org.jgrapht.graph.DefaultDirectedGraph;
26 | import org.jgrapht.graph.DefaultEdge;
27 | import org.jgrapht.traverse.TopologicalOrderIterator;
28 | import java.util.List;
29 | import java.util.Map;
30 | import java.util.Set;
31 | import java.util.stream.Collectors;
32 |
33 | public class RunnableTaskDagBuilder
34 | {
35 | private final List entries;
36 | private final Map tasks;
37 |
38 | public RunnableTaskDagBuilder(Task task)
39 | {
40 | ImmutableList.Builder entriesBuilder = ImmutableList.builder();
41 | ImmutableMap.Builder tasksBuilder = ImmutableMap.builder();
42 | build(task, entriesBuilder, tasksBuilder);
43 |
44 | entries = entriesBuilder.build();
45 | tasks = tasksBuilder.build();
46 | }
47 |
48 | public List getEntries()
49 | {
50 | return entries;
51 | }
52 |
53 | public Map getTasks()
54 | {
55 | return tasks;
56 | }
57 |
58 | private void build(Task task, ImmutableList.Builder entriesBuilder, ImmutableMap.Builder tasksBuilder)
59 | {
60 | DefaultDirectedGraph graph = new DefaultDirectedGraph<>(DefaultEdge.class);
61 | worker(graph, task, null, tasksBuilder, Sets.newHashSet());
62 |
63 | CycleDetector cycleDetector = new CycleDetector<>(graph);
64 | if ( cycleDetector.detectCycles() )
65 | {
66 | throw new RuntimeException("The Task DAG contains cycles: " + task);
67 | }
68 |
69 | TopologicalOrderIterator orderIterator = new TopologicalOrderIterator<>(graph);
70 | while ( orderIterator.hasNext() )
71 | {
72 | TaskId taskId = orderIterator.next();
73 | Set taskIdEdges = graph.edgesOf(taskId);
74 | Set processed = taskIdEdges
75 | .stream()
76 | .map(graph::getEdgeSource)
77 | .filter(edge -> !edge.equals(taskId) && !edge.getId().equals(""))
78 | .collect(Collectors.toSet());
79 | entriesBuilder.add(new RunnableTaskDag(taskId, processed));
80 | }
81 | }
82 |
83 | private void worker(DefaultDirectedGraph graph, Task task, TaskId parentId, ImmutableMap.Builder tasksBuilder, Set usedTasksSet)
84 | {
85 | if ( usedTasksSet.add(task.getTaskId()) )
86 | {
87 | tasksBuilder.put(task.getTaskId(), task);
88 | }
89 |
90 | graph.addVertex(task.getTaskId());
91 | if ( parentId != null )
92 | {
93 | graph.addEdge(parentId, task.getTaskId());
94 | }
95 | task.getChildrenTasks().forEach(child -> worker(graph, child, task.getTaskId(), tasksBuilder, usedTasksSet));
96 | }
97 | }
98 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/details/SchedulerSelector.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.details;
17 |
18 | import com.google.common.annotations.VisibleForTesting;
19 | import com.google.common.base.Preconditions;
20 | import com.nirmata.workflow.admin.WorkflowManagerState;
21 | import com.nirmata.workflow.queue.QueueFactory;
22 | import org.apache.curator.framework.CuratorFramework;
23 | import org.apache.curator.framework.recipes.leader.LeaderSelector;
24 | import org.apache.curator.framework.recipes.leader.LeaderSelectorListener;
25 | import org.apache.curator.framework.recipes.leader.LeaderSelectorListenerAdapter;
26 | import org.apache.curator.utils.CloseableUtils;
27 | import org.slf4j.Logger;
28 | import org.slf4j.LoggerFactory;
29 | import java.io.Closeable;
30 | import java.util.concurrent.CountDownLatch;
31 | import java.util.concurrent.atomic.AtomicReference;
32 |
33 | public class SchedulerSelector implements Closeable
34 | {
35 | private final Logger log = LoggerFactory.getLogger(getClass());
36 | private final WorkflowManagerImpl workflowManager;
37 | private final QueueFactory queueFactory;
38 | private final AutoCleanerHolder autoCleanerHolder;
39 | private final LeaderSelector leaderSelector;
40 | private final AtomicReference scheduler = new AtomicReference<>();
41 |
42 | volatile AtomicReference debugLatch = new AtomicReference<>();
43 |
44 | public SchedulerSelector(WorkflowManagerImpl workflowManager, QueueFactory queueFactory, AutoCleanerHolder autoCleanerHolder)
45 | {
46 | this.workflowManager = workflowManager;
47 | this.queueFactory = queueFactory;
48 | this.autoCleanerHolder = autoCleanerHolder;
49 |
50 | LeaderSelectorListener listener = new LeaderSelectorListenerAdapter()
51 | {
52 | @Override
53 | public void takeLeadership(CuratorFramework client) throws Exception
54 | {
55 | SchedulerSelector.this.takeLeadership();
56 | }
57 | };
58 | leaderSelector = new LeaderSelector(workflowManager.getCurator(), ZooKeeperConstants.getSchedulerLeaderPath(), listener);
59 | leaderSelector.autoRequeue();
60 | }
61 |
62 | public WorkflowManagerState.State getState()
63 | {
64 | Scheduler localScheduler = scheduler.get();
65 | return (localScheduler != null) ? localScheduler.getState() : WorkflowManagerState.State.LATENT;
66 | }
67 |
68 | public void start()
69 | {
70 | leaderSelector.start();
71 | }
72 |
73 | @Override
74 | public void close()
75 | {
76 | CloseableUtils.closeQuietly(leaderSelector);
77 | }
78 |
79 | @VisibleForTesting
80 | LeaderSelector getLeaderSelector()
81 | {
82 | return leaderSelector;
83 | }
84 |
85 | @VisibleForTesting
86 | void debugValidateClosed()
87 | {
88 | Preconditions.checkState(!leaderSelector.hasLeadership());
89 | }
90 |
91 | private void takeLeadership()
92 | {
93 | log.info(workflowManager.getInstanceName() + " is now the scheduler");
94 | try
95 | {
96 | scheduler.set(new Scheduler(workflowManager, queueFactory, autoCleanerHolder));
97 | scheduler.get().run();
98 | }
99 | finally
100 | {
101 | log.info(workflowManager.getInstanceName() + " is no longer the scheduler");
102 | scheduler.set(null);
103 |
104 | CountDownLatch latch = debugLatch.getAndSet(null);
105 | if ( latch != null )
106 | {
107 | latch.countDown();
108 | }
109 | }
110 | }
111 | }
112 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/details/TaskExecutorSpec.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.details;
17 |
18 | import com.google.common.base.Preconditions;
19 | import com.nirmata.workflow.executor.TaskExecutor;
20 | import com.nirmata.workflow.models.TaskType;
21 |
22 | public class TaskExecutorSpec
23 | {
24 | private final TaskExecutor taskExecutor;
25 | private final int qty;
26 | private final TaskType taskType;
27 |
28 | public TaskExecutorSpec(TaskExecutor taskExecutor, int qty, TaskType taskType)
29 | {
30 | this.taskType = Preconditions.checkNotNull(taskType, "taskType cannot be null");
31 | this.taskExecutor = Preconditions.checkNotNull(taskExecutor, "taskExecutor cannot be null");
32 | this.qty = qty;
33 | }
34 |
35 | TaskExecutor getTaskExecutor()
36 | {
37 | return taskExecutor;
38 | }
39 |
40 | int getQty()
41 | {
42 | return qty;
43 | }
44 |
45 | TaskType getTaskType()
46 | {
47 | return taskType;
48 | }
49 | }
50 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/details/WorkflowListenerManagerImpl.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.details;
17 |
18 | import com.nirmata.workflow.events.WorkflowEvent;
19 | import com.nirmata.workflow.events.WorkflowListener;
20 | import com.nirmata.workflow.events.WorkflowListenerManager;
21 | import com.nirmata.workflow.models.RunId;
22 | import com.nirmata.workflow.models.TaskId;
23 | import org.apache.curator.framework.listen.Listenable;
24 | import org.apache.curator.framework.listen.StandardListenerManager;
25 | import org.apache.curator.framework.listen.UnaryListenerManager;
26 | import org.apache.curator.framework.recipes.cache.PathChildrenCache;
27 | import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
28 | import org.apache.curator.utils.CloseableUtils;
29 |
30 | import java.io.IOException;
31 |
32 | public class WorkflowListenerManagerImpl implements WorkflowListenerManager {
33 | private final PathChildrenCache completedTasksCache;
34 | private final PathChildrenCache startedTasksCache;
35 | private final PathChildrenCache runsCache;
36 | private final UnaryListenerManager listenerContainer = StandardListenerManager.standard();
37 |
38 | public WorkflowListenerManagerImpl(WorkflowManagerImpl workflowManager) {
39 | completedTasksCache = new PathChildrenCache(workflowManager.getCurator(), ZooKeeperConstants.getCompletedTaskParentPath(), false);
40 | startedTasksCache = new PathChildrenCache(workflowManager.getCurator(), ZooKeeperConstants.getStartedTasksParentPath(), false);
41 | runsCache = new PathChildrenCache(workflowManager.getCurator(), ZooKeeperConstants.getRunParentPath(), false);
42 | }
43 |
44 | @Override
45 | public void start() {
46 | try {
47 | runsCache.getListenable().addListener((client, event) -> {
48 | RunId runId = new RunId(ZooKeeperConstants.getRunIdFromRunPath(event.getData().getPath()));
49 | if (event.getType() == PathChildrenCacheEvent.Type.CHILD_ADDED) {
50 | postEvent(new WorkflowEvent(WorkflowEvent.EventType.RUN_STARTED, runId));
51 | } else if (event.getType() == PathChildrenCacheEvent.Type.CHILD_UPDATED) {
52 | postEvent(new WorkflowEvent(WorkflowEvent.EventType.RUN_UPDATED, runId));
53 | }
54 | });
55 |
56 | startedTasksCache.getListenable().addListener((client, event) -> {
57 | if (event.getType() == PathChildrenCacheEvent.Type.CHILD_ADDED) {
58 | RunId runId = new RunId(ZooKeeperConstants.getRunIdFromStartedTasksPath(event.getData().getPath()));
59 | TaskId taskId = new TaskId(ZooKeeperConstants.getTaskIdFromStartedTasksPath(event.getData().getPath()));
60 | postEvent(new WorkflowEvent(WorkflowEvent.EventType.TASK_STARTED, runId, taskId));
61 | }
62 | });
63 |
64 | completedTasksCache.getListenable().addListener((client, event) -> {
65 | if (event.getType() == PathChildrenCacheEvent.Type.CHILD_ADDED) {
66 | RunId runId = new RunId(ZooKeeperConstants.getRunIdFromCompletedTasksPath(event.getData().getPath()));
67 | TaskId taskId = new TaskId(ZooKeeperConstants.getTaskIdFromCompletedTasksPath(event.getData().getPath()));
68 | postEvent(new WorkflowEvent(WorkflowEvent.EventType.TASK_COMPLETED, runId, taskId));
69 | }
70 | });
71 |
72 | runsCache.start(PathChildrenCache.StartMode.BUILD_INITIAL_CACHE);
73 | completedTasksCache.start(PathChildrenCache.StartMode.BUILD_INITIAL_CACHE);
74 | startedTasksCache.start(PathChildrenCache.StartMode.BUILD_INITIAL_CACHE);
75 | } catch (Exception e) {
76 | throw new RuntimeException(e);
77 | }
78 | }
79 |
80 | @Override
81 | public void close() throws IOException {
82 | CloseableUtils.closeQuietly(runsCache);
83 | CloseableUtils.closeQuietly(startedTasksCache);
84 | CloseableUtils.closeQuietly(completedTasksCache);
85 | }
86 |
87 | @Override
88 | public Listenable getListenable() {
89 | return listenerContainer;
90 | }
91 |
92 | private void postEvent(WorkflowEvent event) {
93 | listenerContainer.forEach(l -> {
94 | if (l != null) {
95 | l.receiveEvent(event);
96 | }
97 | });
98 | }
99 | }
100 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/details/ZooKeeperConstants.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.details;
17 |
18 | import com.google.common.base.Splitter;
19 | import com.nirmata.workflow.models.RunId;
20 | import com.nirmata.workflow.models.TaskId;
21 | import com.nirmata.workflow.models.TaskType;
22 | import org.apache.curator.utils.ZKPaths;
23 |
24 | public class ZooKeeperConstants
25 | {
26 | private static final String SCHEDULER_LEADER_PATH = "/scheduler-leader";
27 | private static final String RUN_PATH = "/runs";
28 | private static final String COMPLETED_TASKS_PATH = "/tasks-completed";
29 | private static final String STARTED_TASKS_PATH = "/tasks-started";
30 | private static final String QUEUE_PATH_BASE = "/tasks-queue";
31 |
32 | private static final String SEPARATOR = "|";
33 |
34 | public static final int MAX_PAYLOAD = 0xfffff; // see "jute.maxbuffer" at http://zookeeper.apache.org/doc/r3.3.1/zookeeperAdmin.html
35 |
36 | private ZooKeeperConstants()
37 | {
38 | }
39 |
40 | public static void main(String[] args)
41 | {
42 | TaskId taskId = new TaskId();
43 | RunId runId = new RunId();
44 | String runPath = getRunPath(runId);
45 | String completedTaskPath = getCompletedTaskPath(runId, taskId);
46 | String startedTaskPath = getStartedTaskPath(runId, taskId);
47 | TaskType taskType = new TaskType("a", "b", true);
48 |
49 | System.out.println("taskId = " + taskId.getId());
50 | System.out.println("runId = " + runId.getId());
51 | System.out.println();
52 |
53 | System.out.println("getRunIdFromRunPath:\t\t\t\t" + getRunIdFromRunPath(runPath));
54 | System.out.println("getRunIdFromCompletedTasksPath:\t\t" + getRunIdFromCompletedTasksPath(completedTaskPath));
55 | System.out.println("getTaskIdFromCompletedTasksPath:\t" + getTaskIdFromCompletedTasksPath(completedTaskPath));
56 | System.out.println("getTaskIdFromStartedTasksPath:\t\t" + getTaskIdFromStartedTasksPath(startedTaskPath));
57 | System.out.println();
58 |
59 | System.out.println("getSchedulerLeaderPath:\t\t\t\t" + getSchedulerLeaderPath());
60 | System.out.println("getRunParentPath:\t\t\t\t\t" + getRunParentPath());
61 | System.out.println("getRunPath:\t\t\t\t\t\t\t" + getRunPath(runId));
62 | System.out.println("getQueueBasePath:\t\t\t\t\t" + getQueueBasePath(taskType));
63 | System.out.println("getQueuePath:\t\t\t\t\t\t" + getQueuePath(taskType));
64 | System.out.println("getQueueLockPath:\t\t\t\t\t" + getQueueLockPath(taskType));
65 | System.out.println("getCompletedTasksParentPath:\t\t" + getCompletedTaskParentPath());
66 | System.out.println("getCompletedTaskPath:\t\t\t\t" + completedTaskPath);
67 | System.out.println("getStartedTasksParentPath:\t\t\t" + getStartedTasksParentPath());
68 | System.out.println("getStartedTaskPath:\t\t\t\t\t" + startedTaskPath);
69 | }
70 |
71 | public static String getSchedulerLeaderPath()
72 | {
73 | return SCHEDULER_LEADER_PATH;
74 | }
75 |
76 | public static String getRunParentPath()
77 | {
78 | return RUN_PATH;
79 | }
80 |
81 | public static String getQueuePathBase()
82 | {
83 | return QUEUE_PATH_BASE;
84 | }
85 |
86 | public static String getRunPath(RunId runId)
87 | {
88 | return ZKPaths.makePath(RUN_PATH, runId.getId());
89 | }
90 |
91 | public static String getRunIdFromRunPath(String path)
92 | {
93 | return ZKPaths.getPathAndNode(path).getNode();
94 | }
95 |
96 | public static String getQueuePath(TaskType taskType)
97 | {
98 | String path = getQueueBasePath(taskType);
99 | return ZKPaths.makePath(path, "queue");
100 | }
101 |
102 | public static String getQueueLockPath(TaskType taskType)
103 | {
104 | String path = getQueueBasePath(taskType);
105 | return ZKPaths.makePath(path, "queue-lock");
106 | }
107 |
108 | private static String getQueueBasePath(TaskType taskType)
109 | {
110 | String path = ZKPaths.makePath(QUEUE_PATH_BASE, taskType.getType());
111 | path = ZKPaths.makePath(path, taskType.isIdempotent() ? "i" : "ni");
112 | return ZKPaths.makePath(path, taskType.getVersion());
113 | }
114 |
115 | public static String getCompletedTaskParentPath()
116 | {
117 | return COMPLETED_TASKS_PATH;
118 | }
119 |
120 | public static String getRunIdFromCompletedTasksPath(String path)
121 | {
122 | String n = ZKPaths.getNodeFromPath(path);
123 | return Splitter.on(SEPARATOR).splitToList(n).get(0);
124 | }
125 |
126 | public static String getTaskIdFromStartedTasksPath(String path)
127 | {
128 | return getTaskIdFromCompletedTasksPath(path);
129 | }
130 |
131 | public static String getTaskIdFromCompletedTasksPath(String path)
132 | {
133 | String n = ZKPaths.getNodeFromPath(path);
134 | return Splitter.on(SEPARATOR).splitToList(n).get(1);
135 | }
136 |
137 | public static String getCompletedTaskPath(RunId runId, TaskId taskId)
138 | {
139 | return ZKPaths.makePath(getCompletedTaskParentPath(), makeRunTask(runId, taskId));
140 | }
141 |
142 | public static String getStartedTasksParentPath()
143 | {
144 | return STARTED_TASKS_PATH;
145 | }
146 |
147 | public static String getStartedTaskPath(RunId runId, TaskId taskId)
148 | {
149 | String parentPath = getStartedTasksParentPath();
150 | return ZKPaths.makePath(parentPath, makeRunTask(runId, taskId));
151 | }
152 |
153 | public static String getRunIdFromStartedTasksPath(String path)
154 | {
155 | return getRunIdFromCompletedTasksPath(path);
156 | }
157 |
158 | private static String makeRunTask(RunId runId, TaskId taskId)
159 | {
160 | return runId.getId() + SEPARATOR + taskId.getId();
161 | }
162 | }
163 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/details/internalmodels/RunnableTask.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.details.internalmodels;
17 |
18 | import com.google.common.base.Preconditions;
19 | import com.google.common.collect.ImmutableList;
20 | import com.google.common.collect.ImmutableMap;
21 | import com.nirmata.workflow.models.ExecutableTask;
22 | import com.nirmata.workflow.models.RunId;
23 | import com.nirmata.workflow.models.TaskId;
24 | import java.io.Serializable;
25 | import java.time.LocalDateTime;
26 | import java.util.List;
27 | import java.util.Map;
28 | import java.util.Optional;
29 |
30 | public class RunnableTask implements Serializable
31 | {
32 | private final Map tasks;
33 | private final List taskDags;
34 | private final LocalDateTime startTimeUtc;
35 | private final Optional completionTimeUtc;
36 | private final Optional parentRunId;
37 |
38 | public RunnableTask(Map tasks, List taskDags, LocalDateTime startTimeUtc)
39 | {
40 | this(tasks, taskDags, startTimeUtc, null, null);
41 | }
42 |
43 | public RunnableTask(Map tasks, List taskDags, LocalDateTime startTimeUtc, LocalDateTime completionTimeUtc)
44 | {
45 | this(tasks, taskDags, startTimeUtc, completionTimeUtc, null);
46 | }
47 |
48 | public RunnableTask(Map tasks, List taskDags, LocalDateTime startTimeUtc, LocalDateTime completionTimeUtc, RunId parentRunId)
49 | {
50 | this.startTimeUtc = Preconditions.checkNotNull(startTimeUtc, "startTimeUtc cannot be null");
51 | this.completionTimeUtc = Optional.ofNullable(completionTimeUtc);
52 | this.parentRunId = Optional.ofNullable(parentRunId);
53 | tasks = Preconditions.checkNotNull(tasks, "tasks cannot be null");
54 | taskDags = Preconditions.checkNotNull(taskDags, "taskDags cannot be null");
55 |
56 | this.tasks = ImmutableMap.copyOf(tasks);
57 | this.taskDags = ImmutableList.copyOf(taskDags);
58 | }
59 |
60 | public Map getTasks()
61 | {
62 | return tasks;
63 | }
64 |
65 | public List getTaskDags()
66 | {
67 | return taskDags;
68 | }
69 |
70 | public Optional getCompletionTimeUtc()
71 | {
72 | return completionTimeUtc;
73 | }
74 |
75 | public LocalDateTime getStartTimeUtc()
76 | {
77 | return startTimeUtc;
78 | }
79 |
80 | public Optional getParentRunId()
81 | {
82 | return parentRunId;
83 | }
84 |
85 | @Override
86 | public boolean equals(Object o)
87 | {
88 | if ( this == o )
89 | {
90 | return true;
91 | }
92 | if ( o == null || getClass() != o.getClass() )
93 | {
94 | return false;
95 | }
96 |
97 | RunnableTask that = (RunnableTask)o;
98 |
99 | if ( !completionTimeUtc.equals(that.completionTimeUtc) )
100 | {
101 | return false;
102 | }
103 | if ( !parentRunId.equals(that.parentRunId) )
104 | {
105 | return false;
106 | }
107 | if ( !startTimeUtc.equals(that.startTimeUtc) )
108 | {
109 | return false;
110 | }
111 | if ( !taskDags.equals(that.taskDags) )
112 | {
113 | return false;
114 | }
115 | //noinspection RedundantIfStatement
116 | if ( !tasks.equals(that.tasks) )
117 | {
118 | return false;
119 | }
120 |
121 | return true;
122 | }
123 |
124 | @Override
125 | public int hashCode()
126 | {
127 | int result = tasks.hashCode();
128 | result = 31 * result + taskDags.hashCode();
129 | result = 31 * result + startTimeUtc.hashCode();
130 | result = 31 * result + completionTimeUtc.hashCode();
131 | result = 31 * result + parentRunId.hashCode();
132 | return result;
133 | }
134 |
135 | @Override
136 | public String toString()
137 | {
138 | return "RunnableTask{" +
139 | "tasks=" + tasks +
140 | ", taskDags=" + taskDags +
141 | ", startTime=" + startTimeUtc +
142 | ", completionTime=" + completionTimeUtc +
143 | ", parentRunId=" + parentRunId +
144 | '}';
145 | }
146 | }
147 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/details/internalmodels/RunnableTaskDag.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.details.internalmodels;
17 |
18 | import com.google.common.base.Preconditions;
19 | import com.google.common.collect.ImmutableSet;
20 | import com.nirmata.workflow.models.TaskId;
21 | import java.io.Serializable;
22 | import java.util.Collection;
23 |
24 | public class RunnableTaskDag implements Serializable
25 | {
26 | private final TaskId taskId;
27 | private final Collection dependencies;
28 |
29 | public RunnableTaskDag(TaskId taskId, Collection dependencies)
30 | {
31 | this.taskId = Preconditions.checkNotNull(taskId, "taskId cannot be null");
32 | dependencies = Preconditions.checkNotNull(dependencies, "dependencies cannot be null");
33 | this.dependencies = ImmutableSet.copyOf(dependencies);
34 | }
35 |
36 | public TaskId getTaskId()
37 | {
38 | return taskId;
39 | }
40 |
41 | public Collection getDependencies()
42 | {
43 | return dependencies;
44 | }
45 |
46 | @Override
47 | public boolean equals(Object o)
48 | {
49 | if ( this == o )
50 | {
51 | return true;
52 | }
53 | if ( o == null || getClass() != o.getClass() )
54 | {
55 | return false;
56 | }
57 |
58 | RunnableTaskDag that = (RunnableTaskDag)o;
59 |
60 | if ( !dependencies.equals(that.dependencies) )
61 | {
62 | return false;
63 | }
64 | //noinspection RedundantIfStatement
65 | if ( !taskId.equals(that.taskId) )
66 | {
67 | return false;
68 | }
69 |
70 | return true;
71 | }
72 |
73 | @Override
74 | public int hashCode()
75 | {
76 | int result = taskId.hashCode();
77 | result = 31 * result + dependencies.hashCode();
78 | return result;
79 | }
80 |
81 | @Override
82 | public String toString()
83 | {
84 | return "RunnableTaskDagEntryModel{" +
85 | "taskId=" + taskId +
86 | ", dependencies=" + dependencies +
87 | '}';
88 | }
89 | }
90 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/details/internalmodels/StartedTask.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.details.internalmodels;
17 |
18 | import com.google.common.base.Preconditions;
19 | import java.io.Serializable;
20 | import java.time.LocalDateTime;
21 |
22 | public class StartedTask implements Serializable
23 | {
24 | private final String instanceName;
25 | private final LocalDateTime startDateUtc;
26 | private final int progress;
27 |
28 | public StartedTask(String instanceName, LocalDateTime startDateUtc, int progress)
29 | {
30 | this.instanceName = Preconditions.checkNotNull(instanceName, "instanceName cannot be null");
31 | this.startDateUtc = Preconditions.checkNotNull(startDateUtc, "startDateUtc cannot be null");
32 | this.progress = progress;
33 | }
34 |
35 | public String getInstanceName()
36 | {
37 | return instanceName;
38 | }
39 |
40 | public LocalDateTime getStartDateUtc()
41 | {
42 | return startDateUtc;
43 | }
44 |
45 | public int getProgress()
46 | {
47 | return progress;
48 | }
49 |
50 | @Override
51 | public boolean equals(Object o)
52 | {
53 | if ( this == o )
54 | {
55 | return true;
56 | }
57 | if ( o == null || getClass() != o.getClass() )
58 | {
59 | return false;
60 | }
61 |
62 | StartedTask that = (StartedTask)o;
63 |
64 | if ( !instanceName.equals(that.instanceName) )
65 | {
66 | return false;
67 | }
68 | //noinspection RedundantIfStatement
69 | if ( !startDateUtc.equals(that.startDateUtc) )
70 | {
71 | return false;
72 | }
73 |
74 | return true;
75 | }
76 |
77 | @Override
78 | public int hashCode()
79 | {
80 | int result = instanceName.hashCode();
81 | result = 31 * result + startDateUtc.hashCode();
82 | return result;
83 | }
84 |
85 | @Override
86 | public String toString()
87 | {
88 | return "StartedTaskModel{" +
89 | "instanceName='" + instanceName + '\'' +
90 | ", startDateUtc=" + startDateUtc +
91 | ", progress=" + progress +
92 | '}';
93 | }
94 | }
95 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/events/WorkflowEvent.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.events;
17 |
18 | import com.nirmata.workflow.models.RunId;
19 | import com.nirmata.workflow.models.TaskId;
20 | import java.util.Optional;
21 |
22 | /**
23 | * Models a workflow event
24 | */
25 | public class WorkflowEvent
26 | {
27 | public enum EventType
28 | {
29 | /**
30 | * A run has started. {@link WorkflowEvent#getRunId()} is the run id.
31 | */
32 | RUN_STARTED,
33 |
34 | /**
35 | * A run has been updated - usually meaning it has completed.
36 | * {@link WorkflowEvent#getRunId()} is the run id.
37 | */
38 | RUN_UPDATED,
39 |
40 | /**
41 | * A task has started. {@link WorkflowEvent#getRunId()} is the run id.
42 | * {@link WorkflowEvent#getTaskId()} is the task id.
43 | */
44 | TASK_STARTED,
45 |
46 | /**
47 | * A task has completed. {@link WorkflowEvent#getRunId()} is the run id.
48 | * {@link WorkflowEvent#getTaskId()} is the task id.
49 | */
50 | TASK_COMPLETED
51 | }
52 |
53 | private final EventType type;
54 | private final RunId runId;
55 | private final Optional taskId;
56 |
57 | public WorkflowEvent(EventType type, RunId runId)
58 | {
59 | this(type, runId, null);
60 | }
61 |
62 | public WorkflowEvent(EventType type, RunId runId, TaskId taskId)
63 | {
64 | this.type = type;
65 | this.runId = runId;
66 | this.taskId = Optional.ofNullable(taskId);
67 | }
68 |
69 | public EventType getType()
70 | {
71 | return type;
72 | }
73 |
74 | public RunId getRunId()
75 | {
76 | return runId;
77 | }
78 |
79 | public Optional getTaskId()
80 | {
81 | return taskId;
82 | }
83 |
84 | @Override
85 | public boolean equals(Object o)
86 | {
87 | if ( this == o )
88 | {
89 | return true;
90 | }
91 | if ( o == null || getClass() != o.getClass() )
92 | {
93 | return false;
94 | }
95 |
96 | WorkflowEvent that = (WorkflowEvent)o;
97 |
98 | if ( !runId.equals(that.runId) )
99 | {
100 | return false;
101 | }
102 | if ( !taskId.equals(that.taskId) )
103 | {
104 | return false;
105 | }
106 | //noinspection RedundantIfStatement
107 | if ( type != that.type )
108 | {
109 | return false;
110 | }
111 |
112 | return true;
113 | }
114 |
115 | @Override
116 | public int hashCode()
117 | {
118 | int result = type.hashCode();
119 | result = 31 * result + runId.hashCode();
120 | result = 31 * result + taskId.hashCode();
121 | return result;
122 | }
123 |
124 | @Override
125 | public String toString()
126 | {
127 | return "WorkflowEvent{" +
128 | "type=" + type +
129 | ", runId=" + runId +
130 | ", taskId=" + taskId +
131 | '}';
132 | }
133 | }
134 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/events/WorkflowListener.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.events;
17 |
18 | @FunctionalInterface
19 | public interface WorkflowListener
20 | {
21 | /**
22 | * Receive a workflow event
23 | *
24 | * @param event the event
25 | */
26 | void receiveEvent(WorkflowEvent event);
27 | }
28 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/events/WorkflowListenerManager.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.events;
17 |
18 | import org.apache.curator.framework.listen.Listenable;
19 | import java.io.Closeable;
20 |
21 | /**
22 | * A listener manager for events. Once {@link #start()} is called, workflow
23 | * events will be generated as runs and tasks are started, completed, etc.
24 | * When through, call {@link #close()}.
25 | */
26 | public interface WorkflowListenerManager extends Closeable
27 | {
28 | /**
29 | * Start listening for events. NOTE: previous events are not reported.
30 | * i.e. already started runs/tasks are not reported.
31 | */
32 | void start();
33 |
34 | /**
35 | * Return the container to add/remove event listeners
36 | *
37 | * @return container
38 | */
39 | Listenable getListenable();
40 | }
41 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/executor/TaskExecution.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.executor;
17 |
18 | import com.nirmata.workflow.models.TaskExecutionResult;
19 |
20 | /**
21 | * Represents a task execution. A new task execution is allocated for each run of a
22 | * task. The Workflow manager will call {@link #execute()} when the task should perform
23 | * its operation.
24 | */
25 | @FunctionalInterface
26 | public interface TaskExecution
27 | {
28 | /**
29 | * Execute the task and return the result when complete
30 | *
31 | * @return result
32 | */
33 | TaskExecutionResult execute();
34 | }
35 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/executor/TaskExecutionStatus.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.executor;
17 |
18 | /**
19 | * Task execution status
20 | */
21 | public enum TaskExecutionStatus
22 | {
23 | /**
24 | * The task executed successfully
25 | */
26 | SUCCESS()
27 | {
28 | @Override
29 | public boolean isCancelingStatus()
30 | {
31 | return false;
32 | }
33 | },
34 |
35 | /**
36 | * The task failed, but the remaining tasks should still execute
37 | */
38 | FAILED_CONTINUE()
39 | {
40 | @Override
41 | public boolean isCancelingStatus()
42 | {
43 | return false;
44 | }
45 | },
46 |
47 | /**
48 | * The task failed and the remaining tasks in the run should be canceled
49 | */
50 | FAILED_STOP()
51 | {
52 | @Override
53 | public boolean isCancelingStatus()
54 | {
55 | return true;
56 | }
57 | }
58 | ;
59 |
60 | public abstract boolean isCancelingStatus();
61 | }
62 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/executor/TaskExecutor.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.executor;
17 |
18 | import com.nirmata.workflow.WorkflowManager;
19 | import com.nirmata.workflow.models.ExecutableTask;
20 |
21 | /**
22 | * Factory for creating task executions
23 | */
24 | @FunctionalInterface
25 | public interface TaskExecutor
26 | {
27 | /**
28 | * Create a task execution for the given task
29 | *
30 | * @param workflowManager the manager
31 | * @param executableTask the task
32 | * @return the execution
33 | */
34 | TaskExecution newTaskExecution(WorkflowManager workflowManager, ExecutableTask executableTask);
35 | }
36 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/models/ExecutableTask.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.models;
17 |
18 | import com.google.common.base.Preconditions;
19 | import com.google.common.collect.ImmutableMap;
20 | import java.io.Serializable;
21 | import java.util.Map;
22 |
23 | /**
24 | * Models a task that has been scheduled for execution
25 | * as part of a run
26 | */
27 | public class ExecutableTask implements Serializable
28 | {
29 | private final RunId runId;
30 | private final TaskId taskId;
31 | private final TaskType taskType;
32 | private final Map metaData;
33 | private final boolean isExecutable;
34 |
35 | /**
36 | * @param runId the run that this is part of
37 | * @param taskId the task
38 | * @param taskType task type
39 | * @param metaData meta data
40 | * @param isExecutable if false, this is merely a container for child tasks
41 | */
42 | public ExecutableTask(RunId runId, TaskId taskId, TaskType taskType, Map metaData, boolean isExecutable)
43 | {
44 | this.runId = Preconditions.checkNotNull(runId, "runId cannot be null");
45 | metaData = Preconditions.checkNotNull(metaData, "metaData cannot be null");
46 | this.isExecutable = isExecutable;
47 | this.taskId = Preconditions.checkNotNull(taskId, "taskId cannot be null");
48 | this.taskType = Preconditions.checkNotNull(taskType, "taskType cannot be null");
49 | this.metaData = ImmutableMap.copyOf(metaData);
50 | }
51 |
52 | public RunId getRunId()
53 | {
54 | return runId;
55 | }
56 |
57 | public boolean isExecutable()
58 | {
59 | return isExecutable;
60 | }
61 |
62 | public TaskId getTaskId()
63 | {
64 | return taskId;
65 | }
66 |
67 | public TaskType getTaskType()
68 | {
69 | return taskType;
70 | }
71 |
72 | public Map getMetaData()
73 | {
74 | return metaData;
75 | }
76 |
77 | @Override
78 | public boolean equals(Object o)
79 | {
80 | if ( this == o )
81 | {
82 | return true;
83 | }
84 | if ( o == null || getClass() != o.getClass() )
85 | {
86 | return false;
87 | }
88 |
89 | ExecutableTask that = (ExecutableTask)o;
90 |
91 | if ( isExecutable != that.isExecutable )
92 | {
93 | return false;
94 | }
95 | if ( !metaData.equals(that.metaData) )
96 | {
97 | return false;
98 | }
99 | if ( !runId.equals(that.runId) )
100 | {
101 | return false;
102 | }
103 | if ( !taskId.equals(that.taskId) )
104 | {
105 | return false;
106 | }
107 | //noinspection RedundantIfStatement
108 | if ( !taskType.equals(that.taskType) )
109 | {
110 | return false;
111 | }
112 |
113 | return true;
114 | }
115 |
116 | @Override
117 | public int hashCode()
118 | {
119 | int result = runId.hashCode();
120 | result = 31 * result + taskId.hashCode();
121 | result = 31 * result + taskType.hashCode();
122 | result = 31 * result + metaData.hashCode();
123 | result = 31 * result + (isExecutable ? 1 : 0);
124 | return result;
125 | }
126 |
127 | @Override
128 | public String toString()
129 | {
130 | return "ExecutableTask{" +
131 | "runId=" + runId +
132 | ", taskId=" + taskId +
133 | ", taskType=" + taskType +
134 | ", metaData=" + metaData +
135 | ", isExecutable=" + isExecutable +
136 | '}';
137 | }
138 | }
139 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/models/Id.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.models;
17 |
18 | import com.google.common.base.Preconditions;
19 | import java.io.Serializable;
20 | import java.util.UUID;
21 |
22 | /**
23 | * Base for IDs
24 | */
25 | public class Id implements Serializable
26 | {
27 | private final String id;
28 |
29 | protected Id()
30 | {
31 | id = newRandomId();
32 | }
33 |
34 | static String newRandomId()
35 | {
36 | return UUID.randomUUID().toString();
37 | }
38 |
39 | protected Id(String id)
40 | {
41 | this.id = Preconditions.checkNotNull(id, "id cannot be null");
42 | }
43 |
44 | public String getId()
45 | {
46 | return id;
47 | }
48 |
49 | public boolean isValid()
50 | {
51 | return (id.length() > 0);
52 | }
53 |
54 | @Override
55 | public boolean equals(Object o)
56 | {
57 | if ( this == o )
58 | {
59 | return true;
60 | }
61 | if ( o == null || getClass() != o.getClass() )
62 | {
63 | return false;
64 | }
65 |
66 | Id id1 = (Id)o;
67 |
68 | //noinspection RedundantIfStatement
69 | if ( !id.equals(id1.id) )
70 | {
71 | return false;
72 | }
73 |
74 | return true;
75 | }
76 |
77 | @Override
78 | public int hashCode()
79 | {
80 | return id.hashCode();
81 | }
82 |
83 | @Override
84 | public String toString()
85 | {
86 | return "Id{" +
87 | "id='" + id + '\'' +
88 | '}';
89 | }
90 | }
91 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/models/RunId.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.models;
17 |
18 | public class RunId extends Id
19 | {
20 | /**
21 | * Generate a new, globally unique ID that has the given prefix.
22 | * E.g. newRandomIdWithPrefix("test") generates: test-828119e0-cd47-45a1-b120-94d284ecb7b3
23 | *
24 | * @param prefix ID's prefix
25 | * @return new ID
26 | */
27 | public static RunId newRandomIdWithPrefix(String prefix)
28 | {
29 | return new RunId(prefix + "-" + newRandomId());
30 | }
31 |
32 | public static void main(String[] args)
33 | {
34 | System.out.println(newRandomIdWithPrefix("hey"));
35 | System.out.println(newRandomIdWithPrefix("hey"));
36 | System.out.println(newRandomIdWithPrefix("hey"));
37 | System.out.println(newRandomIdWithPrefix("hey"));
38 | }
39 |
40 | public RunId()
41 | {
42 | }
43 |
44 | public RunId(String id)
45 | {
46 | super(id);
47 | }
48 | }
49 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/models/Task.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.models;
17 |
18 | import com.google.common.base.Preconditions;
19 | import com.google.common.collect.ImmutableList;
20 | import com.google.common.collect.ImmutableMap;
21 | import com.google.common.collect.Lists;
22 | import com.google.common.collect.Maps;
23 | import java.io.Serializable;
24 | import java.util.List;
25 | import java.util.Map;
26 | import java.util.Optional;
27 |
28 | /**
29 | * Models a task
30 | */
31 | public class Task implements Serializable
32 | {
33 | private final TaskId taskId;
34 | private final Optional taskType;
35 | private final List childrenTasks;
36 | private final Map metaData;
37 |
38 | /**
39 | * Metadata value used by special purpose executors
40 | */
41 | public static final String META_TASK_SUBMIT_VALUE = "__submit_value";
42 |
43 | /**
44 | * Utility to create a new meta map with the given submit value
45 | *
46 | * @param value the submit value
47 | * @return new meta map
48 | */
49 | public static Map makeSpecialMeta(long value)
50 | {
51 | Map meta = Maps.newHashMap();
52 | meta.put(META_TASK_SUBMIT_VALUE, Long.toString(value));
53 | return meta;
54 | }
55 |
56 | /**
57 | * @param taskId this task's ID - must be unique
58 | * @param taskType the task type
59 | */
60 | public Task(TaskId taskId, TaskType taskType)
61 | {
62 | this(taskId, taskType, Lists.newArrayList(), Maps.newHashMap());
63 | }
64 |
65 | /**
66 | * This constructor makes a "container only" task. i.e. this task does
67 | * not execute anything. It exists solely to contain children tasks.
68 | *
69 | * @param taskId this task's ID - must be unique
70 | * @param childrenTasks child tasks - children are not executed until this task completes
71 | */
72 | public Task(TaskId taskId, List childrenTasks)
73 | {
74 | this(taskId, null, childrenTasks, Maps.newHashMap());
75 | }
76 |
77 | /**
78 | * @param taskId this task's ID - must be unique
79 | * @param taskType the task type
80 | * @param childrenTasks child tasks - children are not executed until this task completes
81 | */
82 | public Task(TaskId taskId, TaskType taskType, List childrenTasks)
83 | {
84 | this(taskId, taskType, childrenTasks, Maps.newHashMap());
85 | }
86 |
87 | /**
88 | * @param taskId this task's ID - must be unique
89 | * @param taskType the task type
90 | * @param childrenTasks child tasks - children are not executed until this task completes
91 | * @param metaData meta data for the task
92 | */
93 | public Task(TaskId taskId, TaskType taskType, List childrenTasks, Map metaData)
94 | {
95 | metaData = Preconditions.checkNotNull(metaData, "metaData cannot be null");
96 | childrenTasks = Preconditions.checkNotNull(childrenTasks, "childrenTasks cannot be null");
97 | this.taskId = Preconditions.checkNotNull(taskId, "taskId cannot be null");
98 | this.taskType = Optional.ofNullable(taskType);
99 |
100 | this.metaData = ImmutableMap.copyOf(metaData);
101 | this.childrenTasks = ImmutableList.copyOf(childrenTasks);
102 | }
103 |
104 | public List getChildrenTasks()
105 | {
106 | return childrenTasks;
107 | }
108 |
109 | public TaskId getTaskId()
110 | {
111 | return taskId;
112 | }
113 |
114 | public TaskType getTaskType()
115 | {
116 | //noinspection ConstantConditions
117 | return taskType.get(); // exception if empty is desired
118 | }
119 |
120 | public boolean isExecutable()
121 | {
122 | return taskType.isPresent();
123 | }
124 |
125 | public Map getMetaData()
126 | {
127 | return metaData;
128 | }
129 |
130 | @Override
131 | public boolean equals(Object o)
132 | {
133 | if ( this == o )
134 | {
135 | return true;
136 | }
137 | if ( o == null || getClass() != o.getClass() )
138 | {
139 | return false;
140 | }
141 |
142 | Task task = (Task)o;
143 |
144 | if ( !childrenTasks.equals(task.childrenTasks) )
145 | {
146 | return false;
147 | }
148 | if ( !taskId.equals(task.taskId) )
149 | {
150 | return false;
151 | }
152 | //noinspection RedundantIfStatement
153 | if ( !taskType.equals(task.taskType) )
154 | {
155 | return false;
156 | }
157 |
158 | return true;
159 | }
160 |
161 | @Override
162 | public int hashCode()
163 | {
164 | int result = taskId.hashCode();
165 | result = 31 * result + taskType.hashCode();
166 | result = 31 * result + childrenTasks.hashCode();
167 | return result;
168 | }
169 |
170 | @Override
171 | public String toString()
172 | {
173 | return "Task{" +
174 | "taskId=" + taskId +
175 | ", taskType=" + taskType +
176 | ", childrenTasks=" + childrenTasks +
177 | '}';
178 | }
179 | }
180 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/models/TaskExecutionResult.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.models;
17 |
18 | import com.google.common.base.Preconditions;
19 | import com.google.common.collect.ImmutableMap;
20 | import com.google.common.collect.Maps;
21 | import com.nirmata.workflow.WorkflowManager;
22 | import com.nirmata.workflow.executor.TaskExecutionStatus;
23 | import java.io.Serializable;
24 | import java.time.Clock;
25 | import java.time.LocalDateTime;
26 | import java.util.Map;
27 | import java.util.Optional;
28 |
29 | /**
30 | * Models an execution result.
31 | */
32 | public class TaskExecutionResult implements Serializable
33 | {
34 | private final TaskExecutionStatus status;
35 | private final String message;
36 | private final Map resultData;
37 | private final LocalDateTime completionTimeUtc;
38 | private final Optional subTaskRunId;
39 |
40 | /**
41 | * @param status the execution status
42 | * @param message any message that task
43 | */
44 | public TaskExecutionResult(TaskExecutionStatus status, String message)
45 | {
46 | this(status, message, Maps.newHashMap(), null, null);
47 | }
48 |
49 | /**
50 | * @param status the execution status
51 | * @param message any message that task
52 | * @param resultData result data (can be accessed via {@link WorkflowManager#getTaskExecutionResult(RunId, TaskId)})
53 | */
54 | public TaskExecutionResult(TaskExecutionStatus status, String message, Map resultData)
55 | {
56 | this(status, message, resultData, null, null);
57 | }
58 |
59 | /**
60 | * @param status the execution status
61 | * @param message any message that task
62 | * @param resultData result data (can be accessed via {@link WorkflowManager#getTaskExecutionResult(RunId, TaskId)})
63 | * @param subTaskRunId if not null, the value of a sub-task started via {@link WorkflowManager#submitSubTask(RunId, Task)}. If
64 | * a sub-task was started, it's vital that the run ID be passed here so that this run can pause until the sub-task
65 | * completes.
66 | */
67 | public TaskExecutionResult(TaskExecutionStatus status, String message, Map resultData, RunId subTaskRunId)
68 | {
69 | this(status, message, resultData, subTaskRunId, null);
70 | }
71 |
72 | /**
73 | * @param status the execution status
74 | * @param message any message that task
75 | * @param resultData result data (can be accessed via {@link WorkflowManager#getTaskExecutionResult(RunId, TaskId)})
76 | * @param subTaskRunId if not null, the value of a sub-task started via {@link WorkflowManager#submitSubTask(RunId, Task)}. If
77 | * a sub-task was started, it's vital that the run ID be passed here so that this run can pause until the sub-task
78 | * completes.
79 | * @param completionTimeUtc the completion time of the task. If null, LocalDateTime.now(Clock.systemUTC()) is used
80 | */
81 | public TaskExecutionResult(TaskExecutionStatus status, String message, Map resultData, RunId subTaskRunId, LocalDateTime completionTimeUtc)
82 | {
83 | this.message = Preconditions.checkNotNull(message, "message cannot be null");
84 | this.status = Preconditions.checkNotNull(status, "status cannot be null");
85 | this.subTaskRunId = Optional.ofNullable(subTaskRunId);
86 | this.completionTimeUtc = (completionTimeUtc != null) ? completionTimeUtc : LocalDateTime.now(Clock.systemUTC());
87 |
88 | resultData = Preconditions.checkNotNull(resultData, "resultData cannot be null");
89 | this.resultData = ImmutableMap.copyOf(resultData);
90 | }
91 |
92 | public String getMessage()
93 | {
94 | return message;
95 | }
96 |
97 | public TaskExecutionStatus getStatus()
98 | {
99 | return status;
100 | }
101 |
102 | public Map getResultData()
103 | {
104 | return resultData;
105 | }
106 |
107 | public Optional getSubTaskRunId()
108 | {
109 | return subTaskRunId;
110 | }
111 |
112 | public LocalDateTime getCompletionTimeUtc()
113 | {
114 | return completionTimeUtc;
115 | }
116 |
117 | @Override
118 | public boolean equals(Object o)
119 | {
120 | if ( this == o )
121 | {
122 | return true;
123 | }
124 | if ( o == null || getClass() != o.getClass() )
125 | {
126 | return false;
127 | }
128 |
129 | TaskExecutionResult that = (TaskExecutionResult)o;
130 |
131 | if ( !completionTimeUtc.equals(that.completionTimeUtc) )
132 | {
133 | return false;
134 | }
135 | if ( !message.equals(that.message) )
136 | {
137 | return false;
138 | }
139 | if ( !resultData.equals(that.resultData) )
140 | {
141 | return false;
142 | }
143 | if ( status != that.status )
144 | {
145 | return false;
146 | }
147 | //noinspection RedundantIfStatement
148 | if ( !subTaskRunId.equals(that.subTaskRunId) )
149 | {
150 | return false;
151 | }
152 |
153 | return true;
154 | }
155 |
156 | @Override
157 | public int hashCode()
158 | {
159 | int result = status.hashCode();
160 | result = 31 * result + message.hashCode();
161 | result = 31 * result + resultData.hashCode();
162 | result = 31 * result + completionTimeUtc.hashCode();
163 | result = 31 * result + subTaskRunId.hashCode();
164 | return result;
165 | }
166 |
167 | @Override
168 | public String toString()
169 | {
170 | return "TaskExecutionResult{" +
171 | "status=" + status +
172 | ", message='" + message + '\'' +
173 | ", resultData=" + resultData +
174 | ", completionTimeUtc=" + completionTimeUtc +
175 | ", subTaskRunId=" + subTaskRunId +
176 | '}';
177 | }
178 | }
179 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/models/TaskId.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.models;
17 |
18 | public class TaskId extends Id
19 | {
20 | public TaskId()
21 | {
22 | }
23 |
24 | public TaskId(String id)
25 | {
26 | super(id);
27 | }
28 | }
29 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/models/TaskMode.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.models;
17 |
18 | public enum TaskMode
19 | {
20 | STANDARD(0),
21 | DELAY(1),
22 | PRIORITY(2)
23 | ;
24 |
25 | public static TaskMode fromCode(int code)
26 | {
27 | for ( TaskMode mode : values() )
28 | {
29 | if ( mode.code == code )
30 | {
31 | return mode;
32 | }
33 | }
34 | return null;
35 | }
36 |
37 | public int getCode()
38 | {
39 | return code;
40 | }
41 |
42 | private final int code;
43 |
44 | TaskMode(int code)
45 | {
46 | this.code = code;
47 | }
48 | }
49 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/models/TaskType.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.models;
17 |
18 | import com.google.common.base.Preconditions;
19 | import java.io.Serializable;
20 |
21 | /**
22 | *
23 | * Models a task type
24 | *
25 | *
26 | *
27 | * Tasks can be idempotent or non-idempotent. Idempotent tasks
28 | * can be restarted/re-executed when a workflow instance crashes
29 | * or there is some other kind of error. non-idempotent tasks
30 | * will only be attempted once.
31 | *
32 | */
33 | public class TaskType implements Serializable
34 | {
35 | private final String type;
36 | private final String version;
37 | private final boolean isIdempotent;
38 | private final TaskMode mode;
39 |
40 | // for backward compatibility
41 | public TaskType(String type, String version, boolean isIdempotent)
42 | {
43 | this(type, version, isIdempotent, TaskMode.STANDARD);
44 | }
45 |
46 | /**
47 | * @param type any value to represent the task type
48 | * @param version the version of this task type
49 | * @param isIdempotent whether or not this task is idempotent (see class description for details)
50 | * @param mode the mode
51 | */
52 | public TaskType(String type, String version, boolean isIdempotent, TaskMode mode)
53 | {
54 | Preconditions.checkArgument(!type.contains("/"), "type cannot contain '/'");
55 | Preconditions.checkArgument(!version.contains("/"), "version cannot contain '/'");
56 | this.mode = Preconditions.checkNotNull(mode, "mode cannot be null");
57 |
58 | this.version = Preconditions.checkNotNull(version, "version cannot be null");
59 | this.type = Preconditions.checkNotNull(type, "type cannot be null");
60 | this.isIdempotent = isIdempotent;
61 | }
62 |
63 | public String getVersion()
64 | {
65 | return version;
66 | }
67 |
68 | public String getType()
69 | {
70 | return type;
71 | }
72 |
73 | public boolean isIdempotent()
74 | {
75 | return isIdempotent;
76 | }
77 |
78 | public TaskMode getMode()
79 | {
80 | return mode;
81 | }
82 |
83 | @SuppressWarnings("SimplifiableIfStatement")
84 | @Override
85 | public boolean equals(Object o)
86 | {
87 | if ( this == o )
88 | {
89 | return true;
90 | }
91 | if ( o == null || getClass() != o.getClass() )
92 | {
93 | return false;
94 | }
95 |
96 | TaskType taskType = (TaskType)o;
97 |
98 | if ( isIdempotent != taskType.isIdempotent )
99 | {
100 | return false;
101 | }
102 | if ( !type.equals(taskType.type) )
103 | {
104 | return false;
105 | }
106 | if ( !version.equals(taskType.version) )
107 | {
108 | return false;
109 | }
110 | return mode == taskType.mode;
111 |
112 | }
113 |
114 | @Override
115 | public int hashCode()
116 | {
117 | int result = type.hashCode();
118 | result = 31 * result + version.hashCode();
119 | result = 31 * result + (isIdempotent ? 1 : 0);
120 | result = 31 * result + mode.hashCode();
121 | return result;
122 | }
123 |
124 | @Override
125 | public String toString()
126 | {
127 | return "TaskType{" +
128 | "type='" + type + '\'' +
129 | ", version='" + version + '\'' +
130 | ", isIdempotent=" + isIdempotent +
131 | ", mode=" + mode +
132 | '}';
133 | }
134 | }
135 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/queue/Queue.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.queue;
17 |
18 | import com.nirmata.workflow.models.ExecutableTask;
19 | import java.io.Closeable;
20 |
21 | public interface Queue extends Closeable
22 | {
23 | void start();
24 |
25 | @Override
26 | void close();
27 |
28 | void put(ExecutableTask executableTask);
29 | }
30 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/queue/QueueConsumer.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.queue;
17 |
18 | import com.google.common.annotations.VisibleForTesting;
19 | import com.nirmata.workflow.admin.WorkflowManagerState;
20 | import java.io.Closeable;
21 |
22 | public interface QueueConsumer extends Closeable
23 | {
24 | void start();
25 |
26 | @Override
27 | void close();
28 |
29 | WorkflowManagerState.State getState();
30 |
31 | @VisibleForTesting
32 | void debugValidateClosed();
33 | }
34 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/queue/QueueFactory.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.queue;
17 |
18 | import com.nirmata.workflow.details.WorkflowManagerImpl;
19 | import com.nirmata.workflow.models.TaskType;
20 |
21 | public interface QueueFactory
22 | {
23 | Queue createQueue(WorkflowManagerImpl workflowManager, TaskType taskType);
24 | QueueConsumer createQueueConsumer(WorkflowManagerImpl workflowManager, TaskRunner taskRunner, TaskType taskType);
25 | }
26 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/queue/TaskRunner.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.queue;
17 |
18 | import com.nirmata.workflow.models.ExecutableTask;
19 |
20 | @FunctionalInterface
21 | public interface TaskRunner
22 | {
23 | void executeTask(ExecutableTask executableTask);
24 | }
25 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/queue/zookeeper/ZooKeeperSimpleQueue.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.queue.zookeeper;
17 |
18 | import com.google.common.base.Preconditions;
19 | import com.nirmata.workflow.details.ZooKeeperConstants;
20 | import com.nirmata.workflow.models.ExecutableTask;
21 | import com.nirmata.workflow.models.Task;
22 | import com.nirmata.workflow.models.TaskType;
23 | import com.nirmata.workflow.queue.Queue;
24 | import com.nirmata.workflow.queue.TaskRunner;
25 | import com.nirmata.workflow.serialization.Serializer;
26 | import org.apache.curator.framework.CuratorFramework;
27 | import org.slf4j.Logger;
28 | import org.slf4j.LoggerFactory;
29 |
30 | public class ZooKeeperSimpleQueue implements Queue
31 | {
32 | private final Logger log = LoggerFactory.getLogger(getClass());
33 | private final SimpleQueue queue;
34 | private final Serializer serializer;
35 |
36 | ZooKeeperSimpleQueue(TaskRunner taskRunner, Serializer serializer, CuratorFramework client, TaskType taskType)
37 | {
38 | this.serializer = Preconditions.checkNotNull(serializer, "serializer cannot be null");
39 | String path = ZooKeeperConstants.getQueuePath(taskType);
40 | String lockPath = ZooKeeperConstants.getQueueLockPath(taskType);
41 | queue = new SimpleQueue(client, taskRunner, serializer, path, lockPath, taskType.getMode(), taskType.isIdempotent());
42 | }
43 |
44 | @Override
45 | public void start()
46 | {
47 | // NOP
48 | }
49 |
50 | @Override
51 | public void close()
52 | {
53 | // NOP
54 | }
55 |
56 | @Override
57 | public void put(ExecutableTask executableTask)
58 | {
59 | long value = 0;
60 | try
61 | {
62 | String valueStr = executableTask.getMetaData().get(Task.META_TASK_SUBMIT_VALUE);
63 | if ( valueStr != null )
64 | {
65 | value = Long.parseLong(valueStr);
66 | }
67 | }
68 | catch ( NumberFormatException ignore )
69 | {
70 | // ignore
71 | }
72 |
73 | try
74 | {
75 | byte[] bytes = serializer.serialize(executableTask);
76 | queue.put(bytes, value);
77 | }
78 | catch ( Exception e )
79 | {
80 | log.error("Could not add to queue for: " + executableTask, e);
81 | throw new RuntimeException(e);
82 | }
83 | }
84 |
85 | SimpleQueue getQueue()
86 | {
87 | return queue;
88 | }
89 |
90 | Serializer getSerializer()
91 | {
92 | return serializer;
93 | }
94 | }
95 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/queue/zookeeper/ZooKeeperSimpleQueueFactory.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.queue.zookeeper;
17 |
18 | import com.nirmata.workflow.details.WorkflowManagerImpl;
19 | import com.nirmata.workflow.models.TaskType;
20 | import com.nirmata.workflow.queue.Queue;
21 | import com.nirmata.workflow.queue.QueueConsumer;
22 | import com.nirmata.workflow.queue.QueueFactory;
23 | import com.nirmata.workflow.queue.TaskRunner;
24 |
25 | public class ZooKeeperSimpleQueueFactory implements QueueFactory
26 | {
27 | @Override
28 | public Queue createQueue(WorkflowManagerImpl workflowManager, TaskType taskType)
29 | {
30 | return internalCreateQueue(workflowManager, taskType, null);
31 | }
32 |
33 | @Override
34 | public QueueConsumer createQueueConsumer(WorkflowManagerImpl workflowManager, TaskRunner taskRunner, TaskType taskType)
35 | {
36 | ZooKeeperSimpleQueue queue = internalCreateQueue(workflowManager, taskType, taskRunner);
37 | return queue.getQueue();
38 | }
39 |
40 | private ZooKeeperSimpleQueue internalCreateQueue(WorkflowManagerImpl workflowManager, TaskType taskType, TaskRunner taskRunner)
41 | {
42 | return new ZooKeeperSimpleQueue(taskRunner, workflowManager.getSerializer(), workflowManager.getCurator(), taskType);
43 | }
44 | }
45 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/serialization/JsonSerializerMapper.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.serialization;
17 |
18 | import com.fasterxml.jackson.databind.JsonNode;
19 | import com.fasterxml.jackson.databind.ObjectMapper;
20 | import com.google.common.base.Preconditions;
21 | import com.google.common.collect.ImmutableMap;
22 | import java.lang.reflect.Method;
23 | import java.util.Map;
24 |
25 | public class JsonSerializerMapper
26 | {
27 | private final Map, Methods> methodsMap;
28 |
29 | private static final String NEW_PREFIX = "new";
30 | private static final String GET_PREFIX = "get";
31 |
32 | private static final String MODELS_PACKAGE = "com.nirmata.workflow.models.";
33 | private static final String INTERNAL_MODELS_PACKAGE = "com.nirmata.workflow.details.internalmodels.";
34 |
35 | private static class Methods
36 | {
37 | private final Method getter;
38 | private final Method newer;
39 |
40 | public Methods(Method getter, Method newer)
41 | {
42 | this.getter = getter;
43 | this.newer = newer;
44 | }
45 | }
46 |
47 | public JsonSerializerMapper()
48 | {
49 | ImmutableMap.Builder, Methods> builder = ImmutableMap.builder();
50 | for ( Method method : JsonSerializer.class.getDeclaredMethods() )
51 | {
52 | if ( method.getName().startsWith(NEW_PREFIX) && (method.getReturnType() == JsonNode.class) && (method.getParameterCount() == 1) )
53 | {
54 | String className = method.getName().substring(NEW_PREFIX.length());
55 | Method getter = getGetter(className);
56 | if ( getter != null )
57 | {
58 | Class clazz = getClazz(className);
59 | if ( (clazz != null) && (method.getParameterTypes()[0] == clazz) )
60 | {
61 | builder.put(clazz, new Methods(getter, method));
62 | }
63 | }
64 | }
65 | }
66 | methodsMap = builder.build();
67 | }
68 |
69 | public JsonNode make(T obj)
70 | {
71 | Methods methods = methodsMap.get(obj.getClass());
72 | methods = Preconditions.checkNotNull(methods, "No serializer found for: " + obj.getClass());
73 | try
74 | {
75 | return (JsonNode)methods.newer.invoke(null, obj);
76 | }
77 | catch ( Exception e )
78 | {
79 | throw new RuntimeException("Could not invoke newer() for: " + obj.getClass(), e);
80 | }
81 | }
82 |
83 | public ObjectMapper getMapper()
84 | {
85 | return JsonSerializer.getMapper();
86 | }
87 |
88 | public T get(JsonNode node, Class clazz)
89 | {
90 | Methods methods = methodsMap.get(clazz);
91 | methods = Preconditions.checkNotNull(methods, "No serializer found for: " + clazz);
92 | try
93 | {
94 | return clazz.cast(methods.getter.invoke(null, node));
95 | }
96 | catch ( Exception e )
97 | {
98 | throw new RuntimeException("Could not invoke getter() for: " + clazz, e);
99 | }
100 | }
101 |
102 | private Method getGetter(String className)
103 | {
104 | try
105 | {
106 | return JsonSerializer.class.getDeclaredMethod(GET_PREFIX + className, JsonNode.class);
107 | }
108 | catch ( NoSuchMethodException ignore )
109 | {
110 | // ignore
111 | }
112 | return null;
113 | }
114 |
115 | private Class getClazz(String className)
116 | {
117 | Class clazz = null;
118 | try
119 | {
120 | clazz = Class.forName(MODELS_PACKAGE + className);
121 | }
122 | catch ( ClassNotFoundException ignore )
123 | {
124 | try
125 | {
126 | clazz = Class.forName(INTERNAL_MODELS_PACKAGE + className);
127 | }
128 | catch ( ClassNotFoundException ignore2 )
129 | {
130 | // ignore
131 | }
132 | }
133 | return clazz;
134 | }
135 | }
136 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/serialization/Serializer.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.serialization;
17 |
18 | public interface Serializer
19 | {
20 | /**
21 | * Serialize the given object into bytes
22 | *
23 | * @param obj the object
24 | * @return byte representation
25 | */
26 | byte[] serialize(T obj);
27 |
28 | /**
29 | * Deserialize the given bytes into an object
30 | *
31 | * @param data bytes
32 | * @param clazz type of object
33 | * @return object
34 | */
35 | T deserialize(byte[] data, Class clazz);
36 | }
37 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/serialization/StandardSerializer.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.serialization;
17 |
18 | import com.fasterxml.jackson.core.JsonProcessingException;
19 | import com.fasterxml.jackson.databind.JsonNode;
20 | import org.slf4j.Logger;
21 | import org.slf4j.LoggerFactory;
22 | import java.io.IOException;
23 | import java.util.Arrays;
24 |
25 | public class StandardSerializer implements Serializer
26 | {
27 | private final Logger log = LoggerFactory.getLogger(getClass());
28 | private final JsonSerializerMapper mapper = new JsonSerializerMapper();
29 |
30 | @Override
31 | public byte[] serialize(T obj)
32 | {
33 | try
34 | {
35 | JsonNode node = mapper.make(obj);
36 | return JsonSerializer.getMapper().writeValueAsBytes(node);
37 | }
38 | catch ( JsonProcessingException e )
39 | {
40 | log.error("Could not serialize: " + obj.getClass(), e);
41 | throw new RuntimeException(e);
42 | }
43 | }
44 |
45 | @Override
46 | public T deserialize(byte[] data, Class clazz)
47 | {
48 | try
49 | {
50 | JsonNode node = JsonSerializer.getMapper().readTree(data);
51 | return mapper.get(node, clazz);
52 | }
53 | catch ( IOException e )
54 | {
55 | log.error("Could not deserialize: " + Arrays.toString(data), e);
56 | throw new RuntimeException(e);
57 | }
58 | }
59 | }
60 |
--------------------------------------------------------------------------------
/src/main/java/com/nirmata/workflow/serialization/TaskLoader.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.serialization;
17 |
18 | import com.nirmata.workflow.models.Task;
19 | import java.io.IOException;
20 | import java.io.Reader;
21 | import java.io.StringReader;
22 |
23 | /**
24 | * Utility for loading Tasks from a file/stream
25 | */
26 | public class TaskLoader
27 | {
28 | /**
29 | * Load a complete task with children from a JSON stream
30 | *
31 | * @param jsonStream the JSON stream
32 | * @return task
33 | */
34 | public static Task load(Reader jsonStream)
35 | {
36 | try
37 | {
38 | return JsonSerializer.getTask(JsonSerializer.getMapper().readTree(jsonStream));
39 | }
40 | catch ( IOException e )
41 | {
42 | throw new RuntimeException(e);
43 | }
44 | }
45 |
46 | /**
47 | * Load a complete task with children from a JSON string
48 | *
49 | * @param json the JSON
50 | * @return task
51 | */
52 | public static Task load(String json)
53 | {
54 | try ( StringReader reader = new StringReader(json) )
55 | {
56 | return load(reader);
57 | }
58 | }
59 |
60 | private TaskLoader()
61 | {
62 | }
63 | }
64 |
--------------------------------------------------------------------------------
/src/site/apt/admin.apt:
--------------------------------------------------------------------------------
1 | Administration APIs
2 |
3 | * WorkflowAdmin
4 |
5 | When a workflow run is executed, various data is written to ZooKeeper to keep track of the task runs and results. The WorkflowAdmin
6 | APIs allow you to get run info as well as clean up old runs. NOTE: the admin APIs can be called from any Workflow server.
7 |
8 | * << getRunInfo();>>>
9 |
10 | Info about all runs completed or currently executing in the workflow manager.
11 |
12 | * <<>>
13 |
14 | Info about the given run.
15 |
16 | * << getTaskInfo(RunId runId);>>>
17 |
18 | Info about all the tasks completed, started or waiting for the given run.
19 |
20 | * << getTaskDetails(RunId runId);>>>
21 |
22 | A map of all task details for the given run.
23 |
24 | * <<>>
25 |
26 | Delete all saved data for the given run. IMPORTANT: you should clean up old run data periodically so that
27 | your ZooKeeper instance doesn't get burdened by old data. Also, each time the WorkflowManager starts, it cycles
28 | through existing run data. Old run data will slow startup performance.
29 |
30 | * RunInfo
31 |
32 | Information about a specific task run.
33 |
34 | * <<>>
35 |
36 | The RunId.
37 |
38 | * <<>>
39 |
40 | The start time (in UTC) of the run.
41 |
42 | * <<>>
43 |
44 | Returns true if the run has completed.
45 |
46 | * <<>>
47 |
48 | The completion time (in UTC) of the run.
49 |
50 | * TaskInfo
51 |
52 | * <<>>
53 |
54 | The TaskId.
55 |
56 | * <<>>
57 |
58 | Returns true if the task has started. Otherwise, it is still waiting on dependent parent tasks.
59 |
60 | * <<>>
61 |
62 | The instance that ran the task.
63 |
64 | * <<>>
65 |
66 | The start time (in UTC) of the task.
67 |
68 | * <<>>
69 |
70 | Returns true if the task has completed.
71 |
72 | * <<>>
73 |
74 | For completed tasks, the result data.
75 |
76 | * TaskDetails
77 |
78 | * <<>>
79 |
80 | The TaskId.
81 |
82 | * <<>>
83 |
84 | Returns true if this is an executable task.
85 |
86 | * <<>>
87 |
88 | The task type (if executable).
89 |
90 | * << getMetaData();>>>
91 |
92 | The task's metadata.
93 |
--------------------------------------------------------------------------------
/src/site/apt/example.apt:
--------------------------------------------------------------------------------
1 | Example
2 |
3 | Here is a simple example that shows a Workflow Manager being started and a workflow of three tasks being submitted:
4 |
5 | * Example.java
6 |
7 | +---------------------------------------
8 | public class Example implements Closeable
9 | {
10 | private final TestingServer testingServer;
11 | private final CuratorFramework curator;
12 |
13 | public static void main(String[] args) throws Exception
14 | {
15 | Example example = new Example();
16 | example.runExample();
17 | example.close();
18 | }
19 |
20 | public Example() throws Exception
21 | {
22 | // for testing purposes, start an in-memory test ZooKeeper instance
23 | testingServer = new TestingServer();
24 |
25 | // allocate the Curator instance
26 | curator = CuratorFrameworkFactory.builder()
27 | .connectString(testingServer.getConnectString())
28 | .retryPolicy(new ExponentialBackoffRetry(100, 3))
29 | .build();
30 | }
31 |
32 | public void runExample()
33 | {
34 | curator.start();
35 |
36 | // for our example, we'll just have one task type
37 | TaskType taskType = new TaskType("my type", "1", true);
38 |
39 | // a task which will have two parents
40 | Task childTask = new Task(new TaskId("child task"), taskType);
41 |
42 | // parent #1
43 | Task parentTask1 = new Task(new TaskId("parent 1"), taskType, Lists.newArrayList(childTask));
44 |
45 | // parent #2
46 | Task parentTask2 = new Task(new TaskId("parent 2"), taskType, Lists.newArrayList(childTask));
47 |
48 | // a root container-only for the parent tasks
49 | Task rootTask = new Task(new TaskId(), Lists.newArrayList(parentTask1, parentTask2));
50 |
51 | // an executor that just logs a message and returns
52 | ExampleTaskExecutor taskExecutor = new ExampleTaskExecutor();
53 |
54 | // allocate the workflow manager with some executors for our type
55 | WorkflowManager workflowManager = WorkflowManagerBuilder.builder()
56 | .addingTaskExecutor(taskExecutor, 10, taskType)
57 | .withCurator(curator, "test", "1")
58 | .build();
59 |
60 | WorkflowListenerManager workflowListenerManager = workflowManager.newWorkflowListenerManager();
61 | try
62 | {
63 | // listen for run completion and count down a latch when it happens
64 | final CountDownLatch doneLatch = new CountDownLatch(1);
65 | WorkflowListener listener = new WorkflowListener()
66 | {
67 | @Override
68 | public void receiveEvent(WorkflowEvent event)
69 | {
70 | if ( event.getType() == WorkflowEvent.EventType.RUN_UPDATED )
71 | {
72 | // note: the run could have had an error. RUN_UPDATED does not guarantee successful completion
73 | doneLatch.countDown();
74 | }
75 | }
76 | };
77 | workflowListenerManager.getListenable().addListener(listener);
78 |
79 | // start the manager and the listeners
80 | workflowManager.start();
81 | workflowListenerManager.start();
82 |
83 | // submit our task
84 | workflowManager.submitTask(rootTask);
85 |
86 | // you should see these messages in the console:
87 | // Executing task: Id{id='parent 1'}
88 | // Executing task: Id{id='parent 2'}
89 | // then
90 | // Executing task: Id{id='child task'}
91 |
92 | // wait for completion
93 | doneLatch.await();
94 | }
95 | catch ( InterruptedException e )
96 | {
97 | Thread.currentThread().interrupt();
98 | }
99 | finally
100 | {
101 | CloseableUtils.closeQuietly(workflowListenerManager);
102 | CloseableUtils.closeQuietly(workflowManager);
103 | }
104 | }
105 |
106 | @Override
107 | public void close()
108 | {
109 | CloseableUtils.closeQuietly(curator);
110 | CloseableUtils.closeQuietly(testingServer);
111 | }
112 | }
113 | +---------------------------------------
114 |
115 | * ExampleTaskExecutor.java
116 |
117 | +---------------------------------------
118 | public class ExampleTaskExecutor implements TaskExecutor
119 | {
120 | @Override
121 | public TaskExecution newTaskExecution(WorkflowManager workflowManager, ExecutableTask executableTask)
122 | {
123 | return new TaskExecution()
124 | {
125 | @Override
126 | public TaskExecutionResult execute()
127 | {
128 | System.out.println("Executing task: " + executableTask.getTaskId());
129 | return new TaskExecutionResult(TaskExecutionStatus.SUCCESS, "My message");
130 | }
131 | };
132 | }
133 | }
134 | +---------------------------------------
135 |
--------------------------------------------------------------------------------
/src/site/apt/index.apt:
--------------------------------------------------------------------------------
1 | Nirmata Workflow
2 |
3 | Nirmata Workflow is a Java {{{http://zookeeper.apache.org/}Apache ZooKeeper}} and {{{http://curator.apache.org/}Apache Curator}}
4 | based library that enables distributed task workflows. You can build a
5 | cluster of servers as large as needed to execute tasks. You can limit servers to certain tasks types and/or allow servers to execute any task.
6 |
7 | A workflow is a {{{https://en.wikipedia.org/wiki/Directed_acyclic_graph}DAG}} of tasks
8 | whereby a group of tasks executes concurrently and when complete a set of children tasks execute and so on.
9 |
10 | [dag.png]
11 |
12 | In the DAG described in the diagram above, Tasks 1, 2 and 3 execute concurrently. When they complete, Tasks 4 and 5 execute. Then, Tasks 6, 7 and 8 execute.
13 | When Task 6 completes, Task 9 executes. When Tasks 6 and 8 complete, Task 10 executes.
14 |
15 | * Features
16 |
17 | * Can model simple to complex task relationships
18 |
19 | * Manages task relationships and distributed scheduling
20 |
21 | * Idempotent and non-idempotent tasks supported
22 |
23 | * Custom task-types can be defined and targeted to specific servers
24 |
25 | * Simple API
26 |
27 | * Supports runtime cluster changes
28 |
29 | * No single point of failure
30 |
31 |
--------------------------------------------------------------------------------
/src/site/apt/limits.apt:
--------------------------------------------------------------------------------
1 | Limits
2 |
3 | * Nirmata Workflow uses Apache ZooKeeper which has certain built-in limits. For example, the entire ZooKeeper database is kept in memory.
4 | So, huge data sets are not supported.
5 |
6 | * A given Nirmata Workflow run should be able to support several thousand tasks.
7 |
8 | * Each task should limit the amount of result data it produces. Result data is meant for a few dozen
9 | values at most.
10 |
11 | * It's vital that you clean up old runs periodically. See the {{{./admin.html}Administration APIs}} for details.
12 |
13 | * For proper high availability and failover, you should run a cluster of ZooKeeper instances (as opposed to just one).
14 |
15 |
--------------------------------------------------------------------------------
/src/site/apt/starting.apt:
--------------------------------------------------------------------------------
1 | Getting Started
2 |
3 | * Background
4 |
5 | The basic unit is a {{{./apidocs/com/nirmata/workflow/models/Task.html}Task}}. Tasks have an id, {{{./apidocs/com/nirmata/workflow/models/TaskType.html}type}},
6 | optional metadata, and zero or more children.
7 | Tasks are submitted to be executed through a {{{./apidocs/com/nirmata/workflow/WorkflowManager.html}WorkflowManager}} instance. WorkflowManager instances
8 | are created using a {{{./apidocs/com/nirmata/workflow/WorkflowManagerBuilder.html}WorkflowManagerBuilder}}. The work of a task is executed by a
9 | {{{./apidocs/com/nirmata/workflow/executor/TaskExecutor.html}TaskExecutor}} instance which you must provide an implementation for.
10 |
11 | * Prerequisites
12 |
13 | Nirmata Workflow uses {{{http://zookeeper.apache.org/}Apache ZooKeeper}} and {{{http://curator.apache.org/}Apache Curator}}. You will need to run
14 | an Apache ZooKeeper cluster in order to use the Workflow Manager.
15 | You will also need to create a Curator client instance for the Workflow Manager to use. Refer to the documentation of each of these projects
16 | for details.
17 |
18 | NOTE: Nirmata Workflow is a Java 8 Library
19 |
20 | * Using in Your Project
21 |
22 | Nirmata Workflow is available from Maven Central:
23 |
24 | *------------*-----------------------------*
25 | | GroupId | com.nirmata.workflow |
26 | *------------*-----------------------------*
27 | | ArtifactId | nirmata-workflow |
28 | *------------*-----------------------------*
29 |
30 | * From Maven
31 |
32 | +---------------------------------------
33 |
34 | com.nirmata.workflow
35 | nirmata-workflow
36 | ${nirmata-workflow-version}
37 |
38 | +---------------------------------------
39 |
40 | * From Gradle
41 |
42 | +---------------------------------------
43 | dependencies {
44 | compile('com.nirmata.workflow:nirmata-workflow:$nirmataWorkflowVersion')
45 | }
46 | +---------------------------------------
47 |
48 |
49 | * Tasks
50 |
51 | You can create task objects programmatically or via a file/stream. To load via file/stream, you can use the utility <<>>.
52 | Here is an {{{./tasks.json}example tasks JSON file}}. It defines 4 tasks. The first task, "root" is a
53 | container only task - i.e. there's nothing to execute; it just has children. Container only tasks don't have a "task type". Then, there is "task1" and "task2"
54 | both of which have a child task, "task3". This means that "task3" will not execute until both "task1" and "task2" complete.
55 |
56 | * Task Types
57 |
58 | Every task has a TaskType. The TaskType is used by your TaskExecutor implementation to do the work of the task. TaskType's have three fields:
59 | "type": a string that contains any value you define; "version": a version string used to version the tasks; "isIdempotent": whether or not
60 | the task is idempotent. TaskType also allows setting a priority or delay before a task executes.
61 |
62 | * TaskExecutor
63 |
64 | The TaskExecutor is a factory that you provide which creates TaskExecution instances. Every time a task needs to be executed, the newTaskExecution()
65 | method of TaskExecutor is called to get a new TaskExecution instance. The TaskExecution's execute() method is then called to do the work of the task
66 | and return a result. A task is executed randomly on one of the servers that is configured to handle tasks with that task type. Nirmata Workflow
67 | runs TaskExecutions in internally managed thread pools.
68 |
69 | * WorkflowManagerBuilder
70 |
71 | WorkflowManagerBuilder is used to build WorkflowManager instances. For most uses, you will have one WorkflowManager per server/JVM in your cluster.
72 | However, you can create as many WorkflowManager instances as you need. The builder has the following methods:
73 |
74 | *--------------------*------------------------------------------------------------------------*
75 | | withCurator | Required to set the {{{http://curator.apache.org/}Curator}} instance. |
76 | | | You also specify a namespace and a version. The namespace and version |
77 | | | combine to create a unique workflow. All instances using the same |
78 | | | namespace and version are logically part of the same workflow. |
79 | *--------------------*------------------------------------------------------------------------*
80 | | addingTaskExecutor | Adds a pool of task executors for a given task type to this instance |
81 | | | of the workflow. The specified number of executors are allocated. Call |
82 | | | this method multiple times to allocate executors for the various types |
83 | | | of tasks that will be used in this workflow. You can choose to have |
84 | | | all workflow instances execute all task types or target certain task |
85 | | | types to certain instances.\ |
86 | | | \ |
87 | | | qty is the maximum concurrency for the given type of task for this |
88 | | | instance. The logical concurrency for a given task type is the total |
89 | | | qty of all instances in the workflow. e.g. if there are 3 instances |
90 | | | in the workflow and instance A has 2 executors for task type "a", |
91 | | | instance B has 3 executors for task type "a" and instance C has no |
92 | | | executors for task type "a", the maximum concurrency for task type "a" |
93 | | | is 5.\ |
94 | | | \ |
95 | | | IMPORTANT: every workflow cluster must have at least one instance that |
96 | | | has task executor(s) for each task type that will be submitted to the |
97 | | | workflow. i.e workflows will stall if there is no executor for a given |
98 | | | task type. |
99 | *--------------------*------------------------------------------------------------------------*
100 | | withInstanceName | optional - used in reporting. This will be the value recorded as tasks |
101 | | | are executed. Via reporting, you can determine which instance has |
102 | | | executed a given task. the Default is: |
103 | | | <<>> |
104 | *--------------------*------------------------------------------------------------------------*
105 | | withAutoCleaner | optional - Sets an auto-cleaner that will run every given period. This |
106 | | | is used to clean old runs. IMPORTANT: the auto cleaner will only run |
107 | | | on the instance that is the current scheduler. |
108 | *--------------------*------------------------------------------------------------------------*
109 | | withSerializer | optional - By default, a JSON serializer is used to store data in |
110 | | | ZooKeeper. Use this to specify an alternate serializer. |
111 | *--------------------*------------------------------------------------------------------------*
112 | | withQueueFactory | optional - Pluggable queue factory. Default uses ZooKeeper for |
113 | | | queuing. |
114 | *--------------------*------------------------------------------------------------------------*
115 |
116 | * WorkflowManager
117 |
118 | WorkflowManager is the main API. Through it, you submit tasks, etc. Once created with WorkflowManagerBuilder, call the <<>> method and
119 | the workflow system will operate asynchronously. You can <<>> tasks at any time and from any instance.
120 |
--------------------------------------------------------------------------------
/src/site/dag source.graffle:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/nirmata/workflow/b92f8b36dd000682e217a77f12b6be16a0558f23/src/site/dag source.graffle
--------------------------------------------------------------------------------
/src/site/resources/css/site.css:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | tt {
17 | background-color: transparent;
18 | }
--------------------------------------------------------------------------------
/src/site/resources/dag.png:
--------------------------------------------------------------------------------
https://raw.githubusercontent.com/nirmata/workflow/b92f8b36dd000682e217a77f12b6be16a0558f23/src/site/resources/dag.png
--------------------------------------------------------------------------------
/src/site/resources/tasks.json:
--------------------------------------------------------------------------------
1 | {
2 | "rootTaskId": "root",
3 | "tasks": [
4 | {
5 | "taskId": "root",
6 | "childrenTaskIds": ["task1", "task2"]
7 | },
8 |
9 | {
10 | "taskId": "task1",
11 | "taskType": {
12 | "type": "test",
13 | "version": "1",
14 | "isIdempotent": true
15 | },
16 | "metaData": [
17 | {
18 | "field1": "value1",
19 | "field2": "value2"
20 | }
21 | ],
22 | "childrenTaskIds": ["task3"]
23 | },
24 |
25 | {
26 | "taskId": "task2",
27 | "taskType": {
28 | "type": "test",
29 | "version": "1",
30 | "isIdempotent": true
31 | },
32 | "metaData": [],
33 | "childrenTaskIds": ["task3"]
34 | },
35 |
36 | {
37 | "taskId": "task3",
38 | "taskType": {
39 | "type": "test",
40 | "version": "1",
41 | "isIdempotent": true
42 | },
43 | "metaData": [],
44 | "childrenTaskIds": []
45 | }
46 | ]
47 | }
--------------------------------------------------------------------------------
/src/site/site.xml:
--------------------------------------------------------------------------------
1 |
18 |
20 |
21 | org.apache.maven.skins
22 | maven-fluido-skin
23 | 1.7
24 |
25 |
26 |
27 | Nirmata Workflow
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 | NirmataOSS/workflow
36 | right
37 | black
38 |
39 |
40 |
41 |
42 |
43 |
51 |
52 |
58 |
59 |
64 |
65 |
66 |
--------------------------------------------------------------------------------
/src/test/java/com/nirmata/workflow/BaseForTests.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow;
17 |
18 | import org.apache.curator.framework.CuratorFramework;
19 | import org.apache.curator.framework.CuratorFrameworkFactory;
20 | import org.apache.curator.retry.RetryOneTime;
21 | import org.apache.curator.test.TestingServer;
22 | import org.apache.curator.test.Timing;
23 | import org.apache.curator.utils.CloseableUtils;
24 | import org.testng.annotations.AfterMethod;
25 | import org.testng.annotations.BeforeMethod;
26 |
27 | public abstract class BaseForTests
28 | {
29 | protected TestingServer server;
30 | protected CuratorFramework curator;
31 | protected final Timing timing = new Timing();
32 |
33 | @BeforeMethod
34 | public void setup() throws Exception
35 | {
36 | server = new TestingServer();
37 |
38 | curator = CuratorFrameworkFactory.builder().connectString(server.getConnectString()).retryPolicy(new RetryOneTime(1)).build();
39 | curator.start();
40 | }
41 |
42 | @AfterMethod
43 | public void teardown() throws Exception
44 | {
45 | CloseableUtils.closeQuietly(curator);
46 | CloseableUtils.closeQuietly(server);
47 | }
48 | }
49 |
--------------------------------------------------------------------------------
/src/test/java/com/nirmata/workflow/ConcurrentTaskChecker.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow;
17 |
18 | import com.google.common.collect.Lists;
19 | import com.google.common.collect.Sets;
20 | import com.nirmata.workflow.models.TaskId;
21 | import org.testng.Assert;
22 | import java.util.HashSet;
23 | import java.util.List;
24 | import java.util.Set;
25 |
26 | class ConcurrentTaskChecker
27 | {
28 | private final Set currentSet = Sets.newHashSet();
29 | private final List all = Lists.newArrayList();
30 | private int count = 0;
31 | private final List> sets = Lists.newArrayList();
32 |
33 | synchronized void reset()
34 | {
35 | currentSet.clear();
36 | all.clear();
37 | sets.clear();
38 | count = 0;
39 | }
40 |
41 | synchronized void add(TaskId taskId)
42 | {
43 | all.add(taskId);
44 | currentSet.add(taskId);
45 | ++count;
46 | }
47 |
48 | synchronized void decrement()
49 | {
50 | if ( --count == 0 )
51 | {
52 | HashSet copy = Sets.newHashSet(currentSet);
53 | currentSet.clear();
54 | count = 0;
55 | sets.add(copy);
56 | }
57 | }
58 |
59 | synchronized List> getSets()
60 | {
61 | return Lists.newArrayList(sets);
62 | }
63 |
64 | synchronized List getAll()
65 | {
66 | return Lists.newArrayList(all);
67 | }
68 |
69 | synchronized void assertNoDuplicates()
70 | {
71 | Assert.assertEquals(all.size(), Sets.newHashSet(all).size()); // no dups
72 | }
73 | }
74 |
--------------------------------------------------------------------------------
/src/test/java/com/nirmata/workflow/TestAutoCleanerHolder.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow;
17 |
18 | import com.google.common.collect.Lists;
19 | import com.nirmata.workflow.admin.RunInfo;
20 | import com.nirmata.workflow.admin.StandardAutoCleaner;
21 | import com.nirmata.workflow.admin.TaskDetails;
22 | import com.nirmata.workflow.admin.TaskInfo;
23 | import com.nirmata.workflow.admin.WorkflowAdmin;
24 | import com.nirmata.workflow.admin.WorkflowManagerState;
25 | import com.nirmata.workflow.details.AutoCleanerHolder;
26 | import com.nirmata.workflow.models.RunId;
27 | import com.nirmata.workflow.models.TaskId;
28 | import org.apache.curator.test.Timing;
29 | import org.testng.Assert;
30 | import org.testng.annotations.Test;
31 | import java.time.Clock;
32 | import java.time.Duration;
33 | import java.time.LocalDateTime;
34 | import java.util.List;
35 | import java.util.Map;
36 | import java.util.concurrent.TimeUnit;
37 |
38 | public class TestAutoCleanerHolder
39 | {
40 | @Test
41 | public void testNull() throws InterruptedException
42 | {
43 | Timing timing = new Timing();
44 | AutoCleanerHolder holder = new AutoCleanerHolder(null, Duration.ofDays(10));
45 | Assert.assertFalse(holder.shouldRun());
46 | Assert.assertFalse(holder.shouldRun());
47 | timing.sleepABit();
48 | Assert.assertFalse(holder.shouldRun());
49 | Assert.assertFalse(holder.shouldRun());
50 |
51 | Assert.assertNotNull(holder.getRunPeriod());
52 | }
53 |
54 | @Test
55 | public void testPeriod() throws InterruptedException
56 | {
57 | AutoCleanerHolder holder = new AutoCleanerHolder(new StandardAutoCleaner(Duration.ofSeconds(2)), Duration.ofSeconds(1));
58 | Assert.assertFalse(holder.shouldRun());
59 | TimeUnit.SECONDS.sleep(2);
60 | Assert.assertTrue(holder.shouldRun());
61 |
62 | RunId runningId = new RunId();
63 | RunId completedId = new RunId();
64 | RunId completedButRecentId = new RunId();
65 | List runs = Lists.newArrayList(
66 | new RunInfo(runningId, LocalDateTime.now(Clock.systemUTC())),
67 | new RunInfo(completedButRecentId, LocalDateTime.now(Clock.systemUTC()), LocalDateTime.now(Clock.systemUTC())),
68 | new RunInfo(completedId, LocalDateTime.now(Clock.systemUTC()), LocalDateTime.now(Clock.systemUTC()).minusSeconds(3))
69 | );
70 |
71 | List cleaned = Lists.newArrayList();
72 | WorkflowAdmin admin = new WorkflowAdmin()
73 | {
74 | @Override
75 | public List getRunInfo()
76 | {
77 | return runs;
78 | }
79 |
80 | @Override
81 | public List getRunIds()
82 | {
83 | throw new UnsupportedOperationException();
84 | }
85 |
86 | @Override
87 | public RunInfo getRunInfo(RunId runId)
88 | {
89 | throw new UnsupportedOperationException();
90 | }
91 |
92 | @Override
93 | public List getTaskInfo(RunId runId)
94 | {
95 | throw new UnsupportedOperationException();
96 | }
97 |
98 | @Override
99 | public WorkflowManagerState getWorkflowManagerState()
100 | {
101 | return new WorkflowManagerState(true, WorkflowManagerState.State.PROCESSING, Lists.newArrayList());
102 | }
103 |
104 | @Override
105 | public boolean clean(RunId runId)
106 | {
107 | cleaned.add(runId);
108 | return true;
109 | }
110 |
111 | @Override
112 | public Map getTaskDetails(RunId runId)
113 | {
114 | throw new UnsupportedOperationException();
115 | }
116 | };
117 |
118 | holder.run(admin);
119 | Assert.assertEquals(cleaned.size(), 1);
120 | Assert.assertEquals(cleaned.get(0), completedId);
121 | }
122 | }
123 |
--------------------------------------------------------------------------------
/src/test/java/com/nirmata/workflow/TestParentNodeDeletion.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow;
17 |
18 | import com.nirmata.workflow.admin.AutoCleaner;
19 | import com.nirmata.workflow.admin.StandardAutoCleaner;
20 | import com.nirmata.workflow.details.ZooKeeperConstants;
21 | import com.nirmata.workflow.executor.TaskExecutionStatus;
22 | import com.nirmata.workflow.executor.TaskExecutor;
23 | import com.nirmata.workflow.models.RunId;
24 | import com.nirmata.workflow.models.Task;
25 | import com.nirmata.workflow.models.TaskExecutionResult;
26 | import com.nirmata.workflow.models.TaskId;
27 | import com.nirmata.workflow.models.TaskType;
28 | import org.apache.curator.framework.CuratorFramework;
29 | import org.apache.curator.framework.CuratorFrameworkFactory;
30 | import org.apache.curator.retry.ExponentialBackoffRetry;
31 | import org.apache.curator.utils.ZKPaths;
32 | import org.apache.zookeeper.KeeperException;
33 | import org.slf4j.Logger;
34 | import org.slf4j.LoggerFactory;
35 | import org.testng.Assert;
36 | import org.testng.annotations.Test;
37 | import java.time.Duration;
38 | import java.util.concurrent.TimeUnit;
39 |
40 | public class TestParentNodeDeletion extends BaseForTests
41 | {
42 | private static final String TASK_NAME = "TestTask";
43 | private static final String VERSION = "v1";
44 | private static final TaskType TASK_TYPE = new TaskType(TASK_NAME, VERSION, true);
45 | private static final int CONCURRENT_TASKS = 5;
46 |
47 | private static final Logger log = LoggerFactory.getLogger(TestParentNodeDeletion.class);
48 |
49 | @Test
50 | public void testParentNodeDeletion() throws Exception
51 | {
52 | final String namespace = "test";
53 | final String workflowPath = "/" + namespace + "-" + VERSION;
54 |
55 | try ( CuratorFramework curator = newCurator() )
56 | {
57 | WorkflowManager workflowManager = buildWorkflow(curator, namespace);
58 | RunId runId = submitTask(workflowManager);
59 | assertTask(workflowManager, runId);
60 |
61 | String runPath = ZKPaths.makePath(workflowPath, ZooKeeperConstants.getRunParentPath());
62 | String startedPath = ZKPaths.makePath(workflowPath, ZooKeeperConstants.getStartedTasksParentPath());
63 | String completedPath = ZKPaths.makePath(workflowPath, ZooKeeperConstants.getCompletedTaskParentPath());
64 | String queuePath = ZKPaths.makePath(workflowPath, ZooKeeperConstants.getQueuePathBase());
65 |
66 | deletePath(curator, runPath);
67 | deletePath(curator, startedPath);
68 | deletePath(curator, completedPath);
69 |
70 | timing.sleepABit();
71 |
72 | runId = submitTask(workflowManager);
73 | assertTask(workflowManager, runId);
74 | }
75 | }
76 |
77 | private void assertTask(WorkflowManager workflowManager, RunId runId) throws Exception
78 | {
79 | Assert.assertTrue(workflowManager.getAdmin().getRunIds().contains(runId));
80 | long start = System.nanoTime();
81 | for(;;)
82 | {
83 | long elapsed = System.nanoTime() - start;
84 | if ( TimeUnit.NANOSECONDS.toMillis(elapsed) > timing.forWaiting().milliseconds() )
85 | {
86 | Assert.fail("Task did not execute within timeout");
87 | }
88 | Assert.assertTrue(workflowManager.getAdmin().getRunIds().contains(runId));
89 | if ( workflowManager.getAdmin().getRunInfo(runId).isComplete() )
90 | {
91 | break;
92 | }
93 | timing.sleepABit();
94 | }
95 | }
96 |
97 | private void deletePath(CuratorFramework curator, String path) throws Exception
98 | {
99 | try
100 | {
101 | curator.delete().deletingChildrenIfNeeded().forPath(path);
102 | }
103 | catch ( KeeperException ignore )
104 | {
105 | // ignore
106 | }
107 | }
108 |
109 | private CuratorFramework newCurator()
110 | {
111 | final CuratorFramework curator = CuratorFrameworkFactory.newClient(server.getConnectString(), timing.session(), timing.connection(), new ExponentialBackoffRetry(1000, 10));
112 | curator.start();
113 | return curator;
114 | }
115 |
116 | private WorkflowManager buildWorkflow(CuratorFramework curator, String namespace)
117 | {
118 | Duration runPeriod = Duration.ofSeconds(5);
119 | AutoCleaner cleaner = new StandardAutoCleaner(Duration.ofSeconds(5));
120 |
121 | final WorkflowManagerBuilder workflowManagerBuilder = WorkflowManagerBuilder.builder().withCurator(curator, namespace, VERSION).withAutoCleaner(cleaner, runPeriod);
122 |
123 | final TaskExecutor taskExecutor = (workflowManager, executableTask) -> () -> {
124 | final String runId = executableTask.getRunId().getId();
125 | final String taskId = executableTask.getTaskId().getId();
126 | log.debug("execute task {} - {}", runId, taskId);
127 |
128 | return new TaskExecutionResult(TaskExecutionStatus.SUCCESS, "");
129 | };
130 |
131 | workflowManagerBuilder.addingTaskExecutor(taskExecutor, CONCURRENT_TASKS, TASK_TYPE);
132 |
133 | final WorkflowManager workflowManager = workflowManagerBuilder.build();
134 | workflowManager.start();
135 |
136 | return workflowManager;
137 | }
138 |
139 | private RunId submitTask(WorkflowManager workflowManager)
140 | {
141 | TaskId taskId = new TaskId();
142 | Task task = new Task(taskId, TASK_TYPE);
143 | RunId runId = workflowManager.submitTask(task);
144 | log.debug("submit task done - runId {}", runId.getId());
145 | return runId;
146 | }
147 | }
148 |
--------------------------------------------------------------------------------
/src/test/java/com/nirmata/workflow/TestTaskExecutor.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow;
17 |
18 | import com.google.common.collect.Maps;
19 | import com.nirmata.workflow.executor.TaskExecution;
20 | import com.nirmata.workflow.executor.TaskExecutionStatus;
21 | import com.nirmata.workflow.executor.TaskExecutor;
22 | import com.nirmata.workflow.models.ExecutableTask;
23 | import com.nirmata.workflow.models.TaskExecutionResult;
24 | import java.util.concurrent.CountDownLatch;
25 |
26 | public class TestTaskExecutor implements TaskExecutor
27 | {
28 | private final ConcurrentTaskChecker checker = new ConcurrentTaskChecker();
29 | private final int latchQty;
30 | private volatile CountDownLatch latch;
31 |
32 | public TestTaskExecutor()
33 | {
34 | this(1);
35 | }
36 |
37 | public TestTaskExecutor(int latchQty)
38 | {
39 | this.latchQty = latchQty;
40 | latch = new CountDownLatch(latchQty);
41 | }
42 |
43 | public CountDownLatch getLatch()
44 | {
45 | return latch;
46 | }
47 |
48 | public ConcurrentTaskChecker getChecker()
49 | {
50 | return checker;
51 | }
52 |
53 | public void reset()
54 | {
55 | checker.reset();
56 | latch = new CountDownLatch(latchQty);
57 | }
58 |
59 | @Override
60 | public TaskExecution newTaskExecution(WorkflowManager workflowManager, ExecutableTask task)
61 | {
62 | return () -> {
63 | try
64 | {
65 | checker.add(task.getTaskId());
66 | doRun(task);
67 | }
68 | catch ( InterruptedException e )
69 | {
70 | Thread.currentThread().interrupt();
71 | throw new RuntimeException(e);
72 | }
73 | finally
74 | {
75 | checker.decrement();
76 | latch.countDown();
77 | }
78 | return new TaskExecutionResult(TaskExecutionStatus.SUCCESS, "hey", Maps.newHashMap());
79 | };
80 | }
81 |
82 | @SuppressWarnings("UnusedParameters")
83 | protected void doRun(ExecutableTask task) throws InterruptedException
84 | {
85 | Thread.sleep(1000);
86 | }
87 | }
88 |
--------------------------------------------------------------------------------
/src/test/java/com/nirmata/workflow/TestWorkflowListenerManager.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow;
17 |
18 | import com.google.common.collect.Queues;
19 | import com.nirmata.workflow.events.WorkflowEvent;
20 | import com.nirmata.workflow.events.WorkflowListenerManager;
21 | import com.nirmata.workflow.models.RunId;
22 | import com.nirmata.workflow.models.Task;
23 | import com.nirmata.workflow.models.TaskId;
24 | import com.nirmata.workflow.models.TaskType;
25 | import org.apache.curator.test.Timing;
26 | import org.apache.curator.utils.CloseableUtils;
27 | import org.testng.Assert;
28 | import org.testng.annotations.Test;
29 | import java.util.concurrent.BlockingQueue;
30 | import java.util.concurrent.TimeUnit;
31 |
32 | public class TestWorkflowListenerManager extends BaseForTests
33 | {
34 | @Test
35 | public void testBasic() throws Exception
36 | {
37 | WorkflowListenerManager workflowListenerManager = null;
38 | TestTaskExecutor taskExecutor = new TestTaskExecutor(6);
39 | TaskType taskType = new TaskType("test", "1", true);
40 | WorkflowManager workflowManager = WorkflowManagerBuilder.builder()
41 | .addingTaskExecutor(taskExecutor, 10, taskType)
42 | .withCurator(curator, "test", "1")
43 | .build();
44 | try
45 | {
46 | Task task = new Task(new TaskId(), taskType);
47 |
48 | BlockingQueue eventQueue = Queues.newLinkedBlockingQueue();
49 | workflowListenerManager = workflowManager.newWorkflowListenerManager();
50 | workflowListenerManager.getListenable().addListener(eventQueue::add);
51 |
52 | workflowManager.start();
53 | workflowListenerManager.start();
54 |
55 | RunId runId = workflowManager.submitTask(task);
56 |
57 | timing.sleepABit();
58 |
59 | WorkflowEvent runStarted = new WorkflowEvent(WorkflowEvent.EventType.RUN_STARTED, runId);
60 | WorkflowEvent taskStarted = new WorkflowEvent(WorkflowEvent.EventType.TASK_STARTED, runId, task.getTaskId());
61 | WorkflowEvent event1 = eventQueue.poll(timing.milliseconds(), TimeUnit.MILLISECONDS);
62 | WorkflowEvent event2 = eventQueue.poll(timing.milliseconds(), TimeUnit.MILLISECONDS);
63 | // due to timing, task start might come first
64 | Assert.assertTrue((event1.equals(runStarted) && event2.equals(taskStarted)) || (event2.equals(runStarted) && event1.equals(taskStarted)));
65 |
66 | Assert.assertEquals(eventQueue.poll(timing.milliseconds(), TimeUnit.MILLISECONDS), new WorkflowEvent(WorkflowEvent.EventType.TASK_COMPLETED, runId, task.getTaskId()));
67 | Assert.assertEquals(eventQueue.poll(timing.milliseconds(), TimeUnit.MILLISECONDS), new WorkflowEvent(WorkflowEvent.EventType.RUN_UPDATED, runId));
68 | }
69 | finally
70 | {
71 | CloseableUtils.closeQuietly(workflowListenerManager);
72 | CloseableUtils.closeQuietly(workflowManager);
73 | }
74 | }
75 | }
76 |
--------------------------------------------------------------------------------
/src/test/java/com/nirmata/workflow/WorkflowManagerStateSampler.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow;
17 |
18 | import com.google.common.base.Preconditions;
19 | import com.google.common.collect.Lists;
20 | import com.nirmata.workflow.admin.WorkflowAdmin;
21 | import com.nirmata.workflow.admin.WorkflowManagerState;
22 | import org.apache.curator.utils.ThreadUtils;
23 | import java.io.Closeable;
24 | import java.time.Duration;
25 | import java.util.LinkedList;
26 | import java.util.List;
27 | import java.util.concurrent.ScheduledExecutorService;
28 | import java.util.concurrent.TimeUnit;
29 | import java.util.concurrent.atomic.AtomicReference;
30 |
31 | public class WorkflowManagerStateSampler implements Closeable
32 | {
33 | private final WorkflowAdmin workflowAdmin;
34 | private final int windowSize;
35 | private final Duration samplePeriod;
36 | private final LinkedList samples; // guarded by sync
37 | private final ScheduledExecutorService service;
38 | private final AtomicReference state = new AtomicReference<>(State.LATENT);
39 |
40 | private enum State
41 | {
42 | LATENT,
43 | STARTED,
44 | CLOSED
45 | }
46 |
47 | public WorkflowManagerStateSampler(WorkflowAdmin workflowAdmin, int windowSize, Duration samplePeriod)
48 | {
49 | Preconditions.checkArgument(windowSize > 0, "windowSize must be greater than 0");
50 | this.workflowAdmin = workflowAdmin;
51 | this.windowSize = windowSize;
52 | this.samplePeriod = samplePeriod;
53 | samples = Lists.newLinkedList();
54 | service = ThreadUtils.newSingleThreadScheduledExecutor(WorkflowManagerStateSampler.class.getName());
55 | }
56 |
57 | public void start()
58 | {
59 | Preconditions.checkState(state.compareAndSet(State.LATENT, State.STARTED), "Already started");
60 | service.scheduleAtFixedRate(this::runLoop, samplePeriod.toMillis(), samplePeriod.toMillis(), TimeUnit.MILLISECONDS);
61 | }
62 |
63 | @Override
64 | public void close()
65 | {
66 | if ( state.compareAndSet(State.STARTED, State.CLOSED) )
67 | {
68 | service.shutdownNow();
69 | }
70 | }
71 |
72 | public List getSamples()
73 | {
74 | synchronized(samples)
75 | {
76 | return Lists.newArrayList(samples);
77 | }
78 | }
79 |
80 | private void runLoop()
81 | {
82 | WorkflowManagerState workflowManagerState = workflowAdmin.getWorkflowManagerState();
83 | synchronized(samples)
84 | {
85 | while ( samples.size() >= windowSize )
86 | {
87 | samples.removeFirst();
88 | }
89 | samples.add(workflowManagerState);
90 | }
91 | }
92 | }
93 |
--------------------------------------------------------------------------------
/src/test/java/com/nirmata/workflow/details/TestDelayPriorityTasks.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.details;
17 |
18 | import com.google.common.collect.Lists;
19 | import com.nirmata.workflow.BaseForTests;
20 | import com.nirmata.workflow.WorkflowManager;
21 | import com.nirmata.workflow.WorkflowManagerBuilder;
22 | import com.nirmata.workflow.executor.TaskExecutionStatus;
23 | import com.nirmata.workflow.executor.TaskExecutor;
24 | import com.nirmata.workflow.models.Task;
25 | import com.nirmata.workflow.models.TaskExecutionResult;
26 | import com.nirmata.workflow.models.TaskId;
27 | import com.nirmata.workflow.models.TaskMode;
28 | import com.nirmata.workflow.models.TaskType;
29 | import com.nirmata.workflow.queue.zookeeper.SimpleQueue;
30 | import org.testng.Assert;
31 | import org.testng.annotations.Test;
32 | import java.util.concurrent.BlockingQueue;
33 | import java.util.concurrent.LinkedBlockingQueue;
34 | import java.util.concurrent.Semaphore;
35 | import java.util.concurrent.TimeUnit;
36 |
37 | public class TestDelayPriorityTasks extends BaseForTests
38 | {
39 | @Test
40 | public void testDelay() throws Exception
41 | {
42 | final long delayMs = TimeUnit.SECONDS.toMillis(5);
43 |
44 | BlockingQueue queue = new LinkedBlockingQueue<>();
45 | TaskExecutor taskExecutor = (workflowManager, executableTask) -> () ->
46 | {
47 | queue.add(System.currentTimeMillis());
48 | return new TaskExecutionResult(TaskExecutionStatus.SUCCESS, "");
49 | };
50 | TaskType taskType = new TaskType("test", "1", true, TaskMode.DELAY);
51 | try ( WorkflowManager workflowManager = WorkflowManagerBuilder.builder()
52 | .addingTaskExecutor(taskExecutor, 10, taskType)
53 | .withCurator(curator, "test", "1")
54 | .build() )
55 | {
56 | workflowManager.start();
57 |
58 | Task task = new Task(new TaskId(), taskType);
59 | workflowManager.submitTask(task);
60 |
61 | Long ticksMs = queue.poll(1, TimeUnit.SECONDS);
62 | Assert.assertNotNull(ticksMs);
63 | Assert.assertTrue((ticksMs - ((WorkflowManagerImpl)workflowManager).debugLastSubmittedTimeMs) < 1000); // should have executed immediately
64 |
65 | task = new Task(new TaskId(), taskType, Lists.newArrayList(), Task.makeSpecialMeta(System.currentTimeMillis() + delayMs));
66 | long startTicks = System.currentTimeMillis();
67 | workflowManager.submitTask(task);
68 | ticksMs = queue.poll(delayMs * 2, TimeUnit.MILLISECONDS);
69 | Assert.assertNotNull(ticksMs);
70 | long elapsed = ticksMs - startTicks;
71 | Assert.assertTrue(elapsed >= delayMs, String.format("Bad timing. Elapsed: %d, delay: %d ", elapsed, delayMs));
72 | }
73 | }
74 |
75 | @Test
76 | public void testPriority() throws Exception
77 | {
78 | BlockingQueue queue = new LinkedBlockingQueue<>();
79 | TaskExecutor taskExecutor = (workflowManager, executableTask) -> () ->
80 | {
81 | queue.add(executableTask.getTaskId().getId());
82 | try
83 | {
84 | Thread.sleep(1);
85 | }
86 | catch ( InterruptedException e )
87 | {
88 | Thread.currentThread().interrupt();
89 | }
90 | return new TaskExecutionResult(TaskExecutionStatus.SUCCESS, "");
91 | };
92 | TaskType taskType = new TaskType("test", "1", true, TaskMode.PRIORITY);
93 | try ( WorkflowManager workflowManager = WorkflowManagerBuilder.builder()
94 | .addingTaskExecutor(taskExecutor, 1, taskType)
95 | .withCurator(curator, "test", "1")
96 | .build() )
97 | {
98 | SimpleQueue.debugQueuedTasks = new Semaphore(0);
99 | ((WorkflowManagerImpl)workflowManager).debugDontStartConsumers = true; // make sure all tasks are added to ZK before they start getting consumed
100 | workflowManager.start();
101 |
102 | Task task1 = new Task(new TaskId("1"), taskType, Lists.newArrayList(), Task.makeSpecialMeta(1));
103 | Task task2 = new Task(new TaskId("2"), taskType, Lists.newArrayList(), Task.makeSpecialMeta(100));
104 | Task task3 = new Task(new TaskId("3"), taskType, Lists.newArrayList(), Task.makeSpecialMeta(50));
105 | Task task4 = new Task(new TaskId("4"), taskType, Lists.newArrayList(), Task.makeSpecialMeta(300));
106 | Task task5 = new Task(new TaskId("5"), taskType, Lists.newArrayList(), Task.makeSpecialMeta(200));
107 | workflowManager.submitTask(task1);
108 | workflowManager.submitTask(task2);
109 | workflowManager.submitTask(task3);
110 | workflowManager.submitTask(task4);
111 | workflowManager.submitTask(task5);
112 |
113 | Assert.assertTrue(SimpleQueue.debugQueuedTasks.tryAcquire(5, 5, TimeUnit.SECONDS));
114 | ((WorkflowManagerImpl)workflowManager).startQueueConsumers();
115 |
116 | Assert.assertEquals(queue.poll(1, TimeUnit.SECONDS), "1");
117 | Assert.assertEquals(queue.poll(1, TimeUnit.SECONDS), "3");
118 | Assert.assertEquals(queue.poll(1, TimeUnit.SECONDS), "2");
119 | Assert.assertEquals(queue.poll(1, TimeUnit.SECONDS), "5");
120 | Assert.assertEquals(queue.poll(1, TimeUnit.SECONDS), "4");
121 | }
122 | finally
123 | {
124 | SimpleQueue.debugQueuedTasks = null;
125 | }
126 | }
127 | }
128 |
--------------------------------------------------------------------------------
/src/test/java/com/nirmata/workflow/details/TestDisruptedScheduler.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.details;
17 |
18 | import com.google.common.collect.Lists;
19 | import com.google.common.collect.Sets;
20 | import com.google.common.io.Resources;
21 | import com.nirmata.workflow.TestTaskExecutor;
22 | import com.nirmata.workflow.WorkflowManager;
23 | import com.nirmata.workflow.WorkflowManagerBuilder;
24 | import com.nirmata.workflow.models.ExecutableTask;
25 | import com.nirmata.workflow.models.Task;
26 | import com.nirmata.workflow.models.TaskId;
27 | import com.nirmata.workflow.models.TaskType;
28 | import com.nirmata.workflow.serialization.JsonSerializerMapper;
29 | import org.apache.curator.framework.CuratorFramework;
30 | import org.apache.curator.framework.CuratorFrameworkFactory;
31 | import org.apache.curator.retry.ExponentialBackoffRetry;
32 | import org.apache.curator.test.TestingCluster;
33 | import org.apache.curator.test.Timing;
34 | import org.apache.curator.utils.CloseableUtils;
35 | import org.testng.Assert;
36 | import org.testng.annotations.AfterMethod;
37 | import org.testng.annotations.BeforeMethod;
38 | import org.testng.annotations.Test;
39 | import java.io.Closeable;
40 | import java.io.IOException;
41 | import java.nio.charset.Charset;
42 | import java.util.List;
43 | import java.util.Optional;
44 | import java.util.Set;
45 | import java.util.concurrent.CountDownLatch;
46 | import java.util.stream.IntStream;
47 |
48 | public class TestDisruptedScheduler
49 | {
50 | private static final TaskType taskType = new TaskType("test", "1", true);
51 | private static final Timing timing = new Timing(1, 2);
52 |
53 | private TestingCluster cluster;
54 | private List clients;
55 | private Set executedTasks;
56 | private CountDownLatch executedTasksLatch;
57 |
58 | @BeforeMethod
59 | public void setup() throws Exception
60 | {
61 | cluster = new TestingCluster(3);
62 | cluster.start();
63 |
64 | clients = Lists.newArrayList();
65 | executedTasks = Sets.newConcurrentHashSet();
66 | executedTasksLatch = new CountDownLatch(6);
67 | }
68 |
69 | @AfterMethod
70 | public void teardown() throws Exception
71 | {
72 | clients.forEach(CloseableUtils::closeQuietly);
73 | clients = null;
74 | CloseableUtils.closeQuietly(cluster);
75 | }
76 |
77 | private static class Client implements Closeable
78 | {
79 | final CuratorFramework curator;
80 | final WorkflowManager workflowManager;
81 |
82 | private Client(int id, TestingCluster cluster, Set executedTasks, CountDownLatch executedTasksLatch)
83 | {
84 | curator = CuratorFrameworkFactory.builder().connectString(cluster.getConnectString()).retryPolicy(new ExponentialBackoffRetry(10, 3)).build();
85 | curator.start();
86 |
87 | TestTaskExecutor taskExecutor = new TestTaskExecutor(6) {
88 | @Override
89 | protected void doRun(ExecutableTask task) throws InterruptedException
90 | {
91 | executedTasks.add(task.getTaskId());
92 | timing.forWaiting().sleepABit();
93 | executedTasksLatch.countDown();
94 | }
95 | };
96 |
97 | workflowManager = WorkflowManagerBuilder.builder()
98 | .addingTaskExecutor(taskExecutor, 10, taskType)
99 | .withCurator(curator, "test", "1")
100 | .withInstanceName("i-" + id)
101 | .build();
102 | workflowManager.start();
103 | }
104 |
105 | @Override
106 | public void close() throws IOException
107 | {
108 | CloseableUtils.closeQuietly(workflowManager);
109 | CloseableUtils.closeQuietly(curator);
110 | }
111 | }
112 |
113 | @Test
114 | public void testDisruptedScheduler() throws Exception
115 | {
116 | final int QTY = 3;
117 | IntStream.range(0, QTY).forEach(i -> clients.add(new Client(i, cluster, executedTasks, executedTasksLatch)));
118 |
119 | Optional clientOptional = Optional.empty();
120 | for ( int i = 0; !clientOptional.isPresent() && (i < 3); ++i )
121 | {
122 | timing.sleepABit();
123 | clientOptional = clients
124 | .stream()
125 | .filter(client -> ((WorkflowManagerImpl)client.workflowManager).getSchedulerSelector().getLeaderSelector().hasLeadership())
126 | .findFirst();
127 | }
128 | Assert.assertTrue(clientOptional.isPresent());
129 | Client scheduler = clientOptional.get();
130 | Client nonScheduler = clients.get(0).equals(scheduler) ? clients.get(1) : clients.get(0);
131 |
132 | String json = Resources.toString(Resources.getResource("tasks.json"), Charset.defaultCharset());
133 | JsonSerializerMapper jsonSerializerMapper = new JsonSerializerMapper();
134 | Task task = jsonSerializerMapper.get(jsonSerializerMapper.getMapper().readTree(json), Task.class);
135 | nonScheduler.workflowManager.submitTask(task); // additional test - submit to the non-scheduler
136 |
137 | while ( executedTasks.size() == 0 ) // wait until some tasks have started
138 | {
139 | Thread.sleep(100);
140 | }
141 |
142 | CountDownLatch latch = new CountDownLatch(1);
143 | ((WorkflowManagerImpl)scheduler.workflowManager).getSchedulerSelector().debugLatch.set(latch);
144 | ((WorkflowManagerImpl)scheduler.workflowManager).getSchedulerSelector().getLeaderSelector().interruptLeadership(); // interrupt the scheduler wherever it is
145 | Assert.assertTrue(executedTasks.size() < 6);
146 | Assert.assertTrue(timing.awaitLatch(latch)); // wait for the scheduler method to exit
147 |
148 | Assert.assertTrue(timing.awaitLatch(executedTasksLatch)); // all tasks should complete
149 | }
150 | }
151 |
--------------------------------------------------------------------------------
/src/test/java/com/nirmata/workflow/details/TestEdges.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.details;
17 |
18 | import com.google.common.collect.Lists;
19 | import com.google.common.collect.Queues;
20 | import com.google.common.collect.Sets;
21 | import com.nirmata.workflow.BaseForTests;
22 | import com.nirmata.workflow.WorkflowManager;
23 | import com.nirmata.workflow.WorkflowManagerBuilder;
24 | import com.nirmata.workflow.executor.TaskExecutionStatus;
25 | import com.nirmata.workflow.executor.TaskExecutor;
26 | import com.nirmata.workflow.models.Task;
27 | import com.nirmata.workflow.models.TaskExecutionResult;
28 | import com.nirmata.workflow.models.TaskId;
29 | import com.nirmata.workflow.models.TaskType;
30 | import org.apache.curator.test.Timing;
31 | import org.apache.curator.utils.CloseableUtils;
32 | import org.testng.Assert;
33 | import org.testng.annotations.Test;
34 | import java.util.List;
35 | import java.util.Queue;
36 | import java.util.Set;
37 | import java.util.concurrent.Semaphore;
38 | import java.util.concurrent.atomic.AtomicInteger;
39 | import java.util.stream.Collectors;
40 | import java.util.stream.IntStream;
41 |
42 | public class TestEdges extends BaseForTests
43 | {
44 | @Test
45 | public void testMultiConcurrent() throws Exception
46 | {
47 | final int RETRIES = 3;
48 |
49 | try
50 | {
51 | IntStream.range(0, RETRIES).forEach(i -> {
52 | System.out.println("Retry " + i);
53 | multiConcurrentRetry();
54 | });
55 | }
56 | finally
57 | {
58 | Scheduler.debugBadRunIdCount = null;
59 | }
60 | }
61 |
62 | private void multiConcurrentRetry()
63 | {
64 | final int WORKFLOW_QTY = 3;
65 | final int ITERATIONS = 10;
66 |
67 | Scheduler.debugBadRunIdCount = new AtomicInteger(0);
68 |
69 | TaskType type1 = new TaskType("type1", "1", true);
70 | TaskType type2 = new TaskType("type2", "1", true);
71 |
72 | Semaphore semaphore = new Semaphore(0);
73 | TaskExecutor taskExecutor = (m, t) -> () -> {
74 | try
75 | {
76 | Thread.sleep(578);
77 | }
78 | catch ( InterruptedException e )
79 | {
80 | Thread.currentThread().interrupt();
81 | }
82 | semaphore.release();
83 | return new TaskExecutionResult(TaskExecutionStatus.SUCCESS, "");
84 | };
85 |
86 | List workflowManagers = IntStream.range(0, WORKFLOW_QTY).mapToObj(i -> {
87 | TaskType type = ((i & 1) != 0) ? type1 : type2;
88 | return WorkflowManagerBuilder.builder()
89 | .addingTaskExecutor(taskExecutor, 10, type)
90 | .withCurator(curator, "test-" + i, "1")
91 | .build();
92 | }).collect(Collectors.toList());
93 | try
94 | {
95 | workflowManagers.stream().forEach(WorkflowManager::start);
96 |
97 | for ( int i = 0; i < ITERATIONS; ++i )
98 | {
99 | Assert.assertEquals(Scheduler.debugBadRunIdCount.get(), 0);
100 | System.out.println("Iteration " + i);
101 | for ( int j = 0; j < WORKFLOW_QTY; ++j )
102 | {
103 | TaskType type = ((j & 1) != 0) ? type1 : type2;
104 | workflowManagers.get(j).submitTask(new Task(new TaskId(), type));
105 | }
106 | Assert.assertTrue(timing.acquireSemaphore(semaphore, WORKFLOW_QTY));
107 | }
108 | }
109 | finally
110 | {
111 | workflowManagers.stream().forEach(CloseableUtils::closeQuietly);
112 | }
113 | }
114 |
115 | @Test
116 | public void testIdempotency() throws Exception
117 | {
118 | TaskType idempotentType = new TaskType("yes", "1", true);
119 | TaskType nonIdempotentType = new TaskType("no", "1", false);
120 |
121 | Task idempotentTask = new Task(new TaskId(), idempotentType);
122 | Task nonIdempotentTask = new Task(new TaskId(), nonIdempotentType);
123 | Task root = new Task(new TaskId(), Lists.newArrayList(idempotentTask, nonIdempotentTask));
124 |
125 | Set thrownTasks = Sets.newConcurrentHashSet();
126 | Queue tasks = Queues.newConcurrentLinkedQueue();
127 | TaskExecutor taskExecutor = (m, t) -> () -> {
128 | if ( thrownTasks.add(t.getTaskId()) )
129 | {
130 | throw new RuntimeException();
131 | }
132 | tasks.add(t.getTaskId());
133 | return new TaskExecutionResult(TaskExecutionStatus.SUCCESS, "");
134 | };
135 | WorkflowManager workflowManager = WorkflowManagerBuilder.builder()
136 | .addingTaskExecutor(taskExecutor, 10, idempotentType)
137 | .addingTaskExecutor(taskExecutor, 10, nonIdempotentType)
138 | .withCurator(curator, "test", "1")
139 | .build();
140 | try
141 | {
142 | workflowManager.start();
143 | workflowManager.submitTask(root);
144 |
145 | timing.sleepABit();
146 |
147 | Assert.assertEquals(tasks.size(), 1);
148 | Assert.assertEquals(tasks.poll(), idempotentTask.getTaskId());
149 | }
150 | finally
151 | {
152 | CloseableUtils.closeQuietly(workflowManager);
153 | }
154 | }
155 | }
156 |
--------------------------------------------------------------------------------
/src/test/java/com/nirmata/workflow/serialization/JDKSerializer.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.serialization;
17 |
18 | import java.io.ByteArrayInputStream;
19 | import java.io.ByteArrayOutputStream;
20 | import java.io.IOException;
21 | import java.io.ObjectInputStream;
22 | import java.io.ObjectOutputStream;
23 |
24 | public class JDKSerializer implements Serializer
25 | {
26 | @Override
27 | public byte[] serialize(T obj)
28 | {
29 | ByteArrayOutputStream bytes = null;
30 | try
31 | {
32 | bytes = new ByteArrayOutputStream();
33 | ObjectOutputStream out = new ObjectOutputStream(bytes);
34 | out.writeObject(obj);
35 | out.close();
36 | }
37 | catch ( IOException e )
38 | {
39 | throw new RuntimeException(e);
40 | }
41 | return bytes.toByteArray();
42 | }
43 |
44 | @Override
45 | public T deserialize(byte[] data, Class clazz)
46 | {
47 | try
48 | {
49 | ObjectInputStream in = new ObjectInputStream(new ByteArrayInputStream(data));
50 | return clazz.cast(in.readObject());
51 | }
52 | catch ( Exception e )
53 | {
54 | throw new RuntimeException(e);
55 | }
56 | }
57 | }
58 |
--------------------------------------------------------------------------------
/src/test/java/com/nirmata/workflow/serialization/TestSerializer.java:
--------------------------------------------------------------------------------
1 | /**
2 | * Copyright 2014 Nirmata, Inc.
3 | *
4 | * Licensed under the Apache License, Version 2.0 (the "License");
5 | * you may not use this file except in compliance with the License.
6 | * You may obtain a copy of the License at
7 | *
8 | * http://www.apache.org/licenses/LICENSE-2.0
9 | *
10 | * Unless required by applicable law or agreed to in writing, software
11 | * distributed under the License is distributed on an "AS IS" BASIS,
12 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | * See the License for the specific language governing permissions and
14 | * limitations under the License.
15 | */
16 | package com.nirmata.workflow.serialization;
17 |
18 | import com.fasterxml.jackson.databind.JsonNode;
19 | import com.google.common.collect.Lists;
20 | import com.google.common.collect.Maps;
21 | import com.google.common.io.Resources;
22 | import com.nirmata.workflow.details.internalmodels.RunnableTask;
23 | import com.nirmata.workflow.details.internalmodels.RunnableTaskDag;
24 | import com.nirmata.workflow.details.internalmodels.StartedTask;
25 | import com.nirmata.workflow.executor.TaskExecutionStatus;
26 | import com.nirmata.workflow.models.ExecutableTask;
27 | import com.nirmata.workflow.models.RunId;
28 | import com.nirmata.workflow.models.Task;
29 | import com.nirmata.workflow.models.TaskExecutionResult;
30 | import com.nirmata.workflow.models.TaskId;
31 | import com.nirmata.workflow.models.TaskType;
32 | import org.testng.Assert;
33 | import org.testng.annotations.Test;
34 | import java.io.IOException;
35 | import java.nio.charset.Charset;
36 | import java.time.Clock;
37 | import java.time.LocalDateTime;
38 | import java.util.Collection;
39 | import java.util.List;
40 | import java.util.Map;
41 | import java.util.Random;
42 | import java.util.stream.Collectors;
43 | import java.util.stream.IntStream;
44 | import java.util.stream.Stream;
45 |
46 | import static com.nirmata.workflow.serialization.JsonSerializer.*;
47 |
48 | public class TestSerializer
49 | {
50 | private static final Random random = new Random();
51 |
52 | @Test
53 | public void testRunnableTaskDag()
54 | {
55 | RunnableTaskDag runnableTaskDag = new RunnableTaskDag(new TaskId(), randomTasks());
56 | JsonNode node = newRunnableTaskDag(runnableTaskDag);
57 | String str = nodeToString(node);
58 | System.out.println(str);
59 |
60 | RunnableTaskDag unRunnableTaskDag = getRunnableTaskDag(fromString(str));
61 | Assert.assertEquals(runnableTaskDag, unRunnableTaskDag);
62 | }
63 |
64 | @Test
65 | public void testExecutableTask()
66 | {
67 | ExecutableTask executableTask = new ExecutableTask(new RunId(), new TaskId(), randomTaskType(), randomMap(), random.nextBoolean());
68 | JsonNode node = newExecutableTask(executableTask);
69 | String str = nodeToString(node);
70 | System.out.println(str);
71 |
72 | ExecutableTask unExecutableTask = getExecutableTask(fromString(str));
73 | Assert.assertEquals(executableTask, unExecutableTask);
74 | }
75 |
76 | @Test
77 | public void testRunnableTask()
78 | {
79 | Map tasks = Stream.generate(() -> "")
80 | .limit(random.nextInt(3) + 1)
81 | .collect(Collectors.toMap(s -> new TaskId(), s -> new ExecutableTask(new RunId(), new TaskId(), randomTaskType(), randomMap(), random.nextBoolean())))
82 | ;
83 | List taskDags = Stream.generate(() -> new RunnableTaskDag(new TaskId(), randomTasks()))
84 | .limit(random.nextInt(3) + 1)
85 | .collect(Collectors.toList())
86 | ;
87 | LocalDateTime completionTime = random.nextBoolean() ? LocalDateTime.now() : null;
88 | RunId parentRunId = random.nextBoolean() ? new RunId() : null;
89 | RunnableTask runnableTask = new RunnableTask(tasks, taskDags, LocalDateTime.now(), completionTime, parentRunId);
90 | JsonNode node = newRunnableTask(runnableTask);
91 | String str = nodeToString(node);
92 | System.out.println(str);
93 |
94 | RunnableTask unRunnableTask = getRunnableTask(fromString(str));
95 | Assert.assertEquals(runnableTask, unRunnableTask);
96 | }
97 |
98 | @Test
99 | public void testTaskExecutionResult()
100 | {
101 | TaskExecutionResult taskExecutionResult = new TaskExecutionResult(TaskExecutionStatus.SUCCESS, Integer.toString(random.nextInt()), randomMap());
102 | JsonNode node = newTaskExecutionResult(taskExecutionResult);
103 | String str = nodeToString(node);
104 | System.out.println(str);
105 |
106 | TaskExecutionResult unTaskExecutionResult = getTaskExecutionResult(fromString(str));
107 | Assert.assertEquals(taskExecutionResult, unTaskExecutionResult);
108 | }
109 |
110 | @Test
111 | public void testStartedTask()
112 | {
113 | StartedTask startedTask = new StartedTask(Integer.toString(random.nextInt()), LocalDateTime.now(Clock.systemUTC()), 0);
114 | JsonNode node = newStartedTask(startedTask);
115 | String str = nodeToString(node);
116 | System.out.println(str);
117 |
118 | StartedTask unStartedTask = getStartedTask(fromString(str));
119 | Assert.assertEquals(startedTask, unStartedTask);
120 | }
121 |
122 | @Test
123 | public void testTaskType()
124 | {
125 | TaskType taskType = randomTaskType();
126 | JsonNode node = newTaskType(taskType);
127 | String str = nodeToString(node);
128 | System.out.println(str);
129 |
130 | TaskType unTaskType = getTaskType(fromString(str));
131 | Assert.assertEquals(taskType, unTaskType);
132 | }
133 |
134 | @Test
135 | public void testTask()
136 | {
137 | Task task = randomTask(0);
138 | JsonNode node = newTask(task);
139 | String str = nodeToString(node);
140 | System.out.println(str);
141 |
142 | Task unTask = TaskLoader.load(str);
143 | Assert.assertEquals(task, unTask);
144 | }
145 |
146 | @Test
147 | public void testLoadedTask() throws IOException
148 | {
149 | TaskType taskType = new TaskType("test", "1", true);
150 | Task task6 = new Task(new TaskId("task6"), taskType);
151 | Task task5 = new Task(new TaskId("task5"), taskType, Lists.newArrayList(task6), Maps.newHashMap());
152 | Task task4 = new Task(new TaskId("task4"), taskType, Lists.newArrayList(task6), Maps.newHashMap());
153 | Task task3 = new Task(new TaskId("task3"), taskType, Lists.newArrayList(task6), Maps.newHashMap());
154 | Task task2 = new Task(new TaskId("task2"), taskType, Lists.newArrayList(task3, task4, task5), Maps.newHashMap());
155 | Task task1 = new Task(new TaskId("task1"), taskType, Lists.newArrayList(task3, task4, task5), Maps.newHashMap());
156 | Task task = new Task(new TaskId("root"), Lists.newArrayList(task1, task2));
157 |
158 | String json = Resources.toString(Resources.getResource("tasks.json"), Charset.defaultCharset());
159 | Task unTask = TaskLoader.load(json);
160 |
161 | Assert.assertEquals(task, unTask);
162 | }
163 |
164 | @Test
165 | public void testJsonSerializerMapper()
166 | {
167 | JsonSerializerMapper mapper = new JsonSerializerMapper();
168 | Task task = randomTask(0);
169 | JsonNode node = mapper.make(task);
170 | String str = nodeToString(node);
171 | System.out.println(str);
172 |
173 | Task unTask = mapper.get(fromString(str), Task.class);
174 | Assert.assertEquals(task, unTask);
175 | }
176 |
177 | @Test
178 | public void testAlternateSerializers()
179 | {
180 | JDKSerializer serializer = new JDKSerializer();
181 |
182 | StartedTask startedTask = new StartedTask(Integer.toString(random.nextInt()), LocalDateTime.now(Clock.systemUTC()), 0);
183 | byte[] bytes = serializer.serialize(startedTask);
184 | StartedTask unStartedTask = serializer.deserialize(bytes, StartedTask.class);
185 | Assert.assertEquals(startedTask, unStartedTask);
186 | }
187 |
188 | private Task randomTask(int index)
189 | {
190 | List childrenTasks = Lists.newArrayList();
191 | boolean shouldHaveChildren = (index == 0) || ((index < 2) && random.nextBoolean());
192 | int childrenQty = shouldHaveChildren ? random.nextInt(5) : 0;
193 | IntStream.range(0, childrenQty).forEach(i -> childrenTasks.add(randomTask(index + 1)));
194 | return new Task(new TaskId(), randomTaskType(), childrenTasks, randomMap());
195 | }
196 |
197 | private TaskType randomTaskType()
198 | {
199 | return new TaskType(Integer.toString(random.nextInt()), Integer.toHexString(random.nextInt()), random.nextBoolean());
200 | }
201 |
202 | private Map randomMap()
203 | {
204 | return Stream.generate(random::nextInt)
205 | .limit(random.nextInt(3) + 1)
206 | .collect(Collectors.toMap(n -> Integer.toString(n * 4), n -> Integer.toString(n * 2)));
207 | }
208 |
209 | private Collection randomTasks()
210 | {
211 | return Stream.generate(TaskId::new)
212 | .limit(random.nextInt(10) + 1)
213 | .collect(Collectors.toList());
214 | }
215 | }
216 |
--------------------------------------------------------------------------------
/src/test/resources/log4j.properties:
--------------------------------------------------------------------------------
1 | #
2 | # Copyright 2014 Nirmata, Inc.
3 | #
4 | # Licensed under the Apache License, Version 2.0 (the "License");
5 | # you may not use this file except in compliance with the License.
6 | # You may obtain a copy of the License at
7 | #
8 | # http://www.apache.org/licenses/LICENSE-2.0
9 | #
10 | # Unless required by applicable law or agreed to in writing, software
11 | # distributed under the License is distributed on an "AS IS" BASIS,
12 | # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 | # See the License for the specific language governing permissions and
14 | # limitations under the License.
15 | #
16 |
17 | log4j.rootLogger=WARN, console
18 | log4j.logger.com.nirmata.workflow=DEBUG
19 |
20 | log4j.appender.console=org.apache.log4j.ConsoleAppender
21 | log4j.appender.console.layout=org.apache.log4j.PatternLayout
22 | log4j.appender.console.layout.ConversionPattern=%-5p %c %x %m [%t]%n
23 |
--------------------------------------------------------------------------------
/src/test/resources/multi-tasks.json:
--------------------------------------------------------------------------------
1 | {
2 | "rootTaskId": "root",
3 | "tasks": [
4 | {
5 | "taskId": "root",
6 | "childrenTaskIds": ["task1", "task2"]
7 | },
8 |
9 | {
10 | "taskId": "task1",
11 | "taskType": {
12 | "type": "type1",
13 | "version": "1",
14 | "isIdempotent": true
15 | },
16 | "metaData": [],
17 | "childrenTaskIds": ["task3", "task4", "task5"]
18 | },
19 |
20 | {
21 | "taskId": "task2",
22 | "taskType": {
23 | "type": "type1",
24 | "version": "1",
25 | "isIdempotent": true
26 | },
27 | "metaData": [],
28 | "childrenTaskIds": ["task3", "task4", "task5"]
29 | },
30 |
31 | {
32 | "taskId": "task3",
33 | "taskType": {
34 | "type": "type2",
35 | "version": "1",
36 | "isIdempotent": true
37 | },
38 | "metaData": [],
39 | "childrenTaskIds": ["task6"]
40 | },
41 |
42 | {
43 | "taskId": "task4",
44 | "taskType": {
45 | "type": "type2",
46 | "version": "1",
47 | "isIdempotent": true
48 | },
49 | "metaData": [],
50 | "childrenTaskIds": ["task6"]
51 | },
52 |
53 | {
54 | "taskId": "task5",
55 | "taskType": {
56 | "type": "type3",
57 | "version": "1",
58 | "isIdempotent": true
59 | },
60 | "metaData": [],
61 | "childrenTaskIds": ["task6"]
62 | },
63 |
64 | {
65 | "taskId": "task6",
66 | "taskType": {
67 | "type": "type3",
68 | "version": "1",
69 | "isIdempotent": true
70 | },
71 | "metaData": [],
72 | "childrenTaskIds": []
73 | }
74 | ]
75 | }
--------------------------------------------------------------------------------
/src/test/resources/tasks.json:
--------------------------------------------------------------------------------
1 | {
2 | "rootTaskId": "root",
3 | "tasks": [
4 | {
5 | "taskId": "root",
6 | "childrenTaskIds": ["task1", "task2"]
7 | },
8 |
9 | {
10 | "taskId": "task1",
11 | "taskType": {
12 | "type": "test",
13 | "version": "1",
14 | "isIdempotent": true
15 | },
16 | "metaData": [],
17 | "childrenTaskIds": ["task3", "task4", "task5"]
18 | },
19 |
20 | {
21 | "taskId": "task2",
22 | "taskType": {
23 | "type": "test",
24 | "version": "1",
25 | "isIdempotent": true
26 | },
27 | "metaData": [],
28 | "childrenTaskIds": ["task3", "task4", "task5"]
29 | },
30 |
31 | {
32 | "taskId": "task3",
33 | "taskType": {
34 | "type": "test",
35 | "version": "1",
36 | "isIdempotent": true
37 | },
38 | "metaData": [],
39 | "childrenTaskIds": ["task6"]
40 | },
41 |
42 | {
43 | "taskId": "task4",
44 | "taskType": {
45 | "type": "test",
46 | "version": "1",
47 | "isIdempotent": true
48 | },
49 | "metaData": [],
50 | "childrenTaskIds": ["task6"]
51 | },
52 |
53 | {
54 | "taskId": "task5",
55 | "taskType": {
56 | "type": "test",
57 | "version": "1",
58 | "isIdempotent": true
59 | },
60 | "metaData": [],
61 | "childrenTaskIds": ["task6"]
62 | },
63 |
64 | {
65 | "taskId": "task6",
66 | "taskType": {
67 | "type": "test",
68 | "version": "1",
69 | "isIdempotent": true
70 | },
71 | "metaData": [],
72 | "childrenTaskIds": []
73 | }
74 | ]
75 | }
--------------------------------------------------------------------------------