blacklistRemovals) {
39 | LOG.debug("Blacklist Additions: {} , Blacklist Removals: {}", blacklistAdditions, blacklistRemovals);
40 | amrmClient.updateBlacklist(blacklistAdditions, blacklistRemovals);
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/twill-yarn/src/main/hadoop26/org/apache/twill/internal/yarn/Hadoop26YarnAMClient.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.internal.yarn;
19 |
20 | import org.apache.hadoop.conf.Configuration;
21 | import org.apache.hadoop.yarn.api.records.ContainerId;
22 | import org.slf4j.Logger;
23 | import org.slf4j.LoggerFactory;
24 |
25 | import java.util.List;
26 |
27 | /**
28 | * Wrapper class for AMRMClient for Hadoop version 2.6 or greater.
29 | */
30 | public final class Hadoop26YarnAMClient extends Hadoop22YarnAMClient {
31 |
32 | private static final Logger LOG = LoggerFactory.getLogger(Hadoop26YarnAMClient.class);
33 |
34 | public Hadoop26YarnAMClient(Configuration conf) {
35 | super(conf);
36 | }
37 |
38 | @Override
39 | protected final ContainerId containerIdLookup(String containerIdStr) {
40 | return (ContainerId.fromString(containerIdStr));
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/twill-yarn/src/main/hadoop26/org/apache/twill/internal/yarn/Hadoop26YarnAppClient.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | package org.apache.twill.internal.yarn;
20 |
21 | import org.apache.hadoop.conf.Configuration;
22 | import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
23 | import org.apache.twill.api.Configs;
24 |
25 | /**
26 | *
27 | * The service implementation of {@link YarnAppClient} for Apache Hadoop 2.6 and beyond.
28 | *
29 | * The {@link VersionDetectYarnAppClientFactory} class will decide to return instance of this class for
30 | * Apache Hadoop 2.6 and beyond.
31 | *
32 | */
33 | @SuppressWarnings("unused")
34 | public class Hadoop26YarnAppClient extends Hadoop23YarnAppClient {
35 |
36 | public Hadoop26YarnAppClient(Configuration configuration) {
37 | super(configuration);
38 | }
39 |
40 | @Override
41 | protected void configureAppSubmissionContext(ApplicationSubmissionContext context) {
42 | super.configureAppSubmissionContext(context);
43 | long interval = configuration.getLong(Configs.Keys.YARN_ATTEMPT_FAILURES_VALIDITY_INTERVAL, -1L);
44 | if (interval > 0) {
45 | context.setAttemptFailuresValidityInterval(interval);
46 | }
47 | }
48 | }
49 |
--------------------------------------------------------------------------------
/twill-yarn/src/main/java/org/apache/twill/filesystem/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | /**
19 | * Contains HDFS location classes.
20 | */
21 | package org.apache.twill.filesystem;
22 |
--------------------------------------------------------------------------------
/twill-yarn/src/main/java/org/apache/twill/internal/ContainerExitCodes.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.internal;
19 |
20 | /**
21 | * Collection of known exit code. Some of the codes are copied from ContainerExitStatus as that class is missing in
22 | * older YARN version.
23 | */
24 | public final class ContainerExitCodes {
25 |
26 | public static final int SUCCESS = 0;
27 |
28 | /**
29 | * When the container exit when it fails to initialize.
30 | */
31 | public static final int INIT_FAILED = 10;
32 |
33 | public static final int INVALID = -1000;
34 |
35 | /**
36 | * Containers killed by the framework, either due to being released by
37 | * the application or being 'lost' due to node failures etc.
38 | */
39 | public static final int ABORTED = -100;
40 |
41 | /**
42 | * When threshold number of the nodemanager-local-directories or
43 | * threshold number of the nodemanager-log-directories become bad.
44 | */
45 | public static final int DISKS_FAILED = -101;
46 |
47 | /**
48 | * Containers preempted by the YARN framework.
49 | */
50 | public static final int PREEMPTED = -102;
51 | }
52 |
--------------------------------------------------------------------------------
/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationMasterInfo.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.internal.appmaster;
19 |
20 | import org.apache.hadoop.yarn.api.records.ApplicationId;
21 | import org.apache.twill.internal.ResourceCapability;
22 |
23 | /**
24 | * Represents information of the application master.
25 | */
26 | public class ApplicationMasterInfo implements ResourceCapability {
27 |
28 | private final ApplicationId appId;
29 | private final int memoryMB;
30 | private final int virtualCores;
31 |
32 | public ApplicationMasterInfo(ApplicationId appId, int memoryMB, int virtualCores) {
33 | this.appId = appId;
34 | this.memoryMB = memoryMB;
35 | this.virtualCores = virtualCores;
36 | }
37 |
38 | /**
39 | * Returns the application ID for the YARN application.
40 | */
41 | public ApplicationId getAppId() {
42 | return appId;
43 | }
44 |
45 | @Override
46 | public int getMemoryMB() {
47 | return memoryMB;
48 | }
49 |
50 | @Override
51 | public int getVirtualCores() {
52 | return virtualCores;
53 | }
54 |
55 | @Override
56 | public String toString() {
57 | return "ApplicationMasterInfo{" +
58 | "appId=" + appId +
59 | ", memoryMB=" + memoryMB +
60 | ", virtualCores=" + virtualCores +
61 | '}';
62 | }
63 | }
64 |
--------------------------------------------------------------------------------
/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ApplicationSubmitter.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.internal.appmaster;
19 |
20 | import org.apache.twill.internal.ProcessController;
21 | import org.apache.twill.internal.yarn.YarnApplicationReport;
22 | import org.apache.twill.internal.yarn.YarnLaunchContext;
23 |
24 | /**
25 | * Interface for submitting a new application to run.
26 | */
27 | public interface ApplicationSubmitter {
28 |
29 | ProcessController submit(YarnLaunchContext launchContext);
30 | }
31 |
--------------------------------------------------------------------------------
/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/BasicEventHandlerContext.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.internal.appmaster;
19 |
20 | import org.apache.twill.api.EventHandlerContext;
21 | import org.apache.twill.api.EventHandlerSpecification;
22 | import org.apache.twill.api.RunId;
23 | import org.apache.twill.internal.TwillRuntimeSpecification;
24 |
25 | /**
26 | *
27 | */
28 | final class BasicEventHandlerContext implements EventHandlerContext {
29 |
30 | private final String applicationName;
31 | private final RunId runId;
32 | private final EventHandlerSpecification specification;
33 |
34 | public BasicEventHandlerContext(TwillRuntimeSpecification twillRuntimeSpec) {
35 | this.applicationName = twillRuntimeSpec.getTwillAppName();
36 | this.runId = twillRuntimeSpec.getTwillAppRunId();
37 | this.specification = twillRuntimeSpec.getTwillSpecification().getEventHandler();
38 | }
39 |
40 | @Override
41 | public String getApplicationName() {
42 | return applicationName;
43 | }
44 |
45 | @Override
46 | public RunId getRunId() {
47 | return runId;
48 | }
49 |
50 | @Override
51 | public EventHandlerSpecification getSpecification() {
52 | return specification;
53 | }
54 | }
55 |
--------------------------------------------------------------------------------
/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/LoggerContextListenerAdapter.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.internal.appmaster;
19 |
20 | import ch.qos.logback.classic.Level;
21 | import ch.qos.logback.classic.Logger;
22 | import ch.qos.logback.classic.LoggerContext;
23 | import ch.qos.logback.classic.spi.LoggerContextListener;
24 |
25 | /**
26 | *
27 | */
28 | abstract class LoggerContextListenerAdapter implements LoggerContextListener {
29 |
30 | private final boolean resetResistant;
31 |
32 | protected LoggerContextListenerAdapter(boolean resetResistant) {
33 | this.resetResistant = resetResistant;
34 | }
35 |
36 | @Override
37 | public final boolean isResetResistant() {
38 | return resetResistant;
39 | }
40 |
41 | @Override
42 | public void onStart(LoggerContext context) {
43 | }
44 |
45 | @Override
46 | public void onReset(LoggerContext context) {
47 | }
48 |
49 | @Override
50 | public void onStop(LoggerContext context) {
51 | }
52 |
53 | @Override
54 | public void onLevelChange(Logger logger, Level level) {
55 | }
56 | }
57 |
--------------------------------------------------------------------------------
/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/ProvisionRequest.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.internal.appmaster;
19 |
20 | import org.apache.twill.api.RuntimeSpecification;
21 |
22 | /**
23 | * Package private class to help AM to track in progress container request.
24 | */
25 | final class ProvisionRequest {
26 | private final RuntimeSpecification runtimeSpec;
27 | private final String requestId;
28 | private int requestCount;
29 | private final AllocationSpecification.Type type;
30 |
31 | ProvisionRequest(RuntimeSpecification runtimeSpec, String requestId, int requestCount) {
32 | this(runtimeSpec, requestId, requestCount, AllocationSpecification.Type.DEFAULT);
33 | }
34 |
35 | ProvisionRequest(RuntimeSpecification runtimeSpec, String requestId, int requestCount,
36 | AllocationSpecification.Type type) {
37 | this.runtimeSpec = runtimeSpec;
38 | this.requestId = requestId;
39 | this.requestCount = requestCount;
40 | this.type = type;
41 | }
42 |
43 | RuntimeSpecification getRuntimeSpec() {
44 | return runtimeSpec;
45 | }
46 |
47 | String getRequestId() {
48 | return requestId;
49 | }
50 |
51 | /**
52 | * Called to notify a container has been provision for this request.
53 | * @return {@code true} if the requested container count has been provisioned.
54 | */
55 | boolean containerAcquired() {
56 | requestCount--;
57 | return requestCount == 0;
58 | }
59 |
60 | AllocationSpecification.Type getType() {
61 | return this.type;
62 | }
63 | }
64 |
--------------------------------------------------------------------------------
/twill-yarn/src/main/java/org/apache/twill/internal/appmaster/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | /**
19 | * This package contains implementation of Twill application master.
20 | */
21 | package org.apache.twill.internal.appmaster;
22 |
--------------------------------------------------------------------------------
/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnAMClientFactory.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.internal.yarn;
19 |
20 | /**
21 | *
22 | */
23 | public interface YarnAMClientFactory {
24 |
25 | YarnAMClient create();
26 | }
27 |
--------------------------------------------------------------------------------
/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnAppClientFactory.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.internal.yarn;
19 |
20 | import org.apache.hadoop.conf.Configuration;
21 |
22 | /**
23 | *
24 | */
25 | public interface YarnAppClientFactory {
26 |
27 | YarnAppClient create(Configuration configuration);
28 | }
29 |
--------------------------------------------------------------------------------
/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnContainerInfo.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.internal.yarn;
19 |
20 | import org.apache.twill.internal.ContainerInfo;
21 |
22 | /**
23 | *
24 | */
25 | public interface YarnContainerInfo extends ContainerInfo {
26 |
27 | T getContainer();
28 | }
29 |
--------------------------------------------------------------------------------
/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnContainerStatus.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.internal.yarn;
19 |
20 | import org.apache.hadoop.yarn.api.records.ContainerState;
21 |
22 | /**
23 | * This interface is for adapting differences in ContainerStatus between Hadoop 2.0 and 2.1.
24 | */
25 | public interface YarnContainerStatus {
26 |
27 | String getContainerId();
28 |
29 | ContainerState getState();
30 |
31 | int getExitStatus();
32 |
33 | String getDiagnostics();
34 | }
35 |
--------------------------------------------------------------------------------
/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnLaunchContext.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.internal.yarn;
19 |
20 | import org.apache.hadoop.security.Credentials;
21 | import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
22 |
23 | import java.nio.ByteBuffer;
24 | import java.util.List;
25 | import java.util.Map;
26 |
27 | /**
28 | * This interface is for adapting ContainerLaunchContext in different Hadoop version.
29 | */
30 | public interface YarnLaunchContext {
31 |
32 | T getLaunchContext();
33 |
34 | void setCredentials(Credentials credentials);
35 |
36 | void setLocalResources(Map localResources);
37 |
38 | void setServiceData(Map serviceData);
39 |
40 | Map getEnvironment();
41 |
42 | void setEnvironment(Map environment);
43 |
44 | List getCommands();
45 |
46 | void setCommands(List commands);
47 |
48 | void setApplicationACLs(Map acls);
49 | }
50 |
--------------------------------------------------------------------------------
/twill-yarn/src/main/java/org/apache/twill/internal/yarn/YarnNMClient.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.internal.yarn;
19 |
20 | import org.apache.twill.common.Cancellable;
21 |
22 | /**
23 | * Abstraction for dealing with API differences in different hadoop yarn version.
24 | */
25 | public interface YarnNMClient {
26 |
27 | /**
28 | * Starts a process based on the given launch context.
29 | *
30 | * @param containerInfo The containerInfo that the new process will launch in.
31 | * @param launchContext Contains information about the process going to start.
32 | * @return A {@link Cancellable} that when {@link Cancellable#cancel()}} is invoked,
33 | * it will try to shutdown the process.
34 | *
35 | */
36 | Cancellable start(YarnContainerInfo containerInfo, YarnLaunchContext launchContext);
37 | }
38 |
--------------------------------------------------------------------------------
/twill-yarn/src/main/java/org/apache/twill/internal/yarn/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | /**
19 | * This package contains class for interacting with Yarn.
20 | */
21 | package org.apache.twill.internal.yarn;
22 |
--------------------------------------------------------------------------------
/twill-yarn/src/main/java/org/apache/twill/yarn/LocationSecureStoreRenewer.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.yarn;
19 |
20 | import org.apache.hadoop.conf.Configuration;
21 | import org.apache.hadoop.security.Credentials;
22 | import org.apache.twill.api.RunId;
23 | import org.apache.twill.api.security.SecureStoreRenewer;
24 | import org.apache.twill.api.security.SecureStoreWriter;
25 | import org.apache.twill.filesystem.LocationFactory;
26 | import org.apache.twill.internal.yarn.YarnUtils;
27 |
28 | import java.io.IOException;
29 |
30 | /**
31 | * Package private class for updating location related secure store.
32 | */
33 | final class LocationSecureStoreRenewer extends SecureStoreRenewer {
34 |
35 | private final Configuration configuration;
36 | private final LocationFactory locationFactory;
37 |
38 | LocationSecureStoreRenewer(Configuration configuration, LocationFactory locationFactory) {
39 | this.configuration = configuration;
40 | this.locationFactory = locationFactory;
41 | }
42 |
43 | @Override
44 | public void renew(String application, RunId runId, SecureStoreWriter secureStoreWriter) throws IOException {
45 | Credentials credentials = new Credentials();
46 | YarnUtils.addDelegationTokens(configuration, locationFactory, credentials);
47 | secureStoreWriter.write(YarnSecureStore.create(credentials));
48 | }
49 | }
50 |
--------------------------------------------------------------------------------
/twill-yarn/src/main/java/org/apache/twill/yarn/YarnSecureStore.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.yarn;
19 |
20 | import org.apache.hadoop.security.Credentials;
21 | import org.apache.twill.api.SecureStore;
22 |
23 | /**
24 | * A {@link SecureStore} for hadoop credentials.
25 | */
26 | public final class YarnSecureStore implements SecureStore {
27 |
28 | private final Credentials credentials;
29 |
30 | public static SecureStore create(Credentials credentials) {
31 | return new YarnSecureStore(credentials);
32 | }
33 |
34 | private YarnSecureStore(Credentials credentials) {
35 | this.credentials = credentials;
36 | }
37 |
38 | @Override
39 | public Credentials getStore() {
40 | return credentials;
41 | }
42 | }
43 |
--------------------------------------------------------------------------------
/twill-yarn/src/main/java/org/apache/twill/yarn/YarnTwillControllerFactory.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.yarn;
19 |
20 | import org.apache.twill.api.RunId;
21 | import org.apache.twill.api.logging.LogHandler;
22 | import org.apache.twill.internal.ProcessController;
23 | import org.apache.twill.internal.yarn.YarnApplicationReport;
24 |
25 | import java.util.concurrent.Callable;
26 | import java.util.concurrent.TimeUnit;
27 |
28 | /**
29 | * Factory for creating {@link YarnTwillController}.
30 | */
31 | interface YarnTwillControllerFactory {
32 |
33 | YarnTwillController create(RunId runId, boolean logCollectionEnabled, Iterable logHandlers,
34 | Callable> startUp,
35 | long startTimeout, TimeUnit startTimeoutUnit);
36 | }
37 |
--------------------------------------------------------------------------------
/twill-yarn/src/main/java/org/apache/twill/yarn/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | /**
19 | * Classes in this package implement the Twill API for Apache Hadoop YARN.
20 | */
21 | package org.apache.twill.yarn;
22 |
--------------------------------------------------------------------------------
/twill-yarn/src/main/resources/logback-template.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 |
24 |
25 |
26 |
27 |
28 |
--------------------------------------------------------------------------------
/twill-yarn/src/test/java/org/apache/twill/filesystem/LocalLocationTest.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.filesystem;
19 |
20 | import org.apache.hadoop.security.UserGroupInformation;
21 |
22 | import java.io.File;
23 | import java.io.IOException;
24 |
25 | /**
26 | *
27 | */
28 | public class LocalLocationTest extends LocationTestBase {
29 |
30 | @Override
31 | protected LocationFactory createLocationFactory(String pathBase) throws Exception {
32 | File basePath = new File(tmpFolder.newFolder(), pathBase);
33 | //noinspection ResultOfMethodCallIgnored
34 | basePath.mkdirs();
35 | return new LocalLocationFactory(basePath);
36 | }
37 |
38 | @Override
39 | protected UserGroupInformation createTestUGI() throws IOException {
40 | // In local location, UGI is not supported, hence using the current user as the testing ugi.
41 | return UserGroupInformation.getCurrentUser();
42 | }
43 | }
44 |
--------------------------------------------------------------------------------
/twill-yarn/src/test/java/org/apache/twill/yarn/BuggyServer.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.yarn;
19 |
20 | import org.slf4j.Logger;
21 | import org.slf4j.LoggerFactory;
22 |
23 | import java.io.BufferedReader;
24 | import java.io.IOException;
25 | import java.io.PrintWriter;
26 |
27 | /**
28 | * Server for testing that will die if you give it a 0.
29 | */
30 | public final class BuggyServer extends SocketServer {
31 |
32 | private static final Logger LOG = LoggerFactory.getLogger(BuggyServer.class);
33 |
34 | @Override
35 | public void handleRequest(BufferedReader reader, PrintWriter writer) throws IOException {
36 | String line = reader.readLine();
37 | LOG.info("Received: " + line + " going to divide by it");
38 | Integer toDivide = Integer.valueOf(line);
39 | writer.println(Integer.toString(100 / toDivide));
40 | }
41 | }
42 |
--------------------------------------------------------------------------------
/twill-yarn/src/test/java/org/apache/twill/yarn/CustomClassLoaderRunnable.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.yarn;
19 |
20 | import com.google.common.util.concurrent.Uninterruptibles;
21 | import org.apache.twill.api.AbstractTwillRunnable;
22 | import org.apache.twill.api.ServiceAnnouncer;
23 | import org.slf4j.Logger;
24 | import org.slf4j.LoggerFactory;
25 |
26 | import java.util.concurrent.CountDownLatch;
27 |
28 | /**
29 | * Runnable for testing custom classloader
30 | */
31 | public final class CustomClassLoaderRunnable extends AbstractTwillRunnable {
32 |
33 | static final String GENERATED_CLASS_NAME = "org.apache.twill.test.Generated";
34 |
35 | private static final Logger LOG = LoggerFactory.getLogger(CustomClassLoaderRunnable.class);
36 |
37 | private final CountDownLatch stopLatch = new CountDownLatch(1);
38 |
39 | @Override
40 | public void run() {
41 | try {
42 | Class cls = Class.forName(GENERATED_CLASS_NAME);
43 | java.lang.reflect.Method announce = cls.getMethod("announce", ServiceAnnouncer.class, String.class, int.class);
44 | announce.invoke(cls.newInstance(), getContext(), System.getProperty("service.name"),
45 | Integer.getInteger("service.port"));
46 | Uninterruptibles.awaitUninterruptibly(stopLatch);
47 | } catch (Exception e) {
48 | LOG.error("Failed to call announce on " + GENERATED_CLASS_NAME, e);
49 | }
50 | }
51 |
52 | @Override
53 | public void stop() {
54 | stopLatch.countDown();
55 | }
56 | }
57 |
--------------------------------------------------------------------------------
/twill-yarn/src/test/java/org/apache/twill/yarn/CustomClassLoaderTestRun.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.yarn;
19 |
20 | import org.apache.twill.api.TwillController;
21 | import org.apache.twill.api.logging.PrinterLogHandler;
22 | import org.junit.Assert;
23 | import org.junit.Test;
24 |
25 | import java.io.PrintWriter;
26 |
27 | /**
28 | * Unit test for testing custom classloader for containers.
29 | */
30 | public class CustomClassLoaderTestRun extends BaseYarnTest {
31 |
32 | @Test
33 | public void testCustomClassLoader() throws Exception {
34 | TwillController controller = getTwillRunner().prepare(new CustomClassLoaderRunnable())
35 | .setClassLoader(CustomClassLoader.class.getName())
36 | .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out, true)))
37 | .setJVMOptions("-Dservice.port=54321")
38 | .setJVMOptions(CustomClassLoaderRunnable.class.getSimpleName(), "-Dservice.name=custom")
39 | .start();
40 |
41 | Assert.assertTrue(waitForSize(controller.discoverService("custom"), 1, 120));
42 | controller.terminate().get();
43 | }
44 | }
45 |
--------------------------------------------------------------------------------
/twill-yarn/src/test/java/org/apache/twill/yarn/DistributeShellTestRun.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.yarn;
19 |
20 | import org.apache.twill.api.TwillController;
21 | import org.apache.twill.api.TwillRunner;
22 | import org.apache.twill.api.logging.PrinterLogHandler;
23 | import org.apache.twill.common.Threads;
24 | import org.junit.Assert;
25 | import org.junit.Ignore;
26 | import org.junit.Test;
27 |
28 | import java.io.PrintWriter;
29 | import java.util.concurrent.CountDownLatch;
30 | import java.util.concurrent.TimeUnit;
31 |
32 | /**
33 | * This test is executed by {@link YarnTestSuite}.
34 | */
35 | public final class DistributeShellTestRun extends BaseYarnTest {
36 |
37 | @Ignore
38 | @Test
39 | public void testDistributedShell() throws InterruptedException {
40 | TwillRunner twillRunner = getTwillRunner();
41 |
42 | TwillController controller = twillRunner.prepare(new DistributedShell("pwd", "ls -al"))
43 | .addLogHandler(new PrinterLogHandler(new PrintWriter(System.out)))
44 | .start();
45 |
46 | final CountDownLatch stopLatch = new CountDownLatch(1);
47 | controller.onTerminated(new Runnable() {
48 | @Override
49 | public void run() {
50 | stopLatch.countDown();
51 | }
52 | }, Threads.SAME_THREAD_EXECUTOR);
53 |
54 | Assert.assertTrue(stopLatch.await(10, TimeUnit.SECONDS));
55 | }
56 | }
57 |
--------------------------------------------------------------------------------
/twill-yarn/src/test/java/org/apache/twill/yarn/EchoServer.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.yarn;
19 |
20 | import org.apache.twill.api.Command;
21 | import org.slf4j.Logger;
22 | import org.slf4j.LoggerFactory;
23 |
24 | import java.io.BufferedReader;
25 | import java.io.IOException;
26 | import java.io.PrintWriter;
27 |
28 | /**
29 | * Test server that echoes back what it receives.
30 | */
31 | public final class EchoServer extends SocketServer {
32 |
33 | private static final Logger LOG = LoggerFactory.getLogger(EchoServer.class);
34 |
35 | @Override
36 | public void handleRequest(BufferedReader reader, PrintWriter writer) throws IOException {
37 | String line = reader.readLine();
38 | LOG.info("Received: " + line);
39 | if (line != null) {
40 | writer.println(line);
41 | }
42 | }
43 |
44 | @Override
45 | public void handleCommand(Command command) throws Exception {
46 | LOG.info("Command received: " + command + " " + getContext().getInstanceCount());
47 | }
48 | }
49 |
--------------------------------------------------------------------------------
/twill-yarn/src/test/java/org/apache/twill/yarn/EnvironmentEchoServer.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.yarn;
19 |
20 | import java.io.BufferedReader;
21 | import java.io.IOException;
22 | import java.io.PrintWriter;
23 |
24 | /**
25 | * Test server that returns back the value of the env key sent in. Used to check env for
26 | * runnables is correctly set.
27 | */
28 | public class EnvironmentEchoServer extends SocketServer {
29 |
30 | @Override
31 | public void handleRequest(BufferedReader reader, PrintWriter writer) throws IOException {
32 | String envKey = reader.readLine();
33 | writer.println(System.getenv(envKey));
34 | }
35 | }
36 |
--------------------------------------------------------------------------------
/twill-yarn/src/test/java/org/apache/twill/yarn/YarnTestSuite.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.yarn;
19 |
20 | import org.junit.runner.RunWith;
21 | import org.junit.runners.Suite;
22 |
23 | /**
24 | * Test suite for all tests with mini yarn cluster.
25 | */
26 | @RunWith(Suite.class)
27 | @Suite.SuiteClasses({
28 | ContainerSizeTestRun.class,
29 | CustomClassLoaderTestRun.class,
30 | DebugTestRun.class,
31 | DistributeShellTestRun.class,
32 | EchoServerTestRun.class,
33 | EnvironmentTestRun.class,
34 | EventHandlerTestRun.class,
35 | FailureRestartTestRun.class,
36 | InitializeFailTestRun.class,
37 | JvmOptionsTestRun.class,
38 | LocalFileTestRun.class,
39 | LogHandlerTestRun.class,
40 | LogLevelChangeTestRun.class,
41 | LogLevelTestRun.class,
42 | PlacementPolicyTestRun.class,
43 | ProvisionTimeoutTestRun.class,
44 | ResourceReportTestRun.class,
45 | ServiceDiscoveryTestRun.class,
46 | SessionExpireTestRun.class,
47 | TaskCompletedTestRun.class,
48 | RestartRunnableTestRun.class,
49 | MaxRetriesTestRun.class
50 | })
51 | public final class YarnTestSuite extends BaseYarnTest {
52 | }
53 |
--------------------------------------------------------------------------------
/twill-yarn/src/test/resources/logback-test.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 | %d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
35 |
--------------------------------------------------------------------------------
/twill-zookeeper/src/main/java/org/apache/twill/internal/zookeeper/BasicACLData.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.internal.zookeeper;
19 |
20 | import org.apache.twill.zookeeper.ACLData;
21 | import org.apache.zookeeper.data.ACL;
22 | import org.apache.zookeeper.data.Stat;
23 |
24 | import java.util.List;
25 |
26 | /**
27 | * A straightforward implementation of {@link ACLData}.
28 | */
29 | final class BasicACLData implements ACLData {
30 |
31 | private final List acl;
32 | private final Stat stat;
33 |
34 | BasicACLData(List acl, Stat stat) {
35 | this.acl = acl;
36 | this.stat = stat;
37 | }
38 |
39 | @Override
40 | public List getACL() {
41 | return acl;
42 | }
43 |
44 | @Override
45 | public Stat getStat() {
46 | return stat;
47 | }
48 | }
49 |
--------------------------------------------------------------------------------
/twill-zookeeper/src/main/java/org/apache/twill/internal/zookeeper/BasicNodeChildren.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.internal.zookeeper;
19 |
20 | import com.google.common.base.Objects;
21 | import org.apache.twill.zookeeper.NodeChildren;
22 | import org.apache.zookeeper.data.Stat;
23 |
24 | import java.util.List;
25 |
26 | /**
27 | *
28 | */
29 | final class BasicNodeChildren implements NodeChildren {
30 |
31 | private final Stat stat;
32 | private final List children;
33 |
34 | BasicNodeChildren(List children, Stat stat) {
35 | this.stat = stat;
36 | this.children = children;
37 | }
38 |
39 | @Override
40 | public Stat getStat() {
41 | return stat;
42 | }
43 |
44 | @Override
45 | public List getChildren() {
46 | return children;
47 | }
48 |
49 | @Override
50 | public boolean equals(Object o) {
51 | if (this == o) {
52 | return true;
53 | }
54 | if (o == null || !(o instanceof NodeChildren)) {
55 | return false;
56 | }
57 |
58 | NodeChildren that = (NodeChildren) o;
59 | return stat.equals(that.getStat()) && children.equals(that.getChildren());
60 | }
61 |
62 | @Override
63 | public int hashCode() {
64 | return Objects.hashCode(children, stat);
65 | }
66 | }
67 |
--------------------------------------------------------------------------------
/twill-zookeeper/src/main/java/org/apache/twill/internal/zookeeper/BasicNodeData.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.internal.zookeeper;
19 |
20 | import com.google.common.base.Objects;
21 | import org.apache.twill.zookeeper.NodeData;
22 | import org.apache.zookeeper.data.Stat;
23 |
24 | import java.util.Arrays;
25 |
26 | /**
27 | * A straightforward implementation for {@link NodeData}.
28 | */
29 | final class BasicNodeData implements NodeData {
30 |
31 | private final byte[] data;
32 | private final Stat stat;
33 |
34 | BasicNodeData(byte[] data, Stat stat) {
35 | this.data = data;
36 | this.stat = stat;
37 | }
38 |
39 | @Override
40 | public Stat getStat() {
41 | return stat;
42 | }
43 |
44 | @Override
45 | public byte[] getData() {
46 | return data;
47 | }
48 |
49 | @Override
50 | public boolean equals(Object o) {
51 | if (this == o) {
52 | return true;
53 | }
54 | if (o == null || !(o instanceof NodeData)) {
55 | return false;
56 | }
57 |
58 | BasicNodeData that = (BasicNodeData) o;
59 |
60 | return stat.equals(that.getStat()) && Arrays.equals(data, that.getData());
61 | }
62 |
63 | @Override
64 | public int hashCode() {
65 | return Objects.hashCode(data, stat);
66 | }
67 | }
68 |
--------------------------------------------------------------------------------
/twill-zookeeper/src/main/java/org/apache/twill/internal/zookeeper/RetryUtils.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.internal.zookeeper;
19 |
20 | import org.apache.zookeeper.KeeperException;
21 |
22 | /**
23 | * Utility class for help determining operation retry condition.
24 | */
25 | final class RetryUtils {
26 |
27 | /**
28 | * Tells if a given operation error code can be retried or not.
29 | * @param code The error code of the operation.
30 | * @return {@code true} if the operation can be retried.
31 | */
32 | public static boolean canRetry(KeeperException.Code code) {
33 | return (code == KeeperException.Code.CONNECTIONLOSS
34 | || code == KeeperException.Code.OPERATIONTIMEOUT
35 | || code == KeeperException.Code.SESSIONEXPIRED
36 | || code == KeeperException.Code.SESSIONMOVED);
37 | }
38 |
39 | /**
40 | * Tells if a given operation exception can be retried or not.
41 | * @param t The exception raised by an operation.
42 | * @return {@code true} if the operation can be retried.
43 | */
44 | public static boolean canRetry(Throwable t) {
45 | return t instanceof KeeperException && canRetry(((KeeperException) t).code());
46 | }
47 |
48 | private RetryUtils() {
49 | }
50 | }
51 |
--------------------------------------------------------------------------------
/twill-zookeeper/src/main/java/org/apache/twill/internal/zookeeper/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | /**
20 | * Internal classes for zookeeper.
21 | */
22 | package org.apache.twill.internal.zookeeper;
23 |
--------------------------------------------------------------------------------
/twill-zookeeper/src/main/java/org/apache/twill/zookeeper/ACLData.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.zookeeper;
19 |
20 | import org.apache.zookeeper.data.ACL;
21 | import org.apache.zookeeper.data.Stat;
22 |
23 | import java.util.List;
24 |
25 | /**
26 | * Represents result of call to {@link ZKClient#getACL(String)}.
27 | */
28 | public interface ACLData {
29 |
30 | /**
31 | * @return list of {@link ACL} for the node.
32 | */
33 | List getACL();
34 |
35 | /**
36 | * @return The {@link Stat} of the node.
37 | */
38 | Stat getStat();
39 | }
40 |
--------------------------------------------------------------------------------
/twill-zookeeper/src/main/java/org/apache/twill/zookeeper/NodeChildren.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.zookeeper;
19 |
20 | import org.apache.zookeeper.data.Stat;
21 |
22 | import java.util.List;
23 |
24 | /**
25 | * Represents result of call to {@link ZKClientService#getChildren(String, org.apache.zookeeper.Watcher)} method.
26 | */
27 | public interface NodeChildren {
28 |
29 | /**
30 | * @return The {@link Stat} of the node.
31 | */
32 | Stat getStat();
33 |
34 | /**
35 | * @return List of children node names.
36 | */
37 | List getChildren();
38 | }
39 |
--------------------------------------------------------------------------------
/twill-zookeeper/src/main/java/org/apache/twill/zookeeper/NodeData.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.zookeeper;
19 |
20 | import org.apache.zookeeper.data.Stat;
21 |
22 | import javax.annotation.Nullable;
23 |
24 | /**
25 | * Represents result of call to {@link ZKClient#getData(String, org.apache.zookeeper.Watcher)}.
26 | */
27 | public interface NodeData {
28 |
29 | /**
30 | * @return The {@link Stat} of the node.
31 | */
32 | Stat getStat();
33 |
34 | /**
35 | * @return Data stored in the node, or {@code null} if there is no data.
36 | */
37 | @Nullable
38 | byte[] getData();
39 | }
40 |
--------------------------------------------------------------------------------
/twill-zookeeper/src/main/java/org/apache/twill/zookeeper/OperationFuture.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.zookeeper;
19 |
20 | import com.google.common.util.concurrent.ListenableFuture;
21 |
22 | /**
23 | * A {@link ListenableFuture} that also provides the requested path for a operation.
24 | *
25 | * @param The result type returned by this Future's {@link #get()} method.
26 | */
27 | public interface OperationFuture extends ListenableFuture {
28 |
29 | /**
30 | * @return The path being requested for the ZooKeeper operation.
31 | */
32 | String getRequestPath();
33 | }
34 |
--------------------------------------------------------------------------------
/twill-zookeeper/src/main/java/org/apache/twill/zookeeper/RetryStrategy.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 | package org.apache.twill.zookeeper;
19 |
20 | /**
21 | * Provides strategy to use for operation retries.
22 | */
23 | public interface RetryStrategy {
24 |
25 | /**
26 | * Defines ZooKeeper operation type that triggers retry.
27 | */
28 | enum OperationType {
29 | CREATE,
30 | EXISTS,
31 | GET_CHILDREN,
32 | GET_DATA,
33 | SET_DATA,
34 | DELETE,
35 | SET_ACL,
36 | GET_ACL
37 | }
38 |
39 | /**
40 | * Returns the number of milliseconds to wait before retrying the operation.
41 | *
42 | * @param failureCount Number of times that the request has been failed.
43 | * @param startTime Timestamp in milliseconds that the request starts.
44 | * @param type Type of operation tried to perform.
45 | * @param path The path that the operation is acting on.
46 | * @return Number of milliseconds to wait before retrying the operation. Returning {@code 0} means
47 | * retry it immediately, while negative means abort the operation.
48 | */
49 | long nextRetry(int failureCount, long startTime, OperationType type, String path);
50 | }
51 |
--------------------------------------------------------------------------------
/twill-zookeeper/src/main/java/org/apache/twill/zookeeper/package-info.java:
--------------------------------------------------------------------------------
1 | /*
2 | * Licensed to the Apache Software Foundation (ASF) under one
3 | * or more contributor license agreements. See the NOTICE file
4 | * distributed with this work for additional information
5 | * regarding copyright ownership. The ASF licenses this file
6 | * to you under the Apache License, Version 2.0 (the
7 | * "License"); you may not use this file except in compliance
8 | * with the License. You may obtain a copy of the License at
9 | *
10 | * http://www.apache.org/licenses/LICENSE-2.0
11 | *
12 | * Unless required by applicable law or agreed to in writing, software
13 | * distributed under the License is distributed on an "AS IS" BASIS,
14 | * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 | * See the License for the specific language governing permissions and
16 | * limitations under the License.
17 | */
18 |
19 | /**
20 | * This package provides functionality for ZooKeeper interactions.
21 | */
22 | package org.apache.twill.zookeeper;
23 |
--------------------------------------------------------------------------------
/twill-zookeeper/src/test/resources/logback-test.xml:
--------------------------------------------------------------------------------
1 |
2 |
18 |
19 |
20 |
21 |
22 |
23 | %d{ISO8601} - %-5p [%t:%C{1}@%L] - %m%n
24 |
25 |
26 |
27 |
28 |
29 |
30 |
31 |
32 |
33 |
34 |
--------------------------------------------------------------------------------