├── .gitignore ├── .travis.yml ├── AUTHORS.txt ├── CHANGELOG.md ├── CONTRIBUTING.md ├── LICENSE ├── README-brief-cn.md ├── README.md ├── pom.xml ├── src ├── main │ ├── java │ │ └── io │ │ │ └── parallec │ │ │ └── core │ │ │ ├── FilterRegex.java │ │ │ ├── HostsSourceType.java │ │ │ ├── ParallecHeader.java │ │ │ ├── ParallecResponseHandler.java │ │ │ ├── ParallelClient.java │ │ │ ├── ParallelTask.java │ │ │ ├── ParallelTaskBuilder.java │ │ │ ├── RequestProtocol.java │ │ │ ├── ResponseOnSingleTask.java │ │ │ ├── actor │ │ │ ├── ActorConfig.java │ │ │ ├── AssistantExecutionManager.java │ │ │ ├── ExecutionManager.java │ │ │ ├── HttpWorker.java │ │ │ ├── OperationWorker.java │ │ │ ├── PingWorker.java │ │ │ ├── SshWorker.java │ │ │ ├── TcpWorker.java │ │ │ ├── UdpWorker.java │ │ │ ├── message │ │ │ │ ├── CancelTaskOnHostRequest.java │ │ │ │ ├── ContinueToSendToBatchSenderAsstManager.java │ │ │ │ ├── InitialRequestToManager.java │ │ │ │ ├── NodeReqResponse.java │ │ │ │ ├── RequestToBatchSenderAsstManager.java │ │ │ │ ├── ResponseCountToBatchSenderAsstManager.java │ │ │ │ ├── ResponseFromManager.java │ │ │ │ ├── ResponseOnSingeRequest.java │ │ │ │ ├── package-info.java │ │ │ │ └── type │ │ │ │ │ ├── ExecutionManagerMsgType.java │ │ │ │ │ ├── OperationWorkerMsgType.java │ │ │ │ │ ├── RequestWorkerMsgType.java │ │ │ │ │ └── package-info.java │ │ │ ├── package-info.java │ │ │ └── poll │ │ │ │ ├── HttpPollerProcessor.java │ │ │ │ ├── PollerData.java │ │ │ │ └── package-info.java │ │ │ ├── app │ │ │ ├── ParallecAppMin.java │ │ │ └── package-info.java │ │ │ ├── bean │ │ │ ├── HttpMeta.java │ │ │ ├── ResponseHeaderMeta.java │ │ │ ├── SetAndCount.java │ │ │ ├── SingleTargetTaskStatus.java │ │ │ ├── StrStrMap.java │ │ │ ├── TargetHostMeta.java │ │ │ ├── TaskRequest.java │ │ │ ├── TaskRunMode.java │ │ │ ├── package-info.java │ │ │ ├── ping │ │ │ │ ├── PingMeta.java │ │ │ │ ├── PingMode.java │ │ │ │ └── package-info.java │ │ │ ├── ssh │ │ │ │ ├── SshLoginType.java │ │ │ │ ├── SshMeta.java │ │ │ │ └── package-info.java │ │ │ ├── tcp │ │ │ │ ├── TcpMeta.java │ │ │ │ └── package-info.java │ │ │ └── udp │ │ │ │ ├── UdpMeta.java │ │ │ │ └── package-info.java │ │ │ ├── commander │ │ │ └── workflow │ │ │ │ ├── InternalDataProvider.java │ │ │ │ ├── VarReplacementProvider.java │ │ │ │ ├── package-info.java │ │ │ │ ├── ping │ │ │ │ ├── PingProvider.java │ │ │ │ └── package-info.java │ │ │ │ └── ssh │ │ │ │ ├── SshProvider.java │ │ │ │ └── package-info.java │ │ │ ├── config │ │ │ ├── HandlerExecutionLocation.java │ │ │ ├── ParallecGlobalConfig.java │ │ │ ├── ParallelTaskConfig.java │ │ │ ├── ParallelTaskConfigDefault.java │ │ │ └── package-info.java │ │ │ ├── exception │ │ │ ├── ActorMessageTypeInvalidException.java │ │ │ ├── ExecutionManagerExecutionException.java │ │ │ ├── HttpRequestCreateException.java │ │ │ ├── ParallelTaskInvalidException.java │ │ │ ├── TargetHostsLoadException.java │ │ │ ├── TcpUdpRequestCreateException.java │ │ │ └── package-info.java │ │ │ ├── monitor │ │ │ ├── MonitorProvider.java │ │ │ └── package-info.java │ │ │ ├── package-info.java │ │ │ ├── resources │ │ │ ├── AsyncHttpClientFactoryEmbed.java │ │ │ ├── HttpClientStore.java │ │ │ ├── HttpClientType.java │ │ │ ├── HttpMethod.java │ │ │ ├── TcpUdpSshPingResourceStore.java │ │ │ └── package-info.java │ │ │ ├── task │ │ │ ├── CapacityAwareTaskScheduler.java │ │ │ ├── ParallelTaskBean.java │ │ │ ├── ParallelTaskManager.java │ │ │ ├── ParallelTaskState.java │ │ │ ├── RequestReplacementType.java │ │ │ └── TaskErrorMeta.java │ │ │ ├── taskbuilder │ │ │ └── targethosts │ │ │ │ ├── ITargetHostsBuilder.java │ │ │ │ ├── TargetHostsBuilder.java │ │ │ │ ├── TargetHostsBuilderHelperCms.java │ │ │ │ └── package-info.java │ │ │ └── util │ │ │ ├── BeanMapper.java │ │ │ ├── DaemonThreadFactory.java │ │ │ ├── PcConstants.java │ │ │ ├── PcDateUtils.java │ │ │ ├── PcErrorMsgUtils.java │ │ │ ├── PcFileNetworkIoUtils.java │ │ │ ├── PcHttpUtils.java │ │ │ ├── PcNumberUtils.java │ │ │ ├── PcStringUtils.java │ │ │ ├── PcTargetHostsUtils.java │ │ │ └── package-info.java │ └── resources │ │ ├── actorconfig.conf │ │ └── logback.xml └── test │ └── java │ └── io │ └── parallec │ └── core │ ├── ParallelClientTest.java │ ├── TestBase.java │ ├── actor │ ├── ExecutionManagerTest.java │ ├── HttpWorkerTest.java │ └── OperationWorkerTest.java │ ├── bean │ └── NodeReqResponseTest.java │ ├── client │ └── ParallelTaskTest.java │ ├── commander │ └── workflow │ │ └── VarReplacementProviderTest.java │ ├── filter │ └── FilterRegexTest.java │ ├── httpclient │ └── async │ │ └── HttpClientStoreTest.java │ ├── main │ ├── http │ │ ├── LoadResourceTest.java │ │ ├── ParallelClientBeanTest.java │ │ ├── ParallelClientHttpBasicMoreOptionsTest.java │ │ ├── ParallelClientHttpBasicTest.java │ │ ├── ParallelClientHttpBasicTimeoutTest.java │ │ ├── ParallelClientHttpFromCmsAsyncTest.java │ │ ├── ParallelClientHttpInvalidInputTest.java │ │ ├── ParallelClientHttpLogTest.java │ │ ├── ParallelClientHttpOtherMethodsTest.java │ │ ├── ParallelClientHttpResponseHeaderTest.java │ │ ├── ParallelClientHttpSmallConcurrencyTest.java │ │ ├── ParallelClientHttpTop100WebsiteTest.java │ │ ├── ParallelClientHttpTop100WebsiteTimeoutTest.java │ │ ├── ParallelClientHttpTop500WebsiteTest.java │ │ ├── ParallelTaskCancelOnTargetHostsTest.java │ │ ├── ParallelTaskCancelWholeTaskTest.java │ │ ├── pollable │ │ │ ├── ParallelClientHttpPollerJobTest.java │ │ │ └── sampleserver │ │ │ │ ├── HttpServerThread.java │ │ │ │ └── ServerWithPollableJobs.java │ │ ├── request │ │ │ └── template │ │ │ │ ├── ParallelClientVarReplacementHostSpecificHeaderTest.java │ │ │ │ ├── ParallelClientVarReplacementHostSpecificTest.java │ │ │ │ └── ParallelClientVarReplacementUniformTest.java │ │ └── scheduler │ │ │ └── ParallelClientSchedulerAndCancelTest.java │ ├── ping │ │ ├── PingOnTmpTest.java │ │ ├── PingTest.java │ │ ├── PingWorkerTest.java │ │ └── package-info.java │ ├── ssh │ │ ├── ParallelClientSshTest.java │ │ ├── SshMetaTest.java │ │ ├── SshProviderMockTest.java │ │ ├── SshProviderRealTest.java │ │ └── SshWorkerTest.java │ ├── tcp │ │ ├── ParallelClientTcpBasicTest.java │ │ ├── TcpClientTest.java │ │ ├── TcpProviderPoc.java │ │ ├── TcpWorkerIdleTest.java │ │ ├── TcpWorkerTest.java │ │ ├── package-info.java │ │ └── sampleserver │ │ │ ├── TcpEchoServer.java │ │ │ ├── TcpServerThread.java │ │ │ └── package-info.java │ └── udp │ │ ├── ParallelClientUdpBasicTest.java │ │ ├── ParallelClientUdpSlowServerTest.java │ │ ├── UdpWorkerTest.java │ │ ├── package-info.java │ │ └── sampleserver │ │ ├── UdpEchoServer.java │ │ ├── UdpServerThread.java │ │ └── package-info.java │ ├── monitor │ └── MonitorProviderTest.java │ ├── pojo │ ├── ParallecPojoClassTest.java │ └── ParallecPojoStrTest.java │ ├── task │ ├── ParallelTaskManagerTest.java │ └── package-info.java │ ├── taskbuilder │ ├── ParallelTaskBuilderTest.java │ └── targethosts │ │ └── TargetHostsBuilderTest.java │ └── util │ ├── FileNetworkIoUtilsTest.java │ ├── PcDateUtilsTest.java │ └── PcErrorMsgUtilsTest.java └── userdata ├── keepthisfile ├── sample_target_hosts_json_path.json ├── sample_target_hosts_top100.txt ├── sample_target_hosts_top1000.txt ├── sample_target_hosts_top100_old.txt ├── sample_target_hosts_top2000.txt ├── sample_target_hosts_top500.txt └── sample_target_hosts_top500_old.txt /.gitignore: -------------------------------------------------------------------------------- 1 | *.class 2 | 3 | # Mobile Tools for Java (J2ME) 4 | .mtj.tmp/ 5 | 6 | # Package Files # 7 | *.jar 8 | *.war 9 | *.ear 10 | 11 | application.log* 12 | 13 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 14 | hs_err_pid* 15 | /target/ 16 | 17 | .DS_Store 18 | 19 | #elastic data 20 | data/ 21 | 22 | #parallec log 23 | parallec_logs/* 24 | !parallec_logs/keepthisfile 25 | 26 | #user data 27 | userdata/* 28 | !userdata/keepthisfile 29 | 30 | !userdata/tasklogs 31 | userdata/tasklogs/* 32 | !userdata/tasklogs/keepthisfile 33 | !userdata/sample_target* 34 | *.jks 35 | *.pem 36 | 37 | #mkdown 38 | .*.md.html 39 | 40 | #IDE 41 | /.idea/ 42 | 43 | /.settings/ 44 | .classpath 45 | .project 46 | /bin/ 47 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: java 2 | sudo: true 3 | jdk: 4 | - openjdk7 5 | 6 | before_install: 7 | - sudo apt-get install iputils-ping 8 | - pip install --user codecov 9 | 10 | after_success: 11 | - codecov 12 | -------------------------------------------------------------------------------- /AUTHORS.txt: -------------------------------------------------------------------------------- 1 | Parallec: Parallel Async HTTP/SSH/TCP/PING Client Library 2 | 3 | URL: https://github.com/eBay/parallec 4 | 5 | Core developers (in order of appearance): 6 | 7 | - Yuanteng (Jeff) Pei (original author) 8 | - Teng Song 9 | 10 | Contributors: 11 | 12 | - Lukasz Kryger http://stackoverflow.com/users/1240557/kryger 13 | - billzwu https://github.com/billzwu 14 | - faisal-hameed https://github.com/faisal-hameed 15 | - fivesmallq https://github.com/fivesmallq 16 | - bryant1410 https://github.com/bryant1410 17 | - djKooks https://github.com/djKooks -------------------------------------------------------------------------------- /CONTRIBUTING.md: -------------------------------------------------------------------------------- 1 | ## Contributing to Parallec 2 | 3 | Contributions via GitHub pull requests are **gladly accepted** from their original 4 | author. Along with any pull requests, please state that the **contribution is 5 | your original work** and that you license the work to the project under the 6 | project's open source license. Whether or not you state this explicitly, by 7 | submitting any copyrighted material via pull request, email, or other means 8 | you **agree to license the material under the project's open source license** and 9 | warrant that you have the legal authority to do so. 10 | -------------------------------------------------------------------------------- /README-brief-cn.md: -------------------------------------------------------------------------------- 1 | 2 | Parallec-logo 3 | 4 | ## Parallec 中文介绍 5 | 6 | [Parallec](http://www.parallec.io/) 是一个基于[Akka](http://akka.io)的快速并行异步HTTP(S)/SSH/TCP/UDP/Ping 客户端的Java库。它是eBay云计算部门开发并在[REST Commander](http://www.restcommander.com/)基础上开源的。Parallec的寓意是**"并行客户端"**： **Paralle**l **C**lient, 发音如 "Para-like". 更多说明请见[这里](http://www.parallec.io/)。 7 | 8 | 主要用途： 9 | 10 | - 管理监控大量HTTP/TCP/UDP服务器，ping海量服务器 11 | - 集群机器状态、配置探索，基于HTTP(S)/TCP/UDP Agent 或者 无agent 的大量远程任务执行，软件部署, 网络监控. HTTP支持SSL 客户端认证 12 | - 海量API聚集到任意数据存储，数据流入口 13 | - 并行工作流，自动检测任务进度，以便支持并行处理异步API 14 | - 可控并行的，多个不同API请求到同一个HTTP服务器。HTTP请求模板可以在URL, HTTP Header, EntityBody不同地方进行变量替换。这样可以很方便的对那些有REST API的数据库或者网络服务器进行并行的CRUD操作。 15 | - 灵活的Web服务器的压力测试，并将结果传到任何地方。 16 | 17 | Parallec内置特别的响应上下文（response context），在处理服务器回复（Response）时能方便快捷的传入，传出任何对象，比如各种客户端（比如elastic search, graphite, kafka, mongodb etc），以便汇集处理数据到任何地方。 [样例程序](http://www.parallec.io/#code-sample)**仅需20行代码**，就可以汇总10000个API的回复以简单可控的并行速度发送汇总到Elastic Search。 18 | 19 | 和REST Commander类似，Parallec非常高效并且可以处理大量响应。 比如其Ping服务器的速度是目前使用最广泛**并行Ping 软件[FPing](http://fping.org/)的2倍**，仅用12秒就可以ping 8000服务器。（[**请看Ping视频演示**](https://github.com/eBay/parallec/wiki/Parallec-pings-8000-servers-in-11.1-seconds)) 同样的对于**并行HTTP**， Parallec发送HTTP请求到8000 Web服务器 并聚集响应只需**12秒** （聚集到内存），或**16秒**（聚集到Elastic Search）（[**请看HTTP视频演示**](https://github.com/eBay/parallec/wiki/Parallec-Aggregates-HTTP-Responses-from-8000-Servers)) 20 | 21 | 通过过去一年内部使用和开源REST Superman/ REST Commander的开发， 反馈，经验教训，我们现在做把REST Commander的核心做成一个易于使用的独立库 （便于从Maven导入，让所有程序都写20行代码变成REST Commander）。在此基础上，我们添加超过[**15个新的功能**](https://github.com/eBay/parallec/blob/master/README.md#compare)，**改写超过70%的代码**，**以90%以上的测试覆盖率**，确保了用户和开源贡献者能够更加放心的开发和使用。与此同时，我们也把整体软件结构进行了优化，使得大部分内部开发可以直接在github开源平台上直接进行。 22 | 23 | **生产环境使用案例**：广泛用于基础设施软件作为轮询和聚合引擎: 24 | 25 | * 应用程序部署/ PaaS：Parallec已集成在eBay主要生产应用程序部署系统（PaaS）中。 Parallec协调10+个API任务，每**个任务针对生产中超过1,000个应用程序池的10到1,000个服务器。 Parallec已经与工作流引擎Winder一起使用来处理复杂的部署工作流程。 26 | * 数据提取/ ETL：Parallec已被eBay以色列的结构化大数据团队用于对单个第三方服务器执行10k-100k API并行调用，具有显着改进的性能和减少的资源。 27 | * 主动探测进行网络故障排除：在eBay的网络/云团队中，Parallec有助于确保极低的错误警报率，从而准确检测交换机软故障。 Parallec在Master组件中充当核心轮询引擎，以检查Agent运行状况，及时有效地消除噪音和错误报警。 28 | * Agent管理/Agent Master：在eBay的网站操作/工具团队中，Parallec作为核心引擎，在100,000个生产服务器上管理和监控一种和 puppet agent / salt minion / kubernetes kubelet 类似的agent，以确保快速大规模的并行的操作。 29 | 30 | “Parallec.io现已经被集成在eBay的应用程序部署系统(PaaS)。Parallec被用于控制10个以上的并行API的任务的工作流，每个任务并行执行到数十到数千台的生产服务器的上。与以前的实现相比，Parallec 处理并行操作代码简洁，方便进行批作业的管理，为我们提高了资源利用率和效率。” 31 | －－ Xiaotang (Sheldon) Shao, Principle Software Engineer, Deployment Platform / PaaS, Cloud Engineering at eBay Inc. 32 | 33 | “eBay 网络智能分析组目前的几个项目都需要并行执行万级到十万级的API到第三方的服务器，这毫无疑问是我们最大的瓶颈。因此我们一致在寻找一个高效的异步I/O的解决方案。使用Parallec，不但显著的提升了性能，同时有效的减少了资源的使用。 ” －－ Tomer Balan, Data Extraction Engineer, Structured Data at eBay Inc (以色列). 34 | 35 | 您可以用5分钟按照[样例程序](http://www.parallec.io/#code-sample)试试Parallec去给多个服务器（网站）发送相同或者不同的请求。我们渴望收到您的意见和建议。您可以通过[提交Github Issue](https://github.com/eBay/parallec/issues/new), 查看[FAQ](http://www.parallec.io/docs/faq/)，登陆论坛 [Parallec.io Google Group](https://groups.google.com/forum/#!forum/parallec)进行反馈或者提问。谢谢！ 36 | 37 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/FilterRegex.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core; 14 | 15 | import io.parallec.core.util.PcConstants; 16 | import io.parallec.core.util.PcDateUtils; 17 | 18 | import java.util.regex.Matcher; 19 | import java.util.regex.Pattern; 20 | 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | 24 | /** 25 | * A simple class to apply a regular expression to . 26 | * @author Yuanteng (Jeff) Pei 27 | */ 28 | public class FilterRegex { 29 | 30 | /** The logger. */ 31 | private static Logger logger = LoggerFactory.getLogger(FilterRegex.class); 32 | 33 | /** The regex. */ 34 | private String regex; 35 | 36 | /** 37 | * Instantiates a new filter regex. 38 | */ 39 | public FilterRegex() { 40 | }; 41 | 42 | /** 43 | * Instantiates a new filter regex. 44 | * 45 | * @param regex1 46 | * the regex1 47 | */ 48 | public FilterRegex(String regex1) { 49 | setRegex(regex1); 50 | } 51 | 52 | public String filter(String input) { 53 | return stringMatcherByPattern(input, regex); 54 | } 55 | 56 | /** 57 | * this remove the linebreak. 58 | * 59 | * @param input 60 | * the input 61 | * @param patternStr 62 | * the pattern str 63 | * @return the string 64 | */ 65 | public static String stringMatcherByPattern(String input, String patternStr) { 66 | 67 | String output = PcConstants.SYSTEM_FAIL_MATCH_REGEX; 68 | 69 | // 20140105: fix the NPE issue 70 | if (patternStr == null) { 71 | logger.error("patternStr is NULL! (Expected when the aggregation rule is not defined at " 72 | + PcDateUtils.getNowDateTimeStrStandard()); 73 | return output; 74 | } 75 | 76 | if (input == null) { 77 | logger.error("input (Expected when the response is null and now try to match on response) is NULL in stringMatcherByPattern() at " 78 | + PcDateUtils.getNowDateTimeStrStandard()); 79 | return output; 80 | } else { 81 | input = input.replace("\n", "").replace("\r", ""); 82 | } 83 | 84 | logger.debug("input: " + input); 85 | logger.debug("patternStr: " + patternStr); 86 | 87 | Pattern patternMetric = Pattern.compile(patternStr, Pattern.MULTILINE); 88 | 89 | final Matcher matcher = patternMetric.matcher(input); 90 | if (matcher.matches()) { 91 | output = matcher.group(1); 92 | } 93 | return output; 94 | } 95 | 96 | /** 97 | * Gets the regex. 98 | * 99 | * @return the regex 100 | */ 101 | public String getRegex() { 102 | return regex; 103 | } 104 | 105 | /** 106 | * Sets the regex. 107 | * 108 | * @param regex 109 | * the new regex 110 | */ 111 | public void setRegex(String regex) { 112 | this.regex = regex; 113 | } 114 | 115 | } 116 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/HostsSourceType.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core; 2 | 3 | /** 4 | * The Enum HostsSourceType. 5 | */ 6 | public enum HostsSourceType { 7 | 8 | /** The local file. */ 9 | LOCAL_FILE, 10 | /** The url. */ 11 | URL 12 | } -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/ParallecHeader.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core; 14 | 15 | import io.parallec.core.util.PcStringUtils; 16 | 17 | import java.util.LinkedHashMap; 18 | import java.util.Map; 19 | 20 | 21 | /** 22 | * Easy to use HTTP Header builder to add header key/value pair. 23 | * 24 | * @author Yuanteng (Jeff) Pei 25 | */ 26 | public class ParallecHeader { 27 | 28 | /** The header map. */ 29 | // will never be null 30 | private final Map headerMap = new LinkedHashMap(); 31 | 32 | /** 33 | * Instantiates a new parallec header. 34 | */ 35 | public ParallecHeader() { 36 | 37 | } 38 | 39 | /** 40 | * Adds the pair. 41 | * 42 | * @param key 43 | * the key 44 | * @param value 45 | * the value 46 | * @return the parallec header 47 | */ 48 | public ParallecHeader addPair(String key, String value) { 49 | this.headerMap.put(key, value); 50 | return this; 51 | } 52 | 53 | /** 54 | * Gets the header map. 55 | * 56 | * @return the header map 57 | */ 58 | public Map getHeaderMap() { 59 | return headerMap; 60 | } 61 | 62 | /** 63 | * Gets the header str. 64 | * 65 | * @return the header str 66 | */ 67 | public String getHeaderStr() { 68 | return PcStringUtils.strMapToStr(headerMap); 69 | } 70 | 71 | /* 72 | * (non-Javadoc) 73 | * 74 | * @see java.lang.Object#toString() 75 | */ 76 | @Override 77 | public String toString() { 78 | return "ParallecHeader [headerMap=" 79 | + PcStringUtils.strMapToStr(headerMap) + "]"; 80 | } 81 | 82 | } 83 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/ParallecResponseHandler.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core; 14 | 15 | import java.util.Map; 16 | 17 | 18 | /** 19 | * The Interface ParallecResponseHandler. 20 | * 21 | */ 22 | public interface ParallecResponseHandler { 23 | 24 | /** 25 | * When timeout / error occurred or response received for the target host, 26 | * will trigger execution of onCompleted(). 27 | * 28 | * This can be run at the execution manager (default) or the worker thread 29 | * based on the config {@link ParallelTaskBuilder#handleInWorker()} 30 | * 31 | *
32 | * When {@link ResponseOnSingleTask#isError()} is true: Fail to receive resposne 33 | * 34 | * @param res 35 | * the response 36 | * @param responseContext 37 | * the response context 38 | */ 39 | 40 | public void onCompleted(ResponseOnSingleTask res, 41 | Map responseContext); 42 | 43 | } 44 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/RequestProtocol.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core; 14 | 15 | 16 | /** 17 | * The Enum RequestProtocol. 18 | * 19 | * Currently values include HTTP, HTTPS, SSH, PING, TCP 20 | * 21 | * @author Yuanteng (Jeff) Pei 22 | */ 23 | public enum RequestProtocol { 24 | 25 | /** The HTTP. */ 26 | HTTP, 27 | /** The HTTPS. */ 28 | HTTPS, 29 | /** The SSH. */ 30 | SSH, 31 | /** The PING (ICMP). */ 32 | PING, 33 | /** The TCP. */ 34 | TCP, 35 | /** The UDP. */ 36 | UDP; 37 | /* 38 | * (non-Javadoc) 39 | * 40 | * @see java.lang.Enum#toString() 41 | */ 42 | @Override 43 | public String toString() { 44 | return super.toString().toLowerCase(); 45 | } 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/actor/ActorConfig.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.actor; 14 | 15 | import io.parallec.core.util.PcConstants; 16 | 17 | import java.util.concurrent.TimeUnit; 18 | 19 | import org.slf4j.Logger; 20 | import org.slf4j.LoggerFactory; 21 | 22 | import scala.concurrent.duration.Duration; 23 | import scala.concurrent.duration.FiniteDuration; 24 | import akka.actor.ActorSystem; 25 | 26 | import com.typesafe.config.Config; 27 | import com.typesafe.config.ConfigFactory; 28 | 29 | 30 | /** 31 | * The static akka actor system 32 | */ 33 | public final class ActorConfig { 34 | 35 | /** The conf. */ 36 | private static Config conf = null; 37 | 38 | /** The actor system. */ 39 | private static ActorSystem actorSystem = null; 40 | 41 | /** The logger. */ 42 | private static Logger logger = LoggerFactory.getLogger(ActorConfig.class); 43 | static { 44 | 45 | // load default 46 | conf = ConfigFactory.load("actorconfig.conf"); 47 | logger.debug("Load Actor config {}", conf.toString()); 48 | actorSystem = ActorSystem.create(PcConstants.ACTOR_SYSTEM, conf); 49 | 50 | } 51 | 52 | /** 53 | * Create and get actor system. 54 | * 55 | * @return the actor system 56 | */ 57 | public static ActorSystem createAndGetActorSystem() { 58 | if (actorSystem == null || actorSystem.isTerminated()) { 59 | actorSystem = ActorSystem.create(PcConstants.ACTOR_SYSTEM, conf); 60 | } 61 | return actorSystem; 62 | } 63 | 64 | 65 | /** The Constant timeOutDuration. */ 66 | // wait for 10 seconds 67 | public static final FiniteDuration timeOutDuration = Duration.create(10, 68 | TimeUnit.SECONDS); 69 | 70 | /** 71 | * Shut down actor system force. 72 | */ 73 | public static void shutDownActorSystemForce() { 74 | if (!actorSystem.isTerminated()) { 75 | logger.info("shutting down actor system..."); 76 | actorSystem.shutdown(); 77 | actorSystem.awaitTermination(timeOutDuration); 78 | logger.info("Actor system has been shut down."); 79 | } else { 80 | logger.info("Actor system has been terminated already. NO OP."); 81 | } 82 | 83 | } 84 | 85 | 86 | 87 | 88 | /** 89 | * Gets the timeoutduration. 90 | * 91 | * @return the timeoutduration 92 | */ 93 | public static FiniteDuration getTimeoutduration() { 94 | return timeOutDuration; 95 | } 96 | 97 | } 98 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/actor/message/CancelTaskOnHostRequest.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.actor.message; 14 | 15 | import java.util.ArrayList; 16 | import java.util.List; 17 | 18 | 19 | /** 20 | * The Class CancelTaskOnHostRequestToManager. 21 | */ 22 | public class CancelTaskOnHostRequest { 23 | 24 | private final List targetHosts = new ArrayList(); 25 | 26 | public CancelTaskOnHostRequest(List targetHosts) { 27 | super(); 28 | this.targetHosts.addAll(targetHosts); 29 | } 30 | 31 | public List getTargetHosts() { 32 | return targetHosts; 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/actor/message/ContinueToSendToBatchSenderAsstManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.actor.message; 14 | 15 | 16 | /** 17 | * The Class ContinueToSendToBatchSenderAsstManager. 18 | */ 19 | public class ContinueToSendToBatchSenderAsstManager { 20 | 21 | /** The processed worker count. */ 22 | public final int processedWorkerCount; 23 | 24 | /** 25 | * Instantiates a new continue to send to batch sender asst manager. 26 | * 27 | * @param processedWorkerCount 28 | * the processed worker count 29 | */ 30 | public ContinueToSendToBatchSenderAsstManager(int processedWorkerCount) { 31 | super(); 32 | this.processedWorkerCount = processedWorkerCount; 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/actor/message/InitialRequestToManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.actor.message; 14 | 15 | import io.parallec.core.ParallelTask; 16 | 17 | /** 18 | * The Class InitialRequestToManager. 19 | */ 20 | public class InitialRequestToManager { 21 | 22 | /** 23 | * Instantiates a new initial request to manager with parallel task 24 | * 25 | * @param parallelTask 26 | * the parallel task 27 | */ 28 | public InitialRequestToManager(ParallelTask parallelTask) { 29 | super(); 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/actor/message/RequestToBatchSenderAsstManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.actor.message; 14 | 15 | import io.parallec.core.config.ParallecGlobalConfig; 16 | 17 | import java.util.List; 18 | 19 | import akka.actor.ActorRef; 20 | 21 | /** 22 | * The Class RequestToBatchSenderAsstManager. 23 | */ 24 | public class RequestToBatchSenderAsstManager { 25 | 26 | /** The task id. */ 27 | private final String taskId; 28 | 29 | /** The asst manager retry interval millis. */ 30 | private final long asstManagerRetryIntervalMillis; 31 | 32 | /** The max concurrency. */ 33 | private final int maxConcurrency; 34 | 35 | /** 36 | * The workers: list of actor ref. cannot be empty because coming from 37 | * */ 38 | private final List workers; 39 | 40 | /** The sender. */ 41 | private final ActorRef sender; 42 | 43 | /** used for pojo test only. */ 44 | public RequestToBatchSenderAsstManager() { 45 | super(); 46 | this.taskId = null; 47 | this.asstManagerRetryIntervalMillis = 250L; 48 | this.workers = null; 49 | this.sender = null; 50 | this.maxConcurrency = ParallecGlobalConfig.concurrencyDefault; 51 | } 52 | 53 | /** 54 | * Instantiates a new request to batch sender asst manager. 55 | * 56 | * @param directorJobId the director job id 57 | * @param asstManagerRetryIntervalMillis the asst manager retry interval millis 58 | * @param workers the workers 59 | * @param sender the sender 60 | * @param maxConcurrency the max concurrency 61 | */ 62 | public RequestToBatchSenderAsstManager(String directorJobId, 63 | long asstManagerRetryIntervalMillis, List workers, 64 | ActorRef sender, int maxConcurrency) { 65 | super(); 66 | this.taskId = directorJobId; 67 | this.asstManagerRetryIntervalMillis = asstManagerRetryIntervalMillis; 68 | this.workers = workers; 69 | this.sender = sender; 70 | this.maxConcurrency = maxConcurrency; 71 | } 72 | 73 | /** 74 | * Gets the max concurrency. 75 | * 76 | * @return the max concurrency 77 | */ 78 | public int getMaxConcurrency() { 79 | return maxConcurrency; 80 | } 81 | 82 | /** 83 | * Gets the task id. 84 | * 85 | * @return the task id 86 | */ 87 | public String getTaskId() { 88 | return taskId; 89 | } 90 | 91 | /** 92 | * Gets the workers. 93 | * 94 | * @return the workers 95 | */ 96 | public List getWorkers() { 97 | return workers; 98 | } 99 | 100 | /** 101 | * Gets the sender. 102 | * 103 | * @return the sender 104 | */ 105 | public ActorRef getSender() { 106 | return sender; 107 | } 108 | 109 | /** 110 | * Gets the asst manager retry interval millis. 111 | * 112 | * @return the asst manager retry interval millis 113 | */ 114 | public long getAsstManagerRetryIntervalMillis() { 115 | return asstManagerRetryIntervalMillis; 116 | } 117 | 118 | } 119 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/actor/message/ResponseCountToBatchSenderAsstManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.actor.message; 14 | 15 | 16 | /** 17 | * Receive count from the manager to asst manager; for flow control. 18 | * 19 | * @author Yuanteng (Jeff) Pei 20 | */ 21 | public class ResponseCountToBatchSenderAsstManager { 22 | 23 | /** The response count. */ 24 | private final int responseCount; 25 | 26 | /** 27 | * Instantiates a new response count to batch sender asst manager. 28 | * 29 | * @param responseCount 30 | * the response count 31 | */ 32 | public ResponseCountToBatchSenderAsstManager(int responseCount) { 33 | super(); 34 | this.responseCount = responseCount; 35 | } 36 | 37 | /** 38 | * Gets the response count. 39 | * 40 | * @return the response count 41 | */ 42 | public int getResponseCount() { 43 | return responseCount; 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/actor/message/ResponseFromManager.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.actor.message; 14 | 15 | /** 16 | * The Class BatchResponseFromManager. 17 | */ 18 | public class ResponseFromManager { 19 | 20 | private int responseCount; 21 | 22 | public ResponseFromManager(int responseCount) { 23 | super(); 24 | this.responseCount = responseCount; 25 | } 26 | 27 | public int getResponseCount() { 28 | return responseCount; 29 | } 30 | 31 | public void setResponseCount(int responseCount) { 32 | this.responseCount = responseCount; 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/actor/message/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Actor messages. 3 | */ 4 | package io.parallec.core.actor.message; -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/actor/message/type/ExecutionManagerMsgType.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.actor.message.type; 2 | 3 | /** 4 | * The Enum ExecutionManagerMsgType. 5 | */ 6 | public enum ExecutionManagerMsgType { 7 | /** The operation timeout. */ 8 | OPERATION_TIMEOUT, 9 | /** The cancel. */ 10 | CANCEL 11 | } -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/actor/message/type/OperationWorkerMsgType.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.actor.message.type; 14 | 15 | 16 | /** 17 | * The Enum OperationWorkerMsgType. 18 | */ 19 | public enum OperationWorkerMsgType { 20 | 21 | /** The poll progress. */ 22 | POLL_PROGRESS, 23 | /** The operation timeout. */ 24 | OPERATION_TIMEOUT, 25 | /** The process request. */ 26 | PROCESS_REQUEST, 27 | /** The cancel. */ 28 | CANCEL 29 | } -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/actor/message/type/RequestWorkerMsgType.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.actor.message.type; 14 | 15 | 16 | /** 17 | * The Enum RequestWorkerMsgType. for SSH/HTTP Workers only. 18 | */ 19 | public enum RequestWorkerMsgType { 20 | 21 | /** The process request. */ 22 | PROCESS_REQUEST, 23 | /** The check ssh progress. */ 24 | CHECK_FUTURE_STATE, 25 | /** The cancel. */ 26 | CANCEL, 27 | /** The process on exception. */ 28 | PROCESS_ON_EXCEPTION, 29 | /** The process on timeout. */ 30 | PROCESS_ON_TIMEOUT 31 | } -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/actor/message/type/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Actor message types. 3 | */ 4 | package io.parallec.core.actor.message.type; -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/actor/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Actors to execute parallel task and return responses. 3 | */ 4 | package io.parallec.core.actor; -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/actor/poll/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Classes for handling Async APIs with polling progress APIs 3 | * with success / failure criteria 4 | */ 5 | package io.parallec.core.actor.poll; -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/app/ParallecAppMin.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.app; 14 | 15 | import io.parallec.core.ParallecResponseHandler; 16 | import io.parallec.core.ParallelClient; 17 | import io.parallec.core.ResponseOnSingleTask; 18 | 19 | import java.util.Map; 20 | 21 | /** 22 | * The Class ParallecAppMin to demonstrate Parallec with 10 lines. 23 | */ 24 | public class ParallecAppMin { 25 | 26 | /** 27 | * The main method. 28 | * 29 | * @param args 30 | * the arguments 31 | */ 32 | public static void main(String[] args) { 33 | 34 | ParallelClient pc = new ParallelClient(); 35 | pc.prepareHttpGet("/validateInternals.html") 36 | .setConcurrency(1000) 37 | .setTargetHostsFromString( 38 | "www.parallec.io www.jeffpei.com www.restcommander.com") 39 | .execute(new ParallecResponseHandler() { 40 | 41 | @Override 42 | public void onCompleted(ResponseOnSingleTask res, 43 | Map responseContext) { 44 | System.out.println(res); 45 | } 46 | }); 47 | pc.releaseExternalResources(); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/app/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * The package for storing sample applications using Parallec. 3 | * @author Yuanteng (Jeff) Pei 4 | * 5 | */ 6 | package io.parallec.core.app; -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/bean/ResponseHeaderMeta.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.bean; 14 | 15 | import java.util.List; 16 | 17 | 18 | // TODO: Auto-generated Javadoc 19 | /** 20 | * The class used to represent the metadata for getting the response headers. 21 | * Which keys are needed to get from response header. 22 | * 23 | * @author Yuanteng (Jeff) Pei 24 | */ 25 | /** 26 | * @author ypei 27 | * 28 | */ 29 | public class ResponseHeaderMeta { 30 | 31 | 32 | /** If true, will get all the entries in the response headers. */ 33 | private boolean getAll; 34 | 35 | /** The key set for retrieving the http response headers. */ 36 | private List keys; 37 | 38 | 39 | /** 40 | * Instantiates a new response header meta. 41 | * 42 | * @param keys the keys would like to save 43 | * @param getAll when true: will get all the key value pair, regardless of the keys list provided. 44 | */ 45 | public ResponseHeaderMeta(List keys, boolean getAll) { 46 | super(); 47 | this.keys = keys; 48 | this.getAll = getAll; 49 | } 50 | 51 | 52 | /** 53 | * Checks if is gets the all. 54 | * 55 | * @return true, if is gets the all 56 | */ 57 | public boolean isGetAll() { 58 | return getAll; 59 | } 60 | 61 | /** 62 | * Sets the gets the all. 63 | * 64 | * @param getAll the new gets the all 65 | */ 66 | public void setGetAll(boolean getAll) { 67 | this.getAll = getAll; 68 | } 69 | 70 | /** 71 | * Gets the keys. 72 | * 73 | * @return the keys 74 | */ 75 | public List getKeys() { 76 | return keys; 77 | } 78 | 79 | /** 80 | * Sets the keys. 81 | * 82 | * @param keys the new keys 83 | */ 84 | public void setKeys(List keys) { 85 | this.keys = keys; 86 | } 87 | 88 | /* (non-Javadoc) 89 | * @see java.lang.Object#toString() 90 | */ 91 | @Override 92 | public String toString() { 93 | return "ResponseHeaderMeta [getAll=" + getAll + ", keys=" + keys + "]"; 94 | } 95 | 96 | 97 | } 98 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/bean/SetAndCount.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.bean; 14 | 15 | import java.util.LinkedHashSet; 16 | import java.util.Set; 17 | 18 | /** 19 | * The class used to represent the target hosts and a count when aggregation 20 | * 21 | * @author Yuanteng (Jeff) Pei 22 | */ 23 | public class SetAndCount { 24 | 25 | 26 | /** The count. */ 27 | private int count; 28 | 29 | /** The set. */ 30 | private final Set set = new LinkedHashSet(); 31 | 32 | /** 33 | * Instantiates a new sets the and count. 34 | * 35 | * @param set 36 | * the set 37 | */ 38 | public SetAndCount(Set set) { 39 | super(); 40 | this.set.addAll(set); 41 | this.count = set.size(); 42 | } 43 | 44 | /** 45 | * Gets the count. 46 | * 47 | * @return the count 48 | */ 49 | public int getCount() { 50 | return count; 51 | } 52 | 53 | /** 54 | * Sets the count. 55 | * 56 | * @param count 57 | * the new count 58 | */ 59 | public void setCount(int count) { 60 | this.count = count; 61 | } 62 | 63 | /** 64 | * Gets the sets the. 65 | * 66 | * @return the sets the 67 | */ 68 | public Set getSet() { 69 | return set; 70 | } 71 | 72 | /** 73 | * "SetAndCount [count=" + count + ", set=" + set + "]"; 74 | */ 75 | @Override 76 | public String toString() { 77 | return "SetAndCount [count=" + count + ", set=" + set + "]"; 78 | } 79 | 80 | } 81 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/bean/SingleTargetTaskStatus.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.bean; 14 | 15 | /** 16 | * The Enum SingleTargetTaskStatus. 17 | * 18 | * @author Yuanteng (Jeff) Pei 19 | */ 20 | public enum SingleTargetTaskStatus { 21 | 22 | /** The in progress. */ 23 | IN_PROGRESS, /** The completed. */ 24 | COMPLETED 25 | } 26 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/bean/StrStrMap.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.bean; 14 | 15 | import java.util.LinkedHashMap; 16 | import java.util.Map; 17 | 18 | import org.slf4j.Logger; 19 | import org.slf4j.LoggerFactory; 20 | 21 | /** 22 | * This allow you to replace multiple variables with multiple values key: the 23 | * variable name : e.g. GET /checkProgress/$JOBID value: the actual value string 24 | * that will replace the JOBID 25 | * 26 | * @author Yuanteng (Jeff) Pei 27 | */ 28 | public class StrStrMap { 29 | 30 | /** The map. */ 31 | private final Map map = new LinkedHashMap(); 32 | 33 | /** The logger. */ 34 | private static Logger logger = LoggerFactory.getLogger(StrStrMap.class); 35 | 36 | /** 37 | * Gets the map. 38 | * 39 | * @return the map 40 | */ 41 | public Map getMap() { 42 | return map; 43 | } 44 | 45 | /** 46 | * Instantiates a new str str map. 47 | */ 48 | public StrStrMap() { 49 | } 50 | 51 | /** 52 | * Instantiates a new str str map. 53 | * 54 | * @param map 55 | * the map 56 | */ 57 | public StrStrMap(Map map) { 58 | this.map.putAll(map); 59 | } 60 | 61 | /** 62 | * Adds the pair. 63 | * 64 | * @param key 65 | * the key 66 | * @param value 67 | * the value 68 | * @return the str str map 69 | */ 70 | public StrStrMap addPair(String key, String value) { 71 | if (key == null || value == null) { 72 | logger.error("invalid key value as null. ignore pair"); 73 | 74 | } else { 75 | 76 | this.map.put(key, value); 77 | } 78 | 79 | return this; 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/bean/TargetHostMeta.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.bean; 14 | 15 | import io.parallec.core.util.PcDateUtils; 16 | 17 | import java.util.ArrayList; 18 | import java.util.List; 19 | import java.util.UUID; 20 | 21 | /** 22 | * The metadata about the targetHost, which is mainly a list of hostnames (IP or FQDN) 23 | * the primary key is the targetHostId, generated with timestamp 24 | * 25 | * @author Yuanteng (Jeff) Pei 26 | */ 27 | public class TargetHostMeta { 28 | 29 | /** The target host id. */ 30 | private String targetHostId; 31 | 32 | /** The target host list. */ 33 | private final List hosts = new ArrayList(); 34 | 35 | /** 36 | * Instantiates a new target host meta. 37 | * 38 | * @param hosts the hosts 39 | */ 40 | public TargetHostMeta(List hosts) { 41 | super(); 42 | 43 | final String uuid = UUID.randomUUID().toString().substring(0, 12); 44 | this.hosts.addAll(hosts); 45 | 46 | this.setTargetHostId("THM_" + hosts.size() + "_" 47 | + PcDateUtils.getNowDateTimeStrConciseNoZone() + "_" + uuid); 48 | 49 | } 50 | 51 | /** 52 | * Instantiates a new target host meta. 53 | */ 54 | public TargetHostMeta() { 55 | }; 56 | 57 | /** 58 | * Gets the node list. 59 | * 60 | * @return the node list 61 | */ 62 | public List getHosts() { 63 | return hosts; 64 | } 65 | public String getTargetHostId() { 66 | return targetHostId; 67 | } 68 | 69 | public void setTargetHostId(String targetHostId) { 70 | this.targetHostId = targetHostId; 71 | } 72 | 73 | @Override 74 | public String toString() { 75 | return "TargetHostMeta [targetHostId=" + targetHostId + ", hosts size:" 76 | + hosts.size() + "]"; 77 | } 78 | 79 | 80 | 81 | } 82 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/bean/TaskRunMode.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.bean; 2 | 3 | /** 4 | * The Enum Mode of running Parallel Task: async or sync. 5 | */ 6 | public enum TaskRunMode { 7 | 8 | /** The sync. */ 9 | SYNC, 10 | /** The async. */ 11 | ASYNC 12 | } -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/bean/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Beans. 3 | */ 4 | package io.parallec.core.bean; -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/bean/ping/PingMeta.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.bean.ping; 14 | 15 | import io.parallec.core.config.ParallecGlobalConfig; 16 | import io.parallec.core.exception.ParallelTaskInvalidException; 17 | 18 | import org.slf4j.Logger; 19 | import org.slf4j.LoggerFactory; 20 | 21 | 22 | /** 23 | * The Ping metadata on mode,timeout and retries. 24 | * 25 | * @author Yuanteng (Jeff) Pei 26 | * 27 | */ 28 | public class PingMeta { 29 | /** The logger. */ 30 | private static Logger logger = LoggerFactory.getLogger(PingMeta.class); 31 | 32 | private PingMode mode; 33 | private Integer pingTimeoutMillis; 34 | private Integer numRetries; 35 | 36 | 37 | public PingMeta() { 38 | super(); 39 | this.mode = null; 40 | this.pingTimeoutMillis = null; 41 | this.numRetries = null; 42 | } 43 | 44 | public boolean validation() throws ParallelTaskInvalidException { 45 | 46 | if (this.mode == null) { 47 | logger.info("SET DEFAULT PING MODE: INET_ADDRESS_REACHABLE_NEED_ROOT." 48 | + "WARNING. MUST run as ROOT for accuracy." 49 | + " "); 50 | this.mode= ParallecGlobalConfig.pingModeDefault; 51 | } 52 | 53 | if (this.pingTimeoutMillis == null) { 54 | logger.info("SET DEFAULT PING TIMEOUT: 500ms "); 55 | this.pingTimeoutMillis= ParallecGlobalConfig.pingTimeoutMillisDefault; 56 | } 57 | 58 | if (this.numRetries == null) { 59 | logger.info("SET DEFAULT PING NUM OF RETRIES: 1 "); 60 | this.numRetries= ParallecGlobalConfig.pingNumRetriesDefault; 61 | } 62 | return true; 63 | 64 | } 65 | 66 | 67 | public PingMode getMode() { 68 | return mode; 69 | } 70 | 71 | public void setMode(PingMode mode) { 72 | this.mode = mode; 73 | } 74 | 75 | public Integer getPingTimeoutMillis() { 76 | return pingTimeoutMillis; 77 | } 78 | 79 | public void setPingTimeoutMillis(Integer pingTimeoutMillis) { 80 | this.pingTimeoutMillis = pingTimeoutMillis; 81 | } 82 | 83 | public Integer getNumRetries() { 84 | return numRetries; 85 | } 86 | 87 | public void setNumRetries(Integer numRetries) { 88 | this.numRetries = numRetries; 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/bean/ping/PingMode.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.bean.ping; 14 | 15 | /** 16 | * The Enum of Ping Mode. Process or INET_ADDRESS_REACHABLE based. 17 | * Default as InetAddress mode. InetAddress requires Root privilege. 18 | * @author Yuanteng (Jeff) Pei 19 | */ 20 | public enum PingMode { 21 | 22 | /** Start a process to run the real ping command from the OS 23 | * current did not specify the timeout. 24 | * This is slower due to creating process for each request that limits the concurrency. 25 | * Suggest maximum concurrency less than 400. 26 | * if not adjust your ulimit. Only use this when you do not run as Root.*/ 27 | PROCESS, 28 | 29 | /** The inet address reachable. please only use in linux model with Root privilege. 30 | * This is the way faster approach. 31 | * This normally needs ROOT privilege to really use ICMP protocol. 32 | * Test whether that address is reachable. 33 | * Best effort is made by the implementation to try to reach the host, 34 | * but firewalls and server configuration may block requests resulting in a unreachable status 35 | * while some specific ports may be accessible. 36 | * A typical implementation will use ICMP ECHO REQUESTs if the privilege can be obtained, 37 | * otherwise it will try to establish a TCP connection on port 7 (Echo) of the destination host. 38 | * 39 | * */ 40 | INET_ADDRESS_REACHABLE_NEED_ROOT 41 | } -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/bean/ping/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Beans related to ping. 3 | * 4 | */ 5 | package io.parallec.core.bean.ping; -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/bean/ssh/SshLoginType.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.bean.ssh; 14 | 15 | /** 16 | * The Enum SshLoginType. login via password, or the private/public key pair 17 | */ 18 | public enum SshLoginType { 19 | 20 | /** The password. */ 21 | PASSWORD, 22 | /** The key. */ 23 | KEY 24 | } -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/bean/ssh/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Beans related to ssh. 3 | */ 4 | package io.parallec.core.bean.ssh; -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/bean/tcp/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Beans related to TCP. 3 | */ 4 | package io.parallec.core.bean.tcp; -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/bean/udp/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | /** 5 | * @author ypei 6 | * 7 | */ 8 | package io.parallec.core.bean.udp; -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/commander/workflow/InternalDataProvider.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.commander.workflow; 14 | 15 | import io.parallec.core.ParallelTask; 16 | import io.parallec.core.actor.message.NodeReqResponse; 17 | import io.parallec.core.bean.HttpMeta; 18 | import io.parallec.core.bean.TargetHostMeta; 19 | import io.parallec.core.util.PcConstants; 20 | import io.parallec.core.util.PcDateUtils; 21 | 22 | import java.util.Map; 23 | import java.util.Map.Entry; 24 | 25 | import org.slf4j.Logger; 26 | import org.slf4j.LoggerFactory; 27 | 28 | /** 29 | * generate the node data; make sure it is not null. 30 | * 31 | * @author Yuanteng (Jeff) Pei 32 | * 33 | */ 34 | public class InternalDataProvider { 35 | 36 | /** The logger. */ 37 | private static Logger logger = LoggerFactory 38 | .getLogger(InternalDataProvider.class); 39 | 40 | /** The Constant instance. */ 41 | private static final InternalDataProvider instance = new InternalDataProvider(); 42 | 43 | /** 44 | * Gets the single instance of InternalDataProvider. 45 | * 46 | * @return single instance of InternalDataProvider 47 | */ 48 | public static InternalDataProvider getInstance() { 49 | return instance; 50 | } 51 | 52 | /** 53 | * Instantiates a new internal data provider. 54 | */ 55 | private InternalDataProvider() { 56 | } 57 | 58 | /** 59 | * Generate node data map. 60 | * 61 | * @param task 62 | * the job info 63 | */ 64 | public void genNodeDataMap(ParallelTask task) { 65 | 66 | TargetHostMeta targetHostMeta = task.getTargetHostMeta(); 67 | HttpMeta httpMeta = task.getHttpMeta(); 68 | 69 | String entityBody = httpMeta.getEntityBody(); 70 | String requestContent = HttpMeta 71 | .replaceDefaultFullRequestContent(entityBody); 72 | 73 | Map parallelTaskResult = task 74 | .getParallelTaskResult(); 75 | for (String fqdn : targetHostMeta.getHosts()) { 76 | NodeReqResponse nodeReqResponse = new NodeReqResponse(fqdn); 77 | nodeReqResponse.setDefaultReqestContent(requestContent); 78 | parallelTaskResult.put(fqdn, nodeReqResponse); 79 | } 80 | }// end func. 81 | 82 | /** 83 | * Filter unsafe or unnecessary request. 84 | * 85 | * @param nodeDataMapValidSource 86 | * the node data map valid source 87 | * @param nodeDataMapValidSafe 88 | * the node data map valid safe 89 | */ 90 | public void filterUnsafeOrUnnecessaryRequest( 91 | Map nodeDataMapValidSource, 92 | Map nodeDataMapValidSafe) { 93 | 94 | for (Entry entry : nodeDataMapValidSource 95 | .entrySet()) { 96 | 97 | String hostName = entry.getKey(); 98 | NodeReqResponse nrr = entry.getValue(); 99 | 100 | Map map = nrr.getRequestParameters(); 101 | 102 | /** 103 | * 20130507: will generally apply to all requests: if have this 104 | * field and this field is false 105 | */ 106 | if (map.containsKey(PcConstants.NODE_REQUEST_WILL_EXECUTE)) { 107 | Boolean willExecute = Boolean.parseBoolean(map 108 | .get(PcConstants.NODE_REQUEST_WILL_EXECUTE)); 109 | 110 | if (!willExecute) { 111 | logger.info("NOT_EXECUTE_COMMAND " + " on target: " 112 | + hostName + " at " 113 | + PcDateUtils.getNowDateTimeStrStandard()); 114 | continue; 115 | } 116 | } 117 | 118 | // now safely to add this node in. 119 | nodeDataMapValidSafe.put(hostName, nrr); 120 | }// end for loop 121 | 122 | } 123 | 124 | } 125 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/commander/workflow/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Work flow and variable replacement utilities. 3 | */ 4 | package io.parallec.core.commander.workflow; -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/commander/workflow/ping/PingProvider.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.commander.workflow.ping; 2 | 3 | import io.parallec.core.bean.ping.PingMeta; 4 | import io.parallec.core.bean.ping.PingMode; 5 | 6 | import java.net.InetAddress; 7 | 8 | import org.slf4j.Logger; 9 | import org.slf4j.LoggerFactory; 10 | 11 | public class PingProvider { 12 | /** The logger. */ 13 | private static Logger logger = LoggerFactory.getLogger(PingProvider.class); 14 | 15 | private static PingProvider instance = new PingProvider(); 16 | 17 | private PingProvider(){ 18 | } 19 | 20 | public static PingProvider getInstance(){ 21 | return instance; 22 | } 23 | 24 | public boolean isReachableByPing(String targetHost, PingMeta pingMeta) { 25 | 26 | boolean live = isReachableByPingHelper(targetHost, pingMeta); 27 | int retryLeft = pingMeta.getNumRetries(); 28 | while(!live && retryLeft>0){ 29 | live = isReachableByPingHelper(targetHost, pingMeta); 30 | retryLeft--; 31 | } 32 | return live; 33 | 34 | } 35 | 36 | public boolean isReachableByPingHelper(String targetHost, PingMeta pingMeta) { 37 | try { 38 | 39 | if(pingMeta.getMode()==PingMode.INET_ADDRESS_REACHABLE_NEED_ROOT){ 40 | InetAddress address = InetAddress.getByName(targetHost); 41 | return address.isReachable(pingMeta.getPingTimeoutMillis()); 42 | }else{ 43 | String cmd = ""; 44 | String os = System.getProperty("os.name").toLowerCase(); 45 | if (os.indexOf("win")>=0) { 46 | // For Windows 47 | cmd = "ping -n 1 -w " + pingMeta.getPingTimeoutMillis() + " " + targetHost; 48 | } else { 49 | // For Linux (-W) and OSX (-t) 50 | String timeoutArg = os.indexOf("mac")>=0 ? "-t" : "-W"; 51 | cmd = "ping -c 1 " 52 | + timeoutArg 53 | + " "+ (int) (pingMeta.getPingTimeoutMillis()/1000) + " " + targetHost; 54 | } 55 | Process myProcess = Runtime.getRuntime().exec(cmd); 56 | myProcess.waitFor(); 57 | if (myProcess.exitValue() == 0) { 58 | return true; 59 | } else { 60 | return false; 61 | } 62 | 63 | } 64 | 65 | } catch (Exception e) { 66 | logger.info("Bad hostname {} with err {} ", targetHost, e.getLocalizedMessage()); 67 | return false; 68 | } 69 | } 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/commander/workflow/ping/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Work flow utilities on ping. 3 | */ 4 | package io.parallec.core.commander.workflow.ping; -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/commander/workflow/ssh/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Work flow utilities on ssh. 3 | */ 4 | package io.parallec.core.commander.workflow.ssh; -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/config/HandlerExecutionLocation.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.config; 2 | 3 | public enum HandlerExecutionLocation { 4 | 5 | OPERATION_WORKER, MANAGER 6 | } 7 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/config/ParallelTaskConfigDefault.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.config; 14 | 15 | 16 | /** 17 | * The default values for {@link ParallelTaskConfig} 18 | * 19 | */ 20 | public class ParallelTaskConfigDefault { 21 | 22 | /** The asst manager retry interval millis. */ 23 | public static long asstManagerRetryIntervalMillis = 250L; 24 | 25 | /** The actor max operation timeout sec. */ 26 | public static int actorMaxOperationTimeoutSec = 15; 27 | 28 | /** 29 | * The command manager internal timeout and cancel itself time in seconds 30 | * Note this may need to be adjusted for long polling jobs. 31 | */ 32 | public static long timeoutInManagerSec = 600; 33 | 34 | /** The timeout the director send to the manager to cancel it from outside. */ 35 | public static long timeoutAskManagerSec = timeoutInManagerSec + 10; 36 | 37 | /** The print http true header map. */ 38 | public static boolean printHttpTrueHeaderMap = true; 39 | 40 | /** The print poller. */ 41 | public static boolean printPoller = true; 42 | 43 | /** The save response to task. */ 44 | public static boolean saveResponseToTask = false; 45 | 46 | /** The auto save log to local. */ 47 | public static boolean autoSaveLogToLocal = false; 48 | 49 | /** The enable capacity aware task scheduler. */ 50 | public static boolean enableCapacityAwareTaskScheduler = false; 51 | 52 | /** The handler execution location default as in Manager after aggregation. */ 53 | public static HandlerExecutionLocation handlerExecutionLocationDefault = HandlerExecutionLocation.MANAGER; 54 | /** 55 | * Instantiates a new parallel task config. 56 | */ 57 | public ParallelTaskConfigDefault() { 58 | super(); 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/config/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Configurations on global and task levels. 3 | */ 4 | package io.parallec.core.config; -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/exception/ActorMessageTypeInvalidException.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.exception; 14 | 15 | /** 16 | * The Class ActorMessageTypeInvalidException. 17 | * 18 | * @author Yuanteng (Jeff) Pei 19 | */ 20 | public class ActorMessageTypeInvalidException extends RuntimeException { 21 | 22 | /** The Constant serialVersionUID. */ 23 | private static final long serialVersionUID = 1L; 24 | 25 | /** 26 | * Instantiates a new actor message type invalid exception. 27 | * 28 | * @param error 29 | * the error 30 | */ 31 | public ActorMessageTypeInvalidException(String error) { 32 | super(error); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/exception/ExecutionManagerExecutionException.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.exception; 14 | 15 | /** 16 | * The Class ExecutionManagerExecutionException. 17 | */ 18 | public class ExecutionManagerExecutionException extends RuntimeException { 19 | 20 | /** The Constant serialVersionUID. */ 21 | private static final long serialVersionUID = 1L; 22 | 23 | /** The type. */ 24 | private ManagerExceptionType type; 25 | 26 | /** 27 | * The Enum ManagerExceptionType. 28 | */ 29 | public enum ManagerExceptionType { 30 | 31 | /** The timeout. */ 32 | TIMEOUT, 33 | /** The cancel. */ 34 | CANCEL 35 | } 36 | 37 | /** 38 | * Instantiates a new command manager execution exception. 39 | * 40 | * @param error 41 | * the error 42 | * @param type 43 | * the type 44 | */ 45 | public ExecutionManagerExecutionException(String error, 46 | ManagerExceptionType type) { 47 | super(error); 48 | this.setType(type); 49 | } 50 | 51 | /** 52 | * Gets the type. 53 | * 54 | * @return the type 55 | */ 56 | public ManagerExceptionType getType() { 57 | return type; 58 | } 59 | 60 | /** 61 | * Sets the type. 62 | * 63 | * @param type 64 | * the new type 65 | */ 66 | public void setType(ManagerExceptionType type) { 67 | this.type = type; 68 | } 69 | 70 | } 71 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/exception/HttpRequestCreateException.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.exception; 14 | 15 | 16 | /** 17 | * The Class HttpRequestCreateException. 18 | */ 19 | public class HttpRequestCreateException extends RuntimeException { 20 | 21 | /** The Constant serialVersionUID. */ 22 | private static final long serialVersionUID = 1L; 23 | 24 | /** 25 | * Instantiates a new http request create exception. 26 | * 27 | * @param error 28 | * the error 29 | * @param t 30 | * the t 31 | */ 32 | public HttpRequestCreateException(String error, Exception t) { 33 | super(error, t); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/exception/ParallelTaskInvalidException.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.exception; 14 | 15 | 16 | /** 17 | * The Class ParallelTaskInvalidException. 18 | */ 19 | public class ParallelTaskInvalidException extends RuntimeException { 20 | 21 | /** The Constant serialVersionUID. */ 22 | private static final long serialVersionUID = 1L; 23 | 24 | /** 25 | * Instantiates a new parallel task invalid exception. 26 | * 27 | * @param error 28 | * the error 29 | * @param t 30 | * the t 31 | */ 32 | public ParallelTaskInvalidException(String error, Exception t) { 33 | super(error, t); 34 | } 35 | 36 | /** 37 | * Instantiates a new parallel task invalid exception. 38 | * 39 | * @param error 40 | * the error 41 | */ 42 | public ParallelTaskInvalidException(String error) { 43 | super(error); 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/exception/TargetHostsLoadException.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.exception; 14 | 15 | 16 | /** 17 | * The Class TargetHostsLoadException. 18 | */ 19 | public class TargetHostsLoadException extends RuntimeException { 20 | 21 | /** The Constant serialVersionUID. */ 22 | private static final long serialVersionUID = 1L; 23 | 24 | /** 25 | * Instantiates a new target hosts load exception. 26 | * 27 | * @param error 28 | * the error 29 | * @param t 30 | * the t 31 | */ 32 | public TargetHostsLoadException(String error, Exception t) { 33 | super(error, t); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/exception/TcpUdpRequestCreateException.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.exception; 14 | 15 | 16 | /** 17 | * The Class HttpRequestCreateException. 18 | */ 19 | public class TcpUdpRequestCreateException extends RuntimeException { 20 | 21 | /** The Constant serialVersionUID. */ 22 | private static final long serialVersionUID = 1L; 23 | 24 | /** 25 | * Instantiates a new TCP / UDP request create exception. 26 | * 27 | * @param error 28 | * the error 29 | * @param t 30 | * the t 31 | */ 32 | public TcpUdpRequestCreateException(String error, Exception t) { 33 | super(error, t); 34 | } 35 | 36 | } 37 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/exception/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Exceptions. 3 | */ 4 | package io.parallec.core.exception; -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/monitor/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * On demand monitoring on threads and memory. 3 | */ 4 | package io.parallec.core.monitor; -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Main package, including most user-facing classes such as the ParallelClient, 3 | * ParallelTask etc. 4 | */ 5 | package io.parallec.core; -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/resources/HttpClientType.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.resources; 14 | 15 | 16 | /** 17 | * The Enum of HttyClient Type. Default is the EMBED_FAST. 18 | */ 19 | public enum HttpClientType { 20 | 21 | /** The embed fast. */ 22 | EMBED_FAST, 23 | /** The embed slow. */ 24 | EMBED_SLOW, 25 | /** The custom fast. */ 26 | CUSTOM_FAST, 27 | /** The custom slow. */ 28 | CUSTOM_SLOW 29 | } -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/resources/HttpMethod.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.resources; 14 | 15 | 16 | /** 17 | * The Enum HttpMethod. 18 | */ 19 | public enum HttpMethod { 20 | 21 | /** The head. */ 22 | HEAD, 23 | /** The get. */ 24 | GET, 25 | /** The post. */ 26 | POST, 27 | /** The put. */ 28 | PUT, 29 | /** The delete. */ 30 | DELETE, 31 | /** The options. */ 32 | OPTIONS, 33 | /** The na. */ 34 | NA 35 | } 36 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/resources/TcpUdpSshPingResourceStore.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.resources; 14 | 15 | import java.util.concurrent.ExecutorService; 16 | import java.util.concurrent.Executors; 17 | 18 | import org.jboss.netty.channel.ChannelFactory; 19 | import org.jboss.netty.channel.socket.DatagramChannelFactory; 20 | import org.jboss.netty.channel.socket.nio.NioClientSocketChannelFactory; 21 | import org.jboss.netty.channel.socket.nio.NioDatagramChannelFactory; 22 | import org.jboss.netty.util.HashedWheelTimer; 23 | 24 | /** 25 | * Provide external resources needed for netty based TCP/UDP worker: ChannelFactory 26 | * and HashedWheelTimer timer 27 | * 28 | * @author Yuanteng (Jeff) Pei 29 | */ 30 | public class TcpUdpSshPingResourceStore { 31 | 32 | /** The singleton instance. */ 33 | private final static TcpUdpSshPingResourceStore instance = new TcpUdpSshPingResourceStore(); 34 | 35 | public static TcpUdpSshPingResourceStore getInstance() { 36 | return instance; 37 | } 38 | 39 | private ChannelFactory channelFactory; 40 | private DatagramChannelFactory datagramChannelFactory; 41 | private HashedWheelTimer timer = null; 42 | 43 | private ExecutorService threadPoolForSshPing = Executors 44 | .newCachedThreadPool(); 45 | 46 | public void shutdown() { 47 | if (channelFactory != null) { 48 | channelFactory.releaseExternalResources(); 49 | } 50 | 51 | if (datagramChannelFactory != null) { 52 | channelFactory.releaseExternalResources(); 53 | } 54 | if (timer != null) 55 | timer.stop(); 56 | } 57 | 58 | /** 59 | * Instantiates a new http client store. 60 | */ 61 | private TcpUdpSshPingResourceStore() { 62 | init(); 63 | } 64 | 65 | /** 66 | * Initialize; cached threadpool is safe as it is releasing resources automatically if idle 67 | */ 68 | public synchronized void init() { 69 | channelFactory = new NioClientSocketChannelFactory( 70 | Executors.newCachedThreadPool(), 71 | Executors.newCachedThreadPool()); 72 | 73 | datagramChannelFactory = new NioDatagramChannelFactory( 74 | Executors.newCachedThreadPool()); 75 | 76 | timer = new HashedWheelTimer(); 77 | } 78 | 79 | /** 80 | * close and clean up the http client, then create the new ones. 81 | */ 82 | public synchronized void reinit() { 83 | 84 | // first shutdown existing ones. 85 | shutdown(); 86 | init(); 87 | } 88 | 89 | public ChannelFactory getChannelFactory() { 90 | return channelFactory; 91 | } 92 | 93 | public void setChannelFactory(ChannelFactory channelFactory) { 94 | this.channelFactory = channelFactory; 95 | } 96 | 97 | public HashedWheelTimer getTimer() { 98 | return timer; 99 | } 100 | 101 | public void setTimer(HashedWheelTimer timer) { 102 | this.timer = timer; 103 | } 104 | 105 | public ExecutorService getThreadPoolForSshPing() { 106 | return threadPoolForSshPing; 107 | } 108 | 109 | public void setThreadPoolForSshPing(ExecutorService threadPoolForSshPing) { 110 | this.threadPoolForSshPing = threadPoolForSshPing; 111 | } 112 | 113 | public DatagramChannelFactory getDatagramChannelFactory() { 114 | return datagramChannelFactory; 115 | } 116 | 117 | public void setDatagramChannelFactory(DatagramChannelFactory datagramChannelFactory) { 118 | this.datagramChannelFactory = datagramChannelFactory; 119 | } 120 | 121 | } 122 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/resources/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * External resources on async http client stores, thread pools for netty/ping/ssh. 3 | */ 4 | package io.parallec.core.resources; -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/task/CapacityAwareTaskScheduler.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.task; 14 | 15 | import io.parallec.core.ParallelTask; 16 | import io.parallec.core.ParallelTaskBuilder; 17 | import io.parallec.core.config.ParallecGlobalConfig; 18 | 19 | import java.util.Queue; 20 | 21 | import org.slf4j.Logger; 22 | import org.slf4j.LoggerFactory; 23 | 24 | /** 25 | * 26 | * A task scheduler runnable to check if there are capacity to run a task 27 | * from the waitQ. 28 | * 29 | * Note that this scheduler is not enabled by default. 30 | * 31 | * Enable it by {@link ParallelTaskBuilder#setEnableCapacityAwareTaskScheduler} 32 | * 33 | * @author Yuanteng (Jeff) Pei 34 | */ 35 | public class CapacityAwareTaskScheduler implements Runnable { 36 | 37 | /** The Constant logger. */ 38 | private static final Logger logger = LoggerFactory 39 | .getLogger(CapacityAwareTaskScheduler.class); 40 | 41 | /* 42 | * (non-Javadoc) 43 | * 44 | * @see java.lang.Runnable#run() 45 | */ 46 | public void run() { 47 | try { 48 | 49 | Queue waitQ = ParallelTaskManager.getInstance() 50 | .getWaitQ(); 51 | 52 | logger.debug( 53 | "TASK_WAIT_Q: Current waitQ has task count: {} in Thread scheduler", 54 | waitQ.size()); 55 | 56 | final ParallelTask task = waitQ.peek(); 57 | if (task != null) { 58 | 59 | int totalUsedCapacityBefore = ParallelTaskManager.getInstance() 60 | .getTotalUsedCapacity(); 61 | 62 | int capacityThisTask = task.capacityUsed(); 63 | int capacityRemain = ParallecGlobalConfig.maxCapacity 64 | - totalUsedCapacityBefore; 65 | int totalUsedCapacityNew = totalUsedCapacityBefore 66 | + capacityThisTask; 67 | 68 | logger.info( 69 | "TASK_WAIT_Q: Exists Task in WaitQ with head of queue task id {}" 70 | + "....Current used capacity {}, and remaining capacity is {}", 71 | task.getTaskId(), totalUsedCapacityBefore, 72 | capacityRemain); 73 | // condition to add to the in progress map 74 | if (totalUsedCapacityNew <= ParallecGlobalConfig.maxCapacity) { 75 | logger.info( 76 | "TASK_WAIT_Q: Sufficent capacity. Execute new task from wait queue. Task capacity {}" 77 | + " with total used capacity is now {}, capacityRemain will be {}", 78 | capacityThisTask, totalUsedCapacityNew, 79 | ParallecGlobalConfig.maxCapacity 80 | - totalUsedCapacityNew); 81 | 82 | waitQ.poll(); // dequeue 83 | Runnable taskRunnable = new Runnable() { 84 | @Override 85 | public void run() { 86 | ParallelTaskManager.getInstance() 87 | .generateUpdateExecuteTask(task); 88 | } 89 | }; 90 | new Thread(taskRunnable).start(); 91 | } else { 92 | logger.info( 93 | "TASK_WAIT_Q: Skip execution new task. Insufficent capacity. " 94 | + "Head of queue task needs capacity {}. However capacityRemain is only {}.", 95 | capacityThisTask, capacityRemain); 96 | } 97 | 98 | } 99 | 100 | } catch (Exception e) { 101 | logger.error("TASK_WAIT_Q: fail in EvalTaskInWaitQRunner ", e); 102 | } 103 | } 104 | 105 | } 106 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/task/ParallelTaskState.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.task; 2 | 3 | // Job Id to Job Data 4 | /** 5 | * The Enum ParallelTaskState. 6 | */ 7 | // finishNotGathered is slave are ready but not send to Director 8 | public enum ParallelTaskState { 9 | 10 | /** The waiting. */ 11 | WAITING, 12 | /** The in progress. */ 13 | IN_PROGRESS, 14 | /** The completed without error. */ 15 | COMPLETED_WITHOUT_ERROR, 16 | /** The completed with error. */ 17 | COMPLETED_WITH_ERROR 18 | } -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/task/RequestReplacementType.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.task; 14 | 15 | 16 | /** 17 | * The Enum RequestReplacementType. 18 | */ 19 | public enum RequestReplacementType { 20 | 21 | /** The no variable replacement. */ 22 | NO_REPLACEMENT, 23 | /** The uniform variable replacement. */ 24 | UNIFORM_VAR_REPLACEMENT, 25 | /** The target host specific variable replacement. */ 26 | TARGET_HOST_SPECIFIC_VAR_REPLACEMENT 27 | 28 | } -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/task/TaskErrorMeta.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.task; 14 | 15 | import io.parallec.core.util.PcDateUtils; 16 | 17 | 18 | /** 19 | * The Class TaskErrorMeta. 20 | */ 21 | public class TaskErrorMeta { 22 | 23 | /** 24 | * The Enum TaskErrorType. 25 | */ 26 | public enum TaskErrorType { 27 | 28 | /** The validation error. */ 29 | VALIDATION_ERROR, 30 | /** The global timeout. */ 31 | GLOBAL_TIMEOUT, 32 | /** The command manager error. */ 33 | COMMAND_MANAGER_ERROR, 34 | /** User canceled */ 35 | USER_CANCELED, 36 | /** The unknown. */ 37 | UNKNOWN 38 | 39 | } 40 | 41 | /** The type. */ 42 | private TaskErrorType type; 43 | 44 | /** The error time. */ 45 | private String errorTime; 46 | 47 | /** The throwable. */ 48 | private Throwable throwable; 49 | 50 | /** The details. */ 51 | private String details; 52 | 53 | /** 54 | * Instantiates a new task error meta. 55 | * 56 | * @param type 57 | * the type 58 | * @param details 59 | * the details 60 | */ 61 | public TaskErrorMeta(TaskErrorType type, String details) { 62 | 63 | this.type = type; 64 | this.errorTime = PcDateUtils.getNowDateTimeStrStandard(); 65 | this.details = details; 66 | } 67 | 68 | /** 69 | * Instantiates a new task error meta. 70 | * 71 | * @param type 72 | * the type 73 | * @param details 74 | * the details 75 | * @param throwable 76 | * the throwable 77 | */ 78 | public TaskErrorMeta(TaskErrorType type, String details, Throwable throwable) { 79 | 80 | this.type = type; 81 | this.errorTime = PcDateUtils.getNowDateTimeStrStandard(); 82 | this.details = details; 83 | this.throwable = throwable; 84 | } 85 | 86 | /** 87 | * Gets the type. 88 | * 89 | * @return the type 90 | */ 91 | public TaskErrorType getType() { 92 | return type; 93 | } 94 | 95 | /** 96 | * Sets the type. 97 | * 98 | * @param type 99 | * the new type 100 | */ 101 | public void setType(TaskErrorType type) { 102 | this.type = type; 103 | } 104 | 105 | /** 106 | * Gets the error time. 107 | * 108 | * @return the error time 109 | */ 110 | public String getErrorTime() { 111 | return errorTime; 112 | } 113 | 114 | /** 115 | * Sets the error time. 116 | * 117 | * @param errorTime 118 | * the new error time 119 | */ 120 | public void setErrorTime(String errorTime) { 121 | this.errorTime = errorTime; 122 | } 123 | 124 | /** 125 | * Gets the throwable. 126 | * 127 | * @return the throwable 128 | */ 129 | public Throwable getThrowable() { 130 | return throwable; 131 | } 132 | 133 | /** 134 | * Sets the throwable. 135 | * 136 | * @param throwable 137 | * the new throwable 138 | */ 139 | public void setThrowable(Throwable throwable) { 140 | this.throwable = throwable; 141 | } 142 | 143 | /** 144 | * Gets the details. 145 | * 146 | * @return the details 147 | */ 148 | public String getDetails() { 149 | return details; 150 | } 151 | 152 | /** 153 | * Sets the details. 154 | * 155 | * @param details 156 | * the new details 157 | */ 158 | public void setDetails(String details) { 159 | this.details = details; 160 | } 161 | 162 | } 163 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/taskbuilder/targethosts/ITargetHostsBuilder.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.taskbuilder.targethosts; 14 | 15 | import io.parallec.core.HostsSourceType; 16 | import io.parallec.core.exception.TargetHostsLoadException; 17 | 18 | import java.util.List; 19 | 20 | /** 21 | * The Interface to build the target hosts. 22 | * Will remove duplicates. 23 | */ 24 | public interface ITargetHostsBuilder { 25 | 26 | /** 27 | * Sets the target hosts from list. 28 | * Will remove duplicate from it. 29 | * 30 | * Will create a new array list because the original one may be unmodifiable. 31 | * 32 | * @param targetHosts 33 | * the target hosts 34 | * @return the list 35 | */ 36 | public List setTargetHostsFromList(List targetHosts); 37 | 38 | /** 39 | * Sets the target hosts from string. 40 | * 41 | * @param targetHostsStr 42 | * the target hosts str 43 | * @return the list 44 | */ 45 | public List setTargetHostsFromString(String targetHostsStr); 46 | 47 | /** 48 | * Sets the target hosts from json path. 49 | * 50 | * @param jsonPath 51 | * the json path 52 | * @param sourcePath 53 | * the source path 54 | * @param sourceType 55 | * the source type 56 | * @return the list 57 | * @throws TargetHostsLoadException 58 | * the target hosts load exception 59 | */ 60 | public List setTargetHostsFromJsonPath(String jsonPath, 61 | String sourcePath, HostsSourceType sourceType) 62 | throws TargetHostsLoadException; 63 | 64 | /** 65 | * Sets the target hosts from line by line text. 66 | * 67 | * @param sourcePath 68 | * the source path 69 | * @param sourceType 70 | * the source type 71 | * @return the list 72 | * @throws TargetHostsLoadException 73 | * the target hosts load exception 74 | */ 75 | // read from a URL/File line by line 76 | public List setTargetHostsFromLineByLineText(String sourcePath, 77 | HostsSourceType sourceType) throws TargetHostsLoadException; 78 | 79 | /** 80 | * Sets the target hosts from cms query url. 81 | * 82 | * @param cmsQueryUrl 83 | * the cms query url 84 | * @return the list 85 | * @throws TargetHostsLoadException 86 | * the target hosts load exception 87 | */ 88 | // from query of CMS (CMS is internal name of the project YiDB. aka YiDB 89 | // http://yidb.org/ ); 90 | public List setTargetHostsFromCmsQueryUrl(String cmsQueryUrl) 91 | throws TargetHostsLoadException; 92 | 93 | /** 94 | * Sets the target hosts from cms query url. 95 | * 96 | * @param cmsQueryUrl 97 | * the cms query url 98 | * @param projection 99 | * the projection 100 | * @return the list 101 | * @throws TargetHostsLoadException 102 | * the target hosts load exception 103 | */ 104 | public List setTargetHostsFromCmsQueryUrl(String cmsQueryUrl, 105 | String projection) throws TargetHostsLoadException; 106 | 107 | /** 108 | * Sets the target hosts from cms query url with projection and authorization token. 109 | * 110 | * @param cmsQueryUrl 111 | * the cms query url 112 | * @param projection 113 | * the projection 114 | * @param token 115 | * the cms authorization token 116 | * @return the list 117 | * @throws TargetHostsLoadException 118 | * the target hosts load exception 119 | */ 120 | public List setTargetHostsFromCmsQueryUrl(String cmsQueryUrl, 121 | String projection, String token) throws TargetHostsLoadException; 122 | 123 | } 124 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/taskbuilder/targethosts/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Target host builder. 3 | */ 4 | package io.parallec.core.taskbuilder.targethosts; -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/util/BeanMapper.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.util; 2 | 3 | import io.parallec.core.ParallelTask; 4 | import io.parallec.core.task.ParallelTaskBean; 5 | 6 | import org.springframework.beans.BeanUtils; 7 | 8 | /** 9 | * TODO 10 | * 11 | * @author Yuanteng (Jeff) Pei 12 | * 13 | */ 14 | public class BeanMapper { 15 | 16 | public static void copy(final ParallelTask source, 17 | final ParallelTaskBean target) { 18 | BeanUtils.copyProperties(source, target, new String[] { "state", 19 | "logger", "responseContext", " handler", " parallelTaskResult", 20 | "executionManager", "replacementVarMapNodeSpecific", 21 | "replacementVarMap", "submitTime", "executeStartTime", 22 | "executionEndTime" }); 23 | } 24 | } 25 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/util/DaemonThreadFactory.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.util; 14 | 15 | import java.util.concurrent.ThreadFactory; 16 | 17 | 18 | /** 19 | * A factory for creating DaemonThread objects. 20 | */ 21 | public class DaemonThreadFactory implements ThreadFactory { 22 | 23 | /** The instance. */ 24 | private static DaemonThreadFactory instance = new DaemonThreadFactory(); 25 | 26 | /** 27 | * Gets the single instance of DaemonThreadFactory. 28 | * 29 | * @return single instance of DaemonThreadFactory 30 | */ 31 | public static DaemonThreadFactory getInstance() { 32 | return instance; 33 | } 34 | 35 | /** 36 | * Instantiates a new daemon thread factory. 37 | */ 38 | private DaemonThreadFactory() { 39 | 40 | } 41 | 42 | /* 43 | * (non-Javadoc) 44 | * 45 | * @see java.util.concurrent.ThreadFactory#newThread(java.lang.Runnable) 46 | */ 47 | @Override 48 | public Thread newThread(Runnable arg0) { 49 | Thread t = new Thread(arg0); 50 | t.setDaemon(true); 51 | if (t.getPriority() != Thread.NORM_PRIORITY) { 52 | t.setPriority(Thread.NORM_PRIORITY); 53 | } 54 | return t; 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/util/PcErrorMsgUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.util; 14 | 15 | import java.util.EnumMap; 16 | import java.util.Map; 17 | 18 | 19 | /** 20 | * The Class PcErrorMsgUtils. 21 | */ 22 | public class PcErrorMsgUtils { 23 | 24 | /** 25 | * The Enum ERROR_TYPE. 26 | */ 27 | public enum ERROR_TYPE { 28 | /** The connection exception. */ 29 | CONNECTION_EXCEPTION, 30 | 31 | } 32 | 33 | /** The Constant errorMapOrig. */ 34 | public static final Map errorMapOrig = new EnumMap(ERROR_TYPE.class); 35 | 36 | /** The Constant errorMapReplace. */ 37 | public static final Map errorMapReplace = new EnumMap(ERROR_TYPE.class); 38 | 39 | static { 40 | errorMapOrig.put(ERROR_TYPE.CONNECTION_EXCEPTION, 41 | "java.net.ConnectException"); 42 | errorMapReplace.put(ERROR_TYPE.CONNECTION_EXCEPTION, 43 | "java.net.ConnectException"); 44 | } 45 | 46 | /** 47 | * Replace error msg. 48 | * 49 | * @param origMsg 50 | * the orig msg 51 | * @return the string 52 | */ 53 | public static String replaceErrorMsg(String origMsg) { 54 | 55 | String replaceMsg = origMsg; 56 | for (ERROR_TYPE errorType : ERROR_TYPE.values()) { 57 | 58 | if (origMsg == null) { 59 | replaceMsg = PcConstants.NA; 60 | return replaceMsg; 61 | } 62 | 63 | if (origMsg.contains(errorMapOrig.get(errorType))) { 64 | replaceMsg = errorMapReplace.get(errorType); 65 | break; 66 | } 67 | 68 | } 69 | 70 | return replaceMsg; 71 | 72 | } 73 | 74 | } 75 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/util/PcHttpUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.util; 14 | 15 | import io.parallec.core.actor.message.NodeReqResponse; 16 | 17 | import java.util.Map; 18 | import java.util.Map.Entry; 19 | 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | 23 | import com.ning.http.client.AsyncHttpClient.BoundRequestBuilder; 24 | 25 | 26 | /** 27 | * Adding http header logic TODO; should finally be data driven and specific for 28 | * each command. Now it is defined here and user can easily change. 29 | * 30 | * @author Yuanteng (Jeff) Pei 31 | * 32 | */ 33 | public class PcHttpUtils { 34 | 35 | /** The logger. */ 36 | private static Logger logger = LoggerFactory.getLogger(PcHttpUtils.class); 37 | 38 | /** 39 | * currently not to use MalformedURLException or MalformedURLException as 40 | * our logic includes add HTTP :// etc Assuming not null . 41 | * 42 | * @param url 43 | * the url 44 | * @return true, if is url valid 45 | */ 46 | public static boolean isUrlValid(String url) { 47 | 48 | return (!url.trim().contains(" ")); 49 | 50 | } 51 | 52 | /** 53 | * !!!! ASSUMPTION: all VAR exists in HTTP Header must of type: 54 | * APIVARREPLACE_NAME_PREFIX_HTTP_HEADER 55 | * 56 | * 20140310 This may be costly (O(n^2)) of the updated related # of headers; 57 | * # of parameters in the requests. 58 | * 59 | * Better to only do it when there are some replacement in the request 60 | * Parameters. a prefix : 61 | * 62 | * TOBE tested 63 | * 64 | * @param httpHeaderMap 65 | * the http header map 66 | * @param requestParameters 67 | * the request parameters 68 | */ 69 | 70 | public static void replaceHttpHeaderMapNodeSpecific( 71 | Map httpHeaderMap, 72 | Map requestParameters) { 73 | 74 | boolean needToReplaceVarInHttpHeader = false; 75 | for (String parameter : requestParameters.keySet()) { 76 | if (parameter.contains(PcConstants.NODE_REQUEST_PREFIX_REPLACE_VAR)) { 77 | needToReplaceVarInHttpHeader = true; 78 | break; 79 | } 80 | } 81 | 82 | if (!needToReplaceVarInHttpHeader) { 83 | logger.debug("No need to replace. Since there are no HTTP header variables. "); 84 | return; 85 | } 86 | // replace all the values in the (not the keys) in the header map. 87 | for (Entry entry : httpHeaderMap.entrySet()) { 88 | String key = entry.getKey(); 89 | String valueOriginal = entry.getValue(); 90 | String valueUpdated = NodeReqResponse.replaceStrByMap( 91 | requestParameters, valueOriginal); 92 | httpHeaderMap.put(key, valueUpdated); 93 | } 94 | } 95 | 96 | /** 97 | * Adds the headers. 98 | * 99 | * @param builder 100 | * the builder 101 | * @param headerMap 102 | * the header map 103 | */ 104 | public static void addHeaders(BoundRequestBuilder builder, 105 | Map headerMap) { 106 | for (Entry entry : headerMap.entrySet()) { 107 | String name = entry.getKey(); 108 | String value = entry.getValue(); 109 | builder.addHeader(name, value); 110 | } 111 | 112 | } 113 | 114 | } 115 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/util/PcNumberUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.util; 14 | 15 | import java.text.DecimalFormat; 16 | 17 | /** 18 | * The Class PcNumberUtils. 19 | * @author ypei 20 | */ 21 | public class PcNumberUtils { 22 | 23 | /** 24 | * Gets the string from double. 25 | * 26 | * @param number the number 27 | * @return the string from double 28 | */ 29 | public static String getStringFromDouble(double number) { 30 | DecimalFormat df = new DecimalFormat("#.####"); 31 | return df.format(number); 32 | } 33 | 34 | } 35 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/util/PcStringUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.util; 14 | 15 | import java.util.LinkedHashSet; 16 | import java.util.Map; 17 | import java.util.Map.Entry; 18 | 19 | import org.apache.commons.lang3.exception.ExceptionUtils; 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | 23 | import com.google.gson.Gson; 24 | import com.google.gson.GsonBuilder; 25 | 26 | /** 27 | * The Class PcStringUtils. 28 | */ 29 | public class PcStringUtils { 30 | 31 | /** The logger. */ 32 | @SuppressWarnings("unused") 33 | private static Logger logger = LoggerFactory.getLogger(PcStringUtils.class); 34 | 35 | /** 36 | * Prints the stack trace. 37 | * 38 | * @param t 39 | * the throwable 40 | * @return the string 41 | */ 42 | public static String printStackTrace(Throwable t) { 43 | return t == null ? PcConstants.NA : ExceptionUtils.getStackTrace(t); 44 | 45 | } 46 | 47 | /** 48 | * Str map to str. 49 | * 50 | * @param map 51 | * the map 52 | * @return the string 53 | */ 54 | public static String strMapToStr(Map map) { 55 | 56 | StringBuilder sb = new StringBuilder(); 57 | 58 | if (map == null || map.isEmpty()) 59 | return sb.toString(); 60 | 61 | for (Entry entry : map.entrySet()) { 62 | 63 | sb.append("< " + entry.getKey() + ", " + entry.getValue() + "> "); 64 | } 65 | return sb.toString(); 66 | 67 | } 68 | 69 | /** 70 | * Get the aggregated result human readable string for easy display. 71 | * 72 | * 73 | * @param aggregateResultMap the aggregate result map 74 | * @return the aggregated result human 75 | */ 76 | public static String getAggregatedResultHuman(Map> aggregateResultMap){ 77 | 78 | StringBuilder res = new StringBuilder(); 79 | 80 | for (Entry> entry : aggregateResultMap 81 | .entrySet()) { 82 | LinkedHashSet valueSet = entry.getValue(); 83 | res.append("[" + entry.getKey() + " COUNT: " +valueSet.size() + " ]:\n"); 84 | for(String str: valueSet){ 85 | res.append("\t" + str + "\n"); 86 | } 87 | res.append("###################################\n\n"); 88 | } 89 | 90 | return res.toString(); 91 | 92 | } 93 | 94 | /** 95 | * Render json. 96 | * 97 | * @param o 98 | * the o 99 | * @return the string 100 | */ 101 | public static String renderJson(Object o) { 102 | 103 | Gson gson = new GsonBuilder().disableHtmlEscaping().setPrettyPrinting() 104 | .create(); 105 | return gson.toJson(o); 106 | } 107 | 108 | } 109 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/util/PcTargetHostsUtils.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.util; 14 | 15 | import java.util.ArrayList; 16 | import java.util.HashSet; 17 | import java.util.LinkedHashSet; 18 | import java.util.List; 19 | 20 | import org.slf4j.Logger; 21 | import org.slf4j.LoggerFactory; 22 | 23 | 24 | /** 25 | * The Class PcTargetHostsUtils. 26 | * 27 | * @author Yuanteng (Jeff) Pei 28 | */ 29 | public class PcTargetHostsUtils { 30 | 31 | /** The logger. */ 32 | private static Logger logger = LoggerFactory 33 | .getLogger(PcTargetHostsUtils.class); 34 | 35 | /** 36 | * Gets the node list from string line seperate or space seperate. 37 | * 38 | * @param listStr 39 | * the list str 40 | * @param removeDuplicate 41 | * the remove duplicate 42 | * @return the node list from string line seperate or space seperate 43 | */ 44 | public static List getNodeListFromStringLineSeperateOrSpaceSeperate( 45 | String listStr, boolean removeDuplicate) { 46 | 47 | List nodes = new ArrayList(); 48 | 49 | for (String token : listStr.split("[\\r?\\n| +]+")) { 50 | 51 | // 20131025: fix if fqdn has space in the end. 52 | if (token != null && !token.trim().isEmpty()) { 53 | nodes.add(token.trim()); 54 | 55 | } 56 | } 57 | 58 | if (removeDuplicate) { 59 | removeDuplicateNodeList(nodes); 60 | } 61 | logger.info("Target hosts size : " + nodes.size()); 62 | 63 | return nodes; 64 | 65 | } 66 | 67 | /** 68 | * Removes the duplicate node list. 69 | * 70 | * @param list 71 | * the list 72 | * @return the int 73 | */ 74 | public static int removeDuplicateNodeList(List list) { 75 | 76 | int originCount = list.size(); 77 | // add elements to all, including duplicates 78 | HashSet hs = new LinkedHashSet(); 79 | hs.addAll(list); 80 | list.clear(); 81 | list.addAll(hs); 82 | 83 | return originCount - list.size(); 84 | } 85 | 86 | } 87 | -------------------------------------------------------------------------------- /src/main/java/io/parallec/core/util/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * Utility functions. 3 | */ 4 | package io.parallec.core.util; -------------------------------------------------------------------------------- /src/main/resources/actorconfig.conf: -------------------------------------------------------------------------------- 1 | akka { 2 | log-dead-letters = 0 3 | log-dead-letters-during-shutdown = off 4 | } -------------------------------------------------------------------------------- /src/main/resources/logback.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 11 | 12 | %d{HH:mm:ss.SSS} [%thread] %-5level %logger{5} - %msg%n 13 | 14 | 15 | 16 | 17 | 18 | parallec_logs/parallec.log 19 | 20 | 21 | parallec_logs/parallec.%i.log 22 | 1 23 | 5 24 | 25 | 26 | 27 | 50MB 28 | 29 | 30 | UTF-8 31 | %d %-4relative [%thread] %-5level %logger{35} - %msg%n 32 | 33 | 34 | 35 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/ParallelClientTest.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | import org.apache.http.util.Asserts; 7 | import org.junit.AfterClass; 8 | import org.junit.BeforeClass; 9 | import org.junit.Test; 10 | 11 | public class ParallelClientTest extends TestBase { 12 | private static ParallelClient pc; 13 | 14 | @BeforeClass 15 | public static void setUp() throws Exception { 16 | pc = new ParallelClient(); 17 | } 18 | 19 | @AfterClass 20 | public static void shutdown() throws Exception { 21 | pc.releaseExternalResources(); 22 | } 23 | 24 | @Test 25 | public void testReInit() { 26 | 27 | pc.releaseExternalResources(); 28 | 29 | Map responseContext = new HashMap(); 30 | pc 31 | .prepareHttpGet("/validateInternals.html") 32 | .setConcurrency(1700) 33 | .handleInWorker() 34 | .setTargetHostsFromString( 35 | "www.parallec.io www.jeffpei.com www.restcommander.com") 36 | .execute(new ParallecResponseHandler() { 37 | 38 | @Override 39 | public void onCompleted(ResponseOnSingleTask res, 40 | Map responseContext) { 41 | String cpu = new FilterRegex( 42 | ".*CPU-Usage-Percent\\s*(.*?).*") 43 | .filter(res.getResponseContent()); 44 | String memory = new FilterRegex( 45 | ".*Memory-Used-KB\\s*(.*?).*") 46 | .filter(res.getResponseContent()); 47 | 48 | Map metricMap = new HashMap(); 49 | metricMap.put("CpuUsage", cpu); 50 | metricMap.put("MemoryUsage", memory); 51 | 52 | logger.info("cpu:" + cpu + " memory: " + memory 53 | + " host: " + res.getHost()); 54 | responseContext.put(res.getHost(), cpu); 55 | logger.debug(res.toString()); 56 | 57 | } 58 | }); 59 | 60 | for (Object o : responseContext.values()) { 61 | Double cpuDouble = Double.parseDouble((String) o); 62 | Asserts.check(cpuDouble <= 100.0 && cpuDouble >= 0.0, 63 | " Fail to extract cpu values"); 64 | } 65 | 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/TestBase.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core; 2 | 3 | import io.parallec.core.config.ParallecGlobalConfig; 4 | 5 | import org.slf4j.Logger; 6 | import org.slf4j.LoggerFactory; 7 | 8 | public class TestBase { 9 | public final String FILEPATH_TMP1 = "userdata/target_hosts_tmp1.txt"; 10 | public final String FILEPATH_TMP2 = "userdata/target_hosts_tmp2.txt"; 11 | 12 | public final String FILEPATH_TOP_100 = "userdata/sample_target_hosts_top100_old.txt"; 13 | 14 | public final String FILEPATH_TOP_100_NEW = "userdata/sample_target_hosts_top100.txt"; 15 | 16 | public final String FILEPATH_TOP_500 = "userdata/sample_target_hosts_top500_old.txt"; 17 | public final String FILEPATH_TOP_500_NEW = "userdata/sample_target_hosts_top500.txt"; 18 | 19 | public final String FILEPATH_TOP_1000 = "userdata/sample_target_hosts_top1000.txt"; 20 | public final String FILEPATH_TOP_2000 = "userdata/sample_target_hosts_top2000.txt"; 21 | public final String FILEPATH_TOP_10000 = "userdata/sample_target_hosts_top10k.txt"; 22 | 23 | public final String FILEPATH_JSON_PATH = "userdata/sample_target_hosts_json_path.json"; 24 | 25 | public final String URL_TOP_100 = "http://www.restcommander.com/docs/sample_target_hosts_top100.txt"; 26 | public final String URL_JSON_PATH = "http://www.parallec.io/userdata/sample_target_hosts_json_path.json"; 27 | public final String URL_CMS_QUERY_SINGLE_PAGE = "http://www.parallec.io/cms/repositories/cmsdb/branches/main/query/sample_cms_query_results_single_page.json"; 28 | public final String URL_CMS_QUERY_MULTI_PAGE = "http://www.parallec.io/cms/repositories/cmsdb/branches/main/query/sample_cms_query_results_multi_page_1.json"; 29 | 30 | public final String URL_CMS_QUERY_SINGLE_PAGE_DUPLICATE = "http://www.parallec.io/cms/repositories/cmsdb/branches/main/query/sample_cms_query_results_single_page_duplicate.json"; 31 | public final String URL_CMS_QUERY_SINGLE_PAGE_EMPTY = "http://www.parallec.io/cms/repositories/cmsdb/branches/main/query/sample_cms_query_results_single_page_empty.json"; 32 | 33 | 34 | 35 | public final HostsSourceType SOURCE_LOCAL = HostsSourceType.LOCAL_FILE; 36 | public final HostsSourceType SOURCE_URL = HostsSourceType.URL; 37 | 38 | // ssh 39 | 40 | public final static String userName = "someUser"; 41 | public final static int PORT_DEFAULT = 22; 42 | public final static String passwd = "mypassword"; 43 | public final static String hostIpSample = "192.168.1.155"; 44 | public final static String hostIpSample2 = "192.168.1.156"; 45 | public final static String commandSshLineValid = "date"; 46 | 47 | public final static String LOCALHOST="localhost"; 48 | 49 | public final static int sshConnectionTimeoutMillis = ParallecGlobalConfig.sshConnectionTimeoutMillisDefault; 50 | 51 | protected static final Logger logger = LoggerFactory 52 | .getLogger(TestBase.class); 53 | } 54 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/actor/OperationWorkerTest.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.actor; 2 | 3 | import io.parallec.core.ParallelClient; 4 | import io.parallec.core.RequestProtocol; 5 | import io.parallec.core.TestBase; 6 | import io.parallec.core.actor.message.type.RequestWorkerMsgType; 7 | import io.parallec.core.bean.TaskRequest; 8 | import io.parallec.core.resources.HttpClientStore; 9 | import io.parallec.core.resources.HttpMethod; 10 | 11 | import java.util.HashMap; 12 | 13 | import org.junit.AfterClass; 14 | import org.junit.BeforeClass; 15 | import org.junit.Test; 16 | 17 | import akka.actor.ActorRef; 18 | import akka.actor.Props; 19 | 20 | public class OperationWorkerTest extends TestBase { 21 | 22 | private static ParallelClient pc; 23 | 24 | @BeforeClass 25 | public static void setUp() throws Exception { 26 | pc = new ParallelClient(); 27 | } 28 | 29 | @AfterClass 30 | public static void shutdown() throws Exception { 31 | pc.releaseExternalResources(); 32 | } 33 | 34 | /** 35 | * a request. expected to 36 | */ 37 | @Test 38 | public void testOperationWorkerWrongMsgType() { 39 | ActorRef asyncWorker = null; 40 | try { 41 | // Start new job 42 | 43 | 44 | int actorMaxOperationTimeoutSec = 15; 45 | 46 | asyncWorker = ActorConfig.createAndGetActorSystem().actorOf( 47 | Props.create( 48 | OperationWorker.class, 49 | new TaskRequest(actorMaxOperationTimeoutSec, 50 | RequestProtocol.valueOf("HTTP" 51 | .toUpperCase()), 52 | "www.parallec.io", null, 80, "", "", 53 | HttpMethod.GET, false, null, 54 | null, 55 | new HashMap(), 56 | null,null, null,null,null), HttpClientStore 57 | .getInstance().getEmbedClientFast(), null 58 | 59 | )); 60 | // bad type 61 | asyncWorker.tell(RequestWorkerMsgType.CANCEL, asyncWorker); 62 | 63 | } catch (Throwable ex) { 64 | logger.error("Exception in test : " + ex); 65 | ex.printStackTrace(); 66 | } 67 | }// end func 68 | 69 | } 70 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/bean/NodeReqResponseTest.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.bean; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | import org.junit.Test; 7 | 8 | import io.parallec.core.TestBase; 9 | import io.parallec.core.actor.message.NodeReqResponse; 10 | 11 | public class NodeReqResponseTest extends TestBase { 12 | 13 | @Test 14 | public void testReplaceVar() { 15 | Map requestParameters = new HashMap(); 16 | requestParameters.put("REPLACE-VAR_AGENT_VERSION", "0.1.911"); 17 | 18 | requestParameters.put("REPLACE-VAR_updateWisb", "WWWWWWWII"); 19 | 20 | // String sourceContent = 21 | // "{\"manifest\": \"agent_selfupdate-$AGENT_VERSION\", \"updateWisb\":\"True\"}"; 22 | String sourceContent = "{\"manifest\": \"agent_selfupdate-$AGENT_VERSION\", \"$updateWisb\":\"True\"}"; 23 | 24 | String afterReplacement = NodeReqResponse.replaceStrByMap( 25 | requestParameters, sourceContent); 26 | 27 | logger.info(afterReplacement); 28 | } 29 | 30 | } 31 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/client/ParallelTaskTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.client; 14 | 15 | import io.parallec.core.ParallelClient; 16 | import io.parallec.core.ParallelTask; 17 | import io.parallec.core.TestBase; 18 | import io.parallec.core.actor.ExecutionManagerTest; 19 | import io.parallec.core.exception.ParallelTaskInvalidException; 20 | import io.parallec.core.resources.HttpClientStore; 21 | import io.parallec.core.task.ParallelTaskState; 22 | 23 | import org.junit.AfterClass; 24 | import org.junit.BeforeClass; 25 | import org.junit.Test; 26 | 27 | /** 28 | * The most basic test with hitting the same URL at 3 different websites. 29 | * require Internet access for testing. 30 | * 31 | *

32 | * This example shows 1. Basic request construction 2. how to use response 33 | * context to pass value during the response handler out to a global space 34 | *

35 | */ 36 | public class ParallelTaskTest extends TestBase { 37 | 38 | /** The pc. */ 39 | private static ParallelClient pc; 40 | 41 | /** 42 | * Sets the up. 43 | * 44 | * @throws Exception 45 | * the exception 46 | */ 47 | @BeforeClass 48 | public static void setUp() throws Exception { 49 | pc = new ParallelClient(); 50 | } 51 | 52 | /** 53 | * Shutdown. 54 | * 55 | * @throws Exception 56 | * the exception 57 | */ 58 | @AfterClass 59 | public static void shutdown() throws Exception { 60 | pc.releaseExternalResources(); 61 | } 62 | 63 | @Test 64 | public void testCancelException() { 65 | ParallelTask task = ExecutionManagerTest.genParallelTask(); 66 | task.setState(null); 67 | task.cancel(true); 68 | } 69 | 70 | @Test 71 | public void testGetProgress() { 72 | ParallelTask task = ExecutionManagerTest.genParallelTask(); 73 | task.setState(ParallelTaskState.IN_PROGRESS); 74 | task.setRequestNum(0); 75 | logger.info("progress: {}",task.getProgress()); 76 | task.setState(ParallelTaskState.COMPLETED_WITHOUT_ERROR); 77 | logger.info("progress: {}",task.getProgress()); 78 | 79 | } 80 | 81 | 82 | @Test 83 | public void testCancelNullManager() { 84 | ParallelTask task = ExecutionManagerTest.genParallelTask(); 85 | task.setState(ParallelTaskState.IN_PROGRESS); 86 | task.executionManager = null; 87 | task.cancel(true); 88 | } 89 | 90 | 91 | @Test 92 | public void testValidation() { 93 | ParallelTask task = ExecutionManagerTest.genParallelTask(); 94 | logger.info(task.toString()); 95 | task.setConfig(null); 96 | task.setAsyncHttpClient( HttpClientStore.getInstance() 97 | .getCurrentDefaultClient()); 98 | task.getHttpMeta().setHttpMethod(null); 99 | try { 100 | 101 | task.validateWithFillDefault(); 102 | } catch (ParallelTaskInvalidException e) { 103 | 104 | logger.info("EXPECTED Exception {}", e.getLocalizedMessage()); 105 | } 106 | } 107 | 108 | @Test 109 | public void testValidationPoller() { 110 | ParallelTask task = ExecutionManagerTest.genParallelTask(); 111 | task.getHttpMeta().setRequestUrlPostfix(null); 112 | task.getHttpMeta().setPollable(true); 113 | task.getHttpMeta().setHttpPollerProcessor(null); 114 | try { 115 | task.validateWithFillDefault(); 116 | } catch (ParallelTaskInvalidException e) { 117 | 118 | logger.info("EXPECTED Exception {}", e.getLocalizedMessage()); 119 | } 120 | } 121 | 122 | } 123 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/commander/workflow/VarReplacementProviderTest.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.commander.workflow; 2 | 3 | import io.parallec.core.FilterRegex; 4 | import io.parallec.core.ParallecResponseHandler; 5 | import io.parallec.core.ParallelClient; 6 | import io.parallec.core.ResponseOnSingleTask; 7 | import io.parallec.core.TestBase; 8 | import io.parallec.core.bean.StrStrMap; 9 | 10 | import java.util.HashMap; 11 | import java.util.Map; 12 | 13 | import org.junit.AfterClass; 14 | import org.junit.BeforeClass; 15 | import org.junit.Test; 16 | 17 | public class VarReplacementProviderTest extends TestBase { 18 | 19 | 20 | private static ParallelClient pc; 21 | 22 | @BeforeClass 23 | public static void setUp() throws Exception { 24 | pc = new ParallelClient(); 25 | } 26 | 27 | @AfterClass 28 | public static void shutdown() throws Exception { 29 | pc.releaseExternalResources(); 30 | } 31 | 32 | @Test 33 | public void testVarReplacementProviderWithNullMap() { 34 | 35 | Map replacementVarMapNodeSpecific = new HashMap(); 36 | replacementVarMapNodeSpecific.put("www.jeffpei.com", 37 | new StrStrMap()); 38 | replacementVarMapNodeSpecific.put("www.restcommander.com", 39 | new StrStrMap().addPair("JOB_ID", "job_c")); 40 | 41 | pc.prepareHttpGet("/$JOB_ID.html") 42 | .setConcurrency(1700) 43 | .setTargetHostsFromString( 44 | "www.parallec.io www.jeffpei.com www.restcommander.com") 45 | .setReplacementVarMapNodeSpecific(replacementVarMapNodeSpecific) 46 | .execute(new ParallecResponseHandler() { 47 | @Override 48 | public void onCompleted(ResponseOnSingleTask res, 49 | Map responseContext) { 50 | String extractedString = new FilterRegex( 51 | ".*JobProgress\\s*(.*?)[\\s\\S]*") 52 | .filter(res.getResponseContent()); 53 | logger.info("ExtracedString: progress:" 54 | + extractedString + " host: " + res.getHost()); 55 | logger.debug(res.toString()); 56 | } 57 | }); 58 | 59 | } 60 | } 61 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/filter/FilterRegexTest.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.filter; 2 | 3 | import io.parallec.core.FilterRegex; 4 | import io.parallec.core.TestBase; 5 | import io.parallec.core.util.PcConstants; 6 | 7 | import java.util.regex.Matcher; 8 | import java.util.regex.Pattern; 9 | 10 | import org.junit.Test; 11 | 12 | public class FilterRegexTest extends TestBase { 13 | 14 | @Test 15 | public void testFilterRegex() { 16 | String completeRegex = ".*\"progress\"\\s*:\\s*(100).*}"; 17 | FilterRegex.stringMatcherByPattern(null, completeRegex); 18 | 19 | FilterRegex.stringMatcherByPattern(completeRegex, null); 20 | } 21 | 22 | @Test 23 | public void testRegex() { 24 | 25 | String completeRegex = ".*\"progress\"\\s*:\\s*(100).*}"; 26 | Pattern patternMetric = Pattern.compile(completeRegex, 27 | Pattern.MULTILINE); 28 | String response = "{\"status\": \"/status/e40c0f1e-ddc2-4987-aaa7-b638a9978782\", \"progress\": 100, \"error\": 300}"; 29 | 30 | final Matcher matcher = patternMetric.matcher(response); 31 | String matchStr = PcConstants.NA; 32 | if (matcher.matches()) { 33 | matchStr = matcher.group(1); 34 | } 35 | logger.info(matchStr + ""); 36 | } 37 | 38 | } 39 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/httpclient/async/HttpClientStoreTest.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.httpclient.async; 2 | 3 | import io.parallec.core.ParallelClient; 4 | import io.parallec.core.TestBase; 5 | import io.parallec.core.resources.HttpClientStore; 6 | import io.parallec.core.resources.HttpClientType; 7 | 8 | import org.junit.AfterClass; 9 | import org.junit.BeforeClass; 10 | import org.junit.Test; 11 | 12 | public class HttpClientStoreTest extends TestBase { 13 | 14 | private static ParallelClient pc; 15 | 16 | @BeforeClass 17 | public static void setUp() throws Exception { 18 | pc = new ParallelClient(); 19 | } 20 | 21 | @AfterClass 22 | public static void shutdown() throws Exception { 23 | pc.releaseExternalResources(); 24 | } 25 | 26 | @Test 27 | public void testDirectorForException() { 28 | try { 29 | HttpClientStore.getInstance().init(); 30 | 31 | HttpClientStore.getInstance().getClientByType(HttpClientType.EMBED_FAST); 32 | HttpClientStore.getInstance().getCustomClientFast(); 33 | HttpClientStore.getInstance().getCustomClientSlow(); 34 | HttpClientStore.getInstance().getEmbedClientFast(); 35 | HttpClientStore.getInstance().getEmbedClientSlow(); 36 | HttpClientStore.getInstance().setHttpClientTypeCurrentDefault(HttpClientType.CUSTOM_FAST); 37 | } catch (Exception ex) { 38 | logger.error("Expected Exception : " + ex); 39 | } 40 | }// end func 41 | 42 | } 43 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/http/LoadResourceTest.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.main.http; 2 | 3 | import io.parallec.core.ParallelClient; 4 | import io.parallec.core.TestBase; 5 | 6 | import java.io.IOException; 7 | import java.io.InputStreamReader; 8 | import java.io.Reader; 9 | import java.io.StringWriter; 10 | 11 | import org.junit.AfterClass; 12 | import org.junit.BeforeClass; 13 | import org.junit.Test; 14 | 15 | public class LoadResourceTest extends TestBase { 16 | 17 | private static ParallelClient pc; 18 | 19 | @BeforeClass 20 | public static void setUp() throws Exception { 21 | pc = new ParallelClient(); 22 | } 23 | 24 | @AfterClass 25 | public static void shutdown() throws Exception { 26 | pc.releaseExternalResources(); 27 | } 28 | 29 | @Test 30 | public void testLoadResource() { 31 | 32 | Reader r = new InputStreamReader(getClass().getClassLoader() 33 | .getResourceAsStream("actorconfig.conf")); 34 | StringWriter sw = new StringWriter(); 35 | char[] buffer = new char[1024]; 36 | try { 37 | for (int n; (n = r.read(buffer)) != -1;) 38 | sw.write(buffer, 0, n); 39 | } catch (IOException e) { 40 | e.printStackTrace(); 41 | } 42 | String str = sw.toString(); 43 | System.out.println(str); 44 | } 45 | 46 | 47 | } 48 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/http/ParallelClientBeanTest.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.main.http; 2 | 3 | import io.parallec.core.ParallelClient; 4 | import io.parallec.core.TestBase; 5 | import io.parallec.core.resources.HttpClientType; 6 | 7 | import org.junit.AfterClass; 8 | import org.junit.BeforeClass; 9 | import org.junit.Test; 10 | 11 | public class ParallelClientBeanTest extends TestBase { 12 | 13 | private static ParallelClient pc; 14 | 15 | @BeforeClass 16 | public static void setUp() throws Exception { 17 | pc = new ParallelClient(); 18 | } 19 | 20 | @AfterClass 21 | public static void shutdown() throws Exception { 22 | pc.releaseExternalResources(); 23 | } 24 | 25 | @Test 26 | public void parallecClient() { 27 | pc.reinitHttpClients(); 28 | pc.cleanInprogressJobMap(); 29 | pc.cleanWaitTaskQueue(); 30 | pc.setCustomClientFast(null); 31 | pc.setCustomClientSlow(null); 32 | pc.setHttpClientTypeCurrentDefault(HttpClientType.EMBED_FAST); 33 | pc.getRunningJobCount(); 34 | 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/http/ParallelClientHttpBasicTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.main.http; 14 | 15 | import io.parallec.core.FilterRegex; 16 | import io.parallec.core.ParallecResponseHandler; 17 | import io.parallec.core.ParallelClient; 18 | import io.parallec.core.ParallelTask; 19 | import io.parallec.core.ResponseOnSingleTask; 20 | import io.parallec.core.TestBase; 21 | 22 | import java.util.HashMap; 23 | import java.util.Map; 24 | 25 | import org.apache.http.util.Asserts; 26 | import org.junit.AfterClass; 27 | import org.junit.BeforeClass; 28 | import org.junit.Test; 29 | 30 | /** 31 | * The most basic test with hitting the same URL at 3 different websites. 32 | * require Internet access for testing. 33 | * 34 | *

35 | * This example shows 1. Basic request construction 2. how to use response 36 | * context to pass value during the response handler out to a global space 37 | *

38 | */ 39 | public class ParallelClientHttpBasicTest extends TestBase { 40 | 41 | /** The pc. */ 42 | private static ParallelClient pc; 43 | 44 | /** 45 | * Sets the up. 46 | * 47 | * @throws Exception 48 | * the exception 49 | */ 50 | @BeforeClass 51 | public static void setUp() throws Exception { 52 | pc = new ParallelClient(); 53 | } 54 | 55 | /** 56 | * Shutdown. 57 | * 58 | * @throws Exception 59 | * the exception 60 | */ 61 | @AfterClass 62 | public static void shutdown() throws Exception { 63 | pc.releaseExternalResources(); 64 | } 65 | 66 | /** 67 | * Hit websites min sync. 68 | */ 69 | @Test 70 | public void hitWebsitesMinSync() { 71 | 72 | Map responseContext = new HashMap(); 73 | ParallelTask task = pc 74 | .prepareHttpGet("/validateInternals.html") 75 | .setResponseContext(responseContext) 76 | .setConcurrency(1700) 77 | //.handleInWorker() 78 | .setTargetHostsFromString( 79 | "www.parallec.io www.jeffpei.com www.restcommander.com") 80 | .execute(new ParallecResponseHandler() { 81 | 82 | @Override 83 | public void onCompleted(ResponseOnSingleTask res, 84 | Map responseContext) { 85 | String cpu = new FilterRegex( 86 | ".*CPU-Usage-Percent\\s*(.*?).*") 87 | .filter(res.getResponseContent()); 88 | String memory = new FilterRegex( 89 | ".*Memory-Used-KB\\s*(.*?).*") 90 | .filter(res.getResponseContent()); 91 | 92 | Map metricMap = new HashMap(); 93 | metricMap.put("CpuUsage", cpu); 94 | metricMap.put("MemoryUsage", memory); 95 | 96 | logger.info("cpu:" + cpu + " memory: " + memory 97 | + " host: " + res.getHost()); 98 | responseContext.put(res.getHost(), cpu); 99 | logger.debug(res.toString()); 100 | 101 | } 102 | }); 103 | Asserts.check(responseContext.values().size() >= 3, 104 | " Fail to get 3 results"); 105 | for (Object o : responseContext.values()) { 106 | Double cpuDouble = Double.parseDouble((String) o); 107 | Asserts.check(cpuDouble <= 100.0 && cpuDouble >= 0.0, 108 | " Fail to extract cpu values"); 109 | } 110 | logger.info("Task Pretty Print: \n{}", task.prettyPrintInfo()); 111 | } 112 | 113 | } 114 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/http/ParallelClientHttpBasicTimeoutTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.main.http; 14 | 15 | import io.parallec.core.ParallecResponseHandler; 16 | import io.parallec.core.ParallelClient; 17 | import io.parallec.core.ParallelTask; 18 | import io.parallec.core.ResponseOnSingleTask; 19 | import io.parallec.core.TestBase; 20 | import io.parallec.core.config.ParallelTaskConfig; 21 | 22 | import java.util.HashMap; 23 | import java.util.Map; 24 | 25 | import org.apache.http.util.Asserts; 26 | import org.junit.AfterClass; 27 | import org.junit.BeforeClass; 28 | import org.junit.Test; 29 | 30 | /** 31 | * The most basic test with hitting the same URL at 3 different websites. 32 | * require Internet access for testing. 33 | * 34 | *

35 | * This example shows 1. Basic request construction 2. how to use response 36 | * context to pass value during the response handler out to a global space 37 | *

38 | */ 39 | public class ParallelClientHttpBasicTimeoutTest extends TestBase { 40 | 41 | /** The pc. */ 42 | private static ParallelClient pc; 43 | 44 | /** 45 | * Sets the up. 46 | * 47 | * @throws Exception 48 | * the exception 49 | */ 50 | @BeforeClass 51 | public static void setUp() throws Exception { 52 | pc = new ParallelClient(); 53 | } 54 | 55 | /** 56 | * Shutdown. 57 | * 58 | * @throws Exception 59 | * the exception 60 | */ 61 | @AfterClass 62 | public static void shutdown() throws Exception { 63 | pc.releaseExternalResources(); 64 | } 65 | 66 | /** 67 | * Hit websites min sync. 68 | */ 69 | @Test 70 | public void hitWebsitesMinSyncTimeout() { 71 | 72 | ParallelTaskConfig config = new ParallelTaskConfig(); 73 | config.setTimeoutInManagerSec(0); 74 | 75 | Map responseContext = new HashMap(); 76 | ParallelTask task = pc 77 | .prepareHttpGet("/validateInternals.html") 78 | .setConcurrency(1700) 79 | .setResponseContext(responseContext) 80 | .handleInWorker() 81 | .setConfig(config) 82 | .setTargetHostsFromString( 83 | "www.parallec.io www.jeffpei.com www.restcommander.com") 84 | .execute(new ParallecResponseHandler() { 85 | 86 | @Override 87 | public void onCompleted(ResponseOnSingleTask res, 88 | Map responseContext) { 89 | 90 | ; 91 | } 92 | }); 93 | 94 | Asserts.check(task.getParallelTaskResult().get("www.parallec.io") 95 | .getSingleTaskResponse().getErrorMessage().contains("Execution manager timeout"), 96 | " Fail to get 3 results"); 97 | logger.info("Task Pretty Print: \n{}", task.prettyPrintInfo()); 98 | } 99 | 100 | } 101 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/http/ParallelClientHttpInvalidInputTest.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.main.http; 2 | 3 | import io.parallec.core.ParallecResponseHandler; 4 | import io.parallec.core.ParallelClient; 5 | import io.parallec.core.ResponseOnSingleTask; 6 | import io.parallec.core.TestBase; 7 | 8 | import java.util.ArrayList; 9 | import java.util.List; 10 | import java.util.Map; 11 | 12 | import org.junit.AfterClass; 13 | import org.junit.BeforeClass; 14 | import org.junit.Test; 15 | 16 | public class ParallelClientHttpInvalidInputTest extends TestBase { 17 | 18 | private static ParallelClient pc; 19 | 20 | @BeforeClass 21 | public static void setUp() throws Exception { 22 | pc = new ParallelClient(); 23 | } 24 | 25 | @AfterClass 26 | public static void shutdown() throws Exception { 27 | pc.releaseExternalResources(); 28 | } 29 | 30 | @Test 31 | public void hitWebsitesUrlWithValidation() { 32 | 33 | // miss target hosts 34 | 35 | boolean validate = pc.prepareHttpGet(" /validateInternals.html ") 36 | .setConcurrency(1700) 37 | // .setTargetHostsFromString("www.parallec.io www.jeffpei.com www.restcommander.com") 38 | .validation(); 39 | System.out.println("validation: " + validate); 40 | 41 | boolean validate2 = pc 42 | .prepareHttpGet(" /validateInternals.html ") 43 | .setConcurrency(1700) 44 | .async() 45 | .sync() 46 | .setTargetHostsFromString( 47 | "www.parallec.io www.jeffpei.com www.restcommander.com") 48 | .validation(); 49 | System.out.println("validation2: " + validate2); 50 | } 51 | 52 | @Test 53 | public void hitWebsitesUrlEndingWSInput() { 54 | 55 | // hitWebsitesUrlEndingWSInput 56 | pc.prepareHttpGet(" /validateInternals.html ") 57 | .setConcurrency(1700) 58 | .setTargetHostsFromString( 59 | "www.parallec.io www.jeffpei.com www.restcommander.com") 60 | .execute(new ParallecResponseHandler() { 61 | 62 | @Override 63 | public void onCompleted(ResponseOnSingleTask res, 64 | Map responseContext) { 65 | logger.info(res.toString()); 66 | } 67 | }); 68 | 69 | } 70 | 71 | @Test 72 | public void hitWebsitesInvalidInput() { 73 | 74 | // miss target hosts 75 | pc.prepareHttpGet("/validateInternals.html").setConcurrency(1700) 76 | 77 | .execute(new ParallecResponseHandler() { 78 | 79 | @Override 80 | public void onCompleted(ResponseOnSingleTask res, 81 | Map responseContext) { 82 | } 83 | }); 84 | 85 | // space in between URL 86 | pc.prepareHttpGet("/validate Internals.html") 87 | .setConcurrency(1700) 88 | .setTargetHostsFromString( 89 | "www.restcommander.com www.jeffpei.com") 90 | .execute(new ParallecResponseHandler() { 91 | 92 | @Override 93 | public void onCompleted(ResponseOnSingleTask res, 94 | Map responseContext) { 95 | } 96 | }); 97 | 98 | // space in between single target host 99 | 100 | List badHostList = new ArrayList(); 101 | badHostList.add("www.restco mmander.com"); 102 | pc.prepareHttpGet("/validateInternals.html").setConcurrency(1700) 103 | .setTargetHostsFromList(badHostList) 104 | .execute(new ParallecResponseHandler() { 105 | 106 | @Override 107 | public void onCompleted(ResponseOnSingleTask res, 108 | Map responseContext) { 109 | } 110 | }); 111 | 112 | } 113 | 114 | } 115 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/http/ParallelClientHttpLogTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.main.http; 14 | 15 | import io.parallec.core.FilterRegex; 16 | import io.parallec.core.ParallecResponseHandler; 17 | import io.parallec.core.ParallelClient; 18 | import io.parallec.core.ParallelTask; 19 | import io.parallec.core.ResponseOnSingleTask; 20 | import io.parallec.core.TestBase; 21 | import io.parallec.core.util.PcStringUtils; 22 | 23 | import java.util.HashMap; 24 | import java.util.Map; 25 | 26 | import org.apache.http.util.Asserts; 27 | import org.junit.AfterClass; 28 | import org.junit.BeforeClass; 29 | import org.junit.Test; 30 | 31 | /** 32 | * The most basic test with hitting the same URL at 3 different websites. 33 | * require Internet access for testing. 34 | * 35 | *

36 | * This example shows 1. Basic request construction 2. how to use response 37 | * context to pass value during the response handler out to a global space 38 | *

39 | */ 40 | public class ParallelClientHttpLogTest extends TestBase { 41 | 42 | /** The pc. */ 43 | private static ParallelClient pc; 44 | 45 | /** 46 | * Sets the up. 47 | * 48 | * @throws Exception 49 | * the exception 50 | */ 51 | @BeforeClass 52 | public static void setUp() throws Exception { 53 | pc = new ParallelClient(); 54 | } 55 | 56 | /** 57 | * Shutdown. 58 | * 59 | * @throws Exception 60 | * the exception 61 | */ 62 | @AfterClass 63 | public static void shutdown() throws Exception { 64 | pc.releaseExternalResources(); 65 | } 66 | 67 | /** 68 | * Hit websites min sync. 69 | */ 70 | @Test 71 | public void hitWebsitesMinSync() { 72 | Map responseContext = new HashMap(); 73 | ParallelTask pt = pc 74 | .prepareHttpGet("/validateInternals.html") 75 | .setConcurrency(1700) 76 | .setTargetHostsFromString( 77 | "www.parallec.io www.jeffpei.com www.restcommander.com") 78 | .execute(new ParallecResponseHandler() { 79 | 80 | @Override 81 | public void onCompleted(ResponseOnSingleTask res, 82 | Map responseContext) { 83 | String cpu = new FilterRegex( 84 | ".*CPU-Usage-Percent\\s*(.*?).*") 85 | .filter(res.getResponseContent()); 86 | String memory = new FilterRegex( 87 | ".*Memory-Used-KB\\s*(.*?).*") 88 | .filter(res.getResponseContent()); 89 | 90 | Map metricMap = new HashMap(); 91 | metricMap.put("CpuUsage", cpu); 92 | metricMap.put("MemoryUsage", memory); 93 | 94 | logger.info("cpu:" + cpu + " memory: " + memory 95 | + " host: " + res.getHost()); 96 | responseContext.put(res.getHost(), cpu); 97 | 98 | // logger.info(res.toString()); 99 | 100 | } 101 | }); 102 | logger.info("Response details: " 103 | + PcStringUtils.renderJson(pt.getParallelTaskResult())); 104 | for (Object o : responseContext.values()) { 105 | 106 | Double cpuDouble = Double.parseDouble((String) o); 107 | Asserts.check(cpuDouble <= 100.0 && cpuDouble >= 0.0, 108 | " Fail to extract cpu values"); 109 | } 110 | 111 | } 112 | 113 | } 114 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/http/ParallelClientHttpOtherMethodsTest.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.main.http; 2 | 3 | import io.parallec.core.ParallecResponseHandler; 4 | import io.parallec.core.ParallelClient; 5 | import io.parallec.core.ResponseOnSingleTask; 6 | import io.parallec.core.TestBase; 7 | 8 | import java.util.Map; 9 | 10 | import org.junit.AfterClass; 11 | import org.junit.BeforeClass; 12 | import org.junit.Test; 13 | 14 | public class ParallelClientHttpOtherMethodsTest extends TestBase { 15 | 16 | private static ParallelClient pc; 17 | 18 | @BeforeClass 19 | public static void setUp() throws Exception { 20 | pc = new ParallelClient(); 21 | } 22 | 23 | @AfterClass 24 | public static void shutdown() throws Exception { 25 | pc.releaseExternalResources(); 26 | } 27 | 28 | @Test 29 | public void hitWebsitesInvalidHttpMethods() { 30 | 31 | pc.prepareHttpPut("/validateInternals.html") 32 | .setConcurrency(1700) 33 | .setTargetHostsFromString( 34 | "www.parallec.io www.jeffpei.com www.restcommander.com") 35 | .execute(new ParallecResponseHandler() { 36 | @Override 37 | public void onCompleted(ResponseOnSingleTask res, 38 | Map responseContext) { 39 | logger.info("expected wrong HTTP methods {}", 40 | res.toString()); 41 | } 42 | }); 43 | 44 | pc.prepareHttpDelete("/validateInternals.html") 45 | .setConcurrency(1700) 46 | .setTargetHostsFromString( 47 | "www.parallec.io www.jeffpei.com www.restcommander.com") 48 | .execute(new ParallecResponseHandler() { 49 | @Override 50 | public void onCompleted(ResponseOnSingleTask res, 51 | Map responseContext) { 52 | logger.info("expected wrong HTTP methods {}", 53 | res.toString()); 54 | } 55 | }); 56 | 57 | pc.prepareHttpHead("/validateInternals.html") 58 | .setConcurrency(1700) 59 | .setTargetHostsFromString( 60 | "www.parallec.io www.jeffpei.com www.restcommander.com") 61 | .execute(new ParallecResponseHandler() { 62 | @Override 63 | public void onCompleted(ResponseOnSingleTask res, 64 | Map responseContext) { 65 | logger.info("expected wrong HTTP methods {}", 66 | res.toString()); 67 | } 68 | }); 69 | 70 | pc.prepareHttpOptions("/validateInternals.html") 71 | .setConcurrency(1700) 72 | .setTargetHostsFromString( 73 | "www.parallec.io www.jeffpei.com www.restcommander.com") 74 | .execute(new ParallecResponseHandler() { 75 | @Override 76 | public void onCompleted(ResponseOnSingleTask res, 77 | Map responseContext) { 78 | logger.info("expected wrong HTTP methods {}", 79 | res.toString()); 80 | } 81 | }); 82 | 83 | }// end func 84 | 85 | } 86 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/http/ParallelClientHttpSmallConcurrencyTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.main.http; 14 | 15 | import io.parallec.core.ParallecResponseHandler; 16 | import io.parallec.core.ParallelClient; 17 | import io.parallec.core.ResponseOnSingleTask; 18 | import io.parallec.core.TestBase; 19 | 20 | import java.util.Map; 21 | 22 | import org.junit.AfterClass; 23 | import org.junit.BeforeClass; 24 | import org.junit.Test; 25 | 26 | /** 27 | * The most basic test with hitting the same URL at 3 different websites. 28 | * require Internet access for testing. 29 | * 30 | *

31 | * This example shows 1. Basic request construction 2. how to use response 32 | * context to pass value during the response handler out to a global space 33 | *

34 | */ 35 | public class ParallelClientHttpSmallConcurrencyTest extends TestBase { 36 | 37 | /** The pc. */ 38 | private static ParallelClient pc; 39 | 40 | /** 41 | * Sets the up. 42 | * 43 | * @throws Exception 44 | * the exception 45 | */ 46 | @BeforeClass 47 | public static void setUp() throws Exception { 48 | pc = new ParallelClient(); 49 | } 50 | 51 | /** 52 | * Shutdown. 53 | * 54 | * @throws Exception 55 | * the exception 56 | */ 57 | @AfterClass 58 | public static void shutdown() throws Exception { 59 | pc.releaseExternalResources(); 60 | } 61 | 62 | /** 63 | * Hit websites min sync. 64 | */ 65 | @Test 66 | public void hitWebsitesMinSync() { 67 | pc.prepareHttpGet("") 68 | .setConcurrency(10) 69 | .setTargetHostsFromLineByLineText(FILEPATH_TOP_100, 70 | SOURCE_LOCAL).execute(new ParallecResponseHandler() { 71 | 72 | @Override 73 | public void onCompleted(ResponseOnSingleTask res, 74 | Map responseContext) { 75 | logger.info(res.getStatusCode()); 76 | 77 | } 78 | }); 79 | 80 | } 81 | 82 | } 83 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/http/ParallelClientHttpTop100WebsiteTest.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.main.http; 2 | 3 | import io.parallec.core.ParallecResponseHandler; 4 | import io.parallec.core.ParallelClient; 5 | import io.parallec.core.ParallelTask; 6 | import io.parallec.core.ResponseOnSingleTask; 7 | import io.parallec.core.TestBase; 8 | import io.parallec.core.util.PcFileNetworkIoUtils; 9 | import io.parallec.core.util.PcStringUtils; 10 | 11 | import java.net.InetAddress; 12 | import java.net.UnknownHostException; 13 | import java.util.ArrayList; 14 | import java.util.List; 15 | import java.util.Map; 16 | 17 | import org.junit.AfterClass; 18 | import org.junit.BeforeClass; 19 | import org.junit.Ignore; 20 | import org.junit.Test; 21 | 22 | public class ParallelClientHttpTop100WebsiteTest extends TestBase { 23 | 24 | private static ParallelClient pc; 25 | 26 | @BeforeClass 27 | public static void setUp() throws Exception { 28 | pc = new ParallelClient(); 29 | } 30 | 31 | @AfterClass 32 | public static void shutdown() throws Exception { 33 | pc.releaseExternalResources(); 34 | } 35 | 36 | 37 | @Test 38 | public void hitTop100WebsitesMin() { 39 | 40 | ParallelTask pt = pc 41 | .prepareHttpGet("") 42 | .setConcurrency(1000) 43 | .setSaveResponseToTask(true) 44 | .setTargetHostsFromLineByLineText(FILEPATH_TOP_100, 45 | SOURCE_LOCAL).execute(new ParallecResponseHandler() { 46 | 47 | @Override 48 | public void onCompleted(ResponseOnSingleTask res, 49 | Map responseContext) { 50 | logger.info("Responose Code:" + res.getStatusCode() 51 | + " host: " + res.getHost()); 52 | } 53 | }); 54 | logger.info("Result Summary\n{}", 55 | PcStringUtils.renderJson(pt.getAggregateResultFullSummary())); 56 | 57 | pt.saveLogToLocal(); 58 | } 59 | 60 | @Ignore 61 | // @Test 62 | public void hitTop100WebsitesViaIps() { 63 | 64 | List hostNames = PcFileNetworkIoUtils 65 | .getListFromLineByLineText(FILEPATH_TOP_100, SOURCE_LOCAL); 66 | 67 | List hostIps = new ArrayList(); 68 | int count = 0; 69 | for (String hostName : hostNames) { 70 | logger.info("get ip for host # {}", ++count); 71 | InetAddress address; 72 | try { 73 | address = InetAddress.getByName(hostName); 74 | hostIps.add(address.getHostAddress()); 75 | } catch (UnknownHostException e) { 76 | e.printStackTrace(); 77 | } 78 | } 79 | 80 | ParallelTask pt = pc.prepareHttpGet("").setConcurrency(1000) 81 | .setTargetHostsFromList(hostIps) 82 | // .setTargetHostsFromLineByLineText(FILEPATH_TOP_100, 83 | // SOURCE_LOCAL) 84 | .execute(new ParallecResponseHandler() { 85 | 86 | @Override 87 | public void onCompleted(ResponseOnSingleTask res, 88 | Map responseContext) { 89 | logger.info("Responose Code:" + res.getStatusCode() 90 | + " host: " + res.getHost()); 91 | } 92 | }); 93 | logger.info("Result Summary\n{}", 94 | PcStringUtils.renderJson(pt.getAggregateResultCountSummary())); 95 | 96 | } 97 | 98 | } 99 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/http/ParallelClientHttpTop100WebsiteTimeoutTest.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.main.http; 2 | 3 | import io.parallec.core.ParallecResponseHandler; 4 | import io.parallec.core.ParallelClient; 5 | import io.parallec.core.ParallelTask; 6 | import io.parallec.core.ResponseOnSingleTask; 7 | import io.parallec.core.TestBase; 8 | import io.parallec.core.config.ParallelTaskConfig; 9 | 10 | import java.util.Map; 11 | 12 | import org.junit.AfterClass; 13 | import org.junit.BeforeClass; 14 | import org.junit.Test; 15 | 16 | public class ParallelClientHttpTop100WebsiteTimeoutTest extends TestBase { 17 | 18 | private static ParallelClient pc; 19 | 20 | @BeforeClass 21 | public static void setUp() throws Exception { 22 | pc = new ParallelClient(); 23 | } 24 | 25 | @AfterClass 26 | public static void shutdown() throws Exception { 27 | pc.releaseExternalResources(); 28 | } 29 | 30 | @Test 31 | public void hitTop100WebsitesTimeout() { 32 | // timeout early 33 | ParallelTaskConfig config = new ParallelTaskConfig(); 34 | config.setTimeoutInManagerSec(1); 35 | 36 | ParallelTask pt = pc 37 | .prepareHttpGet("") 38 | .setConcurrency(1000) 39 | .setSaveResponseToTask(true) 40 | .setConfig(config) 41 | .setTargetHostsFromLineByLineText(FILEPATH_TOP_100, 42 | SOURCE_LOCAL).execute(new ParallecResponseHandler() { 43 | 44 | @Override 45 | public void onCompleted(ResponseOnSingleTask res, 46 | Map responseContext) { 47 | logger.info("Responose Code:" + res.getStatusCode() 48 | + " host: " + res.getHost()); 49 | } 50 | }); 51 | logger.info("completed {} out of {} tasks in early timeout.", 52 | pt.getResponsedNum(), pt.getRequestNum()); 53 | 54 | } 55 | 56 | } 57 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/http/ParallelClientHttpTop500WebsiteTest.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.main.http; 2 | 3 | import io.parallec.core.ParallecResponseHandler; 4 | import io.parallec.core.ParallelClient; 5 | import io.parallec.core.ParallelTask; 6 | import io.parallec.core.ResponseOnSingleTask; 7 | import io.parallec.core.TestBase; 8 | import io.parallec.core.util.PcStringUtils; 9 | 10 | import java.util.Map; 11 | 12 | import org.junit.AfterClass; 13 | import org.junit.BeforeClass; 14 | import org.junit.Ignore; 15 | import org.junit.Test; 16 | 17 | public class ParallelClientHttpTop500WebsiteTest extends TestBase { 18 | 19 | private static ParallelClient pc; 20 | 21 | @BeforeClass 22 | public static void setUp() throws Exception { 23 | pc = new ParallelClient(); 24 | } 25 | 26 | @AfterClass 27 | public static void shutdown() throws Exception { 28 | pc.releaseExternalResources(); 29 | } 30 | 31 | 32 | /** 33 | * Add throttling of 50 when need to hit around 500 web sites testing wait 34 | * and retry part 35 | */ 36 | @Test 37 | public void hitTop500WebsitesThrottling() { 38 | 39 | ParallelTask pt = pc 40 | .prepareHttpGet("") 41 | .async() 42 | .setConcurrency(500) 43 | .setTargetHostsFromLineByLineText(FILEPATH_TOP_500, 44 | SOURCE_LOCAL).execute(new ParallecResponseHandler() { 45 | @Override 46 | public void onCompleted(ResponseOnSingleTask res, 47 | Map responseContext) { 48 | logger.info("Responose Code:" + res.getStatusCode() 49 | + " host: " + res.getHost()); 50 | } 51 | }); 52 | 53 | while (!pt.isCompleted()) { 54 | try { 55 | Thread.sleep(100L); 56 | System.err.println(String.format( 57 | "POLL_JOB_PROGRESS (%.5g%%) PT jobid: %s", 58 | pt.getProgress(), pt.getTaskId())); 59 | pc.logHealth(); 60 | } catch (InterruptedException e) { 61 | e.printStackTrace(); 62 | } 63 | } 64 | logger.info("Result Summary\n{}", 65 | PcStringUtils.renderJson(pt.getAggregateResultCountSummary())); 66 | 67 | }// end func 68 | 69 | /** 70 | * Local ISP seems cannot be more than 500 concurrency or will fail 71 | */ 72 | // @Test 73 | @Ignore 74 | public void hitTop1000WebsitesThrottling() { 75 | 76 | long startTime = System.currentTimeMillis(); 77 | ParallelTask pt = pc 78 | .prepareHttpGet("") 79 | .async() 80 | .setConcurrency(500) 81 | .setTargetHostsFromLineByLineText(FILEPATH_TOP_1000, 82 | SOURCE_LOCAL).execute(new ParallecResponseHandler() { 83 | @Override 84 | public void onCompleted(ResponseOnSingleTask res, 85 | Map responseContext) { 86 | logger.info("Responose Code:" + res.getStatusCode() 87 | + " host: " + res.getHost()); 88 | } 89 | }); 90 | 91 | while (!pt.isCompleted()) { 92 | try { 93 | Thread.sleep(1000L); 94 | pc.logHealth(); 95 | // System.err.println(String.format("POLL_JOB_PROGRESS (%.5g%%) PT jobid: %s", 96 | // pt.getProgress(), pt.getTaskId() )); 97 | } catch (InterruptedException e) { 98 | e.printStackTrace(); 99 | } 100 | } 101 | 102 | long endTime = System.currentTimeMillis(); 103 | 104 | String secondElapsedStr = new Double((endTime - startTime) / 1000.0) 105 | .toString(); 106 | 107 | logger.info("Result Summary\n{}", 108 | PcStringUtils.renderJson(pt.getAggregateResultCountSummary())); 109 | logger.info("Hit 10K website use {} seconds.", secondElapsedStr); 110 | 111 | } 112 | 113 | } 114 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/http/pollable/sampleserver/HttpServerThread.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.main.http.pollable.sampleserver; 2 | 3 | import java.io.IOException; 4 | import java.util.logging.Logger; 5 | 6 | public class HttpServerThread extends Thread { 7 | private static Logger logger = Logger.getLogger(HttpServerThread.class 8 | .getName()); 9 | private volatile boolean shutdown = false; 10 | 11 | @Override 12 | public void run() { 13 | 14 | try { 15 | ServerWithPollableJobs server = new ServerWithPollableJobs(); 16 | while (!this.isShutdown()) { 17 | ; 18 | } 19 | server.stop(); 20 | logger.info("Server Stopped.."); 21 | } catch (IOException ioe) { 22 | System.err.println("Couldn't start server:\n" + ioe); 23 | } 24 | 25 | } 26 | 27 | public boolean isShutdown() { 28 | return shutdown; 29 | } 30 | 31 | public void setShutdown(boolean shutdown) { 32 | this.shutdown = shutdown; 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/http/request/template/ParallelClientVarReplacementHostSpecificHeaderTest.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.main.http.request.template; 2 | 3 | import io.parallec.core.ParallecHeader; 4 | import io.parallec.core.ParallecResponseHandler; 5 | import io.parallec.core.ParallelClient; 6 | import io.parallec.core.ResponseOnSingleTask; 7 | import io.parallec.core.TestBase; 8 | import io.parallec.core.main.http.pollable.sampleserver.HttpServerThread; 9 | 10 | import java.util.ArrayList; 11 | import java.util.HashMap; 12 | import java.util.List; 13 | import java.util.Map; 14 | 15 | import org.apache.http.util.Asserts; 16 | import org.junit.AfterClass; 17 | import org.junit.BeforeClass; 18 | import org.junit.Test; 19 | 20 | public class ParallelClientVarReplacementHostSpecificHeaderTest extends 21 | TestBase { 22 | 23 | private static ParallelClient pc; 24 | private static HttpServerThread serverThread; 25 | 26 | @BeforeClass 27 | public static void setUp() throws Exception { 28 | pc = new ParallelClient(); 29 | serverThread = new HttpServerThread(); 30 | serverThread.start(); 31 | 32 | //add sleep to make sure the server starts first 33 | try { 34 | Thread.sleep(500L); 35 | } catch (Exception e) { 36 | ; 37 | } 38 | } 39 | 40 | @AfterClass 41 | public static void shutdown() throws Exception { 42 | pc.releaseExternalResources(); 43 | serverThread.setShutdown(true); 44 | } 45 | 46 | /** 47 | * Testing the replacement in the header values. the sample server would 48 | * response with the header values 49 | */ 50 | @Test 51 | public void differentRequestsToSameTargetHostWithHeaderReplacement() { 52 | List replaceList = new ArrayList(); 53 | replaceList.add("111"); 54 | replaceList.add("222"); 55 | 56 | Map responseContext = new HashMap(); 57 | responseContext.put("temp", null); 58 | 59 | pc.prepareHttpGet("/testHeaders") 60 | .setHttpHeaders(new ParallecHeader().addPair("sample", "$SAMPLE")) 61 | .setConcurrency(1700) 62 | .setHttpPort(10080) 63 | .setReplaceVarMapToSingleTargetSingleVar("SAMPLE", replaceList, 64 | "localhost").setResponseContext(responseContext) 65 | .execute(new ParallecResponseHandler() { 66 | 67 | @Override 68 | public void onCompleted(ResponseOnSingleTask res, 69 | Map responseContext) { 70 | logger.info(res.toString()); 71 | 72 | responseContext.put("sampleValue", 73 | res.getResponseContent()); 74 | } 75 | }); 76 | 77 | int tempGlobal = Integer.parseInt((String) responseContext 78 | .get("sampleValue")); 79 | Asserts.check( 80 | tempGlobal <= 222 && tempGlobal >= 111, 81 | " Fail to extract o Fail different request to same server with header replacement test"); 82 | 83 | } 84 | 85 | } 86 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/http/request/template/ParallelClientVarReplacementUniformTest.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.main.http.request.template; 2 | 3 | import io.parallec.core.FilterRegex; 4 | import io.parallec.core.ParallecResponseHandler; 5 | import io.parallec.core.ParallelClient; 6 | import io.parallec.core.ResponseOnSingleTask; 7 | import io.parallec.core.TestBase; 8 | 9 | import java.util.HashMap; 10 | import java.util.Map; 11 | 12 | import org.apache.http.util.Asserts; 13 | import org.junit.AfterClass; 14 | import org.junit.BeforeClass; 15 | import org.junit.Ignore; 16 | import org.junit.Test; 17 | 18 | public class ParallelClientVarReplacementUniformTest extends TestBase { 19 | 20 | private static ParallelClient pc; 21 | 22 | @BeforeClass 23 | public static void setUp() throws Exception { 24 | pc = new ParallelClient(); 25 | } 26 | 27 | @AfterClass 28 | public static void shutdown() throws Exception { 29 | pc.releaseExternalResources(); 30 | } 31 | 32 | // TODO 33 | @Ignore 34 | public void testVarReplacementTargetHostSpecificHttpHeaderReplacement() { 35 | 36 | } 37 | 38 | /** 39 | * same requests to different target URLs 40 | * 41 | * demonstrate the uniform variable replacement 42 | * 43 | * here: the "$URL_VARIABLE" is replaced by validateInternals 44 | */ 45 | @Test 46 | public void hitWebsitesMinTargeUniformReplacement() { 47 | 48 | Map replacementVarMap = new HashMap(); 49 | replacementVarMap.put("URL_VARIABLE", "validateInternals"); 50 | 51 | pc.prepareHttpGet("/$URL_VARIABLE.html") 52 | .setConcurrency(1700) 53 | .setTargetHostsFromString( 54 | "www.parallec.io www.jeffpei.com www.restcommander.com") 55 | .setReplacementVarMap(replacementVarMap) 56 | .execute(new ParallecResponseHandler() { 57 | @Override 58 | public void onCompleted(ResponseOnSingleTask res, 59 | Map responseContext) { 60 | String cpu = new FilterRegex( 61 | ".*CPU-Usage-Percent\\s*(.*?).*") 62 | .filter(res.getResponseContent()); 63 | String memory = new FilterRegex( 64 | ".*Memory-Used-KB\\s*(.*?).*") 65 | .filter(res.getResponseContent()); 66 | 67 | Map metricMap = new HashMap(); 68 | metricMap.put("CpuUsage", cpu); 69 | metricMap.put("MemoryUsage", memory); 70 | 71 | logger.info("cpu:" + cpu + " memory: " + memory 72 | + " host: " + res.getHost()); 73 | Double cpuDouble = Double.parseDouble(cpu); 74 | Asserts.check(cpuDouble <= 100.0 && cpuDouble >= 0.0, 75 | " Fail to extract cpu values"); 76 | } 77 | }); 78 | 79 | } 80 | 81 | } 82 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/ping/PingOnTmpTest.java: -------------------------------------------------------------------------------- 1 | /* 2 | Copyright [2013-2015] eBay Software Foundation 3 | Licensed under the Apache License, Version 2.0 (the "License"); 4 | you may not use this file except in compliance with the License. 5 | You may obtain a copy of the License at 6 | http://www.apache.org/licenses/LICENSE-2.0 7 | Unless required by applicable law or agreed to in writing, software 8 | distributed under the License is distributed on an "AS IS" BASIS, 9 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 10 | See the License for the specific language governing permissions and 11 | limitations under the License. 12 | */ 13 | package io.parallec.core.main.ping; 14 | 15 | import io.parallec.core.ParallecResponseHandler; 16 | import io.parallec.core.ParallelClient; 17 | import io.parallec.core.ParallelTask; 18 | import io.parallec.core.ResponseOnSingleTask; 19 | import io.parallec.core.TestBase; 20 | import io.parallec.core.monitor.MonitorProvider; 21 | import io.parallec.core.util.PcStringUtils; 22 | 23 | import java.util.Map; 24 | 25 | import org.junit.AfterClass; 26 | import org.junit.BeforeClass; 27 | import org.junit.Ignore; 28 | 29 | /** 30 | * The most basic test with hitting the same URL at 3 different websites. 31 | * require Internet access for testing. 32 | * 33 | *

34 | * This example shows 1. Basic request construction 2. how to use response 35 | * context to pass value during the response handler out to a global space 36 | *

37 | */ 38 | public class PingOnTmpTest extends TestBase { 39 | 40 | /** The pc. */ 41 | private static ParallelClient pc; 42 | 43 | /** 44 | * Sets the up. 45 | * 46 | * @throws Exception 47 | * the exception 48 | */ 49 | @BeforeClass 50 | public static void setUp() throws Exception { 51 | pc = new ParallelClient(); 52 | } 53 | 54 | /** 55 | * Shutdown. 56 | * 57 | * @throws Exception 58 | * the exception 59 | */ 60 | @AfterClass 61 | public static void shutdown() throws Exception { 62 | pc.releaseExternalResources(); 63 | } 64 | 65 | /** 66 | * Ping websites min async. 67 | */ 68 | @Ignore 69 | // @Test 70 | public void pingWebsitesMinBasicSyncOn1500() { 71 | 72 | ParallelTask task = pc.preparePing().setConcurrency(1500) 73 | .setPingNumRetries(0) 74 | .setTargetHostsFromLineByLineText(FILEPATH_TMP1, SOURCE_LOCAL) 75 | // .setTargetHostsFromString( 76 | // "www.parallec.io www.jeffpei.com www.restcommander.com bad.c21tom") 77 | .execute(new ParallecResponseHandler() { 78 | @Override 79 | public void onCompleted(ResponseOnSingleTask res, 80 | Map responseContext) { 81 | logger.info(res.toString()); 82 | } 83 | }); 84 | 85 | logger.info("Task Pretty Print: \n{}", 86 | PcStringUtils.renderJson(task.getAggregateResultCountSummary())); 87 | logger.info("Total Duration: " + task.getDurationSec()); 88 | }// end func 89 | 90 | 91 | /** 92 | * Ping websites min async. 93 | */ 94 | @Ignore 95 | // @Test 96 | public void pingWebsitesMinAsync() { 97 | 98 | ParallelTask task = pc.preparePing().async().setConcurrency(1500) 99 | .setTargetHostsFromLineByLineText(FILEPATH_TMP1, SOURCE_LOCAL) 100 | // .setTargetHostsFromString( 101 | // "www.parallec.io www.jeffpei.com www.restcommander.com bad.c21tom") 102 | .execute(new ParallecResponseHandler() { 103 | @Override 104 | public void onCompleted(ResponseOnSingleTask res, 105 | Map responseContext) { 106 | logger.info(res.toString()); 107 | } 108 | }); 109 | 110 | while (!task.isCompleted()) { 111 | try { 112 | Thread.sleep(100L); 113 | System.err.println("LIVE_THREAD_COUNT: " 114 | + MonitorProvider.getInstance().getLiveThreadCount()); 115 | // System.err.println(String.format("POLL_JOB_PROGRESS (%.5g%%)", 116 | // task.getProgress())); 117 | } catch (InterruptedException e) { 118 | e.printStackTrace(); 119 | } 120 | } 121 | logger.info("Task Pretty Print: \n{}", 122 | PcStringUtils.renderJson(task.getAggregateResultCountSummary())); 123 | 124 | }// end func 125 | } 126 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/ping/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * @author Yuanteng (Jeff) Pei 3 | * 4 | */ 5 | package io.parallec.core.main.ping; -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/ssh/ParallelClientSshTest.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.main.ssh; 2 | 3 | import io.parallec.core.ParallecResponseHandler; 4 | import io.parallec.core.ParallelClient; 5 | import io.parallec.core.ResponseOnSingleTask; 6 | import io.parallec.core.TestBase; 7 | 8 | import java.util.HashMap; 9 | import java.util.Map; 10 | 11 | import org.apache.http.util.Asserts; 12 | import org.junit.AfterClass; 13 | import org.junit.BeforeClass; 14 | import org.junit.Ignore; 15 | import org.junit.Test; 16 | 17 | public class ParallelClientSshTest extends TestBase { 18 | 19 | private static ParallelClient pc; 20 | 21 | @BeforeClass 22 | public static void setUp() throws Exception { 23 | pc = new ParallelClient(); 24 | } 25 | 26 | @AfterClass 27 | public static void shutdown() throws Exception { 28 | pc.releaseExternalResources(); 29 | } 30 | 31 | @Test 32 | public void sshWorkerFakeVmPasswordTest() { 33 | 34 | Map responseContext = new HashMap(); 35 | pc.prepareSsh().setConcurrency(500) 36 | .setTargetHostsFromString(hostIpSample) 37 | .setSshCommandLine("df -h; ds; ").setSshUserName(userName) 38 | .setSshPassword(passwd).setSshConnectionTimeoutMillis(5000) 39 | .setResponseContext(responseContext) 40 | .execute(new ParallecResponseHandler() { 41 | 42 | @Override 43 | public void onCompleted(ResponseOnSingleTask res, 44 | Map responseContext) { 45 | logger.info("Responose:" + res.toString() + " host: " 46 | + res.getHost() + " errmsg: " 47 | + res.getErrorMessage()); 48 | responseContext.put("errorMessage", 49 | res.getErrorMessage()); 50 | 51 | } 52 | 53 | }); 54 | 55 | String errMessage = (String) responseContext.get("errorMessage"); 56 | Asserts.check(errMessage.contains("socket is not established"), 57 | "fail. error is not expected. not sure if ssh flow was executed"); 58 | 59 | }// end func 60 | 61 | @Test 62 | public void sshWorkerFakeVmPasswordInvalidPollerTest() { 63 | 64 | Map responseContext = new HashMap(); 65 | pc.prepareSsh().setConcurrency(300) 66 | .setTargetHostsFromString(hostIpSample) 67 | .setSshCommandLine("df -h; ds; ").setSshUserName(userName) 68 | .setHttpPollable(true).setSshPassword(passwd) 69 | .setResponseContext(responseContext) 70 | .execute(new ParallecResponseHandler() { 71 | 72 | @Override 73 | public void onCompleted(ResponseOnSingleTask res, 74 | Map responseContext) { 75 | logger.info("Responose:" + res.toString() + " host: " 76 | + res.getHost() + " errmsg: " 77 | + res.getErrorMessage()); 78 | 79 | } 80 | 81 | }); 82 | 83 | }// end func 84 | 85 | // @Test 86 | @Ignore 87 | public void sshWorkerRealVmPasswordTest() { 88 | pc.prepareSsh().setConcurrency(300) 89 | .setTargetHostsFromString(SshProviderRealTest.vmIp) 90 | .setSshCommandLine("df -h; ds; ").setSshUserName("parallec") 91 | .setSshPassword("parallec") 92 | 93 | .execute(new ParallecResponseHandler() { 94 | 95 | @Override 96 | public void onCompleted(ResponseOnSingleTask res, 97 | Map responseContext) { 98 | logger.info("Responose:" + res.toString() + " host: " 99 | + res.getHost() + " errmsg: " 100 | + res.getErrorMessage()); 101 | } 102 | }); 103 | } 104 | 105 | } 106 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/ssh/SshMetaTest.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.main.ssh; 2 | 3 | import io.parallec.core.TestBase; 4 | import io.parallec.core.bean.ssh.SshLoginType; 5 | import io.parallec.core.bean.ssh.SshMeta; 6 | import io.parallec.core.commander.workflow.ssh.SshProvider; 7 | import io.parallec.core.exception.ParallelTaskInvalidException; 8 | 9 | import org.junit.Test; 10 | 11 | public class SshMetaTest extends TestBase { 12 | public static SshProvider sshProvider; 13 | 14 | public static SshMeta sshMetaPassword = new SshMeta(commandSshLineValid, 15 | userName, PORT_DEFAULT, SshLoginType.PASSWORD, null, passwd, false, 16 | null, sshConnectionTimeoutMillis, false); 17 | public static SshMeta sshMetaKey = new SshMeta(commandSshLineValid, 18 | userName, PORT_DEFAULT, SshLoginType.KEY, 19 | "userdata/fake-privkey.txt", null, false, null, 20 | sshConnectionTimeoutMillis, false); 21 | public static SshMeta sshMetaKeyNotExist = new SshMeta(commandSshLineValid, 22 | userName, PORT_DEFAULT, SshLoginType.KEY, 23 | "userdata/noneexisting.txt", null, false, null, 24 | sshConnectionTimeoutMillis, false); 25 | 26 | private static void resetPass() { 27 | sshMetaPassword = new SshMeta(commandSshLineValid, userName, 28 | PORT_DEFAULT, SshLoginType.PASSWORD, null, passwd, false, null, 29 | sshConnectionTimeoutMillis, false); 30 | } 31 | 32 | private static void resetKey() { 33 | 34 | sshMetaKey = new SshMeta(commandSshLineValid, userName, PORT_DEFAULT, 35 | SshLoginType.KEY, "userdata/fake-privkey.txt", null, false, 36 | null, sshConnectionTimeoutMillis, false); 37 | } 38 | 39 | private static void expectException(SshMeta meta) { 40 | 41 | try { 42 | meta.validation(); 43 | } catch (ParallelTaskInvalidException t) { 44 | logger.info("expected " + t); 45 | } 46 | 47 | } 48 | 49 | @Test 50 | public void Validationtest() throws Exception { 51 | sshMetaPassword.setCommandLine(null); 52 | 53 | expectException(sshMetaPassword); 54 | resetPass(); 55 | 56 | sshMetaPassword.setUserName(null); 57 | expectException(sshMetaPassword); 58 | resetPass(); 59 | 60 | sshMetaPassword.setSshLoginType(null); 61 | expectException(sshMetaPassword); 62 | resetPass(); 63 | 64 | sshMetaPassword.setPassword(null); 65 | expectException(sshMetaPassword); 66 | resetPass(); 67 | 68 | sshMetaKey.setPrivKeyRelativePath(null); 69 | expectException(sshMetaKey); 70 | resetKey(); 71 | 72 | sshMetaKey.setPrivKeyUsePassphrase(true); 73 | sshMetaKey.setPassphrase(null); 74 | expectException(sshMetaKey); 75 | resetKey(); 76 | 77 | sshMetaKey.getPrivKeyAbsPath(); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/ssh/SshProviderRealTest.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.main.ssh; 2 | 3 | import io.parallec.core.TestBase; 4 | import io.parallec.core.actor.message.ResponseOnSingeRequest; 5 | import io.parallec.core.bean.ssh.SshLoginType; 6 | import io.parallec.core.bean.ssh.SshMeta; 7 | import io.parallec.core.commander.workflow.ssh.SshProvider; 8 | 9 | import org.junit.Ignore; 10 | 11 | public class SshProviderRealTest extends TestBase { 12 | 13 | private static int PORT_DEFAULT = 22; 14 | 15 | public static String vmIp = "52.25.191.169"; 16 | 17 | @Ignore 18 | // @Test 19 | public void testExecuteSshCommandRealHostPassword() { 20 | 21 | SshMeta sshMeta = new SshMeta("date; ifconfig; ds", "parallec", 22 | PORT_DEFAULT, SshLoginType.PASSWORD, null, "parallec", false, 23 | null, sshConnectionTimeoutMillis, false); 24 | 25 | SshProvider instance = new SshProvider(sshMeta, vmIp); 26 | ResponseOnSingeRequest response = instance.executeSshCommand(); 27 | logger.info(response.toString()); 28 | } 29 | 30 | // @Test 31 | @Ignore 32 | public void testExecuteSshCommandRealHostKey() { 33 | 34 | SshMeta sshMeta = new SshMeta(" ps -ef; ds; ifconfig", "ubuntu", 35 | PORT_DEFAULT, SshLoginType.KEY, "userdata/vm-keys.pem", 36 | null, false, null, sshConnectionTimeoutMillis, false); 37 | SshProvider instance = new SshProvider(sshMeta, vmIp); 38 | ResponseOnSingeRequest response = instance.executeSshCommand(); 39 | logger.info("PrivKey absolute path: " + sshMeta.getPrivKeyAbsPath()); 40 | logger.info(response.toString()); 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/tcp/ParallelClientTcpBasicTest.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.main.tcp; 2 | 3 | import io.parallec.core.ParallecResponseHandler; 4 | import io.parallec.core.ParallelClient; 5 | import io.parallec.core.ResponseOnSingleTask; 6 | import io.parallec.core.TestBase; 7 | import io.parallec.core.exception.ParallelTaskInvalidException; 8 | import io.parallec.core.main.tcp.sampleserver.TcpServerThread; 9 | 10 | import java.util.HashMap; 11 | import java.util.Map; 12 | 13 | import org.apache.http.util.Asserts; 14 | import org.junit.AfterClass; 15 | import org.junit.BeforeClass; 16 | import org.junit.Test; 17 | 18 | public class ParallelClientTcpBasicTest extends TestBase { 19 | 20 | private static ParallelClient pc; 21 | private static TcpServerThread serverThread; 22 | 23 | @BeforeClass 24 | public static void setUp() throws Exception { 25 | pc = new ParallelClient(); 26 | 27 | serverThread = new TcpServerThread(false); 28 | serverThread.start(); 29 | 30 | try { 31 | Thread.sleep(500L); 32 | } catch (Exception e) { 33 | ; 34 | } 35 | } 36 | 37 | @AfterClass 38 | public static void shutdown() throws Exception { 39 | pc.releaseExternalResources(); 40 | serverThread.setShutdown(true); 41 | 42 | try { 43 | Thread.sleep(500L); 44 | } catch (Exception e) { 45 | ; 46 | } 47 | } 48 | 49 | 50 | /** 51 | * different requests to different target URLs 52 | * http://www.jeffpei.com/job_b.html http://www.restsuperman.com/job_c.html 53 | */ 54 | @Test(timeout = 50000) 55 | public void TcpTest() { 56 | Map responseContext = new HashMap(); 57 | pc.prepareTcp("hadoopmon").setConcurrency(300) 58 | .setTargetHostsFromString("localhost") 59 | .setTcpPort(10081) 60 | .setResponseContext(responseContext) 61 | .execute(new ParallecResponseHandler() { 62 | 63 | @Override 64 | public void onCompleted(ResponseOnSingleTask res, 65 | Map responseContext) { 66 | logger.info("Responose:" + res.getResponseContent() + " host: " 67 | + res.getHost() + " errmsg: " 68 | + res.getErrorMessage()); 69 | responseContext.put("resp", 70 | res.getResponseContent()); 71 | 72 | } 73 | 74 | }); 75 | 76 | String resp = (String) responseContext.get("resp"); 77 | Asserts.check(resp.contains("AT_TCP_SERVER"), 78 | "fail.TcpTest with whole PC flow"); 79 | 80 | 81 | } 82 | 83 | @Test(timeout = 50000) 84 | public void TcpExpectedRefusedConnectionTest() { 85 | Map responseContext = new HashMap(); 86 | pc.prepareTcp("hadoopmon").setConcurrency(300) 87 | .setTargetHostsFromString("localhost") 88 | .setTcpPort(10099) 89 | .setResponseContext(responseContext) 90 | .execute(new ParallecResponseHandler() { 91 | 92 | @Override 93 | public void onCompleted(ResponseOnSingleTask res, 94 | Map responseContext) { 95 | logger.info("Responose:" + res.getResponseContent() + " host: " 96 | + res.getHost() + " errmsg: " 97 | + res.getErrorMessage()); 98 | responseContext.put("resp", 99 | res.getStatusCode()); 100 | 101 | } 102 | 103 | }); 104 | 105 | String resp = (String) responseContext.get("resp"); 106 | Asserts.check(resp.contains("FAILURE"), 107 | "fail.TcpTest with expected wrong port"); 108 | 109 | 110 | } 111 | 112 | @Test 113 | public void TcpTestFunctionsExceptions() { 114 | 115 | try{ 116 | 117 | pc.prepareTcp("hadoopmon").setConcurrency(300) 118 | .setTargetHostsFromString("localhost") 119 | .setHttpPollable(true) 120 | .validation() 121 | ; 122 | }catch(ParallelTaskInvalidException e){ 123 | logger.info("EXPECTED error" + e.getLocalizedMessage()); 124 | } 125 | 126 | 127 | } 128 | 129 | } 130 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/tcp/TcpClientTest.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.main.tcp; 2 | 3 | import io.parallec.core.ParallelClient; 4 | import io.parallec.core.TestBase; 5 | import io.parallec.core.main.tcp.sampleserver.TcpServerThread; 6 | 7 | import org.junit.AfterClass; 8 | import org.junit.BeforeClass; 9 | import org.junit.Ignore; 10 | 11 | public class TcpClientTest extends TestBase { 12 | 13 | private static ParallelClient pc; 14 | private static TcpServerThread serverThread; 15 | 16 | @BeforeClass 17 | public static void setUp() throws Exception { 18 | pc = new ParallelClient(); 19 | 20 | serverThread = new TcpServerThread(false); 21 | serverThread.start(); 22 | 23 | //add sleep to make sure the server starts first 24 | try { 25 | Thread.sleep(500L); 26 | } catch (Exception e) { 27 | ; 28 | } 29 | } 30 | 31 | @AfterClass 32 | public static void shutdown() throws Exception { 33 | pc.releaseExternalResources(); 34 | serverThread.setShutdown(true); 35 | 36 | try { 37 | Thread.sleep(500L); 38 | } catch (Exception e) { 39 | ; 40 | } 41 | } 42 | 43 | 44 | /** 45 | * different requests to different target URLs 46 | * http://www.jeffpei.com/job_b.html http://www.restsuperman.com/job_c.html 47 | */ 48 | //@Test(timeout = 50000) 49 | @Ignore 50 | public void telClientTest() { 51 | try { 52 | TcpProviderPoc.getInstance().request(); 53 | } catch (Exception e) { 54 | // TODO Auto-generated catch block 55 | e.printStackTrace(); 56 | } 57 | 58 | } 59 | 60 | 61 | } 62 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/tcp/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | /** 5 | * @author Yuanteng (Jeff) Pei 6 | * 7 | */ 8 | package io.parallec.core.main.tcp; -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/tcp/sampleserver/TcpServerThread.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.main.tcp.sampleserver; 2 | 3 | import io.parallec.core.util.PcStringUtils; 4 | 5 | import java.io.IOException; 6 | import java.util.logging.Logger; 7 | 8 | public class TcpServerThread extends Thread { 9 | private static Logger logger = Logger.getLogger(TcpServerThread.class 10 | .getName()); 11 | private volatile boolean shutdown = false; 12 | 13 | private boolean idle; 14 | public TcpServerThread(boolean idle){ 15 | this.idle = idle; 16 | } 17 | private TcpEchoServer server = null; 18 | @Override 19 | public void run() { 20 | 21 | try { 22 | server = new TcpEchoServer(idle); 23 | server.serve(); 24 | while (!this.isShutdown()) { 25 | ; 26 | } 27 | server.stop(); 28 | logger.info("TCP Server Stopped.."); 29 | } catch (Exception e) { 30 | logger.info("Couldn't start TCP server: " + PcStringUtils.printStackTrace(e)); 31 | } 32 | 33 | } 34 | 35 | public boolean isShutdown() { 36 | return shutdown; 37 | } 38 | 39 | //interrupt is useful 40 | public void setShutdown(boolean shutdown) { 41 | this.shutdown = shutdown; 42 | 43 | if(shutdown){ 44 | try { 45 | this.server.getServerSocket().close(); 46 | } catch (IOException e) { 47 | e.printStackTrace(); 48 | } 49 | } 50 | this.interrupt(); 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/tcp/sampleserver/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | /** 5 | * @author Yuanteng (Jeff) Pei 6 | * 7 | */ 8 | package io.parallec.core.main.tcp.sampleserver; -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/udp/ParallelClientUdpBasicTest.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.main.udp; 2 | 3 | import io.parallec.core.ParallecResponseHandler; 4 | import io.parallec.core.ParallelClient; 5 | import io.parallec.core.ResponseOnSingleTask; 6 | import io.parallec.core.TestBase; 7 | import io.parallec.core.exception.ParallelTaskInvalidException; 8 | import io.parallec.core.main.udp.sampleserver.UdpServerThread; 9 | 10 | import java.util.HashMap; 11 | import java.util.Map; 12 | 13 | import org.apache.http.util.Asserts; 14 | import org.junit.AfterClass; 15 | import org.junit.BeforeClass; 16 | import org.junit.Test; 17 | 18 | public class ParallelClientUdpBasicTest extends TestBase { 19 | 20 | private static ParallelClient pc; 21 | private static UdpServerThread serverThread; 22 | 23 | @BeforeClass 24 | public static void setUp() throws Exception { 25 | pc = new ParallelClient(); 26 | boolean similateSlowResponse = false; 27 | serverThread = new UdpServerThread(similateSlowResponse); 28 | serverThread.start(); 29 | 30 | try { 31 | Thread.sleep(500L); 32 | } catch (Exception e) { 33 | ; 34 | } 35 | } 36 | 37 | @AfterClass 38 | public static void shutdown() throws Exception { 39 | pc.releaseExternalResources(); 40 | serverThread.setShutdown(true); 41 | 42 | try { 43 | Thread.sleep(500L); 44 | } catch (Exception e) { 45 | ; 46 | } 47 | } 48 | 49 | 50 | /** 51 | * http://www.jeffpei.com/job_b.html http://www.restsuperman.com/job_c.html 52 | */ 53 | @Test(timeout = 50000) 54 | public void UdpTest() { 55 | Map responseContext = new HashMap(); 56 | pc.prepareUdp("hadoopmonudp").setConcurrency(300) 57 | .setTargetHostsFromString("localhost") 58 | .setUdpPort(10091) 59 | .setResponseContext(responseContext) 60 | .execute(new ParallecResponseHandler() { 61 | 62 | @Override 63 | public void onCompleted(ResponseOnSingleTask res, 64 | Map responseContext) { 65 | logger.info("Responose:" + res.getResponseContent() + " host: " 66 | + res.getHost() + " errmsg: " 67 | + res.getErrorMessage()); 68 | responseContext.put("resp", 69 | res.getResponseContent()); 70 | 71 | } 72 | 73 | }); 74 | 75 | String resp = (String) responseContext.get("resp"); 76 | Asserts.check(resp.contains("AT_UDP_SERVER"), 77 | "fail.UdpTest with whole PC flow"); 78 | 79 | 80 | } 81 | 82 | @Test(timeout = 50000) 83 | public void UdpExpectedRefusedConnectionTest() { 84 | Map responseContext = new HashMap(); 85 | pc.prepareUdp("hadoopmon").setConcurrency(300) 86 | .setTargetHostsFromString("localhost") 87 | .setUdpPort(10099) 88 | .setResponseContext(responseContext) 89 | .execute(new ParallecResponseHandler() { 90 | 91 | @Override 92 | public void onCompleted(ResponseOnSingleTask res, 93 | Map responseContext) { 94 | logger.info("Responose:" + res.getResponseContent() + " host: " 95 | + res.getHost() + " errmsg: " 96 | + res.getErrorMessage()); 97 | responseContext.put("resp", 98 | res.getStatusCode()); 99 | 100 | } 101 | 102 | }); 103 | 104 | String resp = (String) responseContext.get("resp"); 105 | Asserts.check(resp.contains("FAILURE"), 106 | "fail.TcpTest with expected wrong port"); 107 | 108 | 109 | } 110 | 111 | @Test 112 | public void UdpTestFunctionsExceptions() { 113 | 114 | try{ 115 | 116 | pc.prepareUdp("hadoopmonudp").setConcurrency(300) 117 | .setTargetHostsFromString("localhost") 118 | .setHttpPollable(true) 119 | .validation() 120 | ; 121 | }catch(ParallelTaskInvalidException e){ 122 | logger.info("EXPECTED error" + e.getLocalizedMessage()); 123 | } 124 | 125 | 126 | } 127 | 128 | } 129 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/udp/ParallelClientUdpSlowServerTest.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.main.udp; 2 | 3 | import io.parallec.core.ParallecResponseHandler; 4 | import io.parallec.core.ParallelClient; 5 | import io.parallec.core.ResponseOnSingleTask; 6 | import io.parallec.core.TestBase; 7 | import io.parallec.core.main.udp.sampleserver.UdpServerThread; 8 | 9 | import java.util.HashMap; 10 | import java.util.Map; 11 | 12 | import org.apache.http.util.Asserts; 13 | import org.junit.AfterClass; 14 | import org.junit.BeforeClass; 15 | import org.junit.Test; 16 | 17 | public class ParallelClientUdpSlowServerTest extends TestBase { 18 | 19 | private static ParallelClient pc; 20 | private static UdpServerThread serverThread; 21 | 22 | @BeforeClass 23 | public static void setUp() throws Exception { 24 | pc = new ParallelClient(); 25 | boolean similateSlowResponse = true; 26 | serverThread = new UdpServerThread(similateSlowResponse); 27 | serverThread.start(); 28 | 29 | try { 30 | Thread.sleep(500L); 31 | } catch (Exception e) { 32 | ; 33 | } 34 | } 35 | 36 | @AfterClass 37 | public static void shutdown() throws Exception { 38 | pc.releaseExternalResources(); 39 | serverThread.setShutdown(true); 40 | 41 | try { 42 | Thread.sleep(500L); 43 | } catch (Exception e) { 44 | ; 45 | } 46 | } 47 | 48 | /** 49 | * should throw exception of idle timeout. 50 | * 51 | * UDP server will sleep 5 seconds. however this 52 | */ 53 | @Test(timeout = 50000) 54 | public void UdpTestWithSlowServer() { 55 | Map responseContext = new HashMap(); 56 | pc.prepareUdp("hadoopmonudp").setConcurrency(300) 57 | .setTargetHostsFromString("localhost") 58 | .setUdpPort(10091) 59 | .setResponseContext(responseContext) 60 | .setUdpIdleTimeoutSec(1) 61 | .execute(new ParallecResponseHandler() { 62 | 63 | @Override 64 | public void onCompleted(ResponseOnSingleTask res, 65 | Map responseContext) { 66 | logger.info("Responose:" + res.getResponseContent() + " host: " 67 | + res.getHost() + " errmsg: " 68 | + res.getErrorMessage()); 69 | responseContext.put("respErrMsg", 70 | res.getErrorMessage()); 71 | 72 | } 73 | 74 | }); 75 | 76 | String respErrMsg = (String) responseContext.get("respErrMsg"); 77 | Asserts.check(respErrMsg.contains("UDP idle (read) timeout"), 78 | "fail.UdpTest with slow server idle timeout flow"); 79 | 80 | } 81 | 82 | 83 | /** 84 | * should throw exception of idle timeout. 85 | * 86 | * UDP server will sleep 5 seconds. now set idle timeout to be 12 sec 87 | */ 88 | @Test(timeout = 50000) 89 | public void UdpTestWithSlowServerWithLongerIdleTimeout() { 90 | Map responseContext = new HashMap(); 91 | pc.prepareUdp("hadoopmonudp").setConcurrency(300) 92 | .setTargetHostsFromString("localhost") 93 | .setUdpPort(10091) 94 | .setResponseContext(responseContext) 95 | .setUdpIdleTimeoutSec(10) 96 | .execute(new ParallecResponseHandler() { 97 | 98 | @Override 99 | public void onCompleted(ResponseOnSingleTask res, 100 | Map responseContext) { 101 | logger.info("Responose:" + res.getResponseContent() + " host: " 102 | + res.getHost() + " errmsg: " 103 | + res.getErrorMessage()); 104 | responseContext.put("resp", 105 | res.getResponseContent()); 106 | 107 | } 108 | 109 | }); 110 | String resp = (String) responseContext.get("resp"); 111 | Asserts.check(resp.contains("hadoopmonudp"), 112 | "fail.UdpTest with slow server longer idle timeout flow"); 113 | 114 | } 115 | 116 | 117 | } 118 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/udp/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | /** 5 | * @author ypei 6 | * 7 | */ 8 | package io.parallec.core.main.udp; -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/udp/sampleserver/UdpServerThread.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.main.udp.sampleserver; 2 | 3 | import io.parallec.core.util.PcStringUtils; 4 | 5 | import java.util.logging.Logger; 6 | 7 | public class UdpServerThread extends Thread { 8 | private static Logger logger = Logger.getLogger(UdpServerThread.class 9 | .getName()); 10 | private volatile boolean shutdown = false; 11 | 12 | private boolean similateSlowResponse; 13 | public UdpServerThread(boolean similateSlowResponse){ 14 | this.similateSlowResponse = similateSlowResponse; 15 | } 16 | private UdpEchoServer server = null; 17 | @Override 18 | public void run() { 19 | 20 | try { 21 | server = new UdpEchoServer(similateSlowResponse); 22 | server.serve(); 23 | while (!this.isShutdown()) { 24 | ; 25 | } 26 | server.stop(); 27 | logger.info("UDP Server Stopped.."); 28 | } catch (Exception e) { 29 | logger.info("Couldn't start UDP server: " + PcStringUtils.printStackTrace(e)); 30 | } 31 | 32 | } 33 | 34 | public boolean isShutdown() { 35 | return shutdown; 36 | } 37 | 38 | //interrupt is useful 39 | public void setShutdown(boolean shutdown) { 40 | this.shutdown = shutdown; 41 | 42 | if(shutdown){ 43 | try { 44 | this.server.getServerSocket().close(); 45 | } catch (Exception e) { 46 | e.printStackTrace(); 47 | } 48 | } 49 | this.interrupt(); 50 | } 51 | 52 | } 53 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/main/udp/sampleserver/package-info.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | /** 5 | * @author Yuanteng (Jeff) Pei 6 | * 7 | */ 8 | package io.parallec.core.main.udp.sampleserver; -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/monitor/MonitorProviderTest.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.monitor; 2 | 3 | import io.parallec.core.ParallelClient; 4 | import io.parallec.core.TestBase; 5 | 6 | import org.junit.AfterClass; 7 | import org.junit.BeforeClass; 8 | import org.junit.Test; 9 | 10 | public class MonitorProviderTest extends TestBase { 11 | 12 | private static ParallelClient pc; 13 | 14 | @BeforeClass 15 | public static void setUp() throws Exception { 16 | pc = new ParallelClient(); 17 | } 18 | 19 | @AfterClass 20 | public static void shutdown() throws Exception { 21 | pc.releaseExternalResources(); 22 | } 23 | 24 | @Test 25 | public void testMonitorProvider() { 26 | 27 | MonitorProvider.getInstance().getJVMMemoryUsage(); 28 | MonitorProvider.getInstance().getThreadUsage(); 29 | MonitorProvider.THRESHOLD_PERCENT = 0; 30 | MonitorProvider.getInstance().getHealthMemory(); 31 | MonitorProvider.getInstance().currentJvmPerformUsage.getSummary(); 32 | MonitorProvider.THRESHOLD_PERCENT = 90; 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/task/ParallelTaskManagerTest.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.task; 2 | 3 | import io.parallec.core.ParallelClient; 4 | import io.parallec.core.TestBase; 5 | 6 | import org.junit.AfterClass; 7 | import org.junit.BeforeClass; 8 | import org.junit.Test; 9 | 10 | public class ParallelTaskManagerTest extends TestBase { 11 | 12 | private static ParallelClient pc; 13 | 14 | @BeforeClass 15 | public static void setUp() throws Exception { 16 | pc = new ParallelClient(); 17 | } 18 | 19 | @AfterClass 20 | public static void shutdown() throws Exception { 21 | pc.releaseExternalResources(); 22 | } 23 | 24 | 25 | @Test 26 | public void testDirectorForException() { 27 | try { 28 | 29 | ParallelTaskManager.getInstance().getRemainingCapacity(); 30 | ParallelTaskManager.getInstance().getTaskFromInProgressMap("1"); 31 | ParallelTaskManager.getInstance().getInprogressTaskMap(); 32 | 33 | ParallelTaskManager.getInstance().sendTaskToExecutionManager(null); 34 | 35 | } catch (Exception ex) { 36 | logger.error("Expected Exception : " + ex.getLocalizedMessage()); 37 | } 38 | }// end func 39 | 40 | } 41 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/task/package-info.java: -------------------------------------------------------------------------------- 1 | 2 | /** 3 | * @author Yuanteng (Jeff) Pei 4 | * 5 | */ 6 | package io.parallec.core.task; -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/util/FileNetworkIoUtilsTest.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.util; 2 | 3 | import io.parallec.core.TestBase; 4 | 5 | import java.io.FileNotFoundException; 6 | 7 | import org.junit.Before; 8 | import org.junit.Test; 9 | 10 | public class FileNetworkIoUtilsTest extends TestBase { 11 | 12 | @Before 13 | public void setUp() throws Exception { 14 | } 15 | 16 | @Test 17 | public void test() throws FileNotFoundException { 18 | PcFileNetworkIoUtils.isFileExist(FILEPATH_TOP_100); 19 | PcFileNetworkIoUtils.readFileToInputStream(FILEPATH_TOP_100); 20 | PcFileNetworkIoUtils.readFileToInputStream(FILEPATH_TOP_100+"badpath"); 21 | PcFileNetworkIoUtils.getListFromLineByLineText(FILEPATH_TOP_100, 22 | SOURCE_LOCAL); 23 | PcFileNetworkIoUtils.getListFromLineByLineText(FILEPATH_TOP_100 + "a", 24 | SOURCE_LOCAL); 25 | 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/util/PcDateUtilsTest.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.util; 2 | 3 | import java.util.Date; 4 | 5 | import org.junit.Before; 6 | import org.junit.Test; 7 | 8 | public class PcDateUtilsTest { 9 | 10 | @Before 11 | public void setUp() throws Exception { 12 | } 13 | 14 | @Test 15 | public void testSuits() { 16 | new PcDateUtils(); 17 | 18 | PcDateUtils.getDateTimeStrConcise(new Date()); 19 | PcDateUtils.getDateTimeStrConcise(null); 20 | 21 | PcDateUtils.getNowDateTimeStrConcise(); 22 | PcDateUtils.getNowDateTimeStrStandard(); 23 | 24 | PcDateUtils.getDateTimeStrStandard(new Date()); 25 | PcDateUtils.getDateTimeStrStandard(new Date(0L)); 26 | PcDateUtils.getDateTimeStrStandard(null); 27 | 28 | PcDateUtils.getDateTimeStr(new Date()); 29 | PcDateUtils.getDateTimeStr(null); 30 | 31 | PcDateUtils.getDateFromConciseStr(""); 32 | PcDateUtils.getDateFromConciseStr(null); 33 | PcDateUtils.getDateFromConciseStr("20150913104745849-0700"); 34 | PcDateUtils.getDateFromConciseStr("20150913104745849aa-0700"); 35 | 36 | PcDateUtils.getDateTimeStrConciseNoZone(new Date()); 37 | PcDateUtils.getDateTimeStrConciseNoZone(null); 38 | 39 | PcDateUtils.getNowDateTimeStrConciseNoZone(); 40 | 41 | PcDateUtils.getNowDateTimeStr(); 42 | 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /src/test/java/io/parallec/core/util/PcErrorMsgUtilsTest.java: -------------------------------------------------------------------------------- 1 | package io.parallec.core.util; 2 | 3 | import io.parallec.core.TestBase; 4 | 5 | import java.io.FileNotFoundException; 6 | 7 | import org.junit.Before; 8 | import org.junit.Test; 9 | 10 | public class PcErrorMsgUtilsTest extends TestBase { 11 | 12 | @Before 13 | public void setUp() throws Exception { 14 | } 15 | 16 | @Test 17 | public void test() throws FileNotFoundException { 18 | PcErrorMsgUtils.replaceErrorMsg(null); 19 | 20 | PcErrorMsgUtils.replaceErrorMsg("java.net.ConnectException abc"); 21 | 22 | } 23 | 24 | } 25 | -------------------------------------------------------------------------------- /userdata/keepthisfile: -------------------------------------------------------------------------------- 1 | do not remove this for git 2 | -------------------------------------------------------------------------------- /userdata/sample_target_hosts_json_path.json: -------------------------------------------------------------------------------- 1 | { 2 | "sample": { 3 | "small-target-hosts": [ 4 | { 5 | "hostName": "parallec.github.io", 6 | "description": "parallec website" 7 | }, 8 | { 9 | "hostName": "www.jeffpei.com", 10 | "description": "my personal website" 11 | }, 12 | { 13 | "hostName": "www.restcommander.com", 14 | "description": "restcommander website" 15 | } 16 | ], 17 | "others": { 18 | "hostName": "www.google.com", 19 | "description": "google website" 20 | } 21 | }, 22 | "totalCount": 3 23 | } -------------------------------------------------------------------------------- /userdata/sample_target_hosts_top100.txt: -------------------------------------------------------------------------------- 1 | google.com 2 | youtube.com 3 | facebook.com 4 | msn.com 5 | twitter.com 6 | bing.com 7 | yelp.com 8 | amazon.com 9 | microsoft.com 10 | answers.com 11 | buzzfeed.com 12 | pinterest.com 13 | live.com 14 | yahoo.com 15 | ebay.com 16 | wikipedia.org 17 | aol.com 18 | wikia.com 19 | paypal.com 20 | ask.com 21 | linkedin.com 22 | about.com 23 | wordpress.com 24 | blogger.com 25 | diply.com 26 | craigslist.org 27 | bustle.com 28 | blogspot.com 29 | urbandictionary.com 30 | whitepages.com 31 | weather.com 32 | walmart.com 33 | mozilla.org 34 | stackexchange.com 35 | adobe.com 36 | wayfair.com 37 | vimeo.com 38 | quizlet.com 39 | sbnation.com 40 | nydailynews.com 41 | nytimes.com 42 | healthline.com 43 | imgur.com 44 | legacy.com 45 | wellsfargo.com 46 | usmagazine.com 47 | dose.com 48 | comcast.net 49 | topix.com 50 | ijreview.com 51 | mapquest.com 52 | godaddy.com 53 | uproxx.com 54 | go.com 55 | babycenter.com 56 | goodreads.com 57 | bizrate.com 58 | southwest.com 59 | refinery29.com 60 | glassdoor.com 61 | 247sports.com 62 | fandango.com 63 | nbcsports.com 64 | imdb.com 65 | instructables.com 66 | theverge.com 67 | thechive.com 68 | chase.com 69 | ibtimes.com 70 | people.com 71 | distractify.com 72 | rare.us 73 | yourdailydish.com 74 | deviantart.com 75 | apple.com 76 | city-data.com 77 | mic.com 78 | vox.com 79 | photobucket.com 80 | att.com 81 | ups.com 82 | hubpages.com 83 | usps.com 84 | al.com 85 | tmz.com 86 | theblaze.com 87 | moviepilot.com 88 | qpolitical.com 89 | rollingstone.com 90 | webmd.com 91 | instagram.com 92 | lifehacker.com 93 | rawstory.com 94 | booking.com 95 | guff.com 96 | ranker.com 97 | zillow.com 98 | yellowpages.com 99 | upworthy.com 100 | americanoverlook.com -------------------------------------------------------------------------------- /userdata/sample_target_hosts_top100_old.txt: -------------------------------------------------------------------------------- 1 | www.facebook.com 2 | www.google.com 3 | www.youtube.com 4 | www.yahoo.com 5 | www.baidu.com 6 | www.wikipedia.org 7 | www.live.com 8 | www.amazon.com 9 | www.twitter.com 10 | www.taobao.com 11 | www.google.co.in 12 | www.linkedin.com 13 | www.yahoo.co.jp 14 | www.msn.com 15 | www.yandex.ru 16 | www.ebay.com 17 | www.google.co.jp 18 | www.google.de 19 | www.wordpress.com 20 | www.bing.com 21 | www.google.com.hk 22 | www.vk.com 23 | www.google.co.uk 24 | www.google.fr 25 | www.babylon.com 26 | www.weibo.com 27 | www.microsoft.com 28 | www.tumblr.com 29 | www.googleusercontent.com 30 | www.mail.ru 31 | www.pinterest.com 32 | www.apple.com 33 | www.google.com.br 34 | www.loc.gov 35 | www.PayPal.com 36 | www.google.ru 37 | www.google.es 38 | www.google.it 39 | www.fc2.com 40 | www.blogger.com 41 | www.imdb.com 42 | www.craigslist.org 43 | www.ask.com 44 | www.conduit.com 45 | www.bbc.co.uk 46 | www.go.com 47 | www.amazon.co.jp 48 | www.google.com.mx 49 | www.google.ca 50 | www.amazon.de 51 | www.adobe.com 52 | www.flickr.com 53 | www.avg.com 54 | www.aol.com 55 | www.rakuten.co.jp 56 | www.cnn.com 57 | www.mywebsearch.com 58 | www.ebay.de 59 | www.amazon.co.uk 60 | www.alibaba.com 61 | www.espn.go.com 62 | www.blogspot.in 63 | www.google.com.tr 64 | www.google.co.id 65 | www.instagram.com 66 | www.huffingtonpost.com 67 | www.about.com 68 | www.stackoverflow.com 69 | www.google.com.au 70 | www.livedoor.com 71 | www.ebay.co.uk 72 | www.netflix.com 73 | www.dailymotion.com 74 | www.imgur.com 75 | www.zedo.com 76 | www.google.pl 77 | www.deviantart.com 78 | www.google.nl 79 | www.dropbox.com 80 | www.ameblo.jp 81 | www.themeforest.net 82 | www.weather.com 83 | www.dailymail.co.uk 84 | www.slideshare.net 85 | www.booking.com 86 | www.yelp.com 87 | www.livejournal.com 88 | www.skype.com 89 | www.wikihow.com 90 | www.walmart.com 91 | www.forbes.com 92 | www.starpulse.com 93 | www.sfgate.com 94 | www.toptenvideos.com 95 | www.walgreens.com 96 | www.liveleak.com 97 | www.discover.com 98 | www.mosthappy.com 99 | www.reverbnation.com 100 | www.irs.gov --------------------------------------------------------------------------------