├── conf ├── varonedaemond ├── varOne-env.sh.template ├── log4j.properties └── varOne-site.xml.template ├── varOne-server ├── .gitignore ├── src │ ├── main │ │ ├── resources │ │ │ └── log4j.properties │ │ └── java │ │ │ └── com │ │ │ └── varone │ │ │ └── web │ │ │ ├── exception │ │ │ ├── VarOneException.java │ │ │ └── VarOneExceptionParser.java │ │ │ ├── aggregator │ │ │ ├── timeperiod │ │ │ │ ├── SecondPeriodIngestor.java │ │ │ │ ├── TimePeriod.java │ │ │ │ └── PeriodDataIngestor.java │ │ │ └── unit │ │ │ │ ├── TimeUnitTransfer.java │ │ │ │ ├── AbstractUnitAggregator.java │ │ │ │ ├── QuantityAggregator.java │ │ │ │ └── AverageAggregator.java │ │ │ ├── vo │ │ │ ├── TimeValueVO.java │ │ │ ├── MetricCompletedTasksVO.java │ │ │ ├── DefaultNodeVO.java │ │ │ ├── MetricPropVO.java │ │ │ ├── HistoryVO.java │ │ │ ├── HistoryDetailStageVO.java │ │ │ ├── SummaryExecutorVO.java │ │ │ ├── StageVO.java │ │ │ ├── JobVO.java │ │ │ ├── DefaultTotalNodeVO.java │ │ │ └── DefaultApplicationVO.java │ │ │ ├── reader │ │ │ ├── metrics │ │ │ │ ├── MetricsReader.java │ │ │ │ └── stub │ │ │ │ │ └── RpcThreadListener.java │ │ │ └── eventlog │ │ │ │ └── EventLogReader.java │ │ │ ├── metrics │ │ │ └── bean │ │ │ │ ├── TimeValuePairBean.java │ │ │ │ ├── NodeBean.java │ │ │ │ └── MetricBean.java │ │ │ ├── listener │ │ │ └── VarOneServletContextListener.java │ │ │ └── resource │ │ │ └── ClusterResource.java │ └── test │ │ └── java │ │ └── com │ │ └── varone │ │ └── web │ │ ├── aggregator │ │ ├── UIDataAggregatorTest.java │ │ └── timeperiod │ │ │ └── TimePeriodTest.java │ │ ├── reader │ │ └── metrics │ │ │ └── impl │ │ │ ├── SparkEventLogParserTest.java │ │ │ └── MetricsRpcReaderImplTest.java │ │ ├── yarn │ │ └── service │ │ │ └── YarnServiceTest.java │ │ └── metrics │ │ └── client │ │ └── MetricsStubMultiThreadTest.java └── build.gradle ├── varOne-web ├── src │ └── main │ │ ├── webapp │ │ ├── app │ │ │ ├── styles │ │ │ │ ├── app.scss │ │ │ │ ├── bootstrap │ │ │ │ │ ├── customizations.scss │ │ │ │ │ └── pre-customizations.scss │ │ │ │ └── spm │ │ │ │ │ └── spm.css │ │ │ ├── alt.js │ │ │ ├── utils │ │ │ │ ├── router.js │ │ │ │ └── data-format.js │ │ │ ├── actions │ │ │ │ ├── varOne-action.js │ │ │ │ ├── menu-action.js │ │ │ │ ├── job-action.js │ │ │ │ ├── cluster-action.js │ │ │ │ ├── node-action.js │ │ │ │ └── history-action.js │ │ │ ├── components │ │ │ │ ├── app.js │ │ │ │ ├── navigator │ │ │ │ │ ├── menu.js │ │ │ │ │ ├── brand.js │ │ │ │ │ ├── nav.js │ │ │ │ │ └── menu-item.js │ │ │ │ ├── history │ │ │ │ │ ├── history-header.js │ │ │ │ │ ├── history-list.js │ │ │ │ │ ├── stage-list.js │ │ │ │ │ └── job-list.js │ │ │ │ ├── nodes │ │ │ │ │ ├── nodes-metrics.js │ │ │ │ │ ├── nodes-header.js │ │ │ │ │ └── nodes-metric.js │ │ │ │ ├── commons │ │ │ │ │ ├── process-bar.js │ │ │ │ │ ├── breadcrumb.js │ │ │ │ │ ├── time-period-pills.js │ │ │ │ │ └── varOne-log-modal.js │ │ │ │ ├── job │ │ │ │ │ ├── job-header.js │ │ │ │ │ ├── job-metrics.js │ │ │ │ │ ├── job-summary-block.js │ │ │ │ │ └── job-metric.js │ │ │ │ ├── cluster │ │ │ │ │ ├── cluster-header.js │ │ │ │ │ ├── cluster-metrics.js │ │ │ │ │ ├── cluster-summary-block.js │ │ │ │ │ └── cluster-metric.js │ │ │ │ └── home.js │ │ │ ├── routes.js │ │ │ ├── stores │ │ │ │ ├── varOne-store.js │ │ │ │ ├── menu-store.js │ │ │ │ ├── node-store.js │ │ │ │ └── history-store.js │ │ │ └── main.js │ │ ├── .babelrc │ │ ├── tests.webpack.js │ │ ├── test │ │ │ ├── utils │ │ │ │ └── stub-app.js │ │ │ └── spec │ │ │ │ ├── components │ │ │ │ └── cluster │ │ │ │ │ ├── cluster-metric.test.js │ │ │ │ │ ├── cluster-metrics.test.js │ │ │ │ │ ├── cluster-summary-block.test.js │ │ │ │ │ ├── cluster-summary.test.js │ │ │ │ │ └── cluster-header.test.js │ │ │ │ ├── stores │ │ │ │ └── cluster-store.test.js │ │ │ │ └── actions │ │ │ │ └── cluster-action.test.js │ │ ├── webpack │ │ │ ├── webpack-dev-server.js │ │ │ ├── production-config.js │ │ │ └── dev-config.js │ │ ├── index.html │ │ ├── WEB-INF │ │ │ └── web.xml │ │ ├── .eslintrc │ │ ├── package.json │ │ └── karma.conf.js │ │ └── resources │ │ ├── varOne.properties │ │ ├── runnpm.sh │ │ ├── log4j.properties │ │ └── index.html ├── build.gradle └── hadoopConf │ └── core-site.xml ├── varOne-rpc ├── build.gradle └── src │ ├── main │ ├── java │ │ └── com │ │ │ └── varone │ │ │ └── hadoop │ │ │ └── rpc │ │ │ ├── IService.java │ │ │ ├── metrics │ │ │ ├── MetricsService.java │ │ │ ├── MetricsServiceResponse.java │ │ │ ├── MetricsServiceRequest.java │ │ │ ├── MetricsServicePB.java │ │ │ └── impl │ │ │ │ ├── pb │ │ │ │ ├── service │ │ │ │ │ └── MetricsServicePBServiceImpl.java │ │ │ │ └── client │ │ │ │ │ └── MetricsServicePBClientImpl.java │ │ │ │ ├── MetricsServiceResponsePBImpl.java │ │ │ │ └── MetricsServiceRequestPBImpl.java │ │ │ └── AbstractServer.java │ └── resources │ │ ├── metricsService.proto │ │ └── metrics.proto │ └── test │ └── java │ └── com │ └── varone │ ├── MetricsMockNode.java │ ├── MetricsMockClient.java │ └── MetricsMockService.java ├── settings.gradle ├── docs └── README.md ├── varOne-node ├── src │ ├── test │ │ └── resources │ │ │ ├── metrics.withoutDir.properties │ │ │ ├── metrics.withoutPeriod.properties │ │ │ ├── metrics.withoutUnit.properties │ │ │ ├── metrics.valid.properties │ │ │ └── csvsink │ │ │ ├── application_1439169262151_0237.1.jvm.PS-MarkSweep.count.csv │ │ │ ├── application_1439169262151_0237.1.jvm.PS-MarkSweep.time.csv │ │ │ ├── application_1439169262151_0237.5.jvm.PS-MarkSweep.count.csv │ │ │ ├── application_1439169262151_0237.5.jvm.PS-MarkSweep.time.csv │ │ │ ├── application_1439169262151_0237.1.executor.filesystem.file.read_ops.csv │ │ │ ├── application_1439169262151_0237.1.executor.filesystem.file.write_ops.csv │ │ │ ├── application_1439169262151_0237.1.executor.filesystem.hdfs.write_ops.csv │ │ │ ├── application_1439169262151_0237.1.executor.threadpool.activeTasks.csv │ │ │ ├── application_1439169262151_0237.5.executor.filesystem.file.read_ops.csv │ │ │ ├── application_1439169262151_0237.5.executor.filesystem.file.write_ops.csv │ │ │ ├── application_1439169262151_0237.5.executor.filesystem.hdfs.write_ops.csv │ │ │ └── application_1439169262151_0237.5.executor.threadpool.activeTasks.csv │ └── main │ │ ├── resources │ │ └── log4j.properties │ │ └── java │ │ └── com │ │ └── varone │ │ └── node │ │ ├── MetricsProperties.java │ │ ├── utils │ │ ├── Consts.java │ │ ├── MetricsPropertiesParser.java │ │ └── MetricsDataTransfer.java │ │ ├── MetricTuple.java │ │ └── reader │ │ ├── MetricsReader.java │ │ ├── FsMetricsReader.java │ │ ├── ThreadPoolMetricsReader.java │ │ └── JvmMetricsReader.java └── build.gradle ├── .gitignore ├── bin ├── varOned-all.sh ├── varOne.sh ├── varOned.sh └── release.sh └── LICENSE /conf/varonedaemond: -------------------------------------------------------------------------------- 1 | localhost 2 | -------------------------------------------------------------------------------- /varOne-server/.gitignore: -------------------------------------------------------------------------------- 1 | /bin/ 2 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/styles/app.scss: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /varOne-web/src/main/resources/varOne.properties: -------------------------------------------------------------------------------- 1 | varOne.node.port= -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/.babelrc: -------------------------------------------------------------------------------- 1 | { 2 | "stage": 0 3 | } 4 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/styles/bootstrap/customizations.scss: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/styles/bootstrap/pre-customizations.scss: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /varOne-web/src/main/resources/runnpm.sh: -------------------------------------------------------------------------------- 1 | cd ${1} 2 | npm run production 3 | cd - 4 | -------------------------------------------------------------------------------- /varOne-rpc/build.gradle: -------------------------------------------------------------------------------- 1 | dependencies { 2 | compile configurations.hadoopConf 3 | } 4 | -------------------------------------------------------------------------------- /settings.gradle: -------------------------------------------------------------------------------- 1 | include ":varOne-node", ":varOne-rpc", ":varOne-web", ":varOne-server" 2 | -------------------------------------------------------------------------------- /docs/README.md: -------------------------------------------------------------------------------- 1 | ## Table of Contents 2 | * [RESTful](/docs/api.md) 3 | * [Development](/docs/development.md) 4 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/alt.js: -------------------------------------------------------------------------------- 1 | const Alt = require('alt'); 2 | const alt = new Alt(); 3 | 4 | module.exports = alt; 5 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/tests.webpack.js: -------------------------------------------------------------------------------- 1 | // Browser ES6 Polyfill 2 | require('babel/polyfill'); 3 | const context = require.context('./test/spec', true, /\.test\.jsx$|\.test\.js$/); 4 | context.keys().forEach(context); 5 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/styles/spm/spm.css: -------------------------------------------------------------------------------- 1 | @import '../../../node_modules/font-awesome/css/font-awesome.css'; 2 | 3 | .breadcrumb > li + li:before { 4 | color: #ccc; 5 | content: " > "; 6 | padding: 0 5px; 7 | } 8 | -------------------------------------------------------------------------------- /varOne-node/src/test/resources/metrics.withoutDir.properties: -------------------------------------------------------------------------------- 1 | # Enable CsvSink for all instances 2 | *.sink.csv.class=org.apache.spark.metrics.sink.CsvSink 3 | 4 | # Polling period for CsvSink 5 | *.sink.csv.period=1 6 | 7 | *.sink.csv.unit=seconds 8 | -------------------------------------------------------------------------------- /varOne-node/src/test/resources/metrics.withoutPeriod.properties: -------------------------------------------------------------------------------- 1 | # Enable CsvSink for all instances 2 | *.sink.csv.class=org.apache.spark.metrics.sink.CsvSink 3 | 4 | *.sink.csv.unit=seconds 5 | 6 | # Polling directory for CsvSink 7 | *.sink.csv.directory=/tmp/csvsink -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/utils/router.js: -------------------------------------------------------------------------------- 1 | import Router, { HistoryLocation } from 'react-router'; 2 | import routes from '../routes'; 3 | 4 | const config = { routes }; 5 | config.location = HistoryLocation; 6 | 7 | export default Router.create(config); 8 | -------------------------------------------------------------------------------- /varOne-node/src/test/resources/metrics.withoutUnit.properties: -------------------------------------------------------------------------------- 1 | # Enable CsvSink for all instances 2 | *.sink.csv.class=org.apache.spark.metrics.sink.CsvSink 3 | 4 | # Polling period for CsvSink 5 | *.sink.csv.period=1 6 | 7 | # Polling directory for CsvSink 8 | *.sink.csv.directory=/tmp/csvsink -------------------------------------------------------------------------------- /varOne-rpc/src/main/java/com/varone/hadoop/rpc/IService.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.hadoop.rpc; 5 | 6 | /** 7 | * @author allen 8 | * 9 | */ 10 | public interface IService { 11 | void startService() throws Exception; 12 | void stopService() throws Exception; 13 | } 14 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/actions/varOne-action.js: -------------------------------------------------------------------------------- 1 | import alt from '../alt'; 2 | class VarOneAction { 3 | 4 | constructor() { 5 | this.generateActions('showLogStatus'); 6 | this.generateActions('showFailMessage'); 7 | } 8 | } 9 | 10 | export default alt.createActions(VarOneAction); 11 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/components/app.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import Home from './home'; 3 | 4 | class App extends React.Component { 5 | 6 | render() { 7 | return ( 8 |
9 | 10 |
11 | ); 12 | } 13 | } 14 | 15 | export default App; 16 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/routes.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import { Route, DefaultRoute } from 'react-router'; 3 | 4 | export default ( 5 | 6 | 7 | 8 | ); 9 | -------------------------------------------------------------------------------- /varOne-node/src/test/resources/metrics.valid.properties: -------------------------------------------------------------------------------- 1 | # Enable CsvSink for all instances 2 | *.sink.csv.class=org.apache.spark.metrics.sink.CsvSink 3 | 4 | # Polling period for CsvSink 5 | *.sink.csv.period=1 6 | 7 | *.sink.csv.unit=seconds 8 | 9 | # Polling directory for CsvSink 10 | *.sink.csv.directory=/tmp/csvsink -------------------------------------------------------------------------------- /varOne-rpc/src/main/java/com/varone/hadoop/rpc/metrics/MetricsService.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.hadoop.rpc.metrics; 5 | 6 | /** 7 | * @author allen 8 | * 9 | */ 10 | public interface MetricsService { 11 | MetricsServiceResponse getMetrics(MetricsServiceRequest request) throws Exception; 12 | } 13 | -------------------------------------------------------------------------------- /varOne-node/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=INFO,stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%p] %c{2}: %m%n 9 | -------------------------------------------------------------------------------- /varOne-rpc/src/main/resources/metricsService.proto: -------------------------------------------------------------------------------- 1 | option java_package = "com.varone.hadoop.rpc.protos"; 2 | option java_outer_classname = "MetricsServiceProtos"; 3 | option java_generic_services = true; 4 | option java_generate_equals_and_hash = true; 5 | 6 | import "metrics.proto"; 7 | 8 | service MetricsService { 9 | rpc getMetrics(MetricsRequestProto) returns (MetricsResponseProto); 10 | } -------------------------------------------------------------------------------- /varOne-server/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=INFO,stdout 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%p] %c{2}: %m%n 9 | -------------------------------------------------------------------------------- /conf/varOne-env.sh.template: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | # 3 | 4 | # export VARONE_WAR_TEMPDIR # The location of jetty temporary directory. 5 | 6 | # export SPARK_HOME # (required) spark location, please make sure metrics.properties has been configured. 7 | # export HADOOP_CONF_DIR # (required) yarn-site.xml is located in configuration directory in HADOOP_CONF_DIR. 8 | # export VARONE_LOG_DIR 9 | -------------------------------------------------------------------------------- /varOne-rpc/src/main/java/com/varone/hadoop/rpc/metrics/MetricsServiceResponse.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.hadoop.rpc.metrics; 5 | 6 | import java.util.List; 7 | 8 | import com.varone.hadoop.rpc.protos.MetricsProtos.MetricsResponseProto.MetricsMapProto; 9 | 10 | /** 11 | * @author allen 12 | * 13 | */ 14 | public interface MetricsServiceResponse { 15 | List getResult(); 16 | void setResult(List result); 17 | } 18 | -------------------------------------------------------------------------------- /varOne-server/src/main/java/com/varone/web/exception/VarOneException.java: -------------------------------------------------------------------------------- 1 | package com.varone.web.exception; 2 | 3 | import javax.ws.rs.WebApplicationException; 4 | import javax.ws.rs.core.MediaType; 5 | import javax.ws.rs.core.Response; 6 | 7 | public class VarOneException extends WebApplicationException{ 8 | public VarOneException(String message){ 9 | super(Response.status(Response.Status.BAD_REQUEST).entity(message).type(MediaType.TEXT_PLAIN).build()); 10 | } 11 | } 12 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/components/navigator/menu.js: -------------------------------------------------------------------------------- 1 | import React, { PropTypes } from 'react'; 2 | 3 | export default class Menu extends React.Component { 4 | 5 | static propTypes = { 6 | children: PropTypes.element 7 | } 8 | 9 | constructor(props) { 10 | super(props); 11 | } 12 | 13 | render() { 14 | return ( 15 | 18 | ); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /varOne-server/src/main/java/com/varone/web/aggregator/timeperiod/SecondPeriodIngestor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.web.aggregator.timeperiod; 5 | 6 | import java.util.List; 7 | 8 | import com.varone.web.metrics.bean.TimeValuePairBean; 9 | 10 | /** 11 | * @author allen 12 | * 13 | */ 14 | public class SecondPeriodIngestor extends PeriodDataIngestor { 15 | 16 | public SecondPeriodIngestor(List dataSet) { 17 | super(dataSet); 18 | } 19 | } 20 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/test/utils/stub-app.js: -------------------------------------------------------------------------------- 1 | /* eslint react/display-name: 0 */ 2 | import React, { Component } from 'react'; 3 | 4 | export default function stubApp() { 5 | return function (DecoratedComponent, props) { 6 | return class Wrapper extends Component { 7 | 8 | render() { 9 | const customProps = { ...this.props, ...props }; 10 | return ( 11 | 12 | ); 13 | } 14 | }; 15 | }; 16 | } 17 | -------------------------------------------------------------------------------- /varOne-node/build.gradle: -------------------------------------------------------------------------------- 1 | def project_rpc = project(":varOne-rpc") 2 | 3 | dependencies { 4 | compile project_rpc 5 | 6 | } 7 | 8 | shadowJar { 9 | //jar filename = [baseName]-[version]-[classifier].[extension] 10 | baseName = "varOned" 11 | version= project.version 12 | classifier = "" //default:all 13 | extension = "jar" 14 | 15 | mergeServiceFiles() 16 | } 17 | 18 | 19 | jar { 20 | manifest { 21 | attributes 'Main-Class': 'com.varone.node.VarOned' 22 | } 23 | } 24 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/components/history/history-header.js: -------------------------------------------------------------------------------- 1 | import React, { PropTypes } from 'react'; 2 | 3 | export default class HistoryHeader extends React.Component { 4 | 5 | static propTypes = { 6 | children: PropTypes.element 7 | } 8 | 9 | render() { 10 | const headerStyle = { 11 | borderBottom: '1px solid #e7ecf1' 12 | }; 13 | 14 | return ( 15 |
16 | { this.props.children } 17 |
18 | ); 19 | } 20 | } 21 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *~ 2 | .DS_Store 3 | .project 4 | .gradle/ 5 | varOne-web/build/ 6 | varOne-node/build/ 7 | varOne-rpc/build/ 8 | varOne-exec/build/ 9 | varOne-server/build/ 10 | node_modules 11 | *.jar 12 | *.war 13 | varOne-web/src/main/webapp/npm-debug.log 14 | .classpath 15 | .settings 16 | varOne-web/bin/ 17 | varOne-node/bin/ 18 | varOne-rpc/bin/ 19 | varOne-exec/bin/ 20 | varOne-server/bin/ 21 | varOne-web/src/main/webapp/dist 22 | varOne.log 23 | conf/*.xml 24 | conf/*.sh 25 | varOne*.tgz 26 | coverage 27 | report 28 | -------------------------------------------------------------------------------- /varOne-server/src/main/java/com/varone/web/vo/TimeValueVO.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.web.vo; 5 | 6 | /** 7 | * @author allen 8 | * 9 | */ 10 | public class TimeValueVO { 11 | private String time; 12 | private String value; 13 | public String getTime() { 14 | return time; 15 | } 16 | public void setTime(String time) { 17 | this.time = time; 18 | } 19 | public String getValue() { 20 | return value; 21 | } 22 | public void setValue(String value) { 23 | this.value = value; 24 | } 25 | 26 | 27 | } 28 | -------------------------------------------------------------------------------- /varOne-rpc/src/main/java/com/varone/hadoop/rpc/metrics/MetricsServiceRequest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.hadoop.rpc.metrics; 5 | 6 | import java.util.List; 7 | 8 | import com.varone.hadoop.rpc.protos.MetricsProtos.MetricsTypeProto; 9 | 10 | /** 11 | * @author allen 12 | * 13 | */ 14 | public interface MetricsServiceRequest { 15 | String getApplicationId(); 16 | void setApplicationId(String applicationId); 17 | 18 | List getMetrics(); 19 | void setMetrics(List metrics); 20 | } 21 | -------------------------------------------------------------------------------- /varOne-rpc/src/test/java/com/varone/MetricsMockNode.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | 8 | /** 9 | * @author allen 10 | * 11 | */ 12 | public class MetricsMockNode { 13 | 14 | /** 15 | * @throws Exception 16 | * 17 | */ 18 | public static void main(String[] args) throws Exception { 19 | 20 | Configuration config = new Configuration(); 21 | 22 | MetricsMockService service = new MetricsMockService(config); 23 | service.start(); 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /varOne-rpc/src/main/java/com/varone/hadoop/rpc/metrics/MetricsServicePB.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.hadoop.rpc.metrics; 5 | 6 | import org.apache.hadoop.ipc.ProtocolInfo; 7 | 8 | import com.varone.hadoop.rpc.protos.MetricsServiceProtos.MetricsService.BlockingInterface; 9 | 10 | 11 | /** 12 | * @author allen 13 | * 14 | */ 15 | @ProtocolInfo( 16 | protocolName = "com.varone.hadoop.rpc.metrics.MetricsRpcService", 17 | protocolVersion = 1) 18 | public interface MetricsServicePB extends BlockingInterface { 19 | 20 | } 21 | -------------------------------------------------------------------------------- /varOne-server/src/main/java/com/varone/web/reader/metrics/MetricsReader.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.web.reader.metrics; 5 | 6 | import java.util.List; 7 | 8 | import com.varone.web.metrics.bean.NodeBean; 9 | 10 | /** 11 | * @author allen 12 | * 13 | */ 14 | public interface MetricsReader { 15 | public List getAllNodeMetrics(String applicationId, List metricsType) throws Exception; 16 | 17 | public NodeBean getNodeMetrics(String node, String applicationId, List metricsType) throws Exception; 18 | } 19 | -------------------------------------------------------------------------------- /varOne-server/src/main/java/com/varone/web/exception/VarOneExceptionParser.java: -------------------------------------------------------------------------------- 1 | package com.varone.web.exception; 2 | 3 | public class VarOneExceptionParser { 4 | 5 | public String parse(Exception e){ 6 | StackTraceElement[] stackTraceElement = e.getStackTrace(); 7 | String message = e.getMessage() + ":" + e + " \n"; 8 | for(StackTraceElement stackTrace : stackTraceElement){ 9 | message = message + "\t" + stackTrace.getClassName() + "(" + stackTrace.getFileName() + ":" + stackTrace.getLineNumber() + ")\n"; 10 | } 11 | return message; 12 | } 13 | 14 | } 15 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/webpack/webpack-dev-server.js: -------------------------------------------------------------------------------- 1 | require('babel/register'); 2 | 3 | const debug = require('debug')('dev'); 4 | const webpack = require('webpack'); 5 | const WebpackDevServer = require('webpack-dev-server'); 6 | 7 | const devConfig = require('./dev-config'); 8 | 9 | const compiler = webpack(devConfig.webpack); 10 | const devServer = new WebpackDevServer(compiler, devConfig.server.options); 11 | 12 | devServer.listen(devConfig.server.port, 'localhost', function () { 13 | debug('webpack-dev-server listen on port %s', devConfig.server.port); 14 | }); 15 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/stores/varOne-store.js: -------------------------------------------------------------------------------- 1 | import alt from '../alt'; 2 | import VarOneAction from '../actions/varOne-action'; 3 | 4 | class VarOneStore { 5 | 6 | constructor() { 7 | this.bindActions(VarOneAction); 8 | this.msg = null; 9 | this.showLogClick = false; 10 | this.failMessage = null; 11 | } 12 | 13 | onShowLogStatus() { 14 | this.showLogClick = true; 15 | } 16 | 17 | onShowFailMessage(failMessage) { 18 | this.failMessage = this.failMessage + failMessage; 19 | } 20 | } 21 | 22 | export default alt.createStore(VarOneStore); 23 | -------------------------------------------------------------------------------- /varOne-server/src/main/java/com/varone/web/aggregator/unit/TimeUnitTransfer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.web.aggregator.unit; 5 | 6 | 7 | /** 8 | * @author allen 9 | * 10 | */ 11 | public class TimeUnitTransfer { 12 | 13 | 14 | public String transferToAxisX(long value) { 15 | // SimpleDateFormat sdf = new SimpleDateFormat("HH:mm:ss"); 16 | int length = String.valueOf(value).length(); 17 | 18 | //without millisecond 19 | if(length == 10) { 20 | // return sdf.format(value * 1000); 21 | return String.valueOf(value * 1000); 22 | } 23 | 24 | return null; 25 | } 26 | 27 | } 28 | -------------------------------------------------------------------------------- /varOne-server/src/main/java/com/varone/web/aggregator/unit/AbstractUnitAggregator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.web.aggregator.unit; 5 | 6 | import com.varone.web.metrics.bean.TimeValuePairBean; 7 | import com.varone.web.vo.TimeValueVO; 8 | 9 | /** 10 | * @author allen 11 | * 12 | */ 13 | public abstract class AbstractUnitAggregator { 14 | 15 | protected double getDoubleValue(String value){ 16 | return Double.parseDouble(value); 17 | } 18 | 19 | protected long getLongValue(String value){ 20 | return Long.parseLong(value); 21 | } 22 | 23 | public abstract void aggregate(String format, TimeValuePairBean newPair, TimeValueVO currPair); 24 | } 25 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/components/nodes/nodes-metrics.js: -------------------------------------------------------------------------------- 1 | import React, { PropTypes } from 'react'; 2 | import NodeMetric from './nodes-metric'; 3 | 4 | class NodeMetrics extends React.Component { 5 | 6 | static propTypes = { 7 | metrics: PropTypes.array 8 | } 9 | 10 | render() { 11 | const metricCharts = this.props.metrics.map((metric, i) => { 12 | return ( 13 | 14 | ); 15 | }); 16 | 17 | return ( 18 |
19 |
20 | { metricCharts } 21 |
22 |
23 | ); 24 | } 25 | } 26 | export default NodeMetrics; 27 | -------------------------------------------------------------------------------- /varOne-server/src/main/java/com/varone/web/reader/metrics/stub/RpcThreadListener.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.web.reader.metrics.stub; 5 | 6 | /** 7 | * @author allen 8 | * 9 | */ 10 | public class RpcThreadListener { 11 | private final static int MIN_THREAD_COUNT = 0; 12 | private int count; 13 | 14 | public RpcThreadListener(){ 15 | this.count = 0; 16 | } 17 | 18 | public synchronized void increaseRpcThreadCount(){ 19 | this.count++; 20 | } 21 | 22 | public synchronized void decreaseRpcThreadCount(){ 23 | this.count--; 24 | if(this.count == MIN_THREAD_COUNT) notify(); 25 | } 26 | 27 | public synchronized boolean isAllComplete(){ 28 | return this.count == MIN_THREAD_COUNT; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /varOne-server/src/main/java/com/varone/web/metrics/bean/TimeValuePairBean.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.web.metrics.bean; 5 | 6 | /** 7 | * @author allen 8 | * 9 | */ 10 | public class TimeValuePairBean { 11 | private long time; 12 | private String value; 13 | /** 14 | * @return the time 15 | */ 16 | public long getTime() { 17 | return time; 18 | } 19 | /** 20 | * @param time the time to set 21 | */ 22 | public void setTime(long time) { 23 | this.time = time; 24 | } 25 | /** 26 | * @return the value 27 | */ 28 | public String getValue() { 29 | return value; 30 | } 31 | /** 32 | * @param value the value to set 33 | */ 34 | public void setValue(String value) { 35 | this.value = value; 36 | } 37 | 38 | 39 | } 40 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/actions/menu-action.js: -------------------------------------------------------------------------------- 1 | import alt from '../alt'; 2 | import VarOneAction from '../actions/varOne-action'; 3 | import request from 'superagent'; 4 | class MenuAction { 5 | constructor() { 6 | this.generateActions('changErrorMessageIcon'); 7 | } 8 | async fetchRunningJob() { 9 | try { 10 | const response = await request.get('/varOne-web/rest/job') 11 | .set('Accept', 'application/json'); 12 | const result = JSON.parse(response.text); 13 | this.dispatch(result); 14 | } catch (e) { 15 | VarOneAction.showFailMessage(e.response.text); 16 | this.changErrorMessageIcon(true); 17 | } 18 | } 19 | 20 | } 21 | 22 | export default alt.createActions(MenuAction); 23 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/components/navigator/brand.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | 3 | class Brand extends React.Component { 4 | render() { 5 | return ( 6 |
7 | 14 | varOne 16 |
17 | ); 18 | } 19 | } 20 | 21 | export default Brand; 22 | -------------------------------------------------------------------------------- /conf/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=INFO,stdout,R 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%p] %c{2}: %m%n 9 | 10 | 11 | log4j.appender.R=org.apache.log4j.RollingFileAppender 12 | log4j.appender.R.File= ${varOne.log.file} 13 | # ***** Max file size is set to 100KB 14 | log4j.appender.R.MaxFileSize=10000KB 15 | # ***** Keep one backup file 16 | log4j.appender.R.MaxBackupIndex=1 17 | # ***** R uses PatternLayout. 18 | log4j.appender.R.layout=org.apache.log4j.PatternLayout 19 | log4j.appender.R.layout.ConversionPattern=%p %t %c - %m%n 20 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/components/commons/process-bar.js: -------------------------------------------------------------------------------- 1 | import React, { PropTypes } from 'react'; 2 | 3 | export default class ProcessBar extends React.Component { 4 | 5 | static propTypes = { 6 | successPercent: PropTypes.string, 7 | children: PropTypes.element 8 | } 9 | 10 | render() { 11 | const style = { 12 | width: this.props.successPercent 13 | }; 14 | 15 | const progressStyle = { 16 | marginBottom: '0px' 17 | }; 18 | return ( 19 |
20 |
24 | { this.props.children } 25 |
26 |
27 | ); 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /varOne-server/src/main/java/com/varone/web/vo/MetricCompletedTasksVO.java: -------------------------------------------------------------------------------- 1 | package com.varone.web.vo; 2 | 3 | public class MetricCompletedTasksVO { 4 | public String metric; 5 | public long min; 6 | public double median; 7 | public long max; 8 | 9 | public String getMetric() { 10 | return metric; 11 | } 12 | public void setMetric(String metric) { 13 | this.metric = metric; 14 | } 15 | public long getMin() { 16 | return min; 17 | } 18 | public void setMin(long min) { 19 | this.min = min; 20 | } 21 | public double getMedian() { 22 | return median; 23 | } 24 | public void setMedian(double median) { 25 | this.median = median; 26 | } 27 | public long getMax() { 28 | return max; 29 | } 30 | public void setMax(long max) { 31 | this.max = max; 32 | } 33 | 34 | 35 | } 36 | -------------------------------------------------------------------------------- /varOne-web/src/main/resources/log4j.properties: -------------------------------------------------------------------------------- 1 | # Root logger option 2 | log4j.rootLogger=INFO,stdout,R 3 | 4 | # Direct log messages to stdout 5 | log4j.appender.stdout=org.apache.log4j.ConsoleAppender 6 | log4j.appender.stdout.Target=System.out 7 | log4j.appender.stdout.layout=org.apache.log4j.PatternLayout 8 | log4j.appender.stdout.layout.ConversionPattern=%d{yy/MM/dd HH:mm:ss} [%p] %c{2}: %m%n 9 | 10 | 11 | log4j.appender.R=org.apache.log4j.RollingFileAppender 12 | log4j.appender.R.File=varOne.log 13 | # ***** Max file size is set to 100KB 14 | log4j.appender.R.MaxFileSize=10000KB 15 | # ***** Keep one backup file 16 | log4j.appender.R.MaxBackupIndex=1 17 | # ***** R uses PatternLayout. 18 | log4j.appender.R.layout=org.apache.log4j.PatternLayout 19 | log4j.appender.R.layout.ConversionPattern=%p %t %c - %m%n -------------------------------------------------------------------------------- /varOne-server/src/main/java/com/varone/web/reader/eventlog/EventLogReader.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.web.reader.eventlog; 5 | 6 | import java.util.List; 7 | import java.util.Map; 8 | 9 | import com.varone.web.eventlog.bean.SparkEventLogBean; 10 | 11 | /** 12 | * @author allen 13 | * 14 | */ 15 | public interface EventLogReader { 16 | 17 | public List getAllSparkAppLog() throws Exception; 18 | 19 | public Map getAllInProgressLog() throws Exception; 20 | 21 | public SparkEventLogBean getInProgressLog(String applicationId) throws Exception; 22 | 23 | public SparkEventLogBean getApplicationJobs(String applicationId) throws Exception; 24 | 25 | public SparkEventLogBean getJobStages(String applicationId, String jobId) throws Exception; 26 | } 27 | -------------------------------------------------------------------------------- /varOne-node/src/main/java/com/varone/node/MetricsProperties.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.node; 5 | 6 | /** 7 | * @author allen 8 | * 9 | */ 10 | public class MetricsProperties { 11 | /* *.sink.csv.period */ 12 | private String csvPeriod; 13 | /* *.sink.csv.unit */ 14 | private String csvUnit; 15 | /* *.sink.csv.directory */ 16 | private String csvDir; 17 | 18 | public String getCsvPeriod() { 19 | return csvPeriod; 20 | } 21 | public void setCsvPeriod(String csvPeriod) { 22 | this.csvPeriod = csvPeriod; 23 | } 24 | public String getCsvUnit() { 25 | return csvUnit; 26 | } 27 | public void setCsvUnit(String csvUnit) { 28 | this.csvUnit = csvUnit; 29 | } 30 | public String getCsvDir() { 31 | return csvDir; 32 | } 33 | public void setCsvDir(String csvDir) { 34 | this.csvDir = csvDir; 35 | } 36 | } 37 | -------------------------------------------------------------------------------- /bin/varOned-all.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | USAGE="Usage: bin/varOned-all.sh [start|stop]" 4 | 5 | if [[ "$1" == "" ]]; then 6 | echo ${USAGE} 7 | exit 1 8 | fi 9 | 10 | bin=$(dirname "${BASH_SOURCE-$0}") 11 | bin=$(cd "${bin}">/dev/null; pwd) 12 | 13 | if [ -L ${BASH_SOURCE-$0} ]; then 14 | FWDIR=$(dirname $(readlink "${BASH_SOURCE-$0}")) 15 | else 16 | FWDIR=$(dirname "${BASH_SOURCE-$0}") 17 | fi 18 | 19 | if [[ -z "${VARONE_HOME}" ]]; then 20 | export VARONE_HOME="$(cd "${FWDIR}/.."; pwd)" 21 | fi 22 | 23 | if [[ -z "${VARONE_CONF_DIR}" ]]; then 24 | export VARONE_CONF_DIR="${VARONE_HOME}/conf" 25 | fi 26 | 27 | export HOSTLIST="${VARONE_CONF_DIR}/varonedaemond" 28 | 29 | remote_cmd="source /etc/profile; cd ${VARONE_HOME}; $bin/varOned.sh ${1}" 30 | 31 | for varoned in `cat "$HOSTLIST"`; do 32 | ssh $varoned "$remote_cmd" 33 | done 34 | -------------------------------------------------------------------------------- /varOne-server/src/main/java/com/varone/web/aggregator/timeperiod/TimePeriod.java: -------------------------------------------------------------------------------- 1 | package com.varone.web.aggregator.timeperiod; 2 | 3 | 4 | public enum TimePeriod { 5 | MINUTE_30("30m"), 6 | HOUR_1("1h"), 7 | HOUR_2("2h"), 8 | HOUR_6("6h"); 9 | 10 | 11 | private String expression; 12 | 13 | TimePeriod(String expression){ 14 | this.expression = expression; 15 | } 16 | 17 | public String expression(){ 18 | return this.expression; 19 | } 20 | 21 | public static TimePeriod fromString(String text) { 22 | if (text != null) { 23 | for (TimePeriod time : TimePeriod.values()) { 24 | if (text.equals(time.expression())) { 25 | return time; 26 | } 27 | } 28 | } 29 | throw new RuntimeException("Time period expression could not be matched: " + text); 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /varOne-server/src/main/java/com/varone/web/aggregator/unit/QuantityAggregator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.web.aggregator.unit; 5 | 6 | import com.varone.web.metrics.bean.TimeValuePairBean; 7 | import com.varone.web.vo.TimeValueVO; 8 | 9 | /** 10 | * @author allen 11 | * 12 | */ 13 | public class QuantityAggregator extends AbstractUnitAggregator { 14 | 15 | /* (non-Javadoc) 16 | * @see com.varone.web.aggregator.unit.AbstractUnitAggregator#aggregate(java.lang.String, 17 | * com.varone.web.metrics.bean.TimeValuePairBean, com.varone.web.vo.TimeValueVO) 18 | */ 19 | @Override 20 | public void aggregate(String format, TimeValuePairBean newPair, TimeValueVO currPair) { 21 | long newValue = Long.parseLong(newPair.getValue()); 22 | long currValue = Long.parseLong(currPair.getValue()); 23 | currPair.setValue(String.valueOf(newValue+currValue)); 24 | } 25 | 26 | } 27 | -------------------------------------------------------------------------------- /varOne-server/src/main/java/com/varone/web/vo/DefaultNodeVO.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.web.vo; 5 | 6 | import java.util.List; 7 | import java.util.Map; 8 | 9 | import javax.xml.bind.annotation.XmlRootElement; 10 | 11 | /** 12 | * @author allen 13 | * 14 | */ 15 | @XmlRootElement 16 | public class DefaultNodeVO { 17 | private List metricProps; 18 | private Map> propToMetrics; 19 | 20 | public List getMetricProps() { 21 | return metricProps; 22 | } 23 | public void setMetricProps(List metricProps) { 24 | this.metricProps = metricProps; 25 | } 26 | public Map> getPropToMetrics() { 27 | return propToMetrics; 28 | } 29 | public void setPropToMetrics(Map> propToMetrics) { 30 | this.propToMetrics = propToMetrics; 31 | } 32 | 33 | 34 | } 35 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/main.js: -------------------------------------------------------------------------------- 1 | // import React from 'react'; 2 | // import {HistoryLocation} from 'react-router'; 3 | // 4 | // import router from './utils/router'; 5 | // 6 | // router.run((Handler) => { 7 | // React.render(, document.getElementById('content')); 8 | // }); 9 | 10 | require('babel/polyfill'); 11 | 12 | import React from 'react'; 13 | import ReactDOM from 'react-dom'; 14 | // import { Router, Route } from 'react-router'; 15 | 16 | 17 | // React.render(( 18 | // 19 | // 20 | // 21 | // 22 | // 23 | // ), document.getElementById('content')); 24 | 25 | 26 | import App from './components/app'; 27 | ReactDOM.render(( 28 | 29 | ), document.getElementById('content')); 30 | -------------------------------------------------------------------------------- /varOne-node/src/main/java/com/varone/node/utils/Consts.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.node.utils; 5 | 6 | /** 7 | * @author allen 8 | * 9 | */ 10 | public class Consts { 11 | public final static String VARONE_SPARK_METRICS_PROP_DIR = "spark.metrics.prop.dir"; 12 | public final static String VARONE_NODE_PORT = "varOne.node.port"; 13 | public final static String VARONE_NODE_THREAD_NUM = "varOne.node.thread.number"; 14 | 15 | public final static Integer VARONE_NODE_DEFAULT_PORT = 8181; 16 | public final static Integer VARONE_NODE_DEFAULT_THREAD_NUM = 5; 17 | 18 | public final static String SPARK_METRICS_CSV_PERIOD = "csv.period"; 19 | public final static String SPARK_METRICS_CSV_UNIT = "csv.unit"; 20 | public final static String SPARK_METRICS_CSV_DIR = "csv.dir"; 21 | /** 22 | * 23 | */ 24 | public Consts() { 25 | // TODO Auto-generated constructor stub 26 | } 27 | 28 | } 29 | -------------------------------------------------------------------------------- /varOne-server/src/main/java/com/varone/web/aggregator/timeperiod/PeriodDataIngestor.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.web.aggregator.timeperiod; 5 | 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | 9 | import com.varone.web.metrics.bean.TimeValuePairBean; 10 | 11 | /** 12 | * @author allen 13 | * 14 | */ 15 | public abstract class PeriodDataIngestor { 16 | 17 | protected List dataSet; 18 | 19 | public PeriodDataIngestor(List dataSet) { 20 | this.dataSet = dataSet; 21 | } 22 | 23 | public List ingest(List periodOfPlotPoint){ 24 | List result = new ArrayList(); 25 | 26 | for(TimeValuePairBean pair: this.dataSet){ 27 | long ts = pair.getTime(); 28 | if(periodOfPlotPoint.contains(ts)){ 29 | result.add(pair); 30 | } 31 | } 32 | return result; 33 | } 34 | } 35 | -------------------------------------------------------------------------------- /varOne-node/src/main/java/com/varone/node/MetricTuple.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.node; 5 | 6 | import java.util.List; 7 | 8 | /** 9 | * @author allen 10 | * 11 | */ 12 | public class MetricTuple { 13 | private long time; 14 | private String value; 15 | 16 | public MetricTuple(String pairStr){ 17 | String[] pair = pairStr.split(","); 18 | this.setTime(Long.parseLong(pair[0])); 19 | this.setValue(pair[1]); 20 | } 21 | 22 | /** 23 | * @return the time 24 | */ 25 | public long getTime() { 26 | return time; 27 | } 28 | /** 29 | * @param time the time to set 30 | */ 31 | public void setTime(long time) { 32 | this.time = time; 33 | } 34 | /** 35 | * @return the value 36 | */ 37 | public String getValue() { 38 | return value; 39 | } 40 | /** 41 | * @param value the value to set 42 | */ 43 | public void setValue(String value) { 44 | this.value = value; 45 | } 46 | 47 | 48 | } 49 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/components/commons/breadcrumb.js: -------------------------------------------------------------------------------- 1 | import React, { PropTypes } from 'react'; 2 | 3 | export default class BreadCrumb extends React.Component { 4 | 5 | static propTypes = { 6 | content: PropTypes.array, 7 | active: PropTypes.string, 8 | onCrumbSelect: PropTypes.func 9 | } 10 | 11 | handleCrumbClick = e => { 12 | this.props.onCrumbSelect(e.currentTarget.innerText); 13 | } 14 | 15 | renderContent = () => { 16 | return this.props.content.map(item => { 17 | if (item === this.props.active) { 18 | return
  • { item }
    • ; 19 | } else { 20 | return
  • { item }
    • ; 21 | } 22 | }); 23 | } 24 | 25 | render() { 26 | const content = this.renderContent(); 27 | return ( 28 |
      29 | { content } 30 |
    31 | ); 32 | } 33 | } 34 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/components/navigator/nav.js: -------------------------------------------------------------------------------- 1 | import React, { PropTypes } from 'react'; 2 | import Brand from './brand'; 3 | import Header from './header'; 4 | 5 | import VarOneLogModal from '../commons/varOne-log-modal'; 6 | 7 | class Nav extends React.Component { 8 | static propTypes = { 9 | jobItemClickCB: PropTypes.func, 10 | dimensionItemClickCB: PropTypes.func 11 | } 12 | 13 | render() { 14 | return ( 15 |
    16 | 27 | 28 |
    29 | ); 30 | } 31 | } 32 | 33 | export default Nav; 34 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/stores/menu-store.js: -------------------------------------------------------------------------------- 1 | import alt from '../alt'; 2 | import Const from '../utils/consts'; 3 | import MenuAction from '../actions/menu-action'; 4 | 5 | class MenuStore { 6 | 7 | constructor() { 8 | this.bindActions(MenuAction); 9 | this.runningJobs = []; 10 | this.errorFlag = false; 11 | this.leftSideMenu = [ 12 | { name: Const.menu.cluster, icon: 'fa-cloud', collapse: false, children: [] }, 13 | { name: Const.menu.nodes, icon: 'fa-cube', collapse: false, children: [] }, 14 | { name: Const.menu.runningJobs, icon: 'fa-th-list', collapse: true, children: [] }, 15 | { name: Const.menu.history, icon: 'fa-history', collapse: false, children: [] } 16 | ]; 17 | } 18 | onChangErrorMessageIcon(errorFlag) { 19 | this.errorFlag = errorFlag; 20 | } 21 | onFetchRunningJob(result) { 22 | this.runningJobs = result; 23 | this.leftSideMenu[2].children = result; 24 | } 25 | } 26 | 27 | export default alt.createStore(MenuStore); 28 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/actions/job-action.js: -------------------------------------------------------------------------------- 1 | import alt from '../alt'; 2 | import VarOneAction from '../actions/varOne-action'; 3 | import MenuAction from '../actions/menu-action'; 4 | import request from 'superagent'; 5 | 6 | class JobAction { 7 | async fetchJobDashBoard(appId, selectMetrics, period) { 8 | try { 9 | if (selectMetrics && selectMetrics.length === 0) selectMetrics = undefined; 10 | else selectMetrics = selectMetrics.join(','); 11 | const response = await request.get(`/varOne-web/rest/job/${appId}`) 12 | .query({ metrics: selectMetrics, period }) 13 | .set('Accept', 'application/json'); 14 | const result = JSON.parse(response.text); 15 | this.dispatch({ result, period }); 16 | } catch (e) { 17 | VarOneAction.showFailMessage(e.response.text); 18 | MenuAction.changErrorMessageIcon(true); 19 | } 20 | } 21 | } 22 | 23 | export default alt.createActions(JobAction); 24 | -------------------------------------------------------------------------------- /varOne-server/src/main/java/com/varone/web/vo/MetricPropVO.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.web.vo; 5 | 6 | import javax.xml.bind.annotation.XmlRootElement; 7 | 8 | /** 9 | * @author allen 10 | * 11 | */ 12 | @XmlRootElement 13 | public class MetricPropVO { 14 | private String title; 15 | private String property; 16 | private String format; 17 | 18 | public MetricPropVO(){} 19 | 20 | public MetricPropVO(String title, String property, String format){ 21 | this.title = title; 22 | this.property = property; 23 | this.format = format; 24 | } 25 | 26 | public String getTitle() { 27 | return title; 28 | } 29 | public void setTitle(String title) { 30 | this.title = title; 31 | } 32 | public String getProperty() { 33 | return property; 34 | } 35 | public void setProperty(String property) { 36 | this.property = property; 37 | } 38 | public String getFormat() { 39 | return format; 40 | } 41 | public void setFormat(String format) { 42 | this.format = format; 43 | } 44 | 45 | } -------------------------------------------------------------------------------- /varOne-rpc/src/main/java/com/varone/hadoop/rpc/AbstractServer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.hadoop.rpc; 5 | 6 | import org.apache.hadoop.conf.Configuration; 7 | 8 | import com.varone.hadoop.rpc.IService; 9 | import com.varone.hadoop.rpc.RPCService; 10 | 11 | /** 12 | * @author allen 13 | * 14 | */ 15 | public abstract class AbstractServer { 16 | 17 | protected IService service; 18 | 19 | 20 | public abstract void constructServer(Configuration config) throws Exception; 21 | 22 | public void constructHadoopServer(Class protocol, Configuration config, Object serverInstance, int threadCount, int port) throws Exception{ 23 | if(port < 0 || port > 65535) throw new Exception("Illegal port number " + port); 24 | this.service = new RPCService(protocol, config, serverInstance, threadCount, port); 25 | } 26 | 27 | public void start() throws Exception{ 28 | this.service.startService(); 29 | } 30 | 31 | public void stop() throws Exception{ 32 | this.service.stopService(); 33 | } 34 | 35 | } 36 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | Spark Admin 13 | 14 | 15 | 16 | 20 | 21 | 22 | 23 | 24 | 25 |
    26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /varOne-server/src/main/java/com/varone/web/metrics/bean/NodeBean.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.web.metrics.bean; 5 | 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | 9 | /** 10 | * @author allen 11 | * 12 | */ 13 | public class NodeBean { 14 | private String host; 15 | private List metrics; 16 | 17 | public NodeBean(){ 18 | this.metrics = new ArrayList(); 19 | } 20 | 21 | /** 22 | * @return the host 23 | */ 24 | public String getHost() { 25 | return host; 26 | } 27 | /** 28 | * @param host the host to set 29 | */ 30 | public void setHost(String host) { 31 | this.host = host; 32 | } 33 | /** 34 | * @return the metrics 35 | */ 36 | public List getMetrics() { 37 | return metrics; 38 | } 39 | /** 40 | * @param metrics the metrics to set 41 | */ 42 | public void setMetrics(List metrics) { 43 | this.metrics = metrics; 44 | } 45 | 46 | public void addMetrics(MetricBean metric){ 47 | this.metrics.add(metric); 48 | } 49 | 50 | } 51 | -------------------------------------------------------------------------------- /varOne-web/src/main/resources/index.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | Spark Admin 13 | 14 | 15 | 16 | 20 | 21 | 22 | 23 | 24 | 25 |
    26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /varOne-server/src/main/java/com/varone/web/metrics/bean/MetricBean.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.web.metrics.bean; 5 | 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | 9 | /** 10 | * @author allen 11 | * 12 | */ 13 | public class MetricBean { 14 | private String name; 15 | private List values; 16 | 17 | public MetricBean(){ 18 | this.values = new ArrayList(); 19 | } 20 | 21 | /** 22 | * @return the name 23 | */ 24 | public String getName() { 25 | return name; 26 | } 27 | /** 28 | * @param name the name to set 29 | */ 30 | public void setName(String name) { 31 | this.name = name; 32 | } 33 | /** 34 | * @return the values 35 | */ 36 | public List getValues() { 37 | return values; 38 | } 39 | /** 40 | * @param values the values to set 41 | */ 42 | public void setValues(List values) { 43 | this.values = values; 44 | } 45 | 46 | public void addPair(TimeValuePairBean pair){ 47 | this.values.add(pair); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/components/job/job-header.js: -------------------------------------------------------------------------------- 1 | import React, { PropTypes } from 'react'; 2 | import TimePeriodPill from '../commons/time-period-pills'; 3 | 4 | class JobHeader extends React.Component { 5 | 6 | static propTypes = { 7 | appId: PropTypes.string, 8 | period: PropTypes.string, 9 | onPeriodSelect: PropTypes.func 10 | } 11 | 12 | render() { 13 | return ( 14 |
    15 |
    16 |

    { this.props.appId }   17 |

    19 |
    20 |
    21 |
    22 | 25 |
    26 |
    27 |
    28 | ); 29 | } 30 | } 31 | 32 | export default JobHeader; 33 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/components/cluster/cluster-header.js: -------------------------------------------------------------------------------- 1 | import React, { PropTypes } from 'react'; 2 | import TimePeriodPill from '../commons/time-period-pills'; 3 | 4 | class ClusterHeader extends React.Component { 5 | 6 | static propTypes = { 7 | period: PropTypes.string, 8 | onPeriodSelect: PropTypes.func 9 | } 10 | 11 | render() { 12 | return ( 13 |
    14 |
    15 |

    Dashboard   16 | 19 |

    20 |
    21 |
    22 |

    23 | 26 |

    27 |
    28 |
    29 | ); 30 | } 31 | } 32 | 33 | export default ClusterHeader; 34 | -------------------------------------------------------------------------------- /varOne-server/src/main/java/com/varone/web/listener/VarOneServletContextListener.java: -------------------------------------------------------------------------------- 1 | package com.varone.web.listener; 2 | 3 | import java.util.List; 4 | 5 | import javax.servlet.ServletContextEvent; 6 | import javax.servlet.ServletContextListener; 7 | 8 | import com.varone.conf.VarOneConfiguration; 9 | import com.varone.web.util.VarOneEnv; 10 | 11 | public class VarOneServletContextListener implements ServletContextListener { 12 | 13 | @Override 14 | public void contextInitialized(ServletContextEvent sce) { 15 | VarOneConfiguration conf = VarOneConfiguration.create(); 16 | VarOneEnv env = new VarOneEnv(conf); 17 | 18 | // File varOneConfPath = env.createVarOneConfPath(); 19 | 20 | env.checkSpecifiedConfigExist(); 21 | env.isEnableEventLog(); 22 | /** 23 | * For first release, the 1 seconds is a requirement for user to 24 | * define in their metrics.properties for the metrics output period 25 | * 26 | **/ 27 | env.isOneSecondsPeriod(); 28 | } 29 | 30 | @Override 31 | public void contextDestroyed(ServletContextEvent sce) { 32 | 33 | 34 | 35 | } 36 | 37 | } 38 | -------------------------------------------------------------------------------- /varOne-server/src/main/java/com/varone/web/vo/HistoryVO.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.web.vo; 5 | 6 | import javax.xml.bind.annotation.XmlRootElement; 7 | 8 | 9 | /** 10 | * @author allen 11 | * 12 | */ 13 | @XmlRootElement 14 | public class HistoryVO { 15 | private String name; 16 | private String id; 17 | private String startTime; 18 | private String endTime; 19 | private String user; 20 | public String getName() { 21 | return name; 22 | } 23 | public void setName(String name) { 24 | this.name = name; 25 | } 26 | public String getId() { 27 | return id; 28 | } 29 | public void setId(String id) { 30 | this.id = id; 31 | } 32 | public String getUser() { 33 | return user; 34 | } 35 | public void setUser(String user) { 36 | this.user = user; 37 | } 38 | public String getStartTime() { 39 | return startTime; 40 | } 41 | public void setStartTime(String startTime) { 42 | this.startTime = startTime; 43 | } 44 | public String getEndTime() { 45 | return endTime; 46 | } 47 | public void setEndTime(String endTime) { 48 | this.endTime = endTime; 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/actions/cluster-action.js: -------------------------------------------------------------------------------- 1 | import alt from '../alt'; 2 | import VarOneAction from '../actions/varOne-action'; 3 | import MenuAction from '../actions/menu-action'; 4 | import request from 'superagent'; 5 | 6 | class ClusterAction { 7 | constructor() { 8 | this.generateActions('fetchTotalNodeDashBoardSuccessful'); 9 | } 10 | 11 | async fetchTotalNodeDashBoard(selectMetrics, period) { 12 | try { 13 | if (selectMetrics && selectMetrics.length === 0) selectMetrics = undefined; 14 | else selectMetrics = selectMetrics.join(','); 15 | const response = await request.get('/varOne-web/rest/cluster') 16 | .query({ metrics: selectMetrics, period }) 17 | .set('Accept', 'application/json'); 18 | const result = JSON.parse(response.text); 19 | this.actions.fetchTotalNodeDashBoardSuccessful({ result, period }); 20 | } catch (e) { 21 | VarOneAction.showFailMessage(e.response.text); 22 | MenuAction.changErrorMessageIcon(true); 23 | } 24 | } 25 | } 26 | 27 | export default alt.createActions(ClusterAction); 28 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2015 SparkMonitor 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy 6 | of this software and associated documentation files (the "Software"), to deal 7 | in the Software without restriction, including without limitation the rights 8 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell 9 | copies of the Software, and to permit persons to whom the Software is 10 | furnished to do so, subject to the following conditions: 11 | 12 | The above copyright notice and this permission notice shall be included in all 13 | copies or substantial portions of the Software. 14 | 15 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 16 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 17 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 18 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 19 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 20 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 21 | SOFTWARE. 22 | -------------------------------------------------------------------------------- /varOne-web/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'war' 2 | 3 | dependencies { 4 | } 5 | 6 | war { 7 | exec { 8 | executable "sh" 9 | 10 | String arg1 = new File(project(':varOne-web').rootDir, '/varOne-web/src/main/resources/runnpm.sh'); 11 | 12 | String arg2 = new File(project(':varOne-web').rootDir, '/varOne-web/src/main/webapp') 13 | String command = arg1 + " " + arg2 14 | args "-c", command 15 | } 16 | 17 | manifest { 18 | attributes 'Main-Class': 'com.varone.exec.TomcatServer' 19 | } 20 | 21 | from new File(project(':varOne-web').buildDir, 'classes/main/') 22 | from new File(project(':varOne-web').buildDir, 'resources/main/index.html') 23 | 24 | exclude('**/node_modules') 25 | exclude('**/webpack') 26 | exclude('**/app'); 27 | exclude('**/build') 28 | exclude('.babelrc') 29 | exclude('.bootstraprc') 30 | exclude('karma.conf.js') 31 | exclude('package.json') 32 | exclude('**/test') 33 | exclude('**/report') 34 | exclude('tests.webpack.js') 35 | exclude('package.json') 36 | 37 | from('src/main/webapp/dist') 38 | 39 | exclude('dist') 40 | } 41 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/WEB-INF/web.xml: -------------------------------------------------------------------------------- 1 | 6 | Archetype Created Web Application 7 | 8 | 9 | 10 | jersey-serlvet 11 | 12 | com.sun.jersey.spi.container.servlet.ServletContainer 13 | 14 | 15 | com.sun.jersey.config.property.packages 16 | com.varone.web.resource 17 | 18 | 1 19 | 20 | 21 | 22 | jersey-serlvet 23 | /rest/* 24 | 25 | 26 | 27 | 28 | -1 29 | 30 | 31 | 32 | com.varone.web.listener.VarOneServletContextListener 33 | 34 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/components/commons/time-period-pills.js: -------------------------------------------------------------------------------- 1 | import React, { PropTypes } from 'react'; 2 | import Const from '../../utils/consts'; 3 | 4 | export default class TimePeriodPills extends React.Component { 5 | 6 | static propTypes = { 7 | active: PropTypes.string, 8 | onPeriodSelect: PropTypes.func 9 | } 10 | 11 | handlePillClick = e => { 12 | this.props.onPeriodSelect(e.currentTarget.textContent); 13 | } 14 | 15 | renderContent = () => { 16 | return Const.shared.timeperiod.map(period => { 17 | if (period === this.props.active) { 18 | return ( 19 |
  • 23 | { period } 24 |
    • 25 | ); 26 | } else { 27 | return ( 28 |
  • 32 | { period } 33 |
    • 34 | ); 35 | } 36 | }); 37 | } 38 | 39 | render() { 40 | return ( 41 |
      42 | { this.renderContent() } 43 |
    44 | ); 45 | } 46 | } 47 | -------------------------------------------------------------------------------- /conf/varOne-site.xml.template: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | varOne.server.addr 8 | 0.0.0.0 9 | varOne Web Server address 10 | 11 | 12 | 13 | varOne.server.port 14 | 8080 15 | varOne Web Server port, default is 8080 16 | 17 | 18 | 19 | varOne.node.port 20 | 8181 21 | varOne daemond port, default is 8181 22 | 23 | 24 | 25 | varOne.node.thread.number 26 | 5 27 | The number of RPC handler for varOne daemond, default is 5 28 | 29 | 30 | 31 | varOne.server.context.path 32 | /varOne-web 33 | Context Path of the varOne Web Application 34 | 35 | 36 | 37 | varOne.war.tempdir 38 | webapps 39 | Location of jetty temporary directory 40 | 41 | 42 | 43 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/components/job/job-metrics.js: -------------------------------------------------------------------------------- 1 | import React, { PropTypes } from 'react'; 2 | import JobMetric from './job-metric'; 3 | 4 | class JobMetrics extends React.Component { 5 | 6 | static propTypes = { 7 | metrics: PropTypes.array 8 | } 9 | 10 | render() { 11 | const { metrics } = this.props; 12 | const metricCharts = []; 13 | for (const metric in metrics) { 14 | if (metrics.hasOwnProperty(metric)) { 15 | metricCharts.push(( 16 | 17 | )); 18 | } 19 | } 20 | 21 | const rowNum = Math.ceil(metricCharts.length / 2); 22 | let mIdx = 0; 23 | 24 | const metricRows = Array.from(new Array(rowNum), (x, i) => { 25 | return ( 26 |
    27 |
    28 | { metricCharts[mIdx++] } 29 |
    30 |
    31 | { metricCharts[mIdx++] } 32 |
    33 |
    34 | ); 35 | }); 36 | 37 | return ( 38 |
    39 |
    40 | { metricRows } 41 |
    42 |
    43 | ); 44 | } 45 | } 46 | 47 | export default JobMetrics; 48 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/components/cluster/cluster-metrics.js: -------------------------------------------------------------------------------- 1 | import React, { PropTypes } from 'react'; 2 | import ClusterMetric from './cluster-metric'; 3 | 4 | class ClusterMetrics extends React.Component { 5 | 6 | static propTypes = { 7 | metrics: PropTypes.array 8 | } 9 | 10 | render() { 11 | const { metrics } = this.props; 12 | const metricCharts = []; 13 | for (const metric in metrics) { 14 | if (metrics.hasOwnProperty(metric)) { 15 | metricCharts.push(( 16 | 17 | )); 18 | } 19 | } 20 | 21 | const rowNum = Math.ceil(metricCharts.length / 2); 22 | let mIdx = 0; 23 | 24 | const metricRows = Array.from(new Array(rowNum), (x, i) => { 25 | return ( 26 |
    27 |
    28 | { metricCharts[mIdx++] } 29 |
    30 |
    31 | { metricCharts[mIdx++] } 32 |
    33 |
    34 | ); 35 | }); 36 | 37 | return ( 38 |
    39 |
    40 | { metricRows } 41 |
    42 |
    43 | ); 44 | } 45 | } 46 | 47 | export default ClusterMetrics; 48 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/stores/node-store.js: -------------------------------------------------------------------------------- 1 | import alt from '../alt'; 2 | import Const from '../utils/consts'; 3 | import NodeAction from '../actions/node-action'; 4 | 5 | class NodeStore { 6 | 7 | constructor() { 8 | this.bindActions(NodeAction); 9 | this.nodes = []; 10 | this.data = null; 11 | this.period = Const.shared.timeperiod[0]; 12 | } 13 | 14 | onFetchNodes(result) { 15 | this.nodes = result; 16 | } 17 | 18 | onFetchNodeDashBoard({ node, result, period }) { 19 | const data = {}; 20 | data.metrics = []; 21 | 22 | result.metricProps.forEach(metric => { 23 | const o = {}; 24 | o.id = metric.property; 25 | o.title = metric.title; 26 | o.format = metric.format; 27 | o.x = [ 'x' ]; 28 | const values = result.propToMetrics[metric.property]; 29 | const metricsValues = [ node ]; 30 | for (let i = 0; i < values.length; i++) { 31 | o.x.push( 32 | parseInt(values[i].time, 10) 33 | ); 34 | metricsValues.push(values[i].value); 35 | } 36 | o.value = metricsValues; 37 | data.metrics.push(o); 38 | }); 39 | 40 | this.data = data; 41 | this.period = period; 42 | } 43 | 44 | onChangeTimePeriod(period) { 45 | this.period = period; 46 | } 47 | } 48 | 49 | export default alt.createStore(NodeStore); 50 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/utils/data-format.js: -------------------------------------------------------------------------------- 1 | const K = 1024; 2 | const M = 1048576; 3 | const G = 1073741824; 4 | const T = 1099511627776; 5 | 6 | 7 | function getRoundToN(num, pos) { 8 | const N = Math.pow(10, pos); 9 | return Math.round(num * N) / N; 10 | } 11 | 12 | 13 | export function byteFormatter(d) { 14 | let result = d; 15 | if (result < M) { 16 | result = result / K; 17 | result = getRoundToN(result, 2); 18 | return result + ' KB'; 19 | } else if (result >= M && result < G) { 20 | result = result / M; 21 | result = getRoundToN(result, 2); 22 | return result + ' MB'; 23 | } else if (result >= G && result < T) { 24 | result = result / G; 25 | result = getRoundToN(result, 2); 26 | return result + ' GB'; 27 | } else { 28 | result = result / T; 29 | result = getRoundToN(result, 2); 30 | return result + ' TB'; 31 | } 32 | } 33 | 34 | export function percentageFormatter(d) { 35 | let result = d; 36 | result = getRoundToN(result, 2); 37 | return result * 100 + ' %'; 38 | } 39 | 40 | export function millisFormatter(d) { 41 | const second = d / 1000; 42 | if (second < 60) { 43 | return second + ' second'; 44 | } else if (second >= 60) { 45 | return second / 60 + ' minute'; 46 | } 47 | } 48 | 49 | export function dateFormatter(timestamp) { 50 | return new Date(timestamp * 1); 51 | } 52 | -------------------------------------------------------------------------------- /varOne-server/src/main/java/com/varone/web/vo/HistoryDetailStageVO.java: -------------------------------------------------------------------------------- 1 | package com.varone.web.vo; 2 | 3 | import java.util.List; 4 | 5 | import javax.xml.bind.annotation.XmlRootElement; 6 | 7 | @XmlRootElement 8 | public class HistoryDetailStageVO { 9 | 10 | private int completeTaskSize; 11 | private List aggregatorExecutor; 12 | private List tasks; 13 | private List metricCompletedTasks; 14 | 15 | public List getAggregatorExecutor() { 16 | return aggregatorExecutor; 17 | } 18 | 19 | public void setAggregatorExecutor(List aggregatorExecutor) { 20 | this.aggregatorExecutor = aggregatorExecutor; 21 | } 22 | 23 | public List getTasks() { 24 | return tasks; 25 | } 26 | 27 | public void setTasks(List tasks) { 28 | this.tasks = tasks; 29 | } 30 | 31 | public int getCompleteTaskSize() { 32 | return completeTaskSize; 33 | } 34 | 35 | public void setCompleteTaskSize(int completeTaskSize) { 36 | this.completeTaskSize = completeTaskSize; 37 | } 38 | 39 | public List getMetricCompletedTasks() { 40 | return metricCompletedTasks; 41 | } 42 | 43 | public void setMetricCompletedTasks( 44 | List metricCompletedTasks) { 45 | this.metricCompletedTasks = metricCompletedTasks; 46 | } 47 | 48 | 49 | } 50 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/actions/node-action.js: -------------------------------------------------------------------------------- 1 | import alt from '../alt'; 2 | import VarOneAction from '../actions/varOne-action'; 3 | import MenuAction from '../actions/menu-action'; 4 | import request from 'superagent'; 5 | 6 | class NodeAction { 7 | 8 | constructor() { 9 | this.generateActions('changeTimePeriod'); 10 | } 11 | 12 | async fetchNodes() { 13 | try { 14 | const response = await request.get('/varOne-web/rest/nodes') 15 | .set('Accept', 'application/json'); 16 | const result = JSON.parse(response.text); 17 | this.dispatch(result); 18 | } catch (e) { 19 | VarOneAction.showFailMessage(e.response.text); 20 | MenuAction.changErrorMessageIcon(true); 21 | } 22 | } 23 | 24 | async fetchNodeDashBoard(node, selectMetrics, period) { 25 | try { 26 | if (selectMetrics && selectMetrics.length === 0) selectMetrics = undefined; 27 | else selectMetrics = selectMetrics.join(','); 28 | const response = await request.get(`/varOne-web/rest/nodes/${node}`) 29 | .query({ metrics: selectMetrics, period }) 30 | .set('Accept', 'application/json'); 31 | const result = JSON.parse(response.text); 32 | this.dispatch({ node, result, period }); 33 | } catch (e) { 34 | VarOneAction.showFailMessage(e.response.text); 35 | MenuAction.changErrorMessageIcon(true); 36 | } 37 | } 38 | 39 | } 40 | 41 | export default alt.createActions(NodeAction); 42 | -------------------------------------------------------------------------------- /varOne-node/src/main/java/com/varone/node/reader/MetricsReader.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.node.reader; 5 | 6 | import java.io.File; 7 | import java.io.IOException; 8 | import java.util.ArrayList; 9 | import java.util.Collection; 10 | import java.util.LinkedHashMap; 11 | import java.util.List; 12 | import java.util.Map; 13 | 14 | import org.apache.commons.io.FileUtils; 15 | 16 | import com.varone.node.MetricTuple; 17 | import com.varone.node.MetricsType; 18 | 19 | /** 20 | * @author allen 21 | * 22 | */ 23 | public abstract class MetricsReader { 24 | 25 | protected MetricsType metricType; 26 | protected MetricsReader reader; 27 | 28 | public MetricsReader(MetricsReader reader, MetricsType metricType) { 29 | this.reader = reader; 30 | this.metricType = metricType; 31 | } 32 | 33 | protected Map> getMetricsData(Collection filteredFiles) throws IOException{ 34 | Map> result = new LinkedHashMap>(); 35 | 36 | for(File metricFile: filteredFiles){ 37 | List lines = FileUtils.readLines(metricFile); 38 | lines.remove(0); //skip header 39 | List tuples = new ArrayList(lines.size()); 40 | for(String pairStr: lines){ 41 | tuples.add(new MetricTuple(pairStr)); 42 | } 43 | result.put(metricFile.getName(), tuples); 44 | } 45 | 46 | return result; 47 | } 48 | 49 | protected abstract Map> read(String applicationId, String metricsDir) throws IOException; 50 | } 51 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/components/job/job-summary-block.js: -------------------------------------------------------------------------------- 1 | import React, { PropTypes } from 'react'; 2 | import classSet from 'classnames'; 3 | 4 | class JobSummaryBlock extends React.Component { 5 | 6 | static propTypes = { 7 | title: PropTypes.string, 8 | icon: PropTypes.string, 9 | panel: PropTypes.string, 10 | value: PropTypes.number 11 | } 12 | 13 | render() { 14 | const iconClass = classSet('fa', 'fa-5x', this.props.icon); 15 | const panelClass = classSet('panel', this.props.panel); 16 | return ( 17 |
    18 |
    19 |
    20 |
    21 | 22 |
    23 |
    24 |
    { this.props.value }
    25 |
    { this.props.title }
    26 |
    27 |
    28 |
    29 | 30 |
    31 | View Details 32 | 33 | 34 | 35 |
    36 |
    37 |     38 |
    39 | ); 40 | } 41 | } 42 | export default JobSummaryBlock; 43 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/components/cluster/cluster-summary-block.js: -------------------------------------------------------------------------------- 1 | import React, { PropTypes } from 'react'; 2 | import classSet from 'classnames'; 3 | 4 | class ClusterSummaryBlock extends React.Component { 5 | 6 | static propTypes = { 7 | title: PropTypes.string, 8 | icon: PropTypes.string, 9 | panel: PropTypes.string, 10 | value: PropTypes.number, 11 | onSummaryClick: PropTypes.func 12 | } 13 | 14 | handleSummaryClick = () => this.props.onSummaryClick(this.props.title); 15 | 16 | render() { 17 | const iconClass = classSet('fa', 'fa-5x', this.props.icon); 18 | const panelClass = classSet('panel', this.props.panel); 19 | return ( 20 |
    21 |
    22 |
    23 |
    24 | 25 |
    26 |
    27 |
    { this.props.value }
    28 |
    { this.props.title }
    29 |
    30 |
    31 |
    32 | 33 |
    34 | View Details 35 | 36 |
    37 |
    38 |     39 |
    40 | ); 41 | } 42 | } 43 | export default ClusterSummaryBlock; 44 | -------------------------------------------------------------------------------- /bin/varOne.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | bin=$(dirname "${BASH_SOURCE-$0}") 4 | bin=$(cd "${bin}">/dev/null; pwd) 5 | 6 | if [ -L ${BASH_SOURCE-$0} ]; then 7 | FWDIR=$(dirname $(readlink "${BASH_SOURCE-$0}")) 8 | else 9 | FWDIR=$(dirname "${BASH_SOURCE-$0}") 10 | fi 11 | 12 | if [[ -z "${VARONE_HOME}" ]]; then 13 | export VARONE_HOME="$(cd "${FWDIR}/.."; pwd)" 14 | fi 15 | 16 | if [[ -z "${VARONE_CONF_DIR}" ]]; then 17 | export VARONE_CONF_DIR="${VARONE_HOME}/conf" 18 | fi 19 | 20 | if [[ -f "${VARONE_CONF_DIR}/varOne-env.sh" ]]; then 21 | . "${VARONE_CONF_DIR}/varOne-env.sh" 22 | fi 23 | 24 | if [[ -z "${VARONE_WAR}" ]]; then 25 | if [[ -d "${VARONE_HOME}/varOne-web/src/main/webapp" ]]; then 26 | export VARONE_WAR="${VARONE_HOME}/varOne-web/src/main/webapp" 27 | else 28 | # export VARONE_WAR=${VARONE_HOME}/varOne-web-0.1.0-beta.war 29 | export VARONE_WAR=$(find -L "${VARONE_HOME}" -name "varOne-web*.war") 30 | fi 31 | fi 32 | 33 | if [[ -z "${VARONE_LOG_DIR}" ]]; then 34 | export VARONE_LOG_DIR="${VARONE_HOME}/logs" 35 | fi 36 | HOSTNAME=$(hostname) 37 | VARONE_LOGFILE="${VARONE_LOG_DIR}/varOne-server-${HOSTNAME}.log" 38 | JAVA_OPTS+=" -DvarOne.log.file=${VARONE_LOGFILE}" 39 | 40 | 41 | VARONE_CLASSPATH+="${VARONE_CONF_DIR}:" 42 | VARONE_CLASSPATH+=$(find -L "${VARONE_HOME}" -name "varOne-server*.jar") 43 | VARONE_SERVER=com.varone.server.VarOneServer 44 | 45 | function addJarInDir(){ 46 | if [[ -d "${1}" ]]; then 47 | VARONE_CLASSPATH="${1}/*:${VARONE_CLASSPATH}" 48 | fi 49 | } 50 | 51 | addJarInDir "${VARONE_HOME}/lib" 52 | 53 | $(exec java $JAVA_OPTS -cp $VARONE_CLASSPATH $VARONE_SERVER "$@") 54 | -------------------------------------------------------------------------------- /varOne-node/src/main/java/com/varone/node/reader/FsMetricsReader.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.node.reader; 5 | 6 | import java.io.File; 7 | import java.io.IOException; 8 | import java.util.Collection; 9 | import java.util.List; 10 | import java.util.Map; 11 | 12 | import org.apache.commons.io.FileUtils; 13 | import org.apache.commons.io.filefilter.IOFileFilter; 14 | import org.apache.commons.io.filefilter.RegexFileFilter; 15 | 16 | import com.varone.node.MetricTuple; 17 | import com.varone.node.MetricsType; 18 | 19 | /** 20 | * @author allen 21 | * 22 | */ 23 | public class FsMetricsReader extends MetricsReader { 24 | 25 | 26 | 27 | public FsMetricsReader(MetricsReader reader, MetricsType metricType){ 28 | super(reader, metricType); 29 | } 30 | 31 | /* (non-Javadoc) 32 | * @see com.varone.node.reader.MetricsReader#read() 33 | */ 34 | @Override 35 | protected Map> read(String applicationId, String metricsDir) throws IOException { 36 | File root = new File(metricsDir); 37 | IOFileFilter filter = null; 38 | if(super.metricType.equals(MetricsType.FS)){ 39 | filter = new RegexFileFilter(applicationId+"(.\\d+).executor.filesystem.*\\.csv"); 40 | } else { 41 | filter = new RegexFileFilter(applicationId+"(.\\d+)." + super.metricType.type() + ".csv"); 42 | } 43 | 44 | Collection filteredFiles = FileUtils.listFiles(root, filter, null); 45 | 46 | Map> result = super.getMetricsData(filteredFiles); 47 | 48 | if(null != super.reader) 49 | result.putAll(super.reader.read(applicationId, metricsDir)); 50 | 51 | return result; 52 | } 53 | 54 | } 55 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "parser": "babel-eslint", 3 | "extends": "eslint-config-airbnb", 4 | "plugins": ["react"], 5 | "env": { 6 | "browser": true, 7 | "node": true, 8 | "es6": true, 9 | "mocha": true 10 | }, 11 | "globals": { 12 | "chai": true, 13 | "sinon": true 14 | }, 15 | "rules": { 16 | "react/display-name": [2, { "acceptTranspilerName": true }], 17 | "react/jsx-curly-spacing": [2, "always"], 18 | "react/jsx-no-duplicate-props": 2, 19 | "react/jsx-no-undef": 2, 20 | "react/jsx-quotes": 0, 21 | "react/jsx-uses-react": 2, 22 | "react/jsx-uses-vars": 2, 23 | "react/jsx-closing-bracket-location": [2, "after-props" ], 24 | "react/no-did-mount-set-state": 2, 25 | "react/no-did-update-set-state": 2, 26 | "react/no-multi-comp": 0, 27 | "react/no-unknown-property": 2, 28 | "react/prop-types": 2, 29 | "react/react-in-jsx-scope": 2, 30 | "react/require-extension": 2, 31 | "react/self-closing-comp": 2, 32 | "react/wrap-multilines": 2, 33 | "react/sort-comp": 0, 34 | "react/jsx-no-bind": 0, 35 | 36 | "quotes": [2, "single", "avoid-escape"], 37 | "jsx-quotes": [2, "prefer-single"], 38 | "comma-dangle": [2, "never"], 39 | "indent": [2, 2], 40 | "object-curly-spacing": [2, "always"], 41 | "array-bracket-spacing": [2, "always"], 42 | "no-undef": 2, 43 | "no-underscore-dangle": 0, 44 | "func-names": 0, 45 | "no-else-return": 0, 46 | "no-console": 1, 47 | "no-throw-literal": 0, 48 | "id-length": 0, 49 | "no-unused-expressions": 0, 50 | "no-param-reassign": 0, 51 | "max-len": [1, 100] 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /varOne-node/src/main/java/com/varone/node/reader/ThreadPoolMetricsReader.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.node.reader; 5 | 6 | import java.io.File; 7 | import java.io.IOException; 8 | import java.util.Collection; 9 | import java.util.List; 10 | import java.util.Map; 11 | 12 | import org.apache.commons.io.FileUtils; 13 | import org.apache.commons.io.filefilter.IOFileFilter; 14 | import org.apache.commons.io.filefilter.RegexFileFilter; 15 | 16 | import com.varone.node.MetricTuple; 17 | import com.varone.node.MetricsType; 18 | 19 | /** 20 | * @author allen 21 | * 22 | */ 23 | public class ThreadPoolMetricsReader extends MetricsReader { 24 | 25 | public ThreadPoolMetricsReader(MetricsReader reader, MetricsType metricType) { 26 | super(reader, metricType); 27 | } 28 | 29 | /* (non-Javadoc) 30 | * @see com.varone.node.reader.MetricsReader#read(java.lang.String, java.lang.String) 31 | */ 32 | @Override 33 | protected Map> read(String applicationId, String metricsDir) throws IOException { 34 | File root = new File(metricsDir); 35 | IOFileFilter filter = null; 36 | 37 | if(super.metricType.equals(MetricsType.THREAD_POOL)){ 38 | filter = new RegexFileFilter(applicationId+"(.\\d+).executor.threadpool.*\\.csv"); 39 | } else { 40 | filter = new RegexFileFilter(applicationId+"(.\\d+)." + super.metricType.type() + ".csv"); 41 | } 42 | 43 | Collection filteredFiles = FileUtils.listFiles(root, filter, null); 44 | 45 | Map> result = super.getMetricsData(filteredFiles); 46 | 47 | if(null != super.reader) 48 | result.putAll(super.reader.read(applicationId, metricsDir)); 49 | 50 | return result; 51 | } 52 | 53 | } 54 | -------------------------------------------------------------------------------- /varOne-node/src/main/java/com/varone/node/reader/JvmMetricsReader.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.node.reader; 5 | 6 | import java.io.File; 7 | import java.io.IOException; 8 | import java.util.Collection; 9 | import java.util.List; 10 | import java.util.Map; 11 | 12 | import org.apache.commons.io.FileUtils; 13 | import org.apache.commons.io.filefilter.IOFileFilter; 14 | import org.apache.commons.io.filefilter.RegexFileFilter; 15 | 16 | import com.varone.node.MetricTuple; 17 | import com.varone.node.MetricsType; 18 | 19 | /** 20 | * @author allen 21 | * 22 | */ 23 | public class JvmMetricsReader extends MetricsReader { 24 | 25 | /** 26 | * @param reader 27 | */ 28 | public JvmMetricsReader(MetricsReader reader, MetricsType metricType) { 29 | super(reader, metricType); 30 | } 31 | 32 | /* (non-Javadoc) 33 | * @see com.varone.node.reader.MetricsReader#read(java.lang.String, java.lang.String) 34 | */ 35 | @Override 36 | protected Map> read(String applicationId, 37 | String metricsDir) throws IOException { 38 | File root = new File(metricsDir); 39 | IOFileFilter filter = null; 40 | 41 | if(super.metricType.equals(MetricsType.JVM)){ 42 | filter = new RegexFileFilter(applicationId+"(.\\d+).jvm.(?!total).*\\.csv"); 43 | } else { 44 | filter = new RegexFileFilter(applicationId+"(.\\d+)." + super.metricType.type() + ".csv"); 45 | } 46 | 47 | Collection filteredFiles = FileUtils.listFiles(root, filter, null); 48 | 49 | Map> result = super.getMetricsData(filteredFiles); 50 | 51 | if(null != super.reader) 52 | result.putAll(super.reader.read(applicationId, metricsDir)); 53 | 54 | return result; 55 | } 56 | 57 | } 58 | -------------------------------------------------------------------------------- /varOne-server/src/main/java/com/varone/web/aggregator/unit/AverageAggregator.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.web.aggregator.unit; 5 | 6 | import java.util.List; 7 | 8 | import com.varone.web.metrics.bean.TimeValuePairBean; 9 | import com.varone.web.vo.TimeValueVO; 10 | 11 | /** 12 | * @author allen 13 | * 14 | */ 15 | public class AverageAggregator extends AbstractUnitAggregator { 16 | 17 | /* (non-Javadoc) 18 | * @see com.varone.web.aggregator.unit.AbstractUnitAggregator#aggregate(java.lang.String, 19 | * com.varone.web.metrics.bean.TimeValuePairBean, com.varone.web.vo.TimeValueVO) 20 | */ 21 | @Override 22 | public void aggregate(String format, TimeValuePairBean newPair, TimeValueVO currPair) { 23 | String[] valueAndCount = currPair.getValue().split(","); 24 | 25 | double newValue = Double.parseDouble(newPair.getValue()); 26 | double currValue = Double.parseDouble(valueAndCount[0]); 27 | valueAndCount[1] = String.valueOf(Integer.valueOf(valueAndCount[1])+1); 28 | if(format.equals("MILLIS") || format.equals("OPS")){ 29 | currPair.setValue(String.valueOf((long)newValue+currValue)+","+valueAndCount[1]); 30 | } else { 31 | currPair.setValue(String.valueOf(newValue+currValue)+","+valueAndCount[1]); 32 | } 33 | } 34 | 35 | public void calculateAvg(List periodData){ 36 | for(TimeValueVO timeValue: periodData){ 37 | String[] valueAndCount = timeValue.getValue().split(","); 38 | double sum = Double.parseDouble(valueAndCount[0]); 39 | int count = Integer.parseInt(valueAndCount[1]); 40 | if(sum > 0){ 41 | double avg = sum / count; 42 | timeValue.setValue(String.valueOf(avg)); 43 | } else { 44 | timeValue.setValue("0"); 45 | } 46 | } 47 | } 48 | 49 | } 50 | -------------------------------------------------------------------------------- /varOne-server/src/test/java/com/varone/web/aggregator/UIDataAggregatorTest.java: -------------------------------------------------------------------------------- 1 | package com.varone.web.aggregator; 2 | 3 | import java.util.List; 4 | 5 | import org.apache.hadoop.conf.Configuration; 6 | import org.junit.Test; 7 | 8 | import com.varone.conf.VarOneConfiguration; 9 | import com.varone.web.eventlog.bean.SparkEventLogBean; 10 | import com.varone.web.reader.eventlog.impl.EventLogHdfsReaderImpl; 11 | import com.varone.web.util.VarOneEnv; 12 | import com.varone.web.vo.HistoryDetailStageVO; 13 | import com.varone.web.vo.TasksVO; 14 | 15 | public class UIDataAggregatorTest { 16 | 17 | @Test 18 | public void testgetHistoryDetialStage() throws Exception{ 19 | VarOneConfiguration conf = VarOneConfiguration.create(); 20 | VarOneEnv env = new VarOneEnv(conf); 21 | Configuration config = env.loadHadoopConfiguration(); 22 | 23 | 24 | UIDataAggregator aggregator = new UIDataAggregator(); 25 | EventLogHdfsReaderImpl hdfsReader = new EventLogHdfsReaderImpl(config, env.getEventLogDir()); 26 | SparkEventLogBean sparkEventLog = hdfsReader.getHistoryStageDetails("application_1449569227858_0664"); 27 | 28 | HistoryDetailStageVO historyDetailsVO = aggregator.aggregateHistoryDetialStage(sparkEventLog, 1); 29 | List taskVOList = historyDetailsVO.getTasks(); 30 | 31 | /*for(TasksVO taskVO : taskVOList){ 32 | System.out.println(taskVO.getIndex() + " " + taskVO.getGcTime()); 33 | }*/ 34 | 35 | } 36 | 37 | 38 | @Test 39 | public void testMedian(){ 40 | int[] a = {1, 2, 3, 4, 5, 6}; 41 | double result = 0; 42 | int n = a.length; 43 | 44 | if((a.length % 2) == 0){ 45 | result = (a[n / 2] + a[(n / 2) - 1]) / 2.0; 46 | }else{ 47 | result = a[(n - 1) / 2]; 48 | } 49 | System.out.println(result); 50 | } 51 | } 52 | -------------------------------------------------------------------------------- /varOne-server/src/test/java/com/varone/web/reader/metrics/impl/SparkEventLogParserTest.java: -------------------------------------------------------------------------------- 1 | package com.varone.web.reader.metrics.impl; 2 | 3 | import static org.junit.Assert.*; 4 | 5 | import java.util.ArrayList; 6 | import java.util.List; 7 | 8 | import org.junit.Test; 9 | 10 | import com.varone.web.eventlog.bean.SparkEventLogBean; 11 | import com.varone.web.eventlog.bean.SparkEventLogBean.BlockManager; 12 | import com.varone.web.reader.eventlog.parser.SparkEventLogParser; 13 | 14 | public class SparkEventLogParserTest { 15 | 16 | 17 | //{"Event":"SparkListenerBlockManagerAdded","Block Manager ID":{"Executor ID":"1","Host":"server-a2","Port":34086},"Maximum Memory":556038881,"Timestamp":1450686022248} 18 | @Test 19 | public void testBlockManagerTag() throws Exception{ 20 | List eventNames = new ArrayList(); 21 | eventNames.add(SparkEventLogParser.BLOCKMANAGER_ADD); 22 | 23 | SparkEventLogParser eventLogParser = new SparkEventLogParser(eventNames); 24 | boolean flag = eventLogParser.parseLine("{\"Event\":\"SparkListenerBlockManagerAdded\",\"Block Manager ID\":{\"Executor ID\":\"1\",\"Host\":\"server-a2\",\"Port\":34086},\"Maximum Memory\":556038881,\"Timestamp\":1450686022248}"); 25 | SparkEventLogBean result = eventLogParser.result(); 26 | 27 | List blockManagerList = result.getBlockManager(); 28 | 29 | for(BlockManager blockManager : blockManagerList){ 30 | assertEquals("1", blockManager.getBlockManagerID().getId()); 31 | assertEquals("server-a2", blockManager.getBlockManagerID().getHost()); 32 | assertEquals(34086, blockManager.getBlockManagerID().getPort()); 33 | assertEquals(1450686022248L, blockManager.getTimestamp()); 34 | assertEquals(556038881L, blockManager.getMaxMemory()); 35 | } 36 | 37 | 38 | } 39 | 40 | } 41 | -------------------------------------------------------------------------------- /varOne-web/hadoopConf/core-site.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | fs.defaultFS 23 | hdfs://server-a1:9000 24 | 25 | 26 | 27 | hadoop.tmp.dir 28 | /dfs/1/tmp 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | hadoop.proxyuser.nfsserver.groups 38 | * 39 | 40 | The 'nfsserver' user is allowed to proxy all members of the 'nfs-users1' and 41 | 'nfs-users2' groups. Set this to '*' to allow nfsserver user to proxy any group. 42 | 43 | 44 | 45 | 46 | hadoop.proxyuser.nfsserver.hosts 47 | * 48 | 49 | This is the host where the nfs gateway is running. Set this to '*' to allow 50 | requests from any hosts to be proxied. 51 | 52 | 53 | 54 | 55 | -------------------------------------------------------------------------------- /varOne-rpc/src/main/java/com/varone/hadoop/rpc/metrics/impl/pb/service/MetricsServicePBServiceImpl.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.hadoop.rpc.metrics.impl.pb.service; 5 | 6 | import com.google.protobuf.RpcController; 7 | import com.google.protobuf.ServiceException; 8 | import com.varone.hadoop.rpc.metrics.MetricsService; 9 | import com.varone.hadoop.rpc.metrics.MetricsServicePB; 10 | import com.varone.hadoop.rpc.metrics.MetricsServiceResponse; 11 | import com.varone.hadoop.rpc.metrics.impl.MetricsServiceRequestPBImpl; 12 | import com.varone.hadoop.rpc.metrics.impl.MetricsServiceResponsePBImpl; 13 | import com.varone.hadoop.rpc.protos.MetricsProtos.MetricsRequestProto; 14 | import com.varone.hadoop.rpc.protos.MetricsProtos.MetricsResponseProto; 15 | 16 | /** 17 | * @author allen 18 | * 19 | */ 20 | public class MetricsServicePBServiceImpl implements MetricsServicePB { 21 | 22 | protected MetricsService metricsService; 23 | 24 | /** 25 | * 26 | */ 27 | public MetricsServicePBServiceImpl(MetricsService metricsService) { 28 | this.metricsService = metricsService; 29 | } 30 | 31 | /* (non-Javadoc) 32 | * @see com.varone.hadoop.rpc.protos.MetricsServiceProtos.Metrics.BlockingInterface#getMetrics(com.google.protobuf.RpcController, com.varone.hadoop.rpc.protos.MetricsProtos.MetricsRequestProto) 33 | */ 34 | @Override 35 | public MetricsResponseProto getMetrics(RpcController controller, 36 | MetricsRequestProto proto) throws ServiceException { 37 | MetricsServiceRequestPBImpl request = new MetricsServiceRequestPBImpl(proto); 38 | try{ 39 | MetricsServiceResponse response = this.metricsService.getMetrics(request); 40 | return ((MetricsServiceResponsePBImpl)response).getProto(); 41 | } catch (Exception e) { 42 | throw new ServiceException(e); 43 | } 44 | } 45 | 46 | } 47 | -------------------------------------------------------------------------------- /varOne-server/src/main/java/com/varone/web/vo/SummaryExecutorVO.java: -------------------------------------------------------------------------------- 1 | package com.varone.web.vo; 2 | 3 | public class SummaryExecutorVO { 4 | private String executeId; 5 | private String address; 6 | private Long taskTime; 7 | private int totalTasks; 8 | private int failedTasks; 9 | private int succeededTasks; 10 | private String inputSize; 11 | private String records; 12 | private long maxMemory; 13 | 14 | public String getExecuteId() { 15 | return executeId; 16 | } 17 | public void setExecuteId(String executeId) { 18 | this.executeId = executeId; 19 | } 20 | public String getAddress() { 21 | return address; 22 | } 23 | public void setAddress(String address) { 24 | this.address = address; 25 | } 26 | public Long getTaskTime() { 27 | return taskTime; 28 | } 29 | public void setTaskTime(Long taskTime) { 30 | this.taskTime = taskTime; 31 | } 32 | public int getTotalTasks() { 33 | return totalTasks; 34 | } 35 | public void setTotalTasks(int totalTasks) { 36 | this.totalTasks = totalTasks; 37 | } 38 | public int getFailedTasks() { 39 | return failedTasks; 40 | } 41 | public void setFailedTasks(int failedTasks) { 42 | this.failedTasks = failedTasks; 43 | } 44 | public int getSucceededTasks() { 45 | return succeededTasks; 46 | } 47 | public void setSucceededTasks(int succeededTasks) { 48 | this.succeededTasks = succeededTasks; 49 | } 50 | public String getInputSize() { 51 | return inputSize; 52 | } 53 | public void setInputSize(String inputSize) { 54 | this.inputSize = inputSize; 55 | } 56 | public String getRecords() { 57 | return records; 58 | } 59 | public void setRecords(String records) { 60 | this.records = records; 61 | } 62 | public long getMaxMemory() { 63 | return maxMemory; 64 | } 65 | public void setMaxMemory(long maxMemory) { 66 | this.maxMemory = maxMemory; 67 | } 68 | 69 | 70 | 71 | 72 | } 73 | -------------------------------------------------------------------------------- /varOne-rpc/src/main/java/com/varone/hadoop/rpc/metrics/impl/MetricsServiceResponsePBImpl.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.hadoop.rpc.metrics.impl; 5 | 6 | import java.util.List; 7 | 8 | import com.varone.hadoop.rpc.metrics.MetricsServiceResponse; 9 | import com.varone.hadoop.rpc.protos.MetricsProtos.MetricsResponseProto; 10 | import com.varone.hadoop.rpc.protos.MetricsProtos.MetricsResponseProtoOrBuilder; 11 | import com.varone.hadoop.rpc.protos.MetricsProtos.MetricsResponseProto.MetricsMapProto; 12 | 13 | /** 14 | * @author allen 15 | * 16 | */ 17 | public class MetricsServiceResponsePBImpl implements MetricsServiceResponse { 18 | protected MetricsResponseProto proto = MetricsResponseProto.getDefaultInstance(); 19 | protected MetricsResponseProto.Builder builder = null; 20 | protected boolean viaProto = false; 21 | 22 | 23 | protected List result; 24 | 25 | /** 26 | * 27 | */ 28 | public MetricsServiceResponsePBImpl() { 29 | this.builder = MetricsResponseProto.newBuilder(); 30 | } 31 | 32 | public MetricsServiceResponsePBImpl(MetricsResponseProto proto) { 33 | this.proto = proto; 34 | this.viaProto = true; 35 | } 36 | 37 | /* (non-Javadoc) 38 | * @see com.varone.hadoop.rpc.metrics.MetricsServiceResponse#getResult() 39 | */ 40 | @Override 41 | public List getResult() { 42 | MetricsResponseProtoOrBuilder p = viaProto? this.proto:this.builder; 43 | this.result = p.getResultList(); 44 | return this.result; 45 | } 46 | 47 | /* (non-Javadoc) 48 | * @see com.varone.hadoop.rpc.metrics.MetricsServiceResponse#setResult(java.util.List) 49 | */ 50 | @Override 51 | public void setResult(List result) { 52 | this.builder.clearResult(); 53 | this.builder.addAllResult(result); 54 | this.result = result; 55 | } 56 | 57 | public MetricsResponseProto getProto() { 58 | return this.builder.build(); 59 | } 60 | 61 | } 62 | -------------------------------------------------------------------------------- /varOne-server/src/main/java/com/varone/web/vo/StageVO.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.web.vo; 5 | 6 | /** 7 | * @author allen 8 | * 9 | */ 10 | public class StageVO { 11 | private int id; 12 | private String description; 13 | private String submitTime; 14 | private String duration; 15 | private String tasksSuccessVSTotal; 16 | private String tasksSuccessPercent; 17 | private String readAmount; 18 | private String writeAmount; 19 | 20 | public int getId() { 21 | return id; 22 | } 23 | 24 | public void setId(int id) { 25 | this.id = id; 26 | } 27 | 28 | public String getDescription() { 29 | return description; 30 | } 31 | 32 | public void setDescription(String description) { 33 | this.description = description; 34 | } 35 | 36 | public String getSubmitTime() { 37 | return submitTime; 38 | } 39 | 40 | public void setSubmitTime(String submitTime) { 41 | this.submitTime = submitTime; 42 | } 43 | 44 | public String getDuration() { 45 | return duration; 46 | } 47 | 48 | public void setDuration(String duration) { 49 | this.duration = duration; 50 | } 51 | 52 | public String getTasksSuccessVSTotal() { 53 | return tasksSuccessVSTotal; 54 | } 55 | 56 | public void setTasksSuccessVSTotal(String tasksSuccessVSTotal) { 57 | this.tasksSuccessVSTotal = tasksSuccessVSTotal; 58 | } 59 | 60 | public String getTasksSuccessPercent() { 61 | return tasksSuccessPercent; 62 | } 63 | 64 | public void setTasksSuccessPercent(String tasksSuccessPercent) { 65 | this.tasksSuccessPercent = tasksSuccessPercent; 66 | } 67 | 68 | public String getReadAmount() { 69 | return readAmount; 70 | } 71 | 72 | public void setReadAmount(String readAmount) { 73 | this.readAmount = readAmount; 74 | } 75 | 76 | public String getWriteAmount() { 77 | return writeAmount; 78 | } 79 | 80 | public void setWriteAmount(String writeAmount) { 81 | this.writeAmount = writeAmount; 82 | } 83 | 84 | } 85 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/test/spec/components/cluster/cluster-metric.test.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom'; 3 | import TestUtils from 'react-addons-test-utils'; 4 | import stubApp from '../../../utils/stub-app'; 5 | import ClusterMetric from '../../../../app/components/cluster/cluster-metric'; 6 | 7 | const should = chai.should(); 8 | 9 | const mockMetric = { 10 | id: 'mock_id', 11 | title: 'mock_title', 12 | value: [ [ 's1', '0', '0', '0' ], [ 's1', '0', '20', '40' ], [ 's1', '0', '5', '9' ] ], 13 | x: [ 'x', 1454292960000, 1454292990000, 1454293020000 ], 14 | format: 'OPS' 15 | }; 16 | 17 | describe('Cluster Metric', () => { 18 | let node; 19 | let instance; 20 | let spy; 21 | 22 | beforeEach(() => { 23 | spy = sinon.spy(); 24 | ClusterMetric.prototype.defaultCharting = spy; 25 | const stub = stubApp()(ClusterMetric, { 26 | metric: mockMetric 27 | }); 28 | node = window.document.createElement('div'); 29 | instance = ReactDOM.render(React.createElement(stub), node); 30 | }); 31 | 32 | afterEach(() => { 33 | if (instance) ReactDOM.unmountComponentAtNode(node); 34 | }); 35 | 36 | it('should render correctly', () => { 37 | const dom = ReactDOM.findDOMNode(instance); 38 | 39 | dom.tagName.should.eql('DIV'); 40 | (dom.className.indexOf('panel-info') !== -1).should.be.ok; 41 | dom.childNodes.length.should.eql(2); 42 | 43 | const barChartIcon = TestUtils.findRenderedDOMComponentWithClass(instance, 'fa-bar-chart-o'); 44 | const heading = dom.childNodes[0]; 45 | const chartPanel = dom.childNodes[1].childNodes[0]; 46 | 47 | should.exist(barChartIcon); 48 | should.exist(chartPanel); 49 | heading.childNodes.length.should.eql(3); 50 | heading.childNodes[1].textContent.should.eql(mockMetric.title); 51 | (chartPanel.id === mockMetric.id).should.be.ok; 52 | spy.should.have.been.calledOnce; 53 | }); 54 | }); 55 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/components/history/history-list.js: -------------------------------------------------------------------------------- 1 | import React, { PropTypes } from 'react'; 2 | import { BootstrapTable, TableHeaderColumn } from 'react-bootstrap-table'; 3 | import { dateFormatter } from '../../utils/data-format'; 4 | 5 | class AppNameLink extends React.Component { 6 | 7 | static propTypes = { 8 | app: PropTypes.object, 9 | onLinkSelect: PropTypes.func 10 | } 11 | 12 | handleAppSelect = () => { 13 | const { id } = this.props.app; 14 | this.props.onLinkSelect(id); 15 | } 16 | 17 | render() { 18 | const { name } = this.props.app; 19 | return ({ name }); 20 | } 21 | } 22 | 23 | class HistoryList extends React.Component { 24 | 25 | static propTypes = { 26 | histories: PropTypes.array, 27 | onHistorySelect: PropTypes.func 28 | } 29 | 30 | appNameFormatter = (cell, row) => { 31 | return (); 32 | } 33 | 34 | dateUnitFormatter = cell => { 35 | return dateFormatter(cell); 36 | } 37 | 38 | render() { 39 | return ( 40 | 45 | App Name 48 | App ID 49 | Start Time 52 | End Time 55 | User 56 | 57 | ); 58 | } 59 | } 60 | 61 | export default HistoryList; 62 | -------------------------------------------------------------------------------- /bin/varOned.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | USAGE="Usage: bin/varOned.sh [start|stop]" 4 | 5 | if [[ "$1" == "" ]]; then 6 | echo ${USAGE} 7 | exit 1 8 | elif [[ "$1" != "start" && "$1" != "stop" ]]; then 9 | echo ${USAGE} 10 | exit 1 11 | fi 12 | 13 | bin=$(dirname "${BASH_SOURCE-$0}") 14 | bin=$(cd "${bin}">/dev/null; pwd) 15 | 16 | if [ -L ${BASH_SOURCE-$0} ]; then 17 | FWDIR=$(dirname $(readlink "${BASH_SOURCE-$0}")) 18 | else 19 | FWDIR=$(dirname "${BASH_SOURCE-$0}") 20 | fi 21 | 22 | if [[ -z "${VARONE_HOME}" ]]; then 23 | export VARONE_HOME="$(cd "${FWDIR}/.."; pwd)" 24 | fi 25 | 26 | if [[ -z "${VARONE_CONF_DIR}" ]]; then 27 | export VARONE_CONF_DIR="${VARONE_HOME}/conf" 28 | fi 29 | 30 | if [[ -f "${VARONE_CONF_DIR}/varOne-env.sh" ]]; then 31 | . "${VARONE_CONF_DIR}/varOne-env.sh" 32 | fi 33 | 34 | if [[ $SPARK_HOME == "" ]]; then 35 | echo "Make sure you have set SPARK_HOME in ${VARONE_CONF_DIR}/varOne-env.sh" 36 | fi 37 | 38 | if [[ -z "${VARONE_LOG_DIR}" ]]; then 39 | export VARONE_LOG_DIR="${VARONE_HOME}/logs" 40 | fi 41 | 42 | if [[ ! -d "${VARONE_LOG_DIR}" ]]; then 43 | mkdir $VARONE_LOG_DIR 44 | fi 45 | 46 | HOSTNAME=$(hostname) 47 | VARONE_LOGFILE="${VARONE_LOG_DIR}/varOned-${HOSTNAME}.log" 48 | 49 | VARONED_CLASSPATH+="${VARONE_CONF_DIR}" 50 | VARONED_SERVER=com.varone.node.VarOned 51 | 52 | function addJarInDir(){ 53 | if [[ -d "${1}" ]]; then 54 | VARONED_CLASSPATH="${1}/*:${VARONED_CLASSPATH}" 55 | fi 56 | } 57 | 58 | addJarInDir "${VARONE_HOME}/lib" 59 | 60 | 61 | if [[ "$1" == "start" ]]; then 62 | $(nohup java -cp $VARONED_CLASSPATH $VARONED_SERVER -d "${SPARK_HOME}/conf" >$VARONE_LOGFILE 2>&1 &) 63 | elif [[ "$1" == "stop" ]]; then 64 | PID="$(ps -ef | grep VarOned | grep -v grep | awk '{ print $2 }')" 65 | if [[ $PID == "" ]]; then 66 | echo "No Process to kill on ${HOSTNAME}." 67 | else 68 | $(exec kill -9 $PID) 69 | echo "Kill process ${PID} on ${HOSTNAME}." 70 | fi 71 | fi 72 | -------------------------------------------------------------------------------- /bin/release.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | USAGE="Usage: bin/release.sh version_num" 3 | 4 | if [[ "$1" == "" ]]; then 5 | echo "ERROR : please give a release number" 6 | echo ${USAGE} 7 | exit 1 8 | fi 9 | 10 | bin=$(dirname "${BASH_SOURCE-$0}") 11 | bin=$(cd "${bin}">/dev/null; pwd) 12 | 13 | if [ -L ${BASH_SOURCE-$0} ]; then 14 | FWDIR=$(dirname $(readlink "${BASH_SOURCE-$0}")) 15 | else 16 | FWDIR=$(dirname "${BASH_SOURCE-$0}") 17 | fi 18 | 19 | if [[ -z "${VARONE_HOME}" ]]; then 20 | export VARONE_HOME="$(cd "${FWDIR}/.."; pwd)" 21 | fi 22 | 23 | cd ${bin}/.. 24 | gradle clean build shadowJar -x test 25 | 26 | if [[ -d "${VARONE_HOME}/varOne-${1}" ]]; then 27 | rm -rf "${VARONE_HOME}/varOne-${1}" 28 | fi 29 | 30 | export VARONE_RELEASE_DIR="${VARONE_HOME}/varOne-${1}" 31 | 32 | mkdir ${VARONE_RELEASE_DIR} 33 | mkdir ${VARONE_RELEASE_DIR}/conf 34 | mkdir ${VARONE_RELEASE_DIR}/bin 35 | cp ${VARONE_HOME}/bin/varOne* "${VARONE_RELEASE_DIR}/bin" 36 | cp "${VARONE_HOME}/conf/varOne-env.sh.template" "${VARONE_RELEASE_DIR}/conf" 37 | cp "${VARONE_HOME}/conf/varOne-site.xml.template" "${VARONE_RELEASE_DIR}/conf" 38 | cp "${VARONE_HOME}/conf/log4j.properties" "${VARONE_RELEASE_DIR}/conf" 39 | cp "${VARONE_HOME}/conf/varonedaemond" "${VARONE_RELEASE_DIR}/conf" 40 | cp -r "${VARONE_HOME}/docs" "${VARONE_RELEASE_DIR}" 41 | cp -r "${VARONE_HOME}/CONTRIBUTING.md" "${VARONE_RELEASE_DIR}" 42 | cp -r "${VARONE_HOME}/LICENSE" "${VARONE_RELEASE_DIR}" 43 | cp -r "${VARONE_HOME}/README.md" "${VARONE_RELEASE_DIR}" 44 | cp -r "${VARONE_HOME}/varOne-server/build/lib" "${VARONE_RELEASE_DIR}" 45 | cp "${VARONE_HOME}/varOne-node/build/libs/varOne-node-${1}.jar" "${VARONE_RELEASE_DIR}" 46 | cp "${VARONE_HOME}/varOne-server/build/libs/varOne-server-${1}.jar" "${VARONE_RELEASE_DIR}" 47 | cp "${VARONE_HOME}/varOne-web/build/libs/varOne-web-${1}.war" "${VARONE_RELEASE_DIR}" 48 | 49 | tar zcvf "varOne-${1}.tgz" varOne-${1} 50 | rm -rf ${VARONE_RELEASE_DIR} 51 | echo "Release varOne@${1} successfully." 52 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/components/navigator/menu-item.js: -------------------------------------------------------------------------------- 1 | import React, { PropTypes } from 'react'; 2 | 3 | export default class MenuItem extends React.Component { 4 | static propTypes = { 5 | text: PropTypes.string, 6 | icon: PropTypes.string, 7 | collapse: PropTypes.bool, 8 | children: PropTypes.array, 9 | jobItemClickCB: PropTypes.func, 10 | dimensionItemClickCB: PropTypes.func 11 | } 12 | 13 | handleSubMenuItemClick(e, applicationId) { 14 | this.props.jobItemClickCB(applicationId); 15 | e.stopPropagation(); 16 | } 17 | 18 | renderSubMenu() { 19 | if (this.props.collapse) { 20 | return ( 21 |
      22 | { 23 | this.props.children.map((item) => { 24 | return ( 25 |
    • this.handleSubMenuItemClick(e, item) }> 26 | { item } 27 |
      • 28 | ); 29 | }) 30 | } 31 |
    32 | ); 33 | } else { 34 | return null; 35 | } 36 | } 37 | 38 | handleMenuItemClick = () => { 39 | if (!this.props.collapse) { 40 | this.props.dimensionItemClickCB(this.props.text); 41 | } 42 | } 43 | 44 | renderDisplayText() { 45 | if (this.props.collapse) { 46 | return ( 47 | 52 | ); 53 | } else { 54 | return { this.props.text }; 55 | } 56 | } 57 | 58 | render() { 59 | const text = this.renderDisplayText(); 60 | const subMenu = this.renderSubMenu(); 61 | return ( 62 |
  • 63 | { text }{ subMenu } 64 |
    • 65 | ); 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /varOne-rpc/src/main/resources/metrics.proto: -------------------------------------------------------------------------------- 1 | option java_package = "com.varone.hadoop.rpc.protos"; 2 | option java_outer_classname = "MetricsProtos"; 3 | option java_generate_equals_and_hash = true; 4 | 5 | enum MetricsTypeProto { 6 | FS = 0; 7 | JVM = 1; 8 | GC = 2; 9 | THREAD_POOL = 3; 10 | 11 | EXEC_FS_LOCAL_LARGEREAD_OPS = 4; 12 | EXEC_FS_LOCAL_READ_BYTES = 5; 13 | EXEC_FS_LOCAL_READ_OPS = 6; 14 | EXEC_FS_LOCAL_WRITE_BYTES = 7; 15 | EXEC_FS_LOCAL_WRITE_OPS = 8; 16 | 17 | 18 | EXEC_FS_HDFS_LARGEREAD_OPS = 9; 19 | EXEC_FS_HDFS_READ_BYTES = 10; 20 | EXEC_FS_HDFS_READ_OPS = 11; 21 | EXEC_FS_HDFS_WRITE_BYTES = 12; 22 | EXEC_FS_HDFS_WRITE_OPS = 13; 23 | 24 | 25 | EXEC_THREADPOOL_ACTIVETASK = 14; 26 | EXEC_THREADPOOL_COMPLETETASK = 15; 27 | EXEC_THREADPOOL_CURRPOOL_SIZE = 16; 28 | EXEC_THREADPOOL_MAXPOOL_SIZE = 17; 29 | 30 | 31 | JVM_HEAP_COMMITED = 18; 32 | JVM_HEAP_INIT = 19; 33 | JVM_HEAP_MAX = 20; 34 | JVM_HEAP_USAGE = 21; 35 | JVM_HEAP_USED = 22; 36 | 37 | 38 | JVM_NON_HEAP_COMMITED = 23; 39 | JVM_NON_HEAP_INIT = 24; 40 | JVM_NON_HEAP_MAX = 25; 41 | JVM_NON_HEAP_USAGE = 26; 42 | JVM_NON_HEAP_USED = 27; 43 | 44 | 45 | JVM_POOLS_CODE_CACHE_USAGE = 28; 46 | JVM_POOLS_PS_EDEN_SPACE_USAGE = 29; 47 | JVM_POOLS_PS_OLD_GEN_USAGE = 30; 48 | JVM_POOLS_PS_PERM_GEN_USAGE = 31; 49 | JVM_POOLS_PS_SURVIVOR_SPACE_USAGE = 32; 50 | 51 | 52 | JVM_PS_MARKSWEEP_COUNT = 33; 53 | JVM_PS_MARKSWEEP_TIME = 34; 54 | JVM_PS_SCAVENGE_COUNT = 35; 55 | JVM_PS_SCAVENGE_TIME = 36; 56 | 57 | } 58 | 59 | message MetricsRequestProto{ 60 | required string applicationId = 1; 61 | repeated MetricsTypeProto metrics = 2; 62 | } 63 | 64 | 65 | message MetricsResponseProto{ 66 | message MetricsMapProto{ 67 | message TupleProto{ 68 | required int64 time = 1; 69 | optional string value = 2; 70 | } 71 | required string metricsName = 1; 72 | repeated TupleProto metricsValues = 2; 73 | } 74 | repeated MetricsMapProto result = 1; 75 | } 76 | -------------------------------------------------------------------------------- /varOne-server/src/main/java/com/varone/web/vo/JobVO.java: -------------------------------------------------------------------------------- 1 | package com.varone.web.vo; 2 | 3 | import javax.xml.bind.annotation.XmlRootElement; 4 | 5 | /** 6 | * @author allen 7 | * 8 | */ 9 | @XmlRootElement 10 | public class JobVO { 11 | private int id; 12 | private String description; 13 | private String submitTime; 14 | private String duration; 15 | private String stagesSuccessVSTotal; 16 | private String stagesSuccessPercent; 17 | private String tasksSuccessVSTotal; 18 | private String tasksSuccessPercent; 19 | public int getId() { 20 | return id; 21 | } 22 | public void setId(int id) { 23 | this.id = id; 24 | } 25 | public String getDescription() { 26 | return description; 27 | } 28 | public void setDescription(String description) { 29 | this.description = description; 30 | } 31 | public String getSubmitTime() { 32 | return submitTime; 33 | } 34 | public void setSubmitTime(String submitTime) { 35 | this.submitTime = submitTime; 36 | } 37 | public String getDuration() { 38 | return duration; 39 | } 40 | public void setDuration(String duration) { 41 | this.duration = duration; 42 | } 43 | public String getStagesSuccessVSTotal() { 44 | return stagesSuccessVSTotal; 45 | } 46 | public void setStagesSuccessVSTotal(String stagesSuccessVSTotal) { 47 | this.stagesSuccessVSTotal = stagesSuccessVSTotal; 48 | } 49 | public String getTasksSuccessVSTotal() { 50 | return tasksSuccessVSTotal; 51 | } 52 | public void setTasksSuccessVSTotal(String tasksSuccessVSTotal) { 53 | this.tasksSuccessVSTotal = tasksSuccessVSTotal; 54 | } 55 | public String getStagesSuccessPercent() { 56 | return stagesSuccessPercent; 57 | } 58 | public void setStagesSuccessPercent(String stagesSuccessPercent) { 59 | this.stagesSuccessPercent = stagesSuccessPercent; 60 | } 61 | public String getTasksSuccessPercent() { 62 | return tasksSuccessPercent; 63 | } 64 | public void setTasksSuccessPercent(String tasksSuccessPercent) { 65 | this.tasksSuccessPercent = tasksSuccessPercent; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/components/home.js: -------------------------------------------------------------------------------- 1 | require('../styles/sb-admin-2.css'); 2 | require('../styles/spm/spm.css'); 3 | require('../../node_modules/c3/c3.min.css'); 4 | require('../../node_modules/react-bootstrap-table/css/react-bootstrap-table-all.min.css'); 5 | import React from 'react'; 6 | import Const from '../utils/consts'; 7 | import Nav from './navigator/nav'; 8 | import ClusterContainer from './cluster/cluster-container'; 9 | import HistoryContainer from './history/history-container'; 10 | import NodesContainer from './nodes/nodes-container'; 11 | import JobContainer from './job/job-container'; 12 | 13 | class Home extends React.Component { 14 | 15 | state = { 16 | container: Const.menu.cluster, 17 | applicationId: null 18 | } 19 | 20 | handleJobItemClick = applicationId => { 21 | this.setState({ 22 | container: Const.menu.runningJobs, 23 | applicationId 24 | }); 25 | } 26 | 27 | handleDimensionItemClick = dimension => { 28 | this.setState({ 29 | container: dimension 30 | }); 31 | } 32 | 33 | handleNodeClick = () => { 34 | this.setState({ 35 | container: Const.menu.nodes 36 | }); 37 | } 38 | 39 | renderContainer() { 40 | if (this.state.container === Const.menu.cluster) { 41 | return ; 42 | } else if (this.state.container === Const.menu.runningJobs) { 43 | return ; 44 | } else if (this.state.container === Const.menu.nodes) { 45 | return ; 46 | } else if (this.state.container === Const.menu.history) { 47 | return ; 48 | } 49 | } 50 | 51 | render() { 52 | return ( 53 |
    54 |
    59 | ); 60 | } 61 | } 62 | 63 | export default Home; 64 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/test/spec/components/cluster/cluster-metrics.test.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom'; 3 | import stubApp from '../../../utils/stub-app'; 4 | import ClusterMetrics from '../../../../app/components/cluster/cluster-metrics'; 5 | 6 | const should = chai.should(); 7 | 8 | const mockMetrics = { 9 | metricProp1: { 10 | id: 'mock_id1', 11 | title: 'mock_title1', 12 | value: [ [ 's1', '0', '0', '0' ], [ 's1', '0', '20', '40' ], [ 's1', '0', '5', '9' ] ], 13 | x: [ 'x', 1454292960000, 1454292990000, 1454293020000 ], 14 | format: 'OPS' 15 | }, 16 | metricProp2: { 17 | id: 'mock_id2', 18 | title: 'mock_title2', 19 | value: [ [ 's1', '0', '0', '0' ], [ 's1', '0', '20', '40' ], [ 's1', '0', '5', '9' ] ], 20 | x: [ 'x', 1454292960000, 1454292990000, 1454293020000 ], 21 | format: 'BYTE' 22 | }, 23 | metricProp3: { 24 | id: 'mock_id3', 25 | title: 'mock_title3', 26 | value: [ [ 's1', '0', '0', '0' ], [ 's1', '0', '20', '40' ], [ 's1', '0', '5', '9' ] ], 27 | x: [ 'x', 1454292960000, 1454292990000, 1454293020000 ], 28 | format: 'OPS' 29 | } 30 | }; 31 | 32 | describe('Cluster Metrics', () => { 33 | let node; 34 | let instance; 35 | 36 | beforeEach(() => { 37 | const stub = stubApp()(ClusterMetrics, { 38 | metrics: mockMetrics 39 | }); 40 | node = window.document.createElement('div'); 41 | instance = ReactDOM.render(React.createElement(stub), node); 42 | }); 43 | 44 | afterEach(() => { 45 | if (instance) ReactDOM.unmountComponentAtNode(node); 46 | }); 47 | 48 | it('should render correctly', () => { 49 | const dom = ReactDOM.findDOMNode(instance); 50 | 51 | dom.tagName.should.eql('DIV'); 52 | (dom.className.indexOf('row') !== -1).should.be.ok; 53 | dom.childNodes.length.should.eql(1); 54 | 55 | const rows = dom.childNodes[0]; 56 | 57 | should.exist(rows); 58 | rows.childNodes.length.should.eql(Math.ceil(Object.keys(mockMetrics).length / 2)); 59 | }); 60 | }); 61 | -------------------------------------------------------------------------------- /varOne-server/build.gradle: -------------------------------------------------------------------------------- 1 | import java.nio.file.*; 2 | 3 | dependencies { 4 | compile project(":varOne-node") 5 | compile('org.apache.cxf:cxf-rt-frontend-jaxrs:2.7.7'){ 6 | exclude group: "javax.ws.rs", module: "javax.ws.rs-api" 7 | } 8 | compile 'org.apache.cxf:cxf-rt-transports-http:2.7.7' 9 | compile('org.apache.cxf:cxf-rt-transports-http-jetty:2.7.7'){ 10 | exclude group: "org.eclipse.jetty", module: "jetty-server" 11 | exclude group: "org.eclipse.jetty", module: "jetty-security" 12 | exclude group: "org.slf4j", module: "slf4j-api" 13 | } 14 | compile('org.apache.cxf:cxf-api:1.9'){ 15 | exclude group: "org.eclipse.jetty", module: "*" 16 | } 17 | compile 'org.eclipse.jetty.aggregate:jetty-all-server:8.1.14.v20131031' 18 | compile 'javax.ws.rs:javax.ws.rs-api:2.0-m10' 19 | compile 'org.slf4j:slf4j-api:1.6.6' 20 | compile 'org.slf4j:slf4j-log4j12:1.6.3' 21 | compile 'com.google.code.gson:gson:2.3' 22 | compile('com.sun.jersey:jersey-json:1.9'){ 23 | exclude group: "com.sun.xml.bind", module: "jaxb-impl" 24 | } 25 | } 26 | jar { 27 | manifest { 28 | attributes 'Main-Class': 'com.varone.server.VarOneServer' 29 | } 30 | } 31 | 32 | task copyToLib(){ 33 | doLast{ 34 | configurations.compile.copy().allDependencies.each { dep -> 35 | String directory = "$buildDir/lib" 36 | if(!new File(directory).exists()){ 37 | new File(directory).mkdirs() 38 | } 39 | configurations.compile.each { File path -> 40 | String dstJarFilePath = directory + "/" + path.name 41 | if(!new File(dstJarFilePath).exists()){ 42 | Files.copy(path.toPath(), FileSystems.getDefault().getPath(dstJarFilePath)) 43 | } 44 | } 45 | } 46 | } 47 | } 48 | jar.dependsOn('copyToLib') 49 | task execVarOneServer(dependsOn: 'classes', type: JavaExec) { 50 | main = 'com.varone.server.VarOneServer' 51 | classpath = files([files(sourceSets.main.runtimeClasspath), files("../conf")]) 52 | standardInput = System.in 53 | } 54 | -------------------------------------------------------------------------------- /varOne-server/src/test/java/com/varone/web/aggregator/timeperiod/TimePeriodTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.web.aggregator.timeperiod; 5 | 6 | import static org.junit.Assert.*; 7 | 8 | import org.junit.Before; 9 | import org.junit.Test; 10 | 11 | /** 12 | * @author allen 13 | * 14 | */ 15 | public class TimePeriodTest { 16 | 17 | /** 18 | * @throws java.lang.Exception 19 | */ 20 | @Before 21 | public void setUp() throws Exception { 22 | } 23 | 24 | /** 25 | * Test method for {@link com.varone.web.aggregator.timeperiod.TimePeriod#fromString(java.lang.String)}. 26 | */ 27 | @Test 28 | public void testFromStringWith30m() { 29 | TimePeriod timePeriod = TimePeriod.fromString("30m"); 30 | assertEquals(timePeriod, TimePeriod.MINUTE_30); 31 | } 32 | 33 | /** 34 | * Test method for {@link com.varone.web.aggregator.timeperiod.TimePeriod#fromString(java.lang.String)}. 35 | */ 36 | @Test 37 | public void testFromStringWith1h() { 38 | TimePeriod timePeriod = TimePeriod.fromString("1h"); 39 | assertEquals(timePeriod, TimePeriod.HOUR_1); 40 | } 41 | 42 | /** 43 | * Test method for {@link com.varone.web.aggregator.timeperiod.TimePeriod#fromString(java.lang.String)}. 44 | */ 45 | @Test 46 | public void testFromStringWith2h() { 47 | TimePeriod timePeriod = TimePeriod.fromString("2h"); 48 | assertEquals(timePeriod, TimePeriod.HOUR_2); 49 | } 50 | 51 | /** 52 | * Test method for {@link com.varone.web.aggregator.timeperiod.TimePeriod#fromString(java.lang.String)}. 53 | */ 54 | @Test 55 | public void testFromStringWith6h() { 56 | TimePeriod timePeriod = TimePeriod.fromString("6h"); 57 | assertEquals(timePeriod, TimePeriod.HOUR_6); 58 | } 59 | 60 | /** 61 | * Test method for {@link com.varone.web.aggregator.timeperiod.TimePeriod#fromString(java.lang.String)}. 62 | */ 63 | @Test 64 | public void testFromStringWithInvalidStr() { 65 | try{ 66 | TimePeriod.fromString("invalid"); 67 | fail("Should not be passed"); 68 | } catch(Exception e){ 69 | assertEquals(e.getMessage(), "Time period expression could not be matched: invalid"); 70 | } 71 | } 72 | 73 | } 74 | -------------------------------------------------------------------------------- /varOne-server/src/test/java/com/varone/web/reader/metrics/impl/MetricsRpcReaderImplTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.web.reader.metrics.impl; 5 | 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | 9 | import junit.framework.TestCase; 10 | 11 | import com.varone.web.metrics.bean.MetricBean; 12 | import com.varone.web.metrics.bean.NodeBean; 13 | import com.varone.web.reader.metrics.MetricsReader; 14 | import com.varone.web.reader.metrics.impl.MetricsRpcReaderImpl; 15 | 16 | /** 17 | * @author allen 18 | * 19 | */ 20 | public class MetricsRpcReaderImplTest extends TestCase { 21 | final static List HOSTS = new ArrayList(){{add("server-a2");add("server-a3");add("server-a4");add("server-a5");}}; 22 | /** 23 | * @param name 24 | */ 25 | public MetricsRpcReaderImplTest(String name) { 26 | super(name); 27 | } 28 | 29 | /* (non-Javadoc) 30 | * @see junit.framework.TestCase#setUp() 31 | */ 32 | protected void setUp() throws Exception { 33 | super.setUp(); 34 | } 35 | 36 | /** 37 | * Test method for {@link com.varone.web.reader.metrics.impl.MetricsRpcReaderImpl#getAllNodeMetrics(java.lang.String, java.util.List)}. 38 | * @throws Exception 39 | */ 40 | public void testGetNodeMetrics() throws Exception { 41 | MetricsReader reader = new MetricsRpcReaderImpl(HOSTS, 8888); 42 | String applicationId = "application_1439169262151_0237"; 43 | List metricsType = new ArrayList(); 44 | 45 | metricsType.add("FS"); 46 | 47 | List nodeMetrics = reader.getAllNodeMetrics(applicationId, metricsType); 48 | 49 | int totalMetricsNum = 0; 50 | assertEquals(nodeMetrics.size(), HOSTS.size()); 51 | for(NodeBean nodeVO: nodeMetrics){ 52 | assertTrue(HOSTS.contains(nodeVO.getHost())); 53 | assertNotNull(nodeVO.getMetrics()); 54 | //TODO 55 | //assertTrue(nodeVO.getMetrics().size() > 0); 56 | totalMetricsNum += nodeVO.getMetrics().size(); 57 | for(MetricBean metric: nodeVO.getMetrics()){ 58 | System.out.println(metric.getName()); 59 | } 60 | } 61 | //TODO 62 | //assertEquals(totalMetricsNum, 50); 63 | } 64 | 65 | } 66 | -------------------------------------------------------------------------------- /varOne-server/src/main/java/com/varone/web/resource/ClusterResource.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.web.resource; 5 | 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | 9 | import javax.ws.rs.GET; 10 | import javax.ws.rs.Path; 11 | import javax.ws.rs.Produces; 12 | import javax.ws.rs.QueryParam; 13 | import javax.ws.rs.core.MediaType; 14 | 15 | import org.apache.log4j.Logger; 16 | 17 | import com.google.gson.Gson; 18 | import com.varone.web.exception.VarOneException; 19 | import com.varone.web.exception.VarOneExceptionParser; 20 | import com.varone.web.facade.SparkMonitorFacade; 21 | import com.varone.web.vo.DefaultTotalNodeVO; 22 | 23 | /** 24 | * @author allen 25 | * 26 | */ 27 | @Produces(MediaType.APPLICATION_JSON) 28 | @Path("/cluster") 29 | public class ClusterResource { 30 | private Logger logger = Logger.getLogger(ClusterResource.class.getName()); 31 | 32 | @GET 33 | @Path("/") 34 | public String fetchClusterDashBoard(@QueryParam("metrics") String metrics, 35 | @QueryParam("period") String period){ 36 | logger.info("start fetchClusterDashBoard method ..."); 37 | logger.debug("metrics = " + metrics + " period = " + period); 38 | try{ 39 | SparkMonitorFacade facade = new SparkMonitorFacade(); 40 | List metricsAsList = new ArrayList(); 41 | if(metrics != null){ 42 | String[] metricsArr = metrics.split(","); 43 | for(String metric: metricsArr){ 44 | if(!metric.trim().equals("")) 45 | metricsAsList.add(metric); 46 | } 47 | } 48 | 49 | DefaultTotalNodeVO result = facade.getDefaultClusterDashBoard(metricsAsList, period); 50 | Gson gson = new Gson(); 51 | String toJson = gson.toJson(result); 52 | 53 | logger.debug("metricsAsList size = " + metricsAsList.size() + " period = " + period); 54 | logger.debug(" toJson = " + toJson); 55 | logger.info("finish fetchClusterDashBoard method ..."); 56 | return toJson; 57 | }catch(Exception e){ 58 | VarOneExceptionParser parser = new VarOneExceptionParser(); 59 | String errorMessage = parser.parse(e); 60 | logger.error(errorMessage); 61 | throw new VarOneException(errorMessage); 62 | } 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/components/nodes/nodes-header.js: -------------------------------------------------------------------------------- 1 | import React, { PropTypes } from 'react'; 2 | import TimePeriodPill from '../commons/time-period-pills'; 3 | 4 | class NodeHeader extends React.Component { 5 | 6 | static propTypes = { 7 | nodes: PropTypes.array, 8 | period: PropTypes.string, 9 | onNodeSelect: PropTypes.func, 10 | onPeriodSelect: PropTypes.func 11 | } 12 | 13 | state = { 14 | selected: 'Select Node' 15 | } 16 | 17 | constructor(props) { 18 | super(props); 19 | } 20 | 21 | handleNodeItemClick = e => { 22 | this.props.onNodeSelect(e.target.text); 23 | this.setState({ 24 | selected: e.target.text 25 | }); 26 | } 27 | 28 | render() { 29 | const nodes = this.props.nodes.map((node) => { 30 | return ( 31 |
  • { node }
    • 32 | ); 33 | }); 34 | 35 | return ( 36 |
    37 |
    38 |

    39 | Nodes   40 |
    41 | 49 |
      50 | { nodes } 51 |
    52 |   53 | 56 |
    57 |

    58 |
    59 |
    60 |

    61 | 64 |

    65 |
    66 |
    67 | ); 68 | } 69 | } 70 | 71 | export default NodeHeader; 72 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/test/spec/components/cluster/cluster-summary-block.test.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom'; 3 | import TestUtils from 'react-addons-test-utils'; 4 | import stubApp from '../../../utils/stub-app'; 5 | import ClusterSummaryBlock from '../../../../app/components/cluster/cluster-summary-block'; 6 | 7 | chai.should(); 8 | 9 | const mockTitle = 'summary_title'; 10 | const mockIcon = 'fa-tasks'; 11 | const mockPanel = 'panel-green'; 12 | const mockValue = 3; 13 | 14 | describe('Cluster Summary Block', () => { 15 | let node; 16 | let instance; 17 | let spy; 18 | 19 | beforeEach(() => { 20 | spy = sinon.spy(); 21 | const stub = stubApp()(ClusterSummaryBlock, { 22 | title: mockTitle, 23 | icon: mockIcon, 24 | panel: mockPanel, 25 | value: mockValue, 26 | onSummaryClick: spy 27 | }); 28 | node = window.document.createElement('div'); 29 | instance = ReactDOM.render(React.createElement(stub), node); 30 | }); 31 | 32 | afterEach(() => { 33 | if (instance) ReactDOM.unmountComponentAtNode(node); 34 | }); 35 | 36 | it('should render correctly', () => { 37 | const dom = ReactDOM.findDOMNode(instance); 38 | const summaryIcon = TestUtils.scryRenderedDOMComponentsWithClass(instance, mockIcon); 39 | const summaryVal = TestUtils.scryRenderedDOMComponentsWithClass(instance, 'huge'); 40 | const summaryTitle = TestUtils.scryRenderedDOMComponentsWithClass(instance, 'medium'); 41 | 42 | dom.tagName.should.eql('DIV'); 43 | dom.childNodes.length.should.eql(2); 44 | (dom.className.indexOf(mockPanel) !== -1).should.be.ok; 45 | summaryIcon.length.should.eql(1); 46 | summaryTitle.length.should.eql(1); 47 | summaryVal.length.should.eql(1); 48 | summaryTitle[0].textContent.should.eql(mockTitle); 49 | summaryVal[0].textContent.should.eql(mockValue.toString()); 50 | }); 51 | 52 | it('should summary block click', () => { 53 | const dom = ReactDOM.findDOMNode(instance); 54 | const summaryLink = dom.childNodes[1]; 55 | TestUtils.Simulate.click(summaryLink); 56 | spy.should.have.been.calledOnce; 57 | spy.should.have.been.calledWith(mockTitle); 58 | }); 59 | }); 60 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/components/commons/varOne-log-modal.js: -------------------------------------------------------------------------------- 1 | import React, { PropTypes } from 'react'; 2 | import VarOneStore from '../../stores/varOne-store'; 3 | import connectToStores from 'alt/utils/connectToStores'; 4 | 5 | 6 | @connectToStores 7 | export default class VarOneLogModal extends React.Component { 8 | 9 | static propTypes = { 10 | msg: PropTypes.string, 11 | port: PropTypes.string, 12 | inputPort: PropTypes.string, 13 | failMessage: PropTypes.string 14 | } 15 | 16 | static getStores() { 17 | return [ VarOneStore ]; 18 | } 19 | 20 | static getPropsFromStores() { 21 | return VarOneStore.getState(); 22 | } 23 | 24 | render() { 25 | const failResultMessage = this.props.failMessage; 26 | if (failResultMessage !== null) { 27 | return ( 28 | 43 | ); 44 | } else { 45 | return ( 46 | 58 | ); 59 | } 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /varOne-server/src/main/java/com/varone/web/vo/DefaultTotalNodeVO.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.web.vo; 5 | 6 | import java.util.List; 7 | import java.util.Map; 8 | 9 | import javax.xml.bind.annotation.XmlRootElement; 10 | 11 | /** 12 | * @author allen 13 | * 14 | */ 15 | @XmlRootElement 16 | public class DefaultTotalNodeVO { 17 | private int nodeNum; 18 | private int jobNum; 19 | private int taskNum; 20 | private int executorNum; 21 | 22 | private List metricProps; 23 | 24 | 25 | private Map taskStartedNumByNode; 26 | private Map executorNumByNode; 27 | 28 | private Map>> propToMetrics; 29 | 30 | public int getNodeNum() { 31 | return nodeNum; 32 | } 33 | public void setNodeNum(int nodeNum) { 34 | this.nodeNum = nodeNum; 35 | } 36 | public int getJobNum() { 37 | return jobNum; 38 | } 39 | public void setJobNum(int jobNum) { 40 | this.jobNum = jobNum; 41 | } 42 | public int getTaskNum() { 43 | return taskNum; 44 | } 45 | public void setTaskNum(int taskNum) { 46 | this.taskNum = taskNum; 47 | } 48 | public int getExecutorNum() { 49 | return executorNum; 50 | } 51 | public void setExecutorNum(int executorNum) { 52 | this.executorNum = executorNum; 53 | } 54 | public Map getTaskStartedNumByNode() { 55 | return taskStartedNumByNode; 56 | } 57 | public void setTaskStartedNumByNode(Map taskNumByNode) { 58 | this.taskStartedNumByNode = taskNumByNode; 59 | } 60 | public Map getExecutorNumByNode() { 61 | return executorNumByNode; 62 | } 63 | public void setExecutorNumByNode(Map executorNumByNode) { 64 | this.executorNumByNode = executorNumByNode; 65 | } 66 | public List getMetricProps() { 67 | return metricProps; 68 | } 69 | public void setMetricProps(List metricProps) { 70 | this.metricProps = metricProps; 71 | } 72 | public Map>> getPropToMetrics() { 73 | return propToMetrics; 74 | } 75 | public void setPropToMetrics(Map>> propToMetrics) { 76 | this.propToMetrics = propToMetrics; 77 | } 78 | 79 | } 80 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/test/spec/stores/cluster-store.test.js: -------------------------------------------------------------------------------- 1 | import alt from '../../../app/alt'; 2 | import Const from '../../../app/utils/consts'; 3 | import ClusterAction from '../../../app/actions/cluster-action'; 4 | import ClusterStore from '../../../app/stores/cluster-store'; 5 | 6 | const should = chai.should(); 7 | 8 | const mockResponse = { 9 | nodeNum: 5, 10 | jobNum: 0, 11 | taskNum: 0, 12 | executorNum: 0, 13 | metricProps: [], 14 | taskStartedNumByNode: { 15 | 'server-a5': 0, 16 | 'server-a1': 0, 17 | 'server-a2': 0, 18 | 'server-a3': 0, 19 | 'server-a4': 0 20 | }, 21 | executorNumByNode: { 22 | 'server-a5': 0, 23 | 'server-a1': 0, 24 | 'server-a2': 0, 25 | 'server-a3': 0, 26 | 'server-a4': 0 27 | }, 28 | propToMetrics: {} 29 | }; 30 | 31 | const mockPeriod = Const.shared.timeperiod[2]; 32 | 33 | describe('Cluster Store', () => { 34 | it('listens for fetchTotalNodeDashBoardSuccessful action', () => { 35 | const action = ClusterAction.FETCH_TOTAL_NODE_DASH_BOARD_SUCCESSFUL; 36 | const taskStartedNumByNodeKeys = Object.keys(mockResponse.taskStartedNumByNode); 37 | const executorNumByNodeKeys = Object.keys(mockResponse.executorNumByNode); 38 | 39 | alt.dispatcher.dispatch({ 40 | action, 41 | data: { result: mockResponse, period: mockPeriod } 42 | }); 43 | 44 | const state = ClusterStore.getState(); 45 | 46 | should.exist(state.data); 47 | should.exist(state.data.metrics); 48 | should.exist(state.data.displaySummaryInfo); 49 | should.exist(state.data.taskStartedNumByNode); 50 | should.exist(state.data.executorNumByNode); 51 | state.data.displaySummaryInfo.length.should.eql(4); 52 | 53 | state.data.taskStartedNumByNode.length.should.eql(taskStartedNumByNodeKeys.length); 54 | state.data.executorNumByNode.length.should.eql(executorNumByNodeKeys.length); 55 | 56 | let nodes = state.data.taskStartedNumByNode.map(x => x[0]); 57 | nodes.every((o) => taskStartedNumByNodeKeys.find(x => x === o)).should.be.ok; 58 | 59 | nodes = state.data.executorNumByNode.map(x => x[0]); 60 | nodes.every((o) => executorNumByNodeKeys.find(x => x === o)).should.be.ok; 61 | 62 | state.period.should.eql(mockPeriod); 63 | }); 64 | }); 65 | -------------------------------------------------------------------------------- /varOne-server/src/test/java/com/varone/web/yarn/service/YarnServiceTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.web.yarn.service; 5 | 6 | import java.io.IOException; 7 | import java.util.Arrays; 8 | import java.util.List; 9 | 10 | import org.apache.hadoop.conf.Configuration; 11 | import org.apache.hadoop.yarn.exceptions.YarnException; 12 | 13 | import com.varone.web.yarn.service.YarnService; 14 | 15 | import junit.framework.TestCase; 16 | 17 | /** 18 | * @author user1 19 | * 20 | */ 21 | public class YarnServiceTest extends TestCase { 22 | private Configuration config; 23 | /** 24 | * @param name 25 | */ 26 | public YarnServiceTest(String name) { 27 | super(name); 28 | this.config = new Configuration(); 29 | this.config.set("fs.default.name", "hdfs://server-a1:9000"); 30 | this.config.set("yarn.resourcemanager.address", "server-a1:8032"); 31 | } 32 | 33 | /* (non-Javadoc) 34 | * @see junit.framework.TestCase#setUp() 35 | */ 36 | protected void setUp() throws Exception { 37 | super.setUp(); 38 | } 39 | 40 | /** 41 | * Test method for {@link com.varone.web.yarn.service.YarnService#getAllNodeHost()}. 42 | * @throws IOException 43 | * @throws YarnException 44 | */ 45 | public void testGetAllNodeHost() throws YarnException, IOException { 46 | 47 | List hosts = Arrays.asList(new String[]{"server-a2","server-a3","server-a4","server-a5","server-a1"}); 48 | 49 | YarnService service = new YarnService(this.config); 50 | List allNodeHost = service.getAllNodeHost(); 51 | assertNotNull(allNodeHost); 52 | assertTrue(allNodeHost.size() > 0); 53 | 54 | for(String hostname: allNodeHost){ 55 | assertTrue(hosts.contains(hostname)); 56 | } 57 | 58 | service.close(); 59 | } 60 | 61 | /** 62 | * Test method for {@link com.varone.web.yarn.service.YarnService#getRunningSparkApplications()}. 63 | * @throws IOException 64 | * @throws YarnException 65 | */ 66 | public void testGetRunningSparkApplication() throws YarnException, IOException { 67 | 68 | YarnService service = new YarnService(this.config); 69 | List runningSparkAppId = service.getRunningSparkApplications(); 70 | //TODO 71 | //assertEquals(runningSparkAppId.size(), 1); 72 | service.close(); 73 | } 74 | 75 | } 76 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/components/cluster/cluster-metric.js: -------------------------------------------------------------------------------- 1 | import React, { PropTypes } from 'react'; 2 | import c3 from 'c3'; 3 | import { byteFormatter, percentageFormatter, millisFormatter } from '../../utils/data-format'; 4 | 5 | export default class ClusterMetric extends React.Component { 6 | 7 | cluster_metrics_chart = null; 8 | 9 | static propTypes = { 10 | metric: PropTypes.object 11 | } 12 | 13 | componentDidMount() { 14 | this.defaultCharting(); 15 | } 16 | 17 | componentDidUpdate() { 18 | this.reloadCharting(); 19 | } 20 | 21 | render() { 22 | return ( 23 |
    24 |
    25 | { this.props.metric.title } 26 |
    27 |
    28 | 29 |
    30 |
    31 |
    32 |
    33 | ); 34 | } 35 | 36 | reloadCharting() { 37 | const columns = [ this.props.metric.x ]; 38 | 39 | for (let i = 0; i < this.props.metric.value.length; i++) { 40 | columns.push(this.props.metric.value[i]); 41 | } 42 | this.cluster_metrics_chart.load({ columns }); 43 | } 44 | 45 | defaultCharting() { 46 | let formatter; 47 | const columns = [ this.props.metric.x ]; 48 | 49 | for (let i = 0; i < this.props.metric.value.length; i++) { 50 | columns.push(this.props.metric.value[i]); 51 | } 52 | 53 | if (this.props.metric.format === 'BYTE') formatter = byteFormatter; 54 | else if (this.props.metric.format === 'PERCENTAGE') formatter = percentageFormatter; 55 | else if (this.props.metric.format === 'MILLIS') formatter = millisFormatter; 56 | 57 | this.cluster_metrics_chart = c3.generate({ 58 | bindto: '#' + this.props.metric.id, 59 | data: { 60 | x: 'x', 61 | columns 62 | }, 63 | axis: { 64 | x: { 65 | type: 'timeseries', 66 | tick: { 67 | format: '%H:%M:%S' 68 | } 69 | }, 70 | y: { 71 | padding: { bottom: 1 }, 72 | tick: { 73 | format: formatter 74 | } 75 | } 76 | } 77 | }); 78 | } 79 | } 80 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "varOne-web", 3 | "version": "1.0.0", 4 | "description": "", 5 | "main": "index.js", 6 | "scripts": { 7 | "dev-test": "babel-node ./node_modules/.bin/karma start", 8 | "dev": "node --harmony ./webpack/webpack-dev-server.js", 9 | "production": "NODE_ENV=production babel-node ./node_modules/.bin/webpack --config ./webpack/production-config.js", 10 | "lint": "eslint app webpack --ext .jsx,.js" 11 | }, 12 | "author": "AllenFang", 13 | "license": "ISC", 14 | "dependencies": { 15 | "alt": "^0.17.3", 16 | "bootstrap-sass": "^3.3.6", 17 | "c3": "^0.4.10", 18 | "classnames": "^2.1.3", 19 | "debug": "^2.2.0", 20 | "font-awesome": "^4.5.0", 21 | "jquery": "^2.2.0", 22 | "react": "^0.14.0", 23 | "react-bootstrap-table": "1.3.3", 24 | "react-dom": "^0.14.1", 25 | "react-router": "^1.0.0-rc1", 26 | "superagent": "^1.4.0" 27 | }, 28 | "devDependencies": { 29 | "babel": "^5.8.23", 30 | "babel-core": "^5.8.24", 31 | "babel-eslint": "^4.1.6", 32 | "babel-loader": "^5.3.2", 33 | "bootstrap-loader": "1.0.2", 34 | "chai": "^3.3.0", 35 | "css-loader": "^0.18.0", 36 | "eslint": "^1.10.3", 37 | "eslint-config-airbnb": "^3.1.0", 38 | "eslint-loader": "^1.2.0", 39 | "eslint-plugin-react": "^3.15.0", 40 | "extract-text-webpack-plugin": "^0.9.1", 41 | "faux-jax": "^4.2.2", 42 | "file-loader": "^0.8.5", 43 | "imports-loader": "^0.6.5", 44 | "karma": "^0.13.10", 45 | "karma-chai-plugins": "^0.6.1", 46 | "karma-chrome-launcher": "^0.2.2", 47 | "karma-coverage": "^0.5.3", 48 | "karma-coveralls": "^1.1.2", 49 | "karma-htmlfile-reporter": "^0.2.2", 50 | "karma-mocha": "^0.2.0", 51 | "karma-sinon": "^1.0.4", 52 | "karma-sourcemap-loader": "^0.3.5", 53 | "karma-webpack": "^1.7.0", 54 | "lodash": "^4.0.1", 55 | "mocha": "^2.3.3", 56 | "node-sass": "^3.4.2", 57 | "react-addons-test-utils": "^0.14.6", 58 | "react-hot-loader": "^1.3.0", 59 | "resolve-url-loader": "^1.4.3", 60 | "sass-loader": "^3.1.2", 61 | "sinon": "^1.17.1", 62 | "style-loader": "^0.13.0", 63 | "url-loader": "^0.5.7", 64 | "webpack": "^1.12.2", 65 | "webpack-dev-server": "^1.11.0" 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /varOne-server/src/test/java/com/varone/web/metrics/client/MetricsStubMultiThreadTest.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.web.metrics.client; 5 | 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | 9 | import com.varone.hadoop.rpc.metrics.MetricsServiceResponse; 10 | import com.varone.hadoop.rpc.protos.MetricsProtos.MetricsTypeProto; 11 | import com.varone.hadoop.rpc.protos.MetricsProtos.MetricsResponseProto.MetricsMapProto; 12 | import com.varone.web.reader.metrics.stub.MetricsStub; 13 | import com.varone.web.reader.metrics.stub.RpcThreadListener; 14 | 15 | 16 | /** 17 | * @author allen 18 | * 19 | */ 20 | public class MetricsStubMultiThreadTest{ 21 | 22 | final static String[] HOSTS = new String[]{"server-a2","server-a3","server-a4","server-a5"}; 23 | 24 | public static void main(String[] args) throws InterruptedException{ 25 | String applicationId = "application_1439169262151_0539"; 26 | 27 | List metricsTypeProto = new ArrayList(); 28 | metricsTypeProto.add(MetricsTypeProto.EXEC_FS_HDFS_READ_BYTES); 29 | 30 | RpcThreadListener rpcThreadListener = new RpcThreadListener(); 31 | List stubs = new ArrayList(HOSTS.length); 32 | 33 | 34 | for(String host: HOSTS){ 35 | MetricsStub stub = new MetricsStub(host, 8888, applicationId, metricsTypeProto, rpcThreadListener); 36 | stubs.add(stub); 37 | Thread t1 = new Thread(stub); 38 | t1.start(); 39 | } 40 | 41 | synchronized(rpcThreadListener){ 42 | if(!rpcThreadListener.isAllComplete()){ 43 | rpcThreadListener.wait(); 44 | } 45 | } 46 | 47 | System.out.println("finish !!!!!!!!!"); 48 | 49 | for(MetricsStub stub: stubs){ 50 | 51 | MetricsServiceResponse response = stub.getNodeMetrics(); 52 | 53 | List result = response.getResult(); 54 | System.out.println(result.size()); 55 | 56 | // for(MetricsMapProto proto: result){ 57 | // System.out.println(proto.getMetricsName()); 58 | // System.out.println("========"); 59 | // List metricsValuesList = proto.getMetricsValuesList(); 60 | // for(TupleProto tuple: metricsValuesList){ 61 | // System.out.println(tuple.getTime()+"=>"+tuple.getValue()); 62 | // } 63 | // 64 | // } 65 | } 66 | } 67 | 68 | } 69 | -------------------------------------------------------------------------------- /varOne-rpc/src/test/java/com/varone/MetricsMockClient.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone; 5 | 6 | import java.io.IOException; 7 | import java.net.InetSocketAddress; 8 | import java.util.ArrayList; 9 | import java.util.List; 10 | 11 | import org.apache.hadoop.conf.Configuration; 12 | 13 | import com.varone.hadoop.rpc.metrics.MetricsServiceRequest; 14 | import com.varone.hadoop.rpc.metrics.MetricsServiceResponse; 15 | import com.varone.hadoop.rpc.metrics.impl.MetricsServiceRequestPBImpl; 16 | import com.varone.hadoop.rpc.metrics.impl.pb.client.MetricsServicePBClientImpl; 17 | import com.varone.hadoop.rpc.protos.MetricsProtos.MetricsTypeProto; 18 | 19 | /** 20 | * @author allen 21 | * 22 | */ 23 | public class MetricsMockClient { 24 | 25 | /** 26 | * @throws Exception 27 | * 28 | */ 29 | public static void main(String[] args) throws Exception { 30 | Configuration config = new Configuration(); 31 | // config.set("hbase.zookeeper.quorum", "server-a1"); 32 | //// hiveConf.set("hbase.zookeeper.quorum", "server-b1"); 33 | // config.set("hbase.zookeeper.property.clientPort", "2181"); 34 | // config.set("fs.default.name", "hdfs://server-a1:9000"); 35 | // config.set("yarn.resourcemanager.address", "server-a1:8032"); 36 | // config.set("yarn.resourcemanager.scheduler.address", "server-a1:8030"); 37 | // config.set("yarn.resourcemanager.resource-tracker.address", "server-a1:8031"); 38 | // config.set("yarn.resourcemanager.admin.address", "server-a1:8033"); 39 | // config.set("mapreduce.framework.name", "yarn"); 40 | // config.set("mapreduce.johistory.address", "server-a1:10020"); 41 | // config.set("yarn.nodemanager.aux-services", "mapreduce_shuffle"); 42 | 43 | MetricsServicePBClientImpl client = new MetricsServicePBClientImpl(1, new InetSocketAddress("user1-pc", 8088), config); 44 | 45 | MetricsServiceRequest request = new MetricsServiceRequestPBImpl(); 46 | request.setApplicationId("application_123456789"); 47 | 48 | List metrics = new ArrayList(); 49 | metrics.add(MetricsTypeProto.FS); 50 | request.setMetrics(metrics); 51 | 52 | MetricsServiceResponse response = client.getMetrics(request); 53 | 54 | System.out.println(response.getResult().get(0).getMetricsName()); 55 | 56 | } 57 | 58 | } 59 | -------------------------------------------------------------------------------- /varOne-rpc/src/main/java/com/varone/hadoop/rpc/metrics/impl/pb/client/MetricsServicePBClientImpl.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.hadoop.rpc.metrics.impl.pb.client; 5 | 6 | import java.io.Closeable; 7 | import java.io.IOException; 8 | import java.net.InetSocketAddress; 9 | 10 | import org.apache.hadoop.conf.Configuration; 11 | import org.apache.hadoop.ipc.ProtobufRpcEngine; 12 | import org.apache.hadoop.ipc.RPC; 13 | import org.apache.hadoop.yarn.ipc.RPCUtil; 14 | 15 | import com.google.protobuf.ServiceException; 16 | import com.varone.hadoop.rpc.metrics.MetricsService; 17 | import com.varone.hadoop.rpc.metrics.MetricsServicePB; 18 | import com.varone.hadoop.rpc.metrics.MetricsServiceRequest; 19 | import com.varone.hadoop.rpc.metrics.MetricsServiceResponse; 20 | import com.varone.hadoop.rpc.metrics.impl.MetricsServiceRequestPBImpl; 21 | import com.varone.hadoop.rpc.metrics.impl.MetricsServiceResponsePBImpl; 22 | import com.varone.hadoop.rpc.protos.MetricsProtos.MetricsRequestProto; 23 | 24 | /** 25 | * @author allen 26 | * 27 | */ 28 | public class MetricsServicePBClientImpl implements MetricsService, Closeable { 29 | 30 | private MetricsServicePB proxy; 31 | 32 | public MetricsServicePBClientImpl(long clientVersion, InetSocketAddress addr, Configuration conf) throws IOException{ 33 | RPC.setProtocolEngine(conf, MetricsServicePB.class, ProtobufRpcEngine.class); 34 | this.proxy = (MetricsServicePB)RPC 35 | .getProxy(MetricsServicePB.class, clientVersion, addr, conf); 36 | } 37 | 38 | /* (non-Javadoc) 39 | * @see java.io.Closeable#close() 40 | */ 41 | @Override 42 | public void close() throws IOException { 43 | if(this.proxy != null) { 44 | RPC.stopProxy(this.proxy); 45 | } 46 | } 47 | 48 | /* (non-Javadoc) 49 | * @see com.varone.hadoop.rpc.metrics.MetricsService#getMetrics(com.varone.hadoop.rpc.metrics.MetricsServiceRequest) 50 | */ 51 | @Override 52 | public MetricsServiceResponse getMetrics(MetricsServiceRequest request) 53 | throws Exception { 54 | MetricsRequestProto requestProto = ((MetricsServiceRequestPBImpl)request).getProto(); 55 | try { 56 | return new MetricsServiceResponsePBImpl(proxy.getMetrics(null, requestProto)); 57 | } catch (ServiceException e) { 58 | RPCUtil.unwrapAndThrowException(e); 59 | return null; 60 | } 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/components/job/job-metric.js: -------------------------------------------------------------------------------- 1 | import React, { PropTypes } from 'react'; 2 | import c3 from 'c3'; 3 | import { byteFormatter, percentageFormatter, millisFormatter } from '../../utils/data-format'; 4 | 5 | export default class JobMetric extends React.Component { 6 | 7 | cluster_metrics_chart = null; 8 | 9 | static propTypes = { 10 | metric: PropTypes.object 11 | } 12 | 13 | componentDidMount() { 14 | this.defaultCharting(); 15 | } 16 | 17 | componentDidUpdate() { 18 | this.reloadCharting(); 19 | } 20 | 21 | render() { 22 | return ( 23 |
    24 |
    25 | { this.props.metric.title } 26 |
    27 |
    28 | 29 |
    30 |
    31 |
    32 |
    33 | ); 34 | } 35 | 36 | reloadCharting() { 37 | const columns = [ this.props.metric.x ]; 38 | 39 | for (let i = 0; i < this.props.metric.value.length; i++) { 40 | columns.push(this.props.metric.value[i]); 41 | } 42 | this.cluster_metrics_chart.load({ columns }); 43 | } 44 | 45 | defaultCharting() { 46 | let formatter; 47 | const columns = [ this.props.metric.x ]; 48 | 49 | for (let i = 0; i < this.props.metric.value.length; i++) { 50 | columns.push(this.props.metric.value[i]); 51 | } 52 | 53 | if (this.props.metric.format === 'BYTE') { 54 | formatter = byteFormatter; 55 | } else if (this.props.metric.format === 'PERCENTAGE') { 56 | formatter = percentageFormatter; 57 | } else if (this.props.metric.format === 'MILLIS') { 58 | formatter = millisFormatter; 59 | } 60 | 61 | this.cluster_metrics_chart = c3.generate({ 62 | bindto: '#' + this.props.metric.id, 63 | data: { 64 | x: 'x', 65 | columns 66 | }, 67 | axis: { 68 | x: { 69 | type: 'timeseries', 70 | tick: { 71 | format: '%H:%M:%S' 72 | } 73 | }, 74 | y: { 75 | padding: { bottom: 1 }, 76 | tick: { 77 | format: formatter 78 | } 79 | } 80 | } 81 | }); 82 | } 83 | } 84 | -------------------------------------------------------------------------------- /varOne-server/src/main/java/com/varone/web/vo/DefaultApplicationVO.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.web.vo; 5 | 6 | import java.util.List; 7 | import java.util.Map; 8 | 9 | import javax.xml.bind.annotation.XmlRootElement; 10 | 11 | /** 12 | * @author allen 13 | * 14 | */ 15 | @XmlRootElement 16 | public class DefaultApplicationVO { 17 | private int executorNum; 18 | private int taskNum; 19 | private int completedTaskNum; 20 | private int failedTaskNum; 21 | 22 | private List metricProps; 23 | 24 | private Map taskStartedNumByNode; 25 | private Map executorNumByNode; 26 | 27 | private Map>> propToMetrics; 28 | 29 | public int getExecutorNum() { 30 | return executorNum; 31 | } 32 | public void setExecutorNum(int executorNum) { 33 | this.executorNum = executorNum; 34 | } 35 | public int getTaskNum() { 36 | return taskNum; 37 | } 38 | public void setTaskNum(int taskNum) { 39 | this.taskNum = taskNum; 40 | } 41 | public int getCompletedTaskNum() { 42 | return completedTaskNum; 43 | } 44 | public void setCompletedTaskNum(int completedTaskNum) { 45 | this.completedTaskNum = completedTaskNum; 46 | } 47 | public int getFailedTaskNum() { 48 | return failedTaskNum; 49 | } 50 | public void setFailedTaskNum(int failedTaskNum) { 51 | this.failedTaskNum = failedTaskNum; 52 | } 53 | public Map getTaskStartedNumByNode() { 54 | return taskStartedNumByNode; 55 | } 56 | public void setTaskStartedNumByNode(Map taskStartedNumByNode) { 57 | this.taskStartedNumByNode = taskStartedNumByNode; 58 | } 59 | public Map getExecutorNumByNode() { 60 | return executorNumByNode; 61 | } 62 | public void setExecutorNumByNode(Map executorNumByNode) { 63 | this.executorNumByNode = executorNumByNode; 64 | } 65 | public List getMetricProps() { 66 | return metricProps; 67 | } 68 | public void setMetricProps(List metricProps) { 69 | this.metricProps = metricProps; 70 | } 71 | public Map>> getPropToMetrics() { 72 | return propToMetrics; 73 | } 74 | public void setPropToMetrics(Map>> propToMetrics) { 75 | this.propToMetrics = propToMetrics; 76 | } 77 | 78 | } 79 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/components/nodes/nodes-metric.js: -------------------------------------------------------------------------------- 1 | import React, { PropTypes } from 'react'; 2 | import c3 from 'c3'; 3 | import { byteFormatter, percentageFormatter, millisFormatter } from '../../utils/data-format'; 4 | 5 | export default class NodeMetric extends React.Component { 6 | 7 | node_metrics_chart = null; 8 | 9 | static propTypes = { 10 | metric: PropTypes.object 11 | } 12 | 13 | componentDidMount() { 14 | this.defaultCharting(); 15 | } 16 | 17 | componentDidUpdate() { 18 | this.reloadCharting(); 19 | } 20 | 21 | render() { 22 | return ( 23 |
    24 |
    25 | { this.props.metric.title } 26 |
    27 |
    28 | 29 |
    30 |
    31 |
    32 | 33 |
    34 | ); 35 | } 36 | 37 | reloadCharting() { 38 | const columns = [ this.props.metric.x, this.props.metric.value ]; 39 | 40 | const currNode = this.props.metric.value[0]; 41 | const datas = this.node_metrics_chart.data(); 42 | const unload = datas.filter(data => data.id !== currNode).map(data => data.id); 43 | 44 | this.node_metrics_chart.load({ columns, unload }); 45 | } 46 | 47 | defaultCharting() { 48 | const columns = [ this.props.metric.x, this.props.metric.value ]; 49 | 50 | let formatter; 51 | if (this.props.metric.format === 'BYTE') { 52 | formatter = byteFormatter; 53 | } else if (this.props.metric.format === 'PERCENTAGE') { 54 | formatter = percentageFormatter; 55 | } else if (this.props.metric.format === 'MILLIS') { 56 | formatter = millisFormatter; 57 | } 58 | 59 | this.node_metrics_chart = c3.generate({ 60 | bindto: '#' + this.props.metric.id, 61 | data: { 62 | x: 'x', 63 | columns 64 | }, 65 | axis: { 66 | x: { 67 | type: 'timeseries', 68 | tick: { 69 | format: '%H:%M:%S' 70 | } 71 | }, 72 | y: { 73 | padding: { bottom: 1 }, 74 | tick: { 75 | format: formatter 76 | } 77 | } 78 | } 79 | }); 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/webpack/production-config.js: -------------------------------------------------------------------------------- 1 | import path from 'path'; 2 | import webpack from 'webpack'; 3 | import ExtractTextPlugin from 'extract-text-webpack-plugin'; 4 | 5 | export default { 6 | entry: [ 7 | 'bootstrap-loader/extractStyles', 8 | path.resolve(__dirname, '../app/main.js') 9 | ], 10 | output: { 11 | path: path.resolve(__dirname, '../dist'), 12 | filename: 'app.js' 13 | }, 14 | module: { 15 | preLoaders: [ 16 | { test: /\.js$/, exclude: /node_modules/, loader: 'eslint' } 17 | ], 18 | loaders: [ 19 | { test: /\.js$/, loaders: [ 'babel' ], exclude: /node_modules/ }, 20 | { test: /\.scss$/, loaders: [ 'style', 'css', 'sass' ] }, 21 | { test: /\.(woff2?|ttf|eot|svg)$/, loader: 'url?limit=10000' }, 22 | { test: /bootstrap-sass\/assets\/javascripts\//, loader: 'imports?jQuery=jquery' }, 23 | { 24 | test: /\.(jpe?g|png|gif|svg|woff|woff2|eot|ttf)(\?v=[0-9].[0-9].[0-9])?$/, 25 | loader: 'file?name=[sha512:hash:base64:7].[ext]', 26 | exclude: /node_modules\/(?!font-awesome)/ 27 | }, 28 | { test: /\.css$/, loader: 'style-loader!css-loader' } 29 | ] 30 | }, 31 | plugins: [ 32 | new webpack.ProvidePlugin({ 33 | $: 'jquery', 34 | jQuery: 'jquery', 35 | 'window.jQuery': 'jquery', 36 | 'root.jQuery': 'jquery' 37 | }), 38 | new ExtractTextPlugin('app.css', { allChunks: true }), 39 | new webpack.optimize.DedupePlugin(), 40 | new webpack.optimize.OccurenceOrderPlugin(), 41 | new webpack.optimize.UglifyJsPlugin({ 42 | compress: { 43 | warnings: false, 44 | screw_ie8: true, 45 | sequences: true, 46 | dead_code: true, 47 | drop_debugger: true, 48 | comparisons: true, 49 | conditionals: true, 50 | evaluate: true, 51 | booleans: true, 52 | loops: true, 53 | unused: true, 54 | hoist_funs: true, 55 | if_return: true, 56 | join_vars: true, 57 | cascade: true, 58 | drop_console: true 59 | }, 60 | output: { 61 | comments: false 62 | } 63 | }) 64 | ], 65 | resolve: { 66 | extensions: [ '', '.js', '.jsx' ], 67 | // root: [path.join(__dirname, "public", "javascripts")], 68 | modulesDirectories: [ 'node_modules' ] 69 | } 70 | 71 | }; 72 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/components/history/stage-list.js: -------------------------------------------------------------------------------- 1 | import React, { PropTypes } from 'react'; 2 | import { BootstrapTable, TableHeaderColumn } from 'react-bootstrap-table'; 3 | import ProcessBar from '../commons/process-bar'; 4 | import { millisFormatter } from '../../utils/data-format'; 5 | 6 | class StageNameLink extends React.Component { 7 | 8 | static propTypes = { 9 | stage: PropTypes.object, 10 | onLinkSelect: PropTypes.func 11 | } 12 | 13 | handleStageSelect = () => { 14 | const { id } = this.props.stage; 15 | this.props.onLinkSelect(id); 16 | } 17 | 18 | render() { 19 | const { description } = this.props.stage; 20 | return { description }; 21 | } 22 | } 23 | 24 | class StageList extends React.Component { 25 | 26 | static propTypes = { 27 | stages: React.PropTypes.array, 28 | onStageSelect: React.PropTypes.func 29 | } 30 | 31 | taskFormatter = (cell, row) => { 32 | return { cell }; 33 | } 34 | 35 | stageNameFormatter = (cell, row) => { 36 | return ; 37 | } 38 | 39 | timeUnitFormatter = (cell) => millisFormatter(cell); 40 | 41 | render() { 42 | return ( 43 | 47 | Stage ID 48 | Description 51 | Submitted 54 | Duration 57 | Tasks: Succeeded/Total 60 | Input 61 | Ouput 62 | 63 | ); 64 | } 65 | } 66 | 67 | export default StageList; 68 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/test/spec/actions/cluster-action.test.js: -------------------------------------------------------------------------------- 1 | import fauxJax from 'faux-jax'; 2 | 3 | import alt from '../../../app/alt'; 4 | import ClusterAction from '../../../app/actions/cluster-action'; 5 | 6 | const should = chai.should(); 7 | 8 | const mockResponse = { 9 | nodeNum: 5, 10 | jobNum: 0, 11 | taskNum: 0, 12 | executorNum: 0, 13 | metricProps: [], 14 | taskStartedNumByNode: { 15 | 'server-a5': 0, 16 | 'server-a1': 0, 17 | 'server-a2': 0, 18 | 'server-a3': 0, 19 | 'server-a4': 0 20 | }, 21 | executorNumByNode: { 22 | 'server-a5': 0, 23 | 'server-a1': 0, 24 | 'server-a2': 0, 25 | 'server-a3': 0, 26 | 'server-a4': 0 27 | }, 28 | propToMetrics: {} 29 | }; 30 | 31 | describe('Cluster Action', () => { 32 | let dispatcherSpy; 33 | let fetchTotalNodeDashBoardSuccessfulSpy; 34 | 35 | beforeEach(() => { 36 | function respond(request) { 37 | request.respond( 38 | 200, 39 | { 'Content-Type': 'application/json' }, 40 | JSON.stringify(mockResponse) 41 | ); 42 | } 43 | fauxJax.install(); 44 | fauxJax.on('request', respond); 45 | dispatcherSpy = sinon.spy(alt.dispatcher, 'dispatch'); 46 | fetchTotalNodeDashBoardSuccessfulSpy = 47 | sinon.spy(ClusterAction, 'fetchTotalNodeDashBoardSuccessful'); 48 | }); 49 | 50 | afterEach(() => { 51 | fauxJax.restore(); 52 | alt.dispatcher.dispatch.restore(); 53 | fetchTotalNodeDashBoardSuccessfulSpy.restore(); 54 | }); 55 | 56 | it('dispatches correct data', async () => { 57 | const selectMetrics = []; 58 | const period = '30m'; 59 | 60 | await ClusterAction.fetchTotalNodeDashBoard(selectMetrics, period); 61 | const args = dispatcherSpy.args[0][0]; 62 | const response = args.data.result; 63 | should.exist(response); 64 | response.nodeNum.should.eql(mockResponse.nodeNum); 65 | Object.keys(response.taskStartedNumByNode).length 66 | .should.eql(Object.keys(mockResponse.taskStartedNumByNode).length); 67 | Object.keys(response.executorNumByNode).length 68 | .should.eql(Object.keys(mockResponse.executorNumByNode).length); 69 | }); 70 | 71 | it('fires fetchTotalNodeDashBoardSuccessful', async () => { 72 | const selectMetrics = []; 73 | const period = '30m'; 74 | 75 | await ClusterAction.fetchTotalNodeDashBoard(selectMetrics, period); 76 | fetchTotalNodeDashBoardSuccessfulSpy.calledOnce.should.be.ok; 77 | }); 78 | }); 79 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/karma.conf.js: -------------------------------------------------------------------------------- 1 | import webpack from 'webpack'; 2 | 3 | let coverage; 4 | let reporters; 5 | if (process.env.CIRCLECI) { 6 | coverage = { 7 | type: 'lcov', 8 | dir: process.env.CIRCLE_ARTIFACTS 9 | }; 10 | reporters = ['coverage', 'coveralls']; 11 | } else { 12 | coverage = { 13 | type: 'html', 14 | dir: 'report/coverage/' 15 | }; 16 | reporters = ['progress', 'html', 'coverage']; 17 | } 18 | 19 | export default function(config) { 20 | config.set({ 21 | browsers: ['Chrome'], 22 | browserNoActivityTimeout: 30000, 23 | frameworks: ['mocha', 'chai', 'sinon-chai'], 24 | files: ['tests.webpack.js'], 25 | preprocessors: { 26 | 'tests.webpack.js': ['webpack', 'coverage'] 27 | }, 28 | htmlReporter: { 29 | outputFile: 'report/spec/spec.html', 30 | pageTitle: 'Unit Tests', 31 | subPageTitle: 'varOne project testing spec' 32 | }, 33 | reporters: reporters, 34 | coverageReporter: coverage, 35 | webpack: { 36 | entry: [ 37 | 'bootstrap-loader', 38 | './app/main.js' 39 | ], 40 | debug: true, 41 | devtool: 'eval-source-map', 42 | module: { 43 | preLoaders: [ 44 | { test: /\.js$/, exclude: /node_modules/, loader: 'eslint' } 45 | ], 46 | loaders: [ 47 | { test: /\.js$/, loader: 'babel', exclude: /node_modules/ }, 48 | { test: /\.scss$/, loaders: [ 'style', 'css', 'sass' ] }, 49 | { test: /\.(woff2?|ttf|eot|svg)$/, loader: 'url?limit=10000' }, 50 | { test: /bootstrap-sass\/assets\/javascripts\//, loader: 'imports?jQuery=jquery' }, 51 | { 52 | test: /\.(jpe?g|png|gif|svg|woff|woff2|eot|ttf)(\?v=[0-9].[0-9].[0-9])?$/, 53 | loader: 'file?name=[sha512:hash:base64:7].[ext]', 54 | exclude: /node_modules\/(?!font-awesome)/ 55 | }, 56 | { test: /\.css$/, loader: 'style-loader!css-loader' } 57 | ] 58 | }, 59 | plugins: [ 60 | new webpack.ProvidePlugin({ 61 | $: 'jquery', 62 | jQuery: 'jquery', 63 | 'window.jQuery': 'jquery', 64 | 'root.jQuery': 'jquery' 65 | }), 66 | new webpack.DefinePlugin({ 67 | 'process.env': { 68 | BROWSER: JSON.stringify(true), 69 | NODE_ENV: JSON.stringify('test') 70 | } 71 | }) 72 | ] 73 | }, 74 | webpackServer: { noInfo: true } 75 | }); 76 | } 77 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/test/spec/components/cluster/cluster-summary.test.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom'; 3 | import TestUtils from 'react-addons-test-utils'; 4 | import stubApp from '../../../utils/stub-app'; 5 | import ClusterSummary from '../../../../app/components/cluster/cluster-summary'; 6 | import Const from '../../../../app/utils/consts'; 7 | 8 | chai.should(); 9 | 10 | const mockSummaryInfos = [ { 11 | displayName: Const.node.summary.NODE_QTY, 12 | value: 5, 13 | icon: 'fa-cloud', 14 | panel: 'panel-primary' 15 | }, { 16 | displayName: Const.node.summary.RUNNING_JOB, 17 | value: 0, 18 | icon: 'fa-tasks', 19 | panel: 'panel-green' 20 | }, { 21 | displayName: Const.node.summary.RUNNING_EXECUTOR, 22 | value: 0, 23 | icon: 'fa-list-ol', 24 | panel: 'panel-yellow' 25 | }, { 26 | displayName: Const.node.summary.RUNNING_TASK, 27 | value: 0, 28 | icon: 'fa-th', 29 | panel: 'panel-red' 30 | } ]; 31 | const mockTaskStartedNumByNode = [ { 32 | server1: 3 33 | }, { 34 | server2: 6 35 | } ]; 36 | const mockExecutorNumByNode = [ { 37 | server1: 1 38 | }, { 39 | server2: 1 40 | } ]; 41 | 42 | describe('Cluster Summary', () => { 43 | let node; 44 | let instance; 45 | let spy; 46 | 47 | beforeEach(() => { 48 | spy = sinon.spy(); 49 | const stub = stubApp()(ClusterSummary, { 50 | summaryInfos: mockSummaryInfos, 51 | taskStartedNumByNode: mockTaskStartedNumByNode, 52 | executorNumByNode: mockExecutorNumByNode, 53 | onNodeClick: spy 54 | }); 55 | node = window.document.createElement('div'); 56 | instance = ReactDOM.render(React.createElement(stub), node); 57 | }); 58 | 59 | afterEach(() => { 60 | if (instance) ReactDOM.unmountComponentAtNode(node); 61 | }); 62 | 63 | it('should render correctly', () => { 64 | const dom = ReactDOM.findDOMNode(instance); 65 | dom.tagName.should.eql('DIV'); 66 | dom.childNodes.length.should.eql(3); 67 | (dom.className.indexOf('summary-header') !== -1).should.be.ok; 68 | 69 | dom.childNodes[0].childNodes.length.should.eql(Math.ceil(mockSummaryInfos.length / 2)); 70 | }); 71 | 72 | it('only nodes summary block should click', () => { 73 | const dom = ReactDOM.findDOMNode(instance); 74 | const nodeSummaryLink = dom.childNodes[0].childNodes[0]. 75 | childNodes[0].childNodes[0].childNodes[1]; 76 | TestUtils.Simulate.click(nodeSummaryLink); 77 | spy.should.have.been.calledOnce; 78 | }); 79 | }); 80 | -------------------------------------------------------------------------------- /varOne-node/src/main/java/com/varone/node/utils/MetricsPropertiesParser.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.node.utils; 5 | 6 | import java.io.FileInputStream; 7 | import java.io.IOException; 8 | import java.io.InputStream; 9 | import java.util.ArrayList; 10 | import java.util.List; 11 | import java.util.Properties; 12 | 13 | import com.varone.node.MetricsProperties; 14 | 15 | /** 16 | * @author allen 17 | * 18 | */ 19 | public class MetricsPropertiesParser { 20 | 21 | public final static String CSV_SINK = "*.sink.csv.class"; 22 | public final static String CSV_PERIOD = "*.sink.csv.period"; 23 | public final static String CSV_UNIT = "*.sink.csv.unit"; 24 | public final static String CSV_DIR = "*.sink.csv.directory"; 25 | 26 | public static MetricsProperties loadProperties(String filePath) { 27 | MetricsProperties metricsProps = null; 28 | Properties prop = new Properties(); 29 | InputStream input = null; 30 | 31 | try{ 32 | input = new FileInputStream(filePath); 33 | 34 | prop.load(input); 35 | 36 | metricsProps = new MetricsProperties(); 37 | 38 | String csvSink; 39 | if(null == (csvSink = prop.getProperty(CSV_SINK)) || 40 | !csvSink.equals("org.apache.spark.metrics.sink.CsvSink")){ 41 | throw new RuntimeException("Should allow output metrics to csv by setting *.sink.csv.class=org.apache.spark.metrics.sink.CsvSink"); 42 | } 43 | 44 | String csvPeriod; 45 | if(null == (csvPeriod = prop.getProperty(CSV_PERIOD))){ 46 | throw new RuntimeException("Can not find '"+CSV_PERIOD+"' in metrics properties file: " + filePath); 47 | } else { 48 | metricsProps.setCsvPeriod(csvPeriod); 49 | } 50 | 51 | String csvUnit; 52 | if(null == (csvUnit = prop.getProperty(CSV_UNIT))){ 53 | throw new RuntimeException("Can not find '"+CSV_UNIT+"' in metrics properties file: " + filePath); 54 | } else { 55 | metricsProps.setCsvUnit(csvUnit); 56 | } 57 | 58 | String csvDir; 59 | if(null == (csvDir = prop.getProperty(CSV_DIR))){ 60 | throw new RuntimeException("Can not find '"+CSV_DIR+"' in metrics properties file: " + filePath); 61 | } else { 62 | metricsProps.setCsvDir(csvDir); 63 | } 64 | 65 | 66 | } catch(IOException e){ 67 | throw new RuntimeException(e); 68 | } finally { 69 | if (input != null) { 70 | try { 71 | input.close(); 72 | } catch(IOException e){ 73 | 74 | } 75 | } 76 | } 77 | 78 | return metricsProps; 79 | 80 | } 81 | } 82 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/actions/history-action.js: -------------------------------------------------------------------------------- 1 | import alt from '../alt'; 2 | import VarOneAction from '../actions/varOne-action'; 3 | import MenuAction from '../actions/menu-action'; 4 | import request from 'superagent'; 5 | 6 | class HistoryAction { 7 | 8 | constructor() { 9 | this.generateActions('switchToJobTab'); 10 | this.generateActions('switchToHistoryTab'); 11 | this.generateActions('switchToStageTab'); 12 | } 13 | 14 | async fetchApplications() { 15 | try { 16 | const response = await request.get('/varOne-web/rest/history') 17 | .set('Accept', 'application/json'); 18 | const result = JSON.parse(response.text); 19 | this.dispatch(result); 20 | } catch (e) { 21 | VarOneAction.showFailMessage(e.response.text); 22 | MenuAction.changErrorMessageIcon(true); 23 | } 24 | } 25 | 26 | async fetchJobs(applicationId) { 27 | try { 28 | const response = await request.get(`/varOne-web/rest/history/${applicationId}/jobs`) 29 | .set('Accept', 'application/json'); 30 | const jobs = JSON.parse(response.text); 31 | const result = { jobs, applicationId }; 32 | this.dispatch(result); 33 | } catch (e) { 34 | VarOneAction.showFailMessage(e.response.text); 35 | MenuAction.changErrorMessageIcon(true); 36 | } 37 | } 38 | 39 | async fetchStages(applicationId, jobId) { 40 | try { 41 | const response = await request.get( 42 | `/varOne-web/rest/history/${applicationId}/${jobId}/stages`) 43 | .set('Accept', 'application/json'); 44 | const stages = JSON.parse(response.text); 45 | const result = { stages, jobId }; 46 | this.dispatch(result); 47 | } catch (e) { 48 | VarOneAction.showFailMessage(e.response.text); 49 | MenuAction.changErrorMessageIcon(true); 50 | } 51 | } 52 | 53 | async fetchStageDetails(applicationId, stageId) { 54 | try { 55 | const response = await request.get(` 56 | /varOne-web/rest/history/${applicationId}/jobs/${stageId}`) 57 | .set('Accept', 'application/json'); 58 | const stageDetails = JSON.parse(response.text); 59 | const result = { stageDetails }; 60 | this.dispatch(result); 61 | } catch (e) { 62 | VarOneAction.showFailMessage(e.response.text); 63 | MenuAction.changErrorMessageIcon(true); 64 | } 65 | } 66 | } 67 | 68 | export default alt.createActions(HistoryAction); 69 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/stores/history-store.js: -------------------------------------------------------------------------------- 1 | import alt from '../alt'; 2 | import Const from '../utils/consts'; 3 | import HistoryAction from '../actions/history-action'; 4 | 5 | class HistoryStore { 6 | 7 | constructor() { 8 | this.bindActions(HistoryAction); 9 | this.selectApplicationId; 10 | this.jobId; 11 | this.histories = []; 12 | this.jobs = []; 13 | this.stages = []; 14 | this.stageDetails = []; 15 | this.tab = Const.history.tab.HISTORY_TAB; 16 | this.breadcrumb = [ Const.history.tab.HISTORY_TAB ]; 17 | } 18 | 19 | onFetchApplications(result) { 20 | this.tab = Const.history.tab.HISTORY_TAB; 21 | this.breadcrumb = [ Const.history.tab.HISTORY_TAB ]; 22 | this.histories = result; 23 | } 24 | 25 | onFetchJobs(result) { 26 | this.tab = Const.history.tab.JOB_TAB; 27 | this.breadcrumb = [ Const.history.tab.HISTORY_TAB, Const.history.tab.JOB_TAB ]; 28 | this.jobs = result.jobs; 29 | this.selectApplicationId = result.applicationId; 30 | } 31 | 32 | onFetchStages(result) { 33 | this.tab = Const.history.tab.STAGE_TAB; 34 | this.breadcrumb = [ 35 | Const.history.tab.HISTORY_TAB, 36 | Const.history.tab.JOB_TAB, 37 | Const.history.tab.STAGE_TAB ]; 38 | this.stages = result.stages; 39 | this.jobId = result.jobId; 40 | } 41 | 42 | onFetchStageDetails(result) { 43 | this.tab = Const.history.tab.STAGE_DETAILS_TAB; 44 | this.breadcrumb = [ 45 | Const.history.tab.HISTORY_TAB, 46 | Const.history.tab.JOB_TAB, 47 | Const.history.tab.STAGE_TAB, 48 | Const.history.tab.STAGE_DETAILS_TAB ]; 49 | this.stageDetails = result.stageDetails.tasks; 50 | this.stageAggregator = result.stageDetails.aggregatorExecutor; 51 | this.completeTaskSize = result.stageDetails.completeTaskSize; 52 | this.metricCompletedTasks = result.stageDetails.metricCompletedTasks; 53 | } 54 | 55 | onSwitchToJobTab() { 56 | this.tab = Const.history.tab.JOB_TAB; 57 | this.breadcrumb = [ Const.history.tab.HISTORY_TAB, Const.history.tab.JOB_TAB ]; 58 | } 59 | 60 | onSwitchToStageTab() { 61 | this.tab = Const.history.tab.STAGE_TAB; 62 | this.breadcrumb = [ 63 | Const.history.tab.HISTORY_TAB, 64 | Const.history.tab.JOB_TAB, 65 | Const.history.tab.STAGE_TAB 66 | ]; 67 | } 68 | 69 | onSwitchToHistoryTab() { 70 | this.tab = Const.history.tab.HISTORY_TAB; 71 | this.breadcrumb = [ Const.history.tab.HISTORY_TAB ]; 72 | } 73 | } 74 | 75 | export default alt.createStore(HistoryStore); 76 | -------------------------------------------------------------------------------- /varOne-node/src/main/java/com/varone/node/utils/MetricsDataTransfer.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.node.utils; 5 | 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | import java.util.Map; 9 | import java.util.Map.Entry; 10 | 11 | import com.varone.hadoop.rpc.protos.MetricsProtos.MetricsTypeProto; 12 | import com.varone.hadoop.rpc.protos.MetricsProtos.MetricsResponseProto.MetricsMapProto; 13 | import com.varone.hadoop.rpc.protos.MetricsProtos.MetricsResponseProto.MetricsMapProto.TupleProto; 14 | import com.varone.node.MetricTuple; 15 | import com.varone.node.MetricsType; 16 | 17 | /** 18 | * @author allen 19 | * 20 | */ 21 | public class MetricsDataTransfer { 22 | 23 | MetricsMapProto.Builder metricsMapBuilder = null; 24 | TupleProto.Builder tupleBuilder = null; 25 | 26 | /** 27 | * 28 | */ 29 | public MetricsDataTransfer() { 30 | this.metricsMapBuilder = MetricsMapProto.newBuilder(); 31 | this.tupleBuilder = TupleProto.newBuilder(); 32 | } 33 | 34 | public List encodeMetricsType(List metricsType) throws Exception{ 35 | List protos = new ArrayList(metricsType.size()); 36 | 37 | for(String type: metricsType){ 38 | MetricsType fromString = MetricsType.fromString(type); 39 | MetricsTypeProto result = MetricsTypeProto.valueOf(fromString.name()); 40 | protos.add(result); 41 | } 42 | 43 | return protos; 44 | } 45 | 46 | 47 | public List decodeMetricsType(List metricsTypeProtos) throws Exception{ 48 | List result = new ArrayList(metricsTypeProtos.size()); 49 | 50 | for(MetricsTypeProto proto: metricsTypeProtos){ 51 | result.add(MetricsType.fromString(proto.name())); 52 | } 53 | 54 | return result; 55 | } 56 | 57 | public List encodeMetricsData(Map> metricsData){ 58 | List mapProtos = new ArrayList(metricsData.size()); 59 | 60 | for(Entry> entry :metricsData.entrySet()){ 61 | this.metricsMapBuilder.clear(); 62 | this.metricsMapBuilder.setMetricsName(entry.getKey()); 63 | for(MetricTuple tuple: entry.getValue()){ 64 | this.tupleBuilder.clear(); 65 | this.tupleBuilder.setTime(tuple.getTime()); 66 | this.tupleBuilder.setValue(tuple.getValue()); 67 | this.metricsMapBuilder.addMetricsValues(this.tupleBuilder.build()); 68 | } 69 | mapProtos.add(this.metricsMapBuilder.build()); 70 | } 71 | 72 | return mapProtos; 73 | } 74 | 75 | } 76 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/test/spec/components/cluster/cluster-header.test.js: -------------------------------------------------------------------------------- 1 | import React from 'react'; 2 | import ReactDOM from 'react-dom'; 3 | import Const from '../../../../app/utils/consts'; 4 | import TestUtils from 'react-addons-test-utils'; 5 | import stubApp from '../../../utils/stub-app'; 6 | import ClusterHeader from '../../../../app/components/cluster/cluster-header'; 7 | 8 | const should = chai.should(); 9 | 10 | const mockPeriod = Const.shared.timeperiod[2]; 11 | 12 | describe('Cluster Header', () => { 13 | let node; 14 | let instance; 15 | let spy; 16 | 17 | beforeEach(() => { 18 | spy = sinon.spy(); 19 | const stub = stubApp()(ClusterHeader, { 20 | period: mockPeriod, 21 | onPeriodSelect: spy 22 | }); 23 | node = window.document.createElement('div'); 24 | instance = ReactDOM.render(React.createElement(stub), node); 25 | }); 26 | 27 | afterEach(() => { 28 | if (instance) ReactDOM.unmountComponentAtNode(node); 29 | }); 30 | 31 | it('should render correctly', () => { 32 | const dom = ReactDOM.findDOMNode(instance); 33 | 34 | dom.tagName.should.eql('DIV'); 35 | (dom.className.indexOf('row') !== -1).should.be.ok; 36 | dom.childNodes.length.should.eql(2); 37 | 38 | const barChartIcon = TestUtils.findRenderedDOMComponentWithClass(instance, 'fa-bar-chart'); 39 | const heading = dom.childNodes[0].childNodes[0]; 40 | const periodPills = dom.childNodes[1].childNodes[0]; 41 | 42 | should.exist(barChartIcon); 43 | (heading.className.indexOf('page-header') !== -1).should.be.ok; 44 | (heading.textContent.indexOf('Dashboard') !== -1).should.be.ok; 45 | 46 | (periodPills.className.indexOf('page-header') !== -1).should.be.ok; 47 | periodPills.childNodes[0].tagName.should.eql('UL'); 48 | periodPills.childNodes[0].childNodes.length.should.eql(Const.shared.timeperiod.length); 49 | }); 50 | 51 | it('should fire handlePillClick if select a unactived period', () => { 52 | const dom = ReactDOM.findDOMNode(instance); 53 | const periodPills = dom.childNodes[1].childNodes[0]; 54 | const hour1 = periodPills.childNodes[0].childNodes[1]; 55 | TestUtils.Simulate.click(hour1); 56 | spy.should.have.been.calledOnce; 57 | }); 58 | 59 | it('should not fire handlePillClick if select an actived period', () => { 60 | const dom = ReactDOM.findDOMNode(instance); 61 | const periodPills = dom.childNodes[1].childNodes[0]; 62 | const hour2 = periodPills.childNodes[0].childNodes[2]; 63 | TestUtils.Simulate.click(hour2); 64 | spy.should.not.have.been.calledOnce; 65 | }); 66 | }); 67 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/webpack/dev-config.js: -------------------------------------------------------------------------------- 1 | import path from 'path'; 2 | import webpack from 'webpack'; 3 | 4 | const WEBPACK_DEV_SERVER_PORT = parseInt(process.env.PORT, 10) + 1 || 3001; 5 | const PUBLIC_PATH = `http://localhost:${WEBPACK_DEV_SERVER_PORT}/assets/`; 6 | 7 | export default { 8 | server: { 9 | port: WEBPACK_DEV_SERVER_PORT, 10 | options: { 11 | publicPath: PUBLIC_PATH, 12 | hot: true, 13 | stats: { 14 | assets: true, 15 | colors: true, 16 | version: false, 17 | hash: false, 18 | timings: true, 19 | chunks: false, 20 | chunkModules: false 21 | } 22 | } 23 | }, 24 | webpack: { 25 | entry: [ 26 | `webpack-dev-server/client?http://localhost:${WEBPACK_DEV_SERVER_PORT}`, 27 | 'webpack/hot/only-dev-server', 28 | 'bootstrap-loader', 29 | './app/main.js' 30 | ], 31 | 32 | publicPath: PUBLIC_PATH, 33 | 34 | output: { 35 | path: path.join(__dirname, '../build'), 36 | filename: 'app.js', 37 | publicPath: PUBLIC_PATH 38 | }, 39 | 40 | module: { 41 | preLoaders: [ 42 | { test: /\.js$/, exclude: /node_modules/, loader: 'eslint' } 43 | ], 44 | loaders: [ 45 | { test: /\.js$/, loaders: [ 'react-hot', 'babel' ], exclude: /node_modules/ }, 46 | { test: /\.scss$/, loaders: [ 'style', 'css', 'sass' ] }, 47 | { test: /\.(woff2?|ttf|eot|svg)$/, loader: 'url?limit=10000' }, 48 | { test: /bootstrap-sass\/assets\/javascripts\//, loader: 'imports?jQuery=jquery' }, 49 | { 50 | test: /\.(jpe?g|png|gif|svg|woff|woff2|eot|ttf)(\?v=[0-9].[0-9].[0-9])?$/, 51 | loader: 'file?name=[sha512:hash:base64:7].[ext]', 52 | exclude: /node_modules\/(?!font-awesome)/ 53 | }, 54 | { test: /\.css$/, loader: 'style-loader!css-loader' } 55 | ] 56 | }, 57 | 58 | plugins: [ 59 | new webpack.HotModuleReplacementPlugin(), 60 | new webpack.NoErrorsPlugin(), 61 | new webpack.ProvidePlugin({ 62 | $: 'jquery', 63 | jQuery: 'jquery', 64 | 'window.jQuery': 'jquery', 65 | 'root.jQuery': 'jquery' 66 | }), 67 | new webpack.DefinePlugin({ 68 | 'process.env': { 69 | BROWSER: JSON.stringify(true), 70 | NODE_ENV: JSON.stringify('development') 71 | } 72 | }) 73 | ], 74 | 75 | resolve: { 76 | extensions: [ '', '.js', '.jsx' ], 77 | // root: [path.join(__dirname, "public", "javascripts")], 78 | modulesDirectories: [ 'node_modules' ] 79 | } 80 | } 81 | }; 82 | -------------------------------------------------------------------------------- /varOne-rpc/src/main/java/com/varone/hadoop/rpc/metrics/impl/MetricsServiceRequestPBImpl.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone.hadoop.rpc.metrics.impl; 5 | 6 | import java.util.List; 7 | 8 | import com.varone.hadoop.rpc.metrics.MetricsServiceRequest; 9 | import com.varone.hadoop.rpc.protos.MetricsProtos.MetricsRequestProto; 10 | import com.varone.hadoop.rpc.protos.MetricsProtos.MetricsRequestProtoOrBuilder; 11 | import com.varone.hadoop.rpc.protos.MetricsProtos.MetricsTypeProto; 12 | 13 | /** 14 | * @author allen 15 | * 16 | */ 17 | public class MetricsServiceRequestPBImpl implements MetricsServiceRequest { 18 | protected MetricsRequestProto proto = MetricsRequestProto.getDefaultInstance(); 19 | protected MetricsRequestProto.Builder builder = null; 20 | protected boolean viaProto = false; 21 | 22 | protected String applicationId; 23 | protected List metrics; 24 | 25 | /** 26 | * 27 | */ 28 | public MetricsServiceRequestPBImpl() { 29 | this.builder = MetricsRequestProto.newBuilder(); 30 | } 31 | 32 | public MetricsServiceRequestPBImpl(MetricsRequestProto proto) { 33 | this.proto = proto; 34 | this.viaProto = true; 35 | } 36 | 37 | /* (non-Javadoc) 38 | * @see com.varone.hadoop.rpc.metrics.MetricsServiceRequest#getApplicationId() 39 | */ 40 | @Override 41 | public String getApplicationId() { 42 | MetricsRequestProtoOrBuilder p = viaProto? this.proto:this.builder; 43 | this.applicationId = p.getApplicationId(); 44 | return this.applicationId; 45 | } 46 | 47 | /* (non-Javadoc) 48 | * @see com.varone.hadoop.rpc.metrics.MetricsServiceRequest#setApplicationId(java.lang.String) 49 | */ 50 | @Override 51 | public void setApplicationId(String applicationId) { 52 | this.builder.clearApplicationId(); 53 | this.builder.setApplicationId(applicationId); 54 | this.applicationId = applicationId; 55 | } 56 | 57 | /* (non-Javadoc) 58 | * @see com.varone.hadoop.rpc.metrics.MetricsServiceRequest#getMetrics() 59 | */ 60 | @Override 61 | public List getMetrics() { 62 | MetricsRequestProtoOrBuilder p = viaProto? this.proto:this.builder; 63 | this.metrics = p.getMetricsList(); 64 | return this.metrics; 65 | } 66 | 67 | /* (non-Javadoc) 68 | * @see com.varone.hadoop.rpc.metrics.MetricsServiceRequest#setMetrics(java.util.List) 69 | */ 70 | @Override 71 | public void setMetrics(List metrics) { 72 | this.builder.clearMetrics(); 73 | this.builder.addAllMetrics(metrics); 74 | this.metrics = metrics; 75 | } 76 | 77 | public MetricsRequestProto getProto() { 78 | return this.builder.build(); 79 | } 80 | 81 | } 82 | -------------------------------------------------------------------------------- /varOne-web/src/main/webapp/app/components/history/job-list.js: -------------------------------------------------------------------------------- 1 | import React, { PropTypes } from 'react'; 2 | import { BootstrapTable, TableHeaderColumn } from 'react-bootstrap-table'; 3 | import ProcessBar from '../commons/process-bar'; 4 | import { dateFormatter, millisFormatter } from '../../utils/data-format'; 5 | 6 | class JobNameLink extends React.Component { 7 | 8 | static propTypes = { 9 | job: PropTypes.object, 10 | onLinkSelect: PropTypes.func 11 | } 12 | 13 | handleJobSelect = () => { 14 | const { id } = this.props.job; 15 | this.props.onLinkSelect(id); 16 | } 17 | 18 | render() { 19 | const { description } = this.props.job; 20 | return ({ description }); 21 | } 22 | } 23 | 24 | class JobList extends React.Component { 25 | 26 | static propTypes = { 27 | jobs: PropTypes.array, 28 | onJobSelect: PropTypes.func 29 | } 30 | 31 | stageFormatter = (cell, row) => { 32 | return ({ cell }); 33 | } 34 | 35 | taskFormatter = (cell, row) => { 36 | return ({ cell }); 37 | } 38 | 39 | jobNameFormatter = (cell, row) => { 40 | return ; 41 | } 42 | 43 | dateUnitFormatter = cell => dateFormatter(cell); 44 | 45 | timeUnitFormatter = cell => millisFormatter(cell); 46 | 47 | render() { 48 | return ( 49 | 53 | Job ID 54 | Description 57 | Submitted 60 | Duration 63 | Stages: Succeeded/Total 66 | 69 | Tasks (for all stages): Succeeded/Total 70 | 71 | 72 | ); 73 | } 74 | } 75 | 76 | export default JobList; 77 | -------------------------------------------------------------------------------- /varOne-rpc/src/test/java/com/varone/MetricsMockService.java: -------------------------------------------------------------------------------- 1 | /** 2 | * 3 | */ 4 | package com.varone; 5 | 6 | import java.util.ArrayList; 7 | import java.util.List; 8 | 9 | import org.apache.hadoop.conf.Configuration; 10 | 11 | import com.varone.hadoop.rpc.AbstractServer; 12 | import com.varone.hadoop.rpc.metrics.MetricsService; 13 | import com.varone.hadoop.rpc.metrics.MetricsServiceRequest; 14 | import com.varone.hadoop.rpc.metrics.MetricsServiceResponse; 15 | import com.varone.hadoop.rpc.metrics.impl.MetricsServiceResponsePBImpl; 16 | import com.varone.hadoop.rpc.protos.MetricsProtos.MetricsTypeProto; 17 | import com.varone.hadoop.rpc.protos.MetricsProtos.MetricsResponseProto.MetricsMapProto; 18 | import com.varone.hadoop.rpc.protos.MetricsProtos.MetricsResponseProto.MetricsMapProto.TupleProto; 19 | 20 | /** 21 | * @author allen 22 | * 23 | */ 24 | public class MetricsMockService extends AbstractServer implements MetricsService { 25 | 26 | /** 27 | * @throws Exception 28 | * 29 | */ 30 | public MetricsMockService(Configuration config) throws Exception { 31 | this.constructServer(config); 32 | } 33 | 34 | /* (non-Javadoc) 35 | * @see com.varone.hadoop.rpc.metrics.MetricsService#getMetrics(com.varone.hadoop.rpc.metrics.MetricsServiceRequest) 36 | */ 37 | @Override 38 | public MetricsServiceResponse getMetrics(MetricsServiceRequest request) 39 | throws Exception { 40 | MetricsServiceResponse response = new MetricsServiceResponsePBImpl(); 41 | 42 | List result = new ArrayList(); 43 | MetricsMapProto.Builder metricsMapBuilder = MetricsMapProto.newBuilder(); 44 | TupleProto.Builder tupleBuilder = TupleProto.newBuilder(); 45 | 46 | metricsMapBuilder.setMetricsName(MetricsTypeProto.FS.name()); 47 | 48 | tupleBuilder.setTime(System.currentTimeMillis()); 49 | tupleBuilder.setValue("121"); 50 | TupleProto tuple1 = tupleBuilder.build(); 51 | metricsMapBuilder.addMetricsValues(tuple1); 52 | 53 | tupleBuilder.clear(); 54 | 55 | tupleBuilder.setTime(System.currentTimeMillis()); 56 | tupleBuilder.setValue("78"); 57 | TupleProto tuple2 = tupleBuilder.build(); 58 | metricsMapBuilder.addMetricsValues(tuple2); 59 | 60 | result.add(metricsMapBuilder.build()); 61 | 62 | response.setResult(result); 63 | return response; 64 | } 65 | 66 | /* (non-Javadoc) 67 | * @see com.varone.hadoop.rpc.AbstractServer#constructServer(org.apache.hadoop.conf.Configuration) 68 | */ 69 | @Override 70 | public void constructServer(Configuration config) throws Exception { 71 | super.constructHadoopServer(MetricsService.class, config, this, 5, 8088); 72 | } 73 | 74 | } 75 | -------------------------------------------------------------------------------- /varOne-node/src/test/resources/csvsink/application_1439169262151_0237.1.jvm.PS-MarkSweep.count.csv: -------------------------------------------------------------------------------- 1 | t,value 2 | 1442285759,0 3 | 1442285760,0 4 | 1442285761,0 5 | 1442285762,0 6 | 1442285763,0 7 | 1442285764,0 8 | 1442285765,0 9 | 1442285766,0 10 | 1442285767,0 11 | 1442285768,0 12 | 1442285769,0 13 | 1442285770,0 14 | 1442285771,0 15 | 1442285772,0 16 | 1442285773,0 17 | 1442285774,0 18 | 1442285775,0 19 | 1442285776,0 20 | 1442285777,0 21 | 1442285778,0 22 | 1442285779,0 23 | 1442285780,0 24 | 1442285781,0 25 | 1442285782,0 26 | 1442285783,0 27 | 1442285784,0 28 | 1442285785,0 29 | 1442285786,0 30 | 1442285787,0 31 | 1442285788,0 32 | 1442285789,0 33 | 1442285790,0 34 | 1442285791,0 35 | 1442285792,0 36 | 1442285793,0 37 | 1442285794,0 38 | 1442285795,0 39 | 1442285796,0 40 | 1442285797,0 41 | 1442285798,0 42 | 1442285799,0 43 | 1442285800,0 44 | 1442285801,0 45 | 1442285802,0 46 | 1442285803,0 47 | 1442285804,0 48 | 1442285805,0 49 | 1442285806,0 50 | 1442285807,0 51 | 1442285808,0 52 | 1442285809,0 53 | 1442285810,0 54 | 1442285811,0 55 | 1442285812,0 56 | 1442285813,0 57 | 1442285814,0 58 | 1442285815,0 59 | 1442285816,0 60 | 1442285817,0 61 | 1442285818,0 62 | 1442285819,0 63 | 1442285820,0 64 | 1442285821,0 65 | 1442285822,0 66 | 1442285823,0 67 | 1442285824,0 68 | 1442285825,0 69 | 1442285826,0 70 | 1442285827,0 71 | 1442285828,0 72 | 1442285829,0 73 | 1442285830,0 74 | 1442285831,0 75 | 1442285832,0 76 | 1442285833,0 77 | 1442285834,0 78 | 1442285835,0 79 | 1442285836,0 80 | 1442285837,0 81 | 1442285838,0 82 | 1442285839,0 83 | 1442285840,0 84 | 1442285841,0 85 | 1442285842,0 86 | 1442285843,0 87 | 1442285844,0 88 | 1442285845,0 89 | 1442285846,0 90 | 1442285847,0 91 | 1442285848,0 92 | 1442285849,0 93 | 1442285850,0 94 | 1442285851,0 95 | 1442285852,0 96 | 1442285853,0 97 | 1442285854,0 98 | 1442285855,0 99 | 1442285856,0 100 | 1442285857,0 101 | 1442285858,0 102 | 1442285859,0 103 | 1442285860,0 104 | 1442285861,0 105 | 1442285862,0 106 | 1442285863,0 107 | 1442285864,0 108 | 1442285865,0 109 | 1442285866,0 110 | 1442285867,0 111 | 1442285868,0 112 | 1442285869,0 113 | 1442285870,0 114 | 1442285871,0 115 | 1442285872,0 116 | 1442285873,0 117 | 1442285874,0 118 | 1442285875,0 119 | 1442285876,0 120 | 1442285877,0 121 | 1442285878,0 122 | 1442285879,0 123 | 1442285880,0 124 | 1442285881,0 125 | 1442285882,0 126 | 1442285883,0 127 | 1442285884,0 128 | 1442285885,0 129 | 1442285886,0 130 | 1442285887,0 131 | 1442285888,0 132 | 1442285889,0 133 | 1442285890,0 134 | 1442285891,0 135 | 1442285892,0 136 | 1442285893,0 137 | 1442285894,0 138 | 1442285895,0 139 | 1442285896,0 140 | -------------------------------------------------------------------------------- /varOne-node/src/test/resources/csvsink/application_1439169262151_0237.1.jvm.PS-MarkSweep.time.csv: -------------------------------------------------------------------------------- 1 | t,value 2 | 1442285759,0 3 | 1442285760,0 4 | 1442285761,0 5 | 1442285762,0 6 | 1442285763,0 7 | 1442285764,0 8 | 1442285765,0 9 | 1442285766,0 10 | 1442285767,0 11 | 1442285768,0 12 | 1442285769,0 13 | 1442285770,0 14 | 1442285771,0 15 | 1442285772,0 16 | 1442285773,0 17 | 1442285774,0 18 | 1442285775,0 19 | 1442285776,0 20 | 1442285777,0 21 | 1442285778,0 22 | 1442285779,0 23 | 1442285780,0 24 | 1442285781,0 25 | 1442285782,0 26 | 1442285783,0 27 | 1442285784,0 28 | 1442285785,0 29 | 1442285786,0 30 | 1442285787,0 31 | 1442285788,0 32 | 1442285789,0 33 | 1442285790,0 34 | 1442285791,0 35 | 1442285792,0 36 | 1442285793,0 37 | 1442285794,0 38 | 1442285795,0 39 | 1442285796,0 40 | 1442285797,0 41 | 1442285798,0 42 | 1442285799,0 43 | 1442285800,0 44 | 1442285801,0 45 | 1442285802,0 46 | 1442285803,0 47 | 1442285804,0 48 | 1442285805,0 49 | 1442285806,0 50 | 1442285807,0 51 | 1442285808,0 52 | 1442285809,0 53 | 1442285810,0 54 | 1442285811,0 55 | 1442285812,0 56 | 1442285813,0 57 | 1442285814,0 58 | 1442285815,0 59 | 1442285816,0 60 | 1442285817,0 61 | 1442285818,0 62 | 1442285819,0 63 | 1442285820,0 64 | 1442285821,0 65 | 1442285822,0 66 | 1442285823,0 67 | 1442285824,0 68 | 1442285825,0 69 | 1442285826,0 70 | 1442285827,0 71 | 1442285828,0 72 | 1442285829,0 73 | 1442285830,0 74 | 1442285831,0 75 | 1442285832,0 76 | 1442285833,0 77 | 1442285834,0 78 | 1442285835,0 79 | 1442285836,0 80 | 1442285837,0 81 | 1442285838,0 82 | 1442285839,0 83 | 1442285840,0 84 | 1442285841,0 85 | 1442285842,0 86 | 1442285843,0 87 | 1442285844,0 88 | 1442285845,0 89 | 1442285846,0 90 | 1442285847,0 91 | 1442285848,0 92 | 1442285849,0 93 | 1442285850,0 94 | 1442285851,0 95 | 1442285852,0 96 | 1442285853,0 97 | 1442285854,0 98 | 1442285855,0 99 | 1442285856,0 100 | 1442285857,0 101 | 1442285858,0 102 | 1442285859,0 103 | 1442285860,0 104 | 1442285861,0 105 | 1442285862,0 106 | 1442285863,0 107 | 1442285864,0 108 | 1442285865,0 109 | 1442285866,0 110 | 1442285867,0 111 | 1442285868,0 112 | 1442285869,0 113 | 1442285870,0 114 | 1442285871,0 115 | 1442285872,0 116 | 1442285873,0 117 | 1442285874,0 118 | 1442285875,0 119 | 1442285876,0 120 | 1442285877,0 121 | 1442285878,0 122 | 1442285879,0 123 | 1442285880,0 124 | 1442285881,0 125 | 1442285882,0 126 | 1442285883,0 127 | 1442285884,0 128 | 1442285885,0 129 | 1442285886,0 130 | 1442285887,0 131 | 1442285888,0 132 | 1442285889,0 133 | 1442285890,0 134 | 1442285891,0 135 | 1442285892,0 136 | 1442285893,0 137 | 1442285894,0 138 | 1442285895,0 139 | 1442285896,0 140 | -------------------------------------------------------------------------------- /varOne-node/src/test/resources/csvsink/application_1439169262151_0237.5.jvm.PS-MarkSweep.count.csv: -------------------------------------------------------------------------------- 1 | t,value 2 | 1442285759,0 3 | 1442285760,0 4 | 1442285761,0 5 | 1442285762,0 6 | 1442285763,0 7 | 1442285764,0 8 | 1442285765,0 9 | 1442285766,0 10 | 1442285767,0 11 | 1442285768,0 12 | 1442285769,0 13 | 1442285770,0 14 | 1442285771,0 15 | 1442285772,0 16 | 1442285773,0 17 | 1442285774,0 18 | 1442285775,0 19 | 1442285776,0 20 | 1442285777,0 21 | 1442285778,0 22 | 1442285779,0 23 | 1442285780,0 24 | 1442285781,0 25 | 1442285782,0 26 | 1442285783,0 27 | 1442285784,0 28 | 1442285785,0 29 | 1442285786,0 30 | 1442285787,0 31 | 1442285788,0 32 | 1442285789,0 33 | 1442285790,0 34 | 1442285791,0 35 | 1442285792,0 36 | 1442285793,0 37 | 1442285794,0 38 | 1442285795,0 39 | 1442285796,0 40 | 1442285797,0 41 | 1442285798,0 42 | 1442285799,0 43 | 1442285800,0 44 | 1442285801,0 45 | 1442285802,0 46 | 1442285803,0 47 | 1442285804,0 48 | 1442285805,0 49 | 1442285806,0 50 | 1442285807,0 51 | 1442285808,0 52 | 1442285809,0 53 | 1442285810,0 54 | 1442285811,0 55 | 1442285812,0 56 | 1442285813,0 57 | 1442285814,0 58 | 1442285815,0 59 | 1442285816,0 60 | 1442285817,0 61 | 1442285818,0 62 | 1442285819,0 63 | 1442285820,0 64 | 1442285821,0 65 | 1442285822,0 66 | 1442285823,0 67 | 1442285824,0 68 | 1442285825,0 69 | 1442285826,0 70 | 1442285827,0 71 | 1442285828,0 72 | 1442285829,0 73 | 1442285830,0 74 | 1442285831,0 75 | 1442285832,0 76 | 1442285833,0 77 | 1442285834,0 78 | 1442285835,0 79 | 1442285836,0 80 | 1442285837,0 81 | 1442285838,0 82 | 1442285839,0 83 | 1442285840,0 84 | 1442285841,0 85 | 1442285842,0 86 | 1442285843,0 87 | 1442285844,0 88 | 1442285845,0 89 | 1442285846,0 90 | 1442285847,0 91 | 1442285848,0 92 | 1442285849,0 93 | 1442285850,0 94 | 1442285851,0 95 | 1442285852,0 96 | 1442285853,0 97 | 1442285854,0 98 | 1442285855,0 99 | 1442285856,0 100 | 1442285857,0 101 | 1442285858,0 102 | 1442285859,0 103 | 1442285860,0 104 | 1442285861,0 105 | 1442285862,0 106 | 1442285863,0 107 | 1442285864,0 108 | 1442285865,0 109 | 1442285866,0 110 | 1442285867,0 111 | 1442285868,0 112 | 1442285869,0 113 | 1442285870,0 114 | 1442285871,0 115 | 1442285872,0 116 | 1442285873,0 117 | 1442285874,0 118 | 1442285875,0 119 | 1442285876,0 120 | 1442285877,0 121 | 1442285878,0 122 | 1442285879,0 123 | 1442285880,0 124 | 1442285881,0 125 | 1442285882,0 126 | 1442285883,0 127 | 1442285884,0 128 | 1442285885,0 129 | 1442285886,0 130 | 1442285887,0 131 | 1442285888,0 132 | 1442285889,0 133 | 1442285890,0 134 | 1442285891,0 135 | 1442285892,0 136 | 1442285893,0 137 | 1442285894,0 138 | 1442285895,0 139 | 1442285896,0 140 | -------------------------------------------------------------------------------- /varOne-node/src/test/resources/csvsink/application_1439169262151_0237.5.jvm.PS-MarkSweep.time.csv: -------------------------------------------------------------------------------- 1 | t,value 2 | 1442285759,0 3 | 1442285760,0 4 | 1442285761,0 5 | 1442285762,0 6 | 1442285763,0 7 | 1442285764,0 8 | 1442285765,0 9 | 1442285766,0 10 | 1442285767,0 11 | 1442285768,0 12 | 1442285769,0 13 | 1442285770,0 14 | 1442285771,0 15 | 1442285772,0 16 | 1442285773,0 17 | 1442285774,0 18 | 1442285775,0 19 | 1442285776,0 20 | 1442285777,0 21 | 1442285778,0 22 | 1442285779,0 23 | 1442285780,0 24 | 1442285781,0 25 | 1442285782,0 26 | 1442285783,0 27 | 1442285784,0 28 | 1442285785,0 29 | 1442285786,0 30 | 1442285787,0 31 | 1442285788,0 32 | 1442285789,0 33 | 1442285790,0 34 | 1442285791,0 35 | 1442285792,0 36 | 1442285793,0 37 | 1442285794,0 38 | 1442285795,0 39 | 1442285796,0 40 | 1442285797,0 41 | 1442285798,0 42 | 1442285799,0 43 | 1442285800,0 44 | 1442285801,0 45 | 1442285802,0 46 | 1442285803,0 47 | 1442285804,0 48 | 1442285805,0 49 | 1442285806,0 50 | 1442285807,0 51 | 1442285808,0 52 | 1442285809,0 53 | 1442285810,0 54 | 1442285811,0 55 | 1442285812,0 56 | 1442285813,0 57 | 1442285814,0 58 | 1442285815,0 59 | 1442285816,0 60 | 1442285817,0 61 | 1442285818,0 62 | 1442285819,0 63 | 1442285820,0 64 | 1442285821,0 65 | 1442285822,0 66 | 1442285823,0 67 | 1442285824,0 68 | 1442285825,0 69 | 1442285826,0 70 | 1442285827,0 71 | 1442285828,0 72 | 1442285829,0 73 | 1442285830,0 74 | 1442285831,0 75 | 1442285832,0 76 | 1442285833,0 77 | 1442285834,0 78 | 1442285835,0 79 | 1442285836,0 80 | 1442285837,0 81 | 1442285838,0 82 | 1442285839,0 83 | 1442285840,0 84 | 1442285841,0 85 | 1442285842,0 86 | 1442285843,0 87 | 1442285844,0 88 | 1442285845,0 89 | 1442285846,0 90 | 1442285847,0 91 | 1442285848,0 92 | 1442285849,0 93 | 1442285850,0 94 | 1442285851,0 95 | 1442285852,0 96 | 1442285853,0 97 | 1442285854,0 98 | 1442285855,0 99 | 1442285856,0 100 | 1442285857,0 101 | 1442285858,0 102 | 1442285859,0 103 | 1442285860,0 104 | 1442285861,0 105 | 1442285862,0 106 | 1442285863,0 107 | 1442285864,0 108 | 1442285865,0 109 | 1442285866,0 110 | 1442285867,0 111 | 1442285868,0 112 | 1442285869,0 113 | 1442285870,0 114 | 1442285871,0 115 | 1442285872,0 116 | 1442285873,0 117 | 1442285874,0 118 | 1442285875,0 119 | 1442285876,0 120 | 1442285877,0 121 | 1442285878,0 122 | 1442285879,0 123 | 1442285880,0 124 | 1442285881,0 125 | 1442285882,0 126 | 1442285883,0 127 | 1442285884,0 128 | 1442285885,0 129 | 1442285886,0 130 | 1442285887,0 131 | 1442285888,0 132 | 1442285889,0 133 | 1442285890,0 134 | 1442285891,0 135 | 1442285892,0 136 | 1442285893,0 137 | 1442285894,0 138 | 1442285895,0 139 | 1442285896,0 140 | -------------------------------------------------------------------------------- /varOne-node/src/test/resources/csvsink/application_1439169262151_0237.1.executor.filesystem.file.read_ops.csv: -------------------------------------------------------------------------------- 1 | t,value 2 | 1442285759,0 3 | 1442285760,0 4 | 1442285761,0 5 | 1442285762,0 6 | 1442285763,0 7 | 1442285764,0 8 | 1442285765,0 9 | 1442285766,0 10 | 1442285767,0 11 | 1442285768,0 12 | 1442285769,0 13 | 1442285770,0 14 | 1442285771,0 15 | 1442285772,0 16 | 1442285773,0 17 | 1442285774,0 18 | 1442285775,0 19 | 1442285776,0 20 | 1442285777,0 21 | 1442285778,0 22 | 1442285779,0 23 | 1442285780,0 24 | 1442285781,0 25 | 1442285782,0 26 | 1442285783,0 27 | 1442285784,0 28 | 1442285785,0 29 | 1442285786,0 30 | 1442285787,0 31 | 1442285788,0 32 | 1442285789,0 33 | 1442285790,0 34 | 1442285791,0 35 | 1442285792,0 36 | 1442285793,0 37 | 1442285794,0 38 | 1442285795,0 39 | 1442285796,0 40 | 1442285797,0 41 | 1442285798,0 42 | 1442285799,0 43 | 1442285800,0 44 | 1442285801,0 45 | 1442285802,0 46 | 1442285803,0 47 | 1442285804,0 48 | 1442285805,0 49 | 1442285806,0 50 | 1442285807,0 51 | 1442285808,0 52 | 1442285809,0 53 | 1442285810,0 54 | 1442285811,0 55 | 1442285812,0 56 | 1442285813,0 57 | 1442285814,0 58 | 1442285815,0 59 | 1442285816,0 60 | 1442285817,0 61 | 1442285818,0 62 | 1442285819,0 63 | 1442285820,0 64 | 1442285821,0 65 | 1442285822,0 66 | 1442285823,0 67 | 1442285824,0 68 | 1442285825,0 69 | 1442285826,0 70 | 1442285827,0 71 | 1442285828,0 72 | 1442285829,0 73 | 1442285830,0 74 | 1442285831,0 75 | 1442285832,0 76 | 1442285833,0 77 | 1442285834,0 78 | 1442285835,0 79 | 1442285836,0 80 | 1442285837,0 81 | 1442285838,0 82 | 1442285839,0 83 | 1442285840,0 84 | 1442285841,0 85 | 1442285842,0 86 | 1442285843,0 87 | 1442285844,0 88 | 1442285845,0 89 | 1442285846,0 90 | 1442285847,0 91 | 1442285848,0 92 | 1442285849,0 93 | 1442285850,0 94 | 1442285851,0 95 | 1442285852,0 96 | 1442285853,0 97 | 1442285854,0 98 | 1442285855,0 99 | 1442285856,0 100 | 1442285857,0 101 | 1442285858,0 102 | 1442285859,0 103 | 1442285860,0 104 | 1442285861,0 105 | 1442285862,0 106 | 1442285863,0 107 | 1442285864,0 108 | 1442285865,0 109 | 1442285866,0 110 | 1442285867,0 111 | 1442285868,0 112 | 1442285869,0 113 | 1442285870,0 114 | 1442285871,0 115 | 1442285872,0 116 | 1442285873,0 117 | 1442285874,0 118 | 1442285875,0 119 | 1442285876,0 120 | 1442285877,0 121 | 1442285878,0 122 | 1442285879,0 123 | 1442285880,0 124 | 1442285881,0 125 | 1442285882,0 126 | 1442285883,0 127 | 1442285884,0 128 | 1442285885,0 129 | 1442285886,0 130 | 1442285887,0 131 | 1442285888,0 132 | 1442285889,0 133 | 1442285890,0 134 | 1442285891,0 135 | 1442285892,0 136 | 1442285893,0 137 | 1442285894,0 138 | 1442285895,0 139 | 1442285896,0 140 | -------------------------------------------------------------------------------- /varOne-node/src/test/resources/csvsink/application_1439169262151_0237.1.executor.filesystem.file.write_ops.csv: -------------------------------------------------------------------------------- 1 | t,value 2 | 1442285759,0 3 | 1442285760,0 4 | 1442285761,0 5 | 1442285762,0 6 | 1442285763,0 7 | 1442285764,0 8 | 1442285765,0 9 | 1442285766,0 10 | 1442285767,0 11 | 1442285768,0 12 | 1442285769,0 13 | 1442285770,0 14 | 1442285771,0 15 | 1442285772,0 16 | 1442285773,0 17 | 1442285774,0 18 | 1442285775,0 19 | 1442285776,0 20 | 1442285777,0 21 | 1442285778,0 22 | 1442285779,0 23 | 1442285780,0 24 | 1442285781,0 25 | 1442285782,0 26 | 1442285783,0 27 | 1442285784,0 28 | 1442285785,0 29 | 1442285786,0 30 | 1442285787,0 31 | 1442285788,0 32 | 1442285789,0 33 | 1442285790,0 34 | 1442285791,0 35 | 1442285792,0 36 | 1442285793,0 37 | 1442285794,0 38 | 1442285795,0 39 | 1442285796,0 40 | 1442285797,0 41 | 1442285798,0 42 | 1442285799,0 43 | 1442285800,0 44 | 1442285801,0 45 | 1442285802,0 46 | 1442285803,0 47 | 1442285804,0 48 | 1442285805,0 49 | 1442285806,0 50 | 1442285807,0 51 | 1442285808,0 52 | 1442285809,0 53 | 1442285810,0 54 | 1442285811,0 55 | 1442285812,0 56 | 1442285813,0 57 | 1442285814,0 58 | 1442285815,0 59 | 1442285816,0 60 | 1442285817,0 61 | 1442285818,0 62 | 1442285819,0 63 | 1442285820,0 64 | 1442285821,0 65 | 1442285822,0 66 | 1442285823,0 67 | 1442285824,0 68 | 1442285825,0 69 | 1442285826,0 70 | 1442285827,0 71 | 1442285828,0 72 | 1442285829,0 73 | 1442285830,0 74 | 1442285831,0 75 | 1442285832,0 76 | 1442285833,0 77 | 1442285834,0 78 | 1442285835,0 79 | 1442285836,0 80 | 1442285837,0 81 | 1442285838,0 82 | 1442285839,0 83 | 1442285840,0 84 | 1442285841,0 85 | 1442285842,0 86 | 1442285843,0 87 | 1442285844,0 88 | 1442285845,0 89 | 1442285846,0 90 | 1442285847,0 91 | 1442285848,0 92 | 1442285849,0 93 | 1442285850,0 94 | 1442285851,0 95 | 1442285852,0 96 | 1442285853,0 97 | 1442285854,0 98 | 1442285855,0 99 | 1442285856,0 100 | 1442285857,0 101 | 1442285858,0 102 | 1442285859,0 103 | 1442285860,0 104 | 1442285861,0 105 | 1442285862,0 106 | 1442285863,0 107 | 1442285864,0 108 | 1442285865,0 109 | 1442285866,0 110 | 1442285867,0 111 | 1442285868,0 112 | 1442285869,0 113 | 1442285870,0 114 | 1442285871,0 115 | 1442285872,0 116 | 1442285873,0 117 | 1442285874,0 118 | 1442285875,0 119 | 1442285876,0 120 | 1442285877,0 121 | 1442285878,0 122 | 1442285879,0 123 | 1442285880,0 124 | 1442285881,0 125 | 1442285882,0 126 | 1442285883,0 127 | 1442285884,0 128 | 1442285885,0 129 | 1442285886,0 130 | 1442285887,0 131 | 1442285888,0 132 | 1442285889,0 133 | 1442285890,0 134 | 1442285891,0 135 | 1442285892,0 136 | 1442285893,0 137 | 1442285894,0 138 | 1442285895,0 139 | 1442285896,0 140 | -------------------------------------------------------------------------------- /varOne-node/src/test/resources/csvsink/application_1439169262151_0237.1.executor.filesystem.hdfs.write_ops.csv: -------------------------------------------------------------------------------- 1 | t,value 2 | 1442285759,0 3 | 1442285760,0 4 | 1442285761,0 5 | 1442285762,0 6 | 1442285763,0 7 | 1442285764,0 8 | 1442285765,0 9 | 1442285766,0 10 | 1442285767,0 11 | 1442285768,0 12 | 1442285769,0 13 | 1442285770,0 14 | 1442285771,0 15 | 1442285772,0 16 | 1442285773,0 17 | 1442285774,0 18 | 1442285775,0 19 | 1442285776,0 20 | 1442285777,0 21 | 1442285778,0 22 | 1442285779,0 23 | 1442285780,0 24 | 1442285781,0 25 | 1442285782,0 26 | 1442285783,0 27 | 1442285784,0 28 | 1442285785,0 29 | 1442285786,0 30 | 1442285787,0 31 | 1442285788,0 32 | 1442285789,0 33 | 1442285790,0 34 | 1442285791,0 35 | 1442285792,0 36 | 1442285793,0 37 | 1442285794,0 38 | 1442285795,0 39 | 1442285796,0 40 | 1442285797,0 41 | 1442285798,0 42 | 1442285799,0 43 | 1442285800,0 44 | 1442285801,0 45 | 1442285802,0 46 | 1442285803,0 47 | 1442285804,0 48 | 1442285805,0 49 | 1442285806,0 50 | 1442285807,0 51 | 1442285808,0 52 | 1442285809,0 53 | 1442285810,0 54 | 1442285811,0 55 | 1442285812,0 56 | 1442285813,0 57 | 1442285814,0 58 | 1442285815,0 59 | 1442285816,0 60 | 1442285817,0 61 | 1442285818,0 62 | 1442285819,0 63 | 1442285820,0 64 | 1442285821,0 65 | 1442285822,0 66 | 1442285823,0 67 | 1442285824,0 68 | 1442285825,0 69 | 1442285826,0 70 | 1442285827,0 71 | 1442285828,0 72 | 1442285829,0 73 | 1442285830,0 74 | 1442285831,0 75 | 1442285832,0 76 | 1442285833,0 77 | 1442285834,0 78 | 1442285835,0 79 | 1442285836,0 80 | 1442285837,0 81 | 1442285838,0 82 | 1442285839,0 83 | 1442285840,0 84 | 1442285841,0 85 | 1442285842,0 86 | 1442285843,0 87 | 1442285844,0 88 | 1442285845,0 89 | 1442285846,0 90 | 1442285847,0 91 | 1442285848,0 92 | 1442285849,0 93 | 1442285850,0 94 | 1442285851,0 95 | 1442285852,0 96 | 1442285853,0 97 | 1442285854,0 98 | 1442285855,0 99 | 1442285856,0 100 | 1442285857,0 101 | 1442285858,0 102 | 1442285859,0 103 | 1442285860,0 104 | 1442285861,0 105 | 1442285862,0 106 | 1442285863,0 107 | 1442285864,0 108 | 1442285865,0 109 | 1442285866,0 110 | 1442285867,0 111 | 1442285868,0 112 | 1442285869,0 113 | 1442285870,0 114 | 1442285871,0 115 | 1442285872,0 116 | 1442285873,0 117 | 1442285874,0 118 | 1442285875,0 119 | 1442285876,0 120 | 1442285877,0 121 | 1442285878,0 122 | 1442285879,0 123 | 1442285880,0 124 | 1442285881,0 125 | 1442285882,0 126 | 1442285883,0 127 | 1442285884,0 128 | 1442285885,0 129 | 1442285886,0 130 | 1442285887,0 131 | 1442285888,0 132 | 1442285889,0 133 | 1442285890,0 134 | 1442285891,0 135 | 1442285892,0 136 | 1442285893,0 137 | 1442285894,0 138 | 1442285895,0 139 | 1442285896,0 140 | -------------------------------------------------------------------------------- /varOne-node/src/test/resources/csvsink/application_1439169262151_0237.1.executor.threadpool.activeTasks.csv: -------------------------------------------------------------------------------- 1 | t,value 2 | 1442285759,0 3 | 1442285760,1 4 | 1442285761,1 5 | 1442285762,1 6 | 1442285763,1 7 | 1442285764,1 8 | 1442285765,1 9 | 1442285766,1 10 | 1442285767,1 11 | 1442285768,1 12 | 1442285769,1 13 | 1442285770,1 14 | 1442285771,1 15 | 1442285772,1 16 | 1442285773,1 17 | 1442285774,1 18 | 1442285775,1 19 | 1442285776,1 20 | 1442285777,1 21 | 1442285778,1 22 | 1442285779,1 23 | 1442285780,1 24 | 1442285781,1 25 | 1442285782,1 26 | 1442285783,1 27 | 1442285784,1 28 | 1442285785,1 29 | 1442285786,1 30 | 1442285787,1 31 | 1442285788,1 32 | 1442285789,1 33 | 1442285790,1 34 | 1442285791,1 35 | 1442285792,1 36 | 1442285793,1 37 | 1442285794,1 38 | 1442285795,1 39 | 1442285796,1 40 | 1442285797,1 41 | 1442285798,1 42 | 1442285799,1 43 | 1442285800,1 44 | 1442285801,1 45 | 1442285802,1 46 | 1442285803,1 47 | 1442285804,1 48 | 1442285805,1 49 | 1442285806,1 50 | 1442285807,1 51 | 1442285808,1 52 | 1442285809,1 53 | 1442285810,1 54 | 1442285811,1 55 | 1442285812,1 56 | 1442285813,1 57 | 1442285814,1 58 | 1442285815,1 59 | 1442285816,1 60 | 1442285817,1 61 | 1442285818,1 62 | 1442285819,1 63 | 1442285820,1 64 | 1442285821,1 65 | 1442285822,1 66 | 1442285823,1 67 | 1442285824,1 68 | 1442285825,1 69 | 1442285826,1 70 | 1442285827,0 71 | 1442285828,1 72 | 1442285829,1 73 | 1442285830,1 74 | 1442285831,1 75 | 1442285832,1 76 | 1442285833,1 77 | 1442285834,1 78 | 1442285835,1 79 | 1442285836,1 80 | 1442285837,1 81 | 1442285838,1 82 | 1442285839,1 83 | 1442285840,1 84 | 1442285841,1 85 | 1442285842,1 86 | 1442285843,1 87 | 1442285844,1 88 | 1442285845,1 89 | 1442285846,1 90 | 1442285847,1 91 | 1442285848,1 92 | 1442285849,1 93 | 1442285850,1 94 | 1442285851,1 95 | 1442285852,1 96 | 1442285853,1 97 | 1442285854,1 98 | 1442285855,1 99 | 1442285856,1 100 | 1442285857,1 101 | 1442285858,1 102 | 1442285859,1 103 | 1442285860,1 104 | 1442285861,1 105 | 1442285862,1 106 | 1442285863,1 107 | 1442285864,1 108 | 1442285865,1 109 | 1442285866,1 110 | 1442285867,1 111 | 1442285868,1 112 | 1442285869,1 113 | 1442285870,1 114 | 1442285871,1 115 | 1442285872,1 116 | 1442285873,1 117 | 1442285874,1 118 | 1442285875,1 119 | 1442285876,1 120 | 1442285877,1 121 | 1442285878,1 122 | 1442285879,1 123 | 1442285880,1 124 | 1442285881,1 125 | 1442285882,1 126 | 1442285883,1 127 | 1442285884,1 128 | 1442285885,1 129 | 1442285886,1 130 | 1442285887,1 131 | 1442285888,1 132 | 1442285889,1 133 | 1442285890,1 134 | 1442285891,1 135 | 1442285892,1 136 | 1442285893,1 137 | 1442285894,1 138 | 1442285895,0 139 | 1442285896,0 140 | -------------------------------------------------------------------------------- /varOne-node/src/test/resources/csvsink/application_1439169262151_0237.5.executor.filesystem.file.read_ops.csv: -------------------------------------------------------------------------------- 1 | t,value 2 | 1442285759,0 3 | 1442285760,0 4 | 1442285761,0 5 | 1442285762,0 6 | 1442285763,0 7 | 1442285764,0 8 | 1442285765,0 9 | 1442285766,0 10 | 1442285767,0 11 | 1442285768,0 12 | 1442285769,0 13 | 1442285770,0 14 | 1442285771,0 15 | 1442285772,0 16 | 1442285773,0 17 | 1442285774,0 18 | 1442285775,0 19 | 1442285776,0 20 | 1442285777,0 21 | 1442285778,0 22 | 1442285779,0 23 | 1442285780,0 24 | 1442285781,0 25 | 1442285782,0 26 | 1442285783,0 27 | 1442285784,0 28 | 1442285785,0 29 | 1442285786,0 30 | 1442285787,0 31 | 1442285788,0 32 | 1442285789,0 33 | 1442285790,0 34 | 1442285791,0 35 | 1442285792,0 36 | 1442285793,0 37 | 1442285794,0 38 | 1442285795,0 39 | 1442285796,0 40 | 1442285797,0 41 | 1442285798,0 42 | 1442285799,0 43 | 1442285800,0 44 | 1442285801,0 45 | 1442285802,0 46 | 1442285803,0 47 | 1442285804,0 48 | 1442285805,0 49 | 1442285806,0 50 | 1442285807,0 51 | 1442285808,0 52 | 1442285809,0 53 | 1442285810,0 54 | 1442285811,0 55 | 1442285812,0 56 | 1442285813,0 57 | 1442285814,0 58 | 1442285815,0 59 | 1442285816,0 60 | 1442285817,0 61 | 1442285818,0 62 | 1442285819,0 63 | 1442285820,0 64 | 1442285821,0 65 | 1442285822,0 66 | 1442285823,0 67 | 1442285824,0 68 | 1442285825,0 69 | 1442285826,0 70 | 1442285827,0 71 | 1442285828,0 72 | 1442285829,0 73 | 1442285830,0 74 | 1442285831,0 75 | 1442285832,0 76 | 1442285833,0 77 | 1442285834,0 78 | 1442285835,0 79 | 1442285836,0 80 | 1442285837,0 81 | 1442285838,0 82 | 1442285839,0 83 | 1442285840,0 84 | 1442285841,0 85 | 1442285842,0 86 | 1442285843,0 87 | 1442285844,0 88 | 1442285845,0 89 | 1442285846,0 90 | 1442285847,0 91 | 1442285848,0 92 | 1442285849,0 93 | 1442285850,0 94 | 1442285851,0 95 | 1442285852,0 96 | 1442285853,0 97 | 1442285854,0 98 | 1442285855,0 99 | 1442285856,0 100 | 1442285857,0 101 | 1442285858,0 102 | 1442285859,0 103 | 1442285860,0 104 | 1442285861,0 105 | 1442285862,0 106 | 1442285863,0 107 | 1442285864,0 108 | 1442285865,0 109 | 1442285866,0 110 | 1442285867,0 111 | 1442285868,0 112 | 1442285869,0 113 | 1442285870,0 114 | 1442285871,0 115 | 1442285872,0 116 | 1442285873,0 117 | 1442285874,0 118 | 1442285875,0 119 | 1442285876,0 120 | 1442285877,0 121 | 1442285878,0 122 | 1442285879,0 123 | 1442285880,0 124 | 1442285881,0 125 | 1442285882,0 126 | 1442285883,0 127 | 1442285884,0 128 | 1442285885,0 129 | 1442285886,0 130 | 1442285887,0 131 | 1442285888,0 132 | 1442285889,0 133 | 1442285890,0 134 | 1442285891,0 135 | 1442285892,0 136 | 1442285893,0 137 | 1442285894,0 138 | 1442285895,0 139 | 1442285896,0 140 | -------------------------------------------------------------------------------- /varOne-node/src/test/resources/csvsink/application_1439169262151_0237.5.executor.filesystem.file.write_ops.csv: -------------------------------------------------------------------------------- 1 | t,value 2 | 1442285759,0 3 | 1442285760,0 4 | 1442285761,0 5 | 1442285762,0 6 | 1442285763,0 7 | 1442285764,0 8 | 1442285765,0 9 | 1442285766,0 10 | 1442285767,0 11 | 1442285768,0 12 | 1442285769,0 13 | 1442285770,0 14 | 1442285771,0 15 | 1442285772,0 16 | 1442285773,0 17 | 1442285774,0 18 | 1442285775,0 19 | 1442285776,0 20 | 1442285777,0 21 | 1442285778,0 22 | 1442285779,0 23 | 1442285780,0 24 | 1442285781,0 25 | 1442285782,0 26 | 1442285783,0 27 | 1442285784,0 28 | 1442285785,0 29 | 1442285786,0 30 | 1442285787,0 31 | 1442285788,0 32 | 1442285789,0 33 | 1442285790,0 34 | 1442285791,0 35 | 1442285792,0 36 | 1442285793,0 37 | 1442285794,0 38 | 1442285795,0 39 | 1442285796,0 40 | 1442285797,0 41 | 1442285798,0 42 | 1442285799,0 43 | 1442285800,0 44 | 1442285801,0 45 | 1442285802,0 46 | 1442285803,0 47 | 1442285804,0 48 | 1442285805,0 49 | 1442285806,0 50 | 1442285807,0 51 | 1442285808,0 52 | 1442285809,0 53 | 1442285810,0 54 | 1442285811,0 55 | 1442285812,0 56 | 1442285813,0 57 | 1442285814,0 58 | 1442285815,0 59 | 1442285816,0 60 | 1442285817,0 61 | 1442285818,0 62 | 1442285819,0 63 | 1442285820,0 64 | 1442285821,0 65 | 1442285822,0 66 | 1442285823,0 67 | 1442285824,0 68 | 1442285825,0 69 | 1442285826,0 70 | 1442285827,0 71 | 1442285828,0 72 | 1442285829,0 73 | 1442285830,0 74 | 1442285831,0 75 | 1442285832,0 76 | 1442285833,0 77 | 1442285834,0 78 | 1442285835,0 79 | 1442285836,0 80 | 1442285837,0 81 | 1442285838,0 82 | 1442285839,0 83 | 1442285840,0 84 | 1442285841,0 85 | 1442285842,0 86 | 1442285843,0 87 | 1442285844,0 88 | 1442285845,0 89 | 1442285846,0 90 | 1442285847,0 91 | 1442285848,0 92 | 1442285849,0 93 | 1442285850,0 94 | 1442285851,0 95 | 1442285852,0 96 | 1442285853,0 97 | 1442285854,0 98 | 1442285855,0 99 | 1442285856,0 100 | 1442285857,0 101 | 1442285858,0 102 | 1442285859,0 103 | 1442285860,0 104 | 1442285861,0 105 | 1442285862,0 106 | 1442285863,0 107 | 1442285864,0 108 | 1442285865,0 109 | 1442285866,0 110 | 1442285867,0 111 | 1442285868,0 112 | 1442285869,0 113 | 1442285870,0 114 | 1442285871,0 115 | 1442285872,0 116 | 1442285873,0 117 | 1442285874,0 118 | 1442285875,0 119 | 1442285876,0 120 | 1442285877,0 121 | 1442285878,0 122 | 1442285879,0 123 | 1442285880,0 124 | 1442285881,0 125 | 1442285882,0 126 | 1442285883,0 127 | 1442285884,0 128 | 1442285885,0 129 | 1442285886,0 130 | 1442285887,0 131 | 1442285888,0 132 | 1442285889,0 133 | 1442285890,0 134 | 1442285891,0 135 | 1442285892,0 136 | 1442285893,0 137 | 1442285894,0 138 | 1442285895,0 139 | 1442285896,0 140 | -------------------------------------------------------------------------------- /varOne-node/src/test/resources/csvsink/application_1439169262151_0237.5.executor.filesystem.hdfs.write_ops.csv: -------------------------------------------------------------------------------- 1 | t,value 2 | 1442285759,0 3 | 1442285760,0 4 | 1442285761,0 5 | 1442285762,0 6 | 1442285763,0 7 | 1442285764,0 8 | 1442285765,0 9 | 1442285766,0 10 | 1442285767,0 11 | 1442285768,0 12 | 1442285769,0 13 | 1442285770,0 14 | 1442285771,0 15 | 1442285772,0 16 | 1442285773,0 17 | 1442285774,0 18 | 1442285775,0 19 | 1442285776,0 20 | 1442285777,0 21 | 1442285778,0 22 | 1442285779,0 23 | 1442285780,0 24 | 1442285781,0 25 | 1442285782,0 26 | 1442285783,0 27 | 1442285784,0 28 | 1442285785,0 29 | 1442285786,0 30 | 1442285787,0 31 | 1442285788,0 32 | 1442285789,0 33 | 1442285790,0 34 | 1442285791,0 35 | 1442285792,0 36 | 1442285793,0 37 | 1442285794,0 38 | 1442285795,0 39 | 1442285796,0 40 | 1442285797,0 41 | 1442285798,0 42 | 1442285799,0 43 | 1442285800,0 44 | 1442285801,0 45 | 1442285802,0 46 | 1442285803,0 47 | 1442285804,0 48 | 1442285805,0 49 | 1442285806,0 50 | 1442285807,0 51 | 1442285808,0 52 | 1442285809,0 53 | 1442285810,0 54 | 1442285811,0 55 | 1442285812,0 56 | 1442285813,0 57 | 1442285814,0 58 | 1442285815,0 59 | 1442285816,0 60 | 1442285817,0 61 | 1442285818,0 62 | 1442285819,0 63 | 1442285820,0 64 | 1442285821,0 65 | 1442285822,0 66 | 1442285823,0 67 | 1442285824,0 68 | 1442285825,0 69 | 1442285826,0 70 | 1442285827,0 71 | 1442285828,0 72 | 1442285829,0 73 | 1442285830,0 74 | 1442285831,0 75 | 1442285832,0 76 | 1442285833,0 77 | 1442285834,0 78 | 1442285835,0 79 | 1442285836,0 80 | 1442285837,0 81 | 1442285838,0 82 | 1442285839,0 83 | 1442285840,0 84 | 1442285841,0 85 | 1442285842,0 86 | 1442285843,0 87 | 1442285844,0 88 | 1442285845,0 89 | 1442285846,0 90 | 1442285847,0 91 | 1442285848,0 92 | 1442285849,0 93 | 1442285850,0 94 | 1442285851,0 95 | 1442285852,0 96 | 1442285853,0 97 | 1442285854,0 98 | 1442285855,0 99 | 1442285856,0 100 | 1442285857,0 101 | 1442285858,0 102 | 1442285859,0 103 | 1442285860,0 104 | 1442285861,0 105 | 1442285862,0 106 | 1442285863,0 107 | 1442285864,0 108 | 1442285865,0 109 | 1442285866,0 110 | 1442285867,0 111 | 1442285868,0 112 | 1442285869,0 113 | 1442285870,0 114 | 1442285871,0 115 | 1442285872,0 116 | 1442285873,0 117 | 1442285874,0 118 | 1442285875,0 119 | 1442285876,0 120 | 1442285877,0 121 | 1442285878,0 122 | 1442285879,0 123 | 1442285880,0 124 | 1442285881,0 125 | 1442285882,0 126 | 1442285883,0 127 | 1442285884,0 128 | 1442285885,0 129 | 1442285886,0 130 | 1442285887,0 131 | 1442285888,0 132 | 1442285889,0 133 | 1442285890,0 134 | 1442285891,0 135 | 1442285892,0 136 | 1442285893,0 137 | 1442285894,0 138 | 1442285895,0 139 | 1442285896,0 140 | -------------------------------------------------------------------------------- /varOne-node/src/test/resources/csvsink/application_1439169262151_0237.5.executor.threadpool.activeTasks.csv: -------------------------------------------------------------------------------- 1 | t,value 2 | 1442285759,0 3 | 1442285760,1 4 | 1442285761,1 5 | 1442285762,1 6 | 1442285763,1 7 | 1442285764,1 8 | 1442285765,1 9 | 1442285766,1 10 | 1442285767,0 11 | 1442285768,1 12 | 1442285769,1 13 | 1442285770,1 14 | 1442285771,1 15 | 1442285772,1 16 | 1442285773,1 17 | 1442285774,1 18 | 1442285775,1 19 | 1442285776,1 20 | 1442285777,1 21 | 1442285778,0 22 | 1442285779,1 23 | 1442285780,1 24 | 1442285781,1 25 | 1442285782,1 26 | 1442285783,1 27 | 1442285784,1 28 | 1442285785,1 29 | 1442285786,1 30 | 1442285787,1 31 | 1442285788,1 32 | 1442285789,1 33 | 1442285790,1 34 | 1442285791,1 35 | 1442285792,1 36 | 1442285793,1 37 | 1442285794,1 38 | 1442285795,1 39 | 1442285796,1 40 | 1442285797,1 41 | 1442285798,1 42 | 1442285799,1 43 | 1442285800,1 44 | 1442285801,1 45 | 1442285802,1 46 | 1442285803,1 47 | 1442285804,1 48 | 1442285805,1 49 | 1442285806,1 50 | 1442285807,1 51 | 1442285808,1 52 | 1442285809,1 53 | 1442285810,1 54 | 1442285811,1 55 | 1442285812,1 56 | 1442285813,1 57 | 1442285814,1 58 | 1442285815,1 59 | 1442285816,1 60 | 1442285817,1 61 | 1442285818,1 62 | 1442285819,1 63 | 1442285820,1 64 | 1442285821,1 65 | 1442285822,1 66 | 1442285823,1 67 | 1442285824,1 68 | 1442285825,1 69 | 1442285826,1 70 | 1442285827,1 71 | 1442285828,1 72 | 1442285829,1 73 | 1442285830,1 74 | 1442285831,1 75 | 1442285832,1 76 | 1442285833,1 77 | 1442285834,1 78 | 1442285835,1 79 | 1442285836,1 80 | 1442285837,1 81 | 1442285838,1 82 | 1442285839,1 83 | 1442285840,1 84 | 1442285841,1 85 | 1442285842,1 86 | 1442285843,1 87 | 1442285844,1 88 | 1442285845,1 89 | 1442285846,1 90 | 1442285847,1 91 | 1442285848,1 92 | 1442285849,1 93 | 1442285850,1 94 | 1442285851,1 95 | 1442285852,1 96 | 1442285853,1 97 | 1442285854,1 98 | 1442285855,1 99 | 1442285856,1 100 | 1442285857,1 101 | 1442285858,1 102 | 1442285859,1 103 | 1442285860,1 104 | 1442285861,1 105 | 1442285862,1 106 | 1442285863,1 107 | 1442285864,1 108 | 1442285865,1 109 | 1442285866,1 110 | 1442285867,1 111 | 1442285868,1 112 | 1442285869,1 113 | 1442285870,1 114 | 1442285871,1 115 | 1442285872,1 116 | 1442285873,1 117 | 1442285874,1 118 | 1442285875,1 119 | 1442285876,1 120 | 1442285877,1 121 | 1442285878,1 122 | 1442285879,1 123 | 1442285880,1 124 | 1442285881,1 125 | 1442285882,1 126 | 1442285883,1 127 | 1442285884,1 128 | 1442285885,1 129 | 1442285886,1 130 | 1442285887,1 131 | 1442285888,1 132 | 1442285889,1 133 | 1442285890,1 134 | 1442285891,1 135 | 1442285892,1 136 | 1442285893,1 137 | 1442285894,1 138 | 1442285895,1 139 | 1442285896,0 140 | --------------------------------------------------------------------------------