├── SQL ├── Readme.txt ├── 06-14-2017-V0.1.sql ├── logdb-08-06-2017-V0.2.sql ├── logdb-07-31-2017-V0.1.sql ├── 06-16-2017-V0.2.sql ├── 06-23-2017-V0.3.sql ├── 06-26-2017-V0.4.sql ├── 06-27-2017-V0.5.sql └── 06-28-2017-V0.6.sql ├── JerseyServices ├── build.sh ├── launch_scripts │ ├── run_training.sh │ ├── run_algr_input.sh │ ├── run_data_input.sh │ ├── run_execution.sh │ ├── run_jersey_testing.sh │ └── run_jersey_all.sh ├── dbConfig.ini ├── src │ ├── main │ │ ├── resources │ │ │ └── db │ │ │ │ └── db-config.properties │ │ └── java │ │ │ └── com │ │ │ └── chaosmonkeys │ │ │ ├── Utilities │ │ │ ├── LogType.java │ │ │ ├── Constants.java │ │ │ ├── ConfigurationHelper.java │ │ │ ├── DbConfigurationHelper.java │ │ │ ├── ZipUtils.java │ │ │ ├── Logger.java │ │ │ ├── MachineIPHelper.java │ │ │ ├── DataVerification.java │ │ │ ├── ArrayUtils.java │ │ │ └── db │ │ │ │ └── DbUtils.java │ │ │ ├── train │ │ │ ├── task │ │ │ │ ├── TaskType.java │ │ │ │ ├── TaskManager.java │ │ │ │ ├── interfaces │ │ │ │ │ └── OnTaskUpdateListener.java │ │ │ │ ├── TrainingTaskInfo.java │ │ │ │ ├── ExecutionTaskInfo.java │ │ │ │ ├── TaskState.java │ │ │ │ ├── BaseTaskInfo.java │ │ │ │ ├── AbsTask.java │ │ │ │ ├── ResourceInfo.java │ │ │ │ ├── TrainingTaskManager.java │ │ │ │ └── ExecutionTaskManager.java │ │ │ ├── Constants.java │ │ │ ├── dto │ │ │ │ ├── ExecutionServiceStatus.java │ │ │ │ └── ExperimentDto.java │ │ │ └── heartbeat │ │ │ │ └── ExecutionServiceHeartBeatsClient.java │ │ │ ├── dao │ │ │ ├── OpLog.java │ │ │ ├── ErrorLog.java │ │ │ ├── Task.java │ │ │ ├── Experiment.java │ │ │ ├── Prediction.java │ │ │ ├── AlgorithmLanguage.java │ │ │ ├── PredictionModel.java │ │ │ ├── Dataset.java │ │ │ └── Algorithm.java │ │ │ ├── DTO │ │ │ ├── StatusInfo.java │ │ │ ├── DbConfigInfo.java │ │ │ ├── RegistrationInfo.java │ │ │ ├── BasicResponse.java │ │ │ ├── ConfigInfo.java │ │ │ └── BaseResponse.java │ │ │ ├── CORSFilter.java │ │ │ ├── datasetinputservice │ │ │ ├── DTO │ │ │ │ ├── DatasetInputServiceStatusInfo.java │ │ │ │ └── DatasetInputServiceWorkState.java │ │ │ └── DatasetInputServiceHeartBeatsClient.java │ │ │ ├── algrinputservice │ │ │ ├── DTO │ │ │ │ ├── AlgorithmInputServiceStatusInfo.java │ │ │ │ └── AlgorithmInputServiceWorkState.java │ │ │ └── AlgorithmInputServiceHeartBeatsClient.java │ │ │ └── HeartBeatsClient.java │ └── test │ │ └── java │ │ └── com │ │ └── chaosmonkeys │ │ ├── utils │ │ └── StringUtilsContainsTest.java │ │ └── DatasetInputServiceTest.java ├── Config │ ├── execution.ini │ ├── training.ini │ ├── algr-input.ini │ ├── jersey-testing.ini │ └── data-input.ini └── JerseyServices.iml ├── CoordinationService ├── exportToCSV.js ├── public │ ├── favicon.ico │ ├── stylesheets │ │ ├── style.styl │ │ ├── style.css │ │ ├── monitoring-table.css │ │ ├── common.css │ │ └── justified-nav.css │ ├── fonts │ │ ├── fontawesome-webfont.ttf │ │ ├── fontawesome-webfont.woff │ │ └── fontawesome-webfont.woff2 │ └── scripts │ │ ├── monitoring.js │ │ ├── common.js │ │ └── models.js ├── views │ ├── error.jade │ ├── layout.jade │ └── monitoring.html ├── config │ ├── dbConfig.json │ └── logDbConfig.json ├── routes │ ├── users.js │ ├── dbUtilities.js │ ├── index.js │ ├── models.js │ ├── datasets.js │ ├── algorithms.js │ ├── download.js │ ├── logUtilities.js │ ├── dynamicIp.js │ ├── experiments.js │ └── heartbeat.js ├── Readme.txt ├── startup.js ├── package.json ├── PostmanQueries │ └── Skeleton.postman_collection.json ├── test │ └── dbUtilitiesTest.js ├── bin │ └── www └── app.js ├── SampleInput ├── Datasets │ └── default.csv └── Algorithms │ └── Main.zip ├── .gitignore └── README.md /SQL/Readme.txt: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /JerseyServices/build.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | mvn clean compile -DskipTests package -------------------------------------------------------------------------------- /CoordinationService/exportToCSV.js: -------------------------------------------------------------------------------- 1 | var mysqlToCsv = require("mysql-to-csv") 2 | 3 | 4 | 5 | 6 | -------------------------------------------------------------------------------- /SampleInput/Datasets/default.csv: -------------------------------------------------------------------------------- 1 | user,description 2 | Gazonk Foo,Here we go 3 | Foo Gazonk,doing some tests 4 | Mr Foo, 5 | -------------------------------------------------------------------------------- /JerseyServices/launch_scripts/run_training.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | java -jar ../target/jersey-service-1.0.jar ../Config/training.ini -------------------------------------------------------------------------------- /SampleInput/Algorithms/Main.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Harpuia/chaos-monkeys-ml-platform/HEAD/SampleInput/Algorithms/Main.zip -------------------------------------------------------------------------------- /JerseyServices/launch_scripts/run_algr_input.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | java -jar ../target/jersey-service-1.0.jar ../Config/algr-input.ini -------------------------------------------------------------------------------- /JerseyServices/launch_scripts/run_data_input.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | java -jar ../target/jersey-service-1.0.jar ../Config/data-input.ini -------------------------------------------------------------------------------- /JerseyServices/launch_scripts/run_execution.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | java -jar ../target/jersey-service-1.0.jar ../Config/execution.ini -------------------------------------------------------------------------------- /CoordinationService/public/favicon.ico: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Harpuia/chaos-monkeys-ml-platform/HEAD/CoordinationService/public/favicon.ico -------------------------------------------------------------------------------- /CoordinationService/views/error.jade: -------------------------------------------------------------------------------- 1 | extends layout 2 | 3 | block content 4 | h1= message 5 | h2= error.status 6 | pre #{error.stack} 7 | -------------------------------------------------------------------------------- /JerseyServices/dbConfig.ini: -------------------------------------------------------------------------------- 1 | [info] 2 | host = 127.0.0.1 3 | port = 3306 4 | userName = root 5 | password = root 6 | dbName = ConfigurationDatabase -------------------------------------------------------------------------------- /JerseyServices/launch_scripts/run_jersey_testing.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | java -jar ../target/jersey-service-1.0.jar ../Config/jersey-testing.ini -------------------------------------------------------------------------------- /CoordinationService/public/stylesheets/style.styl: -------------------------------------------------------------------------------- 1 | body 2 | padding: 50px 3 | font: 14px "Lucida Grande", Helvetica, Arial, sans-serif 4 | a 5 | color: #00B7FF 6 | -------------------------------------------------------------------------------- /CoordinationService/config/dbConfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "dbhost":"127.0.0.1", 3 | "dbuser":"root", 4 | "dbpassword":"root", 5 | "dbname":"configurationdatabase" 6 | } -------------------------------------------------------------------------------- /CoordinationService/public/fonts/fontawesome-webfont.ttf: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Harpuia/chaos-monkeys-ml-platform/HEAD/CoordinationService/public/fonts/fontawesome-webfont.ttf -------------------------------------------------------------------------------- /CoordinationService/public/fonts/fontawesome-webfont.woff: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Harpuia/chaos-monkeys-ml-platform/HEAD/CoordinationService/public/fonts/fontawesome-webfont.woff -------------------------------------------------------------------------------- /CoordinationService/config/logDbConfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "dbhost":"127.0.0.1", 3 | "dbuser":"root", 4 | "dbpassword":"root", 5 | "dbname":"logdatabase", 6 | "port":"3306" 7 | } -------------------------------------------------------------------------------- /CoordinationService/public/fonts/fontawesome-webfont.woff2: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/Harpuia/chaos-monkeys-ml-platform/HEAD/CoordinationService/public/fonts/fontawesome-webfont.woff2 -------------------------------------------------------------------------------- /CoordinationService/public/stylesheets/style.css: -------------------------------------------------------------------------------- 1 | body { 2 | padding: 50px; 3 | font: 14px "Lucida Grande", Helvetica, Arial, sans-serif; 4 | } 5 | a { 6 | color: #00b7ff; 7 | } 8 | -------------------------------------------------------------------------------- /CoordinationService/views/layout.jade: -------------------------------------------------------------------------------- 1 | doctype html 2 | html 3 | head 4 | title= title 5 | link(rel='stylesheet', href='/stylesheets/style.css') 6 | body 7 | block content 8 | -------------------------------------------------------------------------------- /JerseyServices/src/main/resources/db/db-config.properties: -------------------------------------------------------------------------------- 1 | jdbc.baseurl=jdbc:mysql://127.0.0.1:3306/ 2 | jdbc.username=root 3 | jdbc.password=root 4 | jdbc.driver=com.mysql.jdbc.Driver 5 | jdbc.dbName=ConfigurationDatabase -------------------------------------------------------------------------------- /JerseyServices/Config/execution.ini: -------------------------------------------------------------------------------- 1 | [coordination] 2 | IP = http://127.0.0.1:3000 3 | [info] 4 | name = Execution Service for R 5 | type = Exec-R 6 | description = Execution Service for R 7 | ip = http://127.0.0.1 8 | port = 8083 -------------------------------------------------------------------------------- /JerseyServices/Config/training.ini: -------------------------------------------------------------------------------- 1 | [coordination] 2 | IP = http://127.0.0.1:3000 3 | [info] 4 | name = Training Service for R 5 | type = Train-R 6 | description = Training Service for R 7 | ip = http://127.0.0.1 8 | port = 8084 -------------------------------------------------------------------------------- /CoordinationService/public/stylesheets/monitoring-table.css: -------------------------------------------------------------------------------- 1 | td { 2 | padding-left: 10px; 3 | padding-right: 10px; 4 | padding-top: 0px; 5 | padding-bottom: 0px; 6 | text-align: center; 7 | } 8 | 9 | th { 10 | text-align: center; 11 | } -------------------------------------------------------------------------------- /JerseyServices/Config/algr-input.ini: -------------------------------------------------------------------------------- 1 | [coordination] 2 | IP = http://127.0.0.1:3000 3 | [info] 4 | name = Algorithm Input Service for R 5 | type = AlgInput-R 6 | description = Algorithm Input Service for R 7 | ip = http://127.0.0.1 8 | port = 8081 -------------------------------------------------------------------------------- /JerseyServices/Config/jersey-testing.ini: -------------------------------------------------------------------------------- 1 | [coordination] 2 | IP = http://127.0.0.1:3000 3 | [info] 4 | name = All Jersey Services 5 | type = JerseyTesting 6 | description = All Jersey Services (for testing) 7 | ip = http://127.0.0.1 8 | port = 8085 -------------------------------------------------------------------------------- /JerseyServices/Config/data-input.ini: -------------------------------------------------------------------------------- 1 | [coordination] 2 | IP = http://127.0.0.1:3000 3 | [info] 4 | name = Data Input Service for CSV Format 5 | type = DataInput-CSV 6 | description = Data Input Service for CSV Format 7 | ip = http://127.0.0.1 8 | port = 8082 -------------------------------------------------------------------------------- /CoordinationService/routes/users.js: -------------------------------------------------------------------------------- 1 | var express = require('express'); 2 | var router = express.Router(); 3 | 4 | /* GET users listing. */ 5 | router.get('/', function(req, res, next) { 6 | res.send('respond with a resource'); 7 | }); 8 | 9 | module.exports = router; 10 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/Utilities/LogType.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.Utilities; 2 | 3 | /** 4 | * Contains the types of log messages. 5 | */ 6 | public enum LogType { 7 | Error, 8 | Exception, 9 | Information, 10 | Request, 11 | Response 12 | } 13 | -------------------------------------------------------------------------------- /CoordinationService/Readme.txt: -------------------------------------------------------------------------------- 1 | To run this: 2 | - "cd" into the ./express folder 3 | - type the command "npm start" 4 | 5 | To edit and auto-update this: 6 | - install "nodemon" using npm 7 | - "cd" into the ./express folder 8 | - type the command "nodemon npm start" 9 | 10 | To access the web app: 11 | http://127.0.0.1:3000 -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/train/task/TaskType.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.train.task; 2 | 3 | /** 4 | * Enum class for Task Type 5 | * TRAIN 6 | * PREDICATE 7 | */ 8 | public enum TaskType { 9 | TRAIN, // Train and get a model 10 | PREDICATE // Run a predication based on model, dataset and algorithm 11 | } -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/train/task/TaskManager.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.train.task; 2 | 3 | /** 4 | * Interface for task manager 5 | */ 6 | public interface TaskManager { 7 | 8 | boolean submitTask(BaseTaskInfo taskInfo); 9 | boolean cancelTask(String taskId); 10 | boolean cancelTaskByExperimentName(String experimentName); 11 | 12 | } 13 | -------------------------------------------------------------------------------- /JerseyServices/launch_scripts/run_jersey_all.sh: -------------------------------------------------------------------------------- 1 | #! /bin/bash 2 | 3 | # java -jar ../target/jersey-service-1.0.jar ../Config/jersey-all.ini 4 | java -jar ../target/jersey-service-1.0.jar ../Config/algr-input.ini & 5 | java -jar ../target/jersey-service-1.0.jar ../Config/data-input.ini & 6 | java -jar ../target/jersey-service-1.0.jar ../Config/training.ini & 7 | java -jar ../target/jersey-service-1.0.jar ../Config/execution.ini & -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/dao/OpLog.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.dao; 2 | 3 | import org.javalite.activejdbc.Model; 4 | import org.javalite.activejdbc.annotations.DbName; 5 | import org.javalite.activejdbc.annotations.Table; 6 | /** 7 | * class used to manipulate log database using ActiveJDBC ORM 8 | */ 9 | @DbName("logdatabase") 10 | @Table("`operations_log`") 11 | public class OpLog extends Model{ 12 | } 13 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/dao/ErrorLog.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.dao; 2 | 3 | import org.javalite.activejdbc.Model; 4 | import org.javalite.activejdbc.annotations.DbName; 5 | import org.javalite.activejdbc.annotations.Table; 6 | /** 7 | * class used to manipulate log database using ActiveJDBC ORM 8 | */ 9 | @DbName("logdatabase") 10 | @Table("`errors_log`") 11 | public class ErrorLog extends Model{ 12 | } 13 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/dao/Task.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.dao; 2 | 3 | import org.javalite.activejdbc.Model; 4 | import org.javalite.activejdbc.annotations.BelongsTo; 5 | import org.javalite.activejdbc.annotations.DbName; 6 | 7 | @DbName("ConfigurationDatabase") 8 | @BelongsTo(parent = PredictionModel.class, foreignKeyName = "model_id") 9 | public class Task extends Model { 10 | 11 | /** 12 | * Get task type from database 13 | * @return 14 | */ 15 | public String getTaskType(){ 16 | return getString("type"); 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/Utilities/Constants.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.Utilities; 2 | 3 | public class Constants { 4 | private Constants(){} // restrict initialization 5 | 6 | // Constants operating with service types 7 | public static final String TYPE_DATASET_INPUT= "DataInput"; 8 | public static final String TYPE_ALGR_INPUT = "AlgrInput"; 9 | public static final String TYPE_EXECUTION = "Execution"; 10 | public static final String TYPE_TRAINING = "Training"; 11 | 12 | // logging cache number 13 | public static final int MAX_LOG_CACHE_NUM = 20; 14 | } 15 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/train/task/interfaces/OnTaskUpdateListener.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.train.task.interfaces; 2 | 3 | /** 4 | * Listener for invoking callback from Task Runner 5 | */ 6 | public interface OnTaskUpdateListener { 7 | void onInit(String taskId); // when a task has been registered in the manager and not assigned a thread 8 | void onInitialized(String taskId); 9 | void onStarted(String taskId); 10 | void onCancelled(String taskId); 11 | void onSuccess(String taskId); 12 | void onError(Throwable ex, String taskId, String errorMessage); 13 | } 14 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/dao/Experiment.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.dao; 2 | 3 | import org.javalite.activejdbc.Model; 4 | import org.javalite.activejdbc.annotations.DbName; 5 | /** 6 | * class used to manipulate database using ActiveJDBC ORM 7 | */ 8 | @DbName("ConfigurationDatabase") 9 | public class Experiment extends Model { 10 | private volatile String experimentName; 11 | 12 | public String getExperimentName() { 13 | if(null == experimentName){ 14 | experimentName = getString("experiment_name"); 15 | } 16 | return experimentName; 17 | } 18 | } 19 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/train/task/TrainingTaskInfo.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.train.task; 2 | 3 | /** 4 | * Information about a training task 5 | */ 6 | public class TrainingTaskInfo extends BaseTaskInfo { 7 | 8 | /** 9 | * Input workspace and other path information 10 | * make sure they are valid before constructing 11 | * the task info object 12 | */ 13 | public TrainingTaskInfo(){ 14 | super(TaskType.TRAIN); 15 | } 16 | 17 | public TrainingTaskInfo(String name, String language, ResourceInfo resInfo){ 18 | super(TaskType.TRAIN, name, language, resInfo); 19 | } 20 | 21 | } 22 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/train/task/ExecutionTaskInfo.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.train.task; 2 | 3 | /** 4 | * Information about an execution task 5 | */ 6 | public class ExecutionTaskInfo extends BaseTaskInfo{ 7 | /** 8 | * Input workspace and other path information 9 | * make sure they are valid before constructing 10 | * the task info object 11 | */ 12 | public ExecutionTaskInfo(){ 13 | super(TaskType.PREDICATE); 14 | } 15 | 16 | public ExecutionTaskInfo(String name, String language, ResourceInfo resInfo){ 17 | super(TaskType.PREDICATE, name, language, resInfo); 18 | } 19 | 20 | } 21 | 22 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/DTO/StatusInfo.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.DTO; 2 | 3 | /** 4 | * Status sent to the coordination service by the heartbeat. 5 | */ 6 | public class StatusInfo { 7 | private String IP; 8 | private String Status; 9 | 10 | public StatusInfo(String ip, String status) { 11 | this.IP = ip; 12 | this.Status = status; 13 | } 14 | 15 | public String getIP() { 16 | return IP; 17 | } 18 | 19 | public void setIP(String IP) { 20 | this.IP = IP; 21 | } 22 | 23 | public String getStatus() { 24 | return Status; 25 | } 26 | 27 | public void setStatus(String status) { 28 | this.Status = status; 29 | } 30 | 31 | } 32 | -------------------------------------------------------------------------------- /CoordinationService/routes/dbUtilities.js: -------------------------------------------------------------------------------- 1 | var express = require('express'); 2 | var mysql = require('mysql'); 3 | var config = require('../config/dbConfig.json'); 4 | 5 | /* Create DB connection */ 6 | createDbConnection = function () { 7 | var connection = mysql.createConnection({ 8 | host: config.dbhost, 9 | user: config.dbuser, 10 | password: config.dbpassword, 11 | database: config.dbname 12 | }); 13 | return connection; 14 | } 15 | 16 | /* Truncates/pads numbers to two digits */ 17 | var twoDigits = function twoDigits(d) { 18 | if (d) { 19 | if (0 <= d && d < 10) return "0" + d.toString(); 20 | if (-10 < d && d < 0) return "-0" + (-1 * d).toString(); 21 | return d.toString(); 22 | } else 23 | return null; 24 | } -------------------------------------------------------------------------------- /CoordinationService/startup.js: -------------------------------------------------------------------------------- 1 | var express = require('express'); 2 | var router = express.Router(); 3 | var dbUtilities = require('./routes/dbUtilities'); 4 | var log = require('./routes/logUtilities'); 5 | 6 | //Connect to DB 7 | var connection = createDbConnection(); 8 | connection.connect(); 9 | var sql = 'delete from connected_services'; 10 | logMessage(false, log.operationType.QueryData, new Date(), sql); 11 | //Return all services 12 | connection.query(sql, function deleteAllConnectedServices(err, result) { 13 | if (err) { 14 | logMessage(true, log.errorType.DBError, new Date(), err); 15 | } else { 16 | logMessage(false, log.operationType.ResponseReceived, new Date(), 'Query successful: connected_services table cleared.'); 17 | } 18 | }); 19 | 20 | module.exports = router; -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/train/Constants.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.train; 2 | 3 | /** 4 | * Constants for training/execution tasks management 5 | */ 6 | public final class Constants { 7 | 8 | private Constants(){} // restrict initialization 9 | //TODO: move the status to a global shared constant class 10 | // Constants operating with service status 11 | public static final String STATUS_RUN = "RUNNING"; 12 | public static final String STATUS_IDLE = "IDLE"; 13 | 14 | // task type columns in task table 15 | public static final String TYPE_TRAIN = "Training"; 16 | public static final String TYPE_EXECUTION = "Execution"; 17 | 18 | // reserved error message length 19 | public static final int ERR_MSG_LENGTH = 500; 20 | } 21 | -------------------------------------------------------------------------------- /CoordinationService/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "express", 3 | "version": "0.0.0", 4 | "private": true, 5 | "scripts": { 6 | "start": "node ./bin/www", 7 | "test": "mocha" 8 | }, 9 | "dependencies": { 10 | "body-parser": "~1.17.1", 11 | "cookie-parser": "~1.4.3", 12 | "debug": "~2.6.3", 13 | "express": "~4.15.2", 14 | "jade": "~1.11.0", 15 | "mocha": "^3.4.2", 16 | "morgan": "~1.8.1", 17 | "mysql": "^2.13.0", 18 | "mysql-to-csv": "^0.9.3", 19 | "node-cmd": "^3.0.0", 20 | "nodemon": "^1.11.0", 21 | "really-need": "^1.9.2", 22 | "request": "^2.81.0", 23 | "serve-favicon": "~2.4.2", 24 | "should": "^11.2.1", 25 | "socket.io": "^2.0.3", 26 | "stylus": "0.54.5", 27 | "supertest": "^3.0.0", 28 | "zip-folder": "^1.0.0" 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /SQL/06-14-2017-V0.1.sql: -------------------------------------------------------------------------------- 1 | /* Create a database named ConfigurationDatabase. */ 2 | CREATE DATABASE IF NOT EXISTS ConfigurationDatabase; 3 | USE ConfigurationDatabase; 4 | 5 | /*Create a table named Formats which stores all system-supported data formats. */ 6 | DROP TABLE IF EXISTS Formats; 7 | CREATE TABLE Formats ( 8 | ID int NOT NULL AUTO_INCREMENT, 9 | format varchar(255) NOT NULL, 10 | PRIMARY KEY (ID) 11 | ); 12 | INSERT INTO Formats (Format) 13 | VALUES ('JSON'); 14 | INSERT INTO Formats (Format) 15 | VALUES ('CSV'); 16 | 17 | /* Create a table named Dataset which contains the information for each uploaded data set.*/ 18 | DROP TABLE IF EXISTS Dataset; 19 | CREATE TABLE Dataset ( 20 | ID int NOT NULL AUTO_INCREMENT, 21 | project_id varchar(255) NOT NULL, 22 | path varchar(255) NOT NULL, 23 | PRIMARY KEY (ID) 24 | ); 25 | 26 | 27 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/dao/Prediction.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.dao; 2 | 3 | import org.javalite.activejdbc.Model; 4 | import org.javalite.activejdbc.annotations.DbName; 5 | import org.javalite.activejdbc.annotations.Table; 6 | 7 | /** 8 | * ORM for Prediction table data 9 | */ 10 | @DbName("ConfigurationDatabase") 11 | @Table("`predictions`") 12 | public class Prediction extends Model{ 13 | public Prediction setPath(String path){ 14 | set("path", path); 15 | return this; 16 | } 17 | 18 | public Prediction setModelName(String name){ 19 | set("name", name); 20 | return this; 21 | } 22 | public Prediction setDescription(String description){ 23 | set("description", description); 24 | return this; 25 | } 26 | 27 | public Prediction setExperimentId(int id){ 28 | set("experiment_id", id); 29 | return this; 30 | } 31 | } 32 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/dao/AlgorithmLanguage.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.dao; 2 | 3 | import org.javalite.activejdbc.Model; 4 | import org.javalite.activejdbc.annotations.DbName; 5 | import org.javalite.activejdbc.annotations.Table; 6 | 7 | /** 8 | * DAO class used for mapping table algorithm_languages in configurationdatabase 9 | */ 10 | @DbName("ConfigurationDatabase") 11 | @Table("algorithm_languages") 12 | public class AlgorithmLanguage extends Model{ 13 | private String language = ""; 14 | 15 | /** 16 | * Get and cache algorithm language 17 | * @return one supported machine learning developing language 18 | */ 19 | public String getLanguage(){ 20 | if(language.equals("")){ 21 | String tmp = getString("language"); 22 | if(null != tmp){ 23 | language = tmp; 24 | } 25 | } 26 | return language; 27 | } 28 | } 29 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/dao/PredictionModel.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.dao; 2 | 3 | import org.javalite.activejdbc.Model; 4 | import org.javalite.activejdbc.annotations.DbName; 5 | import org.javalite.activejdbc.annotations.Table; 6 | 7 | /** 8 | * ORM for model 9 | */ 10 | @DbName("ConfigurationDatabase") 11 | @Table("`models`") 12 | public class PredictionModel extends Model { 13 | 14 | public PredictionModel setPath(String path){ 15 | set("path", path); 16 | return this; 17 | } 18 | 19 | public PredictionModel setModelName(String name){ 20 | set("name", name); 21 | return this; 22 | } 23 | public PredictionModel setDescription(String description){ 24 | set("description", description); 25 | return this; 26 | } 27 | public PredictionModel setExperimentId(int id){ 28 | set("experiment_id", id); 29 | return this; 30 | } 31 | 32 | } 33 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/CORSFilter.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys; 2 | 3 | import java.io.IOException; 4 | import javax.ws.rs.container.ContainerRequestContext; 5 | import javax.ws.rs.container.ContainerResponseContext; 6 | import javax.ws.rs.container.ContainerResponseFilter; 7 | import javax.ws.rs.ext.Provider; 8 | 9 | @Provider 10 | public class CORSFilter implements ContainerResponseFilter { 11 | 12 | @Override 13 | public void filter(ContainerRequestContext request, 14 | ContainerResponseContext response) throws IOException { 15 | response.getHeaders().add("Access-Control-Allow-Origin", "*"); 16 | response.getHeaders().add("Access-Control-Allow-Headers", 17 | "origin, content-type, accept, authorization"); 18 | response.getHeaders().add("Access-Control-Allow-Credentials", "true"); 19 | response.getHeaders().add("Access-Control-Allow-Methods", 20 | "GET, POST, PUT, DELETE, OPTIONS, HEAD"); 21 | } 22 | } -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/dao/Dataset.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.dao; 2 | 3 | import org.javalite.activejdbc.Model; 4 | import org.javalite.activejdbc.annotations.DbName; 5 | 6 | /** 7 | * class used to manipulate database using ActiveJDBC ORM 8 | */ 9 | @DbName("ConfigurationDatabase") 10 | public class Dataset extends Model { 11 | private volatile String dataSetPath; 12 | 13 | /** 14 | * Query once and retur dataset path 15 | * @return dataset file path 16 | */ 17 | public String getDatasetPath(){ 18 | if(null == dataSetPath){ 19 | dataSetPath = getString("path"); 20 | } 21 | return dataSetPath; 22 | } 23 | 24 | public Dataset setDatasetName(String name){ 25 | set("name",name); 26 | return this; 27 | } 28 | 29 | public Dataset setDatasetPath(String path){ 30 | set("path", path); 31 | return this; 32 | } 33 | 34 | public Dataset setDescription(String description){ 35 | set("description", description); 36 | return this; 37 | } 38 | public Dataset setFormat(String format){ 39 | set("format", format); 40 | return this; 41 | } 42 | 43 | } 44 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/DTO/DbConfigInfo.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.DTO; 2 | 3 | /** 4 | * Created by rachel on 6/23/17. 5 | */ 6 | public class DbConfigInfo { 7 | private String host; 8 | private String userName; 9 | private String password; 10 | private String port; 11 | private String dbName; 12 | 13 | 14 | public String getHost() { 15 | return host; 16 | } 17 | 18 | public void setHost(String host) { 19 | this.host = host; 20 | } 21 | 22 | public String getUserName() { 23 | return userName; 24 | } 25 | 26 | public void setUserName(String userName) { 27 | this.userName = userName; 28 | } 29 | 30 | public String getPassword() { 31 | return password; 32 | } 33 | 34 | public void setPassword(String password) { 35 | this.password = password; 36 | } 37 | 38 | public String getPort() { 39 | return port; 40 | } 41 | 42 | public void setPort(String port) { 43 | this.port = port; 44 | } 45 | 46 | public String getDbName() { 47 | return dbName; 48 | } 49 | 50 | public void setDbName(String dbName) { 51 | this.dbName = dbName; 52 | } 53 | } 54 | -------------------------------------------------------------------------------- /CoordinationService/routes/index.js: -------------------------------------------------------------------------------- 1 | var express = require('express'); 2 | var router = express.Router(); 3 | var path = require('path'); 4 | var utilities = require('./dbUtilities'); 5 | 6 | /* List of active services. */ 7 | router.get('/', function routeRoot(req, res, next) { 8 | res.sendFile(path.join(__dirname + '/../views/monitoring.html')); 9 | }); 10 | 11 | router.get('/monitoring', function routeRoot(req, res, next) { 12 | res.sendFile(path.join(__dirname + '/../views/monitoring.html')); 13 | }); 14 | 15 | router.get('/algorithms', function routeRoot(req, res, next) { 16 | res.sendFile(path.join(__dirname + '/../views/algorithms.html')); 17 | }); 18 | 19 | router.get('/datasets', function routeRoot(req, res, next) { 20 | res.sendFile(path.join(__dirname + '/../views/datasets.html')); 21 | }); 22 | 23 | router.get('/experiments', function routeRoot(req, res, next) { 24 | res.sendFile(path.join(__dirname + '/../views/experiments.html')); 25 | }); 26 | 27 | router.get('/models', function routeRoot(req, res, next) { 28 | res.sendFile(path.join(__dirname + '/../views/models.html')); 29 | }); 30 | 31 | router.get('/tasks', function routeRoot(req, res, next) { 32 | res.sendFile(path.join(__dirname + '/../views/tasks.html')); 33 | }); 34 | 35 | module.exports = router; -------------------------------------------------------------------------------- /CoordinationService/routes/models.js: -------------------------------------------------------------------------------- 1 | var express = require('express'); 2 | var router = express.Router(); 3 | var path = require('path'); 4 | var utilities = require('./dbUtilities'); 5 | var log = require('./logUtilities'); 6 | /* gets the models list */ 7 | router.get('/list', function getModels(req, res, next) { 8 | //Connect to DB 9 | var connection = createDbConnection(); 10 | connection.connect(); 11 | var query = 'SELECT models.id as id, models.name as name, models.description as description, models.path as path, experiments.experiment_name as experiment_name FROM models join experiments on models.experiment_id = experiments.id'; 12 | logMessage(false, log.operationType.QueryData, new Date(), query); 13 | //Return all services 14 | var results = connection.query(query, function getAllModels(err, rows, fields) { 15 | if (err) { // pass the err to error handler 16 | logMessage(true, log.errorType.DBError, new Date(), err); 17 | err.source = 'mysql'; // add error source for tracing 18 | err.status = 500; 19 | next(err); 20 | } 21 | else { 22 | logMessage(false, log.operationType.ResponseReceived, new Date(), "Received the models list"); 23 | } 24 | res.json({ models: rows }); 25 | }); 26 | 27 | //Closing connection 28 | connection.end(); 29 | }); 30 | 31 | module.exports = router; -------------------------------------------------------------------------------- /CoordinationService/PostmanQueries/Skeleton.postman_collection.json: -------------------------------------------------------------------------------- 1 | { 2 | "variables": [], 3 | "info": { 4 | "name": "Skeleton", 5 | "_postman_id": "aeeeddb9-0d91-0a4d-c976-404f3a50f2ee", 6 | "description": "", 7 | "schema": "https://schema.getpostman.com/json/collection/v2.0.0/collection.json" 8 | }, 9 | "item": [ 10 | { 11 | "name": "RegisterRequest", 12 | "request": { 13 | "url": "http://127.0.0.1:3000/registerService", 14 | "method": "POST", 15 | "header": [ 16 | { 17 | "key": "Content-Type", 18 | "value": "application/json", 19 | "description": "" 20 | } 21 | ], 22 | "body": { 23 | "mode": "raw", 24 | "raw": "{\n\t\"ip\":\"127.0.0.36\",\"type\":\"Input\",\"name\":\"Input Service\",\"description\":\"My service yo!\",\"status\":\"On!\"\n}" 25 | }, 26 | "description": "" 27 | }, 28 | "response": [] 29 | }, 30 | { 31 | "name": "SetStatus", 32 | "request": { 33 | "url": "http://127.0.0.1:3000/setStatus", 34 | "method": "POST", 35 | "header": [ 36 | { 37 | "key": "Content-Type", 38 | "value": "application/json", 39 | "description": "" 40 | } 41 | ], 42 | "body": { 43 | "mode": "raw", 44 | "raw": "{\n\t\"ip\":\"127.0.0.20\",\"status\":\"Pending\"\n}" 45 | }, 46 | "description": "" 47 | }, 48 | "response": [] 49 | } 50 | ] 51 | } -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/Utilities/ConfigurationHelper.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.Utilities; 2 | 3 | import com.chaosmonkeys.DTO.ConfigInfo; 4 | import org.ini4j.Wini; 5 | import java.io.File; 6 | import java.io.IOException; 7 | 8 | /** 9 | * Utility functions for loading configuration from file. 10 | */ 11 | public class ConfigurationHelper { 12 | /** 13 | * load service information 14 | * 15 | * @param filename 16 | * @throws IOException 17 | */ 18 | public static ConfigInfo loadBasicInfo(String filename) { 19 | // no exception handling here 20 | Wini ini = null; 21 | try { 22 | ini = new Wini(new File(filename)); 23 | } catch (IOException e) { 24 | Logger.SaveLog(LogType.Exception, "EXCEPTION: Cannot read configuration file "+ filename); 25 | } 26 | ConfigInfo basicInfo = new ConfigInfo(); 27 | basicInfo.setServiceName(ini.get("info", "name", String.class)); 28 | basicInfo.setServiceType(ini.get("info", "type", String.class)); 29 | basicInfo.setServiceDescription(ini.get("info", "description", String.class)); 30 | basicInfo.setCoordinationIP(ini.get("coordination", "IP", String.class)); 31 | basicInfo.setServiceIP(ini.get("info", "ip", String.class)); 32 | basicInfo.setServicePort(ini.get("info", "port", String.class)); 33 | return basicInfo; 34 | } 35 | } 36 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/Utilities/DbConfigurationHelper.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.Utilities; 2 | 3 | import com.chaosmonkeys.DTO.ConfigInfo; 4 | import com.chaosmonkeys.DTO.DbConfigInfo; 5 | import org.ini4j.Wini; 6 | import java.io.File; 7 | import java.io.IOException; 8 | 9 | /** 10 | * Configuration Helper for reading database config, including Host, Database name, username 11 | * password and port number 12 | * 13 | * Rachel 14 | */ 15 | public class DbConfigurationHelper { 16 | /** 17 | * load db connection information 18 | * 19 | * @param filename 20 | * @throws IOException 21 | */ 22 | public static DbConfigInfo loadBasicInfo(String filename) { 23 | 24 | Wini ini = null; 25 | try { 26 | ini = new Wini(new File(filename)); 27 | } catch (IOException e) { 28 | Logger.SaveLog(LogType.Exception, "EXCEPTION: Cannot read database configuration file "+ filename); 29 | } 30 | DbConfigInfo basicInfo = new DbConfigInfo(); 31 | basicInfo.setHost(ini.get("info", "host", String.class)); 32 | basicInfo.setDbName(ini.get("info", "dbName", String.class)); 33 | basicInfo.setUserName(ini.get("info", "userName", String.class)); 34 | basicInfo.setPassword(ini.get("info", "password", String.class)); 35 | basicInfo.setPort(ini.get("info", "port", String.class)); 36 | return basicInfo; 37 | } 38 | 39 | } 40 | 41 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/DTO/RegistrationInfo.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.DTO; 2 | 3 | /** 4 | * Information sent to the Coordination Service when registering. 5 | */ 6 | public class RegistrationInfo { 7 | private String IP; 8 | private String Type; 9 | private String Name; 10 | private String Description; 11 | private String Status; 12 | 13 | public RegistrationInfo() { 14 | 15 | } 16 | 17 | public RegistrationInfo(String IP, String type, String name, String description) { 18 | this.IP = IP; 19 | this.Type = type; 20 | this.Name = name; 21 | this.Description = description; 22 | } 23 | 24 | public String getIP() { 25 | return IP; 26 | } 27 | 28 | public void setIP(String IP) { 29 | this.IP = IP; 30 | } 31 | 32 | public String getType() { 33 | return Type; 34 | } 35 | 36 | public void setType(String type) { 37 | this.Type = type; 38 | } 39 | 40 | public String getName() { 41 | return Name; 42 | } 43 | 44 | public void setName(String name) { 45 | this.Name = name; 46 | } 47 | 48 | public String getDescription() { 49 | return Description; 50 | } 51 | 52 | public void setDescription(String description) { 53 | this.Description = description; 54 | } 55 | 56 | public String getStatus() { 57 | return Status; 58 | } 59 | 60 | public void setStatus(String status) { 61 | Status = status; 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/DTO/BasicResponse.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.DTO; 2 | 3 | /** 4 | * BasicResponse DTO for JSON response with only one val 5 | * such as "{response : OK}" and "{response : ERROR}" 6 | */ 7 | public class BasicResponse { 8 | //Response field 9 | private final String response; 10 | 11 | private BasicResponse(BasicResponseBuilder builder) { 12 | this.response = builder.response; 13 | } 14 | 15 | public BasicResponse() { 16 | BasicResponse bResponse = new BasicResponse.BasicResponseBuilder().build(); 17 | this.response = bResponse.getResponse(); 18 | } 19 | 20 | public BasicResponse(String response) { 21 | BasicResponse bResponse = new BasicResponse.BasicResponseBuilder().build(); 22 | this.response = bResponse.getResponse(); 23 | } 24 | 25 | public String getResponse() { 26 | return response; 27 | } 28 | 29 | public static class BasicResponseBuilder { 30 | private String response; 31 | 32 | public BasicResponseBuilder response(String response) { 33 | this.response = response; 34 | return this; 35 | } 36 | 37 | public BasicResponseBuilder responseOK() { 38 | this.response = "OK"; 39 | return this; 40 | } 41 | 42 | public BasicResponseBuilder responseError() { 43 | this.response = "ERROR"; 44 | return this; 45 | } 46 | 47 | public BasicResponse build() { 48 | return new BasicResponse(this); 49 | } 50 | } 51 | } 52 | 53 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/train/dto/ExecutionServiceStatus.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.train.dto; 2 | 3 | /** 4 | * DTO as heartbeats message 5 | */ 6 | public class ExecutionServiceStatus { 7 | private String IP; 8 | private String Type; 9 | private String Name; 10 | private String Description; 11 | private String status; 12 | 13 | public ExecutionServiceStatus(String IP, String status) { 14 | this.IP = IP; 15 | this.status = status; 16 | } 17 | 18 | public ExecutionServiceStatus(String IP, String type, String name, String description, String status) { 19 | this.IP = IP; 20 | Type = type; 21 | Name = name; 22 | Description = description; 23 | this.status = status; 24 | } 25 | 26 | public String getIP() { 27 | return IP; 28 | } 29 | 30 | public void setIP(String IP) { 31 | this.IP = IP; 32 | } 33 | 34 | public String getStatus() { 35 | return status; 36 | } 37 | 38 | public void setStatus(String status) { 39 | this.status = status; 40 | } 41 | 42 | public String getType() { 43 | return Type; 44 | } 45 | 46 | public void setType(String type) { 47 | Type = type; 48 | } 49 | 50 | public String getName() { 51 | return Name; 52 | } 53 | 54 | public void setName(String name) { 55 | Name = name; 56 | } 57 | 58 | public String getDescription() { 59 | return Description; 60 | } 61 | 62 | public void setDescription(String description) { 63 | Description = description; 64 | } 65 | } 66 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/DTO/ConfigInfo.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.DTO; 2 | 3 | /** 4 | * DTO for basic info loaded from the configuration file. 5 | */ 6 | public class ConfigInfo { 7 | private String serviceName; 8 | private String serviceType; 9 | private String serviceDescription; 10 | private String coordinationIP; 11 | private String serviceIP; 12 | private String servicePort; 13 | 14 | public String getServiceName() { 15 | return serviceName; 16 | } 17 | 18 | public void setServiceName(String serviceName) { 19 | this.serviceName = serviceName; 20 | } 21 | 22 | public String getServiceType() { 23 | return serviceType; 24 | } 25 | 26 | public void setServiceType(String serviceType) { 27 | this.serviceType = serviceType; 28 | } 29 | 30 | public String getServiceDescription() { 31 | return serviceDescription; 32 | } 33 | 34 | public void setServiceDescription(String serviceDescription) { 35 | this.serviceDescription = serviceDescription; 36 | } 37 | 38 | public String getCoordinationIP() { 39 | return coordinationIP; 40 | } 41 | 42 | public void setCoordinationIP(String coordinationIP) { 43 | this.coordinationIP = coordinationIP; 44 | } 45 | 46 | public String getServiceIP() { 47 | return serviceIP; 48 | } 49 | 50 | public void setServiceIP(String serviceIP) { 51 | this.serviceIP = serviceIP; 52 | } 53 | 54 | public String getServicePort() { 55 | return servicePort; 56 | } 57 | 58 | public void setServicePort(String servicePort) { 59 | this.servicePort = servicePort; 60 | } 61 | } 62 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/train/task/TaskState.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.train.task; 2 | 3 | import java.util.Arrays; 4 | import java.util.List; 5 | 6 | /** 7 | * The enumeration for the current execution-state of a {@link AbsTask}. 8 | * 9 | */ 10 | public enum TaskState { 11 | IDLE(0), INITIALIZING(1), INITIALIZED(2), STARTED(3), CANCELLED(4), SUCCESS(5), ERROR(6); 12 | 13 | private final int value; 14 | private static List strValList = Arrays.asList("IDLE", "INITIALIZING","INITIALIZED", "STARTED", "CANCELLED", "SUCCESS", "ERROR"); 15 | 16 | TaskState(int value){ 17 | this.value = value; 18 | } 19 | public int value(){ 20 | return this.value; 21 | } 22 | 23 | /** 24 | * Obtain string value 25 | * @return 26 | */ 27 | public String StringValue(){ 28 | return strValList.get(this.value); 29 | } 30 | 31 | @Override 32 | public String toString() { 33 | return StringValue(); 34 | } 35 | 36 | /** 37 | * Judge whether a state string represented Finished 38 | * @param status 39 | * @return 40 | */ 41 | public static boolean isFinished(String status){ 42 | int idx = strValList.indexOf(status); 43 | if(idx > 3){ 44 | return true; 45 | } 46 | return false; 47 | } 48 | 49 | /** 50 | * Determine if a string is a valid status string 51 | * @return 52 | */ 53 | public static boolean isValidStatus(String str){ 54 | if(null != str && 0 != str.length()){ 55 | if(strValList.contains(str)){ 56 | return true; 57 | } 58 | } 59 | return false; 60 | } 61 | } -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/train/dto/ExperimentDto.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.train.dto; 2 | 3 | /** 4 | * DTO Used for consuming json from POST request 5 | * Execution Resource 6 | */ 7 | public class ExperimentDto { 8 | // member variables 9 | private int id = -1; 10 | private int project_id = -1; 11 | private int task_id = -1; 12 | private String experiment_name; 13 | 14 | /** 15 | * constructor 16 | */ 17 | // Must have no-argument constructor 18 | public ExperimentDto(){ 19 | } 20 | 21 | public ExperimentDto(String experiment_name){ 22 | this.experiment_name = experiment_name; 23 | } 24 | public ExperimentDto(int id, int project_id, int task_id, String experiment_name) { 25 | this.id = id; 26 | this.project_id = project_id; 27 | this.task_id = task_id; 28 | this.experiment_name = experiment_name; 29 | } 30 | 31 | 32 | /** 33 | * Getter and Setter 34 | * @return 35 | */ 36 | public int getId() { 37 | return id; 38 | } 39 | 40 | public void setId(int id) { 41 | this.id = id; 42 | } 43 | 44 | public int getProject_id() { 45 | return project_id; 46 | } 47 | 48 | public void setProject_id(int project_id) { 49 | this.project_id = project_id; 50 | } 51 | 52 | public int getTask_id() { 53 | return task_id; 54 | } 55 | 56 | public void setTask_id(int task_id) { 57 | this.task_id = task_id; 58 | } 59 | 60 | public String getExperiment_name() { 61 | return experiment_name; 62 | } 63 | 64 | public void setExperiment_name(String experiment_name) { 65 | this.experiment_name = experiment_name; 66 | } 67 | } 68 | -------------------------------------------------------------------------------- /CoordinationService/test/dbUtilitiesTest.js: -------------------------------------------------------------------------------- 1 | var utilities = require('../routes/dbUtilities'); 2 | var should = require('should'); 3 | 4 | describe('twoDigits() in DB Utilities', function () { 5 | it('should pad single-digit numbers to two digits', function () { 6 | var pos = 5; 7 | var neg = -5; 8 | var zero = 0; 9 | 10 | var posRet = utilities.twoDigits(pos); 11 | var negRet = utilities.twoDigits(neg); 12 | var zeroRet = utilities.twoDigits(zero); 13 | 14 | posRet.should.equal('05'); 15 | negRet.should.equal('-05'); 16 | zeroRet.should.equal('00'); 17 | }); 18 | 19 | it('should cast non-single-digit numbers to string', function () { 20 | var posTwoDigits = 10; 21 | var negTwoDigits = -10; 22 | 23 | var posThreeDigits = 100; 24 | var negThreeDigits = -100; 25 | 26 | var posTwoDigitsRet = utilities.twoDigits(posTwoDigits); 27 | var negTwoDigitsRet = utilities.twoDigits(negTwoDigits); 28 | var posThreeDigitsRet = utilities.twoDigits(posThreeDigits); 29 | var negThreeDigitsRet = utilities.twoDigits(negThreeDigits); 30 | 31 | posTwoDigitsRet.should.equal('10'); 32 | negTwoDigitsRet.should.equal('-10'); 33 | posThreeDigitsRet.should.equal('100'); 34 | negThreeDigitsRet.should.equal('-100'); 35 | }); 36 | 37 | it('should return undefined when the parameter is not valid', function () { 38 | var str = 'str'; 39 | var nullVal = null; 40 | var undefinedVal = undefined; 41 | 42 | var strRet = utilities.twoDigits(str); 43 | var nullValRet = utilities.twoDigits(null); 44 | var undefinedValRet = utilities.twoDigits(undefined); 45 | 46 | (typeof strRet).should.equal('undefined'); 47 | (typeof nullValRet).should.equal('undefined'); 48 | (typeof undefinedValRet).should.equal('undefined'); 49 | }); 50 | }); 51 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/train/task/BaseTaskInfo.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.train.task; 2 | 3 | import com.chaosmonkeys.Utilities.StringUtils; 4 | 5 | import java.io.File; 6 | 7 | /** 8 | * Base task information class 9 | * training or predication task should inherit this class 10 | * to specify the basic information of a task 11 | * 12 | * Jiawei Li 13 | */ 14 | 15 | public class BaseTaskInfo { 16 | 17 | /** Identifier for a task **/ 18 | public final String TASK_ID; 19 | 20 | public final TaskType taskType; 21 | 22 | protected String experimentName; 23 | protected String experimentLanguage; // Python or R 24 | 25 | private ResourceInfo resourceInfo; 26 | /** 27 | * Make sure subclass invoke this super() constructor 28 | */ 29 | public BaseTaskInfo(TaskType type){ 30 | this.TASK_ID = StringUtils.getUUID(); 31 | this.taskType = type; 32 | } 33 | 34 | public BaseTaskInfo(TaskType type, String experimentName, String experimentLanguage, ResourceInfo resInfo){ 35 | this(type); // assign an UUID as identifier 36 | this.experimentName = experimentName; 37 | this.experimentLanguage = experimentLanguage; 38 | this.resourceInfo = resInfo; 39 | } 40 | 41 | 42 | /** 43 | * Obtain the identifier of the task 44 | * @return task id (an instance of UUID string) 45 | */ 46 | public String getTaskId(){ 47 | return this.TASK_ID; 48 | } 49 | 50 | public ResourceInfo getResourceInfo() { 51 | return resourceInfo; 52 | } 53 | 54 | public void setResourceInfo(ResourceInfo resourceInfo) { 55 | this.resourceInfo = resourceInfo; 56 | } 57 | 58 | public String getExperimentName(){ 59 | return experimentName; 60 | } 61 | public String getExperimentLanguage(){ 62 | return experimentLanguage; 63 | } 64 | } 65 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/datasetinputservice/DTO/DatasetInputServiceStatusInfo.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.datasetinputservice.DTO; 2 | 3 | /** 4 | * DTO for DatasetInputService, 5 | * including IP, name, description and InputServiceStatus 6 | */ 7 | public class DatasetInputServiceStatusInfo { 8 | private String IP; 9 | private String Type; 10 | private String Name; 11 | private String Description; 12 | private DatasetInputServiceWorkState status; 13 | 14 | public DatasetInputServiceStatusInfo() { 15 | } 16 | 17 | public DatasetInputServiceStatusInfo(String IP, DatasetInputServiceWorkState status) { 18 | this.IP = IP; 19 | this.status = status; 20 | } 21 | 22 | public DatasetInputServiceStatusInfo(String IP, String type, String name, String description, DatasetInputServiceWorkState status) { 23 | this.IP = IP; 24 | Type = type; 25 | Name = name; 26 | Description = description; 27 | this.status = status; 28 | } 29 | 30 | public String getIP() { 31 | return IP; 32 | } 33 | 34 | public void setIP(String IP) { 35 | this.IP = IP; 36 | } 37 | 38 | public String getType() { 39 | return Type; 40 | } 41 | 42 | public void setType(String type) { 43 | this.Type = type; 44 | } 45 | 46 | public String getName() { 47 | return Name; 48 | } 49 | 50 | public void setName(String name) { 51 | this.Name = name; 52 | } 53 | 54 | public String getDescription() { 55 | return Description; 56 | } 57 | 58 | public void setDescription(String description) { 59 | this.Description = description; 60 | } 61 | 62 | public DatasetInputServiceWorkState getStatus() { 63 | return status; 64 | } 65 | 66 | public void setStatus(DatasetInputServiceWorkState status) { 67 | this.status = status; 68 | } 69 | } 70 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/algrinputservice/DTO/AlgorithmInputServiceStatusInfo.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.algrinputservice.DTO; 2 | 3 | /** 4 | * DTO for algorithmInputService heartbeats client, 5 | * including IP, name, description and status 6 | */ 7 | public class AlgorithmInputServiceStatusInfo { 8 | //Basic info of the jersey service 9 | private String IP; 10 | private String Type; 11 | private String Name; 12 | private String Description; 13 | private AlgorithmInputServiceWorkState status; 14 | 15 | 16 | public AlgorithmInputServiceStatusInfo() { 17 | } 18 | 19 | public AlgorithmInputServiceStatusInfo(String IP, AlgorithmInputServiceWorkState status) { 20 | this.IP = IP; 21 | this.status = status; 22 | } 23 | 24 | public AlgorithmInputServiceStatusInfo(String IP, String type, String name, String description, AlgorithmInputServiceWorkState status) { 25 | this.IP = IP; 26 | Type = type; 27 | Name = name; 28 | Description = description; 29 | this.status = status; 30 | } 31 | 32 | public String getIP() { 33 | return IP; 34 | } 35 | 36 | public void setIP(String IP) { 37 | this.IP = IP; 38 | } 39 | 40 | public AlgorithmInputServiceWorkState getStatus() { 41 | return status; 42 | } 43 | 44 | public void setStatus(AlgorithmInputServiceWorkState status) { 45 | this.status = status; 46 | } 47 | 48 | public String getType() { 49 | return Type; 50 | } 51 | 52 | public void setType(String type) { 53 | Type = type; 54 | } 55 | 56 | public String getName() { 57 | return Name; 58 | } 59 | 60 | public void setName(String name) { 61 | Name = name; 62 | } 63 | 64 | public String getDescription() { 65 | return Description; 66 | } 67 | 68 | public void setDescription(String description) { 69 | Description = description; 70 | } 71 | } 72 | -------------------------------------------------------------------------------- /CoordinationService/bin/www: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | /** 4 | * Module dependencies. 5 | */ 6 | 7 | var app = require('../app'); 8 | var debug = require('debug')('express:server'); 9 | var http = require('http'); 10 | 11 | /** 12 | * Get port from environment and store in Express. 13 | */ 14 | 15 | var port = normalizePort(process.env.PORT || '3000'); 16 | app.set('port', port); 17 | 18 | /** 19 | * Create HTTP server. 20 | */ 21 | 22 | var server = http.createServer(app); 23 | 24 | /** 25 | * Listen on provided port, on all network interfaces. 26 | */ 27 | 28 | server.listen(port); 29 | server.on('error', onError); 30 | server.on('listening', onListening); 31 | 32 | /** 33 | * Normalize a port into a number, string, or false. 34 | */ 35 | 36 | function normalizePort(val) { 37 | var port = parseInt(val, 10); 38 | 39 | if (isNaN(port)) { 40 | // named pipe 41 | return val; 42 | } 43 | 44 | if (port >= 0) { 45 | // port number 46 | return port; 47 | } 48 | 49 | return false; 50 | } 51 | 52 | /** 53 | * Event listener for HTTP server "error" event. 54 | */ 55 | 56 | function onError(error) { 57 | if (error.syscall !== 'listen') { 58 | throw error; 59 | } 60 | 61 | var bind = typeof port === 'string' 62 | ? 'Pipe ' + port 63 | : 'Port ' + port; 64 | 65 | // handle specific listen errors with friendly messages 66 | switch (error.code) { 67 | case 'EACCES': 68 | console.error(bind + ' requires elevated privileges'); 69 | process.exit(1); 70 | break; 71 | case 'EADDRINUSE': 72 | console.error(bind + ' is already in use'); 73 | process.exit(1); 74 | break; 75 | default: 76 | throw error; 77 | } 78 | } 79 | 80 | /** 81 | * Event listener for HTTP server "listening" event. 82 | */ 83 | 84 | function onListening() { 85 | var addr = server.address(); 86 | var bind = typeof addr === 'string' 87 | ? 'pipe ' + addr 88 | : 'port ' + addr.port; 89 | debug('Listening on ' + bind); 90 | } 91 | -------------------------------------------------------------------------------- /CoordinationService/public/stylesheets/common.css: -------------------------------------------------------------------------------- 1 | /* Center the loader */ 2 | #uploadingAlgorithm { 3 | position: absolute; 4 | left: 50%; 5 | top: 50%; 6 | z-index: 1; 7 | width: 150px; 8 | height: 150px; 9 | margin: -60px 0 0 -75px; 10 | border: 16px solid #f3f3f3; 11 | border-radius: 50%; 12 | border-top: 16px solid #3498db; 13 | width: 120px; 14 | height: 120px; 15 | -webkit-animation: spin 2s linear infinite; 16 | animation: spin 2s linear infinite; 17 | } 18 | 19 | #uploadingDataset { 20 | position: absolute; 21 | left: 50%; 22 | top: 50%; 23 | z-index: 1; 24 | width: 150px; 25 | height: 150px; 26 | margin: -60px 0 0 -75px; 27 | border: 16px solid #f3f3f3; 28 | border-radius: 50%; 29 | border-top: 16px solid #3498db; 30 | width: 120px; 31 | height: 120px; 32 | -webkit-animation: spin 2s linear infinite; 33 | animation: spin 2s linear infinite; 34 | } 35 | 36 | @-webkit-keyframes spin { 37 | 0% { 38 | -webkit-transform: rotate(0deg); 39 | } 40 | 100% { 41 | -webkit-transform: rotate(360deg); 42 | } 43 | } 44 | 45 | @keyframes spin { 46 | 0% { 47 | transform: rotate(0deg); 48 | } 49 | 100% { 50 | transform: rotate(360deg); 51 | } 52 | } 53 | /* Add animation to "page content" */ 54 | 55 | .animate-bottom { 56 | position: relative; 57 | -webkit-animation-name: animatebottom; 58 | -webkit-animation-duration: 1s; 59 | animation-name: animatebottom; 60 | animation-duration: 1s 61 | } 62 | 63 | @-webkit-keyframes animatebottom { 64 | from { 65 | bottom: -100px; 66 | opacity: 0 67 | } 68 | to { 69 | bottom: 0px; 70 | opacity: 1 71 | } 72 | } 73 | 74 | @keyframes animatebottom { 75 | from { 76 | bottom: -100px; 77 | opacity: 0 78 | } 79 | to { 80 | bottom: 0; 81 | opacity: 1 82 | } 83 | } 84 | 85 | #myDiv { 86 | display: none; 87 | text-align: center; 88 | } -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled class file 2 | *.class 3 | 4 | # Log file 5 | *.log 6 | 7 | # BlueJ files 8 | *.ctxt 9 | 10 | # Mobile Tools for Java (J2ME) 11 | .mtj.tmp/ 12 | 13 | # Package Files # 14 | *.jar 15 | *.war 16 | *.ear 17 | *.tar.gz 18 | *.rar 19 | 20 | # virtual machine crash logs, see http://www.java.com/en/download/help/error_hotspot.xml 21 | hs_err_pid* 22 | 23 | # Logs 24 | logs 25 | *.log 26 | npm-debug.log* 27 | yarn-debug.log* 28 | yarn-error.log* 29 | 30 | # Runtime data 31 | pids 32 | *.pid 33 | *.seed 34 | *.pid.lock 35 | 36 | # Directory for instrumented libs generated by jscoverage/JSCover 37 | lib-cov 38 | 39 | # Coverage directory used by tools like istanbul 40 | coverage 41 | 42 | # nyc test coverage 43 | .nyc_output 44 | 45 | # Grunt intermediate storage (http://gruntjs.com/creating-plugins#storing-task-files) 46 | .grunt 47 | 48 | # Bower dependency directory (https://bower.io/) 49 | bower_components 50 | 51 | # node-waf configuration 52 | .lock-wscript 53 | 54 | # Compiled binary addons (http://nodejs.org/api/addons.html) 55 | build/Release 56 | 57 | # Dependency directories 58 | node_modules/ 59 | jspm_packages/ 60 | 61 | # Typescript v1 declaration files 62 | typings/ 63 | 64 | # Optional npm cache directory 65 | .npm 66 | 67 | # Optional eslint cache 68 | .eslintcache 69 | 70 | # Optional REPL history 71 | .node_repl_history 72 | 73 | # Output of 'npm pack' 74 | *.tgz 75 | 76 | # Yarn Integrity file 77 | .yarn-integrity 78 | 79 | # dotenv environment variables file 80 | .env 81 | 82 | # AWS credentials 83 | cred/ 84 | 85 | # Terraform AWS access key pairs 86 | *.tf 87 | 88 | # Built target 89 | target/ 90 | 91 | # Maven Dependency Reduced Pom 92 | dependency-reduced-pom.xml 93 | 94 | # Maven Archiver 95 | maven-archiver/ 96 | 97 | # IntelliJIDEA Folder 98 | .idea/ 99 | 100 | # VS Code Config Folder 101 | .vscode/ 102 | 103 | # IML File 104 | basic-service.iml 105 | 106 | # Package Lock File 107 | package-lock.json -------------------------------------------------------------------------------- /JerseyServices/src/test/java/com/chaosmonkeys/utils/StringUtilsContainsTest.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.utils; 2 | 3 | import com.chaosmonkeys.Utilities.StringUtils; 4 | import org.junit.Test; 5 | 6 | import static org.junit.Assert.assertFalse; 7 | import static org.junit.Assert.assertTrue; 8 | 9 | /** 10 | * Testing containIgnoreCase 11 | */ 12 | public class StringUtilsContainsTest { 13 | @Test 14 | public void testContainsIgnoreCase_StringString() { 15 | assertFalse(StringUtils.containsIgnoreCase(null, null)); 16 | 17 | // Null tests 18 | assertFalse(StringUtils.containsIgnoreCase(null, "")); 19 | assertFalse(StringUtils.containsIgnoreCase(null, "a")); 20 | assertFalse(StringUtils.containsIgnoreCase(null, "abc")); 21 | 22 | assertFalse(StringUtils.containsIgnoreCase("", null)); 23 | assertFalse(StringUtils.containsIgnoreCase("a", null)); 24 | assertFalse(StringUtils.containsIgnoreCase("abc", null)); 25 | 26 | // Match len = 0 27 | assertTrue(StringUtils.containsIgnoreCase("", "")); 28 | assertTrue(StringUtils.containsIgnoreCase("a", "")); 29 | assertTrue(StringUtils.containsIgnoreCase("abc", "")); 30 | 31 | // Match len = 1 32 | assertFalse(StringUtils.containsIgnoreCase("", "a")); 33 | assertTrue(StringUtils.containsIgnoreCase("a", "a")); 34 | assertTrue(StringUtils.containsIgnoreCase("abc", "a")); 35 | assertFalse(StringUtils.containsIgnoreCase("", "A")); 36 | assertTrue(StringUtils.containsIgnoreCase("a", "A")); 37 | assertTrue(StringUtils.containsIgnoreCase("abc", "A")); 38 | 39 | // Match len > 1 40 | assertFalse(StringUtils.containsIgnoreCase("", "abc")); 41 | assertFalse(StringUtils.containsIgnoreCase("a", "abc")); 42 | assertTrue(StringUtils.containsIgnoreCase("xabcz", "abc")); 43 | assertFalse(StringUtils.containsIgnoreCase("", "ABC")); 44 | assertFalse(StringUtils.containsIgnoreCase("a", "ABC")); 45 | assertTrue(StringUtils.containsIgnoreCase("xabcz", "ABC")); 46 | } 47 | } 48 | -------------------------------------------------------------------------------- /CoordinationService/routes/datasets.js: -------------------------------------------------------------------------------- 1 | var express = require('express'); 2 | var router = express.Router(); 3 | var path = require('path'); 4 | var utilities = require('./dbUtilities'); 5 | var log = require('./logUtilities'); 6 | 7 | /* gets the list of datasets from the database */ 8 | router.get('/list', function getDatasets(req, res, next) { 9 | //Connect to DB 10 | var connection = createDbConnection(); 11 | connection.connect(); 12 | var query = 'select * from datasets'; 13 | logMessage(false, log.operationType.QueryData, new Date(), query); 14 | //Return all services 15 | var results = connection.query(query, function selectAllDatasets(err, rows, fields) { 16 | if (err) { // pass the err to error handler 17 | logMessage(true, log.errorType.DBError, new Date(), err); 18 | err.source = 'mysql'; // add error source for tracing 19 | err.status = 500; 20 | next(err) 21 | } 22 | else { 23 | logMessage(false, log.operationType.ResponseReceived, new Date(), "Received the datasets list"); 24 | } 25 | res.json({ datasets: rows }); 26 | }); 27 | 28 | //Closing connection 29 | connection.end(); 30 | }); 31 | 32 | /* gets the supported formats */ 33 | router.get('/formats', function getFormats(req, res, next) { 34 | //Connect to DB 35 | var connection = createDbConnection(); 36 | connection.connect(); 37 | var query = 'select format from formats'; 38 | logMessage(false, log.operationType.QueryData, new Date(), query); 39 | //Return all services 40 | var results = connection.query(query, function getAllDatasets(err, rows, fields) { 41 | if (err) { // pass the err to error handler 42 | logMessage(true, log.errorType.DBError, new Date(), err); 43 | err.source = 'mysql'; // add error source for tracing 44 | err.status = 500; 45 | next(err) 46 | } 47 | else { 48 | logMessage(false, log.operationType.ResponseReceived, new Date(), "Received the supported formats list"); 49 | } 50 | res.json({ formats: rows }); 51 | }); 52 | 53 | //Closing connection 54 | connection.end(); 55 | }); 56 | 57 | module.exports = router; 58 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/HeartBeatsClient.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys; 2 | 3 | import com.chaosmonkeys.DTO.StatusInfo; 4 | 5 | import com.chaosmonkeys.Utilities.LogType; 6 | import com.chaosmonkeys.Utilities.Logger; 7 | import org.glassfish.jersey.client.ClientConfig; 8 | import org.glassfish.jersey.filter.LoggingFilter; 9 | 10 | import javax.ws.rs.ProcessingException; 11 | import javax.ws.rs.client.*; 12 | import javax.ws.rs.core.MediaType; 13 | import javax.ws.rs.core.Response; 14 | 15 | /** 16 | * Used to send heartbeats message to coordinationService 17 | * HTTP Method: POST 18 | * Content: JSON -> {IP: String, Status: String} 19 | */ 20 | public class HeartBeatsClient { 21 | private static String coordinationIP; 22 | private static boolean isStart = false; 23 | 24 | public void startSendHeartBeat(String ip) { 25 | if (isStart) { 26 | return; 27 | } 28 | coordinationIP = ip; 29 | Thread thread = new Thread(new SendHeartBeatTask()); 30 | thread.start(); 31 | } 32 | 33 | class SendHeartBeatTask implements Runnable { 34 | 35 | @Override 36 | public void run() { 37 | Client client = ClientBuilder.newClient(new ClientConfig().register(LoggingFilter.class)); 38 | WebTarget webTarget = client.target(coordinationIP).path("setStatus"); // target to the 39 | 40 | Invocation.Builder invocationBuilder = webTarget.request(MediaType.APPLICATION_JSON); 41 | while (true) { 42 | try { 43 | StatusInfo status = new StatusInfo(Launcher.getHostIP(), Launcher.getServiceStatus()); 44 | Response response = invocationBuilder.post(Entity.entity(status, MediaType.APPLICATION_JSON)); 45 | } catch (ProcessingException e) { 46 | Logger.SaveLog(LogType.Exception,"EXCEPTION: Heartbeat Connection Timeout, retrying..."); 47 | } 48 | try { 49 | Thread.sleep(3 * 1000); 50 | } catch (InterruptedException e) { 51 | e.printStackTrace(); 52 | } 53 | } 54 | 55 | } 56 | } 57 | } 58 | -------------------------------------------------------------------------------- /CoordinationService/routes/algorithms.js: -------------------------------------------------------------------------------- 1 | var express = require('express'); 2 | var utilities = require('./dbUtilities'); 3 | var router = express.Router(); 4 | var log = require('./logUtilities'); 5 | 6 | /* gets the supported languages */ 7 | router.get('/languages', function getFormats(req, res, next) { 8 | //Connect to DB 9 | var connection = createDbConnection(); 10 | connection.connect(); 11 | //console.log(log.operationType.Upload); 12 | var query = 'select language from algorithm_languages'; 13 | logMessage(false, log.operationType.QueryData, new Date(), query); 14 | //Return all supported languages 15 | var results = connection.query(query, function getSupportedLanguages(err, rows, fields) { 16 | if (err) { // pass the err to error handler 17 | logMessage(true, log.errorType.DBError, new Date(), err); 18 | connection.end(); 19 | err.source = 'mysql'; // add error source for tracing 20 | err.status = 500; 21 | next(err); 22 | } 23 | else { 24 | logMessage(false, log.operationType.ResponseReceived, new Date(), "Received the supported languages"); 25 | } 26 | res.json({ 27 | languages: rows 28 | }); // rows: [ cols:[] ] 29 | }); 30 | //Closing connection 31 | connection.end(); 32 | }); 33 | 34 | /* gets the list of algorithms from the database */ 35 | router.get('/list', function getAlgorithms(req, res, next) { 36 | //Connect to DB 37 | var connection = createDbConnection(); 38 | connection.connect(); 39 | var query = 'select * from algorithms'; 40 | logMessage(false, log.operationType.QueryData, new Date(), query); 41 | //Return all services 42 | var results = connection.query(query, function selectAllAlgorithms(err, rows, fields) { 43 | if (err) { // pass the err to error handler 44 | logMessage(true, log.errorType.DBError, new Date(), err); 45 | err.source = 'mysql'; // add error source for tracing 46 | err.status = 500; 47 | next(err); 48 | } 49 | else { 50 | logMessage(false, log.operationType.ResponseReceived, new Date(), "Received the algorithms list"); 51 | } 52 | res.json({ algorithms: rows }); 53 | }); 54 | 55 | //Closing connection 56 | connection.end(); 57 | }); 58 | 59 | module.exports = router; 60 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/Utilities/ZipUtils.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.Utilities; 2 | 3 | import java.io.*; 4 | import java.util.zip.ZipEntry; 5 | import java.util.zip.ZipInputStream; 6 | 7 | /** 8 | * Utilities that are used to unzip a zip file 9 | */ 10 | public class ZipUtils { 11 | 12 | public ZipUtils(){ 13 | super(); 14 | } 15 | 16 | /** 17 | * Size of the buffer to read/write data 18 | */ 19 | private static final int BUFFER_SIZE = 4096; 20 | 21 | 22 | /** 23 | * Extracts a zip file specified by the zipFilePath to a directory specified by 24 | * destDirectory (will be created if does not exists) 25 | * @param zipFile 26 | * @param destDir 27 | * @throws IOException 28 | */ 29 | public static void unzip(File zipFile, File destDir) throws IOException { 30 | 31 | if (!destDir.exists()) { 32 | destDir.mkdir(); 33 | } 34 | ZipInputStream zipIn = new ZipInputStream(new FileInputStream(zipFile)); 35 | ZipEntry entry = zipIn.getNextEntry(); 36 | // iterates over entries in the zip file 37 | while (entry != null) { 38 | File zipEntryFile = new File(destDir, entry.getName()); 39 | if (!entry.isDirectory()) { 40 | // if the entry is a file, extracts it 41 | extractFile(zipIn, zipEntryFile); 42 | } else { 43 | // if the entry is a directory, make the directory 44 | zipEntryFile.mkdir(); 45 | } 46 | zipIn.closeEntry(); 47 | entry = zipIn.getNextEntry(); 48 | } 49 | zipIn.close(); 50 | } 51 | 52 | 53 | /** 54 | * Extracts a zip entry (file entry) 55 | * @param zipIn 56 | * @param file 57 | * @throws IOException 58 | */ 59 | private static void extractFile(ZipInputStream zipIn, File file) throws IOException { 60 | BufferedOutputStream bos = new BufferedOutputStream(new FileOutputStream(file)); 61 | byte[] bytesIn = new byte[BUFFER_SIZE]; 62 | int read = 0; 63 | while ((read = zipIn.read(bytesIn)) != -1) { 64 | bos.write(bytesIn, 0, read); 65 | } 66 | bos.close(); 67 | } 68 | 69 | } 70 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/dao/Algorithm.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.dao; 2 | 3 | import org.javalite.activejdbc.Model; 4 | import org.javalite.activejdbc.annotations.DbName; 5 | 6 | /** 7 | * Algorithm entity 8 | */ 9 | @DbName("ConfigurationDatabase") 10 | public class Algorithm extends Model { 11 | private volatile String algorithmPath; 12 | private volatile String algorithmLanguage; 13 | /** 14 | * Query once and return algorithm folder path 15 | * @return algorithm path 16 | */ 17 | public String getAlgorithmPath(){ 18 | if(null == algorithmPath){ 19 | algorithmPath = getString("path"); 20 | } 21 | return algorithmPath; 22 | } 23 | 24 | /** 25 | * Query once and return algorithm language 26 | * @return algorithm language 27 | */ 28 | public String getAlgorithmLanguage() { 29 | if(null == algorithmLanguage){ 30 | algorithmLanguage = getString("language"); 31 | } 32 | return algorithmLanguage; 33 | } 34 | 35 | /** 36 | * Set algorithm name and return itself 37 | * @param name algorithm name 38 | * @return current algorithm object 39 | */ 40 | public Algorithm setAlgorithmName(String name){ 41 | set("name",name); 42 | return this; 43 | } 44 | 45 | /** 46 | * set algorithm path and return itself 47 | * @param path algorithm storage folder path 48 | * @return current algorithm object 49 | */ 50 | public Algorithm setAlgorithmPath(String path){ 51 | set("path",path); 52 | return this; 53 | } 54 | 55 | /** 56 | * set algorithm description 57 | * @param description algorithm description 58 | * @return current algorithm object 59 | */ 60 | public Algorithm setAlgorithmDescription(String description){ 61 | set("description", description); 62 | return this; 63 | } 64 | 65 | /** 66 | * set algorithm development language 67 | * @param language language, such as R, Python 68 | * @return current algorithm object 69 | */ 70 | public Algorithm setAlgorithmLanguage(String language){ 71 | set("language",language); 72 | return this; 73 | } 74 | 75 | } 76 | -------------------------------------------------------------------------------- /CoordinationService/app.js: -------------------------------------------------------------------------------- 1 | var express = require('express'); 2 | var path = require('path'); 3 | var favicon = require('serve-favicon'); 4 | var logger = require('morgan'); 5 | var cookieParser = require('cookie-parser'); 6 | var bodyParser = require('body-parser'); 7 | var stylus = require('stylus'); 8 | var request = require('request'); 9 | var index = require('./routes/index'); 10 | var heartbeat = require('./routes/heartbeat'); 11 | var datasets = require('./routes/datasets'); 12 | var tasks = require('./routes/tasks'); 13 | var users = require('./routes/users'); 14 | var experiments = require('./routes/experiments'); 15 | var models = require('./routes/models'); 16 | var algorithms = require('./routes/algorithms'); 17 | var download = require('./routes/download'); 18 | var dynamicIp = require('./routes/dynamicIp'); 19 | var startup = require('./startup'); 20 | 21 | var app = express(); 22 | 23 | //View engine setup (for errors) 24 | app.set('views', path.join(__dirname, 'views')); 25 | app.set('view engine', 'jade'); 26 | 27 | //Modules 28 | app.use(logger('dev')); 29 | app.use(bodyParser.json()); 30 | app.use(bodyParser.urlencoded({ extended: false })); 31 | app.use(cookieParser()); 32 | app.use(stylus.middleware(path.join(__dirname, 'public'))); 33 | app.use(express.static(path.join(__dirname, 'public'))); 34 | 35 | //Mounting middleware 36 | app.use('/', index); 37 | app.use('/', heartbeat); 38 | app.use('/datasets', datasets); 39 | app.use('/tasks', tasks); 40 | app.use('/users', users); 41 | app.use('/experiments', experiments); 42 | app.use('/models', models); 43 | app.use('/algorithms', algorithms); 44 | app.use('/download', download); 45 | app.use('/dynamicIp', dynamicIp); 46 | app.use('/public', express.static(__dirname + '/public')); 47 | app.use('/views', express.static(__dirname + '/views')); 48 | 49 | //Catch 404 and forward to error handler 50 | app.use(function handleNotFoundError(req, res, next) { 51 | var err = new Error('Not Found'); 52 | err.status = 404; 53 | next(err); 54 | }); 55 | 56 | //Error handler 57 | app.use(function handleError(err, req, res, next) { 58 | // set locals, only providing error in development 59 | res.locals.message = err.message; 60 | res.locals.error = req.app.get('env') === 'development' ? err : {}; 61 | 62 | // render the error page 63 | res.status(err.status || 500); 64 | res.render('error'); 65 | }); 66 | 67 | module.exports = app; 68 | -------------------------------------------------------------------------------- /CoordinationService/routes/download.js: -------------------------------------------------------------------------------- 1 | var express = require('express'); 2 | var router = express.Router(); 3 | var path = require('path'); 4 | var utilities = require('./dbUtilities'); 5 | var request = require('request'); 6 | var zipFolder = require('zip-folder'); 7 | var fs = require('fs'); 8 | 9 | /* Downloads a document stored in one of the services */ 10 | router.get('/:entity/:id', function routeRoot(req, res, next) { 11 | //The url should be called as follows: /download/dataset/5, for example 12 | var entityName = req.params['entity']; 13 | var entityId = req.params['id']; 14 | var sql; 15 | var filePath; 16 | var tableName; 17 | switch (entityName) { 18 | case ('model'): 19 | tableName = 'models'; 20 | break; 21 | case ('dataset'): 22 | tableName = 'datasets'; 23 | break; 24 | case ('algorithm'): 25 | tableName = 'algorithms'; 26 | break; 27 | case ('prediction'): 28 | tableName = 'predictions'; 29 | break; 30 | default: 31 | res.send({ error: 'Incorrect parameters.' }); 32 | break; 33 | } 34 | 35 | //Creating sql 36 | sql = 'select path from ' + tableName + ' where id = ' + entityId; 37 | console.log(sql); 38 | 39 | //Connect to DB 40 | var connection = createDbConnection(); 41 | connection.connect(); 42 | 43 | //Reading path 44 | connection.query(sql, function getModelPath(err, result) { 45 | if (err) { 46 | console.log(err); 47 | res.send({ error: err }); 48 | } 49 | else { 50 | if (result && result.length > 0) { 51 | filePath = result[0]['path']; 52 | if (tableName === 'models' || tableName === 'algorithms' || tableName === 'predictions') { 53 | var fullPath = path.join(filePath, 'archive.zip'); 54 | if (!fs.existsSync(path)) { 55 | zipFolder(filePath, fullPath, function (err) { 56 | if (err) { 57 | console.log('oh no!', err); 58 | } else { 59 | console.log('EXCELLENT'); 60 | res.download(fullPath); 61 | } 62 | }); 63 | } else{ 64 | res.download(fullPath); 65 | } 66 | } else { 67 | res.download(filePath); 68 | } 69 | } 70 | } 71 | }); 72 | 73 | //Closing connection 74 | connection.end(); 75 | }); 76 | 77 | module.exports = router; -------------------------------------------------------------------------------- /CoordinationService/routes/logUtilities.js: -------------------------------------------------------------------------------- 1 | var mysql = require('mysql'); 2 | var config = require('../config/logDbConfig.json'); 3 | 4 | /* Create logdatabase connection */ 5 | createLogDbConnection = function () { 6 | var connection = mysql.createConnection({ 7 | host: config.dbhost, 8 | user: config.dbuser, 9 | password: config.dbpassword, 10 | database: config.dbname 11 | }); 12 | return connection; 13 | } 14 | //Error type object 15 | var errorType = { 16 | 'DBError': 'DBError', 17 | 'JerseyError': 'JerseyError', 18 | 'LogicError': 'LogicError' 19 | } 20 | //Operation type object 21 | var operationType = { 22 | 'ResponseReceived': 'ResponseReceived', 23 | 'QueryData': 'QueryData', 24 | 'SubmitExperimentToRun': 'SubmitExperimentToRun' 25 | } 26 | //Insert any error/operation into logdatabase 27 | logMessage = function (ifError, type, timestamp, message) { 28 | var logInfo = { 29 | "timestamp": timestamp, 30 | "type": type, 31 | "message": message 32 | } 33 | if (ifError === true) { 34 | { 35 | console.log(timestamp + " " + type + ": " + message); 36 | var connection = createLogDbConnection(); 37 | connection.connect(); 38 | //Insert errors information into errors_log table 39 | var query = 'insert into errors_log (timestamp,type,message) values(?,?,?)'; 40 | var results = connection.query(query, [timestamp, type, message], function insertLog(err, result) { 41 | if (err) { // pass the err to error handler 42 | console.log(query); 43 | } 44 | connection.end(); 45 | }); 46 | } 47 | } 48 | else { 49 | console.log(timestamp + " " + type + ": " + message); 50 | var connection = createLogDbConnection(); 51 | connection.connect(); 52 | //Insert operations information into operations_log table 53 | var query = 'insert into operations_log (timestamp,type,message) values(?,?,?)'; 54 | var results = connection.query(query, [timestamp, type, message], function insertLog(err, result) { 55 | if (err) { // pass the err to error handler 56 | console.log("eoor occurred", err); 57 | } 58 | connection.end(); 59 | }); 60 | } 61 | } 62 | 63 | module.exports = { 64 | errorType: errorType, 65 | operationType: operationType 66 | }; -------------------------------------------------------------------------------- /CoordinationService/public/scripts/monitoring.js: -------------------------------------------------------------------------------- 1 | $(document).ready(function () { 2 | $('#serverIP').text(location.host); 3 | }); 4 | 5 | //Update view with table values 6 | setInterval(function () { 7 | $.getJSON('http://' + location.host + '/table', function (data) { 8 | var tr = 'IPTypeNameDescriptionStatusLast Contacted (seconds ago)'; 9 | var color; 10 | var displayStatus; 11 | var jsonStatus; 12 | for (i = 0; i < data['table'].length; i++) { 13 | var item = data['table'][i]; 14 | if (item['last_updated'] > 60 && item['last_updated'] < 120) 15 | color = 'class="bg-warning"'; 16 | else if (item['last_updated'] > 120) 17 | color = 'class="bg-danger"'; 18 | else 19 | color = 'class="bg-success"'; 20 | displayStatus = ''; 21 | var itemType = item['type'].split('-')[0]; 22 | if (item['status'] && (itemType === 'DataInput' || itemType === 'AlgInput')) { 23 | jsonStatus = JSON.parse(item['status']); 24 | for (var name in jsonStatus) { 25 | displayStatus += '

' + name + ' :' + jsonStatus[name] + '

'; 26 | } 27 | } else { 28 | displayStatus = 'N/A'; 29 | } 30 | tr += '' + item['ip_address'] + '' + item['type'] + '' + item['name'] + '' + item['description'] + '' + displayStatus + '' + item['last_updated'] + ''; 31 | } 32 | $('#monitoring_table').html(tr); 33 | }) 34 | }, 2000); 35 | /*Download the errors_log table as a CSV file */ 36 | function exportErrLogToCSV() { 37 | $.get("/getErrLog", function (data) { 38 | if (data['errorItems'].length > 0) 39 | window.open('/exportErrLogToCSV'); 40 | else { 41 | var alert = $('#formNoExist')[0]; 42 | var alertText = $('#formNoExistText')[0]; 43 | showSubmissionResult("There is no data in error log!", alert, alertText); 44 | 45 | } 46 | }); 47 | } 48 | /*Download the operations_log table as a CSV file */ 49 | function exportOpLogToCSV() { 50 | $.get("/getOpLog", function (data) { 51 | if (data['operationItems'].length > 0) 52 | window.open('/exportOpLogToCSV'); 53 | else { 54 | var alert = $('#formNoExist')[0]; 55 | var alertText = $('#formNoExistText')[0]; 56 | showSubmissionResult("There is no data in operation log!", alert, alertText); 57 | } 58 | }); 59 | } 60 | /*Show bootstrap alert after clicking a button */ 61 | function showSubmissionResult(message, alert, alertText) { 62 | alertText.innerText = message; 63 | alert.style.display = "block"; 64 | } -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/DTO/BaseResponse.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.DTO; 2 | 3 | /** 4 | * BaseResponse which defines the 5 | * {@Code code, msg, success} 6 | * code : error code, default is 0 (no error for all service functionality) 7 | * msg : could be empty or properly detailed message 8 | * success: bool value to show the whether the request is succeed or not 9 | *

10 | * ! Remember all backend service response DTO should inherit from this BaseResponse to provide these information 11 | */ 12 | 13 | public class BaseResponse { 14 | private int code; 15 | private String msg; 16 | private boolean success; 17 | 18 | public BaseResponse(int code, String msg, boolean success) { 19 | this.code = code; 20 | this.msg = msg; 21 | this.success = success; 22 | } 23 | 24 | public BaseResponse() { 25 | } 26 | 27 | public BaseResponse successful() { 28 | this.code = 0; 29 | this.msg = "request successful"; 30 | this.success = true; 31 | return this; 32 | } 33 | public BaseResponse successful(String message) { 34 | this.code = 0; 35 | this.msg = message; 36 | this.success = true; 37 | return this; 38 | } 39 | public BaseResponse failed() { 40 | this.code = -1; 41 | this.msg = "request failed"; 42 | this.success = false; 43 | return this; 44 | } 45 | 46 | /** 47 | * Input error code and message 48 | * @param errorCode 49 | * @param message 50 | * @return 51 | */ 52 | public BaseResponse failed(int errorCode, String message) { 53 | this.code = errorCode; 54 | this.msg = message; 55 | this.success = false; 56 | return this; 57 | } 58 | 59 | /** 60 | * do not use this unless you do not need to specify the error code which will be returned to the client 61 | * @param message 62 | * @return 63 | */ 64 | public BaseResponse failed(String message) { 65 | this.code = 100; 66 | this.msg = message; 67 | this.success = false; 68 | return this; 69 | } 70 | 71 | public int getCode() { 72 | return code; 73 | } 74 | 75 | public void setCode(int code) { 76 | this.code = code; 77 | } 78 | 79 | public String getMsg() { 80 | return msg; 81 | } 82 | 83 | public void setMsg(String msg) { 84 | this.msg = msg; 85 | } 86 | 87 | public boolean isSuccess() { 88 | return success; 89 | } 90 | 91 | public boolean getSuccess() {return success;} 92 | 93 | public void setSuccess(boolean success) { 94 | this.success = success; 95 | } 96 | } 97 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/datasetinputservice/DTO/DatasetInputServiceWorkState.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.datasetinputservice.DTO; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import java.util.Set; 6 | 7 | /** 8 | * Class that represents the working states of DatasetInputService 9 | * including: datasets is under checking, datasets is under uploading 10 | */ 11 | public class DatasetInputServiceWorkState { 12 | public String getStatus() { 13 | return status; 14 | } 15 | 16 | public void setStatus(String status) { 17 | this.status = status; 18 | } 19 | 20 | private String status; 21 | 22 | private List checking; 23 | private List uploading; 24 | 25 | private String checkingList; 26 | private String uploadingList; 27 | 28 | 29 | // Constructors 30 | 31 | public DatasetInputServiceWorkState() { 32 | 33 | } 34 | 35 | public DatasetInputServiceWorkState(List checking, List uploading, String status) { 36 | this.status = status; 37 | this.checking = checking; 38 | this.uploading = uploading; 39 | } 40 | 41 | public DatasetInputServiceWorkState(Set checking, Set uploading, String status) { 42 | this.status = status; 43 | this.checking = new ArrayList<>(checking); 44 | this.uploading = new ArrayList<>(uploading); 45 | StringBuilder strBuilder = new StringBuilder(); 46 | for (String s : checking) { 47 | strBuilder.append(s); 48 | strBuilder.append("; "); 49 | } 50 | checkingList = strBuilder.toString(); 51 | StringBuilder sb = new StringBuilder(); 52 | for (String s : uploading) { 53 | sb.append(s); 54 | sb.append(";"); 55 | } 56 | uploadingList = sb.toString(); 57 | } 58 | 59 | public List getChecking() { 60 | return checking; 61 | } 62 | 63 | public void setChecking(List checking) { 64 | this.checking = checking; 65 | } 66 | 67 | public List getUploading() { 68 | return uploading; 69 | } 70 | 71 | public void setUploading(List uploading) { 72 | this.uploading = uploading; 73 | } 74 | 75 | public String getCheckingList() { 76 | return checkingList; 77 | } 78 | 79 | public void setCheckingList(String checkingList) { 80 | this.checkingList = checkingList; 81 | } 82 | 83 | public String getUploadingList() { 84 | return uploadingList; 85 | } 86 | 87 | public void setUploadingList(String uploadingList) { 88 | this.uploadingList = uploadingList; 89 | } 90 | } 91 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/train/heartbeat/ExecutionServiceHeartBeatsClient.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.train.heartbeat; 2 | 3 | import com.chaosmonkeys.Launcher; 4 | import com.chaosmonkeys.Utilities.LogType; 5 | import com.chaosmonkeys.Utilities.Logger; 6 | 7 | import com.chaosmonkeys.train.Constants; 8 | import com.chaosmonkeys.train.dto.ExecutionServiceStatus; 9 | import com.chaosmonkeys.train.task.ExecutionTaskManager; 10 | import com.chaosmonkeys.train.task.TrainingTaskManager; 11 | import org.glassfish.jersey.client.ClientConfig; 12 | import org.glassfish.jersey.filter.LoggingFilter; 13 | 14 | import javax.ws.rs.ProcessingException; 15 | import javax.ws.rs.client.*; 16 | import javax.ws.rs.core.MediaType; 17 | import javax.ws.rs.core.Response; 18 | import java.util.concurrent.ExecutorService; 19 | import java.util.concurrent.Executors; 20 | 21 | 22 | /** 23 | * Heartbeats client for Execution Service 24 | */ 25 | public class ExecutionServiceHeartBeatsClient { 26 | 27 | private static String coordinationIP; 28 | private static boolean isStart = false; 29 | 30 | public void startSendHeartBeat(String ip) { 31 | if (isStart) { 32 | return; 33 | } 34 | coordinationIP = ip; 35 | 36 | ExecutorService executorService = Executors.newSingleThreadExecutor(); 37 | executorService.submit(this::sendHeartBeatTask); 38 | } 39 | 40 | private void sendHeartBeatTask(){ 41 | Client client = ClientBuilder.newClient(new ClientConfig().register(LoggingFilter.class)); 42 | WebTarget webTarget = client.target(coordinationIP).path("setStatus"); // target to the 43 | 44 | Invocation.Builder invocationBuilder = webTarget.request(MediaType.APPLICATION_JSON); 45 | while (true) { 46 | try { 47 | int runningNum = TrainingTaskManager.INSTANCE.getRunningTaskNum() + ExecutionTaskManager.INSTANCE.getRunningTaskNum(); 48 | String runState = runningNum > 0 ? Constants.STATUS_RUN : Constants.STATUS_IDLE; 49 | ExecutionServiceStatus status = new ExecutionServiceStatus(Launcher.getServiceHost(), Launcher.getServiceType(), Launcher.getServiceName(), Launcher.getServiceDescription(), runState); 50 | 51 | Response response = invocationBuilder.post(Entity.entity(status, MediaType.APPLICATION_JSON)); 52 | } catch (ProcessingException e) { 53 | Logger.SaveLog(LogType.Exception, "EXCEPTION: Heartbeat Connection Timeout, retrying..."); 54 | } 55 | try { 56 | Thread.sleep(10 * 1000); 57 | } catch (InterruptedException e) { 58 | e.printStackTrace(); 59 | } 60 | } 61 | } 62 | 63 | } 64 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/algrinputservice/DTO/AlgorithmInputServiceWorkState.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.algrinputservice.DTO; 2 | 3 | import java.util.ArrayList; 4 | import java.util.List; 5 | import java.util.Set; 6 | 7 | /** 8 | * Class that represents the working states of AlgorithmInputService 9 | * including: datasets is under checking, datasets is under uploading 10 | */ 11 | public class AlgorithmInputServiceWorkState { 12 | 13 | private String status; 14 | 15 | private List checking; 16 | private List uploading; 17 | 18 | private String checkingList; 19 | private String uploadingList; 20 | 21 | 22 | // Constructors 23 | public AlgorithmInputServiceWorkState() { 24 | 25 | } 26 | 27 | public AlgorithmInputServiceWorkState(List checking, List uploading, String status) { 28 | this.status = status; 29 | this.checking = checking; 30 | this.uploading = uploading; 31 | } 32 | 33 | public AlgorithmInputServiceWorkState(Set checking, Set uploading, String status) { 34 | this.status = status; 35 | this.checking = new ArrayList<>(checking); 36 | this.uploading = new ArrayList<>(uploading); 37 | StringBuilder strBuilder = new StringBuilder(); 38 | for (String s : checking) { 39 | strBuilder.append(s); 40 | strBuilder.append("; "); 41 | } 42 | checkingList = strBuilder.toString(); 43 | StringBuilder sb = new StringBuilder(); 44 | for (String s : uploading) { 45 | sb.append(s); 46 | sb.append(";"); 47 | } 48 | uploadingList = sb.toString(); 49 | } 50 | 51 | // Getter and setters 52 | public String getStatus() { 53 | return status; 54 | } 55 | 56 | public void setStatus(String status) { 57 | this.status = status; 58 | } 59 | 60 | public List getChecking() { 61 | return checking; 62 | } 63 | 64 | public void setChecking(List checking) { 65 | this.checking = checking; 66 | } 67 | 68 | public List getUploading() { 69 | return uploading; 70 | } 71 | 72 | public void setUploading(List uploading) { 73 | this.uploading = uploading; 74 | } 75 | 76 | public String getCheckingList() { 77 | return checkingList; 78 | } 79 | 80 | public void setCheckingList(String checkingList) { 81 | this.checkingList = checkingList; 82 | } 83 | 84 | public String getUploadingList() { 85 | return uploadingList; 86 | } 87 | 88 | public void setUploadingList(String uploadingList) { 89 | this.uploadingList = uploadingList; 90 | } 91 | 92 | 93 | } 94 | -------------------------------------------------------------------------------- /CoordinationService/public/stylesheets/justified-nav.css: -------------------------------------------------------------------------------- 1 | body { 2 | padding-top: 20px; 3 | } 4 | 5 | .footer { 6 | padding-top: 40px; 7 | padding-bottom: 40px; 8 | margin-top: 40px; 9 | border-top: 1px solid #eee; 10 | } 11 | 12 | /* Main marketing message and sign up button */ 13 | .jumbotron { 14 | text-align: center; 15 | background-color: transparent; 16 | } 17 | .jumbotron .btn { 18 | padding: 14px 24px; 19 | font-size: 21px; 20 | } 21 | 22 | /* Customize the nav-justified links to be fill the entire space of the .navbar */ 23 | 24 | .nav-justified { 25 | background-color: #eee; 26 | border: 1px solid #ccc; 27 | border-radius: 5px; 28 | } 29 | .nav-justified > li > a { 30 | padding-top: 15px; 31 | padding-bottom: 15px; 32 | margin-bottom: 0; 33 | font-weight: bold; 34 | color: #777; 35 | text-align: center; 36 | background-color: #e5e5e5; /* Old browsers */ 37 | background-image: -webkit-gradient(linear, left top, left bottom, from(#f5f5f5), to(#e5e5e5)); 38 | background-image: -webkit-linear-gradient(top, #f5f5f5 0%, #e5e5e5 100%); 39 | background-image: -o-linear-gradient(top, #f5f5f5 0%, #e5e5e5 100%); 40 | background-image: linear-gradient(to bottom, #f5f5f5 0%,#e5e5e5 100%); 41 | filter: progid:DXImageTransform.Microsoft.gradient( startColorstr='#f5f5f5', endColorstr='#e5e5e5',GradientType=0 ); /* IE6-9 */ 42 | background-repeat: repeat-x; /* Repeat the gradient */ 43 | border-bottom: 1px solid #d5d5d5; 44 | } 45 | .nav-justified > .active > a, 46 | .nav-justified > .active > a:hover, 47 | .nav-justified > .active > a:focus { 48 | background-color: #ddd; 49 | background-image: none; 50 | -webkit-box-shadow: inset 0 3px 7px rgba(0,0,0,.15); 51 | box-shadow: inset 0 3px 7px rgba(0,0,0,.15); 52 | } 53 | .nav-justified > li:first-child > a { 54 | border-radius: 5px 5px 0 0; 55 | } 56 | .nav-justified > li:last-child > a { 57 | border-bottom: 0; 58 | border-radius: 0 0 5px 5px; 59 | } 60 | 61 | @media (min-width: 768px) { 62 | .nav-justified { 63 | max-height: 52px; 64 | } 65 | .nav-justified > li > a { 66 | border-right: 1px solid #d5d5d5; 67 | border-left: 1px solid #fff; 68 | } 69 | .nav-justified > li:first-child > a { 70 | border-left: 0; 71 | border-radius: 5px 0 0 5px; 72 | } 73 | .nav-justified > li:last-child > a { 74 | border-right: 0; 75 | border-radius: 0 5px 5px 0; 76 | } 77 | } 78 | 79 | /* Responsive: Portrait tablets and up */ 80 | @media screen and (min-width: 768px) { 81 | /* Remove the padding we set earlier */ 82 | .masthead, 83 | .marketing, 84 | .footer { 85 | padding-right: 0; 86 | padding-left: 0; 87 | } 88 | } 89 | .form-group.required .control-label:after { 90 | content:"*"; 91 | color:red; 92 | } 93 | .modal-body { 94 | max-height: calc(100vh - 210px); 95 | overflow-y: auto; 96 | } 97 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/algrinputservice/AlgorithmInputServiceHeartBeatsClient.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.algrinputservice; 2 | 3 | import com.chaosmonkeys.Launcher; 4 | import com.chaosmonkeys.Utilities.LogType; 5 | import com.chaosmonkeys.Utilities.Logger; 6 | import com.chaosmonkeys.algrinputservice.DTO.AlgorithmInputServiceStatusInfo; 7 | import com.chaosmonkeys.algrinputservice.DTO.AlgorithmInputServiceWorkState; 8 | import org.glassfish.jersey.client.ClientConfig; 9 | import org.glassfish.jersey.filter.LoggingFilter; 10 | 11 | import javax.ws.rs.ProcessingException; 12 | import javax.ws.rs.client.*; 13 | import javax.ws.rs.core.MediaType; 14 | import javax.ws.rs.core.Response; 15 | import java.util.concurrent.Executors; 16 | import java.util.concurrent.ScheduledExecutorService; 17 | 18 | /** 19 | * Class for sending input service heartbeats 20 | */ 21 | public class AlgorithmInputServiceHeartBeatsClient { 22 | private static String coordinationIP; 23 | private static boolean isStart = false; 24 | 25 | public void startSendHeartBeat(String ip) { 26 | if (isStart) { 27 | return; 28 | } 29 | coordinationIP = ip; 30 | ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); 31 | 32 | scheduledExecutorService.submit(new SendHeartBeatTask()); 33 | 34 | } 35 | 36 | class SendHeartBeatTask implements Runnable { 37 | 38 | @Override 39 | public void run() { 40 | Client client = ClientBuilder.newClient(new ClientConfig().register(LoggingFilter.class)); 41 | WebTarget webTarget = client.target(coordinationIP).path("setStatus"); // target to the 42 | 43 | Invocation.Builder invocationBuilder = webTarget.request(MediaType.APPLICATION_JSON); 44 | while (true) { 45 | try { 46 | AlgorithmInputServiceWorkState workState = new AlgorithmInputServiceWorkState(AlgorithmResource.checkSet, AlgorithmResource.uploadSet, AlgorithmResource.serviceStatus); 47 | AlgorithmInputServiceStatusInfo serviceStatusInfo = new AlgorithmInputServiceStatusInfo(Launcher.getServiceHost(), Launcher.getServiceType(), Launcher.getServiceName(), Launcher.getServiceDescription(), workState); 48 | 49 | Response response = invocationBuilder.post(Entity.entity(serviceStatusInfo, MediaType.APPLICATION_JSON)); 50 | } catch (ProcessingException e) { 51 | Logger.SaveLog(LogType.Exception, "EXCEPTION: Heartbeat Connection Timeout, retrying..."); 52 | } 53 | try { 54 | Thread.sleep(3 * 1000); 55 | } catch (InterruptedException e) { 56 | e.printStackTrace(); 57 | } 58 | } 59 | 60 | } 61 | } 62 | } 63 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/datasetinputservice/DatasetInputServiceHeartBeatsClient.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.datasetinputservice; 2 | 3 | import com.chaosmonkeys.Launcher; 4 | import com.chaosmonkeys.Utilities.LogType; 5 | import com.chaosmonkeys.Utilities.Logger; 6 | import com.chaosmonkeys.datasetinputservice.DTO.DatasetInputServiceStatusInfo; 7 | import com.chaosmonkeys.datasetinputservice.DTO.DatasetInputServiceWorkState; 8 | import org.glassfish.jersey.client.ClientConfig; 9 | import org.glassfish.jersey.filter.LoggingFilter; 10 | 11 | import javax.ws.rs.ProcessingException; 12 | import javax.ws.rs.client.*; 13 | import javax.ws.rs.core.MediaType; 14 | import javax.ws.rs.core.Response; 15 | import java.util.concurrent.Executors; 16 | import java.util.concurrent.ScheduledExecutorService; 17 | 18 | /** 19 | * Class for sending input service heartbeats 20 | */ 21 | public class DatasetInputServiceHeartBeatsClient { 22 | private static String coordinationIP; 23 | private static boolean isStart = false; 24 | 25 | public void startSendHeartBeat(String ip) { 26 | if (isStart) { 27 | return; 28 | } 29 | coordinationIP = ip; 30 | // start a single thread 31 | ScheduledExecutorService scheduledExecutorService = Executors.newSingleThreadScheduledExecutor(); 32 | SendHeartBeatTask heartbeatTask = new SendHeartBeatTask(); 33 | scheduledExecutorService.submit(heartbeatTask); 34 | 35 | } 36 | 37 | class SendHeartBeatTask implements Runnable { 38 | 39 | @Override 40 | public void run() { 41 | Client client = ClientBuilder.newClient(new ClientConfig().register(LoggingFilter.class)); 42 | WebTarget webTarget = client.target(coordinationIP).path("setStatus"); // target to the 43 | 44 | Invocation.Builder invocationBuilder = webTarget.request(MediaType.APPLICATION_JSON); 45 | while (true) { 46 | try { 47 | DatasetInputServiceWorkState workState = new DatasetInputServiceWorkState(DatasetInputService.checkSet, DatasetInputService.uploadSet, DatasetInputService.serviceStatus); 48 | DatasetInputServiceStatusInfo serviceStatusInfo = new DatasetInputServiceStatusInfo(Launcher.getServiceHost(), Launcher.getServiceType(), Launcher.getServiceName(), Launcher.getServiceDescription(), workState); 49 | 50 | Response response = invocationBuilder.post(Entity.entity(serviceStatusInfo, MediaType.APPLICATION_JSON)); 51 | } catch (ProcessingException e) { 52 | Logger.SaveLog(LogType.Exception, "EXCEPTION: Heartbeat Connection Timeout, retrying..."); 53 | } 54 | try { 55 | Thread.sleep(10 * 1000); 56 | } catch (InterruptedException e) { 57 | e.printStackTrace(); 58 | } 59 | } 60 | 61 | } 62 | } 63 | } 64 | -------------------------------------------------------------------------------- /SQL/logdb-08-06-2017-V0.2.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.13 Distrib 5.7.12, for Win64 (x86_64) 2 | -- 3 | -- Host: localhost Database: logdatabase 4 | -- ------------------------------------------------------ 5 | -- Server version 5.6.35-log 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Table structure for table `errors_log` 20 | -- 21 | 22 | DROP TABLE IF EXISTS `errors_log`; 23 | /*!40101 SET @saved_cs_client = @@character_set_client */; 24 | /*!40101 SET character_set_client = utf8 */; 25 | CREATE TABLE `errors_log` ( 26 | `id` int(11) NOT NULL AUTO_INCREMENT, 27 | `timestamp` datetime DEFAULT NULL, 28 | `type` varchar(30) NOT NULL, 29 | `message` varchar(1000) NOT NULL, 30 | PRIMARY KEY (`id`) 31 | ) ENGINE=InnoDB AUTO_INCREMENT=1097 DEFAULT CHARSET=utf8; 32 | /*!40101 SET character_set_client = @saved_cs_client */; 33 | 34 | -- 35 | -- Dumping data for table `errors_log` 36 | -- 37 | 38 | LOCK TABLES `errors_log` WRITE; 39 | /*!40000 ALTER TABLE `errors_log` DISABLE KEYS */; 40 | /*!40000 ALTER TABLE `errors_log` ENABLE KEYS */; 41 | UNLOCK TABLES; 42 | 43 | -- 44 | -- Table structure for table `operations_log` 45 | -- 46 | 47 | DROP TABLE IF EXISTS `operations_log`; 48 | /*!40101 SET @saved_cs_client = @@character_set_client */; 49 | /*!40101 SET character_set_client = utf8 */; 50 | CREATE TABLE `operations_log` ( 51 | `id` int(11) NOT NULL AUTO_INCREMENT, 52 | `timestamp` datetime DEFAULT NULL, 53 | `type` varchar(30) NOT NULL, 54 | `message` varchar(1000) NOT NULL, 55 | PRIMARY KEY (`id`) 56 | ) ENGINE=InnoDB AUTO_INCREMENT=17847 DEFAULT CHARSET=utf8; 57 | /*!40101 SET character_set_client = @saved_cs_client */; 58 | 59 | -- 60 | -- Dumping data for table `operations_log` 61 | -- 62 | 63 | LOCK TABLES `operations_log` WRITE; 64 | /*!40000 ALTER TABLE `operations_log` DISABLE KEYS */; 65 | /*!40000 ALTER TABLE `operations_log` ENABLE KEYS */; 66 | UNLOCK TABLES; 67 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 68 | 69 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 70 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 71 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 72 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 73 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 74 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 75 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 76 | 77 | -- Dump completed on 2017-08-06 19:10:47 78 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/Utilities/Logger.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.Utilities; 2 | 3 | import com.chaosmonkeys.Launcher; 4 | import com.chaosmonkeys.Utilities.db.DbUtils; 5 | import com.chaosmonkeys.dao.ErrorLog; 6 | import com.chaosmonkeys.dao.OpLog; 7 | import org.javalite.activejdbc.Model; 8 | 9 | import java.sql.Timestamp; 10 | import java.time.Instant; 11 | 12 | /** 13 | * Class that is used to log information and error messages. 14 | */ 15 | public class Logger { 16 | 17 | // service type 18 | private static String serviceType = ""; 19 | // a list to store log info 20 | 21 | /** 22 | * Displaying log and store it to database 23 | * @param type 24 | * @param message 25 | */ 26 | public static void SaveLog(LogType type, String message){ 27 | if(serviceType.equals("")){ 28 | serviceType = Launcher.serviceType; 29 | } 30 | StringBuilder strBuilder = new StringBuilder(); 31 | strBuilder.append(serviceType); 32 | strBuilder.append(" - "); 33 | strBuilder.append(message); 34 | String msg = strBuilder.toString(); 35 | Timestamp nowTime = Timestamp.from(Instant.now()); 36 | //TODO: Implement logic here 37 | System.out.println(msg); 38 | if(type == LogType.Error || type == LogType.Exception){ 39 | DbUtils.openLogConnection(); // connections must be opened before any models 40 | ErrorLog errorLogModel = new ErrorLog(); 41 | errorLogModel.set("timestamp", nowTime); 42 | errorLogModel.set("type", type.toString()); 43 | errorLogModel.set("message", msg); 44 | errorLogModel.saveIt(); 45 | DbUtils.closeLogConnection(); 46 | }else{ 47 | DbUtils.openLogConnection(); 48 | OpLog opLogModel = new OpLog(); 49 | opLogModel.set("timestamp", nowTime); 50 | opLogModel.set("type", type.toString()); 51 | opLogModel.set("message", msg); 52 | opLogModel.saveIt(); 53 | DbUtils.closeLogConnection(); 54 | } 55 | } 56 | 57 | public static void setServiceType(String type){ 58 | Logger.serviceType = type; 59 | } 60 | 61 | public static void Error(String message){ 62 | SaveLog(LogType.Error, message); 63 | } 64 | public static void Exception(String message) { SaveLog(LogType.Exception, message);} 65 | public static void Info(String message){ 66 | SaveLog(LogType.Information, message); 67 | } 68 | public static void Request(String message){SaveLog(LogType.Request, message);} 69 | public static void Response(String message){SaveLog(LogType.Response, message);} 70 | 71 | //TODO: if logging slow down the whole system, we will cache log items and flush them when it reaches maxium cache number to avoid frequently connecting 72 | private class LogItem{ 73 | Timestamp timestamp; 74 | String log; 75 | LogType type; 76 | } 77 | } -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/train/task/AbsTask.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.train.task; 2 | 3 | import com.chaosmonkeys.train.task.interfaces.OnTaskUpdateListener; 4 | 5 | import java.util.concurrent.ExecutorService; 6 | import java.util.concurrent.Executors; 7 | import java.util.concurrent.locks.ReentrantLock; 8 | 9 | /** 10 | * Base class of Async task 11 | */ 12 | public abstract class AbsTask { 13 | 14 | private volatile boolean isCancelled = false; 15 | private volatile TaskState state = TaskState.IDLE; 16 | // error message field used to store error message 17 | private volatile String errorMsg = ""; 18 | private volatile ExecutorService singleExecutorService; 19 | // lock for managing connection 20 | private final ReentrantLock stateUpdateLock = new ReentrantLock(); 21 | 22 | protected OnTaskUpdateListener taskUpdateListener = null; 23 | //private ResultType result; 24 | 25 | public AbsTask(){} 26 | 27 | protected abstract void initialize(); 28 | 29 | protected abstract void performTask(); 30 | 31 | protected abstract void cleanUp(); 32 | 33 | /** 34 | * Obtains a single thread executor for this Task's following runnables 35 | * using double-checked lock to ensure thread-safe 36 | * @return ExecutorService 37 | * @since JDK 1.5 38 | */ 39 | protected ExecutorService getExecutorService(){ 40 | if(null == singleExecutorService){ 41 | synchronized (this){ 42 | if(null == singleExecutorService){ 43 | singleExecutorService = Executors.newSingleThreadExecutor(); 44 | } 45 | } 46 | } 47 | return singleExecutorService; 48 | } 49 | 50 | 51 | protected abstract void cancelWorks(); 52 | 53 | public String getErrorMsg() { 54 | return errorMsg; 55 | } 56 | 57 | public void setErrorMsg(String errorMsg) { 58 | this.errorMsg = errorMsg; 59 | } 60 | public final boolean isCancelled() { 61 | return ( isCancelled || state == TaskState.CANCELLED ); 62 | } 63 | 64 | public final boolean isIDLE(){ 65 | return this.state.value() == TaskState.IDLE.value(); 66 | } 67 | 68 | public final boolean isInitialized(){ 69 | return this.state.value() >= TaskState.INITIALIZED.value(); 70 | } 71 | 72 | public final boolean isFinished() { 73 | return this.state.value() > TaskState.STARTED.value(); 74 | } 75 | public final TaskState getState(){ 76 | return state; 77 | } 78 | 79 | /** 80 | * package - only task pacakge can set state 81 | * using lock to make sure consistent state update 82 | */ 83 | void setState(TaskState tState){ 84 | stateUpdateLock.lock(); 85 | try{ 86 | this.state = tState; 87 | }finally { 88 | stateUpdateLock.unlock(); 89 | } 90 | } 91 | final void setTaskUpdateListener(OnTaskUpdateListener listener){ 92 | this.taskUpdateListener = listener; 93 | } 94 | 95 | } 96 | -------------------------------------------------------------------------------- /SQL/logdb-07-31-2017-V0.1.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.13 Distrib 5.7.17, for osx10.12 (x86_64) 2 | -- 3 | -- Host: localhost Database: logdatabase 4 | -- ------------------------------------------------------ 5 | -- Server version 5.7.18 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Current Database: `logdatabase` 20 | -- 21 | 22 | CREATE DATABASE /*!32312 IF NOT EXISTS*/ `logdatabase` /*!40100 DEFAULT CHARACTER SET latin1 */; 23 | 24 | USE `logdatabase`; 25 | 26 | -- 27 | -- Table structure for table `errors_log` 28 | -- 29 | 30 | DROP TABLE IF EXISTS `errors_log`; 31 | /*!40101 SET @saved_cs_client = @@character_set_client */; 32 | /*!40101 SET character_set_client = utf8 */; 33 | CREATE TABLE `errors_log` ( 34 | `id` int(11) NOT NULL AUTO_INCREMENT, 35 | `timestamp` datetime DEFAULT NULL, 36 | `type` varchar(30) NOT NULL, 37 | `message` varchar(500) NOT NULL, 38 | PRIMARY KEY (`id`) 39 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 40 | /*!40101 SET character_set_client = @saved_cs_client */; 41 | 42 | -- 43 | -- Dumping data for table `errors_log` 44 | -- 45 | 46 | LOCK TABLES `errors_log` WRITE; 47 | /*!40000 ALTER TABLE `errors_log` DISABLE KEYS */; 48 | /*!40000 ALTER TABLE `errors_log` ENABLE KEYS */; 49 | UNLOCK TABLES; 50 | 51 | -- 52 | -- Table structure for table `operations_log` 53 | -- 54 | 55 | DROP TABLE IF EXISTS `operations_log`; 56 | /*!40101 SET @saved_cs_client = @@character_set_client */; 57 | /*!40101 SET character_set_client = utf8 */; 58 | CREATE TABLE `operations_log` ( 59 | `id` int(11) NOT NULL AUTO_INCREMENT, 60 | `timestamp` datetime DEFAULT NULL, 61 | `type` varchar(30) NOT NULL, 62 | `message` varchar(500) NOT NULL, 63 | PRIMARY KEY (`id`) 64 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 65 | /*!40101 SET character_set_client = @saved_cs_client */; 66 | 67 | -- 68 | -- Dumping data for table `operations_log` 69 | -- 70 | 71 | LOCK TABLES `operations_log` WRITE; 72 | /*!40000 ALTER TABLE `operations_log` DISABLE KEYS */; 73 | /*!40000 ALTER TABLE `operations_log` ENABLE KEYS */; 74 | UNLOCK TABLES; 75 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 76 | 77 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 78 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 79 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 80 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 81 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 82 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 83 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 84 | 85 | -- Dump completed on 2017-07-31 19:40:57 86 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Chaos Monkeys Experimentation Platform 2 | 3 | ## Introduction 4 | 5 | This archive includes the sources for this project. The system is tested under Ubuntu 16.04 and Windows 10 with the following packages installed. 6 | 7 | * JDK 1.8 8 | * RScript 3.2.3 9 | * NodeJS 8 10 | * MySQL 5.7 11 | * Maven 3.3.9 12 | 13 | ## Installation 14 | * Open the terminal / CMD. 15 | * Make sure the MySQL server has started. If the server is not on, please follow the instruction below. 16 | 17 | For UNIX-based system, type the following commands in the terminal: 18 | 19 | ``` 20 | $ mysql.server start 21 | ``` 22 | 23 | For Windows users, type the following commands in CMD: 24 | 25 | ``` 26 | C:\ start mysqlId -u root 27 | ``` 28 | 29 | * Run the command line MySQL interface to connect to the MySQL server 30 | 31 | **Windows** 32 | 33 | ``` 34 | C:\ mysql --user=root 35 | ``` 36 | 37 | **UNIX** 38 | 39 | ``` 40 | $ mysql -u root 41 | ``` 42 | 43 | * You need to create a database by 44 | 45 | ``` 46 | mysql> create database configurationdatabase; 47 | mysql> create database logdatabase 48 | ``` 49 | 50 | * At this point exit from MySQL by typing `exit` at the MySQL prompt. In your OS shell (**not the MySQL command line window**), navigate to folder `SQL` and import the sample data by typing the following commands (*root pwd* is the password you set for the MySQL root administrator): 51 | 52 | **Windows** 53 | 54 | ``` 55 | C:\ cd SQL 56 | C:\\SQL mysql -u root -p configurationdatabase < 08-08-2017-V1.4.sql 57 | C:\\SQL mysql -u root -p logdatabase < logdb-08-06-2017-V0.2.sql 58 | ``` 59 | **UNIX** 60 | 61 | ``` 62 | $ cd SQL 63 | $ mysql -u root -p configurationdatabase < 08-08-2017-V1.4.sql 64 | $ mysql -u root -p logdatabase < logdb-08-06-2017-V0.2.sql 65 | ``` 66 | 67 | * Next, you will need to configure the parameters of MySQL in both Coordination Service and the Jersey Services to your customized settings. You can do this by editing the database config files in the path `CoordinationService/config/dbConfig.json`, `CoordinationService\config\logDbConfig.json` and `JerseySerices/dbConfig.ini`. 68 | 69 | ## To Build and Run the Coordination Service 70 | 71 | ``` 72 | $ cd CoordinationServices 73 | $ npm install 74 | $ npm start 75 | ``` 76 | 77 | ## To Build Jersey Services 78 | 79 | ``` 80 | $ bash JerseyServices/build.sh 81 | ``` 82 | 83 | ## To Run All Jersey Services 84 | 85 | ``` 86 | $ cd JerseyServices/launch_scripts/ 87 | $ bash run_jersey_all.sh 88 | ``` 89 | 90 | ## Or, Run Jersey Services Separately 91 | 92 | ### To Run Algorithm Input Service 93 | 94 | ``` 95 | $ cd JerseyServices/launch_scripts/ 96 | $ bash run_algr_input.sh 97 | ``` 98 | 99 | ### To Run Data Input Service 100 | 101 | ``` 102 | $ cd JerseyServices/launch_scripts/ 103 | $ bash run_data_input.sh 104 | ``` 105 | 106 | ### To Run Training Service 107 | 108 | ``` 109 | $ cd JerseyServices/launch_scripts/ 110 | $ bash run_training.sh 111 | ``` 112 | 113 | ### To Run Execution Service 114 | 115 | ``` 116 | $ cd JerseyServices/launch_scripts/ 117 | $ bash run_algr_input.sh 118 | ``` 119 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/Utilities/MachineIPHelper.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.Utilities; 2 | 3 | import java.io.BufferedReader; 4 | import java.io.IOException; 5 | import java.io.InputStreamReader; 6 | import java.net.*; 7 | import java.util.Enumeration; 8 | 9 | /** 10 | * Utilities to detect IP addresses. 11 | */ 12 | public class MachineIPHelper { 13 | /** 14 | * Gets host IP from AWS. 15 | * 16 | * @return Service host IP. 17 | */ 18 | public static String getIPfromAWS() { 19 | URL whatIsMyIP = null; 20 | try { 21 | whatIsMyIP = new URL("http://checkip.amazonaws.com"); 22 | } catch (MalformedURLException e) { 23 | Logger.SaveLog(LogType.Exception,"EXCEPTION: Could not create Amazon URL (my IP)."); 24 | } 25 | 26 | BufferedReader in = null; 27 | try { 28 | in = new BufferedReader(new InputStreamReader( 29 | whatIsMyIP.openStream())); 30 | } catch (IOException e) { 31 | Logger.SaveLog(LogType.Exception,"EXCEPTION: Could not open stream to Amazon (my IP)."); 32 | } 33 | 34 | String ip = null; 35 | try { 36 | ip = in.readLine(); 37 | } catch (IOException e) { 38 | Logger.SaveLog(LogType.Exception,"EXCEPTION: Could not read value (my IP)."); 39 | } 40 | return ip; 41 | } 42 | 43 | /**/ 44 | public static String getRealIp() { 45 | //If no external ip 46 | String localIP = null; 47 | //External ip 48 | String netIP = null; 49 | 50 | //Create network interface 51 | Enumeration netInterfaces = null; 52 | try { 53 | netInterfaces = NetworkInterface.getNetworkInterfaces(); 54 | } catch (SocketException e) { 55 | e.printStackTrace(); 56 | } 57 | 58 | //IP 59 | InetAddress ip = null; 60 | 61 | //Whether the external network IP exists 62 | boolean found = false; 63 | 64 | //Extract connection information 65 | while (netInterfaces.hasMoreElements() && !found) { 66 | NetworkInterface ni = netInterfaces.nextElement(); 67 | Enumeration address = ni.getInetAddresses(); 68 | while (address.hasMoreElements()) { 69 | ip = address.nextElement(); 70 | if (!ip.isSiteLocalAddress() 71 | && !ip.isLoopbackAddress() 72 | && ip.getHostAddress().indexOf(":") == -1) {// external network IP 73 | netIP = ip.getHostAddress(); 74 | found = true; 75 | break; 76 | } else if (ip.isSiteLocalAddress() 77 | && !ip.isLoopbackAddress() 78 | && ip.getHostAddress().indexOf(":") == -1) {// local IP 79 | localIP = ip.getHostAddress(); 80 | } 81 | } 82 | } 83 | 84 | //Deciding which value to return 85 | if (netIP != null && !"".equals(netIP)) { 86 | return netIP; 87 | } else { 88 | return localIP; 89 | } 90 | } 91 | 92 | /** 93 | * Gets the localhost IP 94 | * @return Localhost IP. 95 | */ 96 | public static String getLocalIP() { 97 | InetAddress address = null; 98 | try { 99 | address = InetAddress.getLocalHost(); 100 | } catch (UnknownHostException e) { 101 | Logger.SaveLog(LogType.Exception,"EXCEPTION: Unknown host "); 102 | } 103 | //Get local machine IP address 104 | return address.getHostAddress().toString(); 105 | } 106 | } 107 | -------------------------------------------------------------------------------- /CoordinationService/public/scripts/common.js: -------------------------------------------------------------------------------- 1 | /* Reset the datasets/algorithms upload and tasks/experiments creation modals. */ 2 | function resetModal(modalId) { 3 | $('#' + modalId).on('hidden.bs.modal', function () { 4 | for (i = 0; i < $('form').length; i++) 5 | $('form').get(i).reset(); 6 | var formerror = $('#formError')[0]; 7 | var formsuccess = $('#formSuccess')[0]; 8 | var formuploaderror = $('#formUploadError')[0]; 9 | var formuploadsuccess = $('#formUploadSuccess')[0]; 10 | var formtasksuccess = $('#formTaskSuccess')[0]; 11 | var formtaskerror = $('#formTaskError')[0]; 12 | var formexperimenterror = $('#formExperimentError')[0]; 13 | var formexperimentsuccess = $('#formExperimentSuccess')[0]; 14 | if (typeof formerror != 'undefined') 15 | formerror.style.display = "none"; 16 | if (typeof formsuccess != 'undefined') 17 | formsuccess.style.display = "none"; 18 | if (typeof formuploaderror != 'undefined') 19 | formuploaderror.style.display = "none"; 20 | if (typeof formuploadsuccess != 'undefined') 21 | formuploadsuccess.style.display = "none"; 22 | if (typeof formtaskerror != 'undefined') 23 | formtaskerror.style.display = "none"; 24 | if (typeof formtasksuccess != 'undefined') 25 | formtasksuccess.style.display = "none"; 26 | if (typeof formexperimenterror != 'undefined') 27 | formexperimenterror.style.display = "none"; 28 | if (typeof formexperimentsuccess != 'undefined') 29 | formexperimentsuccess.style.display = "none"; 30 | }); 31 | $('#' + modalId).on('shown.bs.modal', function () { 32 | enableDisabledButtons(modalId); 33 | }); 34 | } 35 | 36 | //Formats MYSQL Datetime for display 37 | function displayDateTime(time) { 38 | if (time) 39 | return time.replace('T', ' ').replace('.000Z', ''); 40 | else 41 | return time; 42 | } 43 | 44 | //Enables disabled buttons under a div 45 | function enableDisabledButtons(divId) { 46 | $('#' + divId + ' :button').attr('disabled', false); 47 | } 48 | 49 | //Parse a String format date into a Date format 50 | function stringToDate(s) { 51 | if (s) 52 | return new Date(Date.parse(s.replace('-', '/', 'g'))); 53 | else 54 | return s; 55 | } 56 | 57 | //Transform milliseconds to HH:MM:SS format 58 | function msToTime(duration) { 59 | var seconds = parseInt((duration / 1000) % 60) 60 | , minutes = parseInt((duration / (1000 * 60)) % 60) 61 | , hours = parseInt((duration / (1000 * 60 * 60)) % 24) 62 | , days = parseInt((duration / (1000 * 60 * 60 * 24))); 63 | 64 | days = (days < 10) ? "0" + days : days; 65 | hours = (hours < 10) ? "0" + hours : hours; 66 | minutes = (minutes < 10) ? "0" + minutes : minutes; 67 | seconds = (seconds < 10) ? "0" + seconds : seconds; 68 | 69 | return days + " days " + hours + " hours " + minutes + " minutes " + seconds + " seconds"; 70 | } 71 | 72 | //Transform a javascript date format to YYYY-MM-DD HH:MM:SS format 73 | function formatDate(datetime) { 74 | var year = datetime.getFullYear(); 75 | var month = datetime.getMonth() + 1; 76 | var day = datetime.getDate(); 77 | var hour = datetime.getHours(); 78 | var minute = datetime.getMinutes(); 79 | var second = datetime.getSeconds(); 80 | return year + "-" + month + "-" + day + " " + hour + ":" + minute + ":" + second; 81 | } 82 | 83 | //Dynamically loads the IP of a service 84 | function getServiceIp(serviceType, callback) { 85 | var coordinationUrl = 'http://' + location.host + '/dynamicIp/get/' + serviceType; 86 | $.getJSON(coordinationUrl, function (data) { 87 | if (data['message'] === 'success') 88 | callback(data['ip']); 89 | }); 90 | } -------------------------------------------------------------------------------- /SQL/06-16-2017-V0.2.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.13 Distrib 5.7.12, for Win64 (x86_64) 2 | -- 3 | -- Host: localhost Database: configurationdatabase 4 | -- ------------------------------------------------------ 5 | -- Server version 5.6.35-log 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Table structure for table `connected_services` 20 | -- 21 | 22 | DROP TABLE IF EXISTS `connected_services`; 23 | /*!40101 SET @saved_cs_client = @@character_set_client */; 24 | /*!40101 SET character_set_client = utf8 */; 25 | CREATE TABLE `connected_services` ( 26 | `id` int(11) NOT NULL AUTO_INCREMENT, 27 | `ip_address` varchar(255) NOT NULL, 28 | `type` varchar(255) NOT NULL, 29 | `name` varchar(50) DEFAULT NULL, 30 | `description` varchar(255) DEFAULT NULL, 31 | `status` varchar(255) DEFAULT NULL, 32 | `last_updated` datetime DEFAULT NULL, 33 | PRIMARY KEY (`id`), 34 | UNIQUE KEY `ip_address_UNIQUE` (`ip_address`) 35 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 36 | /*!40101 SET character_set_client = @saved_cs_client */; 37 | 38 | -- 39 | -- Dumping data for table `connected_services` 40 | -- 41 | 42 | LOCK TABLES `connected_services` WRITE; 43 | /*!40000 ALTER TABLE `connected_services` DISABLE KEYS */; 44 | /*!40000 ALTER TABLE `connected_services` ENABLE KEYS */; 45 | UNLOCK TABLES; 46 | 47 | -- 48 | -- Table structure for table `datasets` 49 | -- 50 | 51 | DROP TABLE IF EXISTS `datasets`; 52 | /*!40101 SET @saved_cs_client = @@character_set_client */; 53 | /*!40101 SET character_set_client = utf8 */; 54 | CREATE TABLE `datasets` ( 55 | `id` int(11) NOT NULL, 56 | `project_id` varchar(255) NOT NULL, 57 | `path` varchar(255) NOT NULL, 58 | `format` varchar(20) DEFAULT NULL, 59 | PRIMARY KEY (`id`) 60 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 61 | /*!40101 SET character_set_client = @saved_cs_client */; 62 | 63 | -- 64 | -- Dumping data for table `datasets` 65 | -- 66 | 67 | LOCK TABLES `datasets` WRITE; 68 | /*!40000 ALTER TABLE `datasets` DISABLE KEYS */; 69 | /*!40000 ALTER TABLE `datasets` ENABLE KEYS */; 70 | UNLOCK TABLES; 71 | 72 | -- 73 | -- Table structure for table `formats` 74 | -- 75 | 76 | DROP TABLE IF EXISTS `formats`; 77 | /*!40101 SET @saved_cs_client = @@character_set_client */; 78 | /*!40101 SET character_set_client = utf8 */; 79 | CREATE TABLE `formats` ( 80 | `id` int(11) NOT NULL AUTO_INCREMENT, 81 | `format` varchar(20) NOT NULL, 82 | PRIMARY KEY (`id`) 83 | ) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8; 84 | /*!40101 SET character_set_client = @saved_cs_client */; 85 | 86 | -- 87 | -- Dumping data for table `formats` 88 | -- 89 | 90 | LOCK TABLES `formats` WRITE; 91 | /*!40000 ALTER TABLE `formats` DISABLE KEYS */; 92 | INSERT INTO `formats` VALUES (1,'JSON'),(2,'CSV'); 93 | /*!40000 ALTER TABLE `formats` ENABLE KEYS */; 94 | UNLOCK TABLES; 95 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 96 | 97 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 98 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 99 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 100 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 101 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 102 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 103 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 104 | 105 | -- Dump completed on 2017-06-16 20:06:30 106 | -------------------------------------------------------------------------------- /CoordinationService/views/monitoring.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | ChaosMonkeys 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 |

42 |
46 | 48 |
49 |
50 |
51 |

Chaos Monkeys Experimentation Platform

52 | 62 |
63 |
64 |
65 |
66 |
67 |
68 |

Connected services

69 |

Coordination Service local IP:

70 | 71 |

72 |

77 | 78 | 79 |

80 | 81 | 82 | 83 | 84 | 85 | 86 | 87 | 88 | 89 |
Color Code
OK (lost contact for less than 1 minute)Possible issue (lost contact for 1 to 2 minutes)Service disconnected (lost contact for 2 to 3 minutes)
90 |

91 |
92 | 93 |
94 |
95 |
96 | 97 | 98 | -------------------------------------------------------------------------------- /SQL/06-23-2017-V0.3.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.13 Distrib 5.7.17, for osx10.12 (x86_64) 2 | -- 3 | -- Host: localhost Database: ConfigurationDatabase 4 | -- ------------------------------------------------------ 5 | -- Server version 5.7.18 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Table structure for table `connected_services` 20 | -- 21 | 22 | DROP TABLE IF EXISTS `connected_services`; 23 | /*!40101 SET @saved_cs_client = @@character_set_client */; 24 | /*!40101 SET character_set_client = utf8 */; 25 | CREATE TABLE `connected_services` ( 26 | `id` int(11) NOT NULL AUTO_INCREMENT, 27 | `ip_address` varchar(255) NOT NULL, 28 | `type` varchar(255) NOT NULL, 29 | `name` varchar(50) DEFAULT NULL, 30 | `description` varchar(255) DEFAULT NULL, 31 | `status` varchar(255) DEFAULT NULL, 32 | `last_updated` datetime DEFAULT NULL, 33 | PRIMARY KEY (`id`), 34 | UNIQUE KEY `ip_address_UNIQUE` (`ip_address`) 35 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 36 | /*!40101 SET character_set_client = @saved_cs_client */; 37 | 38 | -- 39 | -- Dumping data for table `connected_services` 40 | -- 41 | 42 | LOCK TABLES `connected_services` WRITE; 43 | /*!40000 ALTER TABLE `connected_services` DISABLE KEYS */; 44 | /*!40000 ALTER TABLE `connected_services` ENABLE KEYS */; 45 | UNLOCK TABLES; 46 | 47 | -- 48 | -- Table structure for table `datasets` 49 | -- 50 | 51 | DROP TABLE IF EXISTS `datasets`; 52 | /*!40101 SET @saved_cs_client = @@character_set_client */; 53 | /*!40101 SET character_set_client = utf8 */; 54 | CREATE TABLE `datasets` ( 55 | `id` int(11) NOT NULL AUTO_INCREMENT, 56 | `user_id` varchar(255) NOT NULL, 57 | `project_id` varchar(255) NOT NULL, 58 | `name` varchar(255) NOT NULL, 59 | `path` varchar(255) NOT NULL, 60 | `description` varchar(255) DEFAULT NULL, 61 | `format` varchar(20) DEFAULT NULL, 62 | PRIMARY KEY (`id`) 63 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 64 | /*!40101 SET character_set_client = @saved_cs_client */; 65 | 66 | -- 67 | -- Dumping data for table `datasets` 68 | -- 69 | 70 | LOCK TABLES `datasets` WRITE; 71 | /*!40000 ALTER TABLE `datasets` DISABLE KEYS */; 72 | /*!40000 ALTER TABLE `datasets` ENABLE KEYS */; 73 | UNLOCK TABLES; 74 | 75 | -- 76 | -- Table structure for table `formats` 77 | -- 78 | 79 | DROP TABLE IF EXISTS `formats`; 80 | /*!40101 SET @saved_cs_client = @@character_set_client */; 81 | /*!40101 SET character_set_client = utf8 */; 82 | CREATE TABLE `formats` ( 83 | `id` int(11) NOT NULL AUTO_INCREMENT, 84 | `format` varchar(20) NOT NULL, 85 | PRIMARY KEY (`id`) 86 | ) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8; 87 | /*!40101 SET character_set_client = @saved_cs_client */; 88 | 89 | -- 90 | -- Dumping data for table `formats` 91 | -- 92 | 93 | LOCK TABLES `formats` WRITE; 94 | /*!40000 ALTER TABLE `formats` DISABLE KEYS */; 95 | INSERT INTO `formats` VALUES (1,'JSON'),(2,'CSV'); 96 | /*!40000 ALTER TABLE `formats` ENABLE KEYS */; 97 | UNLOCK TABLES; 98 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 99 | 100 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 101 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 102 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 103 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 104 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 105 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 106 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 107 | 108 | -- Dump completed on 2017-06-23 20:04:29 109 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/Utilities/DataVerification.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.Utilities; 2 | 3 | import com.google.gson.Gson; 4 | import com.google.gson.JsonSyntaxException; 5 | import java.io.*; 6 | import java.nio.file.Path; 7 | import java.util.Optional; 8 | 9 | 10 | public class DataVerification { 11 | 12 | private String fileName; 13 | private File datasetFile; 14 | private Path datasetPath; 15 | private Gson gson; 16 | 17 | 18 | public DataVerification(String fileName) { 19 | this.fileName = fileName; 20 | gson = new Gson(); 21 | 22 | } 23 | 24 | public DataVerification(File file){ 25 | this(file.getName()); 26 | this.datasetFile = file; 27 | this.datasetPath = file.toPath(); 28 | } 29 | 30 | public static void main(String[] args) { 31 | DataVerification dv = new DataVerification("test.json"); 32 | Logger.Info("datasets verification result: " + dv.isFileValidate()); 33 | } 34 | 35 | /** 36 | * Pass a filename and see if it is valid json or csv filename 37 | * @param fileName 38 | * @return 39 | */ 40 | public static boolean isFileExtensionValidate(String fileName){ 41 | Optional extensionOptional = Optional.ofNullable(getFileExtension(fileName)); 42 | boolean valid = false; 43 | if(extensionOptional.isPresent()) { 44 | String str = extensionOptional.get(); 45 | if (str.toLowerCase().equals("json") || str.toLowerCase().equals("csv")) { 46 | return true; 47 | } 48 | } 49 | return false; 50 | } 51 | 52 | public boolean isFileValidate() { 53 | Optional extensionOptional = Optional.ofNullable(getFileExtension(this.fileName)); 54 | boolean valid = false; 55 | if(extensionOptional.isPresent()) { 56 | String str = extensionOptional.get(); 57 | if (str.toLowerCase().equals("json") || str.toLowerCase().equals("csv")) { 58 | return true; 59 | } 60 | } 61 | return false; 62 | } 63 | 64 | public static String getFileExtension(String fileName) { 65 | if (fileName == null || fileName.length() == 0){ 66 | return null; 67 | } 68 | if (fileName.lastIndexOf(".") != -1 && fileName.lastIndexOf(".") != 0){ 69 | return fileName.substring(fileName.lastIndexOf(".") + 1); 70 | } else{ 71 | return ""; 72 | } 73 | } 74 | 75 | 76 | public boolean isJSONValid() { 77 | InputStream inputStream=null; 78 | InputStreamReader reader=null; 79 | try { 80 | inputStream = new FileInputStream(datasetFile); 81 | reader = new InputStreamReader(inputStream); 82 | gson.fromJson(reader, Object.class); 83 | reader.close(); 84 | return true; 85 | 86 | } catch (FileNotFoundException ex) { 87 | try{ 88 | if(inputStream!=null) 89 | inputStream.close(); 90 | Logger.Exception("file not found"); 91 | return false; 92 | } catch (IOException IOExp) { 93 | return false; 94 | } 95 | } catch (JsonSyntaxException ex) { 96 | try{ 97 | if(reader!=null) 98 | reader.close(); 99 | Logger.Exception("JSON syntax error"); 100 | return false; 101 | } catch (IOException IOEx) { 102 | return false; 103 | } 104 | }catch (IOException ex) { 105 | try{ 106 | reader.close(); 107 | Logger.Exception("IOException"); 108 | return false; 109 | } catch (IOException IOEx) { 110 | return false; 111 | } 112 | 113 | } 114 | 115 | } 116 | 117 | public boolean isCSVValid() { 118 | if(getFileExtension(fileName).toLowerCase().equals("csv")){ 119 | return true; 120 | } 121 | return false; 122 | } 123 | 124 | 125 | } 126 | 127 | 128 | -------------------------------------------------------------------------------- /CoordinationService/routes/dynamicIp.js: -------------------------------------------------------------------------------- 1 | var express = require('express'); 2 | var router = express.Router(); 3 | var mysql = require('mysql'); 4 | var utilities = require('./dbUtilities'); 5 | var log = require('./logUtilities'); 6 | 7 | /* Convention: serviceType must be in the format specified by "ServiceTypes.xlsx", i.e. "DataInput-", "AlgInput-", "Train-" or "Exec-" */ 8 | router.get('/get/:serviceType', function loadIp(req, res, next) { 9 | //Getting the elements of the serviceType value 10 | var serviceType = req.params['serviceType']; 11 | loadServiceIp(serviceType, function (value) { 12 | res.send(value); 13 | }); 14 | }); 15 | 16 | //Allows to get experiment IP from experiment id 17 | router.get('/getExperimentIp/:experimentTaskId', function loadIp(req, res, next) { 18 | //Getting the elements of the serviceType value 19 | var taskId = req.params['experimentTaskId']; 20 | 21 | //Querying for experiment type 22 | //Connect to DB 23 | var connection = createDbConnection(); 24 | connection.connect(); 25 | var query = 'select tasks.`type` as `type`, algorithms.`language` as `language` from tasks inner join algorithms on tasks.algorithm_id=algorithms.id where tasks.id=' + taskId; 26 | logMessage(false, log.operationType.QueryData, new Date(), query); 27 | //Return all services 28 | var results = connection.query(query, function getTaskType(err, rows, fields) { 29 | if (err) { // pass the err to error handler 30 | logMessage(true, log.errorType.DBError, new Date(), err); 31 | } 32 | else { 33 | logMessage(false, log.operationType.ResponseReceived, new Date(), "Received the task type"); 34 | if (rows.length > 0) { 35 | //Deciding which service to call 36 | var taskType = rows[0]['type']; 37 | var taskLanguage = rows[0]['language']; 38 | var serviceType = '-' + taskLanguage; 39 | if (taskType === 'Training') 40 | serviceType = 'Train' + serviceType; 41 | else 42 | serviceType = 'Exec' + serviceType; 43 | 44 | //Getting service IP 45 | loadServiceIp(serviceType, function (value) { 46 | res.send(value); 47 | }); 48 | } else { 49 | res.send({ message: 'No service of type ' + serviceType + ' found.', ip: undefined }) 50 | } 51 | } 52 | }); 53 | //Closing connection 54 | connection.end(); 55 | }); 56 | 57 | loadServiceIp = function (serviceType, callback) { 58 | if (serviceType) { 59 | var typeElements = serviceType.split('-'); 60 | if (typeElements.length === 2 && (typeElements[0] === 'DataInput' || typeElements[0] === 'AlgInput' || typeElements[0] === 'Train' || typeElements[0] === 'Exec')) { 61 | //Connect to DB 62 | var connection = createDbConnection(); 63 | connection.connect(); 64 | var query = 'select type, ip_address from connected_services where `type`="' + serviceType + '"'; 65 | logMessage(false, log.operationType.QueryData, new Date(), query); 66 | //Return all services 67 | connection.query(query, function selectServiceTypes(err, rows, fields) { 68 | if (err) { // pass the err to error handler 69 | logMessage(true, log.errorType.DBError, new Date(), err); 70 | callback(null); 71 | } 72 | else { 73 | if (rows.length > 0) { 74 | logMessage(false, log.operationType.ResponseReceived, new Date(), "Received the service types list"); 75 | callback({ message: 'success', ip: rows[0]['ip_address'] }); 76 | } 77 | else { 78 | logMessage(true, log.errorType.LogicError, new Date(), 'No service of type ' + serviceType + ' found.'); 79 | callback({ message: 'No service of type ' + serviceType + ' found.', ip: undefined }); 80 | } 81 | } 82 | }); 83 | connection.end(); 84 | } else { 85 | logMessage(true, log.errorType.LogicError, new Date(), 'Unknown service type.'); 86 | callback({ message: 'Unknown service type.', ip: undefined }); 87 | } 88 | } else { 89 | logMessage(true, log.errorType.LogicError, new Date(), 'Unknown service type.'); 90 | callback({ message: 'Unknown service type.', ip: undefined }); 91 | } 92 | } 93 | 94 | module.exports = router; -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/Utilities/ArrayUtils.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.Utilities; 2 | 3 | import java.lang.reflect.Array; 4 | 5 | /** 6 | *

Operations on arrays, primitive arrays (like {@code int[]}) and 7 | * primitive wrapper arrays (like {@code Integer[]}). 8 | * 9 | *

This class tries to handle {@code null} input gracefully. 10 | * An exception will not be thrown for a {@code null} 11 | * array input. However, an Object array that contains a {@code null} 12 | * element may throw an exception. Each method documents its behaviour. 13 | * Portion code from commons-lang 14 | */ 15 | public class ArrayUtils { 16 | /** 17 | *

Checks if an array of Objects is empty or {@code null}. 18 | * 19 | * @param array the array to test 20 | * @return {@code true} if the array is empty or {@code null} 21 | */ 22 | public static boolean isEmpty(final Object[] array) { 23 | return getLength(array) == 0; 24 | } 25 | 26 | /** 27 | *

Checks if an array of primitive longs is empty or {@code null}. 28 | * 29 | * @param array the array to test 30 | * @return {@code true} if the array is empty or {@code null} 31 | */ 32 | public static boolean isEmpty(final long[] array) { 33 | return getLength(array) == 0; 34 | } 35 | 36 | /** 37 | *

Checks if an array of primitive ints is empty or {@code null}. 38 | * 39 | * @param array the array to test 40 | * @return {@code true} if the array is empty or {@code null} 41 | */ 42 | public static boolean isEmpty(final int[] array) { 43 | return getLength(array) == 0; 44 | } 45 | 46 | /** 47 | *

Checks if an array of primitive shorts is empty or {@code null}. 48 | * 49 | * @param array the array to test 50 | * @return {@code true} if the array is empty or {@code null} 51 | */ 52 | public static boolean isEmpty(final short[] array) { 53 | return getLength(array) == 0; 54 | } 55 | 56 | /** 57 | *

Checks if an array of primitive chars is empty or {@code null}. 58 | * 59 | * @param array the array to test 60 | * @return {@code true} if the array is empty or {@code null} 61 | 62 | */ 63 | public static boolean isEmpty(final char[] array) { 64 | return getLength(array) == 0; 65 | } 66 | 67 | /** 68 | *

Checks if an array of primitive bytes is empty or {@code null}. 69 | * 70 | * @param array the array to test 71 | * @return {@code true} if the array is empty or {@code null} 72 | 73 | */ 74 | public static boolean isEmpty(final byte[] array) { 75 | return getLength(array) == 0; 76 | } 77 | 78 | /** 79 | *

Checks if an array of primitive doubles is empty or {@code null}. 80 | * 81 | * @param array the array to test 82 | * @return {@code true} if the array is empty or {@code null} 83 | */ 84 | public static boolean isEmpty(final double[] array) { 85 | return getLength(array) == 0; 86 | } 87 | 88 | /** 89 | *

Checks if an array of primitive floats is empty or {@code null}. 90 | * 91 | * @param array the array to test 92 | * @return {@code true} if the array is empty or {@code null} 93 | 94 | */ 95 | public static boolean isEmpty(final float[] array) { 96 | return getLength(array) == 0; 97 | } 98 | 99 | /** 100 | *

Checks if an array of primitive booleans is empty or {@code null}. 101 | * 102 | * @param array the array to test 103 | * @return {@code true} if the array is empty or {@code null} 104 | */ 105 | public static boolean isEmpty(final boolean[] array) { 106 | return getLength(array) == 0; 107 | } 108 | 109 | //----------------------------------------------------------------------- 110 | /** 111 | *

Returns the length of the specified array. 112 | * This method can deal with {@code Object} arrays and with primitive arrays. 113 | * 114 | *

If the input array is {@code null}, {@code 0} is returned. 115 | * 116 | * 

117 |      * ArrayUtils.getLength(null)            = 0
118 |      * ArrayUtils.getLength([])              = 0
119 |      * ArrayUtils.getLength([null])          = 1
120 |      * ArrayUtils.getLength([true, false])   = 2
121 |      * ArrayUtils.getLength([1, 2, 3])       = 3
122 |      * ArrayUtils.getLength(["a", "b", "c"]) = 3
123 |      *
124 | * 125 | * @param array the array to retrieve the length from, may be null 126 | * @return The length of the array, or {@code 0} if the array is {@code null} 127 | * @throws IllegalArgumentException if the object argument is not an array. 128 | 129 | */ 130 | public static int getLength(final Object array) { 131 | if (array == null) { 132 | return 0; 133 | } 134 | return Array.getLength(array); 135 | } 136 | } 137 | -------------------------------------------------------------------------------- /SQL/06-26-2017-V0.4.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.13 Distrib 5.7.17, for osx10.12 (x86_64) 2 | -- 3 | -- Host: localhost Database: ConfigurationDatabase 4 | -- ------------------------------------------------------ 5 | -- Server version 5.7.18 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Table structure for table `connected_services` 20 | -- 21 | 22 | DROP TABLE IF EXISTS `connected_services`; 23 | /*!40101 SET @saved_cs_client = @@character_set_client */; 24 | /*!40101 SET character_set_client = utf8 */; 25 | CREATE TABLE `connected_services` ( 26 | `id` int(11) NOT NULL AUTO_INCREMENT, 27 | `ip_address` varchar(255) NOT NULL, 28 | `type` varchar(255) NOT NULL, 29 | `name` varchar(50) DEFAULT NULL, 30 | `description` varchar(255) DEFAULT NULL, 31 | `status` varchar(255) DEFAULT NULL, 32 | `last_updated` datetime DEFAULT NULL, 33 | PRIMARY KEY (`id`), 34 | UNIQUE KEY `ip_address_UNIQUE` (`ip_address`) 35 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 36 | /*!40101 SET character_set_client = @saved_cs_client */; 37 | 38 | -- 39 | -- Dumping data for table `connected_services` 40 | -- 41 | 42 | LOCK TABLES `connected_services` WRITE; 43 | /*!40000 ALTER TABLE `connected_services` DISABLE KEYS */; 44 | /*!40000 ALTER TABLE `connected_services` ENABLE KEYS */; 45 | UNLOCK TABLES; 46 | 47 | -- 48 | -- Table structure for table `datasets` 49 | -- 50 | 51 | DROP TABLE IF EXISTS `datasets`; 52 | /*!40101 SET @saved_cs_client = @@character_set_client */; 53 | /*!40101 SET character_set_client = utf8 */; 54 | CREATE TABLE `datasets` ( 55 | `id` int(11) NOT NULL AUTO_INCREMENT, 56 | `user_id` varchar(255) NOT NULL, 57 | `project_id` varchar(255) NOT NULL, 58 | `name` varchar(255) NOT NULL, 59 | `path` varchar(255) NOT NULL, 60 | `description` varchar(255) DEFAULT NULL, 61 | `format` varchar(20) NOT NULL, 62 | PRIMARY KEY (`id`) 63 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 64 | /*!40101 SET character_set_client = @saved_cs_client */; 65 | 66 | -- 67 | -- Dumping data for table `datasets` 68 | -- 69 | 70 | LOCK TABLES `datasets` WRITE; 71 | /*!40000 ALTER TABLE `datasets` DISABLE KEYS */; 72 | /*!40000 ALTER TABLE `datasets` ENABLE KEYS */; 73 | UNLOCK TABLES; 74 | 75 | -- 76 | -- Table structure for table `experiments` 77 | -- 78 | 79 | DROP TABLE IF EXISTS `experiments`; 80 | /*!40101 SET @saved_cs_client = @@character_set_client */; 81 | /*!40101 SET character_set_client = utf8 */; 82 | CREATE TABLE `experiments` ( 83 | `id` int(11) NOT NULL AUTO_INCREMENT, 84 | `project_id` varchar(255) NOT NULL, 85 | `task_id` varchar(255) NOT NULL, 86 | `experiment_name` varchar(255) NOT NULL, 87 | `start` datetime NOT NULL, 88 | `end` datetime NOT NULL, 89 | `last_status` varchar(25) NOT NULL, 90 | `last_updated` datetime NOT NULL, 91 | `description` varchar(255) DEFAULT NULL, 92 | PRIMARY KEY (`id`) 93 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 94 | /*!40101 SET character_set_client = @saved_cs_client */; 95 | 96 | -- 97 | -- Dumping data for table `experiments` 98 | -- 99 | 100 | LOCK TABLES `experiments` WRITE; 101 | /*!40000 ALTER TABLE `experiments` DISABLE KEYS */; 102 | /*!40000 ALTER TABLE `experiments` ENABLE KEYS */; 103 | UNLOCK TABLES; 104 | 105 | -- 106 | -- Table structure for table `formats` 107 | -- 108 | 109 | DROP TABLE IF EXISTS `formats`; 110 | /*!40101 SET @saved_cs_client = @@character_set_client */; 111 | /*!40101 SET character_set_client = utf8 */; 112 | CREATE TABLE `formats` ( 113 | `id` int(11) NOT NULL AUTO_INCREMENT, 114 | `format` varchar(20) NOT NULL, 115 | PRIMARY KEY (`id`) 116 | ) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8; 117 | /*!40101 SET character_set_client = @saved_cs_client */; 118 | 119 | -- 120 | -- Dumping data for table `formats` 121 | -- 122 | 123 | LOCK TABLES `formats` WRITE; 124 | /*!40000 ALTER TABLE `formats` DISABLE KEYS */; 125 | INSERT INTO `formats` VALUES (1,'JSON'),(2,'CSV'); 126 | /*!40000 ALTER TABLE `formats` ENABLE KEYS */; 127 | UNLOCK TABLES; 128 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 129 | 130 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 131 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 132 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 133 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 134 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 135 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 136 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 137 | 138 | -- Dump completed on 2017-06-26 19:37:29 139 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/train/task/ResourceInfo.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.train.task; 2 | 3 | import com.chaosmonkeys.Utilities.FileUtils; 4 | import com.chaosmonkeys.train.Constants; 5 | 6 | import java.io.File; 7 | 8 | /** 9 | * Class used to store experiment required 10 | * resource info 11 | * paths are optional 12 | * Files are necessary to be initialize 13 | */ 14 | public class ResourceInfo { 15 | 16 | // Resource Folder Reference 17 | protected File datasetFile; 18 | protected File algorithmFolder; 19 | protected File modelFolder; 20 | protected File workspaceFolder; 21 | 22 | 23 | //** Constructor --------------------------------------------------------------------------- 24 | 25 | 26 | public ResourceInfo(File datasetFile, File algorithmFolder, File workspaceFolder) { 27 | this.datasetFile = datasetFile; 28 | this.algorithmFolder = algorithmFolder; 29 | this.workspaceFolder = workspaceFolder; 30 | } 31 | public ResourceInfo(File datasetFile, File algorithmFolder, File modelFolder, File workspaceFolder) { 32 | this(datasetFile, algorithmFolder, workspaceFolder); 33 | if(null != modelFolder){ 34 | this.modelFolder = modelFolder; 35 | } 36 | } 37 | 38 | //** Check --------------------------------------------------------------------------------- 39 | 40 | /** 41 | * Check whether ResourceInfo are ready 42 | * @param type 43 | * @return 44 | */ 45 | public boolean checkRequirement(TaskType type){ 46 | 47 | boolean modelValid = false; 48 | 49 | boolean datasetValid = FileUtils.checkDatasetFileValid(datasetFile); 50 | boolean algrValid = FileUtils.checkFolderValid(algorithmFolder); 51 | boolean workspaceValid = FileUtils.checkFolderValid(workspaceFolder); 52 | 53 | if(type == TaskType.TRAIN){ 54 | modelValid = true; 55 | }else{ 56 | modelValid = FileUtils.checkFolderValid(modelFolder); 57 | } 58 | boolean valid = datasetValid && algrValid && workspaceValid && modelValid; 59 | return valid; 60 | } 61 | /** 62 | * Check whether ResourceInfo are ready 63 | * @param type Constants.TYPE_TRAIN or Constants.TYPE_EXECUTION 64 | * @return 65 | */ 66 | public boolean checkRequirement(String type){ 67 | 68 | boolean modelValid = false; 69 | 70 | boolean datasetValid = FileUtils.checkDatasetFileValid(datasetFile); 71 | boolean algrValid = FileUtils.checkFolderValid(algorithmFolder); 72 | boolean workspaceValid = FileUtils.checkFolderValid(workspaceFolder); 73 | 74 | if(type.equals(Constants.TYPE_TRAIN)){ 75 | modelValid = true; 76 | }else{ 77 | modelValid = FileUtils.checkFolderValid(modelFolder); 78 | } 79 | boolean valid = datasetValid && algrValid && workspaceValid && modelValid; 80 | return valid; 81 | } 82 | 83 | //** Builder ------------------------------------------------------------------------------- 84 | public static class ResourceInfoBuilder { 85 | private File datasetFolder; 86 | private File algorithmFolder; 87 | private File workspaceFolder; 88 | private File modelFolder; 89 | 90 | public ResourceInfoBuilder setDatasetFolder(File datasetFolder) { 91 | this.datasetFolder = datasetFolder; 92 | return this; 93 | } 94 | 95 | public ResourceInfoBuilder setAlgorithmFolder(File algorithmFolder) { 96 | this.algorithmFolder = algorithmFolder; 97 | return this; 98 | } 99 | 100 | public ResourceInfoBuilder setWorkspaceFolder(File workspaceFolder) { 101 | this.workspaceFolder = workspaceFolder; 102 | return this; 103 | } 104 | public ResourceInfoBuilder setModelFolder(File modelFolder) { 105 | this.modelFolder = modelFolder; 106 | return this; 107 | } 108 | 109 | public ResourceInfo build() { 110 | return new ResourceInfo(datasetFolder, algorithmFolder, modelFolder, workspaceFolder); 111 | } 112 | } 113 | 114 | //** Getter and Setter --------------------------------------------------------------------- 115 | 116 | public File getDatasetFile() { 117 | return datasetFile; 118 | } 119 | 120 | public void setDatasetFile(File datasetFile) { 121 | this.datasetFile = datasetFile; 122 | } 123 | 124 | public File getAlgorithmFolder() { 125 | return algorithmFolder; 126 | } 127 | 128 | public void setAlgorithmFolder(File algorithmFolder) { 129 | this.algorithmFolder = algorithmFolder; 130 | } 131 | 132 | public File getWorkspaceFolder() { 133 | return workspaceFolder; 134 | } 135 | 136 | public void setWorkspaceFolder(File workspaceFolder) { 137 | this.workspaceFolder = workspaceFolder; 138 | } 139 | 140 | public File getModelFolder() { 141 | return modelFolder; 142 | } 143 | 144 | public void setModelFolder(File modelFolder) { 145 | this.modelFolder = modelFolder; 146 | } 147 | 148 | 149 | } 150 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/train/task/TrainingTaskManager.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.train.task; 2 | 3 | 4 | import com.chaosmonkeys.Utilities.db.DbUtils; 5 | import com.chaosmonkeys.dao.Experiment; 6 | import com.chaosmonkeys.train.task.interfaces.OnTaskUpdateListener; 7 | 8 | import java.sql.Timestamp; 9 | import java.time.Instant; 10 | import java.util.Map; 11 | import java.util.concurrent.ConcurrentHashMap; 12 | import java.util.concurrent.atomic.AtomicInteger; 13 | 14 | /** 15 | * Traning task manager used for handling task callback 16 | * adding task, invoking updating database and provide proper info for 17 | * heartbeats 18 | */ 19 | 20 | public enum TrainingTaskManager implements TaskManager{ 21 | // singleton 22 | INSTANCE; 23 | 24 | private java.util.concurrent.atomic.AtomicInteger runningTaskNum = new AtomicInteger(); 25 | // taskId -> Task 26 | private Map taskMap = new ConcurrentHashMap<>(); 27 | // map experiment name to task ID because the frontend would like to use experiment name as identifier 28 | private Map taskNameIdMap = new ConcurrentHashMap<>(); 29 | 30 | public boolean submitTask(BaseTaskInfo taskInfo){ 31 | TrainingTaskInfo trainTaskInfo = (TrainingTaskInfo) taskInfo; 32 | TrainingTask trainingTask = new TrainingTask(trainTaskInfo, mOnTaskUpdateListener); 33 | taskMap.put(trainTaskInfo.getTaskId(), trainingTask); 34 | taskNameIdMap.put(trainTaskInfo.getExperimentName(), trainTaskInfo.getTaskId()); 35 | trainingTask.initialize(); 36 | 37 | // TODO: discover some error case which may return false 38 | return true; 39 | } 40 | 41 | @Override 42 | public boolean cancelTask(String taskId){ 43 | TrainingTask trainTask = taskMap.get(taskId); 44 | trainTask.cancelWorks(); 45 | return true; 46 | } 47 | 48 | @Override 49 | public boolean cancelTaskByExperimentName(String experimentName){ 50 | String taskId = taskNameIdMap.get(experimentName); 51 | TrainingTask trainTask = taskMap.get(taskId); 52 | trainTask.cancelWorks(); 53 | return true; 54 | } 55 | 56 | 57 | 58 | /** 59 | * Update experiment task status in database 60 | * @param taskId task ID, the UUID one 61 | * @param state new task state 62 | */ 63 | public TrainingTask updateTaskStatus(String taskId, TaskState state){ 64 | TrainingTask task = taskMap.get(taskId); 65 | TrainingTaskInfo trainTaskInfo = task.getTaskInfo(); 66 | // set initializing status and update database record 67 | DbUtils.openConnection(); 68 | task.setState(state); 69 | Experiment exp = DbUtils.getExperimentModelByName(trainTaskInfo.getExperimentName()); 70 | exp.set("last_status",task.getState().StringValue()); 71 | //TODO: handle locale problem in the future 72 | Timestamp nowTime = Timestamp.from(Instant.now()); 73 | exp.setTimestamp("last_updated",nowTime); 74 | if(state == TaskState.INITIALIZING){ 75 | exp.setTimestamp("start",nowTime); 76 | } 77 | if(state.value() > TaskState.STARTED.value()){ 78 | exp.setTimestamp("end",nowTime); 79 | } 80 | if(state == TaskState.ERROR){ 81 | String errorMsg = task.getErrorMsg(); 82 | exp.set("error_message",errorMsg); // experiment error message field 83 | } 84 | //TODO: add error handler for connection error 85 | exp.save(); 86 | DbUtils.closeConnection(); 87 | return task; 88 | } 89 | 90 | OnTaskUpdateListener mOnTaskUpdateListener = new OnTaskUpdateListener() { 91 | // start initializing 92 | @Override 93 | public void onInit(String taskId) { 94 | updateTaskStatus(taskId, TaskState.INITIALIZING); 95 | runningTaskNum.incrementAndGet(); 96 | } 97 | // task workspace has been initialized, the manager should update record and let the task start if available 98 | @Override 99 | public void onInitialized(String taskId) { 100 | TrainingTask task = updateTaskStatus(taskId, TaskState.INITIALIZED); 101 | // task is permitted to run 102 | task.performTask(); 103 | } 104 | 105 | @Override 106 | public void onStarted(String taskId) { 107 | updateTaskStatus(taskId, TaskState.STARTED); 108 | } 109 | 110 | @Override 111 | public void onCancelled(String taskId) { 112 | TrainingTask task = updateTaskStatus(taskId, TaskState.CANCELLED); 113 | runningTaskNum.decrementAndGet(); 114 | } 115 | 116 | @Override 117 | public void onSuccess(String taskId) { 118 | TrainingTask task = updateTaskStatus(taskId, TaskState.SUCCESS); 119 | runningTaskNum.decrementAndGet(); 120 | task.cleanUp(); 121 | } 122 | 123 | @Override 124 | public void onError(Throwable ex, String taskId, String errorMessage) { 125 | updateTaskStatus(taskId, TaskState.ERROR); 126 | runningTaskNum.decrementAndGet(); 127 | } 128 | }; 129 | 130 | public int getRunningTaskNum(){ 131 | return runningTaskNum.get(); 132 | } 133 | 134 | } 135 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/train/task/ExecutionTaskManager.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.train.task; 2 | 3 | import com.chaosmonkeys.Utilities.db.DbUtils; 4 | import com.chaosmonkeys.dao.Experiment; 5 | import com.chaosmonkeys.train.task.interfaces.OnTaskUpdateListener; 6 | 7 | import java.sql.Timestamp; 8 | import java.time.Instant; 9 | import java.util.Map; 10 | import java.util.concurrent.ConcurrentHashMap; 11 | import java.util.concurrent.atomic.AtomicInteger; 12 | 13 | /** 14 | * Execution/Prediction task manager used for handling task callback 15 | * adding task, invoking updating database and provide proper info for 16 | * heartbeats (maybe) 17 | */ 18 | public enum ExecutionTaskManager implements TaskManager{ 19 | // singleton 20 | INSTANCE; 21 | 22 | AtomicInteger runningTaskNum = new AtomicInteger(); 23 | // taskId -> Task 24 | private Map taskMap = new ConcurrentHashMap<>(); 25 | // map experiment name to task ID because the frontend would like to use experiment name as identifier 26 | private Map taskNameIdMap = new ConcurrentHashMap<>(); 27 | 28 | 29 | @Override 30 | public boolean submitTask(BaseTaskInfo taskInfo) { 31 | ExecutionTaskInfo predictTaskInfo = (ExecutionTaskInfo) taskInfo; 32 | ExecutionTask predictTask = new ExecutionTask(predictTaskInfo, mOnTaskUpdateListener); 33 | // update the reference of experiment name -> taskId -> task 34 | taskMap.put(predictTaskInfo.getTaskId(), predictTask); 35 | taskNameIdMap.put(predictTaskInfo.getExperimentName(), predictTaskInfo.getTaskId()); 36 | predictTask.initialize(); 37 | 38 | return true; 39 | } 40 | 41 | @Override 42 | public boolean cancelTask(String taskId) { 43 | ExecutionTask predictTask = taskMap.get(taskId); 44 | predictTask.cancelWorks(); 45 | return true; 46 | } 47 | 48 | @Override 49 | public boolean cancelTaskByExperimentName(String experimentName) { 50 | String taskId = taskNameIdMap.get(experimentName); 51 | ExecutionTask predictTask = taskMap.get(taskId); 52 | predictTask.cancelWorks(); 53 | return true; 54 | } 55 | 56 | //--------------- 57 | /** 58 | * Update experiment task status in database 59 | * @param taskId abstract task ID (the UUID one) 60 | * @param state new task state 61 | */ 62 | public ExecutionTask updateTaskStatus(String taskId, TaskState state){ 63 | ExecutionTask task = taskMap.get(taskId); 64 | BaseTaskInfo trainTaskInfo = task.getTaskInfo(); 65 | // set initializing status and update database record 66 | DbUtils.openConnection(); 67 | task.setState(state); 68 | Experiment exp = DbUtils.getExperimentModelByName(trainTaskInfo.getExperimentName()); 69 | exp.set("last_status",task.getState().StringValue()); 70 | //TODO: handle locale problem in the future 71 | Timestamp nowTime = Timestamp.from(Instant.now()); 72 | exp.setTimestamp("last_updated",nowTime); 73 | if(state == TaskState.INITIALIZING){ 74 | exp.setTimestamp("start",nowTime); 75 | } 76 | if(state.value() > TaskState.STARTED.value()){ 77 | exp.setTimestamp("end",nowTime); 78 | } 79 | if(state == TaskState.ERROR){ 80 | String errorMsg = task.getErrorMsg(); 81 | exp.set("error_message",errorMsg); 82 | } 83 | //TODO: add error handler for connection error 84 | exp.save(); 85 | DbUtils.closeConnection(); 86 | return task; 87 | } 88 | 89 | /** 90 | * Listener for monitoring and controling the lifecycle of tasks 91 | */ 92 | OnTaskUpdateListener mOnTaskUpdateListener = new OnTaskUpdateListener() { 93 | // start initializing 94 | @Override 95 | public void onInit(String taskId) { 96 | updateTaskStatus(taskId, TaskState.INITIALIZING); 97 | runningTaskNum.incrementAndGet(); 98 | } 99 | // task workspace has been initialized, the manager should update record and let the task start if available 100 | @Override 101 | public void onInitialized(String taskId) { 102 | ExecutionTask task = updateTaskStatus(taskId, TaskState.INITIALIZED); 103 | // task is permitted to run 104 | task.performTask(); 105 | } 106 | 107 | @Override 108 | public void onStarted(String taskId) { 109 | updateTaskStatus(taskId, TaskState.STARTED); 110 | } 111 | 112 | @Override 113 | public void onCancelled(String taskId) { 114 | updateTaskStatus(taskId, TaskState.CANCELLED); 115 | runningTaskNum.decrementAndGet(); 116 | } 117 | 118 | @Override 119 | public void onSuccess(String taskId) { 120 | ExecutionTask task = updateTaskStatus(taskId, TaskState.SUCCESS); 121 | runningTaskNum.decrementAndGet(); 122 | task.cleanUp(); 123 | } 124 | 125 | @Override 126 | public void onError(Throwable ex, String taskId, String errorMessage) { 127 | updateTaskStatus(taskId, TaskState.ERROR); 128 | runningTaskNum.decrementAndGet(); 129 | } 130 | }; 131 | 132 | public int getRunningTaskNum(){ 133 | return runningTaskNum.get(); 134 | } 135 | } 136 | -------------------------------------------------------------------------------- /SQL/06-27-2017-V0.5.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.13 Distrib 5.7.17, for osx10.12 (x86_64) 2 | -- 3 | -- Host: localhost Database: ConfigurationDatabase 4 | -- ------------------------------------------------------ 5 | -- Server version 5.7.18 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Table structure for table `connected_services` 20 | -- 21 | 22 | DROP TABLE IF EXISTS `connected_services`; 23 | /*!40101 SET @saved_cs_client = @@character_set_client */; 24 | /*!40101 SET character_set_client = utf8 */; 25 | CREATE TABLE `connected_services` ( 26 | `id` int(11) NOT NULL AUTO_INCREMENT, 27 | `ip_address` varchar(255) NOT NULL, 28 | `type` varchar(255) NOT NULL, 29 | `name` varchar(50) DEFAULT NULL, 30 | `description` varchar(255) DEFAULT NULL, 31 | `status` varchar(255) DEFAULT NULL, 32 | `last_updated` datetime DEFAULT NULL, 33 | PRIMARY KEY (`id`), 34 | UNIQUE KEY `ip_address_UNIQUE` (`ip_address`) 35 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 36 | /*!40101 SET character_set_client = @saved_cs_client */; 37 | 38 | -- 39 | -- Dumping data for table `connected_services` 40 | -- 41 | 42 | LOCK TABLES `connected_services` WRITE; 43 | /*!40000 ALTER TABLE `connected_services` DISABLE KEYS */; 44 | /*!40000 ALTER TABLE `connected_services` ENABLE KEYS */; 45 | UNLOCK TABLES; 46 | 47 | -- 48 | -- Table structure for table `datasets` 49 | -- 50 | 51 | DROP TABLE IF EXISTS `datasets`; 52 | /*!40101 SET @saved_cs_client = @@character_set_client */; 53 | /*!40101 SET character_set_client = utf8 */; 54 | CREATE TABLE `datasets` ( 55 | `id` int(11) NOT NULL AUTO_INCREMENT, 56 | `user_id` varchar(255) NOT NULL, 57 | `project_id` varchar(255) NOT NULL, 58 | `name` varchar(255) NOT NULL, 59 | `path` varchar(255) NOT NULL, 60 | `description` varchar(255) DEFAULT NULL, 61 | `format` varchar(20) NOT NULL, 62 | PRIMARY KEY (`id`) 63 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 64 | /*!40101 SET character_set_client = @saved_cs_client */; 65 | 66 | -- 67 | -- Dumping data for table `datasets` 68 | -- 69 | 70 | LOCK TABLES `datasets` WRITE; 71 | /*!40000 ALTER TABLE `datasets` DISABLE KEYS */; 72 | /*!40000 ALTER TABLE `datasets` ENABLE KEYS */; 73 | UNLOCK TABLES; 74 | 75 | -- 76 | -- Table structure for table `experiments` 77 | -- 78 | 79 | DROP TABLE IF EXISTS `experiments`; 80 | /*!40101 SET @saved_cs_client = @@character_set_client */; 81 | /*!40101 SET character_set_client = utf8 */; 82 | CREATE TABLE `experiments` ( 83 | `id` int(11) NOT NULL AUTO_INCREMENT, 84 | `project_id` varchar(255) NOT NULL, 85 | `task_id` varchar(255) NOT NULL, 86 | `experiment_name` varchar(255) NOT NULL, 87 | `start` datetime NOT NULL, 88 | `end` datetime NOT NULL, 89 | `last_status` varchar(25) NOT NULL, 90 | `last_updated` datetime NOT NULL, 91 | `description` varchar(255) DEFAULT NULL, 92 | PRIMARY KEY (`id`) 93 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 94 | /*!40101 SET character_set_client = @saved_cs_client */; 95 | 96 | -- 97 | -- Dumping data for table `experiments` 98 | -- 99 | 100 | LOCK TABLES `experiments` WRITE; 101 | /*!40000 ALTER TABLE `experiments` DISABLE KEYS */; 102 | /*!40000 ALTER TABLE `experiments` ENABLE KEYS */; 103 | UNLOCK TABLES; 104 | 105 | -- 106 | -- Table structure for table `formats` 107 | -- 108 | 109 | DROP TABLE IF EXISTS `formats`; 110 | /*!40101 SET @saved_cs_client = @@character_set_client */; 111 | /*!40101 SET character_set_client = utf8 */; 112 | CREATE TABLE `formats` ( 113 | `id` int(11) NOT NULL AUTO_INCREMENT, 114 | `format` varchar(20) NOT NULL, 115 | PRIMARY KEY (`id`) 116 | ) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8; 117 | /*!40101 SET character_set_client = @saved_cs_client */; 118 | 119 | -- 120 | -- Dumping data for table `formats` 121 | -- 122 | 123 | LOCK TABLES `formats` WRITE; 124 | /*!40000 ALTER TABLE `formats` DISABLE KEYS */; 125 | INSERT INTO `formats` VALUES (1,'JSON'),(2,'CSV'); 126 | /*!40000 ALTER TABLE `formats` ENABLE KEYS */; 127 | UNLOCK TABLES; 128 | 129 | LOCK TABLES `datasets` WRITE; 130 | /*!40000 ALTER TABLE `datasets` DISABLE KEYS */; 131 | INSERT INTO `datasets` VALUES (0, 'Ingo Weber', 'Project Amazon', '100-Sleep-Data', 'amazon', 'This data set includes 100 students\' sleeping data', 'JSON'), (1, 'Len Bass', 'Project Amazon', '30-Sqrill-Data', 'amazon', 'This data set includes 100 students\' sleeping data', 'CSV'); 132 | /*!40000 ALTER TABLE `datasets` ENABLE KEYS */; 133 | UNLOCK TABLES; 134 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 135 | 136 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 137 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 138 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 139 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 140 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 141 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 142 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 143 | 144 | -- Dump completed on 2017-06-26 19:37:29 145 | -------------------------------------------------------------------------------- /JerseyServices/JerseyServices.iml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | 48 | 49 | 50 | 51 | 52 | 53 | 54 | 55 | 56 | 57 | 58 | 59 | 60 | -------------------------------------------------------------------------------- /CoordinationService/routes/experiments.js: -------------------------------------------------------------------------------- 1 | var express = require('express'); 2 | var router = express.Router(); 3 | var path = require('path'); 4 | var utilities = require('./dbUtilities'); 5 | var request = require('request'); 6 | var log = require('./logUtilities'); 7 | var dynamicIp = require('./dynamicIp'); 8 | 9 | //Lists all experiments 10 | router.get("/list", function getExperiments(req, res, next) { 11 | var connection = createDbConnection(); 12 | connection.connect(); 13 | var query = 'select experiments.id as id, experiments.task_id as task_id, experiments.experiment_name as experiment_name, experiments.start as start, experiments.end as end, experiments.last_status as last_status, experiments.last_updated as last_updated, experiments.description as description, experiments.error_message as error_message, predictions.id as prediction_id from experiments left join predictions on experiments.id = predictions.experiment_id'; 14 | logMessage(false, log.operationType.QueryData, new Date(), query); 15 | //Return all services 16 | var results = connection.query(query, function getAllExperiments(err, rows, fields) { 17 | if (err) { // pass the err to error handler 18 | logMessage(true, log.errorType.DBError, new Date(), err); 19 | err.source = 'mysql'; // add error source for tracing 20 | err.status = 500; 21 | next(err); 22 | } 23 | else { 24 | logMessage(false, log.operationType.ResponseReceived, new Date(), "Received the experiments list"); 25 | } 26 | res.json({ experimentsData: rows }); 27 | }); 28 | 29 | //Closing connection 30 | connection.end(); 31 | }); 32 | 33 | /* Create a new experiment */ 34 | router.post('/create', function insertNewExperiment(req, res, next) { 35 | //Connect to DB 36 | var connection = createDbConnection(); 37 | connection.connect(); 38 | var query = 'insert into experiments SET ?'; 39 | logMessage(false, log.operationType.QueryData, new Date(), query); 40 | //Insert tasks information into tasks table 41 | connection.query(query, req.body, function insertExperiment(err, result) { 42 | //Closing connection 43 | connection.end(); 44 | if (err) { // pass the err to error handler 45 | logMessage(true, log.errorType.DBError, new Date(), err); 46 | err.source = 'mysql'; // add error source for tracing 47 | err.status = 500; 48 | next(err); 49 | } else { 50 | logMessage(false, log.operationType.ResponseReceived, new Date(), "Created a new experiment"); 51 | next(); 52 | } 53 | }); 54 | 55 | }, function submitExperimentToRun(req, res, next) { 56 | //Options used for submitting the experiment to Jersey backend 57 | var contents = { experiment_name: req.body.experiment_name }; 58 | logMessage(false, log.operationType.SubmitExperimentToRun, new Date(), "Submitting the experiment to Jersey backend"); 59 | var taskId = req.body.task_id; 60 | 61 | //Getting experiment type 62 | //Connect to DB 63 | var connection = createDbConnection(); 64 | connection.connect(); 65 | var query = 'select tasks.`type` as `type`, algorithms.`language` as `language` from tasks inner join algorithms on tasks.algorithm_id=algorithms.id where tasks.id=' + taskId; 66 | logMessage(false, log.operationType.QueryData, new Date(), query); 67 | //Return all services 68 | var results = connection.query(query, function getTaskType(err, rows, fields) { 69 | if (err) { // pass the err to error handler 70 | logMessage(true, log.errorType.DBError, new Date(), err); 71 | } 72 | else { 73 | logMessage(false, log.operationType.ResponseReceived, new Date(), "Received the task type"); 74 | if (rows.length > 0) { 75 | //Deciding which service to call 76 | var taskType = rows[0]['type']; 77 | var taskLanguage = rows[0]['language']; 78 | var serviceType = '-' + taskLanguage; 79 | if (taskType === 'Training') 80 | serviceType = 'Train' + serviceType; 81 | else 82 | serviceType = 'Exec' + serviceType; 83 | 84 | //Getting service IP 85 | loadServiceIp(serviceType, function (value) { 86 | if (value.message === 'success') { 87 | var address = 'http://' + value.ip + '/services/exp/start'; 88 | var options = { 89 | url: address, 90 | method: 'POST', 91 | json: contents 92 | }; 93 | request(options, function (error, response, body) { 94 | if (!error && response.statusCode === 200) { 95 | logMessage(false, log.operationType.ResponseReceived, new Date(), "Received response from Jeysey backend"); 96 | //TODO: put this response in previous middleware when implementing the new design 97 | // we return a success message to the frontend for easy use now ... 98 | res.json({ newexperimentinfo: req.body }); 99 | } else if (!error && response.statusCode === 400) { 100 | logMessage(false, log.operationType.ResponseReceived, new Date(), "Jeysey backend refused this request due to bad request"); 101 | //TODO: delete the experiment record just created 102 | res.status(400).send({ code: body.code, msg: body.msg }); 103 | } else { 104 | logMessage(true, log.errorType.JerseyError, new Date(), error); 105 | res.status(500).send({ code: 399, msg: "Server may not work now, please try later or contact admin" }); 106 | } 107 | }); 108 | } else { 109 | logMessage(true, log.errorType.JerseyError, new Date(), 'Training/Execution service not found.'); 110 | res.status(500).send({ code: 399, msg: "Server may not work now, please try later or contact admin" }); 111 | } 112 | }); 113 | } 114 | } 115 | }); 116 | //Closing connection 117 | connection.end(); 118 | }); 119 | 120 | 121 | module.exports = router; 122 | -------------------------------------------------------------------------------- /JerseyServices/src/main/java/com/chaosmonkeys/Utilities/db/DbUtils.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys.Utilities.db; 2 | 3 | 4 | import com.chaosmonkeys.Utilities.Logger; 5 | import com.chaosmonkeys.dao.Algorithm; 6 | import com.chaosmonkeys.dao.Dataset; 7 | import com.chaosmonkeys.dao.Experiment; 8 | import org.javalite.activejdbc.Base; 9 | import org.javalite.activejdbc.DB; 10 | import org.javalite.activejdbc.Model; 11 | import org.javalite.activejdbc.validation.ValidationException; 12 | 13 | import java.util.List; 14 | import java.util.ResourceBundle; 15 | import java.util.concurrent.locks.ReentrantLock; 16 | 17 | public final class DbUtils { 18 | 19 | // lock for managing connection 20 | private static ReentrantLock connectionLock = new ReentrantLock(); 21 | // lock for managing logging db connection 22 | private static ReentrantLock logConnectionLock = new ReentrantLock(); 23 | 24 | // JDBC connection properties 25 | private static String url; 26 | private static String logUrl; 27 | private static String username; 28 | private static String password; 29 | private static String driver; 30 | private static String dbName; 31 | private static String logDbName; 32 | private static ResourceBundle rb = ResourceBundle.getBundle("db.db-config"); 33 | // using SSL for connection but turning off server verification 34 | private static String options = "?verifyServerCertificate=false&useSSL=true"; 35 | 36 | // initialize the driver 37 | static{ 38 | dbName = rb.getString("jdbc.dbName"); 39 | url = rb.getString("jdbc.baseurl") + rb.getString("jdbc.dbName") + options; 40 | logUrl = rb.getString("jdbc.baseurl") + "logdatabase" + options; 41 | username = rb.getString("jdbc.username"); 42 | password = rb.getString("jdbc.password"); 43 | driver = rb.getString("jdbc.driver"); 44 | logDbName = "logdatabase"; // extract to resources file 45 | try { 46 | Class.forName(driver); 47 | } catch (ClassNotFoundException e) { 48 | e.printStackTrace(); 49 | } 50 | } 51 | 52 | /** 53 | * open ActiveJDBC connection when needed in one thread 54 | * this method only aimed to use the Configuration database 55 | * if you wanna use another database, refactor it or write a 56 | * new method 57 | */ 58 | public static void openConnection(){ 59 | connectionLock.lock(); 60 | new DB(dbName).open(driver, url, username, password); 61 | } 62 | 63 | /** 64 | * Close ActiveJDBC connection in current thread 65 | */ 66 | public static void closeConnection(){ 67 | new DB(dbName).close(); 68 | //TODO: place lock to another proper code 69 | connectionLock.unlock(); 70 | } 71 | 72 | /** 73 | * open ActiveJDBC connection when needed in one thread 74 | * this method only aimed to use the Log database 75 | * if you wanna use another database, refactor it or write a 76 | * new method 77 | */ 78 | public static void openLogConnection(){ 79 | logConnectionLock.lock(); 80 | new DB(logDbName).open(driver, logUrl, username, password); 81 | } 82 | 83 | /** 84 | * Close ActiveJDBC connection for LogDatabase in current thread 85 | */ 86 | public static void closeLogConnection(){ 87 | new DB(logDbName).close(); 88 | //TODO: place lock to another proper code 89 | logConnectionLock.unlock(); 90 | } 91 | 92 | /** 93 | * Insert the data sets information to the ConfigurationDatabase database. 94 | * @param dataName the target database name. 95 | * @param dataDescription the description regarding the input data. 96 | * @param path the input data path. 97 | * @param format the input data format. 98 | * @return false if insert error 99 | */ 100 | public static boolean storeDataSet(String dataName, String dataDescription, String path, String format){ 101 | openConnection(); 102 | Dataset dataset = new Dataset() 103 | .setDatasetName(dataName) 104 | .setDescription(dataDescription) 105 | .setDatasetPath(path) 106 | .setFormat(format); 107 | // this operation will throw exception when the mapping model is wrong with database schema 108 | boolean inserted = insertDAO(dataset); 109 | closeConnection(); 110 | return inserted; 111 | } 112 | 113 | /** 114 | * Store algorithm record in database 115 | * @param name algorithm name 116 | * @param description algorithm description 117 | * @param path 118 | * @param language 119 | * @return 120 | */ 121 | public static boolean storeAlgorithm( String name, String description, String path, String language){ 122 | openConnection(); 123 | Algorithm algr= new Algorithm() 124 | .setAlgorithmName(name) 125 | .setAlgorithmDescription(description) 126 | .setAlgorithmPath(path) 127 | .setAlgorithmLanguage(language); 128 | // this operation will throw exception when the mapping model is wrong with database schema 129 | boolean inserted = insertDAO(algr); 130 | closeConnection(); 131 | return inserted; 132 | } 133 | 134 | private static boolean insertDAO(Model m){ 135 | try { 136 | m.saveIt(); 137 | return true; 138 | }catch (ValidationException validationEx){ 139 | String errString="\nProblem with insert record: " + validationEx; 140 | Logger.Error(errString); 141 | return false; 142 | } 143 | } 144 | 145 | /** 146 | * This method would provide the experiment dao model 147 | * notice you should only use it when you can ensure the model existed in database 148 | * @param experimentName 149 | * @return Experiment DAO model with the specific experiment name 150 | */ 151 | public static Experiment getExperimentModelByName(String experimentName){ 152 | List expList = Experiment.where("experiment_name = ?", experimentName); 153 | Experiment exp = expList.get(0); 154 | return exp; 155 | } 156 | 157 | } 158 | -------------------------------------------------------------------------------- /CoordinationService/routes/heartbeat.js: -------------------------------------------------------------------------------- 1 | var express = require('express'); 2 | var router = express.Router(); 3 | var path = require('path'); 4 | var utilities = require('./dbUtilities'); 5 | var log = require('./logUtilities'); 6 | var mysqlToCsv = require("mysql-to-csv") 7 | var nodeCmd = require("node-cmd"); 8 | var url = require('url'); 9 | var config = require('../config/logDbConfig.json'); 10 | const SERVICE_EXISTS_ERRORCODE = 1062; 11 | 12 | /* gets the table of services, controls liveness of remote services and deregisters non responsive ones after 3 minutes */ 13 | router.get('/table', function getServiceTable(req, res, next) { 14 | //Connect to DB 15 | var connection = createDbConnection(); 16 | connection.connect(); 17 | ip = ''; 18 | result = []; 19 | //Return all services 20 | var sql = 'select * from connected_services'; 21 | logMessage(false, log.operationType.QueryData, new Date(), sql); 22 | connection.query(sql, function getAllConnectedServices(err, rows, fields) { 23 | if (err) { 24 | logMessage(true, log.errorType.DBError, new Date(), err); 25 | err.source = 'mysql'; 26 | err.status = 500; 27 | next(err) 28 | } else { 29 | result = []; 30 | var duration; 31 | for (var i = 0; i < rows.length; i++) { 32 | duration = Date.now() - rows[i]['last_updated']; 33 | rows[i]['last_updated'] = duration / 1000; 34 | ip = rows[i]['ip']; 35 | if (duration > 180000) { 36 | //Remove service from table 37 | //Connect to DB 38 | var deleteConnection = createDbConnection(); 39 | deleteConnection.connect(); 40 | var sqlDelete = 'delete from connected_services where id ="' + rows[i]['id'] + '"'; 41 | logMessage(false, log.operationType.QueryData, new Date(), sqlDelete); 42 | var results = deleteConnection.query(sqlDelete, function deleteInactiveServer(err, resp) { 43 | if (err) { 44 | logMessage(true, log.errorType.DBError, new Date(), err); 45 | err.source = 'mysql'; 46 | err.status = 500; 47 | next(err) 48 | } else { 49 | logMessage(false, log.operationType.ResponseReceived, new Date(), 'Successfully deleted inactive server with IP: ' + ip); 50 | } 51 | }); 52 | deleteConnection.end(); 53 | } else { 54 | result.push(rows[i]); 55 | } 56 | } 57 | res.json({ table: result }); 58 | } 59 | }); 60 | //Closing connection 61 | connection.end(); 62 | }); 63 | 64 | /* sets status */ 65 | router.post('/setStatus', function handleHeartbeatMsg(req, res, next) { 66 | var currentdate = Date.now(); 67 | var newService = req.body; 68 | //Extracting IP address 69 | var parsedUrl = url.parse(newService['ip']); 70 | var ipAddress = parsedUrl.host; 71 | //Connect to DB 72 | var connection = createDbConnection(); 73 | connection.connect(); 74 | var sql = 'insert into connected_services (ip_address, type, name, description, status, last_updated) values (\'' + ipAddress + '\',\'' + newService['type'] + '\',\'' + newService['name'] + '\',\'' + newService['description'] + '\',\'' + JSON.stringify(newService['status']) + '\',' + currentdate + ') on duplicate key update type=\'' + newService['type'] + '\', name=\'' + newService['name'] + '\', description=\'' + newService['description'] + '\', status=\'' + JSON.stringify(newService['status']) + '\', last_updated=' + currentdate; 75 | logMessage(false, log.operationType.QueryData, new Date(), sql); 76 | connection.query(sql, function checkUpdateOperationStatus(err, result) { 77 | if (err) { 78 | res.json({ response: err }); 79 | } else { 80 | res.json({ response: 'success' }); 81 | } 82 | }); 83 | connection.end(); 84 | }); 85 | 86 | /*Get errors_log items */ 87 | router.get('/getErrLog', function getErrLog(req, res, next) { 88 | var connection = createLogDbConnection(); 89 | connection.connect(); 90 | var query = "select * from errors_log"; 91 | //Return query results 92 | var results = connection.query(query, function getLogItems(err, rows, fields) { 93 | if (err) { // pass the err to error handler 94 | err.source = 'mysql'; // add error source for tracing 95 | err.status = 500; 96 | next(err); 97 | } else { 98 | res.send({ errorItems: rows }); 99 | } 100 | }); 101 | connection.end(); 102 | }); 103 | 104 | /*Get operations_log items */ 105 | router.get('/getOpLog', function getOpLog(req, res, next) { 106 | var connection = createLogDbConnection(); 107 | connection.connect(); 108 | var query = "select * from operations_log"; 109 | //Return query results 110 | var results = connection.query(query, function getLogItems(err, rows, fields) { 111 | if (err) { // pass the err to error handler 112 | err.source = 'mysql'; // add error source for tracing 113 | err.status = 500; 114 | next(err); 115 | } else { 116 | res.send({ operationItems: rows }); 117 | } 118 | }); 119 | connection.end(); 120 | }); 121 | 122 | /*Export errors_log table to a CSV file */ 123 | router.get('/exportErrLogToCSV', function exportErr(req, res) { 124 | var query = "select * from errors_log"; 125 | var command = "node exportToCSV.js -h " + config.dbhost + " -P " + config.port + " -d " + config.dbname + " -u " + config.dbuser + " -p " + config.dbpassword + " -q \"" + query + "\" -t false -T csv -o errlog"; 126 | var filepath = path.resolve("output/errlog.csv"); 127 | nodeCmd.get( 128 | command, 129 | function (err, data, stderr) { 130 | if (!err) { 131 | res.download(filepath); 132 | } 133 | }); 134 | }); 135 | 136 | /*Export operations_log table to a CSV file */ 137 | router.get('/exportOpLogToCSV', function exportOp(req, res) { 138 | var query = "select * from operations_log"; 139 | var command = "node exportToCSV.js -h " + config.dbhost + " -P " + config.port + " -d " + config.dbname + " -u " + config.dbuser + " -p " + config.dbpassword + " -q \"" + query + "\" -t false -T csv -o oplog"; 140 | var filepath = path.resolve("output/oplog.csv"); 141 | nodeCmd.get( 142 | command, 143 | function (err, data, stderr) { 144 | if (!err) { 145 | res.download(filepath); 146 | } 147 | } 148 | ); 149 | }); 150 | 151 | module.exports = router; 152 | -------------------------------------------------------------------------------- /CoordinationService/public/scripts/models.js: -------------------------------------------------------------------------------- 1 | //All model data 2 | var modelsData; 3 | 4 | //Selected model id 5 | var selectedModelId; 6 | 7 | //Initialization 8 | $(document).ready(function () { 9 | //Reset task modal 10 | resetModal('createTaskModal'); 11 | 12 | //Showing the loading text 13 | $('#modelsTableBody').text('Loading...'); 14 | 15 | //Load upload types 16 | $.get("models/list", function (data) { 17 | modelsData = data['models']; 18 | var modelsList = ''; 19 | if (!modelsData || modelsData.length === 0) { 20 | $('#modelsTableBody').html('

This list is empty!

'); 21 | } else { 22 | for (i = 0; i < data['models'].length; i++) { 23 | modelsList += '  ' + data['models'][i]['name'] + '' + data['models'][i]['description'] + '
' + data['models'][i]['experiment_name'] + ''; 24 | } 25 | $('#modelsTableBody').html(modelsList); 26 | } 27 | }); 28 | 29 | //Load datasets names 30 | $.get("tasks/datasetsnames", function (data) { 31 | var datasetsnames = ''; 32 | for (i = 0; i < data['datasetsnames'].length; i++) { 33 | datasetsnames += '\n'; 34 | } 35 | $('#taskDatasetsNames').html(datasetsnames); 36 | }); 37 | 38 | //Load algorithms names 39 | $.get("tasks/algorithmsnames", function (data) { 40 | var algorithmsnames = ''; 41 | for (i = 0; i < data['algorithmsnames'].length; i++) { 42 | algorithmsnames += '\n'; 43 | } 44 | $('#algorithmNameInTaskModal').html(algorithmsnames); 45 | }); 46 | }); 47 | 48 | //Displays details for a selected dataset 49 | function displayDetails(modelIndex) { 50 | $('#modelName').text(modelsData[modelIndex]['name']); 51 | $('#modelDescription').text(modelsData[modelIndex]['description']); 52 | $('#modelPath').html('Download'); 53 | 54 | //Load tasks types 55 | $.get("tasks/listByModel/" + modelsData[modelIndex]['id'], function (data) { 56 | if (typeof data != "undefined") { 57 | var tasksList = ''; 58 | if (!data['tasks'] || data['tasks'].length === 0) { 59 | $('#tasksTableBody').text('This list is empty!'); 60 | } else { 61 | for (i = 0; i < data['tasks'].length; i++) { 62 | tasksList += '  ' + data['tasks'][i]['name'] + '' + data['tasks'][i]['description'] + ''; 63 | } 64 | $('#tasksTableBody').html(tasksList); 65 | } 66 | } 67 | }); 68 | // Uncomment the following two lines after the table field is filled. 69 | //$('#datasetProject').text(datasetsData[datasetId]['project_name']); 70 | //$('#datasetTask').text(datasetsData[datasetId]['task_name']); 71 | 72 | $('#detailsModal').modal('show'); 73 | } 74 | 75 | //Creates a task definition from a model 76 | function createTaskFromModel(index) { 77 | selectedModelId = index; 78 | $('#taskModel').val(modelsData[index]['name']); 79 | $('#createTaskModal').modal('show'); 80 | } 81 | 82 | //Submit the task form 83 | function submitTaskForm() { 84 | var success = $('#formTaskSuccess')[0]; 85 | var successText = $('#formTaskSuccessText')[0]; 86 | var alert = $('#formTaskError')[0]; 87 | var alertText = $('#formTaskErrorText')[0]; 88 | var tasksInfoForExecution = { 89 | "dataset_id": $("#taskDatasetsNames").val(), 90 | "algorithm_id": $("#algorithmNameInTaskModal").val(), 91 | "model_id": modelsData[selectedModelId]['id'], 92 | "name": $("#taskName").val(), 93 | "description": $("#taskDescription").val(), 94 | "type": $("#taskType").val() 95 | } 96 | 97 | var objectToSend; 98 | var url; 99 | if (checkRequiredTaskFields()) { 100 | //Disabling submit button 101 | $('#submitTaskButton').prop("disabled", true); 102 | 103 | objectToSend = tasksInfoForExecution; 104 | url = 'http://' + location.host + '/tasks/createExecutionTask'; 105 | $.ajax({ 106 | url: url, 107 | type: "POST", 108 | dataType: "json", 109 | contentType: 'application/json', 110 | data: JSON.stringify(objectToSend), 111 | error: function (request, status, error) { 112 | showSubmissionResult("Oops! An error occurs when creating the task. Please check the error log in log path for possible reasons: " + status + error, alert, alertText); 113 | }, 114 | success: function (data) { 115 | showSubmissionResult("Task: " + data.newtaskinfo.name + " has been created successfully!", success, successText); 116 | } 117 | }); 118 | } 119 | } 120 | 121 | //Checks required fields 122 | function checkRequiredTaskFields() { 123 | var d = $("#taskDatasetsNames")[0]; 124 | var a = $("#algorithmNameInTaskModal")[0]; 125 | var taskName = $("#taskName").val(); 126 | var selectedTaskType = $("#taskType").val(); 127 | var selectedDatasetName = d.options[d.selectedIndex] === undefined ? "" : d.options[d.selectedIndex].text; 128 | var selectedAlgorithmName = a.options[a.selectedIndex] === undefined ? "" : a.options[a.selectedIndex].text; 129 | var alert = $('#formTaskError')[0]; 130 | var alertText = $('#formTaskErrorText')[0]; 131 | 132 | if (selectedTaskType.length == 0) { 133 | showSubmissionResult('Please select a task type.', alert, alertText); 134 | return false; 135 | } 136 | else if (taskName.length == 0) { 137 | showSubmissionResult('Please input a task name.', alert, alertText); 138 | return false; 139 | } 140 | else if (selectedDatasetName.length == 0) { 141 | showSubmissionResult('Please select a dataset.', alert, alertText); 142 | return false; 143 | } 144 | else if (selectedAlgorithmName.length == 0) { 145 | showSubmissionResult('Please select an algorithm.', alert, alertText); 146 | return false; 147 | } 148 | else { 149 | return true; 150 | } 151 | } 152 | 153 | //Show submission result after clicking Submit button 154 | function showSubmissionResult(message, alert, alertText) { 155 | alertText.innerText = message; 156 | alert.style.display = "block"; 157 | } -------------------------------------------------------------------------------- /SQL/06-28-2017-V0.6.sql: -------------------------------------------------------------------------------- 1 | -- MySQL dump 10.13 Distrib 5.7.12, for Win64 (x86_64) 2 | -- 3 | -- Host: localhost Database: configurationdatabase 4 | -- ------------------------------------------------------ 5 | -- Server version 5.6.35-log 6 | 7 | /*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */; 8 | /*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */; 9 | /*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */; 10 | /*!40101 SET NAMES utf8 */; 11 | /*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */; 12 | /*!40103 SET TIME_ZONE='+00:00' */; 13 | /*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */; 14 | /*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */; 15 | /*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */; 16 | /*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */; 17 | 18 | -- 19 | -- Table structure for table `connected_services` 20 | -- 21 | 22 | DROP TABLE IF EXISTS `connected_services`; 23 | /*!40101 SET @saved_cs_client = @@character_set_client */; 24 | /*!40101 SET character_set_client = utf8 */; 25 | CREATE TABLE `connected_services` ( 26 | `id` int(11) NOT NULL AUTO_INCREMENT, 27 | `ip_address` varchar(255) NOT NULL, 28 | `type` varchar(255) NOT NULL, 29 | `name` varchar(50) DEFAULT NULL, 30 | `description` varchar(255) DEFAULT NULL, 31 | `status` varchar(255) DEFAULT NULL, 32 | `last_updated` datetime DEFAULT NULL, 33 | PRIMARY KEY (`id`), 34 | UNIQUE KEY `ip_address_UNIQUE` (`ip_address`) 35 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 36 | /*!40101 SET character_set_client = @saved_cs_client */; 37 | 38 | -- 39 | -- Dumping data for table `connected_services` 40 | -- 41 | 42 | LOCK TABLES `connected_services` WRITE; 43 | /*!40000 ALTER TABLE `connected_services` DISABLE KEYS */; 44 | /*!40000 ALTER TABLE `connected_services` ENABLE KEYS */; 45 | UNLOCK TABLES; 46 | 47 | -- 48 | -- Table structure for table `datasets` 49 | -- 50 | 51 | DROP TABLE IF EXISTS `datasets`; 52 | /*!40101 SET @saved_cs_client = @@character_set_client */; 53 | /*!40101 SET character_set_client = utf8 */; 54 | CREATE TABLE `datasets` ( 55 | `id` int(11) NOT NULL AUTO_INCREMENT, 56 | `user_id` varchar(255) NOT NULL, 57 | `project_id` varchar(255) NOT NULL, 58 | `name` varchar(255) NOT NULL, 59 | `path` varchar(255) NOT NULL, 60 | `description` varchar(255) DEFAULT NULL, 61 | `format` varchar(20) NOT NULL, 62 | PRIMARY KEY (`id`) 63 | ) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8; 64 | /*!40101 SET character_set_client = @saved_cs_client */; 65 | 66 | -- 67 | -- Dumping data for table `datasets` 68 | -- 69 | 70 | LOCK TABLES `datasets` WRITE; 71 | /*!40000 ALTER TABLE `datasets` DISABLE KEYS */; 72 | INSERT INTO `datasets` VALUES (0,'Ingo Weber','Project Amazon','100-Sleep-Data','amazon','This data set includes 100 students\' sleeping data','JSON'),(1,'Len Bass','Project Amazon','30-Sqrill-Data','amazon','This data set includes 100 students\' sleeping data','CSV'); 73 | /*!40000 ALTER TABLE `datasets` ENABLE KEYS */; 74 | UNLOCK TABLES; 75 | 76 | -- 77 | -- Table structure for table `experiments` 78 | -- 79 | 80 | DROP TABLE IF EXISTS `experiments`; 81 | /*!40101 SET @saved_cs_client = @@character_set_client */; 82 | /*!40101 SET character_set_client = utf8 */; 83 | CREATE TABLE `experiments` ( 84 | `id` int(11) NOT NULL AUTO_INCREMENT, 85 | `project_id` varchar(255) NOT NULL, 86 | `task_id` varchar(255) NOT NULL, 87 | `experiment_name` varchar(255) NOT NULL, 88 | `start` datetime NOT NULL, 89 | `end` datetime NOT NULL, 90 | `last_status` varchar(25) NOT NULL, 91 | `last_updated` datetime NOT NULL, 92 | `description` varchar(255) DEFAULT NULL, 93 | PRIMARY KEY (`id`) 94 | ) ENGINE=InnoDB DEFAULT CHARSET=utf8; 95 | /*!40101 SET character_set_client = @saved_cs_client */; 96 | 97 | -- 98 | -- Dumping data for table `experiments` 99 | -- 100 | 101 | LOCK TABLES `experiments` WRITE; 102 | /*!40000 ALTER TABLE `experiments` DISABLE KEYS */; 103 | /*!40000 ALTER TABLE `experiments` ENABLE KEYS */; 104 | UNLOCK TABLES; 105 | 106 | -- 107 | -- Table structure for table `formats` 108 | -- 109 | 110 | DROP TABLE IF EXISTS `formats`; 111 | /*!40101 SET @saved_cs_client = @@character_set_client */; 112 | /*!40101 SET character_set_client = utf8 */; 113 | CREATE TABLE `formats` ( 114 | `id` int(11) NOT NULL AUTO_INCREMENT, 115 | `format` varchar(20) NOT NULL, 116 | PRIMARY KEY (`id`) 117 | ) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8; 118 | /*!40101 SET character_set_client = @saved_cs_client */; 119 | 120 | -- 121 | -- Dumping data for table `formats` 122 | -- 123 | 124 | LOCK TABLES `formats` WRITE; 125 | /*!40000 ALTER TABLE `formats` DISABLE KEYS */; 126 | INSERT INTO `formats` VALUES (1,'JSON'),(2,'CSV'); 127 | /*!40000 ALTER TABLE `formats` ENABLE KEYS */; 128 | UNLOCK TABLES; 129 | 130 | -- 131 | -- Table structure for table `task_types` 132 | -- 133 | 134 | DROP TABLE IF EXISTS `task_types`; 135 | /*!40101 SET @saved_cs_client = @@character_set_client */; 136 | /*!40101 SET character_set_client = utf8 */; 137 | CREATE TABLE `task_types` ( 138 | `id` int(11) NOT NULL AUTO_INCREMENT, 139 | `type` varchar(20) DEFAULT NULL, 140 | PRIMARY KEY (`id`) 141 | ) ENGINE=InnoDB AUTO_INCREMENT=3 DEFAULT CHARSET=utf8; 142 | /*!40101 SET character_set_client = @saved_cs_client */; 143 | 144 | -- 145 | -- Dumping data for table `task_types` 146 | -- 147 | 148 | LOCK TABLES `task_types` WRITE; 149 | /*!40000 ALTER TABLE `task_types` DISABLE KEYS */; 150 | INSERT INTO `task_types` VALUES (1,'Training'),(2,'Execution'); 151 | /*!40000 ALTER TABLE `task_types` ENABLE KEYS */; 152 | UNLOCK TABLES; 153 | 154 | -- 155 | -- Table structure for table `tasks` 156 | -- 157 | 158 | DROP TABLE IF EXISTS `tasks`; 159 | /*!40101 SET @saved_cs_client = @@character_set_client */; 160 | /*!40101 SET character_set_client = utf8 */; 161 | CREATE TABLE `tasks` ( 162 | `id` int(11) NOT NULL AUTO_INCREMENT, 163 | `project_id` int(11) NOT NULL, 164 | `dataset_id` int(11) NOT NULL, 165 | `algorithm_id` int(11) NOT NULL, 166 | `name` varchar(45) DEFAULT NULL, 167 | `description` varchar(500) DEFAULT NULL, 168 | `type` varchar(20) NOT NULL, 169 | PRIMARY KEY (`id`) 170 | ) ENGINE=InnoDB AUTO_INCREMENT=6 DEFAULT CHARSET=utf8; 171 | /*!40101 SET character_set_client = @saved_cs_client */; 172 | 173 | -- 174 | -- Dumping data for table `tasks` 175 | -- 176 | 177 | LOCK TABLES `tasks` WRITE; 178 | /*!40000 ALTER TABLE `tasks` DISABLE KEYS */; 179 | INSERT INTO `tasks` VALUES (1,1,1,1,'Task1','Sample algorithm 1','Training'),(2,2,2,2,'Task2','Sample algorithm 2','Execution'),(3,3,3,3,'Task3','Sample algorithm 3','Training'),(4,4,4,4,'Task4','Sample algorithm 4','Execution'),(5,5,5,5,'Task5','Sample algorithm 5','Training'); 180 | /*!40000 ALTER TABLE `tasks` ENABLE KEYS */; 181 | UNLOCK TABLES; 182 | /*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */; 183 | 184 | /*!40101 SET SQL_MODE=@OLD_SQL_MODE */; 185 | /*!40014 SET FOREIGN_KEY_CHECKS=@OLD_FOREIGN_KEY_CHECKS */; 186 | /*!40014 SET UNIQUE_CHECKS=@OLD_UNIQUE_CHECKS */; 187 | /*!40101 SET CHARACTER_SET_CLIENT=@OLD_CHARACTER_SET_CLIENT */; 188 | /*!40101 SET CHARACTER_SET_RESULTS=@OLD_CHARACTER_SET_RESULTS */; 189 | /*!40101 SET COLLATION_CONNECTION=@OLD_COLLATION_CONNECTION */; 190 | /*!40111 SET SQL_NOTES=@OLD_SQL_NOTES */; 191 | 192 | -- Dump completed on 2017-06-28 16:06:52 193 | -------------------------------------------------------------------------------- /JerseyServices/src/test/java/com/chaosmonkeys/DatasetInputServiceTest.java: -------------------------------------------------------------------------------- 1 | package com.chaosmonkeys; 2 | 3 | 4 | import com.chaosmonkeys.datasetinputservice.DatasetInputService; 5 | import org.glassfish.jersey.client.ClientConfig; 6 | import org.glassfish.jersey.media.multipart.*; 7 | import org.glassfish.jersey.media.multipart.file.FileDataBodyPart; 8 | import org.glassfish.jersey.server.ResourceConfig; 9 | import org.glassfish.jersey.test.JerseyTest; 10 | import org.glassfish.jersey.test.TestProperties; 11 | import org.junit.Rule; 12 | import org.junit.Test; 13 | import org.junit.rules.TemporaryFolder; 14 | 15 | import javax.ws.rs.client.Entity; 16 | import javax.ws.rs.core.Application; 17 | import javax.ws.rs.core.MediaType; 18 | import javax.ws.rs.core.Response; 19 | 20 | import java.io.File; 21 | import java.io.IOException; 22 | 23 | import static org.junit.Assert.assertEquals; 24 | 25 | public class DatasetInputServiceTest extends JerseyTest { 26 | @Rule 27 | public TemporaryFolder folder= new TemporaryFolder(); 28 | 29 | /** 30 | * Control over configuring values of the properties defined and described in the TestProperties class 31 | * 32 | * @return ResourceConfig 33 | */ 34 | @Override 35 | protected Application configure() { 36 | enable(TestProperties.LOG_TRAFFIC); 37 | enable(TestProperties.DUMP_ENTITY); 38 | ResourceConfig config = new ResourceConfig(DatasetInputService.class); 39 | config.register(MultiPartFeature.class); // IMPORTANT: if you want to test multipart feature, remember to add this 40 | return config; 41 | } 42 | 43 | /** 44 | * Some advanced client configuration 45 | * This is for enabling uploading mulitpart feature, avoid 46 | * MessageBodyWriter not found error 47 | * 48 | * @param config 49 | */ 50 | @Override 51 | protected void configureClient(ClientConfig config) { 52 | super.configureClient(config); 53 | // when you are using some advanced feature, remember to register 54 | config.register(MultiPartFeature.class); 55 | } 56 | 57 | /** 58 | * Test to see that the service does not accept GET request 59 | */ 60 | @Test 61 | public void testGetRequest() { 62 | int errorCode = target("services/datasets/upload").request().get().getStatus(); 63 | assertEquals(405, errorCode); 64 | } 65 | 66 | /** 67 | * Test to see that the service does not accept wrong parameter 68 | */ 69 | @Test 70 | public void testUnsupportedMediaType() { 71 | int responseMsg = target("services/datasets/upload").request().post(Entity.text("asb")).getStatus(); 72 | assertEquals(Response.Status.UNSUPPORTED_MEDIA_TYPE.getStatusCode(), responseMsg); 73 | } 74 | 75 | /** 76 | * Test with empty file uploaded (the fileStream is null) 77 | * expected return BAD_REQUEST 78 | */ 79 | @Test 80 | public void testLackOfFormBodyPart(){ 81 | MediaType contentType = MediaType.MULTIPART_FORM_DATA_TYPE; 82 | contentType = Boundary.addBoundary(contentType); 83 | // declare a file as part of the form data 84 | FormDataBodyPart filePart; 85 | try { 86 | // create test file 87 | final File testFile = folder.newFile("testCSV.csv"); 88 | 89 | filePart = new FileDataBodyPart("file", testFile); // pom.xml 90 | 91 | // construct the entire form with all required parameters 92 | MultiPart multipartEntity = new FormDataMultiPart() 93 | .field("project_id","test_id") 94 | .field("format","csv") 95 | .field("description","TEST DATA-SET DESCRIPTION") 96 | .field("user_id", "test_u_id") 97 | .field("name", "testname"); 98 | // request the response Jersey 2.25 will not recognize multipartEntity.getMediaType() 99 | Response response = target("services/datasets/upload").request().post(Entity.entity(multipartEntity, contentType)); 100 | 101 | assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus()); 102 | response.close(); 103 | } catch (IOException e) { 104 | e.printStackTrace(); 105 | } 106 | } 107 | 108 | /** 109 | * Test with invalid file extension operation 110 | * TODO: someone could build a test data util or coleection inside the test class 111 | */ 112 | @Test 113 | public void testInvalidExtensionUpload() { 114 | MediaType contentType = MediaType.MULTIPART_FORM_DATA_TYPE; 115 | contentType = Boundary.addBoundary(contentType); 116 | // declare a file as part of the form data 117 | FormDataBodyPart filePart; 118 | try { 119 | // create test file 120 | final File testFile = folder.newFile("test.txt"); 121 | filePart = new FileDataBodyPart("file", testFile); // pom.xml 122 | 123 | // construct the entire form with all required parameters 124 | MultiPart multipartEntity = new FormDataMultiPart() 125 | .field("project_id", "123") 126 | .field("user_id", "321") 127 | .field("name", "testname") 128 | .field("description", "memeda des") 129 | .field("format","csv") 130 | .bodyPart(filePart); 131 | // request the response Jersey 2.25 will not recognize multipartEntity.getMediaType() 132 | Response response = target("services/datasets/upload").request().post(Entity.entity(multipartEntity, contentType)); 133 | 134 | assertEquals(Response.Status.BAD_REQUEST.getStatusCode(), response.getStatus()); 135 | response.close(); 136 | } catch (IOException e) { 137 | e.printStackTrace(); 138 | } 139 | } 140 | /** 141 | * Test with normal operation 142 | */ 143 | @Test 144 | public void testNormalUpload() { 145 | MediaType contentType = MediaType.MULTIPART_FORM_DATA_TYPE; 146 | contentType = Boundary.addBoundary(contentType); 147 | // declare a file as part of the form data 148 | FormDataBodyPart filePart; 149 | try { 150 | // create test file 151 | final File testFile = folder.newFile("testCSV.csv"); 152 | filePart = new FileDataBodyPart("file", testFile); // pom.xml 153 | 154 | // construct the entire form with all required parameters 155 | MultiPart multipartEntity = new FormDataMultiPart() 156 | .field("project_id", "123") 157 | .field("user_id", "321") 158 | .field("name", "testname") 159 | .field("description", "memeda des") 160 | .field("format","csv") 161 | .bodyPart(filePart); 162 | // request the response Jersey 2.25 will not recognize multipartEntity.getMediaType() 163 | Response response = target("services/datasets/upload").request().post(Entity.entity(multipartEntity, contentType)); 164 | 165 | assertEquals(Response.Status.OK.getStatusCode(), response.getStatus()); 166 | response.close(); 167 | } catch (IOException e) { 168 | e.printStackTrace(); 169 | } 170 | } 171 | } --------------------------------------------------------------------------------