├── Cometd ├── src │ └── main │ │ ├── resources │ │ └── docker-config.yml │ │ ├── docker │ │ └── docker-assembly.xml │ │ └── java │ │ └── com │ │ └── hortonworks │ │ └── iot │ │ └── cometd │ │ └── Main.java ├── .gitignore ├── .DS_Store ├── .settings │ └── .gitignore ├── .project ├── .classpath └── pom.xml ├── MapUI ├── src │ ├── main │ │ ├── resources │ │ │ └── docker-config.yml │ │ ├── webapp │ │ │ ├── WEB-INF │ │ │ │ ├── .gitignore │ │ │ │ └── web.xml │ │ │ ├── .DS_Store │ │ │ ├── images │ │ │ │ ├── car.png │ │ │ │ ├── 487632108.png │ │ │ │ ├── greenMarker.png │ │ │ │ ├── hospitalRed.png │ │ │ │ ├── redMarker.png │ │ │ │ ├── stb_alert.png │ │ │ │ ├── stb_normal.png │ │ │ │ ├── hospitalGreen.png │ │ │ │ ├── stb_alert_25.png │ │ │ │ ├── stb_normal_25.png │ │ │ │ ├── hortonworks-logo-new.png │ │ │ │ └── hortonworks-logo-green.png │ │ │ └── css │ │ │ │ └── mapStyle.css │ │ ├── .DS_Store │ │ ├── java │ │ │ └── com │ │ │ │ └── hortonworks │ │ │ │ └── iot │ │ │ │ └── mapui │ │ │ │ ├── MapUIMain.java │ │ │ │ └── MapUI.java │ │ └── docker │ │ │ └── docker-assembly.xml │ └── .DS_Store ├── .gitignore ├── .DS_Store ├── .settings │ └── .gitignore ├── .classpath ├── .project └── pom.xml ├── DeviceMonitor ├── .gitignore ├── .settings │ └── .gitignore ├── .DS_Store ├── src │ ├── .DS_Store │ └── main │ │ ├── .DS_Store │ │ └── java │ │ ├── .DS_Store │ │ └── com │ │ ├── .DS_Store │ │ └── hortonworks │ │ ├── .DS_Store │ │ ├── events │ │ ├── ExternalRequest.java │ │ ├── DeviceStatus.java │ │ ├── TechnicianDestination.java │ │ ├── TechnicianDetails.java │ │ ├── STBStatus.java │ │ ├── DeviceAlert.java │ │ └── TechnicianStatus.java │ │ ├── bolts │ │ ├── PrintDeviceAlert.java │ │ ├── PrintDeviceStatus.java │ │ ├── PublishAlert.java │ │ ├── PublishTechnicianLocation.java │ │ ├── PublishDeviceStatus.java │ │ ├── RouteTechnician.java │ │ ├── PersistTechnicianLocation.java │ │ ├── IncidentDetector.java │ │ ├── EnrichDeviceStatus.java │ │ └── RecommendTechnician.java │ │ ├── util │ │ ├── DeviceEventJSONScheme.java │ │ ├── TechnicianEventJSONScheme.java │ │ └── Constants.java │ │ └── iot │ │ └── topology │ │ └── DeviceMonitorTopology.java ├── .project ├── .classpath └── pom.xml ├── DeviceMonitorNostradamus ├── .gitignore ├── .settings │ └── .gitignore ├── .DS_Store ├── src │ ├── .DS_Store │ └── main │ │ ├── .DS_Store │ │ └── java │ │ ├── .DS_Store │ │ └── com │ │ └── hortonworks │ │ └── iot │ │ ├── events │ │ ├── DeviceStatus.java │ │ └── STBStatus.java │ │ └── util │ │ └── Constants.java ├── .project ├── .classpath └── pom.xml ├── Model ├── nostradamusSVMModel.zip └── device_manager_svm.xml ├── screenshot └── DeviceManagerScreenShot.png ├── stopSimulation.sh ├── Nifi └── config │ ├── nifi-ambari-config.json │ ├── nifi-env.json │ ├── nifi-flow-env.json │ ├── nifi-bootstrap-env.json │ ├── nifi-logback-env.json │ └── nifi-properties-env.json ├── DEVICE_MANAGER_DEMO_CONTROL ├── package │ └── scripts │ │ ├── params.py │ │ └── master.py ├── quicklinks │ └── quicklinks.json ├── configuration │ └── control-config.xml └── metainfo.xml ├── SliderConfig ├── resources.json ├── appConfig.json └── metainfo.json ├── DEVICE_MANAGER_DEMO_CONTROL_SAM ├── quicklinks │ └── quicklinks.json ├── package │ └── scripts │ │ ├── params.py │ │ └── master.py ├── configuration │ └── control-config.xml └── metainfo.xml ├── startSimulation.sh ├── Zeppelin └── notebook │ └── 2B75QB4J1 │ └── note.json ├── README.md ├── DeviceMonitorNostradamusScala ├── pom.xml └── src │ └── main │ └── scala │ └── com │ └── hortonworks │ └── iot │ └── spark │ └── streaming │ └── SparkNostradamus.scala └── redeployApplication.sh /Cometd/src/main/resources/docker-config.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /MapUI/src/main/resources/docker-config.yml: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /Cometd/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | /classes/ 3 | -------------------------------------------------------------------------------- /DeviceMonitor/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | /classes/ 3 | -------------------------------------------------------------------------------- /DeviceMonitorNostradamus/.gitignore: -------------------------------------------------------------------------------- 1 | /target/ 2 | /classes/ 3 | -------------------------------------------------------------------------------- /MapUI/.gitignore: -------------------------------------------------------------------------------- 1 | /classes/ 2 | /target/ 3 | /tomcat.8090/ 4 | -------------------------------------------------------------------------------- /MapUI/src/main/webapp/WEB-INF/.gitignore: -------------------------------------------------------------------------------- 1 | /lib/ 2 | /classes/ 3 | -------------------------------------------------------------------------------- /Cometd/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/Cometd/.DS_Store -------------------------------------------------------------------------------- /Cometd/.settings/.gitignore: -------------------------------------------------------------------------------- 1 | /org.eclipse.jdt.core.prefs 2 | /org.eclipse.m2e.core.prefs 3 | -------------------------------------------------------------------------------- /MapUI/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/MapUI/.DS_Store -------------------------------------------------------------------------------- /DeviceMonitor/.settings/.gitignore: -------------------------------------------------------------------------------- 1 | /org.eclipse.jdt.core.prefs 2 | /org.eclipse.m2e.core.prefs 3 | -------------------------------------------------------------------------------- /MapUI/src/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/MapUI/src/.DS_Store -------------------------------------------------------------------------------- /DeviceMonitor/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/DeviceMonitor/.DS_Store -------------------------------------------------------------------------------- /MapUI/src/main/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/MapUI/src/main/.DS_Store -------------------------------------------------------------------------------- /DeviceMonitorNostradamus/.settings/.gitignore: -------------------------------------------------------------------------------- 1 | /org.eclipse.jdt.core.prefs 2 | /org.eclipse.m2e.core.prefs 3 | -------------------------------------------------------------------------------- /DeviceMonitor/src/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/DeviceMonitor/src/.DS_Store -------------------------------------------------------------------------------- /Model/nostradamusSVMModel.zip: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/Model/nostradamusSVMModel.zip -------------------------------------------------------------------------------- /MapUI/src/main/webapp/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/MapUI/src/main/webapp/.DS_Store -------------------------------------------------------------------------------- /DeviceMonitor/src/main/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/DeviceMonitor/src/main/.DS_Store -------------------------------------------------------------------------------- /DeviceMonitorNostradamus/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/DeviceMonitorNostradamus/.DS_Store -------------------------------------------------------------------------------- /MapUI/.settings/.gitignore: -------------------------------------------------------------------------------- 1 | /org.eclipse.jdt.core.prefs 2 | /org.eclipse.m2e.core.prefs 3 | /org.eclipse.wst.validation.prefs 4 | -------------------------------------------------------------------------------- /MapUI/src/main/webapp/images/car.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/MapUI/src/main/webapp/images/car.png -------------------------------------------------------------------------------- /DeviceMonitor/src/main/java/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/DeviceMonitor/src/main/java/.DS_Store -------------------------------------------------------------------------------- /DeviceMonitorNostradamus/src/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/DeviceMonitorNostradamus/src/.DS_Store -------------------------------------------------------------------------------- /screenshot/DeviceManagerScreenShot.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/screenshot/DeviceManagerScreenShot.png -------------------------------------------------------------------------------- /DeviceMonitor/src/main/java/com/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/DeviceMonitor/src/main/java/com/.DS_Store -------------------------------------------------------------------------------- /DeviceMonitorNostradamus/src/main/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/DeviceMonitorNostradamus/src/main/.DS_Store -------------------------------------------------------------------------------- /MapUI/src/main/webapp/images/487632108.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/MapUI/src/main/webapp/images/487632108.png -------------------------------------------------------------------------------- /MapUI/src/main/webapp/images/greenMarker.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/MapUI/src/main/webapp/images/greenMarker.png -------------------------------------------------------------------------------- /MapUI/src/main/webapp/images/hospitalRed.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/MapUI/src/main/webapp/images/hospitalRed.png -------------------------------------------------------------------------------- /MapUI/src/main/webapp/images/redMarker.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/MapUI/src/main/webapp/images/redMarker.png -------------------------------------------------------------------------------- /MapUI/src/main/webapp/images/stb_alert.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/MapUI/src/main/webapp/images/stb_alert.png -------------------------------------------------------------------------------- /MapUI/src/main/webapp/images/stb_normal.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/MapUI/src/main/webapp/images/stb_normal.png -------------------------------------------------------------------------------- /MapUI/src/main/webapp/images/hospitalGreen.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/MapUI/src/main/webapp/images/hospitalGreen.png -------------------------------------------------------------------------------- /MapUI/src/main/webapp/images/stb_alert_25.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/MapUI/src/main/webapp/images/stb_alert_25.png -------------------------------------------------------------------------------- /MapUI/src/main/webapp/images/stb_normal_25.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/MapUI/src/main/webapp/images/stb_normal_25.png -------------------------------------------------------------------------------- /DeviceMonitorNostradamus/src/main/java/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/DeviceMonitorNostradamus/src/main/java/.DS_Store -------------------------------------------------------------------------------- /DeviceMonitor/src/main/java/com/hortonworks/.DS_Store: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/DeviceMonitor/src/main/java/com/hortonworks/.DS_Store -------------------------------------------------------------------------------- /MapUI/src/main/webapp/images/hortonworks-logo-new.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/MapUI/src/main/webapp/images/hortonworks-logo-new.png -------------------------------------------------------------------------------- /MapUI/src/main/webapp/images/hortonworks-logo-green.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/vvaks0/DeviceManagerDemo/HEAD/MapUI/src/main/webapp/images/hortonworks-logo-green.png -------------------------------------------------------------------------------- /stopSimulation.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | kill -9 `cat STB_1000_Sim.pid` 4 | kill -9 `cat STB_2000_Sim.pid` 5 | kill -9 `cat STB_3000_Sim.pid` 6 | 7 | kill -9 `cat Technician_1000_Sim.pid` 8 | kill -9 `cat Technician_2000_Sim.pid` 9 | kill -9 `cat Technician_3000_Sim.pid` 10 | -------------------------------------------------------------------------------- /Nifi/config/nifi-ambari-config.json: -------------------------------------------------------------------------------- 1 | "properties" : { 2 | "nifi.ambari_reporting_frequency" : "1 mins", 3 | "nifi.initial_mem" : "512m", 4 | "nifi.install_dir" : "/opt", 5 | "nifi.max_mem" : "512m", 6 | "nifi.node.protocol.port" : "9088", 7 | "nifi.port" : "9090", 8 | "nifi.setup_prebuilt" : "true" 9 | } 10 | -------------------------------------------------------------------------------- /Nifi/config/nifi-env.json: -------------------------------------------------------------------------------- 1 | "properties" : { 2 | "content" : "\n#!/bin/sh\n\n# The java implementation to use.\nexport JAVA_HOME={{jdk64_home}}\n\nexport NIFI_HOME={{nifi_dir}}\n\n#The directory for the NiFi pid file\nexport NIFI_PID_DIR=\"{{nifi_pid_dir}}\"\n\n#The directory for NiFi log files\nexport NIFI_LOG_DIR=\"{{nifi_log_dir}}\"" 3 | } -------------------------------------------------------------------------------- /MapUI/src/main/webapp/WEB-INF/web.xml: -------------------------------------------------------------------------------- 1 | 2 | 5 | 6 | 7 | -------------------------------------------------------------------------------- /DEVICE_MANAGER_DEMO_CONTROL/package/scripts/params.py: -------------------------------------------------------------------------------- 1 | 2 | #!/usr/bin/env python 3 | from resource_management import * 4 | 5 | # server configurations 6 | config = Script.get_config() 7 | 8 | install_dir = config['configurations']['control-config']['democontrol.install_dir'] 9 | download_url = config['configurations']['control-config']['democontrol.download_url'] 10 | nifi_host_ip = config['configurations']['control-config']['democontrol.nifi_host_ip'] 11 | -------------------------------------------------------------------------------- /SliderConfig/resources.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": "http://example.org/specification/v2.0.0", 3 | "metadata": { }, 4 | "global": { }, 5 | "components": {"slider-appmaster": { }, 6 | "MAPUI": { 7 | "yarn.role.priority": "1", 8 | "yarn.component.instances": "1", 9 | "yarn.memory": "256" 10 | }, 11 | "COMETD": { 12 | "yarn.role.priority": "2", 13 | "yarn.component.instances": "1", 14 | "yarn.memory": "256" 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /MapUI/.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | -------------------------------------------------------------------------------- /MapUI/src/main/java/com/hortonworks/iot/mapui/MapUIMain.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks.iot.mapui; 2 | 3 | import org.apache.catalina.startup.Tomcat; 4 | 5 | public class MapUIMain{ 6 | public static void main(String [] args) throws Exception { 7 | //String contextPath = "/"; 8 | String appBase = "."; 9 | 10 | Tomcat tomcat = new Tomcat(); 11 | tomcat.setPort(Integer.parseInt("8090")); 12 | tomcat.getHost().setAppBase(appBase); 13 | tomcat.addWebapp("/MapUI", appBase); 14 | 15 | tomcat.start(); 16 | tomcat.getServer().await(); 17 | } 18 | } -------------------------------------------------------------------------------- /DEVICE_MANAGER_DEMO_CONTROL/quicklinks/quicklinks.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "default", 3 | "description": "default quick links configuration", 4 | "configuration": { 5 | "protocol": 6 | { 7 | "type":"http", 8 | "checks":[] 9 | }, 10 | 11 | "links": [ 12 | { 13 | "name": "device_manager_map_ui", 14 | "label": "Device Manager Map UI", 15 | "requires_user_name": "false", 16 | "component_name": "NIFI_MASTER", 17 | "url":"%@://%@:%@/MapUI/DeviceMap", 18 | "port":{ 19 | "http_default_port": "8090" 20 | } 21 | } 22 | ] 23 | } 24 | } -------------------------------------------------------------------------------- /DEVICE_MANAGER_DEMO_CONTROL_SAM/quicklinks/quicklinks.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "default", 3 | "description": "default quick links configuration", 4 | "configuration": { 5 | "protocol": 6 | { 7 | "type":"http", 8 | "checks":[] 9 | }, 10 | 11 | "links": [ 12 | { 13 | "name": "device_manager_map_ui", 14 | "label": "Device Manager Map UI", 15 | "requires_user_name": "false", 16 | "component_name": "NIFI_MASTER", 17 | "url":"%@://%@:%@/MapUI/DeviceMap", 18 | "port":{ 19 | "http_default_port": "8090" 20 | } 21 | } 22 | ] 23 | } 24 | } -------------------------------------------------------------------------------- /Cometd/.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | Cometd 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | org.eclipse.m2e.core.maven2Builder 15 | 16 | 17 | 18 | 19 | 20 | org.eclipse.jdt.core.javanature 21 | org.eclipse.m2e.core.maven2Nature 22 | 23 | 24 | -------------------------------------------------------------------------------- /DeviceMonitor/.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | DeviceMonitor 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | org.eclipse.m2e.core.maven2Builder 15 | 16 | 17 | 18 | 19 | 20 | org.eclipse.jdt.core.javanature 21 | org.eclipse.m2e.core.maven2Nature 22 | 23 | 24 | -------------------------------------------------------------------------------- /MapUI/src/main/docker/docker-assembly.xml: -------------------------------------------------------------------------------- 1 | 3 | ${project.artifactId} 4 | 5 | 6 | target/MapUI-jar-with-dependencies.jar 7 | / 8 | 9 | 10 | src/main/resources/docker-config.yml 11 | / 12 | 13 | 14 | -------------------------------------------------------------------------------- /Cometd/src/main/docker/docker-assembly.xml: -------------------------------------------------------------------------------- 1 | 3 | ${project.artifactId} 4 | 5 | 6 | target/Cometd-jar-with-dependencies.jar 7 | / 8 | 9 | 10 | src/main/resources/docker-config.yml 11 | / 12 | 13 | 14 | -------------------------------------------------------------------------------- /DeviceMonitorNostradamus/.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | DeviceMonitorNostradamus 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | org.eclipse.m2e.core.maven2Builder 15 | 16 | 17 | 18 | 19 | 20 | org.eclipse.jdt.core.javanature 21 | org.eclipse.m2e.core.maven2Nature 22 | 23 | 24 | -------------------------------------------------------------------------------- /SliderConfig/appConfig.json: -------------------------------------------------------------------------------- 1 | { 2 | "schema": "http://example.org/specification/v2.0.0", 3 | "metadata": {}, 4 | "global": {}, 5 | "components": { 6 | "MAPUI": { 7 | "mapui.commandPath": "/usr/bin/docker", 8 | "mapui.options":"-d --net=host", 9 | "mapui.statusCommand":"docker inspect -f {{.State.Running}} ${CONTAINER_ID} | grep true" 10 | }, 11 | "COMETD": { 12 | "cometd.commandPath": "/usr/bin/docker", 13 | "cometd.options":"-d --net=host", 14 | "cometd.statusCommand":"docker inspect -f {{.State.Running}} ${CONTAINER_ID} | grep true" 15 | } 16 | } 17 | } 18 | -------------------------------------------------------------------------------- /DeviceMonitor/src/main/java/com/hortonworks/events/ExternalRequest.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks.events; 2 | 3 | import java.io.Serializable; 4 | 5 | public class ExternalRequest implements Serializable { 6 | private static final long serialVersionUID = 1L; 7 | private String serialNumber; 8 | private String command; 9 | 10 | public ExternalRequest() {} 11 | 12 | public void setDeviceSerialNumber(String value){ 13 | serialNumber = value; 14 | } 15 | 16 | public String getDeviceSerialNumber(){ 17 | return serialNumber; 18 | } 19 | 20 | public void setCommand(String value){ 21 | command = value; 22 | } 23 | 24 | public String getCommand(){ 25 | return command; 26 | } 27 | } 28 | -------------------------------------------------------------------------------- /DEVICE_MANAGER_DEMO_CONTROL_SAM/package/scripts/params.py: -------------------------------------------------------------------------------- 1 | 2 | #!/usr/bin/env python 3 | from resource_management import * 4 | 5 | # server configurations 6 | config = Script.get_config() 7 | 8 | install_dir = config['configurations']['control-config']['democontrol.install.dir'] 9 | simulator_download_url = config['configurations']['control-config']['democontrol.simulator.git.url'] 10 | device_manager_download_url = config['configurations']['control-config']['democontrol.device.manager.git.url'] 11 | sam_extentions_download_url = config['configurations']['control-config']['democontrol.sam.extentions.git.url'] 12 | google_api_key = config['configurations']['control-config']['democontrol.google.api.key'] 13 | 14 | nifi_host = str(config['clusterHostInfo']['nifi_master_hosts'][0]) 15 | nifi_port = '9090' 16 | 17 | -------------------------------------------------------------------------------- /DeviceMonitor/src/main/java/com/hortonworks/events/DeviceStatus.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks.events; 2 | 3 | import java.io.Serializable; 4 | 5 | public class DeviceStatus implements Serializable { 6 | private static final long serialVersionUID = 1L; 7 | private String serialNumber; 8 | private String status; 9 | private String state; 10 | 11 | public void setSerialNumber(String value){ 12 | serialNumber = value; 13 | } 14 | public void setState(String value){ 15 | state = value; 16 | } 17 | public void setStatus(String value){ 18 | status = value; 19 | } 20 | 21 | public String getSerialNumber(){ 22 | return serialNumber; 23 | } 24 | public String getState(){ 25 | return state; 26 | } 27 | public String getStatus(){ 28 | return status; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /DeviceMonitor/src/main/java/com/hortonworks/events/TechnicianDestination.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks.events; 2 | 3 | public class TechnicianDestination extends TechnicianStatus { 4 | private static final long serialVersionUID = 1L; 5 | private Double destinationLatitude; 6 | private Double destinationLongitude; 7 | 8 | public TechnicianDestination() { 9 | super(); 10 | } 11 | 12 | public void setDestinationLongitude(Double value){ 13 | destinationLongitude = value; 14 | } 15 | public void setDestinationLatitude(Double value){ 16 | destinationLatitude = value; 17 | } 18 | 19 | public Double getDestinationLongitude(){ 20 | return destinationLongitude; 21 | } 22 | public Double getDestinationLatitude(){ 23 | return destinationLatitude; 24 | } 25 | } -------------------------------------------------------------------------------- /MapUI/.project: -------------------------------------------------------------------------------- 1 | 2 | 3 | MapUI 4 | 5 | 6 | 7 | 8 | 9 | org.eclipse.jdt.core.javabuilder 10 | 11 | 12 | 13 | 14 | org.eclipse.wst.common.project.facet.core.builder 15 | 16 | 17 | 18 | 19 | org.eclipse.m2e.core.maven2Builder 20 | 21 | 22 | 23 | 24 | 25 | org.eclipse.jdt.core.javanature 26 | org.eclipse.m2e.core.maven2Nature 27 | org.eclipse.wst.common.project.facet.core.nature 28 | 29 | 30 | -------------------------------------------------------------------------------- /DeviceMonitorNostradamus/src/main/java/com/hortonworks/iot/events/DeviceStatus.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks.iot.events; 2 | 3 | import java.io.Serializable; 4 | 5 | public class DeviceStatus implements Serializable { 6 | private static final long serialVersionUID = 1L; 7 | private String serialNumber; 8 | private String status; 9 | private String state; 10 | 11 | public void setSerialNumber(String value){ 12 | serialNumber = value; 13 | } 14 | public void setState(String value){ 15 | state = value; 16 | } 17 | public void setStatus(String value){ 18 | status = value; 19 | } 20 | 21 | public String getSerialNumber(){ 22 | return serialNumber; 23 | } 24 | public String getState(){ 25 | return state; 26 | } 27 | public String getStatus(){ 28 | return status; 29 | } 30 | } 31 | -------------------------------------------------------------------------------- /DEVICE_MANAGER_DEMO_CONTROL/configuration/control-config.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | democontrol.install_dir 8 | /root 9 | Directory under which demo control utility will be installed 10 | 11 | 12 | 13 | 14 | democontrol.download_url 15 | https://github.com/vakshorton/DataSimulators.git 16 | Online location to download Simulator package 17 | 18 | 19 | 20 | democontrol.nifi_host_ip 21 | 22 | IP address of host where Nifi is listening 23 | 24 | -------------------------------------------------------------------------------- /DeviceMonitor/src/main/java/com/hortonworks/events/TechnicianDetails.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks.events; 2 | 3 | import java.io.Serializable; 4 | 5 | //import java.util.HashMap; 6 | //import java.util.Map; 7 | 8 | public class TechnicianDetails extends TechnicianStatus implements Serializable { 9 | private static final long serialVersionUID = 1L; 10 | private Integer yearsExperience; 11 | private Integer rating; 12 | //private Map parts = new HashMap(); 13 | public TechnicianDetails() { 14 | super(); 15 | } 16 | 17 | public Integer getYearsExperience(){ 18 | return yearsExperience; 19 | } 20 | public Integer getRating(){ 21 | return rating; 22 | } 23 | public void setYearsExperience(Integer value){ 24 | yearsExperience = value; 25 | } 26 | public void setRating(Integer value){ 27 | rating = value; 28 | } 29 | } -------------------------------------------------------------------------------- /SliderConfig/metainfo.json: -------------------------------------------------------------------------------- 1 | { 2 | "schemaVersion": "2.1", 3 | "application": { 4 | "name": "MAPUI", 5 | "components": [ 6 | { 7 | "name": "MAPUI", 8 | "type": "docker", 9 | "dockerContainers": [ 10 | { 11 | "name": "mapui", 12 | "commandPath": "/usr/bin/docker", 13 | "image": "vvaks/mapui", 14 | "ports": [{"containerPort" : "8090"}] 15 | } 16 | ] 17 | }, 18 | { 19 | "name": "COMETD", 20 | "type": "docker", 21 | "dockerContainers": [ 22 | { 23 | "name": "cometd", 24 | "commandPath": "/usr/bin/docker", 25 | "image": "vvaks/cometd", 26 | "ports": [{"containerPort" : "8091"}] 27 | } 28 | ] 29 | } 30 | ] 31 | } 32 | } 33 | -------------------------------------------------------------------------------- /Nifi/config/nifi-flow-env.json: -------------------------------------------------------------------------------- 1 | "properties" : { 2 | "content" : "\n\n 10\n 5\n \n 7c84501d-d10c-407c-b9f3-1d80e38fe36a\n NiFi Flow\n \n \n \n \n \n \n 3b80ba0f-a6c0-48db-b721-4dbc04cef28e\n AmbariReportingTask\n \n org.apache.nifi.reporting.ambari.AmbariReportingTask\n {{nifi_ambari_reporting_frequency}}\n RUNNING\n TIMER_DRIVEN\n \n Metrics Collector URL\n ${ambari.metrics.collector.url}\n \n \n Application ID\n ${ambari.application.id}\n \n \n Hostname\n ${hostname(true)}\n \n \n \n" 3 | } -------------------------------------------------------------------------------- /Cometd/.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /DeviceMonitor/.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | -------------------------------------------------------------------------------- /DeviceMonitor/src/main/java/com/hortonworks/events/STBStatus.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks.events; 2 | 3 | import java.io.Serializable; 4 | 5 | import com.hortonworks.events.DeviceStatus; 6 | 7 | public class STBStatus extends DeviceStatus implements Serializable { 8 | private static final long serialVersionUID = 1L; 9 | private String deviceModel; 10 | private Integer internalTemp; 11 | private Integer signalStrength; 12 | private Double latitude; 13 | private Double longitude; 14 | 15 | public void setDeviceModel(String value){ 16 | deviceModel = value; 17 | } 18 | public void setInternalTemp(Integer value){ 19 | internalTemp = value; 20 | } 21 | public void setSignalStrength(Integer value){ 22 | signalStrength = value; 23 | } 24 | public void setLatitude(Double value){ 25 | latitude = value; 26 | } 27 | public void setLongitude(Double value){ 28 | longitude = value; 29 | } 30 | 31 | public String getDeviceModel(){ 32 | return deviceModel; 33 | } 34 | public Integer getInternalTemp(){ 35 | return internalTemp; 36 | } 37 | public Integer getSignalStrength(){ 38 | return signalStrength; 39 | } 40 | public Double getLatitude(){ 41 | return latitude; 42 | } 43 | public Double getLongitude(){ 44 | return longitude; 45 | } 46 | } -------------------------------------------------------------------------------- /DeviceMonitorNostradamus/src/main/java/com/hortonworks/iot/events/STBStatus.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks.iot.events; 2 | 3 | import java.io.Serializable; 4 | 5 | import com.hortonworks.iot.events.DeviceStatus; 6 | 7 | public class STBStatus extends DeviceStatus implements Serializable { 8 | private static final long serialVersionUID = 1L; 9 | private String deviceModel; 10 | private Integer internalTemp; 11 | private Integer signalStrength; 12 | private Double latitude; 13 | private Double longitude; 14 | 15 | public void setDeviceModel(String value){ 16 | deviceModel = value; 17 | } 18 | public void setInternalTemp(Integer value){ 19 | internalTemp = value; 20 | } 21 | public void setSignalStrength(Integer value){ 22 | signalStrength = value; 23 | } 24 | public void setLatitude(Double value){ 25 | latitude = value; 26 | } 27 | public void setLongitude(Double value){ 28 | longitude = value; 29 | } 30 | 31 | public String getDeviceModel(){ 32 | return deviceModel; 33 | } 34 | public Integer getInternalTemp(){ 35 | return internalTemp; 36 | } 37 | public Integer getSignalStrength(){ 38 | return signalStrength; 39 | } 40 | public Double getLatitude(){ 41 | return latitude; 42 | } 43 | public Double getLongitude(){ 44 | return longitude; 45 | } 46 | } -------------------------------------------------------------------------------- /DeviceMonitor/src/main/java/com/hortonworks/events/DeviceAlert.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks.events; 2 | 3 | import java.io.Serializable; 4 | 5 | public class DeviceAlert implements Serializable { 6 | private static final long serialVersionUID = 1L; 7 | private String serialNumber; 8 | private String deviceModel; 9 | private String alertDescription; 10 | private Double longitude; 11 | private Double latitude; 12 | 13 | public DeviceAlert() {} 14 | 15 | public void setSerialNumber(String value){ 16 | serialNumber = value; 17 | } 18 | public void setDeviceModel(String value){ 19 | deviceModel = value; 20 | } 21 | public void setAlertDescription(String value){ 22 | alertDescription = value; 23 | } 24 | public void setLongitude(Double value){ 25 | longitude = value; 26 | } 27 | public void setLatitude(Double value){ 28 | latitude = value; 29 | } 30 | 31 | public String getSerialNumber(){ 32 | return serialNumber; 33 | } 34 | public String getDeviceModel(){ 35 | return deviceModel; 36 | } 37 | public String getAlertDescription(){ 38 | return alertDescription; 39 | } 40 | public Double getLongitude(){ 41 | return longitude; 42 | } 43 | public Double getLatitude(){ 44 | return latitude; 45 | } 46 | } -------------------------------------------------------------------------------- /DeviceMonitorNostradamus/.classpath: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | -------------------------------------------------------------------------------- /DEVICE_MANAGER_DEMO_CONTROL_SAM/configuration/control-config.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | 7 | democontrol.install.dir 8 | /root 9 | Directory under which demo control utility will be installed 10 | 11 | 12 | 13 | democontrol.device.manager.git.url 14 | https://github.com/vakshorton/DeviceManagerDemo.git 15 | Online location to download Device Manager Application 16 | 17 | 18 | 19 | democontrol.sam.extentions.git.url 20 | https://github.com/vakshorton/sam-custom-extensions.git 21 | Online location to download SAM custom extentions 22 | 23 | 24 | 25 | democontrol.simulator.git.url 26 | https://github.com/vakshorton/DataSimulators.git 27 | Online location to download Simulator package 28 | 29 | 30 | 31 | democontrol.google.api.key 32 | 33 | PASSWORD 34 | Google API Key used to enable Map in Device Manager UI 35 | 36 | password 37 | 38 | 39 | -------------------------------------------------------------------------------- /DEVICE_MANAGER_DEMO_CONTROL/metainfo.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 2.0 4 | 5 | 6 | DEVICE_MANAGER_DEMO_CONTROL 7 | Device Manager Demo Control 8 | Utility to start and stop Device and Technician event generator for the Device Manager Demo 9 | 0.1.0 10 | 11 | 12 | DEVICE_MANAGER_DEMO_CONTROL 13 | Device Manager Demo Control 14 | MASTER 15 | 1 16 | 17 | 18 | PYTHON 19 | 600 20 | 21 | 22 | 23 | 24 | 25 | any 26 | 27 | 28 | 29 | control-config 30 | 31 | false 32 | 33 | 34 | quicklinks.json 35 | true 36 | 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /DEVICE_MANAGER_DEMO_CONTROL_SAM/metainfo.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 2.0 4 | 5 | 6 | DEVICE_MANAGER_DEMO_CONTROL 7 | Device Manager Demo Control 8 | Utility to start and stop Device and Technician event generator for the Device Manager Demo 9 | 0.1.0 10 | 11 | 12 | DEVICE_MANAGER_DEMO_CONTROL 13 | Device Manager Demo Control 14 | MASTER 15 | 1 16 | 17 | 18 | PYTHON 19 | 600 20 | 21 | 22 | 23 | 24 | 25 | any 26 | 27 | 28 | 29 | control-config 30 | 31 | false 32 | 33 | 34 | quicklinks.json 35 | true 36 | 37 | 38 | 39 | 40 | 41 | -------------------------------------------------------------------------------- /DeviceMonitor/src/main/java/com/hortonworks/events/TechnicianStatus.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks.events; 2 | 3 | import java.io.Serializable; 4 | 5 | public class TechnicianStatus implements Serializable { 6 | private static final long serialVersionUID = 1L; 7 | private String technicianId; 8 | private Double longitude; 9 | private Double latitude; 10 | private String status; 11 | private String ipaddress; 12 | private String port; 13 | private String eventType; 14 | 15 | public TechnicianStatus() {} 16 | 17 | public void setTechnicianId(String value){ 18 | technicianId = value; 19 | } 20 | public void setLongitude(Double value){ 21 | longitude = value; 22 | } 23 | public void setLatitude(Double value){ 24 | latitude = value; 25 | } 26 | public void setStatus(String value){ 27 | status = value; 28 | } 29 | public void setIpAddress(String value){ 30 | ipaddress = value; 31 | } 32 | public void setPort(String value){ 33 | port = value; 34 | } 35 | public void setEventType(String value){ 36 | eventType = value; 37 | } 38 | public String getTechnicianId(){ 39 | return technicianId; 40 | } 41 | public Double getLongitude(){ 42 | return longitude; 43 | } 44 | public Double getLatitude(){ 45 | return latitude; 46 | } 47 | public String getStatus(){ 48 | return status; 49 | } 50 | public String getIpAddress(){ 51 | return ipaddress; 52 | } 53 | public String getPort(){ 54 | return port; 55 | } 56 | public String getEventType(){ 57 | return eventType; 58 | } 59 | } -------------------------------------------------------------------------------- /DeviceMonitor/src/main/java/com/hortonworks/bolts/PrintDeviceAlert.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks.bolts; 2 | 3 | import java.util.Map; 4 | 5 | import com.hortonworks.events.DeviceAlert; 6 | 7 | import org.apache.storm.task.OutputCollector; 8 | import org.apache.storm.task.TopologyContext; 9 | import org.apache.storm.topology.OutputFieldsDeclarer; 10 | import org.apache.storm.topology.base.BaseRichBolt; 11 | import org.apache.storm.tuple.Fields; 12 | import org.apache.storm.tuple.Tuple; 13 | import org.apache.storm.tuple.Values; 14 | 15 | /* 16 | import backtype.storm.task.OutputCollector; 17 | import backtype.storm.task.TopologyContext; 18 | import backtype.storm.topology.OutputFieldsDeclarer; 19 | import backtype.storm.topology.base.BaseRichBolt; 20 | import backtype.storm.tuple.Fields; 21 | import backtype.storm.tuple.Tuple; 22 | import backtype.storm.tuple.Values; 23 | */ 24 | 25 | public class PrintDeviceAlert extends BaseRichBolt { 26 | private static final long serialVersionUID = 1L; 27 | private OutputCollector collector; 28 | 29 | public void execute(Tuple tuple) { 30 | DeviceAlert deviceAlert = (DeviceAlert) tuple.getValueByField("DeviceAlert"); 31 | System.out.println("ALERT******"); 32 | System.out.println("Device Serial Number: " + deviceAlert.getSerialNumber()); 33 | System.out.println("Device Model: " + deviceAlert.getDeviceModel()); 34 | System.out.println("Incident Description: " + deviceAlert.getAlertDescription()); 35 | System.out.println("Device Lat: " + deviceAlert.getLatitude()); 36 | System.out.println("Device Lng: " + deviceAlert.getLongitude()); 37 | 38 | collector.emit(tuple, new Values((DeviceAlert)deviceAlert)); 39 | collector.ack(tuple); 40 | } 41 | 42 | public void prepare(Map arg0, TopologyContext arg1, OutputCollector collector) { 43 | this.collector = collector; 44 | } 45 | 46 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 47 | declarer.declare(new Fields("DeviceAlert")); 48 | } 49 | } 50 | -------------------------------------------------------------------------------- /DeviceMonitor/src/main/java/com/hortonworks/bolts/PrintDeviceStatus.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks.bolts; 2 | 3 | import java.util.Map; 4 | 5 | import com.hortonworks.events.STBStatus; 6 | 7 | import org.apache.storm.task.OutputCollector; 8 | import org.apache.storm.task.TopologyContext; 9 | import org.apache.storm.topology.OutputFieldsDeclarer; 10 | import org.apache.storm.topology.base.BaseRichBolt; 11 | import org.apache.storm.tuple.Fields; 12 | import org.apache.storm.tuple.Tuple; 13 | import org.apache.storm.tuple.Values; 14 | 15 | /* 16 | import backtype.storm.task.OutputCollector; 17 | import backtype.storm.task.TopologyContext; 18 | import backtype.storm.topology.OutputFieldsDeclarer; 19 | import backtype.storm.topology.base.BaseRichBolt; 20 | import backtype.storm.tuple.Fields; 21 | import backtype.storm.tuple.Tuple; 22 | import backtype.storm.tuple.Values; 23 | */ 24 | 25 | public class PrintDeviceStatus extends BaseRichBolt { 26 | private static final long serialVersionUID = 1L; 27 | private OutputCollector collector; 28 | 29 | public void execute(Tuple tuple) { 30 | STBStatus deviceStatus = (STBStatus) tuple.getValueByField("DeviceStatus"); 31 | System.out.println("Device Serial Number: " + deviceStatus.getSerialNumber()); 32 | System.out.println("State: " + deviceStatus.getState()); 33 | System.out.println("Status: " + deviceStatus.getStatus()); 34 | System.out.println("Internal Temp: " + deviceStatus.getInternalTemp()); 35 | System.out.println("Signal Strength: " + deviceStatus.getSignalStrength()); 36 | System.out.println("Device Model: " + deviceStatus.getDeviceModel()); 37 | System.out.println("Latitude: " + deviceStatus.getLatitude()); 38 | System.out.println("Longitude: " + deviceStatus.getLongitude()); 39 | //collector.emit(tuple, new Values((DeviceStatus)deviceStatus)); 40 | collector.ack(tuple); 41 | } 42 | 43 | public void prepare(Map arg0, TopologyContext arg1, OutputCollector collector) { 44 | this.collector = collector; 45 | } 46 | 47 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 48 | declarer.declare(new Fields("DeviceStatus")); 49 | } 50 | } 51 | -------------------------------------------------------------------------------- /startSimulation.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | export AMBARI_HOST=$(hostname -f) 4 | 5 | export CLUSTER_NAME=$(curl -u admin:admin -X GET http://$AMBARI_HOST:8080/api/v1/clusters |grep cluster_name|grep -Po ': "(.+)'|grep -Po '[a-zA-Z0-9!$\-]+') 6 | 7 | if [[ -z $CLUSTER_NAME ]]; then 8 | echo "Could not connect to Ambari Server. Please run this script on the same host where Ambari Server is installed." 9 | exit 1 10 | else 11 | echo "*********************************CLUSTER NAME IS: $CLUSTER_NAME" 12 | fi 13 | 14 | getNifiHost () { 15 | NIFI_HOST=$(curl -u admin:admin -X GET http://$AMBARI_HOST:8080/api/v1/clusters/$CLUSTER_NAME/services/NIFI/components/NIFI_MASTER|grep "host_name"|grep -Po ': "([a-zA-Z0-9\-_!?.]+)'|grep -Po '([a-zA-Z0-9\-_!?.]+)') 16 | 17 | echo $NIFI_HOST 18 | } 19 | 20 | NIFI_HOST=$(getNifiHost) 21 | NIFI_HOST_IP=$(getent hosts $NIFI_HOST | awk '{ print $1 }') 22 | 23 | echo "*********************************NIFI HOST: $NIFI_HOST" 24 | echo "*********************************NIFI HOST IP: $NIFI_HOST_IP" 25 | 26 | echo "*********************************Starting Set Top Box Simulations..." 27 | nohup java -jar DeviceSimulator-0.0.1-SNAPSHOT-jar-with-dependencies.jar STB 1000 Simulation $NIFI_HOST_IP > STB_1000_Sim.log 2>&1& 28 | echo $! > STB_1000_Sim.pid 29 | 30 | nohup java -jar DeviceSimulator-0.0.1-SNAPSHOT-jar-with-dependencies.jar STB 2000 Simulation $NIFI_HOST_IP > STB_2000_Sim.log 2>&1& 31 | echo $! > STB_2000_Sim.pid 32 | 33 | nohup java -jar DeviceSimulator-0.0.1-SNAPSHOT-jar-with-dependencies.jar STB 3000 Simulation $NIFI_HOST_IP > STB_3000_Sim.log 2>&1& 34 | echo $! > STB_3000_Sim.pid 35 | 36 | echo "*********************************Starting Technician Simulations..." 37 | nohup java -jar DeviceSimulator-0.0.1-SNAPSHOT-jar-with-dependencies.jar Technician 1000 Simulation $NIFI_HOST_IP > Technician_1000_Sim.log 2>&1& 38 | echo $! > Technician_1000_Sim.pid 39 | 40 | nohup java -jar DeviceSimulator-0.0.1-SNAPSHOT-jar-with-dependencies.jar Technician 2000 Simulation $NIFI_HOST_IP > Technician_2000_Sim.log 2>&1& 41 | echo $! > Technician_2000_Sim.pid 42 | 43 | nohup java -jar DeviceSimulator-0.0.1-SNAPSHOT-jar-with-dependencies.jar Technician 3000 Simulation $NIFI_HOST_IP > Technician_3000_Sim.log 2>&1& 44 | echo $! > Technician_3000_Sim.pid 45 | 46 | exit 0 -------------------------------------------------------------------------------- /Cometd/src/main/java/com/hortonworks/iot/cometd/Main.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks.iot.cometd; 2 | 3 | import java.util.Map; 4 | 5 | import org.cometd.bayeux.server.BayeuxServer; 6 | import org.cometd.bayeux.server.ConfigurableServerChannel; 7 | import org.cometd.bayeux.server.ServerChannel; 8 | import org.cometd.server.CometDServlet; 9 | import org.eclipse.jetty.server.Server; 10 | import org.eclipse.jetty.servlet.ServletContextHandler; 11 | import org.eclipse.jetty.servlet.ServletHolder; 12 | 13 | public class Main { 14 | 15 | public static void main(String[] args) throws Exception { 16 | // Configure Jetty 17 | Integer cometdListenPort = 8091; 18 | Map env = System.getenv(); 19 | System.out.println("********************** ENV: " + env); 20 | if(env.get("COMETD_PORT") != null){ 21 | cometdListenPort = Integer.valueOf((String)env.get("COMETD_PORT")); 22 | } 23 | 24 | System.out.println("********************** Cometd Port: " + cometdListenPort); 25 | Server server = new Server(cometdListenPort); 26 | 27 | ServletContextHandler webAppContext = new ServletContextHandler(ServletContextHandler.SESSIONS); 28 | webAppContext.setContextPath("/"); 29 | 30 | System.out.println("Loading Cometd PubSub....."); 31 | // Configure Cometd 32 | CometDServlet cometdServlet = new CometDServlet(); 33 | webAppContext.addServlet(new ServletHolder(cometdServlet), "/cometd/*"); 34 | server.setHandler(webAppContext); 35 | 36 | System.out.println("Starting Jetty Server....."); 37 | // Start Jetty 38 | server.start(); 39 | 40 | System.out.println("Creating PubSub Topics....."); 41 | // Create channels to publish on 42 | BayeuxServer bayeux = cometdServlet.getBayeux(); 43 | createChannel(bayeux, "/devicestatus"); 44 | createChannel(bayeux, "/technicianstatus"); 45 | createChannel(bayeux, "/alert"); 46 | createChannel(bayeux, "/prediction"); 47 | createChannel(bayeux, "/fraudAlert"); 48 | createChannel(bayeux, "/incomingTransactions"); 49 | createChannel(bayeux, "/accountStatusUpdate"); 50 | createChannel(bayeux, "/bioReactorStatus"); 51 | createChannel(bayeux, "/filtrationStatus"); 52 | createChannel(bayeux, "/sentiment"); 53 | //final ServerChannel channel = bayeux.getChannel(""); 54 | 55 | server.join(); 56 | } 57 | 58 | public static void createChannel(BayeuxServer bayeux, String channelName){ 59 | bayeux.createChannelIfAbsent(channelName, new ServerChannel.Initializer() 60 | { 61 | public void configureChannel(ConfigurableServerChannel channel) 62 | { 63 | channel.setPersistent(true); 64 | System.out.println(channel.getChannelId()); 65 | } 66 | }); 67 | } 68 | } 69 | -------------------------------------------------------------------------------- /Zeppelin/notebook/2B75QB4J1/note.json: -------------------------------------------------------------------------------- 1 | { 2 | "paragraphs": [ 3 | { 4 | "text": "%dep\nz.load(\"org.apache.spark:spark-streaming-kafka_2.10:1.5.2\")", 5 | "dateUpdated": "Dec 16, 2015 10:48:58 PM", 6 | "config": { 7 | "colWidth": 12.0, 8 | "graph": { 9 | "mode": "table", 10 | "height": 300.0, 11 | "optionOpen": false, 12 | "keys": [], 13 | "values": [], 14 | "groups": [], 15 | "scatter": {} 16 | }, 17 | "editorMode": "ace/mode/scala" 18 | }, 19 | "settings": { 20 | "params": {}, 21 | "forms": {} 22 | }, 23 | "jobName": "paragraph_1450305854262_-641518226", 24 | "id": "20151216-224414_1407033887", 25 | "result": { 26 | "code": "SUCCESS", 27 | "type": "TEXT", 28 | "msg": "res0: org.apache.zeppelin.spark.dep.Dependency \u003d org.apache.zeppelin.spark.dep.Dependency@299aef85\n" 29 | }, 30 | "dateCreated": "Dec 16, 2015 10:44:14 PM", 31 | "dateStarted": "Dec 16, 2015 10:48:58 PM", 32 | "dateFinished": "Dec 16, 2015 10:49:37 PM", 33 | "status": "FINISHED", 34 | "progressUpdateIntervalMs": 500 35 | }, 36 | { 37 | "text": "import org.apache.spark.streaming.kafka._\n \nval kafkaStream \u003d KafkaUtils.createStream(streamingContext, \"localhost:2181\", [consumer group id], [per-topic number of Kafka partitions to consume])", 38 | "dateUpdated": "Dec 16, 2015 10:52:37 PM", 39 | "config": { 40 | "colWidth": 12.0, 41 | "graph": { 42 | "mode": "table", 43 | "height": 300.0, 44 | "optionOpen": false, 45 | "keys": [], 46 | "values": [], 47 | "groups": [], 48 | "scatter": {} 49 | }, 50 | "editorMode": "ace/mode/scala" 51 | }, 52 | "settings": { 53 | "params": {}, 54 | "forms": {} 55 | }, 56 | "jobName": "paragraph_1450305904843_828699266", 57 | "id": "20151216-224504_2103656662", 58 | "dateCreated": "Dec 16, 2015 10:45:04 PM", 59 | "status": "READY", 60 | "progressUpdateIntervalMs": 500 61 | } 62 | ], 63 | "name": "Device Management Spark Streaming", 64 | "id": "2B75QB4J1", 65 | "angularObjects": { 66 | "2B9EDS7RX": [], 67 | "2B8V224NM": [], 68 | "2B5YUQKT8": [], 69 | "2B8JH2674": [], 70 | "2B8AE3SQC": [], 71 | "2B8RDV9CU": [], 72 | "2B6XJD72P": [], 73 | "2B889XZ5W": [], 74 | "2B5SJ7RSF": [], 75 | "2B8WH294Q": [], 76 | "2B7K5YMG2": [], 77 | "2B7Y9ZUR6": [], 78 | "2B5Y3SDHE": [], 79 | "2B8B5C112": [] 80 | }, 81 | "config": {}, 82 | "info": {} 83 | } -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # DeviceManagerDemo 2 | The Device Manager Demo is designed to demonstrate a fully functioning modern Data/IoT application. 3 | It is a Lambda architecture built using the Hortonworks Data Platform and Hortonworks Data Flow. 4 | The demo shows how a Telecom can manage customer device outages using predictive maintenance and a connected workforce. 5 | 6 | # Install DeviceManagerDemo 7 | Download and Import Hortonworks Sandbox 2.4 for Virtual Box. 8 | 9 | Should work with VMWare but has not been tested. 10 | 11 | Modify local hosts file so that sandbox.hortonworks.com resolves to 127.0.0.1 (This is important and may break the simulator and UI) 12 | 13 | Configure Virtual Box Port Forward 14 | 15 | 8082 – HDF_HTTP_Ingest 16 | 17 | 8090 - MapUI 18 | 19 | 8091 - Cometd 20 | 21 | 9090 – HDF_Studio 22 | 23 | Start Sandbox, SSH to Sandbox (ssh root@sandbox.hortonworks.com -p 2222) 24 | 25 | Wait for Sandbox to fully boot up, all service need to finish starting 26 | 27 | Change Ambari password to "admin" (ambari-admin-password-reset) 28 | 29 | (!!!!!!!!!!!!!AMBARI PASSWORD MUST BE SET TO "admin" (no quotes)!!!!!!) 30 | 31 | cd /root (use the /root directory to begin the install) 32 | 33 | git clone https://github.com/vakshorton/DeviceManagerDemo.git 34 | 35 | (make sure that git cloned to /root/DeviceManagerDemo) 36 | 37 | cd DeviceManagerDemo 38 | 39 | ./install.sh 40 | 41 | Install script reboot once complete. This is required to refresh session configuration. 42 | 43 | Wait for Sandbox to fully reboot 44 | 45 | Log back in via SSH (ssh root@sandbox.hortonworks.com -p 2222) 46 | 47 | # Start Demo 48 | The script to start demo services should be located in the /root/DeviceManagerDemo directory 49 | 50 | ./startDemoServices.sh 51 | 52 | Slider will download the servlet (UI) docker containers from the docker hub so it may take a few minutes for the application server to start 53 | 54 | Bring up the UI in a Browser: http://sandbox.hortonworks.com:8090/MapUI/DeviceMap 55 | 56 | Start Simulation: 57 | 58 | The Simulator should be located in the DeviceManagerDemo directory 59 | 60 | java -jar DeviceSimulator-0.0.1-SNAPSHOT-jar-with-dependencies.jar Technician 1000 Simulation 61 | 62 | java -jar DeviceSimulator-0.0.1-SNAPSHOT-jar-with-dependencies.jar STB 1000 Simulation 63 | 64 | USAGE: 65 | 66 | java -jar simulator.jar arg1=Simulator-Type{BioReactor|FiltrationSystem} arg2=EntityId{1000} arg3={Simulation|Training} 67 | 68 | Example: 69 | 70 | java -jar DeviceSimulator-0.0.1-SNAPSHOT-jar-with-dependencies.jar Technician 1000 Simulation 71 | 72 | java -jar DeviceSimulator-0.0.1-SNAPSHOT-jar-with-dependencies.jar STB 1000 Simulation 73 | 74 | # MAP UI 75 | ![Alt text](/screenshot/DeviceManagerScreenShot.png "MapUI") 76 | 77 | 78 | -------------------------------------------------------------------------------- /Nifi/config/nifi-bootstrap-env.json: -------------------------------------------------------------------------------- 1 | "properties" : { 2 | "content" : "\n# Java command to use when running NiFi\njava=java\n\n# Username to use when running NiFi. This value will be ignored on Windows.\nrun.as={{nifi_user}}\n\n# Configure where NiFi's lib and conf directories live\nlib.dir=./lib\nconf.dir=./conf\n\n# How long to wait after telling NiFi to shutdown before explicitly killing the Process\ngraceful.shutdown.seconds=20\n\n# Disable JSR 199 so that we can use JSP's without running a JDK\njava.arg.1=-Dorg.apache.jasper.compiler.disablejsr199=true\n\n# JVM memory settings\njava.arg.2=-Xms{{nifi_initial_mem}}\njava.arg.3=-Xmx{{nifi_max_mem}}\n\n# Enable Remote Debugging\n#java.arg.debug=-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8000\n\njava.arg.4=-Djava.net.preferIPv4Stack=true\n\n# allowRestrictedHeaders is required for Cluster/Node communications to work properly\njava.arg.5=-Dsun.net.http.allowRestrictedHeaders=true\njava.arg.6=-Djava.protocol.handler.pkgs=sun.net.www.protocol\n\n# Java 7 and below have issues with Code Cache. The following lines allow us to run well even with\n# many classes loaded in the JVM.\n#java.arg.7=-XX:ReservedCodeCacheSize=256m\n#java.arg.8=-XX:CodeCacheFlushingMinimumFreeSpace=10m\n#java.arg.9=-XX:+UseCodeCacheFlushing\n#java.arg.11=-XX:PermSize=128M\n#java.arg.12=-XX:MaxPermSize=128M\n\n# The G1GC is still considered experimental but has proven to be very advantageous in providing great\n# performance without significant \"stop-the-world\" delays.\n#java.arg.13=-XX:+UseG1GC\n\n#Set headless mode by default\njava.arg.14=-Djava.awt.headless=true\n\n#Ambari Metrics Collector URL - passed in to flow.xml for AmbariReportingTask\njava.arg.15=-Dambari.metrics.collector.url=http://{{metrics_collector_host}}:{{metrics_collector_port}}/ws/v1/timeline/metrics\n\n#Application ID - used in flow.xml - passed into flow.xml for AmbariReportingTask \njava.arg.16=-Dambari.application.id=nifi\n\n\n###\n# Notification Services for notifying interested parties when NiFi is stopped, started, dies\n###\n\n# XML File that contains the definitions of the notification services\nnotification.services.file=./conf/bootstrap-notification-services.xml\n\n# In the case that we are unable to send a notification for an event, how many times should we retry?\nnotification.max.attempts=5\n\n# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is started?\n#nifi.start.notification.services=email-notification\n\n# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi is stopped?\n#nifi.stop.notification.services=email-notification\n\n# Comma-separated list of identifiers that are present in the notification.services.file; which services should be used to notify when NiFi dies?\n#nifi.dead.notification.services=email-notification", 3 | "nifi_group" : "nifi", 4 | "nifi_log_dir" : "/var/log/nifi", 5 | "nifi_pid_dir" : "/var/run/nifi", 6 | "nifi_user" : "nifi" 7 | } -------------------------------------------------------------------------------- /Model/device_manager_svm.xml: -------------------------------------------------------------------------------- 1 | 2 |
3 | 4 | 5 | 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 | 27 | 28 | 29 | 30 | 31 | 32 | 33 | 34 | 35 | 36 | 37 | 38 | 39 | 40 | 41 | 42 | 43 | 44 | 45 | 46 | 47 | -------------------------------------------------------------------------------- /DeviceMonitor/src/main/java/com/hortonworks/bolts/PublishAlert.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks.bolts; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | import org.cometd.client.BayeuxClient; 7 | import org.cometd.client.transport.ClientTransport; 8 | import org.cometd.client.transport.LongPollingTransport; 9 | import org.eclipse.jetty.client.HttpClient; 10 | 11 | import com.hortonworks.events.DeviceAlert; 12 | import com.hortonworks.util.Constants; 13 | 14 | import org.apache.storm.task.OutputCollector; 15 | import org.apache.storm.task.TopologyContext; 16 | import org.apache.storm.topology.OutputFieldsDeclarer; 17 | import org.apache.storm.topology.base.BaseRichBolt; 18 | import org.apache.storm.tuple.Fields; 19 | import org.apache.storm.tuple.Tuple; 20 | import org.apache.storm.tuple.Values; 21 | 22 | /* 23 | import backtype.storm.task.OutputCollector; 24 | import backtype.storm.task.TopologyContext; 25 | import backtype.storm.topology.OutputFieldsDeclarer; 26 | import backtype.storm.topology.base.BaseRichBolt; 27 | import backtype.storm.tuple.Fields; 28 | import backtype.storm.tuple.Tuple; 29 | import backtype.storm.tuple.Values; 30 | */ 31 | 32 | public class PublishAlert extends BaseRichBolt { 33 | private String pubSubUrl; 34 | private String alertChannel; 35 | private BayeuxClient bayuexClient; 36 | private OutputCollector collector; 37 | private Constants constants; 38 | 39 | public void execute(Tuple tuple) { 40 | DeviceAlert deviceAlert = (DeviceAlert) tuple.getValueByField("DeviceAlert"); 41 | 42 | Map data = new HashMap(); 43 | data.put("deviceSerialNumber", deviceAlert.getSerialNumber()); 44 | data.put("alertDescription", deviceAlert.getAlertDescription()); 45 | data.put("deviceModel", deviceAlert.getDeviceModel()); 46 | data.put("latitude", deviceAlert.getLatitude()); 47 | data.put("longitude", deviceAlert.getLongitude()); 48 | bayuexClient.getChannel(alertChannel).publish(data); 49 | 50 | collector.emit(tuple, new Values((DeviceAlert)deviceAlert)); 51 | collector.ack(tuple); 52 | } 53 | 54 | public void prepare(Map arg0, TopologyContext arg1, OutputCollector collector) { 55 | this.collector = collector; 56 | this.constants = new Constants(); 57 | pubSubUrl = constants.getPubSubUrl(); 58 | alertChannel = constants.getAlertChannel(); 59 | 60 | HttpClient httpClient = new HttpClient(); 61 | try { 62 | httpClient.start(); 63 | } catch (Exception e) { 64 | e.printStackTrace(); 65 | } 66 | 67 | // Prepare the transport 68 | Map options = new HashMap(); 69 | ClientTransport transport = new LongPollingTransport(options, httpClient); 70 | 71 | // Create the BayeuxClient 72 | bayuexClient = new BayeuxClient(pubSubUrl, transport); 73 | 74 | bayuexClient.handshake(); 75 | boolean handshaken = bayuexClient.waitFor(3000, BayeuxClient.State.CONNECTED); 76 | if (handshaken) 77 | { 78 | System.out.println("Connected to Cometd Http PubSub Platform"); 79 | } 80 | else{ 81 | System.out.println("Could not connect to Cometd Http PubSub Platform"); 82 | } 83 | } 84 | 85 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 86 | declarer.declare(new Fields("DeviceAlert")); 87 | } 88 | } -------------------------------------------------------------------------------- /DeviceMonitor/src/main/java/com/hortonworks/bolts/PublishTechnicianLocation.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks.bolts; 2 | 3 | import java.io.IOException; 4 | import java.util.HashMap; 5 | import java.util.Map; 6 | 7 | import org.cometd.client.BayeuxClient; 8 | import org.cometd.client.transport.ClientTransport; 9 | import org.cometd.client.transport.LongPollingTransport; 10 | import org.eclipse.jetty.client.HttpClient; 11 | 12 | import com.hortonworks.events.TechnicianStatus; 13 | import com.hortonworks.util.Constants; 14 | 15 | import org.apache.storm.task.OutputCollector; 16 | import org.apache.storm.task.TopologyContext; 17 | import org.apache.storm.topology.OutputFieldsDeclarer; 18 | import org.apache.storm.topology.base.BaseRichBolt; 19 | import org.apache.storm.tuple.Fields; 20 | import org.apache.storm.tuple.Tuple; 21 | import org.apache.storm.tuple.Values; 22 | 23 | /* 24 | import backtype.storm.task.OutputCollector; 25 | import backtype.storm.task.TopologyContext; 26 | import backtype.storm.topology.OutputFieldsDeclarer; 27 | import backtype.storm.topology.base.BaseRichBolt; 28 | import backtype.storm.tuple.Fields; 29 | import backtype.storm.tuple.Tuple; 30 | import backtype.storm.tuple.Values; 31 | */ 32 | 33 | public class PublishTechnicianLocation extends BaseRichBolt { 34 | private String pubSubUrl; 35 | private String techChannel; 36 | private BayeuxClient bayuexClient; 37 | private OutputCollector collector; 38 | private Constants constants; 39 | 40 | public void execute(Tuple tuple) { 41 | TechnicianStatus technicianStatus = (TechnicianStatus) tuple.getValueByField("TechnicianStatus"); 42 | Map data = new HashMap(); 43 | data.put("technicianId", technicianStatus.getTechnicianId()); 44 | data.put("latitude", technicianStatus.getLatitude()); 45 | data.put("longitude", technicianStatus.getLongitude()); 46 | data.put("status", technicianStatus.getStatus()); 47 | data.put("eventType", technicianStatus.getEventType()); 48 | bayuexClient.getChannel(techChannel).publish(data); 49 | 50 | collector.emit(tuple, new Values((TechnicianStatus)technicianStatus)); 51 | collector.ack(tuple); 52 | } 53 | 54 | public void prepare(Map arg0, TopologyContext arg1, OutputCollector collector) { 55 | this.collector = collector; 56 | constants = new Constants(); 57 | pubSubUrl = constants.getPubSubUrl(); 58 | techChannel = constants.getTechnicianChannel(); 59 | 60 | HttpClient httpClient = new HttpClient(); 61 | try { 62 | httpClient.start(); 63 | } catch (Exception e) { 64 | e.printStackTrace(); 65 | } 66 | 67 | // Prepare the transport 68 | Map options = new HashMap(); 69 | ClientTransport transport = new LongPollingTransport(options, httpClient); 70 | 71 | // Create the BayeuxClient 72 | bayuexClient = new BayeuxClient(pubSubUrl, transport); 73 | 74 | bayuexClient.handshake(); 75 | boolean handshaken = bayuexClient.waitFor(1000, BayeuxClient.State.CONNECTED); 76 | if (handshaken) 77 | { 78 | System.out.println("Connected to Cometd Http PubSub Platform"); 79 | } 80 | else{ 81 | System.out.println("Could not connect to Cometd Http PubSub Platform"); 82 | } 83 | } 84 | 85 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 86 | declarer.declare(new Fields("TechnicianStatus")); 87 | } 88 | } -------------------------------------------------------------------------------- /DeviceMonitor/src/main/java/com/hortonworks/bolts/PublishDeviceStatus.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks.bolts; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | 6 | import org.cometd.client.BayeuxClient; 7 | import org.cometd.client.transport.ClientTransport; 8 | import org.cometd.client.transport.LongPollingTransport; 9 | import org.eclipse.jetty.client.HttpClient; 10 | 11 | import com.hortonworks.events.DeviceStatus; 12 | import com.hortonworks.events.STBStatus; 13 | import com.hortonworks.util.Constants; 14 | 15 | import org.apache.storm.task.OutputCollector; 16 | import org.apache.storm.task.TopologyContext; 17 | import org.apache.storm.topology.OutputFieldsDeclarer; 18 | import org.apache.storm.topology.base.BaseRichBolt; 19 | import org.apache.storm.tuple.Fields; 20 | import org.apache.storm.tuple.Tuple; 21 | import org.apache.storm.tuple.Values; 22 | 23 | /* 24 | import backtype.storm.task.OutputCollector; 25 | import backtype.storm.task.TopologyContext; 26 | import backtype.storm.topology.OutputFieldsDeclarer; 27 | import backtype.storm.topology.base.BaseRichBolt; 28 | import backtype.storm.tuple.Fields; 29 | import backtype.storm.tuple.Tuple; 30 | import backtype.storm.tuple.Values; 31 | */ 32 | 33 | public class PublishDeviceStatus extends BaseRichBolt { 34 | private String pubSubUrl; 35 | private String deviceChannel; 36 | private BayeuxClient bayuexClient; 37 | private OutputCollector collector; 38 | private Constants constants; 39 | 40 | public void execute(Tuple tuple) { 41 | STBStatus deviceStatus = (STBStatus) tuple.getValueByField("DeviceStatus"); 42 | 43 | Map data = new HashMap(); 44 | data.put("deviceSerialNumber", deviceStatus.getSerialNumber()); 45 | data.put("internalTemp", deviceStatus.getInternalTemp()); 46 | data.put("signalStrength", deviceStatus.getSignalStrength()); 47 | data.put("deviceModel", deviceStatus.getDeviceModel()); 48 | data.put("latitude", deviceStatus.getLatitude()); 49 | data.put("longitude", deviceStatus.getLongitude()); 50 | data.put("status", deviceStatus.getStatus()); 51 | data.put("state", deviceStatus.getState()); 52 | bayuexClient.getChannel(deviceChannel).publish(data); 53 | 54 | collector.emit(tuple, new Values((DeviceStatus)deviceStatus)); 55 | collector.ack(tuple); 56 | } 57 | 58 | public void prepare(Map arg0, TopologyContext arg1, OutputCollector collector) { 59 | this.collector = collector; 60 | this.constants = new Constants(); 61 | pubSubUrl = constants.getPubSubUrl(); 62 | deviceChannel = constants.getDeviceChannel(); 63 | 64 | HttpClient httpClient = new HttpClient(); 65 | try { 66 | httpClient.start(); 67 | } catch (Exception e) { 68 | e.printStackTrace(); 69 | } 70 | 71 | // Prepare the transport 72 | Map options = new HashMap(); 73 | ClientTransport transport = new LongPollingTransport(options, httpClient); 74 | 75 | // Create the BayeuxClient 76 | bayuexClient = new BayeuxClient(pubSubUrl, transport); 77 | 78 | bayuexClient.handshake(); 79 | boolean handshaken = bayuexClient.waitFor(3000, BayeuxClient.State.CONNECTED); 80 | if (handshaken) 81 | { 82 | System.out.println("Connected to Cometd Http PubSub Platform"); 83 | } 84 | else{ 85 | System.out.println("Could not connect to Cometd Http PubSub Platform"); 86 | } 87 | } 88 | 89 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 90 | declarer.declare(new Fields("DeviceStatus")); 91 | } 92 | } -------------------------------------------------------------------------------- /DeviceMonitor/src/main/java/com/hortonworks/bolts/RouteTechnician.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks.bolts; 2 | 3 | import java.io.IOException; 4 | import java.io.OutputStream; 5 | import java.net.HttpURLConnection; 6 | import java.net.URL; 7 | import java.util.HashMap; 8 | import java.util.Map; 9 | 10 | import org.codehaus.jackson.JsonGenerationException; 11 | import org.codehaus.jackson.map.JsonMappingException; 12 | import org.codehaus.jackson.map.ObjectMapper; 13 | 14 | import com.hortonworks.events.TechnicianDestination; 15 | import com.hortonworks.util.Constants; 16 | 17 | import org.apache.storm.task.OutputCollector; 18 | import org.apache.storm.task.TopologyContext; 19 | import org.apache.storm.topology.OutputFieldsDeclarer; 20 | import org.apache.storm.topology.base.BaseRichBolt; 21 | import org.apache.storm.tuple.Fields; 22 | import org.apache.storm.tuple.Tuple; 23 | import org.apache.storm.tuple.Values; 24 | 25 | /* 26 | import backtype.storm.task.OutputCollector; 27 | import backtype.storm.task.TopologyContext; 28 | import backtype.storm.topology.OutputFieldsDeclarer; 29 | import backtype.storm.topology.base.BaseRichBolt; 30 | import backtype.storm.tuple.Fields; 31 | import backtype.storm.tuple.Tuple; 32 | import backtype.storm.tuple.Values; 33 | */ 34 | 35 | public class RouteTechnician extends BaseRichBolt { 36 | static final long serialVersionUID = 1L; 37 | private TechnicianDestination technicianDestination; 38 | private OutputCollector collector; 39 | private Constants constants; 40 | 41 | public RouteTechnician() { 42 | super(); 43 | } 44 | 45 | public void execute(Tuple tuple) { 46 | technicianDestination = (TechnicianDestination)tuple.getValueByField("TechnicianDestination"); 47 | try{ 48 | URL url = new URL("http://" + constants.getNameNodeHost() +":8084/contentListener"); 49 | HttpURLConnection conn = (HttpURLConnection) url.openConnection(); 50 | conn.setDoOutput(true); 51 | conn.setRequestMethod("POST"); 52 | conn.setRequestProperty("Content-Type", "application/json"); 53 | 54 | System.out.println("To String: " + convertPOJOToJSON(technicianDestination)); 55 | 56 | OutputStream os = conn.getOutputStream(); 57 | os.write(convertPOJOToJSON(technicianDestination).getBytes()); 58 | os.flush(); 59 | 60 | if (conn.getResponseCode() != 200) { 61 | throw new RuntimeException("Failed : HTTP error code : " 62 | + conn.getResponseCode()); 63 | } 64 | 65 | } catch (Exception e) { 66 | e.printStackTrace(); 67 | } 68 | collector.ack(tuple); 69 | } 70 | 71 | public void prepare(Map arg0, TopologyContext arg1, OutputCollector collector) { 72 | this.collector = collector; 73 | this.constants = new Constants(); 74 | } 75 | 76 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 77 | declarer.declare(new Fields("TechnicianDestination")); 78 | } 79 | 80 | public String convertPOJOToJSON(Object pojo) { 81 | String jsonString = ""; 82 | ObjectMapper mapper = new ObjectMapper(); 83 | 84 | try { 85 | jsonString = mapper.writeValueAsString(pojo); 86 | } catch (JsonGenerationException e) { 87 | e.printStackTrace(); 88 | } catch (JsonMappingException e) { 89 | e.printStackTrace(); 90 | } catch (IOException e) { 91 | e.printStackTrace(); 92 | } 93 | return jsonString; 94 | } 95 | } -------------------------------------------------------------------------------- /DEVICE_MANAGER_DEMO_CONTROL/package/scripts/master.py: -------------------------------------------------------------------------------- 1 | import sys, os, pwd, signal, time, shutil 2 | from subprocess import * 3 | from resource_management import * 4 | 5 | class DemoControl(Script): 6 | def install(self, env): 7 | self.configure(env) 8 | import params 9 | 10 | if not os.path.exists(params.install_dir): 11 | os.makedirs(params.install_dir) 12 | os.chdir(params.install_dir) 13 | Execute('git clone ' + params.download_url) 14 | os.chdir(params.install_dir+'/DataSimulators/DeviceSimulator') 15 | Execute('mvn clean package') 16 | os.chdir(params.install_dir+'/DataSimulators/DeviceSimulator/target') 17 | shutil.copy('DeviceSimulator-0.0.1-SNAPSHOT-jar-with-dependencies.jar', params.install_dir) 18 | 19 | def start(self, env): 20 | self.configure(env) 21 | import params 22 | Execute('echo Start Simulation') 23 | Execute('nohup java -jar '+params.install_dir+'/DeviceSimulator-0.0.1-SNAPSHOT-jar-with-dependencies.jar STB 1000 Simulation '+params.nifi_host_ip+' > '+params.install_dir+'/STB_1000_Sim.log 2>&1 & echo $! > /var/run/STB_1000_Sim.pid') 24 | Execute('nohup java -jar '+params.install_dir+'/DeviceSimulator-0.0.1-SNAPSHOT-jar-with-dependencies.jar STB 2000 Simulation '+params.nifi_host_ip+' > '+params.install_dir+'/STB_2000_Sim.log 2>&1 & echo $! > /var/run/STB_2000_Sim.pid') 25 | Execute('nohup java -jar '+params.install_dir+'/DeviceSimulator-0.0.1-SNAPSHOT-jar-with-dependencies.jar STB 3000 Simulation '+params.nifi_host_ip+' > '+params.install_dir+'/STB_3000_Sim.log 2>&1 & echo $! > /var/run/STB_3000_Sim.pid') 26 | 27 | Execute('nohup java -jar '+params.install_dir+'/DeviceSimulator-0.0.1-SNAPSHOT-jar-with-dependencies.jar Technician 1000 Simulation '+params.nifi_host_ip+' > '+params.install_dir+'/Technician_1000_Sim.log 2>&1 & echo $! > /var/run/Technician_1000_Sim.pid') 28 | Execute('nohup java -jar '+params.install_dir+'/DeviceSimulator-0.0.1-SNAPSHOT-jar-with-dependencies.jar Technician 2000 Simulation '+params.nifi_host_ip+' > '+params.install_dir+'/Technician_2000_Sim.log 2>&1 & echo $! > /var/run/Technician_2000_Sim.pid') 29 | Execute('nohup java -jar '+params.install_dir+'/DeviceSimulator-0.0.1-SNAPSHOT-jar-with-dependencies.jar Technician 3000 Simulation '+params.nifi_host_ip+' > '+params.install_dir+'/Technician_3000_Sim.log 2>&1 & echo $! > /var/run/Technician_3000_Sim.pid') 30 | 31 | def stop(self, env): 32 | self.configure(env) 33 | import params 34 | Execute('echo Stop Simulation') 35 | Execute (format('kill -9 `cat /var/run/STB_1000_Sim.pid` >/dev/null 2>&1')) 36 | Execute (format('kill -9 `cat /var/run/STB_2000_Sim.pid` >/dev/null 2>&1')) 37 | Execute (format('kill -9 `cat /var/run/STB_3000_Sim.pid` >/dev/null 2>&1')) 38 | Execute (format('kill -9 `cat /var/run/Technician_1000_Sim.pid` >/dev/null 2>&1')) 39 | Execute (format('kill -9 `cat /var/run/Technician_2000_Sim.pid` >/dev/null 2>&1')) 40 | Execute (format('kill -9 `cat /var/run/Technician_3000_Sim.pid` >/dev/null 2>&1')) 41 | 42 | Execute ('rm -f /var/run/STB_1000_Sim.pid') 43 | Execute ('rm -f /var/run/STB_2000_Sim.pid') 44 | Execute ('rm -f /var/run/STB_3000_Sim.pid') 45 | Execute ('rm -f /var/run/Technician_1000_Sim.pid') 46 | Execute ('rm -f /var/run/Technician_2000_Sim.pid') 47 | Execute ('rm -f /var/run/Technician_3000_Sim.pid') 48 | 49 | def status(self, env): 50 | import params 51 | env.set_params(params) 52 | check_process_status('/var/run/STB_1000_Sim.pid') 53 | 54 | def configure(self, env): 55 | import params 56 | env.set_params(params) 57 | 58 | if __name__ == "__main__": 59 | DemoControl().execute() 60 | -------------------------------------------------------------------------------- /DeviceMonitor/src/main/java/com/hortonworks/util/DeviceEventJSONScheme.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks.util; 2 | 3 | import java.io.IOException; 4 | import java.nio.ByteBuffer; 5 | import java.nio.charset.Charset; 6 | import java.util.List; 7 | 8 | import org.apache.storm.kafka.KeyValueScheme; 9 | import org.apache.storm.kafka.StringScheme; 10 | import org.apache.storm.tuple.Fields; 11 | import org.apache.storm.tuple.Values; 12 | import org.codehaus.jackson.JsonParseException; 13 | import org.codehaus.jackson.map.JsonMappingException; 14 | import org.codehaus.jackson.map.ObjectMapper; 15 | 16 | import com.hortonworks.events.STBStatus; 17 | 18 | /* 19 | import backtype.storm.spout.Scheme; 20 | import backtype.storm.tuple.Fields; 21 | import backtype.storm.tuple.Values; 22 | import storm.kafka.KeyValueScheme; 23 | */ 24 | 25 | /** 26 | * Encodes a byte array into a single UTF-8 string. Very useful for testing and passing raw JSON messages around without 27 | * proper deserialization. 28 | */ 29 | public class DeviceEventJSONScheme implements KeyValueScheme { 30 | private static final long serialVersionUID = 1L; 31 | private static final Charset UTF8 = Charset.forName("UTF-8"); 32 | 33 | public List deserialize(byte[] bytes) { 34 | String eventJSONString = new String(bytes, UTF8); 35 | STBStatus stbStatus = null; 36 | ObjectMapper mapper = new ObjectMapper(); 37 | 38 | try { 39 | stbStatus = mapper.readValue(eventJSONString, STBStatus.class); 40 | } catch (JsonParseException e) { 41 | e.printStackTrace(); 42 | } catch (JsonMappingException e) { 43 | e.printStackTrace(); 44 | } catch (IOException e) { 45 | e.printStackTrace(); 46 | } 47 | return new Values(stbStatus); 48 | } 49 | 50 | public List deserializeKeyAndValue(byte[] key, byte[] value) { 51 | String eventJSONString = new String(value, UTF8); 52 | STBStatus stbStatus = null; 53 | ObjectMapper mapper = new ObjectMapper(); 54 | 55 | try { 56 | stbStatus = mapper.readValue(eventJSONString, STBStatus.class); 57 | } catch (JsonParseException e) { 58 | e.printStackTrace(); 59 | } catch (JsonMappingException e) { 60 | e.printStackTrace(); 61 | } catch (IOException e) { 62 | e.printStackTrace(); 63 | } 64 | return new Values(stbStatus); 65 | } 66 | 67 | public List deserializeKeyAndValue(ByteBuffer key, ByteBuffer value) { 68 | String eventKey = StringScheme.deserializeString(key); 69 | String eventJSONString = StringScheme.deserializeString(value); 70 | STBStatus stbStatus = null; 71 | ObjectMapper mapper = new ObjectMapper(); 72 | 73 | try { 74 | stbStatus = mapper.readValue(eventJSONString, STBStatus.class); 75 | } catch (JsonParseException e) { 76 | e.printStackTrace(); 77 | } catch (JsonMappingException e) { 78 | e.printStackTrace(); 79 | } catch (IOException e) { 80 | e.printStackTrace(); 81 | } 82 | return new Values(stbStatus); 83 | } 84 | public List deserialize(ByteBuffer value) { 85 | String eventJSONString = StringScheme.deserializeString(value); 86 | STBStatus stbStatus = null; 87 | ObjectMapper mapper = new ObjectMapper(); 88 | 89 | try { 90 | stbStatus = mapper.readValue(eventJSONString, STBStatus.class); 91 | } catch (JsonParseException e) { 92 | e.printStackTrace(); 93 | } catch (JsonMappingException e) { 94 | e.printStackTrace(); 95 | } catch (IOException e) { 96 | e.printStackTrace(); 97 | } 98 | return new Values(stbStatus); 99 | } 100 | 101 | public Fields getOutputFields() { 102 | return new Fields("DeviceStatus"); 103 | } 104 | } -------------------------------------------------------------------------------- /DeviceMonitor/src/main/java/com/hortonworks/util/TechnicianEventJSONScheme.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks.util; 2 | 3 | import java.io.IOException; 4 | import java.nio.ByteBuffer; 5 | import java.nio.charset.Charset; 6 | import java.util.List; 7 | 8 | import org.apache.storm.kafka.KeyValueScheme; 9 | import org.apache.storm.kafka.StringScheme; 10 | import org.apache.storm.tuple.Fields; 11 | import org.apache.storm.tuple.Values; 12 | import org.codehaus.jackson.JsonParseException; 13 | import org.codehaus.jackson.map.JsonMappingException; 14 | import org.codehaus.jackson.map.ObjectMapper; 15 | 16 | import com.hortonworks.events.TechnicianStatus; 17 | 18 | /* 19 | import backtype.storm.spout.Scheme; 20 | import backtype.storm.tuple.Fields; 21 | import backtype.storm.tuple.Values; 22 | import storm.kafka.KeyValueScheme; 23 | */ 24 | 25 | public class TechnicianEventJSONScheme implements KeyValueScheme { 26 | 27 | private static final long serialVersionUID = 1L; 28 | private static final Charset UTF8 = Charset.forName("UTF-8"); 29 | 30 | public List deserialize(byte[] bytes) { 31 | String eventJSONString = new String(bytes, UTF8); 32 | TechnicianStatus technicianStatus = null; 33 | ObjectMapper mapper = new ObjectMapper(); 34 | 35 | try { 36 | technicianStatus = mapper.readValue(eventJSONString, TechnicianStatus.class); 37 | } catch (JsonParseException e) { 38 | e.printStackTrace(); 39 | } catch (JsonMappingException e) { 40 | e.printStackTrace(); 41 | } catch (IOException e) { 42 | e.printStackTrace(); 43 | } 44 | return new Values(technicianStatus); 45 | } 46 | 47 | public List deserializeKeyAndValue(byte[] key, byte[] value) { 48 | String eventJSONString = new String(value, UTF8); 49 | TechnicianStatus technicianStatus = null; 50 | ObjectMapper mapper = new ObjectMapper(); 51 | 52 | try { 53 | technicianStatus = mapper.readValue(eventJSONString, TechnicianStatus.class); 54 | } catch (JsonParseException e) { 55 | e.printStackTrace(); 56 | } catch (JsonMappingException e) { 57 | e.printStackTrace(); 58 | } catch (IOException e) { 59 | e.printStackTrace(); 60 | } 61 | return new Values(technicianStatus); 62 | } 63 | 64 | public List deserializeKeyAndValue(ByteBuffer key, ByteBuffer value) { 65 | String eventKey = StringScheme.deserializeString(key); 66 | String eventJSONString = StringScheme.deserializeString(value); 67 | TechnicianStatus technicianStatus = null; 68 | ObjectMapper mapper = new ObjectMapper(); 69 | 70 | try { 71 | technicianStatus = mapper.readValue(eventJSONString, TechnicianStatus.class); 72 | } catch (JsonParseException e) { 73 | e.printStackTrace(); 74 | } catch (JsonMappingException e) { 75 | e.printStackTrace(); 76 | } catch (IOException e) { 77 | e.printStackTrace(); 78 | } 79 | 80 | return new Values(technicianStatus); 81 | } 82 | 83 | public List deserialize(ByteBuffer value) { 84 | String eventJSONString = StringScheme.deserializeString(value); 85 | TechnicianStatus technicianStatus = null; 86 | ObjectMapper mapper = new ObjectMapper(); 87 | 88 | try { 89 | technicianStatus = mapper.readValue(eventJSONString, TechnicianStatus.class); 90 | } catch (JsonParseException e) { 91 | e.printStackTrace(); 92 | } catch (JsonMappingException e) { 93 | e.printStackTrace(); 94 | } catch (IOException e) { 95 | e.printStackTrace(); 96 | } 97 | 98 | return new Values(technicianStatus); 99 | } 100 | public Fields getOutputFields() { 101 | return new Fields("TechnicianStatus"); 102 | } 103 | } -------------------------------------------------------------------------------- /MapUI/src/main/webapp/css/mapStyle.css: -------------------------------------------------------------------------------- 1 | .CSSTableGenerator { 2 | margin:0px; 3 | padding:0px; 4 | top: 200px; 5 | width:100%; 6 | margin: auto; 7 | box-shadow: 10px 10px 5px #888888; 8 | border:1px solid #000000; 9 | position: relative; 10 | 11 | -moz-border-radius-bottomleft:0px; 12 | -webkit-border-bottom-left-radius:0px; 13 | border-bottom-left-radius:0px; 14 | 15 | -moz-border-radius-bottomright:0px; 16 | -webkit-border-bottom-right-radius:0px; 17 | border-bottom-right-radius:0px; 18 | 19 | -moz-border-radius-topright:0px; 20 | -webkit-border-top-right-radius:0px; 21 | border-top-right-radius:0px; 22 | 23 | -moz-border-radius-topleft:0px; 24 | -webkit-border-top-left-radius:0px; 25 | border-top-left-radius:0px; 26 | } 27 | 28 | .CSSTableGenerator table{ 29 | border-collapse: collapse; 30 | border-spacing: 0; 31 | width:100%; 32 | height:100%; 33 | margin:0px;padding:0px; 34 | } 35 | 36 | .CSSTableGenerator tr:last-child td:last-child { 37 | -moz-border-radius-bottomright:0px; 38 | -webkit-border-bottom-right-radius:0px; 39 | border-bottom-right-radius:0px; 40 | } 41 | .CSSTableGenerator table tr:first-child td:first-child { 42 | -moz-border-radius-topleft:0px; 43 | -webkit-border-top-left-radius:0px; 44 | border-top-left-radius:0px; 45 | } 46 | .CSSTableGenerator table tr:first-child td:last-child { 47 | -moz-border-radius-topright:0px; 48 | -webkit-border-top-right-radius:0px; 49 | border-top-right-radius:0px; 50 | }.CSSTableGenerator tr:last-child td:first-child{ 51 | -moz-border-radius-bottomleft:0px; 52 | -webkit-border-bottom-left-radius:0px; 53 | border-bottom-left-radius:0px; 54 | }.CSSTableGenerator tr:hover td{ 55 | background-color:#ffffff; 56 | 57 | 58 | } 59 | .CSSTableGenerator td{ 60 | vertical-align:middle; 61 | 62 | background-color:#ffffff; 63 | 64 | border:1px solid #000000; 65 | border-width:0px 1px 1px 0px; 66 | text-align:left; 67 | padding:7px; 68 | font-size:10px; 69 | font-family:Arial; 70 | font-weight:normal; 71 | color:#000000; 72 | }.CSSTableGenerator tr:last-child td{ 73 | border-width:0px 1px 0px 0px; 74 | }.CSSTableGenerator tr td:last-child{ 75 | border-width:0px 0px 1px 0px; 76 | }.CSSTableGenerator tr:last-child td:last-child{ 77 | border-width:0px 0px 0px 0px; 78 | } 79 | .CSSTableGenerator tr:first-child td{ 80 | background:-o-linear-gradient(bottom, #415866 5%, #415866 100%); background:-webkit-gradient( linear, left top, left bottom, color-stop(0.05, #415866), color-stop(1, #415866) ); 81 | background:-moz-linear-gradient( center top, #415866 5%, #415866 100% ); 82 | filter:progid:DXImageTransform.Microsoft.gradient(startColorstr="#415866", endColorstr="#415866"); background: -o-linear-gradient(top,#415866,bf5f00); 83 | 84 | background-color:#415866; 85 | border:0px solid #000000; 86 | text-align:center; 87 | border-width:0px 0px 1px 1px; 88 | font-size:14px; 89 | font-family:Arial; 90 | font-weight:bold; 91 | color:#ffffff; 92 | } 93 | .CSSTableGenerator tr:first-child:hover td{ 94 | background:-o-linear-gradient(bottom, #415866 5%, #415866 100%); background:-webkit-gradient( linear, left top, left bottom, color-stop(0.05, #415866), color-stop(1, #415866) ); 95 | background:-moz-linear-gradient( center top, #415866 5%, #415866 100% ); 96 | filter:progid:DXImageTransform.Microsoft.gradient(startColorstr="#415866", endColorstr="#415866"); background: -o-linear-gradient(top,#415866,bf5f00); 97 | 98 | background-color:#415866; 99 | } 100 | .CSSTableGenerator tr:first-child td:first-child{ 101 | border-width:0px 0px 1px 0px; 102 | } 103 | .CSSTableGenerator tr:first-child td:last-child{ 104 | border-width:0px 0px 1px 1px; 105 | } -------------------------------------------------------------------------------- /DEVICE_MANAGER_DEMO_CONTROL_SAM/package/scripts/master.py: -------------------------------------------------------------------------------- 1 | import sys, os, pwd, signal, time, shutil 2 | from subprocess import * 3 | from resource_management import * 4 | 5 | class DemoControl(Script): 6 | def install(self, env): 7 | self.configure(env) 8 | import params 9 | 10 | if not os.path.exists(params.install_dir): 11 | os.makedirs(params.install_dir) 12 | os.chdir(params.install_dir) 13 | Execute('git clone ' + params.device_manager_download_url) 14 | Execute('git clone ' + params.sam_extentions_download_url) 15 | Execute(params.install_dir + '/DeviceManagerDemo/device-manager-sam-install.sh ' + params.install_dir + ' '+ params.google_api_key) 16 | Execute('git clone ' + params.simulator_download_url) 17 | os.chdir(params.install_dir+'/DataSimulators/DeviceSimulator') 18 | Execute('mvn clean package') 19 | os.chdir(params.install_dir+'/DataSimulators/DeviceSimulator/target') 20 | shutil.copy('DeviceSimulator-0.0.1-SNAPSHOT-jar-with-dependencies.jar', params.install_dir) 21 | 22 | 23 | def start(self, env): 24 | self.configure(env) 25 | import params 26 | Execute('echo Start Simulation') 27 | Execute('nohup java -jar '+params.install_dir+'/DeviceSimulator-0.0.1-SNAPSHOT-jar-with-dependencies.jar STB 1000 Simulation '+params.nifi_host_ip+' > '+params.install_dir+'/STB_1000_Sim.log 2>&1 & echo $! > /var/run/STB_1000_Sim.pid') 28 | Execute('nohup java -jar '+params.install_dir+'/DeviceSimulator-0.0.1-SNAPSHOT-jar-with-dependencies.jar STB 2000 Simulation '+params.nifi_host_ip+' > '+params.install_dir+'/STB_2000_Sim.log 2>&1 & echo $! > /var/run/STB_2000_Sim.pid') 29 | Execute('nohup java -jar '+params.install_dir+'/DeviceSimulator-0.0.1-SNAPSHOT-jar-with-dependencies.jar STB 3000 Simulation '+params.nifi_host_ip+' > '+params.install_dir+'/STB_3000_Sim.log 2>&1 & echo $! > /var/run/STB_3000_Sim.pid') 30 | 31 | Execute('nohup java -jar '+params.install_dir+'/DeviceSimulator-0.0.1-SNAPSHOT-jar-with-dependencies.jar Technician 1000 Simulation '+params.nifi_host_ip+' > '+params.install_dir+'/Technician_1000_Sim.log 2>&1 & echo $! > /var/run/Technician_1000_Sim.pid') 32 | Execute('nohup java -jar '+params.install_dir+'/DeviceSimulator-0.0.1-SNAPSHOT-jar-with-dependencies.jar Technician 2000 Simulation '+params.nifi_host_ip+' > '+params.install_dir+'/Technician_2000_Sim.log 2>&1 & echo $! > /var/run/Technician_2000_Sim.pid') 33 | Execute('nohup java -jar '+params.install_dir+'/DeviceSimulator-0.0.1-SNAPSHOT-jar-with-dependencies.jar Technician 3000 Simulation '+params.nifi_host_ip+' > '+params.install_dir+'/Technician_3000_Sim.log 2>&1 & echo $! > /var/run/Technician_3000_Sim.pid') 34 | 35 | def stop(self, env): 36 | self.configure(env) 37 | import params 38 | Execute('echo Stop Simulation') 39 | Execute (format('kill -9 `cat /var/run/STB_1000_Sim.pid` >/dev/null 2>&1')) 40 | Execute (format('kill -9 `cat /var/run/STB_2000_Sim.pid` >/dev/null 2>&1')) 41 | Execute (format('kill -9 `cat /var/run/STB_3000_Sim.pid` >/dev/null 2>&1')) 42 | Execute (format('kill -9 `cat /var/run/Technician_1000_Sim.pid` >/dev/null 2>&1')) 43 | Execute (format('kill -9 `cat /var/run/Technician_2000_Sim.pid` >/dev/null 2>&1')) 44 | Execute (format('kill -9 `cat /var/run/Technician_3000_Sim.pid` >/dev/null 2>&1')) 45 | 46 | Execute ('rm -f /var/run/STB_1000_Sim.pid') 47 | Execute ('rm -f /var/run/STB_2000_Sim.pid') 48 | Execute ('rm -f /var/run/STB_3000_Sim.pid') 49 | Execute ('rm -f /var/run/Technician_1000_Sim.pid') 50 | Execute ('rm -f /var/run/Technician_2000_Sim.pid') 51 | Execute ('rm -f /var/run/Technician_3000_Sim.pid') 52 | 53 | def status(self, env): 54 | import params 55 | env.set_params(params) 56 | check_process_status('/var/run/STB_1000_Sim.pid') 57 | 58 | def configure(self, env): 59 | import params 60 | env.set_params(params) 61 | 62 | if __name__ == "__main__": 63 | DemoControl().execute() 64 | -------------------------------------------------------------------------------- /DeviceMonitorNostradamusScala/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | com.hortonworks.spark.streaming 4 | DeviceMonitorNostradamusScala 5 | 0.0.1-SNAPSHOT 6 | 7 | 8 | org.apache.spark 9 | spark-streaming_2.10 10 | 1.6.1 11 | 12 | 13 | org.apache.spark 14 | spark-streaming-kafka_2.10 15 | 1.6.1 16 | 17 | 18 | org.apache.spark 19 | spark-mllib_2.10 20 | 1.6.1 21 | 22 | 23 | org.cometd.java 24 | cometd-java-client 25 | 3.0.7 26 | 27 | 28 | 29 | 30 | 66 | 67 | maven-assembly-plugin 68 | 2.4.1 69 | 70 | 71 | jar-with-dependencies 72 | 73 | 74 | 75 | 76 | make-assembly 77 | package 78 | 79 | single 80 | 81 | 82 | 83 | 84 | 85 | org.scala-tools 86 | maven-scala-plugin 87 | 2.15.2 88 | 89 | 90 | 91 | compile 92 | testCompile 93 | 94 | 95 | 96 | 97 | src/main/scala 98 | src/test/scala 99 | 100 | 101 | 102 | classes/ 103 | 104 | -------------------------------------------------------------------------------- /Cometd/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | com.hortonworks.iot.cometd 4 | Cometd 5 | 0.0.1-SNAPSHOT 6 | 7 | 8 | ${docker.registry.name}vvaks/cometd 9 | 10 | 11 | 12 | org.springframework 13 | spring-core 14 | 2.5.6 15 | 16 | 17 | org.eclipse.jetty 18 | jetty-server 19 | 9.3.6.v20151106 20 | 21 | 22 | org.eclipse.jetty 23 | jetty-servlet 24 | 9.3.6.v20151106 25 | 26 | 27 | org.eclipse.jetty 28 | jetty-webapp 29 | 9.3.6.v20151106 30 | 31 | 32 | org.eclipse.jetty 33 | jetty-util 34 | 9.3.6.v20151106 35 | 36 | 37 | org.cometd.java 38 | cometd-api 39 | 1.1.5 40 | 41 | 42 | org.cometd.java 43 | cometd-java-server 44 | 3.0.7 45 | 46 | 47 | org.slf4j 48 | slf4j-simple 49 | 1.7.13 50 | 51 | 52 | 53 | ${project.artifactId} 54 | src/ 55 | 56 | 57 | src/main/webapp 58 | META-INF/resources 59 | 60 | 61 | src/main/resources 62 | META-INF/resources 63 | 64 | 65 | classes/ 66 | 67 | 68 | maven-compiler-plugin 69 | 3.5.1 70 | 71 | 1.7 72 | 1.7 73 | 74 | 75 | 76 | maven-assembly-plugin 77 | 78 | 79 | jar-with-dependencies 80 | 81 | 82 | 83 | com.hortonworks.iot.cometd.Main 84 | 85 | 86 | 87 | 88 | 89 | package 90 | 91 | single 92 | 93 | 94 | 95 | 96 | 97 | org.jolokia 98 | docker-maven-plugin 99 | 0.13.3 100 | 101 | 102 | 103 | 105 | cometd 106 | ${docker.repository.name} 107 | 108 | 109 | centos:7 110 | vvaks 111 | 112 | docker-assembly.xml 113 | 114 | 115 | 8091 116 | 117 | 118 | 8u31 119 | b13 120 | 121 | 122 | yum install -y wget 123 | wget --no-cookies --no-check-certificate --header "Cookie: oraclelicense=accept-securebackup-cookie" "http://download.oracle.com/otn-pub/java/jdk/$JAVA_VERSION-$BUILD_VERSION/jdk-$JAVA_VERSION-linux-x64.rpm" -O /tmp/jdk-8-linux-x64.rpm 124 | yum install -y /tmp/jdk-8-linux-x64.rpm 125 | yum install -y iproute 126 | yum install -y net-tools 127 | chmod 757 -R /maven 128 | 129 | 130 | java -jar /maven/Cometd-jar-with-dependencies.jar server /maven/docker-config.yml 131 | 132 | 133 | 134 | 135 | 136 | 137 | 138 | 139 | -------------------------------------------------------------------------------- /DeviceMonitor/src/main/java/com/hortonworks/bolts/PersistTechnicianLocation.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks.bolts; 2 | 3 | import java.io.IOException; 4 | import java.util.HashMap; 5 | import java.util.Map; 6 | 7 | import org.apache.hadoop.conf.Configuration; 8 | import org.apache.hadoop.hbase.HBaseConfiguration; 9 | import org.apache.hadoop.hbase.HColumnDescriptor; 10 | import org.apache.hadoop.hbase.HTableDescriptor; 11 | import org.apache.hadoop.hbase.client.Get; 12 | import org.apache.hadoop.hbase.client.HBaseAdmin; 13 | import org.apache.hadoop.hbase.client.HTable; 14 | import org.apache.hadoop.hbase.client.Put; 15 | import org.apache.hadoop.hbase.client.Result; 16 | import org.apache.hadoop.hbase.util.Bytes; 17 | 18 | import com.hortonworks.events.TechnicianStatus; 19 | import com.hortonworks.util.Constants; 20 | 21 | import org.apache.storm.task.OutputCollector; 22 | import org.apache.storm.task.TopologyContext; 23 | import org.apache.storm.topology.OutputFieldsDeclarer; 24 | import org.apache.storm.topology.base.BaseRichBolt; 25 | import org.apache.storm.tuple.Fields; 26 | import org.apache.storm.tuple.Tuple; 27 | import org.apache.storm.tuple.Values; 28 | 29 | /* 30 | import backtype.storm.task.OutputCollector; 31 | import backtype.storm.task.TopologyContext; 32 | import backtype.storm.topology.OutputFieldsDeclarer; 33 | import backtype.storm.topology.base.BaseRichBolt; 34 | import backtype.storm.tuple.Fields; 35 | import backtype.storm.tuple.Tuple; 36 | import backtype.storm.tuple.Values; 37 | */ 38 | 39 | public class PersistTechnicianLocation extends BaseRichBolt { 40 | 41 | private static final long serialVersionUID = 1L; 42 | private OutputCollector collector; 43 | private Constants constants; 44 | private HTable table = null; 45 | 46 | @SuppressWarnings("deprecation") 47 | public void execute(Tuple tuple) { 48 | TechnicianStatus technicianStatus = (TechnicianStatus) tuple.getValueByField("TechnicianStatus"); 49 | 50 | Put p = new Put(Bytes.toBytes(technicianStatus.getTechnicianId())); 51 | p.add(Bytes.toBytes("cf"), Bytes.toBytes("Status"), Bytes.toBytes(technicianStatus.getStatus())); 52 | p.add(Bytes.toBytes("cf"), Bytes.toBytes("Latitude"), Bytes.toBytes(technicianStatus.getLatitude().toString())); 53 | p.add(Bytes.toBytes("cf"), Bytes.toBytes("Longitude"), Bytes.toBytes(technicianStatus.getLongitude().toString())); 54 | p.add(Bytes.toBytes("cf"), Bytes.toBytes("IpAddress"), Bytes.toBytes(technicianStatus.getIpAddress().toString())); 55 | p.add(Bytes.toBytes("cf"), Bytes.toBytes("Port"), Bytes.toBytes(technicianStatus.getPort().toString())); 56 | try { 57 | table.put(p); 58 | } catch (IOException e) { 59 | e.printStackTrace(); 60 | } 61 | System.out.println("TechnicianId: " + technicianStatus.getTechnicianId()); 62 | System.out.println("Technician Status: " + technicianStatus.getStatus()); 63 | System.out.println("Latitude: " + technicianStatus.getLatitude()); 64 | System.out.println("Longitude: " + technicianStatus.getLongitude()); 65 | System.out.println("IpAddress: " + technicianStatus.getIpAddress()); 66 | System.out.println("Port: " + technicianStatus.getPort()); 67 | 68 | collector.emit(tuple, new Values((TechnicianStatus)technicianStatus)); 69 | collector.ack(tuple); 70 | } 71 | 72 | public void prepare(Map arg0, TopologyContext arg1, OutputCollector collector) { 73 | this.collector = collector; 74 | this.constants = new Constants(); 75 | 76 | System.out.println("********************** Zookeeper Host: " + constants.getZkHost()); 77 | System.out.println("********************** Zookeeper Port: " + constants.getZkPort()); 78 | System.out.println("********************** Zookeeper ConnString: " + constants.getZkConnString()); 79 | System.out.println("********************** Zookeeper Kafka Path: " + constants.getZkKafkaPath()); 80 | System.out.println("********************** Zookeeper HBase Path: " + constants.getZkHBasePath()); 81 | System.out.println("********************** Cometd URI: " + constants.getPubSubUrl()); 82 | 83 | Configuration config = HBaseConfiguration.create(); 84 | config.set("hbase.zookeeper.quorum", constants.getZkHost()); 85 | config.set("hbase.zookeeper.property.clientPort", constants.getZkPort()); 86 | config.set("zookeeper.znode.parent", constants.getZkHBasePath()); 87 | 88 | String tableName = "TechnicianEvents"; 89 | try { 90 | HBaseAdmin hbaseAdmin = new HBaseAdmin(config); 91 | if (hbaseAdmin.tableExists(tableName)) { 92 | table = new HTable(config, tableName); 93 | }else{ 94 | System.out.println("********************** Table " + tableName + "does not exist, creating..."); 95 | HTableDescriptor tableDescriptor = new HTableDescriptor(tableName); 96 | HColumnDescriptor cfColumnFamily = new HColumnDescriptor("cf".getBytes()); 97 | tableDescriptor.addFamily(cfColumnFamily); 98 | hbaseAdmin.createTable(tableDescriptor); 99 | System.out.println("********************** Created " + tableName); 100 | table = new HTable(config, tableName); 101 | System.out.println("********************** Acquired " + tableName); 102 | } 103 | } catch (IOException e) { 104 | e.printStackTrace(); 105 | } 106 | } 107 | 108 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 109 | declarer.declare(new Fields("TechnicianStatus")); 110 | } 111 | } -------------------------------------------------------------------------------- /DeviceMonitorNostradamus/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | com.hortonworks.iot 4 | DeviceMonitorNostradamus 5 | 0.0.1-SNAPSHOT 6 | 7 | 8 | org.apache.spark 9 | spark-core_2.10 10 | 1.6.1 11 | 12 | 13 | org.apache.spark 14 | spark-streaming_2.10 15 | 1.6.1 16 | 17 | 18 | org.apache.spark 19 | spark-mllib_2.10 20 | 1.6.1 21 | 22 | 23 | org.apache.spark 24 | spark-streaming-kafka_2.10 25 | 1.6.1 26 | 27 | 28 | org.apache.zookeeper 29 | zookeeper 30 | 3.4.6 31 | 32 | 33 | org.slf4j 34 | slf4j-log4j12 35 | 36 | 37 | log4j 38 | log4j 39 | 40 | 41 | 42 | 43 | org.apache.kafka 44 | kafka_2.10 45 | 0.8.2.1 46 | 47 | 48 | org.apache.zookeeper 49 | zookeeper 50 | 51 | 52 | log4j 53 | log4j 54 | 55 | 56 | 57 | 58 | org.apache.hbase 59 | hbase-client 60 | 1.1.2 61 | 62 | 63 | org.slf4j 64 | slf4j-log4j12 65 | 66 | 67 | log4j 68 | log4j 69 | 70 | 71 | 72 | 73 | org.codehaus.jackson 74 | jackson-mapper-asl 75 | 1.9.13 76 | 77 | 78 | com.google.appengine 79 | appengine-api-1.0-sdk 80 | 1.9.28 81 | 82 | 83 | com.googlecode.json-simple 84 | json-simple 85 | 1.1.1 86 | 87 | 88 | org.cometd.java 89 | cometd-java-client 90 | 3.0.7 91 | 92 | 93 | 94 | 95 | 96 | maven-assembly-plugin 97 | 98 | 99 | jar-with-dependencies 100 | 101 | 102 | 103 | com.hortonworks.iot.spark.streaming.SparkNostradamus 104 | 105 | 106 | 107 | 108 | 109 | package 110 | 111 | single 112 | 113 | 114 | 115 | 116 | 142 | 143 | src/ 144 | 145 | 146 | ${basedir} 147 | 148 | * 149 | 150 | 151 | 152 | src/resources 153 | 154 | * 155 | 156 | 157 | 158 | classes/ 159 | 160 | -------------------------------------------------------------------------------- /Nifi/config/nifi-logback-env.json: -------------------------------------------------------------------------------- 1 | "properties" : { 2 | "content" : "\n\n \n true\n \n \n \n {{nifi_log_dir}}/nifi-app.log\n \n \n {{nifi_log_dir}}/nifi-app_%d{yyyy-MM-dd_HH}.%i.log\n \n 100MB\n \n \n 30\n \n \n %date %level [%thread] %logger{40} %msg%n\n true\n \n \n \n \n {{nifi_log_dir}}/nifi-user.log\n \n \n {{nifi_log_dir}}/nifi-user_%d.log\n \n 30\n \n \n %date %level [%thread] %logger{40} %msg%n\n \n \n\n \n {{nifi_log_dir}}/nifi-bootstrap.log\n \n \n {{nifi_log_dir}}/nifi-bootstrap_%d.log\n \n 5\n \n \n %date %level [%thread] %logger{40} %msg%n\n \n \n\t\n\t\n\t\t\n %date %level [%thread] %logger{40} %msg%n\n \n\t\n \n \n \n \n \n \n \n \n\n \n \n\n \n \n\n \n \n \n \n \n \n \n\n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n \n\n\n \n \n \n \n \n \n \n \n\n \n \n \n \n \n \n\t\n \t\n \n\n\n \n \n \n \n" 3 | } 4 | -------------------------------------------------------------------------------- /redeployApplication.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | echo "*********************************Setting Environment..." 4 | . ~/.bash_profile 5 | 6 | export NIFI_HOST=$1 7 | export NIFI_PORT=$2 8 | export ATLAS_HOST=$3 9 | export ATLAS_PORT=$4 10 | export HIVESERVER_HOST=$5 11 | export HIVESERVER_PORT=$6 12 | export CLUSTER_NAME=$7 13 | 14 | env 15 | 16 | retargetNifiFlowReporter() { 17 | sleep 1 18 | echo "*********************************Getting Nifi Reporting Task Id..." 19 | REPORTING_TASK_ID=$(curl -H "Content-Type: application/json" -X GET http://$NIFI_HOST:$NIFI_PORT/nifi-api/flow/reporting-tasks| grep -Po '("component":{"id":"[0-9a-zA-z\-]+","name":"AtlasFlowReportingTask)'| grep -Po 'id":"([0-9a-zA-z\-]+)'| grep -Po ':"([0-9a-zA-z\-]+)'| grep -Po '([0-9a-zA-z\-]+)') 20 | 21 | echo "*********************************Getting Nifi Reporting Task Revision..." 22 | REPORTING_TASK_REVISION=$(curl -X GET http://$NIFI_HOST:$NIFI_PORT/nifi-api/reporting-tasks/$REPORTING_TASK_ID |grep -Po '\"version\":([0-9]+)'|grep -Po '([0-9]+)') 23 | 24 | echo "*********************************Stopping Nifi Reporting Task..." 25 | PAYLOAD=$(echo "{\"id\":\"$REPORTING_TASK_ID\",\"revision\":{\"version\":$REPORTING_TASK_REVISION},\"component\":{\"id\":\"$REPORTING_TASK_ID\",\"state\":\"STOPPED\"}}") 26 | 27 | curl -d "$PAYLOAD" -H "Content-Type: application/json" -X PUT http://$NIFI_HOST:$NIFI_PORT/nifi-api/reporting-tasks/$REPORTING_TASK_ID 28 | 29 | echo "*********************************Getting Nifi Reporting Task Revision..." 30 | REPORTING_TASK_REVISION=$(curl -X GET http://$NIFI_HOST:$NIFI_PORT/nifi-api/reporting-tasks/$REPORTING_TASK_ID |grep -Po '\"version\":([0-9]+)'|grep -Po '([0-9]+)') 31 | 32 | echo "*********************************Removing Nifi Reporting Task..." 33 | curl -X DELETE http://$NIFI_HOST:$NIFI_PORT/nifi-api/reporting-tasks/$REPORTING_TASK_ID?version=$REPORTING_TASK_REVISION 34 | 35 | echo "*********************************Instantiating Reporting Task..." 36 | PAYLOAD=$(echo "{\"revision\":{\"version\":0},\"component\":{\"name\":\"AtlasFlowReportingTask\",\"type\":\"org.apache.nifi.atlas.reporting.AtlasFlowReportingTask\",\"properties\":{\"Atlas URL\":\"http://$ATLAS_HOST:$ATLAS_PORT\",\"Nifi URL\":\"http://$NIFI_HOST:$NIFI_PORT\"}}}") 37 | 38 | REPORTING_TASK_ID=$(curl -d "$PAYLOAD" -H "Content-Type: application/json" -X POST http://$NIFI_HOST:$NIFI_PORT/nifi-api/controller/reporting-tasks|grep -Po '("component":{"id":")([0-9a-zA-z\-]+)'| grep -Po '(:"[0-9a-zA-z\-]+)'| grep -Po '([0-9a-zA-z\-]+)') 39 | 40 | echo "*********************************Starting Reporting Task..." 41 | PAYLOAD=$(echo "{\"id\":\"$REPORTING_TASK_ID\",\"revision\":{\"version\":1},\"component\":{\"id\":\"$REPORTING_TASK_ID\",\"state\":\"RUNNING\"}}") 42 | 43 | curl -d "$PAYLOAD" -H "Content-Type: application/json" -X PUT http://$NIFI_HOST:$NIFI_PORT/nifi-api/reporting-tasks/$REPORTING_TASK_ID 44 | sleep 1 45 | } 46 | 47 | recreateDeviceManagerTables () { 48 | HQL="CREATE TABLE IF NOT EXISTS telecom_device_status_log_$CLUSTER_NAME ( 49 | serialNumber string, 50 | status string, 51 | state string, 52 | internalTemp int, 53 | signalStrength int, 54 | eventTimeStamp bigint) 55 | CLUSTERED BY (serialNumber) INTO 30 BUCKETS STORED AS ORC;" 56 | 57 | # CREATE DEVICE STATUS LOG TABLE 58 | beeline -u jdbc:hive2://$HIVESERVER_HOST:$HIVESERVER_PORT/default -d org.apache.hive.jdbc.HiveDriver -e "$HQL" -n hive 59 | 60 | HQL="CREATE TABLE IF NOT EXISTS telecom_device_details_$CLUSTER_NAME ( 61 | serialNumber string, 62 | deviceModel string, 63 | latitude string, 64 | longitude string, 65 | ipAddress string, 66 | port string) 67 | CLUSTERED BY (serialNumber) INTO 30 BUCKETS STORED AS ORC;" 68 | 69 | # CREATE DEVICE DETAILS TABLE 70 | beeline -u jdbc:hive2://$HIVESERVER_HOST:$HIVESERVER_PORT/default -d org.apache.hive.jdbc.HiveDriver -e "$HQL" -n hive 71 | } 72 | 73 | #cd $ROOT_PATH/DataPlaneUtils 74 | #mvn clean package 75 | #java -jar target/DataPlaneUtils-0.0.1-SNAPSHOT-jar-with-dependencies.jar 76 | 77 | echo "*********************************Configure HDFS application space..." 78 | export HADOOP_USER_NAME=hdfs 79 | hadoop fs -mkdir /user/root 80 | hadoop fs -chown root:hdfs /user/root 81 | hadoop fs -mkdir /spark-history 82 | hadoop fs -chown spark:hdfs /spark-history 83 | hadoop fs -chmod 777 /spark-history 84 | 85 | echo "*********************************Importing Spark Model..." 86 | #cd /root/DeviceManagerDemo/Model 87 | #unzip nostradamusSVMModel.zip 88 | #cp -rvf nostradamusSVMModel /tmp 89 | #cp -vf DeviceLogTrainingData.csv /tmp 90 | hadoop fs -mkdir /demo/ 91 | hadoop fs -mkdir /demo/data 92 | hadoop fs -mkdir /demo/data/model/ 93 | hadoop fs -mkdir /demo/data/checkpoint 94 | hadoop fs -mkdir /demo/data/training/ 95 | hadoop fs -chmod -R 777 /demo/data/ 96 | hadoop fs -put /tmp/nostradamusSVMModel /demo/data/model/ 97 | hadoop fs -put /tmp/DeviceLogTrainingData.csv /demo/data/training/ 98 | 99 | # Recreate Device Manager tables to reset Atlas qualified name to this cluster 100 | echo "*********************************Recreating Device Manager Tables..." 101 | recreateDeviceManagerTables 102 | 103 | cd /root/Utils/SparkPhoenixETL 104 | rm -Rvf classes* 105 | mvn clean package 106 | mv target/SparkPhoenixETL-0.0.1-SNAPSHOT-jar-with-dependencies.jar /home/spark 107 | 108 | spark-submit --class com.hortonworks.util.SparkPhoenixETL --master yarn-client --executor-cores 2 --driver-memory 2G --executor-memory 2G --num-executors 1 /home/spark/SparkPhoenixETL-0.0.1-SNAPSHOT-jar-with-dependencies.jar $ZK_HOST:2181:/hbase-unsecure $CLUSTER_NAME DeviceManager 109 | 110 | echo "*********************************Redeploying Spark Streaming Application..." 111 | nohup spark-submit --class com.hortonworks.iot.spark.streaming.SparkNostradamus --master yarn-cluster --executor-cores 2 --driver-memory 2G --executor-memory 2G --num-executors 1 /home/spark/DeviceMonitorNostradamusScala-0.0.1-SNAPSHOT-jar-with-dependencies.jar $ZK_HOST:2181 DeviceEvents $COMETD_HOST:8091 > /dev/null 2>&1& 112 | 113 | # Redeploy Storm Topology to send topology meta data to Atlas 114 | echo "*********************************Redeploying Storm Topology..." 115 | storm kill DeviceMonitorTopology-$CLUSTER_NAME 116 | 117 | curl -u admin:admin -X DELETE 'http://'"$ATLAS_HOST:$ATLAS_PORT"'/api/atlas/entities?type=storm_topology&property=qualifiedName&value=DeviceMonitorTopology' 118 | 119 | storm jar /home/storm/DeviceMonitor-0.0.1-SNAPSHOT.jar com.hortonworks.iot.topology.DeviceMonitorTopology $CLUSTER_NAME 120 | 121 | # Start Nifi Flow Reporter to send flow meta data to Atlas 122 | echo "*********************************Retargeting Nifi Flow Reporting Task..." 123 | sleep 5 124 | retargetNifiFlowReporter 125 | 126 | exit 0 -------------------------------------------------------------------------------- /DeviceMonitor/src/main/java/com/hortonworks/util/Constants.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks.util; 2 | 3 | import java.util.Map; 4 | 5 | public class Constants { 6 | private String zkHost = "sandbox.hortonworks.com"; 7 | private String zkPort = "2181"; 8 | private String zkKafkaPath = "/brokers"; 9 | private String zkHBasePath = "/hbase-unsecure"; 10 | private String zkConnString; 11 | private String deviceEventsTopicName = "DeviceEvents"; 12 | private String technicianEventsTopicName = "TechnicianEvent"; 13 | 14 | private String cometdHost = "sandbox.hortonworks.com"; 15 | private String cometdPort = "8091"; 16 | private String pubSubUrl; 17 | private String alertChannel = "/alert"; 18 | private String deviceChannel = "/devicestatus"; 19 | private String technicianChannel = "/technicianstatus"; 20 | 21 | private String nameNodeHost = "sandbox.hortonworks.com"; 22 | private String nameNodePort = "8020"; 23 | private String nameNodeUrl; 24 | private String hivePath = "/demo/data/transaction_logs"; 25 | private String hbasePath = "/apps/hbase/data/"; 26 | 27 | private String atlasHost = "localhost"; 28 | private String atlasPort = "21000"; 29 | 30 | private String hiveMetaStoreURI = "jdbc:mysql://sandbox.hortonworks.com/hive"; 31 | private String hiveDbName = "default"; 32 | 33 | public Constants(){ 34 | Map env = System.getenv(); 35 | //System.out.println("********************** ENV: " + env); 36 | if(env.get("ZK_HOST") != null){ 37 | this.zkHost = (String)env.get("ZK_HOST"); 38 | } 39 | if(env.get("ZK_PORT") != null){ 40 | this.zkPort = (String)env.get("ZK_PORT"); 41 | } 42 | if(env.get("ZK_KAFKA_PATH") != null){ 43 | this.zkKafkaPath = (String)env.get("ZK_KAFKA_PATH"); 44 | } 45 | if(env.get("ZK_HBASE_PATH") != null){ 46 | this.zkHBasePath = (String)env.get("ZK_HBASE_PATH"); 47 | } 48 | if(env.get("NAMENODE_HOST") != null){ 49 | this.setNameNodeHost((String)env.get("NAMENODE_HOST")); 50 | } 51 | if(env.get("NAMENODE_PORT") != null){ 52 | this.setNameNodePort((String)env.get("NAMENODE_PORT")); 53 | } 54 | if(env.get("HIVE_PATH") != null){ 55 | this.hivePath = (String)env.get("HIVE_PATH"); 56 | } 57 | if(env.get("HIVE_METASTORE_URI") != null){ 58 | this.hiveMetaStoreURI = (String)env.get("HIVE_METASTORE_URI"); 59 | } 60 | if(env.get("ATLAS_HOST") != null){ 61 | this.setAtlasHost((String)env.get("ATLAS_HOST")); 62 | } 63 | if(env.get("ATLAS_PORT") != null){ 64 | this.setAtlasPort((String)env.get("ATLAS_PORT")); 65 | } 66 | if(env.get("COMETD_HOST") != null){ 67 | this.cometdHost = (String)env.get("COMETD_HOST"); 68 | } 69 | if(env.get("COMETD_PORT") != null){ 70 | this.cometdPort = (String)env.get("COMETD_PORT"); 71 | } 72 | 73 | this.zkConnString = zkHost+":"+zkPort; 74 | this.pubSubUrl = "http://" + cometdHost + ":" + cometdPort + "/cometd"; 75 | this.nameNodeUrl = "hdfs://" + nameNodeHost + ":" + nameNodePort; 76 | } 77 | 78 | public String getZkHost() { 79 | return zkHost; 80 | } 81 | 82 | public void setZkHost(String zkHost) { 83 | this.zkHost = zkHost; 84 | } 85 | 86 | public String getZkPort() { 87 | return zkPort; 88 | } 89 | 90 | public void setZkPort(String zkPort) { 91 | this.zkPort = zkPort; 92 | } 93 | 94 | public String getZkKafkaPath() { 95 | return zkKafkaPath; 96 | } 97 | 98 | public void setZkKafkaPath(String zkKafkaPath) { 99 | this.zkKafkaPath = zkKafkaPath; 100 | } 101 | 102 | public String getZkHBasePath() { 103 | return zkHBasePath; 104 | } 105 | 106 | public void setZkHBasePath(String zkHBasePath) { 107 | this.zkHBasePath = zkHBasePath; 108 | } 109 | 110 | public String getZkConnString() { 111 | return zkConnString; 112 | } 113 | 114 | public void setZkConnString(String zkConnString) { 115 | this.zkConnString = zkConnString; 116 | } 117 | 118 | public String getPubSubUrl() { 119 | return pubSubUrl; 120 | } 121 | 122 | public void setPubSubUrl(String pubSubUrl) { 123 | this.pubSubUrl = pubSubUrl; 124 | } 125 | 126 | public String getHivePath() { 127 | return hivePath; 128 | } 129 | 130 | public void setHivePath(String hivePath) { 131 | this.hivePath = hivePath; 132 | } 133 | 134 | public String getAtlasHost() { 135 | return atlasHost; 136 | } 137 | 138 | public void setAtlasHost(String atlasHost) { 139 | this.atlasHost = atlasHost; 140 | } 141 | 142 | public String getAtlasPort() { 143 | return atlasPort; 144 | } 145 | 146 | public void setAtlasPort(String atlasPort) { 147 | this.atlasPort = atlasPort; 148 | } 149 | 150 | public String getHiveMetaStoreURI() { 151 | return hiveMetaStoreURI; 152 | } 153 | 154 | public void setHiveMetaStoreURI(String hiveMetaStoreURI) { 155 | this.hiveMetaStoreURI = hiveMetaStoreURI; 156 | } 157 | 158 | public String getHiveDbName() { 159 | return hiveDbName; 160 | } 161 | 162 | public void setHiveDbName(String hiveDbName) { 163 | this.hiveDbName = hiveDbName; 164 | } 165 | 166 | public String getNameNodeUrl() { 167 | return nameNodeUrl; 168 | } 169 | 170 | public void setNameNodeUrl(String nameNodeUrl) { 171 | this.nameNodeUrl = nameNodeUrl; 172 | } 173 | 174 | public String getNameNodePort() { 175 | return nameNodePort; 176 | } 177 | 178 | public void setNameNodePort(String nameNodePort) { 179 | this.nameNodePort = nameNodePort; 180 | } 181 | 182 | public String getNameNodeHost() { 183 | return nameNodeHost; 184 | } 185 | 186 | public void setNameNodeHost(String nameNodeHost) { 187 | this.nameNodeHost = nameNodeHost; 188 | } 189 | 190 | public String getHbasePath() { 191 | return hbasePath; 192 | } 193 | 194 | public void setHbasePath(String hbasePath) { 195 | this.hbasePath = hbasePath; 196 | } 197 | 198 | public String getAlertChannel() { 199 | return alertChannel; 200 | } 201 | 202 | public void setAlertChannel(String alertChannel) { 203 | this.alertChannel = alertChannel; 204 | } 205 | 206 | public String getDeviceChannel() { 207 | return deviceChannel; 208 | } 209 | 210 | public void setDeviceChannel(String deviceChannel) { 211 | this.deviceChannel = deviceChannel; 212 | } 213 | 214 | public String getTechnicianChannel() { 215 | return technicianChannel; 216 | } 217 | 218 | public void setTechnicianChannel(String technicianChannel) { 219 | this.technicianChannel = technicianChannel; 220 | } 221 | 222 | public String getDeviceEventsTopicName() { 223 | return deviceEventsTopicName; 224 | } 225 | 226 | public void setDeviceEventsTopicName(String deviceEventsTopicName) { 227 | this.deviceEventsTopicName = deviceEventsTopicName; 228 | } 229 | 230 | public String getTechnicianEventsTopicName() { 231 | return technicianEventsTopicName; 232 | } 233 | 234 | public void setTechnicianEventsTopicName(String technicianEventsTopicName) { 235 | this.technicianEventsTopicName = technicianEventsTopicName; 236 | } 237 | } 238 | -------------------------------------------------------------------------------- /DeviceMonitor/src/main/java/com/hortonworks/bolts/IncidentDetector.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks.bolts; 2 | 3 | import java.io.IOException; 4 | import java.sql.Connection; 5 | import java.sql.DriverManager; 6 | import java.sql.SQLException; 7 | import java.text.DateFormat; 8 | import java.text.ParseException; 9 | import java.text.SimpleDateFormat; 10 | import java.time.Instant; 11 | import java.util.Date; 12 | import java.util.Map; 13 | 14 | import com.hortonworks.events.DeviceAlert; 15 | import com.hortonworks.events.STBStatus; 16 | import com.hortonworks.util.Constants; 17 | 18 | import org.apache.hadoop.conf.Configuration; 19 | import org.apache.hadoop.hbase.HBaseConfiguration; 20 | import org.apache.hadoop.hbase.HColumnDescriptor; 21 | import org.apache.hadoop.hbase.HTableDescriptor; 22 | import org.apache.hadoop.hbase.client.HBaseAdmin; 23 | import org.apache.hadoop.hbase.client.HTable; 24 | 25 | /* 26 | import backtype.storm.task.OutputCollector; 27 | import backtype.storm.task.TopologyContext; 28 | import backtype.storm.topology.OutputFieldsDeclarer; 29 | import backtype.storm.topology.base.BaseRichBolt; 30 | import backtype.storm.tuple.Fields; 31 | import backtype.storm.tuple.Tuple; 32 | import backtype.storm.tuple.Values; 33 | */ 34 | 35 | import org.apache.storm.task.OutputCollector; 36 | import org.apache.storm.task.TopologyContext; 37 | import org.apache.storm.topology.OutputFieldsDeclarer; 38 | import org.apache.storm.topology.base.BaseRichBolt; 39 | import org.apache.storm.tuple.Fields; 40 | import org.apache.storm.tuple.Tuple; 41 | import org.apache.storm.tuple.Values; 42 | 43 | public class IncidentDetector extends BaseRichBolt { 44 | private static final long serialVersionUID = 1L; 45 | private OutputCollector collector; 46 | private Constants constants; 47 | private Connection conn; 48 | 49 | public void execute(Tuple tuple) { 50 | STBStatus deviceStatus = (STBStatus) tuple.getValueByField("DeviceStatus"); 51 | collector.emit("DeviceStatusLogStream", new Values(deviceStatus.getSerialNumber(), deviceStatus.getDeviceModel(), deviceStatus.getStatus(), deviceStatus.getState(), deviceStatus.getInternalTemp(), deviceStatus.getSignalStrength())); 52 | 53 | if(deviceStatus.getSignalStrength() <= 70){ 54 | DeviceAlert signalStrengthAlert = new DeviceAlert(); 55 | signalStrengthAlert.setAlertDescription("Signal Strength is too low"); 56 | signalStrengthAlert.setSerialNumber(deviceStatus.getSerialNumber()); 57 | signalStrengthAlert.setLongitude(deviceStatus.getLongitude()); 58 | signalStrengthAlert.setLatitude(deviceStatus.getLatitude()); 59 | collector.emit("DeviceAlertStream", new Values(signalStrengthAlert)); 60 | } 61 | else if(deviceStatus.getInternalTemp() >= 105){ 62 | DeviceAlert internalTempAlert = new DeviceAlert(); 63 | internalTempAlert.setAlertDescription("Device internal temp is too high"); 64 | internalTempAlert.setSerialNumber(deviceStatus.getSerialNumber()); 65 | internalTempAlert.setDeviceModel(deviceStatus.getDeviceModel()); 66 | internalTempAlert.setLongitude(deviceStatus.getLongitude()); 67 | internalTempAlert.setLatitude(deviceStatus.getLatitude()); 68 | collector.emit("DeviceAlertStream", new Values(internalTempAlert)); 69 | } 70 | else{ 71 | collector.emit("DeviceStatusStream", new Values(deviceStatus)); 72 | } 73 | 74 | try { 75 | conn.createStatement().executeUpdate("UPSERT INTO \"DeviceStatusLog\" VALUES(" 76 | + "'"+deviceStatus.getSerialNumber()+Instant.now().toEpochMilli()+"', " 77 | + "'"+deviceStatus.getSerialNumber()+"', " 78 | + "'"+deviceStatus.getStatus()+"', " 79 | + "'"+deviceStatus.getState()+"', " 80 | + ""+deviceStatus.getInternalTemp()+", " 81 | + ""+deviceStatus.getSignalStrength()+", " 82 | + ""+Instant.now().toEpochMilli()+")"); 83 | conn.commit(); 84 | } catch (SQLException e) { 85 | e.printStackTrace(); 86 | } 87 | 88 | collector.ack(tuple); 89 | } 90 | 91 | public void prepare(Map arg0, TopologyContext arg1, OutputCollector collector) { 92 | this.collector = collector; 93 | constants = new Constants(); 94 | System.out.println("********************** Zookeeper Host: " + constants.getZkHost()); 95 | System.out.println("********************** Zookeeper Port: " + constants.getZkPort()); 96 | System.out.println("********************** Zookeeper ConnString: " + constants.getZkConnString()); 97 | System.out.println("********************** Zookeeper Kafka Path: " + constants.getZkKafkaPath()); 98 | System.out.println("********************** Zookeeper HBase Path: " + constants.getZkHBasePath()); 99 | System.out.println("********************** Cometd URI: " + constants.getPubSubUrl()); 100 | 101 | Configuration config = HBaseConfiguration.create(); 102 | config.set("hbase.zookeeper.quorum", constants.getZkHost()); 103 | config.set("hbase.zookeeper.property.clientPort", constants.getZkPort()); 104 | config.set("zookeeper.znode.parent", constants.getZkHBasePath()); 105 | 106 | String tableName = "DeviceStatusLog"; 107 | try { 108 | HBaseAdmin hbaseAdmin = new HBaseAdmin(config); 109 | if (hbaseAdmin.tableExists("tableName")) { 110 | System.out.println("********************** Acquired " + tableName); 111 | Class.forName("org.apache.phoenix.jdbc.PhoenixDriver"); 112 | conn = DriverManager.getConnection("jdbc:phoenix:"+ constants.getZkHost() + ":" + constants.getZkPort() + ":" + constants.getZkHBasePath()); 113 | }else{ 114 | System.out.println("********************** Table " + tableName + " does not exist, creating..."); 115 | Class.forName("org.apache.phoenix.jdbc.PhoenixDriver"); 116 | conn = DriverManager.getConnection("jdbc:phoenix:"+ constants.getZkHost() + ":" + constants.getZkPort() + ":" + constants.getZkHBasePath()); 117 | conn.createStatement().execute("CREATE TABLE IF NOT EXISTS \"DeviceStatusLog\" (" 118 | + "\"pk\" VARCHAR PRIMARY KEY, " 119 | + "\"serialNumber\" VARCHAR, " 120 | + "\"state\" VARCHAR, " 121 | + "\"status\" VARCHAR, " 122 | + "\"internalTemp\" INTEGER, " 123 | + "\"signalStrength\" INTEGER, " 124 | + "\"timeStamp\" BIGINT)"); 125 | conn.commit(); 126 | 127 | System.out.println("********************** Created " + tableName); 128 | System.out.println("********************** Acquired " + tableName); 129 | hbaseAdmin.close(); 130 | } 131 | } catch (IOException e) { 132 | e.printStackTrace(); 133 | } catch (ClassNotFoundException e) { 134 | e.printStackTrace(); 135 | } catch (SQLException e) { 136 | e.printStackTrace(); 137 | } 138 | } 139 | 140 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 141 | declarer.declareStream("DeviceStatusStream", new Fields("DeviceStatus")); 142 | declarer.declareStream("DeviceAlertStream", new Fields("DeviceAlert")); 143 | declarer.declareStream("DeviceStatusLogStream", new Fields("serialNumber", "deviceModel", "status", "state", "internalTemp", "signalStrength")); 144 | declarer.declareStream("DeviceStatusLogNullStream", new Fields("serialNumber", "deviceModel", "status", "state", "internalTemp", "signalStrength")); 145 | } 146 | 147 | } 148 | -------------------------------------------------------------------------------- /MapUI/src/main/java/com/hortonworks/iot/mapui/MapUI.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks.iot.mapui; 2 | 3 | import java.io.IOException; 4 | import java.io.PrintWriter; 5 | import java.net.URL; 6 | import java.util.HashMap; 7 | import java.util.Map; 8 | 9 | import javax.servlet.*; 10 | import javax.servlet.annotation.WebServlet; 11 | import javax.servlet.http.*; 12 | 13 | import org.codehaus.jackson.JsonGenerationException; 14 | import org.codehaus.jackson.map.JsonMappingException; 15 | import org.codehaus.jackson.map.ObjectMapper; 16 | import org.cometd.client.BayeuxClient; 17 | import org.cometd.client.transport.ClientTransport; 18 | import org.cometd.client.transport.LongPollingTransport; 19 | import org.eclipse.jetty.client.HttpClient; 20 | 21 | @WebServlet(name = "MapUI", urlPatterns = { "/DeviceMap" }) 22 | public class MapUI extends HttpServlet { 23 | private static final String CONTENT_TYPE = "text/html; charset=windows-1252"; 24 | private String requestType; 25 | private String zkHost = "sandbox.hortonworks.com"; 26 | private String zkPort = "2181"; 27 | private String zkHBasePath = "/hbase-unsecure"; 28 | private String httpHost = "sandbox.hortonworks.com"; 29 | private String httpListenPort = "8082"; 30 | private String httpListenUri = "/contentListener"; 31 | private String cometdHost = "sandbox.hortonworks.com"; 32 | private String cometdListenPort = "8091"; 33 | private String defaultAccountNumber = "19123"; 34 | private String mapAPIKey = "NO_API_KEY_FOUND"; 35 | 36 | public void init(ServletConfig config) throws ServletException { 37 | super.init(config); 38 | System.out.println("Calling Init method and setting request to Initial"); 39 | requestType = "initial"; 40 | } 41 | 42 | public void doTask(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { 43 | response.setContentType(CONTENT_TYPE); 44 | PrintWriter out = response.getWriter(); 45 | System.out.println("First Check of Request Type: " + requestType); 46 | if(request.getParameter("requestType") != null){ 47 | System.out.println("RequestType parameter : " + request.getParameter("requestType")); 48 | requestType = request.getParameter("requestType"); 49 | System.out.println("RequestType set to :" + requestType); 50 | } 51 | 52 | //testPubSub(); 53 | 54 | Map env = System.getenv(); 55 | System.out.println("********************** ENV: " + env); 56 | if(env.get("ZK_HOST") != null){ 57 | this.zkHost = (String)env.get("ZK_HOST"); 58 | } 59 | if(env.get("ZK_PORT") != null){ 60 | this.zkPort = (String)env.get("ZK_PORT"); 61 | } 62 | if(env.get("ZK_HBASE_PATH") != null){ 63 | this.zkHBasePath = (String)env.get("ZK_HBASE_PATH"); 64 | } 65 | if(env.get("COMETD_HOST") != null){ 66 | this.cometdHost = (String)env.get("COMETD_HOST"); 67 | } 68 | if(env.get("COMETD_PORT") != null){ 69 | this.cometdListenPort = (String)env.get("COMETD_PORT"); 70 | } 71 | if(env.get("HTTP_HOST") != null){ 72 | this.httpHost = (String)env.get("HTTP_HOST"); 73 | } 74 | if(env.get("HTTP_PORT") != null){ 75 | this.httpListenPort = (String)env.get("HTTP_PORT"); 76 | } 77 | if(env.get("HTTP_URI") != null){ 78 | this.httpListenUri = (String)env.get("HTTP_URI"); 79 | } 80 | if(env.get("MAP_API_KEY") != null){ 81 | this.mapAPIKey = (String)env.get("MAP_API_KEY"); 82 | } 83 | System.out.println("********************** Zookeeper Host: " + zkHost); 84 | System.out.println("********************** Zookeeper: " + zkPort); 85 | System.out.println("********************** Zookeeper Path: " + zkHBasePath); 86 | System.out.println("********************** Cometd Host: " + cometdHost); 87 | System.out.println("********************** Cometd Port: " + cometdListenPort); 88 | System.out.println("********************** Http Host: " + httpHost); 89 | System.out.println("********************** Http Port: " + httpListenPort); 90 | System.out.println("********************** Http Uri: " + httpListenUri); 91 | System.out.println("********************** Map Api Key: " + mapAPIKey); 92 | 93 | ServletContext sc = this.getServletContext(); 94 | 95 | URL indexUri = this.getClass().getResource("/webapp"); 96 | //System.out.println(indexUri.toExternalForm()); 97 | System.out.println("Checking if Initial: " + requestType); 98 | if(requestType.equalsIgnoreCase("initial")){ 99 | request.setAttribute("cometdHost", cometdHost); 100 | request.setAttribute("cometdPort", cometdListenPort); 101 | request.setAttribute("mapAPIKey", mapAPIKey); 102 | request.getRequestDispatcher("mapui.jsp").forward(request, response); 103 | } 104 | } 105 | public void doGet(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { 106 | this.doTask(request, response); 107 | } 108 | public void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException { 109 | this.doTask(request, response); 110 | } 111 | 112 | public void testPubSub() { 113 | String pubSubUrl = "http://sandbox.hortonworks.com:8091/cometd"; 114 | String deviceChannel = "/devicestatus"; 115 | HttpClient httpClient = new HttpClient(); 116 | try { 117 | httpClient.start(); 118 | } catch (Exception e) { 119 | e.printStackTrace(); 120 | } 121 | 122 | // Prepare the transport 123 | Map options = new HashMap(); 124 | ClientTransport transport = new LongPollingTransport(options, httpClient); 125 | 126 | // Create the BayeuxClient 127 | BayeuxClient bayuexClient = new BayeuxClient(pubSubUrl, transport); 128 | 129 | bayuexClient.handshake(); 130 | boolean handshaken = bayuexClient.waitFor(5000, BayeuxClient.State.CONNECTED); 131 | if (handshaken) 132 | { 133 | System.out.println("Connected to Cometd Http PubSub Platform"); 134 | } 135 | else{ 136 | System.out.println("Could not connect to Cometd Http PubSub Platform"); 137 | } 138 | 139 | bayuexClient.getChannel(deviceChannel).publish("TEST"); 140 | } 141 | 142 | public String convertPOJOToJSON(Object pojo) { 143 | String jsonString = ""; 144 | ObjectMapper mapper = new ObjectMapper(); 145 | 146 | try { 147 | jsonString = mapper.writeValueAsString(pojo); 148 | } catch (JsonGenerationException e) { 149 | e.printStackTrace(); 150 | } catch (JsonMappingException e) { 151 | e.printStackTrace(); 152 | } catch (IOException e) { 153 | e.printStackTrace(); 154 | } 155 | return jsonString; 156 | } 157 | } -------------------------------------------------------------------------------- /MapUI/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4.0.0 4 | BigData 5 | MapUI 6 | 1.0-SNAPSHOT 7 | Project for MapUI 8 | jar 9 | 10 | 11 | ${docker.registry.name}vvaks/mapui 12 | 7.0.57 13 | 14 | 15 | 16 | org.apache.tomcat.embed 17 | tomcat-embed-core 18 | ${tomcat.version} 19 | 20 | 21 | org.apache.tomcat.embed 22 | tomcat-embed-logging-juli 23 | ${tomcat.version} 24 | 25 | 26 | org.apache.tomcat.embed 27 | tomcat-embed-jasper 28 | ${tomcat.version} 29 | 30 | 31 | org.apache.tomcat 32 | tomcat-jasper 33 | ${tomcat.version} 34 | 35 | 36 | org.apache.tomcat 37 | tomcat-jasper-el 38 | ${tomcat.version} 39 | 40 | 41 | org.apache.tomcat 42 | tomcat-jsp-api 43 | ${tomcat.version} 44 | 45 | 46 | org.eclipse.jetty 47 | jetty-client 48 | 9.3.6.v20151106 49 | 50 | 51 | org.eclipse.jetty 52 | jetty-util 53 | 9.3.6.v20151106 54 | 55 | 56 | org.cometd.java 57 | cometd-api 58 | 1.1.5 59 | 60 | 61 | org.cometd.java 62 | cometd-java-client 63 | 3.0.7 64 | 65 | 66 | javax.el 67 | el-api 68 | 2.2 69 | 70 | 71 | javax.servlet 72 | javax.servlet-api 73 | 3.1.0 74 | 75 | 76 | javax.servlet.jsp 77 | jsp-api 78 | 2.2 79 | 80 | 81 | javax.servlet.jsp.jstl 82 | javax.servlet.jsp.jstl-api 83 | 1.2.1 84 | 85 | 86 | org.codehaus.jackson 87 | jackson-core-asl 88 | 1.9.13 89 | 90 | 91 | org.codehaus.jackson 92 | jackson-mapper-asl 93 | 1.9.13 94 | 95 | 100 | 101 | org.slf4j 102 | slf4j-simple 103 | 1.7.13 104 | 105 | 106 | 107 | ${project.artifactId} 108 | src/ 109 | 110 | 111 | src/main/webapp 112 | META-INF/resources 113 | 114 | 115 | src/main/resources 116 | META-INF/resources 117 | 118 | 119 | classes/ 120 | 121 | 122 | maven-assembly-plugin 123 | 124 | 125 | jar-with-dependencies 126 | 127 | 128 | 129 | com.hortonworks.iot.mapui.MapUIMain 130 | 131 | 132 | 133 | 134 | 135 | package 136 | 137 | single 138 | 139 | 140 | 141 | 142 | 159 | 160 | org.apache.maven.plugins 161 | maven-war-plugin 162 | 2.1.1 163 | 164 | webapp/ 165 | MapUI 166 | 167 | 168 | 169 | org.jolokia 170 | docker-maven-plugin 171 | 0.13.3 172 | 173 | 174 | 175 | 177 | mapui 178 | ${docker.repository.name} 179 | 180 | java:8-jre 181 | vvaks 182 | 183 | docker-assembly.xml 184 | 185 | 186 | 8090 187 | 188 | 189 | java -jar \ 190 | /maven/MapUI-jar-with-dependencies.jar server \ 191 | /maven/docker-config.yml 192 | 193 | 194 | 195 | 196 | 197 | 198 | 199 | 200 | 201 | -------------------------------------------------------------------------------- /DeviceMonitor/pom.xml: -------------------------------------------------------------------------------- 1 | 2 | 4.0.0 3 | com.hortonworks.iot 4 | DeviceMonitor 5 | 0.0.1-SNAPSHOT 6 | DeviceMonitor 7 | jar 8 | 9 | 10 | org.apache.storm 11 | storm-core 12 | 1.0.1 13 | provided 14 | 15 | 16 | org.apache.zookeeper 17 | zookeeper 18 | 3.4.6 19 | 20 | 21 | org.slf4j 22 | slf4j-log4j12 23 | 24 | 25 | org.slf4j 26 | slf4j-api 27 | 28 | 29 | log4j 30 | log4j 31 | 32 | 33 | 34 | 35 | org.apache.storm 36 | storm-hdfs 37 | 1.0.1 38 | 39 | 40 | org.slf4j 41 | slf4j-log4j12 42 | 43 | 44 | log4j 45 | log4j 46 | 47 | 48 | 49 | 50 | org.apache.storm 51 | storm-hbase 52 | 1.0.1 53 | 54 | 55 | org.slf4j 56 | slf4j-log4j12 57 | 58 | 59 | log4j 60 | log4j 61 | 62 | 63 | 64 | 65 | org.apache.hbase 66 | hbase-client 67 | 1.1.2 68 | compile 69 | 70 | 71 | org.slf4j 72 | slf4j-log4j12 73 | 74 | 75 | log4j 76 | log4j 77 | 78 | 79 | 80 | 81 | org.apache.phoenix 82 | phoenix-core 83 | 4.7.0-HBase-1.1 84 | 85 | 86 | org.apache.storm 87 | storm-hive 88 | 1.0.1 89 | 90 | 91 | org.slf4j 92 | slf4j-log4j12 93 | 94 | 95 | log4j 96 | log4j 97 | 98 | 99 | org.eclipse.jetty.aggregate 100 | jetty-all 101 | 102 | 103 | 104 | 105 | org.apache.storm 106 | storm-kafka 107 | 1.0.1 108 | 109 | 110 | org.slf4j 111 | slf4j-log4j12 112 | 113 | 114 | org.slf4j 115 | slf4j-api 116 | 117 | 118 | log4j 119 | log4j 120 | 121 | 122 | org.apache.curator 123 | curator-framework 124 | 125 | 126 | 127 | 128 | org.apache.kafka 129 | kafka_2.10 130 | 0.8.2.1 131 | 132 | 133 | org.apache.zookeeper 134 | zookeeper 135 | 136 | 137 | org.slf4j 138 | slf4j-api 139 | 140 | 141 | log4j 142 | log4j 143 | 144 | 145 | 146 | 147 | org.codehaus.jackson 148 | jackson-mapper-asl 149 | 1.9.13 150 | 151 | 152 | com.google.appengine 153 | appengine-api-1.0-sdk 154 | 1.9.28 155 | 156 | 157 | com.googlecode.json-simple 158 | json-simple 159 | 1.1.1 160 | 161 | 162 | org.cometd.java 163 | cometd-java-client 164 | 3.0.7 165 | 166 | 167 | 168 | 169 | 170 | org.apache.maven.plugins 171 | maven-shade-plugin 172 | 1.4 173 | 174 | true 175 | 176 | 177 | 178 | package 179 | 180 | shade 181 | 182 | 183 | 184 | 186 | 188 | 189 | 190 | 191 | 192 | 193 | *:* 194 | 195 | META-INF/*.SF 196 | META-INF/*.DSA 197 | META-INF/*.RSA 198 | 199 | 200 | 201 | 202 | 203 | 204 | 205 | 206 | src/ 207 | 208 | 209 | ${basedir} 210 | 211 | * 212 | 213 | 214 | 215 | src/resources 216 | 217 | * 218 | 219 | 220 | 221 | classes/ 222 | 223 | -------------------------------------------------------------------------------- /Nifi/config/nifi-properties-env.json: -------------------------------------------------------------------------------- 1 | "properties" : { 2 | "content" : " \n# Core Properties #\nnifi.version={{nifi_dirname}}\nnifi.flow.configuration.file=./conf/flow.xml.gz\nnifi.flow.configuration.archive.enabled=true\nnifi.flow.configuration.archive.dir=./conf/archive/\nnifi.flow.configuration.archive.max.time=30 days\nnifi.flow.configuration.archive.max.storage=500 MB\nnifi.flowcontroller.autoResumeState=true\nnifi.flowcontroller.graceful.shutdown.period=10 sec\nnifi.flowservice.writedelay.interval=500 ms\nnifi.administrative.yield.duration=30 sec\n# If a component has no work to do (is \"bored\"), how long should we wait before checking again for work?\nnifi.bored.yield.duration=10 millis\n\nnifi.authority.provider.configuration.file=./conf/authorizers.xml\nnifi.login.identity.provider.configuration.file=./conf/login-identity-providers.xml\nnifi.templates.directory=./conf/templates\nnifi.ui.banner.text=\nnifi.ui.autorefresh.interval=30 sec\nnifi.nar.library.directory=./lib\nnifi.nar.working.directory=./work/nar/\nnifi.documentation.working.directory=./work/docs/components\n\n####################\n# State Management #\n####################\nnifi.state.management.configuration.file=./conf/state-management.xml\n# The ID of the local state provider\nnifi.state.management.provider.local=local-provider\n# The ID of the cluster-wide state provider. This will be ignored if NiFi is not clustered but must be populated if running in a cluster.\nnifi.state.management.provider.cluster=zk-provider\n# Specifies whether or not this instance of NiFi should run an embedded ZooKeeper server\nnifi.state.management.embedded.zookeeper.start=false\n# Properties file that provides the ZooKeeper properties to use if nifi.state.management.embedded.zookeeper.start is set to true\nnifi.state.management.embedded.zookeeper.properties=./conf/zookeeper.properties\n\n\n# H2 Settings\nnifi.database.directory=./database_repository\nnifi.h2.url.append=;LOCK_TIMEOUT=25000;WRITE_DELAY=0;AUTO_SERVER=FALSE\n\n# FlowFile Repository\nnifi.flowfile.repository.implementation=org.apache.nifi.controller.repository.WriteAheadFlowFileRepository\nnifi.flowfile.repository.directory=./flowfile_repository\nnifi.flowfile.repository.partitions=256\nnifi.flowfile.repository.checkpoint.interval=2 mins\nnifi.flowfile.repository.always.sync=false\n\nnifi.swap.manager.implementation=org.apache.nifi.controller.FileSystemSwapManager\nnifi.queue.swap.threshold=20000\nnifi.swap.in.period=5 sec\nnifi.swap.in.threads=1\nnifi.swap.out.period=5 sec\nnifi.swap.out.threads=4\n\n# Content Repository\nnifi.content.repository.implementation=org.apache.nifi.controller.repository.FileSystemRepository\nnifi.content.claim.max.appendable.size=10 MB\nnifi.content.claim.max.flow.files=100\nnifi.content.repository.directory.default=./content_repository\nnifi.content.repository.archive.max.retention.period=12 hours\nnifi.content.repository.archive.max.usage.percentage=50%\nnifi.content.repository.archive.enabled=true\nnifi.content.repository.always.sync=false\nnifi.content.viewer.url=/nifi-content-viewer/\n\n# Provenance Repository Properties\nnifi.provenance.repository.implementation=org.apache.nifi.provenance.PersistentProvenanceRepository\n\n# Persistent Provenance Repository Properties\nnifi.provenance.repository.directory.default=./provenance_repository\nnifi.provenance.repository.max.storage.time=24 hours\nnifi.provenance.repository.max.storage.size=1 GB\nnifi.provenance.repository.rollover.time=30 secs\nnifi.provenance.repository.rollover.size=100 MB\nnifi.provenance.repository.query.threads=2\nnifi.provenance.repository.index.threads=1\nnifi.provenance.repository.compress.on.rollover=true\nnifi.provenance.repository.always.sync=false\nnifi.provenance.repository.journal.count=16\n# Comma-separated list of fields. Fields that are not indexed will not be searchable. Valid fields are: \n# EventType, FlowFileUUID, Filename, TransitURI, ProcessorID, AlternateIdentifierURI, Relationship, Details\nnifi.provenance.repository.indexed.fields=EventType, FlowFileUUID, Filename, ProcessorID, Relationship\n# FlowFile Attributes that should be indexed and made searchable. Some examples to consider are filename, uuid, mime.type\nnifi.provenance.repository.indexed.attributes=\n# Large values for the shard size will result in more Java heap usage when searching the Provenance Repository\n# but should provide better performance\nnifi.provenance.repository.index.shard.size=500 MB\n# Indicates the maximum length that a FlowFile attribute can be when retrieving a Provenance Event from\n# the repository. If the length of any attribute exceeds this value, it will be truncated when the event is retrieved.\nnifi.provenance.repository.max.attribute.length=65536\n\n# Volatile Provenance Respository Properties\nnifi.provenance.repository.buffer.size=100000\n\n# Component Status Repository\nnifi.components.status.repository.implementation=org.apache.nifi.controller.status.history.VolatileComponentStatusRepository\nnifi.components.status.repository.buffer.size=1440\nnifi.components.status.snapshot.frequency=1 min\n\n# Site to Site properties\nnifi.remote.input.host=\nnifi.remote.input.secure=false\nnifi.remote.input.socket.port=\nnifi.remote.input.http.enabled=true\nnifi.remote.input.http.transaction.ttl=30 sec\n\n# web properties #\nnifi.web.war.directory=./lib\nnifi.web.http.host={{nifi_node_host}}\nnifi.web.http.port={{nifi_port}}\nnifi.web.https.host=\nnifi.web.https.port=\nnifi.web.jetty.working.directory=./work/jetty\nnifi.web.jetty.threads=200\n\n# security properties #\nnifi.sensitive.props.key=\nnifi.sensitive.props.key.protected=\nnifi.sensitive.props.algorithm=PBEWITHMD5AND256BITAES-CBC-OPENSSL\nnifi.sensitive.props.provider=BC\nnifi.sensitive.props.additional.keys=\n\nnifi.security.keystore=\nnifi.security.keystoreType=\nnifi.security.keystorePasswd=\nnifi.security.keyPasswd=\nnifi.security.truststore=\nnifi.security.truststoreType=\nnifi.security.truststorePasswd=\nnifi.security.needClientAuth=\nnifi.security.user.authorizer=file-provider\nnifi.security.user.login.identity.provider=\nnifi.security.ocsp.responder.url=\nnifi.security.ocsp.responder.certificate=\n\n# Identity Mapping Properties #\n# These properties allow normalizing user identities such that identities coming from different identity providers\n# (certificates, LDAP, Kerberos) can be treated the same internally in NiFi. The following example demonstrates normalizing\n# DNs from certificates and principals from Kerberos into a common identity string:\n#\n# nifi.security.identity.mapping.pattern.dn=^CN=(.*?), OU=(.*?), O=(.*?), L=(.*?), ST=(.*?), C=(.*?)$\n# nifi.security.identity.mapping.value.dn=$1@$2\n# nifi.security.identity.mapping.pattern.kerb=^(.*?)/instance@(.*?)$\n# nifi.security.identity.mapping.value.kerb=$1@$2\n\n# cluster common properties (all nodes must have same values) #\nnifi.cluster.protocol.heartbeat.interval=5 sec\nnifi.cluster.protocol.is.secure=false\n\n# cluster node properties (only configure for cluster nodes) #\nnifi.cluster.is.node={{nifi_is_node}}\nnifi.cluster.node.address={{nifi_node_host}}\nnifi.cluster.node.protocol.port={{nifi_node_protocol_port}}\nnifi.cluster.node.protocol.threads=10\nnifi.cluster.node.event.history.size=25\nnifi.cluster.node.connection.timeout=5 sec\nnifi.cluster.node.read.timeout=5 sec\nnifi.cluster.firewall.file=\nnifi.cluster.flow.election.max.wait.time=5 mins\nnifi.cluster.flow.election.max.candidates={{nifi_num_nodes}}\n\n# zookeeper properties, used for cluster management #\nnifi.zookeeper.connect.string={{zookeeper_quorum}}\nnifi.zookeeper.connect.timeout=3 secs\nnifi.zookeeper.session.timeout=3 secs\nnifi.zookeeper.root.node=/nifi\n\n# kerberos #\nnifi.kerberos.krb5.file=\n\n# kerberos service principle #\nnifi.kerberos.service.principal=\nnifi.kerberos.service.keytab.location=\n\n# kerberos spnego principle #\nnifi.kerberos.spnego.principal=\nnifi.kerberos.spnego.keytab.location=\nnifi.kerberos.spnego.authentication.expiration=12 hours\n\n# external properties files for variable registry\n# supports a comma delimited list of file locations\nnifi.variable.registry.properties=" 3 | } -------------------------------------------------------------------------------- /DeviceMonitorNostradamusScala/src/main/scala/com/hortonworks/iot/spark/streaming/SparkNostradamus.scala: -------------------------------------------------------------------------------- 1 | package com.hortonworks.iot.spark.streaming 2 | 3 | import org.apache.spark.SparkConf 4 | import org.apache.spark.streaming.{Seconds, StreamingContext} 5 | import org.apache.spark.streaming.kafka._ 6 | import org.apache.spark.mllib.classification.SVMModel 7 | import org.apache.spark.mllib.linalg.Vectors 8 | import org.cometd.client.BayeuxClient 9 | import org.cometd.client.transport.{ClientTransport,LongPollingTransport} 10 | import org.eclipse.jetty.client.HttpClient 11 | import scala.util.parsing.json._ 12 | 13 | object SparkNostradamus { 14 | def main(args: Array[String]) { 15 | val sparkConf = new SparkConf().setAppName("SparkNostradamus") 16 | val ssc = new StreamingContext(sparkConf, Seconds(1)) 17 | ssc.sparkContext.setLogLevel("WARN") 18 | ssc.checkpoint("/demo/data/checkpoint"); 19 | val nostradamus = SVMModel.load(ssc.sparkContext, "/demo/data/model/nostradamusSVMModel"); 20 | val kafkaConsumerGroup = "spark-streaming-group" 21 | val zkQuorum = args(0) 22 | val deviceEventsTopic = args(1) 23 | val cometdUrl = ssc.sparkContext.broadcast("http://" + args(2) + "/cometd") 24 | val predictionChannel = ssc.sparkContext.broadcast("/prediction") 25 | val kafkaTopicConfig = Map(deviceEventsTopic -> 1) 26 | val deviceStreamJSON = KafkaUtils.createStream(ssc, zkQuorum, kafkaConsumerGroup, kafkaTopicConfig) 27 | val deviceStream = deviceStreamJSON.map{ rdd => val deviceStatusEvent = JSON.parseFull(rdd._2).getOrElse("{}").asInstanceOf[Map[String,Any]] 28 | val serialNumber = deviceStatusEvent.get("serialNumber").get.asInstanceOf[String] 29 | val internalTemp = deviceStatusEvent.get("internalTemp").get.asInstanceOf[Int].toDouble 30 | (serialNumber, internalTemp) 31 | }.updateStateByKey(fillFeatureList).foreachRDD(rdd => { 32 | rdd.foreach(featureTupleList => 33 | { 34 | val tempFailPredication = "Temperature pattern indicates imminent device failure. Contact customer or send technician" 35 | val serialNumber = featureTupleList._1 36 | val featureList = featureTupleList._2 37 | if(featureList.size == 10){ 38 | println("**********************Feature list has reached required size... " + featureList) 39 | println("**********************Making a prediction...") 40 | val predictionFeatures = Vectors.dense(featureList.toArray) 41 | val prediction = nostradamus.predict(predictionFeatures) 42 | if(prediction == 1.0){ 43 | println("*********************************************************************************") 44 | println("**********************DEVICE FAILURE IMMINENT: " + serialNumber + " : " + prediction + " : " + predictionFeatures) 45 | println("*********************************************************************************") 46 | val httpClient = new HttpClient 47 | httpClient.start 48 | val bayuexClient = new BayeuxClient(cometdUrl.value, new LongPollingTransport( new java.util.HashMap[String,Object](), httpClient)) 49 | bayuexClient.handshake 50 | println("Attempting to connect to: " + cometdUrl.value) 51 | if (bayuexClient.waitFor(3000, BayeuxClient.State.CONNECTED)){ 52 | println("Connected to Cometd Http PubSub Platform") 53 | val data = new java.util.HashMap[String,String]() 54 | data.put("deviceSerialNumber", serialNumber) 55 | data.put("predictionDescription", tempFailPredication) 56 | bayuexClient.getChannel(predictionChannel.value).publish(data) 57 | }else{ 58 | println("Could not connect to Cometd Http PubSub Platform") 59 | } 60 | }else{ 61 | println("*********************************************************************************") 62 | println("**********************DEVICE FUNCTION NORMAL : " + serialNumber + " : " + prediction + " : " + predictionFeatures) 63 | println("*********************************************************************************") 64 | } 65 | }else{ 66 | println("$$$$$$$$$$$$$$$$$$$$$$$$DEVICE: " + serialNumber + "... Not enough events to make a prediction...") 67 | } 68 | } 69 | ) 70 | } 71 | ) 72 | ssc.start() 73 | ssc.awaitTermination() 74 | } 75 | 76 | def fillFeatureList(incomingEventList: Seq[(Double)], currentEventWindow: Option[List[Double]]): Option[List[Double]] = { 77 | println("@@@@@@@@@@@@@@@@@@@@Current Event List... " + currentEventWindow) 78 | println("@@@@@@@@@@@@@@@@@@@@Incoming Event List... " + incomingEventList) 79 | val updatedEventWindow = currentEventWindow.getOrElse(List()).++(incomingEventList) 80 | println("@@@@@@@@@@@@@@@@@@@@Updated Event List " + updatedEventWindow) 81 | val returnEventWindow = if(updatedEventWindow.size > 10){ 82 | updatedEventWindow.drop(10) 83 | }else{ 84 | updatedEventWindow 85 | } 86 | println("@@@@@@@@@@@@@@@@@@@@Returning Event List " + returnEventWindow) 87 | Some(returnEventWindow) 88 | } 89 | } -------------------------------------------------------------------------------- /DeviceMonitorNostradamus/src/main/java/com/hortonworks/iot/util/Constants.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks.iot.util; 2 | 3 | import java.util.List; 4 | import java.util.Map; 5 | 6 | import com.google.common.collect.Lists; 7 | 8 | public class Constants { 9 | //Properties file which has all the configurable parameters required for execution of this Topology. 10 | public static final String CONFIG_PROPERTIES_FILE = "config.properties"; 11 | 12 | private String zkHost = "sandbox.hortonworks.com"; 13 | private String zkPort = "2181"; 14 | private String zkKafkaPath = "/brokers"; 15 | private String zkHBasePath = "/hbase-unsecure"; 16 | private String zkConnString = "sandbox.hortonworks.com:2181"; 17 | private String deviceTopicName = "DeviceEvents"; 18 | private String technicianTopicName = "TechnicianEvent"; 19 | 20 | private String cometdHost = "sandbox.hortonworks.com"; 21 | private String cometdPort = "8091"; 22 | private String pubSubUrl; 23 | private String technicianChannel = "/technicianstatus"; 24 | private String deviceChannel = "/devicestatus"; 25 | private String alertChannel = "/alert"; 26 | private String predictionChannel = "/prediction"; 27 | 28 | private String nameNodeHost = "sandbox.hortonworks.com"; 29 | private String nameNodePort = "8020"; 30 | private String nameNodeUrl; 31 | private String hivePath = "/demo/data/device_logs"; 32 | private String sparkModelPath = "/demo/data/model/"; 33 | private String sparkCheckpointPath = "/demo/data/checkpoints/"; 34 | 35 | private String atlasHost = "localhost"; 36 | private String atlasPort = "21000"; 37 | 38 | private String hiveMetaStoreURI = "jdbc:mysql://sandbox.hortonworks.com/hive"; 39 | private String hiveDbName = "default"; 40 | 41 | public Constants(){ 42 | Map env = System.getenv(); 43 | //System.out.println("********************** ENV: " + env); 44 | if(env.get("ZK_HOST") != null){ 45 | this.setZkHost((String)env.get("ZK_HOST")); 46 | } 47 | if(env.get("ZK_PORT") != null){ 48 | this.setZkPort((String)env.get("ZK_PORT")); 49 | } 50 | if(env.get("ZK_KAFKA_PATH") != null){ 51 | this.setZkKafkaPath((String)env.get("ZK_KAFKA_PATH")); 52 | } 53 | if(env.get("ZK_HBASE_PATH") != null){ 54 | this.setZkHBasePath((String)env.get("ZK_HBASE_PATH")); 55 | } 56 | if(env.get("NAMENODE_HOST") != null){ 57 | this.setNameNodeHost((String)env.get("NAMENODE_HOST")); 58 | } 59 | if(env.get("NAMENODE_PORT") != null){ 60 | this.setNameNodePort((String)env.get("NAMENODE_PORT")); 61 | } 62 | if(env.get("HIVE_PATH") != null){ 63 | this.setHivePath((String)env.get("HIVE_PATH")); 64 | } 65 | if(env.get("HIVE_METASTORE_URI") != null){ 66 | this.setHiveMetaStoreURI((String)env.get("HIVE_METASTORE_URI")); 67 | } 68 | if(env.get("ATLAS_HOST") != null){ 69 | this.setAtlasHost((String)env.get("ATLAS_HOST")); 70 | } 71 | if(env.get("ATLAS_PORT") != null){ 72 | this.setAtlasPort((String)env.get("ATLAS_PORT")); 73 | } 74 | if(env.get("COMETD_HOST") != null){ 75 | this.setCometdHost((String)env.get("COMETD_HOST")); 76 | } 77 | if(env.get("COMETD_PORT") != null){ 78 | this.setCometdPort((String)env.get("COMETD_PORT")); 79 | } 80 | 81 | this.setZkConnString(zkHost+":"+zkPort); 82 | this.setPubSubUrl("http://" + cometdHost + ":" + cometdPort + "/cometd"); 83 | this.setNameNodeUrl("hdfs://" + nameNodeHost + ":" + nameNodePort); 84 | } 85 | 86 | public String getZkKafkaPath() { 87 | return zkKafkaPath; 88 | } 89 | 90 | public void setZkKafkaPath(String zkKafkaPath) { 91 | this.zkKafkaPath = zkKafkaPath; 92 | } 93 | 94 | public String getZkHBasePath() { 95 | return zkHBasePath; 96 | } 97 | 98 | public void setZkHBasePath(String zkHBasePath) { 99 | this.zkHBasePath = zkHBasePath; 100 | } 101 | 102 | public String getPubSubUrl() { 103 | return pubSubUrl; 104 | } 105 | 106 | public void setPubSubUrl(String pubSubUrl) { 107 | this.pubSubUrl = pubSubUrl; 108 | } 109 | 110 | public String getTechnicianTopicName() { 111 | return technicianTopicName; 112 | } 113 | 114 | public void setTechnicianTopicName(String technicianTopicName) { 115 | this.technicianTopicName = technicianTopicName; 116 | } 117 | 118 | public String getDeviceTopicName() { 119 | return deviceTopicName; 120 | } 121 | 122 | public void setDeviceTopicName(String deviceTopicName) { 123 | this.deviceTopicName = deviceTopicName; 124 | } 125 | 126 | public String getZkConnString() { 127 | return zkConnString; 128 | } 129 | 130 | public void setZkConnString(String zkConnString) { 131 | this.zkConnString = zkConnString; 132 | } 133 | 134 | public String getHivePath() { 135 | return hivePath; 136 | } 137 | 138 | public void setHivePath(String hivePath) { 139 | this.hivePath = hivePath; 140 | } 141 | 142 | public String getSparkModelPath() { 143 | return sparkModelPath; 144 | } 145 | 146 | public void setSparkModelPath(String sparkModelPath) { 147 | this.sparkModelPath = sparkModelPath; 148 | } 149 | 150 | public String getSparkCheckpointPath() { 151 | return sparkCheckpointPath; 152 | } 153 | 154 | public void setSparkCheckpointPath(String sparkCheckpointPath) { 155 | this.sparkCheckpointPath = sparkCheckpointPath; 156 | } 157 | 158 | public String getNameNodeUrl() { 159 | return nameNodeUrl; 160 | } 161 | 162 | public void setNameNodeUrl(String nameNodeUrl) { 163 | this.nameNodeUrl = nameNodeUrl; 164 | } 165 | 166 | public String getAtlasHost() { 167 | return atlasHost; 168 | } 169 | 170 | public void setAtlasHost(String atlasHost) { 171 | this.atlasHost = atlasHost; 172 | } 173 | 174 | public String getAtlasPort() { 175 | return atlasPort; 176 | } 177 | 178 | public void setAtlasPort(String atlasPort) { 179 | this.atlasPort = atlasPort; 180 | } 181 | 182 | public String getHiveMetaStoreURI() { 183 | return hiveMetaStoreURI; 184 | } 185 | 186 | public void setHiveMetaStoreURI(String hiveMetaStoreURI) { 187 | this.hiveMetaStoreURI = hiveMetaStoreURI; 188 | } 189 | 190 | public String getHiveDbName() { 191 | return hiveDbName; 192 | } 193 | 194 | public void setHiveDbName(String hiveDbName) { 195 | this.hiveDbName = hiveDbName; 196 | } 197 | 198 | public String getTechnicianChannel() { 199 | return technicianChannel; 200 | } 201 | 202 | public void setTechnicianChannel(String technicianChannel) { 203 | this.technicianChannel = technicianChannel; 204 | } 205 | 206 | public String getAlertChannel() { 207 | return alertChannel; 208 | } 209 | 210 | public void setAlertChannel(String alertChannel) { 211 | this.alertChannel = alertChannel; 212 | } 213 | 214 | public String getPredictionChannel() { 215 | return predictionChannel; 216 | } 217 | 218 | public void setPredictionChannel(String predictionChannel) { 219 | this.predictionChannel = predictionChannel; 220 | } 221 | 222 | public String getDeviceChannel() { 223 | return deviceChannel; 224 | } 225 | 226 | public void setDeviceChannel(String deviceChannel) { 227 | this.deviceChannel = deviceChannel; 228 | } 229 | 230 | public String getCometdHost() { 231 | return cometdHost; 232 | } 233 | 234 | public void setCometdHost(String cometdHost) { 235 | this.cometdHost = cometdHost; 236 | } 237 | 238 | public String getCometdPort() { 239 | return cometdPort; 240 | } 241 | 242 | public void setCometdPort(String cometdPort) { 243 | this.cometdPort = cometdPort; 244 | } 245 | 246 | public String getNameNodeHost() { 247 | return nameNodeHost; 248 | } 249 | 250 | public void setNameNodeHost(String nameNodeHost) { 251 | this.nameNodeHost = nameNodeHost; 252 | } 253 | 254 | public String getNameNodePort() { 255 | return nameNodePort; 256 | } 257 | 258 | public void setNameNodePort(String nameNodePort) { 259 | this.nameNodePort = nameNodePort; 260 | } 261 | 262 | public String getZkHost() { 263 | return zkHost; 264 | } 265 | 266 | public void setZkHost(String zkHost) { 267 | this.zkHost = zkHost; 268 | } 269 | 270 | public String getZkPort() { 271 | return zkPort; 272 | } 273 | 274 | public void setZkPort(String zkPort) { 275 | this.zkPort = zkPort; 276 | } 277 | } -------------------------------------------------------------------------------- /DeviceMonitor/src/main/java/com/hortonworks/bolts/EnrichDeviceStatus.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks.bolts; 2 | 3 | import java.io.IOException; 4 | import java.sql.Connection; 5 | import java.sql.DriverManager; 6 | import java.sql.SQLException; 7 | import java.util.Map; 8 | 9 | import org.apache.hadoop.conf.Configuration; 10 | import org.apache.hadoop.hbase.HBaseConfiguration; 11 | import org.apache.hadoop.hbase.HColumnDescriptor; 12 | import org.apache.hadoop.hbase.HTableDescriptor; 13 | import org.apache.hadoop.hbase.client.Get; 14 | import org.apache.hadoop.hbase.client.HBaseAdmin; 15 | import org.apache.hadoop.hbase.client.HTable; 16 | import org.apache.hadoop.hbase.client.Put; 17 | import org.apache.hadoop.hbase.client.Result; 18 | import org.apache.hadoop.hbase.util.Bytes; 19 | 20 | import com.hortonworks.events.DeviceStatus; 21 | import com.hortonworks.events.STBStatus; 22 | import com.hortonworks.util.Constants; 23 | 24 | import org.apache.storm.task.OutputCollector; 25 | import org.apache.storm.task.TopologyContext; 26 | import org.apache.storm.topology.OutputFieldsDeclarer; 27 | import org.apache.storm.topology.base.BaseRichBolt; 28 | import org.apache.storm.tuple.Fields; 29 | import org.apache.storm.tuple.Tuple; 30 | import org.apache.storm.tuple.Values; 31 | 32 | /* 33 | import backtype.storm.task.OutputCollector; 34 | import backtype.storm.task.TopologyContext; 35 | import backtype.storm.topology.OutputFieldsDeclarer; 36 | import backtype.storm.topology.base.BaseRichBolt; 37 | import backtype.storm.tuple.Fields; 38 | import backtype.storm.tuple.Tuple; 39 | import backtype.storm.tuple.Values; 40 | */ 41 | 42 | public class EnrichDeviceStatus extends BaseRichBolt { 43 | private static final long serialVersionUID = 1L; 44 | private OutputCollector collector; 45 | private Constants constants; 46 | private HTable table = null; 47 | private Connection conn; 48 | 49 | public void execute(Tuple tuple) { 50 | STBStatus deviceStatus = (STBStatus) tuple.getValueByField("DeviceStatus"); 51 | 52 | Get get = new Get(Bytes.toBytes(deviceStatus.getSerialNumber())); 53 | System.out.println("Build Request..."); 54 | // Reading the data 55 | Result result = null; 56 | try { 57 | result = table.get(get); 58 | } catch (IOException e) { 59 | e.printStackTrace(); 60 | } 61 | //System.out.println("Get Results..."); 62 | // Reading values from Result class object 63 | 64 | if(result.getValue(Bytes.toBytes("cf"),Bytes.toBytes("Longitude")) !=null && result.getValue(Bytes.toBytes("cf"),Bytes.toBytes("Latitude")) != null){ 65 | byte [] deviceModel = result.getValue(Bytes.toBytes("cf"),Bytes.toBytes("DeviceModel")); 66 | byte [] longitude = result.getValue(Bytes.toBytes("cf"),Bytes.toBytes("Longitude")); 67 | byte [] latitude = result.getValue(Bytes.toBytes("cf"),Bytes.toBytes("Latitude")); 68 | 69 | deviceStatus.setDeviceModel(Bytes.toString(deviceModel)); 70 | deviceStatus.setLatitude(Double.parseDouble(Bytes.toString(latitude))); 71 | deviceStatus.setLongitude(Double.parseDouble(Bytes.toString(longitude))); 72 | 73 | collector.emit(tuple, new Values((DeviceStatus)deviceStatus)); 74 | collector.ack(tuple); 75 | } 76 | else{ 77 | System.out.println("Recieved and event from a device that is not in the datastore or location is unknown... dropping event"); 78 | collector.ack(tuple); 79 | } 80 | /*System.out.println(deviceStatus.getSerialNumber()); 81 | System.out.println(deviceStatus.getState()); 82 | System.out.println(deviceStatus.getStatus()); 83 | System.out.println(deviceStatus.getInternalTemp()); 84 | System.out.println(deviceStatus.getSignalStrength());*/ 85 | } 86 | 87 | @SuppressWarnings("deprecation") 88 | public void prepare(Map arg0, TopologyContext arg1, OutputCollector collector) { 89 | this.constants = new Constants(); 90 | 91 | System.out.println("********************** Zookeeper Host: " + constants.getZkHost()); 92 | System.out.println("********************** Zookeeper Port: " + constants.getZkPort()); 93 | System.out.println("********************** Zookeeper ConnString: " + constants.getZkConnString()); 94 | System.out.println("********************** Zookeeper Kafka Path: " + constants.getZkKafkaPath()); 95 | System.out.println("********************** Zookeeper HBase Path: " + constants.getZkHBasePath()); 96 | System.out.println("********************** Cometd URI: " + constants.getPubSubUrl()); 97 | 98 | Configuration config = HBaseConfiguration.create(); 99 | config.set("hbase.zookeeper.quorum", constants.getZkHost()); 100 | config.set("hbase.zookeeper.property.clientPort", constants.getZkPort()); 101 | config.set("zookeeper.znode.parent", constants.getZkHBasePath()); 102 | 103 | String tableName = "DeviceDetails"; 104 | try { 105 | HBaseAdmin hbaseAdmin = new HBaseAdmin(config); 106 | if (hbaseAdmin.tableExists(tableName)) { 107 | table = new HTable(config, tableName); 108 | System.out.println("********************** Acquired " + tableName); 109 | Class.forName("org.apache.phoenix.jdbc.PhoenixDriver"); 110 | conn = DriverManager.getConnection("jdbc:phoenix:"+ constants.getZkHost() + ":" + constants.getZkPort() + ":" + constants.getZkHBasePath()); 111 | }else{ 112 | System.out.println("********************** Table " + tableName + "does not exist, creating..."); 113 | Class.forName("org.apache.phoenix.jdbc.PhoenixDriver"); 114 | conn = DriverManager.getConnection("jdbc:phoenix:"+ constants.getZkHost() + ":" + constants.getZkPort() + ":" + constants.getZkHBasePath()); 115 | conn.createStatement().execute("CREATE TABLE IF NOT EXISTS \"DeviceDetailsBI\" (" 116 | + "\"serialNumber\" VARCHAR PRIMARY KEY, " 117 | + "\"deviceModel\" VARCHAR, " 118 | + "\"latitude\" VARCHAR, " 119 | + "\"longitude\" VARCHAR, " 120 | + "\"ipAddress\" VARCHAR, " 121 | + "\"port\" VARCHAR)"); 122 | conn.commit(); 123 | 124 | conn.createStatement().executeUpdate("UPSERT INTO \"DeviceDetailsBI\" VALUES(" 125 | + "'1000', " 126 | + "'Motorolla', " 127 | + "'39.951694', " 128 | + "'-75.144596', " 129 | + "'192.168.56.1', " 130 | + "'8085')"); 131 | conn.commit(); 132 | 133 | conn.createStatement().executeUpdate("UPSERT INTO \"DeviceDetailsBI\" VALUES(" 134 | + "'2000', " 135 | + "'Motorolla', " 136 | + "'39.970279', " 137 | + "'-75.175152', " 138 | + "'192.168.56.1', " 139 | + "'8087')"); 140 | conn.commit(); 141 | 142 | conn.createStatement().executeUpdate("UPSERT INTO \"DeviceDetailsBI\" VALUES(" 143 | + "'3000', " 144 | + "'Motorolla', " 145 | + "'39.948174', " 146 | + "'-75.170689', " 147 | + "'192.168.56.1', " 148 | + "'8089')"); 149 | conn.commit(); 150 | 151 | HTableDescriptor tableDescriptor = new HTableDescriptor(tableName); 152 | HColumnDescriptor cfColumnFamily = new HColumnDescriptor("cf".getBytes()); 153 | tableDescriptor.addFamily(cfColumnFamily); 154 | hbaseAdmin.createTable(tableDescriptor); 155 | System.out.println("********************** Created " + tableName); 156 | table = new HTable(config, tableName); 157 | System.out.println("********************** Acquired " + tableName); 158 | } 159 | hbaseAdmin.close(); 160 | } catch (IOException e) { 161 | e.printStackTrace(); 162 | } catch (ClassNotFoundException e) { 163 | e.printStackTrace(); 164 | } catch (SQLException e) { 165 | e.printStackTrace(); 166 | } 167 | 168 | Put device1000 = new Put(Bytes.toBytes("1000")); 169 | device1000.add(Bytes.toBytes("cf"), Bytes.toBytes("DeviceModel"), Bytes.toBytes("Motorolla")); 170 | device1000.add(Bytes.toBytes("cf"), Bytes.toBytes("Latitude"), Bytes.toBytes("39.951694")); 171 | device1000.add(Bytes.toBytes("cf"), Bytes.toBytes("Longitude"), Bytes.toBytes("-75.144596")); 172 | device1000.add(Bytes.toBytes("cf"), Bytes.toBytes("IpAddress"), Bytes.toBytes("192.168.56.1")); 173 | device1000.add(Bytes.toBytes("cf"), Bytes.toBytes("Port"), Bytes.toBytes("8085")); 174 | 175 | Put device2000 = new Put(Bytes.toBytes("2000")); 176 | device2000.add(Bytes.toBytes("cf"), Bytes.toBytes("DeviceModel"), Bytes.toBytes("Motorolla")); 177 | device2000.add(Bytes.toBytes("cf"), Bytes.toBytes("Latitude"), Bytes.toBytes("39.970279")); 178 | device2000.add(Bytes.toBytes("cf"), Bytes.toBytes("Longitude"), Bytes.toBytes("-75.175152")); 179 | device2000.add(Bytes.toBytes("cf"), Bytes.toBytes("IpAddress"), Bytes.toBytes("192.168.56.1")); 180 | device2000.add(Bytes.toBytes("cf"), Bytes.toBytes("Port"), Bytes.toBytes("8087")); 181 | 182 | Put device3000 = new Put(Bytes.toBytes("3000")); 183 | device3000.add(Bytes.toBytes("cf"), Bytes.toBytes("DeviceModel"), Bytes.toBytes("Motorolla")); 184 | device3000.add(Bytes.toBytes("cf"), Bytes.toBytes("Latitude"), Bytes.toBytes("39.948174")); 185 | device3000.add(Bytes.toBytes("cf"), Bytes.toBytes("Longitude"), Bytes.toBytes("-75.170689")); 186 | device3000.add(Bytes.toBytes("cf"), Bytes.toBytes("IpAddress"), Bytes.toBytes("192.168.56.1")); 187 | device3000.add(Bytes.toBytes("cf"), Bytes.toBytes("Port"), Bytes.toBytes("8089")); 188 | 189 | try { 190 | table.put(device1000); 191 | table.put(device2000); 192 | table.put(device3000); 193 | } catch (IOException e) { 194 | e.printStackTrace(); 195 | } 196 | this.collector = collector; 197 | } 198 | 199 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 200 | declarer.declare(new Fields("DeviceStatus")); 201 | } 202 | } -------------------------------------------------------------------------------- /DeviceMonitor/src/main/java/com/hortonworks/iot/topology/DeviceMonitorTopology.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks.iot.topology; 2 | 3 | import java.util.HashMap; 4 | import java.util.Map; 5 | import java.util.UUID; 6 | 7 | import org.apache.storm.hbase.bolt.HBaseBolt; 8 | import org.apache.storm.hbase.bolt.mapper.SimpleHBaseMapper; 9 | import org.apache.storm.hdfs.bolt.HdfsBolt; 10 | import org.apache.storm.hdfs.bolt.format.DefaultFileNameFormat; 11 | import org.apache.storm.hdfs.bolt.format.DelimitedRecordFormat; 12 | import org.apache.storm.hdfs.bolt.format.FileNameFormat; 13 | import org.apache.storm.hdfs.bolt.format.RecordFormat; 14 | import org.apache.storm.hdfs.bolt.rotation.FileRotationPolicy; 15 | import org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy; 16 | import org.apache.storm.hdfs.bolt.rotation.FileSizeRotationPolicy.Units; 17 | import org.apache.storm.hdfs.bolt.sync.CountSyncPolicy; 18 | import org.apache.storm.hdfs.bolt.sync.SyncPolicy; 19 | import org.apache.storm.hive.bolt.HiveBolt; 20 | import org.apache.storm.hive.bolt.mapper.DelimitedRecordHiveMapper; 21 | import org.apache.storm.hive.common.HiveOptions; 22 | 23 | import com.hortonworks.bolts.EnrichDeviceStatus; 24 | import com.hortonworks.bolts.IncidentDetector; 25 | import com.hortonworks.bolts.PersistTechnicianLocation; 26 | import com.hortonworks.bolts.PrintDeviceAlert; 27 | import com.hortonworks.bolts.PrintDeviceStatus; 28 | import com.hortonworks.bolts.PublishAlert; 29 | import com.hortonworks.bolts.PublishDeviceStatus; 30 | import com.hortonworks.bolts.PublishTechnicianLocation; 31 | import com.hortonworks.bolts.RecommendTechnician; 32 | import com.hortonworks.bolts.RouteTechnician; 33 | 34 | import com.hortonworks.util.Constants; 35 | import com.hortonworks.util.DeviceEventJSONScheme; 36 | import com.hortonworks.util.TechnicianEventJSONScheme; 37 | 38 | 39 | /* 40 | import backtype.storm.StormSubmitter; 41 | import backtype.storm.Config; 42 | import backtype.storm.LocalCluster; 43 | import backtype.storm.StormSubmitter; 44 | import backtype.storm.generated.AlreadyAliveException; 45 | import backtype.storm.generated.AuthorizationException; 46 | import backtype.storm.generated.InvalidTopologyException; 47 | import backtype.storm.spout.SchemeAsMultiScheme; 48 | import backtype.storm.topology.TopologyBuilder; 49 | import backtype.storm.tuple.Fields; 50 | import storm.kafka.BrokerHosts; 51 | import storm.kafka.KafkaSpout; 52 | import storm.kafka.KeyValueSchemeAsMultiScheme; 53 | import storm.kafka.SpoutConfig; 54 | import storm.kafka.ZkHosts; 55 | */ 56 | 57 | import org.apache.storm.Config; 58 | import org.apache.storm.LocalCluster; 59 | import org.apache.storm.StormSubmitter; 60 | import org.apache.storm.generated.AlreadyAliveException; 61 | import org.apache.storm.generated.AuthorizationException; 62 | import org.apache.storm.generated.InvalidTopologyException; 63 | import org.apache.storm.kafka.BrokerHosts; 64 | import org.apache.storm.kafka.KafkaSpout; 65 | import org.apache.storm.kafka.KeyValueSchemeAsMultiScheme; 66 | import org.apache.storm.kafka.SpoutConfig; 67 | import org.apache.storm.kafka.ZkHosts; 68 | import org.apache.storm.spout.SchemeAsMultiScheme; 69 | import org.apache.storm.task.OutputCollector; 70 | import org.apache.storm.task.TopologyContext; 71 | import org.apache.storm.topology.OutputFieldsDeclarer; 72 | import org.apache.storm.topology.TopologyBuilder; 73 | import org.apache.storm.topology.base.BaseRichBolt; 74 | import org.apache.storm.tuple.Fields; 75 | import org.apache.storm.tuple.Tuple; 76 | import org.apache.storm.tuple.Values; 77 | 78 | public class DeviceMonitorTopology { 79 | static String topologyName = "DeviceMonitorTopology"; 80 | public static void main(String[] args) { 81 | TopologyBuilder builder = new TopologyBuilder(); 82 | Constants constants = new Constants(); 83 | String hostClusterName = null; 84 | String deviceLogTable = "telecom_device_status_log"; 85 | if(args[0] != null){ 86 | hostClusterName = args[0]; 87 | topologyName = "DeviceMonitorTopology-"+hostClusterName; 88 | deviceLogTable = "telecom_device_status_log_"+hostClusterName; 89 | } 90 | // Use pipe as record boundary 91 | RecordFormat format = new DelimitedRecordFormat().withFieldDelimiter(","); 92 | 93 | //Synchronize data buffer with the filesystem every 1000 tuples 94 | SyncPolicy syncPolicy = new CountSyncPolicy(1000); 95 | 96 | // Rotate data files when they reach five MB 97 | FileRotationPolicy rotationPolicy = new FileSizeRotationPolicy(5.0f, Units.MB); 98 | 99 | // Use default, Storm-generated file names 100 | FileNameFormat deviceLogFileNameFormat = new DefaultFileNameFormat().withPath(constants.getHivePath()); 101 | HdfsBolt deviceLogHdfsBolt = new HdfsBolt() 102 | .withFsUrl(constants.getNameNodeUrl()) 103 | .withFileNameFormat(deviceLogFileNameFormat) 104 | .withRecordFormat(format) 105 | .withRotationPolicy(rotationPolicy) 106 | .withSyncPolicy(syncPolicy); 107 | System.out.println("********************** Starting Topology......."); 108 | System.out.println("********************** Name Node Url: " + constants.getNameNodeUrl()); 109 | System.out.println("********************** Zookeeper Host: " + constants.getZkHost()); 110 | System.out.println("********************** Zookeeper Port: " + constants.getZkPort()); 111 | System.out.println("********************** Zookeeper ConnString: " + constants.getZkConnString()); 112 | System.out.println("********************** Zookeeper Kafka Path: " + constants.getZkKafkaPath()); 113 | System.out.println("********************** Zookeeper HBase Path: " + constants.getZkHBasePath()); 114 | System.out.println("********************** Atlas Host: " + constants.getAtlasHost()); 115 | System.out.println("********************** Atlas Port: " + constants.getAtlasPort()); 116 | System.out.println("********************** Metastore URI: " + constants.getHiveMetaStoreURI()); 117 | System.out.println("********************** Cometd URI: " + constants.getPubSubUrl()); 118 | 119 | Config conf = new Config(); 120 | //BrokerHosts hosts = new ZkHosts(Constants.zkConnString); 121 | BrokerHosts hosts = new ZkHosts(constants.getZkConnString(), constants.getZkKafkaPath()); 122 | 123 | SpoutConfig deviceKafkaSpoutConfig = new SpoutConfig(hosts, constants.getDeviceEventsTopicName(), constants.getZkKafkaPath(), UUID.randomUUID().toString()); 124 | deviceKafkaSpoutConfig.scheme = new KeyValueSchemeAsMultiScheme(new DeviceEventJSONScheme()); 125 | deviceKafkaSpoutConfig.ignoreZkOffsets = true; 126 | deviceKafkaSpoutConfig.useStartOffsetTimeIfOffsetOutOfRange = true; 127 | deviceKafkaSpoutConfig.startOffsetTime = kafka.api.OffsetRequest.LatestTime(); 128 | KafkaSpout deviceKafkaSpout = new KafkaSpout(deviceKafkaSpoutConfig); 129 | 130 | SpoutConfig technicianKafkaSpoutConfig = new SpoutConfig(hosts, constants.getTechnicianEventsTopicName(), constants.getZkKafkaPath(), UUID.randomUUID().toString()); 131 | technicianKafkaSpoutConfig.scheme = new KeyValueSchemeAsMultiScheme(new TechnicianEventJSONScheme()); 132 | technicianKafkaSpoutConfig.ignoreZkOffsets = true; 133 | technicianKafkaSpoutConfig.useStartOffsetTimeIfOffsetOutOfRange = true; 134 | technicianKafkaSpoutConfig.startOffsetTime = kafka.api.OffsetRequest.LatestTime(); 135 | KafkaSpout technicianKafkaSpout = new KafkaSpout(technicianKafkaSpoutConfig); 136 | 137 | Map hbConf = new HashMap(); 138 | hbConf.put("hbase.rootdir", constants.getNameNodeUrl() + constants.getHbasePath()); 139 | hbConf.put("hbase.zookeeper.quorum", constants.getZkHost()); 140 | hbConf.put("hbase.zookeeper.property.clientPort", constants.getZkPort()); 141 | hbConf.put("zookeeper.znode.parent", constants.getZkHBasePath()); 142 | conf.put("hbase.conf", hbConf); 143 | conf.put("hbase.rootdir", constants.getNameNodeUrl() + constants.getHbasePath()); 144 | 145 | SimpleHBaseMapper deviceStatusLogMapper = new SimpleHBaseMapper() 146 | .withRowKeyField("serialNumber") 147 | .withColumnFields(new Fields("deviceModel", "status", "state", "internalTemp", "signalStrength")) 148 | .withColumnFamily("deviceStatus"); 149 | 150 | DelimitedRecordHiveMapper deviceStatusLogHiveMapper = new DelimitedRecordHiveMapper() 151 | .withColumnFields(new Fields("deviceModel", "status", "state", "internalTemp", "signalStrength")); 152 | //.withPartitionFields(new Fields("accounttype")); 153 | 154 | HiveOptions deviceStatusLogHiveOptions = new HiveOptions(constants.getHiveMetaStoreURI(), 155 | constants.getHiveDbName(), 156 | deviceLogTable, 157 | deviceStatusLogHiveMapper); 158 | 159 | //HBaseBolt hbasePersistTechnicianLocation = new HBaseBolt("TechnicianEvents", technicianLocationMapper).withConfigKey("hbase.conf"); 160 | //builder.setSpout("DeviceSpout", new DeviceSpout()); 161 | builder.setSpout("DeviceKafkaSpout", deviceKafkaSpout); 162 | builder.setBolt("EnrichDeviceStatus", new EnrichDeviceStatus(), 1).shuffleGrouping("DeviceKafkaSpout"); 163 | builder.setBolt("PublishDeviceStatus", new PublishDeviceStatus(), 1).shuffleGrouping("EnrichDeviceStatus"); 164 | builder.setBolt("DetectIncident", new IncidentDetector(), 1).shuffleGrouping("EnrichDeviceStatus"); 165 | builder.setBolt("PrintDeviceStatus", new PrintDeviceStatus(), 1).shuffleGrouping("DetectIncident", "DeviceStatusStream"); 166 | builder.setBolt("PrintDeviceAlert", new PrintDeviceAlert(), 1).shuffleGrouping("DetectIncident", "DeviceAlertStream"); 167 | builder.setBolt("DeviceLogHDFSBolt", deviceLogHdfsBolt, 1).shuffleGrouping("DetectIncident", "DeviceStatusLogStream"); 168 | builder.setBolt("DeviceLogHBaseBolt", new HBaseBolt("DeviceStatusLog", deviceStatusLogMapper).withConfigKey("hbase.conf"), 1).shuffleGrouping("DetectIncident", "DeviceStatusLogNullStream"); 169 | builder.setBolt("DeviceLogHiveBolt", new HiveBolt(deviceStatusLogHiveOptions),1).shuffleGrouping("DetectIncident", "DeviceStatusLogNullStream"); 170 | builder.setBolt("PublishAlert", new PublishAlert(), 1).shuffleGrouping("PrintDeviceAlert"); 171 | builder.setBolt("RecommendTechnician", new RecommendTechnician(), 1).shuffleGrouping("PrintDeviceAlert"); 172 | builder.setBolt("RouteTechnician", new RouteTechnician(), 1).shuffleGrouping("RecommendTechnician"); 173 | //builder.setBolt("persist", printerHdfsBolt).shuffleGrouping("print"); 174 | 175 | builder.setSpout("TechnicianKafkaSpout", technicianKafkaSpout); 176 | //builder.setBolt("PersistTechnicianLocation", hbasePersistTechnicianLocation).shuffleGrouping("TechnicianKafkaSpout"); 177 | builder.setBolt("PersistTechnicianLocation", new PersistTechnicianLocation(), 1).shuffleGrouping("TechnicianKafkaSpout"); 178 | builder.setBolt("PublishTechnicianLocation", new PublishTechnicianLocation(), 1).shuffleGrouping("PersistTechnicianLocation"); 179 | 180 | conf.setNumWorkers(1); 181 | conf.setMaxSpoutPending(5000); 182 | conf.setMaxTaskParallelism(1); 183 | 184 | //submitToLocal(builder, conf); 185 | submitToCluster(builder, conf); 186 | } 187 | 188 | public static void submitToLocal(TopologyBuilder builder, Config conf){ 189 | LocalCluster cluster = new LocalCluster(); 190 | cluster.submitTopology(topologyName, conf, builder.createTopology()); 191 | } 192 | 193 | public static void submitToCluster(TopologyBuilder builder, Config conf){ 194 | try { 195 | StormSubmitter.submitTopology(topologyName, conf, builder.createTopology()); 196 | } catch (AlreadyAliveException e) { 197 | e.printStackTrace(); 198 | } catch (InvalidTopologyException e) { 199 | e.printStackTrace(); 200 | } catch (AuthorizationException e) { 201 | e.printStackTrace(); 202 | } 203 | } 204 | } 205 | -------------------------------------------------------------------------------- /DeviceMonitor/src/main/java/com/hortonworks/bolts/RecommendTechnician.java: -------------------------------------------------------------------------------- 1 | package com.hortonworks.bolts; 2 | 3 | import java.io.IOException; 4 | import java.util.Map; 5 | import java.util.NavigableSet; 6 | 7 | import org.apache.hadoop.conf.Configuration; 8 | import org.apache.hadoop.hbase.HBaseConfiguration; 9 | import org.apache.hadoop.hbase.KeyValue; 10 | import org.apache.hadoop.hbase.MasterNotRunningException; 11 | import org.apache.hadoop.hbase.ZooKeeperConnectionException; 12 | import org.apache.hadoop.hbase.client.Get; 13 | import org.apache.hadoop.hbase.client.HBaseAdmin; 14 | import org.apache.hadoop.hbase.client.HTable; 15 | import org.apache.hadoop.hbase.client.Result; 16 | import org.apache.hadoop.hbase.client.ResultScanner; 17 | import org.apache.hadoop.hbase.client.Scan; 18 | import org.apache.hadoop.hbase.util.Bytes; 19 | 20 | import com.hortonworks.events.DeviceAlert; 21 | import com.hortonworks.events.DeviceStatus; 22 | import com.hortonworks.events.TechnicianDestination; 23 | import com.hortonworks.events.TechnicianStatus; 24 | import com.hortonworks.util.Constants; 25 | 26 | import org.apache.storm.task.OutputCollector; 27 | import org.apache.storm.task.TopologyContext; 28 | import org.apache.storm.topology.OutputFieldsDeclarer; 29 | import org.apache.storm.topology.base.BaseRichBolt; 30 | import org.apache.storm.tuple.Fields; 31 | import org.apache.storm.tuple.Tuple; 32 | import org.apache.storm.tuple.Values; 33 | 34 | /* 35 | import backtype.storm.task.OutputCollector; 36 | import backtype.storm.task.TopologyContext; 37 | import backtype.storm.topology.OutputFieldsDeclarer; 38 | import backtype.storm.topology.base.BaseRichBolt; 39 | import backtype.storm.tuple.Fields; 40 | import backtype.storm.tuple.Tuple; 41 | import backtype.storm.tuple.Values; 42 | */ 43 | 44 | public class RecommendTechnician extends BaseRichBolt { 45 | private static final long serialVersionUID = 1L; 46 | private OutputCollector collector; 47 | private Constants constants; 48 | private HTable table = null; 49 | 50 | public RecommendTechnician() {} 51 | 52 | public static Double distFrom(Double lat1, Double lng1, Double lat2, Double lng2) { 53 | double earthRadius = 6371000; //meters 54 | double dLat = Math.toRadians(lat2-lat1); 55 | double dLng = Math.toRadians(lng2-lng1); 56 | double a = Math.sin(dLat/2) * Math.sin(dLat/2) + 57 | Math.cos(Math.toRadians(lat1)) * Math.cos(Math.toRadians(lat2)) * 58 | Math.sin(dLng/2) * Math.sin(dLng/2); 59 | double c = 2 * Math.atan2(Math.sqrt(a), Math.sqrt(1-a)); 60 | double dist = (Double)(earthRadius * c); 61 | 62 | return dist; 63 | } 64 | 65 | @SuppressWarnings("deprecation") 66 | public TechnicianDestination nominateTechnician (DeviceAlert deviceAlert) throws IOException { 67 | TechnicianDestination techDestination = new TechnicianDestination(); 68 | TechnicianStatus currentTechStatus = new TechnicianStatus(); 69 | TechnicianStatus recommendedTechStatus = new TechnicianStatus(); 70 | ResultScanner techLocationScanner; 71 | 72 | Scan scan = new Scan(); 73 | scan.addColumn(Bytes.toBytes("cf"),Bytes.toBytes("Status")); 74 | scan.addColumn(Bytes.toBytes("cf"),Bytes.toBytes("Latitude")); 75 | scan.addColumn(Bytes.toBytes("cf"),Bytes.toBytes("Longitude")); 76 | scan.addColumn(Bytes.toBytes("cf"),Bytes.toBytes("IpAddress")); 77 | scan.addColumn(Bytes.toBytes("cf"),Bytes.toBytes("Port")); 78 | 79 | ResultScanner scanner = table.getScanner(scan); 80 | techLocationScanner = scanner; 81 | // Scanning the required columns 82 | /*for (Result result = scanner.next(); (result != null); result = scanner.next()) { 83 | System.out.println("Row Key: " + Bytes.toString(result.getRow())); 84 | for(KeyValue keyValue : result.list()) { 85 | System.out.println("Qualifier : " + Bytes.toString(keyValue.getQualifier()) + " : Value : " + Bytes.toString(keyValue.getValue())); 86 | } 87 | }*/ 88 | 89 | Double currentDistance = null; 90 | Double leastDistance = null; 91 | System.out.println("********************** DEVICE ALERT: " + deviceAlert.getLatitude() + " , " + deviceAlert.getLongitude()); 92 | System.out.println("Recommended Tech: " + recommendedTechStatus.getTechnicianId()); 93 | System.out.println("Recommended Tech Destination: " + techDestination.getTechnicianId()); 94 | 95 | for(Result result = techLocationScanner.next(); (result != null); result = techLocationScanner.next()) { 96 | currentTechStatus.setTechnicianId(Bytes.toString(result.getRow())); 97 | System.out.println(currentTechStatus.getTechnicianId()); 98 | System.out.println("Start of Loop Recommended Tech: " + recommendedTechStatus.getTechnicianId()); 99 | System.out.println("Start of Loop Recommended Tech Destination: " + techDestination.getTechnicianId()); 100 | 101 | for(KeyValue keyValue : result.list()) { 102 | System.out.println("Qualifier : " + Bytes.toString(keyValue.getQualifier()) + " : Value : " + Bytes.toString(keyValue.getValue())); 103 | if(Bytes.toString(keyValue.getQualifier()).equalsIgnoreCase("Status")){ 104 | currentTechStatus.setStatus(Bytes.toString(keyValue.getValue())); 105 | } else if(Bytes.toString(keyValue.getQualifier()).equalsIgnoreCase("Latitude")){ 106 | currentTechStatus.setLatitude(Double.parseDouble(Bytes.toString(keyValue.getValue()))); 107 | } else if(Bytes.toString(keyValue.getQualifier()).equalsIgnoreCase("Longitude")){ 108 | currentTechStatus.setLongitude(Double.parseDouble(Bytes.toString(keyValue.getValue()))); 109 | } else if(Bytes.toString(keyValue.getQualifier()).equalsIgnoreCase("IpAddress")){ 110 | currentTechStatus.setIpAddress(Bytes.toString(keyValue.getValue())); 111 | } else if(Bytes.toString(keyValue.getQualifier()).equalsIgnoreCase("Port")){ 112 | currentTechStatus.setPort(Bytes.toString(keyValue.getValue())); 113 | } 114 | } 115 | System.out.println("Post Lookup Loop Recommended Tech: " + recommendedTechStatus.getTechnicianId()); 116 | System.out.println("Post Lookup Loop Recommended Tech Destination: " + techDestination.getTechnicianId()); 117 | System.out.println("Current Tech Id: " + currentTechStatus.getTechnicianId()); 118 | System.out.println("Current Tech Status: " + currentTechStatus.getStatus()); 119 | System.out.println("Current Tech Latitude: " + currentTechStatus.getLatitude()); 120 | System.out.println("Current Tech Longitude: " + currentTechStatus.getLongitude()); 121 | System.out.println("Current IpAddress: " + currentTechStatus.getIpAddress()); 122 | System.out.println("Current Port: " + currentTechStatus.getPort()); 123 | 124 | System.out.println("Inside Loop Recommended Tech: " + recommendedTechStatus.getTechnicianId()); 125 | System.out.println("Inside Loop Recommended Tech Destination: " + techDestination.getTechnicianId()); 126 | 127 | currentDistance = distFrom(currentTechStatus.getLatitude(), currentTechStatus.getLongitude(), deviceAlert.getLatitude(), deviceAlert.getLongitude()); 128 | if(currentTechStatus.getStatus().equalsIgnoreCase("Available")){ 129 | System.out.println("Current Tech is available for repair"); 130 | if(currentDistance == null || leastDistance == null){ 131 | System.out.println("Current Tech: " + currentTechStatus.getTechnicianId() + " Current Distance: " + currentDistance + " Least Distance:" + leastDistance); 132 | System.out.println("Setting current tech as recommended tech since least distance is null"); 133 | recommendedTechStatus = copyTechnicianStatus(currentTechStatus); 134 | leastDistance = currentDistance; 135 | } 136 | else if(currentDistance < leastDistance){ 137 | System.out.println("Current Tech: " + currentTechStatus.getTechnicianId() + " Current Distance: " + currentDistance + " Least Distance:" + leastDistance); 138 | System.out.println("Setting current tech as recommended tech since they are currently closest to incident"); 139 | recommendedTechStatus = copyTechnicianStatus(currentTechStatus); 140 | leastDistance = currentDistance; 141 | } 142 | else{ 143 | System.out.println("Current Tech: " + currentTechStatus.getTechnicianId() + " Current Distance: " + currentDistance + " Least Distance:" + leastDistance); 144 | System.out.println("Current tech is not recommended as some other tech is closer to the incident"); 145 | } 146 | } 147 | else{ 148 | System.out.println("Current Tech: " + currentTechStatus.getTechnicianId() + " Current Distance: " + currentDistance + " Least Distance:" + leastDistance); 149 | System.out.println("Technician " + currentTechStatus.getTechnicianId() + " is already assigned to a repair"); 150 | } 151 | System.out.println("End of Inside Loop Recommended Tech: " + recommendedTechStatus.getTechnicianId()); 152 | System.out.println("End of Inside Loop Recommended Tech Destination: " + techDestination.getTechnicianId()); 153 | } 154 | 155 | System.out.println("Technician " + recommendedTechStatus.getTechnicianId() + " is recommended for this repair"); 156 | 157 | techDestination.setTechnicianId(recommendedTechStatus.getTechnicianId()); 158 | techDestination.setLatitude(recommendedTechStatus.getLatitude()); 159 | techDestination.setLongitude(recommendedTechStatus.getLongitude()); 160 | techDestination.setDestinationLatitude(deviceAlert.getLatitude()); 161 | techDestination.setDestinationLongitude(deviceAlert.getLongitude()); 162 | techDestination.setStatus(recommendedTechStatus.getStatus()); 163 | techDestination.setIpAddress(recommendedTechStatus.getIpAddress()); 164 | techDestination.setPort(recommendedTechStatus.getPort()); 165 | return techDestination; 166 | } 167 | 168 | public TechnicianStatus copyTechnicianStatus(TechnicianStatus sourceTechStatus){ 169 | TechnicianStatus targetTechStatus = new TechnicianStatus(); 170 | targetTechStatus.setTechnicianId(sourceTechStatus.getTechnicianId()); 171 | targetTechStatus.setIpAddress(sourceTechStatus.getIpAddress()); 172 | targetTechStatus.setPort(sourceTechStatus.getPort()); 173 | targetTechStatus.setLatitude(sourceTechStatus.getLatitude()); 174 | targetTechStatus.setLongitude(sourceTechStatus.getLongitude()); 175 | targetTechStatus.setStatus(sourceTechStatus.getStatus()); 176 | return targetTechStatus ; 177 | } 178 | 179 | public void execute(Tuple tuple) { 180 | DeviceAlert deviceAlert = (DeviceAlert) tuple.getValueByField("DeviceAlert"); 181 | 182 | try{ 183 | TechnicianDestination techDestination = nominateTechnician(deviceAlert); 184 | if(techDestination.getTechnicianId() != null && techDestination.getStatus().equalsIgnoreCase("Available")){ 185 | System.out.println("Emiting Tech Route Request: " + techDestination.getTechnicianId() + " : " + techDestination.toString()); 186 | collector.emit(tuple, new Values(techDestination)); 187 | } 188 | else{ 189 | System.out.println("Recommended Technician is Null or all Technicians are already assigned"); 190 | System.out.println("Need additional Techs in the field....."); 191 | } 192 | collector.ack(tuple); 193 | } 194 | catch(IOException e){ 195 | e.printStackTrace(); 196 | } 197 | } 198 | 199 | public void prepare(Map arg0, TopologyContext arg1, OutputCollector collector) { 200 | this.collector = collector; 201 | this.constants = new Constants(); 202 | Configuration config = HBaseConfiguration.create(); 203 | config.set("hbase.zookeeper.quorum", constants.getZkHost()); 204 | config.set("hbase.zookeeper.property.clientPort", constants.getZkPort()); 205 | config.set("zookeeper.znode.parent", constants.getZkHBasePath());; 206 | 207 | String tableName = "TechnicianEvents"; 208 | HBaseAdmin hbaseAdmin; 209 | try { 210 | hbaseAdmin = new HBaseAdmin(config); 211 | while(!hbaseAdmin.tableExists(tableName)){ 212 | Thread.sleep(1000); 213 | System.out.println("******************** DeviceMonitor prepare() Waiting for HBase Tables to be prepared..."); 214 | } 215 | table = new HTable(config, tableName); 216 | System.out.println("******************** Acquired HBase Table " + tableName); 217 | }catch (MasterNotRunningException e) { 218 | e.printStackTrace(); 219 | } catch (ZooKeeperConnectionException e) { 220 | e.printStackTrace(); 221 | } catch (IOException e) { 222 | e.printStackTrace(); 223 | } catch (InterruptedException e) { 224 | e.printStackTrace(); 225 | } 226 | } 227 | 228 | public void declareOutputFields(OutputFieldsDeclarer declarer) { 229 | declarer.declare(new Fields("TechnicianDestination")); 230 | } 231 | } --------------------------------------------------------------------------------