├── .npmrc ├── veriflier ├── config │ ├── .gitignore │ └── veriflier-sample.json ├── certs │ ├── you_need_to_create_certs │ └── .gitignore ├── .gitignore ├── logs │ └── .gitignore ├── headers │ ├── config.h │ ├── logger.h │ ├── ssl_server.h │ ├── jetmon_server.h │ ├── check_thread.h │ ├── http_checker.h │ ├── client_thread.h │ └── check_controller.h ├── source │ ├── main.cpp │ ├── check_thread.cpp │ ├── config.cpp │ ├── logger.cpp │ ├── jetmon_server.cpp │ ├── ssl_server.cpp │ ├── http_checker.cpp │ ├── check_controller.cpp │ └── client_thread.cpp ├── README.md ├── veriflier.pro ├── veriflier.sh └── LICENSE ├── config ├── .gitignore ├── db-config-sample.conf ├── config-sample.json └── config.readme ├── certs ├── you_need_to_create_certs └── .gitignore ├── docker ├── .dockerignore ├── volumes │ └── statsd │ │ ├── graphite │ │ └── .gitignore │ │ ├── logs │ │ └── .gitignore │ │ └── statsd │ │ └── .gitignore ├── Dockerfile_jetmon ├── .env-sample ├── run-veriflier.sh ├── Dockerfile_veriflier ├── run-jetmon.sh └── docker-compose.yml ├── .gitignore ├── logs └── .gitignore ├── stats └── .gitignore ├── package.json ├── lib ├── config.js ├── wpcom.js ├── statsd.js ├── comms.js ├── dbpools.js ├── server.js ├── database.js ├── httpcheck.js └── jetmon.js ├── binding.gyp ├── src ├── http_checker.h ├── main.cpp └── http_checker.cpp ├── README.md └── LICENSE /.npmrc: -------------------------------------------------------------------------------- 1 | unsafe-perm=true 2 | -------------------------------------------------------------------------------- /veriflier/config/.gitignore: -------------------------------------------------------------------------------- 1 | veriflier.json 2 | 3 | -------------------------------------------------------------------------------- /config/.gitignore: -------------------------------------------------------------------------------- 1 | config.json 2 | db-config.conf 3 | 4 | -------------------------------------------------------------------------------- /certs/you_need_to_create_certs: -------------------------------------------------------------------------------- 1 | create certs to use for SSL 2 | -------------------------------------------------------------------------------- /veriflier/certs/you_need_to_create_certs: -------------------------------------------------------------------------------- 1 | create certs to use for SSL 2 | -------------------------------------------------------------------------------- /docker/.dockerignore: -------------------------------------------------------------------------------- 1 | 2 | build/ 3 | node_modules/ 4 | .DS_Store 5 | .env 6 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | build/ 2 | node_modules/ 3 | .DS_Store 4 | lib/jetmon.node 5 | .env 6 | .idea 7 | -------------------------------------------------------------------------------- /veriflier/.gitignore: -------------------------------------------------------------------------------- 1 | *.o 2 | *.cpp 3 | *.h 4 | .qmake.stash 5 | Makefile 6 | veriflier 7 | -------------------------------------------------------------------------------- /logs/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file 4 | !.gitignore 5 | -------------------------------------------------------------------------------- /stats/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file 4 | !.gitignore 5 | -------------------------------------------------------------------------------- /veriflier/logs/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file 4 | !.gitignore 5 | -------------------------------------------------------------------------------- /certs/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Exceptions 4 | !you_need_to_create_certs 5 | !.gitignore 6 | -------------------------------------------------------------------------------- /docker/volumes/statsd/graphite/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file 4 | !.gitignore 5 | -------------------------------------------------------------------------------- /docker/volumes/statsd/logs/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file 4 | !.gitignore 5 | -------------------------------------------------------------------------------- /docker/volumes/statsd/statsd/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Except this file 4 | !.gitignore 5 | -------------------------------------------------------------------------------- /veriflier/certs/.gitignore: -------------------------------------------------------------------------------- 1 | # Ignore everything in this directory 2 | * 3 | # Exceptions 4 | !you_need_to_create_certs 5 | !.gitignore 6 | -------------------------------------------------------------------------------- /docker/Dockerfile_jetmon: -------------------------------------------------------------------------------- 1 | FROM node:16 2 | 3 | WORKDIR /jetmon 4 | 5 | # RUN apk add --no-cache python3 make g++ 6 | 7 | RUN npm install -g node-gyp 8 | 9 | # Get the dependencies loaded first - this makes rebuilds faster 10 | COPY package.json . 11 | RUN npm install 12 | 13 | COPY . . 14 | 15 | CMD [ "bash", "docker/run-jetmon.sh" ] 16 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "jetmon", 3 | "version": "0.0.1", 4 | "description": "Parallel HTTP health monitoring using HEAD requests for large scale website monitoring.", 5 | "scripts": { 6 | "rebuild-run": "node-gyp rebuild && cp build/Release/jetmon.node lib && node lib/jetmon.js" 7 | }, 8 | "dependencies": { 9 | "log4js": "0.6.38", 10 | "mysql": "2.11.1" 11 | } 12 | } 13 | -------------------------------------------------------------------------------- /lib/config.js: -------------------------------------------------------------------------------- 1 | 2 | const CONFIG_FILE = 'config/config.json'; 3 | 4 | var fs = require( 'fs' ); 5 | 6 | var config = { 7 | _cache: null, 8 | load: function( s_file_name ) { 9 | this._cache = JSON.parse( fs.readFileSync( CONFIG_FILE ).toString() ); 10 | return this._cache; 11 | }, 12 | get: function( key ) { 13 | return this._cache[ key ]; 14 | } 15 | }; 16 | 17 | module.exports = config; 18 | -------------------------------------------------------------------------------- /config/db-config-sample.conf: -------------------------------------------------------------------------------- 1 | $db_servers = array( 2 | 'misc' => array( 3 | array( 'jetmon', 0, 1, 'mysqldb:', 'mysqldb:', '', '', '', null, null, 30 ), 4 | array( 'jetmon', 1, 0, 'mysqldb:', 'mysqldb:', '', '', '', null, null, 30 ), 5 | ) 6 | ); 7 | -------------------------------------------------------------------------------- /docker/.env-sample: -------------------------------------------------------------------------------- 1 | MYSQLDB_USER=root 2 | MYSQLDB_ROOT_PASSWORD=123456 3 | MYSQLDB_DATABASE=jetmon_db 4 | MYSQLDB_LOCAL_PORT=3307 5 | MYSQLDB_DOCKER_PORT=3306 6 | 7 | JETMON_LOCAL_PORT=7800 8 | JETMON_DOCKER_PORT=7800 9 | 10 | JETMON_STATUS_LOCAL_PORT=7802 11 | JETMON_STATUS_DOCKER_PORT=7802 12 | 13 | WPCOM_JETMON_AUTH_TOKEN=change_me 14 | 15 | VERIFLIER_LOCAL_PORT=7801 16 | VERIFLIER_DOCKER_PORT=7801 17 | VERIFLIER_AUTH_TOKEN=veriflier_1_auth_token 18 | -------------------------------------------------------------------------------- /binding.gyp: -------------------------------------------------------------------------------- 1 | { 2 | 'targets':[ { 3 | 'target_name':'jetmon', 4 | 'cflags_cc': [ '-fexceptions','-O3' ], 5 | 'sources':[ 6 | './src/main.cpp', 7 | './src/http_checker.cpp', 8 | ], 9 | 'conditions': [ 10 | ['node_shared_openssl=="false"', { 11 | 'include_dirs': [ 12 | '<(node_root_dir)/deps/openssl/openssl/include' 13 | ], 14 | }] 15 | ] 16 | } ] 17 | } 18 | -------------------------------------------------------------------------------- /veriflier/headers/config.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef __CONFIG_H__ 3 | #define __CONFIG_H__ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | 12 | class Config { 13 | public: 14 | static Config* instance() { return m_instance; } 15 | 16 | int get_int_value( QString name ); 17 | bool get_bool_value( QString name ); 18 | QString get_string_value( QString name ); 19 | 20 | private: 21 | Config(); 22 | static Config *m_instance; 23 | QJsonDocument m_json; 24 | 25 | void load_config_file(); 26 | }; 27 | 28 | #endif // __CONFIG_H__ 29 | 30 | -------------------------------------------------------------------------------- /docker/run-veriflier.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | cd /opt/veriflier 4 | 5 | qmake 6 | make 7 | 8 | mkdir -p certs 9 | if [ ! -f certs/veriflier.key ] && [ ! -f certs/veriflier.crt ]; then 10 | openssl req -newkey rsa:2048 -nodes -keyout certs/veriflier.key -x509 -days 365 -out certs/veriflier.crt -subj "/C=US/ST=California/L=San Francisco/O=Automattic Inc./CN=jetmon" 11 | fi 12 | 13 | if [ ! -f config/veriflier.json ]; then 14 | sed -e "s//${JETMON_PORT}/g" -e "s//${VERIFLIER_PORT}/g" -e "s//${VERIFLIER_AUTH_TOKEN}/g" config/veriflier-sample.json > config/veriflier.json 15 | fi 16 | 17 | exec ./veriflier start 18 | -------------------------------------------------------------------------------- /veriflier/source/main.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | 4 | #include "headers/config.h" 5 | #include "headers/logger.h" 6 | #include "headers/ssl_server.h" 7 | 8 | #include 9 | 10 | int main( int argc, char *argv[] ) 11 | { 12 | QCoreApplication app(argc, argv); 13 | Logger::instance()->startLogger(); 14 | 15 | SSL_Server *ssl = new SSL_Server(); 16 | bool result = ssl->listen( QHostAddress::Any, Config::instance()->get_int_value( "listen_port" ) ); 17 | 18 | if ( ! result ) { 19 | LOG( "failed to open the server port, eXiting." ); 20 | Logger::instance()->stopLogging(); 21 | return -1; 22 | } 23 | 24 | return app.exec(); 25 | } 26 | -------------------------------------------------------------------------------- /veriflier/README.md: -------------------------------------------------------------------------------- 1 | veriflier service 2 | ================= 3 | 4 | Overview 5 | -------- 6 | 7 | The veriflier services check whether sites are reachable from their location, using the same HEAD request as jetmon. This allows deployment to be done in geographically disparate datacenters, providing a true global status of the site being verified. 8 | 9 | Building 10 | -------- 11 | 12 | 1) Ensure you have a Qt5 build environment installed. 13 | 14 | 2) Run the Qt5 'qmake' executable in the veriflier directory. 15 | 16 | 3) Run 'make'. 17 | 18 | Running 19 | ------- 20 | 21 | 1) Modify the install path if necessary in 'veriflier.sh'. 22 | 23 | 2) Run './veriflier start|stop|restart|reload' 24 | 25 | -------------------------------------------------------------------------------- /veriflier/veriflier.pro: -------------------------------------------------------------------------------- 1 | QT += core network 2 | QT -= gui 3 | 4 | TARGET = veriflier 5 | CONFIG += console 6 | CONFIG -= app_bundle 7 | 8 | TEMPLATE = app 9 | 10 | SOURCES += \ 11 | source/client_thread.cpp \ 12 | source/main.cpp \ 13 | source/ssl_server.cpp \ 14 | source/http_checker.cpp \ 15 | source/config.cpp \ 16 | source/logger.cpp \ 17 | source/check_thread.cpp \ 18 | source/check_controller.cpp \ 19 | source/jetmon_server.cpp 20 | 21 | HEADERS += \ 22 | headers/client_thread.h \ 23 | headers/http_checker.h \ 24 | headers/ssl_server.h \ 25 | headers/config.h \ 26 | headers/logger.h \ 27 | headers/check_thread.h \ 28 | headers/check_controller.h \ 29 | headers/jetmon_server.h 30 | -------------------------------------------------------------------------------- /veriflier/config/veriflier-sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "veriflier_name" : "Veriflier 1", 3 | "auth_token" : "", 4 | 5 | "thread_pool_max" : 4, 6 | "max_pending_conns" : 256, 7 | "max_redirects" : 2, 8 | "max_checkers" : 10, 9 | "max_checks" : 1000, 10 | 11 | "net_comms_timeout" : 30000, 12 | "debug" : true, 13 | 14 | "listen_port" : , 15 | "jetmon_server_port" : , 16 | 17 | "privatekey_file" : "./certs/veriflier.key", 18 | "privatecert_file" : "./certs/veriflier.crt", 19 | 20 | "monitors" : [ 21 | { 22 | "name" : "", 23 | "host" : "" 25 | } 26 | ] 27 | } 28 | -------------------------------------------------------------------------------- /docker/Dockerfile_veriflier: -------------------------------------------------------------------------------- 1 | FROM ubuntu:18.04 2 | # Sourced from https://github.com/MatiMoreyra/qt5-docker/blob/master/Dockerfile 3 | 4 | # Install dependencies. 5 | RUN apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y \ 6 | git \ 7 | build-essential \ 8 | cmake \ 9 | qt5-default \ 10 | libfontconfig1 \ 11 | mesa-common-dev \ 12 | libglu1-mesa-dev \ 13 | libgtest-dev 14 | 15 | # Build GTest 16 | RUN cd /usr/src/gtest/ && \ 17 | cmake -DBUILD_SHARED_LIBS=ON && \ 18 | make && \ 19 | cp -a include/gtest /usr/include && \ 20 | cp -a libgtest_main.so libgtest.so /usr/lib/ 21 | 22 | # Cleanup 23 | RUN rm -rf /var/lib/apt/lists/* 24 | 25 | 26 | WORKDIR /opt 27 | 28 | COPY . . 29 | 30 | CMD [ "bash", "docker/run-veriflier.sh" ] 31 | -------------------------------------------------------------------------------- /veriflier/headers/logger.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef __LOGGER_H__ 3 | #define __LOGGER_H__ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | 12 | #define MAX_LOG_FILESIZE 1024 * 1024 * 50 // 50 MB 13 | #define LOGS_TO_KEEP 20 14 | 15 | const QString LOG_FILE_NAME = QDir::currentPath() + "/logs/veriflier.log"; 16 | #define LOG( content ) Logger::write( content ) 17 | 18 | class Logger { 19 | public: 20 | static Logger* instance() { return m_instance; } 21 | 22 | static void startLogger(); 23 | static void stopLogging(); 24 | static void write( QString s_data ); 25 | 26 | private: 27 | Logger() {} 28 | static Logger *m_instance; 29 | static QFile *m_file; 30 | static QMutex *m_mutex; 31 | 32 | static void do_log_rotation(); 33 | }; 34 | 35 | #endif // __CONFIG_H__ 36 | 37 | -------------------------------------------------------------------------------- /docker/run-jetmon.sh: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | cd /jetmon 3 | 4 | mkdir -p logs 5 | touch logs/jetmon.log logs/status-change.log 6 | 7 | mkdir -p stats 8 | touch stats/sitespersec stats/sitesqueue stats/totals 9 | 10 | mkdir -p certs 11 | if [ ! -f certs/jetmon.key ] && [ ! -f certs/jetmon.crt ]; then 12 | openssl req -newkey rsa:2048 -nodes -keyout certs/jetmon.key -x509 -days 365 -out certs/jetmon.crt -subj "/C=US/ST=California/L=San Francisco/O=Automattic Inc./CN=jetmon" 13 | fi 14 | 15 | if [ ! -f config/config.json ]; then 16 | sed -e "s//${WPCOM_JETMON_AUTH_TOKEN}/g" -e "s//${VERIFLIER_PORT}/g" -e "s//${VERIFLIER_AUTH_TOKEN}/g" config/config-sample.json > config/config.json 17 | fi 18 | if [ ! -f config/db-config.conf ]; then 19 | sed -e "s//${MYSQLDB_USER}/g" -e "s//${MYSQLDB_ROOT_PASSWORD}/g" -e "s//${MYSQLDB_DOCKER_PORT}/g" -e "s//${MYSQLDB_DATABASE}/g" config/db-config-sample.conf > config/db-config.conf 20 | fi 21 | 22 | exec npm run rebuild-run 23 | -------------------------------------------------------------------------------- /veriflier/headers/ssl_server.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef __SSL_SERVER_H__ 3 | #define __SSL_SERVER_H__ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | 11 | #include "headers/config.h" 12 | #include "headers/client_thread.h" 13 | #include "headers/check_controller.h" 14 | 15 | #define DEFAULT_MAX_CHECKERS 4 16 | #define DEFAULT_MAX_CHECKS 500 17 | 18 | class SSL_Server : public QTcpServer 19 | { 20 | Q_OBJECT 21 | public: 22 | SSL_Server( QObject *parent = 0 ); 23 | ~SSL_Server(); 24 | 25 | protected: 26 | void incomingConnection( qintptr socketDescriptor ); 27 | 28 | public slots: 29 | void logError( QAbstractSocket::SocketError socketError ); 30 | 31 | private: 32 | QThreadPool *pool; 33 | CheckController *m_checker; 34 | QSslConfiguration *m_ssl_config; 35 | 36 | QDateTime ticker; 37 | QString m_veriflier_name; 38 | QString m_auth_token; 39 | int m_net_timeout; 40 | int m_served_count; 41 | int m_jetmon_server_port; 42 | bool m_debug; 43 | }; 44 | 45 | #endif // __SSL_SERVER_H__ 46 | 47 | -------------------------------------------------------------------------------- /veriflier/headers/jetmon_server.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef __JETMON_SERVER_H__ 3 | #define __JETMON_SERVER_H__ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | #include "headers/logger.h" 11 | 12 | class JetmonServer : public QObject 13 | { 14 | Q_OBJECT 15 | public: 16 | JetmonServer( QObject *parent, const QSslConfiguration *ssl_config, QString jetmon_server, int jetmon_server_port ); 17 | 18 | void sendData( QByteArray status_data ); 19 | QString jetmonServer() { return m_jetmon_server; } 20 | 21 | signals: 22 | void finished( JetmonServer* jetmon_server, int status, int rtt ); 23 | 24 | private slots: 25 | void connected(); 26 | void connectionError( QAbstractSocket::SocketError err ); 27 | void readyRead(); 28 | 29 | private: 30 | QSslSocket *m_socket; 31 | QString m_jetmon_server; 32 | int m_jetmon_server_port; 33 | QDateTime m_timer; 34 | QByteArray m_status_data; 35 | 36 | QJsonDocument parse_json_response( QByteArray &raw_data ); 37 | void closeConnection(); 38 | }; 39 | 40 | #endif // __JETMON_SERVER_H__ 41 | -------------------------------------------------------------------------------- /config/config-sample.json: -------------------------------------------------------------------------------- 1 | { 2 | "DEBUG" : true, 3 | "NUM_WORKERS" : 60, 4 | "NUM_TO_PROCESS" : 40, 5 | "DATASET_SIZE" : 100, 6 | "WORKER_MAX_CHECKS" : 10000, 7 | "WORKER_MAX_MEM_MB" : 53, 8 | 9 | "DB_UPDATES_ENABLE" : false, 10 | 11 | "BUCKET_NO_MIN" : 0, 12 | "BUCKET_NO_MAX" : 512, 13 | "BATCH_SIZE" : 32, 14 | "AUTH_TOKEN" : "", 15 | 16 | "VERIFLIER_BATCH_SIZE" : 200, 17 | "SQL_UPDATE_BATCH" : 1, 18 | "DB_CONFIG_UPDATES_MIN" : 10, 19 | "PEER_OFFLINE_LIMIT" : 3, 20 | 21 | "NUM_OF_CHECKS" : 3, 22 | "TIME_BETWEEN_CHECKS_SEC" : 30, 23 | 24 | "STATS_UPDATE_INTERVAL_MS" : 10000, 25 | "STATSD_SEND_MEM_USAGE" : false, 26 | "TIME_BETWEEN_NOTICES_MIN" : 59, 27 | "MIN_TIME_BETWEEN_ROUNDS_SEC" : 300, 28 | "TIMEOUT_FOR_REQUESTS_SEC" : 60, 29 | "USE_VARIABLE_CHECK_INTERVALS" : false, 30 | 31 | "VERIFIERS": [ 32 | { 33 | "name" : "Veriflier 1", 34 | "host" : "veriflier", 35 | "port" : "", 36 | "auth_token" : "" 37 | } 38 | ] 39 | } 40 | -------------------------------------------------------------------------------- /veriflier/headers/check_thread.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef __CHECKTHREAD_H__ 3 | #define __CHECKTHREAD_H__ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | #include "headers/config.h" 15 | #include "headers/http_checker.h" 16 | 17 | #define HOST_DOWN 0 18 | #define HOST_ONLINE 1 19 | 20 | class CheckThread : public QThread 21 | { 22 | Q_OBJECT 23 | public: 24 | CheckThread( const int net_timeout, const bool debug, 25 | const int thread_index ); 26 | 27 | void performCheck( HealthCheck* hc ); 28 | 29 | protected: 30 | void run(); 31 | 32 | signals: 33 | void resultReady( int thread_index, qint64 blog_id, QString monitor_url, int status, int http_code, int rtt ); 34 | 35 | public slots: 36 | void finishedCheck( HTTP_Checker *checker, HealthCheck* hc ); 37 | 38 | private: 39 | QVector m_checkers; 40 | 41 | int m_net_timeout; 42 | int m_thread_index; 43 | 44 | QDateTime m_timer; 45 | bool m_debug; 46 | 47 | void performHostCheck(); 48 | }; 49 | 50 | #endif // __CHECKTHREAD_H__ 51 | -------------------------------------------------------------------------------- /veriflier/source/check_thread.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include "headers/check_thread.h" 3 | #include "headers/logger.h" 4 | 5 | CheckThread::CheckThread( const int net_timeout, const bool debug, const int thread_index ) 6 | : QThread( 0 ), m_net_timeout( net_timeout ), m_thread_index( thread_index ), m_debug( debug ) 7 | { 8 | ; 9 | } 10 | 11 | void CheckThread::run() { 12 | LOG( "checker #" + QString::number( m_thread_index ) + " running" ); 13 | exec(); 14 | } 15 | 16 | void CheckThread::finishedCheck( HTTP_Checker *checker, HealthCheck *hc ) { 17 | int status = -1; 18 | if ( checker->get_rtt() > 0 && 0 < checker->get_response_code() && 400 > checker->get_response_code() ) 19 | status = HOST_ONLINE; 20 | else 21 | status = HOST_DOWN; 22 | 23 | if ( m_debug ) { 24 | LOG( QString::number( hc->received.msecsTo( QDateTime::currentDateTime() ) ) + 25 | QString( "\t: STAGE 2 :\t" ) + hc->monitor_url + 26 | QString( " status :" ) + QString::number( status ) ); 27 | } 28 | 29 | emit resultReady( hc->thread_index, hc->blog_id, hc->monitor_url, status, checker->get_response_code(), checker->get_rtt() ); 30 | checker->deleteLater(); 31 | } 32 | 33 | void CheckThread::performCheck( HealthCheck *hc ) { 34 | if ( m_debug ) { 35 | LOG( QString::number( hc->received.msecsTo( QDateTime::currentDateTime() ) ) + 36 | QString( " \t: STAGE 1 :\t" ) + QString( hc->monitor_url ) ); 37 | } 38 | 39 | // Start the check on our side 40 | HTTP_Checker *m_checker = new HTTP_Checker( m_net_timeout ); 41 | QObject::connect( m_checker, SIGNAL( finished( HTTP_Checker*, HealthCheck* ) ), this, SLOT( finishedCheck( HTTP_Checker*, HealthCheck* ) ) ); 42 | m_checker->check( hc ); 43 | } 44 | 45 | -------------------------------------------------------------------------------- /veriflier/source/config.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include "headers/config.h" 3 | #include "headers/logger.h" 4 | 5 | Config *Config::m_instance = new Config; 6 | 7 | Config::Config() { 8 | load_config_file(); 9 | } 10 | 11 | void Config::load_config_file() { 12 | QFile file; 13 | file.setFileName( QDir::currentPath() + "/config/veriflier.json" ); 14 | file.open( QIODevice::ReadOnly | QIODevice::Text ); 15 | QString val = file.readAll(); 16 | file.close(); 17 | m_json = QJsonDocument::fromJson( val.toUtf8() ); 18 | } 19 | 20 | int Config::get_int_value( QString name ) { 21 | if ( m_json.isEmpty() || m_json.isNull() ) 22 | return -1; 23 | 24 | QJsonValue value = m_json.object().value( name ); 25 | if ( value.isNull() ) { 26 | LOG( ( QString( "Missing '" ) + name + QString( "' JSON value in config file." ) ).toStdString().c_str() ); 27 | return -1; 28 | } 29 | return value.toInt(); 30 | } 31 | 32 | bool Config::get_bool_value( QString name ) { 33 | if ( m_json.isEmpty() || m_json.isNull() ) 34 | return false; 35 | 36 | QJsonValue value = m_json.object().value( name ); 37 | if ( value.isNull() ) { 38 | LOG( ( QString( "Missing '" ) + name + QString( "' JSON value in config file." ) ).toStdString().c_str() ); 39 | return false; 40 | } 41 | return value.toBool(); 42 | } 43 | 44 | QString Config::get_string_value( QString name ) { 45 | if ( m_json.isEmpty() || m_json.isNull() ) 46 | return QString( "" ); 47 | 48 | QJsonValue value = m_json.object().value( name ); 49 | if ( value.isNull() ) { 50 | LOG( ( QString( "Missing '" ) + name + QString( "' JSON value in config file." ) ).toStdString().c_str() ); 51 | return QString( "" ); 52 | } 53 | return QString( value.toString() ); 54 | } 55 | 56 | -------------------------------------------------------------------------------- /veriflier/headers/http_checker.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef __HTTP_CHECKER_H__ 3 | #define __HTTP_CHECKER_H__ 4 | 5 | #include 6 | #include 7 | 8 | #include 9 | #include 10 | #include 11 | 12 | #include "headers/config.h" 13 | 14 | #define DEFAULT_HTTP_PORT 80 15 | #define DEFAULT_HTTPS_PORT 443 16 | 17 | struct HealthCheck { 18 | int blog_id; 19 | QString monitor_url; 20 | QString jetmon_server; 21 | QDateTime received; 22 | int thread_index; 23 | }; 24 | 25 | class HTTP_Checker: public QObject 26 | { 27 | Q_OBJECT 28 | public: 29 | HTTP_Checker( const int p_net_timeout = 20000 ); 30 | ~HTTP_Checker(); 31 | 32 | void check( HealthCheck* hc ); 33 | int get_rtt() { return m_starttime.msecsTo( QDateTime::currentDateTime() ); } 34 | int get_response_code() { return m_response_code; } 35 | 36 | signals: 37 | void finished( HTTP_Checker* checker, HealthCheck* hc ); 38 | 39 | private slots: 40 | void connected(); 41 | void connectionError( QAbstractSocket::SocketError err ); 42 | void readyRead(); 43 | void timed_out(); 44 | 45 | private: 46 | QSslConfiguration *m_ssl_config; 47 | QAbstractSocket *m_sock; 48 | HealthCheck *m_hc; 49 | QTimer *m_timeout; 50 | 51 | QString m_host_name; 52 | QString m_host_dir; 53 | int m_port; 54 | bool m_is_ssl; 55 | bool m_finished; 56 | QDateTime m_starttime; 57 | QString m_response; 58 | int m_redirects; 59 | int m_response_code; 60 | int m_net_timeout; 61 | 62 | void connect(); 63 | void closeConnection(); 64 | bool send_http_get(); 65 | void process_response(); 66 | void finish_request(); 67 | bool set_redirect_host_values( QString p_content ); 68 | void parse_host_values(); 69 | void parse_response_code( QByteArray a_data ); 70 | }; 71 | 72 | #endif //__HTTP_H__ 73 | 74 | -------------------------------------------------------------------------------- /veriflier/source/logger.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include "headers/logger.h" 3 | 4 | Logger *Logger::m_instance = new Logger; 5 | QFile *Logger::m_file = new QFile; 6 | QMutex *Logger::m_mutex = new QMutex; 7 | 8 | void Logger::stopLogging() { 9 | m_file->close(); 10 | } 11 | 12 | void Logger::startLogger() { 13 | QDir check; 14 | if ( ! check.exists( QDir::currentPath() + "/logs" ) ) 15 | check.mkdir( QDir::currentPath() + "/logs" ); 16 | m_file->setFileName( LOG_FILE_NAME ); 17 | m_file->open( QIODevice::WriteOnly | QIODevice::Text | QIODevice::Append ); 18 | } 19 | 20 | void Logger::write( QString s_data ) { 21 | m_mutex->lock(); 22 | if ( ( QFile( LOG_FILE_NAME ).size() ) > MAX_LOG_FILESIZE ) { 23 | m_file->close(); 24 | Logger::do_log_rotation(); 25 | m_file->setFileName( LOG_FILE_NAME ); 26 | m_file->open( QIODevice::WriteOnly | QIODevice::Text | QIODevice::Append ); 27 | } 28 | if ( m_file->isOpen() ) { 29 | m_file->write( QDateTime::currentDateTime().toString( "yyyy-MM-dd hh:mm:ss").toStdString().c_str() ); 30 | m_file->write( " - " ); 31 | m_file->write( s_data.toStdString().c_str() ); 32 | m_file->write( "\n" ); 33 | m_file->flush(); 34 | } 35 | m_mutex->unlock(); 36 | } 37 | 38 | void Logger::do_log_rotation() { 39 | for ( int del_loop = ( LOGS_TO_KEEP - 1 ); del_loop > 0; del_loop-- ) { 40 | if ( QFile( LOG_FILE_NAME + "." + QString::number( del_loop ) ).exists() ) { 41 | if ( QFile( LOG_FILE_NAME +"." + QString::number( del_loop + 1 ) ).exists() ) 42 | QFile( LOG_FILE_NAME + "." + QString::number( del_loop + 1 ) ).remove(); 43 | QFile( LOG_FILE_NAME + "." + QString::number( del_loop ) ).copy( 44 | LOG_FILE_NAME + "." + QString::number( del_loop + 1 ) ); 45 | } 46 | } 47 | if ( QFile( LOG_FILE_NAME + ".1" ).exists() ) 48 | QFile( LOG_FILE_NAME + ".1" ).remove(); 49 | QFile( LOG_FILE_NAME ).copy( LOG_FILE_NAME + ".1" ); 50 | QFile( LOG_FILE_NAME ).remove(); 51 | } 52 | 53 | -------------------------------------------------------------------------------- /veriflier/headers/client_thread.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef __CLIENT_THREAD_H__ 3 | #define __CLIENT_THREAD_H__ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | #include "headers/config.h" 17 | #include "headers/check_controller.h" 18 | 19 | #define HOST_DOWN 0 20 | #define HOST_ONLINE 1 21 | 22 | class ClientThread : public QRunnable 23 | { 24 | public: 25 | enum QueryType { ServiceRunning, SiteStatusCheck, SiteStatusPostCheck, UnknownQuery }; 26 | 27 | ClientThread( qintptr sock, 28 | const QSslConfiguration *ssl_config, 29 | CheckController *checker, 30 | const QString &veriflier_name, 31 | const QString &auth_token, 32 | const int net_timeout, 33 | const bool debug ); 34 | ~ClientThread(); 35 | 36 | void run(); 37 | 38 | private: 39 | qintptr m_sock; 40 | QSslSocket *m_socket; 41 | const QSslConfiguration *m_ssl_config; 42 | CheckController *m_checker; 43 | QVector m_checks; 44 | 45 | QString m_veriflier_name; 46 | QString m_auth_token; 47 | int m_net_timeout; 48 | QString m_jetmon_server; 49 | 50 | bool m_debug; 51 | bool m_site_status_request; 52 | 53 | void sendOK(); 54 | void sendServiceOK(); 55 | void sendError( const QString errorString ); 56 | 57 | void readRequest(); 58 | 59 | QueryType get_request_type( QByteArray &raw_data ); 60 | 61 | QJsonDocument parse_json_request( QByteArray &raw_data ); 62 | QJsonDocument parse_json_request_post( QByteArray &raw_data ); 63 | 64 | int parse_json_request_post_length( QByteArray &raw_data ); 65 | int get_content_length( QByteArray &raw_data ); 66 | 67 | bool parse_requests( QueryType type, QJsonDocument json_doc ); 68 | 69 | QString get_http_reply_header( const QString &http_code, const QString &p_data); 70 | QString get_http_content( int status, const QString &error = "" ); 71 | }; 72 | #endif // __CLIENT_THREAD_H__ 73 | 74 | -------------------------------------------------------------------------------- /veriflier/veriflier.sh: -------------------------------------------------------------------------------- 1 | #!/bin/bash 2 | 3 | ### BEGIN INIT INFO 4 | # Provides: veriflier 5 | # Required-Start: 6 | # Required-Stop: 7 | # Default-Start: 2 3 4 5 8 | # Default-Stop: 0 1 6 9 | # Short-Description: Start/stop veriflier 10 | # Description: Start/stop the service. 11 | ### END INIT INFO 12 | 13 | # installation directory 14 | INSTALL_DIR=/opt/veriflier 15 | 16 | function startservice { 17 | if [ ! -d ${INSTALL_DIR} ]; then 18 | echo "the jetmon veriflier is not installed in the correct directory: ${INSTALL_DIR}" 19 | exit 1 20 | fi 21 | if [ -f /var/run/veriflier.pid ]; then 22 | pid="`cat /var/run/veriflier.pid`" 23 | if [ -z "$pid" ]; then 24 | pid=0 25 | else 26 | if ! ps -p $pid >/dev/null; then 27 | rm -f /var/run/veriflier.pid 28 | pid=0 29 | fi 30 | fi 31 | if [ $pid -gt 0 ] ; then 32 | echo "veriflier service is already running" 33 | exit 1 34 | fi 35 | fi 36 | # create the required log directory 37 | if [ ! -d "${INSTALL_DIR}/logs" ]; then 38 | mkdir "${INSTALL_DIR}/logs" 39 | fi 40 | 41 | echo "Starting veriflier" 42 | ( 43 | cd "${INSTALL_DIR}" 44 | ( ${INSTALL_DIR}/veriflier >/dev/null 2>&1 )& 45 | echo $! > /var/run/veriflier.pid 46 | ) 47 | } 48 | 49 | function stopservice { 50 | if [ -f /var/run/veriflier.pid ]; then 51 | pid="`cat /var/run/veriflier.pid`" 52 | if [ -z "$pid" ]; then 53 | echo "There was an error loading the pid file." 54 | else 55 | echo "Stopping veriflier with pid $pid" 56 | kill -15 $pid 57 | fi 58 | rm -f /var/run/veriflier.pid 59 | else 60 | echo "There is no veriflier process running." 61 | fi 62 | } 63 | 64 | function reload_config { 65 | pid="`ps -ef | grep 'veriflier' | grep -v 'grep' | awk ' { print $(2) }'`" 66 | if [ -z "$pid" ]; then 67 | pid=0 68 | fi 69 | if [ $pid -gt 0 ]; then 70 | echo "Reloading veriflier config" 71 | kill -SIGHUP $pid 72 | else 73 | echo "There is no veriflier process running." 74 | fi 75 | } 76 | 77 | case "$1" in 78 | start ) 79 | startservice 80 | ;; 81 | stop ) 82 | stopservice 83 | ;; 84 | restart ) 85 | stopservice 86 | sleep 2 87 | startservice 88 | ;; 89 | reload ) 90 | reload_config 91 | ;; 92 | * ) 93 | echo "Usage:$0 start|stop|restart|reload" 94 | exit 1 95 | ;; 96 | esac 97 | exit 0 98 | -------------------------------------------------------------------------------- /veriflier/headers/check_controller.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef __CHECKCONTROLLER_H__ 3 | #define __CHECKCONTROLLER_H__ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | #include "headers/check_thread.h" 14 | #include "headers/jetmon_server.h" 15 | 16 | #define NOT_ASSIGNED -1 17 | #define PRE_ASSIGNED -2 18 | 19 | struct Runner { 20 | CheckThread* ct; 21 | int checking; 22 | }; 23 | 24 | class CheckController : public QObject 25 | { 26 | Q_OBJECT 27 | public: 28 | explicit CheckController( const QSslConfiguration *m_ssl_config, 29 | const int jetmon_server_port, 30 | const int max_runners = 20, 31 | const int max_checks = 50, 32 | const QString &veriflier_name = "", 33 | const QString &auth_token = "", 34 | const int net_timeout = 20000, 35 | const bool debug = false ); 36 | 37 | ~CheckController(); 38 | void addCheck( HealthCheck* hc ); 39 | void addChecks( QVector hcs ); 40 | 41 | public slots: 42 | void startChecking( HealthCheck* hc ); 43 | void finishedChecking( int thread_index, qint64 blog_id, QString monitor_url, int status, int http_code, int rtt ); 44 | void finishedSending( JetmonServer* js, int status, int rtt ); 45 | void ticked(); 46 | 47 | signals: 48 | void startCheck( HealthCheck* hc ); 49 | 50 | private: 51 | QVector m_checks; 52 | QVector m_runners; 53 | 54 | QMap m_check_results; 55 | 56 | const QSslConfiguration *m_ssl_config; 57 | int m_jetmon_server_port; 58 | QSslSocket *m_socket; 59 | QMutex m_check_lock; 60 | QTimer *m_ticker; 61 | 62 | int m_max_checkers; 63 | int m_max_checks; 64 | int m_checking; 65 | int m_checked; 66 | 67 | QString m_veriflier_name; 68 | QString m_auth_token; 69 | int m_net_timeout; 70 | bool m_debug; 71 | 72 | inline bool haveCheck( qint64 blog_id, QString monitor_url ); 73 | int selectRunner(); 74 | void sendResults(); 75 | QString post_http_header( QString jetmon_server, int content_size ); 76 | void sendToJetmonServer( QString jetmon_server, QByteArray status_data ); 77 | QJsonDocument parse_json_response( QByteArray &raw_data ); 78 | int readResponse(); 79 | }; 80 | 81 | #endif // __CHECKCONTROLLER_H__ 82 | -------------------------------------------------------------------------------- /docker/docker-compose.yml: -------------------------------------------------------------------------------- 1 | version: '3.8' 2 | 3 | services: 4 | mysqldb: 5 | image: mysql:5.7 6 | restart: unless-stopped 7 | env_file: 8 | - .env 9 | environment: 10 | - MYSQL_ROOT_PASSWORD=$MYSQLDB_ROOT_PASSWORD 11 | - MYSQL_DATABASE=$MYSQLDB_DATABASE 12 | ports: 13 | - $MYSQLDB_LOCAL_PORT:$MYSQLDB_DOCKER_PORT 14 | volumes: 15 | - db:/var/lib/mysql 16 | jetmon: 17 | hostname: docker.jetmon.dev.com 18 | build: 19 | context: ../ 20 | dockerfile: docker/Dockerfile_jetmon 21 | env_file: 22 | - .env 23 | volumes: 24 | - ../:/jetmon 25 | # Don't sync the node_modules directory back to the client. 26 | - "/jetmon/node_modules" 27 | environment: 28 | - DB_HOST=mysqldb 29 | - DB_USER=$MYSQLDB_USER 30 | - DB_PASSWORD=$MYSQLDB_ROOT_PASSWORD 31 | - DB_NAME=$MYSQLDB_DATABASE 32 | - DB_PORT=$MYSQLDB_DOCKER_PORT 33 | - VERIFLIER_AUTH_TOKEN=$VERIFLIER_AUTH_TOKEN 34 | - VERIFLIER_PORT=$VERIFLIER_DOCKER_PORT 35 | - WPCOM_JETMON_AUTH_TOKEN=$WPCOM_JETMON_AUTH_TOKEN 36 | ports: 37 | - $JETMON_LOCAL_PORT:$JETMON_DOCKER_PORT 38 | - $JETMON_STATUS_LOCAL_PORT:$JETMON_STATUS_DOCKER_PORT 39 | depends_on: 40 | - mysqldb 41 | veriflier: 42 | build: 43 | context: ../ 44 | dockerfile: docker/Dockerfile_veriflier 45 | volumes: 46 | - ../:/opt 47 | ports: 48 | - $VERIFLIER_LOCAL_PORT:$VERIFLIER_DOCKER_PORT 49 | environment: 50 | - VERIFLIER_AUTH_TOKEN=$VERIFLIER_AUTH_TOKEN 51 | - VERIFLIER_PORT=$VERIFLIER_DOCKER_PORT 52 | - JETMON_PORT=$JETMON_DOCKER_PORT 53 | statsd: 54 | image: graphiteapp/graphite-statsd 55 | restart: unless-stopped 56 | ports: 57 | - 8088:80 58 | - 8126:8126 59 | - 8125:8125 60 | - 8125:8125/udp 61 | volumes: 62 | - ./volumes/statsd/graphite/conf:/opt/graphite/conf 63 | - ./volumes/statsd/graphite/storage:/opt/graphite/storage 64 | - ./volumes/statsd/statsd/config:/opt/statsd/config 65 | - ./volumes/statsd/logs:/var/log 66 | 67 | volumes: 68 | db: 69 | -------------------------------------------------------------------------------- /src/http_checker.h: -------------------------------------------------------------------------------- 1 | 2 | #ifndef __HTTP_CHECKER_H__ 3 | #define __HTTP_CHECKER_H__ 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | #include 18 | #include 19 | 20 | #include 21 | #include 22 | #include 23 | #include 24 | #include 25 | #include 26 | 27 | #include 28 | #include 29 | #include 30 | #if (SSLEAY_VERSION_NUMBER >= 0x0907000L) 31 | # include 32 | #endif 33 | 34 | #define HTTP_DEFAULT_PORT 80 35 | #define HTTPS_DEFAULT_PORT 443 36 | #define MAX_TCP_BUFFER 1024 37 | #define NET_COMMS_TIMEOUT 20 38 | #define MAX_REDIRECTS 2 39 | #define MAX_EPOLL_EVENTS 10 40 | 41 | // Enables the printing of debug messages to stderr 42 | #define DEBUG_MODE 0 43 | 44 | // getaddrinfo is much slower than gethostbyname and, although 45 | // it is technically the best way to lookup hosts, only enable 46 | // this on hosts with more than enough CPU compute headroom. 47 | #define USE_GETADDRINFO 1 48 | 49 | // Sets whether we compile and use non-blocking socket IO 50 | #define NON_BLOCKING_IO 0 51 | 52 | class HTTP_Checker { 53 | 54 | public: 55 | HTTP_Checker(); 56 | ~HTTP_Checker(); 57 | 58 | void check( std::string p_host_name, int p_port = HTTP_DEFAULT_PORT ); 59 | time_t get_rtt(); 60 | int get_response_code() { return m_response_code; } 61 | int get_error_code() { return m_error_code; } 62 | 63 | private: 64 | char m_buf[MAX_TCP_BUFFER]; 65 | int m_sock; 66 | std::string m_host_name; 67 | std::string m_host_dir; 68 | int m_port; 69 | bool m_is_ssl; 70 | struct timezone m_tzone; 71 | struct timeval m_tstart; 72 | time_t m_triptime; 73 | time_t m_cutofftime; 74 | int m_response_code; 75 | int m_error_code; 76 | 77 | SSL_CTX *m_ctx; 78 | SSL *m_ssl; 79 | BIO *m_sbio; 80 | 81 | bool init_socket( addrinfo *addr ); 82 | bool init_ssl(); 83 | bool connect(); 84 | #if USE_GETADDRINFO 85 | bool connect_getaddrinfo(); 86 | #else 87 | bool connect_gethostbyname(); 88 | #endif 89 | bool disconnect(); 90 | #if NON_BLOCKING_IO 91 | void disconnect_ssl(); 92 | #endif 93 | std::string send_http_get(); 94 | bool send_bytes( char* p_packet, size_t p_packet_length ); 95 | std::string get_response(); 96 | void set_host_response( int redirects ); 97 | bool set_redirect_host_values( std::string p_content ); 98 | void parse_host_values(); 99 | }; 100 | 101 | #endif //__HTTP_H__ 102 | 103 | -------------------------------------------------------------------------------- /lib/wpcom.js: -------------------------------------------------------------------------------- 1 | 2 | const NETWORK_TIMEOUT_MS = 20000; 3 | 4 | var https = require( 'https' ); 5 | var fs = require( 'fs' ); 6 | var db_mysql = require( './database' ); 7 | var ssl_key = fs.readFileSync( 'certs/jetmon.key' ); 8 | var ssl_cert = fs.readFileSync( 'certs/jetmon.crt' ); 9 | 10 | var wpcom = { 11 | notifyStatusChange : function( serverObject, callBack ) { 12 | if ( global.config.get( 'DB_UPDATES_ENABLE' ) ) { 13 | db_mysql.updateSite( serverObject.blog_id, serverObject.monitor_url, serverObject.site_status ); 14 | } 15 | try { 16 | var o_request = {}; 17 | o_request.blog_id = serverObject.blog_id; 18 | o_request.monitor_url = serverObject.monitor_url; 19 | o_request.status_id = serverObject.site_status; 20 | o_request.last_check = new Date( serverObject.lastCheck ); 21 | o_request.last_status_change = serverObject.last_status_change ? new Date( serverObject.last_status_change ) : null; 22 | o_request.checks = serverObject.checks; 23 | o_request.token = global.config.get( 'AUTH_TOKEN' ); 24 | 25 | var request_str = JSON.stringify( o_request ); 26 | 27 | var options = { 28 | hostname: 'jetpack.wordpress.com', 29 | port: 443, 30 | key: ssl_key, 31 | cert: ssl_cert, 32 | path: '/jetmon/?data=' + request_str, 33 | method: 'GET', 34 | rejectUnauthorized: false, 35 | }; 36 | 37 | logger.trace( 'setting blogid ' + o_request.blog_id + ' status ' + o_request.status_id + ', URL: ' + o_request.monitor_url ); 38 | 39 | var response_handler = function( res ) { 40 | res.setEncoding( 'utf8' ); 41 | var reply_data = {}; 42 | 43 | if ( 200 != res.statusCode ) { 44 | logger.error('incorrect status code from the server: ' + res.statusCode); 45 | } 46 | 47 | res.on( 'data', function ( response_data ) { 48 | try { 49 | reply_data = JSON.parse( response_data ); 50 | } 51 | catch ( Exception ) { 52 | logger.error( 'error parsing the server response.' ); 53 | reply_data.success = false; 54 | } 55 | callBack( reply_data ); 56 | }); 57 | }; 58 | 59 | var error_handler = function( err ) { 60 | logger.error( 'error performing request: ' + err ); 61 | var reply_data = {}; 62 | reply_data.success = false; 63 | callBack( reply_data ); 64 | }; 65 | 66 | var timeout_handler = function() { 67 | logger.error( 'timed out performing a request to the jetpack.wordpress.com server ' ); 68 | var reply_data = {}; 69 | reply_data.success = false; 70 | callBack( reply_data ); 71 | }; 72 | 73 | var request = https.request( options ). 74 | addListener( 'response', response_handler ) 75 | .addListener( 'error', error_handler ) 76 | .addListener( 'timeout', timeout_handler ); 77 | request.setTimeout( NETWORK_TIMEOUT_MS ); 78 | request.end(); 79 | } 80 | catch ( Exception ) { 81 | logger.error( 'notifyStatusChange error: ' + Exception.toString() ); 82 | } 83 | } 84 | } 85 | 86 | module.exports = wpcom; 87 | 88 | -------------------------------------------------------------------------------- /veriflier/source/jetmon_server.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include "../headers/jetmon_server.h" 3 | 4 | JetmonServer::JetmonServer( QObject *parent, const QSslConfiguration *ssl_config, QString jetmon_server, int jetmon_server_port ) : 5 | QObject(parent), m_jetmon_server( jetmon_server ), m_jetmon_server_port( jetmon_server_port ) 6 | { 7 | m_socket = new QSslSocket(); 8 | m_socket->setSslConfiguration( *ssl_config ); 9 | 10 | QObject::connect( m_socket, SIGNAL( connected() ), this, SLOT( connected() ) ); 11 | QObject::connect( m_socket, SIGNAL( readyRead() ), this, SLOT( readyRead() ) ); 12 | QObject::connect( m_socket, SIGNAL( error( QAbstractSocket::SocketError) ), this, SLOT( connectionError( QAbstractSocket::SocketError) ) ); 13 | } 14 | 15 | void JetmonServer::sendData( QByteArray status_data ) { 16 | m_timer = QDateTime::currentDateTime(); 17 | m_status_data = status_data; 18 | m_socket->connectToHostEncrypted( m_jetmon_server, m_jetmon_server_port ); 19 | } 20 | 21 | void JetmonServer::connected() { 22 | if ( m_socket->isEncrypted() || ( ! m_socket->isOpen() ) ) { 23 | emit finished( this, 0, m_timer.msecsTo( QDateTime::currentDateTime() ) ); 24 | return; 25 | } 26 | 27 | LOG( QString::number( m_timer.msecsTo( QDateTime::currentDateTime() ) ) + 28 | QString( "\t\t: SENDING :\tconnected to :" ) + m_jetmon_server ); 29 | m_timer = QDateTime::currentDateTime(); 30 | 31 | m_socket->write( m_status_data ); 32 | m_socket->flush(); 33 | } 34 | 35 | void JetmonServer::connectionError( QAbstractSocket::SocketError err ) { 36 | LOG( "Connection Error[" + QString::number( err ) + "]: " + m_jetmon_server + " : "+ m_socket->errorString() ); 37 | emit finished( this, 0, m_timer.msecsTo( QDateTime::currentDateTime() ) ); 38 | } 39 | 40 | void JetmonServer::readyRead() { 41 | QByteArray a_data = m_socket->readAll(); 42 | this->closeConnection(); 43 | 44 | if ( 0 == a_data.length() ) { 45 | LOG( "NO data returned when reading jetmon response." ); 46 | emit finished( this, 0, m_timer.msecsTo( QDateTime::currentDateTime() ) ); 47 | return; 48 | } 49 | 50 | QJsonDocument json_doc = parse_json_response( a_data ); 51 | 52 | if ( json_doc.isEmpty() || json_doc.isNull() ) { 53 | LOG( "Invalid JSON document format." ); 54 | emit finished( this, 0, m_timer.msecsTo( QDateTime::currentDateTime() ) ); 55 | return; 56 | } 57 | 58 | QJsonValue response = json_doc.object().value( "response" ); 59 | if ( response.isNull() ) { 60 | LOG( "Missing 'response' JSON value." ); 61 | emit finished( this, 0, m_timer.msecsTo( QDateTime::currentDateTime() ) ); 62 | return; 63 | } 64 | 65 | emit finished( this, 1, m_timer.msecsTo( QDateTime::currentDateTime() ) ); 66 | } 67 | 68 | void JetmonServer::closeConnection() { 69 | if ( m_socket->isOpen() ) 70 | m_socket->close(); 71 | m_socket->deleteLater(); 72 | } 73 | 74 | QJsonDocument JetmonServer::parse_json_response( QByteArray &raw_data ) { 75 | QJsonDocument ret_val; 76 | QString s_data = raw_data.data(); 77 | 78 | if ( ( -1 == s_data.indexOf( "{" ) ) || ( -1 == s_data.lastIndexOf( "}" ) ) ) { 79 | LOG( "Invalid JSON response format.\n\n" + s_data ); 80 | return ret_val; 81 | } 82 | 83 | s_data = s_data.mid( s_data.indexOf( "{" ), s_data.lastIndexOf( "}" ) - s_data.indexOf( "{" ) + 1 ); 84 | ret_val = QJsonDocument::fromJson( s_data.toUtf8() ); 85 | return ret_val; 86 | } 87 | 88 | -------------------------------------------------------------------------------- /lib/statsd.js: -------------------------------------------------------------------------------- 1 | const os = require('os'); 2 | const dgram = require('dgram'); 3 | 4 | /** 5 | * Hostnames on prod look like: 6 | * .. 7 | * 8 | * We only need the first 2 pieces and flip them around to make easier to group/filter things in StatsD later. 9 | * @type {string} 10 | */ 11 | const currentHostname = os.hostname().split( '.' ).slice( 0, 2 ).reverse().join( '.' ); 12 | 13 | /** 14 | * Set up the StatD client. 15 | * 16 | * All entries are prefixed with `com.jetpack.jetmon..` 17 | */ 18 | let statsdHostname = '127.0.0.1'; 19 | 20 | /** 21 | * The MTU of the network connection that sends StatsD metrics is used to 22 | * determine the max buffer size. 23 | */ 24 | let statsdMTU = 65536; 25 | 26 | /** 27 | * The number of milliseconds that can elapse before buffered StatsD metrics are 28 | * flushed. 29 | */ 30 | let statsdFlushInterval = 5000; 31 | 32 | /** 33 | * Add a workaround for the local Docker instances, as prod is running statsd proxies on 127.0.0.1, 34 | * while the Docker nodes run it in the `statsd` container. 35 | */ 36 | if ( currentHostname === 'jetmon.docker' ) { 37 | statsdHostname = 'statsd'; 38 | statsdMTU = 1500; 39 | } 40 | 41 | const prefix = 'com.jetpack.jetmon.' + currentHostname + '.'; 42 | 43 | 44 | const statsdClient = { 45 | init: function( prefix, host, port, mtu, flushInterval, logger ) { 46 | this.prefix = prefix; 47 | this.host = host; 48 | this.port = port; 49 | this.maxBufferSize = mtu - 29; // Reduce by 29 to account for packet headers. 50 | this.logger = logger; 51 | 52 | this.buffer = ''; 53 | 54 | this.socket = dgram.createSocket( 'udp4' ); 55 | this.socket.on( 'error', (error) => this.logger.error( error ) ); 56 | 57 | this.interval = setInterval( () => { 58 | this.flush(); 59 | }, flushInterval ); 60 | }, 61 | 62 | increment: function( metric, value = 1, sampleRate = 1) { 63 | this.send( `${metric}:${value}|c|@${sampleRate}` ); 64 | }, 65 | 66 | timing: function( metric, value, sampleRate = 1 ) { 67 | this.send( `${metric}:${value}|ms|@${sampleRate}` ); 68 | }, 69 | 70 | gauge: function( metric, value ) { 71 | this.send( `${metric}:${value}|g` ); 72 | }, 73 | 74 | send: function( message ) { 75 | message = `${this.prefix}${message}\n`; 76 | 77 | // If the total buffer size is already at the maximum size, flush it first 78 | if ( this.buffer.length + message.length >= this.maxBufferSize ) { 79 | this.flush(); 80 | } 81 | 82 | // Append the message to the buffer 83 | this.buffer += message; 84 | }, 85 | 86 | flush: function() { 87 | if ( this.buffer.length > 0 ) { 88 | const buffer = this.buffer; 89 | this.buffer = ''; 90 | try { 91 | this.socket.send( buffer, this.port, this.host, (error) => { 92 | if ( error ) { 93 | this.logger.error( 'Error when sending to statsd: ' + error.toString() ); 94 | } 95 | }); 96 | } 97 | catch ( Exception ) { 98 | this.logger.error( 'Exception when sending to statsd: ' + Exception.toString() ); 99 | } 100 | } 101 | }, 102 | 103 | close: function() { 104 | clearInterval( this.interval ); 105 | this.socket.close(); 106 | } 107 | } 108 | 109 | statsdClient.init( prefix, statsdHostname, 8125, statsdMTU, statsdFlushInterval, logger ); 110 | 111 | module.exports = statsdClient; 112 | -------------------------------------------------------------------------------- /veriflier/source/ssl_server.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include "headers/ssl_server.h" 3 | #include "headers/logger.h" 4 | 5 | using namespace std; 6 | 7 | SSL_Server::SSL_Server( QObject *parent ) : QTcpServer( parent ) { 8 | LOG( "booting veriflier" ); 9 | m_served_count = 0; 10 | 11 | pool = new QThreadPool(this); 12 | pool->setMaxThreadCount( Config::instance()->get_int_value( "thread_pool_max" ) ); 13 | LOG( ( QString( "comms thread pool: " ) + QString::number( pool->maxThreadCount() ) ) ); 14 | 15 | this->setMaxPendingConnections( Config::instance()->get_int_value( "max_pending_conns" ) ); 16 | LOG( ( QString( "max pending conns: " ) + QString::number( this->maxPendingConnections() ) ) ); 17 | 18 | m_veriflier_name = Config::instance()->get_string_value( "veriflier_name" ); 19 | m_auth_token = Config::instance()->get_string_value( "auth_token" ); 20 | m_net_timeout = Config::instance()->get_int_value( "net_comms_timeout" ); 21 | m_debug = Config::instance()->get_bool_value( "debug" ); 22 | m_jetmon_server_port = Config::instance()->get_int_value( "jetmon_server_port" ); 23 | 24 | int max_checkers = Config::instance()->get_int_value( "max_checkers" ); 25 | if ( 0 == max_checkers || -1 == max_checkers ) max_checkers = DEFAULT_MAX_CHECKERS; 26 | LOG( ( QString( "max checkers: " ) + QString::number( max_checkers ) ) ); 27 | 28 | int max_checks = Config::instance()->get_int_value( "max_checks" ); 29 | if ( 0 == max_checks || -1 == max_checks ) max_checks = DEFAULT_MAX_CHECKS; 30 | LOG( ( QString( "max checks: " ) + QString::number( max_checks ) ) ); 31 | 32 | m_ssl_config = new QSslConfiguration(); 33 | m_ssl_config->setPeerVerifyMode( QSslSocket::VerifyNone ); 34 | m_ssl_config->setProtocol( QSsl::AnyProtocol ); 35 | 36 | QFile keyFile( Config::instance()->get_string_value( "privatekey_file" ) ); 37 | keyFile.open( QFile::ReadOnly ); 38 | if ( ! keyFile.isOpen() ) { 39 | LOG( "Error loading private key file" ); 40 | cerr << "Error loading private key file" << endl; 41 | exit(-1); 42 | } 43 | QSslKey ssl_key( &keyFile, QSsl::Rsa ); 44 | m_ssl_config->setPrivateKey( ssl_key ); 45 | keyFile.close(); 46 | 47 | QFile certFile( Config::instance()->get_string_value( "privatecert_file" ) ); 48 | certFile.open( QFile::ReadOnly ); 49 | if ( ! certFile.isOpen() ) { 50 | LOG( "Error loading private certificate file" ); 51 | cerr << "Error loading private key file" << endl; 52 | exit(-1); 53 | } 54 | QSslCertificate ssl_cert( &certFile ); 55 | m_ssl_config->setLocalCertificate( ssl_cert ); 56 | certFile.close(); 57 | 58 | m_checker = new CheckController( m_ssl_config, m_jetmon_server_port, max_checkers, max_checks, 59 | m_veriflier_name, m_auth_token, m_net_timeout, m_debug ); 60 | 61 | connect( this, SIGNAL( acceptError(QAbstractSocket::SocketError) ), this, SLOT( logError(QAbstractSocket::SocketError) ) ); 62 | } 63 | 64 | SSL_Server::~SSL_Server() { 65 | delete m_ssl_config; 66 | delete m_checker; 67 | delete pool; 68 | Logger::instance()->stopLogging(); 69 | } 70 | 71 | void SSL_Server::incomingConnection( qintptr socketDescriptor ) { 72 | m_served_count++; 73 | if ( m_served_count % 50 == 0 ) 74 | LOG( ( QString( "served count: " ) + QString::number( m_served_count ) ).toStdString().c_str() ); 75 | 76 | ClientThread *client = new ClientThread( socketDescriptor, m_ssl_config, m_checker, m_veriflier_name, 77 | m_auth_token, m_net_timeout, m_debug ); 78 | client->setAutoDelete( true ); 79 | pool->start( client ); 80 | } 81 | 82 | void SSL_Server::logError(QAbstractSocket::SocketError socketError) { 83 | LOG( QString( socketError ).toStdString().c_str() ); 84 | } 85 | -------------------------------------------------------------------------------- /lib/comms.js: -------------------------------------------------------------------------------- 1 | 2 | var _https = require( 'https' ); 3 | var _fs = require( 'fs' ); 4 | 5 | const NETWORK_TIMEOUT_MS = 30000; 6 | 7 | var ssl_key = _fs.readFileSync( 'certs/jetmon.key' ); 8 | var ssl_cert = _fs.readFileSync( 'certs/jetmon.crt' ); 9 | 10 | var client = { 11 | get_remote_status: function( veriflier_server, blog_id, s_server, call_back ) { 12 | var o_request = {}; 13 | o_request.auth_token = global.config.get( 'AUTH_TOKEN' ); 14 | o_request.blog_id = blog_id; 15 | o_request.monitor_url = s_server; 16 | var request_str = JSON.stringify( o_request ); 17 | 18 | var options = { 19 | hostname: veriflier_server.host, 20 | port: veriflier_server.port, 21 | key: ssl_key, 22 | cert: ssl_cert, 23 | path: '/get/host-status?d=' + request_str, 24 | method: 'GET', 25 | rejectUnauthorized: false, 26 | }; 27 | 28 | client.perform_request( options, null, call_back ); 29 | }, 30 | 31 | get_remote_status_array: function( veriflier_server, checkArray, call_back ) { 32 | var o_request = {}; 33 | o_request.auth_token = global.config.get( 'AUTH_TOKEN' ); 34 | o_request.checks = checkArray; 35 | var requestData = JSON.stringify( o_request ); 36 | 37 | logger.debug( 'POSTing ' + checkArray.length + ' to ' + 38 | veriflier_server.name + ', ' + requestData.length + ' bytes' ); 39 | 40 | var options = { 41 | hostname: veriflier_server.host, 42 | port: veriflier_server.port, 43 | key: ssl_key, 44 | cert: ssl_cert, 45 | path: '/get/host-status', 46 | method: 'POST', 47 | headers: { 48 | 'Content-Type' : 'application/json', 49 | 'Content-Length': requestData.length, 50 | }, 51 | rejectUnauthorized: false, 52 | }; 53 | 54 | client.perform_request( options, requestData, call_back ); 55 | }, 56 | 57 | perform_request: function( options, postData, call_back ) { 58 | var response_handler = function( res ) { 59 | res.setEncoding( 'utf8' ); 60 | var s_data = ''; 61 | res.on( 'data', function( response_data ) { 62 | s_data += response_data; 63 | }); 64 | res.on( 'end', function() { 65 | var reply_data = {}; 66 | if ( 200 == res.statusCode ) { 67 | try { 68 | reply_data = JSON.parse( s_data ); 69 | } 70 | catch ( Exception ) { 71 | reply_data.status = -1; 72 | reply_data.reply = "PARSE_ERROR"; 73 | } 74 | } else { 75 | reply_data.status = -1; 76 | reply_data.reply = "CODE_ERROR"; 77 | logger.error( res.statusCode + ': error sending status data to ' + 78 | options.hostname + ':' + options.port ); 79 | } 80 | call_back( reply_data ); 81 | }); 82 | }; 83 | 84 | var error_handler = function( err ) { 85 | var reply_data = {}; 86 | reply_data.status = -1; 87 | reply_data.reply = "REQUEST_ERROR"; 88 | call_back( reply_data ); 89 | logger.error( 'error performing request: ' + err ); 90 | }; 91 | 92 | var timeout_handler = function() { 93 | var reply_data = {}; 94 | reply_data.status = -1; 95 | reply_data.reply = "TIMEOUT_ERROR"; 96 | call_back( reply_data ); 97 | logger.error( 'timed out performing a request to server ' + options.hostname ); 98 | }; 99 | 100 | options.secureProtocol = "TLSv1_2_method"; 101 | 102 | var request = _https.request( options ). 103 | addListener( 'response', response_handler ) 104 | .addListener( 'error', error_handler ) 105 | .addListener( 'timeout', timeout_handler ); 106 | request.setTimeout( NETWORK_TIMEOUT_MS ); 107 | if ( null !== postData ) { 108 | request.write( postData ); 109 | } 110 | request.end(); 111 | } 112 | } 113 | 114 | module.exports = client; 115 | -------------------------------------------------------------------------------- /src/main.cpp: -------------------------------------------------------------------------------- 1 | 2 | #ifndef BUILDING_NODE_EXTENSION 3 | #define BUILDING_NODE_EXTENSION 4 | #endif 5 | 6 | #include 7 | #include 8 | #include 9 | 10 | #include 11 | #include 12 | 13 | using namespace v8; 14 | using namespace node; 15 | 16 | #include "http_checker.h" 17 | 18 | struct HTTP_Check_Baton { 19 | CopyablePersistentTraits::CopyablePersistent callback; 20 | HTTP_Checker *http_checker; 21 | std::string server; 22 | int port; 23 | int server_id; 24 | }; 25 | 26 | static void http_check_async_fin( uv_work_t *req, int status ) { 27 | Isolate* isolate = Isolate::GetCurrent(); 28 | HandleScope scope( isolate ); 29 | 30 | HTTP_Check_Baton *baton = static_cast(req->data); 31 | Local argv[4] = { Number::New( isolate, baton->server_id ), 32 | Number::New( isolate, baton->http_checker->get_rtt() ), 33 | Number::New( isolate, baton->http_checker->get_response_code() ), 34 | Number::New( isolate, baton->http_checker->get_error_code() ) }; 35 | 36 | Local cb_func = Local::New( isolate, baton->callback ); 37 | cb_func->Call( isolate->GetCurrentContext(), isolate->GetCurrentContext()->Global(), 4, argv ); 38 | baton->callback.Reset(); 39 | delete baton->http_checker; 40 | delete baton; 41 | delete req; 42 | } 43 | 44 | void http_check_async( uv_work_t *req ) { 45 | HTTP_Check_Baton *baton = static_cast( req->data ); 46 | baton->http_checker->check( baton->server, baton->port ); 47 | } 48 | 49 | void http_check( const FunctionCallbackInfo& args ) { 50 | args.GetReturnValue().SetUndefined(); 51 | Isolate* isolate = args.GetIsolate(); 52 | HandleScope scope( isolate ); 53 | 54 | if ( args.Length() < 4 ) { 55 | isolate->ThrowException( Exception::TypeError( 56 | String::NewFromUtf8( isolate, "Wrong number of arguments" ).ToLocalChecked() ) ); 57 | return; 58 | } 59 | 60 | if ( ! args[1]->IsNumber() ) { 61 | isolate->ThrowException( Exception::TypeError( 62 | String::NewFromUtf8( isolate, "The port number argument is not valid" ).ToLocalChecked() ) ); 63 | return; 64 | } 65 | 66 | if ( ! args[2]->IsNumber() ) { 67 | isolate->ThrowException( Exception::TypeError( 68 | String::NewFromUtf8( isolate, "The server id argument is not valid" ).ToLocalChecked() ) ); 69 | return; 70 | } 71 | 72 | if ( ! args[3]->IsFunction() ) { 73 | isolate->ThrowException( Exception::TypeError( 74 | String::NewFromUtf8( isolate, "You have not provided a callback function as the 4th parameter" ).ToLocalChecked() ) ); 75 | return; 76 | } 77 | 78 | HTTP_Check_Baton *baton = new HTTP_Check_Baton(); 79 | HTTP_Checker *checker = new HTTP_Checker(); 80 | baton->http_checker = checker; 81 | 82 | String::Utf8Value sHost( isolate, args[0] ); 83 | baton->server = *sHost; 84 | 85 | baton->port = args[1]->ToInteger( isolate->GetCurrentContext() ).ToLocalChecked()->Value(); 86 | baton->server_id = (int) args[2]->ToInteger( isolate->GetCurrentContext() ).ToLocalChecked()->Value(); 87 | 88 | CopyablePersistentTraits::CopyablePersistent percy( isolate, args[3].As() ); 89 | baton->callback.Reset( isolate, percy ); 90 | 91 | uv_work_t *req = new uv_work_t(); 92 | req->data = baton; 93 | 94 | uv_queue_work( uv_default_loop(), req, http_check_async, (uv_after_work_cb)http_check_async_fin ); 95 | } 96 | 97 | void Initialise( Local exports) { 98 | SSL_load_error_strings(); 99 | SSL_library_init(); 100 | OpenSSL_add_all_algorithms(); 101 | #if (SSLEAY_VERSION_NUMBER >= 0x0907000L) 102 | OPENSSL_config( NULL ); 103 | #endif 104 | 105 | NODE_SET_METHOD( exports, "http_check", http_check ); 106 | } 107 | 108 | NODE_MODULE( jetmon, Initialise ) 109 | 110 | -------------------------------------------------------------------------------- /config/config.readme: -------------------------------------------------------------------------------- 1 | DEBUG 2 | Set to true to enable more verbose log messages in logs/jetmon.log. 3 | 4 | NUM_WORKERS 5 | The number of forked worker processes to create and maintain. 6 | 7 | NUM_TO_PROCESS 8 | The number of sites that a worker should process in parallel. 9 | 10 | DATASET_SIZE 11 | The maximum number of sites to send to a worker's queue in a single batch. 12 | 13 | WORKER_MAX_CHECKS 14 | The maximum number of checks that a worker can process before it stops accepting work and is scheduled to recycle. 15 | Set to 0 or a negative value to disable recycling workers based on the number of checks. 16 | 17 | WORKER_MAX_MEM_MB 18 | The maximum MB of memory that a worker can consume before it stops accepting work and is scheduled to recycle. 19 | Set to 0 or a negative value to disable recycling workers based on memory usage. 20 | The following comment was in the worker source code from an early dev on why they chose 45MB as the original value. Since then, we moved to a value of 53MB. 21 | Empirically ended up with 45MB per worker. 22 | They don't get killed off all the time, and on a system with 16GB RAM 23 | we end up having ~1.6GB free. 24 | 25 | DB_UPDATES_ENABLE 26 | WARNING: Do not enabled this on production hosts. This should only be enabled on local docker test environments and never in production. 27 | Set to true to allow Jetmon to update the jetpack_monitor_sites table. Without this, it is difficult to test how effective the code is working when in a local docker test environment. 28 | 29 | BUCKET_NO_MIN 30 | The first bucket in the range of jetpack_monitor_sites buckets that this host should process when checking sites. Each host should be configured to have a unique set of buckets that it is responsible for. 31 | The buckets currently range from 0 to 511. 32 | 33 | BUCKET_NO_MAX 34 | The last bucket in the range of jetpack_monitor_sites buckets that this host should process when checking sites. Each host should be configured to have a unique set of buckets that it is responsible for. 35 | The buckets currently range from 0 to 511. 36 | 37 | BATCH_SIZE 38 | The number of buckets returned in each batch when running checks. 39 | 40 | AUTH_TOKEN 41 | A string used to validate communications between different systems over HTTPS. 42 | 43 | VERIFLIER_BATCH_SIZE 44 | The maximum number of sites to send to verifliers in a single batch. 45 | 46 | SQL_UPDATE_BATCH 47 | Unknown. Likely not used currently. 48 | 49 | DB_CONFIG_UPDATES_MIN 50 | How frequently in minutes the database library should check for DB config changes in order to reload. 51 | 52 | PEER_OFFLINE_LIMIT 53 | The minimum number of verifliers that must confirm that a site is down before changing the site status to down. 54 | 55 | NUM_OF_CHECKS 56 | The number of local checks that must fail before a site is checked by the verifliers. 57 | 58 | TIME_BETWEEN_CHECKS_SEC 59 | The minimum amount of time that must elapse between local checks for a specific site. 60 | 61 | STATS_UPDATE_INTERVAL_MS 62 | The minimum delay, in milliseconds, between stats updates to both statsd and stats log files. 63 | 64 | TIME_BETWEEN_NOTICES_MIN 65 | The minimum delay, in minutes, that must pass before a site can transition from SITE_DOWN to SITE_CONFIRMED_DOWN. 66 | 67 | MIN_TIME_BETWEEN_ROUNDS_SEC 68 | The minimum delay, in seconds, between check rounds. 69 | Note: This value has no effect if USE_VARIABLE_CHECK_INTERVALS is set to true. 70 | 71 | TIMEOUT_FOR_REQUESTS_SEC 72 | The amount of time, in seconds, that a site can remain in the queuedRetries array (the queue that holds sites being checked by verifliers) before being purged out of the queue. 73 | 74 | USE_VARIABLE_CHECK_INTERVALS 75 | Set to true to enable the variable check intervals as set for each site in the jetpack_monitor_sites table. 76 | Note: Enabling this disables use of the MIN_TIME_BETWEEN_ROUNDS_SEC config, sets the round loop to execute every minute, and checks each site on the interval as set in the database. 77 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | jetmon.js 2 | ========= 3 | 4 | Overview 5 | -------- 6 | 7 | Parallel HTTP health monitoring using HEAD requests for large scale website monitoring. 8 | 9 | The service relies on confirmation from external servers to verify that sites are indeed offline. This mitigates the Internet weather issue sometimes giving false positives. The code for these servers can be found in the verifliers directory. 10 | 11 | Architecture 12 | -------- 13 | ![jetmon_chart](https://user-images.githubusercontent.com/1758399/201877599-8992b68a-9ca7-4984-9de7-abe99f989d88.png) 14 | 15 | Jetmon will periodically (every 5 minutes) loop over a list of Jetpack sites and perform a HEAD request to check their current status. 16 | 17 | When a status change is detected, Jetmon will notify WPCOM including the related notification data in the request. 18 | 19 | Here are the possible flows, depending on the status change: 20 | 21 | | Previous Status | Current status | Action | 22 | | ---------------- | ---------------- | ---------------------------------------------------------------------------------- | 23 | | DOWN | UP | Notify WPCOM about status change | 24 | | UP | DOWN | Verify status down via the Veriflier services and notify WPCOM about status change | 25 | | DOWN | DOWN (confirmed) | Notify WPCOM about status change | 26 | 27 | ### Jetmon service 28 | 29 | The Jetmon master service is responsible for communicating with the database in order to fetch a list of sites to check. It will spawn and re-allocate workers every five seconds and update stats repeatedly based on `STATS_UPDATE_INTERVAL_MS`. 30 | 31 | The jetmon-workers internally use an Node Addon written in C++ to check the connection by sending a HEAD request to the server. 32 | 33 | 34 | ### Verifliers 35 | 36 | The Veriflier service, which is written in C++ and uses the QT Framework, does something similar to the Node Addon mentioned before, but lives in its own server. Note that the production environment consists of multiple Verifliers, though the local development environment consists of a single Veriflier service. 37 | 38 | ### Notification data 39 | 40 | Here are the current notification data, Jetmon sends to WPCOM upon detecting a site status change: 41 | - `blog_id`: The site's WPCOM ID 42 | - `monitor_url`: The URL Jetmon checked 43 | - `status_id`: The site's current status. Enum: `0` is status down, `1` is status running and `2` status confirmed down. 44 | - `last_check`: The datetime of the last check 45 | - `last_status_change`: The datetime of the last status change 46 | - `checks`: An array of the checks results from both Jetmon and Veriflier services. Each entry consists of: 47 | - `type`: Enum: `1` refers to a Jetmon check, while `2` to a Veriflier check. 48 | - `host`: The server hostname. 49 | - `status`: The site's current status. Enum: `0` is status down, `1` is status running and `2` status confirmed down. 50 | - `rtt`: Round-trip time (RTT) in milliseconds (ms). 51 | - `code`: The HTTP response status code. 52 | 53 | 54 | Installation 55 | ------------ 56 | 57 | 1) Make sure you have installed [Docker](https://docs.docker.com/get-docker/) and [docker-compose](https://docs.docker.com/compose/install/) 58 | 59 | 2) Clone the Jetmon monorepo 60 | 61 | 3) Copy the environment variables file from within the `docker` folder: `cp jetmon/docker/.env-sample jetmon/docker/.env` 62 | 63 | 4) Open `jetmon/docker/.env` and make any modifications you'd like. 64 | 65 | 5) Run `docker compose build` from within the `docker` folder 66 | 67 | Configuration 68 | ------------- 69 | 70 | The Jetmon configuration lives under `config/config.json`. This file is generated on the fly, if not present, each time you run the Jetmon service, using the `config-sample.json` and the corresponding environment variables defined in `docker/.env`. 71 | Feel free to modify your local config file as needed. 72 | 73 | The Veriflier configuration lives under `veriflier/config/veriflier.json`. This file is generated on the fly, if not present, each time you run the Veriflier service, using the `veriflier-sample.json` and the corresponding environment variables defined in `docker/.env`. 74 | 75 | Running 76 | ------- 77 | 78 | Run `docker compose up -d` from within the `docker` folder. 79 | 80 | Database 81 | ------- 82 | 83 | Main Table Schema: 84 | 85 | CREATE TABLE `jetpack_monitor_sites` ( 86 | `jetpack_monitor_site_id` bigint(20) unsigned NOT NULL AUTO_INCREMENT PRIMARY KEY, 87 | `blog_id` bigint(20) unsigned NOT NULL, 88 | `bucket_no` smallint(2) unsigned NOT NULL, 89 | `monitor_url` varchar(300) NOT NULL, 90 | `monitor_active` tinyint(1) unsigned NOT NULL DEFAULT 1, 91 | `site_status` tinyint(1) unsigned NOT NULL DEFAULT 1, 92 | `last_status_change` timestamp NULL DEFAULT current_timestamp(), 93 | `check_interval` tinyint(1) unsigned NOT NULL DEFAULT 5, 94 | INDEX `blog_id_monitor_url` (`blog_id`, `monitor_url`), 95 | INDEX `bucket_no_monitor_active_check_interval` (`bucket_no`, `monitor_active`, `check_interval`) 96 | ); 97 | 98 | -------------------------------------------------------------------------------- /lib/dbpools.js: -------------------------------------------------------------------------------- 1 | 2 | const DATACENTER = 0; 3 | const READ_SLAVE = 1; 4 | const WRITE_MASTER = 2; 5 | const INTERNET_URI = 3; 6 | const INTERNAL_URI = 4; 7 | const DB_NAME = 5; 8 | const DB_USER = 6; 9 | const DB_PASSWORD = 7; 10 | 11 | const DB_CONF_FILE = 'config/db-config.conf'; 12 | const DB_ORIGINAL_FILE = 'config/db-config_original.conf'; 13 | const DB_UPDATE_SCRIPT = '/usr/local/bin/jetmon-config-update.sh'; 14 | 15 | var fs = require( 'fs' ); 16 | var mysql = require( 'mysql' ); 17 | 18 | var reloadConfig = false; 19 | var poolCluster = mysql.createPoolCluster(); 20 | 21 | poolCluster.on( 'remove', function( nodeName ) { 22 | logger.debug( 'node has been removed : ' + nodeName ); 23 | }); 24 | 25 | poolCluster.on( 'error', function( err ) { 26 | logger.error( "pool cluster error:" + err ); 27 | }); 28 | 29 | var configuration = { 30 | reload : function() { 31 | logger.debug( 'reloading the DB config' ); 32 | poolCluster = mysql.createPoolCluster(); 33 | if ( configuration.load() ) { 34 | logger.debug( 'DB config has been reloaded' ); 35 | } else { 36 | logger.error( 'DB config failed to reload' ); 37 | } 38 | }, 39 | 40 | update : function( callBack ) { 41 | var execute = require('child_process').exec; 42 | fs.stat( DB_ORIGINAL_FILE, function( err, stats ) { 43 | if ( err ) { 44 | logger.error( 'stat error on the config file: ' + err ); 45 | callBack( false ); 46 | return; 47 | } 48 | var mDate = stats.mtime.valueOf(); 49 | var result = execute( 50 | DB_UPDATE_SCRIPT, 51 | function( error, stdout, stderr ) { 52 | if ( error ) { 53 | logger.error( 'error updating the config: ' + error ); 54 | callBack( false ); 55 | } else { 56 | if ( 0 === stdout.length ) { 57 | fs.stat( 58 | DB_ORIGINAL_FILE, 59 | function( err, stats ) { 60 | if ( err ) { 61 | logger.error( 'stat error on the config file: ' + error ); 62 | callBack( false ); 63 | } else { 64 | if ( stats.mtime.valueOf() > mDate ) { 65 | callBack( true ); 66 | } else { 67 | callBack( false ); 68 | } 69 | } 70 | }); 71 | } else { 72 | callBack( false ); 73 | } 74 | } 75 | }); 76 | }); 77 | }, 78 | 79 | load: function( callBack ) { 80 | var data = fs.readFileSync( DB_CONF_FILE ); 81 | if ( undefined === data ) { 82 | logger.error( 'error loading the db config file: ' + err ); 83 | if ( undefined !== callBack ) { 84 | callBack( false ); 85 | return; 86 | } else { 87 | return false; 88 | } 89 | } 90 | var aDataLines = data.toString().split( '\n' ); 91 | var slaveUniqueCount = 1; 92 | var backupUniqueCount = 1; 93 | var currentDataset; 94 | var datasetPattern = /^\s'([-\w]+)'\s*=>\s*array\(/; 95 | var serverPattern = /^\s*array\(/; 96 | 97 | for ( var loop = 0; loop < aDataLines.length; loop++ ) { 98 | var match = datasetPattern.exec( aDataLines[loop] ); 99 | if ( match ) 100 | currentDataset = match[1]; 101 | 102 | if ( 'misc' !== currentDataset ) 103 | continue; 104 | 105 | if ( ! serverPattern.test( aDataLines[loop] ) ) 106 | continue; 107 | 108 | var arrSettings = aDataLines[loop].substr( aDataLines[loop].indexOf( "'" ), aDataLines[loop].lastIndexOf( ')' ) - aDataLines[loop].indexOf( "'" ) ); 109 | arrSettings = arrSettings.replace(/\'/g, '' ).replace(/\"/g, '' ).replace(/ /g, '' ).split( ',' ); 110 | if ( 11 != arrSettings.length ) { 111 | continue; 112 | } 113 | var db_config = { 114 | host : arrSettings[ INTERNET_URI ].split( ':' )[0], 115 | port : arrSettings[ INTERNET_URI ].split( ':' )[1], 116 | user : arrSettings[ DB_USER ], 117 | password : arrSettings[ DB_PASSWORD ], 118 | database : arrSettings[ DB_NAME ], 119 | connectionLimit : 5, 120 | supportBigNumbers : true, 121 | }; 122 | 123 | if ( 1 == arrSettings[ WRITE_MASTER ] ) { 124 | db_config['multipleStatements'] = true; 125 | poolCluster.add( 'MISC_MASTER', db_config ); 126 | } else { 127 | var _os = require( 'os' ); 128 | var aHost = _os.hostname().split( '.' ); 129 | if( aHost.length < 4 ) { 130 | logger.error( 'DB config failed to grok the installed DC' ); 131 | if ( undefined !== callBack ) { 132 | callBack( false ); 133 | } else { 134 | return false; 135 | } 136 | } 137 | var installedDC = aHost[ aHost.length - 3 ]; 138 | if ( -1 != arrSettings[ DATACENTER ].indexOf( installedDC ) ) { 139 | poolCluster.add( 'MISC_SLAVE' + slaveUniqueCount++, db_config ); 140 | } else if ( -1 == arrSettings[ DATACENTER ].indexOf( "'bak'" ) ) { 141 | // change to external URI's for non-local DC servers 142 | db_config.host = arrSettings[ INTERNET_URI ].split( ':' )[0]; 143 | db_config.port = arrSettings[ INTERNET_URI ].split( ':' )[1]; 144 | poolCluster.add( 'MISC_FAILOVER' + backupUniqueCount++, db_config ); 145 | } 146 | } 147 | } 148 | 149 | if ( undefined !== callBack ) { 150 | callBack( true ); 151 | } else { 152 | return true; 153 | } 154 | } 155 | } 156 | 157 | exports.cluster = poolCluster; 158 | exports.config = configuration; 159 | 160 | -------------------------------------------------------------------------------- /lib/server.js: -------------------------------------------------------------------------------- 1 | 2 | process.title = 'jetmon-server'; 3 | 4 | var _https = require( 'https' ); 5 | var _url = require( 'url' ); 6 | var _fs = require( 'fs' ); 7 | var config = require( './config' ); 8 | config.load(); 9 | 10 | var o_log4js = require( 'log4js' ); 11 | 12 | o_log4js.configure( { 13 | appenders: [ { 14 | 'type' : 'file', 15 | 'filename' : 'logs/jetmon.log', 16 | 'maxLogSize': 52428800, 17 | 'backups' : 30, 18 | 'category' : 'flog', 19 | 'levels' : 'DEBUG', 20 | } 21 | ] 22 | }); 23 | o_log4js.PatternLayout = '%d{HH:mm:ss,SSS} p m'; 24 | var logger = o_log4js.getLogger( 'flog' ); 25 | 26 | const NETWORK_TIMEOUT_MS = 30000; 27 | 28 | const PortNum = 7800; 29 | 30 | const SITE_DOWN = 0; 31 | const SITE_RUNNING = 1; 32 | const SITE_CONFIRMED_DOWN = 2; 33 | 34 | const HOST_OFFLINE = 0; 35 | const HOST_ONLINE = 1; 36 | 37 | const ERROR = -1; 38 | const SUCCESS = 1; 39 | 40 | var veriflierArray = config.get( 'VERIFIERS' ); 41 | var ssl_key = _fs.readFileSync( 'certs/jetmon.key' ); 42 | var ssl_cert = _fs.readFileSync( 'certs/jetmon.crt' ); 43 | 44 | var https_server = function() { 45 | 46 | var routes = { 47 | '/' : function( request, response ) { 48 | response.writeHead( 404, { 'Content-Type': 'text/html' } ); 49 | response.write( 'Unsupported call\n' ); 50 | response.end(); 51 | }, 52 | 53 | '/put/host-status' : function( request, response ) { 54 | if ( 'POST' === request.method ) { 55 | request.setEncoding( 'utf8' ); 56 | var s_data = ''; 57 | request.on( 'data', function( response_data ) { 58 | s_data += response_data; 59 | }); 60 | request.on( 'end', function() { 61 | var reply_data = {}; 62 | try { 63 | req = JSON.parse( s_data ); 64 | if ( undefined === req.auth_token || undefined === req.checks || 0 == req.checks.length ) { 65 | response.writeHead( 404, { 'Content-Type': 'application/json' } ); 66 | response.write( '{"response":' + ERROR + '}' ); 67 | response.end(); 68 | logger.error( 'invalid JSON POST data provided by ' + 69 | request.connection.remoteAddress ); 70 | return; 71 | } 72 | 73 | var veriflier = false; 74 | for ( var count in veriflierArray ) { 75 | if ( req.auth_token == veriflierArray[ count ].auth_token ) { 76 | veriflier = true; 77 | req.veriflier_host = veriflierArray[ count ].host; 78 | break; 79 | } 80 | } 81 | 82 | if ( false === veriflier ) { 83 | response.writeHead( 503, { 'Content-Type': 'application/json' } ); 84 | response.write( '{"response":' + ERROR + '}' ); 85 | response.end(); 86 | logger.error( 'invalid auth_code provided ' + req.auth_code ); 87 | return; 88 | } 89 | 90 | response.writeHead( 200, { 'Content-Type': 'application/json' } ); 91 | response.write( '{"response":' + SUCCESS + '}' ); 92 | response.end(); 93 | 94 | process.send( { msgtype: 'host_status_array', payload: req } ); 95 | } 96 | catch ( Exception ) { 97 | logger.error( 'error parsing status reply data: ' + Exception.toString() ); 98 | } 99 | }); 100 | } else { 101 | var _get = _url.parse( request.url, true ).query; 102 | if ( ( undefined == _get['d'] ) && ( "" == _get['d'] ) ) { 103 | response.writeHead( 503, { 'Content-Type': 'application/json' } ); 104 | response.write( '{"response":' + ERROR + '}' ); 105 | response.end(); 106 | logger.error( 'malformed request from server:' + _get ); 107 | return; 108 | } 109 | 110 | var req = JSON.parse( _get['d'] ); 111 | if ( undefined === req.auth_token || undefined === req.blog_id || undefined === req.status ) { 112 | response.writeHead( 404, { 'Content-Type': 'application/json' } ); 113 | response.write( '{"response":' + ERROR + '}' ); 114 | response.end(); 115 | logger.error( 'invalid JSON data provided ' + _get['d'] ); 116 | return; 117 | } 118 | 119 | var veriflier = false; 120 | for ( var count in veriflierArray ) { 121 | if ( req.auth_token == veriflierArray[ count ].auth_token ) { 122 | veriflier = true; 123 | req.veriflier_host = veriflierArray[ count ].host; 124 | break; 125 | } 126 | } 127 | 128 | if ( false === veriflier ) { 129 | response.writeHead( 503, { 'Content-Type': 'application/json' } ); 130 | response.write( '{"response":' + ERROR + '}' ); 131 | response.end(); 132 | logger.error( 'invalid auth_code provided ' + req.auth_code ); 133 | return; 134 | } 135 | 136 | response.writeHead( 200, { 'Content-Type': 'application/json' } ); 137 | response.write( '{"response":' + SUCCESS + '}' ); 138 | response.end(); 139 | 140 | process.send( { msgtype: 'host_status', payload: req } ); 141 | } 142 | }, 143 | 144 | '/get/status' : function( request, response ) { 145 | logger.error( 'status confirmation requested' ); 146 | response.writeHead( 200, { 'Content-Type': 'text/plain' } ); 147 | response.write( 'OK' ); 148 | response.end(); 149 | } 150 | }; 151 | 152 | var ssl_options = { 153 | key: ssl_key, 154 | cert: ssl_cert, 155 | }; 156 | 157 | var request_handler = function( request, response ) { 158 | var arr_req = request.url.toString().split( '?' ); 159 | if ( arr_req instanceof Array ) { 160 | if( undefined === routes[ arr_req[0] ] ) { 161 | response.writeHead( 404, { 'Content-Type': 'text/plain' } ); 162 | response.write( 'not found\n' ); 163 | response.end(); 164 | } else { 165 | routes[ arr_req[0] ].call( this, request, response ); 166 | } 167 | } else { 168 | response.writeHead( 404, { 'Content-Type': 'text/plain' } ); 169 | response.write( 'Unsupported call\n' ); 170 | response.end(); 171 | logger.error( 'unsupported call: ' + request.url.toString() ); 172 | } 173 | }; 174 | 175 | var close_handler = function() { 176 | logger.error( process.pid + ': HTTPS server has been shutdown.' ); 177 | }; 178 | 179 | var error_handler = function( err ) { 180 | logger.error( process.pid + ': HTTPS error encountered: ' + err ); 181 | }; 182 | 183 | var _server = _https.createServer( ssl_options ). 184 | addListener( 'request', request_handler ) 185 | .addListener( 'close', close_handler ) 186 | .addListener( 'error', error_handler ) 187 | .listen( PortNum ); 188 | }; 189 | 190 | new https_server(); 191 | 192 | -------------------------------------------------------------------------------- /lib/database.js: -------------------------------------------------------------------------------- 1 | 2 | const SITE_DOWN = 0; 3 | const SITE_RUNNING = 1; 4 | const SITE_CONFIRMED_DOWN = 2; 5 | 6 | const SECONDS = 1000; 7 | const MINUTES = 60 * SECONDS; 8 | const HOURS = 60 * MINUTES; 9 | const DAYS = 24 * HOURS; 10 | 11 | var pool = require( './dbpools' ); 12 | 13 | var fromBucketNo = global.config.get( 'BUCKET_NO_MIN' ); 14 | var toBucketNo = global.config.get( 'BUCKET_NO_MIN' ) + global.config.get( 'BATCH_SIZE' ) - 1; 15 | 16 | var arrUpdateStatements = []; 17 | var arrUpdateQueue = []; 18 | var reloadConfig = false; 19 | 20 | var database = { 21 | init : function( success ) { 22 | if ( ! success ) { 23 | console.error( 'failed to load the DB config file, exiting...' ); 24 | process.exit( 1 ); 25 | } 26 | }, 27 | 28 | updateConfig : function() { 29 | logger.debug( 'checking if the DB config file has been updated.' ); 30 | pool.config.update( 31 | function( updated ) { 32 | if ( updated ) { 33 | logger.debug( 'updated DB config file detected, setting reloadConfig variable.' ); 34 | reloadConfig = true; 35 | } else { 36 | logger.debug( 'no DB config update detected.' ); 37 | } 38 | }); 39 | }, 40 | 41 | execQuery : function( sqlQuery, callBack ) { 42 | if ( reloadConfig ) { 43 | pool.config.reload(); 44 | reloadConfig = false; 45 | } 46 | var poolPrefix = new String( 'USER_' ); 47 | if ( -1 !== sqlQuery.indexOf( 'jetpack_' ) ) { 48 | poolPrefix = 'MISC_'; 49 | } else if ( -1 !== sqlQuery.indexOf( 'languages' ) ) { 50 | poolPrefix = 'GLOBAL_'; 51 | } 52 | // round-robin select the relevant read-only server 53 | pool.cluster.getConnection( 54 | poolPrefix + 'SLAVE*', 55 | function( err, connection ) { 56 | if ( err ) { 57 | logger.error( 'error connecting to local DC slave db: ' + err ); 58 | 59 | // round-robin select a read-only failover server 60 | pool.cluster.getConnection( 61 | poolPrefix + 'FAILOVER*', 62 | function( err, connection ) { 63 | if ( err ) { 64 | callBack( 'error connecting to a remote failover db: ' + err, new Array() ); 65 | } else { 66 | if ( true === global.config.get( 'DEBUG' ) ) { 67 | logger.debug( 'running: ' + sqlQuery ); 68 | } 69 | connection.query( 70 | sqlQuery, 71 | function( error, rows ) { 72 | callBack( error, rows ); 73 | connection.release(); 74 | }); 75 | } 76 | }); 77 | } else { 78 | if ( true === global.config.get( 'DEBUG' ) ) 79 | logger.debug( 'running: ' + sqlQuery ); 80 | connection.query( 81 | sqlQuery, 82 | function( error, rows ) { 83 | callBack( error, rows ); 84 | connection.release(); 85 | }); 86 | } 87 | }); 88 | }, 89 | 90 | getNextBatch : function( afterQueryFunction ) { 91 | if ( global.config.get( 'USE_VARIABLE_CHECK_INTERVALS' ) ) { 92 | /** 93 | * If variable check intervals are enabled, use a different query to 94 | * spread out the sites across one-minute intervals. 95 | */ 96 | var query = 'SELECT `blog_id`, `monitor_url`, `site_status`, `last_status_change` ' + 97 | 'FROM `jetpack_monitor_sites` WHERE `bucket_no` >= ' + 98 | fromBucketNo + ' AND `bucket_no` <= ' + toBucketNo + ' AND `monitor_active` = 1 AND ' + 99 | 'MOD(MINUTE(NOW()) + `jetpack_monitor_site_id`, `check_interval`) = 0;'; 100 | } else { 101 | var query = 'SELECT `blog_id`, `monitor_url`, `site_status`, `last_status_change` ' + 102 | 'FROM `jetpack_monitor_sites` WHERE `bucket_no` >= ' + 103 | fromBucketNo + ' AND `bucket_no` <= ' + toBucketNo + ' AND `monitor_active` = 1'; 104 | } 105 | 106 | database.execQuery( 107 | query, 108 | function( error, rows ) { 109 | if ( error ) { 110 | logger.debug( 'error fetching records: ' + error ); 111 | fromBucketNo -= global.config.get( 'BATCH_SIZE' ); 112 | toBucketNo -= global.config.get( 'BATCH_SIZE' ); 113 | } else { 114 | afterQueryFunction( rows ); 115 | } 116 | }); 117 | 118 | fromBucketNo = fromBucketNo + global.config.get( 'BATCH_SIZE' ); 119 | if ( fromBucketNo >= global.config.get( 'BUCKET_NO_MAX' ) ) { 120 | fromBucketNo = global.config.get( 'BUCKET_NO_MIN' ); 121 | } 122 | toBucketNo = fromBucketNo + global.config.get( 'BATCH_SIZE' ) - 1; 123 | if ( toBucketNo > global.config.get( 'BUCKET_NO_MAX' ) ) { 124 | toBucketNo = global.config.get( 'BUCKET_NO_MAX' ); 125 | } 126 | // if we have 'wrapped' around to the start again, then return that we are finished for this round 127 | return ( global.config.get( 'BUCKET_NO_MIN' ) === fromBucketNo ); 128 | }, 129 | 130 | updateSite : function( blog_id, monitor_url, site_status ) { 131 | var query = "UPDATE `jetpack_monitor_sites` " + 132 | "SET `site_status`=" + Number( site_status ) + ", `last_status_change`=NOW() " + 133 | "WHERE `blog_id`=" + Number( blog_id ) + " AND `monitor_url`='" + monitor_url + "'"; 134 | 135 | database.execQuery( 136 | query, 137 | function( error, rows ) { 138 | if ( error ) { 139 | logger.debug( 'error updating site: ' + error ); 140 | } 141 | } 142 | ); 143 | }, 144 | 145 | getNowDateTime : function() { 146 | var now = new Date(); 147 | return now.getFullYear() + '-' + 148 | ( 1 == ( 1 + now.getMonth() ).length ? '0' + ( 1 + now.getMonth() ) : ( 1 + now.getMonth() ) ) + '-' + 149 | ( 1 == now.getDate().toString().length ? '0' + now.getDate() : now.getDate() ) + ' ' + 150 | ( 1 == now.getHours().toString().length ? '0' + now.getHours() : now.getHours() ) + ':' + 151 | ( 1 == now.getMinutes().toString().length ? '0' + now.getMinutes() : now.getMinutes() ) + ':' + 152 | ( 1 == now.getSeconds().toString().length ? '0' + now.getSeconds() : now.getSeconds() ); 153 | }, 154 | 155 | commitUpdates : function( callBack ) { 156 | if ( 0 == arrUpdateStatements.length ) { 157 | if ( undefined !== callBack ) 158 | callBack(); 159 | return; 160 | } 161 | if ( reloadConfig ) { 162 | pool.config.reload(); 163 | reloadConfig = false; 164 | } 165 | pool.cluster.getConnection( 166 | 'MISC_MASTER', 167 | function( err, conn_write ) { 168 | if ( undefined != err ) { 169 | logger.error( 'error getting a connection: ' + err.code ); 170 | if ( undefined !== callBack ) { 171 | callBack(); 172 | } 173 | return; 174 | } 175 | var up_query = ''; 176 | for ( var uploop = 0; uploop < arrUpdateStatements.length; uploop++ ) { 177 | up_query = up_query + arrUpdateStatements[uploop]; 178 | } 179 | if ( true === global.config.get( 'DEBUG' ) ) { 180 | logger.debug( 'RUNNING batch : ' + up_query ); 181 | } 182 | conn_write.query( 183 | up_query, 184 | function( error, rows ) { 185 | if ( error ) { 186 | logger.error( 'Error updating: ' + error.code ); 187 | } else { 188 | arrUpdateStatements = []; 189 | } 190 | conn_write.release(); 191 | if ( undefined !== callBack ) { 192 | callBack(); 193 | } 194 | }); 195 | }); 196 | } 197 | }; 198 | 199 | // initialise the database settings 200 | pool.config.load( database.init ); 201 | 202 | // set a repeating 'tick' to perform the database config update checks and conditional reloads 203 | setInterval( database.updateConfig, global.config.get( 'DB_CONFIG_UPDATES_MIN' ) * MINUTES ); 204 | 205 | module.exports = database; 206 | 207 | -------------------------------------------------------------------------------- /veriflier/source/http_checker.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include 3 | #include "headers/http_checker.h" 4 | #include "headers/logger.h" 5 | 6 | using namespace std; 7 | 8 | HTTP_Checker::HTTP_Checker( const int p_net_timeout ) : QObject( NULL ), m_ssl_config( NULL ), 9 | m_sock( NULL ), m_timeout( NULL ), m_host_name( "" ), m_host_dir( "" ), 10 | m_port( DEFAULT_HTTP_PORT ), m_is_ssl( false ), m_finished( false ), 11 | m_redirects( 0 ), m_response_code( 0 ), m_net_timeout( p_net_timeout ) 12 | { 13 | m_starttime = QDateTime::currentDateTime(); 14 | m_ssl_config = new QSslConfiguration(); 15 | m_ssl_config->setProtocol( QSsl::SecureProtocols ); 16 | } 17 | 18 | HTTP_Checker::~HTTP_Checker() { 19 | this->closeConnection(); 20 | delete m_ssl_config; 21 | } 22 | 23 | void HTTP_Checker::check( HealthCheck* p_hc ) { 24 | try { 25 | m_hc = p_hc; 26 | m_host_name = m_hc->monitor_url; 27 | m_host_dir = '/'; 28 | 29 | m_timeout = new QTimer( this ); 30 | QObject::connect( m_timeout, SIGNAL( timeout() ), this, SLOT( timed_out() ) ); 31 | m_timeout->start( m_net_timeout ); 32 | 33 | this->parse_host_values(); 34 | this->connect(); 35 | } 36 | catch ( exception &ex ) { 37 | LOG( QString( "exception in HTTP_Checker::check(): for host '" ) + m_host_name + "' : " + ex.what() ); 38 | } 39 | } 40 | 41 | void HTTP_Checker::process_response() { 42 | // if we have been redirected, get the details and make a recursive call 43 | if ( ( 300 < m_response_code ) && ( 400 > m_response_code ) ) { 44 | m_redirects++; 45 | if ( Config::instance()->get_int_value( "max_redirects" ) >= m_redirects && 46 | set_redirect_host_values( m_response.toStdString().c_str() ) ) { 47 | this->closeConnection(); 48 | LOG( QString::number( m_starttime.msecsTo( QDateTime::currentDateTime() ) ) + 49 | " \t: STAGE 1 :\tredirecting to " + m_host_name + m_host_dir ); 50 | m_response_code = 0; 51 | this->connect(); 52 | } else { 53 | // Note we leave the 3xx response code so this site is marked as up 54 | finish_request(); 55 | } 56 | } else { 57 | finish_request(); 58 | } 59 | } 60 | 61 | bool HTTP_Checker::set_redirect_host_values( QString p_content ) { 62 | try { 63 | QString p_lcase_search = p_content.toLower(); 64 | if ( -1 == p_lcase_search.indexOf( "location: " ) ) 65 | return false; 66 | 67 | p_content = p_content.mid( p_lcase_search.indexOf( "location: " ) + 10, p_content.length() - ( p_lcase_search.indexOf( "location: " ) + 10 ) ); 68 | if ( -1 == p_content.indexOf( "\r\n" ) ) 69 | return false; 70 | 71 | p_content.remove( p_content.indexOf( "\r\n" ), p_content.length() - p_content.indexOf( "\r\n" ) ); 72 | 73 | // keep a copy for relative location redirects 74 | QString hostname_backup = m_host_name; 75 | m_host_name = p_content; 76 | m_port = DEFAULT_HTTP_PORT; 77 | m_host_dir = '/'; 78 | m_is_ssl = false; 79 | 80 | this->parse_host_values(); 81 | 82 | // this is a relative location redirect, reinstate hostname 83 | if ( 0 == m_host_name.size() ) 84 | m_host_name = hostname_backup; 85 | 86 | return true; 87 | } 88 | catch( exception &ex ) { 89 | LOG( QString( "exception in HTTP_Checker::set_redirect_host_values(): " ) + ex.what() ); 90 | return false; 91 | } 92 | } 93 | 94 | void HTTP_Checker::parse_host_values() { 95 | if ( -1 != m_host_name.indexOf( "http://" ) ) { 96 | m_host_name.remove( m_host_name.indexOf( "http://" ), 7 ); 97 | m_is_ssl = false; 98 | } 99 | 100 | if ( -1 != m_host_name.indexOf( "https://" ) ) { 101 | m_host_name.remove( m_host_name.indexOf( "https://" ), 8 ); 102 | m_is_ssl = true; 103 | m_port = DEFAULT_HTTPS_PORT; 104 | } 105 | 106 | size_t s_pos = m_host_name.indexOf( '/' ); 107 | size_t q_pos = m_host_name.indexOf( '?' ); 108 | size_t c_pos = m_host_name.indexOf( ':' ); 109 | size_t f_pos = m_host_name.indexOf( '#' ); 110 | 111 | if ( ( c_pos < s_pos ) && ( c_pos < q_pos ) && ( c_pos < f_pos ) ) { 112 | int new_port = m_host_name.mid( c_pos + 1, min( s_pos, (size_t)m_host_name.length() ) ).toInt(); 113 | if ( 0 < new_port ) { 114 | m_port = new_port; 115 | m_host_name.remove( c_pos, min( s_pos, (size_t)m_host_name.length() ) - c_pos ); 116 | // recalc since we've erased some characters 117 | s_pos = m_host_name.indexOf( '/' ); 118 | q_pos = m_host_name.indexOf( '?' ); 119 | f_pos = m_host_name.indexOf( '#' ); 120 | } 121 | } 122 | 123 | if ( string::npos != s_pos || string::npos != q_pos || string::npos != f_pos ) { 124 | int m_pos = min( min( s_pos, q_pos ), f_pos ); 125 | m_host_dir = m_host_name.mid( m_pos, m_host_name.length() - m_pos ); 126 | if ( 0 == m_host_dir.length() || '?' == m_host_dir[0] || '#' == m_host_dir[0] ) { 127 | m_host_dir = "/" + m_host_dir; 128 | } 129 | m_host_name.remove( m_pos, m_host_name.length() - m_pos ); 130 | } 131 | } 132 | 133 | void HTTP_Checker::parse_response_code( QByteArray a_data ) { 134 | if ( 0 == a_data.size() ) { 135 | return; 136 | } 137 | 138 | m_response = a_data.toStdString().c_str(); 139 | if ( m_response.indexOf( " " ) == 8 ) { 140 | m_response_code = m_response.mid( 9, 3 ).toInt(); 141 | } else { 142 | m_response_code = -1; 143 | } 144 | } 145 | 146 | bool HTTP_Checker::send_http_get() { 147 | QString m_buf = "HEAD " + m_host_dir + " HTTP/1.1\r\n"; 148 | m_buf += "Host: " + m_host_name + "\r\n"; 149 | m_buf += "User-Agent: jetmon/1.0 (Jetpack Site Uptime Monitor by WordPress.com)\r\n"; 150 | m_buf += "Connection: close\r\n\r\n"; 151 | 152 | qint64 bytes_sent = m_sock->write( m_buf.toStdString().c_str(), m_buf.length() ); 153 | 154 | return ( bytes_sent == m_buf.length() ); 155 | } 156 | 157 | void HTTP_Checker::connect() { 158 | if ( m_is_ssl ) { 159 | m_sock = new QSslSocket(); 160 | ((QSslSocket*)m_sock)->setSslConfiguration( *m_ssl_config ); 161 | QObject::connect( ((QSslSocket*)m_sock), SIGNAL( connected() ), this, SLOT( connected() ) ); 162 | QObject::connect( ((QSslSocket*)m_sock), SIGNAL( readyRead() ), this, SLOT( readyRead() ) ); 163 | QObject::connect( ((QSslSocket*)m_sock), SIGNAL( error( QAbstractSocket::SocketError) ), this, SLOT( connectionError( QAbstractSocket::SocketError) ) ); 164 | ((QSslSocket*)m_sock)->connectToHostEncrypted( m_host_name, m_port ); 165 | } else { 166 | m_sock = new QTcpSocket(); 167 | QObject::connect( m_sock, SIGNAL( connected() ), this, SLOT( connected() ) ); 168 | QObject::connect( m_sock, SIGNAL( readyRead() ), this, SLOT( readyRead() ) ); 169 | QObject::connect( m_sock, SIGNAL( error( QAbstractSocket::SocketError) ), this, SLOT( connectionError( QAbstractSocket::SocketError) ) ); 170 | m_sock->connectToHost( m_host_name, m_port ); 171 | } 172 | } 173 | 174 | void HTTP_Checker::connected() { 175 | try { 176 | if ( ! m_sock->isOpen() ) { 177 | finish_request(); 178 | return; 179 | } 180 | send_http_get(); 181 | } 182 | catch( exception &ex ) { 183 | LOG( QString( "exception in HTTP_Checker::connected(): for host '" ) + m_host_name + "' : " + ex.what() ); 184 | } 185 | } 186 | 187 | void HTTP_Checker::connectionError( QAbstractSocket::SocketError err ) { 188 | //LOG( "Connection Error[" + QString::number( err ) + "]: " + m_sock->errorString() ); 189 | finish_request(); 190 | } 191 | 192 | void HTTP_Checker::readyRead() { 193 | QByteArray a_data = m_sock->readAll(); 194 | 195 | if ( 0 == a_data.length() ) { 196 | LOG( "NO data received from the check." ); 197 | return; 198 | } 199 | 200 | if ( 0 == m_response_code ) { 201 | parse_response_code( a_data ); 202 | if ( 0 != m_response_code ) { 203 | process_response(); 204 | } 205 | } 206 | } 207 | 208 | void HTTP_Checker::closeConnection() { 209 | if ( m_sock != NULL ) { 210 | if ( m_sock->isOpen() ) 211 | m_sock->close(); 212 | m_sock->deleteLater(); 213 | } 214 | } 215 | 216 | void HTTP_Checker::timed_out() { 217 | if ( m_sock != NULL ) { 218 | if ( m_sock->isOpen() ) 219 | m_sock->disconnectFromHost(); 220 | } 221 | m_timeout->stop(); 222 | finish_request(); 223 | } 224 | 225 | void HTTP_Checker::finish_request() { 226 | if ( ! m_finished ) { 227 | m_finished = true; 228 | emit finished( this, m_hc ); 229 | } 230 | } 231 | 232 | -------------------------------------------------------------------------------- /veriflier/source/check_controller.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include "headers/check_controller.h" 3 | #include "headers/logger.h" 4 | 5 | #include 6 | #include 7 | #include 8 | 9 | CheckController::CheckController( const QSslConfiguration *ssl_config, const int jetmon_server_port, 10 | const int max_runners, const int max_checks, const QString &veriflier_name, 11 | const QString &auth_token, const int net_timeout, const bool debug ) 12 | : m_ssl_config( ssl_config ), m_jetmon_server_port( jetmon_server_port ), m_socket( NULL ), 13 | m_max_checkers( max_runners ), m_max_checks( max_checks ), m_checking( 0 ), m_checked( 0 ), 14 | m_veriflier_name( veriflier_name ), m_auth_token( auth_token ), m_net_timeout( net_timeout ), m_debug( debug ) 15 | { 16 | m_checks.resize( 0 ); 17 | m_ticker = new QTimer( this ); 18 | connect( m_ticker, SIGNAL( timeout() ), this, SLOT( ticked() ) ); 19 | m_ticker->start( 5000 ); 20 | 21 | for ( int thread_index = 0; thread_index < m_max_checkers; thread_index++ ) { 22 | CheckThread *ct = new CheckThread( m_net_timeout, m_debug, thread_index ); 23 | connect( ct, SIGNAL( resultReady(int, qint64, QString, int, int, int) ), this, SLOT( finishedChecking(int, qint64, QString, int, int, int) ) ); 24 | Runner* run = new Runner(); 25 | run->ct = ct; 26 | run->checking = 0; 27 | run->ct->start(); 28 | m_runners.push_back(run); 29 | } 30 | connect( this, SIGNAL(startCheck(HealthCheck*)), this, SLOT(startChecking(HealthCheck*)) ); 31 | } 32 | 33 | CheckController::~CheckController() { 34 | m_ticker->stop(); 35 | delete m_ticker; 36 | delete m_socket; 37 | for ( int checker = 0; checker < m_runners.length(); checker++ ) { 38 | delete m_runners[checker]->ct; 39 | m_runners[checker]->ct = NULL; 40 | } 41 | qDeleteAll(m_runners); 42 | } 43 | 44 | void CheckController::finishedChecking( int thread_index, qint64 blog_id, QString monitor_url, int status, int http_code, int rtt ) { 45 | QJsonDocument json_doc; 46 | QJsonObject json_obj, arr_result; 47 | QJsonArray checkArray; 48 | m_checked++; 49 | m_check_lock.lock(); 50 | for ( int loop = 0; loop < m_checks.size(); loop++ ) { 51 | if ( m_checks[loop]->blog_id == blog_id && m_checks[loop]->monitor_url == monitor_url ) { 52 | if ( 0 > m_checks[loop]->thread_index ) { 53 | LOG( "deleting a blog_id that does not have a check thread assigned?: " + QString::number( blog_id ) + " " + monitor_url ); 54 | } 55 | if ( thread_index != m_checks[loop]->thread_index ) { 56 | LOG( "deleting a blog_id that has a different thread_index linked: " + 57 | QString::number( m_checks[loop]->thread_index ) + " != " + QString::number( thread_index ) ); 58 | } 59 | arr_result.insert( "blog_id", QJsonValue( blog_id ) ); 60 | arr_result.insert( "monitor_url", QJsonValue( monitor_url ) ); 61 | arr_result.insert( "status", QJsonValue( status ) ); 62 | arr_result.insert( "code", QJsonValue( http_code ) ); 63 | arr_result.insert( "rtt", QJsonValue( rtt ) ); 64 | QMap::iterator itr = m_check_results.find( m_checks[loop]->jetmon_server ); 65 | if ( m_check_results.end() != itr ) { 66 | json_doc = itr.value(); 67 | checkArray = json_doc.object()["checks"].toArray(); 68 | } 69 | checkArray.append( arr_result ); 70 | json_obj.insert( "auth_token", QJsonValue( m_auth_token ) ); 71 | json_obj.insert( "checks", checkArray ); 72 | json_doc.setObject( json_obj ); 73 | m_check_results.insert( m_checks[loop]->jetmon_server, json_doc ); 74 | 75 | HealthCheck *ptr = m_checks[loop]; 76 | m_checks.remove( loop ); 77 | delete ptr; 78 | m_runners[thread_index]->checking--; 79 | m_checking--; 80 | break; 81 | } 82 | } 83 | if ( m_checking < ( m_runners.length() * m_max_checks ) ) { 84 | for ( int loop = 0; loop < m_checks.size(); loop++ ) { 85 | if ( NOT_ASSIGNED == m_checks[loop]->thread_index ) { 86 | m_checks[loop]->thread_index = PRE_ASSIGNED; 87 | emit startCheck( m_checks[loop] ); 88 | m_check_lock.unlock(); 89 | return; 90 | } 91 | } 92 | } 93 | m_check_lock.unlock(); 94 | } 95 | 96 | inline bool CheckController::haveCheck( qint64 blog_id, QString monitor_url ) { 97 | for ( int loop = 0; loop < m_checks.size(); loop++ ) { 98 | if ( m_checks[loop]->blog_id == blog_id && m_checks[loop]->monitor_url == monitor_url ) { 99 | return true; 100 | } 101 | } 102 | return false; 103 | } 104 | 105 | void CheckController::startChecking( HealthCheck* hc ) { 106 | m_checking++; 107 | int runner = this->selectRunner(); 108 | m_runners[runner]->checking++; 109 | hc->thread_index = runner; 110 | m_runners[runner]->ct->performCheck( hc ); 111 | } 112 | 113 | int CheckController::selectRunner() { 114 | int min = m_max_checks; 115 | int min_index = 0; 116 | for ( int index = 0; index < m_runners.length(); index++ ) { 117 | if ( m_runners[index]->checking < min ) { 118 | min = m_runners[index]->checking; 119 | min_index = index; 120 | } 121 | } 122 | return min_index; 123 | } 124 | 125 | void CheckController::addCheck( HealthCheck* hc ) { 126 | if ( haveCheck( hc->blog_id, hc->monitor_url ) ) { 127 | LOG( "ERROR:\t: already have this blog in the check list: " + QString::number( hc->blog_id ) + " " + hc->monitor_url ); 128 | return; 129 | } 130 | 131 | m_checks.append( hc ); 132 | if ( m_checking < ( m_runners.length() * m_max_checks ) ) { 133 | hc->thread_index = PRE_ASSIGNED; 134 | emit startCheck( hc ); 135 | } 136 | } 137 | 138 | void CheckController::addChecks( QVector hcs ) { 139 | m_check_lock.lock(); 140 | for ( int loop = 0; loop < hcs.size(); loop++ ) { 141 | this->addCheck( hcs[loop] ); 142 | } 143 | m_check_lock.unlock(); 144 | } 145 | 146 | void CheckController::ticked() { 147 | this->sendResults(); 148 | 149 | if ( m_checks.size() > 0 || m_checked > 0 ) { 150 | LOG( "total - " + QString::number( m_checks.size() ) + 151 | " : checking - " + QString::number( m_checking ) + 152 | " : checked = " + QString::number( m_checked ) ); 153 | for ( int index = 0; index < m_runners.length(); index++ ) { 154 | LOG( "runner " + QString::number( index ) + "\t: checking " + QString::number( m_runners[index]->checking ) ); 155 | } 156 | } 157 | m_checked = 0; 158 | } 159 | 160 | void CheckController::sendResults() { 161 | if ( 0 == m_check_results.size() ) 162 | return; 163 | 164 | m_check_lock.lock(); 165 | QMap sendMap( m_check_results ); 166 | m_check_results.clear(); 167 | m_check_lock.unlock(); 168 | 169 | QMap::const_iterator itr = sendMap.begin(); 170 | while ( itr != sendMap.end() ) { 171 | QByteArray arr_data; 172 | arr_data.append( post_http_header( QString( itr.key().toStdString().c_str() ), itr.value().toJson().size() ) ); 173 | arr_data.append( itr.value().toJson() ); 174 | LOG( "\t\t: SENDING :\t" + QString::number( itr.value().object()["checks"].toArray().size() ) + " results" ); 175 | this->sendToJetmonServer( QString( itr.key().toStdString().c_str() ), arr_data ); 176 | itr++; 177 | } 178 | } 179 | 180 | QString CheckController::post_http_header( QString jetmon_server, int content_size ) { 181 | QString ret_val = "POST /put/host-status"; 182 | ret_val += " HTTP/1.1\r\nHost: "; 183 | ret_val += jetmon_server; 184 | ret_val += "\r\nContent-Type: application/json"; 185 | ret_val += "\r\nContent-Length: " + QString::number( content_size ); 186 | ret_val += "\r\nConnection: Keep-Alive\r\n\r\n"; 187 | 188 | return ret_val; 189 | } 190 | 191 | void CheckController::sendToJetmonServer( QString jetmon_server, QByteArray status_data ) { 192 | JetmonServer * js = new JetmonServer( this, m_ssl_config, jetmon_server, m_jetmon_server_port ); 193 | QObject::connect( js, SIGNAL( finished( JetmonServer*, int, int ) ), SLOT( finishedSending( JetmonServer*, int, int ) ) ); 194 | js->sendData( status_data ); 195 | } 196 | 197 | void CheckController::finishedSending( JetmonServer* js, int status, int rtt ) { 198 | if ( 1 == status ) { 199 | LOG( QString::number( rtt ) + "\t\t: SENDING :\tsent - 1" ); 200 | } else { 201 | LOG( QString::number( rtt ) + "\t\t: SENDING :\tfailed to connect to :" + js->jetmonServer() ); 202 | } 203 | js->deleteLater(); 204 | } 205 | -------------------------------------------------------------------------------- /veriflier/source/client_thread.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include "headers/client_thread.h" 3 | #include "headers/logger.h" 4 | 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | ClientThread::ClientThread( qintptr sock, const QSslConfiguration *ssl_config, 11 | CheckController *checker, const QString &veriflier_name, 12 | const QString &auth_token, const int net_timeout, const bool debug ) 13 | : m_sock( sock ), m_socket( NULL ), m_ssl_config( ssl_config ), m_checker( checker ), 14 | m_veriflier_name( veriflier_name ), m_auth_token( auth_token ), m_net_timeout( net_timeout ), 15 | m_debug( debug ), m_site_status_request( false ) 16 | { 17 | ; 18 | } 19 | 20 | ClientThread::~ClientThread() { 21 | delete m_socket; 22 | } 23 | 24 | void ClientThread::run() { 25 | m_socket = new QSslSocket(); 26 | 27 | if ( ! m_socket->setSocketDescriptor( m_sock ) ) { 28 | LOG ( "Unable to set file descriptor for server SSL connection." ); 29 | return; 30 | } 31 | 32 | m_socket->setSslConfiguration( *m_ssl_config ); 33 | m_socket->startServerEncryption(); 34 | 35 | if ( ! m_socket->waitForEncrypted() ) { 36 | LOG( "Unable to negotiate SSL for server request: " + m_socket->errorString() ); 37 | m_socket->close(); 38 | return; 39 | } 40 | 41 | if ( m_socket->encryptedBytesToWrite() ) { 42 | m_socket->flush(); 43 | } 44 | 45 | // Store the jetmon server's address for our reply 46 | m_jetmon_server = m_socket->peerAddress().toString(); 47 | 48 | if ( m_socket->waitForReadyRead( m_net_timeout ) ) { 49 | this->readRequest(); 50 | } 51 | 52 | if ( ! m_site_status_request ) { 53 | m_socket->close(); 54 | return; 55 | } 56 | 57 | // Tell the Jetmon server we have received and validated the request 58 | this->sendOK(); 59 | m_socket->close(); 60 | 61 | if ( m_debug ) { 62 | LOG( "RECV\t: ------- :\t " + QString::number( m_checks.size() ) ); 63 | } 64 | m_checker->addChecks( m_checks ); 65 | } 66 | 67 | ClientThread::QueryType ClientThread::get_request_type( QByteArray &raw_data ) { 68 | QString s_data = raw_data.data(); 69 | int pos = s_data.indexOf( "HTTP/1." ); 70 | 71 | if ( -1 == pos ) { 72 | LOG( "Invalid HTTP request format." ); 73 | this->sendError( "Invalid HTTP request format." ); 74 | return ClientThread::UnknownQuery; 75 | } 76 | 77 | s_data = s_data.left( pos - 1 ); 78 | 79 | if ( s_data.startsWith( "GET /get/status") ) { 80 | return ClientThread::ServiceRunning; 81 | } 82 | 83 | if ( s_data.startsWith( "GET /get/host-status") ) { 84 | return ClientThread::SiteStatusCheck; 85 | } 86 | 87 | if ( s_data.startsWith( "POST /get/host-status") ) { 88 | return ClientThread::SiteStatusPostCheck; 89 | } 90 | 91 | return ClientThread::UnknownQuery; 92 | } 93 | 94 | QJsonDocument ClientThread::parse_json_request( QByteArray &raw_data ) { 95 | QJsonDocument ret_val; 96 | QString s_data = raw_data.data(); 97 | int pos = s_data.indexOf( "HTTP/1." ); 98 | 99 | if ( -1 == pos ) { 100 | LOG( "Invalid HTTP request format." ); 101 | this->sendError( "Invalid HTTP request format." ); 102 | return ret_val; 103 | } 104 | 105 | s_data = s_data.left( pos - 1 ); 106 | s_data = s_data.right( s_data.length() - s_data.indexOf( "/" ) ); 107 | s_data = s_data.right( s_data.length() - s_data.indexOf( "?d=" ) - 3 ); 108 | 109 | ret_val = QJsonDocument::fromJson( s_data.toUtf8() ); 110 | 111 | return ret_val; 112 | } 113 | 114 | QJsonDocument ClientThread::parse_json_request_post( QByteArray &raw_data ) { 115 | QJsonDocument ret_val; 116 | QString s_data = raw_data.data(); 117 | int pos = s_data.indexOf( " HTTP/" ); 118 | 119 | if ( -1 == pos ) { 120 | LOG( "Invalid HTTP request format." ); 121 | this->sendError( "Invalid HTTP request format." ); 122 | return ret_val; 123 | } 124 | 125 | pos = s_data.indexOf( "\r\n\r\n" ); 126 | 127 | if ( -1 == pos ) { 128 | LOG( "Invalid HTTP request format." ); 129 | this->sendError( "Invalid HTTP request format." ); 130 | return ret_val; 131 | } 132 | 133 | s_data = s_data.right( s_data.length() - pos - 4 ); 134 | ret_val = QJsonDocument::fromJson( s_data.toUtf8() ); 135 | 136 | return ret_val; 137 | } 138 | 139 | int ClientThread::parse_json_request_post_length( QByteArray &raw_data ) { 140 | QString s_data = raw_data.data(); 141 | int pos = s_data.indexOf( " HTTP/" ); 142 | 143 | if ( -1 == pos ) { 144 | LOG( "Invalid HTTP request format." ); 145 | this->sendError( "Invalid HTTP request format." ); 146 | return -1; 147 | } 148 | 149 | pos = s_data.indexOf( "\r\n\r\n" ); 150 | 151 | if ( -1 == pos ) { 152 | LOG( "Invalid HTTP request format." ); 153 | this->sendError( "Invalid HTTP request format." ); 154 | return -1; 155 | } 156 | 157 | return ( s_data.length() - pos - 4 ); 158 | } 159 | 160 | int ClientThread::get_content_length( QByteArray &raw_data ) { 161 | QString s_data = raw_data.data(); 162 | int pos = s_data.indexOf( " HTTP/" ); 163 | 164 | if ( -1 == pos ) { 165 | LOG( "Invalid HTTP request format." ); 166 | return -1; 167 | } 168 | 169 | pos = s_data.indexOf( "Content-Length: " ); 170 | if ( -1 == pos ) { 171 | pos = s_data.indexOf( "GET /" ); 172 | if ( 0 == pos ) 173 | return 0; 174 | LOG( "Unable to get 'Content-Length'." ); 175 | return -1; 176 | } 177 | 178 | pos += 16; 179 | s_data = s_data.right( s_data.length() - pos ); 180 | pos = s_data.indexOf( "\r\n" ); 181 | if ( -1 == pos ) { 182 | LOG( "Unable to get 'Content-Length' termination characters." ); 183 | return -1; 184 | } 185 | 186 | s_data = s_data.left( pos ); 187 | return s_data.toInt(); 188 | } 189 | 190 | void ClientThread::readRequest() { 191 | QByteArray a_data = m_socket->readAll(); 192 | 193 | if ( 0 == a_data.length() ) { 194 | LOG( "NO data received from the jetmon server." ); 195 | return; 196 | } 197 | 198 | QueryType type = get_request_type( a_data ); 199 | 200 | if ( type == ClientThread::UnknownQuery ) { 201 | this->sendError( "Unknown query received: " + QString::number( type ) ); 202 | LOG( "unknown query received: " + QString::number( type ) ); 203 | return; 204 | } 205 | if ( type == ClientThread::ServiceRunning ) { 206 | this->sendServiceOK(); 207 | LOG( "replied to service status check" ); 208 | return; 209 | } 210 | 211 | int content_len = get_content_length( a_data ); 212 | if ( -1 == content_len ) // Failed 213 | return; 214 | 215 | QJsonDocument json_doc; 216 | int current_len = 0; 217 | 218 | if ( 0 == content_len ) { // GET request 219 | json_doc = parse_json_request( a_data ); 220 | } else { 221 | current_len = parse_json_request_post_length( a_data ); 222 | while ( current_len < content_len && m_socket->waitForReadyRead( m_net_timeout ) ) { 223 | if ( m_socket->bytesAvailable() ) { 224 | a_data += m_socket->readAll(); 225 | current_len = parse_json_request_post_length( a_data ); 226 | } 227 | } 228 | json_doc = ( type == ClientThread::SiteStatusPostCheck ? parse_json_request_post( a_data ) : parse_json_request( a_data ) ); 229 | } 230 | 231 | if ( json_doc.isEmpty() || json_doc.isNull() ) { 232 | LOG( "Invalid JSON document format." ); 233 | this->sendError( "Invalid JSON document format." ); 234 | return; 235 | } 236 | 237 | QString client_auth_token = json_doc.object().value( "auth_token" ).toString(""); 238 | if ( "" == client_auth_token ) { 239 | LOG( "Missing 'auth_token' JSON value." ); 240 | this->sendError( "Missing 'auth_token' JSON value." ); 241 | return; 242 | } 243 | 244 | m_site_status_request = parse_requests( type, json_doc ); 245 | } 246 | 247 | bool ClientThread::parse_requests( QueryType type, QJsonDocument json_doc ) { 248 | QJsonValue blog_id, monitor_url; 249 | 250 | if ( type == ClientThread::SiteStatusCheck ) { 251 | blog_id = json_doc.object().value( "blog_id" ); 252 | if ( blog_id.isNull() ) { 253 | LOG( "Missing 'blog_id' JSON value." ); 254 | this->sendError( "Missing 'blog_id' JSON value." ); 255 | return false; 256 | } 257 | 258 | monitor_url = json_doc.object().value( "monitor_url" ); 259 | if ( monitor_url.isNull() ) { 260 | LOG( "Missing 'monitor_url' JSON value." ); 261 | this->sendError( "Missing 'monitor_url' JSON value." ); 262 | return false; 263 | } 264 | 265 | HealthCheck *hc = new HealthCheck(); 266 | hc->thread_index = NOT_ASSIGNED; 267 | hc->received = QDateTime::currentDateTime(); 268 | hc->jetmon_server = m_jetmon_server; 269 | hc->monitor_url = monitor_url.toString(); 270 | hc->blog_id = blog_id.toInt(); 271 | 272 | this->m_checks.append( hc ); 273 | } else { 274 | QJsonArray jArr = json_doc.object()["checks"].toArray(); 275 | if ( jArr.isEmpty() ) { 276 | this->sendError( "Missing 'checks' JSON array." ); 277 | LOG( "Missing 'checks' JSON array." ); 278 | return false; 279 | } 280 | 281 | for ( int loop = 0; loop < jArr.count(); loop++ ) { 282 | blog_id = jArr.at( loop ).toObject().value( "blog_id" ); 283 | if ( blog_id.isNull() ) { 284 | LOG( "Missing 'blog_id' JSON value for array index " + QString::number( loop ) ); 285 | continue; 286 | } 287 | 288 | monitor_url = jArr.at( loop ).toObject().value( "monitor_url" ); 289 | if ( monitor_url.isNull() ) { 290 | LOG( "Missing 'monitor_url' JSON value for array index " + QString::number( loop ) ); 291 | continue; 292 | } 293 | 294 | HealthCheck *hc = new HealthCheck(); 295 | hc->thread_index = NOT_ASSIGNED; 296 | hc->received = QDateTime::currentDateTime(); 297 | hc->jetmon_server = m_jetmon_server; 298 | hc->monitor_url = monitor_url.toString(); 299 | hc->blog_id = blog_id.toInt(); 300 | 301 | this->m_checks.append( hc ); 302 | } 303 | } 304 | 305 | return true; 306 | } 307 | 308 | void ClientThread::sendServiceOK() { 309 | m_socket->write( "OK" ); 310 | m_socket->flush(); 311 | m_socket->waitForBytesWritten( m_net_timeout ); 312 | } 313 | 314 | void ClientThread::sendOK() { 315 | QString s_data = get_http_content( 1 ); 316 | QString s_response = get_http_reply_header( "200 OK", s_data ); 317 | 318 | m_socket->write( s_response.toStdString().c_str() ); 319 | m_socket->flush(); 320 | m_socket->waitForBytesWritten( m_net_timeout ); 321 | } 322 | 323 | void ClientThread::sendError( const QString errorString ) { 324 | QString s_data = get_http_content( -1, errorString ); 325 | QString s_response = get_http_reply_header( "404 Not Found", s_data ); 326 | 327 | m_socket->write( s_response.toStdString().c_str() ); 328 | m_socket->flush(); 329 | m_socket->waitForBytesWritten( m_net_timeout ); 330 | } 331 | 332 | QString ClientThread::get_http_content( int status, const QString &error ) { 333 | QString ret_val = "{\"veriflier\":\""; 334 | ret_val += m_veriflier_name; 335 | ret_val += "\",\"auth_token\":\""; 336 | ret_val += m_auth_token; 337 | ret_val += "\",\"status\":"; 338 | ret_val += QString::number( status ); 339 | if ( error.length() > 0 ) { 340 | ret_val += ",\"error\":\""; 341 | ret_val += error; 342 | ret_val += "\""; 343 | } 344 | ret_val += "}\n"; 345 | 346 | return ret_val; 347 | } 348 | 349 | QString ClientThread::get_http_reply_header( const QString &http_code, const QString &p_data) { 350 | QString ret_val = "HTTP/1.1 "; 351 | ret_val += http_code; 352 | ret_val += "\r\nContent-Type: application/json\r\n"; 353 | ret_val += "Content-Length: "; 354 | ret_val += QString::number( p_data.length() ); 355 | ret_val += "\r\nConnection: close\r\n\r\n"; 356 | ret_val += p_data; 357 | 358 | return ret_val; 359 | } 360 | -------------------------------------------------------------------------------- /lib/httpcheck.js: -------------------------------------------------------------------------------- 1 | 2 | process.title = 'jetmon-worker'; 3 | 4 | const SITE_DOWN = 0; 5 | const SITE_RUNNING = 1; 6 | const SITE_CONFIRMED_DOWN = 2; 7 | 8 | const SUICIDE_SIGNAL = 1; 9 | const EXIT_MAXRAMUSAGE = 2; 10 | const EXIT_MAXCHECKS = 3; 11 | 12 | const DEFAULT_HTTP_PORT = 80; 13 | 14 | const JETMON_CHECK = 1; 15 | const VERIFLIER_CHECK = 2; 16 | 17 | const SECONDS = 1000; 18 | const MINUTES = 60 * SECONDS; 19 | const HOURS = 60 * MINUTES; 20 | const DAYS = 24 * HOURS; 21 | 22 | var _watcher = require( './jetmon.node' ); 23 | var o_log4js = require( 'log4js' ); 24 | 25 | // each worker loads it's own config object 26 | var config = require( './config' ); 27 | config.load(); 28 | 29 | var arrCheck = []; 30 | var running = false; 31 | var askedForWork = false; 32 | var availableForWork = true; 33 | var suicideSignal = false; 34 | var pointer = 0; 35 | 36 | /** 37 | * How many checks are currently being processed by the worker. 38 | * 39 | * @type {number} 40 | */ 41 | var activeChecks = 0; 42 | var totalChecks = 0; 43 | var createdTime = Date.now(); 44 | 45 | // These values will be set in HttpChecker.reloadConfig. 46 | var maxChecks = 0; 47 | var maxMemUsage = 0; 48 | 49 | var workerTotals = {}; 50 | workerTotals[SITE_DOWN] = 0; 51 | workerTotals[SITE_RUNNING] = 0; 52 | workerTotals[SITE_CONFIRMED_DOWN] = 0; 53 | 54 | var checkStats = {}; 55 | 56 | o_log4js.configure( { 57 | appenders: [ { 58 | 'type' : 'file', 59 | 'filename' : 'logs/jetmon.log', 60 | 'maxLogSize': 52428800, 61 | 'backups' : 30, 62 | 'category' : 'flog', 63 | 'levels' : 'DEBUG', 64 | }, 65 | { 66 | 'type' : 'file', 67 | 'filename' : 'logs/status-change.log', 68 | 'maxLogSize': 104857600, 69 | 'backups' : 100, 70 | 'category' : 'slog', 71 | 'levels' : 'DEBUG', 72 | } 73 | ] 74 | }); 75 | o_log4js.PatternLayout = '%d{HH:mm:ss,SSS} p m'; 76 | 77 | global.logger = o_log4js.getLogger( 'flog' ); 78 | var slogger = o_log4js.getLogger( 'slog' ); 79 | 80 | var _os = require( 'os' ); 81 | var hostname = _os.hostname(); 82 | 83 | var HttpChecker = { 84 | reloadConfig: function() { 85 | maxChecks = config.get( 'WORKER_MAX_CHECKS' ) || 0; 86 | maxMemUsage = config.get( 'WORKER_MAX_MEM_MB' ) || 53; 87 | 88 | if ( maxChecks > 0 ) { 89 | // If the number of checks is limited, pre-seed totalChecks to a random value. 90 | // This helps prevent all the workers from trying to recycle at the same time. 91 | totalChecks = Math.floor( Math.random() * maxChecks ); 92 | } else { 93 | maxChecks = Number.MAX_VALUE; 94 | } 95 | if ( maxMemUsage > 0 ) { 96 | maxMemUsage = maxMemUsage * 1024 * 1024; 97 | } else { 98 | maxMemUsage = Number.MAX_VALUE; 99 | } 100 | }, 101 | 102 | checkServers: function() { 103 | try { 104 | var pointerCurrentMax = pointer + config.get( 'NUM_TO_PROCESS' ); 105 | if ( pointerCurrentMax > arrCheck.length ) 106 | pointerCurrentMax = arrCheck.length; 107 | for ( ; pointer < pointerCurrentMax ; pointer++ ) { 108 | activeChecks++; 109 | totalChecks++; 110 | _watcher.http_check( arrCheck[ pointer ].monitor_url, DEFAULT_HTTP_PORT, pointer, HttpChecker.processResultsCallback ); 111 | } 112 | } 113 | catch ( Exception ) { 114 | logger.debug( process.pid + ': ERROR - failed to process the server array: ' + Exception.toString() ); 115 | } 116 | }, 117 | 118 | sendStats: function() { 119 | if ( workerTotals[SITE_DOWN] || workerTotals[SITE_RUNNING] || workerTotals[SITE_CONFIRMED_DOWN] ) 120 | process.send( { msgtype: 'totals', worker_pid: process.pid, work_totals: workerTotals } ); 121 | workerTotals[SITE_DOWN] = 0; 122 | workerTotals[SITE_RUNNING] = 0; 123 | workerTotals[SITE_CONFIRMED_DOWN] = 0; 124 | }, 125 | 126 | processResultsCallback: function( serverArrayIndex, rtt, http_code, error_code ) { 127 | /** 128 | * Reduce the amount of active checks, as the check has finished. 129 | */ 130 | activeChecks--; 131 | 132 | var server = arrCheck[ serverArrayIndex ]; 133 | server.processed = true; 134 | server.lastCheck = new Date().valueOf(); // we use set the value to the milliseconds value 135 | 136 | if ( rtt > 0 && 400 > http_code && 0 != http_code ) { 137 | server.site_status = SITE_RUNNING; 138 | } 139 | else if ( 140 | ( SITE_RUNNING == server.oldStatus ) || 141 | ( 142 | ( SITE_CONFIRMED_DOWN != server.oldStatus ) && 143 | ( new Date().valueOf() < ( server.last_status_change + ( config.get( 'TIME_BETWEEN_NOTICES_MIN' ) * MINUTES ) ) ) 144 | ) 145 | ) { 146 | server.site_status = SITE_DOWN; 147 | } 148 | else { 149 | server.site_status = SITE_CONFIRMED_DOWN; 150 | } 151 | 152 | if ( server.site_status != server.oldStatus ) { 153 | var resO = {}; 154 | resO.type = JETMON_CHECK; 155 | resO.host = hostname; 156 | resO.status = server.site_status; 157 | resO.rtt = Math.round( rtt / 1000 ); 158 | resO.code = http_code; 159 | resO.error_code = error_code; 160 | server.checks.push( resO ); 161 | 162 | // if site is down and it has not been confirmed 163 | if ( server.site_status == SITE_DOWN ) { 164 | process.send( { msgtype: 'recheck', server: server } ); 165 | } else if ( SITE_CONFIRMED_DOWN != server.site_status ) { 166 | process.send( { msgtype: 'notify_status_change', server: server } ); 167 | slogger.trace( 'status_change: ' + JSON.stringify( server ) ); 168 | } else { 169 | process.send( { msgtype: 'notify_still_down', server: server } ); 170 | slogger.trace( 'still_down: ' + JSON.stringify( server ) ); 171 | } 172 | } 173 | 174 | workerTotals[server.site_status]++; 175 | 176 | if ( pointer < arrCheck.length ) { 177 | activeChecks++; 178 | totalChecks++; 179 | _watcher.http_check( arrCheck[ pointer ].monitor_url, DEFAULT_HTTP_PORT, pointer, HttpChecker.processResultsCallback ); 180 | pointer++; 181 | } else { 182 | if ( availableForWork && ( suicideSignal || process.memoryUsage().rss > maxMemUsage || totalChecks > maxChecks ) ) { 183 | availableForWork = false; 184 | process.send( { msgtype: 'stop_work', worker_pid: process.pid } ); 185 | } 186 | 187 | // check if we have any outstanding callbacks 188 | var waiting_for = 0; 189 | for ( var count in arrCheck ) { 190 | if ( ! arrCheck[ count ].processed ) 191 | waiting_for++; 192 | } 193 | 194 | if ( 0 === waiting_for ) { 195 | // No outstanding callbacks and not available for work. Time to die! 196 | if ( false === availableForWork ) { 197 | // HttpChecker.sendStats(); 198 | if ( suicideSignal ) { 199 | process.exit( SUICIDE_SIGNAL ); 200 | } else if ( totalChecks > maxChecks ) { 201 | process.exit( EXIT_MAXCHECKS ); 202 | } else { 203 | process.exit( EXIT_MAXRAMUSAGE ); 204 | } 205 | } 206 | 207 | arrCheck = []; 208 | running = false; 209 | } 210 | 211 | if ( availableForWork && ( false === askedForWork ) ) { 212 | askedForWork = true; 213 | process.send( { msgtype: 'send_work', worker_pid: process.pid } ); 214 | } 215 | } 216 | 217 | 218 | /** 219 | * Store stats data to send to the parent later. 220 | * 221 | * Doing in the end to make sure we send all the data to appropriate consumers first and 222 | * only then we can try to log the data. 223 | */ 224 | let stats_site_status = 'unknown'; 225 | 226 | switch ( server.site_status ) { 227 | case SITE_RUNNING: 228 | stats_site_status = 'up'; 229 | break; 230 | 231 | case SITE_DOWN: 232 | stats_site_status = 'down'; 233 | break; 234 | 235 | case SITE_CONFIRMED_DOWN: 236 | stats_site_status = 'still_down'; 237 | break; 238 | } 239 | 240 | const stats_rtt = Math.round( rtt / 1000 ); 241 | 242 | if ( checkStats[stats_site_status] ) { 243 | checkStats[stats_site_status]['http_code'][http_code] = ( checkStats[stats_site_status]['http_code'][http_code] || 0 ) + 1; 244 | if ( error_code !== 0 ){ 245 | checkStats[stats_site_status]['error_code'][error_code] = ( checkStats[stats_site_status]['error_code'][error_code] || 0 ) + 1; 246 | } 247 | 248 | checkStats[stats_site_status]['rtt']['count']++; 249 | checkStats[stats_site_status]['rtt']['sum'] += stats_rtt; 250 | checkStats[stats_site_status]['rtt']['max'] = Math.max( checkStats[stats_site_status]['rtt']['max'], stats_rtt ); 251 | checkStats[stats_site_status]['rtt']['min'] = Math.min( checkStats[stats_site_status]['rtt']['min'], stats_rtt ); 252 | } else { 253 | checkStats[stats_site_status] = { 254 | 'http_code': {}, 255 | 'error_code': {}, 256 | 'rtt': { 257 | 'count': 1, 258 | 'sum': stats_rtt, 259 | 'max': stats_rtt, 260 | 'min': stats_rtt 261 | } 262 | }; 263 | 264 | checkStats[stats_site_status]['http_code'][http_code] = 1; 265 | if ( error_code !== 0 ) { 266 | checkStats[stats_site_status]['error_code'][error_code] = 1; 267 | } 268 | } 269 | }, 270 | 271 | addToQueue: function( arrData ) { 272 | if ( running ) { 273 | for ( var count in arrData ) { 274 | arrCheck.push( arrData[ count ] ); 275 | } 276 | } else { 277 | arrCheck = arrData; 278 | pointer = 0; 279 | running = true; 280 | setTimeout( HttpChecker.checkServers, 50 ); 281 | } 282 | }, 283 | 284 | /** 285 | * Returns how long the worker has been running. 286 | * 287 | * @returns {number} 288 | */ 289 | getAge: function() { 290 | return Date.now() - createdTime; 291 | } 292 | 293 | }; 294 | 295 | process.on( 'message', function( msg ) { 296 | try { 297 | switch ( msg.request ) 298 | { 299 | case 'queue-add': { 300 | // once we get some work we reset our 'asked state' 301 | askedForWork = false; 302 | HttpChecker.addToQueue( msg.payload ); 303 | break; 304 | } 305 | case 'evaporate' : { 306 | if ( ! running ) { 307 | // HttpChecker.sendStats(); 308 | process.exit( SUICIDE_SIGNAL ); 309 | } else { 310 | suicideSignal = true; 311 | } 312 | break; 313 | } 314 | case 'config-update': { 315 | logger.debug( 'worker pid ' + msg.pid + ': updating config settings.' ); 316 | config.load(); 317 | 318 | HttpChecker.reloadConfig(); 319 | 320 | break; 321 | } 322 | default: { 323 | logger.debug( process.pid + ': INFO: received unknown message "' + msg.request + '"' ); 324 | process.send( { msgtype: 'unknown', worker_pid: msg.pid, payload: 0 } ); 325 | break; 326 | } 327 | } 328 | } 329 | catch ( Exception ) { 330 | logger.error( process.pid + ": ERROR: receiving the Master's message: " + Exception.toString() ); 331 | } 332 | }); 333 | 334 | setInterval( HttpChecker.sendStats, config.get( 'STATS_UPDATE_INTERVAL_MS' ) ); 335 | 336 | setTimeout( function() { 337 | askedForWork = true; 338 | if ( true === process.connected ) { 339 | process.send( { msgtype: 'send_work', worker_pid: process.pid } ); 340 | } 341 | }, 2000 ); 342 | 343 | 344 | /** 345 | * Periodically send stats up to the main Jetmon process, so we can know what the current situation in the worker is. 346 | * 347 | * Currently, sending stats every 1 second. 348 | */ 349 | setInterval( function() { 350 | var message = { 351 | msgtype: 'stats', 352 | worker_pid: process.pid, 353 | stats: { 354 | queueLength: arrCheck.length, 355 | pointer: pointer, 356 | activeChecks: activeChecks, 357 | totalChecks: totalChecks, 358 | memoryUsage: process.memoryUsage().rss, 359 | checkStats: checkStats, 360 | uptime: HttpChecker.getAge(), 361 | } 362 | }; 363 | 364 | checkStats = {}; 365 | 366 | process.send( message ); 367 | }, 1000 ); 368 | 369 | // Ensure that the variable config values are set properly. 370 | HttpChecker.reloadConfig(); 371 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | Jetpack Site Monitor 294 | Copyright (C) 2013 Automattic 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | {signature of Ty Coon}, 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /veriflier/LICENSE: -------------------------------------------------------------------------------- 1 | GNU GENERAL PUBLIC LICENSE 2 | Version 2, June 1991 3 | 4 | Copyright (C) 1989, 1991 Free Software Foundation, Inc., 5 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA 6 | Everyone is permitted to copy and distribute verbatim copies 7 | of this license document, but changing it is not allowed. 8 | 9 | Preamble 10 | 11 | The licenses for most software are designed to take away your 12 | freedom to share and change it. By contrast, the GNU General Public 13 | License is intended to guarantee your freedom to share and change free 14 | software--to make sure the software is free for all its users. This 15 | General Public License applies to most of the Free Software 16 | Foundation's software and to any other program whose authors commit to 17 | using it. (Some other Free Software Foundation software is covered by 18 | the GNU Lesser General Public License instead.) You can apply it to 19 | your programs, too. 20 | 21 | When we speak of free software, we are referring to freedom, not 22 | price. Our General Public Licenses are designed to make sure that you 23 | have the freedom to distribute copies of free software (and charge for 24 | this service if you wish), that you receive source code or can get it 25 | if you want it, that you can change the software or use pieces of it 26 | in new free programs; and that you know you can do these things. 27 | 28 | To protect your rights, we need to make restrictions that forbid 29 | anyone to deny you these rights or to ask you to surrender the rights. 30 | These restrictions translate to certain responsibilities for you if you 31 | distribute copies of the software, or if you modify it. 32 | 33 | For example, if you distribute copies of such a program, whether 34 | gratis or for a fee, you must give the recipients all the rights that 35 | you have. You must make sure that they, too, receive or can get the 36 | source code. And you must show them these terms so they know their 37 | rights. 38 | 39 | We protect your rights with two steps: (1) copyright the software, and 40 | (2) offer you this license which gives you legal permission to copy, 41 | distribute and/or modify the software. 42 | 43 | Also, for each author's protection and ours, we want to make certain 44 | that everyone understands that there is no warranty for this free 45 | software. If the software is modified by someone else and passed on, we 46 | want its recipients to know that what they have is not the original, so 47 | that any problems introduced by others will not reflect on the original 48 | authors' reputations. 49 | 50 | Finally, any free program is threatened constantly by software 51 | patents. We wish to avoid the danger that redistributors of a free 52 | program will individually obtain patent licenses, in effect making the 53 | program proprietary. To prevent this, we have made it clear that any 54 | patent must be licensed for everyone's free use or not licensed at all. 55 | 56 | The precise terms and conditions for copying, distribution and 57 | modification follow. 58 | 59 | GNU GENERAL PUBLIC LICENSE 60 | TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION 61 | 62 | 0. This License applies to any program or other work which contains 63 | a notice placed by the copyright holder saying it may be distributed 64 | under the terms of this General Public License. The "Program", below, 65 | refers to any such program or work, and a "work based on the Program" 66 | means either the Program or any derivative work under copyright law: 67 | that is to say, a work containing the Program or a portion of it, 68 | either verbatim or with modifications and/or translated into another 69 | language. (Hereinafter, translation is included without limitation in 70 | the term "modification".) Each licensee is addressed as "you". 71 | 72 | Activities other than copying, distribution and modification are not 73 | covered by this License; they are outside its scope. The act of 74 | running the Program is not restricted, and the output from the Program 75 | is covered only if its contents constitute a work based on the 76 | Program (independent of having been made by running the Program). 77 | Whether that is true depends on what the Program does. 78 | 79 | 1. You may copy and distribute verbatim copies of the Program's 80 | source code as you receive it, in any medium, provided that you 81 | conspicuously and appropriately publish on each copy an appropriate 82 | copyright notice and disclaimer of warranty; keep intact all the 83 | notices that refer to this License and to the absence of any warranty; 84 | and give any other recipients of the Program a copy of this License 85 | along with the Program. 86 | 87 | You may charge a fee for the physical act of transferring a copy, and 88 | you may at your option offer warranty protection in exchange for a fee. 89 | 90 | 2. You may modify your copy or copies of the Program or any portion 91 | of it, thus forming a work based on the Program, and copy and 92 | distribute such modifications or work under the terms of Section 1 93 | above, provided that you also meet all of these conditions: 94 | 95 | a) You must cause the modified files to carry prominent notices 96 | stating that you changed the files and the date of any change. 97 | 98 | b) You must cause any work that you distribute or publish, that in 99 | whole or in part contains or is derived from the Program or any 100 | part thereof, to be licensed as a whole at no charge to all third 101 | parties under the terms of this License. 102 | 103 | c) If the modified program normally reads commands interactively 104 | when run, you must cause it, when started running for such 105 | interactive use in the most ordinary way, to print or display an 106 | announcement including an appropriate copyright notice and a 107 | notice that there is no warranty (or else, saying that you provide 108 | a warranty) and that users may redistribute the program under 109 | these conditions, and telling the user how to view a copy of this 110 | License. (Exception: if the Program itself is interactive but 111 | does not normally print such an announcement, your work based on 112 | the Program is not required to print an announcement.) 113 | 114 | These requirements apply to the modified work as a whole. If 115 | identifiable sections of that work are not derived from the Program, 116 | and can be reasonably considered independent and separate works in 117 | themselves, then this License, and its terms, do not apply to those 118 | sections when you distribute them as separate works. But when you 119 | distribute the same sections as part of a whole which is a work based 120 | on the Program, the distribution of the whole must be on the terms of 121 | this License, whose permissions for other licensees extend to the 122 | entire whole, and thus to each and every part regardless of who wrote it. 123 | 124 | Thus, it is not the intent of this section to claim rights or contest 125 | your rights to work written entirely by you; rather, the intent is to 126 | exercise the right to control the distribution of derivative or 127 | collective works based on the Program. 128 | 129 | In addition, mere aggregation of another work not based on the Program 130 | with the Program (or with a work based on the Program) on a volume of 131 | a storage or distribution medium does not bring the other work under 132 | the scope of this License. 133 | 134 | 3. You may copy and distribute the Program (or a work based on it, 135 | under Section 2) in object code or executable form under the terms of 136 | Sections 1 and 2 above provided that you also do one of the following: 137 | 138 | a) Accompany it with the complete corresponding machine-readable 139 | source code, which must be distributed under the terms of Sections 140 | 1 and 2 above on a medium customarily used for software interchange; or, 141 | 142 | b) Accompany it with a written offer, valid for at least three 143 | years, to give any third party, for a charge no more than your 144 | cost of physically performing source distribution, a complete 145 | machine-readable copy of the corresponding source code, to be 146 | distributed under the terms of Sections 1 and 2 above on a medium 147 | customarily used for software interchange; or, 148 | 149 | c) Accompany it with the information you received as to the offer 150 | to distribute corresponding source code. (This alternative is 151 | allowed only for noncommercial distribution and only if you 152 | received the program in object code or executable form with such 153 | an offer, in accord with Subsection b above.) 154 | 155 | The source code for a work means the preferred form of the work for 156 | making modifications to it. For an executable work, complete source 157 | code means all the source code for all modules it contains, plus any 158 | associated interface definition files, plus the scripts used to 159 | control compilation and installation of the executable. However, as a 160 | special exception, the source code distributed need not include 161 | anything that is normally distributed (in either source or binary 162 | form) with the major components (compiler, kernel, and so on) of the 163 | operating system on which the executable runs, unless that component 164 | itself accompanies the executable. 165 | 166 | If distribution of executable or object code is made by offering 167 | access to copy from a designated place, then offering equivalent 168 | access to copy the source code from the same place counts as 169 | distribution of the source code, even though third parties are not 170 | compelled to copy the source along with the object code. 171 | 172 | 4. You may not copy, modify, sublicense, or distribute the Program 173 | except as expressly provided under this License. Any attempt 174 | otherwise to copy, modify, sublicense or distribute the Program is 175 | void, and will automatically terminate your rights under this License. 176 | However, parties who have received copies, or rights, from you under 177 | this License will not have their licenses terminated so long as such 178 | parties remain in full compliance. 179 | 180 | 5. You are not required to accept this License, since you have not 181 | signed it. However, nothing else grants you permission to modify or 182 | distribute the Program or its derivative works. These actions are 183 | prohibited by law if you do not accept this License. Therefore, by 184 | modifying or distributing the Program (or any work based on the 185 | Program), you indicate your acceptance of this License to do so, and 186 | all its terms and conditions for copying, distributing or modifying 187 | the Program or works based on it. 188 | 189 | 6. Each time you redistribute the Program (or any work based on the 190 | Program), the recipient automatically receives a license from the 191 | original licensor to copy, distribute or modify the Program subject to 192 | these terms and conditions. You may not impose any further 193 | restrictions on the recipients' exercise of the rights granted herein. 194 | You are not responsible for enforcing compliance by third parties to 195 | this License. 196 | 197 | 7. If, as a consequence of a court judgment or allegation of patent 198 | infringement or for any other reason (not limited to patent issues), 199 | conditions are imposed on you (whether by court order, agreement or 200 | otherwise) that contradict the conditions of this License, they do not 201 | excuse you from the conditions of this License. If you cannot 202 | distribute so as to satisfy simultaneously your obligations under this 203 | License and any other pertinent obligations, then as a consequence you 204 | may not distribute the Program at all. For example, if a patent 205 | license would not permit royalty-free redistribution of the Program by 206 | all those who receive copies directly or indirectly through you, then 207 | the only way you could satisfy both it and this License would be to 208 | refrain entirely from distribution of the Program. 209 | 210 | If any portion of this section is held invalid or unenforceable under 211 | any particular circumstance, the balance of the section is intended to 212 | apply and the section as a whole is intended to apply in other 213 | circumstances. 214 | 215 | It is not the purpose of this section to induce you to infringe any 216 | patents or other property right claims or to contest validity of any 217 | such claims; this section has the sole purpose of protecting the 218 | integrity of the free software distribution system, which is 219 | implemented by public license practices. Many people have made 220 | generous contributions to the wide range of software distributed 221 | through that system in reliance on consistent application of that 222 | system; it is up to the author/donor to decide if he or she is willing 223 | to distribute software through any other system and a licensee cannot 224 | impose that choice. 225 | 226 | This section is intended to make thoroughly clear what is believed to 227 | be a consequence of the rest of this License. 228 | 229 | 8. If the distribution and/or use of the Program is restricted in 230 | certain countries either by patents or by copyrighted interfaces, the 231 | original copyright holder who places the Program under this License 232 | may add an explicit geographical distribution limitation excluding 233 | those countries, so that distribution is permitted only in or among 234 | countries not thus excluded. In such case, this License incorporates 235 | the limitation as if written in the body of this License. 236 | 237 | 9. The Free Software Foundation may publish revised and/or new versions 238 | of the General Public License from time to time. Such new versions will 239 | be similar in spirit to the present version, but may differ in detail to 240 | address new problems or concerns. 241 | 242 | Each version is given a distinguishing version number. If the Program 243 | specifies a version number of this License which applies to it and "any 244 | later version", you have the option of following the terms and conditions 245 | either of that version or of any later version published by the Free 246 | Software Foundation. If the Program does not specify a version number of 247 | this License, you may choose any version ever published by the Free Software 248 | Foundation. 249 | 250 | 10. If you wish to incorporate parts of the Program into other free 251 | programs whose distribution conditions are different, write to the author 252 | to ask for permission. For software which is copyrighted by the Free 253 | Software Foundation, write to the Free Software Foundation; we sometimes 254 | make exceptions for this. Our decision will be guided by the two goals 255 | of preserving the free status of all derivatives of our free software and 256 | of promoting the sharing and reuse of software generally. 257 | 258 | NO WARRANTY 259 | 260 | 11. BECAUSE THE PROGRAM IS LICENSED FREE OF CHARGE, THERE IS NO WARRANTY 261 | FOR THE PROGRAM, TO THE EXTENT PERMITTED BY APPLICABLE LAW. EXCEPT WHEN 262 | OTHERWISE STATED IN WRITING THE COPYRIGHT HOLDERS AND/OR OTHER PARTIES 263 | PROVIDE THE PROGRAM "AS IS" WITHOUT WARRANTY OF ANY KIND, EITHER EXPRESSED 264 | OR IMPLIED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF 265 | MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE ENTIRE RISK AS 266 | TO THE QUALITY AND PERFORMANCE OF THE PROGRAM IS WITH YOU. SHOULD THE 267 | PROGRAM PROVE DEFECTIVE, YOU ASSUME THE COST OF ALL NECESSARY SERVICING, 268 | REPAIR OR CORRECTION. 269 | 270 | 12. IN NO EVENT UNLESS REQUIRED BY APPLICABLE LAW OR AGREED TO IN WRITING 271 | WILL ANY COPYRIGHT HOLDER, OR ANY OTHER PARTY WHO MAY MODIFY AND/OR 272 | REDISTRIBUTE THE PROGRAM AS PERMITTED ABOVE, BE LIABLE TO YOU FOR DAMAGES, 273 | INCLUDING ANY GENERAL, SPECIAL, INCIDENTAL OR CONSEQUENTIAL DAMAGES ARISING 274 | OUT OF THE USE OR INABILITY TO USE THE PROGRAM (INCLUDING BUT NOT LIMITED 275 | TO LOSS OF DATA OR DATA BEING RENDERED INACCURATE OR LOSSES SUSTAINED BY 276 | YOU OR THIRD PARTIES OR A FAILURE OF THE PROGRAM TO OPERATE WITH ANY OTHER 277 | PROGRAMS), EVEN IF SUCH HOLDER OR OTHER PARTY HAS BEEN ADVISED OF THE 278 | POSSIBILITY OF SUCH DAMAGES. 279 | 280 | END OF TERMS AND CONDITIONS 281 | 282 | How to Apply These Terms to Your New Programs 283 | 284 | If you develop a new program, and you want it to be of the greatest 285 | possible use to the public, the best way to achieve this is to make it 286 | free software which everyone can redistribute and change under these terms. 287 | 288 | To do so, attach the following notices to the program. It is safest 289 | to attach them to the start of each source file to most effectively 290 | convey the exclusion of warranty; and each file should have at least 291 | the "copyright" line and a pointer to where the full notice is found. 292 | 293 | Jetpack Site Monitor 294 | Copyright (C) 2013 Automattic 295 | 296 | This program is free software; you can redistribute it and/or modify 297 | it under the terms of the GNU General Public License as published by 298 | the Free Software Foundation; either version 2 of the License, or 299 | (at your option) any later version. 300 | 301 | This program is distributed in the hope that it will be useful, 302 | but WITHOUT ANY WARRANTY; without even the implied warranty of 303 | MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 304 | GNU General Public License for more details. 305 | 306 | You should have received a copy of the GNU General Public License along 307 | with this program; if not, write to the Free Software Foundation, Inc., 308 | 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301 USA. 309 | 310 | Also add information on how to contact you by electronic and paper mail. 311 | 312 | If the program is interactive, make it output a short notice like this 313 | when it starts in an interactive mode: 314 | 315 | Gnomovision version 69, Copyright (C) year name of author 316 | Gnomovision comes with ABSOLUTELY NO WARRANTY; for details type `show w'. 317 | This is free software, and you are welcome to redistribute it 318 | under certain conditions; type `show c' for details. 319 | 320 | The hypothetical commands `show w' and `show c' should show the appropriate 321 | parts of the General Public License. Of course, the commands you use may 322 | be called something other than `show w' and `show c'; they could even be 323 | mouse-clicks or menu items--whatever suits your program. 324 | 325 | You should also get your employer (if you work as a programmer) or your 326 | school, if any, to sign a "copyright disclaimer" for the program, if 327 | necessary. Here is a sample; alter the names: 328 | 329 | Yoyodyne, Inc., hereby disclaims all copyright interest in the program 330 | `Gnomovision' (which makes passes at compilers) written by James Hacker. 331 | 332 | {signature of Ty Coon}, 1 April 1989 333 | Ty Coon, President of Vice 334 | 335 | This General Public License does not permit incorporating your program into 336 | proprietary programs. If your program is a subroutine library, you may 337 | consider it more useful to permit linking proprietary applications with the 338 | library. If this is what you want to do, use the GNU Lesser General 339 | Public License instead of this License. 340 | -------------------------------------------------------------------------------- /src/http_checker.cpp: -------------------------------------------------------------------------------- 1 | 2 | #include "http_checker.h" 3 | #include 4 | 5 | using namespace std; 6 | 7 | const int ERROR_STATUS_CODE_UNKNOWN = 999; 8 | const int ERROR_TIMEOUT = 998; 9 | const int ERROR_REDIRECT_LOCATION = 997; 10 | const int ERROR_CONNECT_REDIRECT_HOST = 996; 11 | const int ERROR_CONNECT_HOST = 995; 12 | 13 | HTTP_Checker::HTTP_Checker() : m_sock( -1 ), m_host_name( "" ), m_host_dir( "" ), m_port( HTTP_DEFAULT_PORT ), 14 | m_is_ssl( false ), m_triptime( 0 ), m_response_code( 0 ), m_ctx( NULL ), m_ssl( NULL ), m_sbio( NULL ), m_error_code( 0 ) { 15 | gettimeofday( &m_tstart, &m_tzone ); 16 | memset( m_buf, 0, MAX_TCP_BUFFER ); 17 | m_cutofftime = time( NULL ); 18 | m_cutofftime += NET_COMMS_TIMEOUT; 19 | } 20 | 21 | HTTP_Checker::~HTTP_Checker() { 22 | this->disconnect(); 23 | } 24 | 25 | time_t HTTP_Checker::get_rtt() { 26 | struct timeval m_tend; 27 | gettimeofday( &m_tend, &m_tzone ); 28 | 29 | if ( (m_tend.tv_usec -= m_tstart.tv_usec) < 0 ) { 30 | m_tend.tv_sec--; 31 | m_tend.tv_usec += 1000000; 32 | } 33 | m_tend.tv_sec -= m_tstart.tv_sec; 34 | return m_tend.tv_sec * 1000000 + ( m_tend.tv_usec ); 35 | } 36 | 37 | void HTTP_Checker::check( string p_host_name, int p_port ) { 38 | try { 39 | m_host_name = p_host_name; 40 | m_port = p_port; 41 | m_host_dir = '/'; 42 | 43 | this->parse_host_values(); 44 | if ( connect() ) { 45 | this->set_host_response( 0 ); 46 | } else { 47 | #if DEBUG_MODE 48 | cerr << "Unable to connect to host" << endl; 49 | #endif 50 | m_error_code = ERROR_CONNECT_HOST; 51 | } 52 | } 53 | catch( exception &ex ) { 54 | cerr << "exception in HTTP_Checker::check(): for host '" << p_host_name.c_str() << "'" << endl; 55 | } 56 | } 57 | 58 | void HTTP_Checker::set_host_response( int redirects ) { 59 | try { 60 | string response = this->send_http_get(); 61 | if ( 0 >= response.size() ) { 62 | #if DEBUG_MODE 63 | cerr << "no response - timed out" << endl; 64 | #endif 65 | m_response_code = 0; 66 | m_error_code = ERROR_TIMEOUT; 67 | return; 68 | } 69 | 70 | if ( 8 != response.find_first_of( ' ' ) ) { 71 | #if DEBUG_MODE 72 | cerr << "Status code unknown" << endl; 73 | #endif 74 | m_response_code = 999; 75 | m_error_code = ERROR_STATUS_CODE_UNKNOWN; 76 | return; 77 | } 78 | 79 | string s_response_code = response.substr( 9, 3 ); 80 | m_response_code = atoi( s_response_code.c_str() ); 81 | 82 | // if we have been redirected, get the details and make a recursive call 83 | if ( ( 300 < m_response_code ) && ( 400 > m_response_code ) ) { 84 | redirects++; 85 | if ( MAX_REDIRECTS < redirects ) { 86 | #if DEBUG_MODE 87 | cerr << "Hit max on the redirects" << endl; 88 | #endif 89 | // Note we leave the 3xx response code so this site is marked as up 90 | return; 91 | } 92 | if ( ! set_redirect_host_values( response ) ) { 93 | #if DEBUG_MODE 94 | cerr << "Unable to parse redirect location" << endl; 95 | #endif 96 | m_response_code = 0; 97 | m_error_code = ERROR_REDIRECT_LOCATION; 98 | return; 99 | } 100 | this->disconnect(); 101 | if ( this->connect() ) { 102 | this->set_host_response( redirects ); 103 | } else { 104 | #if DEBUG_MODE 105 | cerr << "Unable to connect to redirect host" << endl; 106 | #endif 107 | m_response_code = 0; 108 | m_error_code = ERROR_CONNECT_REDIRECT_HOST; 109 | } 110 | } 111 | 112 | #if DEBUG_MODE 113 | cerr << m_host_name.c_str() << " : " << m_response_code << endl; 114 | #endif 115 | 116 | } 117 | catch( exception &ex ) { 118 | cerr << "exception in HTTP_Checker::set_host_responses(): for host '" << m_host_name.c_str() << "'" << endl; 119 | } 120 | } 121 | 122 | bool HTTP_Checker::set_redirect_host_values( string p_content ) { 123 | try { 124 | string p_lcase_search = p_content; 125 | 126 | std::transform( p_lcase_search.begin(), p_lcase_search.end(), p_lcase_search.begin(), ::tolower ); 127 | 128 | if ( string::npos == p_lcase_search.find( "location: " ) ) 129 | return false; 130 | 131 | p_content = p_content.substr( p_lcase_search.find( "location: " ) + 10, p_content.length() - ( p_lcase_search.find( "location: " ) + 10 ) ); 132 | 133 | if ( string::npos == p_content.find( "\r\n" ) ) 134 | return false; 135 | 136 | p_content.erase( p_content.find_first_of( "\r\n" ), p_content.length() - p_content.find_first_of( "\r\n" ) ); 137 | 138 | // keep a copy for relative location redirects 139 | string hostname_backup = m_host_name; 140 | m_host_name = p_content; 141 | m_port = HTTP_DEFAULT_PORT; 142 | m_host_dir = '/'; 143 | m_is_ssl = false; 144 | 145 | this->parse_host_values(); 146 | 147 | // this is a relative location redirect, reinstate hostname 148 | if ( 0 == m_host_name.size() ) 149 | m_host_name = hostname_backup; 150 | 151 | return true; 152 | } 153 | catch( exception &ex ) { 154 | cerr << "exception in HTTP_Checker::set_redirect_host_values()" << endl; 155 | return false; 156 | } 157 | } 158 | 159 | void HTTP_Checker::parse_host_values() { 160 | if ( string::npos != m_host_name.find( "http://" ) ) { 161 | m_host_name.erase( m_host_name.find( "http://" ), 7 ); 162 | } 163 | 164 | if ( string::npos != m_host_name.find( "https://" ) ) { 165 | m_host_name.erase( m_host_name.find( "https://" ), 8 ); 166 | m_port = HTTPS_DEFAULT_PORT; 167 | m_is_ssl = true; 168 | } 169 | 170 | size_t s_pos = m_host_name.find_first_of( '/' ); 171 | size_t q_pos = m_host_name.find_first_of( '?' ); 172 | size_t c_pos = m_host_name.find_first_of( ':' ); 173 | size_t f_pos = m_host_name.find_first_of( '#' ); 174 | 175 | if ( ( c_pos < s_pos ) && ( c_pos < q_pos ) && ( c_pos < f_pos ) ) { 176 | int new_port = atoi( m_host_name.substr( c_pos + 1, min( s_pos, m_host_name.length() ) ).c_str() ); 177 | if ( 0 < new_port ) { 178 | m_port = new_port; 179 | m_host_name.erase( c_pos, min( s_pos, m_host_name.length() ) - c_pos ); 180 | // recalc since we've erased some characters 181 | s_pos = m_host_name.find_first_of( '/' ); 182 | q_pos = m_host_name.find_first_of( '?' ); 183 | f_pos = m_host_name.find_first_of( '#' ); 184 | } 185 | } 186 | 187 | if ( string::npos != s_pos || string::npos != q_pos || string::npos != f_pos ) { 188 | size_t m_pos = min( min( s_pos, q_pos ), f_pos ); 189 | m_host_dir = m_host_name.substr( m_pos, m_host_name.length() - m_pos ); 190 | if ( 0 == m_host_dir.length() || '?' == m_host_dir[0] || '#' == m_host_dir[0] ) { 191 | m_host_dir = "/" + m_host_dir; 192 | } 193 | m_host_name.erase( m_pos, m_host_name.length() - m_pos ); 194 | } 195 | } 196 | 197 | string HTTP_Checker::send_http_get() { 198 | string s_tmp = "HEAD " + m_host_dir + " HTTP/1.1\r\n"; 199 | s_tmp += "Host: " + m_host_name + "\r\n"; 200 | s_tmp += "User-Agent: jetmon/1.0 (Jetpack Site Uptime Monitor by WordPress.com)\r\n"; 201 | s_tmp += "Connection: close\r\n\r\n"; 202 | 203 | strcpy( m_buf, s_tmp.c_str() ); 204 | 205 | if ( send_bytes( m_buf, s_tmp.length() ) ) { 206 | s_tmp = get_response(); 207 | } else { 208 | s_tmp = ""; 209 | #if DEBUG_MODE 210 | cerr << "failed to send_bytes()" << endl; 211 | #endif 212 | } 213 | return s_tmp; 214 | } 215 | 216 | string HTTP_Checker::get_response() { 217 | try { 218 | ssize_t received; 219 | fd_set read_fds; 220 | struct timeval tv; 221 | string ret_val = ""; 222 | 223 | do { 224 | tv.tv_sec = 0; 225 | tv.tv_usec = 500000; 226 | FD_ZERO( &read_fds ); 227 | FD_SET( m_sock, &read_fds ); 228 | 229 | ::select( m_sock + 1, &read_fds, NULL, NULL, &tv ); 230 | } while ( ( FD_ISSET( m_sock, &read_fds ) == 0) && ( m_cutofftime > time( NULL ) ) ); 231 | 232 | if ( FD_ISSET( m_sock, &read_fds) ) { 233 | if ( m_is_ssl ) 234 | received = SSL_read( m_ssl, m_buf, MAX_TCP_BUFFER - 1 ); 235 | else 236 | received = ::recv( m_sock, m_buf, MAX_TCP_BUFFER - 1, 0 ); 237 | 238 | while ( received > 0 ) { 239 | if ( received < MAX_TCP_BUFFER ) { 240 | m_buf[ received ] = '\0'; 241 | ret_val += m_buf; 242 | } 243 | do 244 | { 245 | tv.tv_sec = 0; 246 | tv.tv_usec = 500000; 247 | FD_ZERO( &read_fds ); 248 | FD_SET( m_sock, &read_fds ); 249 | 250 | select( m_sock + 1, &read_fds, NULL, NULL, &tv ); 251 | } while( (FD_ISSET( m_sock, &read_fds ) == 0) && ( m_cutofftime > time( NULL ) ) ); 252 | 253 | if( FD_ISSET( m_sock, &read_fds) ) 254 | if ( m_is_ssl ) 255 | received = SSL_read( m_ssl, m_buf, MAX_TCP_BUFFER - 1 ); 256 | else 257 | received = ::recv( m_sock, m_buf, MAX_TCP_BUFFER - 1, 0 ); 258 | else 259 | received = 0; 260 | } 261 | } 262 | return ret_val; 263 | } 264 | catch( exception& ex ) { 265 | cerr << "exception in HTTP_Checker::get_response(): for host '" << m_host_name.c_str() << "'" << endl; 266 | return ""; 267 | } 268 | } 269 | 270 | bool HTTP_Checker::init_socket( addrinfo *addr ) { 271 | if ( NULL != addr ) { 272 | m_sock = ::socket( addr->ai_family, addr->ai_socktype, addr->ai_protocol ); 273 | } else { 274 | m_sock = ::socket( AF_INET, SOCK_STREAM, IPPROTO_TCP ); 275 | } 276 | if ( -1 == m_sock ) { 277 | errno = 0; 278 | #if DEBUG_MODE 279 | cerr << "unable to create socket" << endl; 280 | #endif 281 | return false; 282 | } 283 | 284 | int val = 1; 285 | int ret_val = ::setsockopt( m_sock, SOL_SOCKET, SO_REUSEADDR, &val, sizeof( val ) ); 286 | if( -1 == ret_val ) { 287 | close( m_sock ); 288 | m_sock = -1; 289 | errno = 0; 290 | #if DEBUG_MODE 291 | cerr << "unable to set socket option SO_REUSEADDR" << endl; 292 | #endif 293 | return false; 294 | } 295 | 296 | #if NON_BLOCKING_IO 297 | int flags = fcntl( m_sock, F_GETFL, 0 ); 298 | if ( fcntl( m_sock, F_SETFL, flags | O_NONBLOCK ) ) { 299 | close( m_sock ); 300 | m_sock = -1; 301 | errno = 0; 302 | #if DEBUG_MODE 303 | cerr << "could not fcntl" << endl; 304 | #endif 305 | return false; 306 | } 307 | #endif // NON_BLOCKING_IO 308 | 309 | struct timeval time_out; 310 | time_out.tv_sec = NET_COMMS_TIMEOUT; 311 | time_out.tv_usec = 0; 312 | 313 | ret_val = ::setsockopt( m_sock, SOL_SOCKET, SO_SNDTIMEO, &time_out, sizeof( time_out ) ); 314 | if( -1 == ret_val ) { 315 | close( m_sock ); 316 | m_sock = -1; 317 | errno = 0; 318 | #if DEBUG_MODE 319 | cerr << "unable to set socket option SO_SNDTIMEO" << endl; 320 | #endif 321 | return false; 322 | } 323 | 324 | ret_val = ::setsockopt( m_sock, SOL_SOCKET, SO_RCVTIMEO, &time_out, sizeof( time_out ) ); 325 | if( -1 == ret_val ) { 326 | close( m_sock ); 327 | m_sock = -1; 328 | errno = 0; 329 | #if DEBUG_MODE 330 | cerr << "unable to set socket option SO_RCVTIMEO" << endl; 331 | #endif 332 | return false; 333 | } 334 | 335 | return true; 336 | } 337 | 338 | bool HTTP_Checker::init_ssl() { 339 | m_ctx = SSL_CTX_new( SSLv23_client_method() ); 340 | 341 | if ( NULL == m_ctx ) { 342 | close( m_sock ); 343 | m_sock = -1; 344 | errno = 0; 345 | #if DEBUG_MODE 346 | cerr << "unable to set SSL context" << endl; 347 | #endif 348 | return false; 349 | } 350 | 351 | #ifdef SSL_MODE_RELEASE_BUFFERS 352 | SSL_CTX_set_mode( m_ctx, SSL_MODE_RELEASE_BUFFERS ); 353 | #endif 354 | 355 | if ( ! SSL_CTX_load_verify_locations( m_ctx, NULL, "/etc/ssl/certs" ) ) { 356 | close( m_sock ); 357 | m_sock = -1; 358 | errno = 0; 359 | #if DEBUG_MODE 360 | cerr << "unable to load the cert location" << endl; 361 | #endif 362 | return false; 363 | } 364 | 365 | m_ssl = SSL_new( m_ctx ); 366 | 367 | if ( NULL == m_ssl ) { 368 | close( m_sock ); 369 | m_sock = -1; 370 | errno = 0; 371 | #if DEBUG_MODE 372 | cerr << "unable to set init SSL" << endl; 373 | #endif 374 | return false; 375 | } 376 | 377 | SSL_set_mode( m_ssl, SSL_MODE_AUTO_RETRY ); 378 | return true; 379 | } 380 | 381 | #if USE_GETADDRINFO 382 | bool HTTP_Checker::connect_getaddrinfo() { 383 | try { 384 | addrinfo *res = 0; 385 | struct addrinfo hints; 386 | memset( &hints, 0, sizeof( hints ) ); 387 | hints.ai_family = AF_UNSPEC; 388 | hints.ai_flags = AI_ADDRCONFIG; 389 | hints.ai_socktype = SOCK_STREAM; 390 | int con_ret = -1; 391 | int result = -1; 392 | 393 | #if DEBUG_MODE 394 | cerr << "getaddrinfo: looking up " << m_host_name.c_str() << endl; 395 | #endif 396 | string s_lookup_type = "http"; 397 | if ( m_is_ssl ) { 398 | s_lookup_type = "https"; 399 | } 400 | 401 | result = getaddrinfo( m_host_name.c_str(), s_lookup_type.c_str(), &hints, &res ); 402 | if ( EAI_BADFLAGS == result ) { 403 | hints.ai_flags = 0; 404 | result = getaddrinfo( m_host_name.c_str(), s_lookup_type.c_str(), &hints, &res ); 405 | } 406 | 407 | if ( EAI_NONAME == result ) { 408 | #if DEBUG_MODE 409 | cerr << "NXDOMAIN: " << m_host_name.c_str() << endl; 410 | #endif 411 | return false; 412 | } 413 | if ( 0 != result || EAI_FAIL == result ) { 414 | #if DEBUG_MODE 415 | cerr << "Error looking up host: " << m_host_name.c_str() << endl; 416 | #endif 417 | return false; 418 | } 419 | 420 | addrinfo *node = res; 421 | int tried_recs = 0; 422 | while ( node && m_cutofftime > time( NULL ) ) { 423 | if ( ! ( AF_INET == node->ai_family || AF_INET6 == node->ai_family ) ) { 424 | node = node->ai_next; 425 | continue; 426 | } 427 | tried_recs++; 428 | if ( ! init_socket( node ) ) { 429 | #if DEBUG_MODE 430 | cerr << "socket init failed" << endl; 431 | #endif 432 | node = node->ai_next; 433 | continue; 434 | } 435 | 436 | #if NON_BLOCKING_IO // NON_BLOCKING_IO 437 | struct epoll_event ev; 438 | struct epoll_event events[MAX_EPOLL_EVENTS]; 439 | 440 | int e_fd = epoll_create1( 0 ); 441 | if ( -1 == e_fd ) { 442 | #if DEBUG_MODE 443 | cerr << "epoll_create failed" << endl; 444 | #endif 445 | close( m_sock ); 446 | m_sock = -1; 447 | errno = 0; 448 | node = node->ai_next; 449 | continue; 450 | } 451 | 452 | ev.data.fd = m_sock; 453 | ev.events = EPOLLOUT | EPOLLIN | EPOLLERR | EPOLLHUP; 454 | int c_fd = epoll_ctl( e_fd, EPOLL_CTL_ADD, m_sock, &ev ); 455 | if ( 0 != c_fd ) { 456 | #if DEBUG_MODE 457 | cerr << "epoll_ctl failed" << endl; 458 | #endif 459 | close( e_fd ); 460 | close( m_sock ); 461 | m_sock = -1; 462 | errno = 0; 463 | node = node->ai_next; 464 | continue; 465 | } 466 | 467 | #endif // NON_BLOCKING_IO 468 | 469 | con_ret = ::connect( m_sock, node->ai_addr, node->ai_addrlen ); 470 | 471 | #if NON_BLOCKING_IO // NON_BLOCKING_IO 472 | 473 | if ( con_ret < 0 && errno != EINPROGRESS ) { 474 | #if DEBUG_MODE 475 | cerr << "socket connect failed" << endl; 476 | #endif 477 | close( e_fd ); 478 | close( m_sock ); 479 | m_sock = -1; 480 | con_ret = -1; 481 | errno = 0; 482 | node = node->ai_next; 483 | continue; 484 | } 485 | 486 | if ( con_ret == 0 ) { 487 | close( e_fd ); 488 | break; 489 | } 490 | 491 | int timeout = m_cutofftime - time( NULL ); 492 | if ( timeout < 0 ) { 493 | #if DEBUG_MODE 494 | cerr << "timed out for " << m_host_name.c_str() << endl; 495 | #endif 496 | errno = 0; 497 | con_ret = -1; 498 | close( e_fd ); 499 | break; 500 | } 501 | int num_events = epoll_wait( e_fd, events, MAX_EPOLL_EVENTS, timeout * 1000 ); 502 | for ( int i = 0; i < num_events; i++ ) { 503 | if ( events[i].events & EPOLLERR || events[i].events & EPOLLHUP ) { 504 | #if DEBUG_MODE 505 | cerr << "epoll error or HUP" << endl; 506 | #endif 507 | con_ret = -1; 508 | break; 509 | } else if ( events[i].events & EPOLLOUT ) { 510 | con_ret = 0; 511 | break; 512 | } 513 | } 514 | close( e_fd ); 515 | #endif // NON_BLOCKING_IO 516 | 517 | if ( con_ret == 0 ) { 518 | break; 519 | } 520 | #if DEBUG_MODE 521 | cerr << "failed to connect to " << m_host_name.c_str() << endl; 522 | #endif 523 | close( m_sock ); 524 | m_sock = -1; 525 | con_ret = -1; 526 | errno = 0; 527 | node = node->ai_next; 528 | } 529 | #if DEBUG_MODE 530 | if ( 0 == tried_recs && 0 == node ) { 531 | cerr << "unknown address types for: " << m_host_name.c_str() << endl; 532 | } 533 | #endif 534 | freeaddrinfo( res ); 535 | return ( 0 == con_ret ); 536 | } 537 | catch( exception& ex ) { 538 | cerr << "exception in HTTP_Checker::connect(): for host '" << m_host_name.c_str() << "'" << endl; 539 | return false; 540 | } 541 | } 542 | 543 | #else // USE_GETADDRINFO 544 | 545 | 546 | bool HTTP_Checker::connect_gethostbyname() { 547 | try { 548 | struct sockaddr_in m_addr; 549 | char *tmp = (char *)malloc( MAX_TCP_BUFFER ); 550 | struct hostent hostbuf, *hp; 551 | int herr, hres; 552 | 553 | #if DEBUG_MODE 554 | cerr << "gethostbyname: looking up " << m_host_name.c_str() << endl; 555 | #endif 556 | hres = gethostbyname_r( m_host_name.c_str(), &hostbuf, tmp, MAX_TCP_BUFFER, &hp, &herr ); 557 | if ( ERANGE == hres ) { 558 | #if DEBUG_MODE 559 | cerr << "realloc for DNS results" << endl; 560 | #endif 561 | tmp = (char *)realloc( tmp, ( MAX_TCP_BUFFER * 2 ) ); 562 | if ( NULL == tmp ) { 563 | #if DEBUG_MODE 564 | cerr << "realloc error!" << endl; 565 | #endif 566 | return false; 567 | } 568 | hres = gethostbyname_r( m_host_name.c_str(), &hostbuf, tmp, ( MAX_TCP_BUFFER * 2 ), &hp, &herr ); 569 | } 570 | 571 | if ( hp ) { 572 | m_addr.sin_port = htons( m_port ); 573 | m_addr.sin_family = hp->h_addrtype; 574 | bcopy( hp->h_addr, (caddr_t)&m_addr.sin_addr, hp->h_length ); 575 | } else { 576 | #if DEBUG_MODE 577 | cerr << "NXDOMAIN: " << m_host_name.c_str() << endl; 578 | #endif 579 | free( tmp ); 580 | return false; 581 | } 582 | 583 | if ( ! init_socket( NULL ) ) { 584 | #if DEBUG_MODE 585 | cerr << "socket init failed" << endl; 586 | #endif 587 | free( tmp ); 588 | return false; 589 | } 590 | 591 | #if NON_BLOCKING_IO // NON_BLOCKING_IO 592 | int e_fd = epoll_create1( 0 ); 593 | if ( -1 == e_fd ) { 594 | #if DEBUG_MODE 595 | cerr << "epoll_create failed" << endl; 596 | #endif 597 | close( m_sock ); 598 | m_sock = -1; 599 | errno = 0; 600 | free( tmp ); 601 | return false; 602 | } 603 | 604 | struct epoll_event ev; 605 | struct epoll_event events[MAX_EPOLL_EVENTS]; 606 | 607 | ev.data.fd = m_sock; 608 | ev.events = EPOLLOUT | EPOLLIN | EPOLLERR | EPOLLHUP; 609 | int c_fd = epoll_ctl( e_fd, EPOLL_CTL_ADD, m_sock, &ev ); 610 | if ( 0 != c_fd ) { 611 | #if DEBUG_MODE 612 | cerr << "epoll_ctl failed" << endl; 613 | #endif 614 | close( e_fd ); 615 | close( m_sock ); 616 | m_sock = -1; 617 | errno = 0; 618 | free( tmp ); 619 | return false; 620 | } 621 | 622 | #endif // NON_BLOCKING_IO 623 | 624 | int con_ret = ::connect( m_sock, (struct sockaddr *)&m_addr, sizeof( struct sockaddr ) ); 625 | free( tmp ); 626 | 627 | #if NON_BLOCKING_IO 628 | if ( con_ret < 0 && errno != EINPROGRESS ) { 629 | #if DEBUG_MODE 630 | cerr << "failed to connect to " << m_host_name.c_str() << endl; 631 | #endif 632 | close( e_fd ); 633 | close( m_sock ); 634 | m_sock = -1; 635 | return false; 636 | } else if ( 0 != con_ret ) { 637 | int timeout = m_cutofftime - time( NULL ); 638 | if ( timeout < 0 ) { 639 | #if DEBUG_MODE 640 | cerr << "timed out for " << m_host_name.c_str() << endl; 641 | #endif 642 | errno = 0; 643 | close( e_fd ); 644 | close( m_sock ); 645 | m_sock = -1; 646 | return false; 647 | } 648 | 649 | int num_events = epoll_wait( e_fd, events, MAX_EPOLL_EVENTS, timeout * 1000 ); 650 | for ( int i = 0; i < num_events; i++ ) { 651 | if ( events[i].events & EPOLLERR || events[i].events & EPOLLHUP ) { 652 | #if DEBUG_MODE 653 | cerr << "epoll error or HUP for " << m_host_name.c_str() << endl; 654 | #endif 655 | con_ret = -1; 656 | break; 657 | } else if ( events[i].events & EPOLLOUT ) { 658 | con_ret = 0; 659 | break; 660 | } 661 | } 662 | } 663 | 664 | close( e_fd ); 665 | #endif // NON_BLOCKING_IO 666 | 667 | return ( 0 == con_ret ); 668 | } 669 | catch( exception& ex ) { 670 | cerr << "exception in HTTP_Checker::connect(): for host '" << m_host_name.c_str() << "'" << endl; 671 | return false; 672 | } 673 | } 674 | 675 | #endif // USE_GETADDRINFO 676 | 677 | bool HTTP_Checker::connect() { 678 | try { 679 | #if USE_GETADDRINFO 680 | if ( ! this->connect_getaddrinfo() ) { 681 | #else 682 | if ( ! this->connect_gethostbyname() ) { 683 | #endif 684 | #if DEBUG_MODE 685 | int so_error; 686 | socklen_t len = sizeof so_error; 687 | ::getsockopt( m_sock, SOL_SOCKET, SO_ERROR, &so_error, &len ); 688 | if ( 0 != so_error ) { 689 | cerr << "socket connect error: " << m_host_name.c_str() << " : " << strerror( so_error ) << endl; 690 | } 691 | #endif 692 | if ( -1 != m_sock ) { 693 | close( m_sock ); 694 | m_sock = -1; 695 | } 696 | errno = 0; 697 | return false; 698 | } 699 | 700 | #if DEBUG_MODE 701 | cerr << "connected!" << endl; 702 | #endif 703 | 704 | if ( m_is_ssl ) { 705 | if ( ! this->init_ssl() ) 706 | return false; 707 | 708 | m_sbio = BIO_new_socket( m_sock, BIO_NOCLOSE ); 709 | if ( NULL == m_sbio ) { 710 | #if DEBUG_MODE 711 | cerr << "The SSL socket alloc failed" << endl; 712 | #endif 713 | close( m_sock ); 714 | m_sock = -1; 715 | errno = 0; 716 | return false; 717 | } 718 | 719 | SSL_set_bio( m_ssl, m_sbio, m_sbio ); 720 | SSL_set_tlsext_host_name( m_ssl, m_host_name.c_str() ); 721 | 722 | #if NON_BLOCKING_IO 723 | int status; 724 | bool want_read = false; 725 | bool want_write = false; 726 | do { 727 | status = SSL_connect( m_ssl ); 728 | switch ( SSL_get_error( m_ssl, status ) ) { 729 | case SSL_ERROR_NONE: 730 | status = 0; 731 | break; 732 | case SSL_ERROR_WANT_WRITE: 733 | want_write = true; 734 | status = 1; 735 | break; 736 | case SSL_ERROR_WANT_READ: 737 | want_read = true; 738 | status = 1; 739 | break; 740 | case SSL_ERROR_ZERO_RETURN: 741 | // The peer has notified us that it is shutting down via 742 | // the SSL "close_notify" message so we need to shutdown, too. 743 | status = -1; 744 | break; 745 | case SSL_ERROR_SYSCALL: 746 | if ( EWOULDBLOCK == errno && -1 == status ) { 747 | // Although the SSL_ERROR_WANT_READ/WRITE isn't getting 748 | // set correctly, the read/write state should be valid. 749 | errno = 0; 750 | status = 1; 751 | if ( SSL_want_write( m_ssl ) ) { 752 | want_write = true; 753 | } else if ( SSL_want_read( m_ssl ) ) { 754 | want_read = true; 755 | } else { 756 | status = -1; 757 | } 758 | } else { 759 | status = -1; 760 | } 761 | break; 762 | default: 763 | status = -1; 764 | break; 765 | } 766 | 767 | if ( 1 == status ) { 768 | if ( ! want_read && ! want_write ) { 769 | #if DEBUG_MODE 770 | cerr << "The SSL connect failed for " << m_host_name.c_str() << endl; 771 | #endif 772 | return false; 773 | } 774 | 775 | fd_set read_fds, write_fds; 776 | if ( want_read ) { 777 | FD_ZERO( &read_fds ); 778 | FD_SET( m_sock, &read_fds ); 779 | } 780 | if ( want_write ) { 781 | FD_ZERO( &write_fds ); 782 | FD_SET( m_sock, &write_fds ); 783 | } 784 | 785 | struct timeval tv; 786 | tv.tv_sec = m_cutofftime - time( NULL ); 787 | tv.tv_usec = 0; 788 | status = ::select( m_sock + 1, &read_fds, &write_fds, NULL, &tv ); 789 | 790 | // 0 is timeout, -1 is error, or one or both handles could be set 791 | if ( status >= 1 ) { 792 | status = 1; 793 | } else { 794 | status = -1; 795 | } 796 | } 797 | } while ( 1 == status && ! SSL_is_init_finished( m_ssl ) && m_cutofftime > time( NULL ) ); 798 | 799 | if ( 0 != status || ! SSL_is_init_finished( m_ssl ) ) { 800 | #if DEBUG_MODE 801 | cerr << "The SSL handshake failed for " << m_host_name.c_str() 802 | << ERR_error_string( ERR_get_error(), NULL ) << endl; 803 | #endif 804 | return false; 805 | } 806 | 807 | #else // NON_BLOCKING_IO 808 | int status = SSL_connect( m_ssl ); 809 | 810 | if ( 1 != status ) { 811 | #if DEBUG_MODE 812 | cerr << "The SSL handshake failed for " << m_host_name.c_str() 813 | << ERR_error_string( ERR_get_error(), NULL ) << endl; 814 | #endif 815 | return false; 816 | } 817 | #endif // NON_BLOCKING_IO 818 | 819 | X509* cert = SSL_get_peer_certificate( m_ssl ); 820 | if ( cert ) { 821 | X509_free( cert ); 822 | } 823 | } 824 | return true; 825 | } 826 | catch( exception& ex ) { 827 | cerr << "exception in HTTP_Checker::connect(): for host '" << m_host_name.c_str() << "'" << endl; 828 | return false; 829 | } 830 | } 831 | 832 | #if NON_BLOCKING_IO 833 | void HTTP_Checker::disconnect_ssl() { 834 | int status; 835 | // attempt shutdown for a max of 3 seconds 836 | time_t waittime = time( NULL ) + 3; 837 | if ( m_cutofftime < waittime ) { 838 | waittime = m_cutofftime; 839 | } 840 | do { 841 | #if DEBUG_MODE 842 | cerr << "SSL shutdown handshake for " << m_host_name.c_str() << endl; 843 | #endif 844 | status = SSL_shutdown( m_ssl ); 845 | switch ( status ) { 846 | case 1: 847 | #if DEBUG_MODE 848 | cerr << "clean shutdown : " << m_host_name.c_str() << endl; 849 | #endif 850 | return; 851 | case -1: 852 | #if DEBUG_MODE 853 | cerr << "shutdown failed: " << m_host_name.c_str() << endl; 854 | #endif 855 | ERR_print_errors_fp( stderr ); 856 | return; 857 | default: 858 | #if DEBUG_MODE 859 | cerr << "shutdown not yet finished : " << m_host_name.c_str() << endl; 860 | #endif 861 | break; 862 | } 863 | switch ( SSL_get_error( m_ssl, status ) ) { 864 | case SSL_ERROR_WANT_WRITE: 865 | case SSL_ERROR_WANT_READ: 866 | #if DEBUG_MODE 867 | cerr << "want read/write : " << m_host_name.c_str() << endl; 868 | #endif 869 | fd_set read_fds, write_fds; 870 | FD_ZERO( &read_fds ); 871 | FD_ZERO( &write_fds ); 872 | FD_SET( m_sock, &read_fds ); 873 | FD_SET( m_sock, &write_fds ); 874 | 875 | struct timeval tv; 876 | tv.tv_sec = waittime - time( NULL ); 877 | tv.tv_usec = 0; 878 | #if DEBUG_MODE 879 | cerr << "selecting : " << m_host_name.c_str() << endl; 880 | #endif 881 | status = ::select( m_sock + 1, &read_fds, &write_fds, NULL, &tv ); 882 | #if DEBUG_MODE 883 | cerr << "select result : " << status << endl; 884 | #endif 885 | if ( status >= 1 ) { 886 | status = 1; 887 | } 888 | break; 889 | case SSL_ERROR_SYSCALL: 890 | // From the man page: 891 | // The output of SSL_get_error(3) may be misleading, as an erroneous 892 | // SSL_ERROR_SYSCALL may be flagged even though no error occurred. 893 | status = 1; 894 | break; 895 | default: 896 | #if DEBUG_MODE 897 | cerr << "generic error for : " << m_host_name.c_str() << endl; 898 | #endif 899 | status = 1; 900 | break; 901 | } 902 | } while ( 1 == status && waittime > time( NULL ) ); 903 | } 904 | #endif // NON_BLOCKING_IO 905 | 906 | bool HTTP_Checker::disconnect() { 907 | try { 908 | if ( m_is_ssl ) { 909 | if ( NULL != m_ssl ) { 910 | #if NON_BLOCKING_IO 911 | this->disconnect_ssl(); 912 | #else 913 | // attempt shutdown for a max of 3 seconds 914 | time_t waittime = time( NULL ) + 3; 915 | if ( m_cutofftime < waittime ) { 916 | waittime = m_cutofftime; 917 | } 918 | int res = SSL_shutdown( m_ssl ); 919 | while ( 1 != res && waittime > time( NULL ) ) { 920 | res = SSL_shutdown( m_ssl ); 921 | sleep( 1 ); 922 | } 923 | #if DEBUG_MODE 924 | if ( 1 == res ) { 925 | cerr << "client exited gracefully: " << m_host_name.c_str() << endl; 926 | } else { 927 | cerr << "error in shutdown for " << m_host_name.c_str() << endl; 928 | ERR_print_errors_fp( stderr ); 929 | } 930 | #endif // DEBUG_MODE 931 | #endif // NON_BLOCKING_IO 932 | SSL_free( m_ssl ); 933 | m_ssl = NULL; 934 | } 935 | if ( NULL != m_ctx ) { 936 | SSL_CTX_free( m_ctx ); 937 | m_ctx = NULL; 938 | } 939 | } 940 | if ( m_sock > 0 ) { 941 | if ( ::shutdown( m_sock, SHUT_RDWR ) != 0 ) { 942 | errno = 0; 943 | } 944 | ::close( m_sock ); 945 | m_sock = -1; 946 | } 947 | return true; 948 | } 949 | catch( exception &ex ) { 950 | cerr << "exception in HTTP_Checker::disconnect(): for host '" << m_host_name.c_str() << "'" << endl; 951 | return false; 952 | } 953 | } 954 | 955 | bool HTTP_Checker::send_bytes( char* p_packet, size_t p_packet_length ) { 956 | try { 957 | ssize_t bytes_left = p_packet_length; 958 | ssize_t bytes_sent = 0; 959 | int send_attempts = 5; 960 | ssize_t bytes_to_send = 0; 961 | 962 | do 963 | { 964 | if ( bytes_left < MAX_TCP_BUFFER ) 965 | bytes_to_send = bytes_left; 966 | else 967 | bytes_to_send = MAX_TCP_BUFFER; 968 | 969 | if ( m_is_ssl ) 970 | bytes_sent = SSL_write( m_ssl, (const char *)p_packet, (int)bytes_to_send ); 971 | else 972 | bytes_sent = ::send( this->m_sock, (const char *)p_packet, (int)bytes_to_send, 0 ); 973 | 974 | if ( bytes_sent != bytes_to_send ) { 975 | switch ( errno ) { 976 | case ENOTSOCK: { 977 | #if DEBUG_MODE 978 | cerr << "ERROR: socket operation on non-socket irrecoverable; aborting" << endl; 979 | #endif 980 | return false; 981 | } 982 | case EBADF: { 983 | #if DEBUG_MODE 984 | cerr << "ERROR: bad file descriptor is irrecoverable; aborting" << endl; 985 | #endif 986 | return false; 987 | } 988 | case EPIPE: { 989 | #if DEBUG_MODE 990 | cerr << "ERROR: broken pipe is irrecoverable; aborting" << endl; 991 | #endif 992 | return false; 993 | } 994 | default: { 995 | #if DEBUG_MODE 996 | cerr << "ERROR: unknown error (" << errno << "); aborting" << endl; 997 | #endif 998 | return false; 999 | } 1000 | } 1001 | } 1002 | if ( bytes_sent > 0 ) { 1003 | bytes_left -= bytes_sent; 1004 | p_packet += bytes_sent; 1005 | } 1006 | } while( ( bytes_left > 0 ) && --send_attempts && m_cutofftime > time( NULL ) ); 1007 | 1008 | return ( bytes_left == 0 ); 1009 | } 1010 | catch( exception & ex ) { 1011 | cerr << "exception in HTTP_Checker::send_bytes(): for host '" << m_host_name.c_str() << "'" << endl; 1012 | return false; 1013 | } 1014 | } 1015 | 1016 | -------------------------------------------------------------------------------- /lib/jetmon.js: -------------------------------------------------------------------------------- 1 | 2 | process.title = 'jetmon-master'; 3 | 4 | const SITE_DOWN = 0; 5 | const SITE_RUNNING = 1; 6 | const SITE_CONFIRMED_DOWN = 2; 7 | 8 | const HOST_OFFLINE = 0; 9 | const HOST_ONLINE = 1; 10 | 11 | const SUICIDE_SIGNAL = 1; 12 | const EXIT_MAXRAMUSAGE = 2; 13 | const EXIT_MAXCHECKS = 3; 14 | 15 | const NUM_SSL_SERVERS = 4; 16 | 17 | const JETMON_CHECK = 1; 18 | const VERIFLIER_CHECK = 2; 19 | 20 | const STATUS_PORT = 7802; 21 | 22 | const SECONDS = 1000; 23 | const MINUTES = 60 * SECONDS; 24 | const HOURS = 60 * MINUTES; 25 | const DAYS = 24 * HOURS; 26 | 27 | global.config = require( './config' ); 28 | config.load(); 29 | 30 | // This determines how many peers have to confirm that the 31 | // site is down before a notification email is sent 32 | const PEER_OFFLINE_LIMIT = global.config.get( 'PEER_OFFLINE_LIMIT' ) || 3; 33 | 34 | var child_proc = require('child_process'); 35 | var fs = require( 'fs' ); 36 | var o_log4js = require( 'log4js' ); 37 | 38 | o_log4js.configure( { 39 | appenders: [ { 40 | 'type' : 'file', 41 | 'filename' : 'logs/jetmon.log', 42 | 'maxLogSize': 52428800, 43 | 'backups' : 30, 44 | 'category' : 'flog', 45 | 'levels' : 'DEBUG', 46 | }, 47 | { 48 | 'type' : 'file', 49 | 'filename' : 'logs/status-change.log', 50 | 'maxLogSize': 104857600, 51 | 'backups' : 100, 52 | 'category' : 'slog', 53 | 'levels' : 'DEBUG', 54 | } 55 | ] 56 | }); 57 | o_log4js.PatternLayout = '%d{HH:mm:ss,SSS} m'; 58 | global.logger = o_log4js.getLogger( 'flog' ); 59 | var slogger = o_log4js.getLogger( 'slog' ); 60 | 61 | var db_mysql = require( './database' ); 62 | var wpcom = require( './wpcom' ); 63 | var comms = require( './comms' ); 64 | var cluster = require( 'cluster' ); 65 | 66 | const statsdClient = require('./statsd.js'); 67 | 68 | var gCountSuccess = 0; 69 | var gCountError = 0; 70 | var gCountOffline = 0; 71 | var startTime = new Date().valueOf(); 72 | var sitesCount = 0; 73 | var arrObjects = []; 74 | var localRetries = []; 75 | var freeWorkers = []; 76 | var haltedWorkers = []; 77 | var arrWorkers = []; 78 | var workerStats = {}; 79 | var checkStats = {}; 80 | var gettingSites = false; 81 | var inRound = false; 82 | var endOfRound = false; 83 | var roundSitesCount = 0; 84 | 85 | global.queuedRetries = []; 86 | 87 | logger.debug( 'booting jetmon.js' ); 88 | 89 | process.on( 'SIGINT', gracefulShutdown ); 90 | process.on( 'EXIT', gracefulShutdown ); 91 | 92 | process.on( 'SIGHUP', function() { 93 | logger.debug( 'reloading config file' ); 94 | global.config.load(); 95 | 96 | statsdClient.increment('config_reload.count'); 97 | 98 | for ( var count in arrWorkers ) { 99 | if ( undefined !== arrWorkers[ count ] ) 100 | arrWorkers[ count ].send( { pid : arrWorkers[ count ].pid, request : 'config-update' } ); 101 | } 102 | }); 103 | 104 | process.on( 'uncaughtException', function( errDesc ) { 105 | logger.debug( 'uncaughtException error: ' + errDesc ); 106 | }); 107 | 108 | 109 | function spawnWorker() { 110 | var worker = child_proc.fork('./lib/httpcheck.js' ); 111 | 112 | statsdClient.increment('worker.spawn.new.count'); 113 | 114 | worker.on( 'message', workerMsgCallback ); 115 | worker.on( 'exit', function( code, signal ) { 116 | if ( true == worker.exitedAfterDisconnect ) { 117 | logger.debug( 'worker thread pid ' + worker.pid + ' shutting down.' ); 118 | 119 | statsdClient.increment('worker.die.shutdown.count'); 120 | } else { 121 | var respawn = false; 122 | 123 | if ( SUICIDE_SIGNAL == code ) { 124 | logger.debug( 'worker thread pid ' + worker.pid + ' was asked to evaporate.' ); 125 | statsdClient.increment('worker.die.evaporate.count'); 126 | } else if ( EXIT_MAXRAMUSAGE == code ) { 127 | logger.debug( 'worker thread pid ' + worker.pid + ' exited due to reaching mem limit, replacing...' ); 128 | statsdClient.increment('worker.die.memlimit.count'); 129 | respawn = true; 130 | } else if ( EXIT_MAXCHECKS == code ) { 131 | logger.debug( 'worker thread pid ' + worker.pid + ' exited due to reaching check limit, replacing...' ); 132 | statsdClient.increment('worker.die.checklimit.count'); 133 | respawn = true; 134 | } else { 135 | if ( 130 == code ) { 136 | logger.debug( 'worker thread pid ' + worker.pid + ' shutting down.' ); 137 | statsdClient.increment('worker.die.code_130.count'); 138 | } else { 139 | logger.debug( 'worker thread pid ' + worker.pid + ' died (' + code + '), creating a replacement.' ); 140 | statsdClient.increment('worker.die.code_other.count'); 141 | respawn = true; 142 | } 143 | } 144 | 145 | deleteWorker( worker.pid ); 146 | 147 | if ( respawn ) { 148 | spawnWorker(); 149 | } 150 | } 151 | } ); 152 | 153 | // Ensure that the new worker PID is not in any of the existing arrays. 154 | deleteWorker( worker.pid ); 155 | 156 | arrWorkers.push( worker ); 157 | } 158 | 159 | function deleteWorker( pid ) { 160 | if ( ! pid ) 161 | return; 162 | for ( var count in arrWorkers ) { 163 | if ( 164 | ( undefined != arrWorkers[count] ) && 165 | ( arrWorkers[count].pid == pid ) 166 | ) { 167 | arrWorkers.splice( count, 1 ); 168 | if ( workerStats[pid] ) { 169 | delete( workerStats[pid] ); 170 | } 171 | 172 | statsdClient.increment('worker.delete.count'); 173 | 174 | break; 175 | } 176 | } 177 | freeWorkers = freeWorkers.filter( a => a !== pid ); 178 | haltedWorkers = haltedWorkers.filter( a => a !== pid ); 179 | } 180 | 181 | function getWorker( pid ) { 182 | if ( ! pid ) 183 | return null; 184 | for ( var count in arrWorkers ) { 185 | if ( ( undefined != arrWorkers[ count ] ) && 186 | ( arrWorkers[ count ].pid == pid ) ) { 187 | return arrWorkers[ count ]; 188 | } 189 | } 190 | return null; 191 | } 192 | 193 | function gracefulShutdown() { 194 | // Note: calling the 'logger' object during shutdown causes an immediate exit (only use 'console.log') 195 | console.log( 'Caught shutdown signal, disconnecting worker threads.' ); 196 | for ( var count in arrWorkers ) { 197 | if ( undefined !== arrWorkers[ count ] && arrWorkers[ count ].connected ) { 198 | arrWorkers[ count ].disconnect(); 199 | } 200 | } 201 | 202 | console.log( 'committing any outstanding db updates.' ); 203 | db_mysql.commitUpdates( 204 | function() { 205 | printTotalsExit(); 206 | process.exit( 0 ); 207 | }); 208 | } 209 | 210 | function printTotalsExit() { 211 | printTotals(); 212 | process.exit( 0 ); 213 | } 214 | 215 | function printTotals() { 216 | console.log( '' ); 217 | console.log( 'Error: ' + gCountError ); 218 | console.log( 'Offline: ' + gCountOffline ); 219 | console.log( 'Success: ' + gCountSuccess ); 220 | console.log( 'Total: ' + ( gCountSuccess + gCountError + gCountOffline ) ); 221 | var now = new Date().valueOf(); 222 | console.log( 'Time: ' + Math.floor( ( now - startTime ) / 60000 ) + 'm ' + 223 | ( ( ( now - startTime ) % 60000 ) / 1000 ) + 's' ); 224 | } 225 | 226 | function resetVariables() { 227 | startTime = new Date().valueOf(); 228 | endOfRound = false; 229 | } 230 | 231 | function getRoundDuration() { 232 | if ( global.config.get( 'USE_VARIABLE_CHECK_INTERVALS' ) ) { 233 | /** 234 | * If variable check intervals are enabled, rounds must run every 235 | * minute. 236 | */ 237 | return 60; 238 | } else { 239 | return global.config.get( 'MIN_TIME_BETWEEN_ROUNDS_SEC' ); 240 | } 241 | } 242 | 243 | function getMoreSites() { 244 | gettingSites = true; 245 | 246 | if ( ! inRound ) { 247 | inRound = true; 248 | } 249 | 250 | if ( endOfRound ) { 251 | var timeSinceStart = new Date().valueOf() - startTime; 252 | var timeToNextLoop = ( getRoundDuration() * SECONDS ) - timeSinceStart; 253 | 254 | statsdClient.timing( 'round.done_sending_work.time', timeSinceStart ); 255 | 256 | setTimeout( function() { 257 | resetVariables(); 258 | getMoreSites(); 259 | }, 260 | timeToNextLoop 261 | ); 262 | return; 263 | } 264 | 265 | /** 266 | * Write out how many items were still in the queue when we requested new batch of data 267 | */ 268 | statsdClient.increment( 'queue.items_left_in_queue_when_fetching_new.count', arrObjects.length ); 269 | 270 | const startTimeGetDbBatch = new Date().valueOf(); 271 | 272 | endOfRound = db_mysql.getNextBatch( 273 | function( rows ) { 274 | if ( ( undefined === rows ) || ( 0 === rows.length ) ) { 275 | getMoreSites(); 276 | return; 277 | } 278 | 279 | const endTimeGetDbBatch = new Date().valueOf(); 280 | statsdClient.timing( 'db.get_next_batch', endTimeGetDbBatch - startTimeGetDbBatch ); 281 | 282 | for ( var i = 0; i < rows.length; i++ ) { 283 | var server = rows[i]; 284 | server.processed = false; 285 | server.oldStatus = server.site_status; 286 | server.last_status_change = new Date( server.last_status_change ).valueOf(); 287 | server.checks = []; 288 | arrObjects.push( server ); 289 | } 290 | gettingSites = false; 291 | freeWorkersToWork(); 292 | }); 293 | } 294 | 295 | function maybeEndRound() { 296 | if ( inRound && 0 === arrObjects.length && arrWorkers.length === freeWorkers.length ) { 297 | // Still in the round. No work is queued. All the workers are free. The round has ended. 298 | inRound = false; 299 | 300 | var timeSinceStart = new Date().valueOf() - startTime; 301 | var timeToNextLoop = ( getRoundDuration() * SECONDS ) - timeSinceStart; 302 | var sps = roundSitesCount / timeSinceStart * 1000; 303 | 304 | if ( 0 === sps % 1 ) { 305 | sps = sps.toFixed( 0 ); 306 | } else { 307 | sps = sps.toFixed( 1 ); 308 | } 309 | 310 | statsdClient.timing( 'round.complete.time', timeSinceStart ); 311 | statsdClient.timing( 'round.next.time', timeToNextLoop ); 312 | statsdClient.increment( 'round.sites.count', roundSitesCount ); 313 | statsdClient.increment( 'round.sps.count', sps ); 314 | 315 | // TODO: Deprecated. Leave this in temporarily to help track changes 316 | // from the old calculation to the new calculation. 317 | statsdClient.timing( 'round.time', timeSinceStart ); 318 | 319 | roundSitesCount = 0; 320 | } 321 | } 322 | 323 | function freeWorkersToWork() { 324 | if ( 0 == arrObjects.length ) 325 | return; 326 | var tmpWorkers = freeWorkers; // take pointer 327 | freeWorkers = []; // and reset 328 | for ( var i = 0; i < tmpWorkers.length; i++ ) 329 | if ( null !== getWorker( tmpWorkers[i] ) ) 330 | workerMsgCallback( { msgtype: 'send_work', worker_pid: tmpWorkers[i] } ); 331 | } 332 | 333 | function checkHostStatus( veriflier_host, data ) { 334 | for( var loop = 0; loop < queuedRetries.length; loop++ ) { 335 | if ( queuedRetries[ loop ].blog_id != data.blog_id || queuedRetries[ loop ].monitor_url != data.monitor_url ) { 336 | continue; 337 | } 338 | queuedRetries[ loop ].requests_outstanding--; 339 | queuedRetries[ loop ].last_activity = new Date().valueOf(); 340 | var replyO = {}; 341 | replyO.type = VERIFLIER_CHECK; 342 | replyO.host = veriflier_host; 343 | replyO.status = data.status; 344 | replyO.rtt = data.rtt; 345 | replyO.code = data.code; 346 | replyO.error_code = data.error_code; 347 | queuedRetries[ loop ].checks.push( replyO ); 348 | if ( HOST_OFFLINE == data.status ) { 349 | queuedRetries[ loop ].offline_confirms++; 350 | if ( queuedRetries[ loop ].offline_confirms >= PEER_OFFLINE_LIMIT ) { 351 | queuedRetries[ loop ].site_status = SITE_DOWN; 352 | wpcom.notifyStatusChange( queuedRetries[ loop ], 353 | function( reply ) { 354 | if ( ! reply.success ) { 355 | logger.error( 'error posting status change, retrying: ' + ( reply?.data || 'no error message' ) ); 356 | wpcom.notifyStatusChange( queuedRetries[ loop ], 357 | function( reply ) { 358 | if ( reply.success ) 359 | logger.trace( 'posted successfully' ); 360 | else 361 | logger.error( 'error posting status change: ' + ( reply?.data || 'no error message' ) ); 362 | }); 363 | } 364 | }); 365 | slogger.trace( 'site_down: ' + JSON.stringify( queuedRetries[ loop ] ) ); 366 | } 367 | } 368 | break; 369 | } 370 | } 371 | 372 | function sslWorkerCallBack( msg ) { 373 | try { 374 | switch ( msg.msgtype ) { 375 | case 'host_status': { 376 | checkHostStatus( msg.payload.veriflier_host, msg.payload ); 377 | break; 378 | } 379 | case 'host_status_array': { 380 | for( var loop = 0; loop < msg.payload.checks.length; loop++ ) { 381 | checkHostStatus( msg.payload.veriflier_host, msg.payload.checks[ loop ] ); 382 | } 383 | break; 384 | } 385 | default: { 386 | logger.debug( 'Unknown SSL worker message type: ' + msg.msgtype ); 387 | break; 388 | } 389 | } 390 | } 391 | catch ( Exception ) { 392 | logger.error( "Error receiving SSL worker's message: " + Exception.toString() ); 393 | } 394 | } 395 | 396 | function workerMsgCallback( msg ) { 397 | try { 398 | switch ( msg.msgtype ) { 399 | case 'totals': 400 | gCountSuccess += msg.work_totals[SITE_RUNNING]; 401 | gCountError += msg.work_totals[SITE_DOWN]; 402 | gCountOffline += msg.work_totals[SITE_CONFIRMED_DOWN]; 403 | sitesCount += msg.work_totals[SITE_DOWN] + msg.work_totals[SITE_RUNNING] + msg.work_totals[SITE_CONFIRMED_DOWN]; 404 | roundSitesCount += msg.work_totals[SITE_DOWN] + msg.work_totals[SITE_RUNNING] + msg.work_totals[SITE_CONFIRMED_DOWN]; 405 | break; 406 | case 'notify_still_down': 407 | // set new server status and then send via the next case statement 408 | msg.server.site_status = SITE_CONFIRMED_DOWN; 409 | case 'notify_status_change': 410 | wpcom.notifyStatusChange( msg.server, 411 | function( reply ) { 412 | if ( ! reply.success ) { 413 | logger.error( 'error posting status change, retrying: ' + ( reply?.data || 'no error message' ) ); 414 | wpcom.notifyStatusChange( msg.server, 415 | function( reply ) { 416 | if ( reply.success ) 417 | logger.trace( 'posted successfully' ); 418 | else 419 | logger.error( 'error posting status change: ' + ( reply?.data || 'no error message' ) ); 420 | }); 421 | } 422 | }); 423 | break; 424 | case 'stop_work': 425 | /** 426 | * Worker asked to no longer receive work so that it can be recycled. 427 | */ 428 | if ( -1 == haltedWorkers.indexOf( msg.worker_pid ) && null !== getWorker( msg.worker_pid ) ) { 429 | haltedWorkers.push( msg.worker_pid ); 430 | freeWorkers = freeWorkers.filter( a => a !== msg.worker_pid ); 431 | } 432 | 433 | maybeEndRound(); 434 | 435 | break; 436 | case 'send_work': 437 | /** 438 | * Worker asked for work 439 | */ 440 | 441 | /** 442 | * There are more workers than needed, kindly ask the worker to shut down. 443 | */ 444 | if ( arrWorkers.length > global.config.get( 'NUM_WORKERS' ) ) { 445 | var w = getWorker( msg.worker_pid ); 446 | if ( null !== w ) 447 | w.send( { 448 | pid : msg.worker_pid, 449 | request : 'evaporate', 450 | payload : 'pls :)' 451 | } ); 452 | break; 453 | } 454 | 455 | if ( 0 == arrObjects.length ) { 456 | /** 457 | * There are no URLs in the global queue, let's flag the worker as "free" 458 | * and request more sites from the database, if we haven't done so yet. 459 | */ 460 | if ( -1 == haltedWorkers.indexOf( msg.worker_pid ) && -1 == freeWorkers.indexOf( msg.worker_pid ) ) { 461 | freeWorkers.push( msg.worker_pid ); 462 | } 463 | if ( ! gettingSites ) { 464 | gettingSites = true; 465 | getMoreSites(); 466 | } 467 | 468 | maybeEndRound(); 469 | } else { 470 | /** 471 | * There are items in the global queue, let's send them to the worker. 472 | */ 473 | assign_work_to_worker( msg.worker_pid ); 474 | } 475 | break; 476 | case 'recheck': 477 | if ( msg.server.checks.length < config.get( 'NUM_OF_CHECKS' ) ) { 478 | add_server_to_local_retries( msg.server ); 479 | } else { 480 | // we have exhausted our local check limit, ask the verifliers to confirm 481 | host_check_request( msg.server ); 482 | } 483 | break; 484 | 485 | case 'stats': 486 | if ( msg.stats ) { 487 | // Update global checkStats var with data from the worker. 488 | for ( let site_status in msg.stats.checkStats ) { 489 | if ( checkStats[site_status] ) { 490 | for ( let http_code in msg.stats.checkStats[site_status]['http_code'] ) { 491 | if ( checkStats[site_status]['http_code'][http_code] ) { 492 | checkStats[site_status]['http_code'][http_code] += msg.stats.checkStats[site_status]['http_code'][http_code]; 493 | } else { 494 | checkStats[site_status]['http_code'][http_code] = msg.stats.checkStats[site_status]['http_code'][http_code]; 495 | } 496 | } 497 | for ( let error_code in msg.stats.checkStats[site_status]['error_code'] ) { 498 | if ( checkStats[site_status]['error_code'][error_code] ) { 499 | checkStats[site_status]['error_code'][error_code] += msg.stats.checkStats[site_status]['error_code'][error_code]; 500 | } else { 501 | checkStats[site_status]['error_code'][error_code] = msg.stats.checkStats[site_status]['error_code'][error_code]; 502 | } 503 | } 504 | checkStats[site_status]['rtt']['count'] += msg.stats.checkStats[site_status]['rtt']['count']; 505 | checkStats[site_status]['rtt']['sum'] += msg.stats.checkStats[site_status]['rtt']['sum']; 506 | checkStats[site_status]['rtt']['max'] = Math.max( checkStats[site_status]['rtt']['max'], msg.stats.checkStats[site_status]['rtt']['max'] ); 507 | checkStats[site_status]['rtt']['min'] = Math.min( checkStats[site_status]['rtt']['min'], msg.stats.checkStats[site_status]['rtt']['min'] ); 508 | } else { 509 | checkStats[site_status] = msg.stats.checkStats[site_status]; 510 | } 511 | } 512 | 513 | // Remove checkStats as it is not needed for workerStats. 514 | delete msg.stats.checkStats; 515 | 516 | workerStats[msg.worker_pid] = msg.stats; 517 | 518 | const workerUptime = msg.stats.uptime; 519 | if ( workerUptime > 5000 ) { 520 | /** 521 | * Log only if the worker has been up for at least 5 seconds, to make sure we don't log 522 | * empty values at the beginning when the worker has just started, but hasn't received any work. 523 | */ 524 | statsdClient.increment( 'worker.queue.active', msg.stats.activeChecks ) 525 | statsdClient.increment( 'worker.queue.queue_size', msg.stats.queueLength ); 526 | } 527 | 528 | /** 529 | * Check if the worker's queue is less than what we want. 530 | * 531 | * If the worker's queue has less than NUM_TO_PROCESS items in there, we want to 532 | * push more, as it might be waiting for some longer-running ones to finish, before continuing. 533 | * This will keep the worker busier than before. 534 | */ 535 | // Math.max used to make sure that we don't go below zero and make crazy assumptions 536 | const queueLeftToProcess = Math.max( 0, msg.stats.queueLength - msg.stats.pointer ); 537 | const maxParallel = global.config.get( 'NUM_TO_PROCESS' ) 538 | 539 | if ( queueLeftToProcess < maxParallel ) { 540 | assign_work_to_worker( msg.worker_pid, global.config.get( 'DATASET_SIZE' ) - queueLeftToProcess ); 541 | } 542 | } 543 | break; 544 | default: 545 | } 546 | } 547 | catch ( Exception ) { 548 | logger.error( "Error receiving worker's message: ", Exception, msg ); 549 | } 550 | } 551 | 552 | /** 553 | * Add the server to localRetries. 554 | * 555 | * The server is not added if it already exists in the array. As it is already 556 | * processing if it is in the array. 557 | * 558 | * @param object server The server object as received by the recheck message. 559 | * 560 | * @returns {null} 561 | */ 562 | function add_server_to_local_retries( server ) { 563 | var found = false; 564 | for( var loop = 0; loop < localRetries.length; loop++ ) { 565 | if ( localRetries[loop].blog_id == server.blog_id && localRetries[loop].monitor_url == server.monitor_url ) { 566 | found = true; 567 | break; 568 | } 569 | } 570 | if ( ! found ) { 571 | server.processed = false; 572 | localRetries.push( server ); 573 | } 574 | } 575 | 576 | /** 577 | * Get a work batch dataset to send to a worker. 578 | * @param int size The batch size. If the number is invalid, negative or more than `DATASET_SIZE`, return `DATASET_SIZE` items. 579 | * 580 | * @returns {*[]|boolean} 581 | */ 582 | function get_work_dataset( size ) { 583 | // Make sure that we don't give too little or too much work. 584 | if ( !size || size < 1 || size > global.config.get( 'DATASET_SIZE' ) ) { 585 | size = global.config.get( 'DATASET_SIZE' ); 586 | } 587 | 588 | if ( arrObjects.length < 1 ) { 589 | return []; 590 | } 591 | 592 | const data = arrObjects.splice( 0, Math.min( arrObjects.length, size ) ) 593 | 594 | return data; 595 | } 596 | 597 | /** 598 | * Assigns (sends) a variable amount of work to a specific worker. 599 | * 600 | * @param int pid The Worker's PID 601 | * @param int|null size The number of items to send to the worker. @see get_work_dataset() 602 | * @returns {null} 603 | */ 604 | function assign_work_to_worker( pid, size = null ) { 605 | const dataset = get_work_dataset( size ); 606 | if ( !dataset || dataset.length === 0 ) { 607 | return false; 608 | } 609 | 610 | const worker = getWorker( pid ); 611 | if ( !worker ) { 612 | return false; 613 | } 614 | 615 | if ( -1 != haltedWorkers.indexOf( pid ) ) { 616 | return false; 617 | } 618 | 619 | worker.send( { 620 | pid: worker.pid, 621 | request: 'queue-add', 622 | payload: dataset, 623 | } ); 624 | } 625 | 626 | function host_check_request( server ) { 627 | var check_server = {}; 628 | check_server.blog_id = server.blog_id; 629 | check_server.monitor_url = server.monitor_url; 630 | check_server.status_id = server.site_status; 631 | check_server.lastCheck = server.lastCheck; 632 | check_server.last_status_change = server.last_status_change; 633 | check_server.checks = server.checks; 634 | check_server.offline_confirms = 0; 635 | check_server.requests_sent = false; 636 | check_server.requests_outstanding = 0; 637 | check_server.last_activity = new Date().valueOf(); 638 | 639 | queuedRetries.push( check_server ); 640 | } 641 | 642 | function updateStats() { 643 | try { 644 | var sps = sitesCount / global.config.get( 'STATS_UPDATE_INTERVAL_MS' ) * 1000; 645 | 646 | if ( 0 === sps % 1 ) { 647 | sps = sps.toFixed( 0 ); 648 | } else { 649 | sps = sps.toFixed( 1 ); 650 | } 651 | 652 | if ( true === global.config.get( 'DEBUG' ) ) { 653 | var nextLoop = ( getRoundDuration() * SECONDS ) - ( new Date().valueOf() - startTime ); 654 | logger.debug( 'sps = ' + sps + ' - ' + 655 | ( arrWorkers.length - freeWorkers.length ) + ' working, ' + 656 | freeWorkers.length + ' waiting, ' + 657 | haltedWorkers.length + ' halting : ' + 658 | 'next round in ' + ( nextLoop / 1000 ) + 's' ); 659 | if ( nextLoop < -20000 ) { 660 | logger.error( 'restarting the getMoreSites loop' ); 661 | resetVariables(); 662 | setTimeout( getMoreSites, 100 ); 663 | } 664 | } 665 | 666 | var localGCountSuccess = gCountSuccess; 667 | var localGCountError = gCountError; 668 | var localGCountOffline = gCountOffline; 669 | var localSitesCount = sitesCount; 670 | 671 | gCountSuccess = 0; 672 | gCountError = 0; 673 | gCountOffline = 0; 674 | sitesCount = 0; // need this local otherwise the async call below writes 0, due to the 'finally' call setting sitesCount to 0 675 | 676 | var spsFile = fs.createWriteStream( 'stats/sitespersec', { flags : "w" } ); 677 | spsFile.once( 'open', function( fd ) { 678 | spsFile.write( 'sites per second: ' + sps + '\n' ); 679 | spsFile.end(); 680 | }); 681 | var queueFile = fs.createWriteStream( 'stats/sitesqueue', { flags : "w" } ); 682 | queueFile.once( 'open', function( fd ) { 683 | queueFile.write( 'sites in queue: ' + arrObjects.length + '\n' ); 684 | queueFile.end(); 685 | }); 686 | var totalFile = fs.createWriteStream( 'stats/totals', { flags : "w" } ); 687 | totalFile.once( 'open', function( fd ) { 688 | totalFile.write( 'working : ' + ( arrWorkers.length - freeWorkers.length ) + '\n' ); 689 | totalFile.write( 'waiting : ' + freeWorkers.length + '\n' ); 690 | totalFile.write( 'halting : ' + haltedWorkers.length + '\n' ); 691 | totalFile.write( 'error : ' + localGCountError + '\n' ); 692 | totalFile.write( 'offline : ' + localGCountOffline + '\n' ); 693 | totalFile.write( 'success : ' + localGCountSuccess + '\n' ); 694 | totalFile.write( 'total : ' + localSitesCount + '\n' ); 695 | totalFile.end(); 696 | }); 697 | 698 | /** 699 | * Push some of the stats to StatsD 700 | */ 701 | statsdClient.increment( 'stats.sites.sps.count', sps ); 702 | statsdClient.increment( 'stats.sites.error.count', localGCountError ); 703 | statsdClient.increment( 'stats.sites.offline.count', localGCountOffline ); 704 | statsdClient.increment( 'stats.sites.success.count', localGCountSuccess ); 705 | statsdClient.increment( 'stats.sites.total.count', localSitesCount ); 706 | statsdClient.increment( 'stats.sites.queue.count', arrObjects.length ); 707 | 708 | statsdClient.increment( 'stats.workers.free.count', freeWorkers.length ); 709 | statsdClient.increment( 'stats.workers.halting.count', haltedWorkers.length ); 710 | statsdClient.increment( 'stats.workers.working.count', ( arrWorkers.length - freeWorkers.length ) ); 711 | 712 | for ( let site_status in checkStats ) { 713 | for ( let http_code in checkStats[site_status]['http_code'] ) { 714 | statsdClient.increment( `worker.check.${site_status}.code.${http_code}.count`, checkStats[site_status]['http_code'][http_code] ); 715 | } 716 | for ( let error_code in checkStats[site_status]['error_code'] ) { 717 | statsdClient.increment( `worker.check.${site_status}.error_code.${error_code}.count`, checkStats[site_status]['error_code'][error_code] ); 718 | } 719 | 720 | let rtt_avg = Math.round( checkStats[site_status]['rtt']['sum'] / checkStats[site_status]['rtt']['count'] ); 721 | statsdClient.timing( `worker.check.${site_status}.rtt.avg`, rtt_avg ); 722 | statsdClient.timing( `worker.check.${site_status}.rtt.max`, checkStats[site_status]['rtt']['max'] ); 723 | statsdClient.timing( `worker.check.${site_status}.rtt.min`, checkStats[site_status]['rtt']['min'] ); 724 | } 725 | 726 | checkStats = {}; 727 | 728 | if ( global.config.get( 'STATSD_SEND_MEM_USAGE' ) ) { 729 | statsdClient.timing( 'stats.parent.memory', process.memoryUsage().rss ); 730 | 731 | for ( var pid in workerStats ) { 732 | statsdClient.timing( 'stats.workers.memory', workerStats[pid].memoryUsage ); 733 | } 734 | } 735 | } 736 | catch ( Exception ) { 737 | logger.error( 'Error updating stats files: ' + Exception.toString() ); 738 | } 739 | finally { 740 | sitesCount = 0; 741 | setTimeout( updateStats, ( global.config.get( 'STATS_UPDATE_INTERVAL_MS' ) ) ); 742 | } 743 | } 744 | 745 | function processQueuedRetries() { 746 | if ( true === global.config.get( 'DEBUG' ) ) 747 | logger.debug( 'starting checks for ' + queuedRetries.length + ' REMOTE' ); 748 | 749 | var sendRetries = []; 750 | var peerCount = global.config.get( 'VERIFIERS' ).length; 751 | for( var loop = queuedRetries.length - 1; loop >= 0; loop-- ) { 752 | if ( false === queuedRetries[loop].requests_sent ) { 753 | sendRetries.push( queuedRetries[loop] ); 754 | queuedRetries[loop].requests_sent = true; 755 | queuedRetries[loop].requests_outstanding = peerCount; 756 | } else if ( ( queuedRetries[loop].requests_outstanding <= 0 ) || 757 | ( new Date().valueOf() > queuedRetries[loop].last_activity + ( global.config.get( 'TIMEOUT_FOR_REQUESTS_SEC' ) * SECONDS ) ) ) { 758 | if ( true === global.config.get( 'DEBUG' ) ) { 759 | if ( 0 < queuedRetries[loop].requests_outstanding ) 760 | logger.trace( 'TIMED out : ' + queuedRetries[loop].monitor_url + 761 | ', "outstanding": ' + queuedRetries[loop].requests_outstanding + 762 | ', "confirms": ' + queuedRetries[loop].offline_confirms ); 763 | else 764 | logger.trace( 'NORMAL out : ' + queuedRetries[loop].monitor_url + 765 | ', "outstanding": ' + queuedRetries[loop].requests_outstanding + 766 | ', "confirms": ' + queuedRetries[loop].offline_confirms ); 767 | } 768 | queuedRetries.splice( loop, 1 ); 769 | } 770 | } 771 | 772 | var peerArray = global.config.get( 'VERIFIERS' ); 773 | var batchSize = global.config.get( 'VERIFLIER_BATCH_SIZE' ) || 200; 774 | for( var loop = sendRetries.length - 1; loop >= 0; loop -= batchSize ) { 775 | var sending = Math.min( batchSize, sendRetries.length ); 776 | var batchData = sendRetries.splice( sendRetries.length - sending, sending ); 777 | for ( var count in peerArray ) { 778 | comms.get_remote_status_array( 779 | peerArray[ count ], 780 | batchData, 781 | function( res ) { 782 | if ( 1 !== res.status ) { 783 | logger.debug( res.veriflier + ': send ' + res.status ); 784 | } 785 | }); 786 | } 787 | } 788 | 789 | var addedWork = false; 790 | if ( true === global.config.get( 'DEBUG' ) ) 791 | logger.debug( 'starting checks for ' + localRetries.length + ' LOCAL' ); 792 | for( var loop = localRetries.length - 1; loop >= 0; loop-- ) { 793 | if ( new Date().valueOf() < ( localRetries[loop].lastCheck + ( global.config.get( 'TIME_BETWEEN_CHECKS_SEC' ) * SECONDS ) ) ) 794 | continue; 795 | if ( 0 !== freeWorkers.length ) { 796 | var i = 0; 797 | while ( i < freeWorkers.length && null === getWorker( freeWorkers[i] ) ) { 798 | i++; 799 | } 800 | if ( i < freeWorkers.length ) { 801 | var w = getWorker( freeWorkers[i] ); 802 | w.send( { 803 | pid : freeWorkers[i], 804 | request : 'queue-add', 805 | payload : [ localRetries.splice( loop, 1 )[0] ] 806 | } ); 807 | freeWorkers.splice( i, 1 ); 808 | } else { 809 | arrObjects.push( localRetries.splice( loop, 1 )[0] ); 810 | addedWork = true; 811 | } 812 | } else { 813 | arrObjects.push( localRetries.splice( loop, 1 )[0] ); 814 | addedWork = true; 815 | } 816 | } 817 | if ( addedWork ) 818 | freeWorkersToWork(); 819 | } 820 | 821 | 822 | 823 | /** 824 | * Ensures that we're always at NUM_WORKERS count. 825 | * @param first_usage If this call is the initial spawn of workers when Jetmon has started. 826 | */ 827 | function ensure_worker_count( first_usage = false ) { 828 | const max_worker_count = global.config.get( 'NUM_WORKERS' ); 829 | const current_worker_count = arrWorkers.length; 830 | 831 | if ( current_worker_count < max_worker_count ) { 832 | const new_worker_count = max_worker_count - current_worker_count; 833 | 834 | logger.debug( `Missing workers, spawning: ${new_worker_count} new workers` ); 835 | 836 | /** 837 | * Only log the missing worker count if it's not the first spawn 838 | * after Jetmon has started. 839 | * This is done to avoid polluting the data with the occasional NUM_WORKERS peaks. 840 | */ 841 | if ( ! first_usage ) { 842 | statsdClient.increment( 'worker.spawn.missing.count', new_worker_count ); 843 | } 844 | 845 | for ( let loop = 0; loop < new_worker_count; loop++ ) { 846 | spawnWorker(); 847 | } 848 | } 849 | } 850 | 851 | /** 852 | * Spawn the workers and start keeping track of the number of workers. 853 | */ 854 | ensure_worker_count( true ); 855 | setInterval( ensure_worker_count, SECONDS ); 856 | 857 | // Start the SSL cluster 858 | cluster.setupMaster( { 859 | exec : './lib/server', 860 | silent : false, 861 | }); 862 | 863 | cluster.on( 'online', function( worker ) { 864 | logger.debug( 'SSL worker (pid:' + worker.process.pid + ') is online.' ); 865 | }); 866 | 867 | cluster.on( 'disconnect', function( worker ) { 868 | logger.debug( 'SSL worker (pid:' + worker.process.pid + ') has disconnected.' ); 869 | }); 870 | 871 | cluster.on( 'exit', function( worker, code, signal ) { 872 | if ( true == worker.exitedAfterDisconnect ) { 873 | logger.debug( 'SSL worker (pid:' + worker.process.pid + ') is shutting down.' ); 874 | } else { 875 | logger.error( 'SSL worker (pid:' + worker.process.pid + ') died (' + worker.process.exitCode + ').' ); 876 | } 877 | }); 878 | 879 | for ( var i = 0; i < NUM_SSL_SERVERS; i++ ) { 880 | var ssl_server = cluster.fork(); 881 | ssl_server.on( 'message', sslWorkerCallBack ); 882 | } 883 | 884 | // set a repeating 'tick' to perform clean-up and retries allocation 885 | setInterval( processQueuedRetries, SECONDS * 5 ); 886 | 887 | // start the 'recursive' stats logging 888 | updateStats(); 889 | 890 | --------------------------------------------------------------------------------