├── rel-eng ├── packages │ ├── memkeys │ └── .readme └── tito.props ├── m4 └── .gitignore ├── .gitmodules ├── src ├── logging │ ├── logging.h │ ├── test_record.cpp │ ├── level.h │ ├── record.h │ ├── test_level.cpp │ ├── logger.h │ ├── level.cpp │ ├── record.cpp │ └── logger.cpp ├── net │ ├── net.h │ ├── pcap_live.h │ ├── device.h │ ├── packet.h │ ├── pcap.h │ ├── pcap_live.cpp │ ├── capture_engine.h │ ├── device.cpp │ ├── memcache_command.h │ ├── packet.cpp │ ├── pcap.cpp │ ├── memcache_command.cpp │ └── capture_engine.cpp ├── util │ ├── util.h │ ├── exceptions.h │ ├── options.h │ ├── state.cpp │ ├── state.h │ ├── util_time.h │ ├── stat.cpp │ ├── stat.h │ ├── backoff.h │ ├── mqueue.h │ ├── stats.h │ ├── backoff.cpp │ └── stats.cpp ├── report │ ├── csv.h │ ├── report.h │ ├── report_type.h │ ├── report.cpp │ ├── curses.h │ ├── report_type.cpp │ ├── csv.cpp │ └── curses.cpp ├── cli.h ├── common.h ├── memkeys.h ├── main.cpp ├── config.h ├── Makefile.am ├── cli.cpp ├── config.cpp └── memkeys.cpp ├── .travis.yml ├── .gitignore ├── Makefile.am ├── NOTICE ├── memkeys.spec ├── README.md ├── configure.ac └── LICENSE /rel-eng/packages/memkeys: -------------------------------------------------------------------------------- 1 | 0.2-1 / 2 | -------------------------------------------------------------------------------- /m4/.gitignore: -------------------------------------------------------------------------------- 1 | * 2 | 3 | !.gitignore 4 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "gtest"] 2 | path = gtest 3 | url = https://github.com/bmatheny/gtest.git 4 | -------------------------------------------------------------------------------- /rel-eng/packages/.readme: -------------------------------------------------------------------------------- 1 | the rel-eng/packages directory contains metadata files 2 | named after their packages. Each file has the latest tagged 3 | version and the project's relative directory. 4 | -------------------------------------------------------------------------------- /rel-eng/tito.props: -------------------------------------------------------------------------------- 1 | [globalconfig] 2 | default_builder = tito.builder.Builder 3 | default_tagger = tito.tagger.VersionTagger 4 | changelog_do_not_remove_cherrypick = 0 5 | changelog_format = %s (%ae) 6 | -------------------------------------------------------------------------------- /src/logging/logging.h: -------------------------------------------------------------------------------- 1 | #ifndef _LOGGING_LOGGING_H 2 | #define _LOGGING_LOGGING_H 3 | 4 | #include "logging/level.h" 5 | #include "logging/record.h" 6 | #include "logging/logger.h" 7 | 8 | #endif 9 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: cpp 2 | compiler: gcc 3 | install: 4 | - sudo apt-get update -qq 5 | - sudo apt-get install -qq libpcap-dev libpcre3-dev lib32ncurses5-dev 6 | before_script: 7 | - ./build-eng/autogen.sh 8 | - ./configure 9 | - cd gtest && make && cd .. 10 | - make 11 | script: 12 | - make check 13 | -------------------------------------------------------------------------------- /src/net/net.h: -------------------------------------------------------------------------------- 1 | #ifndef _NET_NET_H 2 | #define _NET_NET_H 3 | 4 | extern "C" { 5 | #include 6 | } 7 | 8 | #include "net/device.h" 9 | #include "net/memcache_command.h" 10 | #include "net/capture_engine.h" 11 | #include "net/packet.h" 12 | #include "net/pcap.h" 13 | #include "net/pcap_live.h" 14 | 15 | #endif 16 | -------------------------------------------------------------------------------- /src/util/util.h: -------------------------------------------------------------------------------- 1 | #ifndef _UTIL_UTIL_H 2 | #define _UTIL_UTIL_H 3 | 4 | #include "util/backoff.h" 5 | #include "util/exceptions.h" 6 | #include "util/options.h" 7 | #include "util/mqueue.h" 8 | #include "util/util_time.h" 9 | #include "util/stat.h" 10 | #include "util/stats.h" 11 | #include "util/state.h" 12 | 13 | #endif 14 | -------------------------------------------------------------------------------- /src/report/csv.h: -------------------------------------------------------------------------------- 1 | #ifndef _REPORT_CSV_H 2 | #define _REPORT_CSV_H 3 | 4 | #include "config.h" 5 | #include "net/pcap.h" 6 | #include "report/report.h" 7 | #include "util/stats.h" 8 | 9 | namespace mckeys { 10 | 11 | class CsvReport : public Report 12 | { 13 | public: 14 | CsvReport(const Config* cfg, const Pcap* session, Stats* stats); 15 | virtual void render(); 16 | 17 | protected: 18 | void renderStats(std::deque q); 19 | 20 | private: 21 | Stats* stats; 22 | }; 23 | 24 | } // end namespace 25 | 26 | #endif 27 | -------------------------------------------------------------------------------- /src/util/exceptions.h: -------------------------------------------------------------------------------- 1 | #ifndef _UTIL_EXCEPTIONS_H 2 | #define _UTIL_EXCEPTIONS_H 3 | 4 | #include 5 | #include 6 | 7 | namespace mckeys { 8 | 9 | class MemkeysException : public std::runtime_error 10 | { 11 | public: 12 | MemkeysException(const std::string &msg) : std::runtime_error(msg) {} 13 | }; 14 | 15 | class MemkeysConfigurationError : public MemkeysException 16 | { 17 | public: 18 | MemkeysConfigurationError(const std::string &msg) : MemkeysException(msg) {} 19 | }; 20 | 21 | } // end namespace 22 | 23 | #endif 24 | -------------------------------------------------------------------------------- /src/report/report.h: -------------------------------------------------------------------------------- 1 | #ifndef _REPORT_REPORT_H 2 | #define _REPORT_REPORT_H 3 | 4 | #include 5 | #include "config.h" 6 | #include "logging/logger.h" 7 | #include "util/state.h" 8 | 9 | namespace mckeys { 10 | 11 | class Report { 12 | public: 13 | virtual void render() = 0; 14 | virtual bool isShutdown() const; 15 | virtual void shutdown(); 16 | virtual ~Report(); 17 | 18 | protected: 19 | Report(const Config *cfg, const LoggerPtr logger); 20 | 21 | const Config * config; 22 | const LoggerPtr logger; 23 | State state; 24 | std::thread report_thread; 25 | }; 26 | 27 | } // end namespace 28 | 29 | #endif 30 | -------------------------------------------------------------------------------- /src/cli.h: -------------------------------------------------------------------------------- 1 | #ifndef SRC_CLI_H_ 2 | #define SRC_CLI_H_ 3 | 4 | /** 5 | * Copyright 2013 Blake Matheny 6 | */ 7 | #include 8 | 9 | namespace mckeys { 10 | 11 | class Config; 12 | 13 | class Cli { 14 | public: 15 | static void parse(int argc, char ** argv, Config * cfg); 16 | static std::string help(const char * progname); 17 | 18 | private: 19 | static std::string mkHelpLead(const struct option opt, 20 | const std::string &key); 21 | static std::string mkHelpDoc(const struct option opt, 22 | const std::string &desc, 23 | const std::string &key); 24 | }; 25 | 26 | } // namespace mckeys 27 | #endif // SRC_CLI_H_ 28 | -------------------------------------------------------------------------------- /src/net/pcap_live.h: -------------------------------------------------------------------------------- 1 | #ifndef _NET_PCAP_LIVE_H 2 | #define _NET_PCAP_LIVE_H 3 | 4 | #include "net/pcap.h" 5 | #include "net/device.h" 6 | #include "common.h" 7 | 8 | namespace mckeys { 9 | 10 | class PcapLive : public Pcap 11 | { 12 | public: 13 | PcapLive(const Config * cfg); 14 | virtual ~PcapLive(); 15 | 16 | Device getDevice() const 17 | { return device; } 18 | 19 | const char * getInterfaceC() { return getDevice().getDeviceName().c_str(); } 20 | std::string getInterface() const 21 | { return getDevice().getDeviceName(); } 22 | 23 | virtual void open(); 24 | virtual bpf_u_int32 getSubnetMask() const; 25 | virtual bpf_u_int32 getIpAddress() const; 26 | 27 | private: 28 | const Config * config; 29 | const Device device; 30 | 31 | }; 32 | 33 | } 34 | 35 | #endif 36 | -------------------------------------------------------------------------------- /src/report/report_type.h: -------------------------------------------------------------------------------- 1 | #ifndef _REPORT_REPORT_TYPE_H 2 | #define _REPORT_REPORT_TYPE_H 3 | 4 | #include 5 | 6 | namespace mckeys { 7 | 8 | // forward declarations because shit we have some recursive dependencies 9 | class Config; 10 | class Pcap; 11 | class Stats; 12 | class Report; 13 | 14 | class ReportType 15 | { 16 | public: 17 | static ReportType NCURSES; 18 | static ReportType CSV; 19 | 20 | static ReportType fromString(const std::string &name); 21 | 22 | Report* makeReport(const Config* cfg, const Pcap* sess, Stats* stats); 23 | std::string getName() const; 24 | 25 | bool operator==(const ReportType &other) const; 26 | 27 | protected: 28 | std::string name; 29 | ReportType(const std::string &name); 30 | 31 | }; 32 | 33 | } // end namespace 34 | 35 | #endif 36 | -------------------------------------------------------------------------------- /src/util/options.h: -------------------------------------------------------------------------------- 1 | #ifndef _UTIL_OPTIONS_H 2 | #define _UTIL_OPTIONS_H 3 | 4 | #include 5 | 6 | #ifndef no_argument 7 | #define no_argument 0 8 | #endif 9 | 10 | #ifndef required_argument 11 | #define required_argument 1 12 | #endif 13 | 14 | #ifndef optional_argument 15 | #define optional_argument 2 16 | #endif 17 | 18 | #define USIZE_BITS(type) ((1L << (8*sizeof(type))) - 1) 19 | #define REQUIRE_UINT(name, value, type) do { \ 20 | uint64_t usize = USIZE_BITS(type); \ 21 | if (value <= 0) { \ 22 | throw std::range_error(name " must be >= 0"); \ 23 | } else if ((uint64_t)value > usize) { \ 24 | std::ostringstream oss; \ 25 | oss << name << " must be <= " << usize; \ 26 | throw std::range_error(oss.str()); \ 27 | } \ 28 | } while(0); 29 | 30 | #endif 31 | -------------------------------------------------------------------------------- /src/logging/test_record.cpp: -------------------------------------------------------------------------------- 1 | #include "logging/record.h" 2 | #include "logging/level.h" 3 | #include 4 | #include 5 | 6 | namespace mckeys { 7 | 8 | TEST(Record, DefaultConstructor) { 9 | Record rec; 10 | EXPECT_TRUE(rec.getFileName().empty()); 11 | EXPECT_EQ(Level::OFF, rec.getLevel()); 12 | } 13 | 14 | TEST(Record, FileName) { 15 | Record rec; 16 | rec.setFileName("fizz.txt"); 17 | EXPECT_EQ("fizz.txt", rec.getFileName()); 18 | } 19 | 20 | TEST(Record, Level) { 21 | Record rec; 22 | rec.setLevel(Level::DEBUG); 23 | EXPECT_EQ(Level::DEBUG, rec.getLevel()); 24 | rec.setLevel(Level::INFO); 25 | EXPECT_EQ(Level::INFO, rec.getLevel()); 26 | } 27 | 28 | TEST(Record, Timestamp) { 29 | Record rec1; 30 | struct timeval tv = {1369615982, 123000}; 31 | auto actual = rec1.getTimestamp(tv); 32 | std::string expected("20130527-00:53:02.123"); 33 | EXPECT_EQ(expected, actual); 34 | } 35 | 36 | } // end namespace 37 | -------------------------------------------------------------------------------- /src/report/report.cpp: -------------------------------------------------------------------------------- 1 | #include "report/report.h" 2 | #include "config.h" 3 | #include "logging/logger.h" 4 | 5 | namespace mckeys { 6 | 7 | using namespace std; 8 | 9 | bool Report::isShutdown() const { 10 | return state.isTerminated(); 11 | } 12 | void Report::shutdown() { 13 | if (state.checkAndSet(state_RUNNING, state_STOPPING)) { 14 | logger->info(CONTEXT, "Shutting down"); 15 | } else { 16 | logger->warning(CONTEXT, "Could not shutdown, state is %s", 17 | state.getName().c_str()); 18 | } 19 | } 20 | 21 | Report::~Report() 22 | { 23 | // FIXME this should have a timer 24 | report_thread.join(); 25 | logger->debug(CONTEXT, "Report thread shut down"); 26 | if (logger != NULL) { 27 | delete logger; 28 | } 29 | } 30 | 31 | // protected 32 | Report::Report(const Config* cfg, const LoggerPtr logger) 33 | : config(cfg), 34 | logger(logger), 35 | state(state_NEW) 36 | {} 37 | 38 | } // end namespace top 39 | -------------------------------------------------------------------------------- /src/net/device.h: -------------------------------------------------------------------------------- 1 | #ifndef _NET_DEVICE_H 2 | #define _NET_DEVICE_H 3 | 4 | #include 5 | 6 | namespace mckeys { 7 | 8 | class Device; 9 | 10 | class Device { 11 | public: 12 | static Device getDevice(const std::string &name); 13 | 14 | std::string getDeviceName() const { return deviceName_; } 15 | bpf_u_int32 getNetwork() const { return network_; } 16 | bpf_u_int32 getSubnetMask() const { return subnetMask_; } 17 | bpf_u_int32 getIpAddress() const { return ipAddress_; } 18 | bool isLoopback() const { return loopback_; } 19 | 20 | protected: 21 | Device(const std::string &name, const bool loopback, 22 | const bpf_u_int32 network, const bpf_u_int32 mask, 23 | const bpf_u_int32 address); 24 | 25 | private: 26 | const std::string deviceName_; 27 | const bool loopback_; 28 | const bpf_u_int32 network_; 29 | const bpf_u_int32 subnetMask_; 30 | const bpf_u_int32 ipAddress_; 31 | }; 32 | 33 | } // end of namespace 34 | 35 | #endif 36 | -------------------------------------------------------------------------------- /src/common.h: -------------------------------------------------------------------------------- 1 | #ifndef _COMMON_H 2 | #define _COMMON_H 3 | 4 | #ifdef HAVE_CONFIG_H 5 | #include "mconfig.h" 6 | #endif 7 | 8 | #ifndef PACKAGE_NAME 9 | #define PACKAGE_NAME "memkeys" 10 | #endif 11 | #ifndef PACKAGE_STRING 12 | #define PACKAGE_STRING "memkeys (unknown version)" 13 | #endif 14 | #ifndef PACKAGE_BUGREPORT 15 | #define PACKAGE_BUGREPORT "bmatheny@mobocracy.net" 16 | #endif 17 | 18 | // Define _DEBUG if appropriate 19 | #ifdef DEBUGGING 20 | #if DEBUGGING 21 | #define _DEBUG 1 22 | #endif 23 | #endif 24 | 25 | // Define _DEVEL if appropriate 26 | #ifdef DEVELOPMENT 27 | #if DEVELOPMENT 28 | #define _DEVEL 1 29 | #endif 30 | #endif 31 | 32 | #include 33 | #include 34 | #include 35 | #include 36 | #include 37 | 38 | // Need for typecasting in to_string for older versions of g++ 39 | typedef long long unsigned int llui_t; 40 | typedef long long int llsi_t; 41 | 42 | #include "util/util.h" 43 | #include "logging/logging.h" 44 | #include "config.h" 45 | 46 | #endif 47 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | # Compiled Object files 2 | *.slo 3 | *.lo 4 | *.o 5 | 6 | # Compiled Dynamic libraries 7 | *.so 8 | *.dylib 9 | 10 | # Compiled Static libraries 11 | *.lai 12 | *.la 13 | *.a 14 | # Compiled Object files 15 | *.lo 16 | *.o 17 | 18 | # Compiled Dynamic libraries 19 | *.so 20 | 21 | # Compiled Static libraries 22 | *.la 23 | *.a 24 | 25 | # Compiled misc 26 | *.dep 27 | *.gcda 28 | *.gcno 29 | *.gcov 30 | 31 | # Packages 32 | *.tar.gz 33 | *.tar.bz2 34 | 35 | # Logs 36 | *.log 37 | 38 | # Temporary 39 | *.swp 40 | *.~ 41 | *.project 42 | *.cproject 43 | 44 | # Core and executable 45 | core* 46 | nutcracker 47 | 48 | # extracted yaml 49 | !/contrib/yaml-0.1.4.tar.gz 50 | 51 | # Autotools 52 | .deps 53 | .libs 54 | 55 | aclocal.m4 56 | # Directory for autoconf generated files 57 | build-aux/* 58 | autom4te.cache 59 | autoscan.log 60 | libtool 61 | 62 | /config.log 63 | /config.status 64 | /configure.scan 65 | /configure 66 | 67 | Makefile 68 | Makefile.in 69 | 70 | src/memkeys 71 | 72 | *.dirstamp 73 | *.prof 74 | src/testall 75 | -------------------------------------------------------------------------------- /src/memkeys.h: -------------------------------------------------------------------------------- 1 | #ifndef _MEMKEYS_MEMKEYS_H 2 | #define _MEMKEYS_MEMKEYS_H 3 | 4 | #include 5 | 6 | #include "common.h" 7 | #include "net/net.h" 8 | #include "report/report.h" 9 | 10 | namespace mckeys { 11 | 12 | class Memkeys { 13 | public: 14 | static Memkeys * getInstance(const Config * config); 15 | static Memkeys * getInstance(int argc, char ** argv); 16 | virtual ~Memkeys(); 17 | 18 | virtual void run(); 19 | 20 | // called by signal handler 21 | void tryShutdown(); 22 | // called by signal handler after timeout 23 | void forceShutdown(); 24 | 25 | bool isRunning() const 26 | { return state.isRunning(); } 27 | bool isShutdown() const 28 | { return state.isTerminated(); } 29 | std::string getStateName() const 30 | { return state.getName(); } 31 | 32 | protected: 33 | Memkeys(const Config * config); 34 | 35 | const Config * config; 36 | const LoggerPtr logger; 37 | Pcap * session; 38 | CaptureEngine * engine; 39 | State state; 40 | }; 41 | 42 | } // end namespace mctop 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /src/logging/level.h: -------------------------------------------------------------------------------- 1 | #ifndef _LOGGING_LEVEL_H 2 | #define _LOGGING_LEVEL_H 3 | 4 | #include 5 | #include 6 | 7 | namespace mckeys { 8 | 9 | class Level 10 | { 11 | public: 12 | static Level TRACE; 13 | static Level DEBUG; 14 | static Level INFO; 15 | static Level WARNING; 16 | static Level ERROR; 17 | static Level FATAL; 18 | static Level OFF; 19 | static Level fromName(const std::string &name); 20 | static Level fromValue(const uint32_t value); 21 | 22 | std::string getName() const; 23 | uint32_t getValue() const; 24 | 25 | protected: 26 | std::string name; 27 | uint32_t value; 28 | 29 | Level(const std::string &name, const uint32_t value); 30 | }; 31 | 32 | bool operator==(const Level& lhs, const Level& rhs); 33 | bool operator<(const Level& lhs, const Level& rhs); 34 | bool operator<=(const Level& lhs, const Level& rhs); 35 | bool operator>=(const Level& lhs, const Level& rhs); 36 | bool operator>(const Level& lhs, const Level& rhs); 37 | bool operator!=(const Level& lhs, const Level& rhs); 38 | 39 | typedef std::map ValueMap; 40 | typedef std::map NameMap; 41 | 42 | } // end namespace 43 | 44 | #endif 45 | -------------------------------------------------------------------------------- /src/net/packet.h: -------------------------------------------------------------------------------- 1 | #ifndef _NET_PACKET_H 2 | #define _NET_PACKET_H 3 | 4 | #include "net/pcap.h" 5 | 6 | extern "C" { 7 | #include 8 | #include 9 | } 10 | 11 | namespace mckeys { 12 | 13 | class Packet { 14 | public: 15 | 16 | typedef u_char Data; 17 | typedef pcap_pkthdr Header; 18 | 19 | Packet(); 20 | Packet(const Header& header, const Data* data); 21 | Packet(const Packet& packet); // copy constructor 22 | Packet& operator=(const Packet& packet); // assignment operator 23 | virtual ~Packet(); 24 | 25 | const Data* getData() const { return data; } 26 | const Header& getHeader() const { return header; } 27 | 28 | uint32_t capLength() const { return getHeader().caplen; } 29 | int32_t id() const; 30 | uint32_t length() const { return getHeader().len; } 31 | uint64_t timestamp() const { return _timestamp; } 32 | 33 | protected: 34 | Data* copyData(Data* data, const bpf_u_int32 len); 35 | 36 | private: 37 | inline void* __void_data(Data* d); 38 | void copy(const Packet& other); 39 | void destroy(); 40 | 41 | Header header; 42 | Data* data; 43 | uint64_t _timestamp; 44 | 45 | }; 46 | 47 | } // end namespace 48 | 49 | #endif 50 | -------------------------------------------------------------------------------- /Makefile.am: -------------------------------------------------------------------------------- 1 | ACLOCAL_AMFLAGS = ${ACLOCAL_FLAGS} -I m4 2 | 3 | # Keep objects in separate directories when possible 4 | AUTOMAKE_OPTIONS = subdir-objects 5 | 6 | SUBDIRS = . src 7 | DIST_SUBDIRS = $(subdirs) src 8 | 9 | dist_noinst_SCRIPTS = build-aux/* 10 | 11 | EXTRA_DIST = README.md NOTICE LICENSE 12 | 13 | check-local: 14 | @echo "Making lib/libgtest.a lib/libgtest_main.a in gtest" 15 | @cd gtest && $(MAKE) $(AM_MAKEFLAGS) lib/libgtest.la lib/libgtest_main.la 16 | 17 | clean-local: 18 | @if test -e gtest/Makefile; then \ 19 | echo "Making clean in gtest"; \ 20 | cd gtest && $(MAKE) $(AM_MAKEFLAGS) clean; \ 21 | fi 22 | 23 | test: all 24 | 25 | # Deletes all the files generated by autogen.sh. 26 | MAINTAINERCLEANFILES = \ 27 | aclocal.m4 \ 28 | config.guess \ 29 | config.sub \ 30 | configure \ 31 | depcomp \ 32 | install-sh \ 33 | ltmain.sh \ 34 | Makefile.in \ 35 | missing \ 36 | mkinstalldirs \ 37 | config.h.in \ 38 | stamp.h.in \ 39 | m4/ltsugar.m4 \ 40 | m4/libtool.m4 \ 41 | m4/ltversion.m4 \ 42 | m4/lt~obsolete.m4 \ 43 | m4/ltoptions.m4 44 | -------------------------------------------------------------------------------- /src/util/state.cpp: -------------------------------------------------------------------------------- 1 | #include "util/state.h" 2 | #include 3 | 4 | namespace mckeys { 5 | 6 | using namespace std; 7 | 8 | State::State() 9 | : _state(state_NEW), 10 | _mutex() 11 | {} 12 | 13 | State::State(state_t state) 14 | : _state(state), 15 | _mutex() 16 | {} 17 | State::~State() 18 | {} 19 | 20 | bool State::checkAndSet(const state_t expected, const state_t next) 21 | { 22 | _mutex.lock(); 23 | state_t state = _state; 24 | if (state == expected) { 25 | _state = next; 26 | } 27 | _mutex.unlock(); 28 | return (state == expected); 29 | } 30 | 31 | state_t State::getState() const 32 | { 33 | _mutex.lock(); 34 | state_t state = _state; 35 | _mutex.unlock(); 36 | return state; 37 | } 38 | void State::setState(const state_t state) 39 | { 40 | _mutex.lock(); 41 | _state = state; 42 | _mutex.unlock(); 43 | } 44 | string State::getName() const 45 | { 46 | switch (_state) { 47 | case state_NEW: 48 | return "NEW"; 49 | case state_STARTING: 50 | return "STARTING"; 51 | case state_RUNNING: 52 | return "RUNNING"; 53 | case state_STOPPING: 54 | return "STOPPING"; 55 | case state_TERMINATED: 56 | return "TERMINATED"; 57 | default: 58 | return "UNKNOWN"; 59 | } 60 | } 61 | 62 | } // end namespace 63 | -------------------------------------------------------------------------------- /src/util/state.h: -------------------------------------------------------------------------------- 1 | #ifndef _UTIL_STATE_H 2 | #define _UTIL_STATE_H 3 | 4 | #include 5 | #include 6 | 7 | namespace mckeys { 8 | 9 | enum state_t { 10 | state_NEW, state_STARTING, state_RUNNING, state_STOPPING, state_TERMINATED 11 | }; 12 | 13 | class State; 14 | 15 | // FIXME - profiling shows that we may want to use a std::atomic instead of a 16 | // mutex 17 | 18 | // thread safe state representation 19 | class State 20 | { 21 | public: 22 | State(); 23 | State(state_t state); 24 | virtual ~State(); 25 | 26 | bool checkAndSet(const state_t expected, const state_t next); 27 | state_t getState() const; 28 | std::string getName() const; 29 | void setState(const state_t state); 30 | bool isNew() const 31 | { return getState() == state_NEW; } 32 | bool isStarting() const 33 | { return getState() == state_STARTING; } 34 | bool isRunning() const 35 | { return getState() == state_RUNNING; } 36 | bool isStopping() const 37 | { return getState() == state_STOPPING; } 38 | bool isTerminated() const 39 | { return getState() == state_TERMINATED; } 40 | 41 | protected: 42 | state_t _state; 43 | mutable std::mutex _mutex; 44 | 45 | private: 46 | State(const State &state); // no copy constructor 47 | }; 48 | 49 | } // end namespace 50 | 51 | #endif 52 | -------------------------------------------------------------------------------- /src/util/util_time.h: -------------------------------------------------------------------------------- 1 | #ifndef _UTIL_UTIL_TIME_H 2 | #define _UTIL_UTIL_TIME_H 3 | 4 | #include 5 | #include 6 | 7 | extern "C" { 8 | #include 9 | #include 10 | #include 11 | } 12 | 13 | namespace mckeys { 14 | 15 | class UtilTime { 16 | public: 17 | static inline uint64_t currentTimeMillis() { 18 | struct timeval t; 19 | gettimeofday(&t, NULL); 20 | return (1000L * t.tv_sec) + (t.tv_usec / 1000); 21 | } 22 | static inline struct timespec millisToTimespec(long millis) { 23 | struct timespec interval; 24 | interval.tv_sec = UtilTime::millisToSeconds(millis); 25 | interval.tv_nsec = UtilTime::millisToNanos(millis) - UtilTime::secondsToNanos(interval.tv_sec); 26 | return interval; 27 | } 28 | static inline long millisToNanos(long millis) { 29 | return millis*1000000L; 30 | } 31 | static inline long millisToMicros(long millis) { 32 | return millis*1000L; 33 | } 34 | static inline long millisToSeconds(long millis) { 35 | return millis/1000L; 36 | } 37 | static inline long nanosToMillis(long nanos) { 38 | return nanos/1000000L; 39 | } 40 | static inline long microsToMillis(long micros) { 41 | return micros/1000L; 42 | } 43 | static inline long secondsToNanos(long secs) { 44 | return secs*1000000000L; 45 | } 46 | }; 47 | 48 | } // end namespace 49 | 50 | #endif 51 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | memkeys 2 | ======= 3 | Copyright 2013 Tumblr. 4 | 5 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use 6 | this file except in compliance with the License. You may obtain a copy of the 7 | License at http://www.apache.org/licenses/LICENSE-2.0 8 | 9 | Unless required by applicable law or agreed to in writing, software distributed 10 | under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 11 | CONDITIONS OF ANY KIND, either express or implied. See the License for the 12 | specific language governing permissions and limitations under the License. 13 | 14 | src/util/backoff.* 15 | ================== 16 | Ported from http://goo.gl/AfeaV. 17 | 18 | Copyright 2011 Google Inc. 19 | 20 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use 21 | this file except in compliance with the License. You may obtain a copy of the 22 | License at http://www.apache.org/licenses/LICENSE-2.0 23 | 24 | Unless required by applicable law or agreed to in writing, software distributed 25 | under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 26 | CONDITIONS OF ANY KIND, either express or implied. See the License for the 27 | specific language governing permissions and limitations under the License. 28 | 29 | src/util/mqueue.h 30 | ================= 31 | Modified from goo.gl/lrQ71 32 | 33 | No license specified, so public domain assumed. 34 | -------------------------------------------------------------------------------- /src/report/curses.h: -------------------------------------------------------------------------------- 1 | #ifndef _REPORT_CURSES_H 2 | #define _REPORT_CURSES_H 3 | 4 | #include "config.h" 5 | #include "net/pcap.h" 6 | #include "report/report.h" 7 | #include "util/stats.h" 8 | 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | namespace mckeys { 15 | 16 | class CursesReport : public Report 17 | { 18 | public: 19 | typedef std::vector StatColumns; 20 | typedef std::map CommandMap; 21 | 22 | CursesReport(const Config* cfg, const Pcap* session, Stats* stats); 23 | virtual ~CursesReport(); 24 | 25 | virtual void render(); 26 | 27 | protected: 28 | void handleKeyPress(char key); 29 | void renderHeader(); 30 | void renderFooter(); 31 | void renderTimeStat(const uint64_t render_time) const; 32 | void renderStats(std::deque q, uint32_t qsize); 33 | std::string createStatLine(const Stat &stat) const; 34 | std::string createRenderTime(const uint64_t render_time) const; 35 | void setpos(int y, int x) const; 36 | 37 | private: 38 | void renderInit(); // initialize rendering system - only once before first rendering 39 | void renderDone(); // tear down render system 40 | void initFooterText(); // done once by renderInit 41 | const Pcap* session; 42 | Stats* stats; 43 | int statColumnWidth; 44 | uint32_t keyColumnWidth; 45 | std::string footerText; 46 | StatColumns statColumns; 47 | CommandMap cmdMap; 48 | SortOrder sortOrder; 49 | SortMode sortMode; 50 | }; 51 | 52 | } // end namespace 53 | 54 | #endif 55 | -------------------------------------------------------------------------------- /src/net/pcap.h: -------------------------------------------------------------------------------- 1 | #ifndef _NET_PCAP_H 2 | #define _NET_PCAP_H 3 | 4 | extern "C" { 5 | #include 6 | } 7 | 8 | #include "common.h" 9 | 10 | namespace mckeys { 11 | 12 | typedef pcap_t * PcapPtr; 13 | typedef pcap_handler PcapCallback; 14 | typedef struct { 15 | llui_t received; 16 | llui_t dropped; 17 | llui_t if_dropped; 18 | double drop_pct; 19 | } PcapStats; 20 | 21 | // Pcap is not intended to be directly instantiated. Concrete implementations 22 | // should be used as: 23 | // PcapImpl impl(...); 24 | // impl.setFilter("port 11211"); // optional 25 | // impl.open(); 26 | // impl.startCapture(handler, packet_count); 27 | // setup call back handler to run 28 | // impl.stopCapture() 29 | // impl.close(); 30 | class Pcap { 31 | public: 32 | virtual ~Pcap(); 33 | 34 | // be sure to set the state to STARTING once the session is open 35 | // you also need to set handle 36 | virtual void open() = 0; 37 | PcapStats getStats() const; 38 | std::string getStatsString() const; 39 | void close(); 40 | 41 | virtual bpf_u_int32 getSubnetMask() const = 0; 42 | virtual bpf_u_int32 getIpAddress() const = 0; 43 | 44 | void setFilter(const std::string &filter); 45 | void startCapture(PcapCallback cb, int cnt = -1 /* loop forever */, u_char *userData = NULL); 46 | void stopCapture(); 47 | 48 | protected: 49 | Pcap(); 50 | 51 | std::string getPcapError() const; 52 | 53 | static char errorBuffer[]; 54 | PcapPtr handle; 55 | LoggerPtr logger; 56 | State state; 57 | }; 58 | 59 | } 60 | 61 | #endif 62 | -------------------------------------------------------------------------------- /src/report/report_type.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | 3 | #include "common.h" 4 | #include "report/report_type.h" 5 | #include "report/csv.h" 6 | #include "report/curses.h" 7 | 8 | namespace mckeys { 9 | 10 | using namespace std; 11 | 12 | // protected constructor 13 | ReportType::ReportType(const string &name) : name(name) 14 | {} 15 | 16 | // static values 17 | ReportType ReportType::CSV = ReportType("CSV"); 18 | ReportType ReportType::NCURSES = ReportType("CURSES"); 19 | 20 | // static methods 21 | ReportType ReportType::fromString(const string &name) { 22 | string reportName = ""; 23 | for (uint32_t i = 0; i < name.length(); i++) { 24 | reportName += toupper(name.at(i)); 25 | } 26 | if (reportName == "CURSES") { 27 | return ReportType::NCURSES; 28 | } else if (reportName == "CSV") { 29 | return ReportType::CSV; 30 | } else { 31 | throw range_error("No such report type with name " + name); 32 | } 33 | } 34 | 35 | // public methods 36 | string ReportType::getName() const { 37 | return name; 38 | } 39 | bool ReportType::operator==(const ReportType &other) const { 40 | return (getName() == other.getName()); 41 | } 42 | 43 | Report* ReportType::makeReport(const Config* cfg, const Pcap* sess, Stats* stats) { 44 | if (cfg->getReportType() == ReportType::NCURSES) { 45 | return new CursesReport(cfg, sess, stats); 46 | } else if (cfg->getReportType() == ReportType::CSV) { 47 | return new CsvReport(cfg, sess, stats); 48 | } else { 49 | throw range_error("Unsupported report type " + cfg->getReportType().getName()); 50 | } 51 | } 52 | 53 | } // end namespace 54 | -------------------------------------------------------------------------------- /src/main.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "common.h" 6 | #include "cli.h" 7 | #include "memkeys.h" 8 | 9 | using namespace std; 10 | using namespace mckeys; 11 | 12 | int handleConfigurationError(LoggerPtr logger, const char *progname) { 13 | try { 14 | throw; 15 | } catch (const MemkeysConfigurationError &er) { 16 | logger->fatal(CONTEXT, 17 | "Error configuring %s: %s", PACKAGE_NAME, er.what()); 18 | cout << Cli::help(progname); 19 | return EXIT_FAILURE; 20 | } catch (const MemkeysException &ex) { 21 | logger->fatal(CONTEXT, "Error setting up application: %s", ex.what()); 22 | return EXIT_FAILURE; 23 | } catch (...) { 24 | logger->fatal("Unexpected failure"); 25 | throw; 26 | } 27 | } 28 | 29 | int main(int argc, char ** argv) { 30 | LoggerPtr logger = Logger::getLogger("main"); 31 | Memkeys * app = NULL; 32 | int rc = EXIT_SUCCESS; 33 | 34 | logger->setLevel(Level::INFO); 35 | 36 | // configure and initialize the app 37 | try { 38 | app = Memkeys::getInstance(argc, argv); 39 | } catch(...) { 40 | return handleConfigurationError(logger, argv[0]); 41 | } 42 | 43 | // run the app 44 | try { 45 | app->run(); 46 | } catch (const exception &ex) { 47 | logger->fatal(CONTEXT, "Error running application: %s", ex.what()); 48 | rc = EXIT_FAILURE; 49 | } 50 | logger->info(CONTEXT, "Shutting down"); 51 | 52 | // handle cleanup 53 | delete logger; 54 | delete app; 55 | delete Config::getInstance(); 56 | delete Logger::getRootLogger(); 57 | return rc; 58 | } 59 | -------------------------------------------------------------------------------- /src/logging/record.h: -------------------------------------------------------------------------------- 1 | #ifndef _LOGGING_RECORD_H 2 | #define _LOGGING_RECORD_H 3 | 4 | #include "logging/level.h" 5 | 6 | #include 7 | #include 8 | 9 | extern "C" { 10 | #include 11 | } 12 | 13 | namespace mckeys { 14 | 15 | class Record 16 | { 17 | public: 18 | Record(); 19 | Record(const std::string &fname, 20 | const uint32_t ln, 21 | const std::string &name); 22 | Record(const std::string &fname, 23 | const uint32_t ln, 24 | const std::string &name, 25 | const std::exception &ex); 26 | 27 | std::string getFileName() const; 28 | void setFileName(const std::string &fname); 29 | 30 | Level getLevel() const; 31 | void setLevel(const Level &level); 32 | 33 | uint32_t getLineNumber() const; 34 | void setLineNumber(const uint32_t ln); 35 | 36 | std::string getLoggerName() const; 37 | void setLoggerName(const std::string &lname); 38 | 39 | std::string getMessage() const; 40 | void setMessage(const std::string &message); 41 | 42 | std::string getMethodName() const; 43 | void setMethodName(const std::string &name); 44 | 45 | std::string getThrownMessage() const; 46 | void setThrownMessage(const std::string &ex); 47 | bool hasThrown() const; 48 | 49 | std::string getTimestamp() const; 50 | std::string getTimestamp(struct timeval ts) const; 51 | 52 | private: 53 | std::string _filename; 54 | Level _level; 55 | uint32_t _lineNumber; 56 | std::string _loggerName; 57 | std::string _message; 58 | std::string _methodName; 59 | std::string _thrownMessage; 60 | }; 61 | 62 | } // end namespace 63 | 64 | #endif 65 | -------------------------------------------------------------------------------- /src/config.h: -------------------------------------------------------------------------------- 1 | #ifndef CONFIG_H 2 | #define CONFIG_H 3 | 4 | #include 5 | #include "logging/logger.h" 6 | #include "report/report_type.h" 7 | 8 | namespace mckeys { 9 | 10 | class Config 11 | { 12 | public: 13 | static Config * getInstance(); 14 | virtual ~Config(); 15 | 16 | void setDiscardThreshold(const double threshold); 17 | double getDiscardThreshold() const; 18 | 19 | void setInterface(const std::string &value); 20 | std::string getInterface() const; 21 | 22 | void setLogfile(const std::string &value); 23 | std::string getLogfile() const; 24 | 25 | void setPort(const int port); 26 | uint16_t getPort() const; 27 | std::string getPortAsString() const; 28 | 29 | int getReadTimeout() const 30 | { return _readTimeout; } 31 | 32 | void setRefreshInterval(const int interval); 33 | uint16_t getRefreshInterval() const; 34 | 35 | void setReportType(const std::string &value); 36 | ReportType getReportType() const; 37 | 38 | int getSnapLength() const 39 | { return _snapLength; } 40 | 41 | bool isPromiscuous() const 42 | { return _isPromiscuous; } 43 | 44 | void increaseVerbosity(); 45 | void makeLessVerbose(); 46 | Level verbosity() const; 47 | 48 | std::string toString() const; 49 | 50 | private: 51 | Config(); 52 | void adjustLoggerLevel(const Level &newLevel); 53 | 54 | double discardThreshold; 55 | std::string interface; 56 | bool _isPromiscuous; 57 | uint16_t port; 58 | int _readTimeout; 59 | uint16_t refreshInterval; 60 | int _snapLength; 61 | LoggerPtr logger; 62 | std::string logfile; 63 | ReportType reportType; 64 | }; 65 | 66 | } // end namespace 67 | 68 | #endif 69 | -------------------------------------------------------------------------------- /src/net/pcap_live.cpp: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | #include "net/net.h" 3 | 4 | namespace mckeys { 5 | 6 | using namespace std; 7 | 8 | PcapLive::PcapLive(const Config * cfg) 9 | : Pcap(), 10 | config(cfg), 11 | // TODO this should be provided to constructor, otherwise exception thrown 12 | // in constructor 13 | device(Device::getDevice(cfg->getInterface())) 14 | { 15 | logger->debug(CONTEXT, "Using device %s for capture", getInterfaceC()); 16 | } 17 | PcapLive::~PcapLive() 18 | { 19 | logger->info(CONTEXT, "Shutting down"); 20 | } 21 | 22 | bpf_u_int32 PcapLive::getIpAddress() const 23 | { 24 | return getDevice().getIpAddress(); 25 | } 26 | 27 | bpf_u_int32 PcapLive::getSubnetMask() const 28 | { 29 | return getDevice().getSubnetMask(); 30 | } 31 | 32 | void PcapLive::open() 33 | { 34 | if (!state.checkAndSet(state_NEW, state_STARTING)) { 35 | logger->warning(CONTEXT, "Device already open"); 36 | } 37 | const char *dev = getInterfaceC(); 38 | logger->info(CONTEXT, "Opening device %s for capture", dev); 39 | handle = pcap_open_live(dev, /* device to capture on */ 40 | config->getSnapLength(), /* how many bytes per packet */ 41 | config->isPromiscuous(), /* promiscuous */ 42 | config->getReadTimeout(), /* read timeout, in ms */ 43 | errorBuffer); 44 | if (handle == NULL) { 45 | string msg = "Could not open device " + getInterface() + " for reading: " + errorBuffer; 46 | logger->error(CONTEXT, msg.c_str()); 47 | state.setState(state_NEW); 48 | throw MemkeysException(msg); 49 | } 50 | } 51 | 52 | } // end namespace 53 | -------------------------------------------------------------------------------- /src/util/stat.cpp: -------------------------------------------------------------------------------- 1 | #include "util/stat.h" 2 | 3 | #include 4 | 5 | namespace mckeys { 6 | 7 | using namespace std; 8 | using std::isinf; 9 | 10 | Stat::Stat() 11 | : key(), _created(0), _size(0), _count(0) 12 | {} 13 | Stat::Stat(const string &key, const uint32_t size) 14 | : key(key), 15 | _created(UtilTime::currentTimeMillis()), 16 | _size(size), 17 | _count(1) 18 | {} 19 | Stat::~Stat() {} 20 | // std::atomic member variables require that your class implement a copy 21 | // constructor and the assignment operator, if you want to use the Stat class 22 | // with most STL collections 23 | Stat::Stat(const Stat &stat) 24 | : key(stat.getKey()), 25 | _created(stat.getCreated()), 26 | _size(stat.getSize()), 27 | _count(stat.getCount()) 28 | {} 29 | // TODO implement copy/destroy 30 | Stat& Stat::operator=(const Stat& rhs) 31 | { 32 | if (this != &rhs) { 33 | key = rhs.getKey(); 34 | _created = rhs.getCreated(); 35 | setSize(rhs.getSize()); 36 | _count.store(rhs.getCount()); 37 | } 38 | return *this; 39 | } 40 | 41 | uint64_t Stat::getCreated() const { 42 | return _created; 43 | } 44 | const string Stat::getKey() const { 45 | return key; 46 | } 47 | uint32_t Stat::getSize() const { 48 | return _size.load(); 49 | } 50 | void Stat::setSize(const uint32_t size) { 51 | _size.store(size); 52 | } 53 | 54 | double Stat::bandwidth(const uint64_t elapsed_t) const { 55 | return (getSize() * requestRate(elapsed_t) * 8) / 1000.0; 56 | } 57 | 58 | double Stat::requestRate(const uint64_t elapsed_t) const { 59 | double rate = (getCount() / (double)elapsed_t); 60 | if (isinf(rate)) { 61 | return 1.0; 62 | } else { 63 | return rate; 64 | } 65 | } 66 | 67 | void Stat::increment() { 68 | _count += 1; 69 | } 70 | 71 | } // end namespace 72 | -------------------------------------------------------------------------------- /src/net/capture_engine.h: -------------------------------------------------------------------------------- 1 | #ifndef _NET_CAPTURE_ENGINE_H 2 | #define _NET_CAPTURE_ENGINE_H 3 | 4 | #include 5 | #include 6 | 7 | #include "common.h" 8 | #include "net/memcache_command.h" 9 | #include "net/packet.h" 10 | #include "net/pcap.h" 11 | #include "report/report.h" 12 | 13 | // Basic holder for userdata when processing packets. Provides a logger, report 14 | // instance, and a config 15 | 16 | namespace mckeys { 17 | 18 | class CaptureEngine { 19 | 20 | public: 21 | typedef std::vector*> Packets; 22 | 23 | CaptureEngine(const Config* config, const Pcap* session); 24 | ~CaptureEngine(); 25 | 26 | void enqueue(const Packet& packet); 27 | 28 | bpf_u_int32 getIpAddress() const 29 | { return session->getIpAddress(); } 30 | PcapStats getStats() const 31 | { return session->getStats(); } 32 | std::string getStatsString() const 33 | { return session->getStatsString(); } 34 | 35 | bool isShutdown() const; 36 | void shutdown(); 37 | 38 | protected: 39 | // run in threads 40 | void processPackets(int worker_id, mqueue* work_queue); 41 | 42 | // called by processPackets to parse an enqueued packet 43 | MemcacheCommand parse(const Packet& mc) const; 44 | // called by processPackets if appropriate 45 | void enqueue(const MemcacheCommand& mc); 46 | 47 | const LoggerPtr logger; 48 | const Config* config; 49 | const Pcap* session; 50 | mqueue *barrier; 51 | Stats* stats; 52 | Report* report; 53 | volatile bool _is_terminated; 54 | // there is one worker thread per queue 55 | uint8_t queue_count; 56 | Packets packets; 57 | std::thread *worker_threads; 58 | std::mutex barrier_lock; 59 | 60 | private: 61 | CaptureEngine(); // no default constructor 62 | 63 | }; 64 | 65 | } // end namespace 66 | 67 | #endif 68 | -------------------------------------------------------------------------------- /src/net/device.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "util/util.h" 5 | #include "net/net.h" 6 | 7 | extern "C" { 8 | #include 9 | } 10 | 11 | namespace mckeys { 12 | 13 | using namespace std; 14 | 15 | /** 16 | * Internal to compilation unit. 17 | */ 18 | static char errbuf[PCAP_ERRBUF_SIZE] = {0}; 19 | 20 | Device Device::getDevice(const string &name) 21 | { 22 | pcap_if_t *alldevs; 23 | bpf_u_int32 address = 0; 24 | bpf_u_int32 network = 0; 25 | bpf_u_int32 mask = 0; 26 | bool loopback = 0; 27 | int status = pcap_findalldevs(&alldevs, errbuf); 28 | if (status != 0) { 29 | throw MemkeysException(string("No readable devices: ") + errbuf); 30 | } 31 | // TODO -> move to find_address_iface(name); 32 | for (pcap_if_t *d = alldevs; d != NULL; d = d->next) { 33 | if (string(d->name) == name) { 34 | if (d->flags & PCAP_IF_LOOPBACK) { 35 | loopback = true; 36 | } 37 | for (pcap_addr_t *a = d->addresses; a != NULL; a = a->next) { 38 | if (a->addr->sa_family == AF_INET) { 39 | address = ((struct sockaddr_in*)(a->addr))->sin_addr.s_addr; 40 | } 41 | } 42 | } 43 | } 44 | pcap_freealldevs(alldevs); 45 | if (pcap_lookupnet(name.c_str(), &network, &mask, errbuf) < 0) { 46 | throw MemkeysException(string("Invalid device ") + name + ": " + errbuf); 47 | } 48 | return Device(name, loopback, network, mask, address); 49 | } 50 | 51 | // protected 52 | Device::Device(const string &name, 53 | const bool loopback, 54 | const bpf_u_int32 network, 55 | const bpf_u_int32 mask, 56 | const bpf_u_int32 address) 57 | : deviceName_(name), 58 | loopback_(loopback), 59 | network_(network), 60 | subnetMask_(mask), 61 | ipAddress_(address) 62 | {} 63 | 64 | } //end namespace 65 | -------------------------------------------------------------------------------- /src/report/csv.cpp: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | #include "report/csv.h" 3 | 4 | namespace mckeys { 5 | 6 | using namespace std; 7 | 8 | CsvReport::CsvReport(const Config* cfg, const Pcap* session, Stats* stats) 9 | : Report(cfg, Logger::getLogger("csvReport")), 10 | stats(stats) 11 | { 12 | (void)session; 13 | if (state.checkAndSet(state_NEW, state_STARTING)) { 14 | report_thread = thread(&CsvReport::render, this); 15 | } else { 16 | logger->warning(CONTEXT, "Incorrect API usage"); 17 | } 18 | } 19 | 20 | static void printHeader() { 21 | cout << "key,count,elapsed,rate,size,bandwidth" << endl; 22 | } 23 | 24 | void CsvReport::render() 25 | { 26 | static bool first = false; 27 | struct timespec ts = UtilTime::millisToTimespec(config->getRefreshInterval()); 28 | 29 | if (!state.checkAndSet(state_STARTING, state_RUNNING)) { 30 | logger->error(CONTEXT, "render already started"); 31 | return; 32 | } 33 | 34 | while(state.isRunning()) { 35 | deque q = stats->getLeaders(mode_REQRATE, sort_DESC); 36 | uint32_t qsize = q.size(); 37 | logger->debug(CONTEXT, "Rendering report with %u data points", qsize); 38 | if (qsize > 0) { 39 | if (!first) { 40 | first = true; 41 | printHeader(); 42 | } else { 43 | cout << ",,,,,," << endl; 44 | } 45 | renderStats(q); 46 | } 47 | nanosleep(&ts, NULL); 48 | } 49 | } 50 | 51 | void CsvReport::renderStats(deque q) { 52 | for (deque::iterator it = q.begin(); it != q.end(); ++it) { 53 | Stat stat = *it; 54 | cout << stat.getKey() << ","; 55 | cout << stat.getCount() << ","; 56 | cout << stat.elapsed() << ","; 57 | cout << std::fixed << std::setprecision(2) << stat.requestRate() << ","; 58 | cout << stat.getSize() << ","; 59 | cout << std::fixed << std::setprecision(2) << stat.bandwidth() << endl; 60 | } 61 | } 62 | 63 | } // end namespace 64 | -------------------------------------------------------------------------------- /memkeys.spec: -------------------------------------------------------------------------------- 1 | Name: memkeys 2 | Version: 0.2 3 | Release: 1%{?dist} 4 | Summary: Shows your memcache key usage in realtime 5 | 6 | Group: Development/Tools 7 | License: Apache 2.0 8 | URL: https://github.com/tumblr/memkeys 9 | Source0: %{name}-%{version}.tar.gz 10 | BuildRoot: %{_tmppath}/%{name}-%{version}-%{release}-root-%(%{__id_u} -n) 11 | 12 | # Uses 'preview' GCC 4.4 on RHEL 5 for needed features. 13 | %if 0%{?rhel} < 6 14 | BuildRequires: gcc44 15 | BuildRequires: gcc44-c++ 16 | %else 17 | BuildRequires: gcc 18 | BuildRequires: gcc-c++ 19 | %endif 20 | 21 | BuildRequires: libtool >= 1.5.22 22 | BuildRequires: autoconf >= 2.61 23 | BuildRequires: automake >= 1.9.6 24 | BuildRequires: libpcap-devel >= 0.9.4-15 25 | BuildRequires: pcre-devel >= 6.6-6 26 | BuildRequires: ncurses-devel >= 5.5-24.20060715 27 | 28 | Requires: libpcap >= 0.9.4-15 29 | Requires: pcre >= 6.6-6 30 | Requires: ncurses >= 5.5-24.20060715 31 | 32 | 33 | %description 34 | memkeys provides an interface to track the usage of memcache keys in realtime, 35 | originally inspired by mctop from mctop from etsy. It provides an ncurses 36 | interface alongside a CSV reporting interface, and is designed to drop less 37 | than 3 percent of memcache packets at production load, tested on a saturated 1 38 | Gigabit link. 39 | 40 | 41 | %prep 42 | %setup -q 43 | 44 | 45 | %build 46 | %if 0%{?rhel} < 6 47 | export CC=gcc44 48 | export CXX=g++44 49 | %endif 50 | 51 | ./build-eng/autogen.sh 52 | %configure 53 | 54 | make %{?_smp_mflags} 55 | 56 | 57 | %install 58 | rm -rf $RPM_BUILD_ROOT 59 | make install DESTDIR=$RPM_BUILD_ROOT 60 | 61 | 62 | %clean 63 | rm -rf $RPM_BUILD_ROOT 64 | 65 | 66 | %files 67 | %defattr(-,root,root,-) 68 | %doc LICENSE NOTICE README.md 69 | %{_bindir}/memkeys 70 | 71 | 72 | %changelog 73 | * Sun Apr 28 2013 Steve Salevan 0.2-1 74 | - Initial specfile writeup 75 | 76 | 77 | -------------------------------------------------------------------------------- /src/Makefile.am: -------------------------------------------------------------------------------- 1 | MAINTAINERCLEANFILES = Makefile.in 2 | 3 | AUTOMAKE_OPTIONS = subdir-objects 4 | 5 | # Need for getopt_long 6 | AM_CPPFLAGS = -D_GNU_SOURCE 7 | 8 | # Always build with Wall and pedantic errors 9 | AM_CFLAGS = -Wall -pedantic-errors 10 | AM_CXXFLAGS = -Wall -pedantic-errors 11 | 12 | # Different optimization levels 13 | if DEBUGGING 14 | AM_CFLAGS += -g3 -O0 15 | AM_CXXFLAGS += -g3 -O0 16 | else 17 | AM_CFLAGS += -O2 18 | AM_CXXFLAGS += -O2 19 | endif 20 | 21 | if DEVELOPMENT 22 | AM_CFLAGS += -Wextra 23 | AM_CXXFLAGS += -Wextra -Wwrite-strings -Woverloaded-virtual -Wno-sign-compare 24 | endif 25 | 26 | # Build a dummy library so we can link tests against it 27 | noinst_LTLIBRARIES = libmemkeys.la 28 | libmemkeys_la_SOURCES = \ 29 | util/util.h \ 30 | util/backoff.h util/backoff.cpp \ 31 | util/exceptions.h \ 32 | util/mqueue.h \ 33 | util/options.h \ 34 | util/util_time.h \ 35 | util/stat.h util/stat.cpp \ 36 | util/stats.h util/stats.cpp \ 37 | util/state.h util/state.cpp \ 38 | logging/record.h logging/record.cpp \ 39 | logging/level.h logging/level.cpp \ 40 | logging/logger.h logging/logger.cpp \ 41 | config.h config.cpp \ 42 | net/device.h net/device.cpp \ 43 | net/pcap.h net/pcap.cpp \ 44 | net/packet.h net/packet.cpp \ 45 | net/pcap_live.h net/pcap_live.cpp \ 46 | report/report_type.h report/report_type.cpp \ 47 | report/report.h report/report.cpp \ 48 | report/csv.h report/csv.cpp \ 49 | report/curses.h report/curses.cpp \ 50 | net/memcache_command.h net/memcache_command.cpp \ 51 | net/capture_engine.h net/capture_engine.cpp \ 52 | cli.h cli.cpp \ 53 | memkeys.h memkeys.cpp 54 | 55 | # memkeys binary 56 | bin_PROGRAMS = memkeys 57 | memkeys_LDADD = libmemkeys.la 58 | memkeys_SOURCES = main.cpp 59 | 60 | # Magic needed for linking the dummy memkeys library for building our test suite 61 | check_PROGRAMS = testall 62 | testall_LDADD = libmemkeys.la \ 63 | $(top_builddir)/gtest/lib/libgtest.la \ 64 | $(top_builddir)/gtest/lib/libgtest_main.la 65 | testall_CXXFLAGS = -I$(top_srcdir)/gtest/include \ 66 | -I$(top_builddir)/gtest/include 67 | testall_SOURCES = logging/test_level.cpp \ 68 | logging/test_record.cpp 69 | 70 | TESTS = testall 71 | -------------------------------------------------------------------------------- /src/net/memcache_command.h: -------------------------------------------------------------------------------- 1 | #ifndef _NET_MEMCACHE_CMD_H 2 | #define _NET_MEMCACHE_CMD_H 3 | 4 | #include 5 | 6 | #include "net/packet.h" 7 | #include "net/pcap.h" 8 | 9 | namespace mckeys { 10 | 11 | enum memcache_command_t { 12 | MC_UNKNOWN, MC_REQUEST, MC_RESPONSE 13 | }; 14 | 15 | /** 16 | * Usage: auto mc = MemcacheCommand::create(pkt, captureAddress); 17 | */ 18 | class MemcacheCommand 19 | { 20 | public: 21 | static MemcacheCommand create(const Packet& pkt, 22 | const bpf_u_int32 captureAddress); 23 | 24 | bool isCommand() const 25 | { return (isRequest() || isResponse()); } 26 | bool isRequest() const 27 | { return (cmdType_ == MC_REQUEST); } 28 | bool isResponse() const 29 | { return (cmdType_ == MC_RESPONSE); } 30 | 31 | // only when isRequest is true 32 | std::string getCommandName() const { return commandName_; } 33 | 34 | // sometimes when isResponse is true, sometimes when isRequest is true 35 | std::string getObjectKey() const { return objectKey_; } 36 | 37 | // only when isResponse is true 38 | uint32_t getObjectSize() const { return objectSize_; } 39 | 40 | // source address for request 41 | std::string getSourceAddress() const { return sourceAddress_; } 42 | 43 | protected: 44 | // no assignment operator 45 | MemcacheCommand& operator=(const MemcacheCommand& mc) = delete; 46 | 47 | // Default constructor is protected 48 | MemcacheCommand(); 49 | 50 | MemcacheCommand(const memcache_command_t cmdType, 51 | const std::string sourceAddress, 52 | const std::string commandName, 53 | const std::string objectKey, 54 | uint32_t objectSize); 55 | 56 | static MemcacheCommand makeRequest(u_char *data, 57 | int dataLength, 58 | std::string sourceAddress); 59 | static MemcacheCommand makeResponse(u_char *data, 60 | int dataLength, 61 | std::string sourceAddress); 62 | 63 | const memcache_command_t cmdType_; 64 | const std::string sourceAddress_; 65 | const std::string commandName_; 66 | const std::string objectKey_; 67 | const uint32_t objectSize_; 68 | }; 69 | 70 | } // end namespace 71 | 72 | #endif 73 | -------------------------------------------------------------------------------- /src/logging/test_level.cpp: -------------------------------------------------------------------------------- 1 | #include "logging/level.h" 2 | #include "util/options.h" 3 | #include 4 | #include 5 | #include 6 | 7 | namespace mckeys { 8 | 9 | TEST(Level, Names) { 10 | ASSERT_EQ("TRACE", Level::TRACE.getName()); 11 | ASSERT_EQ("DEBUG", Level::DEBUG.getName()); 12 | ASSERT_EQ("INFO", Level::INFO.getName()); 13 | ASSERT_EQ("WARNING", Level::WARNING.getName()); 14 | ASSERT_EQ("ERROR", Level::ERROR.getName()); 15 | ASSERT_EQ("FATAL", Level::FATAL.getName()); 16 | ASSERT_EQ("OFF", Level::OFF.getName()); 17 | } 18 | 19 | #define NAME_TEST(lvl, ucn, id) do { \ 20 | ASSERT_EQ(lvl, Level::fromName(ucn)); \ 21 | ASSERT_EQ(lvl, Level::fromValue(id)); \ 22 | } while(0) 23 | TEST(Level, NameMap) { 24 | NAME_TEST(Level::TRACE, "TRACE", 0); 25 | NAME_TEST(Level::DEBUG, "DEBUG", 1); 26 | NAME_TEST(Level::INFO, "INFO", 2); 27 | NAME_TEST(Level::WARNING, "WARNING", 3); 28 | NAME_TEST(Level::ERROR, "ERROR", 4); 29 | NAME_TEST(Level::FATAL, "FATAL", 5); 30 | NAME_TEST(Level::OFF, "OFF", USIZE_BITS(uint32_t)); 31 | ASSERT_THROW(Level::fromName("trace"), std::range_error); 32 | ASSERT_THROW(Level::fromValue(1942), std::range_error); 33 | } 34 | 35 | TEST(Level, Equality) { 36 | ASSERT_LT(Level::TRACE, Level::DEBUG); 37 | ASSERT_LT(Level::DEBUG, Level::INFO); 38 | ASSERT_LT(Level::INFO, Level::WARNING); 39 | ASSERT_LT(Level::WARNING, Level::ERROR); 40 | ASSERT_LT(Level::ERROR, Level::FATAL); 41 | ASSERT_LT(Level::FATAL, Level::OFF); 42 | 43 | ASSERT_GT(Level::OFF, Level::FATAL); 44 | ASSERT_GT(Level::FATAL, Level::ERROR); 45 | ASSERT_GT(Level::ERROR, Level::WARNING); 46 | ASSERT_GT(Level::WARNING, Level::INFO); 47 | ASSERT_GT(Level::INFO, Level::DEBUG); 48 | ASSERT_GT(Level::DEBUG, Level::TRACE); 49 | 50 | ASSERT_EQ(Level::OFF, Level::OFF); 51 | ASSERT_EQ(Level::FATAL, Level::FATAL); 52 | ASSERT_EQ(Level::ERROR, Level::ERROR); 53 | ASSERT_EQ(Level::WARNING, Level::WARNING); 54 | ASSERT_EQ(Level::INFO, Level::INFO); 55 | ASSERT_EQ(Level::DEBUG, Level::DEBUG); 56 | ASSERT_EQ(Level::TRACE, Level::TRACE); 57 | 58 | ASSERT_NE(Level::OFF, Level::FATAL); 59 | ASSERT_NE(Level::OFF, Level::ERROR); 60 | ASSERT_NE(Level::OFF, Level::WARNING); 61 | ASSERT_NE(Level::OFF, Level::INFO); 62 | ASSERT_NE(Level::OFF, Level::DEBUG); 63 | ASSERT_NE(Level::OFF, Level::TRACE); 64 | } 65 | 66 | } // end namespace 67 | -------------------------------------------------------------------------------- /src/util/stat.h: -------------------------------------------------------------------------------- 1 | #ifndef _UTIL_STAT_H 2 | #define _UTIL_STAT_H 3 | 4 | #ifdef HAVE_CONFIG_H 5 | #include "mconfig.h" 6 | #endif 7 | 8 | #include 9 | 10 | #ifdef HAVE_ATOMIC 11 | #include 12 | #else 13 | #include 14 | #endif 15 | 16 | #include 17 | #include 18 | #include 19 | 20 | #include "util/util_time.h" 21 | 22 | namespace mckeys { 23 | 24 | /** 25 | * A Stat is part of a collection of stats. 26 | * Each stat has a unique key, and a possibly changing size. 27 | * Every time you see this key, call increment on the associated stat. 28 | * You can then easily determine the requestRate, associated bandwidth, and the 29 | * total seen count. 30 | * 31 | * Calls wrt count() and size are atomic. Size may be mutated. 32 | * 33 | * Also see Stats 34 | */ 35 | class Stat { 36 | public: 37 | static inline size_t hashKey(const std::string &key) { 38 | std::hash hash_fn; 39 | return hash_fn(key); 40 | } 41 | 42 | Stat(const std::string &key, const uint32_t size); 43 | Stat(); // needed for updating a map. fucking hell. 44 | ~Stat(); 45 | // std::atomic member variables require that your class implement a copy 46 | // constructor and the assignment operator, if you want to use the Stat class 47 | // with most STL collections 48 | Stat(const Stat& stat); 49 | Stat& operator=(const Stat& rhs); 50 | 51 | uint64_t getCreated() const; 52 | const std::string getKey() const; 53 | uint32_t getSize() const; 54 | void setSize(const uint32_t sz); 55 | 56 | double bandwidth(const uint64_t e) const; 57 | inline double bandwidth() const { 58 | return bandwidth(elapsed()); 59 | } 60 | 61 | uint64_t getCount() const { 62 | return _count.load(); 63 | } 64 | 65 | inline uint64_t elapsed() const { 66 | return (UtilTime::currentTimeMillis() - _created)/1000; 67 | } 68 | 69 | void increment(); 70 | 71 | double requestRate(const uint64_t e) const; 72 | inline double requestRate() const { 73 | return requestRate(elapsed()); 74 | } 75 | 76 | private: 77 | std::string key; 78 | uint64_t _created; 79 | mutable std::atomic_uint_fast32_t _size; 80 | mutable std::atomic_uint_fast64_t _count; 81 | }; 82 | 83 | typedef std::unordered_map StatCollection; 84 | typedef std::pair StatPair; 85 | 86 | } // end namespace 87 | 88 | #endif 89 | -------------------------------------------------------------------------------- /src/util/backoff.h: -------------------------------------------------------------------------------- 1 | #ifndef _UTIL_BACKOFF_H 2 | #define _UTIL_BACKOFF_H 3 | 4 | #include 5 | 6 | namespace mckeys { 7 | 8 | // Modeled off http://goo.gl/AfeaV 9 | // Provides a configurable backoff policy for something that needs it 10 | class Backoff { 11 | public: 12 | // 0.5 seconds 13 | const static uint32_t DEFAULT_INITIAL_INTERVAL_MILLIS = 5; 14 | // 15 minutes 15 | const static uint32_t DEFAULT_MAX_ELAPSED_TIME_MILLIS = 1000; 16 | // 1 minute 17 | const static uint32_t DEFAULT_MAX_INTERVAL_MILLIS = 1000; 18 | // 1.5 which is 50% increase per back off 19 | const static double DEFAULT_MULTIPLIER; // = 1.5; 20 | // 0.5 which results in a random period ranging between 50% below and 50% 21 | // above the retry interval 22 | const static double DEFAULT_RANDOMIZATION_FACTOR; // = 0.5; 23 | 24 | Backoff(); 25 | virtual ~Backoff(); 26 | 27 | // The current retry interval in milliseconds 28 | uint32_t getCurrentIntervalMillis() const; 29 | 30 | // The elapsed time in ms since the instance was created. 31 | // Reset when reset() is called. 32 | uint64_t getElapsedTimeMillis() const; 33 | 34 | // The initial retry interval in milliseconds. 35 | uint32_t getInitialIntervalMillis() const; 36 | Backoff& setInitialIntervalMillis(const uint32_t millis); 37 | 38 | // The maximum elapsed time in milliseconds 39 | uint32_t getMaxElapsedTimeMillis() const; 40 | Backoff& setMaxElapsedTimeMillis(const uint32_t millis); 41 | 42 | // The max value of the back off period in milliseconds 43 | uint32_t getMaxIntervalMillis() const; 44 | Backoff& setMaxIntervalMillis(const uint32_t millis); 45 | 46 | // Returns the value to multiply the current interval with for each retry 47 | // attempt. 48 | double getMultiplier() const; 49 | Backoff& setMultiplier(const double multiplier); 50 | 51 | // Gets the number of milliseconds to wait before retrying 52 | uint64_t getNextBackOffMillis(); 53 | 54 | // Returns the randomization factor to use for creating a range around the 55 | // retry interval 56 | double getRandomizationFactor() const; 57 | Backoff& setRandomizationFactor(const double factor); 58 | 59 | void reset(); 60 | 61 | private: 62 | uint32_t initialIntervalMillis; 63 | uint32_t maxElapsedTimeMillis; 64 | uint32_t maxIntervalMillis; 65 | double multiplier; 66 | double randomizationFactor; 67 | uint64_t startTime; 68 | uint32_t currentRetryInterval; 69 | }; 70 | 71 | } // end namespace 72 | 73 | #endif 74 | -------------------------------------------------------------------------------- /src/util/mqueue.h: -------------------------------------------------------------------------------- 1 | #ifndef _UTIL_MQUEUE_H 2 | #define _UTIL_MQUEUE_H 3 | 4 | // Implementation taken from 5 | // http://www.soa-world.de/echelon/uploaded_files/lockfreequeue.cpp 6 | // http://www.soa-world.de/echelon/2011/02/a-simple-lock-free-queue-in-c.html 7 | // 8 | // Provides a lock free queue for single producer, single consumer applications 9 | #include 10 | 11 | namespace mckeys { 12 | 13 | // You can't put the implementation for template classes into a cpp file. 14 | // Awesome sauce! 15 | template 16 | class mqueue 17 | { 18 | public: 19 | mqueue() { 20 | first = new Node(T()); 21 | divider = first; 22 | last = first; 23 | } 24 | ~mqueue() { 25 | uint32_t deleted = 0; 26 | while(first != NULL) // release the list 27 | { 28 | Node* tmp = first; 29 | first = tmp->next; 30 | deleted += 1; 31 | delete tmp; 32 | } 33 | } 34 | 35 | void produce(const T& t) { 36 | last->next = new Node(t); // add the new item 37 | asm volatile("" ::: "memory"); // prevend compiler reordering 38 | // gcc atomic, cast to void to prevent "value computed is not used" 39 | (void)__sync_lock_test_and_set(&last, last->next); 40 | while(first != divider) // trim unused nodes 41 | { 42 | Node* tmp = first; 43 | first = first->next; 44 | delete tmp; 45 | } 46 | } 47 | 48 | bool consume(T& result) { 49 | if(divider != last) // if queue is nonempty 50 | { 51 | result = divider->next->value; // C: copy it back 52 | asm volatile("" ::: "memory"); // prevent compiler reordering 53 | // gcc atomic, cast to void to prevent "value computed is not used" 54 | (void)__sync_lock_test_and_set(÷r, divider->next); 55 | return true; // and report success 56 | } 57 | return false; // else report empty 58 | } 59 | 60 | // TODO add count for use in shutdown 61 | 62 | private: 63 | struct Node 64 | { 65 | Node(T val) : value(val), next(NULL) { } 66 | T value; 67 | Node* next; 68 | }; 69 | Node* first; // for producer only 70 | Node* divider; // shared 71 | Node* last; // shared 72 | }; 73 | 74 | } // end namespace 75 | 76 | #endif 77 | -------------------------------------------------------------------------------- /src/logging/logger.h: -------------------------------------------------------------------------------- 1 | #ifndef _LOGGING_LOGGER_H 2 | #define _LOGGING_LOGGER_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "logging/level.h" 12 | #include "logging/record.h" 13 | 14 | #define CONTEXT mckeys::Record(__FILE__, __LINE__, __FUNCTION__) 15 | #define CONTEXT_EX(ex) mckeys::Record(__FILE__, __LINE__, __FUNCTION__, ex) 16 | 17 | #define LOG_WITH_VARARGS(level, rec, fmt, logger) { \ 18 | va_list argp; \ 19 | char buffer[1024]; \ 20 | va_start(argp, fmt); \ 21 | vsnprintf(buffer, 1024, fmt.c_str(), argp); \ 22 | rec.setLevel(level); \ 23 | rec.setLoggerName(logger->getName()); \ 24 | rec.setMessage(buffer); \ 25 | logger->log(level, rec); \ 26 | va_end(argp); \ 27 | } 28 | 29 | namespace mckeys { 30 | 31 | class Logger; 32 | typedef Logger* LoggerPtr; 33 | 34 | class Logger 35 | { 36 | public: 37 | static LoggerPtr getLogger(const std::string &name); 38 | static LoggerPtr getRootLogger(); 39 | 40 | virtual ~Logger(); 41 | 42 | Level getLevel() const; 43 | void setLevel(const Level &level); 44 | 45 | std::string getName() const; 46 | 47 | void setParent(const LoggerPtr &logger); 48 | LoggerPtr getParent() const; 49 | 50 | void setUseParent(const bool use_parent); 51 | bool getUseParent() const; 52 | 53 | void setHandler(std::ostream* handler); 54 | 55 | bool isRootLogger() const; 56 | bool isTrace() const { return getLevel() <= Level::TRACE; } 57 | bool isDebug() const { return getLevel() <= Level::DEBUG; } 58 | bool isInfo() const { return getLevel() <= Level::INFO; } 59 | bool isWarning() const { return getLevel() <= Level::WARNING; } 60 | bool isError() const { return getLevel() <= Level::ERROR; } 61 | bool isFatal() const { return getLevel() <= Level::FATAL; } 62 | 63 | void log(const Level &level, const std::string &msg); 64 | void log(const Level &level, const Record &rec); 65 | 66 | void trace(const std::string &msg); 67 | void trace(Record rec, std::string fmt, ...); 68 | void debug(const std::string &msg); 69 | void debug(Record rec, std::string fmt, ...); 70 | void info(const std::string &msg); 71 | void info(Record rec, std::string fmt, ...); 72 | void warning(const std::string &msg); 73 | void warning(Record rec, std::string fmt, ...); 74 | void error(const std::string &msg); 75 | void error(Record rec, std::string fmt, ...); 76 | void fatal(const std::string &msg); 77 | void fatal(Record rec, std::string fmt, ...); 78 | protected: 79 | Logger(const std::string &name); 80 | std::string format(const Record &rec); 81 | std::ostream& getHandler(); 82 | 83 | private: 84 | const std::string _name; 85 | LoggerPtr _parent; 86 | bool _useParent; 87 | Level _level; 88 | std::mutex _writeMutex; 89 | static std::ostream* _handler; 90 | }; 91 | 92 | typedef std::map Loggers; 93 | 94 | } // end namespace 95 | 96 | #endif 97 | -------------------------------------------------------------------------------- /src/net/packet.cpp: -------------------------------------------------------------------------------- 1 | #include "net/packet.h" 2 | 3 | namespace mckeys { 4 | using namespace std; 5 | 6 | Packet::Packet() : header(), data(NULL), _timestamp(0) 7 | { 8 | memset(&header, 0, sizeof(header)); 9 | } 10 | Packet::Packet(const Header& header, const Data* data) 11 | : header(header), 12 | data(copyData(const_cast(data), header.caplen)), 13 | _timestamp(UtilTime::currentTimeMillis()) 14 | {} 15 | 16 | // Copy constructor 17 | Packet::Packet(const Packet& other) 18 | { 19 | copy(other); 20 | } 21 | // Assignment operator 22 | Packet& Packet::operator=(const Packet& packet) { 23 | if (this != &packet) { 24 | destroy(); 25 | copy(packet); 26 | } 27 | return (*this); 28 | } 29 | // Destructor 30 | Packet::~Packet() { 31 | destroy(); 32 | } 33 | 34 | int32_t Packet::id() const { 35 | return header.ts.tv_sec + header.ts.tv_usec + length() + capLength(); 36 | } 37 | 38 | /////////////////////////////////////////////////////////////////////////////// 39 | // Protected Methods // 40 | /////////////////////////////////////////////////////////////////////////////// 41 | 42 | /** 43 | * This horrible bit of hack has something to do with memory alignment. Using 44 | * memcpy without all the cast business results in a segfault due to 45 | * uninitialized memory. 46 | * @warning This is likely not portable 47 | * @protected 48 | */ 49 | Packet::Data* Packet::copyData(Data* in, const bpf_u_int32 len) { 50 | if (len == 0) { 51 | return NULL; 52 | } 53 | void * d = reinterpret_cast(in); 54 | void * nd = malloc(len); 55 | memset(nd, 0, len); 56 | memcpy(nd, const_cast(d), len); 57 | return reinterpret_cast(nd); 58 | } 59 | 60 | /////////////////////////////////////////////////////////////////////////////// 61 | // Private Methods // 62 | /////////////////////////////////////////////////////////////////////////////// 63 | 64 | /** 65 | * Cast data to a void. 66 | * @private 67 | */ 68 | inline void* Packet::__void_data(Data* d) { 69 | return reinterpret_cast(d); 70 | } 71 | 72 | /** 73 | * Copy the contents of one packet to this packet, respecting the malloc() 74 | * required for a proper copy. 75 | * @param Packet other the packet to copy data from 76 | * @result Nothing, mutates the current packet 77 | * @private 78 | */ 79 | void Packet::copy(const Packet& other) { 80 | header = other.header; 81 | if (other.data == NULL) { 82 | data = NULL; 83 | } else { 84 | data = copyData(other.data, other.capLength()); 85 | } 86 | _timestamp = other._timestamp; 87 | } 88 | 89 | /** 90 | * Destroy the current packet. Free the memory associated with data if needed. 91 | * @private 92 | */ 93 | void Packet::destroy() { 94 | if (data != NULL) { 95 | free(__void_data(data)); 96 | } 97 | } 98 | 99 | } // end namespace 100 | -------------------------------------------------------------------------------- /src/util/stats.h: -------------------------------------------------------------------------------- 1 | #ifndef _UTIL_STATS_H 2 | #define _UTIL_STATS_H 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include "config.h" 12 | #include "util/mqueue.h" 13 | #include "util/stat.h" 14 | #include "util/state.h" 15 | 16 | namespace mckeys { 17 | 18 | typedef enum { 19 | sort_DESC, sort_ASC 20 | } SortOrder; 21 | 22 | typedef enum { 23 | mode_CALLS, mode_SIZE, mode_REQRATE, mode_BANDWIDTH 24 | } SortMode; 25 | 26 | class SortByCount { 27 | public: 28 | bool operator() (const Stat &first, const Stat &second) { 29 | return (first.getCount() > second.getCount()); 30 | } 31 | }; 32 | class SortBySize { 33 | public: 34 | bool operator() (const Stat &first, const Stat &second) { 35 | return (first.getSize() > second.getSize()); 36 | } 37 | }; 38 | class SortByReqRate { 39 | public: 40 | bool operator() (const Stat &first, const Stat &second) { 41 | return (first.requestRate() > second.requestRate()); 42 | } 43 | }; 44 | class SortByBandwidth { 45 | public: 46 | bool operator() (const Stat &first, const Stat &second) { 47 | return (first.bandwidth() > second.bandwidth()); 48 | } 49 | }; 50 | 51 | typedef std::pair Elem; 52 | 53 | // Keep track of a collection of Stat items 54 | class Stats 55 | { 56 | public: 57 | static std::string getSortModeString(const SortMode& sortMode); 58 | static std::string getSortOrderString(const SortOrder& sortOrder); 59 | 60 | Stats(const Config *config, mqueue *mq); 61 | ~Stats(); 62 | 63 | void start(); 64 | void shutdown(); 65 | 66 | void increment(const std::string &key, const uint32_t size); 67 | void printStats(const uint16_t size); 68 | uint32_t getStatCount(); 69 | 70 | // This will want to take a sort mode (what value to sort on) and a sort order 71 | // (asc,desc) as arguments 72 | // FIXME this sucks big time 73 | template 74 | std::deque getLeaders() { 75 | std::priority_queue, T> pq; 76 | std::deque holder; 77 | _mutex.lock(); 78 | for (StatCollection::iterator it = _collection.begin(); 79 | it != _collection.end(); ++it) 80 | { 81 | pq.push(it->second); 82 | } 83 | _mutex.unlock(); 84 | while(!pq.empty()) { 85 | holder.push_front(pq.top()); 86 | pq.pop(); 87 | } 88 | return holder; 89 | } 90 | std::deque getLeaders(const SortMode mode, const SortOrder order); 91 | 92 | protected: 93 | // These are run in threads 94 | void collect(); 95 | void prune(); 96 | 97 | private: 98 | const Config * config; 99 | mqueue * barrier; 100 | StatCollection _collection; 101 | mutable std::mutex _mutex; 102 | LoggerPtr logger; 103 | State state; 104 | std::thread reaper_thread; 105 | std::thread poller_thread; 106 | }; 107 | 108 | } 109 | 110 | #endif 111 | -------------------------------------------------------------------------------- /src/logging/level.cpp: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | #include 3 | #include 4 | 5 | #include "logging/level.h" 6 | #include "util/util.h" 7 | 8 | namespace mckeys { 9 | 10 | using namespace std; 11 | 12 | static ValueMap valueMap; 13 | #define ADD_TO_VALUE_MAP(level) valueMap.insert(ValueMap::value_type(level.getValue(), level)) 14 | 15 | static NameMap nameMap; 16 | #define ADD_TO_NAME_MAP(level) nameMap.insert(NameMap::value_type(level.getName(), level)) 17 | 18 | // protected constructor 19 | Level::Level(const string &name, const uint32_t value) 20 | : name(name), 21 | value(value) 22 | {} 23 | 24 | // static values 25 | Level Level::TRACE = Level("TRACE", 0); 26 | Level Level::DEBUG = Level("DEBUG", 1); 27 | Level Level::INFO = Level("INFO", 2); 28 | Level Level::WARNING = Level("WARNING", 3); 29 | Level Level::ERROR = Level("ERROR", 4); 30 | Level Level::FATAL = Level("FATAL", 5); 31 | Level Level::OFF = Level("OFF", USIZE_BITS(uint32_t)); 32 | Level Level::fromName(const string &name) 33 | { 34 | if (nameMap.empty()) { 35 | ADD_TO_NAME_MAP(Level::TRACE); 36 | ADD_TO_NAME_MAP(Level::DEBUG); 37 | ADD_TO_NAME_MAP(Level::INFO); 38 | ADD_TO_NAME_MAP(Level::WARNING); 39 | ADD_TO_NAME_MAP(Level::ERROR); 40 | ADD_TO_NAME_MAP(Level::FATAL); 41 | ADD_TO_NAME_MAP(Level::OFF); 42 | } 43 | NameMap::iterator it = nameMap.find(name); 44 | if (it != nameMap.end()) { 45 | return it->second; 46 | } else { 47 | throw range_error("No such level with name " + name); 48 | } 49 | } 50 | 51 | Level Level::fromValue(const uint32_t value) 52 | { 53 | if (valueMap.empty()) { 54 | ADD_TO_VALUE_MAP(Level::TRACE); 55 | ADD_TO_VALUE_MAP(Level::DEBUG); 56 | ADD_TO_VALUE_MAP(Level::INFO); 57 | ADD_TO_VALUE_MAP(Level::WARNING); 58 | ADD_TO_VALUE_MAP(Level::ERROR); 59 | ADD_TO_VALUE_MAP(Level::FATAL); 60 | ADD_TO_VALUE_MAP(Level::OFF); 61 | } 62 | ValueMap::iterator it = valueMap.find(value); 63 | if (it != valueMap.end()) { 64 | return it->second; 65 | } else { 66 | throw range_error("No such level with value " + to_string((llui_t)value)); 67 | } 68 | } 69 | 70 | string Level::getName() const 71 | { 72 | return name; 73 | } 74 | uint32_t Level::getValue() const 75 | { 76 | return value; 77 | } 78 | 79 | /* Comparison Operators */ 80 | bool operator==(const Level& lhs, const Level& rhs) { 81 | return (lhs.getValue() == rhs.getValue()); 82 | } 83 | bool operator<(const Level& lhs, const Level& rhs) { 84 | return (lhs.getValue() < rhs.getValue()); 85 | } 86 | bool operator<=(const Level& lhs, const Level& rhs) { 87 | return (lhs.getValue() <= rhs.getValue()); 88 | } 89 | bool operator>(const Level& lhs, const Level& rhs) { 90 | return (lhs.getValue() >= rhs.getValue()); 91 | } 92 | bool operator>=(const Level& lhs, const Level& rhs) { 93 | return (lhs.getValue() >= rhs.getValue()); 94 | } 95 | bool operator!=(const Level& lhs, const Level& rhs) { 96 | return (lhs.getValue() != rhs.getValue()); 97 | } 98 | 99 | 100 | } // end namespace mckeys 101 | -------------------------------------------------------------------------------- /src/logging/record.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | extern "C" { 5 | #include 6 | } 7 | 8 | #include "logging/record.h" 9 | 10 | namespace mckeys { 11 | 12 | using namespace std; 13 | 14 | /** 15 | * Constructor for a new log record. 16 | */ 17 | Record::Record() 18 | : _filename(), 19 | _level(Level::OFF), 20 | _lineNumber(0), 21 | _loggerName(), 22 | _message(), 23 | _methodName() 24 | {} 25 | Record::Record(const string &fname, const uint32_t ln, const string &name) 26 | : _filename(fname), 27 | _level(Level::OFF), 28 | _lineNumber(ln), 29 | _loggerName(), 30 | _message(), 31 | _methodName(name) 32 | {} 33 | Record::Record(const string &fname, const uint32_t ln, const string &name, 34 | const exception &ex) 35 | : _filename(fname), 36 | _level(Level::OFF), 37 | _lineNumber(ln), 38 | _loggerName(), 39 | _message(), 40 | _methodName(name), 41 | _thrownMessage(ex.what()) 42 | {} 43 | 44 | /** 45 | * Manage filename. 46 | */ 47 | string Record::getFileName() const 48 | { 49 | return _filename; 50 | } 51 | void Record::setFileName(const string &filename) 52 | { 53 | _filename = filename; 54 | } 55 | 56 | /** 57 | * Manage level. 58 | */ 59 | Level Record::getLevel() const 60 | { 61 | return _level; 62 | } 63 | void Record::setLevel(const Level &level) 64 | { 65 | _level = level; 66 | } 67 | 68 | /** 69 | * Manage line number. 70 | */ 71 | uint32_t Record::getLineNumber() const 72 | { 73 | return _lineNumber; 74 | } 75 | void Record::setLineNumber(const uint32_t lineNumber) 76 | { 77 | _lineNumber = lineNumber; 78 | } 79 | 80 | /* 81 | * Manage logger name. 82 | */ 83 | string Record::getLoggerName() const 84 | { 85 | return _loggerName; 86 | } 87 | void Record::setLoggerName(const string &name) 88 | { 89 | _loggerName = name; 90 | } 91 | 92 | /** 93 | * Manage the message. 94 | */ 95 | string Record::getMessage() const 96 | { 97 | return _message; 98 | } 99 | void Record::setMessage(const string &msg) 100 | { 101 | _message = msg; 102 | } 103 | 104 | /** 105 | * Manage method name. 106 | */ 107 | string Record::getMethodName() const 108 | { 109 | return _methodName; 110 | } 111 | void Record::setMethodName(const string &name) 112 | { 113 | _methodName = name; 114 | } 115 | 116 | /** 117 | * Manage thrown messages. 118 | * FIXME ran into a weird issue where when we actually set the message things 119 | * got screwed up which is why we only keep the message. 120 | */ 121 | string Record::getThrownMessage() const 122 | { 123 | return _thrownMessage; 124 | } 125 | void Record::setThrownMessage(const std::string &exmsg) 126 | { 127 | _thrownMessage = exmsg; 128 | } 129 | bool Record::hasThrown() const { 130 | return (!_thrownMessage.empty()); 131 | } 132 | 133 | string Record::getTimestamp() const 134 | { 135 | struct timeval tv; 136 | gettimeofday(&tv, NULL); 137 | return Record::getTimestamp(tv); 138 | } 139 | 140 | string Record::getTimestamp(struct timeval tv) const { 141 | struct tm *timeinfo = gmtime(&(tv.tv_sec)); 142 | char buffer[80] = {0}; 143 | 144 | strftime(buffer, 80, "%Y%m%d-%H:%M:%S", timeinfo); 145 | 146 | char result[100] = {0}; 147 | std::snprintf(result, 100, "%s.%03ld", buffer, (long)tv.tv_usec / 1000); 148 | return result; 149 | } 150 | 151 | } // end namespace 152 | -------------------------------------------------------------------------------- /src/util/backoff.cpp: -------------------------------------------------------------------------------- 1 | #include "util/backoff.h" 2 | #include "util/util_time.h" 3 | #include // rand & srand 4 | 5 | namespace mckeys { 6 | 7 | using namespace std; 8 | 9 | // 1.5 which is 50% increase per back off 10 | const double Backoff::DEFAULT_MULTIPLIER = 1.5; 11 | // 0.5 which results in a random period ranging between 50% below and 50% above 12 | // the retry interval 13 | const double Backoff::DEFAULT_RANDOMIZATION_FACTOR = 0.5; 14 | 15 | Backoff::Backoff() 16 | : initialIntervalMillis(Backoff::DEFAULT_INITIAL_INTERVAL_MILLIS), 17 | maxElapsedTimeMillis(Backoff::DEFAULT_MAX_ELAPSED_TIME_MILLIS), 18 | maxIntervalMillis(Backoff::DEFAULT_MAX_INTERVAL_MILLIS), 19 | multiplier(Backoff::DEFAULT_MULTIPLIER), 20 | randomizationFactor(Backoff::DEFAULT_RANDOMIZATION_FACTOR), 21 | currentRetryInterval(0) 22 | { 23 | srand(time(NULL)); 24 | reset(); 25 | } 26 | 27 | Backoff::~Backoff() { 28 | } 29 | 30 | // The current retry interval in milliseconds 31 | uint32_t Backoff::getCurrentIntervalMillis() const { 32 | return currentRetryInterval; 33 | } 34 | 35 | // The elapsed time in ms since the instance was created. 36 | // Reset when reset() is called. 37 | uint64_t Backoff::getElapsedTimeMillis() const { 38 | return UtilTime::currentTimeMillis() - startTime; 39 | } 40 | 41 | // The initial retry interval in milliseconds. 42 | uint32_t Backoff::getInitialIntervalMillis() const { 43 | return initialIntervalMillis; 44 | } 45 | Backoff& Backoff::setInitialIntervalMillis(const uint32_t millis) { 46 | initialIntervalMillis = millis; 47 | return (*this); 48 | } 49 | 50 | // The maximum elapsed time in milliseconds 51 | uint32_t Backoff::getMaxElapsedTimeMillis() const { 52 | return maxElapsedTimeMillis; 53 | } 54 | Backoff& Backoff::setMaxElapsedTimeMillis(const uint32_t millis) { 55 | maxElapsedTimeMillis = millis; 56 | return (*this); 57 | } 58 | 59 | // The max value of the back off period in milliseconds 60 | uint32_t Backoff::getMaxIntervalMillis() const { 61 | return maxIntervalMillis; 62 | } 63 | Backoff& Backoff::setMaxIntervalMillis(const uint32_t millis) { 64 | maxIntervalMillis = millis; 65 | return (*this); 66 | } 67 | 68 | // Returns the value to multiply the current interval with for each retry 69 | // attempt. 70 | double Backoff::getMultiplier() const { 71 | return multiplier; 72 | } 73 | Backoff& Backoff::setMultiplier(const double mult) { 74 | multiplier = mult; 75 | return (*this); 76 | } 77 | 78 | // Gets the number of milliseconds to wait before retrying 79 | uint64_t Backoff::getNextBackOffMillis() { 80 | if (getElapsedTimeMillis() > getMaxElapsedTimeMillis()) { 81 | return currentRetryInterval; 82 | } 83 | uint32_t curint = getCurrentIntervalMillis(); 84 | double delta = getRandomizationFactor() * curint; 85 | double minInterval = curint - delta; 86 | double maxInterval = curint + delta; 87 | double random = (float) rand()/RAND_MAX; 88 | int randInterval = (int)(minInterval + (random * (maxInterval - minInterval + 1))); 89 | if (curint >= (getMaxIntervalMillis() / getMultiplier())) { 90 | currentRetryInterval = getMaxIntervalMillis(); 91 | } else { 92 | currentRetryInterval *= getMultiplier(); 93 | } 94 | return randInterval; 95 | } 96 | 97 | // Returns the randomization factor to use for creating a range around the 98 | // retry interval 99 | double Backoff::getRandomizationFactor() const { 100 | return randomizationFactor; 101 | } 102 | Backoff& Backoff::setRandomizationFactor(const double factor) { 103 | randomizationFactor = factor; 104 | return (*this); 105 | } 106 | 107 | void Backoff::reset() { 108 | startTime = UtilTime::currentTimeMillis(); 109 | currentRetryInterval = getInitialIntervalMillis(); 110 | } 111 | 112 | } // end namespace 113 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # memkeys 2 | 3 | Show your memcache key usage in realtime. 4 | 5 | This was originally inspired by [mctop](https://github.com/etsy/mctop) from etsy. 6 | I found that under load mctop would drop between 50 and 75 percent of packets. 7 | Under the same load memkeys will typically drop less than 3 percent of packets. 8 | This is on a machine saturating a 1Gb network link. 9 | 10 | Build Status: [![Build Status](https://travis-ci.org/bmatheny/memkeys.png?branch=master)](https://travis-ci.org/bmatheny/memkeys) 11 | 12 | ## Command line options 13 | 14 | Usage: memkeys -i NIC [options] 15 | -d, --discard=THRESH Discard keys where req/s rate is below THRESH 16 | -i, --interface=NIC Network interface to capture traffic on (required) 17 | -p, --port=PORT Network port to capture memcache traffic on (default 11211) 18 | -r, --refresh=INTERVAL Refresh the stats display every INTERVAL ms (default 500) 19 | -l, --logfile=FILE Output logs to FILE 20 | -R, --report=REPORT Output data in REPORT format (CSV or curses, default curses) 21 | 22 | -h, --help This help 23 | -v, --verbose Increase verbosity. May be used multiple times. 24 | -V, --version Show program info and exit. 25 | 26 | ## Running 27 | 28 | You will most likely want to run with something like: 29 | 30 | memkeys -i eth0 -l /tmp/memkeys.log 31 | 32 | If you are running memkeys on a very high traffic machine you will want to 33 | specify a discard threshold, otherwise the memory footprint will grow quite 34 | large. 35 | 36 | memkeys -i eth0 -d 10.0 -l /tmp/memkeys.log 37 | 38 | If you are running memkeys on a proxy you may want to use `-i lo` to listen on 39 | localhost. 40 | 41 | ## Screenshot 42 | 43 | ![Screenshot](https://raw.github.com/wiki/bmatheny/memkeys/misc/screenshot.png) 44 | 45 | ## Development/Build 46 | 47 | Build is based on autoconf. 48 | 49 | Install gperftools and gperftools-devel if you want to build with 50 | `--enable-profiling`. You will typically want to configure with 51 | `--enable-debug`, and possibly with `--enable-development`. The latter two 52 | options will enable additional error logging. If you are actually doing 53 | development you should definitely add `--enable-development` as doing so will 54 | add some additional compiler flags to help catch errors. 55 | 56 | You will need libpcap-devel, libpcrecpp, and libncurses-devel. 57 | 58 | On Ubuntu with all packages: 59 | 60 | sudo apt-get install autoconf libpcap-dev libpcre3-dev and lib32ncurses5-dev google-perftools libgoogle-perftools-dev 61 | ./build-eng/autogen.sh 62 | make 63 | make check 64 | 65 | The memkeys should then be present in /usr/local/bin. 66 | 67 | memkeys was developed on CentOS 5.8 with the following software tools: 68 | 69 | GCC version : g++44 (GCC) 4.4.6 20110731 (Red Hat 4.4.6-3) 70 | GNU gmake : 3.81 71 | ld : 2.17.50.0.6-20 72 | coreutils : 5.97 73 | libtool : 2.4.2 74 | autoconf : 2.68 75 | automake : 1.9.6 76 | 77 | The following library versions were used: 78 | 79 | libpcap-devel : 0.9.4-15 80 | pcre-devel : 6.6-6 81 | ncurses-devel : 5.5-24.20060715 82 | 83 | This should build fine against newer versions of the above tools. Different 84 | libraries are untested. 85 | 86 | # License 87 | 88 | Copyright 2013 Tumblr. 89 | 90 | Licensed under the Apache License, Version 2.0 (the "License"); you may not use 91 | this file except in compliance with the License. You may obtain a copy of the 92 | License at http://www.apache.org/licenses/LICENSE-2.0 93 | 94 | Unless required by applicable law or agreed to in writing, software distributed 95 | under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR 96 | CONDITIONS OF ANY KIND, either express or implied. See the License for the 97 | specific language governing permissions and limitations under the License. 98 | -------------------------------------------------------------------------------- /src/net/pcap.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "net/net.h" 5 | #include "util/util.h" 6 | 7 | namespace mckeys { 8 | 9 | using namespace std; 10 | 11 | // protected and static 12 | char Pcap::errorBuffer[PCAP_ERRBUF_SIZE] = {0}; 13 | 14 | Pcap::~Pcap() 15 | { 16 | close(); 17 | logger->debug(CONTEXT, "Deleting logger"); 18 | delete logger; 19 | } 20 | 21 | void Pcap::setFilter(const string &filter) 22 | { 23 | int rc = 0; 24 | struct bpf_program bpf; 25 | if (!state.isStarting()) { 26 | string emsg = "No pcap session open, can't apply filter"; 27 | logger->error(CONTEXT, emsg.c_str()); 28 | throw MemkeysException(emsg); 29 | } 30 | logger->info(CONTEXT, 31 | "Applying filter (%s) to pcap session", filter.c_str()); 32 | rc = pcap_compile(handle, &bpf, 33 | const_cast(filter.c_str()), 34 | true, /*optimize*/ 35 | getSubnetMask()); 36 | if (rc == -1) { 37 | string msg = "Couldn't parse pcap filter " + filter + ": " + getPcapError(); 38 | logger->error(CONTEXT, msg.c_str()); 39 | throw MemkeysException(msg); 40 | } 41 | rc = pcap_setfilter(handle, &bpf); 42 | if (rc == -1) { 43 | string msg = "Couldn't install pcap filter " + filter + ": " + getPcapError(); 44 | logger->error(CONTEXT, msg.c_str()); 45 | throw MemkeysException(msg); 46 | } 47 | pcap_freecode(&bpf); 48 | } 49 | 50 | void Pcap::startCapture(PcapCallback cb, 51 | int cnt /* default to forever */, 52 | u_char *userData) 53 | { 54 | if (!state.isStarting()) { 55 | string msg = "No pcap session available"; 56 | logger->error(CONTEXT, msg.c_str()); 57 | throw MemkeysException(msg); 58 | } 59 | state.setState(state_RUNNING); 60 | // NOTE - With this, a CPU gets slammed. Without this, you drop a shit ton of 61 | // traffic. 62 | if (pcap_setnonblock(handle, true, errorBuffer) < 0) { 63 | logger->error(CONTEXT, "Could not set interface to be non blocking: %s", 64 | errorBuffer); 65 | } 66 | int rc = pcap_loop(handle, cnt, cb, userData); 67 | if (rc == -1 && !(state.isStopping() || state.isTerminated())) { 68 | string msg = "Could not start capture loop: "; 69 | msg.append(getPcapError()); 70 | logger->error(CONTEXT, msg.c_str()); 71 | throw MemkeysException(msg); 72 | } 73 | } 74 | 75 | PcapStats Pcap::getStats() const 76 | { 77 | pcap_stat stat; 78 | PcapStats stats; 79 | memset(&stat, 0, sizeof(stat)); 80 | memset(&stats, 0, sizeof(stats)); 81 | if (state.isRunning()) { 82 | pcap_stats(handle, &stat); 83 | } 84 | stats.received = stat.ps_recv; 85 | stats.dropped = stat.ps_drop; 86 | stats.if_dropped = stat.ps_ifdrop; 87 | stats.drop_pct = 100.0 * (stats.dropped / (double)(stats.received + 1.0)); 88 | return stats; 89 | } 90 | 91 | string Pcap::getStatsString() const 92 | { 93 | ostringstream statout; 94 | PcapStats stats = getStats(); 95 | statout << "packets (recv/dropped): "; 96 | statout << stats.received << " / "; 97 | statout << stats.dropped << " ("; 98 | statout << std::fixed << std::setprecision(2) << stats.drop_pct; 99 | statout << "%)"; 100 | return statout.str(); 101 | } 102 | 103 | void Pcap::stopCapture() 104 | { 105 | if (handle != NULL && state.checkAndSet(state_RUNNING, state_STOPPING)) { 106 | logger->debug(CONTEXT, "Stopping capture loop"); 107 | pcap_breakloop(handle); 108 | } 109 | } 110 | 111 | void Pcap::close() 112 | { 113 | if (handle != NULL && state.checkAndSet(state_STOPPING, state_TERMINATED)) { 114 | logger->info(CONTEXT, "Closing pcap session"); 115 | // This is important. The ordering of shutdown MUST be: 116 | // start loop with pcap_loop 117 | // call pcap_breakloop 118 | // return from pcap_loop 119 | // call pcap_close 120 | pcap_close(handle); 121 | handle = NULL; 122 | } 123 | } 124 | 125 | // protected 126 | Pcap::Pcap() 127 | : handle(NULL), 128 | logger(Logger::getLogger("pcap")), 129 | state() 130 | {} 131 | 132 | std::string Pcap::getPcapError() const 133 | { 134 | if (handle == NULL) { 135 | return "No pcap session (session handle is NULL)"; 136 | } 137 | char * err = pcap_geterr(handle); 138 | if (err == NULL) { 139 | return "No pcap error (pcap_geterr returned NULL)"; 140 | } 141 | return err; 142 | } 143 | 144 | } // end namespace 145 | -------------------------------------------------------------------------------- /src/net/memcache_command.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | #include 5 | 6 | #include "net/net.h" 7 | 8 | extern "C" { 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | } 15 | 16 | static inline std::string ipv4addressToString(const void * src) { 17 | char ip[INET_ADDRSTRLEN]; 18 | inet_ntop(AF_INET, src, ip, INET_ADDRSTRLEN); 19 | return std::string(ip); 20 | } 21 | 22 | namespace mckeys { 23 | 24 | using namespace std; 25 | 26 | // Like getInstance. Used for creating commands from packets. 27 | MemcacheCommand MemcacheCommand::create(const Packet& pkt, 28 | const bpf_u_int32 captureAddress) 29 | { 30 | static ssize_t ether_header_sz = sizeof(struct ether_header); 31 | static ssize_t ip_sz = sizeof(struct ip); 32 | static ssize_t tcphdr_sz = sizeof(struct tcphdr); 33 | 34 | const struct ether_header* ethernetHeader; 35 | const struct ip* ipHeader; 36 | const struct tcphdr* tcpHeader; 37 | 38 | const Packet::Header* pkthdr = &pkt.getHeader(); 39 | const Packet::Data* packet = pkt.getData(); 40 | 41 | bool possible_request = false; 42 | u_char *data; 43 | uint32_t dataLength = 0; 44 | 45 | string sourceAddress = ""; 46 | 47 | // must be an IP packet 48 | // TODO add support for dumping localhost 49 | ethernetHeader = (struct ether_header*)packet; 50 | auto etype = ntohs(ethernetHeader->ether_type); 51 | if (etype != ETHERTYPE_IP) { 52 | return MemcacheCommand(); 53 | } 54 | 55 | // must be TCP - TODO add support for UDP 56 | ipHeader = (struct ip*)(packet + ether_header_sz); 57 | auto itype = ipHeader->ip_p; 58 | if (itype != IPPROTO_TCP) { 59 | return MemcacheCommand(); 60 | } 61 | sourceAddress = ipv4addressToString(&(ipHeader->ip_src)); 62 | 63 | // The packet was destined for our capture address, this is a request 64 | // This bit of optimization lets us ignore a reasonably large percentage of 65 | // traffic 66 | if (ipHeader->ip_dst.s_addr == captureAddress) { 67 | possible_request = true; 68 | } 69 | // FIXME will remove once we add back the direction parsing 70 | (void)possible_request; 71 | 72 | tcpHeader = (struct tcphdr*)(packet + ether_header_sz + ip_sz); 73 | (void)tcpHeader; 74 | data = (u_char*)(packet + ether_header_sz + ip_sz + tcphdr_sz); 75 | dataLength = pkthdr->len - (ether_header_sz + ip_sz + tcphdr_sz); 76 | if (dataLength > pkthdr->caplen) { 77 | dataLength = pkthdr->caplen; 78 | } 79 | 80 | // TODO revert to detecting request/response and doing the right thing 81 | return MemcacheCommand::makeResponse(data, dataLength, sourceAddress); 82 | } 83 | 84 | // protected default constructor 85 | MemcacheCommand::MemcacheCommand() 86 | : cmdType_(MC_UNKNOWN), 87 | sourceAddress_(), 88 | commandName_(), 89 | objectKey_(), 90 | objectSize_(0) 91 | {} 92 | 93 | // protected constructor 94 | MemcacheCommand::MemcacheCommand(const memcache_command_t cmdType, 95 | const string sourceAddress, 96 | const string commandName, 97 | const string objectKey, 98 | uint32_t objectSize) 99 | : cmdType_(cmdType), 100 | sourceAddress_(sourceAddress), 101 | commandName_(commandName), 102 | objectKey_(objectKey), 103 | objectSize_(objectSize) 104 | {} 105 | 106 | // static protected 107 | MemcacheCommand MemcacheCommand::makeRequest(u_char*, int, string) 108 | { 109 | // don't care about requests right now 110 | return MemcacheCommand(); 111 | } 112 | 113 | // static protected 114 | MemcacheCommand MemcacheCommand::makeResponse(u_char *data, int length, 115 | string sourceAddress) 116 | { 117 | static pcrecpp::RE re("VALUE (\\S+) \\d+ (\\d+)", 118 | pcrecpp::RE_Options(PCRE_MULTILINE)); 119 | string key; 120 | int size = -1; 121 | string input = ""; 122 | for (int i = 0; i < length; i++) { 123 | int cid = (int)data[i]; 124 | if (isprint(cid) || cid == 10 || cid == 13) { 125 | input += (char)data[i]; 126 | } 127 | } 128 | if (input.length() < 11) { 129 | return MemcacheCommand(); 130 | } 131 | re.PartialMatch(input, &key, &size); 132 | if (size >= 0) { 133 | return MemcacheCommand(MC_RESPONSE, sourceAddress, "", key, size); 134 | } else { 135 | return MemcacheCommand(); 136 | } 137 | } 138 | 139 | } // end namespace 140 | -------------------------------------------------------------------------------- /src/cli.cpp: -------------------------------------------------------------------------------- 1 | /** 2 | * Copyright 2013 Blake Matheny 3 | */ 4 | extern "C" { 5 | #include 6 | } 7 | 8 | #include 9 | #include 10 | #include "common.h" 11 | #include "cli.h" 12 | 13 | namespace mckeys { 14 | 15 | static const struct option longopts[] = { 16 | {"discard", required_argument, 0, 'd'}, 17 | {"interface", required_argument, 0, 'i'}, 18 | {"port", required_argument, 0, 'p'}, 19 | {"refresh", required_argument, 0, 'r'}, 20 | {"help", no_argument, 0, 'h'}, 21 | {"verbose", no_argument, 0, 'v'}, 22 | {"version", no_argument, 0, 'V'}, 23 | {"logfile", required_argument, 0, 'l'}, 24 | {"report", required_argument, 0, 'R'} 25 | }; 26 | static const char * argp = "d:i:l:p:r:R:hvV"; 27 | 28 | using std::cout; using std::endl; 29 | using std::string; using std::ostringstream; 30 | 31 | string Cli::help(const char * progname) { 32 | ostringstream txt; 33 | string pname = progname; 34 | string tab = " "; 35 | txt << "Usage: " << pname << " -i NIC [options]" << endl; 36 | txt << mkHelpDoc(longopts[0], 37 | "Discard keys where req/s rate is below THRESH", "THRESH"); 38 | txt << mkHelpDoc(longopts[1], 39 | "Network interface to capture traffic on (required)", "NIC"); 40 | txt << mkHelpDoc( 41 | longopts[2], 42 | "Network port to capture memcache traffic on (default 11211)", 43 | "PORT"); 44 | txt << mkHelpDoc( 45 | longopts[3], 46 | "Refresh the stats display every INTERVAL ms (default 500)", 47 | "INTERVAL"); 48 | txt << mkHelpDoc(longopts[7], "Output logs to FILE", "FILE"); 49 | txt << mkHelpDoc( 50 | longopts[8], 51 | "Output data in REPORT format (CSV or curses, default curses)", 52 | "REPORT"); 53 | txt << endl; 54 | txt << mkHelpDoc(longopts[4], "This help", ""); 55 | txt << mkHelpDoc(longopts[5], 56 | "Increase verbosity. May be used multiple times.", ""); 57 | txt << mkHelpDoc(longopts[6], "Show program info and exit.", ""); 58 | return txt.str(); 59 | } 60 | 61 | /** 62 | * Parse the command line arguments, updating a config as appropriate. 63 | */ 64 | void Cli::parse(int argc, char ** argv, Config * cfg) { 65 | int c; 66 | char * progname = argv[0]; 67 | while (1) { 68 | int option_index = 0; 69 | c = getopt_long(argc, argv, argp, longopts, &option_index); 70 | if (c == -1) 71 | break; 72 | switch (c) { 73 | case 'd': 74 | cfg->setDiscardThreshold(::atof(optarg)); 75 | break; 76 | case 'i': 77 | cfg->setInterface(optarg); 78 | break; 79 | case 'l': 80 | cfg->setLogfile(optarg); 81 | break; 82 | case 'p': 83 | cfg->setPort(::atoi(optarg)); 84 | break; 85 | case 'r': 86 | cfg->setRefreshInterval(::atoi(optarg)); 87 | break; 88 | case 'R': 89 | cfg->setReportType(optarg); 90 | break; 91 | case 'h': 92 | cout << Cli::help(progname); 93 | exit(EXIT_SUCCESS); 94 | case 'v': 95 | cfg->increaseVerbosity(); 96 | break; 97 | case 'V': 98 | cout << "Package: " << PACKAGE_STRING << endl; 99 | cout << "Author : " << PACKAGE_BUGREPORT << endl; 100 | cout << "License: Apache 2.0" << endl; 101 | exit(EXIT_SUCCESS); 102 | case '?': 103 | cout << "Unknown flag specified" << endl; 104 | cout << Cli::help(progname); 105 | exit(EXIT_FAILURE); 106 | default: 107 | cout << "Unknown flag specified" << endl; 108 | cout << Cli::help(progname); 109 | exit(EXIT_FAILURE); 110 | } 111 | } 112 | } 113 | 114 | string Cli::mkHelpLead(const struct option opt, const string &key) { 115 | ostringstream txt; 116 | static const char c = 0; 117 | ssize_t len = 0, alloc = 33; 118 | char * os = NULL; 119 | txt << " -" << static_cast(opt.val) << ", --" << opt.name; 120 | if (opt.has_arg == 1) { 121 | txt << "=" << key; 122 | } else if (opt.has_arg == 2) { 123 | txt << "[=" << key << "]"; 124 | } 125 | len = txt.str().length(); 126 | alloc += len; 127 | os = reinterpret_cast(malloc(sizeof(c)*alloc)); 128 | snprintf(os, alloc, "%-32s", txt.str().c_str()); 129 | return string(os); 130 | } 131 | 132 | string Cli::mkHelpDoc(const struct option opt, const string &desc, 133 | const string &key) { 134 | ostringstream txt; 135 | string lead = mkHelpLead(opt, key); 136 | txt << lead << desc << endl; 137 | return txt.str(); 138 | } 139 | 140 | } // namespace mckeys 141 | -------------------------------------------------------------------------------- /src/config.cpp: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | #include 3 | #include 4 | 5 | namespace mckeys { 6 | 7 | static Config * instance = NULL; 8 | 9 | using namespace std; 10 | 11 | // static 12 | Config * Config::getInstance() 13 | { 14 | if (instance == NULL) 15 | instance = new Config(); 16 | 17 | return instance; 18 | } 19 | 20 | Config::~Config() 21 | { 22 | logger->trace(CONTEXT, "Deleting logger"); 23 | delete logger; 24 | instance = NULL; 25 | } 26 | 27 | /** 28 | * Set the discard threshold for stats. When displaying cache keys, only keys 29 | * being requested at or above this rate will be displayed. 30 | */ 31 | void Config::setDiscardThreshold(const double threshold) 32 | { 33 | if (threshold < 0.0) { 34 | throw range_error("threshold must be >= 0"); 35 | } 36 | logger->debug(CONTEXT, "Setting discard threshold to %.02f", threshold); 37 | discardThreshold = threshold; 38 | } 39 | double Config::getDiscardThreshold() const 40 | { 41 | return discardThreshold; 42 | } 43 | 44 | void Config::setInterface(const string &value) 45 | { 46 | logger->debug(CONTEXT, "Setting interface to %s", value.c_str()); 47 | interface = value; 48 | } 49 | string Config::getInterface() const 50 | { 51 | return interface; 52 | } 53 | 54 | void Config::setLogfile(const string &value) { 55 | ofstream* ofs = new ofstream(value.c_str(), std::ofstream::out); 56 | if (ofs->fail()) { 57 | delete ofs; 58 | throw MemkeysConfigurationError("Can't open " + value + " for writing"); 59 | } 60 | logfile = value; 61 | logger->setHandler(ofs); 62 | logger->info(CONTEXT, "Now logging to %s", value.c_str()); 63 | } 64 | string Config::getLogfile() const 65 | { 66 | return logfile; 67 | } 68 | 69 | /** 70 | * Set the port to use for listening. 71 | */ 72 | void Config::setPort(const int _port) 73 | { 74 | REQUIRE_UINT("port", _port, uint16_t); 75 | logger->debug(CONTEXT, "Setting port to %d", _port); 76 | port = (uint16_t)_port; 77 | } 78 | uint16_t Config::getPort() const 79 | { 80 | return port; 81 | } 82 | string Config::getPortAsString() const 83 | { 84 | return to_string((llui_t)getPort()); 85 | } 86 | 87 | /** 88 | * Set the rate (in MS) at which the UI is refreshed. 89 | */ 90 | void Config::setRefreshInterval(const int interval) 91 | { 92 | REQUIRE_UINT("refreshInterval", interval, uint16_t); 93 | refreshInterval = (uint16_t)interval; 94 | } 95 | uint16_t Config::getRefreshInterval() const 96 | { 97 | return refreshInterval; 98 | } 99 | 100 | /** 101 | * Which report should be used by default? 102 | */ 103 | void Config::setReportType(const std::string &value) { 104 | reportType = ReportType::fromString(value); 105 | } 106 | ReportType Config::getReportType() const { 107 | return reportType; 108 | } 109 | 110 | /** 111 | * Manage how verbose we get. 112 | */ 113 | void Config::makeLessVerbose() 114 | { 115 | Level level = logger->getLevel(); 116 | if (level >= Level::FATAL) { 117 | logger->warning(CONTEXT, "Log level already at or above FATAL"); 118 | return; 119 | } 120 | adjustLoggerLevel(Level::fromValue(level.getValue() + 1)); 121 | } 122 | void Config::increaseVerbosity() 123 | { 124 | Level level = logger->getLevel(); 125 | if (level <= Level::TRACE) { 126 | logger->warning(CONTEXT, "Log level already at or below TRACE"); 127 | return; 128 | } 129 | adjustLoggerLevel(Level::fromValue(level.getValue() - 1)); 130 | } 131 | Level Config::verbosity() const 132 | { 133 | return logger->getLevel(); 134 | } 135 | 136 | string Config::toString() const 137 | { 138 | ostringstream configs; 139 | string lfile; 140 | if (logfile.empty()) { 141 | lfile = "stdout"; 142 | } else { 143 | lfile = logfile; 144 | } 145 | configs << setw(20) << "Discard Threshold"; 146 | configs << ": " << getDiscardThreshold() << endl; 147 | configs << setw(20) << "Interface"; 148 | configs << ": " << getInterface() << endl; 149 | configs << setw(20) << "Port"; 150 | configs << ": " << getPort() << endl; 151 | configs << setw(20) << "Refresh Interval"; 152 | configs << ": " << getRefreshInterval() << "ms" << endl; 153 | configs << setw(20) << "Verbosity"; 154 | configs << ": " << verbosity().getName() << endl; 155 | configs << setw(20) << "Logfile"; 156 | configs << ": " << lfile; 157 | return configs.str(); 158 | } 159 | 160 | // private constructor 161 | Config::Config() 162 | : discardThreshold(0.0), 163 | interface(""), 164 | _isPromiscuous(true), 165 | port(11211), 166 | _readTimeout(1000), 167 | refreshInterval(500), 168 | _snapLength(1518), 169 | logger(Logger::getLogger("config")), 170 | logfile(), 171 | reportType(ReportType::NCURSES) 172 | {} 173 | 174 | void Config::adjustLoggerLevel(const Level &newLevel) 175 | { 176 | logger->setLevel(newLevel); 177 | Logger::getRootLogger()->setLevel(newLevel); 178 | } 179 | 180 | } // end namespace 181 | -------------------------------------------------------------------------------- /src/memkeys.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include 4 | 5 | #include "cli.h" 6 | #include "memkeys.h" 7 | 8 | extern "C" { // need for signal handling 9 | #include 10 | #include 11 | #include 12 | } 13 | 14 | using namespace std; 15 | 16 | namespace mckeys { 17 | 18 | // forward declarations 19 | static void process(u_char *userData, const struct pcap_pkthdr* pkthdr, const u_char* packet); 20 | static void signal_cb(int signum); 21 | 22 | // Compilation unit values 23 | static Memkeys * instance = NULL; 24 | 25 | // Static initializer 26 | Memkeys * Memkeys::getInstance(const Config * config) 27 | { 28 | if (instance != NULL) { 29 | return instance; 30 | } 31 | if (config->getInterface().empty()) { 32 | throw MemkeysConfigurationError("No interface was specified"); 33 | } 34 | instance = new Memkeys(config); 35 | // FIXME? This should account for offline vs live 36 | instance->session = new PcapLive(config); 37 | instance->engine = new CaptureEngine(config, instance->session); 38 | return instance; 39 | } 40 | // Used for initializing app main() 41 | Memkeys * Memkeys::getInstance(int argc, char ** argv) 42 | { 43 | Config * cfg = Config::getInstance(); 44 | LoggerPtr mainLogger = Logger::getLogger("main"); 45 | try { 46 | Cli::parse(argc, argv, cfg); 47 | } catch (const exception &ex) { 48 | throw MemkeysConfigurationError(ex.what()); 49 | } 50 | mainLogger->setLevel(Level::INFO); 51 | mainLogger->info(CONTEXT, 52 | "Starting application %s. PID %d", argv[0], getpid()); 53 | Logger::getRootLogger()->setLevel(cfg->verbosity()); 54 | mainLogger->debug("Configuration\n" + cfg->toString()); 55 | return Memkeys::getInstance(cfg); 56 | } 57 | 58 | // Destructor 59 | Memkeys::~Memkeys() 60 | { 61 | if (session != NULL) { 62 | logger->trace(CONTEXT, "Deleting pcap session"); 63 | delete session; 64 | } 65 | if (engine != NULL) { 66 | logger->trace(CONTEXT, "Deleting capture engine"); 67 | delete engine; 68 | } 69 | delete logger; 70 | } 71 | 72 | // Run the capture loop 73 | void Memkeys::run() 74 | { 75 | signal(SIGINT, signal_cb); 76 | state.setState(state_STARTING); 77 | session->open(); 78 | logger->debug("My address: " + to_string((llsi_t)session->getIpAddress())); 79 | session->setFilter(string("tcp port ") + config->getPortAsString()); 80 | try { 81 | state.setState(state_RUNNING); 82 | session->startCapture(process, -1, (u_char*)engine); 83 | logger->info(CONTEXT, "Finished packet capture"); 84 | } catch (...) { 85 | logger->error(CONTEXT, "There was an unexpected error capturing data"); 86 | } 87 | state.setState(state_TERMINATED); 88 | } 89 | 90 | // Call engine and capture shutdown 91 | void Memkeys::tryShutdown() 92 | { 93 | if (state.checkAndSet(state_RUNNING, state_STOPPING)) { 94 | if (engine != NULL) { 95 | logger->info(engine->getStatsString()); 96 | } 97 | 98 | if (engine != NULL && !engine->isShutdown()) { 99 | logger->info(CONTEXT, "Shutting down engine"); 100 | engine->shutdown(); 101 | } 102 | 103 | if (session != NULL) { 104 | logger->info(CONTEXT, "Stopping packet capture"); 105 | session->stopCapture(); 106 | } 107 | } else { 108 | logger->info(CONTEXT, "Shutdown already called"); 109 | } 110 | } 111 | 112 | void Memkeys::forceShutdown() 113 | { 114 | if (session != NULL && state.isStopping()) { 115 | logger->warning(CONTEXT, "Forcibly closing capture session"); 116 | session->close(); 117 | } 118 | } 119 | 120 | // protected, see getInstance 121 | Memkeys::Memkeys(const Config * config) 122 | : config(config), 123 | logger(Logger::getLogger("memkeys")), 124 | session(NULL), 125 | engine(NULL), 126 | state() 127 | { } 128 | 129 | // Process packets as they come in 130 | // Make sure this avoids blocking and stays as fast as possible 131 | // Any real work needs to be delegated to a thread 132 | static void process(u_char *userData, const struct pcap_pkthdr* header, 133 | const u_char* packet) 134 | { 135 | // FIXME just make this a global static to avoid the weird casting issues 136 | static CaptureEngine * ce = (CaptureEngine*)userData; 137 | static Backoff backoff; // used for not killing the logger 138 | 139 | if (ce->isShutdown()) { 140 | uint64_t backoffMs = backoff.getNextBackOffMillis(); 141 | struct timespec waitTime = UtilTime::millisToTimespec(backoffMs); 142 | nanosleep(&waitTime, NULL); 143 | return; 144 | } 145 | 146 | Packet p(*header, packet); 147 | ce->enqueue(p); 148 | } 149 | 150 | // Signal handler for handling shutdowns 151 | static void signal_cb(int signum) 152 | { 153 | if (instance == NULL || instance->isShutdown()) { 154 | return; 155 | } 156 | LoggerPtr logger = Logger::getLogger("capture-engine"); 157 | // alarm means timeout has expired 158 | if (signum == SIGALRM) { 159 | logger->warning(CONTEXT, "Alarm expired, forcing shutdown"); 160 | instance->forceShutdown(); 161 | } else if (instance->isRunning()) { 162 | logger->info(CONTEXT, "Shutting down due to signal"); 163 | instance->tryShutdown(); 164 | // if we haven't shut down in 2 seconds, alarm and force it 165 | signal(SIGALRM, signal_cb); 166 | alarm(2); 167 | } else { 168 | logger->warning(CONTEXT, "Ignoring signal, already shutting down: %s", 169 | instance->getStateName().c_str()); 170 | } 171 | } 172 | 173 | } // end namespace 174 | -------------------------------------------------------------------------------- /configure.ac: -------------------------------------------------------------------------------- 1 | # Initialize autoconf 2 | AC_INIT([memkeys], [0.2], [bmatheny@mobocracy.net]) 3 | AC_PREREQ([2.63]) 4 | AC_CONFIG_SRCDIR([src/main.cpp]) 5 | AC_CONFIG_MACRO_DIR([m4]) 6 | AC_CONFIG_AUX_DIR([build-aux]) 7 | AC_CONFIG_HEADERS([build-aux/mconfig.h:build-aux/mconfig.h.in]) 8 | 9 | # Initialize automake 10 | AM_INIT_AUTOMAKE([1.9.6 foreign]) 11 | 12 | # Checks for language 13 | AC_LANG([C++]) 14 | 15 | # Checks for programs 16 | CXXFLAGS="$CXXFLAGS -std=c++0x" # Will fail if not g++44 or later 17 | AC_PROG_CXX([g++44 g++ gcc cxx cc++ c++]) 18 | AC_PROG_CXXCPP 19 | CXXCPP="$CXXCPP -std=c++0x" # This must come after AC_PROG_CXXCPP 20 | AC_PROG_LIBTOOL 21 | 22 | # Check for typedefs, structures, and compiler characteristics 23 | AC_TYPE_UINT16_T 24 | AC_TYPE_UINT32_T 25 | AC_TYPE_UINT64_T 26 | 27 | # Platform specific setup 28 | AC_CANONICAL_HOST 29 | case "$host" in 30 | *linux*) 31 | # getconf LEVEL1_DCACHE_LINESIZE 32 | AC_DEFINE([linux], 1, [Linux]) 33 | ;; 34 | darwin*) 35 | # sysctl -a hw.cachelinesize 36 | AC_DEFINE([__APPLE__], 1, [Apple Hardware]) 37 | ;; 38 | esac 39 | 40 | 41 | # Check for headers 42 | AC_CHECK_HEADERS([getopt.h]) 43 | AC_CHECK_HEADERS([pcrecpp.h]) 44 | AC_CHECK_HEADERS([cstdatomic]) 45 | AC_CHECK_HEADERS([atomic]) 46 | 47 | # Checks for libraries 48 | AC_CHECK_LIB([pthread], [pthread_create]) 49 | AC_CHECK_LIB([getopt],[getopt_long]) 50 | AC_CHECK_LIB([gnugetopt],[getopt_long]) 51 | 52 | PCRECPP="" 53 | AC_ARG_WITH([libpcrecpp], 54 | AS_HELP_STRING([--with-libpcrecpp=DIR], [libpcre base directory]), 55 | [with_libpcrecpp="$withval"], 56 | [with_libpcrecpp="no"]) 57 | if test "x$with_libpcrecpp" != "xno"; then 58 | LDFLAGS="${LDFLAGS} -L${with_libpcrecpp}/lib" 59 | CPPFLAGS="${CPPFLAGS} -I${with_libpcrecpp}/include" 60 | fi 61 | # Fail if pcrecpp is not found 62 | AC_CHECK_LIB([pcrecpp], [main],, [AC_MSG_ERROR([libpcrecpp not found but required])]) 63 | 64 | AC_ARG_WITH(libpcap_includes, 65 | AS_HELP_STRING([--with-libpcap-includes=DIR], [libpcap include directory]), 66 | [with_libpcap_includes="$withval"], 67 | [with_libpcap_includes="no"]) 68 | AC_ARG_WITH(libpcap_libraries, 69 | AS_HELP_STRING([--with-libpcap-libraries=DIR], [libpcap library directory]), 70 | [with_libpcap_libraries="$withval"], 71 | [with_libpcap_libraries="no"]) 72 | if test "x$with_libpcap_includes" != "xno"; then 73 | CPPFLAGS="${CPPFLAGS} -I${with_libpcap_includes}" 74 | fi 75 | if test "x$with_libpcap_libraries" != "xno"; then 76 | LDFLAGS="${LDFLAGS} -L${with_libpcap_libraries}" 77 | fi 78 | LPCAP="" 79 | AC_CHECK_LIB(pcap, pcap_datalink,, LPCAP="no") 80 | if test "x$LPCAP" = "xno"; then 81 | echo 82 | echo " ERROR! Libpcap library/headers (libpcap.a (or .so)/pcap.h)" 83 | echo " not found, go get it from http://www.tcpdump.org" 84 | echo " or use the --with-libpcap-* options, if you have it installed" 85 | echo " in unusual place. Also check if your libpcap depends on another" 86 | echo " shared library that may be installed in an unusual place" 87 | exit 1 88 | fi 89 | 90 | CURSES="" 91 | AC_CHECK_LIB(ncurses, initscr,, CURSES="no") 92 | AC_CHECK_HEADERS([ncurses.h]) 93 | if test "x$CURSES" = "xno"; then 94 | echo 95 | echo " ERROR! ncurses library/headers (libncurses.a (or .so)/ncurses.h)" 96 | echo " not found." 97 | exit 1 98 | fi 99 | 100 | AC_MSG_CHECKING(whether to create a profiled build) 101 | AC_ARG_ENABLE(profiling, 102 | AS_HELP_STRING([--enable-profiling], [Turn on profiling (default=no)]), 103 | [ case "${enableval}" in 104 | yes) 105 | AC_MSG_RESULT(yes) 106 | AC_CHECK_LIB([profiler], [ProfilerStop]) 107 | ;; 108 | *) 109 | AC_MSG_RESULT(no) 110 | ;; 111 | esac], 112 | [AC_MSG_RESULT(no)]) 113 | 114 | # Enable debugging 115 | AC_MSG_CHECKING(whether to create a debug build) 116 | AC_ARG_ENABLE(debug, 117 | AS_HELP_STRING([--enable-debug], [Turn on debugging (default=no)]), 118 | [ case "${enableval}" in 119 | yes) 120 | AC_MSG_RESULT(yes) 121 | # Created in mconfig.h for use as #ifdef DEBUG 122 | AC_DEFINE([DEBUGGING], 1, [Defined if you are debugging]) 123 | debug=true 124 | ;; 125 | no) 126 | AC_MSG_RESULT(no) 127 | debug=false 128 | ;; 129 | *) 130 | AC_MSG_ERROR([bad value ${enableval} for --enable-debug]) 131 | ;; 132 | esac], 133 | [ 134 | AC_MSG_RESULT(no) 135 | debug=false 136 | ]) 137 | # For use in Makefile.am as if DEBUG 138 | AM_CONDITIONAL([DEBUGGING], [test x$debug = xtrue]) 139 | 140 | # Enable development 141 | AC_MSG_CHECKING(whether to create a development build) 142 | AC_ARG_ENABLE(development, 143 | AS_HELP_STRING([--enable-development], [Turn on development (default=no)]), 144 | [ case "${enableval}" in 145 | yes) 146 | AC_MSG_RESULT(yes) 147 | # Created in mconfig.h for use as #ifdef DEVELOPMENT 148 | AC_DEFINE([DEVELOPMENT], 1, [Defined if you are development]) 149 | development=true 150 | ;; 151 | no) 152 | AC_MSG_RESULT(no) 153 | development=false 154 | ;; 155 | *) 156 | AC_MSG_ERROR([bad value ${enableval} for --enable-development]) 157 | ;; 158 | esac], 159 | [ 160 | AC_MSG_RESULT(no) 161 | development=false 162 | ]) 163 | # For use in Makefile.am as if DEVELOPMENT 164 | AM_CONDITIONAL([DEVELOPMENT], [test x$development = xtrue]) 165 | 166 | # Check functions 167 | AC_CHECK_FUNCS([getopt_long], , 168 | [AC_MSG_ERROR([no getopt_long function found])]) 169 | 170 | # Build gtest 171 | export CFLAGS 172 | export CXXFLAGS 173 | AC_CONFIG_SUBDIRS([gtest]) 174 | 175 | AC_CONFIG_FILES([Makefile 176 | src/Makefile]) 177 | AC_OUTPUT 178 | -------------------------------------------------------------------------------- /src/net/capture_engine.cpp: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | #include "net/net.h" 3 | #include "report/report.h" 4 | #include "report/curses.h" 5 | 6 | #include 7 | 8 | namespace mckeys { 9 | 10 | using namespace std; 11 | 12 | CaptureEngine::CaptureEngine(const Config * config, const Pcap * session) 13 | : logger(Logger::getLogger("capture-engine")), 14 | config(config), 15 | session(session), 16 | barrier(new mqueue()), 17 | stats(new Stats(config, barrier)), 18 | report(config->getReportType().makeReport(config, session, stats)), 19 | _is_terminated(false), 20 | queue_count(3), 21 | packets(), 22 | worker_threads(new thread[queue_count]), 23 | barrier_lock() 24 | { 25 | packets.reserve(queue_count); 26 | for (int i = 0; i < queue_count; i++) { 27 | packets.insert(packets.begin() + i, new mqueue()); 28 | worker_threads[i] = thread(&CaptureEngine::processPackets, this, i, packets.at(i)); 29 | } 30 | stats->start(); 31 | } 32 | 33 | CaptureEngine::~CaptureEngine() 34 | { 35 | if (isShutdown()) { 36 | logger->info(CONTEXT, "Capture engine successfully shut down"); 37 | } else { 38 | logger->error(CONTEXT, "Capture engine not successfully shut down"); 39 | } 40 | delete report; 41 | delete stats; 42 | delete barrier; 43 | // FIXME we should wrap these joins in a timer 44 | for (int i = 0; i < queue_count; i++) { 45 | worker_threads[i].join(); 46 | logger->info(CONTEXT, "Worker thread %d dead", i); 47 | } 48 | while (!packets.empty()) { 49 | delete packets.back(), packets.pop_back(); 50 | } 51 | delete[] worker_threads; 52 | delete logger; 53 | } 54 | 55 | void CaptureEngine::enqueue(const Packet& packet) { 56 | #ifdef _DEVEL 57 | logger->trace(CONTEXT, 58 | "Produced packet: %ld", packet.id()); 59 | #endif 60 | packets.at(packet.id() % queue_count)->produce(packet); 61 | } 62 | 63 | bool CaptureEngine::isShutdown() const 64 | { 65 | return (_is_terminated == true); 66 | } 67 | 68 | void CaptureEngine::shutdown() 69 | { 70 | _is_terminated = true; 71 | stats->shutdown(); 72 | report->shutdown(); 73 | } 74 | 75 | /////////////////////////////////////////////////////////////////////////////// 76 | // Protected Methods // 77 | /////////////////////////////////////////////////////////////////////////////// 78 | 79 | /** 80 | * Process packets found in the packet queue. 81 | * This method pulls packets off of the packets queue, parses them into memcache 82 | * commands, and enqueues them via CaptureEngine::enqueue(MemcacheCommand) if 83 | * the parsed command is a response. 84 | * 85 | * This method is run in a thread and continues running until isShutdown() 86 | * returns true. 87 | * 88 | * @TODO create several worker threads to process packets in parallel 89 | * @protected 90 | */ 91 | void CaptureEngine::processPackets(int worker_id, mqueue* work_queue) { 92 | static int64_t pktCount = 0; 93 | static llui_t resCount = 0; 94 | static bool isDebug = logger->isDebug(); 95 | 96 | Backoff backoff; 97 | backoff.setMaxElapsedTimeMillis(10); 98 | backoff.setMaxIntervalMillis(10); 99 | uint64_t backoffMs = 0; 100 | struct timespec waitTime; 101 | 102 | logger->info(CONTEXT, "Worker %d starting capture processing", worker_id); 103 | 104 | while(!isShutdown()) { 105 | Packet packet; 106 | if (work_queue->consume(packet)) { 107 | pktCount += 1; 108 | if (backoffMs > 0) { 109 | backoffMs = 0; 110 | backoff.reset(); 111 | } 112 | #ifdef _DEVEL 113 | logger->trace(CONTEXT, 114 | "worker %d Consumed packet %ld", worker_id, packet.id()); 115 | #endif 116 | MemcacheCommand mc = parse(packet); 117 | if (mc.isResponse()) { 118 | enqueue(mc); 119 | resCount += 1; 120 | #ifdef _DEBUG 121 | logger->trace(CONTEXT, 122 | "worker %d, packet %ld, key %s", worker_id, packet.id(), 123 | mc.getObjectKey().c_str()); 124 | #endif 125 | } else { 126 | #ifdef _DEVEL 127 | logger->trace(CONTEXT, "worker %d not a memcache response", worker_id); 128 | #endif 129 | } 130 | if ((pktCount % 10000) == 0 && isDebug) { 131 | string out = getStatsString(); 132 | uint64_t now = UtilTime::currentTimeMillis(); 133 | llui_t tdiff = now - packet.timestamp(); 134 | out.append(", memcache replies = "); 135 | out.append(to_string(resCount)); 136 | out.append(", time diff = "); 137 | out.append(to_string(tdiff)); 138 | logger->debug(out); 139 | } 140 | } else { 141 | backoffMs = backoff.getNextBackOffMillis(); 142 | waitTime = UtilTime::millisToTimespec(backoffMs); 143 | #ifdef _DEVEL 144 | logger->trace(CONTEXT, 145 | "worker %d no packet to consume, will sleep %lu ms", 146 | worker_id, backoffMs); 147 | #endif 148 | } 149 | if (backoffMs > 0) { 150 | nanosleep(&waitTime, NULL); 151 | } 152 | } 153 | logger->info(CONTEXT, "Worker %d stopped processing packets", worker_id); 154 | } 155 | 156 | /** 157 | * Parse a packet into a memcache command. 158 | * @param Packet packet the packet to parse. 159 | * @return MemcacheCommand the command representation of the packet 160 | * @protected 161 | */ 162 | MemcacheCommand CaptureEngine::parse(const Packet& packet) const 163 | { 164 | static const auto address = getIpAddress(); 165 | return MemcacheCommand::create(packet, address); 166 | } 167 | 168 | /** 169 | * Enqueue a parsed memcache command for stats processing. 170 | * @param MemcacheCommand mc the command to enqueue 171 | * @protected 172 | */ 173 | void CaptureEngine::enqueue(const MemcacheCommand& mc) 174 | { 175 | Elem e(mc.getObjectKey(), mc.getObjectSize()); 176 | #ifdef _DEBUG 177 | logger->trace(CONTEXT, 178 | "Produced stat: %s, %d", e.first.c_str(), e.second); 179 | #endif 180 | barrier_lock.lock(); 181 | barrier->produce(e); 182 | barrier_lock.unlock(); 183 | } 184 | 185 | } // end namespace 186 | -------------------------------------------------------------------------------- /src/logging/logger.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "logging/logger.h" 5 | 6 | namespace mckeys { 7 | 8 | using namespace std; 9 | 10 | /** 11 | * Interal to this logging implementations. 12 | */ 13 | static Loggers loggers; 14 | static string ROOT_LOGGER_NAME = ""; 15 | 16 | /** 17 | * Static methods. This is how you get a handle on a logger. 18 | */ 19 | ostream* Logger::_handler = NULL; 20 | 21 | LoggerPtr Logger::getLogger(const string &name) 22 | { 23 | Loggers::iterator it = loggers.find(name); 24 | if (it != loggers.end()) { 25 | return it->second; 26 | } else { 27 | LoggerPtr logger = new Logger(name); 28 | if (!logger->isRootLogger()) { 29 | LoggerPtr root = Logger::getRootLogger(); 30 | logger->setParent(root); 31 | logger->setUseParent(true); 32 | logger->setLevel(root->getLevel()); 33 | } 34 | logger->trace("Created logger"); 35 | loggers.insert(it, Loggers::value_type(name, logger)); 36 | return logger; 37 | } 38 | } 39 | LoggerPtr Logger::getRootLogger() 40 | { 41 | Loggers::iterator it = loggers.find(ROOT_LOGGER_NAME); 42 | if (it != loggers.end()) { 43 | return it->second; 44 | } else { 45 | LoggerPtr logger = new Logger(ROOT_LOGGER_NAME); 46 | logger->setParent(NULL); 47 | logger->setUseParent(false); 48 | loggers.insert(it, Loggers::value_type(ROOT_LOGGER_NAME, logger)); 49 | return logger; 50 | } 51 | } 52 | 53 | /** 54 | * Destructor. Delete logger from the map. 55 | */ 56 | Logger::~Logger() { 57 | trace("~Logger destroyed"); 58 | // FIXME need to close handler 59 | Loggers::iterator it = loggers.find(getName()); 60 | if (it != loggers.end()) { 61 | loggers.erase(it); 62 | } 63 | } 64 | 65 | /** 66 | * Manage the logging.Level associated with this logger. 67 | */ 68 | Level Logger::getLevel() const 69 | { 70 | return _level; 71 | } 72 | void Logger::setLevel(const Level &level) 73 | { 74 | trace(CONTEXT, "Changing log level from %s to %s", 75 | _level.getName().c_str(), level.getName().c_str()); 76 | _level = level; 77 | } 78 | 79 | /** 80 | * The name associated with this logger. 81 | */ 82 | string Logger::getName() const 83 | { 84 | return _name; 85 | } 86 | 87 | /** 88 | * Manage the parent of this logger. 89 | */ 90 | void Logger::setParent(const LoggerPtr &logger) 91 | { 92 | _parent = logger; 93 | } 94 | LoggerPtr Logger::getParent() const 95 | { 96 | return _parent; 97 | } 98 | 99 | /** 100 | * Whether or not to use the parent logger for logging. 101 | */ 102 | void Logger::setUseParent(const bool use_parent) 103 | { 104 | _useParent = use_parent; 105 | } 106 | bool Logger::getUseParent() const 107 | { 108 | return _useParent; 109 | } 110 | 111 | void Logger::setHandler(std::ostream* handler) { 112 | _handler = handler; 113 | } 114 | 115 | /** 116 | * True if this logger is the root logger, false otherwise. 117 | */ 118 | bool Logger::isRootLogger() const 119 | { 120 | return (getName() == ROOT_LOGGER_NAME); 121 | } 122 | 123 | /** 124 | * Logging for various levels. 125 | */ 126 | void Logger::log(const Level &level, const string &msg) 127 | { 128 | if (level >= getLevel()) { 129 | Record rec = Record(); 130 | rec.setLevel(level); 131 | rec.setLoggerName(getName()); 132 | rec.setMessage(msg); 133 | log(level, rec); 134 | } 135 | } 136 | void Logger::log(const Level &level, const Record &record) 137 | { 138 | if (level >= getLevel()) { 139 | // TODO this should support writing via an appender so users can log to a 140 | // file while seeing stats on their display 141 | string out = format(record); 142 | _writeMutex.lock(); 143 | getHandler() << out << endl; 144 | getHandler().flush(); 145 | _writeMutex.unlock(); 146 | LoggerPtr logger = getParent(); 147 | if (logger != NULL && logger->getUseParent()) { 148 | logger->log(level, record); 149 | } 150 | } 151 | } 152 | 153 | void Logger::trace(const string &msg) 154 | { 155 | log(Level::TRACE, msg); 156 | } 157 | void Logger::trace(Record record, string fmt, ...) 158 | { 159 | LOG_WITH_VARARGS(Level::TRACE, record, fmt, this); 160 | } 161 | 162 | void Logger::debug(const string &msg) 163 | { 164 | log(Level::DEBUG, msg); 165 | } 166 | void Logger::debug(Record record, string fmt, ...) 167 | { 168 | LOG_WITH_VARARGS(Level::DEBUG, record, fmt, this); 169 | } 170 | 171 | void Logger::info(const string &msg) 172 | { 173 | log(Level::INFO, msg); 174 | } 175 | void Logger::info(Record record, string fmt, ...) 176 | { 177 | LOG_WITH_VARARGS(Level::INFO, record, fmt, this); 178 | } 179 | 180 | void Logger::warning(const string &msg) 181 | { 182 | log(Level::WARNING, msg); 183 | } 184 | void Logger::warning(Record record, string fmt, ...) 185 | { 186 | LOG_WITH_VARARGS(Level::WARNING, record, fmt, this); 187 | } 188 | 189 | void Logger::error(const string &msg) 190 | { 191 | log(Level::ERROR, msg); 192 | } 193 | void Logger::error(Record record, string fmt, ...) 194 | { 195 | LOG_WITH_VARARGS(Level::ERROR, record, fmt, this); 196 | } 197 | 198 | void Logger::fatal(const string &msg) 199 | { 200 | log(Level::FATAL, msg); 201 | } 202 | void Logger::fatal(Record record, string fmt, ...) 203 | { 204 | LOG_WITH_VARARGS(Level::FATAL, record, fmt, this); 205 | } 206 | 207 | // protected 208 | Logger::Logger(const string &name) 209 | : _name(name), 210 | _level(Level::WARNING), 211 | _writeMutex() 212 | { 213 | if (_handler == NULL) { 214 | _handler = &cout; 215 | } 216 | } 217 | 218 | // TODO make this configurable 219 | string Logger::format(const Record &rec) 220 | { 221 | ostringstream out; 222 | out << rec.getLevel().getName() << " "; 223 | out << "[" << rec.getTimestamp() << "] "; 224 | if (!rec.getMethodName().empty()) { 225 | out << "[" << rec.getFileName() << ":" << rec.getLineNumber() << "]["; 226 | out << rec.getMethodName() << "] "; 227 | } 228 | if (rec.hasThrown()) { 229 | out << "[exception(" << rec.getThrownMessage() << ")] "; 230 | } 231 | if (isRootLogger()) { 232 | out << "(root): "; 233 | } else { 234 | out << rec.getLoggerName() << ": "; 235 | } 236 | out << rec.getMessage(); 237 | return out.str(); 238 | } 239 | 240 | ostream& Logger::getHandler() { 241 | return (*_handler); 242 | } 243 | 244 | } // end namespace 245 | -------------------------------------------------------------------------------- /src/util/stats.cpp: -------------------------------------------------------------------------------- 1 | #include "common.h" 2 | #include 3 | 4 | namespace mckeys { 5 | 6 | using namespace std; 7 | 8 | // Static methods 9 | string Stats::getSortOrderString(const SortOrder& sortOrder) { 10 | if (sortOrder == sort_ASC) { 11 | return "asc"; 12 | } else { 13 | return "desc"; 14 | } 15 | } 16 | string Stats::getSortModeString(const SortMode& sortMode) { 17 | switch(sortMode) { 18 | case mode_REQRATE: 19 | return "reqrate"; 20 | case mode_CALLS: 21 | return "calls"; 22 | case mode_SIZE: 23 | return "size"; 24 | default: 25 | return "bw"; 26 | } 27 | } 28 | 29 | Stats::Stats(const Config *config, mqueue *mq) 30 | : config(config), 31 | barrier(mq), 32 | _collection(), 33 | _mutex(), 34 | logger(Logger::getLogger("stats")), 35 | state() 36 | {} 37 | Stats::~Stats() { 38 | if (state.checkAndSet(state_STOPPING, state_TERMINATED)) { 39 | logger->info(CONTEXT, "Stats successfully shut down"); 40 | } else { 41 | logger->error(CONTEXT, "Stats not successfully shut down"); 42 | } 43 | delete logger; 44 | } 45 | 46 | void Stats::start() { 47 | if (state.checkAndSet(state_NEW, state_RUNNING)) { 48 | logger->info(CONTEXT, "Starting stats engine"); 49 | reaper_thread = thread(&Stats::prune, this); 50 | poller_thread = thread(&Stats::collect, this); 51 | } else { 52 | logger->warning(CONTEXT, "Stats engine already started"); 53 | } 54 | } 55 | 56 | void Stats::shutdown() { 57 | if (state.checkAndSet(state_RUNNING, state_STOPPING)) { 58 | // FIXME we should wrap these joins in a timer 59 | logger->info(CONTEXT, "Stopping stats engine"); 60 | reaper_thread.join(); 61 | logger->info(CONTEXT, "Repear thread dead"); 62 | poller_thread.join(); 63 | logger->info(CONTEXT, "Poller thread dead"); 64 | } else { 65 | logger->warning(CONTEXT, "Stats engine already stopping"); 66 | } 67 | } 68 | 69 | void Stats::increment(const string &key, const uint32_t size) { 70 | ssize_t _key = Stat::hashKey(key); 71 | _mutex.lock(); 72 | StatCollection::iterator it = _collection.find(_key); 73 | if (it != _collection.end()) { 74 | Stat stat = it->second; 75 | // FIXME this should probably only be done periodically, not every time 76 | // since it is unlikely to change very often 77 | stat.setSize(size); 78 | stat.increment(); 79 | #ifdef _DEBUG 80 | uint64_t ssize = stat.getCount(); 81 | if (ssize >= 2) { 82 | logger->trace(CONTEXT, 83 | "Incremented stat: %s, %d -> %ld", key.c_str(), size, ssize); 84 | } 85 | #endif 86 | _collection[_key] = stat; 87 | } else { 88 | _collection.insert(it, StatCollection::value_type(_key, Stat(key, size))); 89 | } 90 | _mutex.unlock(); 91 | } 92 | 93 | deque Stats::getLeaders(const SortMode mode, const SortOrder order) { 94 | deque holder; 95 | switch (mode) { 96 | case mode_CALLS: 97 | holder = getLeaders(); 98 | break; 99 | case mode_SIZE: 100 | holder = getLeaders(); 101 | break; 102 | case mode_REQRATE: 103 | holder = getLeaders(); 104 | break; 105 | case mode_BANDWIDTH: 106 | holder = getLeaders(); 107 | break; 108 | } 109 | if (order == sort_ASC) { 110 | reverse(holder.begin(), holder.end()); 111 | } 112 | return holder; 113 | } 114 | 115 | void Stats::printStats(const uint16_t size) { 116 | deque q = getLeaders(); 117 | uint32_t qsize = q.size(); 118 | uint32_t i = 0; 119 | if (qsize > 0) { 120 | cout << setw(110) << "Key" << ", "; 121 | cout << setw(10) << "Count" << ", "; 122 | cout << setw(10) << "Elapsed" << ", "; 123 | cout << setw(10) << "Rate" << ", "; 124 | cout << setw(10) << "Size" << ", "; 125 | cout << setw(10) << "BW" << endl; 126 | } 127 | for (deque::iterator it = q.begin(); it != q.end() && i < size; ++it, ++i) { 128 | Stat stat = *it; 129 | cout << setw(110) << stat.getKey() << ", "; 130 | cout << setw(10) << stat.getCount() << ", "; 131 | cout << setw(10) << stat.elapsed() << ", "; 132 | cout << setw(10) << std::setprecision(2) << stat.requestRate() << ", "; 133 | cout << setw(10) << stat.getSize() << ", "; 134 | cout << setw(10) << std::setprecision(2) << stat.bandwidth() << endl; 135 | } 136 | } 137 | 138 | uint32_t Stats::getStatCount() { 139 | return _collection.size(); 140 | } 141 | 142 | /////////////////////////////////////////////////////////////////////////////// 143 | // Protected Methods // 144 | /////////////////////////////////////////////////////////////////////////////// 145 | void Stats::collect() { 146 | static Backoff backoff; 147 | static uint64_t backoffMs; 148 | static struct timespec waitTime; 149 | logger->info(CONTEXT, "Starting stats collection"); 150 | Elem e; 151 | while (state.isRunning()) { 152 | if (barrier->consume(e)) { // got one 153 | #ifdef _DEBUG 154 | logger->trace(CONTEXT, 155 | "Consumed stat: %s, %d", e.first.c_str(), e.second); 156 | #endif 157 | increment(e.first, e.second); 158 | if (backoffMs > 0) { 159 | backoffMs = 0; 160 | backoff.reset(); 161 | } 162 | } else { // did not get one 163 | backoffMs = backoff.getNextBackOffMillis(); 164 | waitTime = UtilTime::millisToTimespec(backoffMs); 165 | #ifdef _DEVEL 166 | logger->trace(CONTEXT, 167 | "No stat to consume, will sleep %lu ms", backoffMs); 168 | #endif 169 | } 170 | if (backoffMs > 0) { 171 | nanosleep(&waitTime, NULL); 172 | } 173 | } 174 | logger->info(CONTEXT, "Stats collect thread stopped"); 175 | } 176 | 177 | void Stats::prune() { 178 | static const double threshold = config->getDiscardThreshold(); 179 | int size_pre = 0, size_post = 0; 180 | StatCollection::iterator it; 181 | logger->info(CONTEXT, "Starting prune with threshold %0.2f", threshold); 182 | // don't do work if we don't need to 183 | if (threshold == 0.0) { 184 | while(state.isRunning()) { 185 | sleep(1); 186 | } 187 | return; 188 | } 189 | while (state.isRunning()) { 190 | _mutex.lock(); 191 | it = _collection.begin(); 192 | size_pre = _collection.size(); 193 | while (it != _collection.end()) { 194 | Stat stat = it->second; 195 | if (stat.requestRate() < threshold) { 196 | _collection.erase(it++); 197 | } else { 198 | ++it; 199 | } 200 | } 201 | _collection.rehash(0); 202 | size_post = _collection.size(); 203 | logger->debug(CONTEXT, 204 | "Stats collection size: %d -> %d", size_pre, size_post); 205 | _mutex.unlock(); 206 | sleep(5); 207 | } 208 | logger->info(CONTEXT, "Stats prune thread stopped"); 209 | } 210 | 211 | } // end namespace 212 | -------------------------------------------------------------------------------- /src/report/curses.cpp: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | 4 | #include "common.h" 5 | #include "report/curses.h" 6 | 7 | extern "C" { 8 | #include 9 | } 10 | 11 | namespace mckeys { 12 | 13 | using namespace std; 14 | 15 | CursesReport::CursesReport(const Config* cfg, const Pcap* session, Stats* stats) 16 | : Report(cfg, Logger::getLogger("cursesReport")), 17 | session(session), 18 | stats(stats), 19 | statColumnWidth(10), 20 | keyColumnWidth(0), 21 | sortOrder(sort_DESC), 22 | sortMode(mode_REQRATE) 23 | { 24 | vector vec = {"calls", "objsize", "req/sec", "bw(kbps)"}; 25 | statColumns.reserve(vec.size()); 26 | for (uint32_t i = 0; i < vec.size(); i++) { 27 | ostringstream oss; 28 | oss << setw(statColumnWidth) << vec.at(i); 29 | statColumns.push_back(oss.str()); 30 | } 31 | cmdMap = { 32 | {'B', "sort by bandwidth"}, 33 | {'C', "sort by calls"}, 34 | {'Q', "quit"}, 35 | {'R', "sort by req/sec"}, 36 | {'S', "sort by size"}, 37 | {'T', "toggle sort order (asc|desc)"} 38 | }; 39 | if (state.checkAndSet(state_NEW, state_STARTING)) { 40 | report_thread = thread(&CursesReport::render, this); 41 | } else { 42 | logger->warning(CONTEXT, "Incorrect API usage"); 43 | } 44 | } 45 | 46 | // Tear down curses interface 47 | CursesReport::~CursesReport() { 48 | renderDone(); 49 | } 50 | 51 | void CursesReport::render() 52 | { 53 | uint64_t render_start_time = 0, render_duration = 0; 54 | struct timespec ts = UtilTime::millisToTimespec(config->getRefreshInterval()); 55 | string render_string; 56 | 57 | if (!state.checkAndSet(state_STARTING, state_RUNNING)) { 58 | logger->error(CONTEXT, "render already started"); 59 | return; 60 | } 61 | renderInit(); 62 | 63 | while(state.isRunning()) { 64 | render_start_time = UtilTime::currentTimeMillis(); 65 | renderHeader(); 66 | renderFooter(); 67 | deque q = stats->getLeaders(sortMode, sortOrder); 68 | uint32_t qsize = q.size(); 69 | logger->debug(CONTEXT, "Rendering report with %u data points", qsize); 70 | if (qsize > 0) { 71 | renderStats(q, qsize); 72 | } 73 | render_duration = UtilTime::currentTimeMillis() - render_start_time; 74 | renderTimeStat(render_duration); 75 | refresh(); 76 | char c = getch(); 77 | handleKeyPress(c); 78 | nanosleep(&ts, NULL); 79 | } 80 | } 81 | 82 | /////////////////////////////////////////////////////////////////////////////// 83 | // Protected Methods // 84 | /////////////////////////////////////////////////////////////////////////////// 85 | void CursesReport::setpos(int y, int x) const { 86 | move(y, x); 87 | } 88 | 89 | void CursesReport::renderHeader() { 90 | ostringstream header; 91 | uint32_t colSize = statColumns.size(); 92 | 93 | keyColumnWidth = COLS - (colSize * statColumnWidth); 94 | attrset(COLOR_PAIR(1)); 95 | setpos(0, 0); 96 | 97 | header << left << setw(keyColumnWidth) << "memcache key"; 98 | for (uint32_t i = 0; i < statColumns.size(); i++) { 99 | header << statColumns.at(i); 100 | } 101 | addstr(header.str().c_str()); 102 | } 103 | 104 | void CursesReport::renderFooter() { 105 | ostringstream footer; 106 | setpos(LINES - 1, 0); 107 | attrset(COLOR_PAIR(2)); 108 | footer << left << setw(COLS) << footerText; 109 | addstr(footer.str().c_str()); 110 | } 111 | 112 | void CursesReport::renderTimeStat(const uint64_t render_time) const { 113 | attrset(COLOR_PAIR(2)); 114 | setpos(LINES-2, COLS-18); 115 | addstr(createRenderTime(render_time).c_str()); 116 | } 117 | 118 | void CursesReport::renderStats(deque q, uint32_t qsize) { 119 | uint32_t i = 0; 120 | uint32_t maxlines = LINES - 3 - 1; 121 | 122 | string pstats = session->getStatsString(); 123 | ostringstream summary; 124 | ostringstream fwSummary; 125 | 126 | setpos(LINES-2, 0); 127 | attrset(COLOR_PAIR(2)); 128 | 129 | string smd = "sort mode: "; 130 | smd.append(Stats::getSortModeString(sortMode)); 131 | smd.append(" ("); 132 | smd.append(Stats::getSortOrderString(sortOrder)); 133 | smd.append(")"); 134 | 135 | summary << left << setw(28) << smd; 136 | summary << "keys: " << setw(14) << stats->getStatCount(); 137 | summary << setw(40) << pstats; 138 | 139 | fwSummary << left << setw(COLS) << summary.str(); 140 | 141 | addstr(fwSummary.str().c_str()); 142 | attrset(COLOR_PAIR(0)); 143 | 144 | for (deque::iterator it = q.begin(); it != q.end() && i < maxlines; ++it, ++i) { 145 | string line; 146 | if (i < qsize) { 147 | Stat stat = *it; 148 | line = createStatLine(stat); 149 | } else { 150 | line.assign(COLS, ' '); 151 | } 152 | setpos(i+1, 0); 153 | addstr(line.c_str()); 154 | } 155 | for (; i < maxlines; i++) { 156 | string line; 157 | line.assign(COLS, ' '); 158 | setpos(i+1, 0); 159 | addstr(line.c_str()); 160 | } 161 | } 162 | 163 | string CursesReport::createRenderTime(const uint64_t rtime) const { 164 | ostringstream tmp; 165 | tmp << "rt: " << setw(8) << fixed << setprecision(3) << rtime << " (ms)"; 166 | return tmp.str(); 167 | } 168 | 169 | string CursesReport::createStatLine(const Stat& stat) const { 170 | ostringstream tmpStat; 171 | 172 | // handle key 173 | string displayKey = stat.getKey(); 174 | if (displayKey.size() > keyColumnWidth) { 175 | displayKey = displayKey.substr(0, keyColumnWidth - 4) + "..."; 176 | } 177 | tmpStat << left << setw(keyColumnWidth) << displayKey; 178 | 179 | // handle int values 180 | vector ints = {stat.getCount(), stat.getSize()}; 181 | for (uint32_t i = 0; i < ints.size(); i++) { 182 | ostringstream dtmp; 183 | dtmp << setw(statColumnWidth) << ints.at(i); 184 | tmpStat << dtmp.str(); 185 | } 186 | 187 | // handle double values 188 | vector doubles = {stat.requestRate(), stat.bandwidth()}; 189 | for (uint32_t i = 0; i < doubles.size(); i++) { 190 | ostringstream dtmp; 191 | dtmp << setw(statColumnWidth) << fixed << setprecision(2) << doubles.at(i); 192 | tmpStat << dtmp.str(); 193 | } 194 | return tmpStat.str(); 195 | } 196 | 197 | void CursesReport::handleKeyPress(char key) { 198 | switch(key) { 199 | case 'B': 200 | case 'b': 201 | sortMode = mode_BANDWIDTH; 202 | break; 203 | case 'C': 204 | case 'c': 205 | sortMode = mode_CALLS; 206 | break; 207 | case 'Q': 208 | case 'q': 209 | raise(SIGINT); 210 | break; 211 | case 'R': 212 | case 'r': 213 | sortMode = mode_REQRATE; 214 | break; 215 | case 'S': 216 | case 's': 217 | sortMode = mode_SIZE; 218 | break; 219 | case 'T': 220 | case 't': 221 | if (sortOrder == sort_DESC) { 222 | sortOrder = sort_ASC; 223 | } else { 224 | sortOrder = sort_DESC; 225 | } 226 | break; 227 | default: 228 | break; 229 | } 230 | } 231 | 232 | /////////////////////////////////////////////////////////////////////////////// 233 | // Private Methods // 234 | /////////////////////////////////////////////////////////////////////////////// 235 | void CursesReport::initFooterText() { 236 | ostringstream footer; 237 | vector tmp; 238 | 239 | // dump command map into tmp as key:value pairs 240 | for (CommandMap::iterator it = cmdMap.begin(); it != cmdMap.end(); ++it) 241 | { 242 | string stmp; 243 | stmp += it->first; 244 | stmp += ":"; 245 | stmp += it->second; 246 | tmp.push_back(stmp); 247 | } 248 | // use ostream_iterator to join each elem in tmp with a pipe 249 | copy(tmp.begin(), tmp.end() - 1, ostream_iterator(footer, " | ")); 250 | // we left off the last elem so append it 251 | footer << *tmp.rbegin(); 252 | footerText = footer.str(); 253 | } 254 | 255 | void CursesReport::renderInit() { 256 | initscr(); 257 | cbreak(); 258 | curs_set(0); 259 | timeout(0); 260 | if (can_change_color()) { 261 | start_color(); 262 | init_pair(0, COLOR_WHITE, COLOR_BLACK); 263 | init_pair(1, COLOR_WHITE, COLOR_BLUE); 264 | init_pair(2, COLOR_WHITE, COLOR_RED); 265 | } 266 | initFooterText(); 267 | } 268 | 269 | void CursesReport::renderDone() { 270 | nocbreak(); 271 | endwin(); 272 | } 273 | 274 | } // end namespace 275 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Apache License 2 | Version 2.0, January 2004 3 | http://www.apache.org/licenses/ 4 | 5 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 6 | 7 | 1. Definitions. 8 | 9 | "License" shall mean the terms and conditions for use, reproduction, 10 | and distribution as defined by Sections 1 through 9 of this document. 11 | 12 | "Licensor" shall mean the copyright owner or entity authorized by 13 | the copyright owner that is granting the License. 14 | 15 | "Legal Entity" shall mean the union of the acting entity and all 16 | other entities that control, are controlled by, or are under common 17 | control with that entity. For the purposes of this definition, 18 | "control" means (i) the power, direct or indirect, to cause the 19 | direction or management of such entity, whether by contract or 20 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 21 | outstanding shares, or (iii) beneficial ownership of such entity. 22 | 23 | "You" (or "Your") shall mean an individual or Legal Entity 24 | exercising permissions granted by this License. 25 | 26 | "Source" form shall mean the preferred form for making modifications, 27 | including but not limited to software source code, documentation 28 | source, and configuration files. 29 | 30 | "Object" form shall mean any form resulting from mechanical 31 | transformation or translation of a Source form, including but 32 | not limited to compiled object code, generated documentation, 33 | and conversions to other media types. 34 | 35 | "Work" shall mean the work of authorship, whether in Source or 36 | Object form, made available under the License, as indicated by a 37 | copyright notice that is included in or attached to the work 38 | (an example is provided in the Appendix below). 39 | 40 | "Derivative Works" shall mean any work, whether in Source or Object 41 | form, that is based on (or derived from) the Work and for which the 42 | editorial revisions, annotations, elaborations, or other modifications 43 | represent, as a whole, an original work of authorship. For the purposes 44 | of this License, Derivative Works shall not include works that remain 45 | separable from, or merely link (or bind by name) to the interfaces of, 46 | the Work and Derivative Works thereof. 47 | 48 | "Contribution" shall mean any work of authorship, including 49 | the original version of the Work and any modifications or additions 50 | to that Work or Derivative Works thereof, that is intentionally 51 | submitted to Licensor for inclusion in the Work by the copyright owner 52 | or by an individual or Legal Entity authorized to submit on behalf of 53 | the copyright owner. For the purposes of this definition, "submitted" 54 | means any form of electronic, verbal, or written communication sent 55 | to the Licensor or its representatives, including but not limited to 56 | communication on electronic mailing lists, source code control systems, 57 | and issue tracking systems that are managed by, or on behalf of, the 58 | Licensor for the purpose of discussing and improving the Work, but 59 | excluding communication that is conspicuously marked or otherwise 60 | designated in writing by the copyright owner as "Not a Contribution." 61 | 62 | "Contributor" shall mean Licensor and any individual or Legal Entity 63 | on behalf of whom a Contribution has been received by Licensor and 64 | subsequently incorporated within the Work. 65 | 66 | 2. Grant of Copyright License. Subject to the terms and conditions of 67 | this License, each Contributor hereby grants to You a perpetual, 68 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 69 | copyright license to reproduce, prepare Derivative Works of, 70 | publicly display, publicly perform, sublicense, and distribute the 71 | Work and such Derivative Works in Source or Object form. 72 | 73 | 3. Grant of Patent License. Subject to the terms and conditions of 74 | this License, each Contributor hereby grants to You a perpetual, 75 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 76 | (except as stated in this section) patent license to make, have made, 77 | use, offer to sell, sell, import, and otherwise transfer the Work, 78 | where such license applies only to those patent claims licensable 79 | by such Contributor that are necessarily infringed by their 80 | Contribution(s) alone or by combination of their Contribution(s) 81 | with the Work to which such Contribution(s) was submitted. If You 82 | institute patent litigation against any entity (including a 83 | cross-claim or counterclaim in a lawsuit) alleging that the Work 84 | or a Contribution incorporated within the Work constitutes direct 85 | or contributory patent infringement, then any patent licenses 86 | granted to You under this License for that Work shall terminate 87 | as of the date such litigation is filed. 88 | 89 | 4. Redistribution. You may reproduce and distribute copies of the 90 | Work or Derivative Works thereof in any medium, with or without 91 | modifications, and in Source or Object form, provided that You 92 | meet the following conditions: 93 | 94 | (a) You must give any other recipients of the Work or 95 | Derivative Works a copy of this License; and 96 | 97 | (b) You must cause any modified files to carry prominent notices 98 | stating that You changed the files; and 99 | 100 | (c) You must retain, in the Source form of any Derivative Works 101 | that You distribute, all copyright, patent, trademark, and 102 | attribution notices from the Source form of the Work, 103 | excluding those notices that do not pertain to any part of 104 | the Derivative Works; and 105 | 106 | (d) If the Work includes a "NOTICE" text file as part of its 107 | distribution, then any Derivative Works that You distribute must 108 | include a readable copy of the attribution notices contained 109 | within such NOTICE file, excluding those notices that do not 110 | pertain to any part of the Derivative Works, in at least one 111 | of the following places: within a NOTICE text file distributed 112 | as part of the Derivative Works; within the Source form or 113 | documentation, if provided along with the Derivative Works; or, 114 | within a display generated by the Derivative Works, if and 115 | wherever such third-party notices normally appear. The contents 116 | of the NOTICE file are for informational purposes only and 117 | do not modify the License. You may add Your own attribution 118 | notices within Derivative Works that You distribute, alongside 119 | or as an addendum to the NOTICE text from the Work, provided 120 | that such additional attribution notices cannot be construed 121 | as modifying the License. 122 | 123 | You may add Your own copyright statement to Your modifications and 124 | may provide additional or different license terms and conditions 125 | for use, reproduction, or distribution of Your modifications, or 126 | for any such Derivative Works as a whole, provided Your use, 127 | reproduction, and distribution of the Work otherwise complies with 128 | the conditions stated in this License. 129 | 130 | 5. Submission of Contributions. Unless You explicitly state otherwise, 131 | any Contribution intentionally submitted for inclusion in the Work 132 | by You to the Licensor shall be under the terms and conditions of 133 | this License, without any additional terms or conditions. 134 | Notwithstanding the above, nothing herein shall supersede or modify 135 | the terms of any separate license agreement you may have executed 136 | with Licensor regarding such Contributions. 137 | 138 | 6. Trademarks. This License does not grant permission to use the trade 139 | names, trademarks, service marks, or product names of the Licensor, 140 | except as required for reasonable and customary use in describing the 141 | origin of the Work and reproducing the content of the NOTICE file. 142 | 143 | 7. Disclaimer of Warranty. Unless required by applicable law or 144 | agreed to in writing, Licensor provides the Work (and each 145 | Contributor provides its Contributions) on an "AS IS" BASIS, 146 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 147 | implied, including, without limitation, any warranties or conditions 148 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 149 | PARTICULAR PURPOSE. You are solely responsible for determining the 150 | appropriateness of using or redistributing the Work and assume any 151 | risks associated with Your exercise of permissions under this License. 152 | 153 | 8. Limitation of Liability. In no event and under no legal theory, 154 | whether in tort (including negligence), contract, or otherwise, 155 | unless required by applicable law (such as deliberate and grossly 156 | negligent acts) or agreed to in writing, shall any Contributor be 157 | liable to You for damages, including any direct, indirect, special, 158 | incidental, or consequential damages of any character arising as a 159 | result of this License or out of the use or inability to use the 160 | Work (including but not limited to damages for loss of goodwill, 161 | work stoppage, computer failure or malfunction, or any and all 162 | other commercial damages or losses), even if such Contributor 163 | has been advised of the possibility of such damages. 164 | 165 | 9. Accepting Warranty or Additional Liability. While redistributing 166 | the Work or Derivative Works thereof, You may choose to offer, 167 | and charge a fee for, acceptance of support, warranty, indemnity, 168 | or other liability obligations and/or rights consistent with this 169 | License. However, in accepting such obligations, You may act only 170 | on Your own behalf and on Your sole responsibility, not on behalf 171 | of any other Contributor, and only if You agree to indemnify, 172 | defend, and hold each Contributor harmless for any liability 173 | incurred by, or claims asserted against, such Contributor by reason 174 | of your accepting any such warranty or additional liability. 175 | 176 | END OF TERMS AND CONDITIONS 177 | 178 | APPENDIX: How to apply the Apache License to your work. 179 | 180 | To apply the Apache License to your work, attach the following 181 | boilerplate notice, with the fields enclosed by brackets "[]" 182 | replaced with your own identifying information. (Don't include 183 | the brackets!) The text should be enclosed in the appropriate 184 | comment syntax for the file format. We also recommend that a 185 | file or class name and description of purpose be included on the 186 | same "printed page" as the copyright notice for easier 187 | identification within third-party archives. 188 | 189 | Copyright [yyyy] [name of copyright owner] 190 | 191 | Licensed under the Apache License, Version 2.0 (the "License"); 192 | you may not use this file except in compliance with the License. 193 | You may obtain a copy of the License at 194 | 195 | http://www.apache.org/licenses/LICENSE-2.0 196 | 197 | Unless required by applicable law or agreed to in writing, software 198 | distributed under the License is distributed on an "AS IS" BASIS, 199 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 200 | See the License for the specific language governing permissions and 201 | limitations under the License. 202 | --------------------------------------------------------------------------------