├── scripts ├── exp │ ├── __init__.py │ ├── cache.sql │ ├── twitter-pg-schema.sql │ ├── memtierexperiments.py │ ├── evictionexperiments.py │ └── testexperiments.py └── lib │ ├── __init__.py │ ├── aggregate.py │ └── ec2.py ├── src ├── pqbase.cc ├── pqrpc.hh ├── pqlog.hh ├── pqinterconnect.thh ├── pqpersistent.thh ├── pqmemory.cc ├── pqbase.hh ├── pqinterconnect.tcc ├── pqdbpool.thh ├── pqclient.tcc ├── pqmemory.hh ├── pqmulticlient.thh ├── pqdatum.hh ├── pqpartition.cc ├── pqremoteclient.thh ├── mpfd.thh ├── pqremoteclient.tcc ├── pqpersistent.tcc ├── pqmulticlient.tcc └── pqdbpool.tcc ├── vis ├── layout.css ├── vis.html ├── jquery.flot.symbol.js ├── vis.js └── jquery.flot.fillbetween.js ├── .gitmodules ├── AUTHORS ├── bootstrap.sh ├── .gitignore ├── lib ├── keyrange.hh ├── compiler.cc ├── time.hh ├── MurmurHash3.h ├── encoding.cc ├── check.hh ├── sp_key.cc ├── str.cc ├── hosts.cc ├── hosts.hh ├── pair.hh ├── bloom.hh ├── hashallocator.hh ├── hashcode.hh ├── hashallocator.cc ├── sock_helper.hh ├── sp_key.hh ├── str.hh ├── local_str.hh ├── local_string.hh ├── local_stack.hh ├── local_vector.hh └── interval.hh ├── app ├── hashtableadapter.thh ├── pqinfo.tcc ├── hackernews.tcc ├── memcacheadapter.thh ├── poptable.tcc ├── redisadapter.thh └── twitter.tcc ├── LICENSE └── README.md /scripts/exp/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /scripts/lib/__init__.py: -------------------------------------------------------------------------------- 1 | -------------------------------------------------------------------------------- /scripts/exp/cache.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS cache; 2 | CREATE TABLE cache (key varchar, value varchar); 3 | -------------------------------------------------------------------------------- /src/pqbase.cc: -------------------------------------------------------------------------------- 1 | #include "pqbase.hh" 2 | 3 | namespace pq { 4 | 5 | const char marker_data[] = "UEI"; 6 | 7 | } // namespace pq 8 | -------------------------------------------------------------------------------- /vis/layout.css: -------------------------------------------------------------------------------- 1 | body { 2 | font-family: sans-serif; 3 | font-size: 16px; 4 | margin: 50px; 5 | max-width: 800px; 6 | } 7 | -------------------------------------------------------------------------------- /.gitmodules: -------------------------------------------------------------------------------- 1 | [submodule "tamer"] 2 | path = tamer 3 | url = git://github.com/kohler/tamer.git 4 | [submodule "memtier_benchmark"] 5 | path = memtier_benchmark 6 | url = git://github.com/bryankate/memtier_benchmark.git 7 | -------------------------------------------------------------------------------- /AUTHORS: -------------------------------------------------------------------------------- 1 | Bryan Kate 2 | bkate@eecs.harvard.edu 3 | 4 | Eddie Kohler 5 | kohler@seas.harvard.edu 6 | 7 | Yandong Mao 8 | ydmao@csail.mit.edu 9 | 10 | Neha Narula 11 | narula@mit.edu 12 | 13 | Robert Morris 14 | rtm@csail.mit.edu 15 | 16 | Mike Kester 17 | kester@eecs.harvard.edu -------------------------------------------------------------------------------- /bootstrap.sh: -------------------------------------------------------------------------------- 1 | #! /bin/sh 2 | 3 | # check to see if we have the submodules checked out 4 | if [ ! -f tamer/boostrap.sh ] || [ ! -f memtier_benchmark/README.md ]; then 5 | git submodule init 6 | git submodule update 7 | fi 8 | 9 | ( cd tamer; ./bootstrap.sh ) 10 | 11 | autoreconf -i 12 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | gmon.out 2 | perf.data 3 | perf.data.old 4 | 5 | /.deps 6 | /GNUmakefile 7 | /autom4te.cache 8 | /config.h 9 | /config.h.in 10 | /config.log 11 | /config.status 12 | /configure 13 | /conftest.* 14 | /obj 15 | /stamp-h 16 | *~ 17 | *.pyc 18 | results 19 | last 20 | drivers 21 | .redis_one.conf 22 | .DS_Store 23 | 24 | .project 25 | .cproject 26 | .pydevproject 27 | .externalToolBuilders 28 | .settings 29 | 30 | *.pem 31 | aws.json 32 | -------------------------------------------------------------------------------- /src/pqrpc.hh: -------------------------------------------------------------------------------- 1 | #ifndef PEQUOD_PQRPC_HH 2 | #define PEQUOD_PQRPC_HH 3 | 4 | // order matters! 5 | enum { 6 | // single key operations 7 | pq_get = 1, 8 | pq_insert = 2, 9 | pq_erase = 3, 10 | pq_notify_insert = 4, 11 | pq_notify_erase = 5, 12 | pq_count = 6, 13 | 14 | // range operations 15 | pq_scan = 7, 16 | pq_subscribe = 8, 17 | pq_unsubscribe = 9, 18 | pq_invalidate = 10, 19 | 20 | // other 21 | pq_add_join = 11, 22 | pq_stats = 12, 23 | pq_control = 13, 24 | pq_noop_get = 14 25 | }; 26 | 27 | enum { 28 | pq_ok = 0, 29 | pq_fail = -1 30 | }; 31 | 32 | #endif 33 | -------------------------------------------------------------------------------- /lib/keyrange.hh: -------------------------------------------------------------------------------- 1 | #ifndef KEYRANGE_HH 2 | #define KEYRANGE_HH 1 3 | #include "string.hh" 4 | #include 5 | 6 | namespace pq { 7 | 8 | class keyrange { 9 | public: 10 | String key; 11 | int owner; 12 | 13 | keyrange(const String &key, int owner) 14 | : key(key), owner(owner) { 15 | } 16 | template 17 | keyrange(const String_base &key, int owner) 18 | : key(key), owner(owner) { 19 | } 20 | }; 21 | 22 | inline String next_key(String key) { 23 | key += '\0'; 24 | return key; 25 | } 26 | 27 | inline std::string next_key(std::string key) { 28 | key.append(1, '\0'); 29 | return key; 30 | } 31 | 32 | } 33 | #endif 34 | -------------------------------------------------------------------------------- /lib/compiler.cc: -------------------------------------------------------------------------------- 1 | #include "compiler.hh" 2 | #include "string.hh" 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | 13 | void print_stacktrace() { 14 | String pid(getpid()); 15 | char program[1024]; 16 | ssize_t r = readlink("/proc/self/exe", program, sizeof(program)); 17 | if(r > 0){ 18 | program[r] = 0; 19 | int child_pid = fork(); 20 | if (!child_pid) { 21 | dup2(2, 1); 22 | execlp("gdb", "gdb", program, pid.c_str(), "--batch", "-n", 23 | "-ex", "bt", NULL); 24 | abort(); 25 | } else 26 | waitpid(child_pid, NULL, 0); 27 | } 28 | } 29 | 30 | void 31 | fail_mandatory_assert(const char *file, int line, const char *assertion, const char *message) 32 | { 33 | if (message) 34 | fprintf(stderr, "assertion \"%s\" [%s] failed: file \"%s\", line %d\n", 35 | message, assertion, file, line); 36 | else 37 | fprintf(stderr, "assertion \"%s\" failed: file \"%s\", line %d\n", 38 | assertion, file, line); 39 | print_stacktrace(); 40 | abort(); 41 | } 42 | -------------------------------------------------------------------------------- /lib/time.hh: -------------------------------------------------------------------------------- 1 | #ifndef PEQUOD_TIME_HH 2 | #define PEQUOD_TIME_HH 3 | #include 4 | #include 5 | #include 6 | #include "compiler.hh" 7 | 8 | extern uint64_t tstamp_adjustment; 9 | 10 | inline double to_real(timeval tv) { 11 | return tv.tv_sec + tv.tv_usec / (double) 1e6; 12 | } 13 | 14 | inline timeval operator+(timeval a, timeval b) { 15 | timeval c; 16 | timeradd(&a, &b, &c); 17 | return c; 18 | } 19 | 20 | inline timeval operator-(timeval a, timeval b) { 21 | timeval c; 22 | timersub(&a, &b, &c); 23 | return c; 24 | } 25 | 26 | inline uint64_t tous(double seconds) { 27 | mandatory_assert(seconds >= 0); 28 | return (uint64_t) (seconds * 1000000); 29 | } 30 | 31 | inline double fromus(uint64_t t) { 32 | return (double) t / 1000000.; 33 | } 34 | 35 | inline uint64_t tv2us(const timeval& tv) { 36 | return (uint64_t) tv.tv_sec * 1000000 + tv.tv_usec; 37 | } 38 | 39 | inline uint64_t tstamp() { 40 | struct timeval tv; 41 | gettimeofday(&tv, 0); 42 | return tv2us(tv); 43 | } 44 | 45 | inline std::ostream& operator<<(std::ostream& s, const timeval& t) { 46 | return s << t.tv_sec << '.' << std::setfill('0') << std::setw(6) << t.tv_usec; 47 | } 48 | 49 | #endif 50 | -------------------------------------------------------------------------------- /lib/MurmurHash3.h: -------------------------------------------------------------------------------- 1 | //----------------------------------------------------------------------------- 2 | // MurmurHash3 was written by Austin Appleby, and is placed in the public 3 | // domain. The author hereby disclaims copyright to this source code. 4 | 5 | #ifndef _MURMURHASH3_H_ 6 | #define _MURMURHASH3_H_ 7 | 8 | //----------------------------------------------------------------------------- 9 | // Platform-specific functions and macros 10 | 11 | // Microsoft Visual Studio 12 | 13 | #if defined(_MSC_VER) && (_MSC_VER < 1600) 14 | 15 | typedef unsigned char uint8_t; 16 | typedef unsigned int uint32_t; 17 | typedef unsigned __int64 uint64_t; 18 | 19 | // Other compilers 20 | 21 | #else // defined(_MSC_VER) 22 | 23 | #include 24 | 25 | #endif // !defined(_MSC_VER) 26 | 27 | //----------------------------------------------------------------------------- 28 | 29 | void MurmurHash3_x86_32 ( const void * key, int len, uint32_t seed, void * out ); 30 | 31 | void MurmurHash3_x86_128 ( const void * key, int len, uint32_t seed, void * out ); 32 | 33 | void MurmurHash3_x64_128 ( const void * key, int len, uint32_t seed, void * out ); 34 | 35 | //----------------------------------------------------------------------------- 36 | 37 | #endif // _MURMURHASH3_H_ 38 | -------------------------------------------------------------------------------- /lib/encoding.cc: -------------------------------------------------------------------------------- 1 | /* 2 | * encoding.{cc,hh} -- encoding transformations 3 | * Eddie Kohler 4 | * 5 | * Copyright (c) 2011 Eddie Kohler 6 | * 7 | * Permission is hereby granted, free of charge, to any person obtaining a 8 | * copy of this software and associated documentation files (the "Software"), 9 | * to deal in the Software without restriction, subject to the conditions 10 | * listed in the Click LICENSE file. These conditions include: you must 11 | * preserve this copyright notice, and you cannot mention the copyright 12 | * holders in advertising related to the Software without their permission. 13 | * The Software is provided WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. This 14 | * notice is a summary of the Click LICENSE file; the license in that file is 15 | * legally binding. 16 | */ 17 | 18 | #ifdef HAVE_CONFIG_H 19 | # include 20 | #endif 21 | #include "encoding.hh" 22 | 23 | namespace Encoding { 24 | 25 | const uint16_t Windows1252::c1_mapping[] = { 26 | 0x20AC, 0x0081, 0x201A, 0x0192, 0x201E, 0x2026, 0x2020, 0x2021, 27 | 0x20C6, 0x2030, 0x0160, 0x2039, 0x0152, 0x008D, 0x017D, 0x008F, 28 | 0x0090, 0x2018, 0x2019, 0x201C, 0x201D, 0x2022, 0x2013, 0x2014, 29 | 0x20DC, 0x2122, 0x0161, 0x203A, 0x0153, 0x009D, 0x017E, 0x0178 30 | }; 31 | 32 | } 33 | -------------------------------------------------------------------------------- /lib/check.hh: -------------------------------------------------------------------------------- 1 | #ifndef TEST_UTIL_HH_ 2 | #define TEST_UTIL_HH_ 3 | 4 | #include 5 | #include 6 | #include 7 | #include "compiler.hh" 8 | 9 | template 10 | inline void check_true(const char* file, int line, 11 | T test, const std::string& msg = "") { 12 | if (!test) { 13 | std::cerr << file << ":" << line << ": Check failed\n"; 14 | if (!msg.empty()) 15 | std::cerr << "\t" << msg << std::endl; 16 | mandatory_assert(0); 17 | } 18 | } 19 | 20 | template 21 | inline void check_eq(const char* file, int line, 22 | const T1& actual, const T2& expected, 23 | const std::string& msg = "") { 24 | if (!(expected == actual)) { 25 | std::cerr << file << ":" << line << ": Check failed\n"; 26 | if (!msg.empty()) 27 | std::cerr << "\t" << msg << std::endl; 28 | std::cerr << "\tActual: " << actual 29 | << "\n\tExpected: " << expected << std::endl; 30 | mandatory_assert(0); 31 | } 32 | } 33 | 34 | #define CHECK_TRUE(test) check_true(__FILE__, __LINE__, (test), #test) 35 | #define CHECK_EQ(actual, expected, ...) check_eq(__FILE__, __LINE__, (actual), (expected), #actual " == " #expected __VA_ARGS__) 36 | 37 | #endif 38 | -------------------------------------------------------------------------------- /src/pqlog.hh: -------------------------------------------------------------------------------- 1 | #ifndef PQLOG_HH_ 2 | #define PQLOG_HH_ 3 | 4 | #include "str.hh" 5 | #include "json.hh" 6 | #include "time.hh" 7 | #include 8 | 9 | namespace pq { 10 | 11 | class Log { 12 | public: 13 | inline Log(uint64_t epoch = 0); 14 | 15 | template 16 | inline void record(Str key, const T& value); 17 | template 18 | inline void record_at(Str key, uint64_t time, const T& value); 19 | inline void clear(); 20 | inline const Json& as_json() const; 21 | inline void write_json(std::ostream& s) const; 22 | 23 | private: 24 | Json log_; 25 | uint64_t epoch_; 26 | }; 27 | 28 | inline Log::Log(uint64_t epoch) : epoch_(epoch) { 29 | } 30 | 31 | template 32 | inline void Log::record(Str key, const T& value) { 33 | record_at(key, tstamp(), value); 34 | } 35 | 36 | template 37 | inline void Log::record_at(Str key, uint64_t time, const T& value) { 38 | assert(time > epoch_); 39 | log_.get_insert(key).push_back(Json::array(time - epoch_, value)); 40 | } 41 | 42 | inline void Log::clear() { 43 | log_ = Json(); 44 | if (epoch_) 45 | epoch_ = tstamp(); 46 | } 47 | 48 | inline const Json& Log::as_json() const { 49 | return log_; 50 | } 51 | 52 | inline void Log::write_json(std::ostream& s) const { 53 | s << log_ << std::endl; 54 | } 55 | 56 | } 57 | 58 | #endif 59 | -------------------------------------------------------------------------------- /lib/sp_key.cc: -------------------------------------------------------------------------------- 1 | #include "sp_key.hh" 2 | #include 3 | 4 | namespace pq { 5 | 6 | String extract_spkey(int field, const String &spkey) { 7 | const char *s = spkey.begin(), *e = spkey.end(); 8 | while (field > 0) { 9 | if ((s = reinterpret_cast(memchr(s, '|', e - s)))) { 10 | ++s; 11 | --field; 12 | } else 13 | return String(); 14 | } 15 | const char *x = reinterpret_cast(memchr(s, '|', e - s)); 16 | return spkey.substring(s, x ? x : e); 17 | } 18 | 19 | String extract_spkey(int field, const std::string &spkey) { 20 | const char *s = spkey.data(), *e = spkey.data() + spkey.length(); 21 | while (field > 0) { 22 | if ((s = reinterpret_cast(memchr(s, '|', e - s)))) { 23 | ++s; 24 | --field; 25 | } else 26 | return String(); 27 | } 28 | const char *x = reinterpret_cast(memchr(s, '|', e - s)); 29 | return String(s, x ? x : e); 30 | } 31 | 32 | String extract_spkey(int field, Str spkey) { 33 | const char *s = spkey.data(), *e = spkey.data() + spkey.length(); 34 | while (field > 0) { 35 | if ((s = reinterpret_cast(memchr(s, '|', e - s)))) { 36 | ++s; 37 | --field; 38 | } else 39 | return String(); 40 | } 41 | const char *x = reinterpret_cast(memchr(s, '|', e - s)); 42 | return String(s, x ? x : e); 43 | } 44 | 45 | } 46 | -------------------------------------------------------------------------------- /app/hashtableadapter.thh: -------------------------------------------------------------------------------- 1 | // -*- mode: c++ -*- 2 | #ifndef HASHTABLE_ADAPTER_HH 3 | #define HASHTABLE_ADAPTER_HH 4 | 5 | #include 6 | #include "hashtable.hh" 7 | #include "str.hh" 8 | #include "string.hh" 9 | 10 | namespace pq { 11 | 12 | class BuiltinHashClient { 13 | public: 14 | BuiltinHashClient() { 15 | h_.rehash(2000000); 16 | } 17 | void set(const Str k, const Str v, tamer::event<> e) { 18 | h_[k] = v; 19 | e(); 20 | } 21 | void append(const Str k, const Str v, tamer::event<> e) { 22 | auto& ev = h_[k]; 23 | ev.append(v); 24 | e(); 25 | } 26 | void get(const Str k, int32_t offset, tamer::event e) { 27 | auto it = h_.find(k); 28 | if (it == h_.end()) 29 | e(String()); 30 | else { 31 | mandatory_assert(it->second.length() >= offset); 32 | e(String(it->second.data() + offset, it->second.length() - offset)); 33 | } 34 | } 35 | void done_get(String) { 36 | } 37 | void increment(const Str k, tamer::event<> e) { 38 | auto& ev = h_[k]; 39 | if (ev.empty()) 40 | ev = String(1); 41 | else 42 | ev = String(ev.to_i() + 1); 43 | e(); 44 | } 45 | void pace(tamer::event<> done) { 46 | done(); 47 | } 48 | private: 49 | HashTable h_; 50 | }; 51 | 52 | } 53 | 54 | #endif 55 | -------------------------------------------------------------------------------- /src/pqinterconnect.thh: -------------------------------------------------------------------------------- 1 | // -*- mode: c++ -*- 2 | #ifndef PEQUOD_INTERCONNECT_HH 3 | #define PEQUOD_INTERCONNECT_HH 4 | #include 5 | #include 6 | #include "mpfd.hh" 7 | #include "pqrpc.hh" 8 | #include "pqremoteclient.hh" 9 | 10 | namespace pq { 11 | using tamer::event; 12 | 13 | class Interconnect : public RemoteClient { 14 | public: 15 | typedef typename RemoteClient::scan_result scan_result; 16 | 17 | inline Interconnect(tamer::fd fd, int machineid); 18 | inline Interconnect(msgpack_fd* fd, int machineid); 19 | 20 | tamed void subscribe(const String& first, const String& last, 21 | int32_t subscriber, event e); 22 | tamed void unsubscribe(const String& first, const String& last, 23 | int32_t subscriber, event<> e); 24 | 25 | tamed void notify_insert(const String& key, const String& value, event<> e); 26 | tamed void notify_erase(const String& key, event<> e); 27 | 28 | tamed void invalidate(const String& first, const String& last, 29 | event<> e); 30 | }; 31 | 32 | 33 | inline Interconnect::Interconnect(tamer::fd fd, int machineid) 34 | : RemoteClient(fd, String("inter") + String(machineid)) { 35 | } 36 | 37 | inline Interconnect::Interconnect(msgpack_fd* fd, int machineid) 38 | : RemoteClient(fd, String("inter") + String(machineid)) { 39 | } 40 | 41 | } // namespace pq 42 | #endif 43 | -------------------------------------------------------------------------------- /scripts/exp/twitter-pg-schema.sql: -------------------------------------------------------------------------------- 1 | DROP TABLE IF EXISTS s; 2 | DROP TABLE IF EXISTS p; 3 | DROP TABLE IF EXISTS t; 4 | 5 | CREATE TABLE s ( 6 | usr INT, 7 | poster INT, 8 | -- online BOOLEAN, 9 | CONSTRAINT primary_s PRIMARY KEY (usr,poster) 10 | ); 11 | CREATE INDEX s_index_usr ON s (usr); 12 | CREATE INDEX s_index_poster ON s (poster); 13 | --CREATE INDEX s_index_online ON s (online); 14 | 15 | CREATE TABLE p ( 16 | poster INT, 17 | value VARCHAR(140), 18 | time INT, 19 | CONSTRAINT primary_p PRIMARY KEY (poster,time) 20 | ); 21 | CREATE INDEX p_index_time ON p (time); 22 | CREATE INDEX p_index_poster ON p (poster); 23 | 24 | CREATE TABLE t ( 25 | usr INT, 26 | poster INT, 27 | value varchar(140), 28 | time INT 29 | ); 30 | CREATE INDEX t_index_usr ON t (usr); 31 | CREATE INDEX t_index_time ON t (time); 32 | 33 | -- Postgres doesn't support materialized views (until 9.3 which is in beta) 34 | -- Instead we use a real table and update it using a trigger function 35 | CREATE OR REPLACE FUNCTION push_posts() 36 | RETURNS TRIGGER AS $BODY$ 37 | BEGIN 38 | INSERT INTO t (usr, poster, value, time) 39 | SELECT s.usr, NEW.poster, NEW.value, NEW.time 40 | FROM s 41 | WHERE s.poster = NEW.poster 42 | -- AND s.online = TRUE 43 | ; 44 | RETURN NULL; 45 | END; 46 | $BODY$ 47 | LANGUAGE plpgsql VOLATILE 48 | COST 100; 49 | 50 | DROP TRIGGER IF EXISTS update_timeline ON p; 51 | CREATE TRIGGER update_timeline 52 | AFTER INSERT ON p 53 | FOR EACH ROW 54 | EXECUTE PROCEDURE push_posts(); 55 | -------------------------------------------------------------------------------- /lib/str.cc: -------------------------------------------------------------------------------- 1 | #include "str.hh" 2 | 3 | const Str Str::maxkey("\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 4 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 5 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 6 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 7 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 8 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 9 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 10 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 11 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 12 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 13 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 14 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 15 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 16 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 17 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 18 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 19 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 20 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 21 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 22 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 23 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 24 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 25 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 26 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 27 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 28 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 29 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 30 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 31 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 32 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 33 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 34 | "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF" 35 | "\xFF", 257); 36 | -------------------------------------------------------------------------------- /lib/hosts.cc: -------------------------------------------------------------------------------- 1 | #include "hosts.hh" 2 | #include "sock_helper.hh" 3 | #include 4 | #include 5 | #include 6 | #include 7 | #include 8 | 9 | using std::string; 10 | using std::vector; 11 | using std::ifstream; 12 | using std::istringstream; 13 | 14 | namespace pq { 15 | 16 | Host::Host(const String &name, uint32_t port, uint32_t seqid) 17 | : name_(name), port_(port), uid_(sock_helper::get_uid(name.c_str(), port)), seqid_(seqid) { 18 | } 19 | 20 | String Host::uid_string() const { 21 | return name_ + ":" + String(port_); 22 | } 23 | 24 | 25 | Hosts::Hosts(const String &hostFile) { 26 | 27 | ifstream infile(hostFile.c_str(), ifstream::in); 28 | string line; 29 | uint32_t seq = 0; 30 | 31 | mandatory_assert(infile && "Host file cannot be opened."); 32 | while(infile.good()) { 33 | 34 | if (!getline(infile, line)) { 35 | break; 36 | } 37 | 38 | boost::trim(line); 39 | 40 | if ((line.empty()) || (line[0] == '#')) { 41 | continue; 42 | } 43 | 44 | string name; 45 | uint32_t port; 46 | istringstream iss(line); 47 | 48 | iss >> name >> port; 49 | hosts_.push_back(Host(name.c_str(), port, seq++)); 50 | } 51 | } 52 | 53 | const Host *Hosts::get_by_uid(uint64_t uid) const { 54 | 55 | for (auto h = hosts_.begin(); h != hosts_.end(); ++h) { 56 | if (h->uid() == uid) { 57 | return &hosts_[h->seqid()]; 58 | } 59 | } 60 | 61 | assert(false && "Could not find host for given uid."); 62 | return NULL; 63 | } 64 | 65 | Hosts *Hosts::get_instance(const String &hostFile) { 66 | return new Hosts(hostFile); 67 | } 68 | 69 | } 70 | -------------------------------------------------------------------------------- /src/pqpersistent.thh: -------------------------------------------------------------------------------- 1 | #ifndef PQ_PERSISTENT_HH 2 | #define PQ_PERSISTENT_HH 3 | #include "str.hh" 4 | #include "string.hh" 5 | #include "pqdbpool.hh" 6 | #include 7 | #if HAVE_POSTGRESQL_LIBPQ_FE_H 8 | #include 9 | #elif HAVE_LIBPQ_FE_H 10 | #include 11 | #endif 12 | 13 | namespace pq { 14 | class Server; 15 | 16 | class PersistentStore { 17 | public: 18 | typedef std::pair Result; 19 | typedef std::vector ResultSet; 20 | 21 | virtual ~PersistentStore() { } 22 | 23 | virtual void put(Str key, Str value, tamer::event<> done) = 0; 24 | virtual void erase(Str key, tamer::event<> done) = 0; 25 | virtual void get(Str key, tamer::event done) = 0; 26 | virtual void scan(Str first, Str last, tamer::event done) = 0; 27 | virtual void flush() = 0; 28 | 29 | virtual void run_monitor(Server& server) = 0; 30 | }; 31 | 32 | 33 | #if HAVE_LIBPQ 34 | 35 | class PostgresStore : public pq::PersistentStore { 36 | public: 37 | PostgresStore(const DBPoolParams& params); 38 | ~PostgresStore(); 39 | 40 | tamed virtual void put(Str key, Str value, tamer::event<> done); 41 | tamed virtual void erase(Str key, tamer::event<> done); 42 | tamed virtual void get(Str key, tamer::event done); 43 | tamed virtual void scan(Str first, Str last, tamer::event done); 44 | virtual void flush(); 45 | 46 | void connect(); 47 | virtual void run_monitor(Server& server); 48 | 49 | private: 50 | enum { pg_update = 0, pg_delete = 1 }; 51 | 52 | DBPoolParams params_; 53 | DBPool* pool_; 54 | PGconn* monitor_; 55 | 56 | tamed void monitor_db(Server& server); 57 | }; 58 | 59 | #endif 60 | 61 | } 62 | 63 | #endif 64 | -------------------------------------------------------------------------------- /src/pqmemory.cc: -------------------------------------------------------------------------------- 1 | #include "pqmemory.hh" 2 | #include 3 | 4 | namespace pq { 5 | 6 | uint64_t mem_overhead_size = 0; 7 | uint64_t mem_other_size = 0; 8 | uint64_t mem_store_size = 0; 9 | 10 | namespace { 11 | struct meminfo { 12 | uint64_t* type; 13 | size_t sz; 14 | }; 15 | } 16 | 17 | void* allocate(size_t sz, uint64_t* type) { 18 | if (sz == 0) 19 | return NULL; 20 | if (!enable_memory_tracking) 21 | return malloc(sz); 22 | 23 | size_t xsz = sz + sizeof(meminfo); 24 | meminfo* mi; 25 | if (xsz < sz || !(mi = (meminfo*)malloc(xsz))) 26 | return NULL; 27 | mi->type = type; 28 | mi->sz = sz; 29 | mem_overhead_size += xsz - sz; 30 | if (type) 31 | *type += sz; 32 | else 33 | mem_other_size += sz; 34 | return (void *) (mi + 1); 35 | } 36 | 37 | void deallocate(void* p) { 38 | if (!p) 39 | return; 40 | if (!enable_memory_tracking) { 41 | free(p); 42 | return; 43 | } 44 | 45 | meminfo* mi = (meminfo*)p - 1; 46 | mem_overhead_size -= sizeof(meminfo); 47 | if (mi->type) 48 | *(mi->type) -= mi->sz; 49 | else 50 | mem_other_size -= mi->sz; 51 | free(mi); 52 | } 53 | 54 | } // namepace pq 55 | 56 | void* operator new(size_t size) { 57 | return pq::allocate(size, nullptr); 58 | } 59 | 60 | void* operator new[](size_t size) { 61 | return pq::allocate(size, nullptr); 62 | } 63 | 64 | void* operator new(size_t size, uint64_t* type) { 65 | return pq::allocate(size, type); 66 | } 67 | 68 | void* operator new[](size_t size, uint64_t* type) { 69 | return pq::allocate(size, type); 70 | } 71 | 72 | void operator delete(void *p) { 73 | pq::deallocate(p); 74 | } 75 | 76 | void operator delete[](void *p) { 77 | pq::deallocate(p); 78 | } 79 | 80 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012-2014, Bryan Kate, Eddie Kohler, Mike Kester 2 | Yandong Mao, Neha Narula, Robert Morris 3 | Copyright (c) 2012-2014, President and Fellows of Harvard College 4 | Copyright (c) 2012-2014, Massachusetts Institute of Technology 5 | All rights reserved. 6 | 7 | The contributors to this work are listed in the AUTHORS file. 8 | 9 | Redistribution and use in source and binary forms, with or without 10 | modification, are permitted provided that the following conditions are met: 11 | * Redistributions of source code must retain the above copyright 12 | notice, this list of conditions and the following disclaimer. 13 | * Redistributions in binary form must reproduce the above copyright 14 | notice, this list of conditions and the following disclaimer in the 15 | documentation and/or other materials provided with the distribution. 16 | * Neither the name of the copyright holders nor the 17 | names of its contributors may be used to endorse or promote products 18 | derived from this software without specific prior written permission. 19 | 20 | THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND 21 | ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED 22 | WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE 23 | DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE 24 | FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 25 | DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR 26 | SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER 27 | CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 28 | OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 29 | OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -------------------------------------------------------------------------------- /lib/hosts.hh: -------------------------------------------------------------------------------- 1 | #ifndef HOSTS_HH_ 2 | #define HOSTS_HH_ 3 | 4 | #include "compiler.hh" 5 | #include "string.hh" 6 | #include 7 | 8 | namespace pq { 9 | 10 | class Host { 11 | public: 12 | Host(const String &name, uint32_t port, uint32_t seqid); 13 | 14 | inline const String &name() const; 15 | inline uint32_t port() const; 16 | inline uint64_t uid() const; 17 | inline int seqid() const; 18 | 19 | String uid_string() const; 20 | 21 | private: 22 | 23 | String name_; 24 | uint32_t port_; 25 | uint64_t uid_; // a unique identifier based on the hostname and port 26 | int seqid_; // this host's index in the Hosts.all() vector 27 | }; 28 | 29 | 30 | class Hosts { 31 | public: 32 | static Hosts *get_instance(const String &hostFile); 33 | 34 | const Host *get_by_uid(uint64_t uid) const; 35 | inline const Host *get_by_seqid(int seqid) const; 36 | inline const std::vector &all() const; 37 | inline int size() const; 38 | inline uint32_t count() const; // same as size(), deprecated 39 | 40 | private: 41 | std::vector hosts_; 42 | 43 | Hosts(const String &hostFile); 44 | }; 45 | 46 | 47 | inline const String &Host::name() const { 48 | return name_; 49 | } 50 | 51 | inline uint32_t Host::port() const { 52 | return port_; 53 | } 54 | 55 | inline uint64_t Host::uid() const { 56 | return uid_; 57 | } 58 | 59 | inline int Host::seqid() const { 60 | return seqid_; 61 | } 62 | 63 | inline const Host *Hosts::get_by_seqid(int seqid) const { 64 | assert((unsigned) seqid < hosts_.size()); 65 | return &hosts_[seqid]; 66 | } 67 | 68 | inline const std::vector &Hosts::all() const { 69 | return hosts_; 70 | } 71 | 72 | inline int Hosts::size() const { 73 | return hosts_.size(); 74 | } 75 | 76 | inline uint32_t Hosts::count() const { 77 | return hosts_.size(); 78 | } 79 | 80 | } 81 | #endif 82 | -------------------------------------------------------------------------------- /vis/vis.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | pequod experiment details 6 | 7 | 8 | 9 | 10 | 11 | 12 | 13 | 14 | 15 | 16 | 17 | 18 | 19 | 20 | 21 | 22 | 23 | 24 | 25 | 26 |
27 | 28 | 53 | 54 | 55 | 56 | -------------------------------------------------------------------------------- /src/pqbase.hh: -------------------------------------------------------------------------------- 1 | #ifndef PEQUOD_BASE_HH 2 | #define PEQUOD_BASE_HH 3 | #include 4 | #include "string.hh" 5 | namespace pq { 6 | 7 | enum { key_capacity = 128 }; 8 | 9 | template 10 | inline T table_name(const String_base& key) { 11 | const char* x = (const char*) memchr(key.data(), '|', key.length()); 12 | if (!x) { 13 | if (key.length() && key.back() == '}') 14 | x = key.end() - 1; 15 | else 16 | x = key.begin(); 17 | } 18 | return static_cast(key).fast_substring(key.data(), x); 19 | } 20 | 21 | template 22 | inline T table_name(const String_base& key, const String_base& key2) { 23 | T t = table_name(key); 24 | if (t.length() && key2.length() > t.length() 25 | && memcmp(key2.data(), t.data(), t.length()) == 0 26 | && (key2[t.length()] | 1) == '}' /* matches | or } */) 27 | return t; 28 | else 29 | return static_cast(key).fast_substring(key.data(), key.data()); 30 | } 31 | 32 | extern const char marker_data[]; 33 | extern const char erase_marker_data[]; 34 | extern const char invalid_marker_data[]; 35 | 36 | inline String unchanged_marker() { 37 | return String::make_stable(marker_data, 1); 38 | } 39 | 40 | inline bool is_unchanged_marker(const String& str) { 41 | return str.data() == marker_data; 42 | } 43 | 44 | inline String erase_marker() { 45 | return String::make_stable(marker_data + 1, 1); 46 | } 47 | 48 | inline bool is_erase_marker(const String& str) { 49 | return str.data() == marker_data + 1; 50 | } 51 | 52 | inline String invalidate_marker() { 53 | return String::make_stable(marker_data + 2, 1); 54 | } 55 | 56 | inline bool is_invalidate_marker(const String& str) { 57 | return str.data() == marker_data + 2; 58 | } 59 | 60 | inline bool is_marker(const String& str) { 61 | return reinterpret_cast(str.data()) - reinterpret_cast(marker_data) < 3; 62 | } 63 | 64 | } // namespace 65 | #endif 66 | -------------------------------------------------------------------------------- /lib/pair.hh: -------------------------------------------------------------------------------- 1 | // -*- c-basic-offset: 4 -*- 2 | #ifndef CLICK_PAIR_HH 3 | #define CLICK_PAIR_HH 4 | #include "hashcode.hh" 5 | 6 | template 7 | struct Pair { 8 | 9 | typedef T first_type; 10 | typedef U second_type; 11 | typedef T key_type; 12 | typedef const T &key_const_reference; 13 | 14 | T first; 15 | U second; 16 | 17 | inline Pair() 18 | : first(), second() { 19 | } 20 | 21 | inline Pair(const T &t, const U &u) 22 | : first(t), second(u) { 23 | } 24 | 25 | inline Pair(const Pair &p) 26 | : first(p.first), second(p.second) { 27 | } 28 | 29 | template 30 | inline Pair(const Pair &p) 31 | : first(p.first), second(p.second) { 32 | } 33 | 34 | typedef hashcode_t (Pair::*unspecified_bool_type)() const; 35 | inline operator unspecified_bool_type() const { 36 | return first || second ? &Pair::hashcode : 0; 37 | } 38 | 39 | inline const T &hashkey() const { 40 | return first; 41 | } 42 | 43 | inline hashcode_t hashcode() const; 44 | 45 | template 46 | Pair &operator=(const Pair &p) { 47 | first = p.first; 48 | second = p.second; 49 | return *this; 50 | } 51 | 52 | }; 53 | 54 | template 55 | inline bool operator==(const Pair &a, const Pair &b) 56 | { 57 | return a.first == b.first && a.second == b.second; 58 | } 59 | 60 | template 61 | inline bool operator!=(const Pair &a, const Pair &b) 62 | { 63 | return a.first != b.first || a.second != b.second; 64 | } 65 | 66 | template 67 | inline bool operator<(const Pair &a, const Pair &b) 68 | { 69 | return a.first < b.first 70 | || (!(b.first < a.first) && a.second < b.second); 71 | } 72 | 73 | template 74 | inline hashcode_t Pair::hashcode() const 75 | { 76 | return (hashcode(first) << 7) ^ hashcode(second); 77 | } 78 | 79 | template 80 | inline Pair make_pair(T t, U u) 81 | { 82 | return Pair(t, u); 83 | } 84 | 85 | #endif 86 | -------------------------------------------------------------------------------- /src/pqinterconnect.tcc: -------------------------------------------------------------------------------- 1 | // -*- mode: c++ -*- 2 | #include "pqinterconnect.hh" 3 | 4 | namespace pq { 5 | 6 | tamed void Interconnect::subscribe(const String& first, const String& last, 7 | int32_t subscriber, event e) { 8 | tvars { Json j; } 9 | twait ["subscribe " + first.substring(0, 2)] { 10 | fd_->call(Json::array(pq_subscribe, seq_, first, last, 11 | Json().set("subscriber", subscriber)), 12 | make_event(j)); 13 | ++seq_; 14 | } 15 | e(scan_result(j && j[2].to_i() == pq_ok ? j[3] : Json::make_array())); 16 | } 17 | 18 | tamed void Interconnect::unsubscribe(const String& first, const String& last, 19 | int32_t subscriber, event<> e) { 20 | tvars { Json j; } 21 | twait ["unsubscribe " + first.substring(0, 2)] { 22 | fd_->call(Json::array(pq_unsubscribe, seq_, first, last, 23 | Json().set("subscriber", subscriber)), 24 | make_event(j)); 25 | ++seq_; 26 | } 27 | e(); 28 | } 29 | 30 | tamed void Interconnect::notify_insert(const String& key, const String& value, 31 | event<> e) { 32 | tvars { Json j; } 33 | twait ["notify+ " + key.substring(0, 2)] { 34 | fd_->call(Json::array(pq_notify_insert, seq_, key, value), make_event(j)); 35 | ++seq_; 36 | } 37 | e(); 38 | } 39 | 40 | tamed void Interconnect::notify_erase(const String& key, event<> e) { 41 | tvars { Json j; } 42 | twait ["notify- " + key.substring(0, 2)] { 43 | fd_->call(Json::array(pq_notify_erase, seq_, key), make_event(j)); 44 | ++seq_; 45 | } 46 | e(); 47 | } 48 | 49 | tamed void Interconnect::invalidate(const String& first, const String& last, 50 | event<> e) { 51 | tvars { Json j; } 52 | twait ["invalidate " + first.substring(0, 2)] { 53 | fd_->call(Json::array(pq_invalidate, seq_, first, last), 54 | make_event(j)); 55 | ++seq_; 56 | } 57 | e(); 58 | } 59 | 60 | } 61 | -------------------------------------------------------------------------------- /src/pqdbpool.thh: -------------------------------------------------------------------------------- 1 | #ifndef PQDBPOOL_HH_ 2 | #define PQDBPOOL_HH_ 3 | 4 | #include "string.hh" 5 | #include "json.hh" 6 | #include "time.hh" 7 | #include 8 | #include 9 | #include 10 | #if HAVE_POSTGRESQL_LIBPQ_FE_H 11 | #include 12 | #elif HAVE_LIBPQ_FE_H 13 | #include 14 | #endif 15 | 16 | namespace pq { 17 | 18 | struct DBPoolParams { 19 | DBPoolParams(); 20 | 21 | String dbname; 22 | String host; 23 | uint32_t port; 24 | uint32_t min; 25 | uint32_t max; 26 | uint32_t pipeline_depth; 27 | uint32_t pipeline_timeout; 28 | }; 29 | 30 | class DBPool { 31 | 32 | public: 33 | DBPool(const String& host, uint32_t port); 34 | DBPool(const DBPoolParams& params); 35 | ~DBPool(); 36 | 37 | void connect(); 38 | void connect_all(const std::vector& sess_init = std::vector()); 39 | void clear(); 40 | 41 | tamed void execute(Str query, tamer::event e); 42 | tamed void add_prepared(const std::vector& statements, tamer::event<> e); 43 | 44 | inline void maybe_flush(); 45 | tamed void flush(); 46 | 47 | private: 48 | typedef std::vector query_pipe_t; 49 | typedef std::vector> event_pipe_t; 50 | 51 | DBPoolParams params_; 52 | query_pipe_t query_buffer_; 53 | event_pipe_t event_buffer_; 54 | uint64_t oldest_; 55 | 56 | #if HAVE_LIBPQ 57 | std::vector conn_; 58 | std::queue pool_; 59 | std::queue> waiting_; 60 | 61 | void next_connection(tamer::event e); 62 | void replace_connection(PGconn* conn); 63 | PGconn* connect_one(); 64 | 65 | tamed void execute_pipeline(PGconn* conn, 66 | const query_pipe_t& queries, 67 | event_pipe_t& events, 68 | tamer::event<> e); 69 | #endif 70 | }; 71 | 72 | 73 | inline void DBPool::maybe_flush() { 74 | if (query_buffer_.size() < params_.pipeline_depth && oldest_ && 75 | ((tstamp() - oldest_) < params_.pipeline_timeout)) 76 | return; 77 | 78 | flush(); 79 | } 80 | 81 | } 82 | 83 | #endif 84 | -------------------------------------------------------------------------------- /src/pqclient.tcc: -------------------------------------------------------------------------------- 1 | // -*- mode: c++ -*- 2 | #include "pqclient.hh" 3 | 4 | namespace pq { 5 | 6 | tamed void DirectClient::get(const String& key, event e) { 7 | tvars { 8 | Table::iterator it; 9 | } 10 | 11 | twait [key] { server_.validate(key, make_event(it)); } 12 | auto itend = it.table_end(); 13 | if (it != itend && it->key() == key) 14 | e(it->value()); 15 | else 16 | e(String()); 17 | } 18 | 19 | tamed void DirectClient::count(const String& first, const String& last, 20 | event e) { 21 | count(first, last, last, e); 22 | } 23 | 24 | tamed void DirectClient::count(const String& first, const String& last, const String& scanlast, 25 | event e) { 26 | tvars { 27 | Table::iterator it; 28 | } 29 | 30 | twait [first + "," + last] { 31 | server_.validate(first, last, make_event(it)); 32 | } 33 | e(std::distance(it, server_.table_for(first, last).lower_bound(scanlast))); 34 | } 35 | 36 | tamed void DirectClient::add_count(const String& first, const String& last, 37 | event e) { 38 | add_count(first, last, last, e); 39 | } 40 | 41 | tamed void DirectClient::add_count(const String& first, const String& last, const String& scanlast, 42 | event e) { 43 | tvars { 44 | Table::iterator it; 45 | } 46 | 47 | twait [first + "," + last] { 48 | server_.validate(first, last, make_event(it)); 49 | } 50 | e(e.result() + std::distance(it, server_.table_for(first, last).lower_bound(scanlast))); 51 | } 52 | 53 | tamed void DirectClient::scan(const String& first, const String& last, 54 | event e) { 55 | scan(first, last, last, e); 56 | } 57 | 58 | tamed void DirectClient::scan(const String& first, const String& last, const String& scanlast, 59 | event e) { 60 | tvars { 61 | Table::iterator it; 62 | } 63 | 64 | twait [first + "," + last] { 65 | server_.validate(first, last, make_event(it)); 66 | } 67 | e(scan_result(it, server_.table_for(first, last).lower_bound(scanlast))); 68 | } 69 | 70 | } 71 | -------------------------------------------------------------------------------- /src/pqmemory.hh: -------------------------------------------------------------------------------- 1 | #ifndef PQ_MEMORY_HH_ 2 | #define PQ_MEMORY_HH_ 1 3 | 4 | #include "compiler.hh" 5 | #include 6 | #include 7 | 8 | void* operator new(size_t, int64_t* type); 9 | void* operator new[](size_t, int64_t* type); 10 | 11 | namespace pq { 12 | 13 | void* allocate(size_t, uint64_t* type); 14 | void deallocate(void* p); 15 | 16 | enum { enable_memory_tracking = 1 }; 17 | 18 | template 19 | class Allocator { 20 | public: 21 | typedef T* pointer; 22 | typedef const T* const_pointer; 23 | typedef T& reference; 24 | typedef const T& const_reference; 25 | typedef T value_type; 26 | typedef size_t size_type; 27 | typedef ptrdiff_t difference_type; 28 | 29 | inline pointer allocate(size_type n, const void* = 0) { 30 | return reinterpret_cast(pq::allocate(n* sizeof(T), mem_type)); 31 | } 32 | 33 | inline void deallocate(pointer p, size_type) { 34 | pq::deallocate(p); 35 | } 36 | 37 | inline size_type max_size() const throw() { 38 | return size_type(-1) / sizeof(T); 39 | } 40 | 41 | inline pointer address(reference t) const { 42 | return std::addressof(t); 43 | } 44 | 45 | inline const_pointer address(const_reference t) const { 46 | return std::addressof(t); 47 | } 48 | 49 | inline void destroy(pointer p) { 50 | p->~T(); 51 | } 52 | 53 | template 54 | inline void construct(pointer p, Args&&... args) { 55 | ::new((void*)p) T(std::forward(args)...); 56 | } 57 | 58 | template 59 | struct rebind { 60 | typedef Allocator other; 61 | }; 62 | }; 63 | 64 | 65 | extern uint64_t mem_overhead_size; 66 | extern uint64_t mem_other_size; 67 | extern uint64_t mem_store_size; 68 | 69 | template 70 | struct heap_type { 71 | typedef pq::Allocator store; 72 | }; 73 | 74 | 75 | // report rusage ru_maxrss in megabytes 76 | inline uint32_t maxrss_mb(int32_t rss) { 77 | #ifdef __APPLE__ 78 | return (uint32_t)rss >> 20; // apple reports in bytes 79 | #else 80 | return (uint32_t)rss >> 10; // linux reports in kilobytes 81 | #endif 82 | } 83 | 84 | } 85 | 86 | #endif 87 | -------------------------------------------------------------------------------- /lib/bloom.hh: -------------------------------------------------------------------------------- 1 | #ifndef BLOOM_HH 2 | #define BLOOM_HH 3 | 4 | #include "compiler.hh" 5 | #include "MurmurHash3.h" 6 | #include 7 | #include 8 | 9 | 10 | class BloomFilter { 11 | 12 | public: 13 | inline BloomFilter(uint32_t m, double err); 14 | inline BloomFilter(uint32_t m, uint32_t k); 15 | 16 | inline bool check(const char* buff, size_t len) const; 17 | inline bool add(const char* buff, size_t len); 18 | 19 | private: 20 | typedef std::pair hash_t; 21 | 22 | uint32_t k_; 23 | std::vector bits_; 24 | 25 | inline hash_t hash(const char* buff, size_t len) const; 26 | inline bool check(const hash_t hash) const; 27 | inline bool add(const hash_t hash); 28 | }; 29 | 30 | 31 | inline BloomFilter::BloomFilter(uint32_t m, uint32_t k) : k_(k), bits_(m) { 32 | } 33 | 34 | inline BloomFilter::BloomFilter(uint32_t m, double err) { 35 | 36 | if (m < 1 || !err) 37 | mandatory_assert(false && "Invalid BloomFilter params."); 38 | 39 | double ln2 = log(2); 40 | double bpe = -(log(err) / (ln2 * ln2)); 41 | k_ = (int)ceil(bpe * ln2); 42 | bits_ = std::vector((int)bpe * m); 43 | } 44 | 45 | inline bool BloomFilter::check(const char* buff, size_t len) const { 46 | return check(hash(buff, len)); 47 | } 48 | 49 | inline bool BloomFilter::check(const hash_t hash) const { 50 | uint32_t hits = 0; 51 | 52 | for (uint32_t i = 0; i < k_; ++i) 53 | if (bits_[(hash.first + i * hash.second) % bits_.size()]) 54 | ++hits; 55 | 56 | return (hits == k_); 57 | } 58 | 59 | inline bool BloomFilter::add(const char* buff, size_t len) { 60 | return add(hash(buff, len)); 61 | } 62 | 63 | inline bool BloomFilter::add(const hash_t hash) { 64 | uint32_t hits = 0; 65 | 66 | for (uint32_t i = 0; i < k_; ++i) { 67 | auto bit = bits_[(hash.first + i * hash.second) % bits_.size()]; 68 | 69 | if (bit) 70 | ++hits; 71 | else 72 | bit = true; 73 | } 74 | 75 | return (hits != k_); 76 | } 77 | 78 | inline BloomFilter::hash_t BloomFilter::hash(const char* buff, size_t len) const { 79 | hash_t hash(0,0); 80 | 81 | MurmurHash3_x86_32(buff, len, 112181, &hash.first); 82 | MurmurHash3_x86_32(buff, len, hash.first, &hash.second); 83 | return hash; 84 | } 85 | 86 | #endif 87 | -------------------------------------------------------------------------------- /lib/hashallocator.hh: -------------------------------------------------------------------------------- 1 | #ifndef CLICK_HASHALLOCATOR_HH 2 | #define CLICK_HASHALLOCATOR_HH 3 | #include 4 | #include 5 | #if HAVE_VALGRIND && HAVE_VALGRIND_MEMCHECK_H 6 | # include 7 | #endif 8 | 9 | class HashAllocator { public: 10 | 11 | HashAllocator(size_t size); 12 | ~HashAllocator(); 13 | 14 | inline void increase_size(size_t new_size) { 15 | assert(!_free && !_buffer && new_size >= _size); 16 | _size = new_size; 17 | } 18 | 19 | inline void *allocate(); 20 | inline void deallocate(void *p); 21 | 22 | void swap(HashAllocator &x); 23 | 24 | private: 25 | 26 | struct link { 27 | link *next; 28 | }; 29 | 30 | struct buffer { 31 | buffer *next; 32 | size_t pos; 33 | size_t maxpos; 34 | }; 35 | 36 | enum { 37 | min_buffer_size = 1024, 38 | #if CLICK_LINUXMODULE 39 | max_buffer_size = 16384, 40 | #else 41 | max_buffer_size = 1048576, 42 | #endif 43 | min_nelements = 8 44 | }; 45 | 46 | link *_free; 47 | buffer *_buffer; 48 | size_t _size; 49 | 50 | void *hard_allocate(); 51 | 52 | HashAllocator(const HashAllocator &x); 53 | HashAllocator &operator=(const HashAllocator &x); 54 | 55 | }; 56 | 57 | 58 | template 59 | class SizedHashAllocator : public HashAllocator { public: 60 | 61 | SizedHashAllocator() 62 | : HashAllocator(size) { 63 | } 64 | 65 | }; 66 | 67 | 68 | inline void *HashAllocator::allocate() 69 | { 70 | if (link *l = _free) { 71 | #ifdef VALGRIND_MEMPOOL_ALLOC 72 | VALGRIND_MEMPOOL_ALLOC(this, l, _size); 73 | VALGRIND_MAKE_MEM_DEFINED(&l->next, sizeof(l->next)); 74 | #endif 75 | _free = l->next; 76 | #ifdef VALGRIND_MAKE_MEM_DEFINED 77 | VALGRIND_MAKE_MEM_UNDEFINED(&l->next, sizeof(l->next)); 78 | #endif 79 | return l; 80 | } else if (_buffer && _buffer->pos < _buffer->maxpos) { 81 | void *data = reinterpret_cast(_buffer) + _buffer->pos; 82 | _buffer->pos += _size; 83 | #ifdef VALGRIND_MEMPOOL_ALLOC 84 | VALGRIND_MEMPOOL_ALLOC(this, data, _size); 85 | #endif 86 | return data; 87 | } else 88 | return hard_allocate(); 89 | } 90 | 91 | inline void HashAllocator::deallocate(void *p) 92 | { 93 | if (p) { 94 | reinterpret_cast(p)->next = _free; 95 | _free = reinterpret_cast(p); 96 | #ifdef VALGRIND_MEMPOOL_FREE 97 | VALGRIND_MEMPOOL_FREE(this, p); 98 | #endif 99 | } 100 | } 101 | 102 | #endif 103 | -------------------------------------------------------------------------------- /vis/jquery.flot.symbol.js: -------------------------------------------------------------------------------- 1 | /* 2 | Flot plugin that adds some extra symbols for plotting points. 3 | 4 | The symbols are accessed as strings through the standard symbol 5 | choice: 6 | 7 | series: { 8 | points: { 9 | symbol: "square" // or "diamond", "triangle", "cross" 10 | } 11 | } 12 | 13 | */ 14 | 15 | (function ($) { 16 | function processRawData(plot, series, datapoints) { 17 | // we normalize the area of each symbol so it is approximately the 18 | // same as a circle of the given radius 19 | 20 | var handlers = { 21 | square: function (ctx, x, y, radius, shadow) { 22 | // pi * r^2 = (2s)^2 => s = r * sqrt(pi)/2 23 | var size = radius * Math.sqrt(Math.PI) / 2; 24 | ctx.rect(x - size, y - size, size + size, size + size); 25 | }, 26 | diamond: function (ctx, x, y, radius, shadow) { 27 | // pi * r^2 = 2s^2 => s = r * sqrt(pi/2) 28 | var size = radius * Math.sqrt(Math.PI / 2); 29 | ctx.moveTo(x - size, y); 30 | ctx.lineTo(x, y - size); 31 | ctx.lineTo(x + size, y); 32 | ctx.lineTo(x, y + size); 33 | ctx.lineTo(x - size, y); 34 | }, 35 | triangle: function (ctx, x, y, radius, shadow) { 36 | // pi * r^2 = 1/2 * s^2 * sin (pi / 3) => s = r * sqrt(2 * pi / sin(pi / 3)) 37 | var size = radius * Math.sqrt(2 * Math.PI / Math.sin(Math.PI / 3)); 38 | var height = size * Math.sin(Math.PI / 3); 39 | ctx.moveTo(x - size/2, y + height/2); 40 | ctx.lineTo(x + size/2, y + height/2); 41 | if (!shadow) { 42 | ctx.lineTo(x, y - height/2); 43 | ctx.lineTo(x - size/2, y + height/2); 44 | } 45 | }, 46 | cross: function (ctx, x, y, radius, shadow) { 47 | // pi * r^2 = (2s)^2 => s = r * sqrt(pi)/2 48 | var size = radius * Math.sqrt(Math.PI) / 2; 49 | ctx.moveTo(x - size, y - size); 50 | ctx.lineTo(x + size, y + size); 51 | ctx.moveTo(x - size, y + size); 52 | ctx.lineTo(x + size, y - size); 53 | } 54 | } 55 | 56 | var s = series.points.symbol; 57 | if (handlers[s]) 58 | series.points.symbol = handlers[s]; 59 | } 60 | 61 | function init(plot) { 62 | plot.hooks.processDatapoints.push(processRawData); 63 | } 64 | 65 | $.plot.plugins.push({ 66 | init: init, 67 | name: 'symbols', 68 | version: '1.0' 69 | }); 70 | })(jQuery); 71 | -------------------------------------------------------------------------------- /lib/hashcode.hh: -------------------------------------------------------------------------------- 1 | #ifndef CLICK_HASHCODE_HH 2 | #define CLICK_HASHCODE_HH 3 | #include 4 | #include 5 | 6 | // Notes about the hashcode template: On GCC 4.3.0, "template <>" is required 7 | // on the specializations or they aren't used. Just plain overloaded 8 | // functions aren't used. The specializations must be e.g. "const char &", 9 | // not "char", or GCC complains about a specialization not matching the 10 | // general template. The main template takes a const reference for two 11 | // reasons. First, providing both "hashcode_t hashcode(T)" and "hashcode_t 12 | // hashcode(const T&)" leads to ambiguity errors. Second, providing only 13 | // "hashcode_t hashcode(T)" is slower by looks like 8% when T is a String, 14 | // because of copy constructors; for types with more expensive non-default 15 | // copy constructors this would probably be worse. 16 | 17 | typedef size_t hashcode_t; ///< Typical type for a hashcode() value. 18 | 19 | template 20 | inline hashcode_t hashcode(T const &x) { 21 | return x.hashcode(); 22 | } 23 | 24 | template <> 25 | inline hashcode_t hashcode(char const &x) { 26 | return x; 27 | } 28 | 29 | template <> 30 | inline hashcode_t hashcode(signed char const &x) { 31 | return x; 32 | } 33 | 34 | template <> 35 | inline hashcode_t hashcode(unsigned char const &x) { 36 | return x; 37 | } 38 | 39 | template <> 40 | inline hashcode_t hashcode(short const &x) { 41 | return x; 42 | } 43 | 44 | template <> 45 | inline hashcode_t hashcode(unsigned short const &x) { 46 | return x; 47 | } 48 | 49 | template <> 50 | inline hashcode_t hashcode(int const &x) { 51 | return x; 52 | } 53 | 54 | template <> 55 | inline hashcode_t hashcode(unsigned const &x) { 56 | return x; 57 | } 58 | 59 | template <> 60 | inline hashcode_t hashcode(long const &x) { 61 | return x; 62 | } 63 | 64 | template <> 65 | inline hashcode_t hashcode(unsigned long const &x) { 66 | return x; 67 | } 68 | 69 | template <> 70 | inline hashcode_t hashcode(long long const &x) { 71 | return (x >> 32) ^ x; 72 | } 73 | 74 | template <> 75 | inline hashcode_t hashcode(unsigned long long const &x) { 76 | return (x >> 32) ^ x; 77 | } 78 | 79 | #if HAVE_INT64_TYPES && !HAVE_INT64_IS_LONG && !HAVE_INT64_IS_LONG_LONG 80 | template <> 81 | inline hashcode_t hashcode(int64_t const &x) { 82 | return (x >> 32) ^ x; 83 | } 84 | 85 | template <> 86 | inline hashcode_t hashcode(uint64_t const &x) { 87 | return (x >> 32) ^ x; 88 | } 89 | #endif 90 | 91 | template 92 | inline hashcode_t hashcode(T * const &x) { 93 | return reinterpret_cast(x) >> 3; 94 | } 95 | 96 | template 97 | inline typename T::key_const_reference hashkey(const T &x) { 98 | return x.hashkey(); 99 | } 100 | 101 | #endif 102 | -------------------------------------------------------------------------------- /scripts/exp/memtierexperiments.py: -------------------------------------------------------------------------------- 1 | # list of experiments exported to calling script 2 | exps = [] 3 | 4 | # add experiments to the global list. 5 | # a local function that keeps the internal variables from becoming global 6 | def define_experiments(): 7 | serverCmd = "./obj/pqserver" 8 | popCmd = "./obj/poptable" 9 | clientCmd = "./obj/memtier_benchmark" 10 | 11 | exp = {'name': "logn", 'defs': []} 12 | sizes = [100, 1000, 10000, 100000, 1000000, 10000000, 100000000] 13 | popBase = "%s --padding=10" % (popCmd) 14 | clientBase = "%s --ratio 0:1 --key-padding 10" % (clientCmd) 15 | 16 | for s in sizes: 17 | exp['defs'].append( 18 | {'name': "pequod_%d" % s, 19 | 'cachecmd': "%s" % (serverCmd), 20 | 'populatecmd': "%s --maxkey=%d" % (popBase, s), 21 | 'clientcmd': "%s --key-maximum %d -P pequod" % (clientBase, s-1)}) 22 | 23 | exp['defs'].append( 24 | {'name': "redis_%d" % s, 25 | 'def_redis_compare': True, 26 | 'populatecmd': "%s --maxkey=%d --redis" % (popBase, s), 27 | 'clientcmd': "%s --key-maximum %d -P redis" % (clientBase, s-1)}) 28 | 29 | exp['defs'].append( 30 | {'name': "memcache_%d" % s, 31 | 'def_memcache_compare': True, 32 | 'def_memcache_args': "-m 81920 -M", 33 | 'populatecmd': "%s --maxkey=%d --memcached" % (popBase, s), 34 | 'clientcmd': "%s --key-maximum %d -P memcache_binary" % (clientBase, s-1)}) 35 | exps.append(exp) 36 | 37 | 38 | exp = {'name': "latency", 'defs': []} 39 | popBase = "%s --padding=10" % (popCmd) 40 | clientBase = "%s --ratio 0:1 --key-padding 10" % (clientCmd) 41 | 42 | exp['defs'].append( 43 | {'name': "pequod", 44 | 'cachecmd': "%s" % (serverCmd), 45 | 'populatecmd': "%s --maxkey=1000000" % (popBase), 46 | 'clientcmd': "%s --key-maximum 999999 -P pequod" % (clientBase)}) 47 | 48 | exp['defs'].append( 49 | {'name': "redis", 50 | 'def_redis_compare': True, 51 | 'populatecmd': "%s --maxkey=1000000 --redis" % (popBase), 52 | 'clientcmd': "%s --key-maximum 999999 -P redis" % (clientBase)}) 53 | 54 | exp['defs'].append( 55 | {'name': "memcache", 56 | 'def_memcache_compare': True, 57 | 'def_memcache_args': "-m 81920 -M", 58 | 'populatecmd': "%s --maxkey=1000000 --memcached" % (popBase), 59 | 'clientcmd': "%s --key-maximum 999999 -P memcache_binary" % (clientBase)}) 60 | exps.append(exp) 61 | 62 | 63 | exp = {'name': "noop", 'defs': []} 64 | 65 | exp['defs'].append({'name': "pequod", 66 | 'cachecmd': "%s" % (serverCmd), 67 | 'clientcmd': "%s --ratio 0:1 --key-padding 10 -P pequod --noop-get" % (clientCmd)}) 68 | exps.append(exp) 69 | 70 | 71 | define_experiments() 72 | -------------------------------------------------------------------------------- /app/pqinfo.tcc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "clp.h" 4 | #include "string.hh" 5 | #include "hosts.hh" 6 | #include "pqremoteclient.hh" 7 | #include "json.hh" 8 | #include "sock_helper.hh" 9 | 10 | using std::cout; 11 | using std::cerr; 12 | using std::endl; 13 | using namespace pq; 14 | 15 | tamed void get_info(const String& host, uint32_t port, Json& result) { 16 | tvars { 17 | tamer::fd fd; 18 | struct sockaddr_in sin; 19 | RemoteClient *rclient; 20 | Json j, rj; 21 | } 22 | 23 | rj.set("host", host); 24 | rj.set("port", port); 25 | 26 | sock_helper::make_sockaddr(host.c_str(), port, sin); 27 | twait { tamer::tcp_connect(sin.sin_addr, port, make_event(fd)); } 28 | if (!fd) { 29 | cerr << "Could not connect to server (" << host << ":" << port << ")" << endl; 30 | exit(-1); 31 | } 32 | 33 | rclient = new RemoteClient(fd, ""); 34 | 35 | twait { rclient->stats(make_event(j)); } 36 | if (j["id"]) 37 | rj.set("id", j["id"]); 38 | rj.set("stats", j); 39 | 40 | twait { rclient->control(Json().set("client_status", true), make_event(j)); } 41 | rj.set("mpfd_status", j); 42 | 43 | twait { rclient->control(Json().set("tamer_blocking", true), make_event(j)); } 44 | rj.set("tamer_blocking", j); 45 | 46 | result.push_back(rj); 47 | delete rclient; 48 | } 49 | 50 | static char envstr[] = "TAMER_NOLIBEVENT=1"; 51 | static Clp_Option options[] = {{ "host", 'h', 1000, Clp_ValStringNotOption, 0 }, 52 | { "port", 'p', 1001, Clp_ValInt, 0 }, 53 | { "hostfile", 'H', 1002, Clp_ValStringNotOption, 0 }}; 54 | 55 | int main(int argc, char** argv) { 56 | putenv(envstr); 57 | tamer::initialize(); 58 | 59 | String host = "localhost"; 60 | uint32_t port = 7000; 61 | String hostfile; 62 | const pq::Hosts* hosts = nullptr; 63 | Clp_Parser* clp = Clp_NewParser(argc, argv, sizeof(options) / sizeof(options[0]), options); 64 | Json result = Json::array(); 65 | 66 | while (Clp_Next(clp) != Clp_Done) { 67 | if (clp->option->long_name == String("host")) 68 | host = clp->val.s; 69 | else if (clp->option->long_name == String("port")) 70 | port = clp->val.i; 71 | else if (clp->option->long_name == String("hostfile")) 72 | hostfile = clp->val.s; 73 | else 74 | assert(false && "Not a parsable option."); 75 | } 76 | 77 | if (hostfile) { 78 | hosts = Hosts::get_instance(hostfile); 79 | for (auto& h : hosts->all()) 80 | get_info(h.name(), h.port(), result); 81 | } 82 | else 83 | get_info(host, port, result); 84 | 85 | tamer::loop(); 86 | tamer::cleanup(); 87 | 88 | if (result.size() == 1) 89 | result = result[0]; 90 | 91 | cout << result.unparse(Json::indent_depth(4)) << endl; 92 | return 0; 93 | } 94 | -------------------------------------------------------------------------------- /lib/hashallocator.cc: -------------------------------------------------------------------------------- 1 | // -*- related-file-name: "../include/click/hashallocator.hh" -*- 2 | /* 3 | * hashallocator.cc -- simple allocator for HashTable 4 | * Eddie Kohler 5 | * 6 | * Copyright (c) 2008 Meraki, Inc. 7 | * 8 | * Permission is hereby granted, free of charge, to any person obtaining a 9 | * copy of this software and associated documentation files (the "Software"), 10 | * to deal in the Software without restriction, subject to the conditions 11 | * listed in the Click LICENSE file. These conditions include: you must 12 | * preserve this copyright notice, and you cannot mention the copyright 13 | * holders in advertising related to the Software without their permission. 14 | * The Software is provided WITHOUT ANY WARRANTY, EXPRESS OR IMPLIED. This 15 | * notice is a summary of the Click LICENSE file; the license in that file is 16 | * legally binding. 17 | */ 18 | 19 | #include "hashallocator.hh" 20 | #include "compiler.hh" 21 | #include 22 | 23 | HashAllocator::HashAllocator(size_t size) 24 | : _free(0), _buffer(0), _size(size) 25 | { 26 | #ifdef VALGRIND_CREATE_MEMPOOL 27 | VALGRIND_CREATE_MEMPOOL(this, 0, 0); 28 | #endif 29 | } 30 | 31 | HashAllocator::~HashAllocator() 32 | { 33 | while (buffer *b = _buffer) { 34 | _buffer = b->next; 35 | delete[] reinterpret_cast(b); 36 | } 37 | #ifdef VALGRIND_DESTROY_MEMPOOL 38 | VALGRIND_DESTROY_MEMPOOL(this); 39 | #endif 40 | } 41 | 42 | void *HashAllocator::hard_allocate() 43 | { 44 | size_t nelements; 45 | 46 | if (!_buffer) 47 | nelements = (min_buffer_size - sizeof(buffer)) / _size; 48 | else { 49 | size_t shift = sizeof(size_t) * 8 - ffs_msb(_buffer->maxpos + _size); 50 | size_t new_size = 1 << (shift + 1); 51 | if (new_size > max_buffer_size) 52 | new_size = max_buffer_size; 53 | nelements = (new_size - sizeof(buffer)) / _size; 54 | } 55 | if (nelements < min_nelements) 56 | nelements = min_nelements; 57 | 58 | buffer *b = reinterpret_cast(new char[sizeof(buffer) + _size * nelements]); 59 | if (b) { 60 | b->next = _buffer; 61 | _buffer = b; 62 | b->maxpos = sizeof(buffer) + _size * nelements; 63 | b->pos = sizeof(buffer) + _size; 64 | void *data = reinterpret_cast(_buffer) + sizeof(buffer); 65 | #ifdef VALGRIND_MEMPOOL_ALLOC 66 | VALGRIND_MEMPOOL_ALLOC(this, data, _size); 67 | #endif 68 | return data; 69 | } else 70 | return 0; 71 | } 72 | 73 | void HashAllocator::swap(HashAllocator &x) 74 | { 75 | size_t xsize = _size; 76 | _size = x._size; 77 | x._size = xsize; 78 | 79 | link *xfree = _free; 80 | _free = x._free; 81 | x._free = xfree; 82 | 83 | buffer *xbuffer = _buffer; 84 | _buffer = x._buffer; 85 | x._buffer = xbuffer; 86 | 87 | #ifdef VALGRIND_MOVE_MEMPOOL 88 | VALGRIND_MOVE_MEMPOOL(this, reinterpret_cast(100)); 89 | VALGRIND_MOVE_MEMPOOL(&x, this); 90 | VALGRIND_MOVE_MEMPOOL(reinterpret_cast(100), &x); 91 | #endif 92 | } 93 | -------------------------------------------------------------------------------- /scripts/lib/aggregate.py: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env python 2 | 3 | import os, sys, glob, json 4 | 5 | def usage(): 6 | print sys.argv[0] + " PATH_TO_OUTPUT_FILES" + " EXPNAME" 7 | 8 | 9 | def aggregate(expname, *file_names): 10 | need_server_data = True 11 | output = {} 12 | output["client_logs"] = [] 13 | if expname == "karma": 14 | output["npost"] = 0 15 | output["nvote"] = 0 16 | output["ncomment"] = 0 17 | output["nread"] = 0 18 | else: 19 | output["nposts"] = 0 20 | output["nbackposts"] = 0 21 | output["nsubscribes"] = 0 22 | output["nchecks"] = 0 23 | output["nfull"] = 0 24 | output["nposts_read"] = 0 25 | output["nactive"] = 0 26 | output["nlogouts"] = 0 27 | output["user_time"] = 0 28 | output["system_time"] = 0 29 | output["wall_time"] = 0 30 | count = 0 31 | for f in file_names: 32 | fd=open(f) 33 | structured_obj = json.load(fd) 34 | output["client_logs"].append(structured_obj["log"]) 35 | if "server_logs" in structured_obj and need_server_data: 36 | output["server_logs"] = structured_obj["server_logs"] 37 | need_server_data = False 38 | if expname == "karma": 39 | output["npost"] += structured_obj["npost"] 40 | output["nvote"] += structured_obj["nvote"] 41 | output["ncomment"] += structured_obj["ncomment"] 42 | output["nread"] += structured_obj["nread"] 43 | else: 44 | output["nposts"] += structured_obj["nposts"] 45 | output["nbackposts"] += structured_obj["nbackposts"] 46 | output["nsubscribes"] += structured_obj["nsubscribes"] 47 | output["nchecks"] += structured_obj["nchecks"] 48 | output["nfull"] += structured_obj["nfull"] 49 | output["nposts_read"] += structured_obj["nposts_read"] 50 | output["nactive"] += structured_obj["nactive"] 51 | output["nlogouts"] += structured_obj["nlogouts"] 52 | output["user_time"] += structured_obj["user_time"] 53 | output["system_time"] += structured_obj["system_time"] 54 | output["wall_time"] += structured_obj["wall_time"] 55 | fd.close() 56 | 57 | output["user_time"] /= float(len(file_names)) 58 | output["system_time"] /= float(len(file_names)) 59 | output["wall_time"] /= float(len(file_names)) 60 | return output 61 | 62 | def aggregate_dir(dir, expname): 63 | if dir[-1] != '/': 64 | dir += '/' 65 | files = glob.glob(dir + "output_app_*") 66 | files = [os.path.abspath(x) for x in files] 67 | combined_json = aggregate(expname, *files) 68 | fdout = open(dir + "aggregate_output_app.json", 'w') 69 | json.dump(combined_json, fdout, indent=4, separators=(',', ': ')) 70 | 71 | if __name__ == "__main__": 72 | if len(sys.argv) != 3: 73 | usage() 74 | exit() 75 | base_path = sys.argv[1] 76 | expname = sys.argv[2] 77 | aggregate_dir(base_path, expname) 78 | -------------------------------------------------------------------------------- /lib/sock_helper.hh: -------------------------------------------------------------------------------- 1 | #ifndef SOCK_HELPER_HH 2 | #define SOCK_HELPER_HH 3 | 4 | #include 5 | #include 6 | #include 7 | #include 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | #include 16 | #include 17 | 18 | namespace pq { 19 | 20 | class sock_helper { 21 | public: 22 | static int connect(const char *host, int port) { 23 | int fd = socket(AF_INET, SOCK_STREAM, 0); 24 | assert(fd >= 0); 25 | struct sockaddr_in sin; 26 | make_sockaddr(host, port, sin); 27 | if (::connect(fd, (sockaddr *)&sin, sizeof(sin)) != 0) { 28 | perror((std::string("failed to connect to ") + 29 | std::string(host) + ":" + std::to_string(port)).c_str()); 30 | exit(EXIT_FAILURE); 31 | } 32 | return fd; 33 | } 34 | static int listen(int port, int backlog = 0) { 35 | int fd = socket(AF_INET, SOCK_STREAM, 0); 36 | assert(fd >= 0); 37 | int yes = 1; 38 | int r = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &yes, sizeof(yes)); 39 | mandatory_assert(r == 0); 40 | struct sockaddr_in sin; 41 | sin.sin_family = AF_INET; 42 | sin.sin_addr.s_addr = INADDR_ANY; 43 | sin.sin_port = htons(port); 44 | r = ::bind(fd, (struct sockaddr *) &sin, sizeof(sin)); 45 | mandatory_assert(r == 0); 46 | r = ::listen(fd, backlog ? backlog : 100); 47 | mandatory_assert(r == 0); 48 | return fd; 49 | } 50 | static int accept(int fd) { 51 | struct sockaddr_in sin; 52 | socklen_t sinlen; 53 | return accept(fd, sin, sinlen); 54 | } 55 | static int accept(int fd, struct sockaddr_in &sin, socklen_t &sinlen) { 56 | sinlen = sizeof(sin); 57 | int rfd = ::accept(fd, (struct sockaddr *) &sin, &sinlen); 58 | return rfd; 59 | } 60 | static void make_nodelay(int fd) { 61 | int yes = 1; 62 | int r = setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &yes, sizeof(yes)); 63 | mandatory_assert(r == 0); 64 | } 65 | static void make_nonblock(int fd) { 66 | int r = fcntl(fd, F_SETFL, fcntl(fd, F_GETFL, 0) | O_NONBLOCK); 67 | mandatory_assert(r == 0); 68 | } 69 | static uint64_t get_uid(const char *host, int port) { 70 | sockaddr_in sin; 71 | make_sockaddr(host, port, sin); 72 | return((uint64_t)(sin.sin_addr.s_addr) << 32) | port; 73 | } 74 | static void make_sockaddr(const char *host, int port, struct sockaddr_in &sin) { 75 | bzero(&sin, sizeof(sin)); 76 | sin.sin_family = AF_INET; 77 | in_addr_t a = inet_addr(host); 78 | if (a != INADDR_NONE) 79 | sin.sin_addr.s_addr = a; 80 | else { 81 | struct hostent *hp = gethostbyname(host); 82 | if (hp == NULL || hp->h_length != 4 || hp->h_addrtype != AF_INET) { 83 | fprintf(stderr, "Cannot get an IPV4 address for host %s\n", host); 84 | exit(-1); 85 | } 86 | sin.sin_addr.s_addr = ((struct in_addr *)(hp->h_addr))->s_addr; 87 | } 88 | sin.sin_port = htons(port); 89 | } 90 | }; 91 | 92 | } // namespace gstore 93 | #endif 94 | -------------------------------------------------------------------------------- /app/hackernews.tcc: -------------------------------------------------------------------------------- 1 | #include "hackernews.hh" 2 | #include "pqclient.hh" 3 | #include "pqmulticlient.hh" 4 | #include "redisadapter.hh" 5 | #include "memcacheadapter.hh" 6 | #include "pqdbpool.hh" 7 | 8 | namespace pq { 9 | 10 | typedef pq::PQHackerNewsShim local_shim_type; 11 | 12 | tamed void run_hn_local(HackernewsPopulator& hp, Server& server) { 13 | tvars { 14 | DirectClient* client = new DirectClient(server); 15 | local_shim_type* shim = new local_shim_type(*client, hp.writearound()); 16 | HackernewsRunner* hr = new HackernewsRunner(*shim, hp); 17 | } 18 | 19 | twait { hr->populate(make_event()); } 20 | twait { hr->run(make_event()); } 21 | delete hr; 22 | delete shim; 23 | delete client; 24 | server.control(Json().set("quit", true)); 25 | } 26 | 27 | typedef pq::PQHackerNewsShim remote_shim_type; 28 | 29 | tamed void run_hn_remote(HackernewsPopulator& hp, int client_port, 30 | const Hosts* hosts, const Hosts* dbhosts, 31 | const Partitioner* part) { 32 | tvars { 33 | MultiClient* mc = new MultiClient(hosts, part, client_port); 34 | remote_shim_type* shim = new remote_shim_type(*mc, hp.writearound()); 35 | HackernewsRunner* hr = new HackernewsRunner(*shim, hp); 36 | double start, midway, end; 37 | } 38 | twait { mc->connect(make_event()); } 39 | mc->set_wrlowat(1 << 11); 40 | twait { hr->populate(make_event()); } 41 | midway = tstamp(); 42 | twait { hr->run(make_event()); } 43 | end = tstamp(); 44 | std::cerr << "Populate took " << (midway-start)/1000000 << " Run took " << (end-midway)/1000000 << "\n"; 45 | delete hr; 46 | delete shim; 47 | delete mc; 48 | } 49 | 50 | 51 | typedef pq::SQLHackerNewsShim db_shim_type; 52 | 53 | tamed void run_hn_remote_db(HackernewsPopulator& hp, const DBPoolParams& dbparams) { 54 | tvars { 55 | pq::DBPool* client = new pq::DBPool(dbparams); 56 | db_shim_type* shim = new db_shim_type(*client); 57 | pq::HackernewsRunner* hr = new pq::HackernewsRunner(*shim, hp); 58 | } 59 | 60 | client->connect(); 61 | twait { hr->populate(make_event()); } 62 | twait { hr->run(make_event()); } 63 | delete hr; 64 | delete shim; 65 | delete client; 66 | } 67 | 68 | 69 | #if HAVE_HIREDIS_HIREDIS_H 70 | typedef pq::HashHackerNewsShim redis_shim_type; 71 | 72 | tamed void run_hn_remote_redis(HackernewsPopulator& hp) { 73 | tvars { 74 | pq::RedisClient* client = new pq::RedisClient(); 75 | redis_shim_type* shim = new redis_shim_type(*client); 76 | pq::HackernewsRunner* hr = new pq::HackernewsRunner(*shim, hp); 77 | } 78 | 79 | client->connect(); 80 | twait { hr->populate(make_event()); } 81 | twait { hr->run(make_event()); } 82 | delete hr; 83 | delete shim; 84 | delete client; 85 | } 86 | #else 87 | void run_hn_remote_redis(HackernewsPopulator&) { 88 | mandatory_assert(false && "Not configured for Redis!"); 89 | } 90 | #endif 91 | 92 | 93 | #if HAVE_MEMCACHED_PROTOCOL_BINARY_H 94 | typedef pq::HashHackerNewsShim memcache_shim_type; 95 | 96 | tamed void run_hn_remote_memcache(HackernewsPopulator& hp) { 97 | tvars { 98 | pq::MemcacheClient* client = new pq::MemcacheClient(); 99 | memcache_shim_type* shim = new memcache_shim_type(*client); 100 | pq::HackernewsRunner* hr = new pq::HackernewsRunner(*shim, hp); 101 | } 102 | 103 | twait { client->connect(make_event()); } 104 | twait { hr->populate(make_event()); } 105 | twait { hr->run(make_event()); } 106 | delete hr; 107 | delete shim; 108 | delete client; 109 | } 110 | #else 111 | void run_hn_remote_memcache(HackernewsPopulator&) { 112 | mandatory_assert(false && "Not configured for Memcache!"); 113 | } 114 | #endif 115 | 116 | } 117 | -------------------------------------------------------------------------------- /scripts/exp/evictionexperiments.py: -------------------------------------------------------------------------------- 1 | # list of experiments exported to calling script 2 | exps = [] 3 | 4 | # add experiments to the global list. 5 | # a local function that keeps the internal variables from becoming global 6 | def define_experiments(): 7 | binary = True 8 | binaryflag = "" if binary else "--no-binary" 9 | partfunc = "twitternew" if binary else "twitternew-text" 10 | fetch = "--fetch" 11 | 12 | serverCmd = "./obj/pqserver --round-robin=1024" 13 | appCmd = "./obj/pqserver --twitternew --verbose" 14 | clientCmd = "%s %s %s" % (appCmd, binaryflag, fetch) 15 | 16 | # remote eviction experiment 17 | # run on a real network using a two-tier deployment (1 cache, 1 backing, 1 client). 18 | exp = {'name': "evict-policy", 'defs': []} 19 | users = "--graph=/mnt/pequod/twitter_graph_1.8M.dat" 20 | evict = "--evict-periodic --mem-lo=20000 --mem-hi=22500" 21 | points = [0, 5] 22 | 23 | for urate in points: 24 | clientBase = "%s %s --popduration=0 --duration=1000000000 --checklimit=62795845 " \ 25 | "--pactive=70 --ppost=1 --pread=100 --psubscribe=%d --plogout=%d " \ 26 | "--prevalidate --log-rtt" % (clientCmd, users, 2 * urate, urate) 27 | 28 | suffix = "-simple" if urate == 0 else "" 29 | 30 | exp['defs'].append( 31 | {'name': "no-evict%s" % (suffix), 32 | 'def_part': partfunc, 33 | 'backendcmd': "%s" % (serverCmd), 34 | 'cachecmd': "%s" % (serverCmd), 35 | 'clientcmd': "%s" % (clientBase)}) 36 | 37 | ''' 38 | exp['defs'].append( 39 | {'name': "rand-tomb%s" % (suffix), 40 | 'def_part': partfunc, 41 | 'backendcmd': "%s" % (serverCmd), 42 | 'cachecmd': "%s %s --evict-rand --evict-tomb" % (serverCmd, evict), 43 | 'clientcmd': "%s" % (clientBase)}) 44 | 45 | exp['defs'].append( 46 | {'name': "rand-notomb%s" % (suffix), 47 | 'def_part': partfunc, 48 | 'backendcmd': "%s" % (serverCmd), 49 | 'cachecmd': "%s %s --evict-rand --no-evict-tomb" % (serverCmd, evict), 50 | 'clientcmd': "%s" % (clientBase)}) 51 | ''' 52 | 53 | exp['defs'].append( 54 | {'name': "lru-tomb%s" % (suffix), 55 | 'def_part': partfunc, 56 | 'backendcmd': "%s" % (serverCmd), 57 | 'cachecmd': "%s %s --no-evict-multi --evict-tomb" % (serverCmd, evict), 58 | 'clientcmd': "%s" % (clientBase)}) 59 | 60 | exp['defs'].append( 61 | {'name': "lru-notomb%s" % (suffix), 62 | 'def_part': partfunc, 63 | 'backendcmd': "%s" % (serverCmd), 64 | 'cachecmd': "%s %s --no-evict-multi --no-evict-tomb" % (serverCmd, evict), 65 | 'clientcmd': "%s" % (clientBase)}) 66 | 67 | exp['defs'].append( 68 | {'name': "multi-sink-tomb%s" % (suffix), 69 | 'def_part': partfunc, 70 | 'backendcmd': "%s" % (serverCmd), 71 | 'cachecmd': "%s %s --evict-multi --evict-tomb --evict-pref-sink" % (serverCmd, evict), 72 | 'clientcmd': "%s" % (clientBase)}) 73 | 74 | exp['defs'].append( 75 | {'name': "multi-sink-notomb%s" % (suffix), 76 | 'def_part': partfunc, 77 | 'backendcmd': "%s" % (serverCmd), 78 | 'cachecmd': "%s %s --evict-multi --no-evict-tomb --evict-pref-sink" % (serverCmd, evict), 79 | 'clientcmd': "%s" % (clientBase)}) 80 | 81 | exp['defs'].append( 82 | {'name': "multi-remote-tomb%s" % (suffix), 83 | 'def_part': partfunc, 84 | 'backendcmd': "%s" % (serverCmd), 85 | 'cachecmd': "%s %s --evict-multi --evict-tomb --no-evict-pref-sink" % (serverCmd, evict), 86 | 'clientcmd': "%s" % (clientBase)}) 87 | 88 | exp['defs'].append( 89 | {'name': "multi-remote-notomb%s" % (suffix), 90 | 'def_part': partfunc, 91 | 'backendcmd': "%s" % (serverCmd), 92 | 'cachecmd': "%s %s --evict-multi --no-evict-tomb --no-evict-pref-sink" % (serverCmd, evict), 93 | 'clientcmd': "%s" % (clientBase)}) 94 | exps.append(exp) 95 | 96 | define_experiments() 97 | -------------------------------------------------------------------------------- /app/memcacheadapter.thh: -------------------------------------------------------------------------------- 1 | // -*- mode: c++ -*- 2 | #ifndef MEMCACHE_ADAPTER_HH 3 | #define MEMCACHE_ADAPTER_HH 4 | 5 | #include "str.hh" 6 | #include "string.hh" 7 | #include "json.hh" 8 | #include "hosts.hh" 9 | #include "partitioner.hh" 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include 15 | 16 | namespace pq { 17 | 18 | enum { use_yandong_modified_server = 0 }; 19 | 20 | struct MemcacheResponse { 21 | MemcacheResponse(uint32_t s) : seq(s) { } 22 | uint32_t seq; 23 | uint16_t status; 24 | uint64_t intval; 25 | String strval; 26 | Json jsonval; 27 | }; 28 | 29 | 30 | class MemcacheClient { 31 | public: 32 | MemcacheClient(); 33 | MemcacheClient(String host, uint32_t port); 34 | ~MemcacheClient(); 35 | 36 | tamed void connect(tamer::event<> done); 37 | void clear(); 38 | 39 | tamed void get(Str key, tamer::event e); 40 | tamed void get(Str key, int32_t offset, tamer::event e); 41 | tamed void set(Str key, Str value, tamer::event<> e); 42 | tamed void append(Str key, Str value, tamer::event<> e); 43 | tamed void increment(Str key, tamer::event<> e); 44 | tamed void length(Str key, tamer::event e); 45 | tamed void stats(tamer::event e); 46 | tamed void pace(tamer::event<> e); 47 | 48 | // only here for interface compliance 49 | void done_get(Str key); 50 | 51 | private: 52 | enum { wbuffsz_lo = 2 << 19, wbuffsz_hi = 2 << 20 }; 53 | enum { nout_lo = 2 << 15, nout_hi = 2 << 16 }; 54 | 55 | String host_; 56 | uint32_t port_; 57 | tamer::fd sock_; 58 | uint32_t seq_; 59 | volatile bool reading_; 60 | std::deque> rdwait_; 61 | uint32_t wbuffsz_; 62 | tamer::event<> pacer_; 63 | 64 | tamed void check_pace(tamer::event<> e); 65 | 66 | #if HAVE_MEMCACHED_PROTOCOL_BINARY_H 67 | uint32_t prep_command(uint8_t* data, uint8_t op, Str key, Str value, uint32_t seq); 68 | tamed void send_command(const uint8_t* data, uint32_t len, tamer::event e); 69 | tamed void read_loop(); 70 | #endif 71 | }; 72 | 73 | 74 | class MemcacheMultiClient { 75 | public: 76 | MemcacheMultiClient(const Hosts* hosts, const Partitioner* part); 77 | ~MemcacheMultiClient(); 78 | 79 | tamed void connect(tamer::event<> e); 80 | void clear(); 81 | 82 | inline void get(Str k, tamer::event e); 83 | inline void get(Str k, int32_t begin, tamer::event e); 84 | inline void set(Str k, Str v, tamer::event<> e); 85 | inline void append(Str k, Str v, tamer::event<> e); 86 | inline void increment(Str k, tamer::event<> e); 87 | inline void length(Str k, tamer::event e); 88 | tamed void stats(tamer::event e); 89 | tamed void pace(tamer::event<> e); 90 | 91 | // only here for interface compliance 92 | inline void done_get(Str k); 93 | 94 | private: 95 | const Hosts* hosts_; 96 | const Partitioner* part_; 97 | std::vector clients_; 98 | 99 | inline MemcacheClient* cache_for(Str k); 100 | }; 101 | 102 | inline void MemcacheMultiClient::get(Str k, tamer::event e) { 103 | cache_for(k)->get(k, e); 104 | } 105 | 106 | inline void MemcacheMultiClient::get(Str k, int32_t begin, tamer::event e) { 107 | cache_for(k)->get(k, begin, e); 108 | } 109 | 110 | inline void MemcacheMultiClient::set(Str k, Str v, tamer::event<> e) { 111 | cache_for(k)->set(k, v, e); 112 | } 113 | 114 | inline void MemcacheMultiClient::append(Str k, Str v, tamer::event<> e) { 115 | cache_for(k)->append(k, v, e); 116 | } 117 | 118 | inline void MemcacheMultiClient::increment(Str k, tamer::event<> e) { 119 | cache_for(k)->increment(k, e); 120 | } 121 | 122 | inline void MemcacheMultiClient::length(Str k, tamer::event e) { 123 | cache_for(k)->length(k, e); 124 | } 125 | 126 | inline void MemcacheMultiClient::done_get(Str) { 127 | } 128 | 129 | inline MemcacheClient* MemcacheMultiClient::cache_for(Str key) { 130 | int32_t owner = part_->owner(key); 131 | assert(owner >= 0 && owner < (int32_t)clients_.size() && "Make sure the partition function is correct."); 132 | return clients_[owner]; 133 | } 134 | 135 | } 136 | 137 | #endif 138 | -------------------------------------------------------------------------------- /src/pqmulticlient.thh: -------------------------------------------------------------------------------- 1 | #ifndef PQMULTICLIENT_HH_ 2 | #define PQMULTICLIENT_HH_ 3 | 4 | #include "pqremoteclient.hh" 5 | #include "pqdbpool.hh" 6 | #include "hosts.hh" 7 | #include "partitioner.hh" 8 | #include "sock_helper.hh" 9 | #include 10 | #include 11 | #include 12 | #include 13 | 14 | namespace pq { 15 | 16 | // todo: make DB client templated 17 | class MultiClient { 18 | public: 19 | MultiClient(const Hosts* hosts, const Partitioner* part, int colocateCacheServer); 20 | MultiClient(const Hosts* hosts, const Partitioner* part, int colocateCacheServer, 21 | const Hosts* dbhosts, const DBPoolParams* dbparams); 22 | ~MultiClient(); 23 | 24 | tamed void connect(tamer::event<> done); 25 | tamed void restart(tamer::event<> done); 26 | inline void clear(); 27 | 28 | typedef RemoteClient::iterator iterator; 29 | typedef RemoteClient::scan_result scan_result; 30 | 31 | tamed void add_join(const String& first, const String& last, 32 | const String& joinspec, event e); 33 | 34 | tamed void get(const String& key, event e); 35 | tamed void insert(const String& key, const String& value, event<> e); 36 | tamed void erase(const String& key, event<> e); 37 | 38 | tamed void insert_db(const String& key, const String& value, event<> e); 39 | tamed void erase_db(const String& key, event<> e); 40 | 41 | tamed void count(const String& first, const String& last, 42 | event e); 43 | tamed void count(const String& first, const String& last, 44 | const String& scanlast, event e); 45 | tamed void add_count(const String& first, const String& last, 46 | event e); 47 | tamed void add_count(const String& first, const String& last, 48 | const String& scanlast, event e); 49 | tamed void scan(const String& first, const String& last, 50 | event e); 51 | tamed void scan(const String& first, const String& last, 52 | const String& scanlast, event e); 53 | 54 | tamed void stats(event e); 55 | tamed void control(const Json& cmd, event e); 56 | 57 | tamed void pace(tamer::event<> done); 58 | tamed void flush(tamer::event<> done); 59 | 60 | inline void set_wrlowat(size_t limit); 61 | inline void set_rand_cache(bool rc); 62 | 63 | private: 64 | inline RemoteClient* cache_for(const String &key, bool randCache = false); 65 | inline DBPool* backend_for(const String &key) const; 66 | 67 | const Hosts* hosts_; 68 | const Partitioner* part_; 69 | std::vector clients_; 70 | RemoteClient* localNode_; 71 | int colocateCacheServer_; 72 | const Hosts* dbhosts_; 73 | const DBPoolParams* dbparams_; 74 | std::vector dbclients_; 75 | bool rand_cache_; 76 | std::default_random_engine gen_; 77 | }; 78 | 79 | inline void MultiClient::clear() { 80 | for (auto &c : clients_) 81 | delete c; 82 | clients_.clear(); 83 | localNode_ = nullptr; 84 | 85 | for (auto &c : dbclients_) 86 | delete c; 87 | dbclients_.clear(); 88 | } 89 | 90 | inline DBPool* MultiClient::backend_for(const String &key) const { 91 | int32_t owner = part_->owner(key); 92 | assert(owner >= 0 && owner < (int32_t)dbclients_.size() && "Make sure the partition function is correct."); 93 | return dbclients_[owner]; 94 | } 95 | 96 | inline RemoteClient* MultiClient::cache_for(const String &key, bool randCache) { 97 | if (colocateCacheServer_ >= 0) 98 | return localNode_; 99 | else { 100 | int owner; 101 | if (randCache) 102 | owner = part_->rand_cache(gen_); 103 | else 104 | owner = part_->owner(key); 105 | assert(owner >= 0 && owner < (int32_t)clients_.size() && "Make sure the partition function is correct."); 106 | return clients_[owner]; 107 | } 108 | } 109 | 110 | inline void MultiClient::set_wrlowat(size_t limit) { 111 | for (auto &c : clients_) 112 | c->set_wrlowat(limit); 113 | if (localNode_) 114 | localNode_->set_wrlowat(limit); 115 | } 116 | 117 | inline void MultiClient::set_rand_cache(bool rc) { 118 | rand_cache_ = rc; 119 | } 120 | 121 | } 122 | 123 | #endif 124 | -------------------------------------------------------------------------------- /src/pqdatum.hh: -------------------------------------------------------------------------------- 1 | #ifndef PEQUOD_DATUM_HH 2 | #define PEQUOD_DATUM_HH 3 | #include 4 | #include "pqbase.hh" 5 | #include "local_str.hh" 6 | 7 | namespace pq { 8 | class Sink; 9 | class Table; 10 | 11 | typedef boost::intrusive::set_base_hook< 12 | boost::intrusive::link_mode, 13 | boost::intrusive::optimize_size > pequod_set_base_hook; 14 | typedef boost::intrusive::set_member_hook< 15 | boost::intrusive::link_mode, 16 | boost::intrusive::optimize_size > pequod_set_member_hook; 17 | 18 | template class KeyHook { 19 | public: 20 | inline const T& key_holder() const { 21 | return *static_cast(this); 22 | } 23 | }; 24 | 25 | class Datum : public pequod_set_base_hook, public KeyHook { 26 | public: 27 | static const char table_marker[]; 28 | 29 | explicit inline Datum(Str key); 30 | inline Datum(Str key, const Sink* owner); 31 | inline Datum(Str key, const String& value); 32 | 33 | inline bool is_table() const; 34 | inline const Table& table() const; 35 | inline Table& table(); 36 | 37 | inline bool valid() const; 38 | inline void invalidate(); 39 | 40 | inline const Sink* owner() const; 41 | 42 | inline void ref(); 43 | inline void deref(); 44 | 45 | typedef Str key_type; 46 | inline key_type key() const; 47 | inline const String& value() const; 48 | inline String& value(); 49 | 50 | static const Datum empty_datum; 51 | static const Datum max_datum; 52 | 53 | private: 54 | LocalStr<24> key_; 55 | String value_; 56 | int refcount_; 57 | int owner_position_; 58 | const Sink* owner_; 59 | public: 60 | pequod_set_member_hook member_hook_; 61 | 62 | friend class Sink; 63 | }; 64 | 65 | struct KeyCompare { 66 | template 67 | inline bool operator()(const KeyHook& a, const String_base& b) const { 68 | return a.key_holder().key() < b; 69 | } 70 | template 71 | inline bool operator()(const KeyHook& a, Str b) const { 72 | return a.key_holder().key() < b; 73 | } 74 | template 75 | inline bool operator()(const String_base& a, const KeyHook& b) const { 76 | return a < b.key_holder().key(); 77 | } 78 | template 79 | inline bool operator()(Str a, const KeyHook& b) const { 80 | return a < b.key_holder().key(); 81 | } 82 | }; 83 | 84 | struct DatumDispose { 85 | inline void operator()(Datum* ptr) { 86 | delete ptr; 87 | } 88 | }; 89 | 90 | typedef boost::intrusive::set ServerStore; 91 | 92 | 93 | inline bool operator<(const Datum& a, const Datum& b) { 94 | return a.key() < b.key(); 95 | } 96 | inline bool operator==(const Datum& a, const Datum& b) { 97 | return a.key() == b.key(); 98 | } 99 | inline bool operator>(const Datum& a, const Datum& b) { 100 | return a.key() > b.key(); 101 | } 102 | 103 | inline Datum::Datum(Str key) 104 | : key_(key), refcount_(0), owner_{nullptr} { 105 | } 106 | 107 | inline Datum::Datum(Str key, const Sink* owner) 108 | : key_{key}, refcount_{0}, owner_{owner} { 109 | } 110 | 111 | inline Datum::Datum(Str key, const String& value) 112 | : key_(key), value_(value), refcount_(0), owner_{nullptr} { 113 | } 114 | 115 | inline bool Datum::is_table() const { 116 | return value_.data() == table_marker; 117 | } 118 | 119 | inline bool Datum::valid() const { 120 | return !is_invalidate_marker(value_); 121 | } 122 | 123 | inline void Datum::invalidate() { 124 | value_ = invalidate_marker(); 125 | if (refcount_ == 0) 126 | delete this; 127 | } 128 | 129 | inline const Sink* Datum::owner() const { 130 | return owner_; 131 | } 132 | 133 | inline void Datum::ref() { 134 | ++refcount_; 135 | } 136 | 137 | inline void Datum::deref() { 138 | if (--refcount_ == 0 && !valid()) 139 | delete this; 140 | } 141 | 142 | inline Str Datum::key() const { 143 | return key_; 144 | } 145 | 146 | inline const String& Datum::value() const { 147 | return value_; 148 | } 149 | 150 | inline String& Datum::value() { 151 | return value_; 152 | } 153 | 154 | inline std::ostream& operator<<(std::ostream& stream, const Datum& d) { 155 | return stream << d.key() << "=" << (d.valid() ? d.value() : String("INVALID")); 156 | } 157 | 158 | } // namespace 159 | #endif 160 | -------------------------------------------------------------------------------- /lib/sp_key.hh: -------------------------------------------------------------------------------- 1 | #ifndef SP_KEY_HH_ 2 | #define SP_KEY_HH_ 3 | #include "straccum.hh" 4 | #include "str.hh" 5 | #include 6 | #include 7 | #include 8 | #include 9 | 10 | 11 | namespace pq { 12 | 13 | inline void append_spkey(StringAccum &) { 14 | } 15 | 16 | template 17 | inline void append_spkey(StringAccum &sa, const T &x, ARGS... args) { 18 | sa << '|' << x; 19 | append_spkey(sa, args...); 20 | } 21 | 22 | inline String make_spkey() { 23 | return String(); 24 | } 25 | 26 | template 27 | inline String make_spkey(const String &prefix, ARGS... args) { 28 | StringAccum sa(prefix); 29 | append_spkey(sa, args...); 30 | return sa.take_string(); 31 | } 32 | 33 | template 34 | inline String make_spkey_first(const String &prefix, ARGS... args) { 35 | StringAccum sa(prefix); 36 | append_spkey(sa, args...); 37 | sa << '|'; 38 | return sa.take_string(); 39 | } 40 | 41 | template 42 | inline String make_spkey_last(const String &prefix, ARGS... args) { 43 | StringAccum sa(prefix); 44 | append_spkey(sa, args...); 45 | sa << '}'; 46 | return sa.take_string(); 47 | } 48 | 49 | String extract_spkey(int field, const String &spkey); 50 | String extract_spkey(int field, const std::string &spkey); 51 | String extract_spkey(int field, Str spkey); 52 | 53 | 54 | class SPKey { 55 | 56 | public: 57 | 58 | SPKey() { } 59 | 60 | SPKey(const String &key) : key_(key) { 61 | unmarshall(); 62 | } 63 | 64 | template 65 | SPKey(const String &prefix, ARGS... args) : prefix_(prefix) { 66 | set_args(args...); 67 | marshall(); 68 | } 69 | 70 | SPKey(const String &prefix, const std::vector &args) : prefix_(prefix), args_(args) { 71 | marshall(); 72 | } 73 | 74 | const String &key() const { 75 | return key_; 76 | } 77 | 78 | const String &prefix() const { 79 | return prefix_; 80 | } 81 | 82 | const std::vector &args() const { 83 | return args_; 84 | } 85 | 86 | template 87 | static String construct(const String &prefix, ARGS... args) { 88 | std::stringstream ss; 89 | ss << prefix; 90 | return append_args(ss, args...).str().c_str(); 91 | } 92 | 93 | static String construct(const String &prefix, const std::vector &args) { 94 | 95 | std::stringstream ss; 96 | ss << prefix; 97 | 98 | for (auto a = args.begin(); a != args.end(); ++a) { 99 | ss << "|" << *a; 100 | } 101 | 102 | return ss.str().c_str(); 103 | } 104 | 105 | private: 106 | 107 | void unmarshall() { 108 | 109 | mandatory_assert(!key_.empty()); 110 | 111 | int32_t p = 0; 112 | int32_t r = key_.find_left("|"); 113 | 114 | do { 115 | 116 | if (r < 0) { 117 | r = key_.length(); 118 | } 119 | 120 | if (prefix_.empty()) { 121 | prefix_ = key_.substring(p, r-p); 122 | } 123 | else { 124 | args_.push_back(key_.substring(p, r-p)); 125 | } 126 | 127 | p = r + 1; 128 | r = key_.find_left("|", p); 129 | } 130 | while(p < key_.length()); 131 | } 132 | 133 | void marshall() { 134 | 135 | mandatory_assert(!prefix_.empty()); 136 | key_ = construct(prefix_, args_); 137 | } 138 | 139 | template 140 | static std::stringstream &append_args(std::stringstream &ss, const String &arg, ARGS... args) { 141 | ss << "|" << arg; 142 | return append_args(ss, args...); 143 | } 144 | 145 | static std::stringstream &append_args(std::stringstream &ss) { 146 | return ss; 147 | } 148 | 149 | template 150 | void set_args(const String &arg, ARGS... rest) { 151 | 152 | args_.push_back(arg); 153 | set_args(rest...); 154 | } 155 | 156 | void set_args() { } 157 | 158 | String key_; 159 | String prefix_; 160 | std::vector args_; 161 | }; 162 | 163 | } 164 | 165 | #endif 166 | -------------------------------------------------------------------------------- /scripts/exp/testexperiments.py: -------------------------------------------------------------------------------- 1 | # list of experiments exported to calling script 2 | exps = [] 3 | 4 | # add experiments to the global list. 5 | # a local function that keeps the internal variables from becoming global 6 | def define_experiments(): 7 | binary = False 8 | binaryflag = "" if binary else "--no-binary" 9 | partfunc = "twitternew" if binary else "twitternew-text" 10 | 11 | serverCmd = "./obj/pqserver" 12 | initCmd = "%s --twitternew --verbose %s --initialize --no-populate --no-execute" % (serverCmd, binaryflag) 13 | populateCmd = "%s --twitternew --verbose %s --no-initialize --no-execute --popduration=0" % (serverCmd, binaryflag) 14 | clientCmd = "%s --twitternew --verbose %s --no-initialize --no-populate" % (serverCmd, binaryflag) 15 | 16 | users = "--nusers=1000" 17 | clientBase = "%s %s --duration=100000" % (clientCmd, users) 18 | 19 | postgresPopulateCmd = "%s --twitternew --verbose --dbshim %s --no-execute --popduration=0 %s" % \ 20 | (serverCmd, binaryflag, users) 21 | postgresClientCmd ="%s --twitternew --verbose --dbshim %s --no-populate %s --duration=100000 " \ 22 | "--psubscribe=0 --plogout=0" % (serverCmd, binaryflag, users) 23 | 24 | exps.append({'name': "basic", 25 | 'defs': [{'def_part': partfunc, 26 | 'backendcmd': "%s" % (serverCmd), 27 | 'cachecmd': "%s" % (serverCmd), 28 | 'initcmd': "%s" % (initCmd), 29 | 'populatecmd': "%s %s" % (populateCmd, users), 30 | 'clientcmd': "%s --fetch" % (clientBase)}]}) 31 | 32 | exps.append({'name': "writearound", 33 | 'defs': [{'def_part': partfunc, 34 | 'def_db_type': "postgres", 35 | 'def_db_writearound': True, 36 | 'def_db_flags': "-c synchronous_commit=off -c fsync=off", 37 | 'def_db_sql_script': "scripts/exp/cache.sql", 38 | 'backendcmd': "%s" % (serverCmd), 39 | 'cachecmd': "%s" % (serverCmd), 40 | 'initcmd': "%s" % (initCmd), 41 | 'populatecmd': "%s %s --dbpool-max=5 --dbpool-depth=10" % (populateCmd, users), 42 | 'clientcmd': "%s --dbpool-max=5 --dbpool-depth=10" % (clientBase)}]}) 43 | 44 | exps.append({'name': "postgres", 45 | 'defs': [{'def_part': partfunc, 46 | 'def_db_type': "postgres", 47 | #'def_db_in_memory': True, 48 | 'def_db_compare': True, 49 | 'def_db_sql_script': "scripts/exp/twitter-pg-schema.sql", 50 | 'def_db_flags': "-c synchronous_commit=off -c fsync=off " + \ 51 | "-c full_page_writes=off -c bgwriter_lru_maxpages=0 " + \ 52 | "-c bgwriter_delay=10000 -c checkpoint_segments=600", 53 | 'populatecmd': "%s --dbpool-max=5 --dbpool-depth=100" % (postgresPopulateCmd), 54 | 'clientcmd': "%s --dbpool-depth=100" % (postgresClientCmd)}]}) 55 | 56 | exps.append({'name': "redis", 57 | 'defs': [{'def_redis_compare': True, 58 | 'populatecmd': "%s %s --push --redis" % (populateCmd, users), 59 | 'clientcmd': "%s --push --redis" % (clientBase)}]}) 60 | 61 | exps.append({'name': "memcache", 62 | 'defs': [{'def_memcache_compare': True, 63 | 'def_memcache_args': "-m 1024 -M", 64 | 'populatecmd': "%s %s --push --memcached" % (populateCmd, users), 65 | 'clientcmd': "%s --push --memcached" % (clientBase)}]}) 66 | 67 | exps.append({'name': "eviction", 68 | 'defs': [{'def_part': partfunc, 69 | 'def_db_type': "postgres", 70 | 'def_db_writearound': True, 71 | 'def_db_sql_script': "scripts/exp/cache.sql", 72 | 'backendcmd': "%s --evict-periodic --mem-lo=20 --mem-hi=25" % (serverCmd), 73 | 'cachecmd': "%s --evict-periodic --mem-lo=15 --mem-hi=20" % (serverCmd), 74 | 'initcmd': "%s" % (initCmd), 75 | 'populatecmd': "%s %s" % (populateCmd, users), 76 | 'clientcmd': "%s" % (clientBase)}]}) 77 | 78 | define_experiments() 79 | -------------------------------------------------------------------------------- /lib/str.hh: -------------------------------------------------------------------------------- 1 | #ifndef STR_HH 2 | #define STR_HH 3 | #include "string_base.hh" 4 | #include 5 | #include 6 | 7 | struct Str : public String_base { 8 | typedef Str substring_type; 9 | typedef Str argument_type; 10 | 11 | const char *s; 12 | int len; 13 | 14 | Str() 15 | : s(0), len(0) { 16 | } 17 | template 18 | Str(const String_base& x) 19 | : s(x.data()), len(x.length()) { 20 | } 21 | Str(const char* s_) 22 | : s(s_), len(strlen(s_)) { 23 | } 24 | Str(const char* s_, int len_) 25 | : s(s_), len(len_) { 26 | } 27 | Str(const unsigned char* s_, int len_) 28 | : s(reinterpret_cast(s_)), len(len_) { 29 | } 30 | Str(const char *first, const char *last) 31 | : s(first), len(last - first) { 32 | precondition(first <= last); 33 | } 34 | Str(const unsigned char *first, const unsigned char *last) 35 | : s(reinterpret_cast(first)), len(last - first) { 36 | precondition(first <= last); 37 | } 38 | Str(const std::string& str) 39 | : s(str.data()), len(str.length()) { 40 | } 41 | Str(const uninitialized_type &unused) { 42 | (void) unused; 43 | } 44 | 45 | static const Str maxkey; 46 | 47 | void assign() { 48 | s = 0; 49 | len = 0; 50 | } 51 | template 52 | void assign(const String_base &x) { 53 | s = x.data(); 54 | len = x.length(); 55 | } 56 | void assign(const char *s_) { 57 | s = s_; 58 | len = strlen(s_); 59 | } 60 | void assign(const char *s_, int len_) { 61 | s = s_; 62 | len = len_; 63 | } 64 | 65 | const char *data() const { 66 | return s; 67 | } 68 | int length() const { 69 | return len; 70 | } 71 | char* mutable_data() { 72 | return const_cast(s); 73 | } 74 | 75 | Str prefix(int lenx) const { 76 | return Str(s, lenx < len ? lenx : len); 77 | } 78 | Str substring(const char *first, const char *last) const { 79 | if (first <= last && first >= s && last <= s + len) 80 | return Str(first, last); 81 | else 82 | return Str(); 83 | } 84 | Str substring(const unsigned char *first, const unsigned char *last) const { 85 | const unsigned char *u = reinterpret_cast(s); 86 | if (first <= last && first >= u && last <= u + len) 87 | return Str(first, last); 88 | else 89 | return Str(); 90 | } 91 | Str fast_substring(const char *first, const char *last) const { 92 | assert(begin() <= first && first <= last && last <= end()); 93 | return Str(first, last); 94 | } 95 | Str fast_substring(const unsigned char *first, const unsigned char *last) const { 96 | assert(ubegin() <= first && first <= last && last <= uend()); 97 | return Str(first, last); 98 | } 99 | Str ltrim() const { 100 | return String_generic::ltrim(*this); 101 | } 102 | Str rtrim() const { 103 | return String_generic::rtrim(*this); 104 | } 105 | Str trim() const { 106 | return String_generic::trim(*this); 107 | } 108 | 109 | operator std::string() const { 110 | return std::string(s, len); 111 | } 112 | 113 | long to_i() const { // XXX does not handle negative 114 | long x = 0; 115 | int p; 116 | for (p = 0; p < len && s[p] >= '0' && s[p] <= '9'; ++p) 117 | x = (x * 10) + s[p] - '0'; 118 | return p == len && p != 0 ? x : -1; 119 | } 120 | 121 | static Str snprintf(char *buf, size_t size, const char *fmt, ...) { 122 | va_list val; 123 | va_start(val, fmt); 124 | int n = vsnprintf(buf, size, fmt, val); 125 | va_end(val); 126 | return Str(buf, n); 127 | } 128 | }; 129 | 130 | struct inline_string : public String_base { 131 | int len; 132 | char s[0]; 133 | 134 | const char *data() const { 135 | return s; 136 | } 137 | int length() const { 138 | return len; 139 | } 140 | 141 | size_t size() const { 142 | return sizeof(inline_string) + len; 143 | } 144 | static size_t size(int len) { 145 | return sizeof(inline_string) + len; 146 | } 147 | 148 | template 149 | static inline_string *allocate(const char *s, int len, ALLOC &ti) { 150 | inline_string *r = (inline_string *) ti.allocate(size(len)); 151 | r->len = len; 152 | memcpy(r->s, s, len); 153 | return r; 154 | } 155 | template 156 | static inline_string *allocate(Str s, ALLOC &ti) { 157 | return allocate(s.s, s.len, ti); 158 | } 159 | template 160 | void deallocate(ALLOC &ti) { 161 | ti.deallocate(this, size()); 162 | } 163 | template 164 | void deallocate_rcu(ALLOC &ti) { 165 | ti.deallocate_rcu(this, size()); 166 | } 167 | }; 168 | 169 | #endif 170 | -------------------------------------------------------------------------------- /vis/vis.js: -------------------------------------------------------------------------------- 1 | var ngraphs = 0; 2 | var graphs; 3 | var lines; 4 | var raw; 5 | var colors = {client: "#444444", backend: "#0060ad", cache: "#dd181f"}; 6 | var symbols = {client: "circle", backend: "square", cache: "triangle"}; 7 | 8 | function clearGraphs() { 9 | graphs = document.getElementById('graphs'); 10 | for (var i = 0; i < ngraphs; ++i) 11 | graphs.removeChild(document.getElementById('graph_' + i.toString())); 12 | ngraphs = 0; 13 | lines = {}; 14 | } 15 | 16 | function redraw(rawjson) { 17 | clearGraphs(); 18 | raw = rawjson; 19 | 20 | if (rawjson.hasOwnProperty('log')) 21 | processLog(rawjson['log'], 'client_0', colors['client'], symbols['client']); 22 | else if (rawjson.hasOwnProperty('client_logs')) 23 | for (var l = 0; l < rawjson['client_logs'].length; ++l) 24 | processLog(rawjson['client_logs'][l], 'client_' + l, colors['client'], symbols['client']); 25 | 26 | if (rawjson.hasOwnProperty('server_logs') && rawjson['server_logs']) { 27 | var b = 0; 28 | var c = 0; 29 | for (var l = 0; l < rawjson['server_logs'].length; ++l) 30 | if (rawjson['server_logs'][l]['backend']) 31 | processLog(rawjson['server_logs'][l].data, 'backend_' + b++, colors['backend'], symbols['backend']); 32 | else 33 | processLog(rawjson['server_logs'][l].data, 'cache_' + c++, colors['cache'], symbols['cache']); 34 | } 35 | 36 | for (g in lines) { 37 | var plotid = createGraph(g); 38 | var alldata = []; 39 | var i = 0; 40 | 41 | for (l in lines[g]) 42 | alldata.push({data: lines[g][l].data, 43 | label : l, 44 | color : lines[g][l].color, 45 | points: { symbol: lines[g][l].symbol }}); 46 | 47 | var options = { series: { lines: { show: true }, 48 | points: { show: true }}, 49 | grid: { hoverable: true, clickable: true }, 50 | xaxis: {axisLabel: "Time (s)", axisLabelPadding: 10}, 51 | yaxis: {axisLabel: g, axisLabelPadding: 10, min: 0}, 52 | legend: {show: false, position: "nw"}} 53 | 54 | if (g.indexOf("_pct") !== -1) 55 | options.yaxis['max'] = 105; 56 | 57 | plotGraph(alldata, plotid, options); 58 | } 59 | } 60 | 61 | function processLog(logjson, pname, clr, sym) { 62 | for (key in logjson) { 63 | if (key.indexOf("time_us") !== -1) 64 | continue; 65 | if (!lines.hasOwnProperty(key)) 66 | lines[key] = {}; 67 | 68 | line = {data: [], color: clr, symbol: sym}; 69 | for (var i = 0; i < logjson[key].length; ++i) 70 | line.data.push([logjson[key][i][0] / 1000000.0, logjson[key][i][1]]); 71 | 72 | lines[key][pname] = line; 73 | } 74 | } 75 | 76 | function createGraph(gname) { 77 | graphs = document.getElementById('graphs'); 78 | 79 | var gtitle = document.createElement('div'); 80 | gtitle.innerHTML = '

' + gname + '

'; 81 | 82 | var gplot = document.createElement('div'); 83 | var plotid = 'plot_' + ngraphs; 84 | gplot.style.width = '900px'; 85 | gplot.style.height = '350px'; 86 | gplot.id = plotid; 87 | 88 | var graph = document.createElement('div'); 89 | graph.id = 'graph_' + ngraphs; 90 | 91 | graph.appendChild(gtitle); 92 | graph.appendChild(gplot); 93 | graphs.appendChild(graph); 94 | 95 | ++ngraphs; 96 | return plotid; 97 | } 98 | 99 | function showTooltip(tid, x, y, contents) { 100 | $('
' + contents + '
').css( { 101 | position: 'absolute', 102 | display: 'none', 103 | top: y + 5, 104 | left: x + 5, 105 | border: '1px solid #000000', 106 | padding: '5px', 107 | color: '#ffffff', 108 | 'font-size': '12px', 109 | 'background-color': '#333333', 110 | opacity: 0.80 111 | }).appendTo("body").fadeIn(200); 112 | } 113 | 114 | function plotGraph(lines, id, options) { 115 | var tid = id + "_tooltip"; 116 | var plotgid = "#" + id; 117 | var plottid = "#" + tid; 118 | $.plot($(plotgid), lines, options); 119 | 120 | var previousPoint = null; 121 | 122 | $(plotgid).bind("plothover", function (event, pos, item) { 123 | if (!item) { 124 | $(plottid).remove(); 125 | previousPoint = null; 126 | return; 127 | } 128 | if (previousPoint != item.dataIndex) { 129 | previousPoint = item.dataIndex; 130 | $(plottid).remove(); 131 | var ttip = item.series.label + ", (" + 132 | item.datapoint[0].toFixed(3) + ", " + 133 | item.datapoint[1].toFixed(3) + ")"; 134 | 135 | showTooltip(tid, item.pageX, item.pageY, ttip); 136 | } 137 | }); 138 | 139 | $(plotgid).bind("plotclick", function (event, pos, item) { 140 | $(plottid).remove(); 141 | }); 142 | } 143 | -------------------------------------------------------------------------------- /lib/local_str.hh: -------------------------------------------------------------------------------- 1 | #ifndef PEQUOD_LOCAL_STR_HH 2 | #define PEQUOD_LOCAL_STR_HH 3 | #include "string_base.hh" 4 | 5 | template 6 | class LocalStr : public String_base > { 7 | public: 8 | enum { local_capacity = C }; 9 | 10 | inline LocalStr(); 11 | inline LocalStr(const LocalStr& x); 12 | inline LocalStr(LocalStr&& x); 13 | template 14 | inline LocalStr(const String_base& x); 15 | inline LocalStr(const char* cstr); 16 | inline ~LocalStr(); 17 | 18 | inline const char* data() const; 19 | inline int length() const; 20 | 21 | inline char* mutable_data(); 22 | inline uint8_t* mutable_udata(); 23 | 24 | inline bool is_local() const; 25 | 26 | inline LocalStr& operator=(const LocalStr& x); 27 | inline LocalStr& operator=(LocalStr&& x); 28 | template 29 | inline LocalStr& operator=(const String_base& x); 30 | 31 | inline void assign_uninitialized(int length); 32 | 33 | private: 34 | union { 35 | int length; 36 | struct { 37 | int length; 38 | char* data; 39 | } rem; 40 | struct { 41 | int length; 42 | char data[local_capacity]; 43 | } loc; 44 | } u_; 45 | 46 | inline void initialize(const char* data, int length); 47 | inline void uninitialize(); 48 | }; 49 | 50 | template 51 | inline void LocalStr::initialize(const char* data, int length) { 52 | u_.length = length; 53 | if (length > local_capacity) { 54 | u_.rem.data = new char[length]; 55 | memcpy(u_.rem.data, data, length); 56 | } else 57 | memcpy(u_.loc.data, data, length); 58 | } 59 | 60 | template 61 | inline void LocalStr::uninitialize() { 62 | if (u_.length > local_capacity) 63 | delete[] u_.rem.data; 64 | } 65 | 66 | template 67 | inline LocalStr::LocalStr() { 68 | u_.length = 0; 69 | } 70 | 71 | template 72 | inline LocalStr::LocalStr(const LocalStr& x) { 73 | initialize(x.data(), x.length()); 74 | } 75 | 76 | template 77 | inline LocalStr::LocalStr(LocalStr&& x) { 78 | u_.length = x.u_.length; 79 | if (u_.length > local_capacity) { 80 | u_.rem.data = x.u_.rem.data; 81 | x.u_.length = 0; 82 | } else 83 | memcpy(u_.loc.data, x.u_.loc.data, u_.length); 84 | } 85 | 86 | template template 87 | inline LocalStr::LocalStr(const String_base& x) { 88 | initialize(x.data(), x.length()); 89 | } 90 | 91 | template 92 | inline LocalStr::LocalStr(const char* cstr) { 93 | initialize(cstr, strlen(cstr)); 94 | } 95 | 96 | template 97 | inline LocalStr::~LocalStr() { 98 | uninitialize(); 99 | } 100 | 101 | template 102 | inline const char* LocalStr::data() const { 103 | return u_.length > local_capacity ? u_.rem.data : u_.loc.data; 104 | } 105 | 106 | template 107 | inline char* LocalStr::mutable_data() { 108 | return u_.length > local_capacity ? u_.rem.data : u_.loc.data; 109 | } 110 | 111 | template 112 | inline uint8_t* LocalStr::mutable_udata() { 113 | return reinterpret_cast(mutable_data()); 114 | } 115 | 116 | template 117 | inline int LocalStr::length() const { 118 | return u_.length; 119 | } 120 | 121 | template 122 | inline bool LocalStr::is_local() const { 123 | return length() <= local_capacity; 124 | } 125 | 126 | template 127 | inline LocalStr& LocalStr::operator=(const LocalStr& x) { 128 | if (&x != this) { 129 | uninitialize(); 130 | initialize(x.data(), x.length()); 131 | } 132 | return *this; 133 | } 134 | 135 | template 136 | inline LocalStr& LocalStr::operator=(LocalStr&& x) { 137 | using std::swap; 138 | swap(u_.length, x.u_.length); 139 | char* old_rem = u_.rem.data; 140 | if (u_.length > local_capacity) 141 | u_.rem.data = x.u_.rem.data; 142 | else 143 | memcpy(u_.loc.data, x.u_.loc.data, u_.length); 144 | if (x.u_.length > local_capacity) 145 | x.u_.rem.data = old_rem; 146 | return *this; 147 | } 148 | 149 | template template 150 | inline LocalStr& LocalStr::operator=(const String_base& x) { 151 | char* old_rem = u_.length > local_capacity ? u_.rem.data : 0; 152 | u_.length = x.length(); 153 | if (u_.length > local_capacity) 154 | u_.rem.data = new char[u_.length]; 155 | memmove(const_cast(data()), x.data(), u_.length); 156 | delete[] old_rem; 157 | return *this; 158 | } 159 | 160 | template 161 | inline void LocalStr::assign_uninitialized(int length) { 162 | if (u_.length > local_capacity 163 | && (length > u_.length || length <= local_capacity)) 164 | delete[] u_.rem.data; 165 | if (length > local_capacity && length > u_.length) 166 | u_.rem.data = new char[length]; 167 | u_.length = length; 168 | } 169 | 170 | #endif 171 | -------------------------------------------------------------------------------- /lib/local_string.hh: -------------------------------------------------------------------------------- 1 | #ifndef LOCAL_STRING_HH 2 | #define LOCAL_STRING_HH 3 | #include "string_base.hh" 4 | #if SIZEOF_VOID_P != 8 && SIZEOF_VOID_P != 4 5 | # error "unknown SIZEOF_VOID_P" 6 | #endif 7 | 8 | class LocalString : public String_base { 9 | public: 10 | inline LocalString(); 11 | inline LocalString(const LocalString& x); 12 | inline LocalString(LocalString&& x); 13 | inline LocalString(const String& x); 14 | inline LocalString(String&& x); 15 | template 16 | inline LocalString(const String_base& x); 17 | inline ~LocalString(); 18 | 19 | inline const char* data() const; 20 | inline int length() const; 21 | 22 | inline LocalString& operator=(const LocalString& x); 23 | inline LocalString& operator=(LocalString&& x); 24 | inline LocalString& operator=(const String& x); 25 | inline LocalString& operator=(String&& x); 26 | template 27 | inline LocalString& operator=(const String_base& x); 28 | 29 | inline void swap(LocalString& x); 30 | 31 | private: 32 | enum { 33 | #if SIZEOF_VOID_P == 8 34 | local_capacity = 23 35 | #else 36 | local_capacity = 15 37 | #endif 38 | }; 39 | 40 | struct remote_rep_type __attribute__((packed)) { 41 | String::rep_type rem; 42 | char padding[3]; 43 | char remote_tag; // should be at same position as local_rep_type::tag 44 | // but not strictly required 45 | }; 46 | 47 | struct local_rep_type { 48 | char data[local_capacity]; 49 | char tag; 50 | }; 51 | 52 | union rep_type { 53 | remote_rep_type rem; 54 | local_rep_type loc; 55 | }; 56 | 57 | rep_type r_; 58 | 59 | }; 60 | 61 | inline LocalString::LocalString() { 62 | static_assert(sizeof(remote_rep_type) == sizeof(local_rep_type), 63 | "odd String::rep_type size"); 64 | r_.loc.tag = 1; 65 | } 66 | 67 | inline LocalString::LocalString(const LocalString& x) 68 | : r_(x.r_) { 69 | if (!r_.loc.tag) 70 | r_.rem.str.ref(); 71 | } 72 | 73 | inline LocalString::LocalString(LocalString&& x) { 74 | using std::swap; 75 | r_.loc.tag = 1; 76 | swap(r_, x.r_); 77 | } 78 | 79 | inline LocalString::LocalString(const String& x) { 80 | if (x.length() <= local_capacity) { 81 | memcpy(r_.loc.data, x.data(), x.length()); 82 | r_.loc.tag = x.length() + 1; 83 | } else { 84 | r_.rem.str = x.internal_rep(); 85 | r_.rem.str.ref(); 86 | r_.loc.tag = 0; 87 | } 88 | } 89 | 90 | inline LocalString::LocalString(String&& x) { 91 | if (x.length() <= local_capacity) { 92 | memcpy(r_.loc.data, x.data(), x.length()); 93 | r_.loc.tag = x.length() + 1; 94 | } else { 95 | r_.rem.str.memo = 0; 96 | x.swap(r_.rem.str); 97 | r_.loc.tag = 0; 98 | } 99 | } 100 | 101 | template 102 | inline LocalString::LocalString(const String_base& x) { 103 | if (x.length() <= local_capacity) { 104 | memcpy(r_.loc.data, x.data(), x.length()); 105 | r_.loc.tag = x.length() + 1; 106 | } else { 107 | String xx(x); 108 | r_.rem.str.memo = 0; 109 | xx.swap(r_.rem.str); 110 | r_.loc.tag = 0; 111 | } 112 | } 113 | 114 | inline LocalString::~LocalString() { 115 | if (!r_.loc.tag) 116 | r_.rem.str.deref(); 117 | } 118 | 119 | inline const char* LocalString::data() const { 120 | return r_.loc.tag ? r_.loc.data : r_.rem.str.data; 121 | } 122 | 123 | inline int LocalString::length() const { 124 | return r_.loc.tag ? r_.loc.tag - 1 : r_.rem.str.length; 125 | } 126 | 127 | inline LocalString& LocalString::operator=(const LocalString& x) { 128 | if (!x.r_.loc.tag) 129 | x.r_.rem.str.ref(); 130 | if (!r_.loc.tag) 131 | r_.rem.str.deref(); 132 | r_ = x.r_; 133 | return *this; 134 | } 135 | 136 | inline LocalString& LocalString::operator=(LocalString&& x) { 137 | swap(x); 138 | return *this; 139 | } 140 | 141 | inline LocalString& LocalString::operator=(const String& x) { 142 | if (!r_.loc.tag) 143 | r_.rem.str.deref(); 144 | if (x.length() <= local_capacity) { 145 | memcpy(r_.loc.data, x.data(), x.length()); 146 | r_.loc.tag = x.length() + 1; 147 | } else { 148 | r_.rem.str = x.internal_rep(); 149 | r_.rem.str.ref(); 150 | r_.loc.tag = 0; 151 | } 152 | return *this; 153 | } 154 | 155 | inline LocalString& LocalString::operator=(String&& x) { 156 | if (!r_.loc.tag) 157 | r_.rem.str.deref(); 158 | if (x.length() <= local_capacity) { 159 | memcpy(r_.loc.data, x.data(), x.length()); 160 | r_.loc.tag = x.length() + 1; 161 | } else { 162 | r_.rem.str.memo = 0; 163 | x.swap(r_.rem.str); 164 | r_.loc.tag = 0; 165 | } 166 | return *this; 167 | } 168 | 169 | template 170 | inline LocalString& LocalString::operator=(const String_base& x) { 171 | if (!r_.loc.tag) 172 | r_.rem.str.deref(); 173 | if (x.length() <= local_capacity) { 174 | memmove(r_.loc.data, x.data(), x.length()); 175 | r_.loc.tag = x.length() + 1; 176 | } else { 177 | String xx(x); 178 | r_.rem.str.memo = 0; 179 | xx.swap(r_.rem.str); 180 | r_.loc.tag = 0; 181 | } 182 | return *this; 183 | } 184 | 185 | inline void LocalString::swap(LocalString& x) { 186 | using std::swap; 187 | swap(r_, x.r_); 188 | } 189 | 190 | #endif 191 | -------------------------------------------------------------------------------- /src/pqpartition.cc: -------------------------------------------------------------------------------- 1 | #include "partitioner.hh" 2 | 3 | namespace pq { 4 | 5 | namespace { 6 | class DefaultPartitioner : public Partitioner { 7 | public: 8 | DefaultPartitioner(uint32_t nservers); 9 | }; 10 | 11 | DefaultPartitioner::DefaultPartitioner(uint32_t nservers) 12 | : Partitioner(0, nservers, 0) { 13 | ps_.add(partition1("", partition1::text, 1, 0, nservers)); 14 | } 15 | 16 | 17 | class UnitTestPartitioner : public Partitioner { 18 | public: 19 | UnitTestPartitioner(uint32_t nservers, int32_t default_owner); 20 | private: 21 | static const char prefixes_[]; 22 | }; 23 | const char UnitTestPartitioner::prefixes_[] = 24 | " !\"#$%&'()*+,-./0123456789:;<=>?@" 25 | "ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`" 26 | "abcdefghijklmnopqrstuvwxyz{|}"; 27 | 28 | UnitTestPartitioner::UnitTestPartitioner(uint32_t nservers, int default_owner) 29 | : Partitioner(default_owner, nservers, 0) { 30 | // make partitions, except leave '~' as default_owner. 31 | // Keys with the same first character are owned by the same server. 32 | for (size_t i = 0; prefixes_[i]; ++i) 33 | ps_.add(partition1(Str(&prefixes_[i], 1), partition1::text, 0, i % nservers, 1)); 34 | } 35 | 36 | 37 | class TwitterPartitioner : public Partitioner { 38 | public: 39 | TwitterPartitioner(uint32_t nservers, uint32_t nbacking, 40 | uint32_t default_owner, bool newtwitter, bool binary); 41 | }; 42 | 43 | TwitterPartitioner::TwitterPartitioner(uint32_t nservers, uint32_t nbacking, 44 | uint32_t default_owner, 45 | bool newtwitter, bool binary) 46 | : Partitioner(default_owner, nservers, nbacking) { 47 | 48 | ps_.add(partition1("c|", partition1::text, 0, 0, 1)); 49 | ps_.add(partition1("prog", partition1::text, 0, 0, 1)); 50 | 51 | if (newtwitter) { 52 | if (binary) { 53 | ps_.add(partition1("cp|", partition1::binary, 24, 0, (nbacking) ? nbacking : nservers)); 54 | ps_.add(partition1("p|", partition1::binary, 24, 0, (nbacking) ? nbacking : nservers)); 55 | ps_.add(partition1("s|", partition1::binary, 24, 0, (nbacking) ? nbacking : nservers)); 56 | ps_.add(partition1("t|", partition1::binary, 24, nbacking, nservers - nbacking)); 57 | } 58 | else { 59 | ps_.add(partition1("cp|", partition1::decimal, 5, 0, (nbacking) ? nbacking : nservers)); 60 | ps_.add(partition1("p|", partition1::decimal, 5, 0, (nbacking) ? nbacking : nservers)); 61 | ps_.add(partition1("s|", partition1::decimal, 5, 0, (nbacking) ? nbacking : nservers)); 62 | ps_.add(partition1("t|", partition1::decimal, 5, nbacking, nservers - nbacking)); 63 | } 64 | } 65 | else { 66 | ps_.add(partition1("f|", partition1::decimal, 5, 0, (nbacking) ? nbacking : nservers)); 67 | ps_.add(partition1("p|", partition1::decimal, 5, 0, (nbacking) ? nbacking : nservers)); 68 | ps_.add(partition1("s|", partition1::decimal, 5, 0, (nbacking) ? nbacking : nservers)); 69 | ps_.add(partition1("t|", partition1::decimal, 5, nbacking, nservers - nbacking)); 70 | } 71 | } 72 | 73 | 74 | class HackerNewsPartitioner : public Partitioner { 75 | public: 76 | HackerNewsPartitioner(uint32_t nservers, uint32_t nbacking, uint32_t default_owner); 77 | }; 78 | 79 | HackerNewsPartitioner::HackerNewsPartitioner(uint32_t nservers, uint32_t nbacking, 80 | uint32_t default_owner) 81 | : Partitioner(default_owner, nservers, nbacking) { 82 | 83 | ps_.add(partition1("a|", partition1::decimal, 5, 0, (nbacking) ? nbacking : nservers)); 84 | ps_.add(partition1("c|", partition1::decimal, 5, 0, (nbacking) ? nbacking : nservers)); 85 | ps_.add(partition1("v|", partition1::decimal, 5, 0, (nbacking) ? nbacking : nservers)); 86 | ps_.add(partition1("k|", partition1::decimal, 5, nbacking, nservers - nbacking)); 87 | ps_.add(partition1("ma|", partition1::decimal, 5, nbacking, nservers - nbacking)); 88 | } 89 | 90 | } 91 | 92 | 93 | Partitioner *Partitioner::make(const String &name, uint32_t nservers, 94 | uint32_t default_owner) { 95 | return Partitioner::make(name, 0, nservers, default_owner); 96 | } 97 | 98 | Partitioner *Partitioner::make(const String &name, uint32_t nbacking, 99 | uint32_t nservers, uint32_t default_owner) { 100 | if (name == "default" || nservers == 1) 101 | return new DefaultPartitioner(nservers); 102 | else if (name == "unit") 103 | return new UnitTestPartitioner(nservers, default_owner); 104 | else if (name == "twitter") 105 | return new TwitterPartitioner(nservers, nbacking, default_owner, false, false); 106 | else if (name == "twitternew") 107 | return new TwitterPartitioner(nservers, nbacking, default_owner, true, true); 108 | else if (name == "twitternew-text") 109 | return new TwitterPartitioner(nservers, nbacking, default_owner, true, false); 110 | else if (name == "hackernews") 111 | return new HackerNewsPartitioner(nservers, nbacking, default_owner); 112 | else 113 | assert(0 && "Unknown partition name"); 114 | return 0; 115 | } 116 | 117 | } 118 | -------------------------------------------------------------------------------- /lib/local_stack.hh: -------------------------------------------------------------------------------- 1 | #ifndef GSTORE_LOCAL_STACK_HH 2 | #define GSTORE_LOCAL_STACK_HH 1 3 | #include "compiler.hh" 4 | #include 5 | 6 | template > 7 | class local_stack { 8 | public: 9 | typedef bool (local_stack::*unspecified_bool_type)() const; 10 | typedef T *iterator; 11 | typedef const T *const_iterator; 12 | 13 | inline local_stack(const A &allocator = A()); 14 | inline ~local_stack(); 15 | 16 | inline int size() const; 17 | inline bool empty() const; 18 | inline operator unspecified_bool_type() const; 19 | inline bool operator!() const; 20 | 21 | inline iterator begin(); 22 | inline iterator end(); 23 | inline const_iterator begin() const; 24 | inline const_iterator end() const; 25 | inline const_iterator cbegin() const; 26 | inline const_iterator cend() const; 27 | 28 | inline T& operator[](int i); 29 | inline const T& operator[](int i) const; 30 | inline T& top(); 31 | inline const T& top() const; 32 | 33 | inline void push(const T& x); 34 | inline void push(T&& x); 35 | inline void pop(); 36 | 37 | private: 38 | struct rep : public A { 39 | T *v_; 40 | int size_; 41 | int capacity_; 42 | char lv_[sizeof(T) * N]; 43 | 44 | inline rep(const A& a); 45 | }; 46 | rep r_; 47 | 48 | void grow(); 49 | }; 50 | 51 | template 52 | inline local_stack::rep::rep(const A &a) 53 | : A(a), v_(reinterpret_cast(lv_)), size_(0), capacity_(N) { 54 | } 55 | 56 | template 57 | inline local_stack::local_stack(const A &allocator) 58 | : r_(allocator) { 59 | } 60 | 61 | template 62 | inline local_stack::~local_stack() { 63 | for (int i = 0; i < r_.size_; ++i) 64 | r_.destroy(&r_.v_[i]); 65 | if (r_.v_ != reinterpret_cast(r_.lv_)) 66 | r_.deallocate(r_.v_, r_.capacity_); 67 | } 68 | 69 | template 70 | inline int local_stack::size() const { 71 | return r_.size_; 72 | } 73 | 74 | template 75 | inline bool local_stack::empty() const { 76 | return r_.size_ == 0; 77 | } 78 | 79 | template 80 | inline local_stack::operator unspecified_bool_type() const { 81 | return empty() ? 0 : &local_stack::empty; 82 | } 83 | 84 | template 85 | inline bool local_stack::operator!() const { 86 | return empty(); 87 | } 88 | 89 | template 90 | void local_stack::grow() { 91 | T *m = r_.allocate(r_.capacity_ * 2); 92 | for (int i = 0; i < r_.size_; ++i) { 93 | r_.construct(&m[i], std::move(r_.v_[i])); 94 | r_.destroy(&r_.v_[i]); 95 | } 96 | if (r_.v_ != reinterpret_cast(r_.lv_)) 97 | r_.deallocate(r_.v_, r_.capacity_); 98 | r_.v_ = m; 99 | r_.capacity_ *= 2; 100 | } 101 | 102 | template 103 | inline typename local_stack::iterator local_stack::begin() { 104 | return r_.v_; 105 | } 106 | 107 | template 108 | inline typename local_stack::iterator local_stack::end() { 109 | return r_.v_ + r_.size_; 110 | } 111 | 112 | template 113 | inline typename local_stack::const_iterator local_stack::begin() const { 114 | return r_.v_; 115 | } 116 | 117 | template 118 | inline typename local_stack::const_iterator local_stack::end() const { 119 | return r_.v_ + r_.size_; 120 | } 121 | 122 | template 123 | inline typename local_stack::const_iterator local_stack::cbegin() const { 124 | return r_.v_; 125 | } 126 | 127 | template 128 | inline typename local_stack::const_iterator local_stack::cend() const { 129 | return r_.v_ + r_.size_; 130 | } 131 | 132 | template 133 | inline T &local_stack::operator[](int i) { 134 | return r_.v_[i]; 135 | } 136 | 137 | template 138 | inline const T &local_stack::operator[](int i) const { 139 | return r_.v_[i]; 140 | } 141 | 142 | template 143 | inline T &local_stack::top() { 144 | return r_.v_[r_.size_ - 1]; 145 | } 146 | 147 | template 148 | inline const T &local_stack::top() const { 149 | return r_.v_[r_.size_ - 1]; 150 | } 151 | 152 | template 153 | inline void local_stack::push(const T &x) { 154 | if (r_.size_ == r_.capacity_) 155 | grow(); 156 | r_.construct(&r_.v_[r_.size_], x); 157 | ++r_.size_; 158 | } 159 | 160 | template 161 | inline void local_stack::push(T &&x) { 162 | if (r_.size_ == r_.capacity_) 163 | grow(); 164 | r_.construct(&r_.v_[r_.size_], std::move(x)); 165 | ++r_.size_; 166 | } 167 | 168 | template 169 | inline void local_stack::pop() { 170 | --r_.size_; 171 | } 172 | 173 | #endif 174 | -------------------------------------------------------------------------------- /app/poptable.tcc: -------------------------------------------------------------------------------- 1 | #include 2 | #include 3 | #include "clp.h" 4 | #include "string.hh" 5 | #include "pqremoteclient.hh" 6 | #include "memcacheadapter.hh" 7 | #include "redisadapter.hh" 8 | #include "json.hh" 9 | #include "sock_helper.hh" 10 | 11 | using std::cout; 12 | using std::cerr; 13 | using std::endl; 14 | using namespace pq; 15 | 16 | enum { mode_pequod = 0, mode_redis = 1, mode_memcached = 2 }; 17 | 18 | tamed void populate(const Json& params) { 19 | tvars { 20 | tamer::fd fd; 21 | struct sockaddr_in sin; 22 | RemoteClient* pclient = nullptr; 23 | MemcacheClient* mclient = nullptr; 24 | RedisClient* rclient = nullptr; 25 | Json j; 26 | char key[128]; 27 | uint32_t ksz; 28 | uint32_t padding = params["padding"].as_i(); 29 | String value = String::make_fill('.', params["valsize"].as_i()); 30 | int32_t i; 31 | tamer::gather_rendezvous gr; 32 | } 33 | 34 | memset(key, 0, 128); 35 | 36 | switch(params["mode"].as_i()) { 37 | case mode_pequod: 38 | sock_helper::make_sockaddr(params["host"].as_s().c_str(), params["port"].as_i(), sin); 39 | twait { tamer::tcp_connect(sin.sin_addr, params["port"].as_i(), make_event(fd)); } 40 | if (!fd) { 41 | cerr << "Could not connect to pequod server!" << endl; 42 | exit(-1); 43 | } 44 | 45 | pclient = new RemoteClient(fd, ""); 46 | break; 47 | 48 | case mode_memcached: 49 | mclient = new MemcacheClient(params["host"].as_s(), params["port"].as_i()); 50 | twait { mclient->connect(make_event()); } 51 | break; 52 | 53 | case mode_redis: 54 | rclient = new RedisClient(params["host"].as_s(), params["port"].as_i()); 55 | rclient->connect(); 56 | break; 57 | } 58 | 59 | for (i = params["minkey"].as_i(); i < params["maxkey"].as_i(); ++i) { 60 | ksz = sprintf(key, "%s%0*u", params["prefix"].as_s().c_str(), padding, i); 61 | 62 | switch(params["mode"].as_i()) { 63 | case mode_pequod: 64 | pclient->insert(Str(key, ksz), value, gr.make_event()); 65 | twait { pclient->pace(make_event()); } 66 | break; 67 | 68 | case mode_memcached: 69 | mclient->set(Str(key, ksz), value, gr.make_event()); 70 | twait { mclient->pace(make_event()); } 71 | break; 72 | 73 | case mode_redis: 74 | rclient->set(Str(key, ksz), value, gr.make_event()); 75 | twait { rclient->pace(make_event()); } 76 | break; 77 | } 78 | } 79 | twait(gr); 80 | 81 | delete pclient; 82 | delete mclient; 83 | delete rclient; 84 | } 85 | 86 | static char envstr[] = "TAMER_NOLIBEVENT=1"; 87 | static Clp_Option options[] = {{ "host", 'h', 1000, Clp_ValStringNotOption, 0 }, 88 | { "port", 'p', 1001, Clp_ValInt, 0 }, 89 | { "minkey", 0, 1002, Clp_ValInt, 0 }, 90 | { "maxkey", 0, 1003, Clp_ValInt, 0 }, 91 | { "prefix", 0, 1004, Clp_ValStringNotOption, 0 }, 92 | { "padding", 0, 1005, Clp_ValInt, 0 }, 93 | { "valsize", 0, 1006, Clp_ValInt, 0 }, 94 | { "memcached", 0, 1007, 0, Clp_Negate }, 95 | { "redis", 0, 1008, 0, Clp_Negate}}; 96 | 97 | int main(int argc, char** argv) { 98 | putenv(envstr); 99 | tamer::initialize(); 100 | 101 | Json params = Json().set("host", "localhost") 102 | .set("port", 9000) 103 | .set("prefix", "m|") 104 | .set("minkey", 0) 105 | .set("maxkey", 1000000) 106 | .set("padding", 0) 107 | .set("valsize", 1024) 108 | .set("mode", mode_pequod); 109 | Clp_Parser* clp = Clp_NewParser(argc, argv, sizeof(options) / sizeof(options[0]), options); 110 | 111 | while (Clp_Next(clp) != Clp_Done) { 112 | if (clp->option->long_name == String("host")) 113 | params.set("host", clp->val.s); 114 | else if (clp->option->long_name == String("port")) 115 | params.set("port", clp->val.i); 116 | else if (clp->option->long_name == String("minkey")) 117 | params.set("minkey", clp->val.i); 118 | else if (clp->option->long_name == String("maxkey")) 119 | params.set("maxkey", clp->val.i); 120 | else if (clp->option->long_name == String("prefix")) 121 | params.set("prefix", clp->val.s); 122 | else if (clp->option->long_name == String("padding")) 123 | params.set("padding", clp->val.i); 124 | else if (clp->option->long_name == String("valsize")) 125 | params.set("valsize", clp->val.i); 126 | else if (clp->option->long_name == String("memcached")) 127 | params.set("mode", mode_memcached); 128 | else if (clp->option->long_name == String("redis")) 129 | params.set("mode", mode_redis); 130 | else 131 | assert(false && "Not a parsable option."); 132 | } 133 | 134 | populate(params); 135 | tamer::loop(); 136 | tamer::cleanup(); 137 | 138 | return 0; 139 | } 140 | -------------------------------------------------------------------------------- /lib/local_vector.hh: -------------------------------------------------------------------------------- 1 | #ifndef GSTORE_LOCAL_VECTOR_HH 2 | #define GSTORE_LOCAL_VECTOR_HH 1 3 | #include "compiler.hh" 4 | #include 5 | 6 | template > 7 | class local_vector { 8 | public: 9 | typedef bool (local_vector::*unspecified_bool_type)() const; 10 | typedef T* iterator; 11 | typedef const T* const_iterator; 12 | 13 | inline local_vector(const A& allocator = A()); 14 | inline ~local_vector(); 15 | 16 | inline int size() const; 17 | inline bool empty() const; 18 | inline operator unspecified_bool_type() const; 19 | inline bool operator!() const; 20 | 21 | inline iterator begin(); 22 | inline iterator end(); 23 | inline const_iterator begin() const; 24 | inline const_iterator end() const; 25 | inline const_iterator cbegin() const; 26 | inline const_iterator cend() const; 27 | 28 | inline T& operator[](int i); 29 | inline const T& operator[](int i) const; 30 | inline T& back(); 31 | inline const T& back() const; 32 | 33 | inline void clear(); 34 | inline void push_back(const T& x); 35 | inline void push_back(T&& x); 36 | inline void pop_back(); 37 | 38 | private: 39 | struct rep : public A { 40 | T *v_; 41 | int size_; 42 | int capacity_; 43 | char lv_[sizeof(T) * N]; 44 | 45 | inline rep(const A& a); 46 | }; 47 | rep r_; 48 | 49 | void grow(); 50 | }; 51 | 52 | template 53 | inline local_vector::rep::rep(const A& a) 54 | : A(a), v_(reinterpret_cast(lv_)), size_(0), capacity_(N) { 55 | } 56 | 57 | template 58 | inline local_vector::local_vector(const A& allocator) 59 | : r_(allocator) { 60 | } 61 | 62 | template 63 | inline local_vector::~local_vector() { 64 | for (int i = 0; i < r_.size_; ++i) 65 | r_.destroy(&r_.v_[i]); 66 | if (r_.v_ != reinterpret_cast(r_.lv_)) 67 | r_.deallocate(r_.v_, r_.capacity_); 68 | } 69 | 70 | template 71 | inline int local_vector::size() const { 72 | return r_.size_; 73 | } 74 | 75 | template 76 | inline bool local_vector::empty() const { 77 | return r_.size_ == 0; 78 | } 79 | 80 | template 81 | inline local_vector::operator unspecified_bool_type() const { 82 | return empty() ? 0 : &local_vector::empty; 83 | } 84 | 85 | template 86 | inline bool local_vector::operator!() const { 87 | return empty(); 88 | } 89 | 90 | template 91 | void local_vector::grow() { 92 | T *m = r_.allocate(r_.capacity_ * 2); 93 | for (int i = 0; i < r_.size_; ++i) { 94 | r_.construct(&m[i], std::move(r_.v_[i])); 95 | r_.destroy(&r_.v_[i]); 96 | } 97 | if (r_.v_ != reinterpret_cast(r_.lv_)) 98 | r_.deallocate(r_.v_, r_.capacity_); 99 | r_.v_ = m; 100 | r_.capacity_ *= 2; 101 | } 102 | 103 | template 104 | inline typename local_vector::iterator local_vector::begin() { 105 | return r_.v_; 106 | } 107 | 108 | template 109 | inline typename local_vector::iterator local_vector::end() { 110 | return r_.v_ + r_.size_; 111 | } 112 | 113 | template 114 | inline typename local_vector::const_iterator local_vector::begin() const { 115 | return r_.v_; 116 | } 117 | 118 | template 119 | inline typename local_vector::const_iterator local_vector::end() const { 120 | return r_.v_ + r_.size_; 121 | } 122 | 123 | template 124 | inline typename local_vector::const_iterator local_vector::cbegin() const { 125 | return r_.v_; 126 | } 127 | 128 | template 129 | inline typename local_vector::const_iterator local_vector::cend() const { 130 | return r_.v_ + r_.size_; 131 | } 132 | 133 | template 134 | inline T& local_vector::operator[](int i) { 135 | return r_.v_[i]; 136 | } 137 | 138 | template 139 | inline const T& local_vector::operator[](int i) const { 140 | return r_.v_[i]; 141 | } 142 | 143 | template 144 | inline T& local_vector::back() { 145 | return r_.v_[r_.size_ - 1]; 146 | } 147 | 148 | template 149 | inline const T& local_vector::back() const { 150 | return r_.v_[r_.size_ - 1]; 151 | } 152 | 153 | template 154 | inline void local_vector::push_back(const T& x) { 155 | if (r_.size_ == r_.capacity_) 156 | grow(); 157 | r_.construct(&r_.v_[r_.size_], x); 158 | ++r_.size_; 159 | } 160 | 161 | template 162 | inline void local_vector::push_back(T&& x) { 163 | if (r_.size_ == r_.capacity_) 164 | grow(); 165 | r_.construct(&r_.v_[r_.size_], std::move(x)); 166 | ++r_.size_; 167 | } 168 | 169 | template 170 | inline void local_vector::pop_back() { 171 | assert(r_.size_ > 0); 172 | --r_.size_; 173 | r_.destroy(&r_.v_[r_.size_]); 174 | } 175 | 176 | template 177 | inline void local_vector::clear() { 178 | for (int i = 0; i != r_.size_; ++i) 179 | r_.destroy(&r_.v_[i]); 180 | r_.size_ = 0; 181 | } 182 | 183 | #endif 184 | -------------------------------------------------------------------------------- /app/redisadapter.thh: -------------------------------------------------------------------------------- 1 | // -*- mode: c++ -*- 2 | #ifndef REDIS_ADAPTER_HH 3 | #define REDIS_ADAPTER_HH 4 | 5 | #include "json.hh" 6 | #include "str.hh" 7 | #include "string.hh" 8 | #include "hosts.hh" 9 | #include "partitioner.hh" 10 | #include 11 | #include 12 | #if HAVE_HIREDIS_HIREDIS_H 13 | #include 14 | #include 15 | #endif 16 | 17 | namespace pq { 18 | 19 | class RedisClient { 20 | public: 21 | typedef std::vector result_set; 22 | 23 | RedisClient(); 24 | RedisClient(String host, uint32_t port); 25 | ~RedisClient(); 26 | 27 | void connect(); 28 | void clear(); 29 | 30 | tamed void get(Str k, tamer::event e); 31 | tamed void get(Str k, int32_t begin, tamer::event e); 32 | tamed void getrange(Str k, int32_t begin, int32_t end, tamer::event e); 33 | tamed void set(Str k, Str v, tamer::event<> e); 34 | tamed void append(Str k, Str v, tamer::event<> e); 35 | tamed void increment(Str k, tamer::event<> e); 36 | tamed void length(Str k, tamer::event e); 37 | tamed void sadd(Str k, Str v, tamer::event<> e); 38 | tamed void smembers(Str k, tamer::event e); 39 | tamed void zadd(Str k, Str v, int32_t score, tamer::event<> e); 40 | tamed void zrangebyscore(Str k, int32_t begin, int32_t end, tamer::event e); 41 | 42 | tamed void pace(tamer::event<> e); 43 | 44 | // only here for interface compliance 45 | void done_get(Str k); 46 | 47 | private: 48 | #if HAVE_HIREDIS_HIREDIS_H 49 | enum { wbuffsz_lo = 2 << 19, wbuffsz_hi = 2 << 20 }; 50 | enum { nout_lo = 2 << 15, nout_hi = 2 << 16 }; 51 | 52 | redisAsyncContext* ctx_; 53 | #else 54 | void* ctx_; 55 | #endif 56 | 57 | String host_; 58 | uint32_t port_; 59 | tamer::event<> pacer_; 60 | uint32_t nout_; 61 | 62 | void check_pace(); 63 | }; 64 | 65 | #if HAVE_HIREDIS_HIREDIS_H 66 | void redis_cb_void(redisAsyncContext* c, void* reply, void* privdata); 67 | void redis_cb_string(redisAsyncContext* c, void* reply, void* privdata); 68 | void redis_cb_int32(redisAsyncContext* c, void* reply, void* privdata); 69 | void redis_cb_set(redisAsyncContext* c, void* reply, void* privdata); 70 | #endif 71 | 72 | class RedisMultiClient { 73 | public: 74 | typedef RedisClient::result_set result_set; 75 | 76 | RedisMultiClient(const Hosts* hosts, const Partitioner* part); 77 | ~RedisMultiClient(); 78 | 79 | void connect(); 80 | void clear(); 81 | 82 | inline void get(Str k, tamer::event e); 83 | inline void get(Str k, int32_t begin, tamer::event e); 84 | inline void getrange(Str k, int32_t begin, int32_t end, tamer::event e); 85 | inline void set(Str k, Str v, tamer::event<> e); 86 | inline void append(Str k, Str v, tamer::event<> e); 87 | inline void increment(Str k, tamer::event<> e); 88 | inline void length(Str k, tamer::event e); 89 | inline void sadd(Str k, Str v, tamer::event<> e); 90 | inline void smembers(Str k, tamer::event e); 91 | inline void zadd(Str k, Str v, int32_t score, tamer::event<> e); 92 | inline void zrangebyscore(Str k, int32_t begin, int32_t end, tamer::event e); 93 | 94 | tamed void pace(tamer::event<> e); 95 | 96 | private: 97 | const Hosts* hosts_; 98 | const Partitioner* part_; 99 | std::vector clients_; 100 | 101 | inline RedisClient* cache_for(Str k); 102 | }; 103 | 104 | inline void RedisMultiClient::get(Str k, tamer::event e) { 105 | cache_for(k)->get(k, e); 106 | } 107 | 108 | inline void RedisMultiClient::get(Str k, int32_t begin, tamer::event e) { 109 | cache_for(k)->get(k, begin, e); 110 | } 111 | 112 | inline void RedisMultiClient::getrange(Str k, int32_t begin, int32_t end, tamer::event e) { 113 | cache_for(k)->getrange(k, begin, end, e); 114 | } 115 | 116 | inline void RedisMultiClient::set(Str k, Str v, tamer::event<> e) { 117 | cache_for(k)->set(k, v, e); 118 | } 119 | 120 | inline void RedisMultiClient::append(Str k, Str v, tamer::event<> e) { 121 | cache_for(k)->append(k, v, e); 122 | } 123 | 124 | inline void RedisMultiClient::increment(Str k, tamer::event<> e) { 125 | cache_for(k)->increment(k, e); 126 | } 127 | 128 | inline void RedisMultiClient::length(Str k, tamer::event e) { 129 | cache_for(k)->length(k, e); 130 | } 131 | 132 | inline void RedisMultiClient::sadd(Str k, Str v, tamer::event<> e) { 133 | cache_for(k)->sadd(k, v, e); 134 | } 135 | 136 | inline void RedisMultiClient::smembers(Str k, tamer::event e) { 137 | cache_for(k)->smembers(k, e); 138 | } 139 | 140 | inline void RedisMultiClient::zadd(Str k, Str v, int32_t score, tamer::event<> e) { 141 | cache_for(k)->zadd(k, v, score, e); 142 | } 143 | 144 | inline void RedisMultiClient::zrangebyscore(Str k, int32_t begin, int32_t end, 145 | tamer::event e) { 146 | cache_for(k)->zrangebyscore(k, begin, end, e); 147 | } 148 | 149 | inline RedisClient* RedisMultiClient::cache_for(Str key) { 150 | int32_t owner = part_->owner(key); 151 | assert(owner >= 0 && owner < (int32_t)clients_.size() && "Make sure the partition function is correct."); 152 | return clients_[owner]; 153 | } 154 | 155 | 156 | #if HAVE_HIREDIS_HIREDIS_H 157 | struct RedisAdapterState { 158 | RedisAdapterState(redisAsyncContext *c); 159 | 160 | redisAsyncContext *context; 161 | volatile bool reading; 162 | volatile bool writing; 163 | tamer::event read_event; 164 | tamer::event write_event; 165 | }; 166 | 167 | tamed void redis_tamer_add_read(void* privdata); 168 | void redis_tamer_del_read(void* privdata); 169 | tamed void redis_tamer_add_write(void* privdata); 170 | void redis_tamer_del_write(void* privdata); 171 | void redis_tamer_cleanup(void* privdata); 172 | int32_t redis_tamer_attach(redisAsyncContext* ac); 173 | #endif 174 | 175 | } 176 | 177 | #endif 178 | -------------------------------------------------------------------------------- /src/pqremoteclient.thh: -------------------------------------------------------------------------------- 1 | // -*- mode: c++ -*- 2 | #ifndef PEQUOD_REMOTECLIENT_HH 3 | #define PEQUOD_REMOTECLIENT_HH 4 | #include 5 | #include 6 | #include "mpfd.hh" 7 | #include "pqrpc.hh" 8 | #include 9 | namespace pq { 10 | using tamer::event; 11 | 12 | class RemoteClient { 13 | public: 14 | inline RemoteClient(tamer::fd fd, String desc); 15 | inline RemoteClient(msgpack_fd* mpfd, String desc); 16 | inline ~RemoteClient(); 17 | 18 | tamed void add_join(const String& first, const String& last, 19 | const String& joinspec, event e); 20 | 21 | tamed void get(const String& key, event e); 22 | tamed void noop_get(const String& key, event e); 23 | tamed void insert(const String& key, const String& value, event<> e); 24 | tamed void erase(const String& key, event<> e); 25 | 26 | tamed void insert_db(const String& key, const String& value, event<> e); 27 | tamed void erase_db(const String& key, event<> e); 28 | 29 | tamed void count(const String& first, const String& last, 30 | event e); 31 | tamed void count(const String& first, const String& last, 32 | const String& scanlast, event e); 33 | tamed void add_count(const String& first, const String& last, 34 | event e); 35 | tamed void add_count(const String& first, const String& last, 36 | const String& scanlast, event e); 37 | 38 | template 39 | inline void pace(tamer::preevent done); 40 | 41 | inline void set_wrlowat(size_t limit); 42 | 43 | class iterator; 44 | class scanpair { 45 | public: 46 | scanpair() = default; 47 | scanpair(const Json* it) 48 | : it_(it) { 49 | } 50 | const String& key() const { 51 | return it_[0].as_s(); 52 | } 53 | const String& value() const { 54 | return it_[1].as_s(); 55 | } 56 | private: 57 | const Json* it_; 58 | friend class iterator; 59 | }; 60 | class iterator : public std::iterator { 61 | public: 62 | iterator() = default; 63 | explicit iterator(const Json* it) 64 | : it_(it) { 65 | } 66 | 67 | void operator++() { 68 | it_.it_ += 2; 69 | } 70 | void operator--() { 71 | it_.it_ -= 2; 72 | } 73 | size_t operator-(iterator x) const { 74 | return (it_.it_ - x.it_.it_) / 2; 75 | } 76 | 77 | bool operator==(iterator x) const { 78 | return it_.it_ == x.it_.it_; 79 | } 80 | bool operator!=(iterator x) const { 81 | return it_.it_ != x.it_.it_; 82 | } 83 | 84 | scanpair operator*() const { 85 | return it_; 86 | } 87 | const scanpair* operator->() const { 88 | return &it_; 89 | } 90 | private: 91 | scanpair it_; 92 | }; 93 | class scan_result { 94 | public: 95 | scan_result() = default; 96 | inline scan_result(Json&& x) 97 | : result_(std::move(x)) { 98 | } 99 | inline iterator begin() const { 100 | return iterator(result_.array_data()); 101 | } 102 | inline iterator end() const { 103 | return iterator(result_.array_data() + result_.size()); 104 | } 105 | inline void flush() { 106 | result_.clear(); 107 | } 108 | inline size_t size() const { 109 | return result_.size() / 2; 110 | } 111 | private: 112 | mutable Json result_; 113 | }; 114 | 115 | tamed void scan(const String& first, const String& last, 116 | event e); 117 | tamed void scan(const String& first, const String& last, 118 | const String& scanlast, event e); 119 | 120 | tamed void stats(event e); 121 | tamed void control(const Json& cmd, event e); 122 | 123 | inline msgpack_fd* fd() const; 124 | inline String description() const; 125 | 126 | protected: 127 | msgpack_fd* fd_; 128 | unsigned long seq_; 129 | bool alloc_; 130 | String description_; 131 | 132 | inline std::string twait_description(const char* prefix, 133 | const String& first = String(), 134 | const String& last = String()) const; 135 | }; 136 | 137 | 138 | inline RemoteClient::RemoteClient(tamer::fd fd, String desc) 139 | : fd_(new msgpack_fd(fd)), seq_(0), alloc_(true), description_(desc) { 140 | fd_->set_description(description_); 141 | } 142 | 143 | inline RemoteClient::RemoteClient(msgpack_fd* fd, String desc) 144 | : fd_(fd), seq_(0), alloc_(false), description_(desc) { 145 | fd_->set_description(description_); 146 | } 147 | 148 | inline RemoteClient::~RemoteClient() { 149 | if (alloc_) 150 | delete fd_; 151 | } 152 | 153 | inline msgpack_fd* RemoteClient::fd() const { 154 | return fd_; 155 | } 156 | 157 | inline String RemoteClient::description() const { 158 | return description_; 159 | } 160 | 161 | template 162 | inline void RemoteClient::pace(tamer::preevent done) { 163 | fd_->pace(std::move(done)); 164 | } 165 | 166 | inline void RemoteClient::set_wrlowat(size_t limit) { 167 | assert(fd_); 168 | fd_->set_wrlowat(limit); 169 | } 170 | 171 | inline std::string RemoteClient::twait_description(const char* prefix, 172 | const String& first, 173 | const String& last) const { 174 | std::stringstream buf; 175 | if (description_) 176 | buf << description_ << " "; 177 | if (prefix) 178 | buf << prefix << (first || last ? " " : ""); 179 | if (first) 180 | buf << first; 181 | if (last) 182 | buf << "," << last; 183 | return buf.str(); 184 | } 185 | 186 | } // namespace pq 187 | #endif 188 | -------------------------------------------------------------------------------- /src/mpfd.thh: -------------------------------------------------------------------------------- 1 | // -*- mode: c++ -*- 2 | #ifndef PEQUOD_MPFD_HH 3 | #define PEQUOD_MPFD_HH 4 | #include 5 | #include 6 | #include 7 | #include "msgpack.hh" 8 | #include 9 | #include 10 | 11 | class msgpack_fd { 12 | public: 13 | typedef bool (msgpack_fd::*unspecified_bool_type)() const; 14 | 15 | inline msgpack_fd(); 16 | explicit inline msgpack_fd(tamer::fd fd); 17 | inline msgpack_fd(tamer::fd rfd, tamer::fd wfd); 18 | ~msgpack_fd(); 19 | 20 | inline void initialize(tamer::fd fd); 21 | void initialize(tamer::fd rfd, tamer::fd wfd); 22 | inline void set_description(std::string description); 23 | 24 | inline bool valid() const; 25 | inline operator unspecified_bool_type() const; 26 | inline bool operator!() const; 27 | 28 | inline size_t wrlowat() const; 29 | inline void set_wrlowat(size_t wrlowat); 30 | 31 | void write(const Json& j); 32 | template 33 | void read_request(tamer::preevent done); 34 | inline void call(const Json& j, tamer::event reply); 35 | void flush(tamer::event done); 36 | void flush(tamer::event<> done); 37 | template 38 | inline void pace(tamer::preevent done); 39 | 40 | inline size_t sent_bytes() const; 41 | inline size_t recv_bytes() const; 42 | Json status() const; 43 | 44 | private: 45 | tamer::fd wfd_; 46 | tamer::fd rfd_; 47 | 48 | enum { wrcap = 1 << 17, wrhiwat = wrcap - 2048 }; 49 | struct wrelem { 50 | StringAccum sa; 51 | int pos; 52 | }; 53 | struct flushelem { 54 | tamer::event e; 55 | size_t wpos; 56 | }; 57 | std::deque wrelem_; 58 | size_t wrpos_; 59 | size_t wrsize_; 60 | size_t wrlowat_; 61 | size_t wrtotal_; 62 | bool wrblocked_; 63 | std::deque flushelem_; 64 | tamer::event<> wrwake_; 65 | tamer::event<> wrkill_; 66 | 67 | enum { rdcap = 1 << 17, rdbatch = 1024 }; 68 | String rdbuf_; 69 | size_t rdpos_; 70 | size_t rdlen_; 71 | size_t rdtotal_; 72 | int rdquota_; 73 | msgpack::streaming_parser rdparser_; 74 | 75 | struct replyelem { 76 | tamer::event e; 77 | size_t wpos; 78 | }; 79 | std::deque > rdreqwait_; 80 | std::deque rdreqq_; 81 | std::deque rdreplywait_; 82 | unsigned long rdreply_seq_; 83 | tamer::event<> rdwake_; 84 | tamer::event<> rdkill_; 85 | 86 | enum { wrpacelim = 1 << 20, rdpacelim = 1 << 14 }; 87 | enum { wrpacerecover = 1 << 19, rdpacerecover = 1 << 13 }; 88 | tamer::event<> pacer_; 89 | 90 | std::string description_; 91 | 92 | void check() const; 93 | bool dispatch(bool exit_on_request); 94 | inline bool read_until_request(bool exit_on_request); 95 | bool read_one_message(); 96 | void write_once(); 97 | inline bool need_pace() const; 98 | inline bool pace_recovered() const; 99 | inline void check_coroutines(); 100 | tamed void writer_coroutine(); 101 | tamed void reader_coroutine(); 102 | void clear_write(); 103 | void clear_read(); 104 | 105 | msgpack_fd(const msgpack_fd&) = delete; 106 | msgpack_fd& operator=(const msgpack_fd&) = delete; 107 | void construct(); 108 | }; 109 | 110 | inline msgpack_fd::msgpack_fd() { 111 | construct(); 112 | } 113 | 114 | inline msgpack_fd::msgpack_fd(tamer::fd rfd, tamer::fd wfd) { 115 | construct(); 116 | initialize(rfd, wfd); 117 | } 118 | 119 | inline msgpack_fd::msgpack_fd(tamer::fd fd) { 120 | construct(); 121 | initialize(fd); 122 | } 123 | 124 | inline void msgpack_fd::initialize(tamer::fd fd) { 125 | initialize(fd, fd); 126 | } 127 | 128 | inline void msgpack_fd::set_description(std::string description) { 129 | description_ = description; 130 | } 131 | 132 | inline bool msgpack_fd::valid() const { 133 | return rfd_.valid() && wfd_.valid(); 134 | } 135 | 136 | inline msgpack_fd::operator unspecified_bool_type() const { 137 | return valid() ? &msgpack_fd::valid : 0; 138 | } 139 | 140 | inline bool msgpack_fd::operator!() const { 141 | return !valid(); 142 | } 143 | 144 | template 145 | void msgpack_fd::read_request(tamer::preevent receiver) { 146 | if (!rdreqq_.empty()) { 147 | swap(*receiver.result_pointer(), rdreqq_.front()); 148 | rdreqq_.pop_front(); 149 | receiver.unblock(); 150 | } else if (read_until_request(true)) { 151 | swap(*receiver.result_pointer(), rdparser_.result()); 152 | receiver.unblock(); 153 | } else 154 | rdreqwait_.push_back(std::move(receiver)); 155 | } 156 | 157 | inline void msgpack_fd::call(const Json& j, tamer::event done) { 158 | assert(j.is_a() && j[1].is_i()); 159 | unsigned long seq = j[1].as_i(); 160 | assert(rdreplywait_.empty() || seq == rdreply_seq_ + rdreplywait_.size()); 161 | write(j); 162 | if (rdreplywait_.empty()) 163 | rdreply_seq_ = seq; 164 | if (done || !rdreplywait_.empty()) 165 | rdreplywait_.push_back(replyelem{std::move(done), wrpos_ + wrsize_}); 166 | read_until_request(false); 167 | } 168 | 169 | inline bool msgpack_fd::read_until_request(bool exit_on_request) { 170 | while (rdquota_ && read_one_message()) 171 | if (dispatch(exit_on_request)) 172 | return true; 173 | return false; 174 | } 175 | 176 | inline bool msgpack_fd::need_pace() const { 177 | return wrsize_ > wrpacelim || rdreplywait_.size() > rdpacelim; 178 | } 179 | 180 | inline bool msgpack_fd::pace_recovered() const { 181 | return wrsize_ <= wrpacerecover && rdreplywait_.size() <= rdpacerecover; 182 | } 183 | 184 | template 185 | inline void msgpack_fd::pace(tamer::preevent done) { 186 | if (need_pace()) 187 | pacer_ = tamer::distribute(std::move(pacer_), std::move(done)); 188 | else 189 | done(); 190 | } 191 | 192 | inline size_t msgpack_fd::sent_bytes() const { 193 | return wrtotal_; 194 | } 195 | 196 | inline size_t msgpack_fd::recv_bytes() const { 197 | return rdtotal_; 198 | } 199 | 200 | inline size_t msgpack_fd::wrlowat() const { 201 | return wrlowat_; 202 | } 203 | 204 | inline void msgpack_fd::set_wrlowat(size_t wrlowat) { 205 | wrlowat_ = wrlowat; 206 | } 207 | 208 | #endif 209 | -------------------------------------------------------------------------------- /src/pqremoteclient.tcc: -------------------------------------------------------------------------------- 1 | // -*- mode: c++ -*- 2 | #include "pqremoteclient.hh" 3 | 4 | namespace pq { 5 | 6 | tamed void RemoteClient::add_join(const String& first, const String& last, 7 | const String& joinspec, event e) { 8 | tvars { Json j, rj; unsigned long seq = this->seq_; } 9 | rj.set("range", Json::array(first, last)); 10 | twait { 11 | fd_->call(Json::array(pq_add_join, seq_, first, last, joinspec), 12 | make_event(j)); 13 | ++seq_; 14 | } 15 | if (j[2].is_i() && j[2].as_i() == pq_ok) 16 | rj.set("ok", true); 17 | if (j[3].is_s()) 18 | rj.set("message", j[3]); 19 | e(rj); 20 | } 21 | 22 | tamed void RemoteClient::get(const String& key, event e) { 23 | tvars { Json j; unsigned long seq = this->seq_; } 24 | twait [twait_description("get", key)] { 25 | fd_->call(Json::array(pq_get, seq_, key), make_event(j)); 26 | ++seq_; 27 | } 28 | assert(j[0] == -pq_get && j[1] == seq); 29 | e(j && j[2].to_i() == pq_ok ? j[3].to_s() : String()); 30 | } 31 | 32 | tamed void RemoteClient::noop_get(const String& key, event e) { 33 | tvars { Json j; unsigned long seq = this->seq_; } 34 | twait [twait_description("noop_get", key)] { 35 | fd_->call(Json::array(pq_noop_get, seq_, key), make_event(j)); 36 | ++seq_; 37 | } 38 | assert(j[0] == -pq_noop_get && j[1] == seq); 39 | e(j && j[2].to_i() == pq_ok ? j[3].to_s() : String()); 40 | } 41 | 42 | tamed void RemoteClient::insert(const String& key, const String& value, 43 | event<> e) { 44 | tvars { Json j; unsigned long seq = this->seq_; } 45 | twait [twait_description("insert", key)] { 46 | fd_->call(Json::array(pq_insert, seq_, key, value), make_event(j)); 47 | ++seq_; 48 | } 49 | assert(j[0] == -pq_insert && j[1] == seq); 50 | e(); 51 | } 52 | 53 | tamed void RemoteClient::erase(const String& key, event<> e) { 54 | tvars { Json j; unsigned long seq = this->seq_; } 55 | twait [twait_description("erase", key)] { 56 | fd_->call(Json::array(pq_erase, seq_, key), make_event(j)); 57 | ++seq_; 58 | } 59 | e(); 60 | } 61 | 62 | tamed void RemoteClient::insert_db(const String& key, const String& value, event<> e) { 63 | (void)key; 64 | (void)value; 65 | (void)e; 66 | mandatory_assert(false && "Not supported."); 67 | } 68 | 69 | tamed void RemoteClient::erase_db(const String& key, event<> e) { 70 | (void)key; 71 | (void)e; 72 | mandatory_assert(false && "Not supported."); 73 | } 74 | 75 | tamed void RemoteClient::count(const String& first, const String& last, 76 | event e) { 77 | tvars { Json j; unsigned long seq = this->seq_; } 78 | twait [twait_description("count", first, last)] { 79 | fd_->call(Json::array(pq_count, seq_, first, last), make_event(j)); 80 | ++seq_; 81 | } 82 | assert(j[0] == -pq_count && j[1] == seq); 83 | e(j && j[2].to_i() == pq_ok ? j[3].to_u64() : 0); 84 | } 85 | 86 | tamed void RemoteClient::count(const String& first, const String& last, 87 | const String& scanlast, event e) { 88 | tvars { Json j; unsigned long seq = this->seq_; } 89 | twait [twait_description("count", first, last)] { 90 | fd_->call(Json::array(pq_count, seq_, first, last, scanlast), make_event(j)); 91 | ++seq_; 92 | } 93 | assert(j[0] == -pq_count && j[1] == seq); 94 | e(j && j[2].to_i() == pq_ok ? j[3].to_u64() : 0); 95 | } 96 | 97 | tamed void RemoteClient::add_count(const String& first, const String& last, 98 | event e) { 99 | tvars { Json j; unsigned long seq = this->seq_; } 100 | twait [twait_description("count", first, last)] { 101 | fd_->call(Json::array(pq_count, seq_, first, last), make_event(j)); 102 | ++seq_; 103 | } 104 | assert(j[0] == -pq_count && j[1] == seq); 105 | if (e && j && j[2].to_i() == pq_ok) 106 | e(e.result() + j[3].to_u64()); 107 | else 108 | e(0); 109 | } 110 | 111 | tamed void RemoteClient::add_count(const String& first, const String& last, 112 | const String& scanlast, event e) { 113 | tvars { Json j; unsigned long seq = this->seq_; } 114 | twait [twait_description("count", first, last)] { 115 | fd_->call(Json::array(pq_count, seq_, first, last, scanlast), make_event(j)); 116 | ++seq_; 117 | } 118 | assert(j[0] == -pq_count && j[1] == seq); 119 | if (e && j && j[2].to_i() == pq_ok) 120 | e(e.result() + j[3].to_u64()); 121 | else 122 | e(0); 123 | } 124 | 125 | tamed void RemoteClient::scan(const String& first, const String& last, 126 | event e) { 127 | tvars { Json j; } 128 | twait [twait_description("scan", first, last)] { 129 | fd_->call(Json::array(pq_scan, seq_, first, last), make_event(j)); 130 | ++seq_; 131 | } 132 | e(scan_result(j && j[2].to_i() == pq_ok ? j[3] : Json::make_array())); 133 | } 134 | 135 | tamed void RemoteClient::scan(const String& first, const String& last, 136 | const String& scanlast, event e) { 137 | tvars { Json j; } 138 | twait [twait_description("scan", first, last)] { 139 | fd_->call(Json::array(pq_scan, seq_, first, last, scanlast), make_event(j)); 140 | ++seq_; 141 | } 142 | e(scan_result(j && j[2].to_i() == pq_ok ? j[3] : Json::make_array())); 143 | } 144 | 145 | tamed void RemoteClient::stats(event e) { 146 | tvars { Json j; unsigned long seq = this->seq_; } 147 | twait [twait_description("stats")] { 148 | assert(fd_->valid()); 149 | fd_->call(Json::array(pq_stats, seq_), make_event(j)); 150 | ++seq_; 151 | } 152 | 153 | e(j && j[2].to_i() == pq_ok ? j[3] : Json::make_object()); 154 | } 155 | 156 | tamed void RemoteClient::control(const Json& cmd, event e) { 157 | tvars { Json j; } 158 | twait [twait_description("control")] { 159 | assert(fd_->valid()); 160 | fd_->call(Json::array(pq_control, seq_, cmd), make_event(j)); 161 | ++seq_; 162 | } 163 | 164 | assert(j && j[2].to_i() == pq_ok); 165 | e(j[3]); 166 | } 167 | 168 | } 169 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # Pequod # 2 | 3 | This is the source release for Pequod, a fast, distributed key-value cache 4 | with builtin support for materialized views. Pequod is a research prototype 5 | and should not be used in any production environment. For background on 6 | Pequod's design, see the publication: 7 | 8 | [Easy Freshness with Pequod Cache Joins](https://www.usenix.org/system/files/conference/nsdi14/nsdi14-paper-kate.pdf) 9 | 10 | 11 | ## Contents ## 12 | 13 | * `PQDIR` This directory. 14 | * `PQDIR/src` Pequod source. 15 | * `PQDIR/lib` Supporting source. 16 | * `PQDIR/app` Experiment application source. 17 | * `PQDIR/scripts` Testing scripts. 18 | * `PQDIR/vis` Debugging visualization. 19 | 20 | 21 | ## Building ## 22 | 23 | Pequod builds on Linux and Mac OSX. To build: 24 | 25 | $ ./bootstrap.sh 26 | $ ./configure 27 | $ make 28 | 29 | Pequod requires a C++11 compatible compiler, and the Apple-supplied compiler might 30 | not be suitable for building on OSX. To use an alternate compiler (such as one 31 | installed with `homebrew`), specify the `CXX` variable at configuration time: 32 | 33 | $ ./configure CXX='g++-4.8 -std=gnu++11' 34 | 35 | For better deubgging with `gdb`, disable compiler optimizations: 36 | 37 | $ make NO_OPT=1 38 | 39 | For performance measurements, you should disable debugging features: 40 | 41 | $ ./configure --disable-tamer-debug 42 | $ make NDEBUG=1 43 | 44 | Pequod looks for alternate malloc libraries (jemalloc and tcmalloc) and will 45 | use one if found. 46 | 47 | Integration with other tools, like Postgres, memcached and Redis, is determined 48 | automatically at configuration time depending on the presence of client 49 | libraries on the host system. 50 | 51 | See `./configure --help` for more configure options. 52 | 53 | 54 | ## Testing ## 55 | 56 | After compilation, test the build by running unit tests: 57 | 58 | $ ./obj/pqserver --tests 59 | 60 | To run an individual test, name it explicitly: 61 | 62 | $ ./obj/pqserver test_simple 63 | 64 | Running an application in a single process (such as twitternew): 65 | 66 | $ ./obj/pqserver --twitternew 67 | 68 | will produce JSON output similar to: 69 | 70 | 
 71 | {
 72 |   "log": {
 73 |     "mem_max_rss_mb": [
 74 |       [6,129]
 75 |     ],
 76 |     "utime_us": [
 77 |       [6,1018973]
 78 |     ],
 79 |     "stime_us": [
 80 |       [6,53985]
 81 |     ],
 82 |     "cpu_pct": [
 83 |       [6,0]
 84 |     ]
 85 |   },
 86 |   "server_logs": null,
 87 |   "server_stats": [
 88 |     {
 89 |       "store_size": 720901,
 90 |       "source_ranges_size": 8484,
 91 |       "join_ranges_size": 1,
 92 |       "valid_ranges_size": 3483,
 93 |       "server_max_rss_mb": 155,
 94 |       "server_user_time": 0.539933,
 95 |       "server_system_time": 0.011884,
 96 |       "server_wall_time": 0.551801,
 97 |       "server_wall_time_insert": 0.132986,
 98 |       "server_wall_time_validate": 0.107239999999,
 99 |       "server_wall_time_evict": 0,
100 |       "server_wall_time_other": 0.311575000002,
101 |       "tables": [
102 |         {"name":"p","ninsert":1230,"store_size":1230,"source_ranges_size":5001,"nvalidate":1800,"nsubtables":5000},
103 |         {"name":"s","ninsert":563575,"store_size":563305,"source_ranges_size":3483,"nvalidate":3483,"nsubtables":5000},
104 |         {"name":"t","nmodify":156892,"nmodify_nohint":3483,"store_size":156366,"sink_ranges_size":3483,"nsubtables":3483}
105 |       ]
106 |     }
107 |   ],
108 |   "nposts": 1230,
109 |   "nbackposts": 0,
110 |   "nsubscribes": 13173,
111 |   "nchecks": 79075,
112 |   "nfull": 9766,
113 |   "nposts_read": 145572,
114 |   "nactive": 3483,
115 |   "nlogouts": 6522,
116 |   "user_time": 0.524327,
117 |   "system_time": 0.011857,
118 |   "wall_time": 0.536175
119 | }
120 |
121 | 122 | 123 | ## Network Testing ## 124 | 125 | Pequod can be started as a server, listening on a port for clients: 126 | 127 | $ ./obj/pqserver -kl=7777 & 128 | 129 | and to connect a client to a server and run an application: 130 | 131 | $ ./obj/pqserver -c=7777 --twitternew 132 | 133 | which will produce JSON results similar to the inline example above. 134 | 135 | Helper scripts can execute larger deployments on a multiprocessor and on 136 | Amazon EC2. They are in the `scripts` directory. For example, the command: 137 | 138 | $ ./scripts/local.py -c 3 basic 139 | 140 | will run a short Twitter benchmark using 3 Pequod cache servers and 1 client. 141 | The results of the experiment will be stored in `PQDIR/results` in a unique 142 | directory and linked to `PQDIR/last`. 143 | 144 | The scripts take as input the experiment definitions (see `PQDIR/scripts/exp`) 145 | using the `-e` parameter. The number of cache servers to run is given by `-c`, 146 | and if a two-tier deployment is desired, `-b` will designate the number of base 147 | servers. The number of clients used to execute the test workload is given with `-g`. 148 | 149 | The JSON output is directed to `output_app_0.json` if one client is used, and 150 | `aggregate_output_app.json` if more than one client is used. A log file 151 | (prefixed with `fart_`) is generated for each Pequod server and client in the 152 | experiment. 153 | 154 | The above command should print something like: 155 | 156 | 
157 | Running experiment in test 'basic'.
158 | ./obj/pqserver -H=results/exp_2014_04_07-19_24_12/hosts.txt -B=1 -P=twitternew-text -kl=7000
159 | ./obj/pqserver -H=results/exp_2014_04_07-19_24_12/hosts.txt -B=1 -P=twitternew-text -kl=7001
160 | ./obj/pqserver -H=results/exp_2014_04_07-19_24_12/hosts.txt -B=1 -P=twitternew-text -kl=7002
161 | ./obj/pqserver -H=results/exp_2014_04_07-19_24_12/hosts.txt -B=1 -P=twitternew-text -kl=7003
162 | Initializing cache servers.
163 | ./obj/pqserver --twitternew --verbose --no-binary --initialize --no-populate --no-execute -H=results/exp_2014_04_07-19_24_12/hosts.txt -B=1
164 | Populating backend.
165 | ./obj/pqserver --twitternew --verbose --no-binary --no-initialize --no-execute --popduration=0 --nusers=1000 -H=results/exp_2014_04_07-19_24_12/hosts.txt -B=1
166 | Starting app clients.
167 | ./obj/pqserver --twitternew --verbose --no-binary --no-initialize --no-populate --nusers=1000 --duration=100000 --fetch -H=results/exp_2014_04_07-19_24_12/hosts.txt -B=1
168 | Done experiment. Results are stored at results/exp_2014_04_07-19_24_12/basic/
169 |
170 | 171 | For other script options, refer to the python code. 172 | 173 | 174 | ## 3rd Party Benchmarks ## 175 | 176 | The 3rd party benchmark tool `memtier_benchmark` is included as a submodule. It 177 | is modified to include Pequod as a protocol option. It can be built after 178 | Pequod is built (it relies on Pequod files to be generated first): 179 | 180 | $ make memtier 181 | 182 | The benchmark can be executed manually against a running Pequod server, but it 183 | is probably easier to use the `PQDIR/scripts/memtier.py` script. See the Python code 184 | for details. 185 | 186 | 187 | ## Support ## 188 | 189 | This is research code, and you use it at your own risk. Requests for support by email may be ignored. 190 | 191 | 192 | ## License ## 193 | 194 | Pequod is released under the BSD license. See the `PQDIR/LICENSE` file for details. -------------------------------------------------------------------------------- /scripts/lib/ec2.py: -------------------------------------------------------------------------------- 1 | import sys 2 | import os.path 3 | from os import listdir, system, chmod 4 | import stat 5 | from time import sleep 6 | import subprocess 7 | from subprocess import Popen 8 | import shlex 9 | import json 10 | import boto 11 | from boto.ec2.connection import EC2Connection 12 | 13 | REGION = 'us-east-1' 14 | 15 | with open(os.path.join(os.path.dirname(os.path.realpath(__file__)), "aws.json")) as fp: 16 | cred = json.load(fp) 17 | AWS_ACCESS_KEY_ID = cred['AWS_ACCESS_KEY_ID'] 18 | AWS_SECRET_ACCESS_KEY = cred['AWS_SECRET_ACCESS_KEY'] 19 | 20 | KEY_NAME = "pequod-" + REGION 21 | SSH_KEY = os.path.join(os.path.dirname(os.path.realpath(__file__)), KEY_NAME + ".pem") 22 | 23 | # note: instance have 10Gb ethernet only if launched in the same placement group 24 | INSTANCE_TYPES = {'m1.small': {'bid': 0.06, 'hvm': False}, # use for script testing 25 | 'c3.8xlarge': {'bid': 0.76, 'hvm': True}, # 32 cores, 60GB RAM, 10Gb ethernet 26 | 'r3.8xlarge': {'bid': 0.76, 'hvm': True}} # 32 cores, 244GB RAM, 10Gb ethernet 27 | 28 | AMI_IDS = {'us-west-2': {'basic': 'ami-ccf297fc', 'hvm': 'ami-f8f297c8'}, 29 | 'us-east-1': {'basic': 'ami-bba18dd2', 'hvm': 'ami-e9a18d80'}} 30 | 31 | INSTANCE_TYPE_BACKING = 'r3.8xlarge' 32 | INSTANCE_TYPE_CACHE = 'r3.8xlarge' 33 | INSTANCE_TYPE_CLIENT = 'r3.8xlarge' 34 | 35 | 36 | conn = None 37 | 38 | def connect(): 39 | global conn 40 | if conn is None: 41 | conn = boto.ec2.connect_to_region(REGION, 42 | aws_access_key_id=AWS_ACCESS_KEY_ID, 43 | aws_secret_access_key=AWS_SECRET_ACCESS_KEY) 44 | chmod(SSH_KEY, stat.S_IREAD | stat.S_IWRITE) 45 | 46 | def checkout_machines(num, type, spot=False): 47 | print "Checking out %s machines." % (num) 48 | ami_type = 'hvm' if INSTANCE_TYPES[type]['hvm'] else 'basic' 49 | reservation = conn.run_instances(AMI_IDS[REGION][ami_type], 50 | min_count = num, 51 | max_count = num, 52 | key_name = KEY_NAME, 53 | instance_type = type, 54 | security_groups = ['pequod'], 55 | placement_group='pequod') 56 | return [i for i in reservation.instances] 57 | 58 | def request_machines(num, type): 59 | print "Requesting %s spot instances." % (num) 60 | ami_type = 'hvm' if INSTANCE_TYPES[type]['hvm'] else 'basic' 61 | return conn.request_spot_instances(INSTANCE_TYPES[type]['bid'], 62 | AMI_IDS[REGION][ami_type], 63 | count = num, 64 | key_name = KEY_NAME, 65 | instance_type = type, 66 | security_groups = ['pequod'], 67 | placement_group='pequod', 68 | launch_group='pequod') 69 | 70 | def startup_machines(instances): 71 | for i in instances: 72 | print "Starting instance %s" % (i.id) 73 | conn.start_instances([i.id]) 74 | 75 | def shutdown_machines(instances): 76 | for i in instances: 77 | if is_spot_instance(i): 78 | terminate_machines([i]) 79 | else: 80 | print "Shutting down instance %s" % (i.id) 81 | conn.stop_instances([i.id], True) 82 | 83 | def terminate_machines(instances): 84 | for i in instances: 85 | if (i.update(True) == "terminated"): 86 | continue 87 | 88 | print "Terminating instance %s" % (i.id) 89 | conn.terminate_instances([i.id]) 90 | 91 | if is_spot_instance(i): 92 | conn.cancel_spot_instance_requests([i.spot_instance_request_id]) 93 | 94 | def get_all_instances(): 95 | reservations = conn.get_all_instances() 96 | instances = [i for r in reservations for i in r.instances] 97 | return instances 98 | 99 | def get_running_instances(type=None, tag=None): 100 | return filter_instances("running", type, tag) 101 | 102 | def get_stopped_instances(type=None): 103 | return filter_instances("stopped", type, None) 104 | 105 | def get_instances(instance_ids): 106 | ret = [] 107 | all = get_all_instances() 108 | 109 | for i in instance_ids: 110 | for a in all: 111 | if a.id == i: 112 | ret.append(a) 113 | return ret 114 | 115 | def filter_instances(status, type, tag): 116 | instances = get_all_instances() 117 | filtered = [] 118 | for i in instances: 119 | if i.update(True) == status: 120 | if type and i.get_attribute('instanceType')['instanceType'] != type: 121 | continue 122 | if tag: 123 | if tag[0] not in i.tags or i.tags[tag[0]] != tag[1]: 124 | continue 125 | filtered.append(i) 126 | return filtered 127 | 128 | def is_running(instance): 129 | return instance.update(True) == "running" 130 | 131 | def is_spot_instance(instance): 132 | return instance.spot_instance_request_id 133 | 134 | def get_all_spot_requests(): 135 | return conn.get_all_spot_instance_requests() 136 | 137 | def get_open_spot_requests(): 138 | return filter_spot_requests('open') 139 | 140 | def get_active_spot_requests(): 141 | return filter_spot_requests('active') 142 | 143 | def update_spot_requests(request_ids): 144 | return conn.get_all_spot_instance_requests(request_ids) 145 | 146 | def filter_spot_requests(state): 147 | spots = [] 148 | all = conn.get_all_spot_instance_requests() 149 | for r in all: 150 | if r.state == state: 151 | spots.append(r) 152 | return spots 153 | 154 | def get_all_spot_instances(): 155 | spots = [] 156 | for i in get_all_instances(): 157 | if is_spot_instance(i): 158 | spots.append(i) 159 | return spots 160 | 161 | def cancel_spot_requests(requests): 162 | for r in requests: 163 | if r.state != "cancelled": 164 | print "Canceling spot request " + r.id 165 | conn.cancel_spot_instance_requests([r.id]) 166 | 167 | def scp_to(machine, tofile, fromfile): 168 | cmd = "scp -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %s -r %s ec2-user@%s:%s" % \ 169 | (SSH_KEY, fromfile, machine, tofile) 170 | Popen(cmd, shell=True).wait() 171 | 172 | def scp_from(machine, fromfile, tofile): 173 | cmd = "scp -q -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -i %s -r ec2-user@%s:%s %s" % \ 174 | (SSH_KEY, machine, fromfile, tofile) 175 | Popen(cmd, shell=True).wait() 176 | 177 | def run_ssh_command_bg(machine, cmd, tty=False): 178 | sshcmd = "ssh -A -q " + ("-t " if tty else "") + \ 179 | "-o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o ControlPath=none -i " + \ 180 | "%s ec2-user@%s \"%s\"" % (SSH_KEY, machine, cmd) 181 | return Popen(shlex.split(sshcmd)) 182 | 183 | def run_ssh_command(machine, cmd, tty=False): 184 | return run_ssh_command_bg(machine, cmd, tty).wait() 185 | -------------------------------------------------------------------------------- /src/pqpersistent.tcc: -------------------------------------------------------------------------------- 1 | #include "pqpersistent.hh" 2 | #include "pqserver.hh" 3 | 4 | namespace pq { 5 | 6 | #if HAVE_LIBPQ 7 | 8 | PostgresStore::PostgresStore(const DBPoolParams& params) 9 | : params_(params), pool_(nullptr), monitor_(nullptr) { 10 | } 11 | 12 | PostgresStore::~PostgresStore() { 13 | delete pool_; 14 | if (monitor_) 15 | PQfinish(monitor_); 16 | } 17 | 18 | void PostgresStore::connect() { 19 | std::vector statements({ 20 | "PREPARE kv_put(text,text) AS " 21 | "WITH upsert AS " 22 | "(UPDATE cache SET value=$2 WHERE key=$1 RETURNING cache.*) " 23 | "INSERT INTO cache " 24 | "SELECT * FROM (SELECT $1 k, $2 v) AS tmp_table " 25 | "WHERE CAST(tmp_table.k AS TEXT) NOT IN (SELECT key FROM upsert)", 26 | "PREPARE kv_erase(text) AS " 27 | "DELETE FROM cache WHERE key=$1", 28 | "PREPARE kv_get(text) AS " 29 | "SELECT value FROM cache WHERE key=$1", 30 | "PREPARE kv_scan(text,text) AS " 31 | "SELECT key, value FROM cache WHERE key >= $1 AND key < $2"}); 32 | 33 | pool_ = new DBPool(params_); 34 | pool_->connect_all(statements); 35 | } 36 | 37 | tamed void PostgresStore::put(Str key, Str value, tamer::event<> done) { 38 | tvars { 39 | String q = "EXECUTE kv_put('" + key + "','" + value + "')"; 40 | Json j; 41 | } 42 | 43 | twait { pool_->execute(q, make_event(j)); } 44 | done(); 45 | } 46 | 47 | tamed void PostgresStore::erase(Str key, tamer::event<> done) { 48 | tvars { 49 | String q = "EXECUTE kv_erase('" + key + "')"; 50 | Json j; 51 | } 52 | 53 | twait { pool_->execute(q, make_event(j)); } 54 | done(); 55 | } 56 | 57 | tamed void PostgresStore::get(Str key, tamer::event done) { 58 | tvars { 59 | String q = "EXECUTE kv_get('" + key + "')"; 60 | Json j; 61 | } 62 | 63 | twait { pool_->execute(q, make_event(j)); } 64 | 65 | if (j.is_a() && j.size() && j[0].size()) 66 | done(j[0][0].as_s()); 67 | else 68 | // todo: distinguish between no value and empty value! 69 | done(""); 70 | } 71 | 72 | tamed void PostgresStore::scan(Str first, Str last, tamer::event done) { 73 | tvars { 74 | String q = "EXECUTE kv_scan('" + first + "','" + last + "')"; 75 | Json j; 76 | } 77 | 78 | twait { pool_->execute(q, make_event(j)); } 79 | 80 | ResultSet& rs = done.result(); 81 | for (auto it = j.abegin(); it < j.aend(); ++it ) 82 | rs.push_back(Result((*it)[0].as_s(), (*it)[1].as_s())); 83 | 84 | done.unblocker().trigger(); 85 | } 86 | 87 | void PostgresStore::flush() { 88 | pool_->flush(); 89 | } 90 | 91 | void PostgresStore::run_monitor(Server& server) { 92 | String cs = "dbname=" + params_.dbname + " host=" + params_.host + " port=" + String(params_.port); 93 | monitor_ = PQconnectdb(cs.c_str()); 94 | mandatory_assert(monitor_); 95 | mandatory_assert(PQstatus(monitor_) != CONNECTION_BAD); 96 | 97 | PGresult* res; 98 | res = PQexec(monitor_, 99 | "CREATE OR REPLACE FUNCTION notify_upsert_listener() " 100 | "RETURNS trigger AS " 101 | "$BODY$ " 102 | "BEGIN " 103 | "PERFORM pg_notify('backend_queue', '{ \"op\":0, \"k\":\"' || CAST (NEW.key AS TEXT) || '\", \"v\":\"' || CAST (NEW.value AS TEXT) || '\" }'); " 104 | "RETURN NULL; " 105 | "END; " 106 | "$BODY$ " 107 | "LANGUAGE plpgsql VOLATILE " 108 | "COST 100"); 109 | 110 | mandatory_assert(res); 111 | mandatory_assert(PQresultStatus(res) == PGRES_COMMAND_OK); 112 | PQclear(res); 113 | 114 | res = PQexec(monitor_, 115 | "CREATE OR REPLACE FUNCTION notify_delete_listener() " 116 | "RETURNS trigger AS " 117 | "$BODY$ " 118 | "BEGIN " 119 | "PERFORM pg_notify('backend_queue', '{ \"op\":1, \"k\":\"' || CAST (OLD.key AS TEXT) || '\" }'); " 120 | "RETURN NULL; " 121 | "END; " 122 | "$BODY$ " 123 | "LANGUAGE plpgsql VOLATILE " 124 | "COST 100"); 125 | 126 | mandatory_assert(res); 127 | mandatory_assert(PQresultStatus(res) == PGRES_COMMAND_OK); 128 | PQclear(res); 129 | 130 | res = PQexec(monitor_, "DROP TRIGGER IF EXISTS notify_upsert_cache ON cache"); 131 | 132 | mandatory_assert(res); 133 | mandatory_assert(PQresultStatus(res) == PGRES_COMMAND_OK); 134 | PQclear(res); 135 | 136 | res = PQexec(monitor_, "DROP TRIGGER IF EXISTS notify_delete_cache ON cache"); 137 | 138 | mandatory_assert(res); 139 | mandatory_assert(PQresultStatus(res) == PGRES_COMMAND_OK); 140 | PQclear(res); 141 | 142 | res = PQexec(monitor_, 143 | "CREATE TRIGGER notify_upsert_cache " 144 | "AFTER INSERT OR UPDATE ON cache " 145 | "FOR EACH ROW " 146 | "EXECUTE PROCEDURE notify_upsert_listener()"); 147 | 148 | mandatory_assert(res); 149 | mandatory_assert(PQresultStatus(res) == PGRES_COMMAND_OK); 150 | PQclear(res); 151 | 152 | res = PQexec(monitor_, 153 | "CREATE TRIGGER notify_delete_cache " 154 | "AFTER DELETE ON cache " 155 | "FOR EACH ROW " 156 | "EXECUTE PROCEDURE notify_delete_listener()"); 157 | 158 | mandatory_assert(res); 159 | mandatory_assert(PQresultStatus(res) == PGRES_COMMAND_OK); 160 | PQclear(res); 161 | 162 | res = PQexec(monitor_, "LISTEN backend_queue"); 163 | 164 | mandatory_assert(res); 165 | mandatory_assert(PQresultStatus(res) == PGRES_COMMAND_OK); 166 | PQclear(res); 167 | 168 | monitor_db(server); 169 | } 170 | 171 | tamed void PostgresStore::monitor_db(Server& server) { 172 | tvars { 173 | int32_t err; 174 | Json j; 175 | } 176 | 177 | while(true) { 178 | // XXX: we might need to yield once in a while... 179 | 180 | do { 181 | twait { tamer::at_fd_read(PQsocket(monitor_), make_event()); } 182 | err = PQconsumeInput(monitor_); 183 | mandatory_assert(err == 1 && "Error reading data from DB."); 184 | } while(PQisBusy(monitor_)); 185 | 186 | // there should be no results on this connection 187 | mandatory_assert(!PQgetResult(monitor_)); 188 | 189 | // process notifications 190 | PGnotify* n = PQnotifies(monitor_); 191 | while(n) { 192 | 193 | j.assign_parse(n->extra); 194 | assert(j && j.is_o()); 195 | 196 | switch(j["op"].as_i()) { 197 | case pg_update: 198 | server.insert(j["k"].as_s(), j["v"].as_s()); 199 | break; 200 | 201 | case pg_delete: 202 | server.erase(j["k"].as_s()); 203 | break; 204 | 205 | default: 206 | mandatory_assert(false && "Unknown DB operation."); 207 | break; 208 | } 209 | 210 | PQfreemem(n); 211 | n = PQnotifies(monitor_); 212 | } 213 | } 214 | } 215 | 216 | #endif 217 | } 218 | -------------------------------------------------------------------------------- /vis/jquery.flot.fillbetween.js: -------------------------------------------------------------------------------- 1 | /* 2 | Flot plugin for computing bottoms for filled line and bar charts. 3 | 4 | The case: you've got two series that you want to fill the area 5 | between. In Flot terms, you need to use one as the fill bottom of the 6 | other. You can specify the bottom of each data point as the third 7 | coordinate manually, or you can use this plugin to compute it for you. 8 | 9 | In order to name the other series, you need to give it an id, like this 10 | 11 | var dataset = [ 12 | { data: [ ... ], id: "foo" } , // use default bottom 13 | { data: [ ... ], fillBetween: "foo" }, // use first dataset as bottom 14 | ]; 15 | 16 | $.plot($("#placeholder"), dataset, { line: { show: true, fill: true }}); 17 | 18 | As a convenience, if the id given is a number that doesn't appear as 19 | an id in the series, it is interpreted as the index in the array 20 | instead (so fillBetween: 0 can also mean the first series). 21 | 22 | Internally, the plugin modifies the datapoints in each series. For 23 | line series, extra data points might be inserted through 24 | interpolation. Note that at points where the bottom line is not 25 | defined (due to a null point or start/end of line), the current line 26 | will show a gap too. The algorithm comes from the jquery.flot.stack.js 27 | plugin, possibly some code could be shared. 28 | */ 29 | 30 | (function ($) { 31 | var options = { 32 | series: { fillBetween: null } // or number 33 | }; 34 | 35 | function init(plot) { 36 | function findBottomSeries(s, allseries) { 37 | var i; 38 | for (i = 0; i < allseries.length; ++i) { 39 | if (allseries[i].id == s.fillBetween) 40 | return allseries[i]; 41 | } 42 | 43 | if (typeof s.fillBetween == "number") { 44 | i = s.fillBetween; 45 | 46 | if (i < 0 || i >= allseries.length) 47 | return null; 48 | 49 | return allseries[i]; 50 | } 51 | 52 | return null; 53 | } 54 | 55 | function computeFillBottoms(plot, s, datapoints) { 56 | if (s.fillBetween == null) 57 | return; 58 | 59 | var other = findBottomSeries(s, plot.getData()); 60 | if (!other) 61 | return; 62 | 63 | var ps = datapoints.pointsize, 64 | points = datapoints.points, 65 | otherps = other.datapoints.pointsize, 66 | otherpoints = other.datapoints.points, 67 | newpoints = [], 68 | px, py, intery, qx, qy, bottom, 69 | withlines = s.lines.show, 70 | withbottom = ps > 2 && datapoints.format[2].y, 71 | withsteps = withlines && s.lines.steps, 72 | fromgap = true, 73 | i = 0, j = 0, l; 74 | 75 | while (true) { 76 | if (i >= points.length) 77 | break; 78 | 79 | l = newpoints.length; 80 | 81 | if (points[i] == null) { 82 | // copy gaps 83 | for (m = 0; m < ps; ++m) 84 | newpoints.push(points[i + m]); 85 | i += ps; 86 | } 87 | else if (j >= otherpoints.length) { 88 | // for lines, we can't use the rest of the points 89 | if (!withlines) { 90 | for (m = 0; m < ps; ++m) 91 | newpoints.push(points[i + m]); 92 | } 93 | i += ps; 94 | } 95 | else if (otherpoints[j] == null) { 96 | // oops, got a gap 97 | for (m = 0; m < ps; ++m) 98 | newpoints.push(null); 99 | fromgap = true; 100 | j += otherps; 101 | } 102 | else { 103 | // cases where we actually got two points 104 | px = points[i]; 105 | py = points[i + 1]; 106 | qx = otherpoints[j]; 107 | qy = otherpoints[j + 1]; 108 | bottom = 0; 109 | 110 | if (px == qx) { 111 | for (m = 0; m < ps; ++m) 112 | newpoints.push(points[i + m]); 113 | 114 | //newpoints[l + 1] += qy; 115 | bottom = qy; 116 | 117 | i += ps; 118 | j += otherps; 119 | } 120 | else if (px > qx) { 121 | // we got past point below, might need to 122 | // insert interpolated extra point 123 | if (withlines && i > 0 && points[i - ps] != null) { 124 | intery = py + (points[i - ps + 1] - py) * (qx - px) / (points[i - ps] - px); 125 | newpoints.push(qx); 126 | newpoints.push(intery) 127 | for (m = 2; m < ps; ++m) 128 | newpoints.push(points[i + m]); 129 | bottom = qy; 130 | } 131 | 132 | j += otherps; 133 | } 134 | else { // px < qx 135 | if (fromgap && withlines) { 136 | // if we come from a gap, we just skip this point 137 | i += ps; 138 | continue; 139 | } 140 | 141 | for (m = 0; m < ps; ++m) 142 | newpoints.push(points[i + m]); 143 | 144 | // we might be able to interpolate a point below, 145 | // this can give us a better y 146 | if (withlines && j > 0 && otherpoints[j - otherps] != null) 147 | bottom = qy + (otherpoints[j - otherps + 1] - qy) * (px - qx) / (otherpoints[j - otherps] - qx); 148 | 149 | //newpoints[l + 1] += bottom; 150 | 151 | i += ps; 152 | } 153 | 154 | fromgap = false; 155 | 156 | if (l != newpoints.length && withbottom) 157 | newpoints[l + 2] = bottom; 158 | } 159 | 160 | // maintain the line steps invariant 161 | if (withsteps && l != newpoints.length && l > 0 162 | && newpoints[l] != null 163 | && newpoints[l] != newpoints[l - ps] 164 | && newpoints[l + 1] != newpoints[l - ps + 1]) { 165 | for (m = 0; m < ps; ++m) 166 | newpoints[l + ps + m] = newpoints[l + m]; 167 | newpoints[l + 1] = newpoints[l - ps + 1]; 168 | } 169 | } 170 | 171 | datapoints.points = newpoints; 172 | } 173 | 174 | plot.hooks.processDatapoints.push(computeFillBottoms); 175 | } 176 | 177 | $.plot.plugins.push({ 178 | init: init, 179 | options: options, 180 | name: 'fillbetween', 181 | version: '1.0' 182 | }); 183 | })(jQuery); 184 | -------------------------------------------------------------------------------- /src/pqmulticlient.tcc: -------------------------------------------------------------------------------- 1 | // -*- mode: c++ -*- 2 | #include "pqmulticlient.hh" 3 | 4 | namespace pq { 5 | 6 | MultiClient::MultiClient(const Hosts* hosts, const Partitioner* part, int colocateCacheServer) 7 | : hosts_(hosts), part_(part), localNode_(nullptr), 8 | colocateCacheServer_(colocateCacheServer), 9 | dbhosts_(nullptr), dbparams_(nullptr), rand_cache_(false) { 10 | gen_.seed(112181); 11 | } 12 | 13 | MultiClient::MultiClient(const Hosts* hosts, const Partitioner* part, int colocateCacheServer, 14 | const Hosts* dbhosts, const DBPoolParams* dbparams) 15 | : hosts_(hosts), part_(part), localNode_(nullptr), 16 | colocateCacheServer_(colocateCacheServer), 17 | dbhosts_(dbhosts), dbparams_(dbparams), rand_cache_(false) { 18 | gen_.seed(112181); 19 | } 20 | 21 | MultiClient::~MultiClient() { 22 | clear(); 23 | } 24 | 25 | tamed void MultiClient::connect(tamer::event<> done) { 26 | tvars { 27 | tamer::fd fd; 28 | struct sockaddr_in sin; 29 | const Host* h; 30 | int32_t i; 31 | } 32 | 33 | // for backward compatibility, allow connection to a single server. 34 | // in this case we interpret the colocated server to be a port number 35 | if (!hosts_ && !part_ && colocateCacheServer_ > 0) { 36 | twait { tamer::tcp_connect(in_addr{htonl(INADDR_LOOPBACK)}, 37 | colocateCacheServer_, make_event(fd)); } 38 | localNode_ = new RemoteClient(fd, "local"); 39 | } 40 | else { 41 | for (i = 0; i < hosts_->size(); ++i) { 42 | h = hosts_->get_by_seqid(i); 43 | sock_helper::make_sockaddr(h->name().c_str(), h->port(), sin); 44 | twait { tamer::tcp_connect(sin.sin_addr, h->port(), make_event(fd)); } 45 | if (!fd) { 46 | std::cerr << "connection error (" 47 | << h->name() << ":" << h->port() << "): " 48 | << strerror(-fd.error()) << std::endl; 49 | exit(1); 50 | } 51 | 52 | clients_.push_back(new RemoteClient(fd, String(i))); 53 | } 54 | 55 | if (colocateCacheServer_ >= 0) { 56 | mandatory_assert(colocateCacheServer_ < hosts_->size()); 57 | localNode_ = clients_[colocateCacheServer_]; 58 | } 59 | } 60 | 61 | if (dbhosts_ && part_) { 62 | for (i = 0; i < dbhosts_->size(); ++i) { 63 | h = dbhosts_->get_by_seqid(i); 64 | DBPoolParams par = (dbparams_) ? *dbparams_ : DBPoolParams(); 65 | par.host = h->name(); 66 | par.port = h->port(); 67 | DBPool* pool = new DBPool(par); 68 | pool->connect(); 69 | dbclients_.push_back(pool); 70 | } 71 | } 72 | 73 | done(); 74 | } 75 | 76 | tamed void MultiClient::restart(tamer::event<> done) { 77 | clear(); 78 | twait { connect(make_event()); } 79 | done(); 80 | } 81 | 82 | tamed void MultiClient::add_join(const String& first, const String& last, 83 | const String& joinspec, event e) { 84 | tvars { 85 | Json rj; 86 | } 87 | 88 | if (localNode_) 89 | localNode_->add_join(first, last, joinspec, e); 90 | else { 91 | rj = Json::make_array_reserve(clients_.size()); 92 | twait { 93 | for (uint32_t i = 0; i < clients_.size(); ++i) 94 | if (!part_->is_backend(i)) 95 | clients_[i]->add_join(first, last, joinspec, 96 | make_event(rj[i].value())); 97 | } 98 | 99 | Json ret; 100 | for (auto it = rj.abegin(); it != rj.aend(); ++it) 101 | if ((*it)["message"]) { 102 | ret.set("message", rj); 103 | break; 104 | } 105 | e(ret); 106 | } 107 | } 108 | 109 | tamed void MultiClient::get(const String& key, event e) { 110 | cache_for(key)->get(key, e); 111 | } 112 | 113 | tamed void MultiClient::insert(const String& key, const String& value, event<> e) { 114 | cache_for(key)->insert(key, value, e); 115 | } 116 | 117 | tamed void MultiClient::erase(const String& key, event<> e) { 118 | cache_for(key)->erase(key, e); 119 | } 120 | 121 | tamed void MultiClient::insert_db(const String& key, const String& value, event<> e) { 122 | tvars { 123 | Json j; 124 | String query; 125 | } 126 | 127 | query = "WITH upsert AS (UPDATE cache SET value=\'" + value + "\' " + 128 | "WHERE key=\'" + key + "\'" + " RETURNING cache.* ) " 129 | "INSERT INTO cache " 130 | "SELECT * FROM (SELECT \'" + key + "\' k, \'" + value + "\' v) AS tmp_table " 131 | "WHERE CAST(tmp_table.k AS TEXT) NOT IN (SELECT key FROM upsert)"; 132 | 133 | twait { backend_for(key)->execute(query, make_event(j)); } 134 | e(); 135 | } 136 | 137 | tamed void MultiClient::erase_db(const String& key, event<> e) { 138 | tvars { 139 | Json j; 140 | String query; 141 | } 142 | 143 | query = "DELETE FROM cache WHERE key=\'" + key + "\'"; 144 | 145 | twait { backend_for(key)->execute(query, make_event(j)); } 146 | e(); 147 | } 148 | 149 | tamed void MultiClient::count(const String& first, const String& last, 150 | event e) { 151 | cache_for(first, rand_cache_)->count(first, last, e); 152 | } 153 | 154 | tamed void MultiClient::count(const String& first, const String& last, 155 | const String& scanlast, event e) { 156 | cache_for(first, rand_cache_)->count(first, last, scanlast, e); 157 | } 158 | 159 | tamed void MultiClient::add_count(const String& first, const String& last, 160 | event e) { 161 | cache_for(first, rand_cache_)->add_count(first, last, e); 162 | } 163 | 164 | tamed void MultiClient::add_count(const String& first, const String& last, 165 | const String& scanlast, event e) { 166 | cache_for(first, rand_cache_)->add_count(first, last, scanlast, e); 167 | } 168 | 169 | tamed void MultiClient::scan(const String& first, const String& last, 170 | event e) { 171 | cache_for(first, rand_cache_)->scan(first, last, e); 172 | } 173 | 174 | tamed void MultiClient::scan(const String& first, const String& last, 175 | const String& scanlast, event e) { 176 | cache_for(first, rand_cache_)->scan(first, last, scanlast, e); 177 | } 178 | 179 | tamed void MultiClient::stats(event e) { 180 | tvars { 181 | Json j; 182 | } 183 | 184 | if (localNode_) 185 | localNode_->stats(e); 186 | else { 187 | j = Json::make_array_reserve(clients_.size()); 188 | twait ["stats"] { 189 | for (uint32_t i = 0; i < clients_.size(); ++i) 190 | clients_[i]->stats(make_event(j[i].value())); 191 | } 192 | e(j); 193 | } 194 | } 195 | 196 | tamed void MultiClient::control(const Json& cmd, tamer::event done) { 197 | tvars { 198 | Json j; 199 | } 200 | 201 | if (localNode_) 202 | localNode_->control(cmd, done); 203 | else { 204 | j = Json::make_array_reserve(clients_.size()); 205 | twait ["control"] { 206 | for (uint32_t i = 0; i < clients_.size(); ++i) 207 | clients_[i]->control(cmd, make_event(j[i].value())); 208 | } 209 | done(j); 210 | } 211 | } 212 | 213 | tamed void MultiClient::pace(tamer::event<> done) { 214 | twait ["pace"] { 215 | for (auto& r : clients_) 216 | r->pace(make_event()); 217 | } 218 | done(); 219 | } 220 | 221 | tamed void MultiClient::flush(tamer::event<> done) { 222 | for (auto& d : dbclients_) 223 | d->flush(); 224 | done(); 225 | } 226 | 227 | } 228 | -------------------------------------------------------------------------------- /lib/interval.hh: -------------------------------------------------------------------------------- 1 | #ifndef GSTORE_XINTERVAL_HH 2 | #define GSTORE_XINTERVAL_HH 1 3 | #include "compiler.hh" 4 | #include "straccum.hh" 5 | #include 6 | 7 | template 8 | class interval { 9 | public: 10 | typedef T endpoint_type; 11 | typedef bool (interval::*unspecified_bool_type)() const; 12 | 13 | inline interval(); 14 | inline interval(const T& first, const T& last); 15 | inline interval(T&& first, T&& last); 16 | inline interval(const interval& x); 17 | inline interval(interval&& x); 18 | 19 | inline const T& ibegin() const; 20 | inline const T& iend() const; 21 | 22 | inline operator unspecified_bool_type() const; 23 | inline bool operator!() const; 24 | inline bool empty() const; 25 | 26 | static inline bool contains(const T& first, const T& last, const T& x); 27 | template 28 | static inline bool contains(const T& first, const T& last, 29 | const X& xfirst, const X& xlast); 30 | template 31 | static inline bool overlaps(const T& first, const T& last, 32 | const X& xfirst, const X& xlast); 33 | 34 | inline bool contains(const T& x) const; 35 | inline bool contains(const interval& x) const; 36 | template inline bool contains(const interval& x) const; 37 | template inline bool contains(const X& first, const X& last) const; 38 | inline bool overlaps(const interval& x) const; 39 | template inline bool overlaps(const interval& x) const; 40 | template inline bool overlaps(const X& first, const X& last) const; 41 | 42 | inline String unparse() const; 43 | 44 | private: 45 | T ibegin_; 46 | T iend_; 47 | }; 48 | 49 | template 50 | struct default_comparator > { 51 | inline int operator()(const interval& a, const interval& b) const; 52 | }; 53 | 54 | 55 | template 56 | inline interval::interval() 57 | : ibegin_(), iend_() { 58 | } 59 | 60 | template 61 | inline interval::interval(const T& ibegin, const T& iend) 62 | : ibegin_(ibegin), iend_(iend < ibegin ? ibegin : iend) { 63 | } 64 | 65 | template 66 | inline interval::interval(T&& ibegin, T&& iend) 67 | : ibegin_(std::move(ibegin)), 68 | iend_(std::move(iend < ibegin ? ibegin : iend)) { 69 | } 70 | 71 | template 72 | inline interval::interval(const interval& x) 73 | : ibegin_(x.ibegin()), iend_(x.iend()) { 74 | } 75 | 76 | template 77 | inline interval::interval(interval&& x) 78 | : ibegin_(std::move(x.ibegin_)), iend_(std::move(x.iend_)) { 79 | } 80 | 81 | template 82 | inline interval make_interval(const T& ibegin, const T& iend) { 83 | return interval(ibegin, iend); 84 | } 85 | 86 | template 87 | inline const T& interval::ibegin() const { 88 | return ibegin_; 89 | } 90 | 91 | template 92 | inline const T& interval::iend() const { 93 | return iend_; 94 | } 95 | 96 | template 97 | inline interval::operator unspecified_bool_type() const { 98 | return ibegin_ == iend_ ? 0 : &interval::empty; 99 | } 100 | 101 | template 102 | inline bool interval::operator!() const { 103 | return ibegin_ == iend_; 104 | } 105 | 106 | template 107 | inline bool interval::empty() const { 108 | return ibegin_ == iend_; 109 | } 110 | 111 | template 112 | inline bool interval::contains(const T& first, const T& last, const T& x) { 113 | return !(x < first) && x < last; 114 | } 115 | 116 | template template 117 | inline bool interval::contains(const T& first, const T& last, 118 | const X& xfirst, const X& xlast) { 119 | return !(xfirst < first || last < xlast); 120 | } 121 | 122 | template template 123 | inline bool interval::overlaps(const T& first, const T& last, 124 | const X& xfirst, const X& xlast) { 125 | return xfirst < last && first < xlast; 126 | } 127 | 128 | template 129 | inline bool interval::contains(const T& x) const { 130 | return contains(ibegin(), iend(), x); 131 | } 132 | 133 | template 134 | inline bool interval::contains(const interval& x) const { 135 | return contains(ibegin(), iend(), x.ibegin(), x.iend()); 136 | } 137 | 138 | template template 139 | inline bool interval::contains(const interval& x) const { 140 | return contains(ibegin(), iend(), x.ibegin(), x.iend()); 141 | } 142 | 143 | template template 144 | inline bool interval::contains(const X& first, const X& last) const { 145 | return contains(ibegin(), iend(), first, last); 146 | } 147 | 148 | template 149 | inline bool interval::overlaps(const interval& x) const { 150 | return overlaps(ibegin(), iend(), x.ibegin(), x.iend()); 151 | } 152 | 153 | template template 154 | inline bool interval::overlaps(const interval& x) const { 155 | return overlaps(ibegin(), iend(), x.ibegin(), x.iend()); 156 | } 157 | 158 | template template 159 | inline bool interval::overlaps(const X& first, const X& last) const { 160 | return overlaps(ibegin(), iend(), first, last); 161 | } 162 | 163 | template 164 | inline bool operator==(const interval& a, const interval& b) { 165 | return a.ibegin() == b.ibegin() && a.iend() == b.iend(); 166 | } 167 | 168 | template 169 | inline bool operator!=(const interval& a, const interval& b) { 170 | return !(a.ibegin() == b.ibegin()) || !(a.iend() == b.iend()); 171 | } 172 | 173 | template 174 | inline bool operator<(const interval& a, const interval& b) { 175 | int cmp = default_comparator()(a.ibegin(), b.ibegin()); 176 | return cmp < 0 || (cmp == 0 && a.iend() < b.iend()); 177 | } 178 | 179 | template 180 | inline bool operator<=(const interval& a, const interval& b) { 181 | int cmp = default_comparator()(a.ibegin(), b.ibegin()); 182 | return cmp < 0 || (cmp == 0 && !(b.iend() < a.iend())); 183 | } 184 | 185 | template 186 | inline bool operator>=(const interval& a, const interval& b) { 187 | int cmp = default_comparator()(a.ibegin(), b.ibegin()); 188 | return cmp > 0 || (cmp == 0 && !(a.iend() < b.iend())); 189 | } 190 | 191 | template 192 | inline bool operator>(const interval& a, const interval& b) { 193 | int cmp = default_comparator()(a.ibegin(), b.ibegin()); 194 | return cmp > 0 || (cmp == 0 && b.iend() < a.iend()); 195 | } 196 | 197 | template 198 | inline interval operator&(const interval& a, const interval& b) { 199 | return interval(a.ibegin() > b.ibegin() ? a.ibegin() : b.ibegin(), 200 | a.iend() < b.iend() ? a.iend() : b.iend()); 201 | } 202 | 203 | template 204 | inline interval operator|(const interval& a, const interval& b) { 205 | return interval(a.ibegin() < b.ibegin() ? a.ibegin() : b.ibegin(), 206 | a.iend() > b.iend() ? a.iend() : b.iend()); 207 | } 208 | 209 | template 210 | inline int default_comparator >::operator()(const interval& a, 211 | const interval& b) const { 212 | int cmp = default_comparator()(a.ibegin(), b.ibegin()); 213 | return cmp ? cmp : default_comparator()(a.iend(), b.iend()); 214 | }; 215 | 216 | template 217 | std::ostream& operator<<(std::ostream& s, const interval& x) { 218 | return (s << '[' << x.ibegin() << ", " << x.iend() << ')'); 219 | } 220 | 221 | template 222 | StringAccum& operator<<(StringAccum& sa, const interval& x) { 223 | return (sa << '[' << x.ibegin() << ", " << x.iend() << ')'); 224 | } 225 | 226 | template 227 | String interval::unparse() const { 228 | StringAccum sa; 229 | sa << *this; 230 | return sa.take_string(); 231 | } 232 | 233 | #endif 234 | -------------------------------------------------------------------------------- /app/twitter.tcc: -------------------------------------------------------------------------------- 1 | // -*- mode: c++ -*- 2 | #include "twitter.hh" 3 | #include "json.hh" 4 | #include "pqjoin.hh" 5 | #include "pqclient.hh" 6 | #include "pqmulticlient.hh" 7 | #include "memcacheadapter.hh" 8 | #include 9 | #include 10 | #include 11 | #include 12 | #include 13 | #include 14 | #include "time.hh" 15 | namespace pq { 16 | const char TwitterPopulator::tweet_data[] = "................................................................................................................................................................"; 17 | 18 | TwitterPopulator::TwitterPopulator(const Json& param) 19 | : nusers_(param["nusers"].as_i(5000)), 20 | push_(param["push"].as_b(false)), 21 | pull_(param["pull"].as_b(false)), 22 | log_(param["log"].as_b(false)), 23 | fetch_(param["fetch"].as_b(false)), 24 | synchronous_(param["synchronous"].as_b(false)), 25 | min_followers_(param["min_followers"].as_i(10)), 26 | min_subs_(param["min_subscriptions"].as_i(20)), 27 | max_subs_(param["max_subscriptions"].as_i(200)), 28 | shape_(param["shape"].as_d(55)), 29 | duration_(param["duration"].as_i(1000000)), 30 | ppost_(param["ppost"].as_i(2)), 31 | psubscribe_(param["psubscribe"].as_i(3)), 32 | celebrity_(param["celebrity"].as_i(0)), 33 | celebrity_type_(param["celebrity_type"].as_i(0)) { 34 | } 35 | 36 | uint32_t* TwitterPopulator::subscribe_probabilities(generator_type& gen) { 37 | uint32_t expected_subs = (min_subs_ + max_subs_) / 2; 38 | double* follow_shape = new double[nusers_]; 39 | while (1) { 40 | double shape_effect = 0; 41 | for (uint32_t i = 0; i != nusers_; ++i) { 42 | double x = pow((double) (i + 1) / nusers_, shape_); 43 | follow_shape[i] = x; 44 | shape_effect += x; 45 | } 46 | max_followers_ = (expected_subs - min_followers_) / (shape_effect / nusers_); 47 | if (max_followers_ <= nusers_ / 4) 48 | break; 49 | // change shape, try again 50 | shape_ = shape_ * 0.7; 51 | std::cerr << "RETRYING with shape " << shape_ << "...\n"; 52 | } 53 | 54 | rng_type rng(gen); 55 | std::random_shuffle(follow_shape, follow_shape + nusers_, rng); 56 | 57 | uint32_t* sub_prob = new uint32_t[nusers_]; 58 | assert(gen.min() == 0); 59 | double scale = gen.max() / (expected_subs * (double) nusers_); 60 | double pos = 0; 61 | for (uint32_t i = 0; i != nusers_; ++i) { 62 | pos += min_followers_ + follow_shape[i] * max_followers_; 63 | sub_prob[i] = (uint32_t) (pos * scale); 64 | } 65 | sub_prob[nusers_ - 1] = gen.max(); 66 | 67 | delete[] follow_shape; 68 | return sub_prob; 69 | } 70 | 71 | void TwitterPopulator::create_subscriptions(generator_type& gen) { 72 | uint32_t* sub_prob = subscribe_probabilities(gen); 73 | uint32_t* subvec = new uint32_t[(nusers_ + 31) / 32]; 74 | rng_type rng(gen); 75 | subs_.clear(); 76 | std::vector > followers; 77 | 78 | for (uint32_t i = 0; i != nusers_; ++i) { 79 | memset(subvec, 0, sizeof(uint32_t) * ((nusers_ + 31) / 32)); 80 | uint32_t nsubs = min_subs_ + rng(max_subs_ - min_subs_ + 1); 81 | for (uint32_t j = 0; j != nsubs; ++j) { 82 | // pick follow 83 | uint32_t other; 84 | do { 85 | other = std::upper_bound(sub_prob, sub_prob + nusers_, gen()) - sub_prob; 86 | } while (subvec[other / 32] & (1U << (other % 32))); 87 | subs_.push_back(std::make_pair(i, other)); 88 | followers.push_back(std::make_pair(other, i)); 89 | subvec[other / 32] |= 1U << (other % 32); 90 | } 91 | } 92 | 93 | delete[] sub_prob; 94 | delete[] subvec; 95 | 96 | followers_.clear(); 97 | followers_.reserve(followers.size()); 98 | follower_ptrs_.clear(); 99 | follower_ptrs_.reserve(nusers_ + 1); 100 | 101 | std::sort(followers.begin(), followers.end()); 102 | for (auto& sub : followers) { 103 | while (follower_ptrs_.size() <= sub.first) 104 | follower_ptrs_.push_back(followers_.size()); 105 | followers_.push_back(sub.second); 106 | } 107 | while (follower_ptrs_.size() <= nusers_) 108 | follower_ptrs_.push_back(followers_.size()); 109 | } 110 | 111 | 112 | void TwitterPopulator::print_subscription_statistics(std::ostream& stream) const { 113 | using namespace boost::accumulators; 114 | 115 | uint32_t* num_followers = new uint32_t[nusers_]; 116 | memset(num_followers, 0, nusers_ * sizeof(uint32_t)); 117 | for (auto& sub : subs_) 118 | ++num_followers[sub.second]; 119 | std::sort(num_followers, num_followers + nusers_); 120 | 121 | accumulator_set > acc; 122 | for (uint32_t i = 0; i != nusers_; ++i) 123 | acc(num_followers[i]); 124 | 125 | stream << nusers_ << " USERS HAVE SUBSCRIBERS:\n" 126 | << " zero: " << (std::upper_bound(num_followers, num_followers + nusers_, 0) - num_followers) 127 | << " mean: " << mean(acc) 128 | << " sdev: " << sqrt(variance(acc)) << "\n" 129 | << " min " << num_followers[0] 130 | << ", 1% " << num_followers[(int) (0.01 * nusers_)] 131 | << ", 10% " << num_followers[(int) (0.10 * nusers_)] 132 | << ", 25% " << num_followers[(int) (0.25 * nusers_)] 133 | << ", 50% " << num_followers[(int) (0.50 * nusers_)] 134 | << ", 75% " << num_followers[(int) (0.75 * nusers_)] 135 | << ", 90% " << num_followers[(int) (0.90 * nusers_)] 136 | << ", 99% " << num_followers[(int) (0.99 * nusers_)] 137 | << ", max " << num_followers[nusers_ - 1] << "\n"; 138 | 139 | delete[] num_followers; 140 | } 141 | 142 | typedef TwitterShim local_shim_type; 143 | 144 | tamed void run_twitter_local(TwitterPopulator& tp, Server& server) { 145 | tvars { 146 | DirectClient* client = new DirectClient(server); 147 | local_shim_type* shim = new local_shim_type(*client); 148 | TwitterRunner* tr = new TwitterRunner(*shim, tp); 149 | } 150 | 151 | twait { tr->populate(make_event()); } 152 | twait { tr->run(make_event()); } 153 | delete tr; 154 | delete shim; 155 | delete client; 156 | server.control(Json().set("quit", true)); 157 | } 158 | 159 | typedef TwitterShim remote_shim_type; 160 | 161 | tamed void run_twitter_remote(TwitterPopulator& tp, int client_port, 162 | const Hosts* hosts, const Hosts* dbhosts, 163 | const Partitioner* part) { 164 | tvars { 165 | MultiClient* mc = new MultiClient(hosts, part, client_port); 166 | remote_shim_type* shim = new remote_shim_type(*mc); 167 | TwitterRunner* tr = new TwitterRunner(*shim, tp); 168 | } 169 | twait { mc->connect(make_event()); } 170 | twait { tr->populate(make_event()); } 171 | twait { tr->run(make_event()); } 172 | delete tr; 173 | delete shim; 174 | delete mc; 175 | } 176 | 177 | #if HAVE_MEMCACHED_PROTOCOL_BINARY_H 178 | typedef pq::TwitterHashShim memcache_shim_type; 179 | 180 | tamed void run_twitter_memcache(TwitterPopulator& tp, 181 | const Hosts* hosts, const Partitioner* part) { 182 | tvars { 183 | MemcacheMultiClient* mc = new MemcacheMultiClient(hosts, part); 184 | memcache_shim_type* shim = new memcache_shim_type(*mc); 185 | TwitterRunner* tr = new TwitterRunner(*shim, tp); 186 | } 187 | twait { mc->connect(make_event()); } 188 | twait { tr->populate(make_event()); } 189 | twait { tr->run(make_event()); } 190 | delete tr; 191 | delete shim; 192 | delete mc; 193 | } 194 | #else 195 | void run_twitter_memcache(TwitterPopulator&, const Hosts*, const Partitioner*) { 196 | mandatory_assert(false && "Not configured for Memcache!"); 197 | } 198 | #endif 199 | 200 | } // namespace pq 201 | -------------------------------------------------------------------------------- /src/pqdbpool.tcc: -------------------------------------------------------------------------------- 1 | #include "pqdbpool.hh" 2 | #include "str.hh" 3 | #include 4 | 5 | using namespace tamer; 6 | 7 | namespace pq { 8 | 9 | DBPoolParams::DBPoolParams() 10 | : dbname("pequod"), host("127.0.0.1"), port(10000), 11 | min(1), max(1), pipeline_depth(1), pipeline_timeout(2000) { 12 | } 13 | 14 | DBPool::DBPool(const String& host, uint32_t port) { 15 | params_.host = host; 16 | params_.port = port; 17 | } 18 | 19 | DBPool::DBPool(const DBPoolParams& params) : params_(params) { 20 | } 21 | 22 | DBPool::~DBPool() { 23 | clear(); 24 | } 25 | 26 | void DBPool::connect() { 27 | #if HAVE_LIBPQ 28 | for (uint32_t i = 0; i < params_.min; ++i) { 29 | PGconn* conn = connect_one(); 30 | assert(conn); 31 | 32 | conn_.push_back(conn); 33 | pool_.push(conn); 34 | } 35 | #else 36 | mandatory_assert(false && "Database not configured."); 37 | #endif 38 | } 39 | 40 | void DBPool::connect_all(const std::vector& sess_init) { 41 | #if HAVE_LIBPQ 42 | mandatory_assert(conn_.empty() && "Must connect all at once!"); 43 | 44 | for (uint32_t i = 0; i < params_.max; ++i) { 45 | PGconn* conn = connect_one(); 46 | assert(conn); 47 | 48 | for (auto s : sess_init) { 49 | PGresult* res = PQexec(conn, s.c_str()); 50 | mandatory_assert(res); 51 | mandatory_assert(PQresultStatus(res) == PGRES_COMMAND_OK); 52 | PQclear(res); 53 | } 54 | 55 | conn_.push_back(conn); 56 | pool_.push(conn); 57 | } 58 | #else 59 | (void)sess_init; 60 | mandatory_assert(false && "Database not configured."); 61 | #endif 62 | } 63 | 64 | void DBPool::clear() { 65 | #if HAVE_LIBPQ 66 | while(!pool_.empty()) 67 | pool_.pop(); 68 | 69 | while(!conn_.empty()) { 70 | PGconn* conn = conn_.back(); 71 | conn_.pop_back(); 72 | PQfinish(conn); 73 | } 74 | #endif 75 | } 76 | 77 | #if HAVE_LIBPQ 78 | tamed void DBPool::execute(Str query, event e) { 79 | tvars { 80 | PGconn* conn; 81 | } 82 | 83 | if (query_buffer_.empty()) 84 | oldest_ = tstamp(); 85 | 86 | query_buffer_.push_back(query); 87 | event_buffer_.push_back(e); 88 | 89 | maybe_flush(); 90 | } 91 | 92 | tamed void DBPool::flush() { 93 | tvars { 94 | PGconn* conn; 95 | query_pipe_t queries = this->query_buffer_; 96 | event_pipe_t events = this->event_buffer_; 97 | } 98 | 99 | if (query_buffer_.empty()) 100 | return; 101 | 102 | query_buffer_.clear(); 103 | event_buffer_.clear(); 104 | oldest_ = 0; 105 | 106 | twait { next_connection(make_event(conn)); } 107 | twait { execute_pipeline(conn, queries, events, make_event()); } 108 | replace_connection(conn); 109 | } 110 | 111 | tamed void DBPool::execute_pipeline(PGconn* conn, 112 | const query_pipe_t& queries, 113 | event_pipe_t& events, 114 | event<> e) { 115 | tvars { 116 | int32_t err; 117 | PGresult* result; 118 | uint32_t r = 0; 119 | ExecStatusType status; 120 | Json ret; 121 | } 122 | 123 | { 124 | std::stringstream ss; 125 | for (auto& q : queries) 126 | ss << q << "; "; 127 | 128 | // might block. documentation says it is rare but we could use the 129 | // non-blocking write calls in libpq 130 | err = PQsendQuery(conn, ss.str().c_str()); 131 | mandatory_assert(err == 1 && "Could not send query to DB."); 132 | } 133 | 134 | while(true) { 135 | while(PQisBusy(conn)) { 136 | twait { tamer::at_fd_read(PQsocket(conn), make_event()); } 137 | err = PQconsumeInput(conn); 138 | mandatory_assert(err == 1 && "Error reading data from DB."); 139 | } 140 | 141 | result = PQgetResult(conn); 142 | 143 | if (!result) 144 | break; 145 | 146 | status = PQresultStatus(result); 147 | ret.clear(); 148 | 149 | switch(status) { 150 | case PGRES_COMMAND_OK: 151 | // command (e.g. insert, delete) returns no data 152 | break; 153 | case PGRES_TUPLES_OK: { 154 | int32_t nrows = PQntuples(result); 155 | int32_t ncols = PQnfields(result); 156 | 157 | for (int32_t r = 0; r < nrows; ++r) { 158 | ret.push_back(Json::make_array_reserve(ncols)); 159 | for (int32_t c = 0; c < ncols; ++c) { 160 | if (PQgetisnull(result, r, c)) 161 | ret[r][c] = Json::null_json; 162 | else 163 | ret[r][c] = Str(PQgetvalue(result, r, c), 164 | PQgetlength(result, r, c)); 165 | } 166 | } 167 | break; 168 | } 169 | default: { 170 | std::cerr << "Error getting result of DB query. " << std::endl 171 | << " Status: " << PQresStatus(status) << std::endl 172 | << " Message: " << PQresultErrorMessage(result) << std::endl; 173 | mandatory_assert(false); 174 | break; 175 | } 176 | } 177 | 178 | events[r++](ret); 179 | PQclear(result); 180 | } 181 | 182 | e(); 183 | } 184 | 185 | void DBPool::next_connection(tamer::event e) { 186 | if (!pool_.empty()) { 187 | e(pool_.front()); 188 | pool_.pop(); 189 | } 190 | else if (conn_.size() < params_.max) { 191 | conn_.push_back(connect_one()); 192 | e(conn_.back()); 193 | } 194 | else 195 | waiting_.push(e); 196 | } 197 | 198 | void DBPool::replace_connection(PGconn* conn) { 199 | if (!waiting_.empty()) { 200 | waiting_.front()(conn); 201 | waiting_.pop(); 202 | } 203 | else 204 | pool_.push(conn); 205 | } 206 | 207 | PGconn* DBPool::connect_one() { 208 | String cs = "dbname=" + params_.dbname + " host=" + params_.host + " port=" + String(params_.port); 209 | PGconn* conn = PQconnectdb(cs.c_str()); 210 | mandatory_assert(conn); 211 | mandatory_assert(PQstatus(conn) != CONNECTION_BAD); 212 | return conn; 213 | } 214 | 215 | tamed void DBPool::add_prepared(const std::vector& statements, tamer::event<> e) { 216 | tvars { 217 | std::vector local_conns; 218 | std::vector::iterator c; 219 | std::vector> events; 220 | int32_t i, outstanding_count; 221 | PGconn* temp_conn; 222 | Json j; 223 | } 224 | 225 | if (!query_buffer_.empty()) 226 | flush(); 227 | 228 | // force max connections 229 | for ( i = params_.max - conn_.size(); i > 0; --i) { 230 | local_conns.push_back(connect_one()); 231 | conn_.push_back(local_conns.back()); 232 | } 233 | 234 | // wait for existing outstanding connections 235 | outstanding_count = params_.max - local_conns.size(); 236 | for (i = 0; i < outstanding_count; ++i) { 237 | twait { next_connection(make_event(temp_conn)); } 238 | local_conns.push_back(temp_conn); 239 | } 240 | 241 | // add the prepared statements to each connection 242 | for (c = local_conns.begin(); c != local_conns.end(); ++c ) { 243 | twait { 244 | events.clear(); 245 | for (i = 0; i < (int32_t)statements.size(); ++i) 246 | events.push_back(make_event(j)); 247 | execute_pipeline(*c, statements, events, make_event()); 248 | } 249 | 250 | replace_connection(*c); 251 | } 252 | 253 | e(); 254 | } 255 | 256 | #else 257 | 258 | tamed void DBPool::execute(Str query, event e) { 259 | mandatory_assert(false && "Database not configured."); 260 | } 261 | 262 | tamed void DBPool::add_prepared(const std::vector& statements, tamer::event<> e) { 263 | mandatory_assert(false && "Database not configured."); 264 | } 265 | 266 | tamed void DBPool::flush() { 267 | mandatory_assert(false && "Database not configured."); 268 | } 269 | 270 | #endif 271 | 272 | } 273 | --------------------------------------------------------------------------------