├── ruby ├── .rspec ├── spec │ ├── spec_helper.rb │ └── chapter01_spec.rb ├── Gemfile ├── README.md ├── Gemfile.lock └── chapter01.rb ├── java ├── settings.gradle ├── libs │ └── commons-csv-20070730.jar ├── gradle │ └── wrapper │ │ ├── gradle-wrapper.jar │ │ └── gradle-wrapper.properties ├── README.md ├── build.gradle ├── gradlew.bat ├── src │ └── main │ │ └── java │ │ ├── Chapter01.java │ │ ├── Chapter04.java │ │ ├── Chapter02.java │ │ ├── Chapter09.java │ │ ├── Chapter08.java │ │ └── Chapter05.java └── gradlew ├── .gitignore ├── node ├── readme.md ├── package.json ├── run-mocha-tests.js └── ch01 │ ├── test.js │ ├── main.js │ └── mocha-test.js ├── gawk ├── README ├── ch01 │ └── ch01.awk └── ch04 │ └── ch04.awk ├── README.md ├── LICENSE.txt ├── python ├── chA_listing_source.py ├── ch01_listing_source.py ├── ch02_listing_source.py └── ch04_listing_source.py └── excerpt_errata.html /ruby/.rspec: -------------------------------------------------------------------------------- 1 | --color 2 | -------------------------------------------------------------------------------- /ruby/spec/spec_helper.rb: -------------------------------------------------------------------------------- 1 | require 'pry' 2 | require 'redis' 3 | -------------------------------------------------------------------------------- /java/settings.gradle: -------------------------------------------------------------------------------- 1 | rootProject.name = 'redis-in-action' 2 | 3 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | *.swp 2 | /java/build 3 | GeoLiteCity-*.csv 4 | .idea 5 | *.iml 6 | node_modules 7 | -------------------------------------------------------------------------------- /ruby/Gemfile: -------------------------------------------------------------------------------- 1 | source 'https://rubygems.org' 2 | 3 | gem 'pry' 4 | gem 'redis' 5 | gem 'rspec' 6 | -------------------------------------------------------------------------------- /java/libs/commons-csv-20070730.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fuqi365/https---github.com-josiahcarlson-redis-in-action/HEAD/java/libs/commons-csv-20070730.jar -------------------------------------------------------------------------------- /java/gradle/wrapper/gradle-wrapper.jar: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/fuqi365/https---github.com-josiahcarlson-redis-in-action/HEAD/java/gradle/wrapper/gradle-wrapper.jar -------------------------------------------------------------------------------- /java/gradle/wrapper/gradle-wrapper.properties: -------------------------------------------------------------------------------- 1 | #Sun Jan 25 13:22:54 EET 2015 2 | distributionBase=GRADLE_USER_HOME 3 | distributionPath=wrapper/dists 4 | zipStoreBase=GRADLE_USER_HOME 5 | zipStorePath=wrapper/dists 6 | distributionUrl=https\://services.gradle.org/distributions/gradle-2.0-all.zip 7 | -------------------------------------------------------------------------------- /ruby/README.md: -------------------------------------------------------------------------------- 1 | Redis in Action (the Ruby version) 2 | ================================== 3 | 4 | Chapter 1 completed. 5 | 6 | ``` 7 | bundle install 8 | ``` 9 | 10 | To run tests similar to the Python code 11 | 12 | ``` 13 | bundle exec rspec spec/chapter01_spec.rb 14 | ``` 15 | 16 | Only tested on Mac OS X and Ruby 2.1.5 17 | -------------------------------------------------------------------------------- /node/readme.md: -------------------------------------------------------------------------------- 1 | Redis in Action (the node.js way) 2 | ================================= 3 | 4 | Chapter 1 completed. 5 | 6 | npm install 7 | 8 | To run tests similar to the Python code 9 | 10 | node ch01/test.js 11 | 12 | To run mocha based tests covering a little bit more ground. 13 | 14 | node run-mocha-tests.js 15 | 16 | Only tested on Mac OS X and node 0.10.x. 17 | -------------------------------------------------------------------------------- /gawk/README: -------------------------------------------------------------------------------- 1 | To run the gawk version, you must have gawk and gawkextlib. 2 | 3 | To get gawkextlib, you can: 4 | git clone git://git.code.sf.net/u/paulinohuerta/gawkextlib u-paulinohuerta-gawkextlib 5 | 6 | The README file in that repository will explain how to build the Redis extensions for gawk. 7 | 8 | To run the code from chapter 1 (after your Redis client is properly installed): 9 | 10 | $ /path-to-gawk/gawk -f ch01.awk /dev/null 11 | -------------------------------------------------------------------------------- /java/README.md: -------------------------------------------------------------------------------- 1 | ##Prerequisites 2 | 3 | * A running Redis instance as mentioned in the book 4 | * JDK 6 or higher installed (no other Java related software is needed, since the build script takes care of all the rest). 5 | 6 | ##Running 7 | 8 | Open a command-line/terminal in the `java` directory and do one of the following: 9 | 10 | * Windows: 11 | `gradlew.bat -Pchapter=1 run`. Use numbers 1 through 9 depending on the chapter's examples you want to run 12 | 13 | * Linux/Mac: 14 | `./gradlew -Pchapter=1 run`. Use numbers 1 through 9 depending on the chapter's examples you want to run -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | redis-in-action 2 | 3 | =============== 4 | 5 | This project intends to hold the various implementations of code from the book Redis in Action, 6 | written by Josiah Carlson, published by Manning Publications, which is available for purchase: 7 | http://manning.com/carlson/ 8 | 9 | If you would like to read the Errata, it is available as PDF at the above url, or if you would 10 | like to see it as HTML; the most recent version in this repository is (hopefully always) available: 11 | http://htmlpreview.github.io/?https://github.com/josiahcarlson/redis-in-action/blob/master/excerpt_errata.html 12 | -------------------------------------------------------------------------------- /java/build.gradle: -------------------------------------------------------------------------------- 1 | apply plugin: 'java' 2 | apply plugin: 'application' 3 | 4 | version = '1.1' 5 | 6 | repositories { 7 | mavenCentral() 8 | 9 | flatDir { 10 | dirs 'libs' 11 | } 12 | } 13 | 14 | dependencies { 15 | 16 | compile 'redis.clients:jedis:2.1.0' 17 | compile 'org.javatuples:javatuples:1.2' 18 | compile 'com.google.code.gson:gson:2.2.2' 19 | 20 | compile name: 'commons-csv-20070730' 21 | 22 | } 23 | 24 | //set the main class from the command line property 'mainClass' (e.g. gradle -PmainClass=Chapter02) 25 | def chapterPrefix = 'Chapter0' 26 | def defaultChapter = '1' 27 | mainClassName = chapterPrefix + (project.hasProperty('chapter') ? chapter : defaultChapter) 28 | -------------------------------------------------------------------------------- /node/package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "redis-in-action-node", 3 | "version": "0.0.1", 4 | "description": "Redis in Action (Node.js sample code)", 5 | "main": "run-mocha-tests.js", 6 | "scripts": { 7 | "test": "./run-mocha-tests.js" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "https://github.com/danielsundman/redis-in-action" 12 | }, 13 | "keywords": [ 14 | "redis", 15 | "node" 16 | ], 17 | "author": "", 18 | "license": "BSD", 19 | "bugs": { 20 | "url": "https://github.com/danielsundman/redis-in-action/issues" 21 | }, 22 | "dependencies": { 23 | "redis": "~0.8.4" 24 | }, 25 | "devDependencies": { 26 | "mocha": "~1.12.0", 27 | "should": "~1.2.2" 28 | } 29 | } 30 | -------------------------------------------------------------------------------- /ruby/Gemfile.lock: -------------------------------------------------------------------------------- 1 | GEM 2 | remote: https://rubygems.org/ 3 | specs: 4 | coderay (1.0.9) 5 | diff-lcs (1.2.5) 6 | method_source (0.8.2) 7 | pry (0.9.12.2) 8 | coderay (~> 1.0.5) 9 | method_source (~> 0.8) 10 | slop (~> 3.4) 11 | redis (3.2.0) 12 | rspec (3.1.0) 13 | rspec-core (~> 3.1.0) 14 | rspec-expectations (~> 3.1.0) 15 | rspec-mocks (~> 3.1.0) 16 | rspec-core (3.1.7) 17 | rspec-support (~> 3.1.0) 18 | rspec-expectations (3.1.2) 19 | diff-lcs (>= 1.2.0, < 2.0) 20 | rspec-support (~> 3.1.0) 21 | rspec-mocks (3.1.3) 22 | rspec-support (~> 3.1.0) 23 | rspec-support (3.1.2) 24 | slop (3.4.6) 25 | 26 | PLATFORMS 27 | ruby 28 | 29 | DEPENDENCIES 30 | pry 31 | redis 32 | rspec 33 | -------------------------------------------------------------------------------- /node/run-mocha-tests.js: -------------------------------------------------------------------------------- 1 | #!/usr/local/bin/node 2 | // Code adapted from http://ronderksen.nl/2012/05/03/debugging-mocha-tests-in-webstorm/ 3 | 4 | var Mocha = require('mocha'), 5 | path = require('path'), 6 | fs = require('fs'); 7 | 8 | var mocha = new Mocha({ 9 | reporter: 'dot', 10 | ui: 'bdd', 11 | timeout: 999999 12 | }); 13 | 14 | var testDir = './ch01/'; 15 | 16 | fs.readdir(testDir, function (err, files) { 17 | 18 | if (err) { 19 | console.log(err); 20 | return; 21 | } 22 | 23 | files.forEach(function (file) { 24 | if (file === 'mocha-test.js') { 25 | mocha.addFile(testDir + file); 26 | } 27 | }); 28 | 29 | var runner = mocha.run(function () { 30 | console.log('finished'); 31 | }); 32 | 33 | runner.on('pass', function (test) { 34 | // console.log('... %s passed', test.title); 35 | }); 36 | 37 | runner.on('fail', function (test) { 38 | console.log('... %s failed', test.title); 39 | }); 40 | }); -------------------------------------------------------------------------------- /LICENSE.txt: -------------------------------------------------------------------------------- 1 | Copyright (c) 2012-2013 Josiah Carlson, and any contributors 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 4 | 5 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 6 | 7 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /node/ch01/test.js: -------------------------------------------------------------------------------- 1 | var ch01 = require('./main'), 2 | client = require('redis').createClient(); 3 | 4 | client.flushdb(); 5 | 6 | ch01.postArticle(client, 'username', 'A title', 'http://www.google.com', function(err, articleId) { 7 | 8 | console.log('We posted a new article with id', articleId); 9 | 10 | client.hgetall('article:' + articleId, function(err, hash) { 11 | console.log("It's HASH looks like", hash); 12 | }); 13 | 14 | ch01.articleVote(client, 'other_user', 'article:' + articleId, function(err, votes) { 15 | console.log('We voted for the article, it now has votes', votes); 16 | 17 | client.hgetall('article:' + articleId, function(err, hash) { 18 | console.log("After voting, it's HASH looks like", hash); 19 | }); 20 | 21 | ch01.getArticles(client, 1, null, function(err, articles) { 22 | console.log('The current highest-scoring articles are'); 23 | console.log(articles); 24 | 25 | ch01.addRemoveGroups(client, articleId, ['new-group'], null, function() { 26 | ch01.getGroupArticles(client, 'new-group', 1, null, function(err, articles) { 27 | console.log('We added the article to a new group, other articles include:', articles); 28 | client.quit(); 29 | }); 30 | }); 31 | }); 32 | }); 33 | }); 34 | 35 | 36 | -------------------------------------------------------------------------------- /ruby/spec/chapter01_spec.rb: -------------------------------------------------------------------------------- 1 | require 'spec_helper' 2 | require './chapter01' 3 | 4 | describe 'Chapter 1' do 5 | let(:client) { Redis.new(db: 15) } 6 | 7 | before do 8 | client.flushdb 9 | end 10 | 11 | it do 12 | article_id = post_article(client, 'username', 'A title', 'http://www.google.com') 13 | puts 'We posted a new article with id:', article_id 14 | expect(article_id).to be_truthy 15 | 16 | puts 'Its HASH looks like:' 17 | article = client.hgetall("article:#{article_id}") 18 | puts article 19 | expect(hash).to be_truthy 20 | 21 | article_vote(client, 'other_user', "article:#{article_id}") 22 | puts 'We voted for the article, it now has votes:' 23 | votes = client.hget("article:#{article_id}", 'votes').to_i 24 | puts votes 25 | expect(votes > 1).to be_truthy 26 | 27 | puts 'The currently highest-scoring articles are:' 28 | articles = get_articles(client, 1) 29 | puts articles 30 | expect(articles.count >= 1).to be_truthy 31 | 32 | add_remove_groups(client, article_id, ['new-group']) 33 | puts 'We added the article to a new group, other articles include:' 34 | articles = get_group_articles(client, 'new-group', 1) 35 | puts articles 36 | expect(articles.count >= 1).to be_truthy 37 | end 38 | end 39 | -------------------------------------------------------------------------------- /ruby/chapter01.rb: -------------------------------------------------------------------------------- 1 | ONE_WEEK_IN_SECONDS = 7 * 86400 2 | VOTE_SCORE = 432 3 | ARTICLES_PER_PAGE = 25 4 | 5 | def article_vote(client, user, article) 6 | cutoff = Time.now.to_i - ONE_WEEK_IN_SECONDS 7 | return if client.zscore('time:', article) < cutoff 8 | 9 | article_id = article.split(':')[-1] 10 | 11 | if client.sadd("voted:#{article_id}", user) 12 | client.zincrby('score:', VOTE_SCORE, article) 13 | client.hincrby(article, 'votes', 1) 14 | end 15 | end 16 | 17 | def post_article(client, user, title, link) 18 | article_id = client.incr('article:') 19 | 20 | voted = "voted:#{article_id}" 21 | client.sadd(voted, user) 22 | client.expire(voted, ONE_WEEK_IN_SECONDS) 23 | 24 | now = Time.now.to_i 25 | article = "article:#{article_id}" 26 | 27 | client.mapped_hmset( 28 | article, 29 | { 30 | title: title, 31 | link: link, 32 | poster: user, 33 | time: now, 34 | votes: 1, 35 | } 36 | ) 37 | 38 | client.zadd('score:', now + VOTE_SCORE, article) 39 | client.zadd('time:', now, article) 40 | 41 | article_id 42 | end 43 | 44 | def get_articles(client, page, order = 'score:') 45 | start = (page - 1) * ARTICLES_PER_PAGE 46 | ids = client.zrevrange(order, start, start + ARTICLES_PER_PAGE - 1) 47 | 48 | ids.inject([]) { |articles, id| 49 | articles << client.hgetall(id).merge(id: id) 50 | } 51 | end 52 | 53 | def add_remove_groups(client, article_id, to_add = [], to_remove = []) 54 | article = "article:#{article_id}" 55 | 56 | to_add.each do |group| 57 | client.sadd("group:#{group}", article) 58 | end 59 | 60 | to_remove.each do |group| 61 | client.srem("group:#{group}", article) 62 | end 63 | end 64 | 65 | def get_group_articles(client, group, page, order = 'score:') 66 | key = order + group 67 | 68 | unless client.exists(key) 69 | client.zinterstore(key, ["group:#{group}", order], aggregate: 'max') 70 | client.expire(key, 60) 71 | end 72 | 73 | get_articles(client, page, key) 74 | end 75 | -------------------------------------------------------------------------------- /java/gradlew.bat: -------------------------------------------------------------------------------- 1 | @if "%DEBUG%" == "" @echo off 2 | @rem ########################################################################## 3 | @rem 4 | @rem Gradle startup script for Windows 5 | @rem 6 | @rem ########################################################################## 7 | 8 | @rem Set local scope for the variables with windows NT shell 9 | if "%OS%"=="Windows_NT" setlocal 10 | 11 | @rem Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 12 | set DEFAULT_JVM_OPTS= 13 | 14 | set DIRNAME=%~dp0 15 | if "%DIRNAME%" == "" set DIRNAME=. 16 | set APP_BASE_NAME=%~n0 17 | set APP_HOME=%DIRNAME% 18 | 19 | @rem Find java.exe 20 | if defined JAVA_HOME goto findJavaFromJavaHome 21 | 22 | set JAVA_EXE=java.exe 23 | %JAVA_EXE% -version >NUL 2>&1 24 | if "%ERRORLEVEL%" == "0" goto init 25 | 26 | echo. 27 | echo ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 28 | echo. 29 | echo Please set the JAVA_HOME variable in your environment to match the 30 | echo location of your Java installation. 31 | 32 | goto fail 33 | 34 | :findJavaFromJavaHome 35 | set JAVA_HOME=%JAVA_HOME:"=% 36 | set JAVA_EXE=%JAVA_HOME%/bin/java.exe 37 | 38 | if exist "%JAVA_EXE%" goto init 39 | 40 | echo. 41 | echo ERROR: JAVA_HOME is set to an invalid directory: %JAVA_HOME% 42 | echo. 43 | echo Please set the JAVA_HOME variable in your environment to match the 44 | echo location of your Java installation. 45 | 46 | goto fail 47 | 48 | :init 49 | @rem Get command-line arguments, handling Windowz variants 50 | 51 | if not "%OS%" == "Windows_NT" goto win9xME_args 52 | if "%@eval[2+2]" == "4" goto 4NT_args 53 | 54 | :win9xME_args 55 | @rem Slurp the command line arguments. 56 | set CMD_LINE_ARGS= 57 | set _SKIP=2 58 | 59 | :win9xME_args_slurp 60 | if "x%~1" == "x" goto execute 61 | 62 | set CMD_LINE_ARGS=%* 63 | goto execute 64 | 65 | :4NT_args 66 | @rem Get arguments from the 4NT Shell from JP Software 67 | set CMD_LINE_ARGS=%$ 68 | 69 | :execute 70 | @rem Setup the command line 71 | 72 | set CLASSPATH=%APP_HOME%\gradle\wrapper\gradle-wrapper.jar 73 | 74 | @rem Execute Gradle 75 | "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" org.gradle.wrapper.GradleWrapperMain %CMD_LINE_ARGS% 76 | 77 | :end 78 | @rem End local scope for the variables with windows NT shell 79 | if "%ERRORLEVEL%"=="0" goto mainEnd 80 | 81 | :fail 82 | rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of 83 | rem the _cmd.exe /c_ return code! 84 | if not "" == "%GRADLE_EXIT_CONSOLE%" exit 1 85 | exit /b 1 86 | 87 | :mainEnd 88 | if "%OS%"=="Windows_NT" endlocal 89 | 90 | :omega 91 | -------------------------------------------------------------------------------- /gawk/ch01/ch01.awk: -------------------------------------------------------------------------------- 1 | @load "redis" 2 | BEGIN{ 3 | ONE_WEEK_IN_SECONDS = 7*86400 4 | VOTE_SCORE = 432 5 | ARTICLES_PER_PAGE = 25 6 | c=connectRedis() 7 | select(c,12) 8 | articleId=postArticle(c, "username", "A title", "http://www.google.com") 9 | print "We posted a new article with id: "articleId 10 | print "Its HASH looks like:" 11 | hgetall(c,"article:"articleId,RET) 12 | for(i=1;i<=length(RET);i+=2) { 13 | print " "RET[i]": "RET[i+1] 14 | } 15 | print 16 | articleVote(c, "other_user", "article:"articleId) 17 | votes = hget(c,"article:"articleId, "votes") 18 | print "We voted for the article, it now has votes: "votes 19 | print "The currently highest-scoring articles are:" 20 | getArticles(c, 1, articles) # articles is an array 21 | dumparray(articles,"") 22 | ARR[1]="new-group" 23 | addGroups(c, articleId, ARR) 24 | print "We added the article to a new group, other articles include:" 25 | delete(articles) 26 | getGroupArticles(c, "new-group", 1, articles) 27 | dumparray(articles,"") 28 | } 29 | 30 | function getGroupArticles(c, group, page, articles) { 31 | return getGroupArticles1(c, group, page, "score:", articles) 32 | } 33 | 34 | function getGroupArticles1(c, group, page, order, articles) { 35 | key=order""group 36 | if(!exists(c,key)) { 37 | ARI[1]="group:"group 38 | ARI[2]=order 39 | zinterstore(c,key,ARI,"aggregate max") 40 | expire(c,key, 60) 41 | } 42 | getArticles1(c, page, key, articles) 43 | } 44 | 45 | function getArticles(c, page, articles) { 46 | getArticles1(conn, page, "score:", articles) 47 | } 48 | 49 | function getArticles1(c, page, order, articles) { 50 | start = (page - 1) * ARTICLES_PER_PAGE 51 | end = start + ARTICLES_PER_PAGE - 1 52 | delete(RET) 53 | zrevrange(c,order,RET,start,end) 54 | for(i in RET) { 55 | hgetall(c,RET[i],AR) 56 | for(j=1;j<=length(AR);j+=2) { 57 | articles[i][AR[j]]=AR[j+1] 58 | } 59 | articles[i]["id"]=RET[i] 60 | } 61 | } 62 | 63 | function addGroups(c, articleId, TOADD) { 64 | article = "article:"articleId 65 | for(i in TOADD) { 66 | sadd(c,"group:"TOADD[i], article) 67 | } 68 | } 69 | 70 | function postArticle(c, user,title,link) { 71 | articleId=incr(c,"article:") 72 | voted="voted:"articleId; 73 | sadd(c,voted,user) 74 | expire(c,voted,ONE_WEEK_IN_SECONDS) 75 | now=systime() 76 | article = "article:"articleId 77 | AR[1]="title" 78 | AR[2]=title 79 | AR[3]="link" 80 | AR[4]="http://www.google.com" 81 | AR[5]="user" 82 | AR[6]=user 83 | AR[7]="now" 84 | AR[8]=now 85 | AR[9]="votes" 86 | AR[10]=1 87 | hmset(c,article,AR) 88 | zadd(c,"score:",now + VOTE_SCORE,article) 89 | zadd(c,"time:",now,article) 90 | return articleId 91 | } 92 | 93 | function articleVote(c, user, article) { 94 | cutoff= systime() - ONE_WEEK_IN_SECONDS 95 | if(zscore(c,"time:",article) < cutoff){ 96 | return 97 | } 98 | articleId = substr(article,index(article,":")+1) 99 | if (sadd(c,"voted:"articleId,user) == 1) { 100 | zincrby(c,"score:", VOTE_SCORE, article) 101 | hincrby(c,article, "votes", 1) 102 | } 103 | } 104 | 105 | function dumparray(array,e, i) 106 | { 107 | for (i in array){ 108 | if (isarray(array[i])){ 109 | print " id: "array[i]["id"] 110 | dumparray(array[i],"") 111 | } 112 | else { 113 | if(e){ 114 | printf("%s[%s] = %s\n",e,i, array[i]) 115 | } 116 | else { 117 | if(i=="id") 118 | continue 119 | else 120 | printf(" %s = %s\n",i, array[i]) 121 | } 122 | } 123 | } 124 | } 125 | -------------------------------------------------------------------------------- /node/ch01/main.js: -------------------------------------------------------------------------------- 1 | var ONE_WEEK_IN_SECONDS = 7 * 24 * 60 * 60, 2 | VOTE_SCORE = 432; 3 | 4 | var today = todayOrig = function() { 5 | return new Date(); 6 | }; 7 | 8 | var articleVote = function(client, user, article, cb) { 9 | var cutoff = (today() / 1000) - ONE_WEEK_IN_SECONDS; 10 | client.zscore('time:', article, function(err, result) { 11 | if (err || result < cutoff) return cb(err || new Error('cutoff')); 12 | var articleId = article.substring(article.indexOf(':') + 1); 13 | client.sadd('voted:' + articleId, user, function(err, result) { 14 | if (err) return cb(err); 15 | if (result === 1) { 16 | client.zincrby('score:', VOTE_SCORE, article, function(err) { 17 | if (err) return cb(err); 18 | client.hincrby(article, 'votes', 1, function(err, result) { 19 | return cb(err, result); 20 | }); 21 | }); 22 | } else { 23 | return cb(new Error(user + ' already voted for ' + article)); 24 | } 25 | }); 26 | }); 27 | }; 28 | 29 | var getNextArticleId = function(client, cb) { 30 | client.incr("article:", function(err, id) { 31 | if (err) return cb(err); 32 | cb(null, id); 33 | }); 34 | }; 35 | 36 | var postArticle = function(client, user, title, link, cb) { 37 | getNextArticleId(client, function(err, id) { 38 | if (err) return cb(err); 39 | var voted = "voted:" + id; 40 | client.sadd(voted, user, function(err) { 41 | if (err) return cb(err); 42 | client.expire(voted, ONE_WEEK_IN_SECONDS, function(err) { 43 | if (err) return cb(err); 44 | var now = today() / 1000; 45 | var article = 'article:' + id; 46 | var articleData = { 47 | title: title, 48 | link: link, 49 | user: user, 50 | now: '' + now, 51 | votes: '1' 52 | }; 53 | client.hmset(article, articleData, function(err) { 54 | if (err) return cb(err); 55 | client.zadd('score:', now + VOTE_SCORE, article, function(err) { 56 | if (err) return cb(err); 57 | client.zadd('time:', now, article, function(err) { 58 | cb(err, id); 59 | }); 60 | }); 61 | }); 62 | }); 63 | }); 64 | }); 65 | }; 66 | 67 | var ARTICLES_PER_PAGE = 25; 68 | var getArticles = function(client, page, order, cb) { 69 | order = order || 'score:'; 70 | 71 | var start = (page-1) * ARTICLES_PER_PAGE; 72 | var end = start + ARTICLES_PER_PAGE - 1; 73 | 74 | client.zrevrange(order, start, end, function(err, ids) { 75 | if (err) return cb(err); 76 | var articles = []; 77 | var count = ids.length; 78 | ids.forEach(function(id) { 79 | client.hgetall(id, function(err, articleData) { 80 | if (err) return cb(err); 81 | articleData['id'] = id; 82 | articles.push(articleData); 83 | count -= 1; 84 | if (count === 0) { 85 | return cb(null, articles); 86 | } 87 | }); 88 | }); 89 | if (ids.length === 0) { 90 | cb(null, []); 91 | } 92 | }); 93 | }; 94 | 95 | var addRemoveGroups = function(client, articleId, toAdd, toRemove, cb) { 96 | toAdd = toAdd || []; 97 | toRemove = toRemove || []; 98 | var article = 'article:' + articleId; 99 | 100 | var toAddCount = 0, toRemoveCount = 0; 101 | toAdd.forEach(function(group) { 102 | client.sadd('group:' + group, article, function(err) { 103 | if (err) return cb(err); 104 | toAddCount += 1; 105 | if (toAddCount === toAdd.length && toRemoveCount === toRemove.length) return cb(null); 106 | }); 107 | }); 108 | 109 | toRemove.forEach(function(group) { 110 | client.srem('group:' + group, article, function(err) { 111 | if (err) return cb(err); 112 | toRemoveCount += 1; 113 | if (toAddCount === toAdd.length && toRemoveCount === toRemove.length) return cb(null); 114 | }); 115 | }); 116 | 117 | }; 118 | 119 | var getGroupArticles = function(client, group, page, order, cb) { 120 | order = order || 'score:'; 121 | var key = order + group; 122 | client.exists(key, function(err, exists) { 123 | if (err) return cb(err); 124 | if (!exists) { 125 | var args = [key, '2', 'group:' + group, order, 'aggregate', 'max']; 126 | client.zinterstore(args, function(err) { 127 | if (err) return cb(err); 128 | client.expire(key, 60, function(err) { 129 | if (err) return cb(err); 130 | getArticles(client, page, key, cb); 131 | }); 132 | }); 133 | } else { 134 | getArticles(client, page, key, cb); 135 | } 136 | }); 137 | }; 138 | 139 | 140 | module.exports = { 141 | articleVote: articleVote, 142 | postArticle: postArticle, 143 | getArticles: getArticles, 144 | addRemoveGroups: addRemoveGroups, 145 | getGroupArticles: getGroupArticles, 146 | 147 | // Exposed for testing purposes only 148 | ONE_WEEK_IN_SECONDS: ONE_WEEK_IN_SECONDS, 149 | setToday: function(f) { 150 | today = f || todayOrig; 151 | } 152 | }; 153 | -------------------------------------------------------------------------------- /gawk/ch04/ch04.awk: -------------------------------------------------------------------------------- 1 | @load "redis" 2 | BEGIN{ 3 | c=connectRedis() 4 | select(c,12) 5 | testListItem(c, 0) 6 | testPurchaseItem(c) 7 | testBenchmarkUpdateToken(c) 8 | } 9 | 10 | function testListItem(c, nested) { 11 | if (!nested){ 12 | print "\n----- testListItem -----" 13 | } 14 | print "We need to set up just enough state so that a user can list an item" 15 | seller = "userX" 16 | item = "itemX" 17 | sadd(c,"inventory:"seller, item) 18 | delete(AR) 19 | smembers(c,"inventory:"seller,AR) 20 | print "The user's inventory has:" 21 | for(i in AR) { 22 | print " "AR[i] 23 | } 24 | print 25 | print "Listing the item..." 26 | l = listItem(conn, item, seller, 10) 27 | print "Listing the item succeeded? "l 28 | delete(AR) 29 | zrangeWithScores(c,"market:",AR,0, -1) 30 | print "The market contains:" 31 | for(i=1;i<=length(AR);i+=2){ 32 | print " "AR[i]", "AR[i+1] 33 | } 34 | } 35 | 36 | function testPurchaseItem(c) { 37 | print "\n----- testPurchaseItem -----" 38 | testListItem(c, 1) 39 | print "We need to set up just enough state so a user can buy an item" 40 | hset(c,"users:userY", "funds", "125") 41 | delete(AR) 42 | hgetall(c,"users:userY",AR) 43 | print "The user has some money:" 44 | for(i=1;i<=length(AR);i+=2){ 45 | print " "AR[i]": "AR[i+1] 46 | } 47 | print 48 | print "Let's purchase an item" 49 | p = purchaseItem(c, "userY", "itemX", "userX", 10) 50 | print "Purchasing an item succeeded? "p 51 | delete(AR) 52 | hgetall(c,"users:userY",AR) 53 | print "Their money is now:" 54 | for(i=1;i<=length(AR);i+=2){ 55 | print " "AR[i]": "AR[i+1] 56 | } 57 | buyer = "userY" 58 | delete(AR) 59 | smembers(c,"inventory:"buyer,AR) 60 | print "Their inventory is now:" 61 | for(member in AR) { 62 | print " "AR[member] 63 | } 64 | } 65 | 66 | function testBenchmarkUpdateToken(c) { 67 | print "\n----- testBenchmarkUpdate -----" 68 | benchmarkUpdateToken(c, 5) 69 | } 70 | 71 | function listItem(c, itemId, sellerId, price) { 72 | inventory = "inventory:"sellerId 73 | item = itemId"."sellerId 74 | now=systime()*1000 75 | end=now+5000 76 | while (now < end) { 77 | watch(c,inventory) 78 | if(!sismember(c,inventory, itemId)){ 79 | unwatch(c) 80 | return 0 81 | } 82 | multi(c) 83 | zadd(c,"market:", price, item) 84 | srem(c,inventory, itemId) 85 | if(exec(c,R)==0) { 86 | now=systime()*1000 87 | continue 88 | } 89 | return 1 90 | } 91 | return 0 92 | } 93 | 94 | function purchaseItem(c, buyerId, itemId, sellerId, lprice) { 95 | buyer = "users:"buyerId 96 | seller = "users:"sellerId 97 | item = itemId"."sellerId 98 | inventory = "inventory:"buyerId 99 | end=(systime()*1000)+10000 100 | while ((systime()*1000) < end){ 101 | watch(c,"market:"buyer) 102 | price = zscore(c,"market:", item) 103 | funds = hget(c,buyer, "funds") 104 | if (price != lprice || price > funds){ 105 | unwatch() 106 | return 0 107 | } 108 | multi(c) 109 | hincrby(c, seller, "funds", price) 110 | hincrby(c, buyer, "funds", -price) 111 | sadd(c, inventory, itemId) 112 | zrem(c, "market:", item) 113 | if(exec(c,R)==0) { 114 | continue 115 | } 116 | return 1 117 | } 118 | return 0 119 | } 120 | 121 | function benchmarkUpdateToken(c, duration) { 122 | for(i=1;i<=2;i++) { 123 | if(i==1){ 124 | method="updateToken" 125 | } 126 | else { 127 | method="updateTokenPipeline" 128 | p=pipeline(c) 129 | } 130 | count = 0 131 | start=systime()*1000 132 | end = start + (duration * 1000) 133 | while((systime()*1000) < end){ 134 | count++; 135 | if(i==1){ 136 | updateToken(c, "token", "user", "item") 137 | } 138 | else { 139 | updateTokenPipeline(c, "token", "user", "item", p) 140 | } 141 | } 142 | delta=(systime()*1000) - start 143 | print method" "count" "(delta / 1000)" "(count / (delta / 1000)) 144 | } 145 | } 146 | 147 | function updateToken(c, token, user, item) { 148 | timestamp=systime() 149 | hset(c,"login:", token, user) 150 | zadd(c,"recent:", timestamp, token) 151 | if (item) { 152 | zadd(c,"viewed:"token, timestamp, item) 153 | zremrangebyrank(c, "viewed:"token, 0, -26) 154 | zincrby(c, "viewed:", -1, item) 155 | } 156 | } 157 | 158 | function updateTokenPipeline(c, token, user, item, p) { 159 | timestamp=systime() 160 | hset(p,"login:", token, user) 161 | zadd(p,"recent:", timestamp, token) 162 | if (item) { 163 | zadd(p,"viewed:"token, timestamp, item) 164 | zremrangebyrank(p,"viewed:"token, 0, -26) 165 | zincrby(p,"viewed:", -1, item) 166 | } 167 | for(ERRNO="" ; ERRNO=="" ; getReply(p,REPLY)) 168 | ; 169 | } 170 | -------------------------------------------------------------------------------- /java/src/main/java/Chapter01.java: -------------------------------------------------------------------------------- 1 | import redis.clients.jedis.Jedis; 2 | import redis.clients.jedis.ZParams; 3 | 4 | import java.util.*; 5 | 6 | public class Chapter01 { 7 | private static final int ONE_WEEK_IN_SECONDS = 7 * 86400; 8 | private static final int VOTE_SCORE = 432; 9 | private static final int ARTICLES_PER_PAGE = 25; 10 | 11 | public static final void main(String[] args) { 12 | new Chapter01().run(); 13 | } 14 | 15 | public void run() { 16 | Jedis conn = new Jedis("localhost"); 17 | conn.select(15); 18 | 19 | String articleId = postArticle( 20 | conn, "username", "A title", "http://www.google.com"); 21 | System.out.println("We posted a new article with id: " + articleId); 22 | System.out.println("Its HASH looks like:"); 23 | Map articleData = conn.hgetAll("article:" + articleId); 24 | for (Map.Entry entry : articleData.entrySet()){ 25 | System.out.println(" " + entry.getKey() + ": " + entry.getValue()); 26 | } 27 | 28 | System.out.println(); 29 | 30 | articleVote(conn, "other_user", "article:" + articleId); 31 | String votes = conn.hget("article:" + articleId, "votes"); 32 | System.out.println("We voted for the article, it now has votes: " + votes); 33 | assert Integer.parseInt(votes) > 1; 34 | 35 | System.out.println("The currently highest-scoring articles are:"); 36 | List> articles = getArticles(conn, 1); 37 | printArticles(articles); 38 | assert articles.size() >= 1; 39 | 40 | addGroups(conn, articleId, new String[]{"new-group"}); 41 | System.out.println("We added the article to a new group, other articles include:"); 42 | articles = getGroupArticles(conn, "new-group", 1); 43 | printArticles(articles); 44 | assert articles.size() >= 1; 45 | } 46 | 47 | public String postArticle(Jedis conn, String user, String title, String link) { 48 | String articleId = String.valueOf(conn.incr("article:")); 49 | 50 | String voted = "voted:" + articleId; 51 | conn.sadd(voted, user); 52 | conn.expire(voted, ONE_WEEK_IN_SECONDS); 53 | 54 | long now = System.currentTimeMillis() / 1000; 55 | String article = "article:" + articleId; 56 | HashMap articleData = new HashMap(); 57 | articleData.put("title", title); 58 | articleData.put("link", link); 59 | articleData.put("user", user); 60 | articleData.put("now", String.valueOf(now)); 61 | articleData.put("votes", "1"); 62 | conn.hmset(article, articleData); 63 | conn.zadd("score:", now + VOTE_SCORE, article); 64 | conn.zadd("time:", now, article); 65 | 66 | return articleId; 67 | } 68 | 69 | public void articleVote(Jedis conn, String user, String article) { 70 | long cutoff = (System.currentTimeMillis() / 1000) - ONE_WEEK_IN_SECONDS; 71 | if (conn.zscore("time:", article) < cutoff){ 72 | return; 73 | } 74 | 75 | String articleId = article.substring(article.indexOf(':') + 1); 76 | if (conn.sadd("voted:" + articleId, user) == 1) { 77 | conn.zincrby("score:", VOTE_SCORE, article); 78 | conn.hincrBy(article, "votes", 1l); 79 | } 80 | } 81 | 82 | 83 | public List> getArticles(Jedis conn, int page) { 84 | return getArticles(conn, page, "score:"); 85 | } 86 | 87 | public List> getArticles(Jedis conn, int page, String order) { 88 | int start = (page - 1) * ARTICLES_PER_PAGE; 89 | int end = start + ARTICLES_PER_PAGE - 1; 90 | 91 | Set ids = conn.zrevrange(order, start, end); 92 | List> articles = new ArrayList>(); 93 | for (String id : ids){ 94 | Map articleData = conn.hgetAll(id); 95 | articleData.put("id", id); 96 | articles.add(articleData); 97 | } 98 | 99 | return articles; 100 | } 101 | 102 | public void addGroups(Jedis conn, String articleId, String[] toAdd) { 103 | String article = "article:" + articleId; 104 | for (String group : toAdd) { 105 | conn.sadd("group:" + group, article); 106 | } 107 | } 108 | 109 | public List> getGroupArticles(Jedis conn, String group, int page) { 110 | return getGroupArticles(conn, group, page, "score:"); 111 | } 112 | 113 | public List> getGroupArticles(Jedis conn, String group, int page, String order) { 114 | String key = order + group; 115 | if (!conn.exists(key)) { 116 | ZParams params = new ZParams().aggregate(ZParams.Aggregate.MAX); 117 | conn.zinterstore(key, params, "group:" + group, order); 118 | conn.expire(key, 60); 119 | } 120 | return getArticles(conn, page, key); 121 | } 122 | 123 | private void printArticles(List> articles){ 124 | for (Map article : articles){ 125 | System.out.println(" id: " + article.get("id")); 126 | for (Map.Entry entry : article.entrySet()){ 127 | if (entry.getKey().equals("id")){ 128 | continue; 129 | } 130 | System.out.println(" " + entry.getKey() + ": " + entry.getValue()); 131 | } 132 | } 133 | } 134 | } 135 | -------------------------------------------------------------------------------- /java/gradlew: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env bash 2 | 3 | ############################################################################## 4 | ## 5 | ## Gradle start up script for UN*X 6 | ## 7 | ############################################################################## 8 | 9 | # Add default JVM options here. You can also use JAVA_OPTS and GRADLE_OPTS to pass JVM options to this script. 10 | DEFAULT_JVM_OPTS="" 11 | 12 | APP_NAME="Gradle" 13 | APP_BASE_NAME=`basename "$0"` 14 | 15 | # Use the maximum available, or set MAX_FD != -1 to use that value. 16 | MAX_FD="maximum" 17 | 18 | warn ( ) { 19 | echo "$*" 20 | } 21 | 22 | die ( ) { 23 | echo 24 | echo "$*" 25 | echo 26 | exit 1 27 | } 28 | 29 | # OS specific support (must be 'true' or 'false'). 30 | cygwin=false 31 | msys=false 32 | darwin=false 33 | case "`uname`" in 34 | CYGWIN* ) 35 | cygwin=true 36 | ;; 37 | Darwin* ) 38 | darwin=true 39 | ;; 40 | MINGW* ) 41 | msys=true 42 | ;; 43 | esac 44 | 45 | # For Cygwin, ensure paths are in UNIX format before anything is touched. 46 | if $cygwin ; then 47 | [ -n "$JAVA_HOME" ] && JAVA_HOME=`cygpath --unix "$JAVA_HOME"` 48 | fi 49 | 50 | # Attempt to set APP_HOME 51 | # Resolve links: $0 may be a link 52 | PRG="$0" 53 | # Need this for relative symlinks. 54 | while [ -h "$PRG" ] ; do 55 | ls=`ls -ld "$PRG"` 56 | link=`expr "$ls" : '.*-> \(.*\)$'` 57 | if expr "$link" : '/.*' > /dev/null; then 58 | PRG="$link" 59 | else 60 | PRG=`dirname "$PRG"`"/$link" 61 | fi 62 | done 63 | SAVED="`pwd`" 64 | cd "`dirname \"$PRG\"`/" >&- 65 | APP_HOME="`pwd -P`" 66 | cd "$SAVED" >&- 67 | 68 | CLASSPATH=$APP_HOME/gradle/wrapper/gradle-wrapper.jar 69 | 70 | # Determine the Java command to use to start the JVM. 71 | if [ -n "$JAVA_HOME" ] ; then 72 | if [ -x "$JAVA_HOME/jre/sh/java" ] ; then 73 | # IBM's JDK on AIX uses strange locations for the executables 74 | JAVACMD="$JAVA_HOME/jre/sh/java" 75 | else 76 | JAVACMD="$JAVA_HOME/bin/java" 77 | fi 78 | if [ ! -x "$JAVACMD" ] ; then 79 | die "ERROR: JAVA_HOME is set to an invalid directory: $JAVA_HOME 80 | 81 | Please set the JAVA_HOME variable in your environment to match the 82 | location of your Java installation." 83 | fi 84 | else 85 | JAVACMD="java" 86 | which java >/dev/null 2>&1 || die "ERROR: JAVA_HOME is not set and no 'java' command could be found in your PATH. 87 | 88 | Please set the JAVA_HOME variable in your environment to match the 89 | location of your Java installation." 90 | fi 91 | 92 | # Increase the maximum file descriptors if we can. 93 | if [ "$cygwin" = "false" -a "$darwin" = "false" ] ; then 94 | MAX_FD_LIMIT=`ulimit -H -n` 95 | if [ $? -eq 0 ] ; then 96 | if [ "$MAX_FD" = "maximum" -o "$MAX_FD" = "max" ] ; then 97 | MAX_FD="$MAX_FD_LIMIT" 98 | fi 99 | ulimit -n $MAX_FD 100 | if [ $? -ne 0 ] ; then 101 | warn "Could not set maximum file descriptor limit: $MAX_FD" 102 | fi 103 | else 104 | warn "Could not query maximum file descriptor limit: $MAX_FD_LIMIT" 105 | fi 106 | fi 107 | 108 | # For Darwin, add options to specify how the application appears in the dock 109 | if $darwin; then 110 | GRADLE_OPTS="$GRADLE_OPTS \"-Xdock:name=$APP_NAME\" \"-Xdock:icon=$APP_HOME/media/gradle.icns\"" 111 | fi 112 | 113 | # For Cygwin, switch paths to Windows format before running java 114 | if $cygwin ; then 115 | APP_HOME=`cygpath --path --mixed "$APP_HOME"` 116 | CLASSPATH=`cygpath --path --mixed "$CLASSPATH"` 117 | 118 | # We build the pattern for arguments to be converted via cygpath 119 | ROOTDIRSRAW=`find -L / -maxdepth 1 -mindepth 1 -type d 2>/dev/null` 120 | SEP="" 121 | for dir in $ROOTDIRSRAW ; do 122 | ROOTDIRS="$ROOTDIRS$SEP$dir" 123 | SEP="|" 124 | done 125 | OURCYGPATTERN="(^($ROOTDIRS))" 126 | # Add a user-defined pattern to the cygpath arguments 127 | if [ "$GRADLE_CYGPATTERN" != "" ] ; then 128 | OURCYGPATTERN="$OURCYGPATTERN|($GRADLE_CYGPATTERN)" 129 | fi 130 | # Now convert the arguments - kludge to limit ourselves to /bin/sh 131 | i=0 132 | for arg in "$@" ; do 133 | CHECK=`echo "$arg"|egrep -c "$OURCYGPATTERN" -` 134 | CHECK2=`echo "$arg"|egrep -c "^-"` ### Determine if an option 135 | 136 | if [ $CHECK -ne 0 ] && [ $CHECK2 -eq 0 ] ; then ### Added a condition 137 | eval `echo args$i`=`cygpath --path --ignore --mixed "$arg"` 138 | else 139 | eval `echo args$i`="\"$arg\"" 140 | fi 141 | i=$((i+1)) 142 | done 143 | case $i in 144 | (0) set -- ;; 145 | (1) set -- "$args0" ;; 146 | (2) set -- "$args0" "$args1" ;; 147 | (3) set -- "$args0" "$args1" "$args2" ;; 148 | (4) set -- "$args0" "$args1" "$args2" "$args3" ;; 149 | (5) set -- "$args0" "$args1" "$args2" "$args3" "$args4" ;; 150 | (6) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" ;; 151 | (7) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" ;; 152 | (8) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" ;; 153 | (9) set -- "$args0" "$args1" "$args2" "$args3" "$args4" "$args5" "$args6" "$args7" "$args8" ;; 154 | esac 155 | fi 156 | 157 | # Split up the JVM_OPTS And GRADLE_OPTS values into an array, following the shell quoting and substitution rules 158 | function splitJvmOpts() { 159 | JVM_OPTS=("$@") 160 | } 161 | eval splitJvmOpts $DEFAULT_JVM_OPTS $JAVA_OPTS $GRADLE_OPTS 162 | JVM_OPTS[${#JVM_OPTS[*]}]="-Dorg.gradle.appname=$APP_BASE_NAME" 163 | 164 | exec "$JAVACMD" "${JVM_OPTS[@]}" -classpath "$CLASSPATH" org.gradle.wrapper.GradleWrapperMain "$@" 165 | -------------------------------------------------------------------------------- /node/ch01/mocha-test.js: -------------------------------------------------------------------------------- 1 | describe('Redis in Action - Chapter 1', function() { 2 | var ch01 = require('./main'), 3 | redis = require('redis'), 4 | should = require('should'); 5 | 6 | var client; 7 | 8 | before(function() { 9 | client = redis.createClient(); 10 | client.flushdb(); 11 | }); 12 | 13 | after(function() { 14 | client.quit(); 15 | }); 16 | 17 | describe('Post', function() { 18 | it('should be possible to post an article and read it back given the id returned', function(done) { 19 | var before = new Date().getTime() / 1000; 20 | ch01.postArticle(client, 'username', 'A title', 'http://www.google.com', function(err, id) { 21 | client.hgetall('article:' + id, function(err, result) { 22 | result.title.should.equal('A title'); 23 | result.link.should.equal('http://www.google.com'); 24 | result.user.should.equal('username'); 25 | parseInt(result.votes, 10).should.equal(1); 26 | parseFloat(result.now).should.be.above(before); 27 | done(); 28 | }); 29 | }); 30 | }); 31 | }); 32 | 33 | describe('Empty Get', function() { 34 | beforeEach(function() { 35 | client.flushdb(); 36 | }); 37 | it('getArticles should return empty list when db is empty', function(done) { 38 | ch01.getArticles(client, 1, null, function(err, articles) { 39 | should.not.exist(err); 40 | articles.length.should.equal(0); 41 | done(); 42 | }); 43 | }); 44 | }); 45 | 46 | describe('Vote and Get', function() { 47 | var ids; 48 | var voteForArticles = function(done) { 49 | ch01.articleVote(client, 'user2', 'article:' + ids[1], function() { 50 | ch01.articleVote(client, 'user2', 'article:' + ids[2], function() { 51 | ch01.articleVote(client, 'user3', 'article:' + ids[2], function(err) { 52 | done(err); 53 | }); 54 | }); 55 | }); 56 | }; 57 | beforeEach(function(done) { 58 | ids = []; 59 | var cb = function(err, id) { 60 | ids.push(id); 61 | if (ids.length === 3) { 62 | voteForArticles(done); 63 | } 64 | }; 65 | 66 | client.flushdb(); // Empty db so we know what's there 67 | 68 | ch01.postArticle(client, 'username', 'a0', 'link0', cb); 69 | ch01.postArticle(client, 'username', 'a1', 'link1', cb); 70 | ch01.postArticle(client, 'username', 'a2', 'link1', cb); 71 | }); 72 | 73 | it('should return articles sorted according to number of votes', function(done) { 74 | ch01.getArticles(client, 1, null, function(err, articles) { 75 | should.not.exist(err); 76 | 77 | articles.length.should.equal(3); 78 | 79 | articles[0].id.should.equal('article:' + ids[2]); 80 | parseInt(articles[0].votes, 10).should.equal(3); 81 | 82 | articles[1].id.should.equal('article:' + ids[1]); 83 | parseInt(articles[1].votes, 10).should.equal(2); 84 | 85 | articles[2].id.should.equal('article:' + ids[0]); 86 | parseInt(articles[2].votes, 10).should.equal(1); 87 | 88 | done(); 89 | }); 90 | }); 91 | 92 | it('should not be possible to vote for the same article twice', function(done) { 93 | ch01.articleVote(client, 'user2', 'article:' + ids[1], function(err) { 94 | should.exist(err); 95 | err.should.be.an.instanceOf(Error); 96 | err.message.should.equal('user2 already voted for article:' + ids[1]); 97 | done(); 98 | }); 99 | }); 100 | 101 | }); 102 | 103 | describe('Vote after cutoff', function() { 104 | var articleId; 105 | beforeEach(function(done) { 106 | ch01.postArticle(client, 'username', 'a0', 'link0', function(err, id) { 107 | articleId = id; 108 | // Set today to be one week and one millisecond later 109 | ch01.setToday(function() { 110 | return new Date(new Date().getTime() + ch01.ONE_WEEK_IN_SECONDS * 1000 + 1); 111 | }); 112 | done(err); 113 | }); 114 | }); 115 | afterEach(function() { 116 | ch01.setToday(); 117 | }); 118 | 119 | it('should not be possible to vote for an article after the cutoff', function(done) { 120 | ch01.articleVote(client, 'user2', 'article:' + articleId, function(err) { 121 | should.exist(err); 122 | err.should.be.an.instanceOf(Error); 123 | err.message.should.equal('cutoff'); 124 | done(); 125 | }); 126 | }); 127 | 128 | }); 129 | 130 | describe('Groups - create and remove', function() { 131 | 132 | beforeEach(function(done) { 133 | ch01.addRemoveGroups(client, '1', ['x'], null, function(err) { 134 | done(err); 135 | }); 136 | }); 137 | 138 | it('should be possible to create group0', function(done) { 139 | ch01.addRemoveGroups(client, '1', ['group0'], [], function(err) { 140 | should.not.exist(err); 141 | client.smembers('group:group0', function(err, result) { 142 | should.not.exist(err); 143 | result.length.should.equal(1); 144 | result[0].should.equal('article:1'); 145 | done(); 146 | }); 147 | 148 | }); 149 | }); 150 | it('should be possible to remove group x', function(done) { 151 | ch01.addRemoveGroups(client, '1', undefined, ['x'], function(err) { 152 | should.not.exist(err); 153 | client.smembers('group:x', function(err, result) { 154 | should.not.exist(err); 155 | result.length.should.equal(0); 156 | done(); 157 | }); 158 | }); 159 | }); 160 | it('should be possible to add group1 and remove group x at the same time', function(done) { 161 | ch01.addRemoveGroups(client, '1', ['group1'], ['x'], function(err) { 162 | should.not.exist(err); 163 | client.smembers('group:group1', function(err, result) { 164 | should.not.exist(err); 165 | result.length.should.equal(1); 166 | result[0].should.equal('article:1'); 167 | 168 | client.smembers('group:x', function(err, result) { 169 | should.not.exist(err); 170 | result.length.should.equal(0); 171 | done(); 172 | }); 173 | }); 174 | }); 175 | }); 176 | }); 177 | 178 | describe('Groups', function() { 179 | 180 | var ids; 181 | var voteForArticles = function(done) { 182 | ch01.articleVote(client, 'user2', 'article:' + ids[1], function() { 183 | ch01.articleVote(client, 'user2', 'article:' + ids[2], function() { 184 | ch01.articleVote(client, 'user3', 'article:' + ids[2], function() { 185 | addGroups(done); 186 | }); 187 | }); 188 | }); 189 | }; 190 | var addGroups = function(done) { 191 | ch01.addRemoveGroups(client, ids[0], ['g0', 'g1'], null, function() { 192 | ch01.addRemoveGroups(client, ids[1], ['g1'], null, function() { 193 | ch01.addRemoveGroups(client, ids[2], ['g0', 'g1', 'g2'], null, function(err) { 194 | done(err); 195 | }); 196 | }); 197 | }); 198 | }; 199 | beforeEach(function(done) { 200 | ids = []; 201 | var cb = function(err, id) { 202 | ids.push(id); 203 | if (ids.length === 3) { 204 | voteForArticles(done); 205 | } 206 | }; 207 | 208 | client.flushdb(); // Empty db so we know what's there 209 | 210 | ch01.postArticle(client, 'username', 'a0', 'link0', cb); 211 | ch01.postArticle(client, 'username', 'a1', 'link1', cb); 212 | ch01.postArticle(client, 'username', 'a2', 'link1', cb); 213 | }); 214 | 215 | it('group g0 should contain article 2 and 0', function(done) { 216 | ch01.getGroupArticles(client, 'g0', 1, null, function(err, articles) { 217 | should.not.exist(err); 218 | articles.length.should.equal(2); 219 | articles[0].id.should.equal('article:' + ids[2]); 220 | articles[1].id.should.equal('article:' + ids[0]); 221 | done(); 222 | }); 223 | }); 224 | 225 | it('group g1 should contain all three articles', function(done) { 226 | ch01.getGroupArticles(client, 'g1', 1, null, function(err, articles) { 227 | should.not.exist(err); 228 | articles.length.should.equal(3); 229 | articles[0].id.should.equal('article:' + ids[2]); 230 | articles[1].id.should.equal('article:' + ids[1]); 231 | articles[2].id.should.equal('article:' + ids[0]); 232 | done(); 233 | }); 234 | }); 235 | 236 | it('group g2 should only contain article 1', function(done) { 237 | ch01.getGroupArticles(client, 'g2', 1, null, function(err, articles) { 238 | should.not.exist(err); 239 | articles.length.should.equal(1); 240 | articles[0].id.should.equal('article:' + ids[2]); 241 | done(); 242 | }); 243 | }); 244 | 245 | }); 246 | 247 | }); -------------------------------------------------------------------------------- /java/src/main/java/Chapter04.java: -------------------------------------------------------------------------------- 1 | import redis.clients.jedis.Jedis; 2 | import redis.clients.jedis.Pipeline; 3 | import redis.clients.jedis.Transaction; 4 | import redis.clients.jedis.Tuple; 5 | 6 | import java.lang.reflect.Method; 7 | import java.util.List; 8 | import java.util.Map; 9 | import java.util.Set; 10 | 11 | public class Chapter04 { 12 | public static final void main(String[] args) { 13 | new Chapter04().run(); 14 | } 15 | 16 | public void run() { 17 | Jedis conn = new Jedis("localhost"); 18 | conn.select(15); 19 | 20 | testListItem(conn, false); 21 | testPurchaseItem(conn); 22 | testBenchmarkUpdateToken(conn); 23 | } 24 | 25 | public void testListItem(Jedis conn, boolean nested) { 26 | if (!nested){ 27 | System.out.println("\n----- testListItem -----"); 28 | } 29 | 30 | System.out.println("We need to set up just enough state so that a user can list an item"); 31 | String seller = "userX"; 32 | String item = "itemX"; 33 | conn.sadd("inventory:" + seller, item); 34 | Set i = conn.smembers("inventory:" + seller); 35 | 36 | System.out.println("The user's inventory has:"); 37 | for (String member : i){ 38 | System.out.println(" " + member); 39 | } 40 | assert i.size() > 0; 41 | System.out.println(); 42 | 43 | System.out.println("Listing the item..."); 44 | boolean l = listItem(conn, item, seller, 10); 45 | System.out.println("Listing the item succeeded? " + l); 46 | assert l; 47 | Set r = conn.zrangeWithScores("market:", 0, -1); 48 | System.out.println("The market contains:"); 49 | for (Tuple tuple : r){ 50 | System.out.println(" " + tuple.getElement() + ", " + tuple.getScore()); 51 | } 52 | assert r.size() > 0; 53 | } 54 | 55 | public void testPurchaseItem(Jedis conn) { 56 | System.out.println("\n----- testPurchaseItem -----"); 57 | testListItem(conn, true); 58 | 59 | System.out.println("We need to set up just enough state so a user can buy an item"); 60 | conn.hset("users:userY", "funds", "125"); 61 | Map r = conn.hgetAll("users:userY"); 62 | System.out.println("The user has some money:"); 63 | for (Map.Entry entry : r.entrySet()){ 64 | System.out.println(" " + entry.getKey() + ": " + entry.getValue()); 65 | } 66 | assert r.size() > 0; 67 | assert r.get("funds") != null; 68 | System.out.println(); 69 | 70 | System.out.println("Let's purchase an item"); 71 | boolean p = purchaseItem(conn, "userY", "itemX", "userX", 10); 72 | System.out.println("Purchasing an item succeeded? " + p); 73 | assert p; 74 | r = conn.hgetAll("users:userY"); 75 | System.out.println("Their money is now:"); 76 | for (Map.Entry entry : r.entrySet()){ 77 | System.out.println(" " + entry.getKey() + ": " + entry.getValue()); 78 | } 79 | assert r.size() > 0; 80 | 81 | String buyer = "userY"; 82 | Set i = conn.smembers("inventory:" + buyer); 83 | System.out.println("Their inventory is now:"); 84 | for (String member : i){ 85 | System.out.println(" " + member); 86 | } 87 | assert i.size() > 0; 88 | assert i.contains("itemX"); 89 | assert conn.zscore("market:", "itemX.userX") == null; 90 | } 91 | 92 | public void testBenchmarkUpdateToken(Jedis conn) { 93 | System.out.println("\n----- testBenchmarkUpdate -----"); 94 | benchmarkUpdateToken(conn, 5); 95 | } 96 | 97 | public boolean listItem( 98 | Jedis conn, String itemId, String sellerId, double price) { 99 | 100 | String inventory = "inventory:" + sellerId; 101 | String item = itemId + '.' + sellerId; 102 | long end = System.currentTimeMillis() + 5000; 103 | 104 | while (System.currentTimeMillis() < end) { 105 | conn.watch(inventory); 106 | if (!conn.sismember(inventory, itemId)){ 107 | conn.unwatch(); 108 | return false; 109 | } 110 | 111 | Transaction trans = conn.multi(); 112 | trans.zadd("market:", price, item); 113 | trans.srem(inventory, itemId); 114 | List results = trans.exec(); 115 | // null response indicates that the transaction was aborted due to 116 | // the watched key changing. 117 | if (results == null){ 118 | continue; 119 | } 120 | return true; 121 | } 122 | return false; 123 | } 124 | 125 | public boolean purchaseItem( 126 | Jedis conn, String buyerId, String itemId, String sellerId, double lprice) { 127 | 128 | String buyer = "users:" + buyerId; 129 | String seller = "users:" + sellerId; 130 | String item = itemId + '.' + sellerId; 131 | String inventory = "inventory:" + buyerId; 132 | long end = System.currentTimeMillis() + 10000; 133 | 134 | while (System.currentTimeMillis() < end){ 135 | conn.watch("market:", buyer); 136 | 137 | double price = conn.zscore("market:", item); 138 | double funds = Double.parseDouble(conn.hget(buyer, "funds")); 139 | if (price != lprice || price > funds){ 140 | conn.unwatch(); 141 | return false; 142 | } 143 | 144 | Transaction trans = conn.multi(); 145 | trans.hincrBy(seller, "funds", (int)price); 146 | trans.hincrBy(buyer, "funds", (int)-price); 147 | trans.sadd(inventory, itemId); 148 | trans.zrem("market:", item); 149 | List results = trans.exec(); 150 | // null response indicates that the transaction was aborted due to 151 | // the watched key changing. 152 | if (results == null){ 153 | continue; 154 | } 155 | return true; 156 | } 157 | 158 | return false; 159 | } 160 | 161 | public void benchmarkUpdateToken(Jedis conn, int duration) { 162 | try{ 163 | @SuppressWarnings("rawtypes") 164 | Class[] args = new Class[]{ 165 | Jedis.class, String.class, String.class, String.class}; 166 | Method[] methods = new Method[]{ 167 | this.getClass().getDeclaredMethod("updateToken", args), 168 | this.getClass().getDeclaredMethod("updateTokenPipeline", args), 169 | }; 170 | for (Method method : methods){ 171 | int count = 0; 172 | long start = System.currentTimeMillis(); 173 | long end = start + (duration * 1000); 174 | while (System.currentTimeMillis() < end){ 175 | count++; 176 | method.invoke(this, conn, "token", "user", "item"); 177 | } 178 | long delta = System.currentTimeMillis() - start; 179 | System.out.println( 180 | method.getName() + ' ' + 181 | count + ' ' + 182 | (delta / 1000) + ' ' + 183 | (count / (delta / 1000))); 184 | } 185 | }catch(Exception e){ 186 | throw new RuntimeException(e); 187 | } 188 | } 189 | 190 | public void updateToken(Jedis conn, String token, String user, String item) { 191 | long timestamp = System.currentTimeMillis() / 1000; 192 | conn.hset("login:", token, user); 193 | conn.zadd("recent:", timestamp, token); 194 | if (item != null) { 195 | conn.zadd("viewed:" + token, timestamp, item); 196 | conn.zremrangeByRank("viewed:" + token, 0, -26); 197 | conn.zincrby("viewed:", -1, item); 198 | } 199 | } 200 | 201 | public void updateTokenPipeline(Jedis conn, String token, String user, String item) { 202 | long timestamp = System.currentTimeMillis() / 1000; 203 | Pipeline pipe = conn.pipelined(); 204 | pipe.hset("login:", token, user); 205 | pipe.zadd("recent:", timestamp, token); 206 | if (item != null){ 207 | pipe.zadd("viewed:" + token, timestamp, item); 208 | pipe.zremrangeByRank("viewed:" + token, 0, -26); 209 | pipe.zincrby("viewed:", -1, item); 210 | } 211 | pipe.exec(); 212 | } 213 | } 214 | -------------------------------------------------------------------------------- /python/chA_listing_source.py: -------------------------------------------------------------------------------- 1 | 2 | ''' 3 | # 4 | ~:$ wget -q http://redis.googlecode.com/files/redis-2.6.2.tar.gz #A 5 | ~:$ tar -xzf redis-2.6.2.tar.gz #B 6 | ~:$ cd redis-2.6.2/ 7 | ~/redis-2.6.2:$ make #C 8 | cd src && make all #D 9 | [trimmed] #D 10 | make[1]: Leaving directory `~/redis-2.6.2/src' #D 11 | ~/redis-2.6.2:$ sudo make install #E 12 | cd src && make install #F 13 | [trimmed] #F 14 | make[1]: Leaving directory `~/redis-2.6.2/src' #F 15 | ~/redis-2.6.2:$ redis-server redis.conf #G 16 | [13792] 26 Aug 17:53:16.523 * Max number of open files set to 10032 #H 17 | [trimmed] #H 18 | [13792] 26 Aug 17:53:16.529 * The server is now ready to accept #H 19 | connections on port 6379 #H 20 | # 21 | #A Download the most recent version of Redis 2.6 (we use some features of Redis 2.6 in other chapters, but you can use the most recent version you are comfortable with by finding the download link: http://redis.io/download ) 22 | #B Extract the source code 23 | #C Compile Redis 24 | #D Watch compilation messages go by, you shouldn't see any errors 25 | #E Install Redis 26 | #F Watch installation messages go by, you shouldn't see any errors 27 | #G Start Redis server 28 | #H See the confirmation that Redis has started 29 | #END 30 | ''' 31 | 32 | ''' 33 | # 34 | ~:$ wget -q http://peak.telecommunity.com/dist/ez_setup.py #A 35 | ~:$ sudo python ez_setup.py #B 36 | Downloading http://pypi.python.org/packages/2.7/s/setuptools/... #B 37 | [trimmed] #B 38 | Finished processing dependencies for setuptools==0.6c11 #B 39 | ~:$ sudo python -m easy_install redis hiredis #C 40 | Searching for redis #D 41 | [trimmed] #D 42 | Finished processing dependencies for redis #D 43 | Searching for hiredis #E 44 | [trimmed] #E 45 | Finished processing dependencies for hiredis #E 46 | ~:$ 47 | # 48 | #A Download the setuptools ez_setup module 49 | #B Run the ez_setup module to download and install setuptools 50 | #C Run setuptools' easy_install package to install the redis and hiredis packages 51 | #D The redis package offers a somewhat standard interface to Redis from Python 52 | #E The hiredis package is a C accelerator library for the Python Redis library 53 | #END 54 | ''' 55 | 56 | ''' 57 | # 58 | ~:$ curl -O http://rudix.googlecode.com/hg/Ports/rudix/rudix.py #A 59 | [trimmed] 60 | ~:$ sudo python rudix.py install rudix #B 61 | Downloading rudix.googlecode.com/files/rudix-12.6-0.pkg #C 62 | [trimmed] #C 63 | installer: The install was successful. #C 64 | All done #C 65 | ~:$ sudo rudix install redis #D 66 | Downloading rudix.googlecode.com/files/redis-2.4.15-0.pkg #E 67 | [trimmed] #E 68 | installer: The install was successful. #E 69 | All done #E 70 | ~:$ redis-server #F 71 | [699] 13 Jul 21:18:09 # Warning: no config file specified, using the#G 72 | default config. In order to specify a config file use 'redis-server #G 73 | /path/to/redis.conf' #G 74 | [699] 13 Jul 21:18:09 * Server started, Redis version 2.4.15 #G 75 | [699] 13 Jul 21:18:09 * The server is now ready to accept connections#G 76 | on port 6379 #G 77 | [699] 13 Jul 21:18:09 - 0 clients connected (0 slaves), 922304 bytes#G 78 | in use #G 79 | # 80 | #A Download the bootstrap script that installs Rudix 81 | #B Tell Rudix to install itself 82 | #C Rudix is downloading and installing itself 83 | #D Tell Rudix to install Redis 84 | #E Rudix is downloading and installing Redis - note that we use some features from Redis 2.6, which is not yet available from Rudix 85 | #F Start the Redis server 86 | #G Redis started, and is running with the default configuration 87 | #END 88 | ''' 89 | 90 | ''' 91 | # 92 | ~:$ sudo rudix install pip #A 93 | Downloading rudix.googlecode.com/files/pip-1.1-1.pkg #B 94 | [trimmed] #B 95 | installer: The install was successful. #B 96 | All done #B 97 | ~:$ sudo pip install redis #C 98 | Downloading/unpacking redis #D 99 | [trimmed] #D 100 | Cleaning up... #D 101 | ~:$ 102 | # 103 | #A Because we have Rudix installed, we can install a Python package manager called pip 104 | #B Rudix is installing pip 105 | #C We can now use pip to install the Python Redis client library 106 | #D Pip is installing the Redis client library for Python 107 | #END 108 | ''' 109 | 110 | ''' 111 | # 112 | C:\Users\josiah>c:\python27\python #A 113 | Python 2.7.3 (default, Apr 10 2012, 23:31:26) [MSC v.1500 32 bit... 114 | Type "help", "copyright", "credits" or "license" for more information. 115 | >>> from urllib import urlopen #B 116 | >>> data = urlopen('http://peak.telecommunity.com/dist/ez_setup.py') #C 117 | >>> open('ez_setup.py', 'wb').write(data.read()) #D 118 | >>> exit() #E 119 | 120 | C:\Users\josiah>c:\python27\python ez_setup.py #F 121 | Downloading http://pypi.python.org/packages/2.7/s/setuptools/... #G 122 | [trimmed] #G 123 | Finished processing dependencies for setuptools==0.6c11 #G 124 | 125 | C:\Users\josiah>c:\python27\python -m easy_install redis #H 126 | Searching for redis #H 127 | [trimmed] #H 128 | Finished processing dependencies for redis #H 129 | C:\Users\josiah> 130 | # 131 | #A Start Python by itself in interactive mode 132 | #B Import the urlopen factory function from the urllib module 133 | #C Fetch a module that will help us install other packages 134 | #D Write the downloaded module to a file on disk 135 | #E Quit the Python interpreter by running the builtin exit() function 136 | #F Run the ez_setup helper module 137 | #G The ez_setup helper downloads and installs setuptools, which will make it easy to download and install the Redis client library 138 | #H Use setuptools' easy_install module to download and install Redis 139 | #END 140 | ''' 141 | 142 | 143 | ''' 144 | # 145 | ~:$ python #A 146 | Python 2.6.5 (r265:79063, Apr 16 2010, 13:09:56) 147 | [GCC 4.4.3] on linux2 148 | Type "help", "copyright", "credits" or "license" for more information. 149 | >>> import redis #B 150 | >>> conn = redis.Redis() #C 151 | >>> conn.set('hello', 'world') #D 152 | True #D 153 | >>> conn.get('hello') #E 154 | 'world' #E 155 | # 156 | #A Start Python so that we can verify everything is up and running correctly 157 | #B Import the redis library, it will automatically use the hiredis C accelerator library if it is available 158 | #C Create a connection to Redis 159 | #D Set a value and see that it was set 160 | #E Get the value we just set 161 | #END 162 | ''' 163 | -------------------------------------------------------------------------------- /python/ch01_listing_source.py: -------------------------------------------------------------------------------- 1 | 2 | import time 3 | import unittest 4 | 5 | ''' 6 | # 7 | $ redis-cli #A 8 | redis 127.0.0.1:6379> set hello world #D 9 | OK #E 10 | redis 127.0.0.1:6379> get hello #F 11 | "world" #G 12 | redis 127.0.0.1:6379> del hello #H 13 | (integer) 1 #I 14 | redis 127.0.0.1:6379> get hello #J 15 | (nil) 16 | redis 127.0.0.1:6379> 17 | # 18 | #A Start the redis-cli client up 19 | #D Set the key 'hello' to the value 'world' 20 | #E If a SET command succeeds, it returns 'OK', which turns into True on the Python side 21 | #F Now get the value stored at the key 'hello' 22 | #G It is still 'world', like we just set it 23 | #H Let's delete the key/value pair 24 | #I If there was a value to delete, DEL returns the number of items that were deleted 25 | #J There is no more value, so trying to fetch the value returns nil, which turns into None on the Python side 26 | #END 27 | ''' 28 | 29 | 30 | ''' 31 | # 32 | redis 127.0.0.1:6379> rpush list-key item #A 33 | (integer) 1 #A 34 | redis 127.0.0.1:6379> rpush list-key item2 #A 35 | (integer) 2 #A 36 | redis 127.0.0.1:6379> rpush list-key item #A 37 | (integer) 3 #A 38 | redis 127.0.0.1:6379> lrange list-key 0 -1 #B 39 | 1) "item" #B 40 | 2) "item2" #B 41 | 3) "item" #B 42 | redis 127.0.0.1:6379> lindex list-key 1 #C 43 | "item2" #C 44 | redis 127.0.0.1:6379> lpop list-key #D 45 | "item" #D 46 | redis 127.0.0.1:6379> lrange list-key 0 -1 #D 47 | 1) "item2" #D 48 | 2) "item" #D 49 | redis 127.0.0.1:6379> 50 | # 51 | #A When we push items onto a LIST, the command returns the current length of the list 52 | #B We can fetch the entire list by passing a range of 0 for the start index, and -1 for the last index 53 | #C We can fetch individual items from the list with LINDEX 54 | #D By popping an item from the list, it is no longer available 55 | #END 56 | ''' 57 | 58 | 59 | ''' 60 | # 61 | redis 127.0.0.1:6379> sadd set-key item #A 62 | (integer) 1 #A 63 | redis 127.0.0.1:6379> sadd set-key item2 #A 64 | (integer) 1 #A 65 | redis 127.0.0.1:6379> sadd set-key item3 #A 66 | (integer) 1 #A 67 | redis 127.0.0.1:6379> sadd set-key item #A 68 | (integer) 0 #A 69 | redis 127.0.0.1:6379> smembers set-key #B 70 | 1) "item" #B 71 | 2) "item2" #B 72 | 3) "item3" #B 73 | redis 127.0.0.1:6379> sismember set-key item4 #C 74 | (integer) 0 #C 75 | redis 127.0.0.1:6379> sismember set-key item #C 76 | (integer) 1 #C 77 | redis 127.0.0.1:6379> srem set-key item2 #D 78 | (integer) 1 #D 79 | redis 127.0.0.1:6379> srem set-key item2 #D 80 | (integer) 0 #D 81 | redis 127.0.0.1:6379> smembers set-key 82 | 1) "item" 83 | 2) "item3" 84 | redis 127.0.0.1:6379> 85 | # 86 | #A When adding an item to a SET, Redis will return a 1 if the item is new to the set and 0 if it was already in the SET 87 | #B We can fetch all of the items in the SET, which returns them as a sequence of items, which is turned into a Python set from Python 88 | #C We can also ask Redis whether an item is in the SET, which turns into a boolean in Python 89 | #D When we attempt to remove items, our commands return the number of items that were removed 90 | #END 91 | ''' 92 | 93 | 94 | ''' 95 | # 96 | redis 127.0.0.1:6379> hset hash-key sub-key1 value1 #A 97 | (integer) 1 #A 98 | redis 127.0.0.1:6379> hset hash-key sub-key2 value2 #A 99 | (integer) 1 #A 100 | redis 127.0.0.1:6379> hset hash-key sub-key1 value1 #A 101 | (integer) 0 #A 102 | redis 127.0.0.1:6379> hgetall hash-key #B 103 | 1) "sub-key1" #B 104 | 2) "value1" #B 105 | 3) "sub-key2" #B 106 | 4) "value2" #B 107 | redis 127.0.0.1:6379> hdel hash-key sub-key2 #C 108 | (integer) 1 #C 109 | redis 127.0.0.1:6379> hdel hash-key sub-key2 #C 110 | (integer) 0 #C 111 | redis 127.0.0.1:6379> hget hash-key sub-key1 #D 112 | "value1" #D 113 | redis 127.0.0.1:6379> hgetall hash-key 114 | 1) "sub-key1" 115 | 2) "value1" 116 | # 117 | #A When we add items to a hash, again we get a return value that tells us whether the item is new in the hash 118 | #B We can fetch all of the items in the HASH, which gets translated into a dictionary on the Python side of things 119 | #C When we delete items from the hash, the command returns whether the item was there before we tried to remove it 120 | #D We can also fetch individual fields from hashes 121 | #END 122 | ''' 123 | 124 | 125 | ''' 126 | # 127 | redis 127.0.0.1:6379> zadd zset-key 728 member1 #A 128 | (integer) 1 #A 129 | redis 127.0.0.1:6379> zadd zset-key 982 member0 #A 130 | (integer) 1 #A 131 | redis 127.0.0.1:6379> zadd zset-key 982 member0 #A 132 | (integer) 0 #A 133 | redis 127.0.0.1:6379> zrange zset-key 0 -1 withscores #B 134 | 1) "member1" #B 135 | 2) "728" #B 136 | 3) "member0" #B 137 | 4) "982" #B 138 | redis 127.0.0.1:6379> zrangebyscore zset-key 0 800 withscores #C 139 | 1) "member1" #C 140 | 2) "728" #C 141 | redis 127.0.0.1:6379> zrem zset-key member1 #D 142 | (integer) 1 #D 143 | redis 127.0.0.1:6379> zrem zset-key member1 #D 144 | (integer) 0 #D 145 | redis 127.0.0.1:6379> zrange zset-key 0 -1 withscores 146 | 1) "member0" 147 | 2) "982" 148 | # 149 | #A When we add items to a ZSET, the the command returns the number of new items 150 | #B We can fetch all of the items in the ZSET, which are ordered by the scores, and scores are turned into floats in Python 151 | #C We can also fetch a subsequence of items based on their scores 152 | #D When we remove items, we again find the number of items that were removed 153 | #END 154 | ''' 155 | 156 | # 157 | ONE_WEEK_IN_SECONDS = 7 * 86400 #A 158 | VOTE_SCORE = 432 #A 159 | 160 | def article_vote(conn, user, article): 161 | cutoff = time.time() - ONE_WEEK_IN_SECONDS #B 162 | if conn.zscore('time:', article) < cutoff: #C 163 | return 164 | 165 | article_id = article.partition(':')[-1] #D 166 | if conn.sadd('voted:' + article_id, user): #E 167 | conn.zincrby('score:', article, VOTE_SCORE) #E 168 | conn.hincrby(article, 'votes', 1) #E 169 | # 170 | #A Prepare our constants 171 | #B Calculate the cutoff time for voting 172 | #C Check to see if the article can still be voted on (we could use the article HASH here, but scores are returned as floats so we don't have to cast it) 173 | #D Get the id portion from the article:id identifier 174 | #E If the user hasn't voted for this article before, increment the article score and vote count (note that our HINCRBY and ZINCRBY calls should be in a Redis transaction, but we don't introduce them until chapter 3 and 4, so ignore that for now) 175 | #END 176 | 177 | # 178 | def post_article(conn, user, title, link): 179 | article_id = str(conn.incr('article:')) #A 180 | 181 | voted = 'voted:' + article_id 182 | conn.sadd(voted, user) #B 183 | conn.expire(voted, ONE_WEEK_IN_SECONDS) #B 184 | 185 | now = time.time() 186 | article = 'article:' + article_id 187 | conn.hmset(article, { #C 188 | 'title': title, #C 189 | 'link': link, #C 190 | 'poster': user, #C 191 | 'time': now, #C 192 | 'votes': 1, #C 193 | }) #C 194 | 195 | conn.zadd('score:', article, now + VOTE_SCORE) #D 196 | conn.zadd('time:', article, now) #D 197 | 198 | return article_id 199 | # 200 | #A Generate a new article id 201 | #B Start with the posting user having voted for the article, and set the article voting information to automatically expire in a week (we discuss expiration in chapter 3) 202 | #C Create the article hash 203 | #D Add the article to the time and score ordered zsets 204 | #END 205 | 206 | # 207 | ARTICLES_PER_PAGE = 25 208 | 209 | def get_articles(conn, page, order='score:'): 210 | start = (page-1) * ARTICLES_PER_PAGE #A 211 | end = start + ARTICLES_PER_PAGE - 1 #A 212 | 213 | ids = conn.zrevrange(order, start, end) #B 214 | articles = [] 215 | for id in ids: #C 216 | article_data = conn.hgetall(id) #C 217 | article_data['id'] = id #C 218 | articles.append(article_data) #C 219 | 220 | return articles 221 | # 222 | #A Set up the start and end indexes for fetching the articles 223 | #B Fetch the article ids 224 | #C Get the article information from the list of article ids 225 | #END 226 | 227 | # 228 | def add_remove_groups(conn, article_id, to_add=[], to_remove=[]): 229 | article = 'article:' + article_id #A 230 | for group in to_add: 231 | conn.sadd('group:' + group, article) #B 232 | for group in to_remove: 233 | conn.srem('group:' + group, article) #C 234 | # 235 | #A Construct the article information like we did in post_article 236 | #B Add the article to groups that it should be a part of 237 | #C Remove the article from groups that it should be removed from 238 | #END 239 | 240 | # 241 | def get_group_articles(conn, group, page, order='score:'): 242 | key = order + group #A 243 | if not conn.exists(key): #B 244 | conn.zinterstore(key, #C 245 | ['group:' + group, order], #C 246 | aggregate='max', #C 247 | ) 248 | conn.expire(key, 60) #D 249 | return get_articles(conn, page, key) #E 250 | # 251 | #A Create a key for each group and each sort order 252 | #B If we haven't sorted these articles recently, we should sort them 253 | #C Actually sort the articles in the group based on score or recency 254 | #D Tell Redis to automatically expire the ZSET in 60 seconds 255 | #E Call our earlier get_articles() function to handle pagination and article data fetching 256 | #END 257 | 258 | #--------------- Below this line are helpers to test the code ---------------- 259 | 260 | class TestCh01(unittest.TestCase): 261 | def setUp(self): 262 | import redis 263 | self.conn = redis.Redis(db=15) 264 | 265 | def tearDown(self): 266 | del self.conn 267 | print 268 | print 269 | 270 | def test_article_functionality(self): 271 | conn = self.conn 272 | import pprint 273 | 274 | article_id = str(post_article(conn, 'username', 'A title', 'http://www.google.com')) 275 | print "We posted a new article with id:", article_id 276 | print 277 | self.assertTrue(article_id) 278 | 279 | print "Its HASH looks like:" 280 | r = conn.hgetall('article:' + article_id) 281 | print r 282 | print 283 | self.assertTrue(r) 284 | 285 | article_vote(conn, 'other_user', 'article:' + article_id) 286 | print "We voted for the article, it now has votes:", 287 | v = int(conn.hget('article:' + article_id, 'votes')) 288 | print v 289 | print 290 | self.assertTrue(v > 1) 291 | 292 | print "The currently highest-scoring articles are:" 293 | articles = get_articles(conn, 1) 294 | pprint.pprint(articles) 295 | print 296 | 297 | self.assertTrue(len(articles) >= 1) 298 | 299 | add_remove_groups(conn, article_id, ['new-group']) 300 | print "We added the article to a new group, other articles include:" 301 | articles = get_group_articles(conn, 'new-group', 1) 302 | pprint.pprint(articles) 303 | print 304 | self.assertTrue(len(articles) >= 1) 305 | 306 | to_del = ( 307 | conn.keys('time:*') + conn.keys('voted:*') + conn.keys('score:*') + 308 | conn.keys('article:*') + conn.keys('group:*') 309 | ) 310 | if to_del: 311 | conn.delete(*to_del) 312 | 313 | if __name__ == '__main__': 314 | unittest.main() 315 | -------------------------------------------------------------------------------- /python/ch02_listing_source.py: -------------------------------------------------------------------------------- 1 | 2 | import json 3 | import threading 4 | import time 5 | import unittest 6 | import urlparse 7 | import uuid 8 | 9 | QUIT = False 10 | 11 | # 12 | def check_token(conn, token): 13 | return conn.hget('login:', token) #A 14 | # 15 | #A Fetch and return the given user, if available 16 | #END 17 | 18 | # 19 | def update_token(conn, token, user, item=None): 20 | timestamp = time.time() #A 21 | conn.hset('login:', token, user) #B 22 | conn.zadd('recent:', token, timestamp) #C 23 | if item: 24 | conn.zadd('viewed:' + token, item, timestamp) #D 25 | conn.zremrangebyrank('viewed:' + token, 0, -26) #E 26 | # 27 | #A Get the timestamp 28 | #B Keep a mapping from the token to the logged-in user 29 | #C Record when the token was last seen 30 | #D Record that the user viewed the item 31 | #E Remove old items, keeping the most recent 25 32 | #END 33 | 34 | # 35 | QUIT = False 36 | LIMIT = 10000000 37 | 38 | def clean_sessions(conn): 39 | while not QUIT: 40 | size = conn.zcard('recent:') #A 41 | if size <= LIMIT: #B 42 | time.sleep(1) #B 43 | continue 44 | 45 | end_index = min(size - LIMIT, 100) #C 46 | tokens = conn.zrange('recent:', 0, end_index-1) #C 47 | 48 | session_keys = [] #D 49 | for token in tokens: #D 50 | session_keys.append('viewed:' + token) #D 51 | 52 | conn.delete(*session_keys) #E 53 | conn.hdel('login:', *tokens) #E 54 | conn.zrem('recent:', *tokens) #E 55 | # 56 | #A Find out how many tokens are known 57 | #B We are still under our limit, sleep and try again 58 | #C Fetch the token ids that should be removed 59 | #D Prepare the key names for the tokens to delete 60 | #E Remove the oldest tokens 61 | #END 62 | 63 | # 64 | def add_to_cart(conn, session, item, count): 65 | if count <= 0: 66 | conn.hrem('cart:' + session, item) #A 67 | else: 68 | conn.hset('cart:' + session, item, count) #B 69 | # 70 | #A Remove the item from the cart 71 | #B Add the item to the cart 72 | #END 73 | 74 | # 75 | def clean_full_sessions(conn): 76 | while not QUIT: 77 | size = conn.zcard('recent:') 78 | if size <= LIMIT: 79 | time.sleep(1) 80 | continue 81 | 82 | end_index = min(size - LIMIT, 100) 83 | sessions = conn.zrange('recent:', 0, end_index-1) 84 | 85 | session_keys = [] 86 | for sess in sessions: 87 | session_keys.append('viewed:' + sess) 88 | session_keys.append('cart:' + sess) #A 89 | 90 | conn.delete(*session_keys) 91 | conn.hdel('login:', *sessions) 92 | conn.zrem('recent:', *sessions) 93 | # 94 | #A The required added line to delete the shopping cart for old sessions 95 | #END 96 | 97 | # 98 | def cache_request(conn, request, callback): 99 | if not can_cache(conn, request): #A 100 | return callback(request) #A 101 | 102 | page_key = 'cache:' + hash_request(request) #B 103 | content = conn.get(page_key) #C 104 | 105 | if not content: 106 | content = callback(request) #D 107 | conn.setex(page_key, content, 300) #E 108 | 109 | return content #F 110 | # 111 | #A If we cannot cache the request, immediately call the callback 112 | #B Convert the request into a simple string key for later lookups 113 | #C Fetch the cached content if we can, and it is available 114 | #D Generate the content if we can't cache the page, or if it wasn't cached 115 | #E Cache the newly generated content if we can cache it 116 | #F Return the content 117 | #END 118 | 119 | # 120 | def schedule_row_cache(conn, row_id, delay): 121 | conn.zadd('delay:', row_id, delay) #A 122 | conn.zadd('schedule:', row_id, time.time()) #B 123 | # 124 | #A Set the delay for the item first 125 | #B Schedule the item to be cached now 126 | #END 127 | 128 | 129 | # 130 | def cache_rows(conn): 131 | while not QUIT: 132 | next = conn.zrange('schedule:', 0, 0, withscores=True) #A 133 | now = time.time() 134 | if not next or next[0][1] > now: 135 | time.sleep(.05) #B 136 | continue 137 | 138 | row_id = next[0][0] 139 | delay = conn.zscore('delay:', row_id) #C 140 | if delay <= 0: 141 | conn.zrem('delay:', row_id) #D 142 | conn.zrem('schedule:', row_id) #D 143 | conn.delete('inv:' + row_id) #D 144 | continue 145 | 146 | row = Inventory.get(row_id) #E 147 | conn.zadd('schedule:', row_id, now + delay) #F 148 | conn.set('inv:' + row_id, json.dumps(row.to_dict())) #F 149 | # 150 | #A Find the next row that should be cached (if any), including the timestamp, as a list of tuples with zero or one items 151 | #B No rows can be cached now, so wait 50 milliseconds and try again 152 | #C Get the delay before the next schedule 153 | #D The item shouldn't be cached anymore, remove it from the cache 154 | #E Get the database row 155 | #F Update the schedule and set the cache value 156 | #END 157 | 158 | # 159 | def update_token(conn, token, user, item=None): 160 | timestamp = time.time() 161 | conn.hset('login:', token, user) 162 | conn.zadd('recent:', token, timestamp) 163 | if item: 164 | conn.zadd('viewed:' + token, item, timestamp) 165 | conn.zremrangebyrank('viewed:' + token, 0, -26) 166 | conn.zincrby('viewed:', item, -1) #A 167 | # 168 | #A The line we need to add to update_token() 169 | #END 170 | 171 | # 172 | def rescale_viewed(conn): 173 | while not QUIT: 174 | conn.zremrangebyrank('viewed:', 0, -20001) #A 175 | conn.zinterstore('viewed:', {'viewed:': .5}) #B 176 | time.sleep(300) #C 177 | # 178 | #A Remove any item not in the top 20,000 viewed items 179 | #B Rescale all counts to be 1/2 of what they were before 180 | #C Do it again in 5 minutes 181 | #END 182 | 183 | # 184 | def can_cache(conn, request): 185 | item_id = extract_item_id(request) #A 186 | if not item_id or is_dynamic(request): #B 187 | return False 188 | rank = conn.zrank('viewed:', item_id) #C 189 | return rank is not None and rank < 10000 #D 190 | # 191 | #A Get the item id for the page, if any 192 | #B Check whether the page can be statically cached, and whether this is an item page 193 | #C Get the rank of the item 194 | #D Return whether the item has a high enough view count to be cached 195 | #END 196 | 197 | 198 | #--------------- Below this line are helpers to test the code ---------------- 199 | 200 | def extract_item_id(request): 201 | parsed = urlparse.urlparse(request) 202 | query = urlparse.parse_qs(parsed.query) 203 | return (query.get('item') or [None])[0] 204 | 205 | def is_dynamic(request): 206 | parsed = urlparse.urlparse(request) 207 | query = urlparse.parse_qs(parsed.query) 208 | return '_' in query 209 | 210 | def hash_request(request): 211 | return str(hash(request)) 212 | 213 | class Inventory(object): 214 | def __init__(self, id): 215 | self.id = id 216 | 217 | @classmethod 218 | def get(cls, id): 219 | return Inventory(id) 220 | 221 | def to_dict(self): 222 | return {'id':self.id, 'data':'data to cache...', 'cached':time.time()} 223 | 224 | class TestCh02(unittest.TestCase): 225 | def setUp(self): 226 | import redis 227 | self.conn = redis.Redis(db=15) 228 | 229 | def tearDown(self): 230 | conn = self.conn 231 | to_del = ( 232 | conn.keys('login:*') + conn.keys('recent:*') + conn.keys('viewed:*') + 233 | conn.keys('cart:*') + conn.keys('cache:*') + conn.keys('delay:*') + 234 | conn.keys('schedule:*') + conn.keys('inv:*')) 235 | if to_del: 236 | self.conn.delete(*to_del) 237 | del self.conn 238 | global QUIT, LIMIT 239 | QUIT = False 240 | LIMIT = 10000000 241 | print 242 | print 243 | 244 | def test_login_cookies(self): 245 | conn = self.conn 246 | global LIMIT, QUIT 247 | token = str(uuid.uuid4()) 248 | 249 | update_token(conn, token, 'username', 'itemX') 250 | print "We just logged-in/updated token:", token 251 | print "For user:", 'username' 252 | print 253 | 254 | print "What username do we get when we look-up that token?" 255 | r = check_token(conn, token) 256 | print r 257 | print 258 | self.assertTrue(r) 259 | 260 | 261 | print "Let's drop the maximum number of cookies to 0 to clean them out" 262 | print "We will start a thread to do the cleaning, while we stop it later" 263 | 264 | LIMIT = 0 265 | t = threading.Thread(target=clean_sessions, args=(conn,)) 266 | t.setDaemon(1) # to make sure it dies if we ctrl+C quit 267 | t.start() 268 | time.sleep(1) 269 | QUIT = True 270 | time.sleep(2) 271 | if t.isAlive(): 272 | raise Exception("The clean sessions thread is still alive?!?") 273 | 274 | s = conn.hlen('login:') 275 | print "The current number of sessions still available is:", s 276 | self.assertFalse(s) 277 | 278 | def test_shoppping_cart_cookies(self): 279 | conn = self.conn 280 | global LIMIT, QUIT 281 | token = str(uuid.uuid4()) 282 | 283 | print "We'll refresh our session..." 284 | update_token(conn, token, 'username', 'itemX') 285 | print "And add an item to the shopping cart" 286 | add_to_cart(conn, token, "itemY", 3) 287 | r = conn.hgetall('cart:' + token) 288 | print "Our shopping cart currently has:", r 289 | print 290 | 291 | self.assertTrue(len(r) >= 1) 292 | 293 | print "Let's clean out our sessions and carts" 294 | LIMIT = 0 295 | t = threading.Thread(target=clean_full_sessions, args=(conn,)) 296 | t.setDaemon(1) # to make sure it dies if we ctrl+C quit 297 | t.start() 298 | time.sleep(1) 299 | QUIT = True 300 | time.sleep(2) 301 | if t.isAlive(): 302 | raise Exception("The clean sessions thread is still alive?!?") 303 | 304 | r = conn.hgetall('cart:' + token) 305 | print "Our shopping cart now contains:", r 306 | 307 | self.assertFalse(r) 308 | 309 | def test_cache_request(self): 310 | conn = self.conn 311 | token = str(uuid.uuid4()) 312 | 313 | def callback(request): 314 | return "content for " + request 315 | 316 | update_token(conn, token, 'username', 'itemX') 317 | url = 'http://test.com/?item=itemX' 318 | print "We are going to cache a simple request against", url 319 | result = cache_request(conn, url, callback) 320 | print "We got initial content:", repr(result) 321 | print 322 | 323 | self.assertTrue(result) 324 | 325 | print "To test that we've cached the request, we'll pass a bad callback" 326 | result2 = cache_request(conn, url, None) 327 | print "We ended up getting the same response!", repr(result2) 328 | 329 | self.assertEquals(result, result2) 330 | 331 | self.assertFalse(can_cache(conn, 'http://test.com/')) 332 | self.assertFalse(can_cache(conn, 'http://test.com/?item=itemX&_=1234536')) 333 | 334 | def test_cache_rows(self): 335 | import pprint 336 | conn = self.conn 337 | global QUIT 338 | 339 | print "First, let's schedule caching of itemX every 5 seconds" 340 | schedule_row_cache(conn, 'itemX', 5) 341 | print "Our schedule looks like:" 342 | s = conn.zrange('schedule:', 0, -1, withscores=True) 343 | pprint.pprint(s) 344 | self.assertTrue(s) 345 | 346 | print "We'll start a caching thread that will cache the data..." 347 | t = threading.Thread(target=cache_rows, args=(conn,)) 348 | t.setDaemon(1) 349 | t.start() 350 | 351 | time.sleep(1) 352 | print "Our cached data looks like:" 353 | r = conn.get('inv:itemX') 354 | print repr(r) 355 | self.assertTrue(r) 356 | print 357 | print "We'll check again in 5 seconds..." 358 | time.sleep(5) 359 | print "Notice that the data has changed..." 360 | r2 = conn.get('inv:itemX') 361 | print repr(r2) 362 | print 363 | self.assertTrue(r2) 364 | self.assertTrue(r != r2) 365 | 366 | print "Let's force un-caching" 367 | schedule_row_cache(conn, 'itemX', -1) 368 | time.sleep(1) 369 | r = conn.get('inv:itemX') 370 | print "The cache was cleared?", not r 371 | print 372 | self.assertFalse(r) 373 | 374 | QUIT = True 375 | time.sleep(2) 376 | if t.isAlive(): 377 | raise Exception("The database caching thread is still alive?!?") 378 | 379 | # We aren't going to bother with the top 10k requests are cached, as 380 | # we already tested it as part of the cached requests test. 381 | 382 | if __name__ == '__main__': 383 | unittest.main() 384 | -------------------------------------------------------------------------------- /java/src/main/java/Chapter02.java: -------------------------------------------------------------------------------- 1 | import com.google.gson.Gson; 2 | import redis.clients.jedis.Jedis; 3 | import redis.clients.jedis.Tuple; 4 | 5 | import java.net.MalformedURLException; 6 | import java.net.URL; 7 | import java.util.*; 8 | 9 | public class Chapter02 { 10 | public static final void main(String[] args) 11 | throws InterruptedException 12 | { 13 | new Chapter02().run(); 14 | } 15 | 16 | public void run() 17 | throws InterruptedException 18 | { 19 | Jedis conn = new Jedis("localhost"); 20 | conn.select(15); 21 | 22 | testLoginCookies(conn); 23 | testShopppingCartCookies(conn); 24 | testCacheRows(conn); 25 | testCacheRequest(conn); 26 | } 27 | 28 | public void testLoginCookies(Jedis conn) 29 | throws InterruptedException 30 | { 31 | System.out.println("\n----- testLoginCookies -----"); 32 | String token = UUID.randomUUID().toString(); 33 | 34 | updateToken(conn, token, "username", "itemX"); 35 | System.out.println("We just logged-in/updated token: " + token); 36 | System.out.println("For user: 'username'"); 37 | System.out.println(); 38 | 39 | System.out.println("What username do we get when we look-up that token?"); 40 | String r = checkToken(conn, token); 41 | System.out.println(r); 42 | System.out.println(); 43 | assert r != null; 44 | 45 | System.out.println("Let's drop the maximum number of cookies to 0 to clean them out"); 46 | System.out.println("We will start a thread to do the cleaning, while we stop it later"); 47 | 48 | CleanSessionsThread thread = new CleanSessionsThread(0); 49 | thread.start(); 50 | Thread.sleep(1000); 51 | thread.quit(); 52 | Thread.sleep(2000); 53 | if (thread.isAlive()){ 54 | throw new RuntimeException("The clean sessions thread is still alive?!?"); 55 | } 56 | 57 | long s = conn.hlen("login:"); 58 | System.out.println("The current number of sessions still available is: " + s); 59 | assert s == 0; 60 | } 61 | 62 | public void testShopppingCartCookies(Jedis conn) 63 | throws InterruptedException 64 | { 65 | System.out.println("\n----- testShopppingCartCookies -----"); 66 | String token = UUID.randomUUID().toString(); 67 | 68 | System.out.println("We'll refresh our session..."); 69 | updateToken(conn, token, "username", "itemX"); 70 | System.out.println("And add an item to the shopping cart"); 71 | addToCart(conn, token, "itemY", 3); 72 | Map r = conn.hgetAll("cart:" + token); 73 | System.out.println("Our shopping cart currently has:"); 74 | for (Map.Entry entry : r.entrySet()){ 75 | System.out.println(" " + entry.getKey() + ": " + entry.getValue()); 76 | } 77 | System.out.println(); 78 | 79 | assert r.size() >= 1; 80 | 81 | System.out.println("Let's clean out our sessions and carts"); 82 | CleanFullSessionsThread thread = new CleanFullSessionsThread(0); 83 | thread.start(); 84 | Thread.sleep(1000); 85 | thread.quit(); 86 | Thread.sleep(2000); 87 | if (thread.isAlive()){ 88 | throw new RuntimeException("The clean sessions thread is still alive?!?"); 89 | } 90 | 91 | r = conn.hgetAll("cart:" + token); 92 | System.out.println("Our shopping cart now contains:"); 93 | for (Map.Entry entry : r.entrySet()){ 94 | System.out.println(" " + entry.getKey() + ": " + entry.getValue()); 95 | } 96 | assert r.size() == 0; 97 | } 98 | 99 | public void testCacheRows(Jedis conn) 100 | throws InterruptedException 101 | { 102 | System.out.println("\n----- testCacheRows -----"); 103 | System.out.println("First, let's schedule caching of itemX every 5 seconds"); 104 | scheduleRowCache(conn, "itemX", 5); 105 | System.out.println("Our schedule looks like:"); 106 | Set s = conn.zrangeWithScores("schedule:", 0, -1); 107 | for (Tuple tuple : s){ 108 | System.out.println(" " + tuple.getElement() + ", " + tuple.getScore()); 109 | } 110 | assert s.size() != 0; 111 | 112 | System.out.println("We'll start a caching thread that will cache the data..."); 113 | 114 | CacheRowsThread thread = new CacheRowsThread(); 115 | thread.start(); 116 | 117 | Thread.sleep(1000); 118 | System.out.println("Our cached data looks like:"); 119 | String r = conn.get("inv:itemX"); 120 | System.out.println(r); 121 | assert r != null; 122 | System.out.println(); 123 | 124 | System.out.println("We'll check again in 5 seconds..."); 125 | Thread.sleep(5000); 126 | System.out.println("Notice that the data has changed..."); 127 | String r2 = conn.get("inv:itemX"); 128 | System.out.println(r2); 129 | System.out.println(); 130 | assert r2 != null; 131 | assert !r.equals(r2); 132 | 133 | System.out.println("Let's force un-caching"); 134 | scheduleRowCache(conn, "itemX", -1); 135 | Thread.sleep(1000); 136 | r = conn.get("inv:itemX"); 137 | System.out.println("The cache was cleared? " + (r == null)); 138 | assert r == null; 139 | 140 | thread.quit(); 141 | Thread.sleep(2000); 142 | if (thread.isAlive()){ 143 | throw new RuntimeException("The database caching thread is still alive?!?"); 144 | } 145 | } 146 | 147 | public void testCacheRequest(Jedis conn) { 148 | System.out.println("\n----- testCacheRequest -----"); 149 | String token = UUID.randomUUID().toString(); 150 | 151 | Callback callback = new Callback(){ 152 | public String call(String request){ 153 | return "content for " + request; 154 | } 155 | }; 156 | 157 | updateToken(conn, token, "username", "itemX"); 158 | String url = "http://test.com/?item=itemX"; 159 | System.out.println("We are going to cache a simple request against " + url); 160 | String result = cacheRequest(conn, url, callback); 161 | System.out.println("We got initial content:\n" + result); 162 | System.out.println(); 163 | 164 | assert result != null; 165 | 166 | System.out.println("To test that we've cached the request, we'll pass a bad callback"); 167 | String result2 = cacheRequest(conn, url, null); 168 | System.out.println("We ended up getting the same response!\n" + result2); 169 | 170 | assert result.equals(result2); 171 | 172 | assert !canCache(conn, "http://test.com/"); 173 | assert !canCache(conn, "http://test.com/?item=itemX&_=1234536"); 174 | } 175 | 176 | public String checkToken(Jedis conn, String token) { 177 | return conn.hget("login:", token); 178 | } 179 | 180 | public void updateToken(Jedis conn, String token, String user, String item) { 181 | long timestamp = System.currentTimeMillis() / 1000; 182 | conn.hset("login:", token, user); 183 | conn.zadd("recent:", timestamp, token); 184 | if (item != null) { 185 | conn.zadd("viewed:" + token, timestamp, item); 186 | conn.zremrangeByRank("viewed:" + token, 0, -26); 187 | conn.zincrby("viewed:", -1, item); 188 | } 189 | } 190 | 191 | public void addToCart(Jedis conn, String session, String item, int count) { 192 | if (count <= 0) { 193 | conn.hdel("cart:" + session, item); 194 | } else { 195 | conn.hset("cart:" + session, item, String.valueOf(count)); 196 | } 197 | } 198 | 199 | public void scheduleRowCache(Jedis conn, String rowId, int delay) { 200 | conn.zadd("delay:", delay, rowId); 201 | conn.zadd("schedule:", System.currentTimeMillis() / 1000, rowId); 202 | } 203 | 204 | public String cacheRequest(Jedis conn, String request, Callback callback) { 205 | if (!canCache(conn, request)){ 206 | return callback != null ? callback.call(request) : null; 207 | } 208 | 209 | String pageKey = "cache:" + hashRequest(request); 210 | String content = conn.get(pageKey); 211 | 212 | if (content == null && callback != null){ 213 | content = callback.call(request); 214 | conn.setex(pageKey, 300, content); 215 | } 216 | 217 | return content; 218 | } 219 | 220 | public boolean canCache(Jedis conn, String request) { 221 | try { 222 | URL url = new URL(request); 223 | HashMap params = new HashMap(); 224 | if (url.getQuery() != null){ 225 | for (String param : url.getQuery().split("&")){ 226 | String[] pair = param.split("=", 2); 227 | params.put(pair[0], pair.length == 2 ? pair[1] : null); 228 | } 229 | } 230 | 231 | String itemId = extractItemId(params); 232 | if (itemId == null || isDynamic(params)) { 233 | return false; 234 | } 235 | Long rank = conn.zrank("viewed:", itemId); 236 | return rank != null && rank < 10000; 237 | }catch(MalformedURLException mue){ 238 | return false; 239 | } 240 | } 241 | 242 | public boolean isDynamic(Map params) { 243 | return params.containsKey("_"); 244 | } 245 | 246 | public String extractItemId(Map params) { 247 | return params.get("item"); 248 | } 249 | 250 | public String hashRequest(String request) { 251 | return String.valueOf(request.hashCode()); 252 | } 253 | 254 | public interface Callback { 255 | public String call(String request); 256 | } 257 | 258 | public class CleanSessionsThread 259 | extends Thread 260 | { 261 | private Jedis conn; 262 | private int limit; 263 | private boolean quit; 264 | 265 | public CleanSessionsThread(int limit) { 266 | this.conn = new Jedis("localhost"); 267 | this.conn.select(15); 268 | this.limit = limit; 269 | } 270 | 271 | public void quit() { 272 | quit = true; 273 | } 274 | 275 | public void run() { 276 | while (!quit) { 277 | long size = conn.zcard("recent:"); 278 | if (size <= limit){ 279 | try { 280 | sleep(1000); 281 | }catch(InterruptedException ie){ 282 | Thread.currentThread().interrupt(); 283 | } 284 | continue; 285 | } 286 | 287 | long endIndex = Math.min(size - limit, 100); 288 | Set tokenSet = conn.zrange("recent:", 0, endIndex - 1); 289 | String[] tokens = tokenSet.toArray(new String[tokenSet.size()]); 290 | 291 | ArrayList sessionKeys = new ArrayList(); 292 | for (String token : tokens) { 293 | sessionKeys.add("viewed:" + token); 294 | } 295 | 296 | conn.del(sessionKeys.toArray(new String[sessionKeys.size()])); 297 | conn.hdel("login:", tokens); 298 | conn.zrem("recent:", tokens); 299 | } 300 | } 301 | } 302 | 303 | public class CleanFullSessionsThread 304 | extends Thread 305 | { 306 | private Jedis conn; 307 | private int limit; 308 | private boolean quit; 309 | 310 | public CleanFullSessionsThread(int limit) { 311 | this.conn = new Jedis("localhost"); 312 | this.conn.select(15); 313 | this.limit = limit; 314 | } 315 | 316 | public void quit() { 317 | quit = true; 318 | } 319 | 320 | public void run() { 321 | while (!quit) { 322 | long size = conn.zcard("recent:"); 323 | if (size <= limit){ 324 | try { 325 | sleep(1000); 326 | }catch(InterruptedException ie){ 327 | Thread.currentThread().interrupt(); 328 | } 329 | continue; 330 | } 331 | 332 | long endIndex = Math.min(size - limit, 100); 333 | Set sessionSet = conn.zrange("recent:", 0, endIndex - 1); 334 | String[] sessions = sessionSet.toArray(new String[sessionSet.size()]); 335 | 336 | ArrayList sessionKeys = new ArrayList(); 337 | for (String sess : sessions) { 338 | sessionKeys.add("viewed:" + sess); 339 | sessionKeys.add("cart:" + sess); 340 | } 341 | 342 | conn.del(sessionKeys.toArray(new String[sessionKeys.size()])); 343 | conn.hdel("login:", sessions); 344 | conn.zrem("recent:", sessions); 345 | } 346 | } 347 | } 348 | 349 | public class CacheRowsThread 350 | extends Thread 351 | { 352 | private Jedis conn; 353 | private boolean quit; 354 | 355 | public CacheRowsThread() { 356 | this.conn = new Jedis("localhost"); 357 | this.conn.select(15); 358 | } 359 | 360 | public void quit() { 361 | quit = true; 362 | } 363 | 364 | public void run() { 365 | Gson gson = new Gson(); 366 | while (!quit){ 367 | Set range = conn.zrangeWithScores("schedule:", 0, 0); 368 | Tuple next = range.size() > 0 ? range.iterator().next() : null; 369 | long now = System.currentTimeMillis() / 1000; 370 | if (next == null || next.getScore() > now){ 371 | try { 372 | sleep(50); 373 | }catch(InterruptedException ie){ 374 | Thread.currentThread().interrupt(); 375 | } 376 | continue; 377 | } 378 | 379 | String rowId = next.getElement(); 380 | double delay = conn.zscore("delay:", rowId); 381 | if (delay <= 0) { 382 | conn.zrem("delay:", rowId); 383 | conn.zrem("schedule:", rowId); 384 | conn.del("inv:" + rowId); 385 | continue; 386 | } 387 | 388 | Inventory row = Inventory.get(rowId); 389 | conn.zadd("schedule:", now + delay, rowId); 390 | conn.set("inv:" + rowId, gson.toJson(row)); 391 | } 392 | } 393 | } 394 | 395 | public static class Inventory { 396 | private String id; 397 | private String data; 398 | private long time; 399 | 400 | private Inventory (String id) { 401 | this.id = id; 402 | this.data = "data to cache..."; 403 | this.time = System.currentTimeMillis() / 1000; 404 | } 405 | 406 | public static Inventory get(String id) { 407 | return new Inventory(id); 408 | } 409 | } 410 | } 411 | -------------------------------------------------------------------------------- /python/ch04_listing_source.py: -------------------------------------------------------------------------------- 1 | 2 | import os 3 | import time 4 | import unittest 5 | import uuid 6 | 7 | import redis 8 | 9 | ''' 10 | # 11 | save 60 1000 #A 12 | stop-writes-on-bgsave-error no #A 13 | rdbcompression yes #A 14 | dbfilename dump.rdb #A 15 | 16 | appendonly no #B 17 | appendfsync everysec #B 18 | no-appendfsync-on-rewrite no #B 19 | auto-aof-rewrite-percentage 100 #B 20 | auto-aof-rewrite-min-size 64mb #B 21 | 22 | dir ./ #C 23 | # 24 | #A Snapshotting persistence options 25 | #B Append-only file persistence options 26 | #C Shared option, where to store the snapshot or append-only file 27 | #END 28 | ''' 29 | 30 | # 31 | def process_logs(conn, path, callback): #K 32 | current_file, offset = conn.mget( #A 33 | 'progress:file', 'progress:position') #A 34 | 35 | pipe = conn.pipeline() 36 | 37 | def update_progress(): #H 38 | pipe.mset({ #I 39 | 'progress:file': fname, #I 40 | 'progress:position': offset #I 41 | }) 42 | pipe.execute() #J 43 | 44 | for fname in sorted(os.listdir(path)): #B 45 | if fname < current_file: #C 46 | continue 47 | 48 | inp = open(os.path.join(path, fname), 'rb') 49 | if fname == current_file: #D 50 | inp.seek(int(offset, 10)) #D 51 | else: 52 | offset = 0 53 | 54 | current_file = None 55 | 56 | for lno, line in enumerate(inp): #L 57 | callback(pipe, line) #E 58 | offset += int(offset) + len(line) #F 59 | 60 | if not (lno+1) % 1000: #G 61 | update_progress() #G 62 | update_progress() #G 63 | 64 | inp.close() 65 | # 66 | #A Get the current progress 67 | #B Iterate over the logfiles in sorted order 68 | #C Skip over files that are before the current file 69 | #D If we are continuing a file, skip over the parts that we've already processed 70 | #E Handle the log line 71 | #F Update our information about the offset into the file 72 | #G Write our progress back to Redis every 1000 lines, or when we are done with a file 73 | #H This closure is meant primarily to reduce the number of duplicated lines later 74 | #I We want to update our file and line number offsets into the logfile 75 | #J This will execute any outstanding log updates, as well as to actually write our file and line number updates to Redis 76 | #K Our function will be provided with a callback that will take a connection and a log line, calling methods on the pipeline as necessary 77 | #L The enumerate function iterates over a sequence (in this case lines from a file), and produces pairs consisting of a numeric sequence starting from 0, and the original data 78 | #END 79 | 80 | # 81 | def wait_for_sync(mconn, sconn): 82 | identifier = str(uuid.uuid4()) 83 | mconn.zadd('sync:wait', identifier, time.time()) #A 84 | 85 | while not sconn.info()['master_link_status'] != 'up': #B 86 | time.sleep(.001) 87 | 88 | while not sconn.zscore('sync:wait', identifier): #C 89 | time.sleep(.001) 90 | 91 | deadline = time.time() + 1.01 #D 92 | while time.time() < deadline: #D 93 | if sconn.info()['aof_pending_bio_fsync'] == 0: #E 94 | break #E 95 | time.sleep(.001) 96 | 97 | mconn.zrem('sync:wait', identifier) #F 98 | mconn.zremrangebyscore('sync:wait', 0, time.time()-900) #F 99 | # 100 | #A Add the token to the master 101 | #B Wait for the slave to sync (if necessary) 102 | #C Wait for the slave to receive the data change 103 | #D Wait up to 1 second 104 | #E Check to see if the data is known to be on disk 105 | #F Clean up our status and clean out older entries that may have been left there 106 | #END 107 | 108 | ''' 109 | # 110 | user@vpn-master ~:$ ssh root@machine-b.vpn #A 111 | Last login: Wed Mar 28 15:21:06 2012 from ... #A 112 | root@machine-b ~:$ redis-cli #B 113 | redis 127.0.0.1:6379> SAVE #C 114 | OK #C 115 | redis 127.0.0.1:6379> QUIT #C 116 | root@machine-b ~:$ scp \\ #D 117 | > /var/local/redis/dump.rdb machine-c.vpn:/var/local/redis/ #D 118 | dump.rdb 100% 525MB 8.1MB/s 01:05 #D 119 | root@machine-b ~:$ ssh machine-c.vpn #E 120 | Last login: Tue Mar 27 12:42:31 2012 from ... #E 121 | root@machine-c ~:$ sudo /etc/init.d/redis-server start #E 122 | Starting Redis server... #E 123 | root@machine-c ~:$ exit 124 | root@machine-b ~:$ redis-cli #F 125 | redis 127.0.0.1:6379> SLAVEOF machine-c.vpn 6379 #F 126 | OK #F 127 | redis 127.0.0.1:6379> QUIT 128 | root@machine-b ~:$ exit 129 | user@vpn-master ~:$ 130 | # 131 | #A Connect to machine B on our vpn network 132 | #B Start up the command line redis client to do a few simple operations 133 | #C Start a SAVE, and when it is done, QUIT so that we can continue 134 | #D Copy the snapshot over to the new master, machine C 135 | #E Connect to the new master and start Redis 136 | #F Tell machine B's Redis that it should use C as the new master 137 | #END 138 | ''' 139 | 140 | # 141 | def list_item(conn, itemid, sellerid, price): 142 | inventory = "inventory:%s"%sellerid 143 | item = "%s.%s"%(itemid, sellerid) 144 | end = time.time() + 5 145 | pipe = conn.pipeline() 146 | 147 | while time.time() < end: 148 | try: 149 | pipe.watch(inventory) #A 150 | if not pipe.sismember(inventory, itemid):#B 151 | pipe.unwatch() #E 152 | return None 153 | 154 | pipe.multi() #C 155 | pipe.zadd("market:", item, price) #C 156 | pipe.srem(inventory, itemid) #C 157 | pipe.execute() #F 158 | return True 159 | except redis.exceptions.WatchError: #D 160 | pass #D 161 | return False 162 | # 163 | #A Watch for changes to the users's inventory 164 | #B Verify that the user still has the item to be listed 165 | #E If the item is not in the user's inventory, stop watching the inventory key and return 166 | #C Actually list the item 167 | #F If execute returns without a WatchError being raised, then the transaction is complete and the inventory key is no longer watched 168 | #D The user's inventory was changed, retry 169 | #END 170 | 171 | # 172 | def purchase_item(conn, buyerid, itemid, sellerid, lprice): 173 | buyer = "users:%s"%buyerid 174 | seller = "users:%s"%sellerid 175 | item = "%s.%s"%(itemid, sellerid) 176 | inventory = "inventory:%s"%buyerid 177 | end = time.time() + 10 178 | pipe = conn.pipeline() 179 | 180 | while time.time() < end: 181 | try: 182 | pipe.watch("market:", buyer) #A 183 | 184 | price = pipe.zscore("market:", item) #B 185 | funds = int(pipe.hget(buyer, "funds")) #B 186 | if price != lprice or price > funds: #B 187 | pipe.unwatch() #B 188 | return None 189 | 190 | pipe.multi() #C 191 | pipe.hincrby(seller, "funds", int(price)) #C 192 | pipe.hincrby(buyer, "funds", int(-price)) #C 193 | pipe.sadd(inventory, itemid) #C 194 | pipe.zrem("market:", item) #C 195 | pipe.execute() #C 196 | return True 197 | except redis.exceptions.WatchError: #D 198 | pass #D 199 | 200 | return False 201 | # 202 | #A Watch for changes to the market and to the buyer's account information 203 | #B Check for a sold/repriced item or insufficient funds 204 | #C Transfer funds from the buyer to the seller, and transfer the item to the buyer 205 | #D Retry if the buyer's account or the market changed 206 | #END 207 | 208 | 209 | # 210 | def update_token(conn, token, user, item=None): 211 | timestamp = time.time() #A 212 | conn.hset('login:', token, user) #B 213 | conn.zadd('recent:', token, timestamp) #C 214 | if item: 215 | conn.zadd('viewed:' + token, item, timestamp) #D 216 | conn.zremrangebyrank('viewed:' + token, 0, -26) #E 217 | conn.zincrby('viewed:', item, -1) #F 218 | # 219 | #A Get the timestamp 220 | #B Keep a mapping from the token to the logged-in user 221 | #C Record when the token was last seen 222 | #D Record that the user viewed the item 223 | #E Remove old items, keeping the most recent 25 224 | #F Update the number of times the given item had been viewed 225 | #END 226 | 227 | # 228 | def update_token_pipeline(conn, token, user, item=None): 229 | timestamp = time.time() 230 | pipe = conn.pipeline(False) #A 231 | pipe.hset('login:', token, user) 232 | pipe.zadd('recent:', token, timestamp) 233 | if item: 234 | pipe.zadd('viewed:' + token, item, timestamp) 235 | pipe.zremrangebyrank('viewed:' + token, 0, -26) 236 | pipe.zincrby('viewed:', item, -1) 237 | pipe.execute() #B 238 | # 239 | #A Set up the pipeline 240 | #B Execute the commands in the pipeline 241 | #END 242 | 243 | # 244 | def benchmark_update_token(conn, duration): 245 | for function in (update_token, update_token_pipeline): #A 246 | count = 0 #B 247 | start = time.time() #B 248 | end = start + duration #B 249 | while time.time() < end: 250 | count += 1 251 | function(conn, 'token', 'user', 'item') #C 252 | delta = time.time() - start #D 253 | print function.__name__, count, delta, count / delta #E 254 | # 255 | #A Execute both the update_token() and the update_token_pipeline() functions 256 | #B Set up our counters and our ending conditions 257 | #C Call one of the two functions 258 | #D Calculate the duration 259 | #E Print information about the results 260 | #END 261 | 262 | ''' 263 | # 264 | $ redis-benchmark -c 1 -q #A 265 | PING (inline): 34246.57 requests per second 266 | PING: 34843.21 requests per second 267 | MSET (10 keys): 24213.08 requests per second 268 | SET: 32467.53 requests per second 269 | GET: 33112.59 requests per second 270 | INCR: 32679.74 requests per second 271 | LPUSH: 33333.33 requests per second 272 | LPOP: 33670.04 requests per second 273 | SADD: 33222.59 requests per second 274 | SPOP: 34482.76 requests per second 275 | LPUSH (again, in order to bench LRANGE): 33222.59 requests per second 276 | LRANGE (first 100 elements): 22988.51 requests per second 277 | LRANGE (first 300 elements): 13888.89 requests per second 278 | LRANGE (first 450 elements): 11061.95 requests per second 279 | LRANGE (first 600 elements): 9041.59 requests per second 280 | # 281 | #A We run with the '-q' option to get simple output, and '-c 1' to use a single client 282 | #END 283 | ''' 284 | 285 | #--------------- Below this line are helpers to test the code ---------------- 286 | 287 | class TestCh04(unittest.TestCase): 288 | def setUp(self): 289 | import redis 290 | self.conn = redis.Redis(db=15) 291 | self.conn.flushdb() 292 | 293 | def tearDown(self): 294 | self.conn.flushdb() 295 | del self.conn 296 | print 297 | print 298 | 299 | # We can't test process_logs, as that would require writing to disk, which 300 | # we don't want to do. 301 | 302 | # We also can't test wait_for_sync, as we can't guarantee that there are 303 | # multiple Redis servers running with the proper configuration 304 | 305 | def test_list_item(self): 306 | import pprint 307 | conn = self.conn 308 | 309 | print "We need to set up just enough state so that a user can list an item" 310 | seller = 'userX' 311 | item = 'itemX' 312 | conn.sadd('inventory:' + seller, item) 313 | i = conn.smembers('inventory:' + seller) 314 | print "The user's inventory has:", i 315 | self.assertTrue(i) 316 | print 317 | 318 | print "Listing the item..." 319 | l = list_item(conn, item, seller, 10) 320 | print "Listing the item succeeded?", l 321 | self.assertTrue(l) 322 | r = conn.zrange('market:', 0, -1, withscores=True) 323 | print "The market contains:" 324 | pprint.pprint(r) 325 | self.assertTrue(r) 326 | self.assertTrue(any(x[0] == 'itemX.userX' for x in r)) 327 | 328 | def test_purchase_item(self): 329 | self.test_list_item() 330 | conn = self.conn 331 | 332 | print "We need to set up just enough state so a user can buy an item" 333 | buyer = 'userY' 334 | conn.hset('users:userY', 'funds', 125) 335 | r = conn.hgetall('users:userY') 336 | print "The user has some money:", r 337 | self.assertTrue(r) 338 | self.assertTrue(r.get('funds')) 339 | print 340 | 341 | print "Let's purchase an item" 342 | p = purchase_item(conn, 'userY', 'itemX', 'userX', 10) 343 | print "Purchasing an item succeeded?", p 344 | self.assertTrue(p) 345 | r = conn.hgetall('users:userY') 346 | print "Their money is now:", r 347 | self.assertTrue(r) 348 | i = conn.smembers('inventory:' + buyer) 349 | print "Their inventory is now:", i 350 | self.assertTrue(i) 351 | self.assertTrue('itemX' in i) 352 | self.assertEquals(conn.zscore('market:', 'itemX.userX'), None) 353 | 354 | def test_benchmark_update_token(self): 355 | benchmark_update_token(self.conn, 5) 356 | 357 | if __name__ == '__main__': 358 | unittest.main() 359 | -------------------------------------------------------------------------------- /java/src/main/java/Chapter09.java: -------------------------------------------------------------------------------- 1 | import org.javatuples.Pair; 2 | import redis.clients.jedis.Jedis; 3 | import redis.clients.jedis.Pipeline; 4 | import redis.clients.jedis.ZParams; 5 | 6 | import java.io.IOException; 7 | import java.io.InputStream; 8 | import java.text.SimpleDateFormat; 9 | import java.util.*; 10 | import java.util.zip.CRC32; 11 | 12 | public class Chapter09 { 13 | private static final String[] COUNTRIES = ( 14 | "ABW AFG AGO AIA ALA ALB AND ARE ARG ARM ASM ATA ATF ATG AUS AUT AZE BDI " + 15 | "BEL BEN BES BFA BGD BGR BHR BHS BIH BLM BLR BLZ BMU BOL BRA BRB BRN BTN " + 16 | "BVT BWA CAF CAN CCK CHE CHL CHN CIV CMR COD COG COK COL COM CPV CRI CUB " + 17 | "CUW CXR CYM CYP CZE DEU DJI DMA DNK DOM DZA ECU EGY ERI ESH ESP EST ETH " + 18 | "FIN FJI FLK FRA FRO FSM GAB GBR GEO GGY GHA GIB GIN GLP GMB GNB GNQ GRC " + 19 | "GRD GRL GTM GUF GUM GUY HKG HMD HND HRV HTI HUN IDN IMN IND IOT IRL IRN " + 20 | "IRQ ISL ISR ITA JAM JEY JOR JPN KAZ KEN KGZ KHM KIR KNA KOR KWT LAO LBN " + 21 | "LBR LBY LCA LIE LKA LSO LTU LUX LVA MAC MAF MAR MCO MDA MDG MDV MEX MHL " + 22 | "MKD MLI MLT MMR MNE MNG MNP MOZ MRT MSR MTQ MUS MWI MYS MYT NAM NCL NER " + 23 | "NFK NGA NIC NIU NLD NOR NPL NRU NZL OMN PAK PAN PCN PER PHL PLW PNG POL " + 24 | "PRI PRK PRT PRY PSE PYF QAT REU ROU RUS RWA SAU SDN SEN SGP SGS SHN SJM " + 25 | "SLB SLE SLV SMR SOM SPM SRB SSD STP SUR SVK SVN SWE SWZ SXM SYC SYR TCA " + 26 | "TCD TGO THA TJK TKL TKM TLS TON TTO TUN TUR TUV TWN TZA UGA UKR UMI URY " + 27 | "USA UZB VAT VCT VEN VGB VIR VNM VUT WLF WSM YEM ZAF ZMB ZWE").split(" "); 28 | 29 | private static final Map STATES = new HashMap(); 30 | static { 31 | STATES.put("CAN", "AB BC MB NB NL NS NT NU ON PE QC SK YT".split(" ")); 32 | STATES.put("USA", ( 33 | "AA AE AK AL AP AR AS AZ CA CO CT DC DE FL FM GA GU HI IA ID IL IN " + 34 | "KS KY LA MA MD ME MH MI MN MO MP MS MT NC ND NE NH NJ NM NV NY OH " + 35 | "OK OR PA PR PW RI SC SD TN TX UT VA VI VT WA WI WV WY").split(" ")); 36 | } 37 | 38 | private static final SimpleDateFormat ISO_FORMAT = 39 | new SimpleDateFormat("yyyy-MM-dd'T'HH:00:00"); 40 | static{ 41 | ISO_FORMAT.setTimeZone(TimeZone.getTimeZone("UTC")); 42 | } 43 | 44 | public static final void main(String[] args) { 45 | new Chapter09().run(); 46 | } 47 | 48 | public void run() { 49 | Jedis conn = new Jedis("localhost"); 50 | conn.select(15); 51 | conn.flushDB(); 52 | 53 | testLongZiplistPerformance(conn); 54 | testShardKey(conn); 55 | testShardedHash(conn); 56 | testShardedSadd(conn); 57 | testUniqueVisitors(conn); 58 | testUserLocation(conn); 59 | } 60 | 61 | public void testLongZiplistPerformance(Jedis conn) { 62 | System.out.println("\n----- testLongZiplistPerformance -----"); 63 | 64 | longZiplistPerformance(conn, "test", 5, 10, 10); 65 | assert conn.llen("test") == 5; 66 | } 67 | 68 | public void testShardKey(Jedis conn) { 69 | System.out.println("\n----- testShardKey -----"); 70 | 71 | String base = "test"; 72 | assert "test:0".equals(shardKey(base, "1", 2, 2)); 73 | assert "test:1".equals(shardKey(base, "125", 1000, 100)); 74 | 75 | for (int i = 0; i < 50; i++) { 76 | String key = shardKey(base, "hello:" + i, 1000, 100); 77 | String[] parts = key.split(":"); 78 | assert Integer.parseInt(parts[parts.length - 1]) < 20; 79 | 80 | key = shardKey(base, String.valueOf(i), 1000, 100); 81 | parts = key.split(":"); 82 | assert Integer.parseInt(parts[parts.length - 1]) < 10; 83 | } 84 | } 85 | 86 | public void testShardedHash(Jedis conn) { 87 | System.out.println("\n----- testShardedHash -----"); 88 | 89 | for (int i = 0; i < 50; i++) { 90 | String istr = String.valueOf(i); 91 | shardHset(conn, "test", "keyname:" + i, istr, 1000, 100); 92 | assert istr.equals(shardHget(conn, "test", "keyname:" + i, 1000, 100)); 93 | shardHset(conn, "test2", istr, istr, 1000, 100); 94 | assert istr.equals(shardHget(conn, "test2", istr, 1000, 100)); 95 | } 96 | } 97 | 98 | public void testShardedSadd(Jedis conn) { 99 | System.out.println("\n----- testShardedSadd -----"); 100 | 101 | for (int i = 0; i < 50; i++) { 102 | shardSadd(conn, "testx", String.valueOf(i), 50, 50); 103 | } 104 | assert conn.scard("testx:0") + conn.scard("testx:1") == 50; 105 | } 106 | 107 | public void testUniqueVisitors(Jedis conn) { 108 | System.out.println("\n----- testUniqueVisitors -----"); 109 | 110 | DAILY_EXPECTED = 10000; 111 | 112 | for (int i = 0; i < 179; i++) { 113 | countVisit(conn, UUID.randomUUID().toString()); 114 | } 115 | assert "179".equals(conn.get("unique:" + ISO_FORMAT.format(new Date()))); 116 | 117 | conn.flushDB(); 118 | Calendar yesterday = Calendar.getInstance(); 119 | yesterday.add(Calendar.DATE, -1); 120 | conn.set("unique:" + ISO_FORMAT.format(yesterday.getTime()), "1000"); 121 | for (int i = 0; i < 183; i++) { 122 | countVisit(conn, UUID.randomUUID().toString()); 123 | } 124 | assert "183".equals(conn.get("unique:" + ISO_FORMAT.format(new Date()))); 125 | } 126 | 127 | public void testUserLocation(Jedis conn) { 128 | System.out.println("\n----- testUserLocation -----"); 129 | 130 | int i = 0; 131 | for (String country : COUNTRIES) { 132 | if (STATES.containsKey(country)){ 133 | for (String state : STATES.get(country)) { 134 | setLocation(conn, i, country, state); 135 | i++; 136 | } 137 | }else{ 138 | setLocation(conn, i, country, ""); 139 | i++; 140 | } 141 | } 142 | 143 | Pair,Map>> _aggs = 144 | aggregateLocation(conn); 145 | 146 | long[] userIds = new long[i + 1]; 147 | for (int j = 0; j <= i; j++) { 148 | userIds[j] = j; 149 | } 150 | Pair,Map>> aggs = 151 | aggregateLocationList(conn, userIds); 152 | 153 | assert _aggs.equals(aggs); 154 | 155 | Map countries = aggs.getValue0(); 156 | Map> states = aggs.getValue1(); 157 | for (String country : aggs.getValue0().keySet()){ 158 | if (STATES.containsKey(country)) { 159 | assert STATES.get(country).length == countries.get(country); 160 | for (String state : STATES.get(country)){ 161 | assert states.get(country).get(state) == 1; 162 | } 163 | }else{ 164 | assert countries.get(country) == 1; 165 | } 166 | } 167 | } 168 | 169 | public double longZiplistPerformance( 170 | Jedis conn, String key, int length, int passes, int psize) 171 | { 172 | conn.del(key); 173 | for (int i = 0; i < length; i++) { 174 | conn.rpush(key, String.valueOf(i)); 175 | } 176 | 177 | Pipeline pipeline = conn.pipelined(); 178 | long time = System.currentTimeMillis(); 179 | for (int p = 0; p < passes; p++) { 180 | for (int pi = 0; pi < psize; pi++) { 181 | pipeline.rpoplpush(key, key); 182 | } 183 | pipeline.sync(); 184 | } 185 | 186 | return (passes * psize) / (System.currentTimeMillis() - time); 187 | } 188 | 189 | public String shardKey(String base, String key, long totalElements, int shardSize) { 190 | long shardId = 0; 191 | if (isDigit(key)) { 192 | shardId = Integer.parseInt(key, 10) / shardSize; 193 | }else{ 194 | CRC32 crc = new CRC32(); 195 | crc.update(key.getBytes()); 196 | long shards = 2 * totalElements / shardSize; 197 | shardId = Math.abs(((int)crc.getValue()) % shards); 198 | } 199 | return base + ':' + shardId; 200 | } 201 | 202 | public Long shardHset( 203 | Jedis conn, String base, String key, String value, long totalElements, int shardSize) 204 | { 205 | String shard = shardKey(base, key, totalElements, shardSize); 206 | return conn.hset(shard, key, value); 207 | } 208 | 209 | public String shardHget( 210 | Jedis conn, String base, String key, int totalElements, int shardSize) 211 | { 212 | String shard = shardKey(base, key, totalElements, shardSize); 213 | return conn.hget(shard, key); 214 | } 215 | 216 | public Long shardSadd( 217 | Jedis conn, String base, String member, long totalElements, int shardSize) 218 | { 219 | String shard = shardKey(base, "x" + member, totalElements, shardSize); 220 | return conn.sadd(shard, member); 221 | } 222 | 223 | private int SHARD_SIZE = 512; 224 | public void countVisit(Jedis conn, String sessionId) { 225 | Calendar today = Calendar.getInstance(); 226 | String key = "unique:" + ISO_FORMAT.format(today.getTime()); 227 | long expected = getExpected(conn, key, today); 228 | long id = Long.parseLong(sessionId.replace("-", "").substring(0, 15), 16); 229 | if (shardSadd(conn, key, String.valueOf(id), expected, SHARD_SIZE) != 0) { 230 | conn.incr(key); 231 | } 232 | } 233 | 234 | private long DAILY_EXPECTED = 1000000; 235 | private Map EXPECTED = new HashMap(); 236 | 237 | public long getExpected(Jedis conn, String key, Calendar today) { 238 | if (!EXPECTED.containsKey(key)) { 239 | String exkey = key + ":expected"; 240 | String expectedStr = conn.get(exkey); 241 | 242 | long expected = 0; 243 | if (expectedStr == null) { 244 | Calendar yesterday = (Calendar)today.clone(); 245 | yesterday.add(Calendar.DATE, -1); 246 | expectedStr = conn.get( 247 | "unique:" + ISO_FORMAT.format(yesterday.getTime())); 248 | expected = expectedStr != null ? Long.parseLong(expectedStr) : DAILY_EXPECTED; 249 | 250 | expected = (long)Math.pow(2, (long)(Math.ceil(Math.log(expected * 1.5) / Math.log(2)))); 251 | if (conn.setnx(exkey, String.valueOf(expected)) == 0) { 252 | expectedStr = conn.get(exkey); 253 | expected = Integer.parseInt(expectedStr); 254 | } 255 | }else{ 256 | expected = Long.parseLong(expectedStr); 257 | } 258 | 259 | EXPECTED.put(key, expected); 260 | } 261 | 262 | return EXPECTED.get(key); 263 | } 264 | 265 | private long USERS_PER_SHARD = (long)Math.pow(2, 20); 266 | 267 | public void setLocation( 268 | Jedis conn, long userId, String country, String state) 269 | { 270 | String code = getCode(country, state); 271 | 272 | long shardId = userId / USERS_PER_SHARD; 273 | int position = (int)(userId % USERS_PER_SHARD); 274 | int offset = position * 2; 275 | 276 | Pipeline pipe = conn.pipelined(); 277 | pipe.setrange("location:" + shardId, offset, code); 278 | 279 | String tkey = UUID.randomUUID().toString(); 280 | pipe.zadd(tkey, userId, "max"); 281 | pipe.zunionstore( 282 | "location:max", 283 | new ZParams().aggregate(ZParams.Aggregate.MAX), 284 | tkey, 285 | "location:max"); 286 | pipe.del(tkey); 287 | pipe.sync(); 288 | } 289 | 290 | public Pair,Map>> aggregateLocation(Jedis conn) { 291 | Map countries = new HashMap(); 292 | Map> states = new HashMap>(); 293 | 294 | long maxId = conn.zscore("location:max", "max").longValue(); 295 | long maxBlock = maxId; 296 | 297 | byte[] buffer = new byte[(int)Math.pow(2, 17)]; 298 | for (int shardId = 0; shardId <= maxBlock; shardId++) { 299 | InputStream in = new RedisInputStream(conn, "location:" + shardId); 300 | try{ 301 | int read = 0; 302 | while ((read = in.read(buffer, 0, buffer.length)) != -1){ 303 | for (int offset = 0; offset < read - 1; offset += 2) { 304 | String code = new String(buffer, offset, 2); 305 | updateAggregates(countries, states, code); 306 | } 307 | } 308 | }catch(IOException ioe) { 309 | throw new RuntimeException(ioe); 310 | }finally{ 311 | try{ 312 | in.close(); 313 | }catch(Exception e){ 314 | // ignore 315 | } 316 | } 317 | } 318 | 319 | return new Pair,Map>>(countries, states); 320 | } 321 | 322 | public Pair,Map>> aggregateLocationList( 323 | Jedis conn, long[] userIds) 324 | { 325 | Map countries = new HashMap(); 326 | Map> states = new HashMap>(); 327 | 328 | Pipeline pipe = conn.pipelined(); 329 | for (int i = 0; i < userIds.length; i++) { 330 | long userId = userIds[i]; 331 | long shardId = userId / USERS_PER_SHARD; 332 | int position = (int)(userId % USERS_PER_SHARD); 333 | int offset = position * 2; 334 | 335 | pipe.substr("location:" + shardId, offset, offset + 1); 336 | 337 | if ((i + 1) % 1000 == 0) { 338 | updateAggregates(countries, states, pipe.syncAndReturnAll()); 339 | } 340 | } 341 | 342 | updateAggregates(countries, states, pipe.syncAndReturnAll()); 343 | 344 | return new Pair,Map>>(countries, states); 345 | } 346 | 347 | public void updateAggregates( 348 | Map countries, Map> states, List codes) 349 | { 350 | for (Object code : codes) { 351 | updateAggregates(countries, states, (String)code); 352 | } 353 | } 354 | 355 | public void updateAggregates( 356 | Map countries, Map> states, String code) 357 | { 358 | if (code.length() != 2) { 359 | return; 360 | } 361 | 362 | int countryIdx = (int)code.charAt(0) - 1; 363 | int stateIdx = (int)code.charAt(1) - 1; 364 | 365 | if (countryIdx < 0 || countryIdx >= COUNTRIES.length) { 366 | return; 367 | } 368 | 369 | String country = COUNTRIES[countryIdx]; 370 | Long countryAgg = countries.get(country); 371 | if (countryAgg == null){ 372 | countryAgg = Long.valueOf(0); 373 | } 374 | countries.put(country, countryAgg + 1); 375 | 376 | if (!STATES.containsKey(country)) { 377 | return; 378 | } 379 | if (stateIdx < 0 || stateIdx >= STATES.get(country).length){ 380 | return; 381 | } 382 | 383 | String state = STATES.get(country)[stateIdx]; 384 | Map stateAggs = states.get(country); 385 | if (stateAggs == null){ 386 | stateAggs = new HashMap(); 387 | states.put(country, stateAggs); 388 | } 389 | Long stateAgg = stateAggs.get(state); 390 | if (stateAgg == null){ 391 | stateAgg = Long.valueOf(0); 392 | } 393 | stateAggs.put(state, stateAgg + 1); 394 | } 395 | 396 | public String getCode(String country, String state) { 397 | int cindex = bisectLeft(COUNTRIES, country); 398 | if (cindex > COUNTRIES.length || !country.equals(COUNTRIES[cindex])) { 399 | cindex = -1; 400 | } 401 | cindex++; 402 | 403 | int sindex = -1; 404 | if (state != null && STATES.containsKey(country)) { 405 | String[] states = STATES.get(country); 406 | sindex = bisectLeft(states, state); 407 | if (sindex > states.length || !state.equals(states[sindex])) { 408 | sindex--; 409 | } 410 | } 411 | sindex++; 412 | 413 | return new String(new char[]{(char)cindex, (char)sindex}); 414 | } 415 | 416 | private int bisectLeft(String[] values, String key) { 417 | int index = Arrays.binarySearch(values, key); 418 | return index < 0 ? Math.abs(index) - 1 : index; 419 | } 420 | 421 | private boolean isDigit(String string) { 422 | for(char c : string.toCharArray()) { 423 | if (!Character.isDigit(c)){ 424 | return false; 425 | } 426 | } 427 | return true; 428 | } 429 | 430 | public class RedisInputStream 431 | extends InputStream 432 | { 433 | private Jedis conn; 434 | private String key; 435 | private int pos; 436 | 437 | public RedisInputStream(Jedis conn, String key){ 438 | this.conn = conn; 439 | this.key = key; 440 | } 441 | 442 | @Override 443 | public int available() 444 | throws IOException 445 | { 446 | long len = conn.strlen(key); 447 | return (int)(len - pos); 448 | } 449 | 450 | @Override 451 | public int read() 452 | throws IOException 453 | { 454 | byte[] block = conn.substr(key.getBytes(), pos, pos); 455 | if (block == null || block.length == 0){ 456 | return -1; 457 | } 458 | pos++; 459 | return (int)(block[0] & 0xff); 460 | } 461 | 462 | @Override 463 | public int read(byte[] buf, int off, int len) 464 | throws IOException 465 | { 466 | byte[] block = conn.substr(key.getBytes(), pos, pos + (len - off - 1)); 467 | if (block == null || block.length == 0){ 468 | return -1; 469 | } 470 | System.arraycopy(block, 0, buf, off, block.length); 471 | pos += block.length; 472 | return block.length; 473 | } 474 | 475 | @Override 476 | public void close() { 477 | // no-op 478 | } 479 | } 480 | } 481 | -------------------------------------------------------------------------------- /java/src/main/java/Chapter08.java: -------------------------------------------------------------------------------- 1 | import redis.clients.jedis.Jedis; 2 | import redis.clients.jedis.Pipeline; 3 | import redis.clients.jedis.Transaction; 4 | import redis.clients.jedis.Tuple; 5 | 6 | import java.lang.reflect.Method; 7 | import java.util.*; 8 | 9 | public class Chapter08 { 10 | private static int HOME_TIMELINE_SIZE = 1000; 11 | private static int POSTS_PER_PASS = 1000; 12 | private static int REFILL_USERS_STEP = 50; 13 | 14 | public static final void main(String[] args) 15 | throws InterruptedException 16 | { 17 | new Chapter08().run(); 18 | } 19 | 20 | public void run() 21 | throws InterruptedException 22 | { 23 | Jedis conn = new Jedis("localhost"); 24 | conn.select(15); 25 | conn.flushDB(); 26 | 27 | testCreateUserAndStatus(conn); 28 | conn.flushDB(); 29 | 30 | testFollowUnfollowUser(conn); 31 | conn.flushDB(); 32 | 33 | testSyndicateStatus(conn); 34 | conn.flushDB(); 35 | 36 | testRefillTimeline(conn); 37 | } 38 | 39 | public void testCreateUserAndStatus(Jedis conn) { 40 | System.out.println("\n----- testCreateUserAndStatus -----"); 41 | 42 | assert createUser(conn, "TestUser", "Test User") == 1; 43 | assert createUser(conn, "TestUser", "Test User2") == -1; 44 | 45 | assert createStatus(conn, 1, "This is a new status message") == 1; 46 | assert "1".equals(conn.hget("user:1", "posts")); 47 | } 48 | 49 | public void testFollowUnfollowUser(Jedis conn) { 50 | System.out.println("\n----- testFollowUnfollowUser -----"); 51 | 52 | assert createUser(conn, "TestUser", "Test User") == 1; 53 | assert createUser(conn, "TestUser2", "Test User2") == 2; 54 | 55 | assert followUser(conn, 1, 2); 56 | assert conn.zcard("followers:2") == 1; 57 | assert conn.zcard("followers:1") == 0; 58 | assert conn.zcard("following:1") == 1; 59 | assert conn.zcard("following:2") == 0; 60 | assert "1".equals(conn.hget("user:1", "following")); 61 | assert "0".equals(conn.hget("user:2", "following")); 62 | assert "0".equals(conn.hget("user:1", "followers")); 63 | assert "1".equals(conn.hget("user:2", "followers")); 64 | 65 | assert !unfollowUser(conn, 2, 1); 66 | assert unfollowUser(conn, 1, 2); 67 | assert conn.zcard("followers:2") == 0; 68 | assert conn.zcard("followers:1") == 0; 69 | assert conn.zcard("following:1") == 0; 70 | assert conn.zcard("following:2") == 0; 71 | assert "0".equals(conn.hget("user:1", "following")); 72 | assert "0".equals(conn.hget("user:2", "following")); 73 | assert "0".equals(conn.hget("user:1", "followers")); 74 | assert "0".equals(conn.hget("user:2", "followers")); 75 | } 76 | 77 | public void testSyndicateStatus(Jedis conn) 78 | throws InterruptedException 79 | { 80 | System.out.println("\n----- testSyndicateStatus -----"); 81 | 82 | assert createUser(conn, "TestUser", "Test User") == 1; 83 | assert createUser(conn, "TestUser2", "Test User2") == 2; 84 | 85 | assert followUser(conn, 1, 2); 86 | assert conn.zcard("followers:2") == 1; 87 | assert "1".equals(conn.hget("user:1", "following")); 88 | assert postStatus(conn, 2, "this is some message content") == 1; 89 | assert getStatusMessages(conn, 1).size() == 1; 90 | 91 | for(int i = 3; i < 11; i++) { 92 | assert createUser(conn, "TestUser" + i, "Test User" + i) == i; 93 | followUser(conn, i, 2); 94 | } 95 | 96 | POSTS_PER_PASS = 5; 97 | 98 | assert postStatus(conn, 2, "this is some other message content") == 2; 99 | Thread.sleep(100); 100 | assert getStatusMessages(conn, 9).size() == 2; 101 | 102 | assert unfollowUser(conn, 1, 2); 103 | assert getStatusMessages(conn, 1).size() == 0; 104 | } 105 | 106 | public void testRefillTimeline(Jedis conn) 107 | throws InterruptedException 108 | { 109 | System.out.println("\n----- testRefillTimeline -----"); 110 | 111 | assert createUser(conn, "TestUser", "Test User") == 1; 112 | assert createUser(conn, "TestUser2", "Test User2") == 2; 113 | assert createUser(conn, "TestUser3", "Test User3") == 3; 114 | 115 | assert followUser(conn, 1, 2); 116 | assert followUser(conn, 1, 3); 117 | 118 | HOME_TIMELINE_SIZE = 5; 119 | 120 | for (int i = 0; i < 10; i++) { 121 | assert postStatus(conn, 2, "message") != -1; 122 | assert postStatus(conn, 3, "message") != -1; 123 | Thread.sleep(50); 124 | } 125 | 126 | assert getStatusMessages(conn, 1).size() == 5; 127 | assert unfollowUser(conn, 1, 2); 128 | assert getStatusMessages(conn, 1).size() < 5; 129 | 130 | refillTimeline(conn, "following:1", "home:1"); 131 | List> messages = getStatusMessages(conn, 1); 132 | assert messages.size() == 5; 133 | for (Map message : messages) { 134 | assert "3".equals(message.get("uid")); 135 | } 136 | 137 | long statusId = Long.valueOf(messages.get(messages.size() -1).get("id")); 138 | assert deleteStatus(conn, 3, statusId); 139 | assert getStatusMessages(conn, 1).size() == 4; 140 | assert conn.zcard("home:1") == 5; 141 | cleanTimelines(conn, 3, statusId); 142 | assert conn.zcard("home:1") == 4; 143 | } 144 | 145 | public String acquireLockWithTimeout( 146 | Jedis conn, String lockName, int acquireTimeout, int lockTimeout) 147 | { 148 | String id = UUID.randomUUID().toString(); 149 | lockName = "lock:" + lockName; 150 | 151 | long end = System.currentTimeMillis() + (acquireTimeout * 1000); 152 | while (System.currentTimeMillis() < end) { 153 | if (conn.setnx(lockName, id) >= 1) { 154 | conn.expire(lockName, lockTimeout); 155 | return id; 156 | }else if (conn.ttl(lockName) <= 0){ 157 | conn.expire(lockName, lockTimeout); 158 | } 159 | 160 | try{ 161 | Thread.sleep(1); 162 | }catch(InterruptedException ie){ 163 | Thread.interrupted(); 164 | } 165 | } 166 | 167 | return null; 168 | } 169 | 170 | public boolean releaseLock(Jedis conn, String lockName, String identifier) { 171 | lockName = "lock:" + lockName; 172 | while (true) { 173 | conn.watch(lockName); 174 | if (identifier.equals(conn.get(lockName))) { 175 | Transaction trans = conn.multi(); 176 | trans.del(lockName); 177 | List result = trans.exec(); 178 | // null response indicates that the transaction was aborted due 179 | // to the watched key changing. 180 | if (result == null){ 181 | continue; 182 | } 183 | return true; 184 | } 185 | 186 | conn.unwatch(); 187 | break; 188 | } 189 | 190 | return false; 191 | } 192 | 193 | public long createUser(Jedis conn, String login, String name) { 194 | String llogin = login.toLowerCase(); 195 | String lock = acquireLockWithTimeout(conn, "user:" + llogin, 10, 1); 196 | if (lock == null){ 197 | return -1; 198 | } 199 | 200 | if (conn.hget("users:", llogin) != null) { 201 | return -1; 202 | } 203 | 204 | long id = conn.incr("user:id:"); 205 | Transaction trans = conn.multi(); 206 | trans.hset("users:", llogin, String.valueOf(id)); 207 | Map values = new HashMap(); 208 | values.put("login", login); 209 | values.put("id", String.valueOf(id)); 210 | values.put("name", name); 211 | values.put("followers", "0"); 212 | values.put("following", "0"); 213 | values.put("posts", "0"); 214 | values.put("signup", String.valueOf(System.currentTimeMillis())); 215 | trans.hmset("user:" + id, values); 216 | trans.exec(); 217 | releaseLock(conn, "user:" + llogin, lock); 218 | return id; 219 | } 220 | 221 | @SuppressWarnings("unchecked") 222 | public boolean followUser(Jedis conn, long uid, long otherUid) { 223 | String fkey1 = "following:" + uid; 224 | String fkey2 = "followers:" + otherUid; 225 | 226 | if (conn.zscore(fkey1, String.valueOf(otherUid)) != null) { 227 | return false; 228 | } 229 | 230 | long now = System.currentTimeMillis(); 231 | 232 | Transaction trans = conn.multi(); 233 | trans.zadd(fkey1, now, String.valueOf(otherUid)); 234 | trans.zadd(fkey2, now, String.valueOf(uid)); 235 | trans.zcard(fkey1); 236 | trans.zcard(fkey2); 237 | trans.zrevrangeWithScores("profile:" + otherUid, 0, HOME_TIMELINE_SIZE - 1); 238 | 239 | List response = trans.exec(); 240 | long following = (Long)response.get(response.size() - 3); 241 | long followers = (Long)response.get(response.size() - 2); 242 | Set statuses = (Set)response.get(response.size() - 1); 243 | 244 | trans = conn.multi(); 245 | trans.hset("user:" + uid, "following", String.valueOf(following)); 246 | trans.hset("user:" + otherUid, "followers", String.valueOf(followers)); 247 | if (statuses.size() > 0) { 248 | for (Tuple status : statuses){ 249 | trans.zadd("home:" + uid, status.getScore(), status.getElement()); 250 | } 251 | } 252 | trans.zremrangeByRank("home:" + uid, 0, 0 - HOME_TIMELINE_SIZE - 1); 253 | trans.exec(); 254 | 255 | return true; 256 | } 257 | 258 | @SuppressWarnings("unchecked") 259 | public boolean unfollowUser(Jedis conn, long uid, long otherUid) { 260 | String fkey1 = "following:" + uid; 261 | String fkey2 = "followers:" + otherUid; 262 | 263 | if (conn.zscore(fkey1, String.valueOf(otherUid)) == null) { 264 | return false; 265 | } 266 | 267 | Transaction trans = conn.multi(); 268 | trans.zrem(fkey1, String.valueOf(otherUid)); 269 | trans.zrem(fkey2, String.valueOf(uid)); 270 | trans.zcard(fkey1); 271 | trans.zcard(fkey2); 272 | trans.zrevrange("profile:" + otherUid, 0, HOME_TIMELINE_SIZE - 1); 273 | 274 | List response = trans.exec(); 275 | long following = (Long)response.get(response.size() - 3); 276 | long followers = (Long)response.get(response.size() - 2); 277 | Set statuses = (Set)response.get(response.size() - 1); 278 | 279 | trans = conn.multi(); 280 | trans.hset("user:" + uid, "following", String.valueOf(following)); 281 | trans.hset("user:" + otherUid, "followers", String.valueOf(followers)); 282 | if (statuses.size() > 0){ 283 | for (String status : statuses) { 284 | trans.zrem("home:" + uid, status); 285 | } 286 | } 287 | 288 | trans.exec(); 289 | return true; 290 | } 291 | 292 | public long createStatus(Jedis conn, long uid, String message) { 293 | return createStatus(conn, uid, message, null); 294 | } 295 | public long createStatus( 296 | Jedis conn, long uid, String message, Map data) 297 | { 298 | Transaction trans = conn.multi(); 299 | trans.hget("user:" + uid, "login"); 300 | trans.incr("status:id:"); 301 | 302 | List response = trans.exec(); 303 | String login = (String)response.get(0); 304 | long id = (Long)response.get(1); 305 | 306 | if (login == null) { 307 | return -1; 308 | } 309 | 310 | if (data == null){ 311 | data = new HashMap(); 312 | } 313 | data.put("message", message); 314 | data.put("posted", String.valueOf(System.currentTimeMillis())); 315 | data.put("id", String.valueOf(id)); 316 | data.put("uid", String.valueOf(uid)); 317 | data.put("login", login); 318 | 319 | trans = conn.multi(); 320 | trans.hmset("status:" + id, data); 321 | trans.hincrBy("user:" + uid, "posts", 1); 322 | trans.exec(); 323 | return id; 324 | } 325 | 326 | public long postStatus(Jedis conn, long uid, String message) { 327 | return postStatus(conn, uid, message, null); 328 | } 329 | public long postStatus( 330 | Jedis conn, long uid, String message, Map data) 331 | { 332 | long id = createStatus(conn, uid, message, data); 333 | if (id == -1){ 334 | return -1; 335 | } 336 | 337 | String postedString = conn.hget("status:" + id, "posted"); 338 | if (postedString == null) { 339 | return -1; 340 | } 341 | 342 | long posted = Long.parseLong(postedString); 343 | conn.zadd("profile:" + uid, posted, String.valueOf(id)); 344 | 345 | syndicateStatus(conn, uid, id, posted, 0); 346 | return id; 347 | } 348 | 349 | public void syndicateStatus( 350 | Jedis conn, long uid, long postId, long postTime, double start) 351 | { 352 | Set followers = conn.zrangeByScoreWithScores( 353 | "followers:" + uid, 354 | String.valueOf(start), "inf", 355 | 0, POSTS_PER_PASS); 356 | 357 | Transaction trans = conn.multi(); 358 | for (Tuple tuple : followers){ 359 | String follower = tuple.getElement(); 360 | start = tuple.getScore(); 361 | trans.zadd("home:" + follower, postTime, String.valueOf(postId)); 362 | trans.zrange("home:" + follower, 0, -1); 363 | trans.zremrangeByRank( 364 | "home:" + follower, 0, 0 - HOME_TIMELINE_SIZE - 1); 365 | } 366 | trans.exec(); 367 | 368 | if (followers.size() >= POSTS_PER_PASS) { 369 | try{ 370 | Method method = getClass().getDeclaredMethod( 371 | "syndicateStatus", Jedis.class, Long.TYPE, Long.TYPE, Long.TYPE, Double.TYPE); 372 | executeLater("default", method, uid, postId, postTime, start); 373 | }catch(Exception e){ 374 | throw new RuntimeException(e); 375 | } 376 | } 377 | } 378 | 379 | public boolean deleteStatus(Jedis conn, long uid, long statusId) { 380 | String key = "status:" + statusId; 381 | String lock = acquireLockWithTimeout(conn, key, 1, 10); 382 | if (lock == null) { 383 | return false; 384 | } 385 | 386 | try{ 387 | if (!String.valueOf(uid).equals(conn.hget(key, "uid"))) { 388 | return false; 389 | } 390 | 391 | Transaction trans = conn.multi(); 392 | trans.del(key); 393 | trans.zrem("profile:" + uid, String.valueOf(statusId)); 394 | trans.zrem("home:" + uid, String.valueOf(statusId)); 395 | trans.hincrBy("user:" + uid, "posts", -1); 396 | trans.exec(); 397 | 398 | return true; 399 | }finally{ 400 | releaseLock(conn, key, lock); 401 | } 402 | } 403 | 404 | public List> getStatusMessages(Jedis conn, long uid) { 405 | return getStatusMessages(conn, uid, 1, 30); 406 | } 407 | 408 | @SuppressWarnings("unchecked") 409 | public List> getStatusMessages( 410 | Jedis conn, long uid, int page, int count) 411 | { 412 | Set statusIds = conn.zrevrange( 413 | "home:" + uid, (page - 1) * count, page * count - 1); 414 | 415 | Transaction trans = conn.multi(); 416 | for (String id : statusIds) { 417 | trans.hgetAll("status:" + id); 418 | } 419 | 420 | List> statuses = new ArrayList>(); 421 | for (Object result : trans.exec()) { 422 | Map status = (Map)result; 423 | if (status != null && status.size() > 0){ 424 | statuses.add(status); 425 | } 426 | } 427 | return statuses; 428 | } 429 | 430 | public void refillTimeline(Jedis conn, String incoming, String timeline) { 431 | refillTimeline(conn, incoming, timeline, 0); 432 | } 433 | 434 | @SuppressWarnings("unchecked") 435 | public void refillTimeline( 436 | Jedis conn, String incoming, String timeline, double start) 437 | { 438 | if (start == 0 && conn.zcard(timeline) >= 750) { 439 | return; 440 | } 441 | 442 | Set users = conn.zrangeByScoreWithScores( 443 | incoming, String.valueOf(start), "inf", 0, REFILL_USERS_STEP); 444 | 445 | Pipeline pipeline = conn.pipelined(); 446 | for (Tuple tuple : users){ 447 | String uid = tuple.getElement(); 448 | start = tuple.getScore(); 449 | pipeline.zrevrangeWithScores( 450 | "profile:" + uid, 0, HOME_TIMELINE_SIZE - 1); 451 | } 452 | 453 | List response = pipeline.syncAndReturnAll(); 454 | List messages = new ArrayList(); 455 | for (Object results : response) { 456 | messages.addAll((Set)results); 457 | } 458 | 459 | Collections.sort(messages); 460 | messages = messages.subList(0, HOME_TIMELINE_SIZE); 461 | 462 | Transaction trans = conn.multi(); 463 | if (messages.size() > 0) { 464 | for (Tuple tuple : messages) { 465 | trans.zadd(timeline, tuple.getScore(), tuple.getElement()); 466 | } 467 | } 468 | trans.zremrangeByRank(timeline, 0, 0 - HOME_TIMELINE_SIZE - 1); 469 | trans.exec(); 470 | 471 | if (users.size() >= REFILL_USERS_STEP) { 472 | try{ 473 | Method method = getClass().getDeclaredMethod( 474 | "refillTimeline", Jedis.class, String.class, String.class, Double.TYPE); 475 | executeLater("default", method, incoming, timeline, start); 476 | }catch(Exception e){ 477 | throw new RuntimeException(e); 478 | } 479 | } 480 | } 481 | 482 | public void cleanTimelines(Jedis conn, long uid, long statusId) { 483 | cleanTimelines(conn, uid, statusId, 0, false); 484 | } 485 | public void cleanTimelines( 486 | Jedis conn, long uid, long statusId, double start, boolean onLists) 487 | { 488 | String key = "followers:" + uid; 489 | String base = "home:"; 490 | if (onLists) { 491 | key = "list:out:" + uid; 492 | base = "list:statuses:"; 493 | } 494 | Set followers = conn.zrangeByScoreWithScores( 495 | key, String.valueOf(start), "inf", 0, POSTS_PER_PASS); 496 | 497 | Transaction trans = conn.multi(); 498 | for (Tuple tuple : followers) { 499 | start = tuple.getScore(); 500 | String follower = tuple.getElement(); 501 | trans.zrem(base + follower, String.valueOf(statusId)); 502 | } 503 | trans.exec(); 504 | 505 | Method method = null; 506 | try{ 507 | method = getClass().getDeclaredMethod( 508 | "cleanTimelines", Jedis.class, 509 | Long.TYPE, Long.TYPE, Double.TYPE, Boolean.TYPE); 510 | }catch(Exception e){ 511 | throw new RuntimeException(e); 512 | } 513 | 514 | if (followers.size() >= POSTS_PER_PASS) { 515 | executeLater("default", method, uid, statusId, start, onLists); 516 | 517 | }else if (!onLists) { 518 | executeLater("default", method, uid, statusId, 0, true); 519 | } 520 | } 521 | 522 | public void executeLater(String queue, Method method, Object... args) { 523 | MethodThread thread = new MethodThread(this, method, args); 524 | thread.start(); 525 | } 526 | 527 | public class MethodThread 528 | extends Thread 529 | { 530 | private Object instance; 531 | private Method method; 532 | private Object[] args; 533 | 534 | public MethodThread(Object instance, Method method, Object... args) { 535 | this.instance = instance; 536 | this.method = method; 537 | this.args = args; 538 | } 539 | 540 | public void run() { 541 | Jedis conn = new Jedis("localhost"); 542 | conn.select(15); 543 | 544 | Object[] args = new Object[this.args.length + 1]; 545 | System.arraycopy(this.args, 0, args, 1, this.args.length); 546 | args[0] = conn; 547 | 548 | try{ 549 | method.invoke(instance, args); 550 | }catch(Exception e){ 551 | throw new RuntimeException(e); 552 | } 553 | } 554 | } 555 | } 556 | -------------------------------------------------------------------------------- /excerpt_errata.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | Errata for Redis in Action 5 | 24 | 25 | 26 |
27 |

Errata for Redis in Action

28 | booktitle 29 |

Last updated December 14, 2015

30 |

You can view the most recent version of this errata here

31 |

Known errata

32 | 70 |

Recent updates

71 | 88 | 89 |

Chapter 2

90 |

Section 2.5, Page 35, Listing 2.10 (2014/05/04)

91 |

There is a bug on the third line of the rescale_viewed() function definition.

92 |

The full function definition reads:

93 |
94 | 
def rescale_viewed(conn):
 95 |     while not QUIT:
 96 |         conn.zremrangebyrank('viewed:', 20000, -1)
 97 |         conn.zinterstore('viewed:', {'viewed:': .5})
 98 |         time.sleep(300)
99 |
100 | 101 |

The third line, updated inline, should read:

102 |
103 | 
def rescale_viewed(conn):
104 |     while not QUIT:
105 |         conn.zremrangebyrank('viewed:', 0, -20001)
106 |         conn.zinterstore('viewed:', {'viewed:': .5})
107 |         time.sleep(300)
108 |
109 | 110 |

You can see the updated code in-context by visiting: http://goo.gl/1hlS3m

111 | 112 |

Chapter 5

113 |

Section 5.1.2, Page 93, Listing 5.2 (2015/10/10)

114 |

There is a bug caused by a missing elif clause inside the try block of the log_common() function definition.

115 |

The original full function definition reads:

116 |
117 | 
def log_common(conn, name, message, severity=logging.INFO, timeout=5):
118 |     severity = str(SEVERITY.get(severity, severity)).lower()
119 |     destination = 'common:%s:%s'%(name, severity)
120 |     start_key = destination + ':start'
121 |     pipe = conn.pipeline()
122 |     end = time.time() + timeout
123 |     while time.time() < end:
124 |         try:
125 |             pipe.watch(start_key)
126 |             now = datetime.utcnow().timetuple()
127 |             hour_start = datetime(*now[:4]).isoformat()
128 | 
129 |             existing = pipe.get(start_key)
130 |             pipe.multi()
131 |             if existing and existing < hour_start:
132 |                 pipe.rename(destination, destination + ':last')
133 |                 pipe.rename(start_key, destination + ':pstart')
134 |                 pipe.set(start_key, hour_start)
135 | 
136 |             pipe.zincrby(destination, message)
137 |             log_recent(pipe, name, message, severity, pipe)
138 |             return
139 |         except redis.exceptions.WatchError:
140 |             continue
141 |
142 | 143 |

And with the added elif block (prefixed by a comment line below), the function should read:

144 |
145 | 
def log_common(conn, name, message, severity=logging.INFO, timeout=5):
146 |     severity = str(SEVERITY.get(severity, severity)).lower()
147 |     destination = 'common:%s:%s'%(name, severity)
148 |     start_key = destination + ':start'
149 |     pipe = conn.pipeline()
150 |     end = time.time() + timeout
151 |     while time.time() < end:
152 |         try:
153 |             pipe.watch(start_key)
154 |             now = datetime.utcnow().timetuple()
155 |             hour_start = datetime(*now[:4]).isoformat()
156 | 
157 |             existing = pipe.get(start_key)
158 |             pipe.multi()
159 |             if existing and existing < hour_start:
160 |                 pipe.rename(destination, destination + ':last')
161 |                 pipe.rename(start_key, destination + ':pstart')
162 |                 pipe.set(start_key, hour_start)
163 |             # add the following two lines
164 |             elif not existing:
165 |                 pipe.set(start_key, hour_start)
166 | 
167 |             pipe.zincrby(destination, message)
168 |             log_recent(pipe, name, message, severity, pipe)
169 |             return
170 |         except redis.exceptions.WatchError:
171 |             continue
172 |
173 | 174 |

You can see the change in-context by visiting: https://goo.gl/UN5kMw

175 | 176 | 177 |

Chapter 6

178 |

Section 6.2.3, Page 121, Listing 6.9 (2013/08/24, 2014/02/16, 2014/04/22)

179 |

There is one printing errata (wrong in the printed version, correct in the source code), three other bugs, and three cleanups in this listing for purchase_item_with_lock().

180 |

Printing errata: there is a missing line between the two lines that read:

181 |
182 | 
locked = acquire_lock(conn, market)
183 |     return False
184 |
185 | 186 |

With the missing line replaced, it should read:

187 |
188 | 
locked = acquire_lock(conn, market)
189 | if not locked:
190 |     return False
191 |
192 | 193 |

You can see the code in-context by visiting: http://goo.gl/QpcbuC

194 | 195 |

Bug: there is an extra pipe.watch(buyer) call that breaks the remaining behavior, which can be removed. The lines that read:

196 |
197 | 
try:
198 |     pipe.watch(buyer)
199 |     pipe.zscore("market:", item)
200 |     pipe.hget(buyer, 'funds')
201 |
202 | 203 |

Should instead read:

204 |
205 | 
try:
206 |     pipe.zscore("market:", item)
207 |     pipe.hget(buyer, 'funds')
208 |
209 | 210 |

Further, there are missing 'funds' arguments to the pipe.hincrby() calls later, *and* a misnamed argument. The lines that read:

211 |
212 | 
pipe.hincrby(seller, int(price))
213 | pipe.hincrby(buyerid, int(-price))
214 |
215 | 216 |

Should instead read:

217 |
218 | 
pipe.hincrby(seller, 'funds', int(price))
219 | pipe.hincrby(buyer, 'funds', int(-price))
220 |
221 | 222 |

Note the addition of the 'funds' arguments and the renaming of the buyerid argument to buyer.

223 |

You can see the change in-context by visiting: http://goo.gl/kQltLd

224 | 225 |

Bug: the calls to acquire_lock() and release_lock() reference the variable market when they should instead pass the string 'market:'. 226 | 227 |

There are also several additional (but unnecessary) cleanups that can be done to this listing. These changes include: 1) removing the while loop, 2) removing the try/except clause, 3) removing the unnecessary pipe.unwatch() call.

228 |

You can see the change for these cleanups and the fixed market -> 'market:' reference inline by visiting: http://goo.gl/LxGvV8

229 | 230 |
231 |

Section 6.5.2, Page 145, Listing 6.28 (2015/04/10)

232 |

There are two bugs on the last line of the leave_chat() function. The first bug is where the oldest argument to the conn.zremrangebyscore() call should really be oldest[0][1] (this was discovered on 2015-04-10). The second bug is what reads as 'chat:' should read 'msgs:' (this was discovered on 2015-12-14). The lines that read:

233 |
234 | 
        'chat:' + chat_id, 0, 0, withscores=True)
235 |         conn.zremrangebyscore('chat:' + chat_id, 0, oldest)
236 |
237 | 238 |

Should instead read:

239 |
240 | 
        'chat:' + chat_id, 0, 0, withscores=True)
241 |         conn.zremrangebyscore('msgs:' + chat_id, 0, oldest[0][1])
242 |
243 | 244 |

You can see the code in-context by visiting: https://goo.gl/T8zCp2

245 | 246 |

Chapter 7

247 |

Section 7.3.4, Page 177, Listing 7.15 (2014/04/22)

248 |

There is a printing errata and bug in the record_click() function.

249 |

Printing errata and bug: in the printed book, there is a missing else: line between the two pipeline.incr() calls below, and the first pipeline.incr() call is missing a '%s' for the string templating to work. So lines 13-15 in the book:

250 |
251 | 
    if action and type == 'cpa':
252 |         pipeline.incr('type:cpa:actions:' % type)
253 |         pipeline.incr('type:%s:clicks:' % type)
254 |
255 | 256 |

Should instead read:

257 |
258 | 
    if action and type == 'cpa':
259 |         pipeline.incr('type:%s:actions:' % type)
260 |     else:
261 |         pipeline.incr('type:%s:clicks:' % type)
262 |
263 | 264 |

You can see the change in-context with the else: clause by visiting: http://goo.gl/XXiNTD

265 | 266 | 267 |

Chapter 8

268 |

Section 8.1.1, Page 187, Listing 8.1 (2015/04/13)

269 |

There is a bug in the create_user() between lines 7 and 8, where there is a missing release_lock() call prior to return. In practice, this may cause a 1 second delay in deleting a status message for a user if someone were trying to create a new account with the same user name.

270 | 271 |

Lines 7-8 of create_user() originally read:

272 |
273 | 
    if conn.hget('users:', llogin):
274 |         return None
275 |
276 | 277 |

Lines 7-8 should be replaced with these 3 lines:

278 |
279 | 
    if conn.hget('users:', llogin):
280 |         release_lock(conn, 'user:' + llogin, lock)
281 |         return None
282 |
283 | 284 |

You can see the code in-context by visiting: http://goo.gl/quX7ee

285 | 286 |
287 |

Section 8.3, Page 190-191, Listing 8.4 (2015/04/13)

288 |

There is a race condition in the follow_user() function, caused by directly setting the size of the following/follower lists instead of incrementally changing them. This is fixed as part of other updates to follow_user() in section 10.3.3, listing 10.12, and we are just bringing a couple of these changes back to chapter 8.

289 | 290 |

Lines 12-21 originally read:

291 |
292 | 
    pipeline.zadd(fkey1, other_uid, now)
293 |     pipeline.zadd(fkey2, uid, now)
294 |     pipeline.zcard(fkey1)
295 |     pipeline.zcard(fkey2)
296 |     pipeline.zrevrange('profile:%s'%other_uid,
297 |         0, HOME_TIMELINE_SIZE-1, withscores=True)
298 |     following, followers, status_and_score = pipeline.execute()[-3:]
299 | 
300 |     pipeline.hset('user:%s'%uid, 'following', following)
301 |     pipeline.hset('user:%s'%other_uid, 'followers', followers)
302 |
303 | 304 |

With 2 lines deleted and 2 lines changed, lines 12-19 should now read:

305 |
306 | 
    pipeline.zadd(fkey1, other_uid, now)
307 |     pipeline.zadd(fkey2, uid, now)
308 |     pipeline.zrevrange('profile:%s'%other_uid,
309 |         0, HOME_TIMELINE_SIZE-1, withscores=True)
310 |     following, followers, status_and_score = pipeline.execute()[-3:]
311 | 
312 |     pipeline.hincrby('user:%s'%uid, 'following', int(following))
313 |     pipeline.hincrby('user:%s'%other_uid, 'followers', int(followers))
314 |
315 | 316 |

You can see the code in-context by visiting: http://goo.gl/mvmxwV

317 | 318 |
319 |

Section 8.3, Page 191-192, Listing 8.5 (2015/04/13)

320 |

There is a race condition in the unfollow_user() function, caused by directly setting the size of the following/follower lists instead of incrementally changing them. This is the exact same bug that occurs in the follow_user() function, and has the same fix.

321 | 322 |

Lines 12-21 originally read:

323 |
324 | 
    pipeline.zrem(fkey1, other_uid, now)
325 |     pipeline.zrem(fkey2, uid, now)
326 |     pipeline.zcard(fkey1)
327 |     pipeline.zcard(fkey2)
328 |     pipeline.zrevrange('profile:%s'%other_uid,
329 |         0, HOME_TIMELINE_SIZE-1, withscores=True)
330 |     following, followers, status_and_score = pipeline.execute()[-3:]
331 | 
332 |     pipeline.hset('user:%s'%uid, 'following', following)
333 |     pipeline.hset('user:%s'%other_uid, 'followers', followers)
334 |
335 | 336 |

With 2 lines deleted and 2 lines changed, lines 12-19 should now read:

337 |
338 | 
    pipeline.zrem(fkey1, other_uid, now)
339 |     pipeline.zrem(fkey2, uid, now)
340 |     pipeline.zrevrange('profile:%s'%other_uid,
341 |         0, HOME_TIMELINE_SIZE-1, withscores=True)
342 |     following, followers, status_and_score = pipeline.execute()[-3:]
343 | 
344 |     pipeline.hincrby('user:%s'%uid, 'following', int(following))
345 |     pipeline.hincrby('user:%s'%other_uid, 'followers', int(followers))
346 |
347 | 348 |

You can see the code in-context by visiting: http://goo.gl/mxcUqZ

349 | 350 |
351 |

Section 8.4, Page 194, Listing 8.8 (2015/04/13)

352 |

There is a bug in the delete_status() function between lines 7 and 8, where there is a missing release_lock() call prior to return. In practice, this may cause a 1 second delay in deleting a status message for a user if someone tried to delete a status message they didn't own.

353 | 354 |

Lines 7-8 of delete_status(conn, uid, status_id) originally read:

355 |
356 | 
    if conn.hget(key, 'uid') != str(uid):
357 |         return None
358 |
359 | 360 |

Lines 7-8 should be replaced with these 3 lines:

361 |
362 | 
    if conn.hget(key, 'uid') != str(uid):
363 |         release_lock(conn, key, lock)
364 |         return None
365 |
366 | 367 |

You can see the code in-context by visiting: http://goo.gl/gL6dPG

368 | 369 |
370 |

Section 8.5.3, Page 201-202, Listing 8.14 (2015/04/13)

371 | 372 |

There is a bug in the streaming delete_status() function between lines 7 and 8, where there is a missing release_lock() call prior to return. In practice, this may cause a 1 second delay in deleting a status message for a user if someone tried to delete a status message they didn't own.

373 | 374 |

Lines 7-8 of delete_status(conn, uid, status_id) originally read:

375 |
376 | 
    if conn.hget(key, 'uid') != str(uid):
377 |         return None
378 |
379 | 380 |

Lines 7-8 should be replaced with these 3 lines:

381 |
382 | 
    if conn.hget(key, 'uid') != str(uid):
383 |         release_lock(conn, key, lock)
384 |         return None
385 |
386 | 387 |

You can see the code in-context by visiting: http://goo.gl/DzweRD

388 | 389 |
390 |

Section 8.5.3, Page 205, Listing 8.19 (2014/07/30)

391 |

There is a bug in the FollowFilter() function on lines 2, 4, and 10 of the code listing, where the argument names is overridden by an empty set, which prevents the FollowFilter() call from actually following anyone.

392 | 393 |

The function originally read:

394 |
395 | 
def FollowFilter(names):
396 |     names = set()
397 |     for name in names:
398 |         names.add('@' + name.lower().lstrip('@'))
399 | 
400 |     def check(status):
401 |         message_words = set(status['message'].lower().split())
402 |         message_words.add('@' + status['login'].lower())
403 | 
404 |         return message_words & names
405 |     return check
406 |
407 | 408 |

The function (with comments here to show the changes) now reads:

409 |
410 | 
def FollowFilter(names):
411 |     nset = set() # first fix here
412 |     for name in names:
413 |         nset.add('@' + name.lower().lstrip('@')) # second fix here
414 | 
415 |     def check(status):
416 |         message_words = set(status['message'].lower().split())
417 |         message_words.add('@' + status['login'].lower())
418 | 
419 |         return message_words & nset # third fix here
420 |     return check
421 |
422 | 423 |

The diff (which might be easier to read) can be found: http://goo.gl/Opbp13

424 |

And the code in-context can be seen: http://goo.gl/GkqXp5

425 | 426 | 427 |

Chapter 11

428 |

Section 11.3.2, Page 260-261, Listing 11.12 (2015/04/22)

429 |

This code listing is a copy of the listing from Chapter 6, Section 6.2.3, Page 121, Listing 6.9, and has all of the same issues except for the printing errata and the market variable reference bug.

430 | 431 |
432 | 433 | 434 | -------------------------------------------------------------------------------- /java/src/main/java/Chapter05.java: -------------------------------------------------------------------------------- 1 | import com.google.gson.Gson; 2 | import com.google.gson.reflect.TypeToken; 3 | import org.apache.commons.csv.CSVParser; 4 | import org.javatuples.Pair; 5 | import redis.clients.jedis.*; 6 | 7 | import java.io.File; 8 | import java.io.FileReader; 9 | import java.text.Collator; 10 | import java.text.SimpleDateFormat; 11 | import java.util.*; 12 | 13 | public class Chapter05 { 14 | public static final String DEBUG = "debug"; 15 | public static final String INFO = "info"; 16 | public static final String WARNING = "warning"; 17 | public static final String ERROR = "error"; 18 | public static final String CRITICAL = "critical"; 19 | 20 | public static final Collator COLLATOR = Collator.getInstance(); 21 | 22 | public static final SimpleDateFormat TIMESTAMP = 23 | new SimpleDateFormat("EEE MMM dd HH:00:00 yyyy"); 24 | private static final SimpleDateFormat ISO_FORMAT = 25 | new SimpleDateFormat("yyyy-MM-dd'T'HH:00:00"); 26 | static{ 27 | ISO_FORMAT.setTimeZone(TimeZone.getTimeZone("UTC")); 28 | } 29 | 30 | public static final void main(String[] args) 31 | throws InterruptedException 32 | { 33 | new Chapter05().run(); 34 | } 35 | 36 | public void run() 37 | throws InterruptedException 38 | { 39 | Jedis conn = new Jedis("localhost"); 40 | conn.select(15); 41 | 42 | testLogRecent(conn); 43 | testLogCommon(conn); 44 | testCounters(conn); 45 | testStats(conn); 46 | testAccessTime(conn); 47 | testIpLookup(conn); 48 | testIsUnderMaintenance(conn); 49 | testConfig(conn); 50 | } 51 | 52 | public void testLogRecent(Jedis conn) { 53 | System.out.println("\n----- testLogRecent -----"); 54 | System.out.println("Let's write a few logs to the recent log"); 55 | for (int i = 0; i < 5; i++) { 56 | logRecent(conn, "test", "this is message " + i); 57 | } 58 | List recent = conn.lrange("recent:test:info", 0, -1); 59 | System.out.println( 60 | "The current recent message log has this many messages: " + 61 | recent.size()); 62 | System.out.println("Those messages include:"); 63 | for (String message : recent){ 64 | System.out.println(message); 65 | } 66 | assert recent.size() >= 5; 67 | } 68 | 69 | public void testLogCommon(Jedis conn) { 70 | System.out.println("\n----- testLogCommon -----"); 71 | System.out.println("Let's write some items to the common log"); 72 | for (int count = 1; count < 6; count++) { 73 | for (int i = 0; i < count; i ++) { 74 | logCommon(conn, "test", "message-" + count); 75 | } 76 | } 77 | Set common = conn.zrevrangeWithScores("common:test:info", 0, -1); 78 | System.out.println("The current number of common messages is: " + common.size()); 79 | System.out.println("Those common messages are:"); 80 | for (Tuple tuple : common){ 81 | System.out.println(" " + tuple.getElement() + ", " + tuple.getScore()); 82 | } 83 | assert common.size() >= 5; 84 | } 85 | 86 | public void testCounters(Jedis conn) 87 | throws InterruptedException 88 | { 89 | System.out.println("\n----- testCounters -----"); 90 | System.out.println("Let's update some counters for now and a little in the future"); 91 | long now = System.currentTimeMillis() / 1000; 92 | for (int i = 0; i < 10; i++) { 93 | int count = (int)(Math.random() * 5) + 1; 94 | updateCounter(conn, "test", count, now + i); 95 | } 96 | 97 | List> counter = getCounter(conn, "test", 1); 98 | System.out.println("We have some per-second counters: " + counter.size()); 99 | System.out.println("These counters include:"); 100 | for (Pair count : counter){ 101 | System.out.println(" " + count); 102 | } 103 | assert counter.size() >= 10; 104 | 105 | counter = getCounter(conn, "test", 5); 106 | System.out.println("We have some per-5-second counters: " + counter.size()); 107 | System.out.println("These counters include:"); 108 | for (Pair count : counter){ 109 | System.out.println(" " + count); 110 | } 111 | assert counter.size() >= 2; 112 | System.out.println(); 113 | 114 | System.out.println("Let's clean out some counters by setting our sample count to 0"); 115 | CleanCountersThread thread = new CleanCountersThread(0, 2 * 86400000); 116 | thread.start(); 117 | Thread.sleep(1000); 118 | thread.quit(); 119 | thread.interrupt(); 120 | counter = getCounter(conn, "test", 86400); 121 | System.out.println("Did we clean out all of the counters? " + (counter.size() == 0)); 122 | assert counter.size() == 0; 123 | } 124 | 125 | public void testStats(Jedis conn) { 126 | System.out.println("\n----- testStats -----"); 127 | System.out.println("Let's add some data for our statistics!"); 128 | List r = null; 129 | for (int i = 0; i < 5; i++){ 130 | double value = (Math.random() * 11) + 5; 131 | r = updateStats(conn, "temp", "example", value); 132 | } 133 | System.out.println("We have some aggregate statistics: " + r); 134 | Map stats = getStats(conn, "temp", "example"); 135 | System.out.println("Which we can also fetch manually:"); 136 | System.out.println(stats); 137 | assert stats.get("count") >= 5; 138 | } 139 | 140 | public void testAccessTime(Jedis conn) 141 | throws InterruptedException 142 | { 143 | System.out.println("\n----- testAccessTime -----"); 144 | System.out.println("Let's calculate some access times..."); 145 | AccessTimer timer = new AccessTimer(conn); 146 | for (int i = 0; i < 10; i++){ 147 | timer.start(); 148 | Thread.sleep((int)((.5 + Math.random()) * 1000)); 149 | timer.stop("req-" + i); 150 | } 151 | System.out.println("The slowest access times are:"); 152 | Set atimes = conn.zrevrangeWithScores("slowest:AccessTime", 0, -1); 153 | for (Tuple tuple : atimes){ 154 | System.out.println(" " + tuple.getElement() + ", " + tuple.getScore()); 155 | } 156 | assert atimes.size() >= 10; 157 | System.out.println(); 158 | } 159 | 160 | public void testIpLookup(Jedis conn) { 161 | System.out.println("\n----- testIpLookup -----"); 162 | String cwd = System.getProperty("user.dir"); 163 | File blocks = new File(cwd + "/GeoLiteCity-Blocks.csv"); 164 | File locations = new File(cwd + "/GeoLiteCity-Location.csv"); 165 | if (!blocks.exists()){ 166 | System.out.println("********"); 167 | System.out.println("GeoLiteCity-Blocks.csv not found at: " + blocks); 168 | System.out.println("********"); 169 | return; 170 | } 171 | if (!locations.exists()){ 172 | System.out.println("********"); 173 | System.out.println("GeoLiteCity-Location.csv not found at: " + locations); 174 | System.out.println("********"); 175 | return; 176 | } 177 | 178 | System.out.println("Importing IP addresses to Redis... (this may take a while)"); 179 | importIpsToRedis(conn, blocks); 180 | long ranges = conn.zcard("ip2cityid:"); 181 | System.out.println("Loaded ranges into Redis: " + ranges); 182 | assert ranges > 1000; 183 | System.out.println(); 184 | 185 | System.out.println("Importing Location lookups to Redis... (this may take a while)"); 186 | importCitiesToRedis(conn, locations); 187 | long cities = conn.hlen("cityid2city:"); 188 | System.out.println("Loaded city lookups into Redis:" + cities); 189 | assert cities > 1000; 190 | System.out.println(); 191 | 192 | System.out.println("Let's lookup some locations!"); 193 | for (int i = 0; i < 5; i++){ 194 | String ip = 195 | randomOctet(255) + '.' + 196 | randomOctet(256) + '.' + 197 | randomOctet(256) + '.' + 198 | randomOctet(256); 199 | System.out.println(Arrays.toString(findCityByIp(conn, ip))); 200 | } 201 | } 202 | 203 | public void testIsUnderMaintenance(Jedis conn) 204 | throws InterruptedException 205 | { 206 | System.out.println("\n----- testIsUnderMaintenance -----"); 207 | System.out.println("Are we under maintenance (we shouldn't be)? " + isUnderMaintenance(conn)); 208 | conn.set("is-under-maintenance", "yes"); 209 | System.out.println("We cached this, so it should be the same: " + isUnderMaintenance(conn)); 210 | Thread.sleep(1000); 211 | System.out.println("But after a sleep, it should change: " + isUnderMaintenance(conn)); 212 | System.out.println("Cleaning up..."); 213 | conn.del("is-under-maintenance"); 214 | Thread.sleep(1000); 215 | System.out.println("Should be False again: " + isUnderMaintenance(conn)); 216 | } 217 | 218 | public void testConfig(Jedis conn) { 219 | System.out.println("\n----- testConfig -----"); 220 | System.out.println("Let's set a config and then get a connection from that config..."); 221 | Map config = new HashMap(); 222 | config.put("db", 15); 223 | setConfig(conn, "redis", "test", config); 224 | 225 | Jedis conn2 = redisConnection("test"); 226 | System.out.println( 227 | "We can run commands from the configured connection: " + (conn2.info() != null)); 228 | } 229 | 230 | public void logRecent(Jedis conn, String name, String message) { 231 | logRecent(conn, name, message, INFO); 232 | } 233 | 234 | public void logRecent(Jedis conn, String name, String message, String severity) { 235 | String destination = "recent:" + name + ':' + severity; 236 | Pipeline pipe = conn.pipelined(); 237 | pipe.lpush(destination, TIMESTAMP.format(new Date()) + ' ' + message); 238 | pipe.ltrim(destination, 0, 99); 239 | pipe.sync(); 240 | } 241 | 242 | public void logCommon(Jedis conn, String name, String message) { 243 | logCommon(conn, name, message, INFO, 5000); 244 | } 245 | 246 | public void logCommon( 247 | Jedis conn, String name, String message, String severity, int timeout) { 248 | String commonDest = "common:" + name + ':' + severity; 249 | String startKey = commonDest + ":start"; 250 | long end = System.currentTimeMillis() + timeout; 251 | while (System.currentTimeMillis() < end){ 252 | conn.watch(startKey); 253 | String hourStart = ISO_FORMAT.format(new Date()); 254 | String existing = conn.get(startKey); 255 | 256 | Transaction trans = conn.multi(); 257 | if (existing != null && COLLATOR.compare(existing, hourStart) < 0){ 258 | trans.rename(commonDest, commonDest + ":last"); 259 | trans.rename(startKey, commonDest + ":pstart"); 260 | trans.set(startKey, hourStart); 261 | } 262 | 263 | trans.zincrby(commonDest, 1, message); 264 | 265 | String recentDest = "recent:" + name + ':' + severity; 266 | trans.lpush(recentDest, TIMESTAMP.format(new Date()) + ' ' + message); 267 | trans.ltrim(recentDest, 0, 99); 268 | List results = trans.exec(); 269 | // null response indicates that the transaction was aborted due to 270 | // the watched key changing. 271 | if (results == null){ 272 | continue; 273 | } 274 | return; 275 | } 276 | } 277 | 278 | public void updateCounter(Jedis conn, String name, int count) { 279 | updateCounter(conn, name, count, System.currentTimeMillis() / 1000); 280 | } 281 | 282 | public static final int[] PRECISION = new int[]{1, 5, 60, 300, 3600, 18000, 86400}; 283 | public void updateCounter(Jedis conn, String name, int count, long now){ 284 | Transaction trans = conn.multi(); 285 | for (int prec : PRECISION) { 286 | long pnow = (now / prec) * prec; 287 | String hash = String.valueOf(prec) + ':' + name; 288 | trans.zadd("known:", 0, hash); 289 | trans.hincrBy("count:" + hash, String.valueOf(pnow), count); 290 | } 291 | trans.exec(); 292 | } 293 | 294 | public List> getCounter( 295 | Jedis conn, String name, int precision) 296 | { 297 | String hash = String.valueOf(precision) + ':' + name; 298 | Map data = conn.hgetAll("count:" + hash); 299 | ArrayList> results = 300 | new ArrayList>(); 301 | for (Map.Entry entry : data.entrySet()) { 302 | results.add(new Pair( 303 | Integer.parseInt(entry.getKey()), 304 | Integer.parseInt(entry.getValue()))); 305 | } 306 | Collections.sort(results); 307 | return results; 308 | } 309 | 310 | public List updateStats(Jedis conn, String context, String type, double value){ 311 | int timeout = 5000; 312 | String destination = "stats:" + context + ':' + type; 313 | String startKey = destination + ":start"; 314 | long end = System.currentTimeMillis() + timeout; 315 | while (System.currentTimeMillis() < end){ 316 | conn.watch(startKey); 317 | String hourStart = ISO_FORMAT.format(new Date()); 318 | 319 | String existing = conn.get(startKey); 320 | Transaction trans = conn.multi(); 321 | if (existing != null && COLLATOR.compare(existing, hourStart) < 0){ 322 | trans.rename(destination, destination + ":last"); 323 | trans.rename(startKey, destination + ":pstart"); 324 | trans.set(startKey, hourStart); 325 | } 326 | 327 | String tkey1 = UUID.randomUUID().toString(); 328 | String tkey2 = UUID.randomUUID().toString(); 329 | trans.zadd(tkey1, value, "min"); 330 | trans.zadd(tkey2, value, "max"); 331 | 332 | trans.zunionstore( 333 | destination, 334 | new ZParams().aggregate(ZParams.Aggregate.MIN), 335 | destination, tkey1); 336 | trans.zunionstore( 337 | destination, 338 | new ZParams().aggregate(ZParams.Aggregate.MAX), 339 | destination, tkey2); 340 | 341 | trans.del(tkey1, tkey2); 342 | trans.zincrby(destination, 1, "count"); 343 | trans.zincrby(destination, value, "sum"); 344 | trans.zincrby(destination, value * value, "sumsq"); 345 | 346 | List results = trans.exec(); 347 | if (results == null){ 348 | continue; 349 | } 350 | return results.subList(results.size() - 3, results.size()); 351 | } 352 | return null; 353 | } 354 | 355 | public Map getStats(Jedis conn, String context, String type){ 356 | String key = "stats:" + context + ':' + type; 357 | Map stats = new HashMap(); 358 | Set data = conn.zrangeWithScores(key, 0, -1); 359 | for (Tuple tuple : data){ 360 | stats.put(tuple.getElement(), tuple.getScore()); 361 | } 362 | stats.put("average", stats.get("sum") / stats.get("count")); 363 | double numerator = stats.get("sumsq") - Math.pow(stats.get("sum"), 2) / stats.get("count"); 364 | double count = stats.get("count"); 365 | stats.put("stddev", Math.pow(numerator / (count > 1 ? count - 1 : 1), .5)); 366 | return stats; 367 | } 368 | 369 | private long lastChecked; 370 | private boolean underMaintenance; 371 | public boolean isUnderMaintenance(Jedis conn) { 372 | if (lastChecked < System.currentTimeMillis() - 1000){ 373 | lastChecked = System.currentTimeMillis(); 374 | String flag = conn.get("is-under-maintenance"); 375 | underMaintenance = "yes".equals(flag); 376 | } 377 | 378 | return underMaintenance; 379 | } 380 | 381 | public void setConfig( 382 | Jedis conn, String type, String component, Map config) { 383 | Gson gson = new Gson(); 384 | conn.set("config:" + type + ':' + component, gson.toJson(config)); 385 | } 386 | 387 | private static final Map> CONFIGS = 388 | new HashMap>(); 389 | private static final Map CHECKED = new HashMap(); 390 | 391 | @SuppressWarnings("unchecked") 392 | public Map getConfig(Jedis conn, String type, String component) { 393 | int wait = 1000; 394 | String key = "config:" + type + ':' + component; 395 | 396 | Long lastChecked = CHECKED.get(key); 397 | if (lastChecked == null || lastChecked < System.currentTimeMillis() - wait){ 398 | CHECKED.put(key, System.currentTimeMillis()); 399 | 400 | String value = conn.get(key); 401 | Map config = null; 402 | if (value != null){ 403 | Gson gson = new Gson(); 404 | config = (Map)gson.fromJson( 405 | value, new TypeToken>(){}.getType()); 406 | }else{ 407 | config = new HashMap(); 408 | } 409 | 410 | CONFIGS.put(key, config); 411 | } 412 | 413 | return CONFIGS.get(key); 414 | } 415 | 416 | public static final Map REDIS_CONNECTIONS = 417 | new HashMap(); 418 | public Jedis redisConnection(String component){ 419 | Jedis configConn = REDIS_CONNECTIONS.get("config"); 420 | if (configConn == null){ 421 | configConn = new Jedis("localhost"); 422 | configConn.select(15); 423 | REDIS_CONNECTIONS.put("config", configConn); 424 | } 425 | 426 | String key = "config:redis:" + component; 427 | Map oldConfig = CONFIGS.get(key); 428 | Map config = getConfig(configConn, "redis", component); 429 | 430 | if (!config.equals(oldConfig)){ 431 | Jedis conn = new Jedis("localhost"); 432 | if (config.containsKey("db")){ 433 | conn.select(((Double)config.get("db")).intValue()); 434 | } 435 | REDIS_CONNECTIONS.put(key, conn); 436 | } 437 | 438 | return REDIS_CONNECTIONS.get(key); 439 | } 440 | 441 | public void importIpsToRedis(Jedis conn, File file) { 442 | FileReader reader = null; 443 | try{ 444 | reader = new FileReader(file); 445 | CSVParser parser = new CSVParser(reader); 446 | int count = 0; 447 | String[] line = null; 448 | while ((line = parser.getLine()) != null){ 449 | String startIp = line.length > 1 ? line[0] : ""; 450 | if (startIp.toLowerCase().indexOf('i') != -1){ 451 | continue; 452 | } 453 | int score = 0; 454 | if (startIp.indexOf('.') != -1){ 455 | score = ipToScore(startIp); 456 | }else{ 457 | try{ 458 | score = Integer.parseInt(startIp, 10); 459 | }catch(NumberFormatException nfe){ 460 | continue; 461 | } 462 | } 463 | 464 | String cityId = line[2] + '_' + count; 465 | conn.zadd("ip2cityid:", score, cityId); 466 | count++; 467 | } 468 | }catch(Exception e){ 469 | throw new RuntimeException(e); 470 | }finally{ 471 | try{ 472 | reader.close(); 473 | }catch(Exception e){ 474 | // ignore 475 | } 476 | } 477 | } 478 | 479 | public void importCitiesToRedis(Jedis conn, File file) { 480 | Gson gson = new Gson(); 481 | FileReader reader = null; 482 | try{ 483 | reader = new FileReader(file); 484 | CSVParser parser = new CSVParser(reader); 485 | String[] line = null; 486 | while ((line = parser.getLine()) != null){ 487 | if (line.length < 4 || !Character.isDigit(line[0].charAt(0))){ 488 | continue; 489 | } 490 | String cityId = line[0]; 491 | String country = line[1]; 492 | String region = line[2]; 493 | String city = line[3]; 494 | String json = gson.toJson(new String[]{city, region, country}); 495 | conn.hset("cityid2city:", cityId, json); 496 | } 497 | }catch(Exception e){ 498 | throw new RuntimeException(e); 499 | }finally{ 500 | try{ 501 | reader.close(); 502 | }catch(Exception e){ 503 | // ignore 504 | } 505 | } 506 | } 507 | 508 | public int ipToScore(String ipAddress) { 509 | int score = 0; 510 | for (String v : ipAddress.split("\\.")){ 511 | score = score * 256 + Integer.parseInt(v, 10); 512 | } 513 | return score; 514 | } 515 | 516 | public String randomOctet(int max) { 517 | return String.valueOf((int)(Math.random() * max)); 518 | } 519 | 520 | public String[] findCityByIp(Jedis conn, String ipAddress) { 521 | int score = ipToScore(ipAddress); 522 | Set results = conn.zrevrangeByScore("ip2cityid:", score, 0, 0, 1); 523 | if (results.size() == 0) { 524 | return null; 525 | } 526 | 527 | String cityId = results.iterator().next(); 528 | cityId = cityId.substring(0, cityId.indexOf('_')); 529 | return new Gson().fromJson(conn.hget("cityid2city:", cityId), String[].class); 530 | } 531 | 532 | public class CleanCountersThread 533 | extends Thread 534 | { 535 | private Jedis conn; 536 | private int sampleCount = 100; 537 | private boolean quit; 538 | private long timeOffset; // used to mimic a time in the future. 539 | 540 | public CleanCountersThread(int sampleCount, long timeOffset){ 541 | this.conn = new Jedis("localhost"); 542 | this.conn.select(15); 543 | this.sampleCount = sampleCount; 544 | this.timeOffset = timeOffset; 545 | } 546 | 547 | public void quit(){ 548 | quit = true; 549 | } 550 | 551 | public void run(){ 552 | int passes = 0; 553 | while (!quit){ 554 | long start = System.currentTimeMillis() + timeOffset; 555 | int index = 0; 556 | while (index < conn.zcard("known:")){ 557 | Set hashSet = conn.zrange("known:", index, index); 558 | index++; 559 | if (hashSet.size() == 0) { 560 | break; 561 | } 562 | String hash = hashSet.iterator().next(); 563 | int prec = Integer.parseInt(hash.substring(0, hash.indexOf(':'))); 564 | int bprec = (int)Math.floor(prec / 60); 565 | if (bprec == 0){ 566 | bprec = 1; 567 | } 568 | if ((passes % bprec) != 0){ 569 | continue; 570 | } 571 | 572 | String hkey = "count:" + hash; 573 | String cutoff = String.valueOf( 574 | ((System.currentTimeMillis() + timeOffset) / 1000) - sampleCount * prec); 575 | ArrayList samples = new ArrayList(conn.hkeys(hkey)); 576 | Collections.sort(samples); 577 | int remove = bisectRight(samples, cutoff); 578 | 579 | if (remove != 0){ 580 | conn.hdel(hkey, samples.subList(0, remove).toArray(new String[0])); 581 | if (remove == samples.size()){ 582 | conn.watch(hkey); 583 | if (conn.hlen(hkey) == 0) { 584 | Transaction trans = conn.multi(); 585 | trans.zrem("known:", hash); 586 | trans.exec(); 587 | index--; 588 | }else{ 589 | conn.unwatch(); 590 | } 591 | } 592 | } 593 | } 594 | 595 | passes++; 596 | long duration = Math.min( 597 | (System.currentTimeMillis() + timeOffset) - start + 1000, 60000); 598 | try { 599 | sleep(Math.max(60000 - duration, 1000)); 600 | }catch(InterruptedException ie){ 601 | Thread.currentThread().interrupt(); 602 | } 603 | } 604 | } 605 | 606 | // mimic python's bisect.bisect_right 607 | public int bisectRight(List values, String key) { 608 | int index = Collections.binarySearch(values, key); 609 | return index < 0 ? Math.abs(index) - 1 : index + 1; 610 | } 611 | } 612 | 613 | public class AccessTimer { 614 | private Jedis conn; 615 | private long start; 616 | 617 | public AccessTimer(Jedis conn){ 618 | this.conn = conn; 619 | } 620 | 621 | public void start(){ 622 | start = System.currentTimeMillis(); 623 | } 624 | 625 | public void stop(String context){ 626 | long delta = System.currentTimeMillis() - start; 627 | List stats = updateStats(conn, context, "AccessTime", delta / 1000.0); 628 | double average = (Double)stats.get(1) / (Double)stats.get(0); 629 | 630 | Transaction trans = conn.multi(); 631 | trans.zadd("slowest:AccessTime", average, context); 632 | trans.zremrangeByRank("slowest:AccessTime", 0, -101); 633 | trans.exec(); 634 | } 635 | } 636 | } 637 | --------------------------------------------------------------------------------