├── .gitignore
├── Procfile
├── Gemfile
├── config.ru
├── Gemfile.lock
├── app.json
├── COPYING
├── setup-oauth.rb
├── oauth.rb
├── app.rb
├── README.txt
└── service.rb


/.gitignore:
--------------------------------------------------------------------------------
1 | /.bundle
2 | 


--------------------------------------------------------------------------------
/Procfile:
--------------------------------------------------------------------------------
1 | web: bundle exec puma -e production -p $PORT
2 | 


--------------------------------------------------------------------------------
/Gemfile:
--------------------------------------------------------------------------------
1 | source "https://rubygems.org"
2 | 
3 | gem "simple_oauth", "~> 0.3.1"
4 | gem "sinatra", "~> 2.0"
5 | gem "puma"
6 | 


--------------------------------------------------------------------------------
/config.ru:
--------------------------------------------------------------------------------
 1 | # Start home_timeline polling
 2 | require_relative "service"
 3 | require_relative "app"
 4 | 
 5 | # HACK: The web app must be already started and accept "GET /webhook" when
 6 | # Service.setup is called
 7 | Thread.start {
 8 |   sleep 1
 9 |   begin
10 |     Net::HTTP.get_response(URI(ENV["TWITTER_EVENT_STREAM_BASE_URL"]))
11 |   rescue
12 |   end
13 |   Service.setup
14 | }
15 | 
16 | # Start web app
17 | use Rack::Deflater
18 | run App
19 | 


--------------------------------------------------------------------------------
/Gemfile.lock:
--------------------------------------------------------------------------------
 1 | GEM
 2 |   remote: https://rubygems.org/
 3 |   specs:
 4 |     mustermann (1.0.3)
 5 |     puma (3.12.0)
 6 |     rack (2.0.5)
 7 |     rack-protection (2.0.3)
 8 |       rack
 9 |     simple_oauth (0.3.1)
10 |     sinatra (2.0.3)
11 |       mustermann (~> 1.0)
12 |       rack (~> 2.0)
13 |       rack-protection (= 2.0.3)
14 |       tilt (~> 2.0)
15 |     tilt (2.0.8)
16 | 
17 | PLATFORMS
18 |   ruby
19 | 
20 | DEPENDENCIES
21 |   puma
22 |   simple_oauth (~> 0.3.1)
23 |   sinatra (~> 2.0)
24 | 
25 | BUNDLED WITH
26 |    1.16.3
27 | 


--------------------------------------------------------------------------------
/app.json:
--------------------------------------------------------------------------------
 1 | {
 2 |   "name": "twitter-event-stream",
 3 |   "description": "rhenium/twitter-event-stream",
 4 |   "website": "https://github.com/rhenium/twitter-event-stream",
 5 |   "repository": "https://github.com/rhenium/twitter-event-stream",
 6 |   "env": {
 7 |     "RACK_ENV": "production",
 8 |     "TWITTER_EVENT_STREAM_BASE_URL": {
 9 |       "description": "The URL where twitter-event-stream is deployed.",
10 |       "value": "https://<app name>.herokuapp.com/"
11 |     },
12 |     "TWITTER_EVENT_STREAM_ENV_NAME": {
13 |       "description": "The \"dev environment\" for the Account Activity API.",
14 |       "value": "<env name>"
15 |     },
16 |     "TWITTER_EVENT_STREAM_CONSUMER_KEY": {
17 |       "description": "A consumer key whitelisted for the Account Activity API.",
18 |       "value": "<dummy>"
19 |     },
20 |     "TWITTER_EVENT_STREAM_CONSUMER_SECRET": {
21 |       "description": "A consumer secret.",
22 |       "value": "<dummy>"
23 |     },
24 |     "TWITTER_EVENT_STREAM_USER_1": {
25 |       "description": "A JSON object containing credentials for an user '1'.",
26 |       "required": false
27 |     }
28 |   }
29 | }
30 | 


--------------------------------------------------------------------------------
/COPYING:
--------------------------------------------------------------------------------
 1 | Copyright (c) 2018 Kazuki Yamaguchi <k@rhe.jp>
 2 | 
 3 | Permission is hereby granted, free of charge, to any person obtaining a copy
 4 | of this software and associated documentation files (the "Software"), to deal
 5 | in the Software without restriction, including without limitation the rights
 6 | to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
 7 | copies of the Software, and to permit persons to whom the Software is
 8 | furnished to do so, subject to the following conditions:
 9 | 
10 | The above copyright notice and this permission notice shall be included in all
11 | copies or substantial portions of the Software.
12 | 
13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
19 | SOFTWARE.
20 | 


--------------------------------------------------------------------------------
/setup-oauth.rb:
--------------------------------------------------------------------------------
 1 | require "json"
 2 | require_relative "oauth"
 3 | 
 4 | if ARGV.size != 2
 5 |   STDERR.puts "Usage: ruby setup-oauth.rb <consumer key> <consumer secret>"
 6 |   exit 1
 7 | end
 8 | 
 9 | oauth_opts = { consumer_key: ARGV[0], consumer_secret: ARGV[1] }
10 | 
11 | puts "#POST /oauth/request_token"
12 | authorize_params = OAuthHelpers.user_post(oauth_opts, "/oauth/request_token",
13 |                                           { "oauth_callback" => "oob" })
14 | extracted = authorize_params.split("&").map { |v| v.split("=") }.to_h
15 | oauth_opts[:token] = extracted["oauth_token"]
16 | oauth_opts[:token_secret] = extracted["oauth_token_secret"]
17 | puts "#=> #{authorize_params}"
18 | puts
19 | 
20 | puts "Visit https://api.twitter.com/oauth/authorize?oauth_token=" \
21 |   "#{extracted["oauth_token"]}"
22 | print "Input PIN code: "
23 | pin = STDIN.gets.chomp
24 | puts
25 | 
26 | puts "#POST /oauth/access_token"
27 | oauth_params = OAuthHelpers.user_post(oauth_opts, "/oauth/access_token",
28 |                                       { "oauth_verifier" => pin })
29 | puts "#=> #{oauth_params}"
30 | puts
31 | 
32 | extracted = oauth_params.split("&").map { |v| v.split("=") }.to_h
33 | user_id = extracted["oauth_token"].split("-")[0].to_i
34 | obj = {
35 |   user_id: user_id,
36 |   requests_per_window: 15,
37 |   token: extracted["oauth_token"],
38 |   token_secret: extracted["oauth_token_secret"],
39 |   rest_consumer_key: oauth_opts[:consumer_key],
40 |   rest_consumer_secret: oauth_opts[:consumer_secret],
41 |   rest_token: extracted["oauth_token"],
42 |   rest_token_secret: extracted["oauth_token_secret"],
43 | }
44 | puts "TWITTER_EVENT_STREAM_USER_#{user_id}='#{JSON.generate(obj)}'"
45 | 


--------------------------------------------------------------------------------
/oauth.rb:
--------------------------------------------------------------------------------
 1 | require "json"
 2 | require "net/http"
 3 | require "simple_oauth"
 4 | 
 5 | module OAuthHelpers
 6 |   class HTTPRequestError < StandardError
 7 |     attr_reader :res
 8 | 
 9 |     def initialize(uri, res)
10 |       super("HTTP request failed: path=#{uri.request_uri} code=#{res.code} " \
11 |             "body=#{res.body}")
12 |       @res = res
13 |     end
14 |   end
15 | 
16 |   module_function
17 | 
18 |   private def http_req_connect(uri_string)
19 |     uri = URI.parse(uri_string)
20 |     Net::HTTP.start(uri.host, uri.port, use_ssl: true) { |http|
21 |       res = yield(http, uri.request_uri)
22 |       raise HTTPRequestError.new(uri, res) if res.code !~ /\A2\d\d\z/
23 |       res.body
24 |     }
25 |   end
26 | 
27 |   def http_get(auth, uri_string, method: :get)
28 |     http_req_connect(uri_string) { |http, path|
29 |       http.send(method, path, { "Authorization" => auth })
30 |     }
31 |   end
32 | 
33 |   def http_post(auth, uri_string, body)
34 |     http_req_connect(uri_string) { |http, path|
35 |       http.post(path, body, { "Authorization" => auth })
36 |     }
37 |   end
38 | 
39 |   def bearer_request_token(oauth)
40 |     ck, cs = oauth[:consumer_key], oauth[:consumer_secret]
41 |     body = http_post("Basic #{["#{ck}:#{cs}"].pack("m0")}",
42 |                      "https://api.twitter.com/oauth2/token",
43 |                      "grant_type=client_credentials")
44 |     hash = JSON.parse(body, symbolize_names: true)
45 |     hash[:access_token]
46 |   end
47 | 
48 |   def bearer_get(token, path)
49 |     http_get("Bearer #{token}", "https://api.twitter.com#{path}")
50 |   end
51 | 
52 |   def user_get(oauth, path, params = {})
53 |     path += "?" + params.map { |k, v| "#{k}=#{v}" }.join("&") if !params.empty?
54 |     uri_string = "https://api.twitter.com#{path}"
55 |     auth = SimpleOAuth::Header.new(:get, uri_string, {}, oauth).to_s
56 |     http_get(auth, uri_string)
57 |   end
58 | 
59 |   def user_delete(oauth, path, params = {})
60 |     path += "?" + params.map { |k, v| "#{k}=#{v}" }.join("&") if !params.empty?
61 |     uri_string = "https://api.twitter.com#{path}"
62 |     auth = SimpleOAuth::Header.new(:delete, uri_string, {}, oauth).to_s
63 |     http_get(auth, uri_string, method: :delete)
64 |   end
65 | 
66 |   def user_post(oauth, path, params = {})
67 |     body = params.map { |k, v| "#{k}=#{v}" }.join("&")
68 |     uri_string = "https://api.twitter.com#{path}"
69 |     auth = SimpleOAuth::Header.new(:post, uri_string, params, oauth).to_s
70 |     http_post(auth, uri_string, body)
71 |   end
72 | end
73 | 


--------------------------------------------------------------------------------
/app.rb:
--------------------------------------------------------------------------------
  1 | require "sinatra/base"
  2 | require "json"
  3 | require_relative "service"
  4 | 
  5 | class App < Sinatra::Base
  6 |   enable :logging
  7 |   set :consumer_key, ENV["TWITTER_EVENT_STREAM_CONSUMER_KEY"]
  8 |   set :consumer_secret, ENV["TWITTER_EVENT_STREAM_CONSUMER_SECRET"]
  9 | 
 10 |   helpers do
 11 |     def get_service
 12 |       asp = request.env["HTTP_X_AUTH_SERVICE_PROVIDER"]
 13 |       vca = request.env["HTTP_X_VERIFY_CREDENTIALS_AUTHORIZATION"]
 14 |       Service.oauth_echo(asp, vca)
 15 |     rescue ServiceError => e
 16 |       halt 403, "authentication failed"
 17 |     end
 18 |   end
 19 | 
 20 |   get "/stream" do
 21 |     content_type "text/event-stream"
 22 |     service = get_service
 23 |     logger.debug("/stream (#{service.user_id}): CONNECT!")
 24 | 
 25 |     # Heroku will kill the connection after 55 seconds of inactivity.
 26 |     # https://devcenter.heroku.com/articles/request-timeout#long-polling-and-streaming-responses
 27 |     queue = Thread::Queue.new
 28 |     th = Thread.start { sleep 15; loop { queue << ":\r\n\r\n"; sleep 30 } }
 29 |     tag = service.subscribe(params["count"].to_i) { |event, data|
 30 |       queue << "event: #{event}\r\ndata: #{JSON.generate(data)}\r\n\r\n"
 31 |     }
 32 | 
 33 |     stream(true) do |out|
 34 |       out.callback {
 35 |         logger.debug("/stream (#{service.user_id}): CLEANUP!")
 36 |         queue.close
 37 |         service.unsubscribe(tag)
 38 |         th.kill; th.join
 39 |       }
 40 |       loop { out << queue.pop }
 41 |     end
 42 |   end
 43 | 
 44 |   get "/1.1/user.json" do
 45 |     content_type :json
 46 |     service = get_service
 47 |     logger.debug("/1.1/user.json (#{service.user_id}): CONNECT!")
 48 | 
 49 |     friend_ids = service.twitter_get("/1.1/friends/ids.json",
 50 |                                      { "user_id" => service.user_id })
 51 | 
 52 |     queue = Thread::Queue.new
 53 |     queue << "#{JSON.generate({ "friends" => friend_ids["ids"] })}\r\n"
 54 | 
 55 |     th = Thread.start { sleep 15; loop { queue << "\r\n"; sleep 30 } }
 56 |     tag = service.subscribe(params["count"].to_i) { |event, data|
 57 |       case event
 58 |       when "twitter_event_stream_home_timeline"
 59 |         queue << data.map { |object| JSON.generate(object) }.join("\r\n")
 60 |       when "twitter_event_stream_message"
 61 |       when "tweet_create_events"
 62 |         queue << data.map { |object| JSON.generate(object) }.join("\r\n")
 63 |       when "favorite_events"
 64 |         queue << data.map { |object|
 65 |           JSON.generate({
 66 |             "event" => "favorite",
 67 |             "created_at" => object["created_at"],
 68 |             "source" => object["user"],
 69 |             "target" => object["favorited_status"]["user"],
 70 |             "target_object" => object["favorited_status"],
 71 |           })
 72 |         }.join("\r\n")
 73 |       when "follow_events", "block_events"
 74 |         queue << data.map { |object|
 75 |           JSON.generate({
 76 |             "event" => object["type"],
 77 |             "created_at" => Time.utc(Integer(object["created_timestamp"]))
 78 |               .strftime("%a %b %d %T %z %Y"),
 79 |             "source" => object["user"],
 80 |             "target" => object["favorited_status"]["user"],
 81 |             "target_object" => object["favorited_status"],
 82 |           })
 83 |         }.join("\r\n")
 84 |       when "mute_events"
 85 |         # Not supported
 86 |       when "direct_message_events", "direct_message_indicate_typing_events",
 87 |         "direct_message_mark_read_events"
 88 |         # Not supported
 89 |       when "tweet_delete_events"
 90 |         queue << data.map { |object|
 91 |           JSON.generate({
 92 |             "delete" => object
 93 |           })
 94 |         }.join("\r\n")
 95 |       else
 96 |         logger.info("/1.1/user.json (#{service.user_id}): " \
 97 |                     "unknown event: #{event}")
 98 |       end
 99 |     }
100 | 
101 |     stream(true) do |out|
102 |       out.callback {
103 |         logger.debug("/1.1/user.json (#{service.user_id}): CLEANUP!")
104 |         queue.close
105 |         service.unsubscribe(tag)
106 |         th.kill; th.join
107 |       }
108 |       loop { out << queue.pop }
109 |     end
110 |   end
111 | 
112 |   get "/webhook" do
113 |     content_type :json
114 |     crc_token = params["crc_token"] or
115 |       halt 400, "crc_token missing"
116 |     mac = OpenSSL::HMAC.digest("sha256", settings.consumer_secret, crc_token)
117 |     response_token = "sha256=#{[mac].pack("m0")}"
118 |     JSON.generate({ "response_token" => response_token })
119 |   end
120 | 
121 |   post "/webhook" do
122 |     content_type :json
123 |     body = request.body.read
124 |     mac = OpenSSL::HMAC.digest("sha256", settings.consumer_secret, body)
125 |     sig = "sha256=#{[mac].pack("m0")}"
126 |     if request.env["HTTP_X_TWITTER_WEBHOOKS_SIGNATURE"] == sig
127 |       Service.feed_webhook(body)
128 |     else
129 |       logger.info "x-twitter-webhooks-signature invalid"
130 |     end
131 |     JSON.generate({ "looks" => "ok" })
132 |   end
133 | 
134 |   get "/" do
135 |     <<~'EOF'
136 |       <!DOCTYPE html>
137 |       <meta charset=UTF-8>
138 |       <meta name=viewport content="width=device-width,initial-scale=1">
139 |       <title>twitter-event-stream</title>
140 |       <style>
141 |         div { max-width: 1200px; margin: 0 auto; }
142 |       </style>
143 |       <div>
144 |         <h1>twitter-event-stream</h1>
145 |         <a href="https://github.com/rhenium/twitter-event-stream">Source Code</a>
146 |       </div>
147 |     EOF
148 |   end
149 | end
150 | 


--------------------------------------------------------------------------------
/README.txt:
--------------------------------------------------------------------------------
  1 | twitter-event-stream
  2 | ====================
  3 | 
  4 | Description
  5 | -----------
  6 | 
  7 | twitter-event-stream provides an HTTP long polling endpoint that works in a
  8 | similar way to the deprecated User Streams API[1]. It uses the REST API and
  9 | the Account Activity API.
 10 | 
 11 | It is no easy work to update a Twitter client application built on top of the
 12 | User Streams API. Even worse, the Account Activity API which pretends to be
 13 | the replacement cannot be used directly by a mobile Twitter client.
 14 | twitter-event-stream allows such applications to continue to work with the
 15 | minimal amount of changes.
 16 | 
 17 | [1] https://twittercommunity.com/t/details-and-what-to-expect-from-the-api-deprecations-this-week-on-august-16-2018/110746
 18 | 
 19 | Setup
 20 | -----
 21 | 
 22 | Configuration
 23 | ~~~~~~~~~~~~~
 24 | 
 25 |  - You have to gain access to (the premium version of) the Account Activity
 26 |    API, and create a "dev environment". The "dev environment name", the base
 27 |    url where twitter-event-stream is deployed, the whitelisted consumer key,
 28 |    and the consumer secret are specified by environment variables.
 29 | 
 30 |      TWITTER_EVENT_STREAM_BASE_URL=<base url>
 31 |      TWITTER_EVENT_STREAM_ENV_NAME=<dev environment name>
 32 |      TWITTER_EVENT_STREAM_CONSUMER_KEY=<consumer key>
 33 |      TWITTER_EVENT_STREAM_CONSUMER_SECRET=<consumer secret>
 34 | 
 35 |    WARNING: twitter-event-stream assumes your dev environment allows only one
 36 |    webhook URL (which is the case for sandbox (free) plan) and removes all the
 37 |    existing webhook URL(s) on startup.
 38 | 
 39 |    NOTE: Subscription are limited to a maximum of 15 users per application in
 40 |    the sandbox plan. Because there is no way to clear subscriptions without
 41 |    having the access token of every subscribing user, it is not possible for
 42 |    twitter-event-stream to do that. It may be necessary to re-create the dev
 43 |    environment manually on developer.twitter.com after removing and adding
 44 |    another user to twitter-event-stream.
 45 | 
 46 |  - Credentials used for fetching home_timeline are stored in environment
 47 |    variables named `TWITTER_EVENT_STREAM_USER_<tag>`. `<tag>` may be any
 48 |    text.
 49 | 
 50 |      TWITTER_EVENT_STREAM_USER_ABC=<value>
 51 | 
 52 |    `<value>` is a JSON encoding of the following object:
 53 | 
 54 |      {
 55 |        "user_id": <user's numerical id>,
 56 |        "requests_per_window": 15,
 57 |        "token": <access token>,
 58 |        "token_secret": <access token secret>
 59 |      }
 60 |      # Increase requests_per_window if your application is granted the
 61 |      # permission to make more requests per 15 minutes window.
 62 | 
 63 |    If you need to use a different consumer key pair for the REST API requests,
 64 |    add the following to the JSON object. The token may be read-only.
 65 | 
 66 |      {
 67 |        "rest_consumer_key": <consumer key>,
 68 |        "rest_consumer_secret": <consumer secret>,
 69 |        "rest_token": <access token>,
 70 |        "rest_token_secret": <access token secret>,
 71 |      }
 72 | 
 73 |    NOTE: `setup-oauth.rb` included in this distribution might be useful to
 74 |    do 3-legged OAuth and make the JSON object.
 75 | 
 76 | Deployment
 77 | ~~~~~~~~~~
 78 | 
 79 |  - Ruby and Bundler are the prerequisites.
 80 | 
 81 |  - Install dependencies by `bundle install`, and then run
 82 |    `bundle exec puma -e production -p $PORT`.
 83 | 
 84 |    * The quickest way to deploy twitter-event-stream would be to use Heroku.
 85 |      Click the link and fill forms: https://heroku.com/deploy
 86 | 
 87 | Usage
 88 | -----
 89 | 
 90 | twitter-event-stream opens two endpoints for a client:
 91 | 
 92 |  - /1.1/user.json
 93 | 
 94 |    The message format is almost identical to the User streams' message format.
 95 |    However, due to the limitation of the Account Activity API, direct messages
 96 |    and some of the event types are not supported.
 97 | 
 98 |  - /stream
 99 | 
100 |    Sends events and home_timeline tweets in the server-sent events format
101 |    (text/event-stream). Events have the structure:
102 | 
103 |      event: <event>\r\n
104 |      data: <payload>\r\n\r\n
105 | 
106 |    `<event>` will be one of the event types received by the webhook:
107 | 
108 |    * `favorite_events` (for example; see Twitter's documentation[2])
109 | 
110 |        event: favorite_events\r\n
111 |        data: [{"id":"...","favorited_status":{...}}]\r\n\r\n
112 | 
113 |    Or, one of the following event types defined by twitter-event-stream:
114 | 
115 |    * `twitter_event_stream_home_timeline`
116 | 
117 |      New items in the home timeline. `<payload>` is an array of Tweet object.
118 | 
119 |        event: twitter_event_stream_home_timeline\r\n
120 |        data: [{"id":...,"text":"..."},...]\r\n\r\n
121 | 
122 |    * `twitter_event_stream_message`
123 | 
124 |      A message from twitter-event-stream, such as error reporting. `<payload>`
125 |      is a String.
126 | 
127 |        event: twitter_event_stream_message\r\n
128 |        data: "Message"\r\n\r\n
129 | 
130 |    Note that comment events are also sent every 30 seconds to keep the HTTP
131 |    connection open:
132 | 
133 |      :\r\n\r\n
134 | 
135 | 
136 | In both endpoints, the Tweet object structure BASICALLY follows the
137 | "Compatibility with additional extended_tweet in payload" mode[3]. Each Tweet
138 | object has `extended_tweet` object containing `full_text` key.
139 | 
140 | 
141 | twitter-event-stream uses "OAuth Echo"[4] to authenticate a client, meaning
142 | an application must provide the following HTTP headers:
143 | 
144 |  - `x-auth-service-provider`
145 | 
146 |     Must be set to
147 |     "https://api.twitter.com/1.1/account/verify_credentials.json".
148 | 
149 |  - `x-verify-credentials-authorization`
150 | 
151 |     The content of the Authorization HTTP header that the client would
152 |     normally send when calling the account/verify_credentials API.
153 | 
154 | [2] https://developer.twitter.com/en/docs/basics/authentication/overview/oauth-echo.html
155 | [3] https://developer.twitter.com/en/docs/tweets/tweet-updates.html
156 | [4] https://developer.twitter.com/en/docs/accounts-and-users/subscribe-account-activity/guides/account-activity-data-objects
157 | 
158 | License
159 | -------
160 | 
161 | twitter-event-stream is licensed under the MIT license. See COPYING.
162 | 


--------------------------------------------------------------------------------
/service.rb:
--------------------------------------------------------------------------------
  1 | require "json"
  2 | require_relative "oauth"
  3 | 
  4 | class ServiceError < StandardError; end
  5 | 
  6 | class Service
  7 |   class << self
  8 |     private :new
  9 | 
 10 |     def setup
 11 |       consumer_key = ENV["TWITTER_EVENT_STREAM_CONSUMER_KEY"]
 12 |       consumer_secret = ENV["TWITTER_EVENT_STREAM_CONSUMER_SECRET"]
 13 | 
 14 |       user_objs = []
 15 |       ENV.each { |k, v|
 16 |         next unless k.start_with?("TWITTER_EVENT_STREAM_USER_")
 17 |         user_objs << JSON.parse(v, symbolize_names: true)
 18 |       }
 19 | 
 20 |       # We assume the webapp is already started at this point: the CRC requires
 21 |       # GET /webhook to respond
 22 |       app_url = ENV["TWITTER_EVENT_STREAM_BASE_URL"]
 23 |       aa_env_name = ENV["TWITTER_EVENT_STREAM_ENV_NAME"]
 24 |       setup_webhook(app_url, aa_env_name, consumer_key, consumer_secret,
 25 |                     user_objs)
 26 | 
 27 |       @users = {}
 28 |       user_objs.each { |obj|
 29 |         @users[obj.fetch(:user_id)] = new(
 30 |           user_id: obj.fetch(:user_id),
 31 |           requests_per_window: obj.fetch(:requests_per_window),
 32 |           rest_oauth: {
 33 |             consumer_key: obj.fetch(:rest_consumer_key) {
 34 |               consumer_key },
 35 |             consumer_secret: obj.fetch(:rest_consumer_secret) {
 36 |               consumer_secret },
 37 |             token: obj.fetch(:rest_token) {
 38 |               obj.fetch(:token) },
 39 |             token_secret: obj.fetch(:rest_token_secret) {
 40 |               obj.fetch(:token_secret) }
 41 |           },
 42 |         )
 43 |       }
 44 |     end
 45 | 
 46 |     private def setup_webhook(app_url, env_name, consumer_key, consumer_secret,
 47 |                               user_objs)
 48 |       oauth = proc { |n|
 49 |         {
 50 |           consumer_key: consumer_key,
 51 |           consumer_secret: consumer_secret,
 52 |           token: user_objs.dig(n, :token),
 53 |           token_secret: user_objs.dig(n, :token_secret),
 54 |         }
 55 |       }
 56 | 
 57 |       if user_objs.empty?
 58 |         warn "setup_webhook: no users configured. cannot setup webhook"
 59 |         return
 60 |       end
 61 | 
 62 |       warn "setup_webhook: get existing webhook URL(s)"
 63 |       app_token = OAuthHelpers.bearer_request_token(oauth[0])
 64 |       body = OAuthHelpers.bearer_get(app_token,
 65 |                                      "/1.1/account_activity/all/webhooks.json")
 66 |       obj = JSON.parse(body, symbolize_names: true)
 67 |       env = obj.dig(:environments).find { |v| v[:environment_name] == env_name }
 68 | 
 69 |       warn "setup_webhook: clear existing webhook URL(s)"
 70 |       env[:webhooks].each do |webhook|
 71 |         warn "setup_webhook: delete id=#{webhook[:id]}: #{webhook[:url]}"
 72 |         path = "/1.1/account_activity/all/#{env_name}/webhooks/" \
 73 |           "#{webhook[:id]}.json"
 74 |         OAuthHelpers.user_delete(oauth[0], path)
 75 |       end
 76 | 
 77 |       warn "setup_webhook: register a webhook URL"
 78 |       webhook_url = app_url + (app_url.end_with?("/") ? "" : "/") + "webhook"
 79 |       path = "/1.1/account_activity/all/#{env_name}/webhooks.json?url=" +
 80 |         CGI.escape(webhook_url)
 81 |       webhook = OAuthHelpers.user_post(oauth[0], path)
 82 |       warn "setup_webhook: => #{webhook}"
 83 | 
 84 |       warn "setup_webhook: add subscriptions"
 85 |       user_objs.each_with_index { |_, n|
 86 |         warn "setup_webhook: add subscription for " \
 87 |           "user_id=#{user_objs.dig(n, :user_id)}"
 88 |         path = "/1.1/account_activity/all/#{env_name}/subscriptions.json"
 89 |         OAuthHelpers.user_post(oauth[n], path)
 90 |       }
 91 |     rescue => e
 92 |       warn "setup_webhook: uncaught exception: #{e.class} (#{e.message})"
 93 |       warn e.backtrace
 94 |     end
 95 | 
 96 |     def oauth_echo(asp, vca)
 97 |       if asp != "https://api.twitter.com/1.1/account/verify_credentials.json"
 98 |         raise ServiceError, "invalid OAuth Echo parameters"
 99 |       end
100 | 
101 |       begin
102 |         body = OAuthHelpers.http_get(vca, asp)
103 |         content = JSON.parse(body, symbolize_names: true)
104 |         get(content[:id])
105 |       rescue OAuthHelpers::HTTPRequestError
106 |         raise ServiceError, "OAuth Echo failed"
107 |       end
108 |     end
109 | 
110 |     def feed_webhook(json)
111 |       hash = JSON.parse(json)
112 |       if user_id = hash["for_user_id"]
113 |         service = get(Integer(user_id))
114 |         service.feed_webhook(hash)
115 |       else
116 |         warn "FIXME\n#{hash}"
117 |       end
118 |     end
119 | 
120 |     private
121 | 
122 |     def get(user_id)
123 |       defined?(@users) and @users[user_id] or
124 |         raise ServiceError, "unauthenticated user: #{user_id}"
125 |     end
126 |   end
127 | 
128 |   attr_reader :user_id
129 | 
130 |   def initialize(user_id:,
131 |                  requests_per_window:,
132 |                  rest_oauth:)
133 |     @user_id = user_id
134 |     @requests_per_window = Integer(requests_per_window)
135 |     @rest_oauth = rest_oauth
136 |     @listeners = {}
137 |     @backfill = []
138 |     start_polling
139 |   end
140 | 
141 |   def subscribe(count, &block)
142 |     @listeners[block] = block
143 |     emit_backfill(count)
144 |     block
145 |   end
146 | 
147 |   def unsubscribe(tag)
148 |     @listeners.delete(tag)
149 |   end
150 | 
151 |   def feed_webhook(hash)
152 |     hash.each do |key, value|
153 |       next if key == "for_user_id"
154 |       emit(key, value)
155 |     end
156 |   end
157 | 
158 |   def twitter_get(path, params)
159 |     JSON.parse(OAuthHelpers.user_get(@rest_oauth, path, params))
160 |   rescue OAuthHelpers::HTTPRequestError => e
161 |     # pp e.res.each_header.to_h
162 |     raise ServiceError, "API request failed: path=#{path} body=#{e.res.body}"
163 |   end
164 | 
165 |   private
166 | 
167 |   def emit(event, object)
168 |     # TODO: backfill
169 |     @backfill.shift if @backfill.size == 100
170 |     @backfill << [event, object]
171 |     @listeners.each { |_, block| block.call(event, object) }
172 |   end
173 | 
174 |   def emit_system(message)
175 |     emit("twitter_event_stream_message", message)
176 |   end
177 | 
178 |   def emit_backfill(count)
179 |     @backfill.last(count).each { |event, object| emit(event, object) }
180 |   end
181 | 
182 |   def start_polling
183 |     @polling_thread = Thread.start {
184 |       request_interval = 15.0 * 60 / @requests_per_window
185 | 
186 |       begin
187 |         last_max = nil
188 |         while true
189 |           t = Time.now
190 |           opts = {
191 |             "tweet_mode" => "extended",
192 |             "count" => 200,
193 |             "since_id" => last_max ? last_max - 1 : 1
194 |           }
195 |           ret = twitter_get("/1.1/statuses/home_timeline.json", opts)
196 | 
197 |           unless ret.empty?
198 |             if last_max
199 |               if last_max != ret.last["id"]
200 |                 emit_system("possible stalled tweets " \
201 |                             "#{last_max}+1...#{ret.last["id"]}")
202 |               else
203 |                 ret.pop
204 |               end
205 |             end
206 | 
207 |             # Fix Tweet object structure so it follows "Compatibility with
208 |             # additional extended_tweet in payload" mode.
209 |             # https://developer.twitter.com/en/docs/tweets/tweet-updates.html
210 |             ret.each { |tweet|
211 |               tweet["extended_tweet"] = {
212 |                 "full_text" => tweet["full_text"],
213 |                 "display_text_range" => tweet["display_text_range"],
214 |                 "entities" => tweet["entities"],
215 |                 "extended_entities" => tweet["extended_entities"],
216 |               }
217 |               tweet["text"] = tweet["full_text"]
218 | 
219 |               # NOTE: full_text should be removed from tweet, and then
220 |               # truncated, entities, extended_entities, and display_text_range
221 |               # should be modified according to the length of full_text. But
222 |               # this is probably not worth doing as clients will anyway process
223 |               # extended_tweet only so it can support >140 characters tweets.
224 |             }
225 | 
226 |             unless ret.empty?
227 |               emit("twitter_event_stream_home_timeline", ret)
228 |               last_max = ret.first["id"]
229 |             end
230 |           end
231 | 
232 |           sleep -(Time.now - t) % request_interval
233 |         end
234 |       rescue => e
235 |         warn "polling_thread (#{user_id}) uncaught exception: " \
236 |           "#{e.class} (#{e.message})"
237 |         warn e.backtrace
238 |         warn "polling_thread (#{user_id}) restarting in #{request_interval}s"
239 |         sleep request_interval
240 |         retry
241 |       end
242 |     }
243 |   end
244 | end
245 | 


--------------------------------------------------------------------------------