├── .github └── PULL_REQUEST_TEMPLATE.md ├── .gitignore ├── Gemfile ├── LICENSE ├── NOTICE ├── README.md ├── app.rb ├── config.ru ├── iam_policy.json ├── lib ├── scatter.rb └── scatter │ ├── cache.rb │ ├── encoded_entity.rb │ ├── parallel.rb │ ├── statistics.rb │ └── version.rb ├── misc ├── latencyHeatmap.png ├── serviceHealth.png └── traces.png ├── static └── javascript │ ├── cookie.js │ ├── scatter.js │ ├── select.js │ ├── settings.js │ └── traces.js └── templates └── scatter.html /.github/PULL_REQUEST_TEMPLATE.md: -------------------------------------------------------------------------------- 1 | *Issue #, if available:* 2 | 3 | *Description of changes:* 4 | 5 | 6 | By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license. 7 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | vendor 2 | .bundle 3 | coverage 4 | tmp 5 | .DS_Store 6 | .elasticbeanstalk 7 | 8 | # Elastic Beanstalk Files 9 | .elasticbeanstalk/* 10 | !.elasticbeanstalk/*.cfg.yml 11 | !.elasticbeanstalk/*.global.yml 12 | -------------------------------------------------------------------------------- /Gemfile: -------------------------------------------------------------------------------- 1 | source "https://rubygems.org" 2 | gem 'puma' 3 | gem 'rack' 4 | gem 'rack-parser' 5 | gem 'rack-test' 6 | gem 'sinatra' 7 | gem 'json' 8 | gem 'rake' 9 | gem 'aws-sdk-xray' 10 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | 2 | Apache License 3 | Version 2.0, January 2004 4 | http://www.apache.org/licenses/ 5 | 6 | TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 7 | 8 | 1. Definitions. 9 | 10 | "License" shall mean the terms and conditions for use, reproduction, 11 | and distribution as defined by Sections 1 through 9 of this document. 12 | 13 | "Licensor" shall mean the copyright owner or entity authorized by 14 | the copyright owner that is granting the License. 15 | 16 | "Legal Entity" shall mean the union of the acting entity and all 17 | other entities that control, are controlled by, or are under common 18 | control with that entity. For the purposes of this definition, 19 | "control" means (i) the power, direct or indirect, to cause the 20 | direction or management of such entity, whether by contract or 21 | otherwise, or (ii) ownership of fifty percent (50%) or more of the 22 | outstanding shares, or (iii) beneficial ownership of such entity. 23 | 24 | "You" (or "Your") shall mean an individual or Legal Entity 25 | exercising permissions granted by this License. 26 | 27 | "Source" form shall mean the preferred form for making modifications, 28 | including but not limited to software source code, documentation 29 | source, and configuration files. 30 | 31 | "Object" form shall mean any form resulting from mechanical 32 | transformation or translation of a Source form, including but 33 | not limited to compiled object code, generated documentation, 34 | and conversions to other media types. 35 | 36 | "Work" shall mean the work of authorship, whether in Source or 37 | Object form, made available under the License, as indicated by a 38 | copyright notice that is included in or attached to the work 39 | (an example is provided in the Appendix below). 40 | 41 | "Derivative Works" shall mean any work, whether in Source or Object 42 | form, that is based on (or derived from) the Work and for which the 43 | editorial revisions, annotations, elaborations, or other modifications 44 | represent, as a whole, an original work of authorship. For the purposes 45 | of this License, Derivative Works shall not include works that remain 46 | separable from, or merely link (or bind by name) to the interfaces of, 47 | the Work and Derivative Works thereof. 48 | 49 | "Contribution" shall mean any work of authorship, including 50 | the original version of the Work and any modifications or additions 51 | to that Work or Derivative Works thereof, that is intentionally 52 | submitted to Licensor for inclusion in the Work by the copyright owner 53 | or by an individual or Legal Entity authorized to submit on behalf of 54 | the copyright owner. For the purposes of this definition, "submitted" 55 | means any form of electronic, verbal, or written communication sent 56 | to the Licensor or its representatives, including but not limited to 57 | communication on electronic mailing lists, source code control systems, 58 | and issue tracking systems that are managed by, or on behalf of, the 59 | Licensor for the purpose of discussing and improving the Work, but 60 | excluding communication that is conspicuously marked or otherwise 61 | designated in writing by the copyright owner as "Not a Contribution." 62 | 63 | "Contributor" shall mean Licensor and any individual or Legal Entity 64 | on behalf of whom a Contribution has been received by Licensor and 65 | subsequently incorporated within the Work. 66 | 67 | 2. Grant of Copyright License. Subject to the terms and conditions of 68 | this License, each Contributor hereby grants to You a perpetual, 69 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 70 | copyright license to reproduce, prepare Derivative Works of, 71 | publicly display, publicly perform, sublicense, and distribute the 72 | Work and such Derivative Works in Source or Object form. 73 | 74 | 3. Grant of Patent License. Subject to the terms and conditions of 75 | this License, each Contributor hereby grants to You a perpetual, 76 | worldwide, non-exclusive, no-charge, royalty-free, irrevocable 77 | (except as stated in this section) patent license to make, have made, 78 | use, offer to sell, sell, import, and otherwise transfer the Work, 79 | where such license applies only to those patent claims licensable 80 | by such Contributor that are necessarily infringed by their 81 | Contribution(s) alone or by combination of their Contribution(s) 82 | with the Work to which such Contribution(s) was submitted. If You 83 | institute patent litigation against any entity (including a 84 | cross-claim or counterclaim in a lawsuit) alleging that the Work 85 | or a Contribution incorporated within the Work constitutes direct 86 | or contributory patent infringement, then any patent licenses 87 | granted to You under this License for that Work shall terminate 88 | as of the date such litigation is filed. 89 | 90 | 4. Redistribution. You may reproduce and distribute copies of the 91 | Work or Derivative Works thereof in any medium, with or without 92 | modifications, and in Source or Object form, provided that You 93 | meet the following conditions: 94 | 95 | (a) You must give any other recipients of the Work or 96 | Derivative Works a copy of this License; and 97 | 98 | (b) You must cause any modified files to carry prominent notices 99 | stating that You changed the files; and 100 | 101 | (c) You must retain, in the Source form of any Derivative Works 102 | that You distribute, all copyright, patent, trademark, and 103 | attribution notices from the Source form of the Work, 104 | excluding those notices that do not pertain to any part of 105 | the Derivative Works; and 106 | 107 | (d) If the Work includes a "NOTICE" text file as part of its 108 | distribution, then any Derivative Works that You distribute must 109 | include a readable copy of the attribution notices contained 110 | within such NOTICE file, excluding those notices that do not 111 | pertain to any part of the Derivative Works, in at least one 112 | of the following places: within a NOTICE text file distributed 113 | as part of the Derivative Works; within the Source form or 114 | documentation, if provided along with the Derivative Works; or, 115 | within a display generated by the Derivative Works, if and 116 | wherever such third-party notices normally appear. The contents 117 | of the NOTICE file are for informational purposes only and 118 | do not modify the License. You may add Your own attribution 119 | notices within Derivative Works that You distribute, alongside 120 | or as an addendum to the NOTICE text from the Work, provided 121 | that such additional attribution notices cannot be construed 122 | as modifying the License. 123 | 124 | You may add Your own copyright statement to Your modifications and 125 | may provide additional or different license terms and conditions 126 | for use, reproduction, or distribution of Your modifications, or 127 | for any such Derivative Works as a whole, provided Your use, 128 | reproduction, and distribution of the Work otherwise complies with 129 | the conditions stated in this License. 130 | 131 | 5. Submission of Contributions. Unless You explicitly state otherwise, 132 | any Contribution intentionally submitted for inclusion in the Work 133 | by You to the Licensor shall be under the terms and conditions of 134 | this License, without any additional terms or conditions. 135 | Notwithstanding the above, nothing herein shall supersede or modify 136 | the terms of any separate license agreement you may have executed 137 | with Licensor regarding such Contributions. 138 | 139 | 6. Trademarks. This License does not grant permission to use the trade 140 | names, trademarks, service marks, or product names of the Licensor, 141 | except as required for reasonable and customary use in describing the 142 | origin of the Work and reproducing the content of the NOTICE file. 143 | 144 | 7. Disclaimer of Warranty. Unless required by applicable law or 145 | agreed to in writing, Licensor provides the Work (and each 146 | Contributor provides its Contributions) on an "AS IS" BASIS, 147 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or 148 | implied, including, without limitation, any warranties or conditions 149 | of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A 150 | PARTICULAR PURPOSE. You are solely responsible for determining the 151 | appropriateness of using or redistributing the Work and assume any 152 | risks associated with Your exercise of permissions under this License. 153 | 154 | 8. Limitation of Liability. In no event and under no legal theory, 155 | whether in tort (including negligence), contract, or otherwise, 156 | unless required by applicable law (such as deliberate and grossly 157 | negligent acts) or agreed to in writing, shall any Contributor be 158 | liable to You for damages, including any direct, indirect, special, 159 | incidental, or consequential damages of any character arising as a 160 | result of this License or out of the use or inability to use the 161 | Work (including but not limited to damages for loss of goodwill, 162 | work stoppage, computer failure or malfunction, or any and all 163 | other commercial damages or losses), even if such Contributor 164 | has been advised of the possibility of such damages. 165 | 166 | 9. Accepting Warranty or Additional Liability. While redistributing 167 | the Work or Derivative Works thereof, You may choose to offer, 168 | and charge a fee for, acceptance of support, warranty, indemnity, 169 | or other liability obligations and/or rights consistent with this 170 | License. However, in accepting such obligations, You may act only 171 | on Your own behalf and on Your sole responsibility, not on behalf 172 | of any other Contributor, and only if You agree to indemnify, 173 | defend, and hold each Contributor harmless for any liability 174 | incurred by, or claims asserted against, such Contributor by reason 175 | of your accepting any such warranty or additional liability. 176 | 177 | END OF TERMS AND CONDITIONS 178 | 179 | APPENDIX: How to apply the Apache License to your work. 180 | 181 | To apply the Apache License to your work, attach the following 182 | boilerplate notice, with the fields enclosed by brackets "[]" 183 | replaced with your own identifying information. (Don't include 184 | the brackets!) The text should be enclosed in the appropriate 185 | comment syntax for the file format. We also recommend that a 186 | file or class name and description of purpose be included on the 187 | same "printed page" as the copyright notice for easier 188 | identification within third-party archives. 189 | 190 | Copyright [yyyy] [name of copyright owner] 191 | 192 | Licensed under the Apache License, Version 2.0 (the "License"); 193 | you may not use this file except in compliance with the License. 194 | You may obtain a copy of the License at 195 | 196 | http://www.apache.org/licenses/LICENSE-2.0 197 | 198 | Unless required by applicable law or agreed to in writing, software 199 | distributed under the License is distributed on an "AS IS" BASIS, 200 | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 201 | See the License for the specific language governing permissions and 202 | limitations under the License. 203 | -------------------------------------------------------------------------------- /NOTICE: -------------------------------------------------------------------------------- 1 | AWS Xray Scatter Sample 2 | Copyright 2017 Amazon.com, Inc. or its affiliates. All Rights Reserved. 3 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # AWS X-Ray Sample App 2 | This sample application uses the AWS X-Ray API's to generate a 3-hour or 7-day latency heatmap and health graph for individual services that make up your application. It uses [Vega-Lite](https://vega.github.io/vega-lite/) to generate the heatmap and health graph. The sample can be run locally or deployed to [AWS Elastic Beanstalk](http://aws.amazon.com/elasticbeanstalk/). 3 | 4 | ## License 5 | This sample application is licensed under the Apache 2.0 License. 6 | 7 | ## Requirements 8 | Ruby 2.3 or later with bundler. 9 | 10 | ## Features 11 | #### Latency Heatmap 12 | ![](misc/latencyHeatmap.png) 13 | 14 | A heatmap showing the latency over time (3-hour or 7-days) for a particular service or edge. You can click+drag to select an area to search for traces that meet that criteria. 15 | 16 | #### Health Graph 17 | ![](misc/serviceHealth.png) 18 | 19 | A service health graph that shows how the health of a particular service or edge has changed over time. You can click+drag to select an area to search for traces that meet that criteria. 20 | 21 | #### Linking to X-Ray console 22 | ![](misc/traces.png) 23 | 24 | Listing of traces that meet the criteria for the selected areas in the heatmap or the health graph. Enables you to easily bring up the individual traces in the X-Ray console. 25 | 26 | ## Getting Started 27 | ### Local 28 | ``` 29 | $ bundle install --path vendor/bundle 30 | $ bundle exec rackup 31 | ``` 32 | Open your browser to http://localhost:9292/ 33 | 34 | *Note: The sample app default to the **us-east-1** region. If your X-Ray data is in another region, click on the ... on the top right to bring up the region selection panel and select a different region.* 35 | 36 | ### AWS Elastic Beanstalk 37 | *Note: Deploying this app to Elastic Beanstalk using the below instructions will enable anyone with the URL for your Elastic Beanstalk environment to view the app. You can restrict access by running the app in a VPC or updating the load balancer security gruop to only allow access from trusted IP addresses.* 38 | 39 | You can get started using the following steps: 40 | 1. [Install the AWS Elastic Beanstalk Command Line Interface (CLI)](http://docs.aws.amazon.com/elasticbeanstalk/latest/dg/eb-cli3-install.html). 41 | 2. Create an IAM Instance Profile named **aws-elasticbeanstalk-sample-role** with the policy in [iam_policy.json](iam_policy.json). For more information on how to create an IAM Instance Profile, see [Create an IAM Instance Profile for Your Amazon EC2 Instances](https://docs.aws.amazon.com/codedeploy/latest/userguide/how-to-create-iam-instance-profile.html). 42 | 3. Run `eb init -r -p "Ruby"` to initialize the folder for use with the CLI. Replace `` with a region identifier such as `us-west-2` (see [Regions and Endpoints](https://docs.amazonaws.cn/en_us/general/latest/gr/rande.html#elasticbeanstalk_region) for a full list of region identifiers). For interactive mode, run `eb init` then, 43 | 1. Pick a region of your choice. 44 | 2. Select the **[ Create New Application ]** option. 45 | 3. Enter the application name of your choice. 46 | 4. Answer **yes** to *It appears you are using Node.js. Is this correct?*. 47 | 7. Choose whether you want SSH access to the Amazon EC2 instances. 48 | *Note: If you choose to enable SSH and do not have an existing SSH key stored on AWS, the EB CLI requires ssh-keygen to be available on the path to generate SSH keys.* 49 | 4. Run `eb create --instance_profile aws-elasticbeanstalk-sample-role` to begin the creation of your environment. 50 | 1. Enter the environment name of your choice. 51 | 2. Enter the CNAME prefix you want to use for this environment. 52 | 5. Once the environment creation process completes, run `eb open` to open the application in a browser. 53 | 6. Run `eb terminate --all` to clean up. 54 | -------------------------------------------------------------------------------- /app.rb: -------------------------------------------------------------------------------- 1 | require 'sinatra/base' 2 | require 'json' 3 | require 'logger' 4 | require 'aws-sdk-xray' 5 | require 'scatter/encoded_entity' 6 | require 'scatter/statistics' 7 | require 'scatter/parallel' 8 | require 'scatter/cache' 9 | 10 | # retry on socket errors 11 | Aws::XRay::Client.remove_plugin Aws::Plugins::HelpfulSocketErrors 12 | Aws.config.update :retry_limit => 6 13 | 14 | module Scatter 15 | class App < Sinatra::Base 16 | enable :logging 17 | set :public_folder, File.dirname(__FILE__) + '/static' 18 | set :template_folder, File.dirname(__FILE__) + '/templates' 19 | 20 | @@timeline_link = %[https://console.aws.amazon.com/xray/home?region=%s#/traces/%s] 21 | @@grace_period = 60 22 | @@region = "us-east-1" 23 | @@xray_client = {} 24 | @@mutex = Mutex.new 25 | @@cache = Cache.new :ttl => 7 * 24 * 3600 26 | @@maximum_time_range = 6 * 24 * 3600 27 | 28 | before do 29 | content_type 'application/json' 30 | end 31 | 32 | get '/' do 33 | logger.info settings.template_folder 34 | 35 | content_type :html, 'charset' => 'utf-8' 36 | send_file File.join(settings.template_folder, 'scatter.html') 37 | end 38 | 39 | # Returns the entities (services and edges) that have been active within a specified time 40 | # range as a JSON document containing a sorted array of EncodedEntity entries. 41 | get '/entities' do 42 | region = fetch 'region', @@region 43 | start_time = Time.at fetch('startTime').to_i 44 | end_time = Time.at fetch('endTime').to_i 45 | 46 | services = Aws::XRay::Client.new(:region => region) 47 | .get_service_graph(:start_time => start_time, :end_time => end_time) 48 | .services 49 | 50 | entities = services.flat_map do |service| 51 | service_entity = EncodedEntity.new :left => service 52 | 53 | edge_entities = service.edges.collect do |edge| 54 | EncodedEntity.new :left => service, :right => services.find { |i| i.reference_id == edge.reference_id } 55 | end 56 | 57 | if service.type == "client" 58 | edge_entities 59 | else 60 | [service_entity] + edge_entities 61 | end 62 | end 63 | 64 | json :entities => entities.sort.collect(&:to_h) 65 | end 66 | 67 | # Returns an array of Statistics entries for a specified time range as a JSON document. 68 | # Each Statistics entry represents a specified duration. 69 | get '/scatter' do 70 | region = fetch 'region', @@region 71 | entity = EncodedEntity.from_raw fetch('entity') 72 | start_time = Time.at fetch('startTime').to_i 73 | end_time = Time.at fetch('endTime').to_i 74 | duration = fetch('duration').to_i 75 | 76 | h = statistics :region => region, 77 | :entity => entity, 78 | :start_time => start_time, 79 | :end_time => end_time, 80 | :duration => duration 81 | 82 | json :statistics => h.collect(&:to_h) 83 | end 84 | 85 | # Returns an array of summaries for traces within a specified time range as a JSON 86 | # document. 87 | # 88 | # The filter expression for **GetTraceSummaries** is generated based on 89 | # **entity** and **type** parameters. 90 | # 91 | # For services: 92 | # 93 | # service(id) { criteria } 94 | # 95 | # For edges: 96 | # 97 | # edge(id, id) { criteria } 98 | # 99 | # Criteria for "status" type: 100 | # 101 | # !ok 102 | # 103 | # Criteria for "responsetime" type: 104 | # 105 | # responsetime value [&& responsetime value] 106 | # 107 | get '/traces/:type' do 108 | type = fetch 'type' 109 | region = fetch 'region', @@region 110 | entity = EncodedEntity.from_raw fetch('entity') 111 | start_time = Time.at fetch('startTime').to_i 112 | end_time = Time.at fetch('endTime').to_i 113 | response_time_min = fetch 'responseTimeMin', nil 114 | response_time_max = fetch 'responseTimeMax', nil 115 | 116 | if end_time - start_time > @@maximum_time_range 117 | start_time = end_time - @@maximum_time_range 118 | end 119 | 120 | filter_expression = case type 121 | when /^status$/i 122 | %[!ok] 123 | when /^responsetime$/i 124 | case 125 | when response_time_min && response_time_max 126 | %[responsetime >= #{response_time_min} && responsetime <= #{response_time_max}] 127 | when response_time_min 128 | %[responsetime >= #{response_time_min}] 129 | when response_time_max 130 | %[responsetime <= #{response_time_max}] 131 | else 132 | %[] 133 | end 134 | else 135 | halt 400, "invalid type: #{type}" 136 | end 137 | filter_expression = %[#{entity.filter_expression} { #{filter_expression} }] 138 | logger.info %[filter expression: #{filter_expression}] 139 | 140 | summaries = Aws::XRay::Client.new(:region => region) 141 | .get_trace_summaries(:start_time => start_time, :end_time => end_time, :filter_expression => filter_expression) 142 | .lazy 143 | .flat_map(&:trace_summaries) 144 | .first(100) 145 | .to_a 146 | 147 | traces = summaries.collect do |i| 148 | annotations = i.annotations.collect do |k, v| 149 | values = v.collect do |i| 150 | i.annotation_value.boolean_value || i.annotation_value.number_value || i.annotation_value.string_value 151 | end 152 | 153 | [k, values] 154 | end.to_h 155 | 156 | { 157 | :trace_id => i.id, 158 | :timestamp => Time.at(i.id.split("-")[1].to_i(16)).iso8601, 159 | :http_url => i.http&.http_url, 160 | :http_status => i.http&.http_status, 161 | :response_time => i.response_time, 162 | :users => i.users.collect(&:user_name), 163 | :annotations => annotations, 164 | :timeline_link => format(@@timeline_link, region, i.id) 165 | } 166 | end 167 | 168 | json :traces => traces 169 | end 170 | 171 | # Returns an array of Statistics entries for a specified time range using. Each Statistics 172 | # entry represents a specified duration. 173 | private 174 | def statistics(entity:, region:, start_time:, end_time:, duration:) 175 | time_range = (start_time.to_i)..(end_time.to_i) 176 | slices = time_range.step(duration).each_cons(2) 177 | xray = Aws::XRay::Client.new(:region => region) 178 | 179 | logger.info %[fetching #{slices.count} histograms from #{Time.at(time_range.begin).strftime("%D %T %Z")} to #{Time.at(time_range.end).strftime("%D %T %Z")} @ #{region}] 180 | 181 | Parallel.parallel slices, :threads => 6 do |(t0, t1), index| 182 | service_graph = @@cache.fetch t0..t1 do 183 | logger.info %[cache miss for histogram: #{index + 1} / #{slices.count}] 184 | 185 | xray.get_service_graph(:start_time => t0, :end_time => t1) 186 | end 187 | 188 | e = service_graph.services 189 | .find { |i| i.name == entity.left.name && i.type == entity.left.type } 190 | 191 | if entity.edge? 192 | e = e&.edges&.find do |edge| 193 | service_graph.services.find do |service| 194 | service.reference_id == edge.reference_id && service.name == entity.right.name && service.type == entity.right.type 195 | end 196 | end 197 | end 198 | 199 | histogram = e&.response_time_histogram 200 | &.to_a 201 | histogram ||= [] 202 | 203 | ok_count = e&.summary_statistics&.ok_count || 0 204 | error_count = e&.summary_statistics&.error_statistics&.other_count || 0 205 | throttle_count = e&.summary_statistics&.error_statistics&.throttle_count || 0 206 | fault_count = e&.summary_statistics&.fault_statistics&.total_count || 0 207 | 208 | Statistics.new :timestamp => t1, 209 | :entries => histogram&.collect(&:to_h)&.reverse, 210 | :duration => t1 - t0, 211 | :ok_count => ok_count, 212 | :error_count => error_count, 213 | :throttle_count => throttle_count, 214 | :fault_count => fault_count, 215 | :index => { 216 | :start_time => e&.start_time&.to_i, 217 | :end_time => e&.end_time&.to_i 218 | } 219 | end 220 | end 221 | 222 | private 223 | def json(document) 224 | document.to_json 225 | end 226 | 227 | private 228 | def logger 229 | env['rack.logger'] 230 | end 231 | 232 | # Returns a cookie or URL parameter for the specified key. If a default value is not 233 | # specified and key is not found the request is halted with HTTP status 400. 234 | private 235 | def fetch(key, default=:missing) 236 | value = request.cookies[key] || params[key] || default 237 | 238 | if value == :missing 239 | logger.error %[missing URL parameter: #{e.key}] 240 | halt 400, "missing URL parameter: #{key}" 241 | end 242 | 243 | value 244 | end 245 | end 246 | end 247 | -------------------------------------------------------------------------------- /config.ru: -------------------------------------------------------------------------------- 1 | require 'rack' 2 | require 'rack/parser' 3 | require 'etc' 4 | 5 | $:.unshift %[#{File.dirname(__FILE__)}/lib] 6 | 7 | use Rack::Parser, :content_types => { 8 | 'application/json' => Proc.new { |body| JSON.parse body } 9 | } 10 | 11 | require './app' 12 | run Scatter::App 13 | -------------------------------------------------------------------------------- /iam_policy.json: -------------------------------------------------------------------------------- 1 | { 2 | "Version": "2012-10-17", 3 | "Statement": [ 4 | { 5 | "Effect": "Allow", 6 | "Action": [ 7 | "xray:BatchGetTraces", 8 | "xray:GetServiceGraph", 9 | "xray:GetTraceGraph", 10 | "xray:GetTraceSummaries" 11 | ], 12 | "Resource": [ 13 | "*" 14 | ] 15 | } 16 | ] 17 | } -------------------------------------------------------------------------------- /lib/scatter.rb: -------------------------------------------------------------------------------- 1 | module Scatter 2 | end 3 | -------------------------------------------------------------------------------- /lib/scatter/cache.rb: -------------------------------------------------------------------------------- 1 | module Scatter 2 | class Cache 3 | class Entry 4 | attr_reader :value, :ttl 5 | 6 | def initialize(value, ttl:) 7 | @value = value 8 | @ttl = Time.now + ttl 9 | end 10 | 11 | def expired? 12 | Time.now > @ttl 13 | end 14 | end 15 | 16 | attr_reader :ttl 17 | 18 | def initialize(ttl:) 19 | @mutex = Mutex.new 20 | @c = {} 21 | @ttl = ttl 22 | end 23 | 24 | def fetch(key) 25 | @mutex.synchronize do 26 | @c.reject! { |k, v| v.expired? } 27 | end 28 | 29 | if v = @mutex.synchronize { @c[key] } 30 | return v.value 31 | end 32 | 33 | v = yield 34 | @mutex.synchronize do 35 | @c[key] = Entry.new v, :ttl => ttl 36 | end 37 | v 38 | end 39 | end 40 | end 41 | -------------------------------------------------------------------------------- /lib/scatter/encoded_entity.rb: -------------------------------------------------------------------------------- 1 | require 'cgi' 2 | 3 | module Scatter 4 | # Represents X-Ray filter expression service keyword. 5 | # 6 | # http://docs.aws.amazon.com/xray/latest/devguide/xray-console-filters.html#console-filters-complex 7 | class Service 8 | include Comparable 9 | 10 | attr_reader :name, :type 11 | 12 | def initialize(name:, type: nil) 13 | @name = name 14 | @type = type 15 | end 16 | 17 | # Returns X-Ray filter expression ID for the service. 18 | def to_id 19 | name = %["#{@name}"] 20 | type = @type ? %["#{@type}"] : %[null] 21 | 22 | %[id(name: #{name}, type: #{type})] 23 | end 24 | 25 | def <=>(other) 26 | case 27 | when self.type == 'client' 28 | -1 29 | when other.type == 'client' 30 | 1 31 | when self.name != other.name 32 | self.name <=> other.name 33 | when self.type && !other.type 34 | 1 35 | when !self.type && other.type 36 | -1 37 | else 38 | self.type <=> other.type 39 | end 40 | end 41 | 42 | def to_h 43 | if @type 44 | { 45 | 'name' => @name, 46 | 'type' => @type 47 | } 48 | else 49 | { 50 | 'name' => @name, 51 | } 52 | end 53 | end 54 | end 55 | 56 | # Represents X-Ray filter expression service or edge keyword. 57 | # 58 | # http://docs.aws.amazon.com/xray/latest/devguide/xray-console-filters.html#console-filters-complex 59 | class EncodedEntity 60 | include Comparable 61 | 62 | attr_accessor :left, :right 63 | 64 | def self.from_raw(input) 65 | entity = self.new 66 | objects = JSON.parse CGI.unescapeHTML(input), :symbolize_names => true 67 | 68 | case objects 69 | when Hash 70 | entity.left = Service.new **objects 71 | when Array 72 | entity.left, entity.right = objects.collect { |i| Service.new **i } 73 | else 74 | logger.error %[invlid entity: #{input}] 75 | halt 400, "invlid entity: #{input}" 76 | end 77 | 78 | entity 79 | end 80 | 81 | def initialize(left: nil, right: nil) 82 | @left = Service.new :name => left.name, :type => left.type if left 83 | @right = Service.new :name => right.name, :type => right.type if right 84 | end 85 | 86 | def edge? 87 | !!left && !!right 88 | end 89 | 90 | def service? 91 | !edge? 92 | end 93 | 94 | # Returns X-Ray filter expression complex keyword for the service for the edge. 95 | def filter_expression 96 | if edge? 97 | %[edge(#{left.to_id}, #{right.to_id})] 98 | else 99 | %[service(#{left.to_id})] 100 | end 101 | end 102 | 103 | def <=>(other) 104 | case 105 | when self.left != other.left 106 | self.left <=> other.left 107 | when !self.right 108 | -1 109 | when !other.right 110 | 1 111 | else 112 | self.right <=> other.right 113 | end 114 | end 115 | 116 | def to_h 117 | if @right 118 | [ 119 | @left.to_h, 120 | @right.to_h 121 | ] 122 | else 123 | @left.to_h 124 | end 125 | end 126 | end 127 | end 128 | -------------------------------------------------------------------------------- /lib/scatter/parallel.rb: -------------------------------------------------------------------------------- 1 | module Scatter 2 | module Parallel 3 | Task = Struct.new :element, :index, :block, :outbox, :stop 4 | Result = Struct.new :value, :exception 5 | 6 | @@global_workers = nil 7 | @@global_inbox = Queue.new 8 | 9 | # == Usage 10 | # 11 | # require 'scatter/parallel' 12 | # require 'open-uri' 13 | # 14 | # include Scatter::Parallel 15 | # bodies = parallel(urls, :threads => 32) do |url, index| 16 | # puts %[#{index} : reading url "#{url}"...] 17 | # open(url).read 18 | # end 19 | # 20 | def parallel(enumerable, **options, &block) 21 | Parallel.parallel enumerable, **options, &block 22 | end 23 | 24 | def self.parallel(enumerable, threads: nil, expand_backtrace: true, global: nil, &block) 25 | raise ArgumentError, %[block is required] unless block 26 | 27 | global = global? if global == nil 28 | if global && threads 29 | raise ArgumentError, %[threads is not supported option when global is enabled] 30 | end 31 | assert_global! if global 32 | 33 | inbox = @@global_inbox 34 | size = nil 35 | 36 | unless global 37 | size = [threads || 16, enumerable.size].min 38 | inbox = Queue.new 39 | initialize_workers :size => size, :inbox => inbox 40 | end 41 | 42 | outbox = Queue.new 43 | enumerable.each_with_index.collect do |element, index| 44 | task = Task.new 45 | task.element = element 46 | task.index = index 47 | task.block = block 48 | task.outbox = outbox 49 | 50 | inbox.push task 51 | end 52 | 53 | results = [] 54 | while results.size < enumerable.size 55 | results << outbox.pop 56 | end 57 | 58 | unless global 59 | stop_workers :size => size, :inbox => inbox 60 | end 61 | 62 | if first_failed_result = results.detect(&:exception) 63 | exception = first_failed_result.exception 64 | exception.set_backtrace(exception.backtrace + extended_backtrace(expand_backtrace)) 65 | 66 | raise exception 67 | end 68 | results.collect(&:value) 69 | end 70 | 71 | def self.enable_global(size: 20) 72 | if global? 73 | raise %[global worker pool has already been initialized] 74 | end 75 | 76 | @@global_workers ||= begin 77 | initialize_workers :size => size, :inbox => @@global_inbox 78 | end 79 | end 80 | 81 | def self.disable_global 82 | assert_global! 83 | 84 | stop_workers :size => @@global_workers.size, :inbox => @@global_inbox 85 | @@global_workers.collect &:kill 86 | @@global_workers = nil 87 | @@global_inbox.clear 88 | end 89 | 90 | def self.global? 91 | @@global_workers != nil 92 | end 93 | 94 | private 95 | def self.initialize_workers(size:, inbox:) 96 | size.times.collect do 97 | Thread.new do 98 | loop do 99 | result = Result.new 100 | 101 | begin 102 | task = inbox.pop 103 | 104 | if task.stop 105 | break 106 | end 107 | 108 | result.value = task.block.call task.element, task.index 109 | rescue Exception => e 110 | result.exception = e 111 | end 112 | 113 | task.outbox.push result 114 | end 115 | end 116 | end 117 | end 118 | 119 | private 120 | def self.stop_workers(size:, inbox:) 121 | size.times do 122 | task = Task.new 123 | task.stop = true 124 | inbox.push task 125 | end 126 | 127 | sleep 0.05 until inbox.num_waiting == 0 128 | end 129 | 130 | private 131 | def self.assert_global! 132 | unless global? 133 | raise %[global worker pool has not been initialized] 134 | end 135 | end 136 | 137 | private 138 | def self.extended_backtrace(expand) 139 | if expand 140 | Thread.current 141 | .backtrace_locations 142 | .drop_while { |i| i.label != __method__.to_s } 143 | .drop(2) 144 | .collect(&:to_s) 145 | else 146 | [] 147 | end 148 | end 149 | end 150 | end 151 | -------------------------------------------------------------------------------- /lib/scatter/statistics.rb: -------------------------------------------------------------------------------- 1 | module Scatter 2 | # Model for statistics entry returned by /scatter 3 | class Statistics 4 | def initialize(timestamp:, entries:, duration:, index:, ok_count:, error_count:, throttle_count:, fault_count:) 5 | @timestamp = timestamp 6 | @entries = entries 7 | @duration = duration.to_f 8 | @index = index 9 | @ok_count = ok_count 10 | @error_count = error_count 11 | @throttle_count = throttle_count 12 | @fault_count = fault_count 13 | end 14 | 15 | def to_h 16 | { 17 | :timestamp => @timestamp, 18 | :histogram => @entries, 19 | :statuses => [ 20 | { 21 | :status => 'ok', 22 | :request_rate => (@ok_count / @duration).round(5), 23 | }, 24 | { 25 | :status => 'error', 26 | :request_rate => (@error_count / @duration).round(5), 27 | }, 28 | { 29 | :status => 'throttle', 30 | :request_rate => (@throttle_count / @duration).round(5), 31 | }, 32 | { 33 | :status => 'fault', 34 | :request_rate => (@fault_count / @duration).round(5), 35 | } 36 | ], 37 | :index => @index 38 | } 39 | end 40 | end 41 | end 42 | -------------------------------------------------------------------------------- /lib/scatter/version.rb: -------------------------------------------------------------------------------- 1 | module Scatter 2 | VERSION = "0.5" 3 | end 4 | -------------------------------------------------------------------------------- /misc/latencyHeatmap.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-xray-scatter-sample/31ccdb56f50d3dec03b5a8134722640e53185a58/misc/latencyHeatmap.png -------------------------------------------------------------------------------- /misc/serviceHealth.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-xray-scatter-sample/31ccdb56f50d3dec03b5a8134722640e53185a58/misc/serviceHealth.png -------------------------------------------------------------------------------- /misc/traces.png: -------------------------------------------------------------------------------- https://raw.githubusercontent.com/aws-samples/aws-xray-scatter-sample/31ccdb56f50d3dec03b5a8134722640e53185a58/misc/traces.png -------------------------------------------------------------------------------- /static/javascript/cookie.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const cookie = {}; 4 | 5 | // Based on https://stackoverflow.com/questions/14573223/set-cookie-and-get-cookie-with-javascript 6 | 7 | cookie.set = function(name, value, days) { 8 | let expires = ""; 9 | if (days) { 10 | let date = new Date(); 11 | date.setTime(date.getTime() + (days * 24 * 60 * 60 * 1000)); 12 | expires = "; expires=" + date.toUTCString(); 13 | } 14 | document.cookie = name + "=" + encodeURIComponent(value) + expires + "; path=/"; 15 | }; 16 | 17 | cookie.get = function(name, defaultValue) { 18 | let nameEQ = name + "="; 19 | let ca = document.cookie.split(';'); 20 | for (let i=0;i < ca.length; i++) { 21 | let c = ca[i]; 22 | while (c.charAt(0)==' ') c = c.substring(1,c.length); 23 | if (c.indexOf(nameEQ) == 0) return decodeURIComponent(c.substring(nameEQ.length,c.length)); 24 | } 25 | return defaultValue; 26 | }; 27 | 28 | cookie.clear = function(name) { 29 | cookie.set(name, "", -1); 30 | }; 31 | -------------------------------------------------------------------------------- /static/javascript/scatter.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const MINUTE = 60000; 4 | const POLL_INTERVAL = 15000; 5 | 6 | const scatter = {}; 7 | 8 | scatter.startTime = null; 9 | scatter.view = null; 10 | scatter.polledAt = null; 11 | scatter.refreshedAt = null; 12 | scatter.error = null; 13 | scatter.statistics = {}; 14 | scatter.append = false; 15 | scatter.timeSlots = 180; 16 | scatter.duration = 60000; 17 | 18 | // Initializing the vega view without data, or clearing it, doesn't seem to work so we'll 19 | // rebuild the whole view when the view changes. 20 | scatter.rebuild = function() { 21 | scatter.timeSlots = (settings.timeSpan == "threeHours") ? 180 : 168; 22 | scatter.duration = (settings.timeSpan == "threeHours") ? 60000 : 3600000; 23 | settings.set("duration", scatter.duration / 1000); 24 | 25 | const endTime = scatter.safeEndTime(); 26 | const startTime = scatter.cutOffTime(endTime); 27 | 28 | scatter.clear(); 29 | scatter.fetch("/scatter", settings.entity, startTime, endTime, {}, function(data) { 30 | const statistics = scatter.mergeStatistics(data.statistics, endTime); 31 | scatter.build(statistics); 32 | }); 33 | }; 34 | 35 | // Start incremental update poller during initialization. 36 | scatter.init = function() { 37 | setInterval(function() { 38 | const endTime = scatter.safeEndTime(); 39 | const cutOffTime = scatter.cutOffTime(endTime); 40 | let startTime = scatter.startTime; 41 | 42 | scatter.polledAt = new Date(); 43 | 44 | if (startTime && startTime < cutOffTime) { 45 | startTime = cutOffTime; 46 | } 47 | 48 | if (settings.entity && startTime && endTime.getTime() - startTime.getTime() >= scatter.duration) { 49 | console.log("fetch", startTime, endTime); 50 | 51 | scatter.appendStatistics(startTime, endTime, cutOffTime, scatter.append); 52 | } 53 | }, POLL_INTERVAL); 54 | 55 | scatter.polledAt = new Date(); 56 | }; 57 | 58 | scatter.clear = function() { 59 | scatter.append = false; 60 | 61 | scatter.startTime = null; 62 | scatter.refreshedAt = null; 63 | scatter.statistics = {}; 64 | 65 | traces.clear(); 66 | scatter.hideChart(); 67 | }; 68 | 69 | // Initialize the Vega Lite view and the event listener for selections lazily with initial set of 70 | // statistics to workaround limitations of Vega Lite. 71 | scatter.build = function(statistics) { 72 | const width = document.documentElement.clientWidth - 175; 73 | let scatterHeight = (document.documentElement.clientHeight - 500 > 750) ? 750 : document.documentElement.clientHeight - 500; 74 | const statusesHeight = 100; 75 | 76 | if (scatterHeight < statusesHeight * 3) { 77 | scatterHeight = statusesHeight * 3; 78 | } 79 | 80 | var spec = { 81 | "$schema": "https://vega.github.io/schema/vega-lite/v2.json", 82 | "config": { 83 | "range": { 84 | "heatmap": [ 85 | "orange", 86 | "red", 87 | "black", 88 | ], 89 | "ramp": [ 90 | "orange", 91 | "red", 92 | "black", 93 | ] 94 | }, 95 | "legend": { 96 | "gradientWidth": 50, 97 | }, 98 | "invalidValues": "filter", 99 | }, 100 | "view": { 101 | "stroke": "transparent" 102 | }, 103 | "vconcat": [{ 104 | "data": { 105 | "name": "histogram", 106 | "values": statistics.histogram, 107 | }, 108 | "width": width, 109 | "height": scatterHeight, 110 | "mark": { 111 | "type": "rect", 112 | // "type": "circle", 113 | // "size": 100, 114 | }, 115 | "encoding": { 116 | "x": { 117 | "field": "time", 118 | "type": "temporal", 119 | "bin": { "maxbins": scatter.timeSlots }, // rect only 120 | "axis": { 121 | "format": (settings.timeSpan == "threeHours") ? "%H:%M" : "%m/%d %H:%M", 122 | "title": "time", 123 | "tickCount": 24 124 | }, 125 | }, 126 | "y": { 127 | "field": "value", 128 | "type": "quantitative", 129 | "bin": { "maxbins": 180 }, 130 | "axis": { 131 | "title": "response time", 132 | "orient": "right", 133 | "tickCount": 24, 134 | }, 135 | }, 136 | "color": { 137 | "aggregate": "count", 138 | "type": "quantitative", 139 | "legend": { 140 | "title": "Count", 141 | }, 142 | } 143 | }, 144 | "selection": { 145 | "brush": { 146 | "type": "interval", 147 | "encodings": [ 148 | "x", 149 | "y" 150 | ], 151 | "mark": { 152 | "fill": "#333", 153 | "fillOpacity": 0.125, 154 | "stroke": "white" 155 | }, 156 | "resolve": "global", 157 | } 158 | }, 159 | }, 160 | { 161 | "data": { 162 | "name": "statuses", 163 | "values": statistics.statuses, 164 | }, 165 | "width": width, 166 | "height": statusesHeight, 167 | "mark": "area", 168 | "encoding": { 169 | "x": { 170 | "field": "time", 171 | "type": "temporal", 172 | "axis": { 173 | "format": (settings.timeSpan == "threeHours") ? "%H:%M" : "%m/%d %H:%M", 174 | "title": "time", 175 | "tickCount": 24 176 | } 177 | }, 178 | "y": { 179 | "field": "request_rate", 180 | "type": "quantitative", 181 | "aggregate": "sum", 182 | "axis": { 183 | "title": "traces / sec", 184 | "tickCount": 8, 185 | "orient": "right", 186 | }, 187 | }, 188 | "color": { 189 | "field": "status", 190 | "type": "nominal", 191 | "scale": { 192 | "range": ["#178B27", "#FA8608", "#600068", "#C40A05"] // ["green", "orange", "purple", "red"] 193 | }, 194 | "legend": { 195 | "title": "Status", 196 | }, 197 | } 198 | }, 199 | "selection": { 200 | "brush": { 201 | "type": "interval", 202 | "encodings": [ 203 | "x" 204 | ], 205 | "mark": { 206 | "fill": "#333", 207 | "fillOpacity": 0.125, 208 | "stroke": "white" 209 | }, 210 | "resolve": "global", 211 | } 212 | }, 213 | } 214 | ] 215 | }; 216 | 217 | var opt = { 218 | "mode": "vega-lite", 219 | "actions": false 220 | }; 221 | 222 | scatter.showChart(); 223 | vega.embed("#chart", spec, opt, function(error, result) { 224 | const view = result.view; 225 | scatter.view = view; 226 | 227 | view.addEventListener("mouseup", function(event, item) { 228 | if (view.data('brush_store')[0]) { 229 | const intervals = view.data('brush_store')[0].intervals; 230 | const traceTimeRange = { 231 | startTime: intervals[0].extent[0], 232 | endTime: intervals[0].extent[1] 233 | } 234 | const correctedTimeRange = scatter.correctTraceTimeRange(traceTimeRange); 235 | 236 | traces.showProgress(); 237 | if (intervals.length == 2) { 238 | let parameters = { 239 | responseTimeMin: intervals[1].extent[1], 240 | responseTimeMax: intervals[1].extent[0] 241 | } 242 | parameters = scatter.correctResponseTimeRange(scatter.statistics.histogram, traceTimeRange.startTime, traceTimeRange.endTime, parameters); 243 | 244 | scatter.fetch("/traces/responsetime", settings.entity, correctedTimeRange.startTime, correctedTimeRange.endTime, parameters, traces.update); 245 | } else { 246 | const traceStartTime = intervals[0].extent[0]; 247 | const traceEndTime = intervals[0].extent[1]; 248 | 249 | scatter.fetch("/traces/status", settings.entity, correctedTimeRange.startTime, correctedTimeRange.endTime, {}, traces.update); 250 | } 251 | } else { 252 | traces.clear(); 253 | } 254 | }); 255 | }); 256 | }; 257 | 258 | scatter.showChart = function() { 259 | document.getElementById('chart').style.display = "block"; 260 | document.getElementById('loading').style.display = "none"; 261 | }; 262 | 263 | scatter.hideChart = function() { 264 | document.getElementById('chart').style.display = "none"; 265 | document.getElementById('loading').style.display = "block"; 266 | }; 267 | 268 | // Incrementally add statistics to the view to avoiding re-rendering the whole view on update. 269 | scatter.appendStatistics = function(startTime, endTime, cutOffTime, append) { 270 | scatter.fetch("/scatter", settings.entity, startTime, endTime, {}, function(data) { 271 | if (!append) { 272 | return; 273 | } 274 | 275 | const statistics = scatter.mergeStatistics(data.statistics, endTime); 276 | 277 | // TODO: named data sources are broken? (https://vega.github.io/vega-lite/docs/data.html#named) 278 | let histogramSet = "source_0" 279 | let statusesSet = "source_1" 280 | 281 | if (scatter.view.data("source_0").length > 0 && scatter.view.data("source_0")[0].status) { 282 | histogramSet = "source_1" 283 | statusesSet = "source_0" 284 | } 285 | 286 | scatter.view 287 | .change(histogramSet, vega.changeset().remove(function(d) { return new Date(d.time) < cutOffTime; }).insert(statistics.histogram)) 288 | .change(statusesSet, vega.changeset().remove(function(d) { return new Date(d.time) < cutOffTime; }).insert(statistics.statuses)) 289 | .run(); 290 | }); 291 | }; 292 | 293 | // Map the data format from the one returned by /scatter to one used by Vega Lite. 294 | scatter.processStatistics = function(statistics) { 295 | const statusLabels = { 296 | ok: "​\u200BOk", 297 | error: "​\u200B\u200BError", 298 | throttle: "​\u200B\u200B\u200BThrottle", 299 | fault: "​\u200B\u200B\u200B\u200BFault", 300 | }; 301 | const data = { 302 | histogram: [], 303 | statuses: [], 304 | index: [] 305 | }; 306 | let emitZeros = false; 307 | 308 | for (let slice of statistics) { 309 | const timestamp = new Date(0); 310 | 311 | timestamp.setUTCSeconds(slice.timestamp); 312 | 313 | if ((slice.index.start_time && slice.index.end_time) || emitZeros) { 314 | emitZeros = true; 315 | 316 | for (let entry of slice.histogram) { 317 | data.histogram.push({ 318 | time: timestamp.toISOString(), 319 | value: entry.value, 320 | count: entry.count, 321 | }); 322 | } 323 | 324 | // There doesn't seem to be any way to create empty datapoints with heatmaps. 325 | // Use zero value instead to ensure that the views stay in sync if the service 326 | // or entity not sending any traces for a period of time. 327 | if (slice.histogram.length == 0) { 328 | data.histogram.push({ 329 | time: timestamp.toISOString(), 330 | value: 0, 331 | count: 0, 332 | }); 333 | } 334 | 335 | for (let entry of slice.statuses) { 336 | data.statuses.push({ 337 | time: timestamp.toISOString(), 338 | request_rate: entry.request_rate, 339 | status: statusLabels[entry.status] ? statusLabels[entry.status] : entry.status 340 | }); 341 | } 342 | } 343 | 344 | if (slice.index.start_time && slice.index.end_time) { 345 | const startTime = new Date(0); 346 | const endTime = new Date(0); 347 | 348 | startTime.setUTCSeconds(slice.index.start_time); 349 | endTime.setUTCSeconds(slice.index.end_time); 350 | 351 | data.index.push({ 352 | time: timestamp.toISOString(), 353 | startTime: startTime.toISOString(), 354 | endTime: endTime.toISOString() 355 | }); 356 | } 357 | } 358 | 359 | return data; 360 | }; 361 | 362 | // Maintain a cache of visibile statistics entries to allow normalization of selected time and 363 | // response time ranges. 364 | // 365 | // Returns normalized new statistics entries used to incrementally update the view. 366 | scatter.mergeStatistics = function(rawStatistics, endTime) { 367 | const cutOffTime = scatter.cutOffTime(endTime); 368 | const statistics = scatter.processStatistics(rawStatistics); 369 | 370 | scatter.append = true; 371 | scatter.refreshedAt = new Date(); 372 | scatter.startTime = endTime; 373 | 374 | _.forEach(statistics, (v, k) => { 375 | if (!scatter.statistics[k]) { 376 | scatter.statistics[k] = []; 377 | } 378 | Array.prototype.push.apply(scatter.statistics[k], v); 379 | 380 | _.remove(scatter.statistics[k], o => { return new Date(o.time) < cutOffTime; }); 381 | }); 382 | 383 | return statistics; 384 | } 385 | 386 | scatter.fetch = function(path, entityName, startTime, endTime, parameters={}, callback) { 387 | let data = null; 388 | 389 | const esc = encodeURIComponent; 390 | const q = _.chain(parameters) 391 | .map((v, k) => `&${esc(k)}=${esc(v)}`) 392 | .join('') 393 | .value(); 394 | 395 | fetch(path + "?entity=" + esc(entityName) + "&startTime=" + startTime.getTime() / 1000 + "&endTime=" + endTime.getTime() / 1000 + q, { credentials: 'include' }).then(function(response) { 396 | if (!response.ok) { 397 | throw Error(response.statusText + " (" + response.status + ")"); 398 | } 399 | 400 | return response.json(); 401 | }).then(data => { 402 | scatter.error = null; 403 | 404 | callback(data); 405 | }).catch(function(error) { 406 | if (error instanceof TypeError && error.message == "Type error") { 407 | error = Error("Could not connect to the server"); 408 | } 409 | 410 | scatter.error = error; 411 | console.log(error); 412 | }); 413 | }; 414 | 415 | // Returns "safe end time" by rounding down to the nearest minute to avoid reading service graph 416 | // durations that are still being updated. 417 | scatter.safeEndTime = function() { 418 | const safeTime = new Date(Math.floor(Date.now() / (scatter.duration * 1.0)) * scatter.duration); 419 | 420 | if (new Date().getTime() - safeTime.getTime() < MINUTE) { 421 | return new Date(safeTime.getTime() - scatter.duration); 422 | } 423 | 424 | return safeTime; 425 | }; 426 | 427 | // Returns earliest start time for given the selected time range (either 3 hours or 7 days). 428 | scatter.cutOffTime = function(endTime) { 429 | return new Date(endTime.getTime() - scatter.timeSlots * scatter.duration) 430 | }; 431 | 432 | // Normalize response time range for **GetTraceSummaries** API. Since X-Ray histograms are 433 | // sparse and use non-linear scale the selected response time range has to be normalized before 434 | // used as **GetTraceSummaries** API range. 435 | // 436 | // Find closest previous and next buckets for the range and add or remove half of the delta 437 | // of the values between the selection and the previous / next buckets to expand the range. 438 | // 439 | // **Note:** this method mutates parameters object. 440 | scatter.correctResponseTimeRange = function(histogram, startTime, endTime, parameters) { 441 | console.log("before", parameters); 442 | 443 | let startIndex = null; 444 | let endIndex = null; 445 | const sortedHistogram = _.sortBy(histogram, o => { return o.value; }) 446 | 447 | for (let i = 0; i < sortedHistogram.length; i += 1) { 448 | const entry = sortedHistogram[i]; 449 | const entryTimestamp = new Date(entry.time); 450 | 451 | if (entryTimestamp >= startTime && entryTimestamp < endTime) { 452 | // console.log(entry); 453 | 454 | if (entry.value < parameters.responseTimeMin) { 455 | startIndex = i; 456 | } 457 | 458 | if (entry.value > parameters.responseTimeMax && endIndex == null) { 459 | endIndex = i; 460 | } 461 | } 462 | } 463 | 464 | if (startIndex) { 465 | const entry = sortedHistogram[startIndex]; 466 | parameters.responseTimeMin -= (parameters.responseTimeMin - entry.value) / 2; 467 | } else { 468 | delete parameters.responseTimeMin; 469 | } 470 | 471 | if (endIndex) { 472 | const entry = sortedHistogram[endIndex]; 473 | parameters.responseTimeMax += (entry.value - parameters.responseTimeMax) / 2; 474 | } else { 475 | delete parameters.responseTimeMax; 476 | } 477 | 478 | console.log("after", parameters); 479 | return parameters; 480 | }; 481 | 482 | // Return normalized response time range for **GetTraceSummaries** API by finding minimum 483 | // start time and maximum end time for the selected time range and service or edge by 484 | // scanning through all service graphs for the selected time ranges. 485 | scatter.correctTraceTimeRange = function(traceTimeRange) { 486 | console.log("before", traceTimeRange); 487 | 488 | let traceStartTime = new Date(traceTimeRange.startTime) 489 | let traceEndTime = new Date(traceTimeRange.endTime) 490 | 491 | for (let i = 0; i < scatter.statistics.index.length; i += 1) { 492 | const entry = scatter.statistics.index[i]; 493 | const entryTimestamp = new Date(entry.time); 494 | 495 | if (entryTimestamp >= traceStartTime && entryTimestamp < traceEndTime) { 496 | const entryStartTime = new Date(entry.startTime); 497 | const entryEndTime = new Date(entry.endTime); 498 | 499 | // console.log(entry); 500 | 501 | if (entryStartTime < traceStartTime) { 502 | traceStartTime = entryStartTime; 503 | } 504 | 505 | if (entryEndTime > traceEndTime) { 506 | traceEndTime = entryEndTime; 507 | } 508 | } 509 | } 510 | 511 | const correctedTimeRange = { 512 | startTime: traceStartTime, 513 | endTime: traceEndTime 514 | }; 515 | 516 | console.log("after", correctedTimeRange); 517 | return correctedTimeRange; 518 | }; 519 | -------------------------------------------------------------------------------- /static/javascript/select.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const select = {}; 4 | 5 | select.init = function() { 6 | select.initEntity(); 7 | select.initTimeSpan(); 8 | 9 | scatter.rebuild(); 10 | }; 11 | 12 | select.entityChange = function(entity) { 13 | console.log("entity", JSON.parse(entity)); 14 | settings.set("entity", entity); 15 | scatter.rebuild(); 16 | }; 17 | 18 | // Initialize service and edge selection 19 | select.initEntity = function() { 20 | const selectElement = document.getElementById("entitySelect"); 21 | const endTime = scatter.safeEndTime(); 22 | const startTime = new Date(endTime.getTime() - 5 * 60000); 23 | let optionSelected = false; 24 | 25 | scatter.fetch("/entities", null, startTime, endTime, {}, function(data) { 26 | const entities = data.entities; 27 | 28 | for (const entity of entities) { 29 | const entityAsJSON = JSON.stringify(entity); 30 | 31 | let label = null; 32 | 33 | if (entity.name) { 34 | label = entity.name; 35 | if (entity.type) { 36 | label += " (" + entity.type + ")" 37 | } 38 | } else { 39 | label = entity[0].name; 40 | if (entity[0].type == "client") { 41 | label = "client" 42 | } else if (entity[0].type) { 43 | label += " (" + entity[0].type + ")" 44 | } 45 | 46 | label += " \u2192 " 47 | 48 | label += entity[1].name; 49 | if (entity[1].type) { 50 | label += " (" + entity[1].type + ")" 51 | } 52 | } 53 | 54 | const selected = (settings.entity === entityAsJSON); 55 | selectElement.options[selectElement.options.length] = new Option(label, entityAsJSON, false, selected); 56 | 57 | if (selected) { 58 | optionSelected = true; 59 | } 60 | } 61 | 62 | selectElement.addEventListener('change', function (e) { 63 | select.entityChange(e.target.value); 64 | }); 65 | 66 | if (!optionSelected) { 67 | if (selectElement.options[0]) { 68 | settings.value = select.entityChange(selectElement.options[0].value); 69 | } else { 70 | settings.value = null; 71 | } 72 | } 73 | 74 | console.log("entity", selectElement.value != "" ? JSON.parse(selectElement.value) : "n/a"); 75 | }); 76 | }; 77 | 78 | // Initialize time span (3 hours or 7 days) selection 79 | select.initTimeSpan = function() { 80 | const selectElement = document.getElementById("timeSpanSelect"); 81 | const endTime = scatter.safeEndTime(); 82 | const startTime = new Date(endTime.getTime() - 5 * 60000); 83 | 84 | selectElement.options[0] = new Option("3 Hours", "threeHours", false, (settings.timeSpan == "threeHours")); 85 | selectElement.options[1] = new Option("7 Days", "sevenDays", false, (settings.timeSpan == "sevenDays")); 86 | 87 | selectElement.addEventListener('change', function (e) { 88 | const timeSpan = e.target.value; 89 | 90 | console.log("time span", timeSpan); 91 | settings.set("timeSpan", timeSpan); 92 | scatter.rebuild(); 93 | }); 94 | 95 | console.log("time span", selectElement.value); 96 | }; 97 | -------------------------------------------------------------------------------- /static/javascript/settings.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const DEFAULT_REGION = "us-east-1"; 4 | const DEFAULT_ENTITY = null 5 | const DEFAULT_TIMESPAN = "threeHours"; 6 | 7 | const settings = {}; 8 | 9 | settings.region = cookie.get("region", DEFAULT_REGION); 10 | settings.entity = cookie.get("entity", DEFAULT_ENTITY); 11 | settings.timeSpan = cookie.get("timeSpan", DEFAULT_TIMESPAN); 12 | settings.rebuild = false; 13 | 14 | settings.set = function(key, value) { 15 | settings[key] = value; 16 | cookie.set(key, value, 21); 17 | }; 18 | 19 | settings.init = function() { 20 | const makeRadioButton = function(name, value, text, checked) { 21 | let label = document.createElement("label"); 22 | let radio = document.createElement("input"); 23 | 24 | radio.type = "radio"; 25 | radio.name = name; 26 | radio.value = value; 27 | if (checked) { 28 | radio.checked = "checked"; 29 | } 30 | 31 | label.appendChild(radio); 32 | label.appendChild(document.createTextNode(text)); 33 | label.style.display = "block"; 34 | 35 | return label; 36 | } 37 | 38 | const regions = [ 39 | "us-east-1", 40 | "us-west-2", 41 | "eu-west-1", 42 | "us-west-1", 43 | "eu-west-2", 44 | "ap-south-1", 45 | "ap-southeast-1", 46 | "us-east-2", 47 | "eu-central-1", 48 | "sa-east-1", 49 | "ap-northeast-1", 50 | "ap-northeast-2", 51 | "ap-southeast-2", 52 | "ca-central-1", 53 | ] 54 | 55 | for (const region of regions) { 56 | let button = makeRadioButton("region", region, region, (settings.region == region)); 57 | document.getElementById("regions").appendChild(button); 58 | } 59 | 60 | // Event listener for region selection (https://stackoverflow.com/questions/8838648/onchange-event-handler-for-radio-button-input-type-radio-doesnt-work-as-one?rq=1) 61 | const regionForm = document.getElementById('regionFormDiv'); 62 | let prev = null; 63 | for (let i = 0; i < regionForm.length; i++) { 64 | regionForm[i].onclick = function() { 65 | if(this !== prev) { 66 | prev = this; 67 | settings.set("region", this.value); 68 | settings.rebuild = true; 69 | } 70 | }; 71 | } 72 | }; 73 | -------------------------------------------------------------------------------- /static/javascript/traces.js: -------------------------------------------------------------------------------- 1 | "use strict"; 2 | 3 | const traces = {}; 4 | 5 | traces.table = null; 6 | 7 | traces.tableOptions = function() { 8 | return { 9 | columns: [ 10 | { data: "http_url" }, 11 | { 12 | data: "response_time", 13 | render: $.fn.dataTable.render.number(',', '.', 3, '', ' s'), 14 | }, 15 | { data: "http_status" }, 16 | { 17 | data: "users", 18 | render: function (data, type, row) { 19 | return data.join(", "); 20 | }, 21 | }, 22 | { 23 | data: "timeline_link", 24 | orderable: false, 25 | render: function (data, type, row) { 26 | return "" + "Console" + ''; 27 | }, 28 | width: "10px" 29 | }, 30 | { 31 | className: 'details-control', 32 | orderable: false, 33 | data: null, 34 | defaultContent: '', 35 | width: "10px" 36 | }, 37 | { data: "timestamp" }, 38 | { data: "annotations" }, 39 | { data: "trace_id" }, 40 | ], 41 | paging: false, 42 | deferRender: true, 43 | select: { 44 | style: 'single', 45 | items: 'cell', 46 | info: false, 47 | }, 48 | columnDefs: [ 49 | { 50 | visible: false, 51 | searchable: true, 52 | targets: [6, 7, 8] // timestamp, annotations and trace_id 53 | } 54 | ], 55 | order: [[6, "asc"]], 56 | dom: "lrtip", 57 | tabIndex: -1, 58 | language: { 59 | emptyTable: "Select a time range to view traces" 60 | }, 61 | // rowGroup: { 62 | // dataSrc: "http_status" 63 | // } 64 | }; 65 | }; 66 | 67 | // Initialize https://datatables.net table used to display traces that match a selection. 68 | traces.init = function() { 69 | traces.table = $('#traceTable').DataTable(traces.tableOptions()); 70 | 71 | $('#traceTable tbody').on('click', 'td.details-control', function() { 72 | var tr = $(this).closest('tr'); 73 | var row = traces.table.row( tr ); 74 | 75 | if (row.child.isShown() ) { 76 | row.child.hide(); 77 | tr.removeClass('shown'); 78 | } 79 | else { 80 | row.child(traces.formatDetailsRow(row.data())).show(); 81 | tr.addClass('shown'); 82 | } 83 | }); 84 | }; 85 | 86 | traces.clear = function() { 87 | if (traces.table) { 88 | traces.table.clear().columns.adjust().draw(); 89 | } 90 | }; 91 | 92 | traces.update = function(data) { 93 | document.getElementById('traceTable_wrapper').style.display = ""; 94 | document.getElementById('tracesLoading').style.display = "none"; 95 | 96 | console.log(data); 97 | 98 | traces.table.clear().rows.add(data.traces).columns.adjust().draw(); 99 | }; 100 | 101 | traces.showProgress = function() { 102 | document.getElementById('traceTable_wrapper').style.display = "none"; 103 | document.getElementById('tracesLoading').style.display = "block"; 104 | traces.clear(); 105 | }; 106 | 107 | traces.formatDetailsRow = function(d) { 108 | console.log(d); 109 | const annotationsFormatted = _.chain(d.annotations) 110 | .mapKeys(function(v, k) { 111 | v = v.join('
') 112 | return "" + k + "" + "→" + "" + v + ""; 113 | }) 114 | .keys(); 115 | 116 | return ''+ 117 | ''+ 118 | ''+ 119 | ''+ 120 | ''+ 121 | ''+ 122 | ''+ 123 | ''+ 124 | ''+ 125 | ''+ 126 | ''+ 127 | ''+ 128 | ''+ 129 | ''+ 130 | ''+ 131 | ''+ 132 | ''+ 133 | ''+ 134 | ''+ 135 | ''+ 136 | ''+ 137 | '
Trace ID'+d.trace_id+'
URL'+d.http_url+'
HTTP Status'+d.http_status+'
Timestamp'+d.timestamp+'
Annotations'+annotationsFormatted.join('')+'
'; 138 | }; 139 | -------------------------------------------------------------------------------- /templates/scatter.html: -------------------------------------------------------------------------------- 1 | 2 | 3 | 4 | 5 | 6 | X-Ray Scatter Demo 7 | 8 | 9 | 11 | 12 | 177 | 178 | 179 | 180 | 198 | 199 |
200 |
201 |

Please wait...

202 |
203 |
204 |
205 |
206 |

Fetching matching traces...

207 |
208 | 209 | 210 | 211 | 212 | 213 | 214 | 215 | 216 | 217 | 218 | 219 | 220 |
Traces
URLResponse Time (s)HTTP StatusUsersConsole
221 |
222 |
223 | 224 |
225 |
226 | 227 | ✕ 228 | 229 |

230 |

231 |

Region

232 |
233 |
234 |
235 | 236 |

237 |

238 |
239 | 240 | 241 | 242 | 243 | 244 | 245 | 246 | 247 | 248 | 249 | 250 | 251 | 252 | 253 | 254 | 317 | 318 | 319 | --------------------------------------------------------------------------------