├── .gitattributes ├── .gitignore ├── .travis.yml ├── LICENSE.md ├── README.md ├── lib └── aws-cloudwatch-statsd-backend.js └── package.json /.gitattributes: -------------------------------------------------------------------------------- 1 | # Auto detect text files and perform LF normalization 2 | * text=auto 3 | 4 | # Custom for Visual Studio 5 | *.cs diff=csharp 6 | *.sln merge=union 7 | *.csproj merge=union 8 | *.vbproj merge=union 9 | *.fsproj merge=union 10 | *.dbproj merge=union 11 | 12 | # Standard to msysgit 13 | *.doc diff=astextplain 14 | *.DOC diff=astextplain 15 | *.docx diff=astextplain 16 | *.DOCX diff=astextplain 17 | *.dot diff=astextplain 18 | *.DOT diff=astextplain 19 | *.pdf diff=astextplain 20 | *.PDF diff=astextplain 21 | *.rtf diff=astextplain 22 | *.RTF diff=astextplain 23 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | ################# 2 | ## Eclipse 3 | ################# 4 | 5 | *.pydevproject 6 | .project 7 | .metadata 8 | bin/ 9 | tmp/ 10 | *.tmp 11 | *.bak 12 | *.swp 13 | *~.nib 14 | local.properties 15 | .classpath 16 | .settings/ 17 | .loadpath 18 | 19 | # External tool builders 20 | .externalToolBuilders/ 21 | 22 | # Locally stored "Eclipse launch configurations" 23 | *.launch 24 | 25 | # CDT-specific 26 | .cproject 27 | 28 | # PDT-specific 29 | .buildpath 30 | 31 | 32 | ################# 33 | ## Visual Studio 34 | ################# 35 | 36 | ## Ignore Visual Studio temporary files, build results, and 37 | ## files generated by popular Visual Studio add-ons. 38 | 39 | # User-specific files 40 | *.suo 41 | *.user 42 | *.sln.docstates 43 | 44 | # Build results 45 | [Dd]ebug/ 46 | [Rr]elease/ 47 | *_i.c 48 | *_p.c 49 | *.ilk 50 | *.meta 51 | *.obj 52 | *.pch 53 | *.pdb 54 | *.pgc 55 | *.pgd 56 | *.rsp 57 | *.sbr 58 | *.tlb 59 | *.tli 60 | *.tlh 61 | *.tmp 62 | *.vspscc 63 | .builds 64 | *.dotCover 65 | 66 | ## TODO: If you have NuGet Package Restore enabled, uncomment this 67 | #packages/ 68 | 69 | # Visual C++ cache files 70 | ipch/ 71 | *.aps 72 | *.ncb 73 | *.opensdf 74 | *.sdf 75 | 76 | # Visual Studio profiler 77 | *.psess 78 | *.vsp 79 | 80 | # ReSharper is a .NET coding add-in 81 | _ReSharper* 82 | 83 | # Installshield output folder 84 | [Ee]xpress 85 | 86 | # DocProject is a documentation generator add-in 87 | DocProject/buildhelp/ 88 | DocProject/Help/*.HxT 89 | DocProject/Help/*.HxC 90 | DocProject/Help/*.hhc 91 | DocProject/Help/*.hhk 92 | DocProject/Help/*.hhp 93 | DocProject/Help/Html2 94 | DocProject/Help/html 95 | 96 | # Click-Once directory 97 | publish 98 | 99 | # Others 100 | [Bb]in 101 | [Oo]bj 102 | sql 103 | TestResults 104 | *.Cache 105 | ClientBin 106 | stylecop.* 107 | ~$* 108 | *.dbmdl 109 | Generated_Code #added for RIA/Silverlight projects 110 | 111 | # Backup & report files from converting an old project file to a newer 112 | # Visual Studio version. Backup files are not needed, because we have git ;-) 113 | _UpgradeReport_Files/ 114 | Backup*/ 115 | UpgradeLog*.XML 116 | 117 | 118 | 119 | ############ 120 | ## Windows 121 | ############ 122 | 123 | # Windows image file caches 124 | Thumbs.db 125 | 126 | # Folder config file 127 | Desktop.ini 128 | 129 | 130 | ############# 131 | ## Python 132 | ############# 133 | 134 | *.py[co] 135 | 136 | # Packages 137 | *.egg 138 | *.egg-info 139 | dist 140 | build 141 | eggs 142 | parts 143 | bin 144 | var 145 | sdist 146 | develop-eggs 147 | .installed.cfg 148 | 149 | # Installer logs 150 | pip-log.txt 151 | 152 | # Unit test / coverage reports 153 | .coverage 154 | .tox 155 | 156 | #Translations 157 | *.mo 158 | 159 | #Mr Developer 160 | .mr.developer.cfg 161 | 162 | # Mac crap 163 | .DS_Store 164 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | 3 | deploy: 4 | provider: npm 5 | email: j.delivuk@gmail.com 6 | api_key: $npm_api_key 7 | -------------------------------------------------------------------------------- /LICENSE.md: -------------------------------------------------------------------------------- 1 | The MIT License (MIT) 2 | 3 | Copyright (c) 2012 Martin Camitz 4 | 5 | Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions: 6 | 7 | The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software. 8 | 9 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | # StatsD backend for AWS CloudWatch 2 | 3 | ## Overview 4 | 5 | [StatsD](https://github.com/etsy/statsd) is a smart Node.js package that collects and aggregates statistics from differents apps sent over the UDP protocol. At a set time interval it forwards the aggregated data to a configured backend. It is pluggable with several backends available, the most popular being [Graphite](https://github.com/graphite-project/graphite-web), a python/django monitoring tool. 6 | 7 | With **aws-cloudwatch-statsd-backend** you can replace Graphite in favour of [AWS Cloudwatch](http://aws.amazon.com/cloudwatch/) for your monitoring purposes, appropriate for sites on the Amazon EC2 cloud. 8 | 9 | Counters, timers, gauges and sets are all supported. 10 | 11 | ## Installation 12 | 13 | You need node.js installed on your system aswell as StatsD. Follow the instructions on their sites or see this [blog post/tutorial](http://blog.simpletask.se/aws-clouwadwatch-statsd-backend/) on how to install these components on a Windows system. 14 | 15 | The CloudWatch backend is an npm package that can be installed with the npm command which comes with your installation of node.js. Go to the [npm site](https://npmjs.org/) for more information. 16 | 17 | npm install aws-cloudwatch-statsd-backend 18 | 19 | The package has two depdencies that should be installed automatically, [awssum](https://npmjs.org/package/awssum) and [fmt](https://npmjs.org/package/fmt). Awssum is a node.js package encapsulating the AWS API. 20 | 21 | ## Configuration 22 | 23 | The StatsD and its backends are configured in a json object placed in a file supplied to StatsD at the command line. For example, start StatsD with the following. 24 | 25 | node ./stats.js ./myConfig.js 26 | 27 | The following demonstrates the minimum config for the CloudWatch backend. 28 | 29 | { 30 | backends: [ "aws-cloudwatch-statsd-backend" ], 31 | cloudwatch: 32 | { 33 | accessKeyId: 'YOUR_ACCESS_KEY_ID', 34 | secretAccessKey:'YOUR_SECRET_ACCESS_KEY', 35 | region:"YOUR_REGION" 36 | } 37 | } 38 | 39 | The access keys can be you personal credentials to AWS but it is highly recommended to create an ad hoc user via Amazon's IAM service and use those credentials. 40 | 41 | The region is for example `EU_WEST_1` or `US_EAST_1`. Region should be in capital letter and separated by `_` instead of `-`. 42 | 43 | The above will create a metric with the default namespace, AwsCloudWatchStatsdBackend, and send an http request to CloudWatch via awssum. 44 | 45 | See the CloudWatch [documentation](http://docs.amazonwebservices.com/AmazonCloudWatch/latest/DeveloperGuide/cloudwatch_concepts.html) for more information on these concepts. 46 | 47 | The metric name, unit and value depends on what you send StatsD with your UDP request. For example, given 48 | 49 | gorets:1|c 50 | 51 | the Unit will be Counter, the metric name gorets. The value will be the aggregated count as calculated by StatsD. 52 | 53 | *ms* corresponds the unit *Milliseconds*. *s and *g* to *None*. 54 | 55 | **Warning** Indescriminate use of CloudWatch metrics can quickly become costly. Amazon charges 50 cents for each combination of namepace, metric name and dimension per month. However, the 10 first per month are free. 56 | 57 | ## Additional configuration options 58 | 59 | The cloudwatch backend provides ways to override the name and namespace by cofiguration. It can also capture these components from the bucket name. 60 | 61 | The following overrides the default and any provided namespace or metric name with the specified. 62 | 63 | { 64 | backends: [ "aws-cloudwatch-statsd-backend" ], 65 | cloudwatch: 66 | { 67 | accessKeyId: 'YOUR_ACCESS_KEY_ID', 68 | secretAccessKey: 'YOUR_SECRET_ACCESS_KEY', 69 | region: 'YOUR_REGION', 70 | namespace: 'App/Controller/Action', 71 | metricName: 'Request' 72 | } 73 | } 74 | 75 | Using the option *processKeyForNamespace* (default is false) you can parse the bucket name for namespace in addition to metric name. The backend will use the last component of a bucket name comprised of slash (/), dot (.) or dash (-) separated parts as the metric name. The remaining leading parts will be used as namespace. Separators will be replaced with slashes (/). 76 | 77 | { 78 | backends: [ "aws-cloudwatch-statsd-backend" ], 79 | cloudwatch: 80 | { 81 | accessKeyId: 'YOUR_ACCESS_KEY_ID', 82 | secretAccessKey: 'YOUR_SECRET_ACCESS_KEY', 83 | region: 'YOUR_REGION', 84 | processKeyForNames:true 85 | } 86 | } 87 | 88 | For example, sending StatsD the following 89 | 90 | App.Controller.Action.Request:1|c 91 | 92 | is will produce the equivalent to the former configuration example. Note that both will be suppressed if overriden as in the former configuration example. 93 | 94 | ## Whitelisting Metrics 95 | 96 | Using cloudwatch will incur a cost for each metric sent. In order to control your costs, you can optionally whitelist (by full metric name) those metrics sent to cloudwatch. For example: 97 | 98 | { 99 | backends: [ "aws-cloudwatch-statsd-backend" ], 100 | cloudwatch: 101 | { 102 | accessKeyId: 'YOUR_ACCESS_KEY_ID', 103 | secretAccessKey: 'YOUR_SECRET_ACCESS_KEY', 104 | region: 'YOUR_REGION', 105 | whitelist: ['YOUR_FULL_METRIC_NAME'] 106 | } 107 | } 108 | 109 | The above configuration would only sent the metric named 'YOUR_FULL_METRIC_NAME' to cloudwatch. As this is an array, you can specify multiple metrics. This is useful if you are using multiple backends e.g. mysql backend and want to send some metrics cloudwatch (due to the associated cost) and all the metrics together to another backend. It is also useful if you want to limit the metrics you use in cloudwatch to those that raise alarms as part of your wider AWS hosted system. 110 | 111 | ## Using AWS Roles to obtain credentials 112 | 113 | A preferable approach to obtaining account credentials is instead to query the Metadata Service to obtain IAM security credentials for a given role. If iamRole is set to 'any' then any available credentials found on the metadata service will instead be used. For example: 114 | 115 | { 116 | backends: [ "aws-cloudwatch-statsd-backend" ], 117 | cloudwatch: 118 | { 119 | iamRole: 'YOUR_ROLE_NAME', 120 | region: 'YOUR_REGION', 121 | whitelist: ['YOUR_FULL_METRIC_NAME'] 122 | } 123 | } 124 | 125 | ## Multi-region support 126 | 127 | If you wish to send cloudwatch metrics to multiple regions at once, instead of 128 | 129 | { 130 | backends: [ "aws-cloudwatch-statsd-backend" ], 131 | cloudwatch: 132 | { 133 | accessKeyId: 'YOUR_ACCESS_KEY_ID', 134 | secretAccessKey:'YOUR_SECRET_ACCESS_KEY', 135 | region:"YOUR_REGION" 136 | } 137 | } 138 | 139 | you can use the `instances` key under `cloudwatch` to configure a list of configurations. 140 | 141 | { 142 | backends: ["aws-cloudwatch-statsd-backend"], 143 | cloudwatch: { 144 | instances: [{ 145 | accessKeyId: 'YOUR_ACCESS_KEY_ID', 146 | secretAccessKey: 'YOUR_SECRET_ACCESS_KEY', 147 | region: "YOUR_REGION_1", 148 | whitelist: ['YOUR_FULL_METRIC_NAME1'] 149 | }, { 150 | accessKeyId: 'YOUR_ACCESS_KEY_ID', 151 | secretAccessKey: 'YOUR_SECRET_ACCESS_KEY', 152 | region: "YOUR_REGION_2", 153 | whitelist: ['YOUR_FULL_METRIC_NAME2'] 154 | }] 155 | } 156 | } 157 | 158 | 159 | ## Tutorial 160 | 161 | This project was launched with a following [blog post/tutorial](http://blog.simpletask.se/post/aggregating-monitoring-statistics-for-aws-cloudwatch) describing the implementation chain from log4net to Cloudwatch on a Windows system. 162 | 163 | Also in the series: 164 | 165 | [Improving the CloudWatch Appender](http://blog.simpletask.se/post/improving-cloudwatch-appender) 166 | 167 | [A CloudWatch Appender for log4net](http://blog.simpletask.se/post/awscloudwatch-log4net-appender) 168 | 169 | [![endorse](http://api.coderwall.com/camitz/endorsecount.png)](http://coderwall.com/camitz) 170 | -------------------------------------------------------------------------------- /lib/aws-cloudwatch-statsd-backend.js: -------------------------------------------------------------------------------- 1 | var util = require('util'); 2 | var AWS = require('aws-sdk'); 3 | 4 | function CloudwatchBackend(startupTime, config, emitter) { 5 | var self = this; 6 | 7 | this.config = config || {}; 8 | AWS.config = this.config; 9 | 10 | function setEmitter() { 11 | self.cloudwatch = new AWS.CloudWatch(self.config); 12 | emitter.on('flush', function(timestamp, metrics) { self.flush(timestamp, metrics); }); 13 | } 14 | 15 | // if iamRole is set attempt to fetch credentials from the Metadata Service 16 | if (this.config.iamRole) { 17 | if (this.config.iamRole == 'any') { 18 | // If the iamRole is set to any, then attempt to fetch any available credentials 19 | ms = new AWS.EC2MetadataCredentials(); 20 | ms.refresh(function(err) { 21 | if (err) { console.log('Failed to fetch IAM role credentials: ' + err); } 22 | self.config.credentials = ms; 23 | setEmitter(); 24 | }); 25 | } else { 26 | // however if it's set to specify a role, query it specifically. 27 | ms = new AWS.MetadataService(); 28 | ms.request('/latest/meta-data/iam/security-credentials/' + this.config.iamRole, function(err, rdata) { 29 | var data = JSON.parse(rdata); 30 | 31 | if (err) { console.log('Failed to fetch IAM role credentials: ' + err); } 32 | self.config.credentials = new AWS.Credentials(data.AccessKeyId, data.SecretAccessKey, data.Token); 33 | setEmitter(); 34 | }); 35 | } 36 | } else { 37 | setEmitter(); 38 | } 39 | }; 40 | 41 | CloudwatchBackend.prototype.processKey = function(key) { 42 | var parts = key.split(/[\.\/-]/); 43 | 44 | return { 45 | metricName: parts[parts.length - 1], 46 | namespace: parts.length > 1 ? parts.splice(0, parts.length - 1).join("/") : null 47 | }; 48 | }; 49 | 50 | CloudwatchBackend.prototype.isBlacklisted = function(key) { 51 | 52 | var blacklisted = false; 53 | 54 | // First check if key is whitelisted 55 | if (this.config.whitelist && this.config.whitelist.length > 0 && this.config.whitelist.indexOf(key) >= 0) { 56 | // console.log("Key (counter) " + key + " is whitelisted"); 57 | return false; 58 | } 59 | 60 | if (this.config.blacklist && this.config.blacklist.length > 0) { 61 | for (var i = 0; i < this.config.blacklist.length; i++) { 62 | if (key.indexOf(this.config.blacklist[i]) >= 0) { 63 | blacklisted = true; 64 | break; 65 | } 66 | } 67 | } 68 | return blacklisted; 69 | }; 70 | 71 | CloudwatchBackend.prototype.chunk = function(arr, chunkSize) { 72 | 73 | var groups = [], 74 | i; 75 | for (i = 0; i < arr.length; i += chunkSize) { 76 | groups.push(arr.slice(i, i + chunkSize)); 77 | } 78 | return groups; 79 | }; 80 | 81 | CloudwatchBackend.prototype.batchSend = function(currentMetricsBatch, namespace) { 82 | 83 | // send off the array (instead of one at a time) 84 | if (currentMetricsBatch.length > 0) { 85 | 86 | // Chunk into groups of 20 87 | var chunkedGroups = this.chunk(currentMetricsBatch, 20); 88 | 89 | for (var i = 0, len = chunkedGroups.length; i < len; i++) { 90 | this.cloudwatch.putMetricData({ 91 | MetricData: chunkedGroups[i], 92 | Namespace: namespace 93 | }, function(err, data) { 94 | if (err) { 95 | // log an error 96 | console.log(util.inspect(err)); 97 | } else { 98 | // Success 99 | // console.log(util.inspect(data)); 100 | } 101 | }); 102 | } 103 | } 104 | }; 105 | 106 | CloudwatchBackend.prototype.flush = function(timestamp, metrics) { 107 | 108 | console.log('Flushing metrics at ' + new Date(timestamp * 1000).toISOString()); 109 | 110 | var counters = metrics.counters; 111 | var gauges = metrics.gauges; 112 | var timers = metrics.timers; 113 | var sets = metrics.sets; 114 | 115 | // put all currently accumulated counter metrics into an array 116 | var currentCounterMetrics = []; 117 | var namespace = "AwsCloudWatchStatsdBackend"; 118 | for (key in counters) { 119 | if (key.indexOf('statsd.') == 0) 120 | continue; 121 | 122 | if (this.isBlacklisted(key)) { 123 | continue; 124 | } 125 | 126 | var names = this.config.processKeyForNamespace ? this.processKey(key) : {}; 127 | namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend"; 128 | var metricName = this.config.metricName || names.metricName || key; 129 | 130 | currentCounterMetrics.push({ 131 | MetricName: metricName, 132 | Unit: 'Count', 133 | Timestamp: new Date(timestamp * 1000).toISOString(), 134 | Value: counters[key] 135 | }); 136 | } 137 | 138 | this.batchSend(currentCounterMetrics, namespace); 139 | 140 | // put all currently accumulated timer metrics into an array 141 | var currentTimerMetrics = []; 142 | for (key in timers) { 143 | if (timers[key].length > 0) { 144 | 145 | if (this.isBlacklisted(key)) { 146 | continue; 147 | } 148 | 149 | var values = timers[key].sort(function(a, b) { 150 | return a - b; 151 | }); 152 | var count = values.length; 153 | var min = values[0]; 154 | var max = values[count - 1]; 155 | 156 | var cumulativeValues = [min]; 157 | for (var i = 1; i < count; i++) { 158 | cumulativeValues.push(values[i] + cumulativeValues[i - 1]); 159 | } 160 | 161 | var sum = min; 162 | var mean = min; 163 | var maxAtThreshold = max; 164 | 165 | var message = ""; 166 | 167 | var key2; 168 | 169 | sum = cumulativeValues[count - 1]; 170 | mean = sum / count; 171 | 172 | var names = this.config.processKeyForNamespace ? this.processKey(key) : {}; 173 | namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend"; 174 | var metricName = this.config.metricName || names.metricName || key; 175 | 176 | currentTimerMetrics.push({ 177 | MetricName: metricName, 178 | Unit: 'Milliseconds', 179 | Timestamp: new Date(timestamp * 1000).toISOString(), 180 | StatisticValues: { 181 | Minimum: min, 182 | Maximum: max, 183 | Sum: sum, 184 | SampleCount: count 185 | } 186 | }); 187 | } 188 | } 189 | 190 | this.batchSend(currentTimerMetrics, namespace); 191 | 192 | // put all currently accumulated gauge metrics into an array 193 | var currentGaugeMetrics = []; 194 | for (key in gauges) { 195 | 196 | if (this.isBlacklisted(key)) { 197 | continue; 198 | } 199 | 200 | var names = this.config.processKeyForNamespace ? this.processKey(key) : {}; 201 | namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend"; 202 | var metricName = this.config.metricName || names.metricName || key; 203 | 204 | currentGaugeMetrics.push({ 205 | MetricName: metricName, 206 | Unit: 'None', 207 | Timestamp: new Date(timestamp * 1000).toISOString(), 208 | Value: gauges[key] 209 | }); 210 | } 211 | 212 | this.batchSend(currentGaugeMetrics, namespace); 213 | 214 | // put all currently accumulated set metrics into an array 215 | var currentSetMetrics = []; 216 | for (key in sets) { 217 | 218 | if (this.isBlacklisted(key)) { 219 | continue; 220 | } 221 | 222 | var names = this.config.processKeyForNamespace ? this.processKey(key) : {}; 223 | namespace = this.config.namespace || names.namespace || "AwsCloudWatchStatsdBackend"; 224 | var metricName = this.config.metricName || names.metricName || key; 225 | 226 | currentSetMetrics.push({ 227 | MetricName: metricName, 228 | Unit: 'None', 229 | Timestamp: new Date(timestamp * 1000).toISOString(), 230 | Value: sets[key].values().length 231 | }); 232 | } 233 | 234 | this.batchSend(currentSetMetrics, namespace); 235 | }; 236 | 237 | exports.init = function(startupTime, config, events) { 238 | var cloudwatch = config.cloudwatch || {}; 239 | var instances = cloudwatch.instances || [cloudwatch]; 240 | for (key in instances) { 241 | instanceConfig = instances[key]; 242 | console.log("Starting cloudwatch reporter instance in region:", instanceConfig.region); 243 | var instance = new CloudwatchBackend(startupTime, instanceConfig, events); 244 | } 245 | return true; 246 | }; 247 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "author": "Martin Camitz", 3 | "description": "A backend for StatsD to emit stats to Amazon's AWS CloudWatch.", 4 | "name": "aws-cloudwatch-statsd-backend", 5 | "version": "1.3.0", 6 | "main": "lib/aws-cloudwatch-statsd-backend.js", 7 | "dependencies": { 8 | "aws-sdk": "~2.82" 9 | }, 10 | "devDependencies": {}, 11 | "optionalDependencies": {}, 12 | "engines": { 13 | "node": "*" 14 | }, 15 | "repository": { 16 | "type": "git", 17 | "url": "https://github.com/camitz/aws-cloudwatch-statsd-backend.git" 18 | }, 19 | "keywords": [ 20 | "statsd", 21 | "cloudwatch", 22 | "aws", 23 | "amazon", 24 | "cloudwatchappender" 25 | ], 26 | "license": {"type": "MIT", "url": "https://github.com/camitz/aws-cloudwatch-statsd-backend/blob/master/LICENSE.md"} 27 | } 28 | --------------------------------------------------------------------------------