├── .gitignore ├── README.md ├── index.js ├── package.json └── test └── test.js /.gitignore: -------------------------------------------------------------------------------- 1 | lib-cov 2 | *.seed 3 | *.log 4 | *.csv 5 | *.dat 6 | *.out 7 | *.pid 8 | *.gz 9 | 10 | pids 11 | logs 12 | results 13 | 14 | npm-debug.log 15 | 16 | node_modules/* 17 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | node-redis-sentinel 2 | =================== 3 | 4 | **I recommend using [ioredis](https://github.com/luin/ioredis) rather than this library. It has inbuilt sentinel support and is likely much more robust** 5 | 6 | Wrapper around [node_redis](https://github.com/mranney/node_redis) creating a client pointing at the master server which autoupdates when the master goes down. 7 | 8 | ```javascript 9 | var sentinel = require('redis-sentinel'); 10 | 11 | // List the sentinel endpoints 12 | var endpoints = [ 13 | {host: '127.0.0.1', port: 26379}, 14 | {host: '127.0.0.1', port: 26380} 15 | ]; 16 | 17 | var opts = {}; // Standard node_redis client options 18 | var masterName = 'mymaster'; 19 | 20 | // masterName and opts are optional - masterName defaults to 'mymaster' 21 | var redisClient = sentinel.createClient(endpoints, masterName, opts); 22 | 23 | // redisClient is a normal redis client, except that if the master goes down 24 | // it will keep checking the sentinels for a new master and then connect to that. 25 | // No need to monitor for reconnects etc - everything handled transparently 26 | // Anything that persists over the normal node_redis reconnect will persist here. 27 | // Anything that doesn't, won't. 28 | 29 | // An equivalent way of doing the above (if you don't want to have to pass the endpoints around all the time) is 30 | var Sentinel = sentinel.Sentinel(endpoints); 31 | var masterClient = Sentinel.createClient(masterName, opts); 32 | ``` 33 | 34 | ## Connection to slaves or the sentinel itself ## 35 | You can get a connection to a slave (chosen at random) or the first available sentinel from the endpoints by passing in the `role` attribute in the options. E.g. 36 | 37 | ```javascript 38 | // The master is the default case if no role is specified. 39 | var masterClient = sentinel.createClient(endpoints, masterName, {role: 'master'}); 40 | var slaveClient = sentinel.createClient(endpoints, masterName, {role: 'slave'}); 41 | var sentinelClient = sentinel.createClient(endpoints, {role: 'sentinel'}); 42 | ``` 43 | 44 | Where you should also transparently get a reconnection to a new slave/sentinel if the existing one goes down. 45 | 46 | ## TODO ## 47 | * We could probably be cleverer with reconnects etc. and there may be issues with the error handling 48 | 49 | ## Licence ## 50 | MIT 51 | 52 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var redis = require('redis'), 2 | net = require('net'), 3 | when = require('when'); 4 | 5 | function Sentinel(endpoints) { 6 | 7 | // Instantiate if needed 8 | if (!(this instanceof Sentinel)) { 9 | return new Sentinel(endpoints); 10 | } 11 | 12 | this.endpoints = endpoints; 13 | this.clients = []; 14 | this.pubsub = []; 15 | } 16 | 17 | /** 18 | * Create a client 19 | * @param {String} masterName the name of the master. Defaults to mymaster 20 | * @param {Object} opts standard redis client options (optional) 21 | * @return {RedisClient} the RedisClient for the desired endpoint 22 | */ 23 | Sentinel.prototype.createClient = function(masterName, opts) { 24 | // When the client is ready create another client and subscribe to the 25 | // switch-master event. Then any time there is a message on the channel it 26 | // must be a master change, so reconnect all clients. This avoids combining 27 | // the pub/sub client with the normal client and interfering with whatever 28 | // the user is trying to do. 29 | if (this.pubsub.length == 0) { 30 | var self = this; 31 | var pubsubOpts = {}; 32 | pubsubOpts.role = "sentinel"; 33 | pubsubClient = this.createClientInternal(masterName, pubsubOpts); 34 | pubsubClient.subscribe("+switch-master", function(error) { 35 | if (error) { 36 | console.error("Unable to subscribe to Sentinel PUBSUB"); 37 | } 38 | }); 39 | pubsubClient.on("message", function(channel, message) { 40 | console.warn("Received +switch-master message from Redis Sentinel.", 41 | " Reconnecting clients."); 42 | self.reconnectAllClients(); 43 | }); 44 | pubsubClient.on("error", function(error) {}); 45 | self.pubsub.push(pubsubClient); 46 | } 47 | return this.createClientInternal(masterName, opts); 48 | } 49 | 50 | Sentinel.prototype.createClientInternal = function(masterName, opts) { 51 | if (typeof masterName !== 'string') { 52 | opts = masterName; 53 | masterName = 'mymaster'; 54 | } 55 | 56 | opts = opts || {}; 57 | var role = opts.role || 'master'; 58 | 59 | var endpoints = this.endpoints; 60 | 61 | 62 | var netClient = new net.Socket(); 63 | var client = new redis.RedisClient(netClient, opts); 64 | this.clients.push(client); 65 | 66 | var self = this; 67 | 68 | client.on('end', function() { 69 | // if we're purposefully ending, forget us 70 | if (this.closing) { 71 | var index = self.clients.indexOf(this); 72 | if (index !== -1) { 73 | self.clients.splice(index, 1); 74 | } 75 | } 76 | }); 77 | 78 | function connectClient(resolver) { 79 | return function(err, host, port) { 80 | if (err) { 81 | return client.emit('error', err); 82 | } 83 | 84 | var connectionOption = { 85 | port: port, 86 | host: host 87 | }; 88 | client.connectionOption = connectionOption; 89 | client.stream.connect(connectionOption.port, connectionOption.host); 90 | 91 | // Hijack the emit method so that we can get in there and 92 | // do any reconnection on errors, before raising it up the 93 | // stack... 94 | var oldEmit = client.emit; 95 | client.emit = function(eventName) { 96 | 97 | // Has an error been hit? 98 | if (eventName === 'error') { 99 | hitError.apply(null, arguments); 100 | } else { 101 | // Not an error - call the real emit... 102 | oldEmit.apply(client, arguments); 103 | } 104 | }; 105 | 106 | client.on('reconnecting', refreshEndpoints); 107 | 108 | function refreshEndpoints() { 109 | client.connectionOption.port = ""; 110 | client.connectionOption.host = ""; 111 | resolver(self.endpoints, masterName, function(_err, ip, port) { 112 | if (_err) { 113 | oldEmit.call(client, 'error', _err); 114 | } else { 115 | // Try reconnecting - remove the old stream first. 116 | client.stream.end(); 117 | 118 | client.connectionOption.port = port; 119 | client.connectionOption.host = ip; 120 | client.connection_gone("sentinel induced refresh"); 121 | } 122 | }); 123 | } 124 | 125 | // Crude but may do for now. On error re-resolve the master 126 | // and retry the connection 127 | function hitError(eventName, err) { 128 | 129 | var _args = arguments; 130 | function reemit() { 131 | oldEmit.apply(client, _args); 132 | } 133 | 134 | // If we are still connected then reraise the error - thats 135 | // not what we are here to handle 136 | if (client.connected) { return reemit(); } 137 | 138 | // In the background the client is going to keep trying to reconnect 139 | // and this error will keep getting raised - lets just keep trying 140 | // to get a new master... 141 | refreshEndpoints(); 142 | } 143 | }; 144 | } 145 | 146 | switch(role){ 147 | case 'sentinel': 148 | resolveSentinelClient(endpoints, masterName, connectClient(resolveSentinelClient)); 149 | break; 150 | 151 | case 'master': 152 | resolveMasterClient(endpoints, masterName, connectClient(resolveMasterClient)); 153 | break; 154 | 155 | case 'slave': 156 | resolveSlaveClient(endpoints, masterName, connectClient(resolveSlaveClient)); 157 | } 158 | 159 | return client; 160 | }; 161 | 162 | 163 | /* 164 | * Ensure that all clients are trying to reconnect. 165 | */ 166 | Sentinel.prototype.reconnectAllClients = function() { 167 | // clients in 'closing' state were purposefully closed, and won't ever 168 | // reconnect; remove those from our clients before proceeding 169 | this.clients = this.clients.filter(function(client) { return !client.closing; }); 170 | 171 | this.clients.forEach(function(client) { 172 | // It is safe to call this multiple times in quick succession, as 173 | // might happen with multiple Sentinel instances. Each client 174 | // records its reconnect state and will only try to reconnect if 175 | // not already doing so. 176 | client.connection_gone("sentinel switch-master"); 177 | }); 178 | }; 179 | 180 | function resolveClient() { 181 | var _i, __slice = [].slice; 182 | 183 | // The following just splits the arguments into the first argument (endpoints), 184 | // the last argument (callback) and then any arguments in the middle (args). 185 | var endpoints = arguments[0]; 186 | var checkEndpointFn = arguments[1]; 187 | var args = 4 <= arguments.length ? __slice.call(arguments, 2, _i = arguments.length - 1) : (_i = 2, []); 188 | var callback = arguments[_i++]; 189 | 190 | /** 191 | * We use the algorithm from http://redis.io/topics/sentinel-clients 192 | * to get a sentinel client and then do 'stuff' with it 193 | */ 194 | var promise = when.resolve(); 195 | 196 | // Because finding the master is going to be an async list we will terminate 197 | // when we find one then use promises... 198 | promise = endpoints.reduce(function(soFar, endpoint) { 199 | return soFar.then(function() { 200 | var deferred = when.defer(); 201 | 202 | // Farily illegible way of passing (endpoint, arg1, arg2, ..., callback) 203 | // to checkEndpointFn 204 | checkEndpointFn.apply(null, [endpoint].concat(args, [function() { 205 | var err = arguments[0]; 206 | if (err) { 207 | deferred.resolve(); 208 | } else { 209 | // This is the endpoint that has responded so stick it on the top of 210 | // the list 211 | var index = endpoints.indexOf(endpoint); 212 | endpoints.splice(index, 1); 213 | endpoints.unshift(endpoint); 214 | 215 | // Callback with whatever other arguments we've been given 216 | var _args = 2 <= arguments.length ? __slice.call(arguments, 1) : []; 217 | callback.apply(null, [null].concat(_args)); 218 | } 219 | }])); 220 | return deferred.promise; 221 | }); 222 | }, promise); 223 | 224 | promise = promise.then(function() { 225 | // If we've got this far then we've failed to find whatever we are looking for from any 226 | // of the sentinels. Callback with an error. 227 | callback(new Error('Failed to find a sentinel from the endpoints')); 228 | }); 229 | 230 | // Catch the failure (if there is one) 231 | promise.catch(function(err) { callback(err); }); 232 | } 233 | 234 | function isSentinelOk(endpoint, callback) { 235 | var client = redis.createClient(endpoint.port, endpoint.host); 236 | var callbackSent = false; 237 | client.on("error", function(err) { 238 | if (!callbackSent) { 239 | callbackSent = true; 240 | callback(err); 241 | } 242 | client.end(); 243 | }); 244 | 245 | // Send a command just to check we can... 246 | client.info(function(err, resp) { 247 | if (callbackSent) { return; } 248 | callbackSent = true; 249 | if (err) { return callback(err); } 250 | callback(null, endpoint.host, String(endpoint.port)); 251 | }); 252 | client.quit(); 253 | } 254 | 255 | function getMasterFromEndpoint(endpoint, masterName, callback) { 256 | var sentinelClient = redis.createClient(endpoint.port, endpoint.host); 257 | var callbackSent = false; 258 | 259 | // If there is an error then callback with it 260 | sentinelClient.on("error", function(err) { 261 | if (!callbackSent) { 262 | callbackSent = true; 263 | callback(err); 264 | } 265 | sentinelClient.end(); 266 | }); 267 | 268 | sentinelClient.send_command('SENTINEL', ['get-master-addr-by-name', masterName], function(err, result) { 269 | if (callbackSent) { return; } 270 | callbackSent = true; 271 | 272 | if (err) { return callback(err); } 273 | 274 | // Test the response 275 | if (result === null) { 276 | callback(new Error("Unknown master name: " + masterName)); 277 | } else { 278 | var ip = result[0]; 279 | var port = result[1]; 280 | callback(null, ip, port); 281 | } 282 | }); 283 | sentinelClient.quit(); 284 | } 285 | 286 | function getSlaveFromEndpoint(endpoint, masterName, callback) { 287 | var sentinelClient = redis.createClient(endpoint.port, endpoint.host); 288 | var callbackSent = false; 289 | 290 | // If there is an error then callback with it 291 | sentinelClient.on("error", function(err) { 292 | if (!callbackSent) { 293 | callbackSent = true; 294 | callback(err); 295 | } 296 | sentinelClient.end(); 297 | }); 298 | 299 | sentinelClient.send_command('SENTINEL', ['slaves', masterName], function(err, result) { 300 | if (callbackSent) { return; } 301 | callbackSent = true; 302 | 303 | if (err) { return callback(err); } 304 | 305 | // Test the response 306 | if (result === null) { 307 | callback(new Error("Unknown master name: " + masterName)); 308 | } else if(result.length === 0){ 309 | callback(new Error("No slaves linked to the master.")); 310 | } else { 311 | var slaveInfoArr = result[Math.floor(Math.random() * result.length)]; //range 0 to result.length -1 312 | if((slaveInfoArr.length % 2) > 0){ 313 | callback(new Error("Corrupted response from the sentinel")); 314 | } else { 315 | var slaveInfo = parseSentinelResponse(slaveInfoArr); 316 | callback(null, slaveInfo.ip, slaveInfo.port); 317 | } 318 | } 319 | }); 320 | sentinelClient.quit(); 321 | } 322 | 323 | function resolveSentinelClient(endpoints, masterName, callback) { 324 | resolveClient(endpoints, isSentinelOk, callback); 325 | } 326 | 327 | function resolveMasterClient(endpoints, masterName, callback) { 328 | resolveClient(endpoints, getMasterFromEndpoint, masterName, callback); 329 | } 330 | 331 | function resolveSlaveClient(endpoints, masterName, callback) { 332 | resolveClient(endpoints, getSlaveFromEndpoint, masterName, callback); 333 | } 334 | 335 | function parseSentinelResponse(resArr){ 336 | var response = {}; 337 | for(var i = 0 ; i < resArr.length ; i+=2){ 338 | response[resArr[i]] = resArr[i+1]; 339 | } 340 | return response; 341 | } 342 | 343 | // Shortcut for quickly getting a client from endpoints 344 | function createClient(endpoints, masterName, options) { 345 | var sentinel = Sentinel(endpoints); 346 | return sentinel.createClient(masterName, options); 347 | } 348 | 349 | module.exports.Sentinel = Sentinel; 350 | module.exports.createClient = createClient; 351 | module.exports.redis = redis; 352 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "redis-sentinel", 3 | "version": "0.3.3", 4 | "description": "Redis sentinel client for nodejs", 5 | "main": "index.js", 6 | "scripts": { 7 | "test": "mocha" 8 | }, 9 | "repository": { 10 | "type": "git", 11 | "url": "git://github.com/ortoo/node-redis-sentinel.git" 12 | }, 13 | "keywords": [ 14 | "redis" 15 | ], 16 | "dependencies": { 17 | "redis": "0.12.x", 18 | "when": "^3.5.1" 19 | }, 20 | "devDependencies": { 21 | "mocha": "*", 22 | "chai": "*" 23 | }, 24 | "author": "Ortoo Technologies", 25 | "license": "MIT", 26 | "readmeFilename": "README.md", 27 | "gitHead": "289f89ee4dd1d675e33afc2bccf95fe483f39392" 28 | } 29 | -------------------------------------------------------------------------------- /test/test.js: -------------------------------------------------------------------------------- 1 | var sentinel = require('../'); 2 | var expect = require('chai').expect; 3 | var redis = require('redis'); 4 | 5 | describe('Redis Sentinel tests', function() { 6 | 7 | describe('initial connection', function() { 8 | 9 | it('should get master correctly with single sentinel', function(done) { 10 | var endpoints = [{ host: '127.0.0.1', port: 26380}]; 11 | var redisClient = sentinel.createClient(endpoints, 'mymaster'); 12 | redisClient.on('ready', function() { 13 | expect(redisClient.connectionOption.host).to.equal('127.0.0.1'); 14 | expect(redisClient.connectionOption.port).to.equal("6379"); 15 | done(); 16 | }); 17 | }); 18 | 19 | it('should get slave correctly with single sentinel', function(done) { 20 | var endpoints = [{ host: '127.0.0.1', port: 26380}]; 21 | var redisClient = sentinel.createClient(endpoints, 'mymaster', {role:'slave'}); 22 | redisClient.on('ready', function() { 23 | expect(redisClient.connectionOption.host).to.equal('127.0.0.1'); 24 | expect(["6381", "6380"]).to.contain(redisClient.connectionOption.port); 25 | done(); 26 | }); 27 | }); 28 | 29 | it('should get sentinel correctly with single sentinel', function(done) { 30 | var endpoints = [{ host: '127.0.0.1', port: 26380}]; 31 | var redisClient = sentinel.createClient(endpoints, {role:'sentinel'}); 32 | redisClient.on('ready', function() { 33 | expect(redisClient.connectionOption.host).to.equal('127.0.0.1'); 34 | expect(redisClient.connectionOption.port).to.equal("26380"); 35 | done(); 36 | }); 37 | }); 38 | 39 | it('should get master correctly with multiple sentinels', function(done) { 40 | var endpoints = [ 41 | { host: '127.0.0.1', port: 26380}, 42 | { host: '127.0.0.1', port: 26379} 43 | ]; 44 | var redisClient = sentinel.createClient(endpoints); 45 | redisClient.on('ready', function() { 46 | expect(redisClient.connectionOption.host).to.equal('127.0.0.1'); 47 | expect(redisClient.connectionOption.port).to.equal("6379"); 48 | done(); 49 | }); 50 | }); 51 | 52 | it('should get slave correctly with multiple sentinels', function(done) { 53 | var endpoints = [ 54 | { host: '127.0.0.1', port: 26380}, 55 | { host: '127.0.0.1', port: 26379} 56 | ]; 57 | var redisClient = sentinel.createClient(endpoints, 'mymaster', {role:'slave'}); 58 | redisClient.on('ready', function() { 59 | expect(redisClient.connectionOption.host).to.equal('127.0.0.1'); 60 | expect(["6381", "6380"]).to.contain(redisClient.connectionOption.port); 61 | done(); 62 | }); 63 | }); 64 | 65 | it('should get sentinel correctly with multiple sentinels', function(done) { 66 | var endpoints = [ 67 | { host: '127.0.0.1', port: 26380}, 68 | { host: '127.0.0.1', port: 26379} 69 | ]; 70 | var redisClient = sentinel.createClient(endpoints, {role: 'sentinel'}); 71 | redisClient.on('ready', function() { 72 | expect(redisClient.connectionOption.host).to.equal('127.0.0.1'); 73 | expect(redisClient.connectionOption.port).to.equal("26380"); 74 | done(); 75 | }); 76 | }); 77 | 78 | 79 | it('should get master correctly with multiple sentinels - one not active', function(done) { 80 | var endpoints = [ 81 | { host: '127.0.0.1', port: 26378}, 82 | { host: '127.0.0.1', port: 26380}, 83 | { host: '127.0.0.1', port: 26379} 84 | ]; 85 | var redisClient = sentinel.createClient(endpoints, 'mymaster'); 86 | redisClient.on('ready', function() { 87 | expect(redisClient.connectionOption.host).to.equal('127.0.0.1'); 88 | expect(redisClient.connectionOption.port).to.equal("6379"); 89 | done(); 90 | }); 91 | }); 92 | 93 | it('should get slave correctly with multiple sentinels - one not active', function(done) { 94 | var endpoints = [ 95 | { host: '127.0.0.1', port: 26378}, 96 | { host: '127.0.0.1', port: 26380}, 97 | { host: '127.0.0.1', port: 26379} 98 | ]; 99 | var redisClient = sentinel.createClient(endpoints, 'mymaster', {role:'slave'}); 100 | redisClient.on('ready', function() { 101 | expect(redisClient.connectionOption.host).to.equal('127.0.0.1'); 102 | expect(["6381", "6380"]).to.contain(redisClient.connectionOption.port); 103 | done(); 104 | }); 105 | }); 106 | 107 | it('should get sentinel correctly with multiple sentinels - one not active', function(done) { 108 | var endpoints = [ 109 | { host: '127.0.0.1', port: 26378}, 110 | { host: '127.0.0.1', port: 26380}, 111 | { host: '127.0.0.1', port: 26379} 112 | ]; 113 | var redisClient = sentinel.createClient(endpoints, {role: 'sentinel'}); 114 | redisClient.on('ready', function() { 115 | expect(redisClient.connectionOption.host).to.equal('127.0.0.1'); 116 | expect(redisClient.connectionOption.port).to.equal("26380"); 117 | done(); 118 | }); 119 | }); 120 | 121 | it('should return an instance of RedisClient', function(){ 122 | var endpoints = [ 123 | { host: 'bad.addr', port: 26378}, 124 | { host: '127.0.0.1', port: 26380}, 125 | { host: '127.0.0.1', port: 26379} 126 | ]; 127 | var redisClient = sentinel.createClient(endpoints); 128 | expect(redisClient).to.be.an.instanceof(redis.RedisClient); 129 | }); 130 | 131 | it('should give an error when no sentinels are active', function(done) { 132 | var endpoints = [ 133 | { host: '127.0.0.1', port: 26378}, 134 | { host: '127.0.0.1', port: 26377}, 135 | { host: '127.0.0.1', port: 26376} 136 | ]; 137 | var redisClient = sentinel.createClient(endpoints, 'mymaster'); 138 | redisClient.on('error', function(err){ 139 | expect(err.message).to.equal('Failed to find a sentinel from the endpoints'); 140 | done(); 141 | }); 142 | }); 143 | }); 144 | 145 | describe('data writing', function() { 146 | 147 | var redisClient; 148 | 149 | before(function() { 150 | var endpoints = [ 151 | { host: '127.0.0.1', port: 26380}, 152 | { host: '127.0.0.1', port: 26379} 153 | ]; 154 | redisClient = sentinel.createClient(endpoints); 155 | redisClient.select(9); 156 | }); 157 | 158 | it('should write a key and return it', function(done) { 159 | redisClient.set('__test__', 'some value', function(err) { 160 | expect(err).to.be.null; 161 | redisClient.get('__test__', function(err, val) { 162 | expect(err).to.be.null; 163 | expect(val).to.equal('some value'); 164 | done(); 165 | }); 166 | }); 167 | }); 168 | }); 169 | 170 | describe('client management', function () { 171 | it('should clear client entries when they quit', function (done) { 172 | var endpoints = [{host: '127.0.0.1', port: 26380}]; 173 | var instance = sentinel.Sentinel(endpoints); 174 | var redisClient1 = instance.createClient('mymaster'); 175 | redisClient1.on('ready', function () { 176 | // one pubsub, one actual 177 | expect(instance.clients.length).to.equal(2); 178 | 179 | var redisClient2 = instance.createClient('mymaster'); 180 | redisClient2.on('ready', function () { 181 | expect(instance.clients.length).to.equal(3); 182 | redisClient2.quit(); 183 | }); 184 | 185 | redisClient2.on('end', function () { 186 | expect(instance.clients.length).to.equal(2); 187 | expect(redisClient2.info()).to.not.be.ok; 188 | 189 | redisClient1.info(function(err, info) { 190 | expect(err).to.be.null; 191 | expect(info).to.be.ok; 192 | 193 | done(); 194 | }); 195 | }); 196 | }); 197 | }); 198 | 199 | it('should eventually clear client entries when reconnecting', function (done) { 200 | var endpoints = [{host: '127.0.0.1', port: 26380}]; 201 | var instance = sentinel.Sentinel(endpoints); 202 | var redisClient1 = instance.createClient('mymaster'); 203 | redisClient1.on('ready', function () { 204 | // one pubsub, one actual 205 | expect(instance.clients.length).to.equal(2); 206 | 207 | var redisClient2 = instance.createClient('mymaster'); 208 | redisClient2.on('ready', function () { 209 | expect(instance.clients.length).to.equal(3); 210 | redisClient2.end(); 211 | 212 | instance.reconnectAllClients(); 213 | expect(instance.clients.length).to.equal(2); 214 | 215 | expect(redisClient2.info()).to.not.be.ok; 216 | 217 | redisClient1.info(function(err, info) { 218 | expect(err).to.be.null; 219 | expect(info).to.be.ok; 220 | 221 | done(); 222 | }); 223 | }); 224 | }); 225 | }); 226 | }); 227 | }); 228 | --------------------------------------------------------------------------------