├── .eslintignore ├── .eslintrc ├── .gitignore ├── .npmignore ├── .travis.yml ├── LICENSE ├── README.md ├── actions ├── addTagsToStream.js ├── createStream.js ├── decreaseStreamRetentionPeriod.js ├── deleteStream.js ├── describeStream.js ├── describeStreamSummary.js ├── getRecords.js ├── getShardIterator.js ├── increaseStreamRetentionPeriod.js ├── listShards.js ├── listStreams.js ├── listTagsForStream.js ├── mergeShards.js ├── putRecord.js ├── putRecords.js ├── removeTagsFromStream.js └── splitShard.js ├── cli.js ├── db └── index.js ├── index.js ├── package-lock.json ├── package.json ├── ssl ├── ca-crt.pem ├── ca-key.pem ├── server-crt.pem ├── server-csr.pem └── server-key.pem ├── test ├── addTagsToStream.js ├── connection.js ├── createStream.js ├── decreaseStreamRetentionPeriod.js ├── deleteStream.js ├── describeStream.js ├── describeStreamSummary.js ├── getRecords.js ├── getShardIterator.js ├── helpers.js ├── increaseStreamRetentionPeriod.js ├── listShards.js ├── listStreams.js ├── listTagsForStream.js ├── mergeShards.js ├── putRecord.js ├── putRecords.js ├── removeTagsFromStream.js └── splitShard.js └── validations ├── addTagsToStream.js ├── createStream.js ├── decreaseStreamRetentionPeriod.js ├── deleteStream.js ├── describeStream.js ├── describeStreamSummary.js ├── getRecords.js ├── getShardIterator.js ├── increaseStreamRetentionPeriod.js ├── index.js ├── listShards.js ├── listStreams.js ├── listTagsForStream.js ├── mergeShards.js ├── putRecord.js ├── putRecords.js ├── removeTagsFromStream.js └── splitShard.js /.eslintignore: -------------------------------------------------------------------------------- 1 | coverage/** 2 | -------------------------------------------------------------------------------- /.eslintrc: -------------------------------------------------------------------------------- 1 | { 2 | "env": { 3 | "browser": true, 4 | "node": true, 5 | "es6": true, 6 | "mocha": true 7 | }, 8 | "rules": { 9 | // standard defaults 10 | "no-alert": 2, 11 | "no-array-constructor": 2, 12 | "no-caller": 2, 13 | "no-catch-shadow": 2, 14 | "no-eval": 2, 15 | "no-extend-native": 2, 16 | "no-extra-bind": 2, 17 | "no-implied-eval": 2, 18 | "no-iterator": 2, 19 | "no-label-var": 2, 20 | "no-labels": 2, 21 | "no-lone-blocks": 2, 22 | "no-loop-func": 2, 23 | "no-multi-spaces": 2, 24 | "no-multi-str": 2, 25 | "no-native-reassign": 2, 26 | "no-new": 2, 27 | "no-new-func": 2, 28 | "no-new-object": 2, 29 | "no-new-wrappers": 2, 30 | "no-octal-escape": 2, 31 | "no-process-exit": 2, 32 | "no-proto": 2, 33 | "no-return-assign": 2, 34 | "no-script-url": 2, 35 | "no-sequences": 2, 36 | "no-shadow-restricted-names": 2, 37 | "no-spaced-func": 2, 38 | "no-trailing-spaces": 2, 39 | "no-undef-init": 2, 40 | "no-unused-expressions": 2, 41 | "no-with": 2, 42 | "comma-spacing": 2, 43 | "dot-notation": 2, 44 | "eol-last": 2, 45 | "key-spacing": 2, 46 | "new-cap": 2, 47 | "semi-spacing": 2, 48 | "space-infix-ops": 2, 49 | "space-unary-ops": 2, 50 | "yoda": 2, 51 | 52 | // relaxed restrictions 53 | "no-mixed-requires": 0, 54 | "no-underscore-dangle": 0, 55 | "no-shadow": 0, 56 | "no-use-before-define": [2, "nofunc"], 57 | "camelcase": [2, {"properties": "never"}], 58 | "curly": 0, 59 | // "curly": [2, "multi"], 60 | // "curly": [2, "multi-line"], 61 | "eqeqeq": 0, 62 | // "eqeqeq": [2, "smart"], 63 | "new-parens": 0, 64 | "quotes": [2, "single", "avoid-escape"], 65 | "semi": [2, "never"], 66 | "strict": 0, 67 | "consistent-return": 0, 68 | 69 | // extra restrictions 70 | "no-empty-character-class": 2, 71 | "no-extra-parens": [2, "functions"], 72 | "no-floating-decimal": 2, 73 | "no-lonely-if": 2, 74 | "no-self-compare": 2, 75 | "no-throw-literal": 2, 76 | 77 | // style 78 | "array-bracket-spacing": [2, "never"], 79 | "brace-style": [2, "1tbs", {"allowSingleLine": true}], 80 | "comma-dangle": [2, "always-multiline"], 81 | "comma-style": [2, "last"], 82 | "consistent-this": [2, "self"], 83 | // "indent": [2, 2], 84 | "object-curly-spacing": [2, "never"], 85 | "operator-assignment": [2, "always"], 86 | "operator-linebreak": [2, "after"], 87 | "keyword-spacing": [2, {"after": true}], 88 | "space-before-blocks": [2, "always"], 89 | "space-before-function-paren": [2, "never"], 90 | "space-in-parens": [2, "never"], 91 | "spaced-comment": [2, "always"] 92 | } 93 | } 94 | -------------------------------------------------------------------------------- /.gitignore: -------------------------------------------------------------------------------- 1 | node_modules 2 | npm-debug.log 3 | coverage* 4 | .tern-port 5 | -------------------------------------------------------------------------------- /.npmignore: -------------------------------------------------------------------------------- 1 | test 2 | coverage 3 | examples 4 | -------------------------------------------------------------------------------- /.travis.yml: -------------------------------------------------------------------------------- 1 | language: node_js 2 | node_js: 3 | - "6" 4 | - "8" 5 | - "10" 6 | - "12" 7 | -------------------------------------------------------------------------------- /LICENSE: -------------------------------------------------------------------------------- 1 | Copyright 2013 Michael Hart (michael.hart.au@gmail.com) 2 | 3 | Permission is hereby granted, free of charge, to any person obtaining a copy of 4 | this software and associated documentation files (the "Software"), to deal in 5 | the Software without restriction, including without limitation the rights to 6 | use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies 7 | of the Software, and to permit persons to whom the Software is furnished to do 8 | so, subject to the following conditions: 9 | 10 | The above copyright notice and this permission notice shall be included in all 11 | copies or substantial portions of the Software. 12 | 13 | THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 14 | IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 15 | FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE 16 | AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER 17 | LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, 18 | OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE 19 | SOFTWARE. 20 | -------------------------------------------------------------------------------- /README.md: -------------------------------------------------------------------------------- 1 | Kinesalite 2 | ---------- 3 | 4 | [![Build Status](https://secure.travis-ci.org/mhart/kinesalite.png?branch=master)](http://travis-ci.org/mhart/kinesalite) 5 | 6 | An implementation of [Amazon's Kinesis](http://docs.aws.amazon.com/kinesis/latest/APIReference/), 7 | focussed1 on correctness and performance, and built on LevelDB 8 | (well, [@rvagg](https://github.com/rvagg)'s awesome [LevelUP](https://github.com/rvagg/node-levelup) to be precise). 9 | 10 | The Kinesis equivalent of [dynalite](https://github.com/mhart/dynalite). 11 | 12 | To read and write from Kinesis streams in Node.js, consider using the [kinesis](https://github.com/mhart/kinesis) 13 | module. 14 | 15 | Example 16 | ------- 17 | 18 | ```sh 19 | $ kinesalite --help 20 | 21 | Usage: kinesalite [--port ] [--path ] [--ssl] [options] 22 | 23 | A Kinesis http server, optionally backed by LevelDB 24 | 25 | Options: 26 | --help Display this help message and exit 27 | --port The port to listen on (default: 4567) 28 | --path The path to use for the LevelDB store (in-memory by default) 29 | --ssl Enable SSL for the web server (default: false) 30 | --createStreamMs Amount of time streams stay in CREATING state (default: 500) 31 | --deleteStreamMs Amount of time streams stay in DELETING state (default: 500) 32 | --updateStreamMs Amount of time streams stay in UPDATING state (default: 500) 33 | --shardLimit Shard limit for error reporting (default: 10) 34 | 35 | Report bugs at github.com/mhart/kinesalite/issues 36 | ``` 37 | 38 | Or programmatically: 39 | 40 | ```js 41 | // Returns a standard Node.js HTTP server 42 | var kinesalite = require('kinesalite'), 43 | kinesaliteServer = kinesalite({path: './mydb', createStreamMs: 50}) 44 | 45 | // Listen on port 4567 46 | kinesaliteServer.listen(4567, function(err) { 47 | if (err) throw err 48 | console.log('Kinesalite started on port 4567') 49 | }) 50 | ``` 51 | 52 | Once running, here's how you use the [AWS SDK](https://github.com/aws/aws-sdk-js) to connect 53 | (after [configuring the SDK](http://docs.aws.amazon.com/AWSJavaScriptSDK/guide/node-configuring.html)): 54 | 55 | ```js 56 | var AWS = require('aws-sdk') 57 | 58 | var kinesis = new AWS.Kinesis({endpoint: 'http://localhost:4567'}) 59 | 60 | kinesis.listStreams(console.log.bind(console)) 61 | ``` 62 | 63 | Or with the [kinesis](https://github.com/mhart/kinesis) module (currently only works in https mode, when kinesalite is started with `--ssl`): 64 | 65 | ```js 66 | var kinesis = require('kinesis') 67 | 68 | kinesis.listStreams({host: 'localhost', port: 4567}, console.log) 69 | ``` 70 | 71 | Installation 72 | ------------ 73 | 74 | With [npm](http://npmjs.org/) do: 75 | 76 | ```sh 77 | $ npm install -g kinesalite 78 | ``` 79 | 80 | Footnotes 81 | --------- 82 | 83 | 1Hi! You're probably American ([and not a New Yorker editor](https://www.newyorker.com/books/page-turner/the-double-l)) if you're worried about this spelling. No worries – 84 | and no need to open a pull request – we have different spellings in the rest of the English speaking world 🐨 85 | -------------------------------------------------------------------------------- /actions/addTagsToStream.js: -------------------------------------------------------------------------------- 1 | var db = require('../db') 2 | 3 | module.exports = function addTagsToStream(store, data, cb) { 4 | 5 | var metaDb = store.metaDb 6 | 7 | metaDb.lock(data.StreamName, function(release) { 8 | cb = release(cb) 9 | 10 | store.getStream(data.StreamName, function(err, stream) { 11 | if (err) return cb(err) 12 | 13 | var keys = Object.keys(data.Tags), values = keys.map(function(key) { return data.Tags[key] }), 14 | all = keys.concat(values) 15 | 16 | if (all.some(function(key) { return /[^\u00C0-\u1FFF\u2C00-\uD7FF\w\.\/\-=+_ @%]/.test(key) })) 17 | return cb(db.clientError('InvalidArgumentException', 18 | 'Some tags contain invalid characters. Valid characters: ' + 19 | 'Unicode letters, digits, white space, _ . / = + - % @.')) 20 | 21 | if (all.some(function(key) { return ~key.indexOf('%') })) 22 | return cb(db.clientError('InvalidArgumentException', 23 | 'Failed to add tags to stream ' + data.StreamName + ' under account ' + metaDb.awsAccountId + 24 | ' because some tags contained illegal characters. The allowed characters are ' + 25 | 'Unicode letters, white-spaces, \'_\',\',\',\'/\',\'=\',\'+\',\'-\',\'@\'.')) 26 | 27 | var newKeys = keys.concat(Object.keys(stream._tags)).reduce(function(obj, key) { 28 | obj[key] = true 29 | return obj 30 | }, {}) 31 | 32 | if (Object.keys(newKeys).length > 50) 33 | return cb(db.clientError('InvalidArgumentException', 34 | 'Failed to add tags to stream ' + data.StreamName + ' under account ' + metaDb.awsAccountId + 35 | ' because a given stream cannot have more than 10 tags associated with it.')) 36 | 37 | keys.forEach(function(key) { 38 | stream._tags[key] = data.Tags[key] 39 | }) 40 | 41 | metaDb.put(data.StreamName, stream, function(err) { 42 | if (err) return cb(err) 43 | 44 | cb() 45 | }) 46 | }) 47 | }) 48 | } 49 | -------------------------------------------------------------------------------- /actions/createStream.js: -------------------------------------------------------------------------------- 1 | var BigNumber = require('bignumber.js'), 2 | db = require('../db') 3 | 4 | var POW_128 = new BigNumber(2).pow(128), 5 | SEQ_ADJUST_MS = 2000 6 | 7 | module.exports = function createStream(store, data, cb) { 8 | 9 | var key = data.StreamName, metaDb = store.metaDb 10 | 11 | metaDb.lock(key, function(release) { 12 | cb = release(cb) 13 | 14 | metaDb.get(key, function(err) { 15 | if (err && err.name != 'NotFoundError') return cb(err) 16 | if (!err) 17 | return cb(db.clientError('ResourceInUseException', 18 | 'Stream ' + key + ' under account ' + metaDb.awsAccountId + ' already exists.')) 19 | 20 | db.sumShards(store, function(err, shardSum) { 21 | if (err) return cb(err) 22 | 23 | if (shardSum + data.ShardCount > store.shardLimit) { 24 | return cb(db.clientError('LimitExceededException', 25 | 'This request would exceed the shard limit for the account ' + metaDb.awsAccountId + ' in ' + 26 | metaDb.awsRegion + '. Current shard count for the account: ' + shardSum + 27 | '. Limit: ' + store.shardLimit + '. Number of additional shards that would have ' + 28 | 'resulted from this request: ' + data.ShardCount + '. Refer to the AWS Service Limits page ' + 29 | '(http://docs.aws.amazon.com/general/latest/gr/aws_service_limits.html) ' + 30 | 'for current limits and how to request higher limits.')) 31 | } 32 | 33 | var i, shards = new Array(data.ShardCount), shardHash = POW_128.div(data.ShardCount).integerValue(BigNumber.ROUND_FLOOR), 34 | createTime = Date.now() - SEQ_ADJUST_MS, stream 35 | for (i = 0; i < data.ShardCount; i++) { 36 | shards[i] = { 37 | HashKeyRange: { 38 | StartingHashKey: shardHash.times(i).toFixed(), 39 | EndingHashKey: (i < data.ShardCount - 1 ? shardHash.times(i + 1) : POW_128).minus(1).toFixed(), 40 | }, 41 | SequenceNumberRange: { 42 | StartingSequenceNumber: db.stringifySequence({shardCreateTime: createTime, shardIx: i}), 43 | }, 44 | ShardId: db.shardIdName(i), 45 | } 46 | } 47 | stream = { 48 | RetentionPeriodHours: 24, 49 | EnhancedMonitoring: [{ShardLevelMetrics: []}], 50 | EncryptionType: 'NONE', 51 | HasMoreShards: false, 52 | Shards: [], 53 | StreamARN: 'arn:aws:kinesis:' + metaDb.awsRegion + ':' + metaDb.awsAccountId + ':stream/' + data.StreamName, 54 | StreamName: data.StreamName, 55 | StreamStatus: 'CREATING', 56 | StreamCreationTimestamp: Math.floor(createTime / 1000), 57 | _seqIx: new Array(Math.ceil(data.ShardCount / 5)), // Hidden data, remove when returning 58 | _tags: Object.create(null), // Hidden data, remove when returning 59 | } 60 | 61 | metaDb.put(key, stream, function(err) { 62 | if (err) return cb(err) 63 | 64 | setTimeout(function() { 65 | 66 | // Shouldn't need to lock/fetch as nothing should have changed 67 | stream.StreamStatus = 'ACTIVE' 68 | stream.Shards = shards 69 | 70 | metaDb.put(key, stream, function(err) { 71 | if (err && !/Database is not open/.test(err)) console.error(err.stack || err) 72 | }) 73 | 74 | }, store.createStreamMs) 75 | 76 | cb() 77 | }) 78 | }) 79 | }) 80 | }) 81 | 82 | } 83 | -------------------------------------------------------------------------------- /actions/decreaseStreamRetentionPeriod.js: -------------------------------------------------------------------------------- 1 | var db = require('../db') 2 | 3 | module.exports = function decreaseStreamRetentionPeriod(store, data, cb) { 4 | 5 | var metaDb = store.metaDb 6 | 7 | metaDb.lock(data.StreamName, function(release) { 8 | cb = release(cb) 9 | 10 | store.getStream(data.StreamName, function(err, stream) { 11 | if (err) return cb(err) 12 | 13 | if (data.RetentionPeriodHours < 24) { 14 | return cb(db.clientError('InvalidArgumentException', 15 | 'Minimum allowed retention period is 24 hours. Requested retention period (' + data.RetentionPeriodHours + 16 | ' hours) is too short.')) 17 | } 18 | 19 | if (stream.RetentionPeriodHours < data.RetentionPeriodHours) { 20 | return cb(db.clientError('InvalidArgumentException', 21 | 'Requested retention period (' + data.RetentionPeriodHours + 22 | ' hours) for stream ' + data.StreamName + 23 | ' can not be longer than existing retention period (' + stream.RetentionPeriodHours + 24 | ' hours). Use IncreaseRetentionPeriod API.')) 25 | } 26 | 27 | stream.RetentionPeriodHours = data.RetentionPeriodHours 28 | 29 | metaDb.put(data.StreamName, stream, function(err) { 30 | if (err) return cb(err) 31 | 32 | cb() 33 | }) 34 | }) 35 | }) 36 | } 37 | -------------------------------------------------------------------------------- /actions/deleteStream.js: -------------------------------------------------------------------------------- 1 | 2 | module.exports = function deleteStream(store, data, cb) { 3 | 4 | var key = data.StreamName, metaDb = store.metaDb 5 | 6 | store.getStream(key, function(err, stream) { 7 | if (err) return cb(err) 8 | 9 | stream.StreamStatus = 'DELETING' 10 | 11 | metaDb.put(key, stream, function(err) { 12 | if (err) return cb(err) 13 | 14 | store.deleteStreamDb(key, function(err) { 15 | if (err) return cb(err) 16 | 17 | setTimeout(function() { 18 | metaDb.del(key, function(err) { 19 | if (err && !/Database is not open/.test(err)) console.error(err.stack || err) 20 | }) 21 | }, store.deleteStreamMs) 22 | 23 | cb() 24 | }) 25 | }) 26 | }) 27 | 28 | } 29 | 30 | 31 | 32 | -------------------------------------------------------------------------------- /actions/describeStream.js: -------------------------------------------------------------------------------- 1 | 2 | module.exports = function describeStream(store, data, cb) { 3 | 4 | store.getStream(data.StreamName, function(err, stream) { 5 | if (err) return cb(err) 6 | 7 | delete stream._seqIx 8 | delete stream._tags 9 | 10 | cb(null, {StreamDescription: stream}) 11 | }) 12 | } 13 | 14 | 15 | 16 | -------------------------------------------------------------------------------- /actions/describeStreamSummary.js: -------------------------------------------------------------------------------- 1 | 2 | module.exports = function describeStreamSummary(store, data, cb) { 3 | 4 | store.getStream(data.StreamName, function(err, stream) { 5 | if (err) return cb(err) 6 | 7 | stream.OpenShardCount = stream.Shards.filter(function(shard) { 8 | return shard.SequenceNumberRange.EndingSequenceNumber == null 9 | }).length 10 | 11 | delete stream._seqIx 12 | delete stream._tags 13 | delete stream.Shards 14 | delete stream.HasMoreShards 15 | 16 | stream.ConsumerCount = 0 17 | 18 | cb(null, {StreamDescriptionSummary: stream}) 19 | }) 20 | } 21 | -------------------------------------------------------------------------------- /actions/getRecords.js: -------------------------------------------------------------------------------- 1 | var crypto = require('crypto'), 2 | once = require('once'), 3 | db = require('../db') 4 | 5 | module.exports = function getRecords(store, data, cb) { 6 | 7 | var metaDb = store.metaDb, shardIx, shardId, iteratorTime, streamName, seqNo, seqObj, pieces, 8 | buffer = Buffer.from(data.ShardIterator, 'base64'), now = Date.now(), 9 | decipher = crypto.createDecipheriv('aes-256-cbc', db.ITERATOR_PWD_KEY, db.ITERATOR_PWD_IV) 10 | 11 | if (buffer.length < 152 || buffer.length > 280 || buffer.toString('base64') != data.ShardIterator) 12 | return cb(invalidShardIterator()) 13 | 14 | if (buffer.slice(0, 8).toString('hex') != '0000000000000001') 15 | return cb(invalidShardIterator()) 16 | 17 | try { 18 | pieces = Buffer.concat([decipher.update(buffer.slice(8)), decipher.final()]).toString('utf8').split('/') 19 | } catch (e) { 20 | return cb(invalidShardIterator()) 21 | } 22 | 23 | if (pieces.length != 5) 24 | return cb(invalidShardIterator()) 25 | 26 | iteratorTime = +pieces[0] 27 | streamName = pieces[1] 28 | shardId = pieces[2] 29 | seqNo = pieces[3] 30 | 31 | shardIx = parseInt(shardId.split('-')[1]) 32 | if (!/^shardId-[\d]{12}$/.test(shardId) || !(shardIx >= 0 && shardIx < 2147483648)) 33 | return cb(invalidShardIterator()) 34 | 35 | if (!(iteratorTime > 0 && iteratorTime <= now)) 36 | return cb(invalidShardIterator()) 37 | 38 | if (!/[a-zA-Z0-9_.-]+/.test(streamName) || !streamName.length || streamName.length > 128) 39 | return cb(invalidShardIterator()) 40 | 41 | if ((now - iteratorTime) > 300000) { 42 | return cb(db.clientError('ExpiredIteratorException', 43 | 'Iterator expired. The iterator was created at time ' + toAmzUtcString(iteratorTime) + 44 | ' while right now it is ' + toAmzUtcString(now) + ' which is further in the future than the ' + 45 | 'tolerated delay of 300000 milliseconds.')) 46 | } 47 | 48 | try { 49 | seqObj = db.parseSequence(seqNo) 50 | } catch (e) { 51 | return cb(invalidShardIterator()) 52 | } 53 | 54 | store.getStream(streamName, function(err, stream) { 55 | if (err) { 56 | if (err.name == 'NotFoundError' && err.body) { 57 | err.body.message = 'Shard ' + shardId + ' in stream ' + streamName + 58 | ' under account ' + metaDb.awsAccountId + ' does not exist' 59 | } 60 | return cb(err) 61 | } 62 | if (shardIx >= stream.Shards.length) { 63 | return cb(db.clientError('ResourceNotFoundException', 64 | 'Shard ' + shardId + ' in stream ' + streamName + 65 | ' under account ' + metaDb.awsAccountId + ' does not exist')) 66 | } 67 | 68 | cb = once(cb) 69 | 70 | var streamDb = store.getStreamDb(streamName), cutoffTime = now - (stream.RetentionPeriodHours * 60 * 60 * 1000), 71 | keysToDelete = [], lastItem, opts 72 | 73 | opts = { 74 | gte: db.shardIxToHex(shardIx) + '/' + seqNo, 75 | lt: db.shardIxToHex(shardIx + 1), 76 | } 77 | 78 | db.lazy(streamDb.createReadStream(opts), cb) 79 | .take(data.Limit || 10000) 80 | .map(function(item) { 81 | lastItem = item.value 82 | lastItem.SequenceNumber = item.key.split('/')[1] 83 | lastItem._seqObj = db.parseSequence(lastItem.SequenceNumber) 84 | lastItem._tooOld = lastItem._seqObj.seqTime < cutoffTime 85 | if (lastItem._tooOld) keysToDelete.push(item.key) 86 | return lastItem 87 | }) 88 | .filter(function(item) { return !item._tooOld }) 89 | .join(function(items) { 90 | var defaultTime = now 91 | if (seqObj.seqTime > defaultTime) { 92 | defaultTime = seqObj.seqTime 93 | } 94 | var nextSeq = db.incrementSequence(lastItem ? lastItem._seqObj : seqObj, lastItem ? null : defaultTime), 95 | nextShardIterator = db.createShardIterator(streamName, shardId, nextSeq), 96 | millisBehind = 0 97 | 98 | if (!items.length && stream.Shards[shardIx].SequenceNumberRange.EndingSequenceNumber) { 99 | var endSeqObj = db.parseSequence(stream.Shards[shardIx].SequenceNumberRange.EndingSequenceNumber) 100 | if (seqObj.seqTime >= endSeqObj.seqTime) { 101 | nextShardIterator = undefined 102 | millisBehind = Math.max(0, now - endSeqObj.seqTime) 103 | } 104 | } 105 | 106 | cb(null, { 107 | MillisBehindLatest: millisBehind, 108 | NextShardIterator: nextShardIterator, 109 | Records: items.map(function(item) { 110 | delete item._seqObj 111 | delete item._tooOld 112 | return item 113 | }), 114 | }) 115 | 116 | if (keysToDelete.length) { 117 | // Do this async 118 | streamDb.batch(keysToDelete.map(function(key) { return {type: 'del', key: key} }), function(err) { 119 | if (err && !/Database is not open/.test(err)) console.error(err.stack || err) 120 | }) 121 | } 122 | }) 123 | }) 124 | } 125 | 126 | function invalidShardIterator() { 127 | return db.clientError('InvalidArgumentException', 'Invalid ShardIterator.') 128 | } 129 | 130 | // Thu Jan 22 01:22:02 UTC 2015 131 | function toAmzUtcString(date) { 132 | var pieces = new Date(date).toUTCString().match(/^(.+), (.+) (.+) (.+) (.+) GMT$/) 133 | return [pieces[1], pieces[3], pieces[2], pieces[5], 'UTC', pieces[4]].join(' ') 134 | } 135 | -------------------------------------------------------------------------------- /actions/getShardIterator.js: -------------------------------------------------------------------------------- 1 | var db = require('../db') 2 | 3 | module.exports = function getShardIterator(store, data, cb) { 4 | 5 | var metaDb = store.metaDb, shardInfo, shardId, shardIx 6 | 7 | try { 8 | shardInfo = db.resolveShardId(data.ShardId) 9 | } catch (e) { 10 | return cb(db.clientError('ResourceNotFoundException', 11 | 'Could not find shard ' + data.ShardId + ' in stream ' + data.StreamName + 12 | ' under account ' + metaDb.awsAccountId + '.')) 13 | } 14 | shardId = shardInfo.shardId 15 | shardIx = shardInfo.shardIx 16 | 17 | store.getStream(data.StreamName, function(err, stream) { 18 | if (err) { 19 | if (err.name == 'NotFoundError' && err.body) { 20 | err.body.message = 'Shard ' + shardId + ' in stream ' + data.StreamName + 21 | ' under account ' + metaDb.awsAccountId + ' does not exist' 22 | } 23 | return cb(err) 24 | } 25 | 26 | if (shardIx >= stream.Shards.length) { 27 | return cb(db.clientError('ResourceNotFoundException', 28 | 'Shard ' + shardId + ' in stream ' + data.StreamName + 29 | ' under account ' + metaDb.awsAccountId + ' does not exist')) 30 | } 31 | 32 | var seqObj, seqStr, iteratorSeq, shardSeq = stream.Shards[shardIx].SequenceNumberRange.StartingSequenceNumber, 33 | shardSeqObj = db.parseSequence(shardSeq), 34 | now = Date.now() 35 | 36 | if (data.StartingSequenceNumber) { 37 | if (data.ShardIteratorType == 'TRIM_HORIZON' || data.ShardIteratorType == 'LATEST') { 38 | return cb(db.clientError('InvalidArgumentException', 39 | 'Must either specify (1) AT_SEQUENCE_NUMBER or AFTER_SEQUENCE_NUMBER and StartingSequenceNumber or ' + 40 | '(2) TRIM_HORIZON or LATEST and no StartingSequenceNumber. ' + 41 | 'Request specified ' + data.ShardIteratorType + ' and also a StartingSequenceNumber.')) 42 | } 43 | try { 44 | seqObj = db.parseSequence(data.StartingSequenceNumber) 45 | } catch (e) { 46 | return cb(db.clientError('InvalidArgumentException', 47 | 'StartingSequenceNumber ' + data.StartingSequenceNumber + ' used in GetShardIterator on shard ' + shardId + 48 | ' in stream ' + data.StreamName + ' under account ' + metaDb.awsAccountId + ' is invalid.')) 49 | } 50 | if (seqObj.shardIx != shardIx) { 51 | return cb(db.clientError('InvalidArgumentException', 52 | 'Invalid StartingSequenceNumber. It encodes ' + db.shardIdName(seqObj.shardIx) + 53 | ', while it was used in a call to a shard with ' + shardId)) 54 | } 55 | if (seqObj.version != shardSeqObj.version || seqObj.shardCreateTime != shardSeqObj.shardCreateTime) { 56 | if (seqObj.version === 0) { 57 | return cb(db.serverError()) 58 | } 59 | return cb(db.clientError('InvalidArgumentException', 60 | 'StartingSequenceNumber ' + data.StartingSequenceNumber + ' used in GetShardIterator on shard ' + shardId + 61 | ' in stream ' + data.StreamName + ' under account ' + metaDb.awsAccountId + ' is invalid ' + 62 | 'because it did not come from this stream.')) 63 | } 64 | if (data.ShardIteratorType == 'AT_SEQUENCE_NUMBER') { 65 | iteratorSeq = data.StartingSequenceNumber 66 | } else { // AFTER_SEQUENCE_NUMBER 67 | iteratorSeq = db.incrementSequence(seqObj) 68 | } 69 | 70 | } else if (data.ShardIteratorType == 'TRIM_HORIZON') { 71 | iteratorSeq = shardSeq 72 | } else if (data.ShardIteratorType == 'LATEST') { 73 | iteratorSeq = db.stringifySequence({ 74 | shardCreateTime: shardSeqObj.shardCreateTime, 75 | seqIx: stream._seqIx[Math.floor(shardIx / 5)], 76 | seqTime: now, 77 | shardIx: shardSeqObj.shardIx, 78 | }) 79 | } else if (data.ShardIteratorType == 'AT_TIMESTAMP') { 80 | if (data.Timestamp == null) { 81 | return cb(db.clientError('InvalidArgumentException', 82 | 'Must specify timestampInMillis parameter for iterator of type AT_TIMESTAMP. Current request has no timestamp parameter.')) 83 | } 84 | var timestampInMillis = data.Timestamp * 1000 85 | if (timestampInMillis > now) { 86 | return cb(db.clientError('InvalidArgumentException', 87 | 'The timestampInMillis parameter cannot be greater than the currentTimestampInMillis. ' + 88 | 'timestampInMillis: ' + timestampInMillis + ', currentTimestampInMillis: ' + now)) 89 | } 90 | var opts = { 91 | gt: db.shardIxToHex(shardIx), 92 | lt: db.shardIxToHex(shardIx + 1), 93 | } 94 | return db.lazy(store.getStreamDb(data.StreamName).createReadStream(opts), cb) 95 | .filter(function(item) { return item.value.ApproximateArrivalTimestamp >= data.Timestamp }) 96 | .head(function(item) { 97 | iteratorSeq = item.key.split('/')[1] 98 | cb(null, {ShardIterator: db.createShardIterator(data.StreamName, shardId, iteratorSeq)}) 99 | }) 100 | } else { 101 | return cb(db.clientError('InvalidArgumentException', 102 | 'Must either specify (1) AT_SEQUENCE_NUMBER or AFTER_SEQUENCE_NUMBER and StartingSequenceNumber or ' + 103 | '(2) TRIM_HORIZON or LATEST and no StartingSequenceNumber. ' + 104 | 'Request specified ' + data.ShardIteratorType + ' and no StartingSequenceNumber.')) 105 | } 106 | cb(null, {ShardIterator: db.createShardIterator(data.StreamName, shardId, iteratorSeq)}) 107 | }) 108 | } 109 | 110 | -------------------------------------------------------------------------------- /actions/increaseStreamRetentionPeriod.js: -------------------------------------------------------------------------------- 1 | var db = require('../db') 2 | 3 | module.exports = function increaseStreamRetentionPeriod(store, data, cb) { 4 | 5 | var metaDb = store.metaDb 6 | 7 | metaDb.lock(data.StreamName, function(release) { 8 | cb = release(cb) 9 | 10 | store.getStream(data.StreamName, function(err, stream) { 11 | if (err) return cb(err) 12 | 13 | if (data.RetentionPeriodHours < 24) { 14 | return cb(db.clientError('InvalidArgumentException', 15 | 'Minimum allowed retention period is 24 hours. Requested retention period (' + data.RetentionPeriodHours + 16 | ' hours) is too short.')) 17 | } 18 | 19 | if (stream.RetentionPeriodHours > data.RetentionPeriodHours) { 20 | return cb(db.clientError('InvalidArgumentException', 21 | 'Requested retention period (' + data.RetentionPeriodHours + 22 | ' hours) for stream ' + data.StreamName + 23 | ' can not be shorter than existing retention period (' + stream.RetentionPeriodHours + 24 | ' hours). Use DecreaseRetentionPeriod API.')) 25 | } 26 | 27 | stream.RetentionPeriodHours = data.RetentionPeriodHours 28 | 29 | metaDb.put(data.StreamName, stream, function(err) { 30 | if (err) return cb(err) 31 | 32 | cb() 33 | }) 34 | }) 35 | }) 36 | } 37 | -------------------------------------------------------------------------------- /actions/listShards.js: -------------------------------------------------------------------------------- 1 | var once = require('once'), 2 | db = require('../db') 3 | 4 | module.exports = function listShards(store, data, cb) { 5 | 6 | if (!data.StreamName && !data.NextToken) { 7 | return cb(db.clientError('InvalidArgumentException', 'Either NextToken or StreamName should be provided.')) 8 | } else if (data.StreamName && data.NextToken) { 9 | return cb(db.clientError('InvalidArgumentException', 'NextToken and StreamName cannot be provided together.')) 10 | } 11 | 12 | // TODO get stream according to the given input [NextToken|StreamCreationTimestamp|StreamName] 13 | store.getStream(data.StreamName, function(err, stream) { 14 | if (err) return cb(err) 15 | 16 | var outputShards = stream.Shards 17 | if (data.ExclusiveStartShardId) { 18 | outputShards = stream.Shards.filter(shard => shard.ShardId > data.ExclusiveStartShardId) 19 | } 20 | 21 | cb(null, {Shards: outputShards}) 22 | }) 23 | } 24 | -------------------------------------------------------------------------------- /actions/listStreams.js: -------------------------------------------------------------------------------- 1 | var once = require('once'), 2 | db = require('../db') 3 | 4 | module.exports = function listStreams(store, data, cb) { 5 | cb = once(cb) 6 | var opts, keys, limit = data.Limit || 10 7 | 8 | if (data.ExclusiveStartStreamName) 9 | opts = {start: data.ExclusiveStartStreamName + '\x00'} 10 | 11 | keys = db.lazy(store.metaDb.createKeyStream(opts), cb) 12 | .take(limit + 1) 13 | 14 | keys.join(function(names) { 15 | cb(null, {StreamNames: names.slice(0, limit), HasMoreStreams: names.length > limit}) 16 | }) 17 | } 18 | 19 | 20 | -------------------------------------------------------------------------------- /actions/listTagsForStream.js: -------------------------------------------------------------------------------- 1 | 2 | module.exports = function listTagsForStream(store, data, cb) { 3 | 4 | store.getStream(data.StreamName, function(err, stream) { 5 | if (err) return cb(err) 6 | 7 | var hasMoreTags, limit = data.Limit || 100, keys = Object.keys(stream._tags).sort(), tags 8 | 9 | if (data.ExclusiveStartTagKey) 10 | keys = keys.filter(function(key) { return key > data.ExclusiveStartTagKey }) 11 | 12 | hasMoreTags = keys.length > limit 13 | tags = keys.slice(0, limit).map(function(key) { return {Key: key, Value: stream._tags[key]} }) 14 | 15 | cb(null, {Tags: tags, HasMoreTags: hasMoreTags}) 16 | }) 17 | } 18 | 19 | 20 | 21 | -------------------------------------------------------------------------------- /actions/mergeShards.js: -------------------------------------------------------------------------------- 1 | var BigNumber = require('bignumber.js'), 2 | db = require('../db') 3 | 4 | module.exports = function mergeShards(store, data, cb) { 5 | 6 | var metaDb = store.metaDb, key = data.StreamName, shardNames = [data.ShardToMerge, data.AdjacentShardToMerge], 7 | shardInfo, shardIds = [], shardIxs = [], i 8 | 9 | for (i = 0; i < shardNames.length; i++) { 10 | try { 11 | shardInfo = db.resolveShardId(shardNames[i]) 12 | } catch (e) { 13 | return cb(db.clientError('ResourceNotFoundException', 14 | 'Could not find shard ' + shardNames[i] + ' in stream ' + key + 15 | ' under account ' + metaDb.awsAccountId + '.')) 16 | } 17 | shardIds[i] = shardInfo.shardId 18 | shardIxs[i] = shardInfo.shardIx 19 | } 20 | 21 | metaDb.lock(key, function(release) { 22 | cb = release(cb) 23 | 24 | store.getStream(key, function(err, stream) { 25 | if (err) return cb(err) 26 | 27 | if (stream.StreamStatus != 'ACTIVE') { 28 | return cb(db.clientError('ResourceInUseException', 29 | 'Stream ' + data.StreamName + ' under account ' + metaDb.awsAccountId + 30 | ' not ACTIVE, instead in state ' + stream.StreamStatus)) 31 | } 32 | 33 | for (i = 0; i < shardIxs.length; i++) { 34 | if (shardIxs[i] >= stream.Shards.length) { 35 | return cb(db.clientError('ResourceNotFoundException', 36 | 'Could not find shard ' + shardIds[i] + ' in stream ' + key + 37 | ' under account ' + metaDb.awsAccountId + '.')) 38 | } 39 | } 40 | 41 | var shards = [stream.Shards[shardIxs[0]], stream.Shards[shardIxs[1]]] 42 | 43 | if (!new BigNumber(shards[0].HashKeyRange.EndingHashKey).plus(1).eq(shards[1].HashKeyRange.StartingHashKey)) { 44 | return cb(db.clientError('InvalidArgumentException', 45 | 'Shards ' + shardIds[0] + ' and ' + shardIds[1] + ' in stream ' + key + 46 | ' under account ' + metaDb.awsAccountId + ' are not an adjacent pair of shards eligible for merging')) 47 | } 48 | 49 | if (stream.StreamStatus != 'ACTIVE') { 50 | return cb(db.clientError('ResourceInUseException', 51 | 'Stream ' + key + ' under account ' + metaDb.awsAccountId + 52 | ' not ACTIVE, instead in state ' + stream.StreamStatus)) 53 | } 54 | 55 | stream.StreamStatus = 'UPDATING' 56 | 57 | metaDb.put(key, stream, function(err) { 58 | if (err) return cb(err) 59 | 60 | setTimeout(function() { 61 | 62 | metaDb.lock(key, function(release) { 63 | cb = release(function(err) { 64 | if (err && !/Database is not open/.test(err)) console.error(err.stack || err) 65 | }) 66 | 67 | store.getStream(key, function(err, stream) { 68 | if (err && err.name == 'NotFoundError') return cb() 69 | if (err) return cb(err) 70 | 71 | var now = Date.now() 72 | 73 | shards = [stream.Shards[shardIxs[0]], stream.Shards[shardIxs[1]]] 74 | 75 | stream.StreamStatus = 'ACTIVE' 76 | 77 | shards[0].SequenceNumberRange.EndingSequenceNumber = db.stringifySequence({ 78 | shardCreateTime: db.parseSequence(shards[0].SequenceNumberRange.StartingSequenceNumber).shardCreateTime, 79 | shardIx: shardIxs[0], 80 | seqIx: new BigNumber('7fffffffffffffff', 16).toFixed(), 81 | seqTime: now, 82 | }) 83 | 84 | shards[1].SequenceNumberRange.EndingSequenceNumber = db.stringifySequence({ 85 | shardCreateTime: db.parseSequence(shards[1].SequenceNumberRange.StartingSequenceNumber).shardCreateTime, 86 | shardIx: shardIxs[1], 87 | seqIx: new BigNumber('7fffffffffffffff', 16).toFixed(), 88 | seqTime: now, 89 | }) 90 | 91 | stream.Shards.push({ 92 | ParentShardId: shardIds[0], 93 | AdjacentParentShardId: shardIds[1], 94 | HashKeyRange: { 95 | StartingHashKey: shards[0].HashKeyRange.StartingHashKey, 96 | EndingHashKey: shards[1].HashKeyRange.EndingHashKey, 97 | }, 98 | SequenceNumberRange: { 99 | StartingSequenceNumber: db.stringifySequence({ 100 | shardCreateTime: now + 1000, 101 | shardIx: stream.Shards.length, 102 | }), 103 | }, 104 | ShardId: db.shardIdName(stream.Shards.length), 105 | }) 106 | 107 | metaDb.put(key, stream, cb) 108 | }) 109 | }) 110 | 111 | }, store.updateStreamMs) 112 | 113 | cb() 114 | }) 115 | }) 116 | }) 117 | } 118 | 119 | -------------------------------------------------------------------------------- /actions/putRecord.js: -------------------------------------------------------------------------------- 1 | var BigNumber = require('bignumber.js'), 2 | db = require('../db') 3 | 4 | module.exports = function putRecord(store, data, cb) { 5 | 6 | var key = data.StreamName, metaDb = store.metaDb, streamDb = store.getStreamDb(data.StreamName) 7 | 8 | metaDb.lock(key, function(release) { 9 | cb = release(cb) 10 | 11 | store.getStream(data.StreamName, function(err, stream) { 12 | if (err) return cb(err) 13 | 14 | if (!~['ACTIVE', 'UPDATING'].indexOf(stream.StreamStatus)) { 15 | return cb(db.clientError('ResourceNotFoundException', 16 | 'Stream ' + data.StreamName + ' under account ' + metaDb.awsAccountId + ' not found.')) 17 | } 18 | 19 | var hashKey, shardIx, shardId, shardCreateTime 20 | 21 | if (data.ExplicitHashKey != null) { 22 | hashKey = new BigNumber(data.ExplicitHashKey) 23 | 24 | if (hashKey.comparedTo(0) < 0 || hashKey.comparedTo(new BigNumber(2).pow(128)) >= 0) { 25 | return cb(db.clientError('InvalidArgumentException', 26 | 'Invalid ExplicitHashKey. ExplicitHashKey must be in the range: [0, 2^128-1]. ' + 27 | 'Specified value was ' + data.ExplicitHashKey)) 28 | } 29 | } else { 30 | hashKey = db.partitionKeyToHashKey(data.PartitionKey) 31 | } 32 | 33 | if (data.SequenceNumberForOrdering != null) { 34 | try { 35 | var seqObj = db.parseSequence(data.SequenceNumberForOrdering) 36 | if (seqObj.seqTime > Date.now()) throw new Error('Sequence time in the future') 37 | } catch (e) { 38 | return cb(db.clientError('InvalidArgumentException', 39 | 'ExclusiveMinimumSequenceNumber ' + data.SequenceNumberForOrdering + ' used in PutRecord on stream ' + 40 | data.StreamName + ' under account ' + metaDb.awsAccountId + ' is invalid.')) 41 | } 42 | } 43 | 44 | for (var i = 0; i < stream.Shards.length; i++) { 45 | if (stream.Shards[i].SequenceNumberRange.EndingSequenceNumber == null && 46 | hashKey.comparedTo(stream.Shards[i].HashKeyRange.StartingHashKey) >= 0 && 47 | hashKey.comparedTo(stream.Shards[i].HashKeyRange.EndingHashKey) <= 0) { 48 | shardIx = i 49 | shardId = stream.Shards[i].ShardId 50 | shardCreateTime = db.parseSequence( 51 | stream.Shards[i].SequenceNumberRange.StartingSequenceNumber).shardCreateTime 52 | break 53 | } 54 | } 55 | 56 | var seqIxIx = Math.floor(shardIx / 5), now = Math.max(Date.now(), shardCreateTime) 57 | 58 | // Ensure that the first record will always be above the stream start sequence 59 | if (!stream._seqIx[seqIxIx]) 60 | stream._seqIx[seqIxIx] = shardCreateTime == now ? 1 : 0 61 | 62 | var seqNum = db.stringifySequence({ 63 | shardCreateTime: shardCreateTime, 64 | shardIx: shardIx, 65 | seqIx: stream._seqIx[seqIxIx], 66 | seqTime: now, 67 | }) 68 | 69 | var streamKey = db.shardIxToHex(shardIx) + '/' + seqNum 70 | 71 | stream._seqIx[seqIxIx]++ 72 | 73 | metaDb.put(key, stream, function(err) { 74 | if (err) return cb(err) 75 | 76 | var record = { 77 | PartitionKey: data.PartitionKey, 78 | Data: data.Data, 79 | ApproximateArrivalTimestamp: now / 1000, 80 | } 81 | 82 | streamDb.put(streamKey, record, function(err) { 83 | if (err) return cb(err) 84 | cb(null, {ShardId: shardId, SequenceNumber: seqNum}) 85 | }) 86 | }) 87 | }) 88 | }) 89 | } 90 | -------------------------------------------------------------------------------- /actions/putRecords.js: -------------------------------------------------------------------------------- 1 | var BigNumber = require('bignumber.js'), 2 | db = require('../db') 3 | 4 | module.exports = function putRecords(store, data, cb) { 5 | 6 | var key = data.StreamName, metaDb = store.metaDb, streamDb = store.getStreamDb(data.StreamName) 7 | 8 | metaDb.lock(key, function(release) { 9 | cb = release(cb) 10 | 11 | store.getStream(data.StreamName, function(err, stream) { 12 | if (err) return cb(err) 13 | 14 | if (!~['ACTIVE', 'UPDATING'].indexOf(stream.StreamStatus)) { 15 | return cb(db.clientError('ResourceNotFoundException', 16 | 'Stream ' + data.StreamName + ' under account ' + metaDb.awsAccountId + ' not found.')) 17 | } 18 | 19 | var batchOps = new Array(data.Records.length), returnRecords = new Array(data.Records.length), 20 | seqPieces = new Array(data.Records.length), record, hashKey, seqPiece, i 21 | 22 | for (i = 0; i < data.Records.length; i++) { 23 | record = data.Records[i] 24 | 25 | if (record.ExplicitHashKey != null) { 26 | hashKey = new BigNumber(record.ExplicitHashKey) 27 | 28 | if (hashKey.comparedTo(0) < 0 || hashKey.comparedTo(new BigNumber(2).pow(128)) >= 0) { 29 | return cb(db.clientError('InvalidArgumentException', 30 | 'Invalid ExplicitHashKey. ExplicitHashKey must be in the range: [0, 2^128-1]. ' + 31 | 'Specified value was ' + record.ExplicitHashKey)) 32 | } 33 | } else { 34 | hashKey = db.partitionKeyToHashKey(record.PartitionKey) 35 | } 36 | 37 | for (var j = 0; j < stream.Shards.length; j++) { 38 | if (stream.Shards[j].SequenceNumberRange.EndingSequenceNumber == null && 39 | hashKey.comparedTo(stream.Shards[j].HashKeyRange.StartingHashKey) >= 0 && 40 | hashKey.comparedTo(stream.Shards[j].HashKeyRange.EndingHashKey) <= 0) { 41 | seqPieces[i] = { 42 | shardIx: j, 43 | shardId: stream.Shards[j].ShardId, 44 | shardCreateTime: db.parseSequence( 45 | stream.Shards[j].SequenceNumberRange.StartingSequenceNumber).shardCreateTime, 46 | } 47 | break 48 | } 49 | } 50 | } 51 | 52 | // This appears to be the order that shards are processed in a PutRecords call 53 | // XXX: No longer true – shards can be processed simultaneously and do not appear to be deterministic 54 | var shardOrder = stream.Shards.length < 18 ? 55 | [15, 16, 14, 13, 10, 12, 11, 7, 5, 9, 8, 6, 4, 3, 2, 1, 0] : stream.Shards.length < 27 ? 56 | [25, 21, 23, 22, 24, 20, 15, 19, 16, 17, 18, 11, 14, 13, 10, 12, 9, 6, 7, 5, 8, 3, 0, 4, 2, 1] : 57 | [46, 45, 49, 47, 48, 40, 42, 41, 43, 44, 35, 38, 39, 37, 36, 31, 34, 33, 32, 30, 28, 26, 27, 29, 25, 22, 24, 20, 23, 21, 15, 16, 17, 19, 18, 11, 13, 12, 14, 10, 9, 7, 8, 6, 5, 1, 3, 0, 4, 2] 58 | 59 | // Unsure of order after shard 49, just process sequentially 60 | for (i = 50; i < stream.Shards.length; i++) { 61 | shardOrder.push(i) 62 | } 63 | 64 | shardOrder.forEach(function(shardIx) { 65 | if (shardIx >= stream.Shards.length) return 66 | 67 | for (i = 0; i < data.Records.length; i++) { 68 | record = data.Records[i] 69 | seqPiece = seqPieces[i] 70 | 71 | if (seqPiece.shardIx != shardIx) continue 72 | 73 | var seqIxIx = Math.floor(shardIx / 5), now = Math.max(Date.now(), seqPiece.shardCreateTime) 74 | 75 | // Ensure that the first record will always be above the stream start sequence 76 | if (!stream._seqIx[seqIxIx]) 77 | stream._seqIx[seqIxIx] = seqPiece.shardCreateTime == now ? 1 : 0 78 | 79 | var seqNum = db.stringifySequence({ 80 | shardCreateTime: seqPiece.shardCreateTime, 81 | shardIx: shardIx, 82 | seqIx: stream._seqIx[seqIxIx], 83 | seqTime: now, 84 | }) 85 | 86 | var streamKey = db.shardIxToHex(shardIx) + '/' + seqNum 87 | 88 | stream._seqIx[seqIxIx]++ 89 | 90 | batchOps[i] = { 91 | type: 'put', 92 | key: streamKey, 93 | value: { 94 | PartitionKey: record.PartitionKey, 95 | Data: record.Data, 96 | ApproximateArrivalTimestamp: now / 1000, 97 | }, 98 | } 99 | 100 | returnRecords[i] = {ShardId: seqPiece.shardId, SequenceNumber: seqNum} 101 | } 102 | }) 103 | 104 | metaDb.put(key, stream, function(err) { 105 | if (err) return cb(err) 106 | 107 | streamDb.batch(batchOps, {}, function(err) { 108 | if (err) return cb(err) 109 | cb(null, {FailedRecordCount: 0, Records: returnRecords}) 110 | }) 111 | }) 112 | }) 113 | }) 114 | } 115 | -------------------------------------------------------------------------------- /actions/removeTagsFromStream.js: -------------------------------------------------------------------------------- 1 | var db = require('../db') 2 | 3 | module.exports = function removeTagsFromStream(store, data, cb) { 4 | 5 | var metaDb = store.metaDb 6 | 7 | metaDb.lock(data.StreamName, function(release) { 8 | cb = release(cb) 9 | 10 | store.getStream(data.StreamName, function(err, stream) { 11 | if (err) return cb(err) 12 | 13 | if (data.TagKeys.some(function(key) { return /[^\u00C0-\u1FFF\u2C00-\uD7FF\w\.\/\-=+_ @%]/.test(key) })) 14 | return cb(db.clientError('InvalidArgumentException', 15 | 'Some tags contain invalid characters. Valid characters: ' + 16 | 'Unicode letters, digits, white space, _ . / = + - % @.')) 17 | 18 | if (data.TagKeys.some(function(key) { return ~key.indexOf('%') })) 19 | return cb(db.clientError('InvalidArgumentException', 20 | 'Failed to remove tags from stream ' + data.StreamName + ' under account ' + metaDb.awsAccountId + 21 | ' because some tags contained illegal characters. The allowed characters are ' + 22 | 'Unicode letters, white-spaces, \'_\',\',\',\'/\',\'=\',\'+\',\'-\',\'@\'.')) 23 | 24 | data.TagKeys.forEach(function(key) { 25 | delete stream._tags[key] 26 | }) 27 | 28 | metaDb.put(data.StreamName, stream, function(err) { 29 | if (err) return cb(err) 30 | 31 | cb() 32 | }) 33 | }) 34 | }) 35 | } 36 | -------------------------------------------------------------------------------- /actions/splitShard.js: -------------------------------------------------------------------------------- 1 | var BigNumber = require('bignumber.js'), 2 | db = require('../db') 3 | 4 | module.exports = function splitShard(store, data, cb) { 5 | 6 | var metaDb = store.metaDb, key = data.StreamName, shardInfo, shardId, shardIx 7 | 8 | try { 9 | shardInfo = db.resolveShardId(data.ShardToSplit) 10 | } catch (e) { 11 | return cb(db.clientError('ResourceNotFoundException', 12 | 'Could not find shard ' + data.ShardToSplit + ' in stream ' + key + 13 | ' under account ' + metaDb.awsAccountId + '.')) 14 | } 15 | shardId = shardInfo.shardId 16 | shardIx = shardInfo.shardIx 17 | 18 | metaDb.lock(key, function(release) { 19 | cb = release(cb) 20 | 21 | store.getStream(key, function(err, stream) { 22 | if (err) return cb(err) 23 | 24 | if (stream.StreamStatus != 'ACTIVE') { 25 | return cb(db.clientError('ResourceInUseException', 26 | 'Stream ' + data.StreamName + ' under account ' + metaDb.awsAccountId + 27 | ' not ACTIVE, instead in state ' + stream.StreamStatus)) 28 | } 29 | 30 | if (shardIx >= stream.Shards.length) { 31 | return cb(db.clientError('ResourceNotFoundException', 32 | 'Could not find shard ' + shardId + ' in stream ' + key + 33 | ' under account ' + metaDb.awsAccountId + '.')) 34 | } 35 | 36 | db.sumShards(store, function(err, shardSum) { 37 | if (err) return cb(err) 38 | 39 | if (shardSum + 1 > store.shardLimit) { 40 | return cb(db.clientError('LimitExceededException', 41 | 'This request would exceed the shard limit for the account ' + metaDb.awsAccountId + ' in ' + 42 | metaDb.awsRegion + '. Current shard count for the account: ' + shardSum + 43 | '. Limit: ' + store.shardLimit + '. Number of additional shards that would have ' + 44 | 'resulted from this request: ' + 1 + '. Refer to the AWS Service Limits page ' + 45 | '(http://docs.aws.amazon.com/general/latest/gr/aws_service_limits.html) ' + 46 | 'for current limits and how to request higher limits.')) 47 | } 48 | 49 | var hashKey = new BigNumber(data.NewStartingHashKey), shard = stream.Shards[shardIx] 50 | 51 | if (hashKey.lte(new BigNumber(shard.HashKeyRange.StartingHashKey).plus(1)) || 52 | hashKey.gte(shard.HashKeyRange.EndingHashKey)) { 53 | return cb(db.clientError('InvalidArgumentException', 54 | 'NewStartingHashKey ' + data.NewStartingHashKey + ' used in SplitShard() on shard ' + shardId + 55 | ' in stream ' + key + ' under account ' + metaDb.awsAccountId + 56 | ' is not both greater than one plus the shard\'s StartingHashKey ' + 57 | shard.HashKeyRange.StartingHashKey + ' and less than the shard\'s EndingHashKey ' + 58 | shard.HashKeyRange.EndingHashKey + '.')) 59 | } 60 | 61 | if (stream.StreamStatus != 'ACTIVE') { 62 | return cb(db.clientError('ResourceInUseException', 63 | 'Stream ' + key + ' under account ' + metaDb.awsAccountId + 64 | ' not ACTIVE, instead in state ' + stream.StreamStatus)) 65 | } 66 | 67 | stream.StreamStatus = 'UPDATING' 68 | 69 | metaDb.put(key, stream, function(err) { 70 | if (err) return cb(err) 71 | 72 | setTimeout(function() { 73 | 74 | metaDb.lock(key, function(release) { 75 | cb = release(function(err) { 76 | if (err && !/Database is not open/.test(err)) console.error(err.stack || err) 77 | }) 78 | 79 | store.getStream(key, function(err, stream) { 80 | if (err && err.name == 'NotFoundError') return cb() 81 | if (err) return cb(err) 82 | 83 | var now = Date.now() 84 | 85 | shard = stream.Shards[shardIx] 86 | 87 | stream.StreamStatus = 'ACTIVE' 88 | 89 | shard.SequenceNumberRange.EndingSequenceNumber = db.stringifySequence({ 90 | shardCreateTime: db.parseSequence(shard.SequenceNumberRange.StartingSequenceNumber).shardCreateTime, 91 | shardIx: shardIx, 92 | seqIx: new BigNumber('7fffffffffffffff', 16).toFixed(), 93 | seqTime: now, 94 | }) 95 | 96 | stream.Shards.push({ 97 | ParentShardId: shardId, 98 | HashKeyRange: { 99 | StartingHashKey: shard.HashKeyRange.StartingHashKey, 100 | EndingHashKey: hashKey.minus(1).toFixed(), 101 | }, 102 | SequenceNumberRange: { 103 | StartingSequenceNumber: db.stringifySequence({ 104 | shardCreateTime: now + 1000, 105 | shardIx: stream.Shards.length, 106 | }), 107 | }, 108 | ShardId: db.shardIdName(stream.Shards.length), 109 | }) 110 | 111 | stream.Shards.push({ 112 | ParentShardId: shardId, 113 | HashKeyRange: { 114 | StartingHashKey: hashKey.toFixed(), 115 | EndingHashKey: shard.HashKeyRange.EndingHashKey, 116 | }, 117 | SequenceNumberRange: { 118 | StartingSequenceNumber: db.stringifySequence({ 119 | shardCreateTime: now + 1000, 120 | shardIx: stream.Shards.length, 121 | }), 122 | }, 123 | ShardId: db.shardIdName(stream.Shards.length), 124 | }) 125 | 126 | metaDb.put(key, stream, cb) 127 | }) 128 | }) 129 | 130 | }, store.updateStreamMs) 131 | 132 | cb() 133 | }) 134 | }) 135 | }) 136 | }) 137 | } 138 | 139 | 140 | -------------------------------------------------------------------------------- /cli.js: -------------------------------------------------------------------------------- 1 | #!/usr/bin/env node 2 | 3 | var argv = require('minimist')(process.argv.slice(2)) 4 | 5 | if (argv.help) { 6 | return console.log([ 7 | '', 8 | 'Usage: kinesalite [--port ] [--path ] [--ssl] [options]', 9 | '', 10 | 'A Kinesis http server, optionally backed by LevelDB', 11 | '', 12 | 'Options:', 13 | '--help Display this help message and exit', 14 | '--port The port to listen on (default: 4567)', 15 | '--path The path to use for the LevelDB store (in-memory by default)', 16 | '--ssl Enable SSL for the web server (default: false)', 17 | '--createStreamMs Amount of time streams stay in CREATING state (default: 500)', 18 | '--deleteStreamMs Amount of time streams stay in DELETING state (default: 500)', 19 | '--updateStreamMs Amount of time streams stay in UPDATING state (default: 500)', 20 | '--shardLimit Shard limit for error reporting (default: 10)', 21 | '', 22 | 'Report bugs at github.com/mhart/kinesalite/issues', 23 | ].join('\n')) 24 | } 25 | 26 | // If we're PID 1, eg in a docker container, SIGINT won't end the process as usual 27 | if (process.pid == 1) process.on('SIGINT', process.exit) 28 | 29 | var server = require('./index.js')(argv).listen(argv.port || 4567, function() { 30 | var address = server.address(), protocol = argv.ssl ? 'https' : 'http' 31 | console.log('Listening at %s://%s:%s', protocol, address.address, address.port) 32 | }) 33 | -------------------------------------------------------------------------------- /db/index.js: -------------------------------------------------------------------------------- 1 | var crypto = require('crypto'), 2 | lazy = require('lazy'), 3 | levelup = require('levelup'), 4 | memdown = require('memdown'), 5 | sub = require('subleveldown'), 6 | lock = require('lock'), 7 | BigNumber = require('bignumber.js'), 8 | once = require('once') 9 | 10 | exports.create = create 11 | exports.lazy = lazyStream 12 | exports.clientError = clientError 13 | exports.serverError = serverError 14 | exports.parseSequence = parseSequence 15 | exports.stringifySequence = stringifySequence 16 | exports.incrementSequence = incrementSequence 17 | exports.shardIxToHex = shardIxToHex 18 | exports.shardIdName = shardIdName 19 | exports.resolveShardId = resolveShardId 20 | exports.partitionKeyToHashKey = partitionKeyToHashKey 21 | exports.createShardIterator = createShardIterator 22 | exports.sumShards = sumShards 23 | exports.ITERATOR_PWD = 'kinesalite' 24 | // see https://github.com/nodejs/node/issues/16746#issuecomment-348027003 25 | exports.ITERATOR_PWD_KEY = Buffer.from('1133a5a833666b49abf28c8ba302930f0b2fb240dccd43cf4dfbc0ca91f17751', 'hex') 26 | exports.ITERATOR_PWD_IV = Buffer.from('7bf139dbabbea2d9995d6fcae1dff7da', 'hex') 27 | 28 | 29 | function create(options) { 30 | options = options || {} 31 | if (options.createStreamMs == null) options.createStreamMs = 500 32 | if (options.deleteStreamMs == null) options.deleteStreamMs = 500 33 | if (options.updateStreamMs == null) options.updateStreamMs = 500 34 | if (options.shardLimit == null) options.shardLimit = 10 35 | 36 | var db = levelup(options.path ? require('leveldown')(options.path) : memdown()), 37 | metaDb = sub(db, 'meta', {valueEncoding: 'json'}), 38 | streamDbs = [] 39 | 40 | metaDb.lock = lock.Lock() // eslint-disable-line new-cap 41 | 42 | // XXX: Is there a better way to get this? 43 | metaDb.awsAccountId = (process.env.AWS_ACCOUNT_ID || '0000-0000-0000').replace(/[^\d]/g, '') 44 | metaDb.awsRegion = process.env.AWS_REGION || process.env.AWS_DEFAULT_REGION || 'us-east-1' 45 | 46 | function getStreamDb(name) { 47 | if (!streamDbs[name]) { 48 | streamDbs[name] = sub(db, 'stream-' + name, {valueEncoding: 'json'}) 49 | streamDbs[name].lock = lock.Lock() // eslint-disable-line new-cap 50 | } 51 | return streamDbs[name] 52 | } 53 | 54 | function deleteStreamDb(name, cb) { 55 | cb = once(cb) 56 | var streamDb = getStreamDb(name) 57 | delete streamDbs[name] 58 | lazyStream(streamDb.createKeyStream(), cb).join(function(keys) { 59 | streamDb.batch(keys.map(function(key) { return {type: 'del', key: key} }), cb) 60 | }) 61 | } 62 | 63 | function getStream(name, cb) { 64 | cb = once(cb) 65 | metaDb.get(name, function(err, stream) { 66 | if (err) { 67 | if (err.name == 'NotFoundError') { 68 | err.statusCode = 400 69 | err.body = { 70 | __type: 'ResourceNotFoundException', 71 | message: 'Stream ' + name + ' under account ' + metaDb.awsAccountId + ' not found.', 72 | } 73 | } 74 | return cb(err) 75 | } 76 | 77 | cb(null, stream) 78 | }) 79 | } 80 | 81 | function recreate() { 82 | var self = this, newStore = create(options) 83 | Object.keys(newStore).forEach(function(key) { 84 | self[key] = newStore[key] 85 | }) 86 | } 87 | 88 | return { 89 | createStreamMs: options.createStreamMs, 90 | deleteStreamMs: options.deleteStreamMs, 91 | updateStreamMs: options.updateStreamMs, 92 | shardLimit: options.shardLimit, 93 | db: db, 94 | metaDb: metaDb, 95 | getStreamDb: getStreamDb, 96 | deleteStreamDb: deleteStreamDb, 97 | getStream: getStream, 98 | recreate: recreate, 99 | } 100 | } 101 | 102 | function lazyStream(stream, errHandler) { 103 | if (errHandler) stream.on('error', errHandler) 104 | var streamAsLazy = lazy(stream) 105 | stream.removeAllListeners('readable') 106 | stream.on('data', streamAsLazy.emit.bind(streamAsLazy, 'data')) 107 | if (stream.destroy) streamAsLazy.on('pipe', stream.destroy.bind(stream)) 108 | return streamAsLazy 109 | } 110 | 111 | function clientError(type, msg, statusCode) { 112 | if (statusCode == null) statusCode = 400 113 | var err = new Error(msg || type) 114 | err.statusCode = statusCode 115 | err.body = {__type: type} 116 | if (msg != null) err.body.message = msg 117 | return err 118 | } 119 | 120 | function serverError(type, msg, statusCode) { 121 | return clientError(type || 'InternalFailure', msg, statusCode || 500) 122 | } 123 | 124 | var POW_2_124 = new BigNumber(2).pow(124) 125 | var POW_2_124_NEXT = new BigNumber(2).pow(125).times(16) 126 | var POW_2_185 = new BigNumber(2).pow(185) 127 | var POW_2_185_NEXT = new BigNumber(2).pow(185).times(1.5) 128 | 129 | function parseSequence(seq) { 130 | var seqNum = new BigNumber(seq) 131 | if (seqNum.lt(POW_2_124)) { 132 | seqNum = seqNum.plus(POW_2_124) 133 | } 134 | var hex = seqNum.toString(16), version = seqNum.lt(POW_2_124_NEXT) ? 0 : 135 | (seqNum.gt(POW_2_185) && seqNum.lt(POW_2_185_NEXT)) ? parseInt(hex.slice(hex.length - 1), 16) : null 136 | if (version == 2) { 137 | var seqIxHex = hex.slice(11, 27), shardIxHex = hex.slice(38, 46) 138 | if (parseInt(seqIxHex[0], 16) > 7) throw new Error('Sequence index too high') 139 | if (parseInt(shardIxHex[0], 16) > 7) shardIxHex = '-' + shardIxHex 140 | var shardCreateSecs = parseInt(hex.slice(1, 10), 16) 141 | if (shardCreateSecs >= 16025175000) throw new Error('Date too large: ' + shardCreateSecs) 142 | return { 143 | shardCreateTime: shardCreateSecs * 1000, 144 | seqIx: new BigNumber(seqIxHex, 16).toFixed(), 145 | byte1: hex.slice(27, 29), 146 | seqTime: parseInt(hex.slice(29, 38), 16) * 1000, 147 | shardIx: parseInt(shardIxHex, 16), 148 | version: version, 149 | } 150 | } else if (version == 1) { 151 | return { 152 | shardCreateTime: parseInt(hex.slice(1, 10), 16) * 1000, 153 | byte1: hex.slice(11, 13), 154 | seqTime: parseInt(hex.slice(13, 22), 16) * 1000, 155 | seqRand: hex.slice(22, 36), 156 | seqIx: parseInt(hex.slice(36, 38), 16), 157 | shardIx: parseInt(hex.slice(38, 46), 16), 158 | version: version, 159 | } 160 | } else if (version === 0) { 161 | var shardCreateSecs = parseInt(hex.slice(1, 10), 16) 162 | if (shardCreateSecs >= 16025175000) throw new Error('Date too large: ' + shardCreateSecs) 163 | return { 164 | shardCreateTime: shardCreateSecs * 1000, 165 | byte1: hex.slice(10, 12), 166 | seqRand: hex.slice(12, 28), 167 | shardIx: parseInt(hex.slice(28, 32), 16), 168 | version: version, 169 | } 170 | } else { 171 | throw new Error('Unknown version: ' + version) 172 | } 173 | } 174 | 175 | function stringifySequence(obj) { 176 | if (obj.version == null || obj.version == 2) { 177 | return new BigNumber([ 178 | '2', 179 | ('00000000' + Math.floor(obj.shardCreateTime / 1000).toString(16)).slice(-9), 180 | (obj.shardIx || 0).toString(16).slice(-1), 181 | ('0000000000000000' + new BigNumber(obj.seqIx || 0).toString(16)).slice(-16), 182 | obj.byte1 || '00', // Unsure what this is 183 | ('00000000' + Math.floor((obj.seqTime || obj.shardCreateTime) / 1000).toString(16)).slice(-9), 184 | shardIxToHex(obj.shardIx), 185 | '2', 186 | ].join(''), 16).toFixed() 187 | } else if (obj.version == 1) { 188 | return new BigNumber([ 189 | '2', 190 | ('00000000' + Math.floor(obj.shardCreateTime / 1000).toString(16)).slice(-9), 191 | (obj.shardIx || 0).toString(16).slice(-1), 192 | obj.byte1 || '00', // Unsure what this is 193 | ('00000000' + Math.floor((obj.seqTime || obj.shardCreateTime) / 1000).toString(16)).slice(-9), 194 | obj.seqRand || '00000000000000', // Just seems to be a random string of hex 195 | ('0' + (obj.seqIx || 0).toString(16)).slice(-2), 196 | shardIxToHex(obj.shardIx), 197 | '1', 198 | ].join(''), 16).toFixed() 199 | } else if (obj.version === 0) { 200 | return new BigNumber([ 201 | '1', 202 | ('00000000' + Math.floor(obj.shardCreateTime / 1000).toString(16)).slice(-9), 203 | obj.byte1 || '00', // Unsure what this is 204 | obj.seqRand || '0000000000000000', // Unsure what this is 205 | shardIxToHex(obj.shardIx).slice(-4), 206 | ].join(''), 16).toFixed() 207 | } else { 208 | throw new Error('Unknown version: ' + obj.version) 209 | } 210 | } 211 | 212 | function incrementSequence(seqObj, seqTime) { 213 | if (typeof seqObj == 'string') seqObj = parseSequence(seqObj) 214 | 215 | return stringifySequence({ 216 | shardCreateTime: seqObj.shardCreateTime, 217 | seqIx: seqObj.seqIx, 218 | seqTime: seqTime || (seqObj.seqTime + 1000), 219 | shardIx: seqObj.shardIx, 220 | }) 221 | } 222 | 223 | function shardIxToHex(shardIx) { 224 | return ('0000000' + (shardIx || 0).toString(16)).slice(-8) 225 | } 226 | 227 | function shardIdName(shardIx) { 228 | return 'shardId-' + (shardIx < 0 ? '-' : '') + ('00000000000' + Math.abs(shardIx)).slice(shardIx < 0 ? -11 : -12) 229 | } 230 | 231 | function resolveShardId(shardId) { 232 | shardId = shardId.split('-')[1] || shardId 233 | var shardIx = /^\d+$/.test(shardId) ? parseInt(shardId) : NaN 234 | if (!(shardIx >= 0 && shardIx <= 2147483647)) throw new Error('INVALID_SHARD_ID') 235 | return { 236 | shardId: shardIdName(shardIx), 237 | shardIx: shardIx, 238 | } 239 | } 240 | 241 | // Will determine ExplicitHashKey, which will determine ShardId based on stream's HashKeyRange 242 | function partitionKeyToHashKey(partitionKey) { 243 | return new BigNumber(crypto.createHash('md5').update(partitionKey, 'utf8').digest('hex'), 16) 244 | } 245 | 246 | // Unsure how shard iterators are encoded 247 | // First eight bytes are always [0, 0, 0, 0, 0, 0, 0, 1] (perhaps version number?) 248 | // Remaining bytes are 16 byte aligned – perhaps AES encrypted? 249 | 250 | // Length depends on name length, given below calculation: 251 | // 152 + (Math.floor((data.StreamName.length + 2) / 16) * 16) 252 | function createShardIterator(streamName, shardId, seq) { 253 | var encryptStr = [ 254 | (new Array(14).join('0') + Date.now()).slice(-14), 255 | streamName, 256 | shardId, 257 | seq, 258 | new Array(37).join('0'), // Not entirely sure what would be making up all this data in production 259 | ].join('/'), 260 | cipher = crypto.createCipheriv('aes-256-cbc', exports.ITERATOR_PWD_KEY, exports.ITERATOR_PWD_IV) 261 | buffer = Buffer.concat([ 262 | Buffer.from([0, 0, 0, 0, 0, 0, 0, 1]), 263 | cipher.update(encryptStr, 'utf8'), 264 | cipher.final(), 265 | ]) 266 | return buffer.toString('base64') 267 | } 268 | 269 | // Sum shards that haven't closed yet 270 | function sumShards(store, cb) { 271 | exports.lazy(store.metaDb.createValueStream(), cb) 272 | .map(function(stream) { 273 | return stream.Shards.filter(function(shard) { 274 | return shard.SequenceNumberRange.EndingSequenceNumber == null 275 | }).length 276 | }) 277 | .sum(function(sum) { return cb(null, sum) }) 278 | } 279 | -------------------------------------------------------------------------------- /index.js: -------------------------------------------------------------------------------- 1 | var https = require('https'), 2 | http = require('http'), 3 | fs = require('fs'), 4 | path = require('path'), 5 | url = require('url'), 6 | crypto = require('crypto'), 7 | cbor = require('cbor'), 8 | uuid = require('uuid'), 9 | validations = require('./validations'), 10 | db = require('./db') 11 | 12 | var MAX_REQUEST_BYTES = 7 * 1024 * 1024 13 | var AMZ_JSON = 'application/x-amz-json-1.1' 14 | var AMZ_CBOR = 'application/x-amz-cbor-1.1' 15 | 16 | 17 | var validApis = ['Kinesis_20131202'], 18 | validOperations = ['AddTagsToStream', 'CreateStream', 'DeleteStream', 'DescribeStream', 'DescribeStreamSummary', 19 | 'GetRecords', 'GetShardIterator', 'ListShards', 'ListStreams', 'ListTagsForStream', 'MergeShards', 'PutRecord', 20 | 'PutRecords', 'RemoveTagsFromStream', 'SplitShard', 'IncreaseStreamRetentionPeriod', 'DecreaseStreamRetentionPeriod'], 21 | actions = {}, 22 | actionValidations = {} 23 | 24 | module.exports = kinesalite 25 | 26 | function kinesalite(options) { 27 | options = options || {} 28 | var server, store = db.create(options), requestHandler = httpHandler.bind(null, store) 29 | 30 | if (options.ssl) { 31 | options.key = options.key || fs.readFileSync(path.join(__dirname, 'ssl', 'server-key.pem')) 32 | options.cert = options.cert || fs.readFileSync(path.join(__dirname, 'ssl', 'server-crt.pem')) 33 | options.ca = options.ca || fs.readFileSync(path.join(__dirname, 'ssl', 'ca-crt.pem')) 34 | server = https.createServer(options, requestHandler) 35 | } else { 36 | server = http.createServer(requestHandler) 37 | } 38 | 39 | // Ensure we close DB when we're closing the server too 40 | var httpServerClose = server.close, httpServerListen = server.listen 41 | server.close = function(cb) { 42 | store.db.close(function(err) { 43 | if (err) return cb(err) 44 | // Recreate the store if the user wants to listen again 45 | server.listen = function() { 46 | store.recreate() 47 | httpServerListen.apply(server, arguments) 48 | } 49 | httpServerClose.call(server, cb) 50 | }) 51 | } 52 | 53 | return server 54 | } 55 | 56 | validOperations.forEach(function(action) { 57 | action = validations.toLowerFirst(action) 58 | actions[action] = require('./actions/' + action) 59 | actionValidations[action] = require('./validations/' + action) 60 | }) 61 | 62 | function sendRaw(req, res, body, statusCode) { 63 | req.removeAllListeners() 64 | res.statusCode = statusCode || 200 65 | if (body != null) { 66 | res.setHeader('Content-Length', Buffer.isBuffer(body) ? body.length : Buffer.byteLength(body, 'utf8')) 67 | } 68 | // AWS doesn't send a 'Connection' header but seems to use keep-alive behaviour 69 | // res.setHeader('Connection', '') 70 | // res.shouldKeepAlive = false 71 | res.end(body) 72 | } 73 | 74 | function sendJson(req, res, data, statusCode) { 75 | var body = data != null ? JSON.stringify(data) : '' 76 | res.setHeader('Content-Type', res.contentType) 77 | sendRaw(req, res, body, statusCode) 78 | } 79 | 80 | function sendCbor(req, res, data, statusCode) { 81 | var body = data != null ? cbor.Encoder.encodeOne(data) : '' 82 | res.setHeader('Content-Type', res.contentType) 83 | sendRaw(req, res, body, statusCode) 84 | } 85 | 86 | function sendResponse(req, res, data, statusCode) { 87 | return res.contentType == AMZ_CBOR ? sendCbor(req, res, data, statusCode) : 88 | sendJson(req, res, data, statusCode) 89 | } 90 | 91 | function sendError(req, res, contentValid, type, msg) { 92 | return contentValid ? sendResponse(req, res, {__type: type, message: msg}, 400) : 93 | typeof msg == 'number' ? sendRaw(req, res, '<' + type + '/>\n', msg) : 94 | sendRaw(req, res, '<' + type + '>\n ' + msg + '\n\n', 403) 95 | } 96 | 97 | function httpHandler(store, req, res) { 98 | var body 99 | req.on('error', function(err) { throw err }) 100 | req.on('data', function(data) { 101 | var newLength = data.length + (body ? body.length : 0) 102 | if (newLength > MAX_REQUEST_BYTES) { 103 | res.setHeader('Transfer-Encoding', 'chunked') 104 | return sendRaw(req, res, null, 413) 105 | } 106 | body = body ? Buffer.concat([body, data], newLength) : data 107 | }) 108 | req.on('end', function() { 109 | 110 | // All responses after this point have a RequestId 111 | res.setHeader('x-amzn-RequestId', uuid.v1()) 112 | if (req.method != 'OPTIONS' || !req.headers.origin) 113 | res.setHeader('x-amz-id-2', crypto.randomBytes(72).toString('base64')) 114 | 115 | // FIRST check if we've got an origin header: 116 | 117 | if (req.headers.origin) { 118 | res.setHeader('Access-Control-Allow-Origin', '*') 119 | 120 | // If it's a valid OPTIONS call, return here 121 | if (req.method == 'OPTIONS') { 122 | if (req.headers['access-control-request-headers']) 123 | res.setHeader('Access-Control-Allow-Headers', req.headers['access-control-request-headers']) 124 | 125 | if (req.headers['access-control-request-method']) 126 | res.setHeader('Access-Control-Allow-Methods', req.headers['access-control-request-method']) 127 | 128 | res.setHeader('Access-Control-Max-Age', 172800) 129 | return sendRaw(req, res, '') 130 | } 131 | 132 | res.setHeader('Access-Control-Expose-Headers', 'x-amzn-RequestId,x-amzn-ErrorType,x-amz-request-id,x-amz-id-2,x-amzn-ErrorMessage,Date') 133 | } 134 | 135 | if (req.method != 'POST') { 136 | return sendError(req, res, false, 'AccessDeniedException', 137 | 'Unable to determine service/operation name to be authorized') 138 | } 139 | 140 | var contentType = (req.headers['content-type'] || '').split(';')[0].trim() 141 | var contentValid = ~[AMZ_JSON, AMZ_CBOR, 'application/json'].indexOf(contentType) 142 | 143 | var target = (req.headers['x-amz-target'] || '').split('.') 144 | var service = target[0] 145 | var operation = target[1] 146 | var serviceValid = service && ~validApis.indexOf(service) 147 | var operationValid = operation && ~validOperations.indexOf(operation) 148 | 149 | // AWS doesn't seem to care about the HTTP path, so no checking needed for that 150 | 151 | // THEN if the method and content-type are ok, see if the JSON parses: 152 | 153 | if (!body) { 154 | res.contentType = contentType == AMZ_JSON ? AMZ_JSON : AMZ_CBOR 155 | return sendResponse(req, res, {__type: serviceValid && operationValid ? 'SerializationException' : 'UnknownOperationException'}, 400) 156 | } 157 | 158 | if (!contentValid) { 159 | if (!service || !operation) { 160 | return sendError(req, res, false, 'AccessDeniedException', 161 | 'Unable to determine service/operation name to be authorized') 162 | } 163 | return sendError(req, res, false, 'UnknownOperationException', 404) 164 | } 165 | 166 | res.contentType = contentType 167 | 168 | var data 169 | if (contentType == AMZ_CBOR) { 170 | try { data = cbor.Decoder.decodeFirstSync(body) } catch (e) { } 171 | } else { 172 | try { data = JSON.parse(body.toString()) } catch (e) { } 173 | } 174 | 175 | if (typeof data != 'object' || data == null) { 176 | if (contentType == 'application/json') { 177 | return sendJson(req, res, { 178 | Output: {__type: 'com.amazon.coral.service#SerializationException'}, 179 | Version: '1.0', 180 | }, 400) 181 | } 182 | return sendResponse(req, res, {__type: 'SerializationException'}, 400) 183 | } 184 | 185 | // After this point, application/json doesn't seem to progress any further 186 | if (contentType == 'application/json') { 187 | return sendJson(req, res, { 188 | Output: {__type: 'com.amazon.coral.service#UnknownOperationException'}, 189 | Version: '1.0', 190 | }, 404) 191 | } 192 | 193 | if (!serviceValid || !operationValid) { 194 | return sendResponse(req, res, {__type: 'UnknownOperationException'}, 400) 195 | } 196 | 197 | // THEN check auth: 198 | 199 | var authHeader = req.headers.authorization 200 | var query = ~req.url.indexOf('?') ? url.parse(req.url, true).query : {} 201 | var authQuery = 'X-Amz-Algorithm' in query 202 | var msg = '', params 203 | 204 | if (authHeader && authQuery) { 205 | return sendError(req, res, contentValid, 'InvalidSignatureException', 206 | 'Found both \'X-Amz-Algorithm\' as a query-string param and \'Authorization\' as HTTP header.') 207 | } 208 | 209 | if (!authHeader && !authQuery) { 210 | return sendError(req, res, contentValid, 'MissingAuthenticationTokenException', 'Missing Authentication Token') 211 | } 212 | 213 | if (authHeader) { 214 | params = ['Credential', 'Signature', 'SignedHeaders'] 215 | var authParams = authHeader.split(/,| /).slice(1).filter(Boolean).reduce(function(obj, x) { 216 | var keyVal = x.trim().split('=') 217 | obj[keyVal[0]] = keyVal[1] 218 | return obj 219 | }, {}) 220 | params.forEach(function(param) { 221 | if (!authParams[param]) 222 | msg += 'Authorization header requires \'' + param + '\' parameter. ' 223 | }) 224 | if (!req.headers['x-amz-date'] && !req.headers.date) 225 | msg += 'Authorization header requires existence of either a \'X-Amz-Date\' or a \'Date\' header. ' 226 | if (msg) msg += 'Authorization=' + authHeader 227 | 228 | } else { 229 | params = ['X-Amz-Algorithm', 'X-Amz-Credential', 'X-Amz-Signature', 'X-Amz-SignedHeaders', 'X-Amz-Date'] 230 | params.forEach(function(param) { 231 | if (!query[param]) 232 | msg += 'AWS query-string parameters must include \'' + param + '\'. ' 233 | }) 234 | if (msg) msg += 'Re-examine the query-string parameters.' 235 | } 236 | 237 | if (msg) { 238 | return sendError(req, res, contentValid, 'IncompleteSignatureException', msg) 239 | } 240 | 241 | // If we've reached here, we're good to go: 242 | 243 | var action = validations.toLowerFirst(operation) 244 | var actionValidation = actionValidations[action] 245 | try { 246 | data = validations.checkTypes(data, actionValidation.types) 247 | validations.checkValidations(data, actionValidation.types, actionValidation.custom, operation) 248 | } catch (e) { 249 | if (e.statusCode) return sendResponse(req, res, e.body, e.statusCode) 250 | throw e 251 | } 252 | 253 | actions[action](store, data, function(err, data) { 254 | if (err && err.statusCode) return sendResponse(req, res, err.body, err.statusCode) 255 | if (err) throw err 256 | sendResponse(req, res, data) 257 | }) 258 | }) 259 | } 260 | 261 | if (require.main === module) kinesalite().listen(4567) 262 | 263 | -------------------------------------------------------------------------------- /package.json: -------------------------------------------------------------------------------- 1 | { 2 | "name": "kinesalite", 3 | "version": "3.3.3", 4 | "description": "An implementation of Amazon's Kinesis built on LevelDB", 5 | "main": "index.js", 6 | "bin": "cli.js", 7 | "scripts": { 8 | "test": "mocha --require should --reporter spec -t $([ $REMOTE ] && echo 30s || echo 4s)" 9 | }, 10 | "repository": "mhart/kinesalite", 11 | "keywords": [ 12 | "kinesis", 13 | "mock", 14 | "kinesilite", 15 | "kinesis-mock", 16 | "test", 17 | "event", 18 | "stream", 19 | "streaming", 20 | "processing", 21 | "aws", 22 | "logs", 23 | "logging" 24 | ], 25 | "author": "Michael Hart ", 26 | "license": "MIT", 27 | "dependencies": { 28 | "async": "^2.6.3", 29 | "bignumber.js": "^9.0.0", 30 | "cbor": "^5.0.2", 31 | "lazy": "^1.0.11", 32 | "levelup": "^4.3.2", 33 | "lock": "^1.1.0", 34 | "memdown": "^5.1.0", 35 | "minimist": "^1.2.5", 36 | "once": "^1.3.3", 37 | "subleveldown": "^4.1.4", 38 | "uuid": "^7.0.3" 39 | }, 40 | "optionalDependencies": { 41 | "leveldown": "^5.2.1" 42 | }, 43 | "devDependencies": { 44 | "aws4": "^1.9.1", 45 | "mocha": "^6.2.3", 46 | "should": "^13.2.3" 47 | } 48 | } 49 | -------------------------------------------------------------------------------- /ssl/ca-crt.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIFKjCCAxICCQDcbDvTGc7KKjANBgkqhkiG9w0BAQUFADBXMQswCQYDVQQGEwJV 3 | UzELMAkGA1UECAwCTlkxETAPBgNVBAcMCE5ldyBZb3JrMRMwEQYDVQQKDApLaW5l 4 | c2FsaXRlMRMwEQYDVQQDDApraW5lc2FsaXRlMB4XDTE1MDcyOTIyMTEwMloXDTQy 5 | MTIxMzIyMTEwMlowVzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAk5ZMREwDwYDVQQH 6 | DAhOZXcgWW9yazETMBEGA1UECgwKS2luZXNhbGl0ZTETMBEGA1UEAwwKa2luZXNh 7 | bGl0ZTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAL7gQiXnFEKRBJQf 8 | HlDs5DmzZ5geTEiuxV3T16Xsk/+AR1Mtv96FeNglqQobN4oKs1SdJCpgONNtvLgR 9 | sku4rr1eVOGq3KyqSM0ErfVt4tPYg6gIiqt8UJZ2XyEa6xAoOSmCZthmAwxeEK2s 10 | +daJ7Ro6VWU2G/v/wkv1XEnyJQcLiV7jhyJYSKSwZO5cZesnSeHQdrPRK5NgQ7Zo 11 | usS+qpQ1UwE+38VxulJxhBwTtfOEl1IMQzT8K9BVTVByUXirt4enGeSRn2VC6BV+ 12 | jROnM/cWwXykFkCRKmaRq3NOq7l/AhrRe22PHBmexU+BjJ0KRlINz1sUTL2pDRma 13 | PC+O3EJJhJEoPLXeFFP04AgztwtzTemJlNvB2T/EmTQ8+CIdb11HZsFy/VA3xary 14 | u3OATyAxxhdKGVWJojh/DPEV5i4oC72XQd6io5l6S3YHniZ/aKWODmedjsSgKfuq 15 | 06R+RuMbz24QdrI2oqqwCYSGYwzr3zwhZyqKYlHDVICpZyuJothUXfrMPMGOqog6 16 | gHY36ce+hWYuLvOWXt9QrboeEa2YJLSSgAJA6tkvUif+hNMZqiYjaIlxSqVm84Oo 17 | 6vaXttAQlWL+IsCDh28ys+FYGcAIYd25nrFM+9MCxhhgRDqG+tpasJjim8sRqPFW 18 | 8FTHO5ExJCmOGh+UvZFKSR+sDdZ9AgMBAAEwDQYJKoZIhvcNAQEFBQADggIBAGlp 19 | THeqcL/9Pl+KhpfjToXl915HVw1S3h7GI+Zr+izx0UzRlW3Oi9B2b1rr4d3OLPK0 20 | /sPex2mVF6TKqp8R8JgS3VSEmQbRAOJozUuEyZYWPHZ9+48bTICoJtcDHaNb7ElA 21 | bwD/6iSxAZj7WDxZmOQmX4eKK0zWxsftfRWMsov8Qj5KmblizP9xMv9TKFvzxW9j 22 | DroCOZ4ENkXUrLOcyLYxSSwDdGnnv2K2MUQoV1jjF1y1as/XM5GHgJIsOiLBWe4q 23 | 69puV4d+8M9MdGkHaKTvXhzfWaU36bsouitCGrIMZb38LaBl5lrIaYCS3dYgnWbV 24 | psLYnjyfxo8owGJ5ZTWsI4K1NSvcwVlCnokjSc6szMD2XnPdgneo1BFRdDS7Y+9U 25 | NOmcZLrPMpAmDPkuggBAKTVX/f96zt5QyD+T9Tqru+9OcHJJZbA8USZW/ueGw8c/ 26 | dfHvxKjiMlzUzr461bDkjpoeEmFedxXkEFEZuQI5fjL1Igx1WXL3zlnC1+cKjkkF 27 | 8UxJofv00LpcVXDrLcAGJOJcKlvcO+UxaxC9YLWViWQY4VOBiRVG9BO3UYN6nDky 28 | akwNnnVxTQacCHSlkvYpvSEYRQN9Yrz9boMh1tSvGT3+4u4aeshLl9vu9cxaZ2fB 29 | g7Tq4f7BTmojDIrqxAaq1HVBvZ3HCBIQeIDBsnP6 30 | -----END CERTIFICATE----- 31 | -------------------------------------------------------------------------------- /ssl/ca-key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | Proc-Type: 4,ENCRYPTED 3 | DEK-Info: DES-EDE3-CBC,9E2EE4FC36E46104 4 | 5 | E6DetZ4FVFAsf3NEyx+syRB6YlDppT7SAIjyVVyl+eJAYSIesQ/yIeJ8Q78VBn1K 6 | /2zN8fAusTZ8Go48wX7e5DnaZbX/q9OfsnHRJc/sEYmEm0w7ak5JFuxR7j6A0Npr 7 | OVtfnYjJFwyIJ5o5HBFuKnu7VirPjm2tYqL9mEd9At0geQoYEWXLKVhpUhVsWr6J 8 | k/hLpD7Ldj5ZraXJHBpu1BI7Vq8tv7X7lw17PoPj20dgJKS3TOKPCEFNu/yOCuw9 9 | iuwLxwqXbVbNzUl11mUHm6AD+Mei/KT9LDqLmT+79mX3Sd68U62jwvRXURbAxK8U 10 | 2ZEbgS34wH/SPzGRsac8EItE6/Fv/5rVDCIwR5v2C+tGl46vlfNARnpGH4Eb5EqD 11 | ATMGFY0CYmD4AVE0oja+tASZlBefIbEtKt+oAuH3QwxLlUhW909t74XZw3IsSfrU 12 | GMuNGf6Ww08UmVH5HifZ1vHhue6JVs9i9sV5vJbyNZMiNFJImdUocUAcJb1WmYaQ 13 | JRk9bzhdwozTuMSOxfLB1uH+cQI4gD+klJJoQP9qVYuGnfTQbYCsmIHbuAr2djiC 14 | +QwBX2HCeOIOT3tXlucSJqugP/j0XuNHhlzV7XDXTCrDH/wf6eGlZHQdOWdXy7Kr 15 | mq6QiSJyM8VYzyEzdipsdH3Bp1cyC44fTSgCU1RNeitDXH94RBMtuqtNLtv3KuI2 16 | 7qR5pekFGvOw7qlC8sCH9HHAyU8x1pZkC+S7ZA9SuE3LWRRjKHw6pgGH5447yCKi 17 | gZaxqqA5JQLbqI3A9VmpRo5M0D9s+oTH5GjDw+WK5oIrJ4p9XhB/SwJEiPBwYY5z 18 | vACnLMOxaUjTiyU6ewZOhDczs0/Y4+sYS0LaG88EonTe3ZBns45UpHMvRCrNrgTY 19 | WUo97Iv4NTUDAADm9aDfN8a4lU33Tx6Wzlskz5ACVmR8E3llmTJaFBZd6ccGQ5FH 20 | Bq43oCbaLMoTY6e0Hzd58VoXTqp+2eTg0wFktkrczjko/XEWER951ebZ95geydZw 21 | mp6qxUI5JH3xz9Pmj15AoVdfjITNNr+H4utSPqqzItAZRA0UwMQzHXUXD3LFjt6V 22 | 7ARKu18lqYV6HhvgW4BTHou03v4JPtEcGujetXiIQlYLjA28FO4KZj64ahFuTw7M 23 | 49rJ8RwFoYYojrsXIjxg+H0m6E1L1TyQSKFbYy5fX2InULIvaui5KouFXS0T7WdE 24 | UKjelr1jqJbTdIY3VsgJqPOCZB6GUJ/VZ8a3ESkQSnWX7TKusFAgwoX04BlR5CXV 25 | ExCyHoHukmvx3tiV6x6IRQKyuhBXgNnIxirDDAv1ZwrQROTfaO1tiiAjuKWZ7a5i 26 | 9cON4yRpcZL+dF2ROFALvEgBv0tB87YNmC2h5k2zjgehK30QnMXtDQ2ZEW4P2Xra 27 | FxAPOt6hCMRO9O4FUZi8oiXwI6qctq7r5ZuZkqRfPxIkRg6ZBHY1l2XIXa2JcLgx 28 | UqtaFOdyhWLr/mlKNFbDLBhCMngHCo7S9j3mJMF/Jnn5ZCOTPvBvQVqi01T0iQOJ 29 | 3CrfZyyhYUlVyeiP6S2Ff3b/AgWQPRt6MFonXUMdhWUctOCAgNPree3qDHvxsIiy 30 | nRjGNqn79pRKgWePzPeaBvctwcg1jihkDx8vooaC/NVkQU/1E2q1vHaDB5AqzXzm 31 | drUzGF3xsKrti35i11P3zYrFK0hapg58dJ7ceMn+VwsLqkI803NAgRn+xKmnovWh 32 | NJ4hkVNSiwVkaGsNVfri3UYnyWx1xFMHaM43ZLFkjvLWmW4zEeXGlgS2PvEYd4Le 33 | W9XgoONvTitBcyIsFeMh0yun6aWnKTBpydbqtdwVlSUAzOwxIUtXbYsK2Sbx/Z13 34 | VTXN30MnrHtqA/hp94qmqtT9oYLSzswJk/C2m9JBIOr099jdos33hjnJRa1G+8pY 35 | 8SRwDmngnvfehNFdMRdQxsZ2aXD40gjbGnGRLVF+ORDbspgQmA6oq4s3tsQcQcaj 36 | IclnZ8ZlDb3HFu28N6qyuetPz4Fc5dETw8i1EzYSE9TK4RHUs2OFJurJBgxEYFo7 37 | eqntoFVCSoHtzXjtQu+v2Cu/EmH65IgVKkL/53Kia8iw9zTPJWCWw/MHQOppzmKK 38 | 5hsBlxGi36UhCuBFjA1Zu0vtYtVkSWetqIrNhaD2VKDhUmEK3JFxxstG8tPVd5EV 39 | MWYLJ85xuGi5fHjROHKNsiyzkbXTkhI7kkEGOWQqop1XTC5mqJSVzcIcfptoLxyV 40 | taRQf6y8X6HSovNdp2iN5MOrZ7stMuiWWZHJua+JHhu9dlz5rWZ4vkU+FkeHJzzR 41 | VsNNxxhMSZDoZNRJGttz+aaEZhVaJSvvmTPLi4NAq6Gp8vHifOyElqYCCy57eddt 42 | +zrldbw4AP5vtrJcKXBUPAXmfweYQTdIm3PIPG5R9hXu3P/RZ/RIbgjJ8YMnShPY 43 | plxf4d272t3EKM9GFtOvjkqW7CnUukIYC/tjpKpmXV7/1mxy+5TCOLwCVa8FmxPa 44 | El/j8p0wd7csPxBeoLnY3B7kvYTLC5c9V2glCofh8OPtEyn+coOvnMxI7H7JNW1R 45 | 7xdbePe67K95qz8II9dtbPl2KvUjv5AyEpQjG7IHA0HnrtozRIAXVRVEfV1hQ6wG 46 | 0GF8ecKQUPlqhYSlcw+LO+J3BEmX45Mmv4jyli6sk+lTAAd5YmW0LcewX1P2R9M8 47 | l584+Rgx8FVmiBlPG77+bvoA8Zg57ML493AUn5eQB6Hqq7WOp591DmuZh3SJ5kAd 48 | MSyhKg+TuE3Lsx4n14TT0qhXWbY2dl//M1KAYnSysMKOOmIlD7z5EICT6tH+n+I/ 49 | lmQy0uWuCgSAgiWDQFSsRuyQcTfu/sKbdOi+KXeOBIhEYJNqGYoK4LtzhM1VGKqj 50 | sX2b2iCfvDOKS4cekui5YUD/Clf7aWqrzt5IkR4cYQjByIoqDezFblCZ7z4ycI4p 51 | NXh09i4s0kXo7SaOs98z4/p979HRjkoflYOXCxKbV2YnpWNCbI+Mx24hy6dd7ZJR 52 | Zxn6m9wlaXgFrTMOjyrlWZJSMSksRzXJdHk6tq4jc592ymVkEDMwmDQPLWUBoVn2 53 | fgWuSC/nu3EqdNDOlfJJxzZezNbxERPCb6+R0A53LkOuBsg6rNdmU1oeBZnD5UD+ 54 | -----END RSA PRIVATE KEY----- 55 | -------------------------------------------------------------------------------- /ssl/server-crt.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE----- 2 | MIIFLzCCAxegAwIBAgIJAJVLVYneGzXHMA0GCSqGSIb3DQEBBQUAMFcxCzAJBgNV 3 | BAYTAlVTMQswCQYDVQQIDAJOWTERMA8GA1UEBwwITmV3IFlvcmsxEzARBgNVBAoM 4 | CktpbmVzYWxpdGUxEzARBgNVBAMMCmtpbmVzYWxpdGUwHhcNMTUwNzI5MjIxMTIy 5 | WhcNMTgwNDIzMjIxMTIyWjBXMQswCQYDVQQGEwJVUzELMAkGA1UECAwCTlkxETAP 6 | BgNVBAcMCE5ldyBZb3JrMRMwEQYDVQQKDApLaW5lc2FsaXRlMRMwEQYDVQQDDApr 7 | aW5lc2FsaXRlMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEA1c/rO2qJ 8 | f64t+vypNo80/yG7bFnF34vT6i8cjqxcXS6OrN+S3eXVd3xIvQKDspMt77pp8O39 9 | oD41xMb2ZdTe6+NVlPmoETja5QJ1qH8yVbG41WdcUguLfzIFiuevZtGyhzd8tKmZ 10 | Zg6Xn5kegq+mnWwNQis0U5MGFY18iRHsVECY+uFqPcek1LbQA5MtFIwQ/J9fGEW1 11 | Q8T+yF8xNtvg57RWWPABgMdalL198GYWNAVmhVH8FaNyxnJEsQznJfOQqZ+6SqVu 12 | aWA39FwUosrLMhpC3/lFKWmTyP3gqzr3C8SsFAPIACFuDch/SL2/7DDWur6idk3K 13 | 8XogWXIBq/5edUGa9lH/fWqNIT9zOgODm+/mPJgTinKh4uaU+B1Inz9kvXlc5kyu 14 | /1n4ASw7lFSD9xBCrUFtlKHbkU87uw+Cev4yf99ztHHTeiJ5mL+A612dZrdwIU0D 15 | FGwsGIEw76ULuTi4XhSvYpG0OswZ22ObonV2qM3l8fytyltvjkp+7VjHCulddCBN 16 | zDQ5kzGdWFrhCy5o2P1u2jCvmdRY0Yo8qZ7GTjd+3k47Ztah7fWVzoH05sZuI/FW 17 | 7NxCq3KBNigrp4t6jDymg8IoToXJbc+7bFaBSvTNv0nNoXkgZVx/QvBqQxbgg+B1 18 | JHHU3xONgo6RfLpm/etCGdOSI4CtIjLk0ZsCAwEAATANBgkqhkiG9w0BAQUFAAOC 19 | AgEAdA0YXPxMjhOqdwJbhbZI/L63CfqEcRoawMbTqiYcNaIPGvM+KPDQQT7YylCe 20 | r82AoIFu9Sh3ctgLaGLxDp2V0W0NQJUvLf+menjh1/c15GWgqIzj3UI61vorc8yj 21 | HZoYVTgiezIfm47IBIasmvZ2TAwzh4o4LrNSNJWal4hclrxpErnh8di8szUGT73u 22 | puWS8ceTEppXSxNBtB7wmqkqBkFDsgBI2TmqTnlrsujJ/URCg2l7GkoyAR5JZc+a 23 | fHNG6qM6UYnEyo1lLGVYK3eCLE/RAHdJ/myc/okmVuWyU1OfxjC8awo2k9cSpvK/ 24 | YK6Gb3JNDrhibsiYGXjNoMoRHs0zLJ7OLNZSKN0mfg4c4OaXB+VtPHqRRiWvCrmr 25 | dTYD6xWVbEMRD5lpahGdSddn0QhnwvsHlgHHSB2+/cee9NnzmnEG/P41Nl/EVzUR 26 | ikNtmfjllZE18c0dWvDQnB09HFbyoYaxKrrBQC3H7TVv8B5+V0C5ZUl0nB4XbdZe 27 | RlBoJWiGe/q0tvJyPLcRySEEN/RnuJx/oLNeNearltn6zOQCz4NA9rNY6MS7Jakr 28 | rz/EGTWxsrPKrAFyjFylicfbrSyWw0pgdHI75bnSTIRkkArkesxNyhweGKQ0QTla 29 | CGdnrESTJIdh/TSLoAsMWF9uyrTpj9RungdQbqVLUCpPF/E= 30 | -----END CERTIFICATE----- 31 | -------------------------------------------------------------------------------- /ssl/server-csr.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN CERTIFICATE REQUEST----- 2 | MIIEtTCCAp0CAQAwVzELMAkGA1UEBhMCVVMxCzAJBgNVBAgMAk5ZMREwDwYDVQQH 3 | DAhOZXcgWW9yazETMBEGA1UECgwKS2luZXNhbGl0ZTETMBEGA1UEAwwKa2luZXNh 4 | bGl0ZTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBANXP6ztqiX+uLfr8 5 | qTaPNP8hu2xZxd+L0+ovHI6sXF0ujqzfkt3l1Xd8SL0Cg7KTLe+6afDt/aA+NcTG 6 | 9mXU3uvjVZT5qBE42uUCdah/MlWxuNVnXFILi38yBYrnr2bRsoc3fLSpmWYOl5+Z 7 | HoKvpp1sDUIrNFOTBhWNfIkR7FRAmPrhaj3HpNS20AOTLRSMEPyfXxhFtUPE/shf 8 | MTbb4Oe0VljwAYDHWpS9ffBmFjQFZoVR/BWjcsZyRLEM5yXzkKmfukqlbmlgN/Rc 9 | FKLKyzIaQt/5RSlpk8j94Ks69wvErBQDyAAhbg3If0i9v+ww1rq+onZNyvF6IFly 10 | Aav+XnVBmvZR/31qjSE/czoDg5vv5jyYE4pyoeLmlPgdSJ8/ZL15XOZMrv9Z+AEs 11 | O5RUg/cQQq1BbZSh25FPO7sPgnr+Mn/fc7Rx03oieZi/gOtdnWa3cCFNAxRsLBiB 12 | MO+lC7k4uF4Ur2KRtDrMGdtjm6J1dqjN5fH8rcpbb45Kfu1YxwrpXXQgTcw0OZMx 13 | nVha4QsuaNj9btowr5nUWNGKPKmexk43ft5OO2bWoe31lc6B9ObGbiPxVuzcQqty 14 | gTYoK6eLeow8poPCKE6FyW3Pu2xWgUr0zb9JzaF5IGVcf0LwakMW4IPgdSRx1N8T 15 | jYKOkXy6Zv3rQhnTkiOArSIy5NGbAgMBAAGgGTAXBgkqhkiG9w0BCQcxCgwIcGFz 16 | c3dvcmQwDQYJKoZIhvcNAQEFBQADggIBAI7C0S7Q1r6OG0qHPimy82Q2NisZN/Se 17 | Vyw0VwijbgWL7NWEWG32GSdorHDPYsB7JoU6VPd0lQ367UHvgDV2kQpdEsoe4c6d 18 | 33KRuikSP3i1+NYxb9yCofEfwOkX8o9aqumRFKsjeTiLZ0MGKUHpwCdLEInbX/QF 19 | LR8n3gnYiOh0vVPye9PvaTWFzXZ8xvoYQJXJNZbjVMjl9AjB4TdRTQURa8rXw6nE 20 | NgZpqpvnuyJZBaNkp5MO25Ejthwh3DOpJz6GHPYNO8Q9BqF4T7eSd5OCMK5k/68D 21 | f+3HGD8MX/CtUpDENJj9SmDUH2Of9sr+ZJhimzQ7htOE929Kfsdhdf2V3W8vcPcW 22 | 2ReD+AxiqwjobntIxWeqtHqDN7xembGecQMY8Su7/hrE91WYxpWsx+6XRXThVGhO 23 | IywVVJgWyBfl5Uy9Zh3VoIESgblkJAX43vWuX/mYwQB0tAIy1GUoCbAxF9kY/d7b 24 | Ab1nHmiNKlCx4Fe7huHplrp5i9CdAkPPR2VbgWmZyZub9PNy6yRt71X+6z1aNBng 25 | OGomx2QCfB1haziGcdLikCVXnX5GwpiBCTQoMSdsPU7bm0YhBxSoAGjbVPyJkvgs 26 | YVgNYEnHnNaGjg0rTRyCS08Mp7yQ0SDIA+CoWtNq8JpIZIgL5vAxInUWieFxq7/T 27 | 9A89CjVVkrGq 28 | -----END CERTIFICATE REQUEST----- 29 | -------------------------------------------------------------------------------- /ssl/server-key.pem: -------------------------------------------------------------------------------- 1 | -----BEGIN RSA PRIVATE KEY----- 2 | MIIJKQIBAAKCAgEA1c/rO2qJf64t+vypNo80/yG7bFnF34vT6i8cjqxcXS6OrN+S 3 | 3eXVd3xIvQKDspMt77pp8O39oD41xMb2ZdTe6+NVlPmoETja5QJ1qH8yVbG41Wdc 4 | UguLfzIFiuevZtGyhzd8tKmZZg6Xn5kegq+mnWwNQis0U5MGFY18iRHsVECY+uFq 5 | Pcek1LbQA5MtFIwQ/J9fGEW1Q8T+yF8xNtvg57RWWPABgMdalL198GYWNAVmhVH8 6 | FaNyxnJEsQznJfOQqZ+6SqVuaWA39FwUosrLMhpC3/lFKWmTyP3gqzr3C8SsFAPI 7 | ACFuDch/SL2/7DDWur6idk3K8XogWXIBq/5edUGa9lH/fWqNIT9zOgODm+/mPJgT 8 | inKh4uaU+B1Inz9kvXlc5kyu/1n4ASw7lFSD9xBCrUFtlKHbkU87uw+Cev4yf99z 9 | tHHTeiJ5mL+A612dZrdwIU0DFGwsGIEw76ULuTi4XhSvYpG0OswZ22ObonV2qM3l 10 | 8fytyltvjkp+7VjHCulddCBNzDQ5kzGdWFrhCy5o2P1u2jCvmdRY0Yo8qZ7GTjd+ 11 | 3k47Ztah7fWVzoH05sZuI/FW7NxCq3KBNigrp4t6jDymg8IoToXJbc+7bFaBSvTN 12 | v0nNoXkgZVx/QvBqQxbgg+B1JHHU3xONgo6RfLpm/etCGdOSI4CtIjLk0ZsCAwEA 13 | AQKCAgA4SLd/uJfhiJSKEaDpv+EPmG2iJC+2nTR33VcDJ9RYozzjvScISsxa29i6 14 | CMNqatwY01W4Gq49VKMK3eVXASZ9qiaJRP89KV5WEsOuS22QqWwioUboQinCngRf 15 | gIXn5yO7/JtTY6IB/61iUgh6FoshvaPv2ze3GGjtm4VA//raYvbidBxnjvqRFF7F 16 | 0iR32DdQiRObbB4/bMg1LvnhmxglZo+kcemzYMcziH5yni0SHKnURrS5bYF1Q/dZ 17 | NDBVCTz3qhh8NfhOvQl9xg+YmxHKaqdYFmJieGxy6IIQEUGfU3KJAkXP4vz5m+IF 18 | 4A15u5GLL0vNIMVfs+p4IU7XW+q5MBL2igDICkAG60xp1oXgcvxEZ7hYVCE5f/X1 19 | AqdWcPOs9d0pFKM5pTaYYgqZ4gSbxEAo2+ukEeb+l/mE7rmCRH2TuehtbuagIy1N 20 | s/I8jh+Zugd4OC2fs6QBPKjThK2RT6h8diYL59tHl6VfPSABCmIEF8VgOcRVeazV 21 | 65EfGGer6lUOoPFRRBqYwv7msZR5X7T8JGQlTkHEkLnehroiF532ZFclEIbN+oOW 22 | vO9kSOXxQErhD9bW9gZx46/T4AQADvWA0Ulzbe3/69MknCZ1I8TDHwoYzLtLUL58 23 | xGZvG1SgkCqoeK6PxdcBDDrAjYb3NifSjHX84SGTCVp4CESegQKCAQEA6hNPzeyz 24 | XIzzAHv2fcjpt1g1214+JG1bFxwYDdih7j6bDrhhFcAcZDdokh0m8jydYkvINt5v 25 | eDftma7UgD6bxiVxvMKD25iNJ6E0ARpFPQjc750FfSEcUD6kwkhUaooRMiKCWJLz 26 | 8OEZjBXP6HhOymYotcw1qcQvVTkftDq/KJPur6PL30ayGUk5lHyt7FlFkna46sKm 27 | gQSqeC40jorBn+0IgEgZYpo+C9hPs7kUOY9t7gaCWTxDp2r9wfj20WIkpN706nTK 28 | sz9E+j3Q8Z2LuTXH29eP/VQ8WY+7Yc64fqFqZZi2Agmoaye0Ukp74+A6l7Y9WzRG 29 | l2ZwvJYLN+3tEQKCAQEA6da7g245xqeLrnH+csyT0XGapq0bNiSojprP6BRjSQuS 30 | /bGnUTkbOPBps6giDXtwGmhs2GV6UJcbrot2bv+3mWVup7SVNEr3R+021+y5tY9g 31 | Yrl+OoReEB7Dc0JZYYvWAgw2GBMkZ9sIRwFj8WKgf4L11C0V52lZhoE50GO3z5ZZ 32 | CPrf5lVb6Mkab5tjQehMo91LNDRHW0ctDXwAKELXDaCeD88N7rfLwznmbPtldabK 33 | XdV6YRVHBQpeiYKdeqPCmsmne9P9ceTa3Vnd0B9myLcFcerZqo0ZrpW/AfFdP7FI 34 | rzbwET4SIUkra1FwE+/O2cHG2aWTvOBQfUG+6mUD6wKCAQEAkIi6J0o3+jbiOXNR 35 | 4Zt0jQx/vOO1dWK880kapToVyvmHXxCEihykq1D3VfmAOeUpvS9UAmVY6R+Hq/Sj 36 | LEj8gN5QG6D0MF1DE6xbYy7of+aAHciUQg524Cm+LGTjEyILuixK1gxvalZkIva4 37 | +S+IXzuzeYYNwLTuES6DoBMCRDkMEhIO6eBKSojHcCOIdT8uUWDTj8n/a/0Ikcre 38 | EwIkyu5e6G8claDuHPxFQpFOprgkxevpAtbOY4sSA3sKVIHIZLFzA254VzQDEI2o 39 | /fgs9YD61omFVIR1+0tgBeXSUAa1nuQrxphWEUxj8MgjNbMYGuOhgTaHPCc6B4iS 40 | hsd4MQKCAQB/u3tsXM6UlDr5zG0YmmV4eBzpRQ/jMH0egLRm6pQdd4QQrnVeKdqT 41 | 7MdgiswnHzFqS3aBclUxJonJe4bzNR4+XajnVP5XtUeEc7CMnFQJOEuHQhoQrvOO 42 | zK8pC6o1XaRGNBNsbTBqxrurxbepSJ2xaYENKJ+Za/OqRHanPYFPlKoH/LpHYIM3 43 | EnstUe0TOGh87knBN3lvA985SW3wkCpW2FDfA9RxfNaCSuNyzpRqgvRx80XJOE02 44 | FKb1aHLLZh7MXLDvNCpyh3eCiC9hG3YS197Sjl3eCvtnYYcX8ZdlTlsM0u6qDITs 45 | x8I++hpF2a3dRztu8kJUXxe4hCxcb1eHAoIBAQCvbEcx1vpyoOSOXAph0lP+QxQ2 46 | RdUbbdRSpFiwLbRLgae9Vjbf7kmNMD2E910VTectW14nQn1QWETZ5CrCCvuEtljW 47 | S5NoHTryQvHJVt7EXwbzbCnLwh5pj+5fKaxl7SFy2OTrJIK7LSCrn1IdRoOZVoGF 48 | QmYIBaUlck4OINd1GTVDDHemem9tah2vi7fRTGQ/KTGHpXFHTKF1Obo0Ib/j6lMs 49 | hwDKyrgWH384LO0TgnmkgqRPNt3qNrfohJz9cp3ASjFFKBD7leyPsv9fSu1MnmJl 50 | mE6F8hY60n9zt0nSfVyEcd3IZRcMbpXq7Z+UyEOESx1lnzpfvx/xhUip08za 51 | -----END RSA PRIVATE KEY----- 52 | -------------------------------------------------------------------------------- /test/addTagsToStream.js: -------------------------------------------------------------------------------- 1 | var helpers = require('./helpers') 2 | 3 | var target = 'AddTagsToStream', 4 | request = helpers.request, 5 | opts = helpers.opts.bind(null, target), 6 | randomName = helpers.randomName, 7 | assertType = helpers.assertType.bind(null, target), 8 | assertValidation = helpers.assertValidation.bind(null, target), 9 | assertNotFound = helpers.assertNotFound.bind(null, target), 10 | assertInvalidArgument = helpers.assertInvalidArgument.bind(null, target) 11 | 12 | describe('addTagsToStream', function() { 13 | 14 | describe('serializations', function() { 15 | 16 | it('should return SerializationException when Tags is not a map', function(done) { 17 | assertType('Tags', 'Map', done) 18 | }) 19 | 20 | it('should return SerializationException when Tags.a is not a string', function(done) { 21 | assertType('Tags.a', 'String', done) 22 | }) 23 | 24 | it('should return SerializationException when StreamName is not a String', function(done) { 25 | assertType('StreamName', 'String', done) 26 | }) 27 | 28 | }) 29 | 30 | describe('validations', function() { 31 | 32 | it('should return ValidationException for no StreamName', function(done) { 33 | assertValidation({}, [ 34 | 'Value null at \'tags\' failed to satisfy constraint: ' + 35 | 'Member must not be null', 36 | 'Value null at \'streamName\' failed to satisfy constraint: ' + 37 | 'Member must not be null', 38 | ], done) 39 | }) 40 | 41 | it('should return ValidationException for empty StreamName', function(done) { 42 | assertValidation({StreamName: '', Tags: {}}, [ 43 | 'Value \'{}\' at \'tags\' failed to satisfy constraint: ' + 44 | 'Member must have length greater than or equal to 1', 45 | 'Value \'\' at \'streamName\' failed to satisfy constraint: ' + 46 | 'Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+', 47 | 'Value \'\' at \'streamName\' failed to satisfy constraint: ' + 48 | 'Member must have length greater than or equal to 1', 49 | ], done) 50 | }) 51 | 52 | it('should return ValidationException for long StreamName', function(done) { 53 | var name = new Array(129 + 1).join('a') 54 | assertValidation({StreamName: name, Tags: {'a;b': 'b'}}, [ 55 | 'Value \'' + name + '\' at \'streamName\' failed to satisfy constraint: ' + 56 | 'Member must have length less than or equal to 128', 57 | ], done) 58 | }) 59 | 60 | it('should return ValidationException for long tag key', function(done) { 61 | var name = new Array(129 + 1).join('a'), tags = {a: '1', b: '2'} 62 | tags[name] = '3' 63 | assertValidation({StreamName: randomName(), Tags: tags}, 64 | new RegExp('^1 validation error detected: ' + 65 | 'Value \'{[ab12, =]*' + name + '=3[ab12, =]*}\' at \'tags\' failed to satisfy constraint: ' + 66 | 'Map keys must satisfy constraint: \\[Member must have length less than or equal to 128, ' + 67 | 'Member must have length greater than or equal to 1\\]$'), done) 68 | }) 69 | 70 | it('should return ValidationException for long tag value', function(done) { 71 | var name = new Array(257 + 1).join('a'), tags = {a: '1', b: '2', c: name} 72 | assertValidation({StreamName: randomName(), Tags: tags}, 73 | new RegExp('^1 validation error detected: ' + 74 | 'Value \'{[ab12, =]*c=' + name + '[ab12, =]*}\' at \'tags\' failed to satisfy constraint: ' + 75 | 'Map value must satisfy constraint: \\[Member must have length less than or equal to 256, ' + 76 | 'Member must have length greater than or equal to 0\\]$'), done) 77 | }) 78 | 79 | it('should return ValidationException for too many tags', function(done) { 80 | var tags = {a: '1', b: '2', c: '3', d: '4', e: '5', f: '6', g: '7', h: '8', i: '9', j: '10', k: '11'} 81 | assertValidation({StreamName: randomName(), Tags: tags}, 82 | new RegExp('^1 validation error detected: ' + 83 | 'Value \'{[a-k0-9, =]+}\' at \'tags\' failed to satisfy constraint: ' + 84 | 'Member must have length less than or equal to 10$'), done) 85 | }) 86 | 87 | it('should return ResourceNotFoundException if stream does not exist', function(done) { 88 | var name1 = randomName() 89 | assertNotFound({StreamName: name1, Tags: {a: 'b'}}, 90 | 'Stream ' + name1 + ' under account ' + helpers.awsAccountId + ' not found.', done) 91 | }) 92 | 93 | it('should return InvalidArgumentException if ; in tag key', function(done) { 94 | assertInvalidArgument({StreamName: helpers.testStream, Tags: {'abc;def': '1'}}, 95 | 'Some tags contain invalid characters. Valid characters: ' + 96 | 'Unicode letters, digits, white space, _ . / = + - % @.', done) 97 | }) 98 | 99 | it('should return InvalidArgumentException if tab in tag key', function(done) { 100 | assertInvalidArgument({StreamName: helpers.testStream, Tags: {'abc\tdef': '1'}}, 101 | 'Some tags contain invalid characters. Valid characters: ' + 102 | 'Unicode letters, digits, white space, _ . / = + - % @.', done) 103 | }) 104 | 105 | it('should return InvalidArgumentException if newline in tag key', function(done) { 106 | assertInvalidArgument({StreamName: helpers.testStream, Tags: {'abc\ndef': '1'}}, 107 | 'Some tags contain invalid characters. Valid characters: ' + 108 | 'Unicode letters, digits, white space, _ . / = + - % @.', done) 109 | }) 110 | 111 | it('should return InvalidArgumentException if comma in tag key', function(done) { 112 | assertInvalidArgument({StreamName: helpers.testStream, Tags: {'abc,def': '1'}}, 113 | 'Some tags contain invalid characters. Valid characters: ' + 114 | 'Unicode letters, digits, white space, _ . / = + - % @.', done) 115 | }) 116 | 117 | it('should return InvalidArgumentException if % in tag key', function(done) { 118 | assertInvalidArgument({StreamName: helpers.testStream, Tags: {'abc%def': '1'}}, 119 | 'Failed to add tags to stream ' + helpers.testStream + ' under account ' + helpers.awsAccountId + 120 | ' because some tags contained illegal characters. The allowed characters are ' + 121 | 'Unicode letters, white-spaces, \'_\',\',\',\'/\',\'=\',\'+\',\'-\',\'@\'.', done) 122 | }) 123 | 124 | it('should return InvalidArgumentException if ; in tag value', function(done) { 125 | assertInvalidArgument({StreamName: helpers.testStream, Tags: {a: 'abc;def'}}, 126 | 'Some tags contain invalid characters. Valid characters: ' + 127 | 'Unicode letters, digits, white space, _ . / = + - % @.', done) 128 | }) 129 | 130 | it('should return InvalidArgumentException if tab in tag value', function(done) { 131 | assertInvalidArgument({StreamName: helpers.testStream, Tags: {a: 'abc\tdef'}}, 132 | 'Some tags contain invalid characters. Valid characters: ' + 133 | 'Unicode letters, digits, white space, _ . / = + - % @.', done) 134 | }) 135 | 136 | it('should return InvalidArgumentException if newline in tag value', function(done) { 137 | assertInvalidArgument({StreamName: helpers.testStream, Tags: {a: 'abc\ndef'}}, 138 | 'Some tags contain invalid characters. Valid characters: ' + 139 | 'Unicode letters, digits, white space, _ . / = + - % @.', done) 140 | }) 141 | 142 | it('should return InvalidArgumentException if comma in tag value', function(done) { 143 | assertInvalidArgument({StreamName: helpers.testStream, Tags: {a: 'abc,def'}}, 144 | 'Some tags contain invalid characters. Valid characters: ' + 145 | 'Unicode letters, digits, white space, _ . / = + - % @.', done) 146 | }) 147 | 148 | it('should return InvalidArgumentException if % in tag value', function(done) { 149 | assertInvalidArgument({StreamName: helpers.testStream, Tags: {a: 'abc%def'}}, 150 | 'Failed to add tags to stream ' + helpers.testStream + ' under account ' + helpers.awsAccountId + 151 | ' because some tags contained illegal characters. The allowed characters are ' + 152 | 'Unicode letters, white-spaces, \'_\',\',\',\'/\',\'=\',\'+\',\'-\',\'@\'.', done) 153 | }) 154 | 155 | }) 156 | 157 | describe('functionality', function() { 158 | 159 | it('should add and remove tags keys', function(done) { 160 | request(opts({ 161 | StreamName: helpers.testStream, 162 | Tags: {a: 'a', 'ü0 _.': 'a', '/=+-@': 'a', b: 'ü0 _./=+-@', c: ''}, 163 | }), function(err, res) { 164 | if (err) return done(err) 165 | res.statusCode.should.equal(200) 166 | 167 | function addTagsUntilInvalid(i, cb) { 168 | var tags = {} 169 | for (j = 0; j < 10; j++) { 170 | tags[i + j] = 'a' 171 | } 172 | if (i >= 40) { 173 | return assertInvalidArgument({ 174 | StreamName: helpers.testStream, 175 | Tags: tags, 176 | }, 'Failed to add tags to stream ' + helpers.testStream + ' under account ' + helpers.awsAccountId + 177 | ' because a given stream cannot have more than 10 tags associated with it.', cb) 178 | } 179 | request(opts({StreamName: helpers.testStream, Tags: tags}), function(err, res) { 180 | if (err) return cb(err) 181 | res.statusCode.should.equal(200) 182 | addTagsUntilInvalid(i + 10, cb) 183 | }) 184 | } 185 | 186 | addTagsUntilInvalid(0, function(err) { 187 | if (err) return done(err) 188 | 189 | request(helpers.opts('ListTagsForStream', {StreamName: helpers.testStream}), function(err, res) { 190 | if (err) return done(err) 191 | res.statusCode.should.equal(200) 192 | res.body.Tags.should.containEql({Key: 'a', Value: 'a'}) 193 | res.body.Tags.should.containEql({Key: 'ü0 _.', Value: 'a'}) 194 | res.body.Tags.should.containEql({Key: '/=+-@', Value: 'a'}) 195 | res.body.Tags.should.containEql({Key: 'b', Value: 'ü0 _./=+-@'}) 196 | res.body.Tags.should.containEql({Key: 'c', Value: ''}) 197 | 198 | request(opts({StreamName: helpers.testStream, Tags: {a: 'b'}}), function(err, res) { 199 | if (err) return done(err) 200 | res.statusCode.should.equal(200) 201 | 202 | request(helpers.opts('ListTagsForStream', {StreamName: helpers.testStream}), function(err, res) { 203 | if (err) return done(err) 204 | res.statusCode.should.equal(200) 205 | res.body.Tags.should.containEql({Key: 'a', Value: 'b'}) 206 | res.body.Tags.should.containEql({Key: 'ü0 _.', Value: 'a'}) 207 | res.body.Tags.should.containEql({Key: '/=+-@', Value: 'a'}) 208 | res.body.Tags.should.containEql({Key: 'b', Value: 'ü0 _./=+-@'}) 209 | res.body.Tags.should.containEql({Key: 'c', Value: ''}) 210 | 211 | function removeAllTags(tagKeys, cb) { 212 | if (!tagKeys.length) return cb() 213 | request(helpers.opts('RemoveTagsFromStream', { 214 | StreamName: helpers.testStream, 215 | TagKeys: tagKeys.slice(0, 10), 216 | }), function(err, res) { 217 | if (err) return cb(err) 218 | res.statusCode.should.equal(200) 219 | removeAllTags(tagKeys.slice(10), cb) 220 | }) 221 | } 222 | 223 | removeAllTags(res.body.Tags.map(function(t) { return t.Key }), done) 224 | }) 225 | }) 226 | }) 227 | }) 228 | }) 229 | }) 230 | 231 | }) 232 | 233 | }) 234 | 235 | -------------------------------------------------------------------------------- /test/createStream.js: -------------------------------------------------------------------------------- 1 | var async = require('async'), 2 | BigNumber = require('bignumber.js'), 3 | helpers = require('./helpers') 4 | 5 | var target = 'CreateStream', 6 | request = helpers.request, 7 | randomName = helpers.randomName, 8 | opts = helpers.opts.bind(null, target), 9 | assertType = helpers.assertType.bind(null, target), 10 | assertValidation = helpers.assertValidation.bind(null, target), 11 | assertLimitExceeded = helpers.assertLimitExceeded.bind(null, target), 12 | assertInUse = helpers.assertInUse.bind(null, target) 13 | 14 | describe('createStream', function() { 15 | 16 | describe('serializations', function() { 17 | 18 | it('should return SerializationException when StreamName is not a String', function(done) { 19 | assertType('StreamName', 'String', done) 20 | }) 21 | 22 | it('should return SerializationException when ShardCount is not an Integer', function(done) { 23 | assertType('ShardCount', 'Integer', done) 24 | }) 25 | 26 | }) 27 | 28 | describe('validations', function() { 29 | 30 | it('should return ValidationException for no StreamName', function(done) { 31 | assertValidation({}, [ 32 | 'Value null at \'shardCount\' failed to satisfy constraint: ' + 33 | 'Member must not be null', 34 | 'Value null at \'streamName\' failed to satisfy constraint: ' + 35 | 'Member must not be null', 36 | ], done) 37 | }) 38 | 39 | it('should return ValidationException for empty StreamName', function(done) { 40 | assertValidation({StreamName: '', ShardCount: 0}, [ 41 | 'Value \'0\' at \'shardCount\' failed to satisfy constraint: ' + 42 | 'Member must have value greater than or equal to 1', 43 | 'Value \'\' at \'streamName\' failed to satisfy constraint: ' + 44 | 'Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+', 45 | 'Value \'\' at \'streamName\' failed to satisfy constraint: ' + 46 | 'Member must have length greater than or equal to 1', 47 | ], done) 48 | }) 49 | 50 | it('should return ValidationException for long StreamName', function(done) { 51 | var name = new Array(129 + 1).join('a') 52 | assertValidation({StreamName: name, ShardCount: 100000000000}, [ 53 | 'Value \'2147483647\' at \'shardCount\' failed to satisfy constraint: ' + 54 | 'Member must have value less than or equal to 100000', 55 | 'Value \'' + name + '\' at \'streamName\' failed to satisfy constraint: ' + 56 | 'Member must have length less than or equal to 128', 57 | ], done) 58 | }) 59 | 60 | it('should return LimitExceededException for large ShardCount', function(done) { 61 | assertLimitExceeded({StreamName: randomName(), ShardCount: 1000}, 62 | 'This request would exceed the shard limit for the account ' + helpers.awsAccountId + ' in ' + 63 | helpers.awsRegion + '. Current shard count for the account: 3. Limit: ' + helpers.shardLimit + '. ' + 64 | 'Number of additional shards that would have resulted from this request: 1000. ' + 65 | 'Refer to the AWS Service Limits page (http://docs.aws.amazon.com/general/latest/gr/aws_service_limits.html) ' + 66 | 'for current limits and how to request higher limits.', done) 67 | }) 68 | 69 | it('should return ResourceInUseException if stream already exists', function(done) { 70 | assertInUse({StreamName: helpers.testStream, ShardCount: 1}, 71 | 'Stream ' + helpers.testStream + ' under account ' + helpers.awsAccountId + ' already exists.', done) 72 | }) 73 | 74 | }) 75 | 76 | describe('functionality', function() { 77 | 78 | it('should create a basic 3-shard stream', function(done) { 79 | this.timeout(100000) 80 | var stream = {StreamName: randomName(), ShardCount: 3} 81 | request(opts(stream), function(err, res) { 82 | if (err) return done(err) 83 | res.statusCode.should.equal(200) 84 | 85 | var createdAt = Date.now() 86 | 87 | res.body.should.equal('') 88 | 89 | request(helpers.opts('DescribeStream', stream), function(err, res) { 90 | if (err) return done(err) 91 | res.statusCode.should.equal(200) 92 | 93 | res.body.StreamDescription.StreamCreationTimestamp.should.be.above((createdAt / 1000) - 10) 94 | res.body.StreamDescription.StreamCreationTimestamp.should.be.below((createdAt / 1000) + 10) 95 | delete res.body.StreamDescription.StreamCreationTimestamp 96 | 97 | res.body.should.eql({ 98 | StreamDescription: { 99 | StreamStatus: 'CREATING', 100 | StreamName: stream.StreamName, 101 | StreamARN: 'arn:aws:kinesis:' + helpers.awsRegion + ':' + helpers.awsAccountId + 102 | ':stream/' + stream.StreamName, 103 | RetentionPeriodHours: 24, 104 | EncryptionType: 'NONE', 105 | EnhancedMonitoring: [{ShardLevelMetrics: []}], 106 | HasMoreShards: false, 107 | Shards: [], 108 | }, 109 | }) 110 | 111 | async.parallel([ 112 | helpers.assertNotFound.bind(helpers, 'GetShardIterator', 113 | {StreamName: stream.StreamName, ShardId: 'shardId-0', ShardIteratorType: 'LATEST'}, 114 | 'Shard shardId-000000000000 in stream ' + stream.StreamName + ' under account ' + helpers.awsAccountId + ' does not exist'), 115 | helpers.assertInUse.bind(helpers, 'MergeShards', 116 | {StreamName: stream.StreamName, ShardToMerge: 'shardId-0', AdjacentShardToMerge: 'shardId-1'}, 117 | 'Stream ' + stream.StreamName + ' under account ' + helpers.awsAccountId + ' not ACTIVE, instead in state CREATING'), 118 | helpers.assertNotFound.bind(helpers, 'PutRecord', 119 | {StreamName: stream.StreamName, PartitionKey: 'a', Data: ''}, 120 | 'Stream ' + stream.StreamName + ' under account ' + helpers.awsAccountId + ' not found.'), 121 | helpers.assertNotFound.bind(helpers, 'PutRecords', 122 | {StreamName: stream.StreamName, Records: [{PartitionKey: 'a', Data: ''}]}, 123 | 'Stream ' + stream.StreamName + ' under account ' + helpers.awsAccountId + ' not found.'), 124 | helpers.assertInUse.bind(helpers, 'SplitShard', 125 | {StreamName: stream.StreamName, ShardToSplit: 'shardId-0', NewStartingHashKey: '2'}, 126 | 'Stream ' + stream.StreamName + ' under account ' + helpers.awsAccountId + ' not ACTIVE, instead in state CREATING'), 127 | ], function(err) { 128 | if (err) return done(err) 129 | 130 | helpers.waitUntilActive(stream.StreamName, function(err, res) { 131 | if (err) return done(err) 132 | 133 | res.body.StreamDescription.Shards[0].SequenceNumberRange.StartingSequenceNumber.should.match(/^\d{56}$/) 134 | res.body.StreamDescription.Shards[1].SequenceNumberRange.StartingSequenceNumber.should.match(/^\d{56}$/) 135 | res.body.StreamDescription.Shards[2].SequenceNumberRange.StartingSequenceNumber.should.match(/^\d{56}$/) 136 | 137 | var startSeq0 = new BigNumber(res.body.StreamDescription.Shards[0].SequenceNumberRange.StartingSequenceNumber), 138 | startSeq1 = new BigNumber(res.body.StreamDescription.Shards[1].SequenceNumberRange.StartingSequenceNumber), 139 | startSeq2 = new BigNumber(res.body.StreamDescription.Shards[2].SequenceNumberRange.StartingSequenceNumber) 140 | 141 | startSeq1.minus(startSeq0).toFixed().should.equal('22300745198530623141535718272648361505980432') 142 | startSeq2.minus(startSeq1).toFixed().should.equal('22300745198530623141535718272648361505980432') 143 | 144 | var startDiff = parseInt(startSeq0.toString(16).slice(2, 10), 16) - (createdAt / 1000) 145 | startDiff.should.be.below(-2) 146 | startDiff.should.be.above(-7) 147 | 148 | delete res.body.StreamDescription.Shards[0].SequenceNumberRange.StartingSequenceNumber 149 | delete res.body.StreamDescription.Shards[1].SequenceNumberRange.StartingSequenceNumber 150 | delete res.body.StreamDescription.Shards[2].SequenceNumberRange.StartingSequenceNumber 151 | 152 | res.body.StreamDescription.StreamCreationTimestamp.should.be.above((createdAt / 1000) - 10) 153 | res.body.StreamDescription.StreamCreationTimestamp.should.be.below((createdAt / 1000) + 10) 154 | delete res.body.StreamDescription.StreamCreationTimestamp 155 | 156 | res.body.should.eql({ 157 | StreamDescription: { 158 | StreamStatus: 'ACTIVE', 159 | StreamName: stream.StreamName, 160 | StreamARN: 'arn:aws:kinesis:' + helpers.awsRegion + ':' + helpers.awsAccountId + 161 | ':stream/' + stream.StreamName, 162 | RetentionPeriodHours: 24, 163 | EncryptionType: 'NONE', 164 | EnhancedMonitoring: [{ShardLevelMetrics: []}], 165 | HasMoreShards: false, 166 | Shards: [{ 167 | ShardId: 'shardId-000000000000', 168 | SequenceNumberRange: {}, 169 | HashKeyRange: { 170 | StartingHashKey: '0', 171 | EndingHashKey: '113427455640312821154458202477256070484', 172 | }, 173 | }, { 174 | ShardId: 'shardId-000000000001', 175 | SequenceNumberRange: {}, 176 | HashKeyRange: { 177 | StartingHashKey: '113427455640312821154458202477256070485', 178 | EndingHashKey: '226854911280625642308916404954512140969', 179 | }, 180 | }, { 181 | ShardId: 'shardId-000000000002', 182 | SequenceNumberRange: {}, 183 | HashKeyRange: { 184 | StartingHashKey: '226854911280625642308916404954512140970', 185 | EndingHashKey: '340282366920938463463374607431768211455', 186 | }, 187 | }], 188 | }, 189 | }) 190 | 191 | request(helpers.opts('ListStreams', {Limit: 1}), function(err, res) { 192 | if (err) return done(err) 193 | res.statusCode.should.equal(200) 194 | 195 | res.body.StreamNames.should.have.length(1) 196 | res.body.HasMoreStreams.should.equal(true) 197 | 198 | request(helpers.opts('DeleteStream', {StreamName: stream.StreamName}), done) 199 | }) 200 | }) 201 | }) 202 | }) 203 | }) 204 | }) 205 | }) 206 | 207 | }) 208 | -------------------------------------------------------------------------------- /test/decreaseStreamRetentionPeriod.js: -------------------------------------------------------------------------------- 1 | var helpers = require('./helpers') 2 | 3 | require('should') 4 | 5 | var target = 'DecreaseStreamRetentionPeriod', 6 | request = helpers.request, 7 | opts = helpers.opts.bind(null, target), 8 | randomName = helpers.randomName, 9 | assertType = helpers.assertType.bind(null, target), 10 | assertValidation = helpers.assertValidation.bind(null, target), 11 | assertNotFound = helpers.assertNotFound.bind(null, target), 12 | assertInvalidArgument = helpers.assertInvalidArgument.bind(null, target) 13 | 14 | describe('decreaseStreamRetentionPeriod', function() { 15 | 16 | describe('serializations', function() { 17 | 18 | it('should return SerializationException when StreamName is not a String', function(done) { 19 | assertType('StreamName', 'String', done) 20 | }) 21 | 22 | it('should return SerializationException when RetentionPeriodHours is not an Integer', function(done) { 23 | assertType('RetentionPeriodHours', 'Integer', done) 24 | }) 25 | 26 | }) 27 | 28 | describe('validations', function() { 29 | 30 | it('should return ValidationException for no StreamName', function(done) { 31 | assertValidation({}, [ 32 | 'Value null at \'retentionPeriodHours\' failed to satisfy constraint: ' + 33 | 'Member must not be null', 34 | 'Value null at \'streamName\' failed to satisfy constraint: ' + 35 | 'Member must not be null', 36 | ], done) 37 | }) 38 | 39 | it('should return ValidationException for empty StreamName', function(done) { 40 | assertValidation({StreamName: '', RetentionPeriodHours: -1}, [ 41 | 'Value \'-1\' at \'retentionPeriodHours\' failed to satisfy constraint: ' + 42 | 'Member must have value greater than or equal to 1', 43 | 'Value \'\' at \'streamName\' failed to satisfy constraint: ' + 44 | 'Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+', 45 | 'Value \'\' at \'streamName\' failed to satisfy constraint: ' + 46 | 'Member must have length greater than or equal to 1', 47 | ], done) 48 | }) 49 | 50 | it('should return ValidationException for long StreamName', function(done) { 51 | var name = new Array(129 + 1).join('a') 52 | assertValidation({StreamName: name, RetentionPeriodHours: 24}, [ 53 | 'Value \'' + name + '\' at \'streamName\' failed to satisfy constraint: ' + 54 | 'Member must have length less than or equal to 128', 55 | ], done) 56 | }) 57 | 58 | it('should return ValidationException for retention period greater than 168', function(done) { 59 | assertValidation({StreamName: 'a', RetentionPeriodHours: 169}, [ 60 | 'Value \'169\' at \'retentionPeriodHours\' failed to satisfy constraint: ' + 61 | 'Member must have value less than or equal to 168', 62 | ], done) 63 | }) 64 | 65 | it('should return InvalidArgumentException for retention period less than 24', function(done) { 66 | assertInvalidArgument({StreamName: helpers.testStream, RetentionPeriodHours: 23}, 67 | 'Minimum allowed retention period is 24 hours. ' + 68 | 'Requested retention period (23 hours) is too short.', done) 69 | }) 70 | 71 | it('should return InvalidArgumentException for retention period greater than current', function(done) { 72 | assertInvalidArgument({StreamName: helpers.testStream, RetentionPeriodHours: 25}, 73 | 'Requested retention period (25 hours) for stream ' + helpers.testStream + 74 | ' can not be longer than existing retention period (24 hours).' + 75 | ' Use IncreaseRetentionPeriod API.', done) 76 | }) 77 | 78 | it('should return ResourceNotFoundException if stream does not exist', function(done) { 79 | var name1 = randomName() 80 | assertNotFound({StreamName: name1, RetentionPeriodHours: 25}, 81 | 'Stream ' + name1 + ' under account ' + helpers.awsAccountId + ' not found.', done) 82 | }) 83 | }) 84 | 85 | describe('functionality', function() { 86 | 87 | it('should decrease stream retention period', function(done) { 88 | this.timeout(100000) 89 | request(helpers.opts('IncreaseStreamRetentionPeriod', { 90 | StreamName: helpers.testStream, 91 | RetentionPeriodHours: 25, 92 | }), function(err, res) { 93 | if (err) return done(err) 94 | res.statusCode.should.equal(200) 95 | 96 | helpers.waitUntilActive(helpers.testStream, function(err, res) { 97 | if (err) return done(err) 98 | 99 | res.body.StreamDescription.RetentionPeriodHours.should.eql(25) 100 | 101 | request(opts({ 102 | StreamName: helpers.testStream, 103 | RetentionPeriodHours: 24, 104 | }), function(err, res) { 105 | if (err) return done(err) 106 | res.statusCode.should.equal(200) 107 | 108 | helpers.waitUntilActive(helpers.testStream, function(err, res) { 109 | if (err) return done(err) 110 | 111 | res.body.StreamDescription.RetentionPeriodHours.should.eql(24) 112 | 113 | done() 114 | }) 115 | }) 116 | }) 117 | }) 118 | }) 119 | 120 | }) 121 | 122 | }) 123 | 124 | -------------------------------------------------------------------------------- /test/deleteStream.js: -------------------------------------------------------------------------------- 1 | var helpers = require('./helpers') 2 | 3 | var target = 'DeleteStream', 4 | request = helpers.request, 5 | randomName = helpers.randomName, 6 | opts = helpers.opts.bind(null, target), 7 | assertType = helpers.assertType.bind(null, target), 8 | assertValidation = helpers.assertValidation.bind(null, target), 9 | assertNotFound = helpers.assertNotFound.bind(null, target) 10 | 11 | describe('deleteStream', function() { 12 | 13 | describe('serializations', function() { 14 | 15 | it('should return SerializationException when StreamName is not a String', function(done) { 16 | assertType('StreamName', 'String', done) 17 | }) 18 | 19 | }) 20 | 21 | describe('validations', function() { 22 | 23 | it('should return ValidationException for no StreamName', function(done) { 24 | assertValidation({}, [ 25 | 'Value null at \'streamName\' failed to satisfy constraint: ' + 26 | 'Member must not be null', 27 | ], done) 28 | }) 29 | 30 | it('should return ValidationException for empty StreamName', function(done) { 31 | assertValidation({StreamName: ''}, [ 32 | 'Value \'\' at \'streamName\' failed to satisfy constraint: ' + 33 | 'Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+', 34 | 'Value \'\' at \'streamName\' failed to satisfy constraint: ' + 35 | 'Member must have length greater than or equal to 1', 36 | ], done) 37 | }) 38 | 39 | it('should return ValidationException for long StreamName', function(done) { 40 | var name = new Array(129 + 1).join('a') 41 | assertValidation({StreamName: name}, [ 42 | 'Value \'' + name + '\' at \'streamName\' failed to satisfy constraint: ' + 43 | 'Member must have length less than or equal to 128', 44 | ], done) 45 | }) 46 | 47 | it('should return ResourceNotFoundException if stream does not exist', function(done) { 48 | var name = randomName() 49 | assertNotFound({StreamName: name}, 50 | 'Stream ' + name + ' under account ' + helpers.awsAccountId + ' not found.', done) 51 | }) 52 | 53 | }) 54 | 55 | describe('functionality', function() { 56 | 57 | it('should allow stream to be deleted while it is being created', function(done) { 58 | this.timeout(100000) 59 | var createTime = Date.now() / 1000 60 | var stream = {StreamName: randomName(), ShardCount: 1} 61 | request(helpers.opts('CreateStream', stream), function(err, res) { 62 | if (err) return done(err) 63 | res.statusCode.should.equal(200) 64 | 65 | request(helpers.opts('DescribeStream', stream), function(err, res) { 66 | if (err) return done(err) 67 | res.statusCode.should.equal(200) 68 | 69 | res.body.StreamDescription.StreamStatus.should.equal('CREATING') 70 | 71 | request(opts(stream), function(err, res) { 72 | if (err) return done(err) 73 | res.statusCode.should.equal(200) 74 | 75 | res.body.should.equal('') 76 | 77 | request(helpers.opts('DescribeStream', stream), function(err, res) { 78 | if (err) return done(err) 79 | res.statusCode.should.equal(200) 80 | 81 | res.body.StreamDescription.StreamCreationTimestamp.should.be.above(createTime - 10) 82 | res.body.StreamDescription.StreamCreationTimestamp.should.be.below(createTime + 10) 83 | delete res.body.StreamDescription.StreamCreationTimestamp 84 | 85 | res.body.should.eql({ 86 | StreamDescription: { 87 | StreamStatus: 'DELETING', 88 | StreamName: stream.StreamName, 89 | StreamARN: 'arn:aws:kinesis:' + helpers.awsRegion + ':' + helpers.awsAccountId + 90 | ':stream/' + stream.StreamName, 91 | RetentionPeriodHours: 24, 92 | EncryptionType: 'NONE', 93 | EnhancedMonitoring: [{ShardLevelMetrics: []}], 94 | HasMoreShards: false, 95 | Shards: [], 96 | }, 97 | }) 98 | 99 | helpers.waitUntilDeleted(stream.StreamName, function(err, res) { 100 | if (err) return done(err) 101 | res.body.__type.should.equal('ResourceNotFoundException') 102 | done() 103 | }) 104 | }) 105 | }) 106 | }) 107 | }) 108 | }) 109 | 110 | }) 111 | 112 | }) 113 | 114 | -------------------------------------------------------------------------------- /test/describeStream.js: -------------------------------------------------------------------------------- 1 | var helpers = require('./helpers') 2 | 3 | var target = 'DescribeStream', 4 | request = helpers.request, 5 | randomName = helpers.randomName, 6 | opts = helpers.opts.bind(null, target), 7 | assertType = helpers.assertType.bind(null, target), 8 | assertValidation = helpers.assertValidation.bind(null, target), 9 | assertNotFound = helpers.assertNotFound.bind(null, target) 10 | 11 | describe('describeStream', function() { 12 | 13 | describe('serializations', function() { 14 | 15 | it('should return SerializationException when StreamName is not a String', function(done) { 16 | assertType('StreamName', 'String', done) 17 | }) 18 | 19 | it('should return SerializationException when Limit is not an Integer', function(done) { 20 | assertType('Limit', 'Integer', done) 21 | }) 22 | 23 | it('should return SerializationException when ExclusiveStartShardId is not a String', function(done) { 24 | assertType('ExclusiveStartShardId', 'String', done) 25 | }) 26 | 27 | }) 28 | 29 | describe('validations', function() { 30 | 31 | it('should return ValidationException for no StreamName', function(done) { 32 | assertValidation({}, [ 33 | 'Value null at \'streamName\' failed to satisfy constraint: ' + 34 | 'Member must not be null', 35 | ], done) 36 | }) 37 | 38 | it('should return ValidationException for empty StreamName', function(done) { 39 | assertValidation({StreamName: '', Limit: 0, ExclusiveStartShardId: ''}, [ 40 | 'Value \'0\' at \'limit\' failed to satisfy constraint: ' + 41 | 'Member must have value greater than or equal to 1', 42 | 'Value \'\' at \'exclusiveStartShardId\' failed to satisfy constraint: ' + 43 | 'Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+', 44 | 'Value \'\' at \'exclusiveStartShardId\' failed to satisfy constraint: ' + 45 | 'Member must have length greater than or equal to 1', 46 | 'Value \'\' at \'streamName\' failed to satisfy constraint: ' + 47 | 'Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+', 48 | 'Value \'\' at \'streamName\' failed to satisfy constraint: ' + 49 | 'Member must have length greater than or equal to 1', 50 | ], done) 51 | }) 52 | 53 | it('should return ValidationException for long StreamName', function(done) { 54 | var name = new Array(129 + 1).join('a') 55 | assertValidation({StreamName: name, Limit: 100000, ExclusiveStartShardId: name}, [ 56 | 'Value \'100000\' at \'limit\' failed to satisfy constraint: ' + 57 | 'Member must have value less than or equal to 10000', 58 | 'Value \'' + name + '\' at \'exclusiveStartShardId\' failed to satisfy constraint: ' + 59 | 'Member must have length less than or equal to 128', 60 | 'Value \'' + name + '\' at \'streamName\' failed to satisfy constraint: ' + 61 | 'Member must have length less than or equal to 128', 62 | ], done) 63 | }) 64 | 65 | it('should return ResourceNotFoundException if stream does not exist', function(done) { 66 | var name = randomName() 67 | assertNotFound({StreamName: name}, 'Stream ' + name + ' under account ' + 68 | helpers.awsAccountId + ' not found.', done) 69 | }) 70 | 71 | }) 72 | 73 | describe('functionality', function() { 74 | 75 | it('should return stream description', function(done) { 76 | request(opts({StreamName: helpers.testStream}), function(err, res) { 77 | if (err) return done(err) 78 | res.statusCode.should.equal(200) 79 | 80 | res.body.StreamDescription.Shards[0].SequenceNumberRange.StartingSequenceNumber.should.match(/^\d{56}$/) 81 | res.body.StreamDescription.Shards[1].SequenceNumberRange.StartingSequenceNumber.should.match(/^\d{56}$/) 82 | res.body.StreamDescription.Shards[2].SequenceNumberRange.StartingSequenceNumber.should.match(/^\d{56}$/) 83 | 84 | delete res.body.StreamDescription.Shards[0].SequenceNumberRange.StartingSequenceNumber 85 | delete res.body.StreamDescription.Shards[1].SequenceNumberRange.StartingSequenceNumber 86 | delete res.body.StreamDescription.Shards[2].SequenceNumberRange.StartingSequenceNumber 87 | 88 | res.body.StreamDescription.StreamCreationTimestamp.should.be.above(new Date('2018-01-01') / 1000) 89 | res.body.StreamDescription.StreamCreationTimestamp.should.be.below(new Date('2118-01-01') / 1000) 90 | delete res.body.StreamDescription.StreamCreationTimestamp 91 | 92 | res.body.should.eql({ 93 | StreamDescription: { 94 | StreamStatus: 'ACTIVE', 95 | StreamName: helpers.testStream, 96 | StreamARN: 'arn:aws:kinesis:' + helpers.awsRegion + ':' + helpers.awsAccountId + 97 | ':stream/' + helpers.testStream, 98 | RetentionPeriodHours: 24, 99 | EncryptionType: 'NONE', 100 | EnhancedMonitoring: [{ShardLevelMetrics: []}], 101 | HasMoreShards: false, 102 | Shards: [{ 103 | ShardId: 'shardId-000000000000', 104 | SequenceNumberRange: {}, 105 | HashKeyRange: { 106 | StartingHashKey: '0', 107 | EndingHashKey: '113427455640312821154458202477256070484', 108 | }, 109 | }, { 110 | ShardId: 'shardId-000000000001', 111 | SequenceNumberRange: {}, 112 | HashKeyRange: { 113 | StartingHashKey: '113427455640312821154458202477256070485', 114 | EndingHashKey: '226854911280625642308916404954512140969', 115 | }, 116 | }, { 117 | ShardId: 'shardId-000000000002', 118 | SequenceNumberRange: {}, 119 | HashKeyRange: { 120 | StartingHashKey: '226854911280625642308916404954512140970', 121 | EndingHashKey: '340282366920938463463374607431768211455', 122 | }, 123 | }], 124 | }, 125 | }) 126 | 127 | done() 128 | }) 129 | }) 130 | 131 | }) 132 | 133 | }) 134 | 135 | -------------------------------------------------------------------------------- /test/describeStreamSummary.js: -------------------------------------------------------------------------------- 1 | var helpers = require('./helpers') 2 | 3 | var target = 'DescribeStreamSummary', 4 | request = helpers.request, 5 | randomName = helpers.randomName, 6 | opts = helpers.opts.bind(null, target), 7 | assertType = helpers.assertType.bind(null, target), 8 | assertValidation = helpers.assertValidation.bind(null, target), 9 | assertNotFound = helpers.assertNotFound.bind(null, target) 10 | 11 | describe('describeStreamSummary', function() { 12 | 13 | describe('serializations', function() { 14 | 15 | it('should return SerializationException when StreamName is not a String', function(done) { 16 | assertType('StreamName', 'String', done) 17 | }) 18 | 19 | }) 20 | 21 | describe('validations', function() { 22 | 23 | it('should return ValidationException for no StreamName', function(done) { 24 | assertValidation({}, [ 25 | 'Value null at \'streamName\' failed to satisfy constraint: ' + 26 | 'Member must not be null', 27 | ], done) 28 | }) 29 | 30 | it('should return ValidationException for empty StreamName', function(done) { 31 | assertValidation({StreamName: ''}, [ 32 | 'Value \'\' at \'streamName\' failed to satisfy constraint: ' + 33 | 'Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+', 34 | 'Value \'\' at \'streamName\' failed to satisfy constraint: ' + 35 | 'Member must have length greater than or equal to 1', 36 | ], done) 37 | }) 38 | 39 | it('should return ValidationException for long StreamName', function(done) { 40 | var name = new Array(129 + 1).join('a') 41 | assertValidation({StreamName: name}, [ 42 | 'Value \'' + name + '\' at \'streamName\' failed to satisfy constraint: ' + 43 | 'Member must have length less than or equal to 128', 44 | ], done) 45 | }) 46 | 47 | it('should return ResourceNotFoundException if stream does not exist', function(done) { 48 | var name = randomName() 49 | assertNotFound({StreamName: name}, 'Stream ' + name + ' under account ' + 50 | helpers.awsAccountId + ' not found.', done) 51 | }) 52 | 53 | }) 54 | 55 | describe('functionality', function() { 56 | 57 | it('should return stream description summary', function(done) { 58 | request(opts({StreamName: helpers.testStream}), function(err, res) { 59 | if (err) return done(err) 60 | res.statusCode.should.equal(200) 61 | 62 | res.body.StreamDescriptionSummary.StreamCreationTimestamp.should.be.above(0) 63 | res.body.StreamDescriptionSummary.StreamCreationTimestamp.should.be.below(Date.now() / 1000) 64 | delete res.body.StreamDescriptionSummary.StreamCreationTimestamp 65 | 66 | res.body.should.eql({ 67 | StreamDescriptionSummary: { 68 | StreamStatus: 'ACTIVE', 69 | StreamName: helpers.testStream, 70 | StreamARN: 'arn:aws:kinesis:' + helpers.awsRegion + ':' + helpers.awsAccountId + 71 | ':stream/' + helpers.testStream, 72 | RetentionPeriodHours: 24, 73 | EncryptionType: 'NONE', 74 | EnhancedMonitoring: [{ShardLevelMetrics: []}], 75 | OpenShardCount: 3, 76 | ConsumerCount: 0, 77 | }, 78 | }) 79 | 80 | done() 81 | }) 82 | }) 83 | 84 | }) 85 | 86 | }) 87 | 88 | -------------------------------------------------------------------------------- /test/increaseStreamRetentionPeriod.js: -------------------------------------------------------------------------------- 1 | var helpers = require('./helpers') 2 | 3 | require('should') 4 | 5 | var target = 'IncreaseStreamRetentionPeriod', 6 | request = helpers.request, 7 | opts = helpers.opts.bind(null, target), 8 | randomName = helpers.randomName, 9 | assertType = helpers.assertType.bind(null, target), 10 | assertValidation = helpers.assertValidation.bind(null, target), 11 | assertNotFound = helpers.assertNotFound.bind(null, target), 12 | assertInvalidArgument = helpers.assertInvalidArgument.bind(null, target) 13 | 14 | describe('increaseStreamRetentionPeriod', function() { 15 | 16 | describe('serializations', function() { 17 | 18 | it('should return SerializationException when StreamName is not a String', function(done) { 19 | assertType('StreamName', 'String', done) 20 | }) 21 | 22 | it('should return SerializationException when RetentionPeriodHours is not an Integer', function(done) { 23 | assertType('RetentionPeriodHours', 'Integer', done) 24 | }) 25 | 26 | }) 27 | 28 | describe('validations', function() { 29 | 30 | it('should return ValidationException for no StreamName', function(done) { 31 | assertValidation({}, [ 32 | 'Value null at \'retentionPeriodHours\' failed to satisfy constraint: ' + 33 | 'Member must not be null', 34 | 'Value null at \'streamName\' failed to satisfy constraint: ' + 35 | 'Member must not be null', 36 | ], done) 37 | }) 38 | 39 | it('should return ValidationException for empty StreamName', function(done) { 40 | assertValidation({StreamName: '', RetentionPeriodHours: -1}, [ 41 | 'Value \'-1\' at \'retentionPeriodHours\' failed to satisfy constraint: ' + 42 | 'Member must have value greater than or equal to 1', 43 | 'Value \'\' at \'streamName\' failed to satisfy constraint: ' + 44 | 'Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+', 45 | 'Value \'\' at \'streamName\' failed to satisfy constraint: ' + 46 | 'Member must have length greater than or equal to 1', 47 | ], done) 48 | }) 49 | 50 | it('should return ValidationException for long StreamName', function(done) { 51 | var name = new Array(129 + 1).join('a') 52 | assertValidation({StreamName: name, RetentionPeriodHours: 24}, [ 53 | 'Value \'' + name + '\' at \'streamName\' failed to satisfy constraint: ' + 54 | 'Member must have length less than or equal to 128', 55 | ], done) 56 | }) 57 | 58 | it('should return ValidationException for retention period greater than 168', function(done) { 59 | assertValidation({StreamName: 'a', RetentionPeriodHours: 169}, [ 60 | 'Value \'169\' at \'retentionPeriodHours\' failed to satisfy constraint: ' + 61 | 'Member must have value less than or equal to 168', 62 | ], done) 63 | }) 64 | 65 | it('should return InvalidArgumentException for retention period less than 24', function(done) { 66 | assertInvalidArgument({StreamName: helpers.testStream, RetentionPeriodHours: 23}, 67 | 'Minimum allowed retention period is 24 hours. ' + 68 | 'Requested retention period (23 hours) is too short.', done) 69 | }) 70 | 71 | it('should return ResourceNotFoundException if stream does not exist', function(done) { 72 | var name1 = randomName() 73 | assertNotFound({StreamName: name1, RetentionPeriodHours: 25}, 74 | 'Stream ' + name1 + ' under account ' + helpers.awsAccountId + ' not found.', done) 75 | }) 76 | }) 77 | 78 | describe('functionality', function() { 79 | 80 | it('should increase stream retention period', function(done) { 81 | this.timeout(100000) 82 | request(opts({ 83 | StreamName: helpers.testStream, 84 | RetentionPeriodHours: 25, 85 | }), function(err, res) { 86 | if (err) return done(err) 87 | res.statusCode.should.equal(200) 88 | 89 | helpers.waitUntilActive(helpers.testStream, function(err, res) { 90 | if (err) return done(err) 91 | 92 | res.body.StreamDescription.RetentionPeriodHours.should.eql(25) 93 | 94 | request(opts({ 95 | StreamName: helpers.testStream, 96 | RetentionPeriodHours: 25, 97 | }), function(err, res) { 98 | if (err) return done(err) 99 | res.statusCode.should.equal(200) 100 | 101 | assertInvalidArgument({StreamName: helpers.testStream, RetentionPeriodHours: 24}, 102 | 'Requested retention period (24 hours) for stream ' + helpers.testStream + 103 | ' can not be shorter than existing retention period (25 hours).' + 104 | ' Use DecreaseRetentionPeriod API.', 105 | function(err) { 106 | if (err) return done(err) 107 | 108 | request(helpers.opts('DecreaseStreamRetentionPeriod', { 109 | StreamName: helpers.testStream, 110 | RetentionPeriodHours: 24, 111 | }), function(err, res) { 112 | if (err) return done(err) 113 | res.statusCode.should.equal(200) 114 | 115 | helpers.waitUntilActive(helpers.testStream, function(err, res) { 116 | if (err) return done(err) 117 | 118 | res.body.StreamDescription.RetentionPeriodHours.should.eql(24) 119 | 120 | done() 121 | }) 122 | }) 123 | }) 124 | }) 125 | }) 126 | }) 127 | }) 128 | 129 | }) 130 | 131 | }) 132 | 133 | -------------------------------------------------------------------------------- /test/listShards.js: -------------------------------------------------------------------------------- 1 | var helpers = require('./helpers') 2 | 3 | var target = 'ListShards', 4 | request = helpers.request, 5 | randomName = helpers.randomName, 6 | opts = helpers.opts.bind(null, target), 7 | assertType = helpers.assertType.bind(null, target), 8 | assertValidation = helpers.assertValidation.bind(null, target), 9 | assertNotFound = helpers.assertNotFound.bind(null, target), 10 | assertInvalidArgument = helpers.assertInvalidArgument.bind(null, target) 11 | 12 | describe('listShards', function() { 13 | 14 | describe('serializations', function() { 15 | 16 | it('should return SerializationException when StreamName is not a String', function(done) { 17 | assertType('StreamName', 'String', done) 18 | }) 19 | 20 | it('should return SerializationException when MaxResults is not an Integer', function(done) { 21 | assertType('MaxResults', 'Integer', done) 22 | }) 23 | 24 | it('should return SerializationException when ExclusiveStartShardId is not a String', function(done) { 25 | assertType('ExclusiveStartShardId', 'String', done) 26 | }) 27 | 28 | it('should return SerializationException when NextToken is not a String', function(done) { 29 | assertType('NextToken', 'String', done) 30 | }) 31 | 32 | it('should return SerializationException when StreamCreationTimestamp is not a Timestamp', function(done) { 33 | assertType('StreamCreationTimestamp', 'Timestamp', done) 34 | }) 35 | 36 | }) 37 | 38 | describe('validations', function() { 39 | 40 | it('should return InvalidArgumentException for no StreamName or NextToken', function(done) { 41 | assertInvalidArgument({}, 'Either NextToken or StreamName should be provided.', done) 42 | }) 43 | 44 | it('should return ValidationException for empty StreamName', function(done) { 45 | assertValidation({StreamName: '', NextToken: '', MaxResults: 0, ExclusiveStartShardId: ''}, [ 46 | 'Value \'0\' at \'maxResults\' failed to satisfy constraint: ' + 47 | 'Member must have value greater than or equal to 1', 48 | 'Value \'\' at \'exclusiveStartShardId\' failed to satisfy constraint: ' + 49 | 'Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+', 50 | 'Value \'\' at \'exclusiveStartShardId\' failed to satisfy constraint: ' + 51 | 'Member must have length greater than or equal to 1', 52 | 'Value \'\' at \'streamName\' failed to satisfy constraint: ' + 53 | 'Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+', 54 | 'Value \'\' at \'streamName\' failed to satisfy constraint: ' + 55 | 'Member must have length greater than or equal to 1', 56 | 'Value \'\' at \'nextToken\' failed to satisfy constraint: ' + 57 | 'Member must have length greater than or equal to 1', 58 | ], done) 59 | }) 60 | 61 | it('should return ValidationException for long StreamName', function(done) { 62 | var name = new Array(129 + 1).join('a') 63 | assertValidation({StreamName: name, MaxResults: 100000, ExclusiveStartShardId: name}, [ 64 | 'Value \'100000\' at \'maxResults\' failed to satisfy constraint: ' + 65 | 'Member must have value less than or equal to 10000', 66 | 'Value \'' + name + '\' at \'exclusiveStartShardId\' failed to satisfy constraint: ' + 67 | 'Member must have length less than or equal to 128', 68 | 'Value \'' + name + '\' at \'streamName\' failed to satisfy constraint: ' + 69 | 'Member must have length less than or equal to 128', 70 | ], done) 71 | }) 72 | 73 | it('should return InvalidArgumentException if both StreamName and NextToken', function(done) { 74 | var name = randomName() 75 | assertInvalidArgument({StreamName: name, NextToken: name}, 76 | 'NextToken and StreamName cannot be provided together.', done) 77 | }) 78 | 79 | it('should return ResourceNotFoundException if stream does not exist', function(done) { 80 | var name = randomName() 81 | assertNotFound({StreamName: name}, 'Stream ' + name + ' under account ' + 82 | helpers.awsAccountId + ' not found.', done) 83 | }) 84 | 85 | }) 86 | 87 | describe('functionality', function() { 88 | 89 | it('should return stream shards', function(done) { 90 | request(opts({StreamName: helpers.testStream}), function(err, res) { 91 | if (err) return done(err) 92 | res.statusCode.should.equal(200) 93 | 94 | res.body.Shards[0].SequenceNumberRange.StartingSequenceNumber.should.match(/^\d{56}$/) 95 | res.body.Shards[1].SequenceNumberRange.StartingSequenceNumber.should.match(/^\d{56}$/) 96 | res.body.Shards[2].SequenceNumberRange.StartingSequenceNumber.should.match(/^\d{56}$/) 97 | 98 | delete res.body.Shards[0].SequenceNumberRange.StartingSequenceNumber 99 | delete res.body.Shards[1].SequenceNumberRange.StartingSequenceNumber 100 | delete res.body.Shards[2].SequenceNumberRange.StartingSequenceNumber 101 | 102 | res.body.should.eql({ 103 | Shards: [{ 104 | ShardId: 'shardId-000000000000', 105 | SequenceNumberRange: {}, 106 | HashKeyRange: { 107 | StartingHashKey: '0', 108 | EndingHashKey: '113427455640312821154458202477256070484', 109 | }, 110 | }, { 111 | ShardId: 'shardId-000000000001', 112 | SequenceNumberRange: {}, 113 | HashKeyRange: { 114 | StartingHashKey: '113427455640312821154458202477256070485', 115 | EndingHashKey: '226854911280625642308916404954512140969', 116 | }, 117 | }, { 118 | ShardId: 'shardId-000000000002', 119 | SequenceNumberRange: {}, 120 | HashKeyRange: { 121 | StartingHashKey: '226854911280625642308916404954512140970', 122 | EndingHashKey: '340282366920938463463374607431768211455', 123 | }, 124 | }], 125 | }) 126 | 127 | done() 128 | }) 129 | }) 130 | 131 | it('should return stream shards starting with shard which follows ExclusiveStartShardId', function(done) { 132 | request(opts({StreamName: helpers.testStream, ExclusiveStartShardId: 'shardId-000000000001'}), function(err, res) { 133 | if (err) return done(err) 134 | res.statusCode.should.equal(200) 135 | 136 | res.body.Shards[0].SequenceNumberRange.StartingSequenceNumber.should.match(/^\d{56}$/) 137 | 138 | delete res.body.Shards[0].SequenceNumberRange.StartingSequenceNumber 139 | 140 | res.body.should.eql({ 141 | Shards: [{ 142 | ShardId: 'shardId-000000000002', 143 | SequenceNumberRange: {}, 144 | HashKeyRange: { 145 | StartingHashKey: '226854911280625642308916404954512140970', 146 | EndingHashKey: '340282366920938463463374607431768211455', 147 | }, 148 | }], 149 | }) 150 | 151 | done() 152 | }) 153 | }) 154 | 155 | }) 156 | 157 | }) 158 | -------------------------------------------------------------------------------- /test/listStreams.js: -------------------------------------------------------------------------------- 1 | var helpers = require('./helpers') 2 | 3 | var target = 'ListStreams', 4 | request = helpers.request, 5 | opts = helpers.opts.bind(null, target), 6 | assertType = helpers.assertType.bind(null, target), 7 | assertValidation = helpers.assertValidation.bind(null, target) 8 | 9 | describe('listStreams', function() { 10 | 11 | describe('serializations', function() { 12 | 13 | it('should return SerializationException when Limit is not an Integer', function(done) { 14 | assertType('Limit', 'Integer', done) 15 | }) 16 | 17 | it('should return SerializationException when ExclusiveStartStreamName is not a String', function(done) { 18 | assertType('ExclusiveStartStreamName', 'String', done) 19 | }) 20 | 21 | }) 22 | 23 | describe('validations', function() { 24 | 25 | it('should return ValidationException for empty ExclusiveStartStreamName', function(done) { 26 | assertValidation({ExclusiveStartStreamName: '', Limit: 0}, [ 27 | 'Value \'0\' at \'limit\' failed to satisfy constraint: ' + 28 | 'Member must have value greater than or equal to 1', 29 | 'Value \'\' at \'exclusiveStartStreamName\' failed to satisfy constraint: ' + 30 | 'Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+', 31 | 'Value \'\' at \'exclusiveStartStreamName\' failed to satisfy constraint: ' + 32 | 'Member must have length greater than or equal to 1', 33 | ], done) 34 | }) 35 | 36 | it('should return ValidationException for long ExclusiveStartStreamName', function(done) { 37 | var name = new Array(129 + 1).join('a') 38 | assertValidation({ExclusiveStartStreamName: name, Limit: 1000000}, [ 39 | 'Value \'1000000\' at \'limit\' failed to satisfy constraint: ' + 40 | 'Member must have value less than or equal to 10000', 41 | 'Value \'' + name + '\' at \'exclusiveStartStreamName\' failed to satisfy constraint: ' + 42 | 'Member must have length less than or equal to 128', 43 | ], done) 44 | }) 45 | 46 | }) 47 | 48 | describe('functionality', function() { 49 | 50 | it('should return all streams', function(done) { 51 | request(opts({}), function(err, res) { 52 | if (err) return done(err) 53 | res.statusCode.should.equal(200) 54 | Object.keys(res.body).sort().should.eql(['HasMoreStreams', 'StreamNames']) 55 | res.body.HasMoreStreams.should.equal(false) 56 | res.body.StreamNames.should.containEql(helpers.testStream) 57 | done() 58 | }) 59 | }) 60 | 61 | it('should return single stream', function(done) { 62 | var name = helpers.testStream.slice(0, helpers.testStream.length - 1) 63 | request(opts({ExclusiveStartStreamName: name, Limit: 1}), function(err, res) { 64 | if (err) return done(err) 65 | res.statusCode.should.equal(200) 66 | Object.keys(res.body).sort().should.eql(['HasMoreStreams', 'StreamNames']) 67 | res.body.StreamNames.should.eql([helpers.testStream]) 68 | done() 69 | }) 70 | }) 71 | 72 | }) 73 | 74 | }) 75 | 76 | -------------------------------------------------------------------------------- /test/listTagsForStream.js: -------------------------------------------------------------------------------- 1 | var helpers = require('./helpers') 2 | 3 | var target = 'ListTagsForStream', 4 | request = helpers.request, 5 | opts = helpers.opts.bind(null, target), 6 | randomName = helpers.randomName, 7 | assertType = helpers.assertType.bind(null, target), 8 | assertValidation = helpers.assertValidation.bind(null, target), 9 | assertNotFound = helpers.assertNotFound.bind(null, target) 10 | 11 | describe('listTagsForStream', function() { 12 | 13 | describe('serializations', function() { 14 | 15 | it('should return SerializationException when Limit is not an Integer', function(done) { 16 | assertType('Limit', 'Integer', done) 17 | }) 18 | 19 | it('should return SerializationException when ExclusiveStartTagKey is not a String', function(done) { 20 | assertType('ExclusiveStartTagKey', 'String', done) 21 | }) 22 | 23 | it('should return SerializationException when StreamName is not a String', function(done) { 24 | assertType('StreamName', 'String', done) 25 | }) 26 | 27 | }) 28 | 29 | describe('validations', function() { 30 | 31 | it('should return ValidationException for no StreamName', function(done) { 32 | assertValidation({}, [ 33 | 'Value null at \'streamName\' failed to satisfy constraint: ' + 34 | 'Member must not be null', 35 | ], done) 36 | }) 37 | 38 | it('should return ValidationException for empty StreamName', function(done) { 39 | assertValidation({StreamName: '', ExclusiveStartTagKey: '', Limit: 0}, [ 40 | 'Value \'0\' at \'limit\' failed to satisfy constraint: ' + 41 | 'Member must have value greater than or equal to 1', 42 | 'Value \'\' at \'exclusiveStartTagKey\' failed to satisfy constraint: ' + 43 | 'Member must have length greater than or equal to 1', 44 | 'Value \'\' at \'streamName\' failed to satisfy constraint: ' + 45 | 'Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+', 46 | 'Value \'\' at \'streamName\' failed to satisfy constraint: ' + 47 | 'Member must have length greater than or equal to 1', 48 | ], done) 49 | }) 50 | 51 | it('should return ValidationException for long StreamName', function(done) { 52 | var name = new Array(129 + 1).join('a') 53 | assertValidation({StreamName: name, ExclusiveStartTagKey: name, Limit: 100}, [ 54 | 'Value \'100\' at \'limit\' failed to satisfy constraint: ' + 55 | 'Member must have value less than or equal to 10', 56 | 'Value \'' + name + '\' at \'exclusiveStartTagKey\' failed to satisfy constraint: ' + 57 | 'Member must have length less than or equal to 128', 58 | 'Value \'' + name + '\' at \'streamName\' failed to satisfy constraint: ' + 59 | 'Member must have length less than or equal to 128', 60 | ], done) 61 | }) 62 | 63 | it('should return ResourceNotFoundException if stream does not exist', function(done) { 64 | var name1 = randomName() 65 | assertNotFound({StreamName: name1, ExclusiveStartTagKey: 'a', Limit: 1}, 66 | 'Stream ' + name1 + ' under account ' + helpers.awsAccountId + ' not found.', done) 67 | }) 68 | 69 | }) 70 | 71 | describe('functionality', function() { 72 | 73 | it('should return empty list by default', function(done) { 74 | request(opts({StreamName: helpers.testStream}), function(err, res) { 75 | if (err) return done(err) 76 | res.statusCode.should.equal(200) 77 | res.body.should.eql({ 78 | Tags: [], 79 | HasMoreTags: false, 80 | }) 81 | done() 82 | }) 83 | }) 84 | 85 | it('should return empty list with limit and start key', function(done) { 86 | request(opts({StreamName: helpers.testStream, ExclusiveStartTagKey: 'a', Limit: 1}), function(err, res) { 87 | if (err) return done(err) 88 | res.statusCode.should.equal(200) 89 | res.body.should.eql({ 90 | Tags: [], 91 | HasMoreTags: false, 92 | }) 93 | done() 94 | }) 95 | }) 96 | 97 | it('should list in alphabetical order', function(done) { 98 | request(helpers.opts('AddTagsToStream', { 99 | StreamName: helpers.testStream, 100 | Tags: {a: 'b', ' ': 'a', 'ÿ': 'a', '_': 'a', '/': 'a', '=': 'a', '+': 'a', Zb: 'z', 0: 'a', '@': 'a'}, // can't do '%' or '\t' 101 | }), function(err, res) { 102 | if (err) return done(err) 103 | res.statusCode.should.equal(200) 104 | 105 | request(opts({StreamName: helpers.testStream}), function(err, res) { 106 | if (err) return done(err) 107 | res.statusCode.should.equal(200) 108 | res.body.should.eql({ 109 | Tags: [ 110 | {Key: ' ', Value: 'a'}, 111 | {Key: '+', Value: 'a'}, 112 | {Key: '/', Value: 'a'}, 113 | {Key: '0', Value: 'a'}, 114 | {Key: '=', Value: 'a'}, 115 | {Key: '@', Value: 'a'}, 116 | {Key: 'Zb', Value: 'z'}, 117 | {Key: '_', Value: 'a'}, 118 | {Key: 'a', Value: 'b'}, 119 | {Key: 'ÿ', Value: 'a'}, 120 | ], 121 | HasMoreTags: false, 122 | }) 123 | 124 | request(opts({ 125 | StreamName: helpers.testStream, 126 | ExclusiveStartTagKey: '@', 127 | Limit: 2, 128 | }), function(err, res) { 129 | if (err) return done(err) 130 | res.statusCode.should.equal(200) 131 | res.body.should.eql({ 132 | Tags: [ 133 | {Key: 'Zb', Value: 'z'}, 134 | {Key: '_', Value: 'a'}, 135 | ], 136 | HasMoreTags: true, 137 | }) 138 | 139 | request(opts({ 140 | StreamName: helpers.testStream, 141 | ExclusiveStartTagKey: '$Z%*(*&@,,.,,ZAC', 142 | }), function(err, res) { 143 | if (err) return done(err) 144 | res.statusCode.should.equal(200) 145 | res.body.should.eql({ 146 | Tags: [ 147 | {Key: '+', Value: 'a'}, 148 | {Key: '/', Value: 'a'}, 149 | {Key: '0', Value: 'a'}, 150 | {Key: '=', Value: 'a'}, 151 | {Key: '@', Value: 'a'}, 152 | {Key: 'Zb', Value: 'z'}, 153 | {Key: '_', Value: 'a'}, 154 | {Key: 'a', Value: 'b'}, 155 | {Key: 'ÿ', Value: 'a'}, 156 | ], 157 | HasMoreTags: false, 158 | }) 159 | 160 | request(opts({ 161 | StreamName: helpers.testStream, 162 | ExclusiveStartTagKey: 'Za$Z%*(*&@,,.,,', 163 | }), function(err, res) { 164 | if (err) return done(err) 165 | res.statusCode.should.equal(200) 166 | res.body.should.eql({ 167 | Tags: [ 168 | {Key: 'Zb', Value: 'z'}, 169 | {Key: '_', Value: 'a'}, 170 | {Key: 'a', Value: 'b'}, 171 | {Key: 'ÿ', Value: 'a'}, 172 | ], 173 | HasMoreTags: false, 174 | }) 175 | 176 | request(helpers.opts('RemoveTagsFromStream', { 177 | StreamName: helpers.testStream, 178 | TagKeys: ['a', ' ', 'ÿ', '_', '/', '=', '+', 'Zb', '0', '@'], 179 | }), done) 180 | }) 181 | }) 182 | }) 183 | }) 184 | }) 185 | }) 186 | 187 | }) 188 | 189 | }) 190 | 191 | 192 | -------------------------------------------------------------------------------- /test/putRecord.js: -------------------------------------------------------------------------------- 1 | var BigNumber = require('bignumber.js'), 2 | helpers = require('./helpers') 3 | 4 | var target = 'PutRecord', 5 | request = helpers.request, 6 | randomName = helpers.randomName, 7 | opts = helpers.opts.bind(null, target), 8 | assertType = helpers.assertType.bind(null, target), 9 | assertValidation = helpers.assertValidation.bind(null, target), 10 | assertNotFound = helpers.assertNotFound.bind(null, target), 11 | assertInternalFailure = helpers.assertInternalFailure.bind(null, target), 12 | assertInvalidArgument = helpers.assertInvalidArgument.bind(null, target) 13 | 14 | describe('putRecord ', function() { 15 | 16 | describe('serializations', function() { 17 | 18 | it('should return SerializationException when Data is not a Blob', function(done) { 19 | assertType('Data', 'Blob', done) 20 | }) 21 | 22 | it('should return SerializationException when ExplicitHashKey is not a String', function(done) { 23 | assertType('ExplicitHashKey', 'String', done) 24 | }) 25 | 26 | it('should return SerializationException when PartitionKey is not a String', function(done) { 27 | assertType('PartitionKey', 'String', done) 28 | }) 29 | 30 | it('should return SerializationException when SequenceNumberForOrdering is not a String', function(done) { 31 | assertType('SequenceNumberForOrdering', 'String', done) 32 | }) 33 | 34 | it('should return SerializationException when StreamName is not a String', function(done) { 35 | assertType('StreamName', 'String', done) 36 | }) 37 | 38 | }) 39 | 40 | describe('validations', function() { 41 | 42 | it('should return ValidationException for no StreamName', function(done) { 43 | assertValidation({}, [ 44 | 'Value null at \'partitionKey\' failed to satisfy constraint: ' + 45 | 'Member must not be null', 46 | 'Value null at \'data\' failed to satisfy constraint: ' + 47 | 'Member must not be null', 48 | 'Value null at \'streamName\' failed to satisfy constraint: ' + 49 | 'Member must not be null', 50 | ], done) 51 | }) 52 | 53 | it('should return ValidationException for empty StreamName', function(done) { 54 | assertValidation({StreamName: '', PartitionKey: '', Data: '', ExplicitHashKey: '', SequenceNumberForOrdering: ''}, [ 55 | 'Value \'\' at \'sequenceNumberForOrdering\' failed to satisfy constraint: ' + 56 | 'Member must satisfy regular expression pattern: 0|([1-9]\\d{0,128})', 57 | 'Value \'\' at \'partitionKey\' failed to satisfy constraint: ' + 58 | 'Member must have length greater than or equal to 1', 59 | 'Value \'\' at \'explicitHashKey\' failed to satisfy constraint: ' + 60 | 'Member must satisfy regular expression pattern: 0|([1-9]\\d{0,38})', 61 | 'Value \'\' at \'streamName\' failed to satisfy constraint: ' + 62 | 'Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+', 63 | 'Value \'\' at \'streamName\' failed to satisfy constraint: ' + 64 | 'Member must have length greater than or equal to 1', 65 | ], done) 66 | }) 67 | 68 | it('should return ValidationException for long StreamName', function(done) { 69 | var name = new Array(129 + 1).join('a'), name2 = new Array(257 + 1).join('a'), 70 | data = Buffer.allocUnsafe(1048577).toString('base64') 71 | assertValidation({StreamName: name, PartitionKey: name2, Data: data, ExplicitHashKey: ''}, [ 72 | 'Value \'' + name2 + '\' at \'partitionKey\' failed to satisfy constraint: ' + 73 | 'Member must have length less than or equal to 256', 74 | 'Value \'\' at \'explicitHashKey\' failed to satisfy constraint: ' + 75 | 'Member must satisfy regular expression pattern: 0|([1-9]\\d{0,38})', 76 | 'Value \'java.nio.HeapByteBuffer[pos=0 lim=1048577 cap=1048577]\' at \'data\' failed to satisfy constraint: ' + 77 | 'Member must have length less than or equal to 1048576', 78 | 'Value \'' + name + '\' at \'streamName\' failed to satisfy constraint: ' + 79 | 'Member must have length less than or equal to 128', 80 | ], done) 81 | }) 82 | 83 | it('should return ResourceNotFoundException if stream does not exist', function(done) { 84 | var name1 = randomName(), name2 = helpers.randomString() 85 | assertNotFound({StreamName: name1, PartitionKey: name2, Data: ''}, 86 | 'Stream ' + name1 + ' under account ' + helpers.awsAccountId + ' not found.', done) 87 | }) 88 | 89 | it('should return InvalidArgumentException for out of bounds ExplicitHashKey', function(done) { 90 | var hashKey = new BigNumber(2).pow(128).toFixed() 91 | assertInvalidArgument({StreamName: helpers.testStream, PartitionKey: 'a', Data: '', ExplicitHashKey: hashKey}, 92 | 'Invalid ExplicitHashKey. ExplicitHashKey must be in the range: [0, 2^128-1]. Specified value was ' + hashKey, done) 93 | }) 94 | 95 | it('should return InvalidArgumentException for version 0 in SequenceNumberForOrdering', function(done) { 96 | var seq = new BigNumber('20000000000000000000000000000000000000000000000', 16).toFixed() 97 | assertInvalidArgument({StreamName: helpers.testStream, PartitionKey: 'a', Data: '', SequenceNumberForOrdering: seq}, 98 | 'ExclusiveMinimumSequenceNumber ' + seq + ' used in PutRecord on stream ' + helpers.testStream + 99 | ' under account ' + helpers.awsAccountId + ' is invalid.', done) 100 | }) 101 | 102 | it('should return InvalidArgumentException for version 4 in SequenceNumberForOrdering', function(done) { 103 | var seq = new BigNumber('20000000000000000000000000000000000000000000004', 16).toFixed() 104 | assertInvalidArgument({StreamName: helpers.testStream, PartitionKey: 'a', Data: '', SequenceNumberForOrdering: seq}, 105 | 'ExclusiveMinimumSequenceNumber ' + seq + ' used in PutRecord on stream ' + helpers.testStream + 106 | ' under account ' + helpers.awsAccountId + ' is invalid.', done) 107 | }) 108 | 109 | it('should return InvalidArgumentException for version 3 in SequenceNumberForOrdering', function(done) { 110 | var seq = new BigNumber('20000000000000000000000000000000000000000000003', 16).toFixed() 111 | assertInvalidArgument({StreamName: helpers.testStream, PartitionKey: 'a', Data: '', SequenceNumberForOrdering: seq}, 112 | 'ExclusiveMinimumSequenceNumber ' + seq + ' used in PutRecord on stream ' + helpers.testStream + 113 | ' under account ' + helpers.awsAccountId + ' is invalid.', done) 114 | }) 115 | 116 | it('should return InvalidArgumentException for initial 3 in SequenceNumberForOrdering', function(done) { 117 | var seq = new BigNumber('30000000000000000000000000000000000000000000001', 16).toFixed() 118 | assertInvalidArgument({StreamName: helpers.testStream, PartitionKey: 'a', Data: '', SequenceNumberForOrdering: seq}, 119 | 'ExclusiveMinimumSequenceNumber ' + seq + ' used in PutRecord on stream ' + helpers.testStream + 120 | ' under account ' + helpers.awsAccountId + ' is invalid.', done) 121 | }) 122 | 123 | it('should return InvalidArgumentException for initial 1 in SequenceNumberForOrdering', function(done) { 124 | var seq = new BigNumber('1ffffffffff7fffffffffffffff000000000007fffffff2', 16).toFixed() 125 | assertInvalidArgument({StreamName: helpers.testStream, PartitionKey: 'a', Data: '', SequenceNumberForOrdering: seq}, 126 | 'ExclusiveMinimumSequenceNumber ' + seq + ' used in PutRecord on stream ' + helpers.testStream + 127 | ' under account ' + helpers.awsAccountId + ' is invalid.', done) 128 | }) 129 | 130 | it('should return InvalidArgumentException for 8 in index in SequenceNumberForOrdering', function(done) { 131 | var seq = new BigNumber('20000000000800000000000000000000000000000000002', 16).toFixed() 132 | assertInvalidArgument({StreamName: helpers.testStream, PartitionKey: 'a', Data: '', SequenceNumberForOrdering: seq}, 133 | 'ExclusiveMinimumSequenceNumber ' + seq + ' used in PutRecord on stream ' + helpers.testStream + 134 | ' under account ' + helpers.awsAccountId + ' is invalid.', done) 135 | }) 136 | 137 | // Not really sure that this is necessary - seems obscure 138 | it.skip('should return InternalFailure for 8 and real time in SequenceNumberForOrdering', function(done) { 139 | var seq = new BigNumber('2ffffffffff7fffffffffffffff000' + Math.floor(Date.now() / 1000).toString(16) + '7fffffff2', 16).toFixed() 140 | assertInternalFailure({StreamName: helpers.testStream, PartitionKey: 'a', Data: '', SequenceNumberForOrdering: seq}, done) 141 | }) 142 | 143 | it('should return InvalidArgumentException if using sequence number with large date', function(done) { 144 | var seq = new BigNumber('13bb2cc3d80000000000000000000000', 16).toFixed() 145 | assertInvalidArgument({StreamName: helpers.testStream, PartitionKey: 'a', Data: '', SequenceNumberForOrdering: seq}, 146 | 'ExclusiveMinimumSequenceNumber 26227199374822427428162556223570313216 used in PutRecord on stream ' + 147 | helpers.testStream + ' under account ' + helpers.awsAccountId + ' is invalid.', done) 148 | }) 149 | 150 | it('should return InvalidArgumentException with SequenceNumberForOrdering all f', function(done) { 151 | var seq = new BigNumber('2ffffffffff7fffffffffffffff000000000007fffffff2', 16).toFixed() 152 | assertInvalidArgument({StreamName: helpers.testStream, PartitionKey: 'a', Data: '', SequenceNumberForOrdering: seq}, 153 | 'ExclusiveMinimumSequenceNumber ' + seq + ' used in PutRecord on stream ' + helpers.testStream + 154 | ' under account ' + helpers.awsAccountId + ' is invalid.', done) 155 | }) 156 | 157 | }) 158 | 159 | describe('functionality', function() { 160 | 161 | it('should work with empty Data', function(done) { 162 | var now = Date.now() 163 | request(opts({StreamName: helpers.testStream, PartitionKey: 'a', Data: ''}), function(err, res) { 164 | if (err) return done(err) 165 | res.statusCode.should.equal(200) 166 | helpers.assertSequenceNumber(res.body.SequenceNumber, 0, now) 167 | delete res.body.SequenceNumber 168 | res.body.should.eql({ShardId: 'shardId-000000000000'}) 169 | done() 170 | }) 171 | }) 172 | 173 | it('should work with large Data', function(done) { 174 | var now = Date.now(), data = Buffer.allocUnsafe(51200).toString('base64') 175 | request(opts({StreamName: helpers.testStream, PartitionKey: 'a', Data: data}), function(err, res) { 176 | if (err) return done(err) 177 | res.statusCode.should.equal(200) 178 | helpers.assertSequenceNumber(res.body.SequenceNumber, 0, now) 179 | delete res.body.SequenceNumber 180 | res.body.should.eql({ShardId: 'shardId-000000000000'}) 181 | done() 182 | }) 183 | }) 184 | 185 | it('should work with final ExplicitHashKey', function(done) { 186 | var hashKey = new BigNumber(2).pow(128).minus(1).toFixed(), now = Date.now() 187 | request(opts({StreamName: helpers.testStream, PartitionKey: 'a', Data: '', ExplicitHashKey: hashKey}), function(err, res) { 188 | if (err) return done(err) 189 | res.statusCode.should.equal(200) 190 | helpers.assertSequenceNumber(res.body.SequenceNumber, 2, now) 191 | delete res.body.SequenceNumber 192 | res.body.should.eql({ShardId: 'shardId-000000000002'}) 193 | done() 194 | }) 195 | }) 196 | 197 | it('should work with ExplicitHashKey just below range', function(done) { 198 | var hashKey = new BigNumber(2).pow(128).div(3).integerValue(BigNumber.ROUND_FLOOR).times(2).minus(1).toFixed(), now = Date.now() 199 | request(opts({StreamName: helpers.testStream, PartitionKey: 'a', Data: '', ExplicitHashKey: hashKey}), function(err, res) { 200 | if (err) return done(err) 201 | res.statusCode.should.equal(200) 202 | helpers.assertSequenceNumber(res.body.SequenceNumber, 1, now) 203 | delete res.body.SequenceNumber 204 | res.body.should.eql({ShardId: 'shardId-000000000001'}) 205 | done() 206 | }) 207 | }) 208 | 209 | it('should work with ExplicitHashKey just above range', function(done) { 210 | var hashKey = new BigNumber(2).pow(128).div(3).integerValue(BigNumber.ROUND_FLOOR).times(2).toFixed(), now = Date.now() 211 | request(opts({StreamName: helpers.testStream, PartitionKey: 'a', Data: '', ExplicitHashKey: hashKey}), function(err, res) { 212 | if (err) return done(err) 213 | res.statusCode.should.equal(200) 214 | helpers.assertSequenceNumber(res.body.SequenceNumber, 2, now) 215 | delete res.body.SequenceNumber 216 | res.body.should.eql({ShardId: 'shardId-000000000002'}) 217 | done() 218 | }) 219 | }) 220 | 221 | it('should work with SequenceNumberForOrdering all 0', function(done) { 222 | var seq = new BigNumber('20000000000000000000000000000000000000000000002', 16).toFixed(), now = Date.now() 223 | request(opts({StreamName: helpers.testStream, PartitionKey: 'a', Data: '', SequenceNumberForOrdering: seq}), function(err, res) { 224 | if (err) return done(err) 225 | res.statusCode.should.equal(200) 226 | helpers.assertSequenceNumber(res.body.SequenceNumber, 0, now) 227 | delete res.body.SequenceNumber 228 | res.body.should.eql({ShardId: 'shardId-000000000000'}) 229 | done() 230 | }) 231 | }) 232 | 233 | it('should work with SequenceNumberForOrdering all 0 with version 1', function(done) { 234 | var seq = new BigNumber('20000000000000000000000000000000000000000000001', 16).toFixed(), now = Date.now() 235 | request(opts({StreamName: helpers.testStream, PartitionKey: 'a', Data: '', SequenceNumberForOrdering: seq}), function(err, res) { 236 | if (err) return done(err) 237 | res.statusCode.should.equal(200) 238 | helpers.assertSequenceNumber(res.body.SequenceNumber, 0, now) 239 | delete res.body.SequenceNumber 240 | res.body.should.eql({ShardId: 'shardId-000000000000'}) 241 | done() 242 | }) 243 | }) 244 | 245 | it('should work with SequenceNumberForOrdering if 8 near end', function(done) { 246 | var seq = new BigNumber('20000000000000000000000000000000000000800000002', 16).toFixed() 247 | request(opts({StreamName: helpers.testStream, PartitionKey: 'a', Data: '', SequenceNumberForOrdering: seq}), function(err, res) { 248 | if (err) return done(err) 249 | res.statusCode.should.equal(200) 250 | helpers.assertSequenceNumber(res.body.SequenceNumber, 0, 0) 251 | delete res.body.SequenceNumber 252 | res.body.should.eql({ShardId: 'shardId-000000000000'}) 253 | done() 254 | }) 255 | }) 256 | 257 | it('should safely put concurrent, sequential records', function(done) { 258 | this.timeout(100000) 259 | 260 | var remaining = 100, seqIxs = [] 261 | 262 | function putRecords() { 263 | var now = Date.now() 264 | request(opts({StreamName: helpers.testStream, PartitionKey: 'a', Data: ''}), function(err, res) { 265 | if (err) return done(err) 266 | res.statusCode.should.equal(200) 267 | 268 | seqIxs.push(parseInt(new BigNumber(res.body.SequenceNumber).toString(16).slice(11, 27), 16)) 269 | 270 | helpers.assertSequenceNumber(res.body.SequenceNumber, 0, now) 271 | delete res.body.SequenceNumber 272 | res.body.should.eql({ShardId: 'shardId-000000000000'}) 273 | 274 | now = Date.now() 275 | request(opts({StreamName: helpers.testStream, PartitionKey: 'b', Data: ''}), function(err, res) { 276 | if (err) return done(err) 277 | res.statusCode.should.equal(200) 278 | 279 | seqIxs.push(parseInt(new BigNumber(res.body.SequenceNumber).toString(16).slice(11, 27), 16)) 280 | 281 | helpers.assertSequenceNumber(res.body.SequenceNumber, 1, now) 282 | delete res.body.SequenceNumber 283 | res.body.should.eql({ShardId: 'shardId-000000000001'}) 284 | 285 | if (!--remaining) checkIxs() 286 | }) 287 | }) 288 | } 289 | 290 | function checkIxs() { 291 | seqIxs.sort(function(a, b) { return a - b }) 292 | for (var j = 1; j < seqIxs.length; j++) { 293 | var diff = seqIxs[j] - seqIxs[j - 1] 294 | diff.should.equal(1) 295 | } 296 | done() 297 | } 298 | 299 | for (var i = 0; i < remaining; i++) { 300 | putRecords() 301 | } 302 | }) 303 | 304 | }) 305 | 306 | }) 307 | -------------------------------------------------------------------------------- /test/putRecords.js: -------------------------------------------------------------------------------- 1 | var BigNumber = require('bignumber.js'), 2 | helpers = require('./helpers') 3 | 4 | var target = 'PutRecords', 5 | request = helpers.request, 6 | randomName = helpers.randomName, 7 | opts = helpers.opts.bind(null, target), 8 | assertType = helpers.assertType.bind(null, target), 9 | assertValidation = helpers.assertValidation.bind(null, target), 10 | assertNotFound = helpers.assertNotFound.bind(null, target), 11 | assertInvalidArgument = helpers.assertInvalidArgument.bind(null, target) 12 | 13 | describe('putRecords', function() { 14 | 15 | describe('serializations', function() { 16 | 17 | it('should return SerializationException when Records is not a list', function(done) { 18 | assertType('Records', 'List', done) 19 | }) 20 | 21 | it('should return SerializationException when Records.0 is not a struct', function(done) { 22 | assertType('Records.0', 'Structure', done) 23 | }) 24 | 25 | it('should return SerializationException when Records.0.Data is not a Blob', function(done) { 26 | assertType('Records.0.Data', 'Blob', done) 27 | }) 28 | 29 | it('should return SerializationException when Records.0.ExplicitHashKey is not a String', function(done) { 30 | assertType('Records.0.ExplicitHashKey', 'String', done) 31 | }) 32 | 33 | it('should return SerializationException when Records.0.PartitionKey is not a String', function(done) { 34 | assertType('Records.0.PartitionKey', 'String', done) 35 | }) 36 | 37 | it('should return SerializationException when StreamName is not a String', function(done) { 38 | assertType('StreamName', 'String', done) 39 | }) 40 | 41 | }) 42 | 43 | describe('validations', function() { 44 | 45 | it('should return ValidationException for no StreamName', function(done) { 46 | assertValidation({}, [ 47 | 'Value null at \'records\' failed to satisfy constraint: ' + 48 | 'Member must not be null', 49 | 'Value null at \'streamName\' failed to satisfy constraint: ' + 50 | 'Member must not be null', 51 | ], done) 52 | }) 53 | 54 | it('should return ValidationException for empty StreamName', function(done) { 55 | assertValidation({StreamName: '', Records: []}, [ 56 | 'Value \'[]\' at \'records\' failed to satisfy constraint: ' + 57 | 'Member must have length greater than or equal to 1', 58 | 'Value \'\' at \'streamName\' failed to satisfy constraint: ' + 59 | 'Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+', 60 | 'Value \'\' at \'streamName\' failed to satisfy constraint: ' + 61 | 'Member must have length greater than or equal to 1', 62 | ], done) 63 | }) 64 | 65 | it('should return ValidationException for empty PartitionKey', function(done) { 66 | assertValidation({StreamName: '', Records: [{PartitionKey: '', Data: '', ExplicitHashKey: ''}]}, [ 67 | 'Value \'\' at \'records.1.member.partitionKey\' failed to satisfy constraint: ' + 68 | 'Member must have length greater than or equal to 1', 69 | 'Value \'\' at \'records.1.member.explicitHashKey\' failed to satisfy constraint: ' + 70 | 'Member must satisfy regular expression pattern: 0|([1-9]\\d{0,38})', 71 | 'Value \'\' at \'streamName\' failed to satisfy constraint: ' + 72 | 'Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+', 73 | 'Value \'\' at \'streamName\' failed to satisfy constraint: ' + 74 | 'Member must have length greater than or equal to 1', 75 | ], done) 76 | }) 77 | 78 | it('should return ValidationException for long StreamName', function(done) { 79 | var name = new Array(129 + 1).join('a'), name2 = new Array(257 + 1).join('a'), 80 | data = Buffer.allocUnsafe(1048577).toString('base64') 81 | assertValidation({StreamName: name, Records: [{PartitionKey: name2, Data: data, ExplicitHashKey: ''}]}, [ 82 | 'Value \'' + name2 + '\' at \'records.1.member.partitionKey\' failed to satisfy constraint: ' + 83 | 'Member must have length less than or equal to 256', 84 | 'Value \'\' at \'records.1.member.explicitHashKey\' failed to satisfy constraint: ' + 85 | 'Member must satisfy regular expression pattern: 0|([1-9]\\d{0,38})', 86 | 'Value \'java.nio.HeapByteBuffer[pos=0 lim=1048577 cap=1048577]\' at \'records.1.member.data\' failed to satisfy constraint: ' + 87 | 'Member must have length less than or equal to 1048576', 88 | 'Value \'' + name + '\' at \'streamName\' failed to satisfy constraint: ' + 89 | 'Member must have length less than or equal to 128', 90 | ], done) 91 | }) 92 | 93 | it('should return ValidationException for too many records', function(done) { 94 | var records = [], strs = [] 95 | for (var i = 0; i < 501; i++) { 96 | records.push({PartitionKey: '', Data: ''}) 97 | strs.push('com.amazonaws.kinesis.v20131202.PutRecordsRequestEntry@c965e310') 98 | } 99 | assertValidation({StreamName: '', Records: records}, [ 100 | 'Value \'[' + strs.join(', ') + ']\' at \'records\' failed to satisfy constraint: ' + 101 | 'Member must have length less than or equal to 500', 102 | 'Value \'\' at \'records.1.member.partitionKey\' failed to satisfy constraint: ' + 103 | 'Member must have length greater than or equal to 1', 104 | 'Value \'\' at \'records.2.member.partitionKey\' failed to satisfy constraint: ' + 105 | 'Member must have length greater than or equal to 1', 106 | 'Value \'\' at \'records.3.member.partitionKey\' failed to satisfy constraint: ' + 107 | 'Member must have length greater than or equal to 1', 108 | 'Value \'\' at \'records.4.member.partitionKey\' failed to satisfy constraint: ' + 109 | 'Member must have length greater than or equal to 1', 110 | 'Value \'\' at \'records.5.member.partitionKey\' failed to satisfy constraint: ' + 111 | 'Member must have length greater than or equal to 1', 112 | 'Value \'\' at \'records.6.member.partitionKey\' failed to satisfy constraint: ' + 113 | 'Member must have length greater than or equal to 1', 114 | 'Value \'\' at \'records.7.member.partitionKey\' failed to satisfy constraint: ' + 115 | 'Member must have length greater than or equal to 1', 116 | 'Value \'\' at \'records.8.member.partitionKey\' failed to satisfy constraint: ' + 117 | 'Member must have length greater than or equal to 1', 118 | 'Value \'\' at \'records.9.member.partitionKey\' failed to satisfy constraint: ' + 119 | 'Member must have length greater than or equal to 1', 120 | ], done) 121 | }) 122 | 123 | it('should return ResourceNotFoundException if stream does not exist', function(done) { 124 | var name1 = randomName() 125 | assertNotFound({StreamName: name1, Records: [{PartitionKey: 'a', Data: ''}]}, 126 | 'Stream ' + name1 + ' under account ' + helpers.awsAccountId + ' not found.', done) 127 | }) 128 | 129 | it('should return InvalidArgumentException for out of bounds ExplicitHashKey', function(done) { 130 | var hashKey = new BigNumber(2).pow(128).toFixed() 131 | assertInvalidArgument({StreamName: helpers.testStream, Records: [ 132 | {PartitionKey: 'a', Data: '', ExplicitHashKey: hashKey}, {PartitionKey: 'a', Data: ''}]}, 133 | 'Invalid ExplicitHashKey. ExplicitHashKey must be in the range: [0, 2^128-1]. Specified value was ' + hashKey, done) 134 | }) 135 | 136 | }) 137 | 138 | describe('functionality', function() { 139 | 140 | it('should work with large Data', function(done) { 141 | var now = Date.now(), data = Buffer.allocUnsafe(51200).toString('base64'), records = [{PartitionKey: 'a', Data: data}] 142 | request(opts({StreamName: helpers.testStream, Records: records}), function(err, res) { 143 | if (err) return done(err) 144 | res.statusCode.should.equal(200) 145 | 146 | helpers.assertSequenceNumber(res.body.Records[0].SequenceNumber, 0, now) 147 | delete res.body.Records[0].SequenceNumber 148 | 149 | res.body.should.eql({ 150 | FailedRecordCount: 0, 151 | Records: [{ 152 | ShardId: 'shardId-000000000000', 153 | }], 154 | }) 155 | 156 | done() 157 | }) 158 | }) 159 | 160 | it('should work with mixed values', function(done) { 161 | var now = Date.now(), 162 | hashKey1 = new BigNumber(2).pow(128).minus(1).toFixed(), 163 | hashKey2 = new BigNumber(2).pow(128).div(3).integerValue(BigNumber.ROUND_FLOOR).times(2).minus(1).toFixed(), 164 | hashKey3 = new BigNumber(2).pow(128).div(3).integerValue(BigNumber.ROUND_FLOOR).times(2).toFixed(), 165 | records = [ 166 | {PartitionKey: 'a', Data: ''}, 167 | {PartitionKey: 'b', Data: ''}, 168 | {PartitionKey: 'e', Data: ''}, 169 | {PartitionKey: 'f', Data: ''}, 170 | {PartitionKey: 'a', Data: '', ExplicitHashKey: hashKey1}, 171 | {PartitionKey: 'a', Data: '', ExplicitHashKey: hashKey2}, 172 | {PartitionKey: 'a', Data: '', ExplicitHashKey: hashKey3}, 173 | ] 174 | request(opts({StreamName: helpers.testStream, Records: records}), function(err, res) { 175 | if (err) return done(err) 176 | res.statusCode.should.equal(200) 177 | 178 | helpers.assertSequenceNumber(res.body.Records[0].SequenceNumber, 0, now) 179 | helpers.assertSequenceNumber(res.body.Records[1].SequenceNumber, 1, now) 180 | helpers.assertSequenceNumber(res.body.Records[2].SequenceNumber, 2, now) 181 | helpers.assertSequenceNumber(res.body.Records[3].SequenceNumber, 1, now) 182 | helpers.assertSequenceNumber(res.body.Records[4].SequenceNumber, 2, now) 183 | helpers.assertSequenceNumber(res.body.Records[5].SequenceNumber, 1, now) 184 | helpers.assertSequenceNumber(res.body.Records[6].SequenceNumber, 2, now) 185 | 186 | var indexOrder = [[2, 4, 6], [1, 3, 5], [0]] 187 | indexOrder.forEach(function(arr) { 188 | var lastIx 189 | arr.forEach(function(i) { 190 | var seqIx = parseInt(new BigNumber(res.body.Records[i].SequenceNumber).toString(16).slice(11, 27), 16), 191 | diff = lastIx != null ? seqIx - lastIx : 1 192 | diff.should.equal(1) 193 | lastIx = seqIx 194 | }) 195 | }) 196 | 197 | delete res.body.Records[0].SequenceNumber 198 | delete res.body.Records[1].SequenceNumber 199 | delete res.body.Records[2].SequenceNumber 200 | delete res.body.Records[3].SequenceNumber 201 | delete res.body.Records[4].SequenceNumber 202 | delete res.body.Records[5].SequenceNumber 203 | delete res.body.Records[6].SequenceNumber 204 | 205 | res.body.should.eql({ 206 | FailedRecordCount: 0, 207 | Records: [{ 208 | ShardId: 'shardId-000000000000', 209 | }, { 210 | ShardId: 'shardId-000000000001', 211 | }, { 212 | ShardId: 'shardId-000000000002', 213 | }, { 214 | ShardId: 'shardId-000000000001', 215 | }, { 216 | ShardId: 'shardId-000000000002', 217 | }, { 218 | ShardId: 'shardId-000000000001', 219 | }, { 220 | ShardId: 'shardId-000000000002', 221 | }], 222 | }) 223 | 224 | done() 225 | }) 226 | }) 227 | 228 | // Use this test to play around with sequence number generation 229 | // aws kinesis create-stream --stream-name test --shard-count 50 230 | it.skip('should print out sequences for many shards', function(done) { 231 | var records = [], numShards = 50, streamName = 'test' 232 | for (var j = 0; j < 2; j++) { 233 | for (var i = 0; i < numShards; i++) { 234 | records.push({ 235 | PartitionKey: 'a', 236 | Data: '', 237 | ExplicitHashKey: new BigNumber(2).pow(128).div(numShards).integerValue(BigNumber.ROUND_FLOOR).times(i + 1).minus(1).toFixed(), 238 | }) 239 | } 240 | } 241 | request(opts({StreamName: streamName, Records: records}), function(err, res) { 242 | if (err) return done(err) 243 | res.statusCode.should.equal(200) 244 | 245 | res.body.Records.sort(function(a, b) { 246 | var seqIxA = parseInt(new BigNumber(a.SequenceNumber).toString(16).slice(11, 27), 16) 247 | var seqIxB = parseInt(new BigNumber(b.SequenceNumber).toString(16).slice(11, 27), 16) 248 | return seqIxA - seqIxB 249 | }) 250 | res.body.Records.forEach(function(record, i) { 251 | var seqIx = parseInt(new BigNumber(record.SequenceNumber).toString(16).slice(11, 27), 16) 252 | console.log(i, seqIx, record.ShardId) 253 | }) 254 | 255 | done() 256 | }) 257 | }) 258 | }) 259 | }) 260 | -------------------------------------------------------------------------------- /test/removeTagsFromStream.js: -------------------------------------------------------------------------------- 1 | var helpers = require('./helpers') 2 | 3 | var target = 'RemoveTagsFromStream', 4 | request = helpers.request, 5 | opts = helpers.opts.bind(null, target), 6 | randomName = helpers.randomName, 7 | assertType = helpers.assertType.bind(null, target), 8 | assertValidation = helpers.assertValidation.bind(null, target), 9 | assertNotFound = helpers.assertNotFound.bind(null, target), 10 | assertInvalidArgument = helpers.assertInvalidArgument.bind(null, target) 11 | 12 | describe('removeTagsFromStream', function() { 13 | 14 | describe('serializations', function() { 15 | 16 | it('should return SerializationException when TagKeys is not a list', function(done) { 17 | assertType('TagKeys', 'List', done) 18 | }) 19 | 20 | it('should return SerializationException when TagKeys.0 is not a string', function(done) { 21 | assertType('TagKeys.0', 'String', done) 22 | }) 23 | 24 | it('should return SerializationException when StreamName is not a String', function(done) { 25 | assertType('StreamName', 'String', done) 26 | }) 27 | 28 | }) 29 | 30 | describe('validations', function() { 31 | 32 | it('should return ValidationException for no StreamName', function(done) { 33 | assertValidation({}, [ 34 | 'Value null at \'tagKeys\' failed to satisfy constraint: ' + 35 | 'Member must not be null', 36 | 'Value null at \'streamName\' failed to satisfy constraint: ' + 37 | 'Member must not be null', 38 | ], done) 39 | }) 40 | 41 | it('should return ValidationException for empty StreamName', function(done) { 42 | assertValidation({StreamName: '', TagKeys: []}, [ 43 | 'Value \'[]\' at \'tagKeys\' failed to satisfy constraint: ' + 44 | 'Member must have length greater than or equal to 1', 45 | 'Value \'\' at \'streamName\' failed to satisfy constraint: ' + 46 | 'Member must satisfy regular expression pattern: [a-zA-Z0-9_.-]+', 47 | 'Value \'\' at \'streamName\' failed to satisfy constraint: ' + 48 | 'Member must have length greater than or equal to 1', 49 | ], done) 50 | }) 51 | 52 | it('should return ValidationException for long StreamName', function(done) { 53 | var name = new Array(129 + 1).join('a') 54 | assertValidation({StreamName: name, TagKeys: ['a']}, [ 55 | 'Value \'' + name + '\' at \'streamName\' failed to satisfy constraint: ' + 56 | 'Member must have length less than or equal to 128', 57 | ], done) 58 | }) 59 | 60 | it('should return ValidationException for long TagKey', function(done) { 61 | var name = new Array(129 + 1).join('a') 62 | assertValidation({StreamName: randomName(), TagKeys: ['a', name, 'b']}, [ 63 | 'Value \'[a, ' + name + ', b]\' at \'tagKeys\' failed to satisfy constraint: ' + 64 | 'Member must satisfy constraint: [Member must have length less than or equal to 128, ' + 65 | 'Member must have length greater than or equal to 1]', 66 | ], done) 67 | }) 68 | 69 | it('should return ValidationException for too many TagKeys', function(done) { 70 | assertValidation({StreamName: randomName(), TagKeys: ['1', '2', '3', '4', '5', '6', '7', '8', '9', '10', '11']}, [ 71 | 'Value \'[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11]\' at \'tagKeys\' failed to satisfy constraint: ' + 72 | 'Member must have length less than or equal to 10', 73 | ], done) 74 | }) 75 | 76 | it('should return ResourceNotFoundException if stream does not exist', function(done) { 77 | var name1 = randomName() 78 | assertNotFound({StreamName: name1, TagKeys: [';']}, 79 | 'Stream ' + name1 + ' under account ' + helpers.awsAccountId + ' not found.', done) 80 | }) 81 | 82 | it('should return InvalidArgumentException if ; in TagKeys', function(done) { 83 | assertInvalidArgument({StreamName: helpers.testStream, TagKeys: ['abc;def']}, 84 | 'Some tags contain invalid characters. Valid characters: ' + 85 | 'Unicode letters, digits, white space, _ . / = + - % @.', done) 86 | }) 87 | 88 | it('should return InvalidArgumentException if tab in TagKeys', function(done) { 89 | assertInvalidArgument({StreamName: helpers.testStream, TagKeys: ['abc\tdef']}, 90 | 'Some tags contain invalid characters. Valid characters: ' + 91 | 'Unicode letters, digits, white space, _ . / = + - % @.', done) 92 | }) 93 | 94 | it('should return InvalidArgumentException if newline in TagKeys', function(done) { 95 | assertInvalidArgument({StreamName: helpers.testStream, TagKeys: ['abc\ndef']}, 96 | 'Some tags contain invalid characters. Valid characters: ' + 97 | 'Unicode letters, digits, white space, _ . / = + - % @.', done) 98 | }) 99 | 100 | it('should return InvalidArgumentException if comma in TagKeys', function(done) { 101 | assertInvalidArgument({StreamName: helpers.testStream, TagKeys: ['abc,def']}, 102 | 'Some tags contain invalid characters. Valid characters: ' + 103 | 'Unicode letters, digits, white space, _ . / = + - % @.', done) 104 | }) 105 | 106 | it('should return InvalidArgumentException if % in TagKeys', function(done) { 107 | assertInvalidArgument({StreamName: helpers.testStream, TagKeys: ['abc%def']}, 108 | 'Failed to remove tags from stream ' + helpers.testStream + ' under account ' + helpers.awsAccountId + 109 | ' because some tags contained illegal characters. The allowed characters are ' + 110 | 'Unicode letters, white-spaces, \'_\',\',\',\'/\',\'=\',\'+\',\'-\',\'@\'.', done) 111 | }) 112 | 113 | }) 114 | 115 | describe('functionality', function() { 116 | 117 | it('should succeed if valid characters in tag keys', function(done) { 118 | request(opts({StreamName: helpers.testStream, TagKeys: ['ü0 _.', '/=+-@']}), function(err, res) { 119 | if (err) return done(err) 120 | res.statusCode.should.equal(200) 121 | res.body.should.equal('') 122 | done() 123 | }) 124 | }) 125 | 126 | it('should add and remove tags keys', function(done) { 127 | request(helpers.opts('AddTagsToStream', { 128 | StreamName: helpers.testStream, 129 | Tags: {a: 'a', 'ü0 _.': 'a', '/=+-@': 'a'}, 130 | }), function(err, res) { 131 | if (err) return done(err) 132 | res.statusCode.should.equal(200) 133 | 134 | request(helpers.opts('ListTagsForStream', {StreamName: helpers.testStream}), function(err, res) { 135 | if (err) return done(err) 136 | res.statusCode.should.equal(200) 137 | res.body.Tags.should.containEql({Key: 'a', Value: 'a'}) 138 | res.body.Tags.should.containEql({Key: 'ü0 _.', Value: 'a'}) 139 | res.body.Tags.should.containEql({Key: '/=+-@', Value: 'a'}) 140 | 141 | request(opts({StreamName: helpers.testStream, TagKeys: ['ü0 _.', '/=+-@', 'b', 'c']}), function(err, res) { 142 | if (err) return done(err) 143 | res.statusCode.should.equal(200) 144 | res.body.should.equal('') 145 | 146 | request(helpers.opts('ListTagsForStream', {StreamName: helpers.testStream}), function(err, res) { 147 | if (err) return done(err) 148 | res.statusCode.should.equal(200) 149 | res.body.Tags.should.containEql({Key: 'a', Value: 'a'}) 150 | res.body.Tags.should.not.containEql({Key: 'ü0 _.', Value: 'a'}) 151 | res.body.Tags.should.not.containEql({Key: '/=+-@', Value: 'a'}) 152 | 153 | request(opts({StreamName: helpers.testStream, TagKeys: ['a']}), done) 154 | }) 155 | }) 156 | }) 157 | }) 158 | }) 159 | 160 | }) 161 | 162 | }) 163 | 164 | -------------------------------------------------------------------------------- /validations/addTagsToStream.js: -------------------------------------------------------------------------------- 1 | exports.types = { 2 | Tags: { 3 | type: 'Map', 4 | notNull: true, 5 | lengthGreaterThanOrEqual: 1, 6 | lengthLessThanOrEqual: 10, 7 | childKeyLengths: [1, 128], 8 | childValueLengths: [0, 256], 9 | children: { 10 | type: 'String', 11 | }, 12 | }, 13 | StreamName: { 14 | type: 'String', 15 | notNull: true, 16 | regex: '[a-zA-Z0-9_.-]+', 17 | lengthGreaterThanOrEqual: 1, 18 | lengthLessThanOrEqual: 128, 19 | }, 20 | } 21 | -------------------------------------------------------------------------------- /validations/createStream.js: -------------------------------------------------------------------------------- 1 | exports.types = { 2 | ShardCount: { 3 | type: 'Integer', 4 | notNull: true, 5 | greaterThanOrEqual: 1, 6 | lessThanOrEqual: 100000, 7 | }, 8 | StreamName: { 9 | type: 'String', 10 | notNull: true, 11 | regex: '[a-zA-Z0-9_.-]+', 12 | lengthGreaterThanOrEqual: 1, 13 | lengthLessThanOrEqual: 128, 14 | }, 15 | } 16 | -------------------------------------------------------------------------------- /validations/decreaseStreamRetentionPeriod.js: -------------------------------------------------------------------------------- 1 | exports.types = { 2 | RetentionPeriodHours: { 3 | type: 'Integer', 4 | notNull: true, 5 | greaterThanOrEqual: 1, 6 | lessThanOrEqual: 168, 7 | }, 8 | StreamName: { 9 | type: 'String', 10 | notNull: true, 11 | regex: '[a-zA-Z0-9_.-]+', 12 | lengthGreaterThanOrEqual: 1, 13 | lengthLessThanOrEqual: 128, 14 | }, 15 | } 16 | -------------------------------------------------------------------------------- /validations/deleteStream.js: -------------------------------------------------------------------------------- 1 | exports.types = { 2 | StreamName: { 3 | type: 'String', 4 | notNull: true, 5 | regex: '[a-zA-Z0-9_.-]+', 6 | lengthGreaterThanOrEqual: 1, 7 | lengthLessThanOrEqual: 128, 8 | }, 9 | } 10 | -------------------------------------------------------------------------------- /validations/describeStream.js: -------------------------------------------------------------------------------- 1 | exports.types = { 2 | Limit: { 3 | type: 'Integer', 4 | greaterThanOrEqual: 1, 5 | lessThanOrEqual: 10000, 6 | }, 7 | ExclusiveStartShardId: { 8 | type: 'String', 9 | regex: '[a-zA-Z0-9_.-]+', 10 | lengthGreaterThanOrEqual: 1, 11 | lengthLessThanOrEqual: 128, 12 | }, 13 | StreamName: { 14 | type: 'String', 15 | notNull: true, 16 | regex: '[a-zA-Z0-9_.-]+', 17 | lengthGreaterThanOrEqual: 1, 18 | lengthLessThanOrEqual: 128, 19 | }, 20 | } 21 | -------------------------------------------------------------------------------- /validations/describeStreamSummary.js: -------------------------------------------------------------------------------- 1 | exports.types = { 2 | StreamName: { 3 | type: 'String', 4 | notNull: true, 5 | regex: '[a-zA-Z0-9_.-]+', 6 | lengthGreaterThanOrEqual: 1, 7 | lengthLessThanOrEqual: 128, 8 | }, 9 | } 10 | -------------------------------------------------------------------------------- /validations/getRecords.js: -------------------------------------------------------------------------------- 1 | exports.types = { 2 | Limit: { 3 | type: 'Integer', 4 | greaterThanOrEqual: 1, 5 | lessThanOrEqual: 10000, 6 | }, 7 | ShardIterator: { 8 | type: 'String', 9 | notNull: true, 10 | lengthGreaterThanOrEqual: 1, 11 | lengthLessThanOrEqual: 512, 12 | }, 13 | } 14 | -------------------------------------------------------------------------------- /validations/getShardIterator.js: -------------------------------------------------------------------------------- 1 | exports.types = { 2 | ShardId: { 3 | type: 'String', 4 | notNull: true, 5 | regex: '[a-zA-Z0-9_.-]+', 6 | lengthGreaterThanOrEqual: 1, 7 | lengthLessThanOrEqual: 128, 8 | }, 9 | ShardIteratorType: { 10 | type: 'String', 11 | notNull: true, 12 | enum: ['AFTER_SEQUENCE_NUMBER', 'LATEST', 'AT_TIMESTAMP', 'AT_SEQUENCE_NUMBER', 'TRIM_HORIZON'], 13 | }, 14 | StartingSequenceNumber: { 15 | type: 'String', 16 | regex: '0|([1-9]\\d{0,128})', 17 | }, 18 | StreamName: { 19 | type: 'String', 20 | notNull: true, 21 | regex: '[a-zA-Z0-9_.-]+', 22 | lengthGreaterThanOrEqual: 1, 23 | lengthLessThanOrEqual: 128, 24 | }, 25 | Timestamp: { 26 | type: 'Timestamp', 27 | }, 28 | } 29 | -------------------------------------------------------------------------------- /validations/increaseStreamRetentionPeriod.js: -------------------------------------------------------------------------------- 1 | exports.types = { 2 | RetentionPeriodHours: { 3 | type: 'Integer', 4 | notNull: true, 5 | greaterThanOrEqual: 1, 6 | lessThanOrEqual: 168, 7 | }, 8 | StreamName: { 9 | type: 'String', 10 | notNull: true, 11 | regex: '[a-zA-Z0-9_.-]+', 12 | lengthGreaterThanOrEqual: 1, 13 | lengthLessThanOrEqual: 128, 14 | }, 15 | } 16 | -------------------------------------------------------------------------------- /validations/index.js: -------------------------------------------------------------------------------- 1 | exports.checkTypes = checkTypes 2 | exports.checkValidations = checkValidations 3 | exports.toLowerFirst = toLowerFirst 4 | 5 | function checkTypes(data, types) { 6 | var key 7 | for (key in data) { 8 | // TODO: deal with nulls 9 | if (!types[key] || data[key] == null) 10 | delete data[key] 11 | } 12 | 13 | return Object.keys(types).reduce(function(newData, key) { 14 | var val = checkType(data[key], types[key]) 15 | if (val != null) newData[key] = val 16 | return newData 17 | }, {}) 18 | 19 | function typeError(msg) { 20 | var err = new Error(msg) 21 | err.statusCode = 400 22 | err.body = { 23 | __type: 'SerializationException', 24 | Message: msg, 25 | } 26 | return err 27 | } 28 | 29 | function checkType(val, type) { 30 | if (val == null) return null 31 | var actualType = type.type || type 32 | switch (actualType) { 33 | case 'Boolean': 34 | switch (typeof val) { 35 | case 'number': 36 | throw typeError('class com.amazon.coral.value.json.numbers.TruncatingBigNumber can not be converted to an Boolean') 37 | case 'string': 38 | // "\'HELLOWTF\' can not be converted to an Boolean" 39 | // seems to convert to uppercase 40 | // 'true'/'false'/'1'/'0'/'no'/'yes' seem to convert fine 41 | val = val.toUpperCase() 42 | throw typeError('\'' + val + '\' can not be converted to an Boolean') 43 | case 'object': 44 | if (Array.isArray(val)) throw typeError('Start of list found where not expected') 45 | throw typeError('Start of structure or map found where not expected.') 46 | } 47 | return val 48 | case 'Short': 49 | case 'Integer': 50 | case 'Long': 51 | case 'Double': 52 | switch (typeof val) { 53 | case 'boolean': 54 | throw typeError('class java.lang.Boolean can not be converted to an ' + actualType) 55 | case 'number': 56 | if (actualType != 'Double') val = Math.floor(val) 57 | if (actualType == 'Short') val = Math.min(val, 32767) 58 | if (actualType == 'Integer') val = Math.min(val, 2147483647) 59 | break 60 | case 'string': 61 | throw typeError('class java.lang.String can not be converted to an ' + actualType) 62 | case 'object': 63 | if (Array.isArray(val)) throw typeError('Start of list found where not expected') 64 | throw typeError('Start of structure or map found where not expected.') 65 | } 66 | return val 67 | case 'String': 68 | switch (typeof val) { 69 | case 'boolean': 70 | throw typeError('class java.lang.Boolean can not be converted to an String') 71 | case 'number': 72 | throw typeError('class com.amazon.coral.value.json.numbers.TruncatingBigNumber can not be converted to an String') 73 | case 'object': 74 | if (Array.isArray(val)) throw typeError('Start of list found where not expected') 75 | throw typeError('Start of structure or map found where not expected.') 76 | } 77 | return val 78 | case 'Blob': 79 | switch (typeof val) { 80 | case 'boolean': 81 | throw typeError('class java.lang.Boolean can not be converted to a Blob') 82 | case 'number': 83 | throw typeError('class com.amazon.coral.value.json.numbers.TruncatingBigNumber can not be converted to a Blob') 84 | case 'object': 85 | if (Buffer.isBuffer(val)) return val 86 | if (Array.isArray(val)) throw typeError('Start of list found where not expected') 87 | throw typeError('Start of structure or map found where not expected.') 88 | } 89 | if (val.length % 4) 90 | throw typeError('\'' + val + '\' can not be converted to a Blob: ' + 91 | 'Base64 encoded length is expected a multiple of 4 bytes but found: ' + val.length) 92 | var match = val.match(/[^a-zA-Z0-9+/=]|\=[^=]/) 93 | if (match) 94 | throw typeError('\'' + val + '\' can not be converted to a Blob: ' + 95 | 'Invalid Base64 character: \'' + match[0][0] + '\'') 96 | // TODO: need a better check than this... 97 | if (Buffer.from(val, 'base64').toString('base64') != val) 98 | throw typeError('\'' + val + '\' can not be converted to a Blob: ' + 99 | 'Invalid last non-pad Base64 character dectected') 100 | return val 101 | case 'Timestamp': 102 | switch (typeof val) { 103 | case 'boolean': 104 | throw typeError('class java.lang.Boolean can not be converted to milliseconds since epoch') 105 | case 'string': 106 | throw typeError('class java.lang.String can not be converted to milliseconds since epoch') 107 | case 'object': 108 | if (Array.isArray(val)) throw typeError('Start of list found where not expected') 109 | throw typeError('Start of structure or map found where not expected.') 110 | } 111 | return val 112 | case 'List': 113 | switch (typeof val) { 114 | case 'boolean': 115 | case 'number': 116 | case 'string': 117 | throw typeError('Expected list or null') 118 | case 'object': 119 | if (!Array.isArray(val)) throw typeError('Start of structure or map found where not expected.') 120 | } 121 | return val.map(function(child) { return checkType(child, type.children) }) 122 | case 'Map': 123 | switch (typeof val) { 124 | case 'boolean': 125 | case 'number': 126 | case 'string': 127 | throw typeError('Expected map or null') 128 | case 'object': 129 | if (Array.isArray(val)) throw typeError('Start of list found where not expected') 130 | } 131 | return Object.keys(val).reduce(function(newVal, key) { 132 | newVal[key] = checkType(val[key], type.children) 133 | return newVal 134 | }, {}) 135 | case 'Structure': 136 | switch (typeof val) { 137 | case 'boolean': 138 | case 'number': 139 | case 'string': 140 | throw typeError('Expected null') 141 | case 'object': 142 | if (Array.isArray(val)) throw typeError('Start of list found where not expected') 143 | } 144 | return checkTypes(val, type.children) 145 | default: 146 | throw new Error('Unknown type: ' + actualType) 147 | } 148 | } 149 | } 150 | 151 | var validateFns = {} 152 | 153 | function checkValidations(data, validations, custom) { 154 | var attr, msg, errors = [] 155 | function validationError(msg) { 156 | var err = new Error(msg) 157 | err.statusCode = 400 158 | err.body = { 159 | __type: 'ValidationException', 160 | message: msg, 161 | } 162 | return err 163 | } 164 | 165 | for (attr in validations) { 166 | if (validations[attr].required && data[attr] == null) { 167 | throw validationError('The paramater \'' + toLowerFirst(attr) + '\' is required but was not present in the request') 168 | } 169 | } 170 | 171 | function checkNonRequireds(data, types, parent) { 172 | for (attr in types) { 173 | checkNonRequired(attr, data[attr], types[attr], parent) 174 | } 175 | } 176 | 177 | checkNonRequireds(data, validations) 178 | 179 | function checkNonRequired(attr, data, validations, parent) { 180 | if (validations == null || typeof validations != 'object') return 181 | for (var validation in validations) { 182 | if (errors.length >= 10) return 183 | if (~['type', 'required', 'memberStr'].indexOf(validation)) continue 184 | if (validation != 'notNull' && data == null) continue 185 | if (validation == 'children') { 186 | if (validations.type == 'List') { 187 | for (var i = 0; i < data.length; i++) { 188 | checkNonRequired('member', data[i], validations.children, 189 | (parent ? parent + '.' : '') + toLowerFirst(attr) + '.' + (i + 1)) 190 | } 191 | continue 192 | } else if (validations.type == 'Map') { 193 | // TODO: Always reverse? 194 | Object.keys(data).reverse().forEach(function(key) { // eslint-disable-line no-loop-func 195 | checkNonRequired('member', data[key], validations.children, 196 | (parent ? parent + '.' : '') + toLowerFirst(attr) + '.' + key) 197 | }) 198 | continue 199 | } 200 | checkNonRequireds(data, validations.children, (parent ? parent + '.' : '') + toLowerFirst(attr)) 201 | continue 202 | } 203 | validateFns[validation](parent, attr, validations[validation], data, validations.type, validations.memberStr, errors) 204 | } 205 | } 206 | 207 | if (errors.length) 208 | throw validationError(errors.length + ' validation error' + (errors.length > 1 ? 's' : '') + ' detected: ' + errors.join('; ')) 209 | 210 | if (custom) { 211 | msg = custom(data) 212 | if (msg) throw validationError(msg) 213 | } 214 | } 215 | 216 | validateFns.notNull = function(parent, key, val, data, type, memberStr, errors) { 217 | validate(data != null, 'Member must not be null', data, type, memberStr, parent, key, errors) 218 | } 219 | validateFns.greaterThanOrEqual = function(parent, key, val, data, type, memberStr, errors) { 220 | validate(data >= val, 'Member must have value greater than or equal to ' + val, data, type, memberStr, parent, key, errors) 221 | } 222 | validateFns.lessThanOrEqual = function(parent, key, val, data, type, memberStr, errors) { 223 | validate(data <= val, 'Member must have value less than or equal to ' + val, data, type, memberStr, parent, key, errors) 224 | } 225 | validateFns.regex = function(parent, key, pattern, data, type, memberStr, errors) { 226 | validate(RegExp('^' + pattern + '$').test(data), 'Member must satisfy regular expression pattern: ' + pattern, data, type, memberStr, parent, key, errors) 227 | } 228 | validateFns.lengthGreaterThanOrEqual = function(parent, key, val, data, type, memberStr, errors) { 229 | if (type == 'Blob') data = Buffer.from(data, 'base64') 230 | var length = (typeof data == 'object' && !Array.isArray(data) && !Buffer.isBuffer(data)) ? 231 | Object.keys(data).length : data.length 232 | validate(length >= val, 'Member must have length greater than or equal to ' + val, data, type, memberStr, parent, key, errors) 233 | } 234 | validateFns.lengthLessThanOrEqual = function(parent, key, val, data, type, memberStr, errors) { 235 | if (type == 'Blob') data = Buffer.from(data, 'base64') 236 | var length = (typeof data == 'object' && !Array.isArray(data) && !Buffer.isBuffer(data)) ? 237 | Object.keys(data).length : data.length 238 | validate(length <= val, 'Member must have length less than or equal to ' + val, data, type, memberStr, parent, key, errors) 239 | } 240 | validateFns.enum = function(parent, key, val, data, type, memberStr, errors) { 241 | validate(~val.indexOf(data), 'Member must satisfy enum value set: [' + val.join(', ') + ']', data, type, memberStr, parent, key, errors) 242 | } 243 | validateFns.childLengths = function(parent, key, val, data, type, memberStr, errors) { 244 | validate(data.every(function(x) { return x.length >= val[0] && x.length <= val[1] }), 245 | 'Member must satisfy constraint: [Member must have length less than or equal to ' + val[1] + 246 | ', Member must have length greater than or equal to ' + val[0] + ']', 247 | data, type, memberStr, parent, key, errors) 248 | } 249 | validateFns.childKeyLengths = function(parent, key, val, data, type, memberStr, errors) { 250 | validate(Object.keys(data).every(function(x) { return x.length >= val[0] && x.length <= val[1] }), 251 | 'Map keys must satisfy constraint: [Member must have length less than or equal to ' + val[1] + 252 | ', Member must have length greater than or equal to ' + val[0] + ']', 253 | data, type, memberStr, parent, key, errors) 254 | } 255 | validateFns.childValueLengths = function(parent, key, val, data, type, memberStr, errors) { 256 | validate(Object.keys(data).every(function(x) { return data[x].length >= val[0] && data[x].length <= val[1] }), 257 | 'Map value must satisfy constraint: [Member must have length less than or equal to ' + val[1] + 258 | ', Member must have length greater than or equal to ' + val[0] + ']', 259 | data, type, memberStr, parent, key, errors) 260 | } 261 | 262 | function validate(predicate, msg, data, type, memberStr, parent, key, errors) { 263 | if (predicate) return 264 | var value = valueStr(data, type, memberStr) 265 | if (value != 'null') value = '\'' + value + '\'' 266 | parent = parent ? parent + '.' : '' 267 | errors.push('Value ' + value + ' at \'' + parent + toLowerFirst(key) + '\' failed to satisfy constraint: ' + msg) 268 | } 269 | 270 | function toLowerFirst(str) { 271 | return str[0].toLowerCase() + str.slice(1) 272 | } 273 | 274 | function valueStr(data, type, memberStr) { 275 | if (data == null) return 'null' 276 | switch (type) { 277 | case 'Blob': 278 | var length = Buffer.from(data, 'base64').length 279 | return 'java.nio.HeapByteBuffer[pos=0 lim=' + length + ' cap=' + length + ']' 280 | case 'List': 281 | return '[' + data.map(function(item) { return memberStr || item }).join(', ') + ']' 282 | case 'Map': 283 | return '{' + Object.keys(data).map(function(key) { return key + '=' + (memberStr || data[key]) }).join(', ') + '}' 284 | default: 285 | return typeof data == 'object' ? JSON.stringify(data) : data 286 | } 287 | } 288 | 289 | -------------------------------------------------------------------------------- /validations/listShards.js: -------------------------------------------------------------------------------- 1 | exports.types = { 2 | ExclusiveStartShardId: { 3 | type: 'String', 4 | regex: '[a-zA-Z0-9_.-]+', 5 | lengthGreaterThanOrEqual: 1, 6 | lengthLessThanOrEqual: 128, 7 | }, 8 | MaxResults: { 9 | type: 'Integer', 10 | greaterThanOrEqual: 1, 11 | lessThanOrEqual: 10000, 12 | }, 13 | NextToken: { 14 | type: 'String', 15 | lengthGreaterThanOrEqual: 1, 16 | }, 17 | StreamCreationTimestamp: { 18 | type: 'Timestamp', 19 | }, 20 | StreamName: { 21 | type: 'String', 22 | regex: '[a-zA-Z0-9_.-]+', 23 | lengthGreaterThanOrEqual: 1, 24 | lengthLessThanOrEqual: 128, 25 | }, 26 | } 27 | -------------------------------------------------------------------------------- /validations/listStreams.js: -------------------------------------------------------------------------------- 1 | exports.types = { 2 | Limit: { 3 | type: 'Integer', 4 | greaterThanOrEqual: 1, 5 | lessThanOrEqual: 10000, 6 | }, 7 | ExclusiveStartStreamName: { 8 | type: 'String', 9 | regex: '[a-zA-Z0-9_.-]+', 10 | lengthGreaterThanOrEqual: 1, 11 | lengthLessThanOrEqual: 128, 12 | }, 13 | } 14 | -------------------------------------------------------------------------------- /validations/listTagsForStream.js: -------------------------------------------------------------------------------- 1 | exports.types = { 2 | Limit: { 3 | type: 'Integer', 4 | greaterThanOrEqual: 1, 5 | lessThanOrEqual: 10, 6 | }, 7 | ExclusiveStartTagKey: { 8 | type: 'String', 9 | lengthGreaterThanOrEqual: 1, 10 | lengthLessThanOrEqual: 128, 11 | }, 12 | StreamName: { 13 | type: 'String', 14 | notNull: true, 15 | regex: '[a-zA-Z0-9_.-]+', 16 | lengthGreaterThanOrEqual: 1, 17 | lengthLessThanOrEqual: 128, 18 | }, 19 | } 20 | -------------------------------------------------------------------------------- /validations/mergeShards.js: -------------------------------------------------------------------------------- 1 | exports.types = { 2 | ShardToMerge: { 3 | type: 'String', 4 | notNull: true, 5 | regex: '[a-zA-Z0-9_.-]+', 6 | lengthGreaterThanOrEqual: 1, 7 | lengthLessThanOrEqual: 128, 8 | }, 9 | AdjacentShardToMerge: { 10 | type: 'String', 11 | notNull: true, 12 | regex: '[a-zA-Z0-9_.-]+', 13 | lengthGreaterThanOrEqual: 1, 14 | lengthLessThanOrEqual: 128, 15 | }, 16 | StreamName: { 17 | type: 'String', 18 | notNull: true, 19 | regex: '[a-zA-Z0-9_.-]+', 20 | lengthGreaterThanOrEqual: 1, 21 | lengthLessThanOrEqual: 128, 22 | }, 23 | } 24 | -------------------------------------------------------------------------------- /validations/putRecord.js: -------------------------------------------------------------------------------- 1 | exports.types = { 2 | SequenceNumberForOrdering: { 3 | type: 'String', 4 | regex: '0|([1-9]\\d{0,128})', 5 | }, 6 | PartitionKey: { 7 | type: 'String', 8 | notNull: true, 9 | lengthGreaterThanOrEqual: 1, 10 | lengthLessThanOrEqual: 256, 11 | }, 12 | ExplicitHashKey: { 13 | type: 'String', 14 | regex: '0|([1-9]\\d{0,38})', 15 | }, 16 | Data: { 17 | type: 'Blob', 18 | notNull: true, 19 | lengthLessThanOrEqual: 1048576, 20 | }, 21 | StreamName: { 22 | type: 'String', 23 | notNull: true, 24 | regex: '[a-zA-Z0-9_.-]+', 25 | lengthGreaterThanOrEqual: 1, 26 | lengthLessThanOrEqual: 128, 27 | }, 28 | } 29 | -------------------------------------------------------------------------------- /validations/putRecords.js: -------------------------------------------------------------------------------- 1 | exports.types = { 2 | Records: { 3 | type: 'List', 4 | notNull: true, 5 | lengthGreaterThanOrEqual: 1, 6 | lengthLessThanOrEqual: 500, 7 | memberStr: 'com.amazonaws.kinesis.v20131202.PutRecordsRequestEntry@c965e310', 8 | children: { 9 | type: 'Structure', 10 | children: { 11 | PartitionKey: { 12 | type: 'String', 13 | notNull: true, 14 | lengthGreaterThanOrEqual: 1, 15 | lengthLessThanOrEqual: 256, 16 | }, 17 | ExplicitHashKey: { 18 | type: 'String', 19 | regex: '0|([1-9]\\d{0,38})', 20 | }, 21 | Data: { 22 | type: 'Blob', 23 | notNull: true, 24 | lengthLessThanOrEqual: 1048576, 25 | }, 26 | }, 27 | }, 28 | }, 29 | StreamName: { 30 | type: 'String', 31 | notNull: true, 32 | regex: '[a-zA-Z0-9_.-]+', 33 | lengthGreaterThanOrEqual: 1, 34 | lengthLessThanOrEqual: 128, 35 | }, 36 | } 37 | -------------------------------------------------------------------------------- /validations/removeTagsFromStream.js: -------------------------------------------------------------------------------- 1 | exports.types = { 2 | TagKeys: { 3 | type: 'List', 4 | notNull: true, 5 | lengthGreaterThanOrEqual: 1, 6 | lengthLessThanOrEqual: 10, 7 | childLengths: [1, 128], 8 | children: { 9 | type: 'String', 10 | }, 11 | }, 12 | StreamName: { 13 | type: 'String', 14 | notNull: true, 15 | regex: '[a-zA-Z0-9_.-]+', 16 | lengthGreaterThanOrEqual: 1, 17 | lengthLessThanOrEqual: 128, 18 | }, 19 | } 20 | -------------------------------------------------------------------------------- /validations/splitShard.js: -------------------------------------------------------------------------------- 1 | exports.types = { 2 | NewStartingHashKey: { 3 | type: 'String', 4 | notNull: true, 5 | regex: '0|([1-9]\\d{0,38})', 6 | }, 7 | StreamName: { 8 | type: 'String', 9 | notNull: true, 10 | regex: '[a-zA-Z0-9_.-]+', 11 | lengthGreaterThanOrEqual: 1, 12 | lengthLessThanOrEqual: 128, 13 | }, 14 | ShardToSplit: { 15 | type: 'String', 16 | notNull: true, 17 | regex: '[a-zA-Z0-9_.-]+', 18 | lengthGreaterThanOrEqual: 1, 19 | lengthLessThanOrEqual: 128, 20 | }, 21 | } 22 | --------------------------------------------------------------------------------